Block-Structured AMR Software Framework
Loading...
Searching...
No Matches
AMReX_FabArray.H
Go to the documentation of this file.
1
2#ifndef BL_FABARRAY_H
3#define BL_FABARRAY_H
4#include <AMReX_Config.H>
5
6#include <AMReX_BLassert.H>
7#include <AMReX_Array.H>
8#include <AMReX_Vector.H>
9#include <AMReX_Box.H>
10#include <AMReX.H>
11#include <AMReX_BoxArray.H>
12#include <AMReX_BoxDomain.H>
13#include <AMReX_FabFactory.H>
15#include <AMReX_Geometry.H>
16#include <AMReX_GpuComplex.H>
18#include <AMReX_Utility.H>
19#include <AMReX_ccse-mpi.H>
20#include <AMReX_BLProfiler.H>
21#include <AMReX_Periodicity.H>
22#include <AMReX_Print.H>
23#include <AMReX_FabArrayBase.H>
24#include <AMReX_MFIter.H>
25#include <AMReX_MakeType.H>
26#include <AMReX_TypeTraits.H>
27#include <AMReX_Concepts.H>
28#include <AMReX_LayoutData.H>
29#include <AMReX_BaseFab.H>
31#include <AMReX_MFParallelFor.H>
33#include <AMReX_ParReduce.H>
34
35#include <AMReX_Gpu.H>
36
41
42#ifdef AMREX_USE_EB
43#include <AMReX_EBFabFactory.H>
44#endif
45
46#ifdef AMREX_USE_OMP
47#include <omp.h>
48#endif
49
50#include <algorithm>
51#include <cstring>
52#include <iterator>
53#include <limits>
54#include <map>
55#include <memory>
56#include <utility>
57#include <set>
58#include <string>
59#include <vector>
60
61namespace amrex {
62
63template <typename T>
64requires (!BaseFabType<T>)
65Long nBytesOwned (T const&) noexcept { return 0; }
66
67template <typename T>
68Long nBytesOwned (BaseFab<T> const& fab) noexcept { return fab.nBytesOwned(); }
69
73struct MFInfo {
74 // alloc: allocate memory or not
75 bool alloc = true;
77 Arena* arena = nullptr;
79
81 MFInfo& SetAlloc (bool a) noexcept { alloc = a; return *this; }
82
84 MFInfo& SetAllocSingleChunk (bool a) noexcept { alloc_single_chunk = a; return *this; }
85
87 MFInfo& SetArena (Arena* ar) noexcept { arena = ar; return *this; }
88
90 MFInfo& SetTag () noexcept { return *this; }
91
93 MFInfo& SetTag (const char* t) noexcept {
94 tags.emplace_back(t);
95 return *this;
96 }
97
99 MFInfo& SetTag (const std::string& t) noexcept {
100 tags.emplace_back(t);
101 return *this;
102 }
103
105 template <typename T, typename... Ts>
106 MFInfo& SetTag (T&& t, Ts&&... ts) noexcept {
107 tags.emplace_back(std::forward<T>(t));
108 return SetTag(std::forward<Ts>(ts)...);
109 }
110};
111
113 using pointer = char*;
114 void operator()(pointer p) const noexcept {
116 }
117};
118using TheFaArenaPointer = std::unique_ptr<char, TheFaArenaDeleter>;
119
120#if defined(AMREX_USE_MPI) && defined(AMREX_USE_GPU)
121#define AMREX_FB_UNPACK_ON_ARRIVAL 1
122
125template <class FAB>
127 using value_type = typename FAB::value_type;
128
137
140 Vector<int> rcv_warp_begin; // first warp of each message in recv_tv, plus the total
141};
142#endif
143
144// Data used in non-blocking fill boundary.
145template <class FAB>
170
171// Data used in non-blocking parallel copy.
172template <class FAB>
192
193template <typename T>
195{
197 Array4<T> const& operator[] (int li) const noexcept {
198 AMREX_IF_ON_DEVICE((return dp[li];))
199 AMREX_IF_ON_HOST((return hp[li];))
200 }
201
203 explicit operator bool() const noexcept {
204 AMREX_IF_ON_DEVICE((return dp != nullptr;))
205 AMREX_IF_ON_HOST((return hp != nullptr;))
206 }
207
208#ifdef AMREX_USE_GPU
209 Array4<T> const* AMREX_RESTRICT dp = nullptr;
210#endif
211 Array4<T> const* AMREX_RESTRICT hp = nullptr;
212};
213
214template <class FAB> class FabArray;
215
216template <BaseFabType DFAB, BaseFabType SFAB>
217requires (std::is_convertible_v<typename SFAB::value_type, typename DFAB::value_type>)
218void
219Copy (FabArray<DFAB>& dst, FabArray<SFAB> const& src, int srccomp, int dstcomp, int numcomp, int nghost)
220{
221 Copy(dst,src,srccomp,dstcomp,numcomp,IntVect(nghost));
222}
223
224template <BaseFabType DFAB, BaseFabType SFAB>
225requires (std::is_convertible_v<typename SFAB::value_type, typename DFAB::value_type>)
226void
227Copy (FabArray<DFAB>& dst, FabArray<SFAB> const& src, int srccomp, int dstcomp, int numcomp, const IntVect& nghost)
228{
229 BL_PROFILE("amrex::Copy()");
230
231 using DT = typename DFAB::value_type;
232
233 if (dst.local_size() == 0) { return; }
234
235 // avoid self copy
236 if constexpr (std::is_same_v<typename SFAB::value_type, typename DFAB::value_type>) {
237 if (dst.atLocalIdx(0).dataPtr(dstcomp) == src.atLocalIdx(0).dataPtr(srccomp)) {
238 return;
239 }
240 }
241
242#ifdef AMREX_USE_GPU
243 if (Gpu::inLaunchRegion() && dst.isFusingCandidate()) {
244 auto const& srcarr = src.const_arrays();
245 auto const& dstarr = dst.arrays();
246 ParallelFor(dst, nghost, numcomp,
247 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
248 {
249 dstarr[box_no](i,j,k,dstcomp+n) = DT(srcarr[box_no](i,j,k,srccomp+n));
250 });
251 if (!Gpu::inNoSyncRegion()) {
253 }
254 } else
255#endif
256 {
257#ifdef AMREX_USE_OMP
258#pragma omp parallel if (Gpu::notInLaunchRegion())
259#endif
260 for (MFIter mfi(dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
261 {
262 const Box& bx = mfi.growntilebox(nghost);
263 if (bx.ok())
264 {
265 auto const& srcFab = src.const_array(mfi);
266 auto const& dstFab = dst.array(mfi);
267 AMREX_HOST_DEVICE_PARALLEL_FOR_4D( bx, numcomp, i, j, k, n,
268 {
269 dstFab(i,j,k,dstcomp+n) = DT(srcFab(i,j,k,srccomp+n));
270 });
271 }
272 }
273 }
274}
275
276template <BaseFabType FAB>
277void
278Add (FabArray<FAB>& dst, FabArray<FAB> const& src, int srccomp, int dstcomp, int numcomp, int nghost)
279{
280 Add(dst,src,srccomp,dstcomp,numcomp,IntVect(nghost));
281}
282
283template <BaseFabType FAB>
284void
285Add (FabArray<FAB>& dst, FabArray<FAB> const& src, int srccomp, int dstcomp, int numcomp, const IntVect& nghost)
286{
287 BL_PROFILE("amrex::Add()");
288
289#ifdef AMREX_USE_GPU
290 if (Gpu::inLaunchRegion() && dst.isFusingCandidate()) {
291 auto const& dstfa = dst.arrays();
292 auto const& srcfa = src.const_arrays();
293 ParallelFor(dst, nghost, numcomp,
294 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
295 {
296 dstfa[box_no](i,j,k,n+dstcomp) += srcfa[box_no](i,j,k,n+srccomp);
297 });
298 if (!Gpu::inNoSyncRegion()) {
300 }
301 } else
302#endif
303 {
304#ifdef AMREX_USE_OMP
305#pragma omp parallel if (Gpu::notInLaunchRegion())
306#endif
307 for (MFIter mfi(dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
308 {
309 const Box& bx = mfi.growntilebox(nghost);
310 if (bx.ok())
311 {
312 auto const srcFab = src.array(mfi);
313 auto dstFab = dst.array(mfi);
314 AMREX_HOST_DEVICE_PARALLEL_FOR_4D( bx, numcomp, i, j, k, n,
315 {
316 dstFab(i,j,k,n+dstcomp) += srcFab(i,j,k,n+srccomp);
317 });
318 }
319 }
320 }
321}
322
379template <class FAB>
381 :
382 public FabArrayBase
383{
384public:
385
386 struct FABType {
387 using value_type = FAB;
388 };
389
390 /*
391 * if FAB is a BaseFab or its child, value_type = FAB::value_type
392 * else value_type = FAB;
393 */
394 using value_type = typename std::conditional_t<IsBaseFab<FAB>::value, FAB, FABType>::value_type;
395
396 using fab_type = FAB;
397
398 //
400 FabArray () noexcept;
401
410 explicit FabArray (Arena* a) noexcept;
411
417 FabArray (const BoxArray& bxs,
418 const DistributionMapping& dm,
419 int nvar,
420 int ngrow,
421#ifdef AMREX_STRICT_MODE
422 const MFInfo& info,
423 const FabFactory<FAB>& factory);
424#else
425 const MFInfo& info = MFInfo(),
426 const FabFactory<FAB>& factory = DefaultFabFactory<FAB>());
427#endif
428
430 FabArray (const BoxArray& bxs,
431 const DistributionMapping& dm,
432 int nvar,
433 const IntVect& ngrow,
434#ifdef AMREX_STRICT_MODE
435 const MFInfo& info,
436 const FabFactory<FAB>& factory);
437#else
438 const MFInfo& info = MFInfo(),
439 const FabFactory<FAB>& factory = DefaultFabFactory<FAB>());
440#endif
441
450 FabArray (const FabArray<FAB>& rhs, MakeType maketype, int scomp, int ncomp);
451
454
455 FabArray (FabArray<FAB>&& rhs) noexcept;
457
458 FabArray (const FabArray<FAB>& rhs) = delete;
460
467 void define (const BoxArray& bxs,
468 const DistributionMapping& dm,
469 int nvar,
470 int ngrow,
471#ifdef AMREX_STRICT_MODE
472 const MFInfo& info,
473 const FabFactory<FAB>& factory);
474#else
475 const MFInfo& info = MFInfo(),
476 const FabFactory<FAB>& factory = DefaultFabFactory<FAB>());
477#endif
478
480 void define (const BoxArray& bxs,
481 const DistributionMapping& dm,
482 int nvar,
483 const IntVect& ngrow,
484#ifdef AMREX_STRICT_MODE
485 const MFInfo& info,
486 const FabFactory<FAB>& factory);
487#else
488 const MFInfo& info = MFInfo(),
489 const FabFactory<FAB>& factory = DefaultFabFactory<FAB>());
490#endif
491
499 const FabFactory<FAB>& Factory () const noexcept { return *m_factory; }
500
508 Arena* arena () const noexcept { return m_dallocator.arena(); }
509
516 const Vector<std::string>& tags () const noexcept { return m_tags; }
517
521 bool hasEBFabFactory () const noexcept {
522#ifdef AMREX_USE_EB
523 const auto *const f = dynamic_cast<EBFArrayBoxFactory const*>(m_factory.get());
524 return (f != nullptr);
525#else
526 return false;
527#endif
528 }
529
532 [[nodiscard]] value_type* singleChunkPtr () noexcept {
533 return m_single_chunk_arena ? (value_type*)m_single_chunk_arena->data() : nullptr;
534 }
535
538 [[nodiscard]] value_type const* singleChunkPtr () const noexcept {
539 return m_single_chunk_arena ? (value_type const*)m_single_chunk_arena->data() : nullptr;
540 }
541
544 [[nodiscard]] std::size_t singleChunkSize () const noexcept { return m_single_chunk_size; }
545
549 bool isAllRegular () const noexcept {
550#ifdef AMREX_USE_EB
551 const auto *const f = dynamic_cast<EBFArrayBoxFactory const*>(m_factory.get());
552 if (f) {
553 return f->isAllRegular();
554 } else {
555 return true;
556 }
557#else
558 return true;
559#endif
560 }
561
571 bool ok () const;
572
580 bool isDefined () const;
581
583 const FAB& operator[] (const MFIter& mfi) const noexcept { return *(this->fabPtr(mfi)); }
584
586 const FAB& get (const MFIter& mfi) const noexcept { return *(this->fabPtr(mfi)); }
587
589 FAB& operator[] (const MFIter& mfi) noexcept { return *(this->fabPtr(mfi)); }
590
592 FAB& get (const MFIter& mfi) noexcept { return *(this->fabPtr(mfi)); }
593
595 const FAB& operator[] (int K) const noexcept { return *(this->fabPtr(K)); }
596
598 const FAB& get (int K) const noexcept { return *(this->fabPtr(K)); }
599
601 FAB& operator[] (int K) noexcept { return *(this->fabPtr(K)); }
602
604 FAB& get (int K) noexcept { return *(this->fabPtr(K)); }
605
607 FAB& atLocalIdx (int L) noexcept { return *m_fabs_v[L]; }
608 const FAB& atLocalIdx (int L) const noexcept { return *m_fabs_v[L]; }
609
611 FAB * fabPtr (const MFIter& mfi) noexcept;
612 FAB const* fabPtr (const MFIter& mfi) const noexcept;
613 FAB * fabPtr (int K) noexcept; // Here K is global index
614 FAB const* fabPtr (int K) const noexcept;
615
620 void prefetchToHost (const MFIter& mfi) const noexcept
621 requires (BaseFabType<FAB>)
622 {
623 // BaseFab::prefetchToHost() does the managed-memory check and the
624 // per-backend dispatch, so guard only against non-GPU builds here.
625#ifdef AMREX_USE_GPU
626 this->fabPtr(mfi)->prefetchToHost();
627#else
629#endif
630 }
631
636 void prefetchToDevice (const MFIter& mfi) const noexcept
637 requires (BaseFabType<FAB>)
638 {
639 // BaseFab::prefetchToDevice() does the managed-memory check and the
640 // per-backend dispatch, so guard only against non-GPU builds here.
641#ifdef AMREX_USE_GPU
642 this->fabPtr(mfi)->prefetchToDevice();
643#else
645#endif
646 }
647
649 Array4<typename FabArray<FAB>::value_type const> array (const MFIter& mfi) const noexcept
650 requires (BaseFabType<FAB>)
651 {
652 return fabPtr(mfi)->const_array();
653 }
654
657 requires (BaseFabType<FAB>)
658 {
659 return fabPtr(mfi)->array();
660 }
661
664 requires (BaseFabType<FAB>)
665 {
666 return fabPtr(K)->const_array();
667 }
668
671 requires (BaseFabType<FAB>)
672 {
673 return fabPtr(K)->array();
674 }
675
678 requires (BaseFabType<FAB>)
679 {
680 return fabPtr(mfi)->const_array();
681 }
682
685 requires (BaseFabType<FAB>)
686 {
687 return fabPtr(K)->const_array();
688 }
689
692 Array4<typename FabArray<FAB>::value_type const> array (const MFIter& mfi, int start_comp) const noexcept
693 requires (BaseFabType<FAB>)
694 {
695 return fabPtr(mfi)->const_array(start_comp);
696 }
697
700 Array4<typename FabArray<FAB>::value_type> array (const MFIter& mfi, int start_comp) noexcept
701 requires (BaseFabType<FAB>)
702 {
703 return fabPtr(mfi)->array(start_comp);
704 }
705
708 Array4<typename FabArray<FAB>::value_type const> array (int K, int start_comp) const noexcept
709 requires (BaseFabType<FAB>)
710 {
711 return fabPtr(K)->const_array(start_comp);
712 }
713
716 Array4<typename FabArray<FAB>::value_type> array (int K, int start_comp) noexcept
717 requires (BaseFabType<FAB>)
718 {
719 return fabPtr(K)->array(start_comp);
720 }
721
724 Array4<typename FabArray<FAB>::value_type const> const_array (const MFIter& mfi, int start_comp) const noexcept
725 requires (BaseFabType<FAB>)
726 {
727 return fabPtr(mfi)->const_array(start_comp);
728 }
729
732 Array4<typename FabArray<FAB>::value_type const> const_array (int K, int start_comp) const noexcept
733 requires (BaseFabType<FAB>)
734 {
735 return fabPtr(K)->const_array(start_comp);
736 }
737
742 requires (BaseFabType<FAB>)
743 {
744 build_arrays();
745 return m_arrays;
746 }
747
752 requires (BaseFabType<FAB>)
753 {
754 build_arrays();
755 return m_const_arrays;
756 }
757
760 requires (BaseFabType<FAB>)
761 {
762 build_arrays();
763 return m_const_arrays;
764 }
765
767 void setFab (int boxno, std::unique_ptr<FAB> elem);
768
770 template <class F=FAB>
771 requires (std::is_move_constructible_v<F>)
772 void setFab (int boxno, FAB&& elem);
773
775 void setFab (const MFIter&mfi, std::unique_ptr<FAB> elem);
776
778 template <class F=FAB>
779 requires (std::is_move_constructible_v<F>)
780 void setFab (const MFIter&mfi, FAB&& elem);
781
783 [[nodiscard]]
784 FAB* release (int K);
785
787 [[nodiscard]]
788 FAB* release (const MFIter& mfi);
789
791 void clear ();
792
807 template <BaseFabType SFAB, BaseFabType DFAB = FAB>
808 requires (std::is_convertible_v<typename SFAB::value_type,
809 typename DFAB::value_type>)
810 void LocalCopy (FabArray<SFAB> const& src, int scomp, int dcomp, int ncomp,
811 IntVect const& nghost);
812
825 template <class F=FAB>
826 requires (BaseFabType<F>)
827 void LocalAdd (FabArray<FAB> const& src, int scomp, int dcomp, int ncomp,
828 IntVect const& nghost);
829
831 template <class F=FAB>
832 requires (BaseFabType<F>)
833 void setVal (value_type val);
834
836 template <class F=FAB>
837 requires (BaseFabType<F>)
838 FabArray<FAB>& operator= (value_type val);
839
845 template <class F=FAB>
846 requires (BaseFabType<F>)
848 int comp,
849 int ncomp,
850 int nghost = 0);
851
852 template <class F=FAB>
853 requires (BaseFabType<F>)
855 int comp,
856 int ncomp,
857 const IntVect& nghost);
858
865 template <class F=FAB>
866 requires (BaseFabType<F>)
868 const Box& region,
869 int comp,
870 int ncomp,
871 int nghost = 0);
872
873 template <class F=FAB>
874 requires (BaseFabType<F>)
876 const Box& region,
877 int comp,
878 int ncomp,
879 const IntVect& nghost);
884 template <class F=FAB>
885 requires (BaseFabType<F>)
886 void setVal (value_type val, int nghost);
887
888 template <class F=FAB>
889 requires (BaseFabType<F>)
890 void setVal (value_type val, const IntVect& nghost);
891
897 template <class F=FAB>
898 requires (BaseFabType<F>)
899 void setVal (value_type val, const Box& region, int nghost);
900
901 template <class F=FAB>
902 requires (BaseFabType<F>)
903 void setVal (value_type val, const Box& region, const IntVect& nghost);
904
905 template <class F=FAB>
906 requires (BaseFabType<F>)
907 void abs (int comp, int ncomp, int nghost = 0);
908
909 template <class F=FAB>
910 requires (BaseFabType<F>)
911 void abs (int comp, int ncomp, const IntVect& nghost);
912
913 template <class F=FAB>
914 requires (BaseFabType<F>)
915 void plus (value_type val, int comp, int num_comp, int nghost = 0);
916
917 template <class F=FAB>
918 requires (BaseFabType<F>)
919 void plus (value_type val, const Box& region, int comp, int num_comp, int nghost = 0);
920
921 template <class F=FAB>
922 requires (BaseFabType<F>)
923 void mult (value_type val, int comp, int num_comp, int nghost = 0);
924
925 template <class F=FAB>
926 requires (BaseFabType<F>)
927 void mult (value_type val, const Box& region, int comp, int num_comp, int nghost = 0);
928
929 template <class F=FAB>
930 requires (BaseFabType<F>)
931 void invert (value_type numerator, int comp, int num_comp, int nghost = 0);
932
933 template <class F=FAB>
934 requires (BaseFabType<F>)
935 void invert (value_type numerator, const Box& region, int comp, int num_comp, int nghost = 0);
936
938 template <class F=FAB>
939 requires (BaseFabType<F>)
941
943 template <class F=FAB>
944 requires (BaseFabType<F>)
945 void setBndry (value_type val, int strt_comp, int ncomp);
946
952 template <class F=FAB>
953 requires (BaseFabType<F>)
954 void setDomainBndry (value_type val, const Geometry& geom);
955
962 template <class F=FAB>
963 requires (BaseFabType<F>)
964 void setDomainBndry (value_type val, int strt_comp, int ncomp, const Geometry& geom);
965
972 template <std::integral Int>
973 requires (BaseFabType<FAB> && (sizeof(Int) >= sizeof(Long)))
975
985 template <typename F=FAB>
986 requires (BaseFabType<F>)
987 typename F::value_type
988 sum (int comp, IntVect const& nghost, bool local = false) const;
989
996 void ParallelAdd (const FabArray<FAB>& src,
997 const Periodicity& period = Periodicity::NonPeriodic())
998 { ParallelCopy(src,period,FabArray::ADD); }
999 void ParallelCopy (const FabArray<FAB>& src,
1000 const Periodicity& period = Periodicity::NonPeriodic(),
1002 { ParallelCopy(src,0,0,nComp(),0,0,period,op); }
1003
1004 [[deprecated("Use FabArray::ParallelCopy() instead.")]]
1005 void copy (const FabArray<FAB>& src,
1006 const Periodicity& period = Periodicity::NonPeriodic(),
1008 { ParallelCopy(src,period,op); }
1009
1019 const Periodicity& period = Periodicity::NonPeriodic())
1020 { ParallelCopy_nowait(src,period,FabArray::ADD); }
1021
1032 const Periodicity& period = Periodicity::NonPeriodic(),
1034 { ParallelCopy_nowait(src,0,0,nComp(),0,0,period,op); }
1035
1044 void ParallelAdd (const FabArray<FAB>& src,
1045 int src_comp,
1046 int dest_comp,
1047 int num_comp,
1048 const Periodicity& period = Periodicity::NonPeriodic())
1049 { ParallelCopy(src,src_comp,dest_comp,num_comp, period, FabArrayBase::ADD); }
1050 void ParallelCopy (const FabArray<FAB>& src,
1051 int src_comp,
1052 int dest_comp,
1053 int num_comp,
1054 const Periodicity& period = Periodicity::NonPeriodic(),
1056 { ParallelCopy(src,src_comp,dest_comp,num_comp,0,0,period,op); }
1057
1058 [[deprecated("Use FabArray::ParallelCopy() instead.")]]
1059 void copy (const FabArray<FAB>& src,
1060 int src_comp,
1061 int dest_comp,
1062 int num_comp,
1063 const Periodicity& period = Periodicity::NonPeriodic(),
1065 { ParallelCopy(src,src_comp,dest_comp,num_comp, period, op); }
1066
1079 int src_comp,
1080 int dest_comp,
1081 int num_comp,
1082 const Periodicity& period = Periodicity::NonPeriodic())
1083 { ParallelCopy_nowait(src,src_comp,dest_comp,num_comp, period, FabArrayBase::ADD); }
1084
1098 int src_comp,
1099 int dest_comp,
1100 int num_comp,
1101 const Periodicity& period = Periodicity::NonPeriodic(),
1103 { ParallelCopy_nowait(src,src_comp,dest_comp,num_comp,0,0,period,op); }
1104
1106 void ParallelAdd (const FabArray<FAB>& src,
1107 int src_comp,
1108 int dest_comp,
1109 int num_comp,
1110 int src_nghost,
1111 int dst_nghost,
1112 const Periodicity& period = Periodicity::NonPeriodic())
1113 { ParallelCopy(src,src_comp,dest_comp,num_comp,IntVect(src_nghost),IntVect(dst_nghost),period,
1115 void ParallelAdd (const FabArray<FAB>& src,
1116 int src_comp,
1117 int dest_comp,
1118 int num_comp,
1119 const IntVect& src_nghost,
1120 const IntVect& dst_nghost,
1121 const Periodicity& period = Periodicity::NonPeriodic())
1122 { ParallelCopy(src,src_comp,dest_comp,num_comp,src_nghost,dst_nghost,period,FabArrayBase::ADD); }
1123 void ParallelCopy (const FabArray<FAB>& src,
1124 int src_comp,
1125 int dest_comp,
1126 int num_comp,
1127 int src_nghost,
1128 int dst_nghost,
1129 const Periodicity& period = Periodicity::NonPeriodic(),
1131 { ParallelCopy(src,src_comp,dest_comp,num_comp,IntVect(src_nghost),IntVect(dst_nghost),period,op); }
1132 void ParallelCopy (const FabArray<FAB>& src,
1133 int scomp,
1134 int dcomp,
1135 int ncomp,
1136 const IntVect& snghost,
1137 const IntVect& dnghost,
1138 const Periodicity& period = Periodicity::NonPeriodic(),
1140 const FabArrayBase::CPC* a_cpc = nullptr,
1141 bool deterministic = false);
1142
1157 int src_comp,
1158 int dest_comp,
1159 int num_comp,
1160 int src_nghost,
1161 int dst_nghost,
1162 const Periodicity& period = Periodicity::NonPeriodic())
1163 { ParallelCopy_nowait(src,src_comp,dest_comp,num_comp,IntVect(src_nghost),
1164 IntVect(dst_nghost),period,FabArrayBase::ADD); }
1165
1180 int src_comp,
1181 int dest_comp,
1182 int num_comp,
1183 const IntVect& src_nghost,
1184 const IntVect& dst_nghost,
1185 const Periodicity& period = Periodicity::NonPeriodic())
1186 { ParallelCopy_nowait(src,src_comp,dest_comp,num_comp,src_nghost,
1187 dst_nghost,period,FabArrayBase::ADD); }
1188
1204 int src_comp,
1205 int dest_comp,
1206 int num_comp,
1207 int src_nghost,
1208 int dst_nghost,
1209 const Periodicity& period = Periodicity::NonPeriodic(),
1211 { ParallelCopy_nowait(src,src_comp,dest_comp,num_comp,IntVect(src_nghost),
1212 IntVect(dst_nghost),period,op); }
1213
1232 int scomp,
1233 int dcomp,
1234 int ncomp,
1235 const IntVect& snghost,
1236 const IntVect& dnghost,
1237 const Periodicity& period = Periodicity::NonPeriodic(),
1239 const FabArrayBase::CPC* a_cpc = nullptr,
1240 bool to_ghost_cells_only = false,
1241 bool deterministic = false);
1242
1264 int scomp,
1265 int dcomp,
1266 int ncomp,
1267 const IntVect& snghost,
1268 const IntVect& dnghost,
1269 const IntVect& offset,
1270 const Periodicity& period = Periodicity::NonPeriodic(),
1272 const FabArrayBase::CPC* a_cpc = nullptr,
1273 bool to_ghost_cells_only = false,
1274 bool deterministic = false);
1275
1280
1293 int scomp,
1294 int dcomp,
1295 int ncomp,
1296 const IntVect& snghost,
1297 const IntVect& dnghost,
1298 const Periodicity& period = Periodicity::NonPeriodic());
1299
1314 int scomp,
1315 int dcomp,
1316 int ncomp,
1317 const IntVect& snghost,
1318 const IntVect& dnghost,
1319 const Periodicity& period = Periodicity::NonPeriodic());
1320
1323
1336 void ParallelCopy (const FabArray<FAB>& src, int src_comp, int dest_comp, int num_comp,
1337 const IntVect& snghost, const IntVect& dnghost,
1338 const IntVect& offset, const Periodicity& period);
1339
1352 void ParallelAdd (const FabArray<FAB>& src, int src_comp, int dest_comp, int num_comp,
1353 const IntVect& snghost, const IntVect& dnghost,
1354 const IntVect& offset, const Periodicity& period);
1355
1356 [[deprecated("Use FabArray::ParallelCopy() instead.")]]
1357 void copy (const FabArray<FAB>& src,
1358 int src_comp,
1359 int dest_comp,
1360 int num_comp,
1361 int src_nghost,
1362 int dst_nghost,
1363 const Periodicity& period = Periodicity::NonPeriodic(),
1365 { ParallelCopy(src,src_comp,dest_comp,num_comp,IntVect(src_nghost),IntVect(dst_nghost),period,op); }
1366
1367 [[deprecated("Use FabArray::ParallelCopy() instead.")]]
1368 void copy (const FabArray<FAB>& src,
1369 int src_comp,
1370 int dest_comp,
1371 int num_comp,
1372 const IntVect& src_nghost,
1373 const IntVect& dst_nghost,
1374 const Periodicity& period = Periodicity::NonPeriodic(),
1376 { ParallelCopy(src,src_comp,dest_comp,num_comp,src_nghost,dst_nghost,period,op); }
1377
1379 void Redistribute (const FabArray<FAB>& src,
1380 int scomp,
1381 int dcomp,
1382 int ncomp,
1383 const IntVect& nghost);
1384
1390 void copyTo (FAB& dest, int nghost = 0) const;
1391
1399 void copyTo (FAB& dest, int scomp, int dcomp, int ncomp, int nghost = 0) const;
1400
1402 void shift (const IntVect& v);
1403
1404 bool defined (int K) const noexcept;
1405 bool defined (const MFIter& mfi) const noexcept;
1406
1419 template <typename BUF=value_type>
1420 void FillBoundary (bool cross = false);
1421
1423 template <typename BUF=value_type>
1424 void FillBoundary (const Periodicity& period, bool cross = false);
1425
1426 template <typename BUF=value_type>
1427 void FillBoundary (const IntVect& nghost, const Periodicity& period, bool cross = false);
1428
1431 template <typename BUF=value_type>
1432 void FillBoundary (int scomp, int ncomp, bool cross = false);
1433
1435 template <typename BUF=value_type>
1436 void FillBoundary (int scomp, int ncomp, const Periodicity& period, bool cross = false);
1437
1439 template <typename BUF=value_type>
1440 void FillBoundary (int scomp, int ncomp, const IntVect& nghost, const Periodicity& period, bool cross = false);
1441
1442 template <typename BUF=value_type>
1443 void FillBoundary_nowait (bool cross = false);
1444
1445 template <typename BUF=value_type>
1446 void FillBoundary_nowait (const Periodicity& period, bool cross = false);
1447
1448 template <typename BUF=value_type>
1449 void FillBoundary_nowait (const IntVect& nghost, const Periodicity& period, bool cross = false);
1450
1451 template <typename BUF=value_type>
1452 void FillBoundary_nowait (int scomp, int ncomp, bool cross = false);
1453
1454 template <typename BUF=value_type>
1455 void FillBoundary_nowait (int scomp, int ncomp, const Periodicity& period, bool cross = false);
1456
1457 template <typename BUF=value_type>
1458 void FillBoundary_nowait (int scomp, int ncomp, const IntVect& nghost, const Periodicity& period, bool cross = false);
1459
1460 template <typename BUF=value_type, class F=FAB>
1461 requires (BaseFabType<F>)
1463
1465
1496 void FillBoundaryAndSync (int scomp, int ncomp, const IntVect& nghost,
1497 const Periodicity& period);
1499 void FillBoundaryAndSync_nowait (int scomp, int ncomp, const IntVect& nghost,
1500 const Periodicity& period);
1502
1539 void OverrideSync (int scomp, int ncomp, const Periodicity& period);
1549 void OverrideSync_nowait (int scomp, int ncomp, const Periodicity& period);
1551
1563 bool deterministic = false);
1565 void SumBoundary (int scomp, int ncomp, const Periodicity& period = Periodicity::NonPeriodic(),
1566 bool deterministic = false);
1568 bool deterministic = false);
1569 void SumBoundary_nowait (int scomp, int ncomp, const Periodicity& period = Periodicity::NonPeriodic(),
1570 bool deterministic = false);
1571
1582 void SumBoundary (int scomp, int ncomp, IntVect const& nghost,
1583 const Periodicity& period = Periodicity::NonPeriodic(),
1584 bool deterministic = false);
1585 void SumBoundary_nowait (int scomp, int ncomp, IntVect const& nghost,
1586 const Periodicity& period = Periodicity::NonPeriodic(),
1587 bool deterministic = false);
1588
1600 void SumBoundary (int scomp, int ncomp, IntVect const& src_nghost, IntVect const& dst_nghost,
1601 const Periodicity& period = Periodicity::NonPeriodic(),
1602 bool deterministic = false);
1603 void SumBoundary_nowait (int scomp, int ncomp, IntVect const& src_nghost, IntVect const& dst_nghost,
1604 const Periodicity& period = Periodicity::NonPeriodic(),
1605 bool deterministic = false);
1607
1618 void EnforcePeriodicity (const Periodicity& period);
1620 void EnforcePeriodicity (int scomp, int ncomp, const Periodicity& period);
1622 void EnforcePeriodicity (int scomp, int ncomp, const IntVect& nghost,
1623 const Periodicity& period);
1624
1625 // covered : ghost cells covered by valid cells of this FabArray
1626 // (including periodically shifted valid cells)
1627 // notcovered: ghost cells not covered by valid cells
1628 // (including ghost cells outside periodic boundaries)
1629 // physbnd : boundary cells outside the domain (excluding periodic boundaries)
1630 // interior : interior cells (i.e., valid cells)
1631 template <class F=FAB>
1632 requires (BaseFabType<F>)
1633 void BuildMask (const Box& phys_domain, const Periodicity& period,
1634 value_type covered, value_type notcovered,
1635 value_type physbnd, value_type interior);
1636
1637 // The following are private functions. But we have to make them public for cuda.
1638
1639 template <typename BUF=value_type, class F=FAB>
1640 requires (BaseFabType<F>)
1641 void FBEP_nowait (int scomp, int ncomp, const IntVect& nghost,
1642 const Periodicity& period, bool cross,
1643 bool enforce_periodicity_only = false,
1644 bool override_sync = false,
1645 IntVect const& sumboundary_src_nghost = IntVect(-1),
1646 bool deterministic = false);
1647
1648 void FB_local_copy_cpu (const FB& TheFB, int scomp, int ncomp);
1649 void FB_local_add_cpu (const FB& TheFB, int scomp, int ncomp);
1650 void PC_local_cpu (const CPC& thecpc, FabArray<FAB> const& src,
1651 int scomp, int dcomp, int ncomp, CpOp op);
1652
1653 template <class F=FAB>
1654 requires (BaseFabType<F>)
1655 void setVal (value_type val, const CommMetaData& thecmd, int scomp, int ncomp);
1656
1657 template <class F=FAB>
1658 requires (BaseFabType<F>)
1660
1661#ifdef AMREX_USE_GPU
1662
1665
1666 void FB_local_copy_gpu (const FB& TheFB, int scomp, int ncomp);
1667 void FB_local_add_gpu (const FB& TheFB, int scomp, int ncomp, bool deterministic);
1668 void PC_local_gpu (const CPC& thecpc, FabArray<FAB> const& src,
1669 int scomp, int dcomp, int ncomp, CpOp op, bool deterministic);
1670
1671 void CMD_local_setVal_gpu (value_type x, const CommMetaData& thecmd, int scomp, int ncomp);
1672 void CMD_remote_setVal_gpu (value_type x, const CommMetaData& thecmd, int scomp, int ncomp);
1673
1674#if defined(__CUDACC__)
1675
1676 void FB_local_copy_cuda_graph_1 (const FB& TheFB, int scomp, int ncomp);
1677 void FB_local_copy_cuda_graph_n (const FB& TheFB, int scomp, int ncomp);
1678
1679#endif
1680#endif
1681
1682#ifdef AMREX_USE_MPI
1683
1684#ifdef AMREX_USE_GPU
1685#if defined(__CUDACC__)
1686
1687 void FB_pack_send_buffer_cuda_graph (const FB& TheFB, int scomp, int ncomp,
1688 Vector<char*>& send_data,
1689 Vector<std::size_t> const& send_size,
1690 Vector<const CopyComTagsContainer*> const& send_cctc);
1691
1692 void FB_unpack_recv_buffer_cuda_graph (const FB& TheFB, int dcomp, int ncomp,
1693 Vector<char*> const& recv_data,
1694 Vector<std::size_t> const& recv_size,
1695 Vector<const CopyComTagsContainer*> const& recv_cctc,
1696 bool is_thread_safe);
1697
1698#endif
1699
1700 template <typename BUF = value_type>
1701 static void pack_send_buffer_gpu (FabArray<FAB> const& src, int scomp, int ncomp,
1702 Vector<char*> const& send_data,
1703 Vector<std::size_t> const& send_size,
1704 Vector<const CopyComTagsContainer*> const& send_cctc,
1705 std::uint64_t id);
1706
1715 template <typename BUF = value_type>
1716 static void unpack_recv_buffer_gpu (FabArray<FAB>& dst, int dcomp, int ncomp,
1717 Vector<char*> const& recv_data,
1718 Vector<std::size_t> const& recv_size,
1719 Vector<const CopyComTagsContainer*> const& recv_cctc,
1720 CpOp op, bool is_thread_safe,
1721 std::uint64_t id, bool deterministic);
1722
1723 template <typename BUF>
1726 Vector<std::size_t> const& recv_size,
1727 Vector<CopyComTagsContainer const*> const& recv_cctc,
1728 int ncomp, std::uint64_t id);
1729
1730 template <typename BUF>
1733 Vector<std::size_t> const& send_size,
1734 Vector<CopyComTagsContainer const*> const& send_cctc,
1735 int ncomp, std::uint64_t id) const;
1736
1737#ifdef AMREX_FB_UNPACK_ON_ARRIVAL
1738 template <typename BUF>
1739 FBOnArrivalMeta<FAB>* FB_on_arrival_meta (const FB& TheFB, int ncomp);
1740
1741 template <typename BUF>
1742 void FB_on_arrival_nowait (const FB& TheFB, int scomp, int ncomp, int SeqNum,
1743 bool sumboundary);
1744
1745 template <typename BUF>
1747#endif
1748
1749#endif
1750
1751 template <typename BUF = value_type>
1752 static void pack_send_buffer_cpu (FabArray<FAB> const& src, int scomp, int ncomp,
1753 Vector<char*> const& send_data,
1754 Vector<std::size_t> const& send_size,
1755 Vector<const CopyComTagsContainer*> const& send_cctc);
1756
1757 template <typename BUF = value_type>
1758 static void unpack_recv_buffer_cpu (FabArray<FAB>& dst, int dcomp, int ncomp,
1759 Vector<char*> const& recv_data,
1760 Vector<std::size_t> const& recv_size,
1761 Vector<const CopyComTagsContainer*> const& recv_cctc,
1762 CpOp op, bool is_thread_safe);
1763
1764#endif
1765
1778 template <typename F=FAB>
1779 requires (BaseFabType<F>)
1780 typename F::value_type
1781 norminf (int comp, int ncomp, IntVect const& nghost, bool local = false,
1782 [[maybe_unused]] bool ignore_covered = false) const;
1783
1796 template <typename IFAB, typename F=FAB>
1797 requires (BaseFabType<F>)
1798 typename F::value_type
1799 norminf (FabArray<IFAB> const& mask, int comp, int ncomp, IntVect const& nghost,
1800 bool local = false) const;
1801
1802protected:
1803
1804 std::unique_ptr<FabFactory<FAB> > m_factory;
1808 std::unique_ptr<detail::SingleChunkArena> m_single_chunk_arena;
1810
1813
1816
1817 //
1819 std::vector<FAB*> m_fabs_v;
1820
1821#ifdef AMREX_USE_GPU
1822 mutable void* m_dp_arrays = nullptr;
1823#endif
1824 mutable void* m_hp_arrays = nullptr;
1827
1829
1831 struct ShMem {
1832
1833 ShMem () noexcept = default;
1834
1835 ~ShMem () { // NOLINT
1836#if defined(BL_USE_MPI3)
1837 if (win != MPI_WIN_NULL) { MPI_Win_free(&win); }
1838#endif
1839#ifdef BL_USE_TEAM
1840 if (alloc) {
1842 }
1843#endif
1844 }
1845 ShMem (ShMem&& rhs) noexcept
1846 : alloc(rhs.alloc), n_values(rhs.n_values), n_points(rhs.n_points)
1847#if defined(BL_USE_MPI3)
1848 , win(rhs.win)
1849#endif
1850 {
1851 rhs.alloc = false;
1852#if defined(BL_USE_MPI3)
1853 rhs.win = MPI_WIN_NULL;
1854#endif
1855 }
1856 ShMem& operator= (ShMem&& rhs) noexcept {
1857 if (&rhs != this) {
1858 alloc = rhs.alloc;
1859 n_values = rhs.n_values;
1860 n_points = rhs.n_points;
1861 rhs.alloc = false;
1862#if defined(BL_USE_MPI3)
1863 win = rhs.win;
1864 rhs.win = MPI_WIN_NULL;
1865#endif
1866 }
1867 return *this;
1868 }
1869 ShMem (const ShMem&) = delete;
1870 ShMem& operator= (const ShMem&) = delete;
1871 bool alloc{false};
1874#if defined(BL_USE_MPI3)
1875 MPI_Win win = MPI_WIN_NULL;
1876#endif
1877 };
1878 ShMem shmem;
1879
1880 bool SharedMemory () const noexcept { return shmem.alloc; }
1881
1882private:
1883 using Iterator = typename std::vector<FAB*>::iterator;
1884
1885 void AllocFabs (const FabFactory<FAB>& factory, Arena* ar,
1887 bool alloc_single_chunk);
1888
1889 void setFab_assert (int K, FAB const& fab) const;
1890
1891 template <class F=FAB>
1892 requires (BaseFabType<F>)
1893 void build_arrays () const;
1894
1895 void clear_fab_caches ();
1896
1897#ifdef AMREX_USE_GPU
1898 std::map<std::uint64_t,
1899 std::unique_ptr<TagVector<Array4CopyTag<value_type>>>>
1900 m_fb_local_copy_handler;
1901
1902 using RecvSendCopyHandlerKey = std::tuple<std::uint64_t,std::size_t,int>;
1903 std::map<RecvSendCopyHandlerKey,
1904 std::unique_ptr<TagVector<CommRecvBufTag<value_type>>>>
1905 m_recv_copy_handler;
1906 mutable std::map<RecvSendCopyHandlerKey,
1907 std::unique_ptr<TagVector<CommSendBufTag<value_type>>>>
1908 m_send_copy_handler;
1909#endif
1910#ifdef AMREX_FB_UNPACK_ON_ARRIVAL
1911 std::map<RecvSendCopyHandlerKey, std::unique_ptr<FBOnArrivalMeta<FAB>>> m_fb_on_arrival_meta;
1912#endif
1913
1914public:
1915
1916#ifdef BL_USE_MPI
1917
1919 template <typename BUF=value_type>
1920 static void PostRcvs (const MapOfCopyComTagContainers& RcvTags,
1921 char*& the_recv_data,
1922 Vector<char*>& recv_data,
1923 Vector<std::size_t>& recv_size,
1924 Vector<int>& recv_from,
1925 Vector<MPI_Request>& recv_reqs,
1926 int ncomp,
1927 int SeqNum);
1928
1929 template <typename BUF=value_type>
1930 [[nodiscard]]
1931 static TheFaArenaPointer PostRcvs (const MapOfCopyComTagContainers& RcvTags,
1932 Vector<char*>& recv_data,
1933 Vector<std::size_t>& recv_size,
1934 Vector<int>& recv_from,
1935 Vector<MPI_Request>& recv_reqs,
1936 int ncomp,
1937 int SeqNum);
1938
1939 template <typename BUF=value_type>
1940 static void PrepareSendBuffers (const MapOfCopyComTagContainers& SndTags,
1941 char*& the_send_data,
1942 Vector<char*>& send_data,
1943 Vector<std::size_t>& send_size,
1944 Vector<int>& send_rank,
1945 Vector<MPI_Request>& send_reqs,
1947 int ncomp);
1948
1949 template <typename BUF=value_type>
1950 [[nodiscard]]
1951 static TheFaArenaPointer PrepareSendBuffers (const MapOfCopyComTagContainers& SndTags,
1952 Vector<char*>& send_data,
1953 Vector<std::size_t>& send_size,
1954 Vector<int>& send_rank,
1955 Vector<MPI_Request>& send_reqs,
1957 int ncomp);
1958
1959 static void PostSnds (Vector<char*> const& send_data,
1960 Vector<std::size_t> const& send_size,
1961 Vector<int> const& send_rank,
1962 Vector<MPI_Request>& send_reqs,
1963 int SeqNum);
1964#endif
1965
1966 std::unique_ptr<FBData<FAB>> fbd;
1967 std::unique_ptr<PCData<FAB>> pcd;
1968
1969 // Pointer to temporary fab used in non-blocking amrex::OverrideSync
1970 std::unique_ptr< FabArray<FAB> > os_temp;
1971
1983 template <class F=FAB>
1984 requires (BaseFabType<F>)
1985 static void Saxpy (FabArray<FAB>& y, value_type a, FabArray<FAB> const& x,
1986 int xcomp, int ycomp, int ncomp, IntVect const& nghost);
1987
1999 template <class F=FAB>
2000 requires (BaseFabType<F>)
2001 static void Xpay (FabArray<FAB>& y, value_type a, FabArray<FAB> const& x,
2002 int xcomp, int ycomp, int ncomp, IntVect const& nghost);
2003
2018 template <class F=FAB>
2019 requires (BaseFabType<F>)
2020 static void LinComb (FabArray<FAB>& dst,
2021 value_type a, const FabArray<FAB>& x, int xcomp,
2022 value_type b, const FabArray<FAB>& y, int ycomp,
2023 int dstcomp, int numcomp, const IntVect& nghost);
2024
2038 template <class F=FAB>
2039 requires (BaseFabType<F>)
2040 static void Saxpy_Xpay (FabArray<FAB>& y, value_type a1, FabArray<FAB> const& x1,
2041 value_type a2, FabArray<FAB> const& x2,
2042 int xcomp, int ycomp, int ncomp, IntVect const& nghost);
2043
2058 template <class F=FAB>
2059 requires (BaseFabType<F>)
2060 static void Saxpy_Saxpy (FabArray<FAB>& y1, value_type a1, FabArray<FAB> const& x1,
2061 FabArray<FAB>& y2, value_type a2, FabArray<FAB> const& x2,
2062 int xcomp, int ycomp, int ncomp, IntVect const& nghost);
2063
2077 template <class F=FAB>
2078 requires (BaseFabType<F>)
2080 FabArray<FAB>& y2, value_type a2, FabArray<FAB> const& x,
2081 int xcomp, int ycomp, int ncomp, IntVect const& nghost);
2082};
2083
2084}
2085
2086#include <AMReX_FabArrayCommI.H>
2087
2088namespace amrex {
2089
2090template <class FAB>
2091bool
2092FabArray<FAB>::defined (int K) const noexcept
2093{
2094 int li = localindex(K);
2095 if (li >= 0 && li < std::ssize(m_fabs_v) && m_fabs_v[li] != nullptr) {
2096 return true;
2097 }
2098 else {
2099 return false;
2100 }
2101}
2102
2103template <class FAB>
2104bool
2105FabArray<FAB>::defined (const MFIter& mfi) const noexcept
2106{
2107 int li = mfi.LocalIndex();
2108 if (li < std::ssize(m_fabs_v) && m_fabs_v[li] != nullptr) {
2109 return true;
2110 }
2111 else {
2112 return false;
2113 }
2114}
2115
2116template <class FAB>
2117FAB*
2118FabArray<FAB>::fabPtr (const MFIter& mfi) noexcept
2119{
2120 AMREX_ASSERT(mfi.LocalIndex() < indexArray.size());
2122 int li = mfi.LocalIndex();
2123 return m_fabs_v[li];
2124}
2125
2126template <class FAB>
2127FAB const*
2128FabArray<FAB>::fabPtr (const MFIter& mfi) const noexcept
2129{
2130 AMREX_ASSERT(mfi.LocalIndex() < indexArray.size());
2132 int li = mfi.LocalIndex();
2133 return m_fabs_v[li];
2134}
2135
2136template <class FAB>
2137FAB*
2139{
2140 int li = localindex(K);
2141 AMREX_ASSERT(li >=0 && li < indexArray.size());
2142 return m_fabs_v[li];
2143}
2144
2145template <class FAB>
2146FAB const*
2147FabArray<FAB>::fabPtr (int K) const noexcept
2148{
2149 int li = localindex(K);
2150 AMREX_ASSERT(li >=0 && li < indexArray.size());
2151 return m_fabs_v[li];
2152}
2153
2154template <class FAB>
2155template <class F>
2156requires (BaseFabType<F>)
2157void
2158FabArray<FAB>::build_arrays () const
2159{
2160 using A = Array4<value_type>;
2161 using AC = Array4<value_type const>;
2162 static_assert(sizeof(A) == sizeof(AC), "sizeof(Array4<T>) != sizeof(Array4<T const>)");
2163 if (!m_hp_arrays && local_size() > 0) {
2164 const int n = local_size();
2165#ifdef AMREX_USE_GPU
2166 m_hp_arrays = (void*)The_Pinned_Arena()->alloc(n*2*sizeof(A));
2167 m_dp_arrays = (void*)The_Arena()->alloc(n*2*sizeof(A));
2168#else
2169 m_hp_arrays = std::malloc(n*2*sizeof(A));
2170#endif
2171 for (int li = 0; li < n; ++li) {
2172 if (m_fabs_v[li]) {
2173 new ((A*)m_hp_arrays+li) A(m_fabs_v[li]->array());
2174 new ((AC*)m_hp_arrays+li+n) AC(m_fabs_v[li]->const_array());
2175 } else {
2176 new ((A*)m_hp_arrays+li) A{};
2177 new ((AC*)m_hp_arrays+li+n) AC{};
2178 }
2179 }
2180 m_arrays.hp = (A*)m_hp_arrays;
2181 m_const_arrays.hp = (AC*)m_hp_arrays + n;
2182#ifdef AMREX_USE_GPU
2183 m_arrays.dp = (A*)m_dp_arrays;
2184 m_const_arrays.dp = (AC*)m_dp_arrays + n;
2185 Gpu::htod_memcpy_async(m_dp_arrays, m_hp_arrays, n*2*sizeof(A));
2186 // The pinned buffer is not modified after this point and is freed
2187 // in stream order by clear_arrays(), so kernels launched on the
2188 // current stream are correctly ordered after the copy without a
2189 // host sync. MFIter and StreamIter synchronize the current stream
2190 // before switching to other streams, so we only need to sync here
2191 // if we are already inside a multi-stream MFIter loop, where later
2192 // iterations may use the arrays on a different stream.
2193 if (!Gpu::inNoSyncRegion() && Gpu::numGpuStreams() > 1 &&
2195 {
2197 }
2198#endif
2199 }
2200}
2201
2202template <class FAB>
2203void
2205{
2206#ifdef AMREX_USE_GPU
2207 // The host-to-device copy of the Array4s issued in build_arrays() is
2208 // asynchronous and is not synchronized when we are in a NoSync region.
2209 // Use stream-ordered frees so that the pinned and device buffers are not
2210 // reused before the copy and any kernels using them have completed.
2211 if (m_hp_arrays) {
2214 }
2215 m_dp_arrays = nullptr;
2216#else
2217 std::free(m_hp_arrays);
2218#endif
2219 m_hp_arrays = nullptr;
2220 m_arrays.hp = nullptr;
2221 m_const_arrays.hp = nullptr;
2222}
2223
2224template <class FAB>
2225void
2226FabArray<FAB>::clear_fab_caches ()
2227{
2228 clear_arrays();
2229#ifdef AMREX_USE_GPU
2230 m_fb_local_copy_handler.clear();
2231 m_recv_copy_handler.clear();
2232 m_send_copy_handler.clear();
2233#endif
2234#ifdef AMREX_FB_UNPACK_ON_ARRIVAL
2235 m_fb_on_arrival_meta.clear();
2236#endif
2237}
2238
2239template <class FAB>
2240[[nodiscard]]
2241FAB*
2243{
2244 const int li = localindex(K);
2245 if (li >= 0 && li < std::ssize(m_fabs_v) && m_fabs_v[li] != nullptr) {
2247 Long nbytes = amrex::nBytesOwned(*m_fabs_v[li]);
2248 if (nbytes > 0) {
2249 for (auto const& t : m_tags) {
2250 updateMemUsage(t, -nbytes, nullptr);
2251 }
2252 }
2253 clear_fab_caches();
2254 return std::exchange(m_fabs_v[li], nullptr);
2255 } else {
2256 return nullptr;
2257 }
2258}
2259
2260template <class FAB>
2261[[nodiscard]]
2262FAB*
2264{
2265 const int li = mfi.LocalIndex();
2266 if (li >= 0 && li < std::ssize(m_fabs_v) && m_fabs_v[li] != nullptr) {
2268 Long nbytes = amrex::nBytesOwned(*m_fabs_v[li]);
2269 if (nbytes > 0) {
2270 for (auto const& t : m_tags) {
2271 updateMemUsage(t, -nbytes, nullptr);
2272 }
2273 }
2274 clear_fab_caches();
2275 return std::exchange(m_fabs_v[li], nullptr);
2276 } else {
2277 return nullptr;
2278 }
2279}
2280
2281template <class FAB>
2282void
2284{
2286 {
2287 define_function_called = false;
2288 clearThisBD();
2289 }
2290
2291 Long nbytes = 0L;
2292 for (auto *x : m_fabs_v) {
2293 if (x) {
2294 nbytes += amrex::nBytesOwned(*x);
2295 m_factory->destroy(x);
2296 }
2297 }
2298 m_fabs_v.clear();
2299 clear_arrays();
2300 m_factory.reset();
2301 m_dallocator.m_arena = m_default_arena;
2302 // no need to clear the non-blocking fillboundary stuff
2303
2304 if (nbytes > 0) {
2305 for (auto const& t : m_tags) {
2306 updateMemUsage(t, -nbytes, nullptr);
2307 }
2308 }
2309
2311 m_single_chunk_arena.reset();
2312 }
2314
2315 m_tags.clear();
2316
2317#ifdef AMREX_USE_GPU
2318 m_fb_local_copy_handler.clear();
2319 m_recv_copy_handler.clear();
2320 m_send_copy_handler.clear();
2321#endif
2322#ifdef AMREX_FB_UNPACK_ON_ARRIVAL
2323 m_fb_on_arrival_meta.clear();
2324#endif
2325
2327}
2328
2329template <class FAB>
2330template <BaseFabType SFAB, BaseFabType DFAB>
2331requires (std::is_convertible_v<typename SFAB::value_type, typename DFAB::value_type>)
2332void
2333FabArray<FAB>::LocalCopy (FabArray<SFAB> const& src, int scomp, int dcomp, int ncomp,
2334 IntVect const& nghost)
2335{
2336 amrex::Copy(*this, src, scomp, dcomp, ncomp, nghost);
2337}
2338
2339template <class FAB>
2340template <class F>
2341requires (BaseFabType<F>)
2342void
2343FabArray<FAB>::LocalAdd (FabArray<FAB> const& src, int scomp, int dcomp, int ncomp,
2344 IntVect const& nghost)
2345{
2346 amrex::Add(*this, src, scomp, dcomp, ncomp, nghost);
2347}
2348
2349template <class FAB>
2350template <class F>
2351requires (BaseFabType<F>)
2352void
2354{
2355 setVal(val,0,n_comp,IntVect(nghost));
2356}
2357
2358template <class FAB>
2359template <class F>
2360requires (BaseFabType<F>)
2361void
2363{
2364 setVal(val,0,n_comp,nghost);
2365}
2366
2367template <class FAB>
2368template <class F>
2369requires (BaseFabType<F>)
2370void
2371FabArray<FAB>::setVal (value_type val, const Box& region, int nghost)
2372{
2373 setVal(val,region,0,n_comp,IntVect(nghost));
2374}
2375
2376template <class FAB>
2377template <class F>
2378requires (BaseFabType<F>)
2379void
2380FabArray<FAB>::setVal (value_type val, const Box& region, const IntVect& nghost)
2381{
2382 setVal(val,region,0,n_comp,nghost);
2383}
2384
2385template <class FAB>
2387 : shmem()
2388{
2389 m_FA_stats.recordBuild();
2390}
2391
2392template <class FAB>
2394 : m_dallocator(a),
2395 m_default_arena(a),
2396 shmem()
2397{
2398 m_FA_stats.recordBuild();
2399}
2400
2401template <class FAB>
2403 const DistributionMapping& dm,
2404 int nvar,
2405 int ngrow,
2406 const MFInfo& info,
2407 const FabFactory<FAB>& factory)
2408 : FabArray<FAB>(bxs,dm,nvar,IntVect(ngrow),info,factory)
2409{}
2410
2411template <class FAB>
2413 const DistributionMapping& dm,
2414 int nvar,
2415 const IntVect& ngrow,
2416 const MFInfo& info,
2417 const FabFactory<FAB>& factory)
2418 : m_factory(factory.clone()),
2419 shmem()
2420{
2421 m_FA_stats.recordBuild();
2422 define(bxs,dm,nvar,ngrow,info,*m_factory);
2423}
2424
2425template <class FAB>
2426FabArray<FAB>::FabArray (const FabArray<FAB>& rhs, MakeType maketype, int scomp, int ncomp)
2427 : m_factory(rhs.Factory().clone()),
2428 shmem()
2429{
2430 m_FA_stats.recordBuild();
2431 define(rhs.boxArray(), rhs.DistributionMap(), ncomp, rhs.nGrowVect(),
2432 MFInfo().SetAlloc(false), *m_factory);
2433
2434 if (maketype == amrex::make_alias)
2435 {
2436 for (int i = 0, n = indexArray.size(); i < n; ++i) {
2437 auto const& rhsfab = *(rhs.m_fabs_v[i]);
2438 m_fabs_v.push_back(m_factory->create_alias(rhsfab, scomp, ncomp));
2439 }
2440 }
2441 else
2442 {
2443 amrex::Abort("FabArray: unknown MakeType");
2444 }
2445}
2446
2447template <class FAB>
2449 : FabArrayBase (static_cast<FabArrayBase&&>(rhs))
2450 , m_factory (std::move(rhs.m_factory))
2451 , m_dallocator (rhs.m_dallocator)
2452 , m_default_arena(rhs.m_default_arena)
2453 , m_single_chunk_arena(std::move(rhs.m_single_chunk_arena))
2454 , m_single_chunk_size(std::exchange(rhs.m_single_chunk_size,0))
2455 , define_function_called(rhs.define_function_called)
2456 , m_fabs_v (std::move(rhs.m_fabs_v))
2457#ifdef AMREX_USE_GPU
2458 , m_dp_arrays (std::exchange(rhs.m_dp_arrays, nullptr))
2459#endif
2460 , m_hp_arrays (std::exchange(rhs.m_hp_arrays, nullptr))
2461 , m_arrays (rhs.m_arrays)
2462 , m_const_arrays(rhs.m_const_arrays)
2463 , m_tags (std::move(rhs.m_tags))
2464 , shmem (std::move(rhs.shmem))
2465#ifdef AMREX_USE_GPU
2466 , m_fb_local_copy_handler(std::move(rhs.m_fb_local_copy_handler))
2467 , m_recv_copy_handler(std::move(rhs.m_recv_copy_handler))
2468 , m_send_copy_handler(std::move(rhs.m_send_copy_handler))
2469#endif
2470#ifdef AMREX_FB_UNPACK_ON_ARRIVAL
2471 , m_fb_on_arrival_meta(std::move(rhs.m_fb_on_arrival_meta))
2472#endif
2473 // no need to worry about the data used in non-blocking FillBoundary.
2474{
2475 m_FA_stats.recordBuild();
2476 rhs.define_function_called = false; // the responsibility of clear BD has been transferred.
2477 rhs.m_fabs_v.clear(); // clear the data pointers so that rhs.clear does not delete them.
2478 rhs.clear();
2479}
2480
2481template <class FAB>
2484{
2485 if (&rhs != this)
2486 {
2487 clear();
2488
2489 FabArrayBase::operator=(static_cast<FabArrayBase&&>(rhs));
2490 m_factory = std::move(rhs.m_factory);
2491 m_dallocator = rhs.m_dallocator;
2492 m_default_arena = rhs.m_default_arena;
2493 m_single_chunk_arena = std::move(rhs.m_single_chunk_arena);
2494 std::swap(m_single_chunk_size, rhs.m_single_chunk_size);
2495 define_function_called = rhs.define_function_called;
2496 std::swap(m_fabs_v, rhs.m_fabs_v);
2497#ifdef AMREX_USE_GPU
2498 std::swap(m_dp_arrays, rhs.m_dp_arrays);
2499#endif
2500 std::swap(m_hp_arrays, rhs.m_hp_arrays);
2501 m_arrays = rhs.m_arrays;
2502 m_const_arrays = rhs.m_const_arrays;
2503 std::swap(m_tags, rhs.m_tags);
2504 shmem = std::move(rhs.shmem);
2505#ifdef AMREX_USE_GPU
2506 std::swap(m_fb_local_copy_handler, rhs.m_fb_local_copy_handler);
2507 std::swap(m_recv_copy_handler, rhs.m_recv_copy_handler);
2508 std::swap(m_send_copy_handler, rhs.m_send_copy_handler);
2509#endif
2510#ifdef AMREX_FB_UNPACK_ON_ARRIVAL
2511 std::swap(m_fb_on_arrival_meta, rhs.m_fb_on_arrival_meta);
2512#endif
2513
2514 rhs.define_function_called = false;
2515 rhs.m_fabs_v.clear();
2516 rhs.m_tags.clear();
2517 rhs.clear();
2518 }
2519 return *this;
2520}
2521
2522template <class FAB>
2524{
2525 m_FA_stats.recordDelete();
2526 clear();
2527}
2528
2529template <class FAB>
2530bool
2532{
2533 if (!define_function_called) { return false; }
2534
2535 int isok = 1;
2536
2537 for (MFIter fai(*this); fai.isValid() && isok; ++fai)
2538 {
2539 if (defined(fai))
2540 {
2541 if (get(fai).box() != fabbox(fai.index()))
2542 {
2543 isok = 0;
2544 }
2545 }
2546 else
2547 {
2548 isok = 0;
2549 }
2550 }
2551
2553
2554 return isok == 1;
2555}
2556
2557template <class FAB>
2558bool
2560{
2562}
2563
2564template <class FAB>
2565void
2567 const DistributionMapping& dm,
2568 int nvar,
2569 int ngrow,
2570 const MFInfo& info,
2571 const FabFactory<FAB>& a_factory)
2572{
2573 define(bxs,dm,nvar,IntVect(ngrow),info,a_factory);
2574}
2575
2576template <class FAB>
2577void
2579 const DistributionMapping& dm,
2580 int nvar,
2581 const IntVect& ngrow,
2582 const MFInfo& info,
2583 const FabFactory<FAB>& a_factory)
2584{
2585 std::unique_ptr<FabFactory<FAB> > factory(a_factory.clone());
2586
2587 auto *default_arena = m_dallocator.m_arena;
2588 clear();
2589
2590 m_factory = std::move(factory);
2591 m_dallocator.m_arena = info.arena ? info.arena : default_arena;
2592
2594
2595 AMREX_ASSERT(ngrow.allGE(0));
2596 AMREX_ASSERT(boxarray.empty());
2597 FabArrayBase::define(bxs, dm, nvar, ngrow);
2598
2599 addThisBD();
2600
2601 if(info.alloc) {
2602 AllocFabs(*m_factory, m_dallocator.m_arena, info.tags, info.alloc_single_chunk);
2603#ifdef BL_USE_TEAM
2605#endif
2606 }
2607}
2608
2609template <class FAB>
2610void
2611FabArray<FAB>::AllocFabs (const FabFactory<FAB>& factory, Arena* ar,
2612 const Vector<std::string>& tags, bool alloc_single_chunk)
2613{
2614 if (shmem.alloc) { alloc_single_chunk = false; }
2615 if constexpr (!IsBaseFab_v<FAB>) { alloc_single_chunk = false; }
2616
2617 const int n = indexArray.size();
2618 const int nworkers = ParallelDescriptor::TeamSize();
2619 shmem.alloc = (nworkers > 1);
2620
2621 bool alloc = !shmem.alloc;
2622
2623 FabInfo fab_info;
2624 fab_info.SetAlloc(alloc).SetShared(shmem.alloc).SetArena(ar);
2625
2626 if (alloc_single_chunk) {
2627 m_single_chunk_size = 0L;
2628 for (int i = 0; i < n; ++i) {
2629 int K = indexArray[i];
2630 const Box& tmpbox = fabbox(K);
2631 m_single_chunk_size += factory.nBytes(tmpbox, n_comp, K);
2632 }
2633 AMREX_ASSERT(m_single_chunk_size >= 0); // 0 is okay.
2634 m_single_chunk_arena = std::make_unique<detail::SingleChunkArena>(ar, m_single_chunk_size);
2635 fab_info.SetArena(m_single_chunk_arena.get());
2636 }
2637
2638 m_fabs_v.reserve(n);
2639
2640 Long nbytes = 0L;
2641 for (int i = 0; i < n; ++i)
2642 {
2643 int K = indexArray[i];
2644 const Box& tmpbox = fabbox(K);
2645 m_fabs_v.push_back(factory.create(tmpbox, n_comp, fab_info, K));
2646 nbytes += amrex::nBytesOwned(*m_fabs_v.back());
2647 }
2648
2649 m_tags.clear();
2650 m_tags.emplace_back("All");
2651 for (auto const& t : m_region_tag) {
2652 m_tags.push_back(t);
2653 }
2654 for (auto const& t : tags) {
2655 m_tags.push_back(t);
2656 }
2657 for (auto const& t: m_tags) {
2658 updateMemUsage(t, nbytes, ar);
2659 }
2660
2661#ifdef BL_USE_TEAM
2662 if (shmem.alloc)
2663 {
2664 const int teamlead = ParallelDescriptor::MyTeamLead();
2665
2666 shmem.n_values = 0;
2667 shmem.n_points = 0;
2668 Vector<Long> offset(n,0);
2669 Vector<Long> nextoffset(nworkers,-1);
2670 for (int i = 0; i < n; ++i) {
2671 int K = indexArray[i];
2672 int owner = distributionMap[K] - teamlead;
2673 Long s = m_fabs_v[i]->size();
2674 if (ownership[i]) {
2675 shmem.n_values += s;
2676 shmem.n_points += m_fabs_v[i]->numPts();
2677 }
2678 if (nextoffset[owner] < 0) {
2679 offset[i] = 0;
2680 nextoffset[owner] = s;
2681 } else {
2682 offset[i] = nextoffset[owner];
2683 nextoffset[owner] += s;
2684 }
2685 }
2686
2687 size_t bytes = shmem.n_values*sizeof(value_type);
2688
2689 value_type *mfp;
2691
2692#if defined (BL_USE_MPI3)
2693
2694 static MPI_Info info = MPI_INFO_NULL;
2695 if (info == MPI_INFO_NULL) {
2696 MPI_Info_create(&info);
2697 MPI_Info_set(info, "alloc_shared_noncontig", "true");
2698 }
2699
2700 const MPI_Comm& team_comm = ParallelDescriptor::MyTeam().get();
2701
2702 BL_MPI_REQUIRE( MPI_Win_allocate_shared(bytes, sizeof(value_type),
2703 info, team_comm, &mfp, &shmem.win) );
2704
2705 for (int w = 0; w < nworkers; ++w) {
2706 MPI_Aint sz;
2707 int disp;
2708 value_type *dptr = 0;
2709 BL_MPI_REQUIRE( MPI_Win_shared_query(shmem.win, w, &sz, &disp, &dptr) );
2710 // AMREX_ASSERT(disp == sizeof(value_type));
2711 dps.push_back(dptr);
2712 }
2713
2714#else
2715
2716 amrex::Abort("BaseFab::define: to allocate shared memory, USE_MPI3 must be true");
2717
2718#endif
2719
2720 for (int i = 0; i < n; ++i) {
2721 int K = indexArray[i];
2722 int owner = distributionMap[K] - teamlead;
2723 value_type *p = dps[owner] + offset[i];
2724 m_fabs_v[i]->setPtr(p, m_fabs_v[i]->size());
2725 }
2726
2727 for (Long i = 0; i < shmem.n_values; i++, mfp++) {
2728 new (mfp) value_type;
2729 }
2730
2731 amrex::update_fab_stats(shmem.n_points, shmem.n_values, sizeof(value_type));
2732 }
2733#endif
2734}
2735
2736template <class FAB>
2737void
2738FabArray<FAB>::setFab_assert (int K, FAB const& fab) const
2739{
2740 amrex::ignore_unused(K,fab);
2741 AMREX_ASSERT(n_comp == fab.nComp());
2742 AMREX_ASSERT(!boxarray.empty());
2743 AMREX_ASSERT(fab.box() == fabbox(K));
2744 AMREX_ASSERT(distributionMap[K] == ParallelDescriptor::MyProc());
2745 AMREX_ASSERT(m_single_chunk_arena == nullptr);
2746}
2747
2748template <class FAB>
2749void
2750FabArray<FAB>::setFab (int boxno, std::unique_ptr<FAB> elem)
2751{
2752 if (n_comp == 0) {
2753 n_comp = elem->nComp();
2754 }
2755
2756 setFab_assert(boxno, *elem);
2757
2758 if (m_fabs_v.empty()) {
2759 m_fabs_v.resize(indexArray.size(),nullptr);
2760 }
2761
2762 const int li = localindex(boxno);
2763 if (m_fabs_v[li]) {
2764 m_factory->destroy(m_fabs_v[li]);
2765 }
2766 m_fabs_v[li] = elem.release();
2767 clear_fab_caches();
2768}
2769
2770template <class FAB>
2771template <class F>
2772requires (std::is_move_constructible_v<F>)
2773void
2774FabArray<FAB>::setFab (int boxno, FAB&& elem)
2775{
2776 if (n_comp == 0) {
2777 n_comp = elem.nComp();
2778 }
2779
2780 setFab_assert(boxno, elem);
2781
2782 if (m_fabs_v.empty()) {
2783 m_fabs_v.resize(indexArray.size(),nullptr);
2784 }
2785
2786 const int li = localindex(boxno);
2787 if (m_fabs_v[li]) {
2788 m_factory->destroy(m_fabs_v[li]);
2789 }
2790 m_fabs_v[li] = new FAB(std::move(elem));
2791 clear_fab_caches();
2792}
2793
2794template <class FAB>
2795void
2796FabArray<FAB>::setFab (const MFIter& mfi, std::unique_ptr<FAB> elem)
2797{
2798 if (n_comp == 0) {
2799 n_comp = elem->nComp();
2800 }
2801
2802 setFab_assert(mfi.index(), *elem);
2803
2804 if (m_fabs_v.empty()) {
2805 m_fabs_v.resize(indexArray.size(),nullptr);
2806 }
2807
2808 const int li = mfi.LocalIndex();
2809 if (m_fabs_v[li]) {
2810 m_factory->destroy(m_fabs_v[li]);
2811 }
2812 m_fabs_v[li] = elem.release();
2813 clear_fab_caches();
2814}
2815
2816template <class FAB>
2817template <class F>
2818requires (std::is_move_constructible_v<F>)
2819void
2820FabArray<FAB>::setFab (const MFIter& mfi, FAB&& elem)
2821{
2822 if (n_comp == 0) {
2823 n_comp = elem.nComp();
2824 }
2825
2826 setFab_assert(mfi.index(), elem);
2827
2828 if (m_fabs_v.empty()) {
2829 m_fabs_v.resize(indexArray.size(),nullptr);
2830 }
2831
2832 const int li = mfi.LocalIndex();
2833 if (m_fabs_v[li]) {
2834 m_factory->destroy(m_fabs_v[li]);
2835 }
2836 m_fabs_v[li] = new FAB(std::move(elem));
2837 clear_fab_caches();
2838}
2839
2840template <class FAB>
2841template <class F>
2842requires (BaseFabType<F>)
2843void
2845{
2846 setBndry(val, 0, n_comp);
2847}
2848
2849template <class FAB>
2850template <class F>
2851requires (BaseFabType<F>)
2852void
2854 int strt_comp,
2855 int ncomp)
2856{
2857 if (n_grow.max() > 0)
2858 {
2859#ifdef AMREX_USE_GPU
2860 if (Gpu::inLaunchRegion()) {
2861 bool use_mfparfor = true;
2862 const int nboxes = local_size();
2863 if (nboxes == 1) {
2864 if (boxarray[indexArray[0]].numPts() > Long(65*65*65)) {
2865 use_mfparfor = false;
2866 }
2867 } else {
2868 for (int i = 0; i < nboxes; ++i) {
2869 const Long npts = boxarray[indexArray[i]].numPts();
2870 if (npts >= Long(64*64*64)) {
2871 use_mfparfor = false;
2872 break;
2873 } else if (npts <= Long(17*17*17)) {
2874 break;
2875 }
2876 }
2877 }
2878 const IntVect nghost = n_grow;
2879 if (use_mfparfor) {
2880 auto const& ma = this->arrays();
2881 ParallelFor(*this, nghost,
2882 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept
2883 {
2884 auto const& a = ma[box_no];
2885 Box vbx(a);
2886 vbx.grow(-nghost);
2887 if (!vbx.contains(i,j,k)) {
2888 for (int n = 0; n < ncomp; ++n) {
2889 a(i,j,k,strt_comp+n) = val;
2890 }
2891 }
2892 });
2893 if (!Gpu::inNoSyncRegion()) {
2895 }
2896 } else {
2897 using Tag = Array4BoxTag<value_type>;
2899 for (MFIter mfi(*this); mfi.isValid(); ++mfi) {
2900 Box const& vbx = mfi.validbox();
2901 auto const& a = this->array(mfi);
2902
2903 Box b;
2904#if (AMREX_SPACEDIM == 3)
2905 if (nghost[2] > 0) {
2906 b = vbx;
2907 b.setRange(2, vbx.smallEnd(2)-nghost[2], nghost[2]);
2908 b.grow(IntVect(nghost[0],nghost[1],0));
2909 tags.emplace_back(Tag{.dfab = a, .dbox = b});
2910 b.shift(2, vbx.length(2)+nghost[2]);
2911 tags.emplace_back(Tag{.dfab = a, .dbox = b});
2912 }
2913#endif
2914#if (AMREX_SPACEDIM >= 2)
2915 if (nghost[1] > 0) {
2916 b = vbx;
2917 b.setRange(1, vbx.smallEnd(1)-nghost[1], nghost[1]);
2918 b.grow(0, nghost[0]);
2919 tags.emplace_back(Tag{.dfab = a, .dbox = b});
2920 b.shift(1, vbx.length(1)+nghost[1]);
2921 tags.emplace_back(Tag{.dfab = a, .dbox = b});
2922 }
2923#endif
2924 if (nghost[0] > 0) {
2925 b = vbx;
2926 b.setRange(0, vbx.smallEnd(0)-nghost[0], nghost[0]);
2927 tags.emplace_back(Tag{.dfab = a, .dbox = b});
2928 b.shift(0, vbx.length(0)+nghost[0]);
2929 tags.emplace_back(Tag{.dfab = a, .dbox = b});
2930 }
2931 }
2932
2933 ParallelFor(tags, ncomp,
2934 [=] AMREX_GPU_DEVICE (int i, int j, int k, int n, Tag const& tag) noexcept
2935 {
2936 tag.dfab(i,j,k,strt_comp+n) = val;
2937 });
2938 if (!Gpu::inNoSyncRegion()) {
2940 }
2941 }
2942 } else
2943#endif
2944 {
2945#ifdef AMREX_USE_OMP
2946#pragma omp parallel
2947#endif
2948 for (MFIter fai(*this); fai.isValid(); ++fai)
2949 {
2950 get(fai).template setComplement<RunOn::Host>(val, fai.validbox(), strt_comp, ncomp);
2951 }
2952 }
2953 }
2954}
2955
2956template <class FAB>
2957template <class F>
2958requires (BaseFabType<F>)
2959void
2961{
2962 setDomainBndry(val, 0, n_comp, geom);
2963}
2964
2965template <class FAB>
2966template <class F>
2967requires (BaseFabType<F>)
2968void
2970 int strt_comp,
2971 int ncomp,
2972 const Geometry& geom)
2973{
2974 BL_PROFILE("FabArray::setDomainBndry()");
2975
2976 Box domain_box = amrex::convert(geom.Domain(), boxArray().ixType());
2977 for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
2978 if (geom.isPeriodic(idim)) {
2979 int n = domain_box.length(idim);
2980 domain_box.grow(idim, n);
2981 }
2982 }
2983
2984#ifdef AMREX_USE_OMP
2985#pragma omp parallel if (Gpu::notInLaunchRegion())
2986#endif
2987 for (MFIter fai(*this); fai.isValid(); ++fai)
2988 {
2989 const Box& gbx = fai.fabbox();
2990 if (! domain_box.contains(gbx))
2991 {
2992 get(fai).template setComplement<RunOn::Device>(val, domain_box, strt_comp, ncomp);
2993 }
2994 }
2995}
2996
2997template <class FAB>
2998template <std::integral Int>
2999requires (BaseFabType<FAB> && (sizeof(Int) >= sizeof(Long)))
3000void
3002{
3003 AMREX_ASSERT(amrex::isMFIterSafe(*this, mem));
3004 for (MFIter mfi(*this, MFItInfo{}.DisableDeviceSync()); mfi.isValid(); ++mfi) {
3005 mem[mfi] += static_cast<Int>((*this)[mfi].nBytesOwned());
3006 }
3007}
3008
3009template <class FAB>
3010template <class F>
3011requires (BaseFabType<F>)
3012typename F::value_type
3013FabArray<FAB>::sum (int comp, IntVect const& nghost, bool local) const
3014{
3015 BL_PROFILE("FabArray::sum()");
3016
3017 using T = typename FAB::value_type;
3018 auto sm = T(0.0);
3019#ifdef AMREX_USE_GPU
3020 if (Gpu::inLaunchRegion())
3021#endif
3022 {
3023 auto const& ma = this->const_arrays();
3024 sm = ParReduce(TypeList<ReduceOpSum>{}, TypeList<T>{}, *this, nghost,
3025 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept
3026 -> GpuTuple<T>
3027 {
3028 return ma[box_no](i,j,k,comp);
3029 });
3030 }
3031#ifdef AMREX_USE_GPU
3032 else {
3033#ifdef AMREX_USE_OMP
3034#pragma omp parallel if (!system::regtest_reduction) reduction(+:sm)
3035#endif
3036 for (MFIter mfi(*this,true); mfi.isValid(); ++mfi)
3037 {
3038 Box const& bx = mfi.growntilebox(nghost);
3039 auto const& a = this->const_array(mfi);
3040 auto tmp = T(0.0);
3041 AMREX_LOOP_3D(bx, i, j, k,
3042 {
3043 tmp += a(i,j,k,comp);
3044 });
3045 sm += tmp; // Do it this way so that it does not break regression tests.
3046 }
3047 }
3048#endif
3049
3050 if (!local) {
3052 }
3053
3054 return sm;
3055}
3056
3057template <class FAB>
3058void
3059FabArray<FAB>::copyTo (FAB& dest, int nghost) const
3060{
3061 copyTo(dest, 0, 0, dest.nComp(), nghost);
3062}
3063
3064template <class FAB>
3065template <class F>
3066requires (BaseFabType<F>)
3067void
3069{
3070 setVal(val,0,n_comp,n_grow);
3071}
3072
3073template <class FAB>
3074template <class F>
3075requires (BaseFabType<F>)
3078{
3079 setVal(val);
3080 return *this;
3081}
3082
3083template <class FAB>
3084template <class F>
3085requires (BaseFabType<F>)
3086void
3088 int comp,
3089 int ncomp,
3090 int nghost)
3091{
3092 setVal(val,comp,ncomp,IntVect(nghost));
3093}
3094
3095template <class FAB>
3096template <class F>
3097requires (BaseFabType<F>)
3098void
3100 int comp,
3101 int ncomp,
3102 const IntVect& nghost)
3103{
3104 AMREX_ASSERT(nghost.allGE(0) && nghost.allLE(n_grow));
3105 AMREX_ALWAYS_ASSERT(comp+ncomp <= n_comp);
3106
3107 BL_PROFILE("FabArray::setVal()");
3108
3109#ifdef AMREX_USE_GPU
3110 if (Gpu::inLaunchRegion() && this->isFusingCandidate()) {
3111 auto const& fa = this->arrays();
3112 ParallelFor(*this, nghost, ncomp,
3113 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
3114 {
3115 fa[box_no](i,j,k,n+comp) = val;
3116 });
3117 if (!Gpu::inNoSyncRegion()) {
3119 }
3120 } else
3121#endif
3122 {
3123#ifdef AMREX_USE_OMP
3124#pragma omp parallel if (Gpu::notInLaunchRegion())
3125#endif
3126 for (MFIter fai(*this,TilingIfNotGPU()); fai.isValid(); ++fai)
3127 {
3128 const Box& bx = fai.growntilebox(nghost);
3129 auto fab = this->array(fai);
3130 AMREX_HOST_DEVICE_PARALLEL_FOR_4D( bx, ncomp, i, j, k, n,
3131 {
3132 fab(i,j,k,n+comp) = val;
3133 });
3134 }
3135 }
3136}
3137
3138template <class FAB>
3139template <class F>
3140requires (BaseFabType<F>)
3141void
3143 const Box& region,
3144 int comp,
3145 int ncomp,
3146 int nghost)
3147{
3148 setVal(val,region,comp,ncomp,IntVect(nghost));
3149}
3150
3151template <class FAB>
3152template <class F>
3153requires (BaseFabType<F>)
3154void
3156 const Box& region,
3157 int comp,
3158 int ncomp,
3159 const IntVect& nghost)
3160{
3161 AMREX_ASSERT(nghost.allGE(0) && nghost.allLE(n_grow));
3162 AMREX_ALWAYS_ASSERT(comp+ncomp <= n_comp);
3163
3164 BL_PROFILE("FabArray::setVal(val,region,comp,ncomp,nghost)");
3165
3166#ifdef AMREX_USE_GPU
3167 if (Gpu::inLaunchRegion() && this->isFusingCandidate()) {
3168 auto const& fa = this->arrays();
3169 ParallelFor(*this, nghost, ncomp,
3170 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
3171 {
3172 if (region.contains(i,j,k)) {
3173 fa[box_no](i,j,k,n+comp) = val;
3174 }
3175 });
3176 if (!Gpu::inNoSyncRegion()) {
3178 }
3179 } else
3180#endif
3181 {
3182#ifdef AMREX_USE_OMP
3183 AMREX_ALWAYS_ASSERT(!omp_in_parallel());
3184#pragma omp parallel if (Gpu::notInLaunchRegion())
3185#endif
3186 for (MFIter fai(*this,TilingIfNotGPU()); fai.isValid(); ++fai)
3187 {
3188 Box b = fai.growntilebox(nghost) & region;
3189
3190 if (b.ok()) {
3191 auto fab = this->array(fai);
3192 AMREX_HOST_DEVICE_PARALLEL_FOR_4D( b, ncomp, i, j, k, n,
3193 {
3194 fab(i,j,k,n+comp) = val;
3195 });
3196 }
3197 }
3198 }
3199}
3200
3201template <class FAB>
3202template <class F>
3203requires (BaseFabType<F>)
3204void
3205FabArray<FAB>::abs (int comp, int ncomp, int nghost)
3206{
3207 abs(comp, ncomp, IntVect(nghost));
3208}
3209
3210template <class FAB>
3211template <class F>
3212requires (BaseFabType<F>)
3213void
3214FabArray<FAB>::abs (int comp, int ncomp, const IntVect& nghost)
3215{
3216 AMREX_ASSERT(nghost.allGE(0) && nghost.allLE(n_grow));
3217 AMREX_ALWAYS_ASSERT(comp+ncomp <= n_comp);
3218 BL_PROFILE("FabArray::abs()");
3219
3220#ifdef AMREX_USE_GPU
3221 if (Gpu::inLaunchRegion() && this->isFusingCandidate()) {
3222 auto const& fa = this->arrays();
3223 ParallelFor(*this, nghost, ncomp,
3224 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
3225 {
3226 fa[box_no](i,j,k,n+comp) = std::abs(fa[box_no](i,j,k,n+comp));
3227 });
3228 if (!Gpu::inNoSyncRegion()) {
3230 }
3231 } else
3232#endif
3233 {
3234#ifdef AMREX_USE_OMP
3235#pragma omp parallel if (Gpu::notInLaunchRegion())
3236#endif
3237 for (MFIter mfi(*this,TilingIfNotGPU()); mfi.isValid(); ++mfi)
3238 {
3239 const Box& bx = mfi.growntilebox(nghost);
3240 auto fab = this->array(mfi);
3241 AMREX_HOST_DEVICE_PARALLEL_FOR_4D( bx, ncomp, i, j, k, n,
3242 {
3243 fab(i,j,k,n+comp) = std::abs(fab(i,j,k,n+comp));
3244 });
3245 }
3246 }
3247}
3248
3249template <class FAB>
3250template <class F>
3251requires (BaseFabType<F>)
3252void
3253FabArray<FAB>::plus (value_type val, int comp, int num_comp, int nghost)
3254{
3255 BL_PROFILE("FabArray::plus()");
3256
3257#ifdef AMREX_USE_GPU
3258 if (Gpu::inLaunchRegion() && this->isFusingCandidate()) {
3259 auto const& fa = this->arrays();
3260 ParallelFor(*this, IntVect(nghost), num_comp,
3261 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
3262 {
3263 fa[box_no](i,j,k,n+comp) += val;
3264 });
3265 if (!Gpu::inNoSyncRegion()) {
3267 }
3268 } else
3269#endif
3270 {
3271#ifdef AMREX_USE_OMP
3272#pragma omp parallel if (Gpu::notInLaunchRegion())
3273#endif
3274 for (MFIter mfi(*this,TilingIfNotGPU()); mfi.isValid(); ++mfi)
3275 {
3276 const Box& bx = mfi.growntilebox(nghost);
3277 auto fab = this->array(mfi);
3278 AMREX_HOST_DEVICE_PARALLEL_FOR_4D( bx, num_comp, i, j, k, n,
3279 {
3280 fab(i,j,k,n+comp) += val;
3281 });
3282 }
3283 }
3284}
3285
3286template <class FAB>
3287template <class F>
3288requires (BaseFabType<F>)
3289void
3290FabArray<FAB>::plus (value_type val, const Box& region, int comp, int num_comp, int nghost)
3291{
3292 BL_PROFILE("FabArray::plus(val, region, comp, num_comp, nghost)");
3293
3294#ifdef AMREX_USE_GPU
3295 if (Gpu::inLaunchRegion() && this->isFusingCandidate()) {
3296 auto const& fa = this->arrays();
3297 ParallelFor(*this, IntVect(nghost), num_comp,
3298 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
3299 {
3300 if (region.contains(i,j,k)) {
3301 fa[box_no](i,j,k,n+comp) += val;
3302 }
3303 });
3304 if (!Gpu::inNoSyncRegion()) {
3306 }
3307 } else
3308#endif
3309 {
3310#ifdef AMREX_USE_OMP
3311#pragma omp parallel if (Gpu::notInLaunchRegion())
3312#endif
3313 for (MFIter mfi(*this,TilingIfNotGPU()); mfi.isValid(); ++mfi)
3314 {
3315 const Box& bx = mfi.growntilebox(nghost) & region;
3316 if (bx.ok()) {
3317 auto fab = this->array(mfi);
3318 AMREX_HOST_DEVICE_PARALLEL_FOR_4D( bx, num_comp, i, j, k, n,
3319 {
3320 fab(i,j,k,n+comp) += val;
3321 });
3322 }
3323 }
3324 }
3325}
3326
3327template <class FAB>
3328template <class F>
3329requires (BaseFabType<F>)
3330void
3331FabArray<FAB>::mult (value_type val, int comp, int num_comp, int nghost)
3332{
3333 BL_PROFILE("FabArray::mult()");
3334
3335#ifdef AMREX_USE_GPU
3336 if (Gpu::inLaunchRegion() && this->isFusingCandidate()) {
3337 auto const& fa = this->arrays();
3338 ParallelFor(*this, IntVect(nghost), num_comp,
3339 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
3340 {
3341 fa[box_no](i,j,k,n+comp) *= val;
3342 });
3343 if (!Gpu::inNoSyncRegion()) {
3345 }
3346 } else
3347#endif
3348 {
3349#ifdef AMREX_USE_OMP
3350#pragma omp parallel if (Gpu::notInLaunchRegion())
3351#endif
3352 for (MFIter mfi(*this,TilingIfNotGPU()); mfi.isValid(); ++mfi)
3353 {
3354 const Box& bx = mfi.growntilebox(nghost);
3355 auto fab = this->array(mfi);
3356 AMREX_HOST_DEVICE_PARALLEL_FOR_4D( bx, num_comp, i, j, k, n,
3357 {
3358 fab(i,j,k,n+comp) *= val;
3359 });
3360 }
3361 }
3362}
3363
3364template <class FAB>
3365template <class F>
3366requires (BaseFabType<F>)
3367void
3368FabArray<FAB>::mult (value_type val, const Box& region, int comp, int num_comp, int nghost)
3369{
3370 BL_PROFILE("FabArray::mult(val, region, comp, num_comp, nghost)");
3371
3372#ifdef AMREX_USE_GPU
3373 if (Gpu::inLaunchRegion() && this->isFusingCandidate()) {
3374 auto const& fa = this->arrays();
3375 ParallelFor(*this, IntVect(nghost), num_comp,
3376 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
3377 {
3378 if (region.contains(i,j,k)) {
3379 fa[box_no](i,j,k,n+comp) *= val;
3380 }
3381 });
3382 if (!Gpu::inNoSyncRegion()) {
3384 }
3385 } else
3386#endif
3387 {
3388#ifdef AMREX_USE_OMP
3389#pragma omp parallel if (Gpu::notInLaunchRegion())
3390#endif
3391 for (MFIter mfi(*this,TilingIfNotGPU()); mfi.isValid(); ++mfi)
3392 {
3393 const Box& bx = mfi.growntilebox(nghost) & region;
3394 if (bx.ok()) {
3395 auto fab = this->array(mfi);
3396 AMREX_HOST_DEVICE_PARALLEL_FOR_4D( bx, num_comp, i, j, k, n,
3397 {
3398 fab(i,j,k,n+comp) *= val;
3399 });
3400 }
3401 }
3402 }
3403}
3404
3405template <class FAB>
3406template <class F>
3407requires (BaseFabType<F>)
3408void
3409FabArray<FAB>::invert (value_type numerator, int comp, int num_comp, int nghost)
3410{
3411 BL_PROFILE("FabArray::invert()");
3412
3413#ifdef AMREX_USE_GPU
3414 if (Gpu::inLaunchRegion() && this->isFusingCandidate()) {
3415 auto const& fa = this->arrays();
3416 ParallelFor(*this, IntVect(nghost), num_comp,
3417 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
3418 {
3419 fa[box_no](i,j,k,n+comp) = numerator / fa[box_no](i,j,k,n+comp);
3420 });
3421 if (!Gpu::inNoSyncRegion()) {
3423 }
3424 } else
3425#endif
3426 {
3427#ifdef AMREX_USE_OMP
3428#pragma omp parallel if (Gpu::notInLaunchRegion())
3429#endif
3430 for (MFIter mfi(*this,TilingIfNotGPU()); mfi.isValid(); ++mfi)
3431 {
3432 const Box& bx = mfi.growntilebox(nghost);
3433 auto fab = this->array(mfi);
3434 AMREX_HOST_DEVICE_PARALLEL_FOR_4D( bx, num_comp, i, j, k, n,
3435 {
3436 fab(i,j,k,n+comp) = numerator / fab(i,j,k,n+comp);
3437 });
3438 }
3439 }
3440}
3441
3442template <class FAB>
3443template <class F>
3444requires (BaseFabType<F>)
3445void
3446FabArray<FAB>::invert (value_type numerator, const Box& region, int comp, int num_comp, int nghost)
3447{
3448 BL_PROFILE("FabArray::invert(numerator, region, comp, num_comp, nghost)");
3449
3450#ifdef AMREX_USE_GPU
3451 if (Gpu::inLaunchRegion() && this->isFusingCandidate()) {
3452 auto const& fa = this->arrays();
3453 ParallelFor(*this, IntVect(nghost), num_comp,
3454 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
3455 {
3456 if (region.contains(i,j,k)) {
3457 fa[box_no](i,j,k,n+comp) = numerator / fa[box_no](i,j,k,n+comp);
3458 }
3459 });
3460 if (!Gpu::inNoSyncRegion()) {
3462 }
3463 } else
3464#endif
3465 {
3466#ifdef AMREX_USE_OMP
3467#pragma omp parallel if (Gpu::notInLaunchRegion())
3468#endif
3469 for (MFIter mfi(*this,TilingIfNotGPU()); mfi.isValid(); ++mfi)
3470 {
3471 const Box& bx = mfi.growntilebox(nghost) & region;
3472 if (bx.ok()) {
3473 auto fab = this->array(mfi);
3474 AMREX_HOST_DEVICE_PARALLEL_FOR_4D( bx, num_comp, i, j, k, n,
3475 {
3476 fab(i,j,k,n+comp) = numerator / fab(i,j,k,n+comp);
3477 });
3478 }
3479 }
3480 }
3481}
3482
3483template <class FAB>
3484void
3486{
3487 clearThisBD(); // The new boxarray will have a different ID.
3488 boxarray.shift(v);
3489 addThisBD();
3490#ifdef AMREX_USE_OMP
3491#pragma omp parallel
3492#endif
3493 for (MFIter fai(*this); fai.isValid(); ++fai)
3494 {
3495 get(fai).shift(v);
3496 }
3497 clear_arrays();
3498}
3499
3500template <class FAB>
3501template <class F>
3502requires (BaseFabType<F>)
3504 int xcomp, int ycomp, int ncomp, IntVect const& nghost)
3505{
3506 AMREX_ASSERT(y.boxArray() == x.boxArray());
3507 AMREX_ASSERT(y.distributionMap == x.distributionMap);
3508 AMREX_ASSERT(y.nGrowVect().allGE(nghost) && x.nGrowVect().allGE(nghost));
3509
3510 BL_PROFILE("FabArray::Saxpy()");
3511
3512#ifdef AMREX_USE_GPU
3513 if (Gpu::inLaunchRegion() && y.isFusingCandidate()) {
3514 auto const& yma = y.arrays();
3515 auto const& xma = x.const_arrays();
3516 ParallelFor(y, nghost, ncomp,
3517 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
3518 {
3519 yma[box_no](i,j,k,ycomp+n) += a * xma[box_no](i,j,k,xcomp+n);
3520 });
3521 if (!Gpu::inNoSyncRegion()) {
3523 }
3524 } else
3525#endif
3526 {
3527#ifdef AMREX_USE_OMP
3528#pragma omp parallel if (Gpu::notInLaunchRegion())
3529#endif
3530 for (MFIter mfi(y,TilingIfNotGPU()); mfi.isValid(); ++mfi)
3531 {
3532 const Box& bx = mfi.growntilebox(nghost);
3533
3534 if (bx.ok()) {
3535 auto const& xfab = x.const_array(mfi);
3536 auto const& yfab = y.array(mfi);
3537 AMREX_HOST_DEVICE_PARALLEL_FOR_4D( bx, ncomp, i, j, k, n,
3538 {
3539 yfab(i,j,k,ycomp+n) += a * xfab(i,j,k,xcomp+n);
3540 });
3541 }
3542 }
3543 }
3544}
3545
3546template <class FAB>
3547template <class F>
3548requires (BaseFabType<F>)
3549void
3551 int xcomp, int ycomp, int ncomp, IntVect const& nghost)
3552{
3553 AMREX_ASSERT(y.boxArray() == x.boxArray());
3554 AMREX_ASSERT(y.distributionMap == x.distributionMap);
3555 AMREX_ASSERT(y.nGrowVect().allGE(nghost) && x.nGrowVect().allGE(nghost));
3556
3557 BL_PROFILE("FabArray::Xpay()");
3558
3559#ifdef AMREX_USE_GPU
3560 if (Gpu::inLaunchRegion() && y.isFusingCandidate()) {
3561 auto const& yfa = y.arrays();
3562 auto const& xfa = x.const_arrays();
3563 ParallelFor(y, nghost, ncomp,
3564 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
3565 {
3566 yfa[box_no](i,j,k,n+ycomp) = xfa[box_no](i,j,k,n+xcomp)
3567 + a * yfa[box_no](i,j,k,n+ycomp);
3568 });
3569 if (!Gpu::inNoSyncRegion()) {
3571 }
3572 } else
3573#endif
3574 {
3575#ifdef AMREX_USE_OMP
3576#pragma omp parallel if (Gpu::notInLaunchRegion())
3577#endif
3578 for (MFIter mfi(y,TilingIfNotGPU()); mfi.isValid(); ++mfi)
3579 {
3580 const Box& bx = mfi.growntilebox(nghost);
3581 auto const& xFab = x.const_array(mfi);
3582 auto const& yFab = y.array(mfi);
3583 AMREX_HOST_DEVICE_PARALLEL_FOR_4D( bx, ncomp, i, j, k, n,
3584 {
3585 yFab(i,j,k,n+ycomp) = xFab(i,j,k,n+xcomp)
3586 + a * yFab(i,j,k,n+ycomp);
3587 });
3588 }
3589 }
3590}
3591
3592template <class FAB>
3593template <class F>
3594requires (BaseFabType<F>)
3596 value_type a2, FabArray<FAB> const& x2,
3597 int xcomp, int ycomp, int ncomp, IntVect const& nghost)
3598{
3599 AMREX_ASSERT(y.boxArray() == x1.boxArray() &&
3600 y.boxArray() == x2.boxArray() &&
3601 y.distributionMap == x1.distributionMap &&
3602 y.distributionMap == x2.distributionMap &&
3603 y.nGrowVect().allGE(nghost) &&
3604 x1.nGrowVect().allGE(nghost) &&
3605 x2.nGrowVect().allGE(nghost));
3606
3607 BL_PROFILE("FabArray::Saxpy_Xpay()");
3608
3609#ifdef AMREX_USE_GPU
3610 if (Gpu::inLaunchRegion() && y.isFusingCandidate()) {
3611 auto const& yma = y.arrays();
3612 auto const& xma1 = x1.const_arrays();
3613 auto const& xma2 = x2.const_arrays();
3614 ParallelFor(y, nghost, ncomp,
3615 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
3616 {
3617 yma[box_no](i,j,k,ycomp+n) = xma2[box_no](i,j,k,xcomp+n)
3618 + a2 * (yma [box_no](i,j,k,ycomp+n)
3619 + a1 * xma1[box_no](i,j,k,xcomp+n));
3620 });
3621 if (!Gpu::inNoSyncRegion()) {
3623 }
3624 } else
3625#endif
3626 {
3627#ifdef AMREX_USE_OMP
3628#pragma omp parallel if (Gpu::notInLaunchRegion())
3629#endif
3630 for (MFIter mfi(y,TilingIfNotGPU()); mfi.isValid(); ++mfi)
3631 {
3632 const Box& bx = mfi.growntilebox(nghost);
3633
3634 if (bx.ok()) {
3635 auto const& xfab1 = x1.const_array(mfi);
3636 auto const& xfab2 = x2.const_array(mfi);
3637 auto const& yfab = y.array(mfi);
3638 AMREX_HOST_DEVICE_PARALLEL_FOR_4D( bx, ncomp, i, j, k, n,
3639 {
3640 yfab(i,j,k,ycomp+n) = xfab2(i,j,k,xcomp+n)
3641 + a2 * (yfab (i,j,k,ycomp+n)
3642 + a1 * xfab1(i,j,k,xcomp+n));
3643 });
3644 }
3645 }
3646 }
3647}
3648
3649template <class FAB>
3650template <class F>
3651requires (BaseFabType<F>)
3653 FabArray<FAB>& y2, value_type a2, FabArray<FAB> const& x2,
3654 int xcomp, int ycomp, int ncomp, IntVect const& nghost)
3655{
3656 AMREX_ASSERT(y1.boxArray() == y2.boxArray() &&
3657 y1.boxArray() == x1.boxArray() &&
3658 y1.boxArray() == x2.boxArray() &&
3662 y1.nGrowVect().allGE(nghost) &&
3663 y2.nGrowVect().allGE(nghost) &&
3664 x1.nGrowVect().allGE(nghost) &&
3665 x2.nGrowVect().allGE(nghost));
3666
3667 BL_PROFILE("FabArray::Saxpy_Saxpy()");
3668
3669#ifdef AMREX_USE_GPU
3670 if (Gpu::inLaunchRegion() && y1.isFusingCandidate()) {
3671 auto const& y1ma = y1.arrays();
3672 auto const& x1ma = x1.const_arrays();
3673 auto const& y2ma = y2.arrays();
3674 auto const& x2ma = x2.const_arrays();
3675 ParallelFor(y1, nghost, ncomp,
3676 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
3677 {
3678 y1ma[box_no](i,j,k,ycomp+n) += a1 * x1ma[box_no](i,j,k,xcomp+n);
3679 y2ma[box_no](i,j,k,ycomp+n) += a2 * x2ma[box_no](i,j,k,xcomp+n);
3680 });
3681 if (!Gpu::inNoSyncRegion()) {
3683 }
3684 } else
3685#endif
3686 {
3687#ifdef AMREX_USE_OMP
3688#pragma omp parallel if (Gpu::notInLaunchRegion())
3689#endif
3690 for (MFIter mfi(y1,TilingIfNotGPU()); mfi.isValid(); ++mfi)
3691 {
3692 const Box& bx = mfi.growntilebox(nghost);
3693
3694 if (bx.ok()) {
3695 auto const& x1fab = x1.const_array(mfi);
3696 auto const& y1fab = y1.array(mfi);
3697 auto const& x2fab = x2.const_array(mfi);
3698 auto const& y2fab = y2.array(mfi);
3699 AMREX_HOST_DEVICE_PARALLEL_FOR_4D( bx, ncomp, i, j, k, n,
3700 {
3701 y1fab(i,j,k,ycomp+n) += a1 * x1fab(i,j,k,xcomp+n);
3702 y2fab(i,j,k,ycomp+n) += a2 * x2fab(i,j,k,xcomp+n);
3703 });
3704 }
3705 }
3706 }
3707}
3708
3709template <class FAB>
3710template <class F>
3711requires (BaseFabType<F>)
3713 FabArray<FAB>& y2, value_type a2, FabArray<FAB> const& x,
3714 int xcomp, int ycomp, int ncomp, IntVect const& nghost)
3715{
3716 AMREX_ASSERT(y1.boxArray() == y2.boxArray() &&
3717 y1.boxArray() == x.boxArray() &&
3719 y1.distributionMap == x.distributionMap &&
3720 y1.nGrowVect().allGE(nghost) &&
3721 y2.nGrowVect().allGE(nghost) &&
3722 x.nGrowVect().allGE(nghost));
3723
3724 BL_PROFILE("FabArray::Saypy_Saxpy()");
3725
3726#ifdef AMREX_USE_GPU
3727 if (Gpu::inLaunchRegion() && y1.isFusingCandidate()) {
3728 auto const& y1ma = y1.arrays();
3729 auto const& y2ma = y2.arrays();
3730 auto const& xma = x.const_arrays();
3731 ParallelFor(y1, nghost, ncomp,
3732 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
3733 {
3734 y1ma[box_no](i,j,k,ycomp+n) += a1 * y2ma[box_no](i,j,k,ycomp+n);
3735 y2ma[box_no](i,j,k,ycomp+n) += a2 * xma[box_no](i,j,k,xcomp+n);
3736 });
3737 if (!Gpu::inNoSyncRegion()) {
3739 }
3740 } else
3741#endif
3742 {
3743#ifdef AMREX_USE_OMP
3744#pragma omp parallel if (Gpu::notInLaunchRegion())
3745#endif
3746 for (MFIter mfi(y1,TilingIfNotGPU()); mfi.isValid(); ++mfi)
3747 {
3748 const Box& bx = mfi.growntilebox(nghost);
3749
3750 if (bx.ok()) {
3751 auto const& xfab = x.const_array(mfi);
3752 auto const& y1fab = y1.array(mfi);
3753 auto const& y2fab = y2.array(mfi);
3754 AMREX_HOST_DEVICE_PARALLEL_FOR_4D( bx, ncomp, i, j, k, n,
3755 {
3756 y1fab(i,j,k,ycomp+n) += a1 * y2fab(i,j,k,ycomp+n);
3757 y2fab(i,j,k,ycomp+n) += a2 * xfab(i,j,k,xcomp+n);
3758 });
3759 }
3760 }
3761 }
3762}
3763
3764template <class FAB>
3765template <class F>
3766requires (BaseFabType<F>)
3767void
3769 value_type a, const FabArray<FAB>& x, int xcomp,
3770 value_type b, const FabArray<FAB>& y, int ycomp,
3771 int dstcomp, int numcomp, const IntVect& nghost)
3772{
3773 AMREX_ASSERT(dst.boxArray() == x.boxArray());
3774 AMREX_ASSERT(dst.distributionMap == x.distributionMap);
3775 AMREX_ASSERT(dst.boxArray() == y.boxArray());
3776 AMREX_ASSERT(dst.distributionMap == y.distributionMap);
3777 AMREX_ASSERT(dst.nGrowVect().allGE(nghost) && x.nGrowVect().allGE(nghost) && y.nGrowVect().allGE(nghost));
3778
3779 BL_PROFILE("FabArray::LinComb()");
3780
3781#ifdef AMREX_USE_GPU
3782 if (Gpu::inLaunchRegion() && dst.isFusingCandidate()) {
3783 auto const& dstma = dst.arrays();
3784 auto const& xma = x.const_arrays();
3785 auto const& yma = y.const_arrays();
3786 ParallelFor(dst, nghost, numcomp,
3787 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
3788 {
3789 dstma[box_no](i,j,k,dstcomp+n) = a*xma[box_no](i,j,k,xcomp+n)
3790 + b*yma[box_no](i,j,k,ycomp+n);
3791 });
3792 if (!Gpu::inNoSyncRegion()) {
3794 }
3795 } else
3796#endif
3797 {
3798#ifdef AMREX_USE_OMP
3799#pragma omp parallel if (Gpu::notInLaunchRegion())
3800#endif
3801 for (MFIter mfi(dst,TilingIfNotGPU()); mfi.isValid(); ++mfi)
3802 {
3803 const Box& bx = mfi.growntilebox(nghost);
3804 auto const& xfab = x.const_array(mfi);
3805 auto const& yfab = y.const_array(mfi);
3806 auto const& dfab = dst.array(mfi);
3807 AMREX_HOST_DEVICE_PARALLEL_FOR_4D( bx, numcomp, i, j, k, n,
3808 {
3809 dfab(i,j,k,dstcomp+n) = a*xfab(i,j,k,xcomp+n) + b*yfab(i,j,k,ycomp+n);
3810 });
3811 }
3812 }
3813}
3814
3815template <class FAB>
3816template <typename BUF>
3817void
3819{
3820 BL_PROFILE("FabArray::FillBoundary()");
3821 if ( n_grow.max() > 0 ) {
3824 }
3825}
3826
3827template <class FAB>
3828template <typename BUF>
3829void
3830FabArray<FAB>::FillBoundary (const Periodicity& period, bool cross)
3831{
3832 BL_PROFILE("FabArray::FillBoundary()");
3833 if ( n_grow.max() > 0 ) {
3834 FillBoundary_nowait<BUF>(0, nComp(), n_grow, period, cross);
3836 }
3837}
3838
3839template <class FAB>
3840template <typename BUF>
3841void
3842FabArray<FAB>::FillBoundary (const IntVect& nghost, const Periodicity& period, bool cross)
3843{
3844 BL_PROFILE("FabArray::FillBoundary()");
3846 "FillBoundary: asked to fill more ghost cells than we have");
3847 if ( nghost.max() > 0 ) {
3848 FillBoundary_nowait<BUF>(0, nComp(), nghost, period, cross);
3850 }
3851}
3852
3853template <class FAB>
3854template <typename BUF>
3855void
3856FabArray<FAB>::FillBoundary (int scomp, int ncomp, bool cross)
3857{
3858 BL_PROFILE("FabArray::FillBoundary()");
3859 if ( n_grow.max() > 0 ) {
3862 }
3863}
3864
3865template <class FAB>
3866template <typename BUF>
3867void
3868FabArray<FAB>::FillBoundary (int scomp, int ncomp, const Periodicity& period, bool cross)
3869{
3870 BL_PROFILE("FabArray::FillBoundary()");
3871 if ( n_grow.max() > 0 ) {
3872 FillBoundary_nowait<BUF>(scomp, ncomp, n_grow, period, cross);
3874 }
3875}
3876
3877template <class FAB>
3878template <typename BUF>
3879void
3880FabArray<FAB>::FillBoundary (int scomp, int ncomp, const IntVect& nghost,
3881 const Periodicity& period, bool cross)
3882{
3883 BL_PROFILE("FabArray::FillBoundary()");
3885 "FillBoundary: asked to fill more ghost cells than we have");
3886 if ( nghost.max() > 0 ) {
3887 FillBoundary_nowait<BUF>(scomp, ncomp, nghost, period, cross);
3889 }
3890}
3891
3892template <class FAB>
3893template <typename BUF>
3894void
3899
3900template <class FAB>
3901template <typename BUF>
3902void
3904{
3905 FillBoundary_nowait<BUF>(0, nComp(), nGrowVect(), period, cross);
3906}
3907
3908template <class FAB>
3909template <typename BUF>
3910void
3911FabArray<FAB>::FillBoundary_nowait (const IntVect& nghost, const Periodicity& period, bool cross)
3912{
3913 FillBoundary_nowait<BUF>(0, nComp(), nghost, period, cross);
3914}
3915
3916template <class FAB>
3917template <typename BUF>
3918void
3919FabArray<FAB>::FillBoundary_nowait (int scomp, int ncomp, bool cross)
3920{
3922}
3923
3924template <class FAB>
3925void
3927{
3928 BL_PROFILE("FabArray::FillBoundaryAndSync()");
3929 if (n_grow.max() > 0 || !is_cell_centered()) {
3932 }
3933}
3934
3935template <class FAB>
3936void
3937FabArray<FAB>::FillBoundaryAndSync (int scomp, int ncomp, const IntVect& nghost,
3938 const Periodicity& period)
3939{
3940 BL_PROFILE("FabArray::FillBoundaryAndSync()");
3941 if (nghost.max() > 0 || !is_cell_centered()) {
3942 FillBoundaryAndSync_nowait(scomp, ncomp, nghost, period);
3944 }
3945}
3946
3947template <class FAB>
3948void
3953
3954template <class FAB>
3955void
3956FabArray<FAB>::FillBoundaryAndSync_nowait (int scomp, int ncomp, const IntVect& nghost,
3957 const Periodicity& period)
3958{
3959 BL_PROFILE("FillBoundaryAndSync_nowait()");
3960 FBEP_nowait(scomp, ncomp, nghost, period, false, false, true);
3961}
3962
3963template <class FAB>
3964void
3966{
3967 BL_PROFILE("FillBoundaryAndSync_finish()");
3969}
3970
3971template <class FAB>
3972void
3974{
3975 BL_PROFILE("FAbArray::OverrideSync()");
3976 if (!is_cell_centered()) {
3977 OverrideSync_nowait(0, nComp(), period);
3979 }
3980}
3981
3982template <class FAB>
3983void
3984FabArray<FAB>::OverrideSync (int scomp, int ncomp, const Periodicity& period)
3985{
3986 BL_PROFILE("FAbArray::OverrideSync()");
3987 if (!is_cell_centered()) {
3988 OverrideSync_nowait(scomp, ncomp, period);
3990 }
3991}
3992
3993template <class FAB>
3994void
3996{
3997 OverrideSync_nowait(0, nComp(), period);
3998}
3999
4000template <class FAB>
4001void
4002FabArray<FAB>::OverrideSync_nowait (int scomp, int ncomp, const Periodicity& period)
4003{
4004 BL_PROFILE("OverrideSync_nowait()");
4005 FBEP_nowait(scomp, ncomp, IntVect(0), period, false, false, true);
4006}
4007
4008template <class FAB>
4009void
4011{
4012 BL_PROFILE("OverrideSync_finish()");
4014}
4015
4016template <class FAB>
4017void
4018FabArray<FAB>::SumBoundary (const Periodicity& period, bool deterministic)
4019{
4020 SumBoundary(0, n_comp, IntVect(0), period, deterministic);
4021}
4022
4023template <class FAB>
4024void
4025FabArray<FAB>::SumBoundary (int scomp, int ncomp, const Periodicity& period, bool deterministic)
4026{
4027 SumBoundary(scomp, ncomp, IntVect(0), period, deterministic);
4028}
4029
4030template <class FAB>
4031void
4032FabArray<FAB>::SumBoundary (int scomp, int ncomp, IntVect const& nghost, const Periodicity& period, bool deterministic)
4033{
4034 SumBoundary(scomp, ncomp, this->nGrowVect(), nghost, period, deterministic);
4035}
4036
4037template <class FAB>
4038void
4039FabArray<FAB>::SumBoundary (int scomp, int ncomp, IntVect const& src_nghost, IntVect const& dst_nghost, const Periodicity& period, bool deterministic)
4040{
4041 BL_PROFILE("FabArray<FAB>::SumBoundary()");
4042
4043 SumBoundary_nowait(scomp, ncomp, src_nghost, dst_nghost, period, deterministic);
4045}
4046
4047template <class FAB>
4048void
4049FabArray<FAB>::SumBoundary_nowait (const Periodicity& period, bool deterministic)
4050{
4051 SumBoundary_nowait(0, n_comp, IntVect(0), period, deterministic);
4052}
4053
4054template <class FAB>
4055void
4056FabArray<FAB>::SumBoundary_nowait (int scomp, int ncomp, const Periodicity& period, bool deterministic)
4057{
4058 SumBoundary_nowait(scomp, ncomp, IntVect(0), period, deterministic);
4059}
4060
4061template <class FAB>
4062void
4063FabArray<FAB>::SumBoundary_nowait (int scomp, int ncomp, IntVect const& nghost, const Periodicity& period, bool deterministic)
4064{
4065 SumBoundary_nowait(scomp, ncomp, this->nGrowVect(), nghost, period, deterministic);
4066}
4067
4068template <class FAB>
4069void
4070FabArray<FAB>::SumBoundary_nowait (int scomp, int ncomp, IntVect const& src_nghost, IntVect const& dst_nghost, const Periodicity& period, bool deterministic)
4071{
4072 BL_PROFILE("FabArray<FAB>::SumBoundary_nowait()");
4073
4074 if ( n_grow == IntVect::TheZeroVector() && boxArray().ixType().cellCentered()) { return; }
4075
4076 AMREX_ALWAYS_ASSERT(src_nghost.allLE(n_grow) && dst_nghost.allLE(n_grow));
4077
4078 FBEP_nowait(scomp, ncomp, dst_nghost, period, false, false, false, src_nghost, deterministic);
4079}
4080
4081template <class FAB>
4082void
4084{
4085 BL_PROFILE("FabArray<FAB>::SumBoundary_finish()");
4087}
4088
4089template <class FAB>
4090void
4092{
4093 BL_PROFILE("FabArray::EnforcePeriodicity");
4094 if (period.isAnyPeriodic()) {
4095 FBEP_nowait(0, nComp(), nGrowVect(), period, false, true);
4096 FillBoundary_finish(); // unsafe unless isAnyPeriodic()
4097 }
4098}
4099
4100template <class FAB>
4101void
4102FabArray<FAB>::EnforcePeriodicity (int scomp, int ncomp, const Periodicity& period)
4103{
4104 BL_PROFILE("FabArray::EnforcePeriodicity");
4105 if (period.isAnyPeriodic()) {
4106 FBEP_nowait(scomp, ncomp, nGrowVect(), period, false, true);
4107 FillBoundary_finish(); // unsafe unless isAnyPeriodic()
4108 }
4109}
4110
4111template <class FAB>
4112void
4113FabArray<FAB>::EnforcePeriodicity (int scomp, int ncomp, const IntVect& nghost,
4114 const Periodicity& period)
4115{
4116 BL_PROFILE("FabArray::EnforcePeriodicity");
4117 if (period.isAnyPeriodic()) {
4118 FBEP_nowait(scomp, ncomp, nghost, period, false, true);
4119 FillBoundary_finish(); // unsafe unless isAnyPeriodic()
4120 }
4121}
4122
4123template <class FAB>
4124template <typename BUF>
4125void
4126FabArray<FAB>::FillBoundary_nowait (int scomp, int ncomp, const Periodicity& period, bool cross)
4127{
4128 FBEP_nowait<BUF>(scomp, ncomp, nGrowVect(), period, cross);
4129}
4130
4131template <class FAB>
4132template <typename BUF>
4133void
4134FabArray<FAB>::FillBoundary_nowait (int scomp, int ncomp, const IntVect& nghost,
4135 const Periodicity& period, bool cross)
4136{
4137 FBEP_nowait<BUF>(scomp, ncomp, nghost, period, cross);
4138}
4139
4140template <class FAB>
4141template <class F>
4142requires (BaseFabType<F>)
4143void
4144FabArray<FAB>::BuildMask (const Box& phys_domain, const Periodicity& period,
4145 value_type covered, value_type notcovered,
4147{
4148 BL_PROFILE("FabArray::BuildMask()");
4149
4150 int ncomp = this->nComp();
4151 const IntVect& ngrow = this->nGrowVect();
4152
4153 Box domain = amrex::convert(phys_domain, boxArray().ixType());
4154 for (int i = 0; i < AMREX_SPACEDIM; ++i) {
4155 if (period.isPeriodic(i)) {
4156 domain.grow(i, ngrow[i]);
4157 }
4158 }
4159
4160#ifdef AMREX_USE_GPU
4161 if (Gpu::inLaunchRegion() && this->isFusingCandidate()) {
4162 auto const& fa = this->arrays();
4163 ParallelFor(*this, ngrow, ncomp,
4164 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept
4165 {
4166 auto const& fab = fa[box_no];
4167 Box vbx(fab);
4168 vbx.grow(-ngrow);
4169 if (vbx.contains(i,j,k)) {
4170 fab(i,j,k,n) = interior;
4171 } else if (domain.contains(i,j,k)) {
4172 fab(i,j,k,n) = notcovered;
4173 } else {
4174 fab(i,j,k,n) = physbnd;
4175 }
4176 });
4177 if (!Gpu::inNoSyncRegion()) {
4179 }
4180 } else
4181#endif
4182 {
4183#ifdef AMREX_USE_OMP
4184#pragma omp parallel if (Gpu::notInLaunchRegion())
4185#endif
4186 for (MFIter mfi(*this,TilingIfNotGPU()); mfi.isValid(); ++mfi)
4187 {
4188 auto const& fab = this->array(mfi);
4189 Box const& fbx = mfi.growntilebox();
4190 Box const& gbx = fbx & domain;
4191 Box const& vbx = mfi.validbox();
4192 AMREX_HOST_DEVICE_FOR_4D(fbx, ncomp, i, j, k, n,
4193 {
4194 if (vbx.contains(i,j,k)) {
4195 fab(i,j,k,n) = interior;
4196 } else if (gbx.contains(i,j,k)) {
4197 fab(i,j,k,n) = notcovered;
4198 } else {
4199 fab(i,j,k,n) = physbnd;
4200 }
4201 });
4202 }
4203 }
4204
4205 const FabArrayBase::FB& TheFB = this->getFB(ngrow,period);
4206 setVal(covered, TheFB, 0, ncomp);
4207}
4208
4209template <class FAB>
4210template <class F>
4211requires (BaseFabType<F>)
4212void
4213FabArray<FAB>::setVal (value_type val, const CommMetaData& thecmd, int scomp, int ncomp)
4214{
4215 BL_PROFILE("FabArray::setVal(val, thecmd, scomp, ncomp)");
4216
4217#ifdef AMREX_USE_GPU
4218 if (Gpu::inLaunchRegion())
4219 {
4220 CMD_local_setVal_gpu(val, thecmd, scomp, ncomp);
4221 CMD_remote_setVal_gpu(val, thecmd, scomp, ncomp);
4222 if (!Gpu::inNoSyncRegion()) {
4224 }
4225 }
4226 else
4227#endif
4228 {
4229 AMREX_ASSERT(thecmd.m_LocTags && thecmd.m_RcvTags);
4230 const CopyComTagsContainer& LocTags = *(thecmd.m_LocTags);
4231 const MapOfCopyComTagContainers& RcvTags = *(thecmd.m_RcvTags);
4232 auto N_locs = static_cast<int>(LocTags.size());
4233#ifdef AMREX_USE_OMP
4234#pragma omp parallel for if (thecmd.m_threadsafe_loc)
4235#endif
4236 for (int i = 0; i < N_locs; ++i) {
4237 const CopyComTag& tag = LocTags[i];
4238 (*this)[tag.dstIndex].template setVal<RunOn::Host>(val, tag.dbox, scomp, ncomp);
4239 }
4240
4241 for (const auto & RcvTag : RcvTags) {
4242 auto N = static_cast<int>(RcvTag.second.size());
4243#ifdef AMREX_USE_OMP
4244#pragma omp parallel for if (thecmd.m_threadsafe_rcv)
4245#endif
4246 for (int i = 0; i < N; ++i) {
4247 const CopyComTag& tag = RcvTag.second[i];
4248 (*this)[tag.dstIndex].template setVal<RunOn::Host>(val, tag.dbox, scomp, ncomp);
4249 }
4250 }
4251 }
4252}
4253
4254template <class FAB>
4255template <class F>
4256requires (BaseFabType<F>)
4259{
4260 BL_PROFILE("FabArray::RecvLayoutMask()");
4261
4262 LayoutData<int> r(this->boxArray(), this->DistributionMap());
4263#ifdef AMREX_USE_OMP
4264#pragma omp parallel if (thecmd.m_threadsafe_rcv)
4265#endif
4266 for (MFIter mfi(r); mfi.isValid(); ++mfi) {
4267 r[mfi] = 0;
4268 }
4269
4270 const CopyComTagsContainer& LocTags = *(thecmd.m_LocTags);
4271 const MapOfCopyComTagContainers& RcvTags = *(thecmd.m_RcvTags);
4272
4273 auto N_locs = static_cast<int>(LocTags.size());
4274 for (int i = 0; i < N_locs; ++i) {
4275 const CopyComTag& tag = LocTags[i];
4276 r[tag.dstIndex] = 1;
4277 }
4278
4279 for (const auto & RcvTag : RcvTags) {
4280 auto N = static_cast<int>(RcvTag.second.size());
4281 for (int i = 0; i < N; ++i) {
4282 const CopyComTag& tag = RcvTag.second[i];
4283 r[tag.dstIndex] = 1;
4284 }
4285 }
4286 return r;
4287}
4288
4289template <class FAB>
4290template <typename F>
4291requires (BaseFabType<F>)
4292typename F::value_type
4293FabArray<FAB>::norminf (int comp, int ncomp, IntVect const& nghost, bool local,
4294 [[maybe_unused]] bool ignore_covered) const
4295{
4296 BL_PROFILE("FabArray::norminf()");
4297
4298 using RT = typename F::value_type;
4299
4300 auto nm0 = RT(0.0);
4301 Math::detail::AbsNanToInf<RT> const absinf{};
4302
4303#ifdef AMREX_USE_EB
4304 if ( this->is_cell_centered() && this->hasEBFabFactory() && ignore_covered )
4305 {
4306 const auto& ebfactory = dynamic_cast<EBFArrayBoxFactory const&>(this->Factory());
4307 auto const& flags = ebfactory.getMultiEBCellFlagFab();
4308#ifdef AMREX_USE_GPU
4309 if (Gpu::inLaunchRegion())
4310#endif
4311 {
4312 auto const& flagsma = flags.const_arrays();
4313 auto const& ma = this->const_arrays();
4314 nm0 = ParReduce(TypeList<ReduceOpMax>{}, TypeList<RT>{}, *this, nghost, ncomp,
4315 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept -> GpuTuple<RT>
4316 {
4317#ifdef AMREX_USE_GPU
4318 return flagsma[box_no](i,j,k).isCovered() ? RT(0.0) : absinf(ma[box_no](i,j,k,comp+n));
4319#else // unconditional load so that the loop vectorizes; zero covered
4320 // values before any arithmetic
4321 RT v = ma[box_no](i,j,k,comp+n);
4322 v = flagsma[box_no](i,j,k).isCovered() ? RT(0.0) : v;
4323 return absinf(v);
4324#endif
4325 });
4326 }
4327#ifdef AMREX_USE_GPU
4328 else {
4329#ifdef AMREX_USE_OMP
4330#pragma omp parallel reduction(max:nm0)
4331#endif
4332 for (MFIter mfi(*this,true); mfi.isValid(); ++mfi) {
4333 Box const& bx = mfi.growntilebox(nghost);
4334 if (flags[mfi].getType(bx) != FabType::covered) {
4335 auto const& flag = flags.const_array(mfi);
4336 auto const& a = this->const_array(mfi);
4337 AMREX_LOOP_4D(bx, ncomp, i, j, k, n,
4338 {
4339 if (!flag(i,j,k).isCovered()) {
4340 nm0 = std::max(nm0, absinf(a(i,j,k,comp+n)));
4341 }
4342 });
4343 }
4344 }
4345 }
4346#endif
4347 }
4348 else
4349#endif
4350 {
4351#ifdef AMREX_USE_GPU
4352 if (Gpu::inLaunchRegion())
4353#endif
4354 {
4355 auto const& ma = this->const_arrays();
4356 nm0 = ParReduce(TypeList<ReduceOpMax>{}, TypeList<RT>{}, *this, nghost, ncomp,
4357 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept -> GpuTuple<RT>
4358 {
4359 return absinf(ma[box_no](i,j,k,comp+n));
4360 });
4361 }
4362#ifdef AMREX_USE_GPU
4363 else {
4364#ifdef AMREX_USE_OMP
4365#pragma omp parallel reduction(max:nm0)
4366#endif
4367 for (MFIter mfi(*this,true); mfi.isValid(); ++mfi) {
4368 Box const& bx = mfi.growntilebox(nghost);
4369 auto const& a = this->const_array(mfi);
4370 AMREX_LOOP_4D(bx, ncomp, i, j, k, n,
4371 {
4372 nm0 = std::max(nm0, absinf(a(i,j,k,comp+n)));
4373 });
4374 }
4375 }
4376#endif
4377 }
4378
4379 nm0 = std::max(nm0, RT(0.0)); // ParReduce returns lowest() if this rank has no boxes
4380
4381 if (!local) {
4383 }
4384
4385 return nm0;
4386}
4387
4388template <class FAB>
4389template <typename IFAB, typename F>
4390requires (BaseFabType<F>)
4391typename F::value_type
4392FabArray<FAB>::norminf (FabArray<IFAB> const& mask, int comp, int ncomp,
4393 IntVect const& nghost, bool local) const
4394{
4395 BL_PROFILE("FabArray::norminf(mask)");
4396
4397 using RT = typename F::value_type;
4398
4399 auto nm0 = RT(0.0);
4400 Math::detail::AbsNanToInf<RT> const absinf{};
4401
4402#ifdef AMREX_USE_GPU
4403 if (Gpu::inLaunchRegion())
4404#endif
4405 {
4406 auto const& ma = this->const_arrays();
4407 auto const& maskma = mask.const_arrays();
4408 nm0 = ParReduce(TypeList<ReduceOpMax>{}, TypeList<RT>{}, *this, IntVect(nghost), ncomp,
4409 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k, int n) noexcept -> GpuTuple<RT>
4410 {
4411#ifdef AMREX_USE_GPU
4412 return maskma[box_no](i,j,k) ? absinf(ma[box_no](i,j,k,comp+n)) : RT(0.0);
4413#else // unconditional load so that the loop vectorizes; zero masked
4414 // values before any arithmetic, which could trap on garbage
4415 RT v = ma[box_no](i,j,k,comp+n);
4416 v = maskma[box_no](i,j,k) ? v : RT(0.0);
4417 return absinf(v);
4418#endif
4419 });
4420 }
4421#ifdef AMREX_USE_GPU
4422 else {
4423#ifdef AMREX_USE_OMP
4424#pragma omp parallel reduction(max:nm0)
4425#endif
4426 for (MFIter mfi(*this,true); mfi.isValid(); ++mfi) {
4427 Box const& bx = mfi.growntilebox(nghost);
4428 auto const& a = this->const_array(mfi);
4429 auto const& mskfab = mask.const_array(mfi);
4430 AMREX_LOOP_4D(bx, ncomp, i, j, k, n,
4431 {
4432 if (mskfab(i,j,k)) {
4433 nm0 = std::max(nm0, absinf(a(i,j,k,comp+n)));
4434 }
4435 });
4436 }
4437 }
4438#endif
4439
4440 nm0 = std::max(nm0, RT(0.0)); // ParReduce returns lowest() if this rank has no boxes
4441
4442 if (!local) {
4444 }
4445
4446 return nm0;
4447}
4448
4450
4451}
4452
4453#endif /*BL_FABARRAY_H*/
Runtime initialization/finalization helpers and global diagnostics.
Fixed-size array types for use on GPU and CPU.
#define BL_PROFILE(a)
Definition AMReX_BLProfiler.H:593
Assertion macros used across AMReX for runtime consistency checks.
#define AMREX_ALWAYS_ASSERT_WITH_MESSAGE(EX, MSG)
Definition AMReX_BLassert.H:49
#define AMREX_ASSERT(EX)
Definition AMReX_BLassert.H:38
#define AMREX_ALWAYS_ASSERT(EX)
Definition AMReX_BLassert.H:50
Helper routines for manipulating BaseFab data and related arrays.
BaseFab container template providing box-based field storage.
Container and helper utilities for sets of boxes on integer index space.
Disjoint-list representation of boxes and associated utilities.
Integer-lattice boxes and helpers for defining index-space regions.
Maps FABs in a FabArray to MPI processes.
#define AMREX_FORCE_INLINE
Definition AMReX_Extension.H:123
#define AMREX_RESTRICT
Definition AMReX_Extension.H:37
Common infrastructure shared by FabArray specializations.
Inline FabArray communication routines: FillBoundary, ParallelCopy, ParallelCopyToGhost,...
Abstract factory interface for creating and destroying FAB objects.
Problem-domain geometry: maps between index space and physical space.
#define AMREX_HOST_DEVICE_FOR_4D(...)
Definition AMReX_GpuLaunchMacrosC.nolint.H:107
#define AMREX_HOST_DEVICE_PARALLEL_FOR_4D(...)
Definition AMReX_GpuLaunchMacrosC.nolint.H:111
#define AMREX_IF_ON_DEVICE(CODE)
Definition AMReX_GpuQualifiers.H:56
#define AMREX_IF_ON_HOST(CODE)
Definition AMReX_GpuQualifiers.H:58
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
#define AMREX_GPU_HOST_DEVICE
Definition AMReX_GpuQualifiers.H:20
Convenience header for the core AMReX GPU facilities.
#define AMREX_LOOP_3D(bx, i, j, k, block)
#define AMREX_LOOP_4D(bx, ncomp, i, j, k, n, block)
while parser_exe_t p
Definition AMReX_Parser_Exe_Body.H:33
Abstract base class for memory arenas.
Definition AMReX_Arena.H:154
virtual void free(void *pt)=0
Free a previously allocated block pointed to by pt.
virtual void * alloc(std::size_t sz)=0
Allocate sz bytes from this arena.
A FortranArrayBox(FAB)-like object.
Definition AMReX_BaseFab.H:243
Reference-counted collection of Boxes.
Definition AMReX_BoxArray.H:681
__host__ __device__ BoxND & grow(int i) noexcept
Grow in all directions by i cells (negative shrinks).
Definition AMReX_Box.H:668
__host__ __device__ IntVectND< dim > length() const noexcept
Return the length of the BoxND.
Definition AMReX_Box.H:167
__host__ __device__ bool contains(const IntVectND< dim > &p) const noexcept
Return true if argument is contained within BoxND.
Definition AMReX_Box.H:233
__host__ __device__ BoxND & shift(int dir, int nzones) noexcept
Shift this BoxND nzones indexing positions in coordinate direction dir.
Definition AMReX_Box.H:538
__host__ __device__ BoxND & setRange(int dir, int sm_index, int n_cells=1) noexcept
Set the entire range in a given direction, starting at sm_index with length n_cells....
Definition AMReX_Box.H:1143
__host__ __device__ bool ok() const noexcept
Return true if high bounds are >= low bounds and the index type is valid.
Definition AMReX_Box.H:229
__host__ __device__ const IntVectND< dim > & smallEnd() const &noexcept
Return the inclusive lower bound of the box.
Definition AMReX_Box.H:124
Default FabFactory that calls new / delete on the FAB type directly.
Definition AMReX_FabFactory.H:124
Calculates the distribution of FABs to MPI processes.
Definition AMReX_DistributionMapping.H:55
Definition AMReX_EBFabFactory.H:32
const FabArray< EBCellFlagFab > & getMultiEBCellFlagFab() const noexcept
EB cell flags for all boxes.
Definition AMReX_EBFabFactory.H:88
IntVect nGrowVect() const noexcept
Definition AMReX_FabArrayBase.H:85
void clearThisBD(bool no_assertion=false) const
clear BD count and caches associated with this BD, if no other is using this BD.
Definition AMReX_FabArrayBase.cpp:2610
void clear()
Clear this object's BoxArray, DistributionMapping, local index/ownership data, and BDKey so the conta...
Definition AMReX_FabArrayBase.cpp:209
friend class MFIter
Definition AMReX_FabArrayBase.H:48
Vector< int > indexArray
Definition AMReX_FabArrayBase.H:537
FabArrayBase()=default
static FabArrayStats m_FA_stats
Definition AMReX_FabArrayBase.H:893
static bool getAllocSingleChunk()
Query whether single-chunk allocation is enabled for new FabArrays.
Definition AMReX_FabArrayBase.H:904
void addThisBD()
add the current BD into BD count database
Definition AMReX_FabArrayBase.cpp:2640
int n_comp
Definition AMReX_FabArrayBase.H:540
const FB & getFB(const IntVect &nghost, const Periodicity &period, bool cross=false, bool enforce_periodicity_only=false, bool override_sync=false, IntVect const &sumboundary_src_nghost=IntVect(-1)) const
Retrieve (and cache) the FillBoundary metadata for the requested halo.
Definition AMReX_FabArrayBase.cpp:1308
bool isFusingCandidate() const noexcept
Is this a good candidate for kernel fusing?
Definition AMReX_FabArrayBase.cpp:2795
bool is_cell_centered() const noexcept
This tests on whether the FabArray is cell-centered.
Definition AMReX_FabArrayBase.cpp:2789
IndexType ixType() const noexcept
Return index type.
Definition AMReX_FabArrayBase.H:91
FabArrayBase & operator=(const FabArrayBase &rhs)=default
void define(const BoxArray &bxs, const DistributionMapping &dm, int nvar, int ngrow)
Definition AMReX_FabArrayBase.cpp:177
static void updateMemUsage(std::string const &tag, Long nbytes, Arena const *ar)
Accumulate memory usage statistics for diagnostics.
Definition AMReX_FabArrayBase.cpp:2717
CopyComTag::CopyComTagsContainer CopyComTagsContainer
Definition AMReX_FabArrayBase.H:236
CopyComTag::MapOfCopyComTagContainers MapOfCopyComTagContainers
Definition AMReX_FabArrayBase.H:237
int localindex(int K) const noexcept
Return local index in the vector of FABs.
Definition AMReX_FabArrayBase.H:124
const DistributionMapping & DistributionMap() const noexcept
Return constant reference to associated DistributionMapping.
Definition AMReX_FabArrayBase.H:135
int local_size() const noexcept
Return the number of local FABs in the FabArray.
Definition AMReX_FabArrayBase.H:118
IntVect n_grow
Definition AMReX_FabArrayBase.H:539
CpOp
parallel copy or add
Definition AMReX_FabArrayBase.H:432
@ ADD
Definition AMReX_FabArrayBase.H:432
@ COPY
Definition AMReX_FabArrayBase.H:432
Box box(int K) const noexcept
Return the Kth Box in the BoxArray. That is, the valid region of the Kth grid.
Definition AMReX_FabArrayBase.H:106
DistributionMapping distributionMap
Definition AMReX_FabArrayBase.H:536
Box fabbox(int K) const noexcept
Return the Kth FABs Box in the FabArray. That is, the region the Kth fab is actually defined on.
Definition AMReX_FabArrayBase.cpp:219
BoxArray boxarray
Definition AMReX_FabArrayBase.H:535
int nComp() const noexcept
Return number of variables (aka components) associated with each point.
Definition AMReX_FabArrayBase.H:88
const BoxArray & boxArray() const noexcept
Return a constant reference to the BoxArray that defines the valid region associated with this FabArr...
Definition AMReX_FabArrayBase.H:100
An Array of FortranArrayBox(FAB)-like Objects.
Definition AMReX_FabArray.H:383
void ParallelCopy(const FabArray< FAB > &src, int src_comp, int dest_comp, int num_comp, const IntVect &snghost, const IntVect &dnghost, const IntVect &offset, const Periodicity &period)
This function copies data from src to this FabArray.
Definition AMReX_FabArrayCommI.H:370
void ParallelCopyToGhost_finish()
Finish the current ParallelCopyToGhost_nowait() operation.
Definition AMReX_FabArrayCommI.H:426
void setFab(int boxno, std::unique_ptr< FAB > elem)
Explicitly set the Kth FAB in the FabArray to point to elem.
Definition AMReX_FabArray.H:2750
void copy(const FabArray< FAB > &src, int src_comp, int dest_comp, int num_comp, int src_nghost, int dst_nghost, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY)
Definition AMReX_FabArray.H:1357
TagVector< CommSendBufTag< value_type > > const * get_send_copy_tag_vector(Vector< char * > const &send_data, Vector< std::size_t > const &send_size, Vector< CopyComTagsContainer const * > const &send_cctc, int ncomp, std::uint64_t id) const
void SumBoundary_nowait(int scomp, int ncomp, const Periodicity &period=Periodicity::NonPeriodic(), bool deterministic=false)
Definition AMReX_FabArray.H:4056
FBOnArrivalMeta< FAB > * FB_on_arrival_meta(const FB &TheFB, int ncomp)
void mult(value_type val, int comp, int num_comp, int nghost=0)
void copy(const FabArray< FAB > &src, int src_comp, int dest_comp, int num_comp, const IntVect &src_nghost, const IntVect &dst_nghost, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY)
Definition AMReX_FabArray.H:1368
void * m_dp_arrays
Definition AMReX_FabArray.H:1822
void ParallelCopy(const FabArray< FAB > &src, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY)
Definition AMReX_FabArray.H:999
void setFab(const MFIter &mfi, std::unique_ptr< FAB > elem)
Explicitly set the FAB associated with mfi in the FabArray to point to elem.
Definition AMReX_FabArray.H:2796
typename std::conditional_t< IsBaseFab< FAB >::value, FAB, FABType >::value_type value_type
Definition AMReX_FabArray.H:394
void capacityOfFabs(LayoutData< Int > &mem) const
std::unique_ptr< FabArray< BaseFab< GpuComplex< Real > > > > os_temp
Definition AMReX_FabArray.H:1970
static void Saxpy_Xpay(FabArray< BaseFab< GpuComplex< Real > > > &y, value_type a1, FabArray< BaseFab< GpuComplex< Real > > > const &x1, value_type a2, FabArray< BaseFab< GpuComplex< Real > > > const &x2, int xcomp, int ycomp, int ncomp, IntVect const &nghost)
void FillBoundary(const IntVect &nghost, const Periodicity &period, bool cross=false)
Definition AMReX_FabArray.H:3842
const FabFactory< FAB > & Factory() const noexcept
Factory used to create FAB instances for this FabArray.
Definition AMReX_FabArray.H:499
void FBEP_nowait(int scomp, int ncomp, const IntVect &nghost, const Periodicity &period, bool cross, bool enforce_periodicity_only=false, bool override_sync=false, IntVect const &sumboundary_src_nghost=IntVect(-1), bool deterministic=false)
void CMD_remote_setVal_gpu(value_type x, const CommMetaData &thecmd, int scomp, int ncomp)
Definition AMReX_FBI.H:723
void Redistribute(const FabArray< FAB > &src, int scomp, int dcomp, int ncomp, const IntVect &nghost)
Copy from src to this. this and src have the same BoxArray, but different DistributionMapping.
Definition AMReX_FabArrayCommI.H:1037
LayoutData< int > RecvLayoutMask(const CommMetaData &thecmd)
void FillBoundary_nowait(const Periodicity &period, bool cross=false)
Definition AMReX_FabArray.H:3903
static void LinComb(FabArray< BaseFab< GpuComplex< Real > > > &dst, value_type a, const FabArray< BaseFab< GpuComplex< Real > > > &x, int xcomp, value_type b, const FabArray< BaseFab< GpuComplex< Real > > > &y, int ycomp, int dstcomp, int numcomp, const IntVect &nghost)
void shift(const IntVect &v)
Shift the boxarray by vector v.
Definition AMReX_FabArray.H:3485
FAB * release(int K)
Release ownership of the FAB. This function is not thread safe.
Definition AMReX_FabArray.H:2242
bool ok() const
Return true if the FabArray is well-defined. That is, the FabArray has a BoxArray and DistributionMap...
Definition AMReX_FabArray.H:2531
const FAB & operator[](const MFIter &mfi) const noexcept
Return a constant reference to the FAB associated with mfi.
Definition AMReX_FabArray.H:583
void ParallelAdd_nowait(const FabArray< FAB > &src, const Periodicity &period=Periodicity::NonPeriodic())
Start an asynchronous ParallelAdd over all components.
Definition AMReX_FabArray.H:1018
FabArray(FabArray< FAB > &&rhs) noexcept
Definition AMReX_FabArray.H:2448
void ParallelCopy(const FabArray< FAB > &src, int scomp, int dcomp, int ncomp, const IntVect &snghost, const IntVect &dnghost, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY, const FabArrayBase::CPC *a_cpc=nullptr, bool deterministic=false)
Definition AMReX_FabArrayCommI.H:350
FabArray(const BoxArray &bxs, const DistributionMapping &dm, int nvar, const IntVect &ngrow, const MFInfo &info=MFInfo(), const FabFactory< FAB > &factory=DefaultFabFactory< FAB >())
Construct a FabArray with a per-direction grow vector.
Definition AMReX_FabArray.H:2412
bool defined(const MFIter &mfi) const noexcept
Definition AMReX_FabArray.H:2105
void prefetchToHost(const MFIter &mfi) const noexcept
Request that the FAB selected by iterator mfi be prefetched to host memory, where supported.
Definition AMReX_FabArray.H:620
Arena * m_default_arena
Definition AMReX_FabArray.H:1807
void OverrideSync_nowait(int scomp, int ncomp, const Periodicity &period)
Definition AMReX_FabArray.H:4002
FabArray(const FabArray< FAB > &rhs)=delete
void ParallelCopyToGhost(const FabArray< FAB > &src, int scomp, int dcomp, int ncomp, const IntVect &snghost, const IntVect &dnghost, const Periodicity &period=Periodicity::NonPeriodic())
Copy source data to destination ghost cells only.
Definition AMReX_FabArrayCommI.H:395
MultiArray4< typename FabArray< FAB >::value_type > arrays() noexcept
Build, if needed, and return mutable Array4 views for local FABs.
Definition AMReX_FabArray.H:741
static void unpack_recv_buffer_gpu(FabArray< FAB > &dst, int dcomp, int ncomp, Vector< char * > const &recv_data, Vector< std::size_t > const &recv_size, Vector< const CopyComTagsContainer * > const &recv_cctc, CpOp op, bool is_thread_safe, std::uint64_t id, bool deterministic)
Unpack received messages into dst on the GPU.
void ParallelCopy_finish()
Finish the current nowait ParallelCopy or ParallelAdd operation.
Definition AMReX_FabArrayCommI.H:723
FAB & get(int K) noexcept
Return a reference to the FAB associated with the Kth element.
Definition AMReX_FabArray.H:604
void ParallelAdd(const FabArray< FAB > &src, int src_comp, int dest_comp, int num_comp, const IntVect &snghost, const IntVect &dnghost, const IntVect &offset, const Periodicity &period)
This function adds data from src to this FabArray.
Definition AMReX_FabArrayCommI.H:382
FAB const * fabPtr(int K) const noexcept
Definition AMReX_FabArray.H:2147
Array4< typename FabArray< FAB >::value_type > array(const MFIter &mfi, int start_comp) noexcept
Definition AMReX_FabArray.H:700
void SumBoundary_nowait(const Periodicity &period=Periodicity::NonPeriodic(), bool deterministic=false)
Definition AMReX_FabArray.H:4049
static void Saxpy_Saxpy(FabArray< BaseFab< GpuComplex< Real > > > &y1, value_type a1, FabArray< BaseFab< GpuComplex< Real > > > const &x1, FabArray< BaseFab< GpuComplex< Real > > > &y2, value_type a2, FabArray< BaseFab< GpuComplex< Real > > > const &x2, int xcomp, int ycomp, int ncomp, IntVect const &nghost)
void clear_arrays()
Free the cached Array4s so that they are rebuilt by the next arrays() call.
Definition AMReX_FabArray.H:2204
void ParallelCopy(const FabArray< FAB > &src, int src_comp, int dest_comp, int num_comp, int src_nghost, int dst_nghost, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY)
Definition AMReX_FabArray.H:1123
Long m_single_chunk_size
Definition AMReX_FabArray.H:1809
FAB & get(const MFIter &mfi) noexcept
Returns a reference to the FAB associated mfi.
Definition AMReX_FabArray.H:592
void OverrideSync_nowait(const Periodicity &period=Periodicity::NonPeriodic())
Split-phase version of OverrideSync.
Definition AMReX_FabArray.H:3995
std::unique_ptr< PCData< BaseFab< GpuComplex< Real > > > > pcd
Definition AMReX_FabArray.H:1967
void define(const BoxArray &bxs, const DistributionMapping &dm, int nvar, int ngrow, const MFInfo &info=MFInfo(), const FabFactory< FAB > &factory=DefaultFabFactory< FAB >())
Define this FabArray identically to that performed by the constructor having an analogous function si...
Definition AMReX_FabArray.H:2566
void setDomainBndry(value_type val, const Geometry &geom)
void ParallelAdd_nowait(const FabArray< FAB > &src, int src_comp, int dest_comp, int num_comp, const IntVect &src_nghost, const IntVect &dst_nghost, const Periodicity &period=Periodicity::NonPeriodic())
Asynchronous ParallelAdd with per-direction ghost-cell growth.
Definition AMReX_FabArray.H:1179
void PC_local_cpu(const CPC &thecpc, FabArray< FAB > const &src, int scomp, int dcomp, int ncomp, CpOp op)
Definition AMReX_PCI.H:8
std::unique_ptr< FBData< BaseFab< GpuComplex< Real > > > > fbd
Definition AMReX_FabArray.H:1966
std::unique_ptr< detail::SingleChunkArena > m_single_chunk_arena
Definition AMReX_FabArray.H:1808
void SumBoundary_nowait(int scomp, int ncomp, IntVect const &src_nghost, IntVect const &dst_nghost, const Periodicity &period=Periodicity::NonPeriodic(), bool deterministic=false)
Definition AMReX_FabArray.H:4070
FabArray(const FabArray< FAB > &rhs, MakeType maketype, int scomp, int ncomp)
Construct a component alias of rhs.
Definition AMReX_FabArray.H:2426
void ParallelAdd(const FabArray< FAB > &src, const Periodicity &period=Periodicity::NonPeriodic())
This function copies data from src to this FabArray. Each FAB in fa is intersected with all FABs in t...
Definition AMReX_FabArray.H:996
FAB fab_type
Definition AMReX_FabArray.H:396
void ParallelCopy_nowait(const FabArray< FAB > &src, int scomp, int dcomp, int ncomp, const IntVect &snghost, const IntVect &dnghost, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY, const FabArrayBase::CPC *a_cpc=nullptr, bool to_ghost_cells_only=false, bool deterministic=false)
Asynchronous ParallelCopy with per-direction ghost-cell growth.
Definition AMReX_FabArrayCommI.H:434
void FB_local_add_cpu(const FB &TheFB, int scomp, int ncomp)
Definition AMReX_FBI.H:467
Array4< typename FabArray< FAB >::value_type const > array(int K) const noexcept
Read-only Array4 view for the FAB identified by global index K.
Definition AMReX_FabArray.H:663
bool SharedMemory() const noexcept
Definition AMReX_FabArray.H:1880
void FB_on_arrival_nowait(const FB &TheFB, int scomp, int ncomp, int SeqNum, bool sumboundary)
Array4< typename FabArray< FAB >::value_type const > const_array(int K) const noexcept
Read-only Array4 view for the FAB identified by global index K.
Definition AMReX_FabArray.H:684
void prefetchToDevice(const MFIter &mfi) const noexcept
Request that the FAB selected by iterator mfi be prefetched to device memory, where supported.
Definition AMReX_FabArray.H:636
void FillBoundary_nowait(const IntVect &nghost, const Periodicity &period, bool cross=false)
Definition AMReX_FabArray.H:3911
Vector< std::string > m_tags
Definition AMReX_FabArray.H:1828
void ParallelCopyToGhost_nowait(const FabArray< FAB > &src, int scomp, int dcomp, int ncomp, const IntVect &snghost, const IntVect &dnghost, const Periodicity &period=Periodicity::NonPeriodic())
Start an asynchronous version of ParallelCopyToGhost().
Definition AMReX_FabArrayCommI.H:412
static void unpack_recv_buffer_cpu(FabArray< FAB > &dst, int dcomp, int ncomp, Vector< char * > const &recv_data, Vector< std::size_t > const &recv_size, Vector< const CopyComTagsContainer * > const &recv_cctc, CpOp op, bool is_thread_safe)
void ParallelAdd_nowait(const FabArray< FAB > &src, int src_comp, int dest_comp, int num_comp, const Periodicity &period=Periodicity::NonPeriodic())
Start an asynchronous ParallelAdd for a component subset.
Definition AMReX_FabArray.H:1078
void FB_local_add_gpu(const FB &TheFB, int scomp, int ncomp, bool deterministic)
Definition AMReX_FBI.H:630
void FB_local_copy_gpu(const FB &TheFB, int scomp, int ncomp)
Definition AMReX_FBI.H:558
void LocalCopy(FabArray< SFAB > const &src, int scomp, int dcomp, int ncomp, IntVect const &nghost)
void FB_on_arrival_finish()
Arena * arena() const noexcept
Arena configured for allocations by this FabArray.
Definition AMReX_FabArray.H:508
DataAllocator m_dallocator
Definition AMReX_FabArray.H:1805
void copy(const FabArray< FAB > &src, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY)
Definition AMReX_FabArray.H:1005
Array4< typename FabArray< FAB >::value_type > array(const MFIter &mfi) noexcept
Mutable Array4 view for the FAB referenced by iterator mfi.
Definition AMReX_FabArray.H:656
void FillBoundaryAndSync_finish()
Definition AMReX_FabArray.H:3965
void ParallelCopy_nowait(const FabArray< FAB > &src, int src_comp, int dest_comp, int num_comp, int src_nghost, int dst_nghost, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY)
Asynchronous ParallelCopy with scalar ghost-cell growth.
Definition AMReX_FabArray.H:1203
void ParallelCopy_nowait(const FabArray< FAB > &src, int scomp, int dcomp, int ncomp, const IntVect &snghost, const IntVect &dnghost, const IntVect &offset, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY, const FabArrayBase::CPC *a_cpc=nullptr, bool to_ghost_cells_only=false, bool deterministic=false)
Asynchronous ParallelCopy with per-direction ghost-cell growth and a spatial offset.
Definition AMReX_FabArrayCommI.H:452
void FillBoundary_nowait(bool cross=false)
Definition AMReX_FabArray.H:3895
void clear()
Releases FAB memory in the FabArray.
Definition AMReX_FabArray.H:2283
void FillBoundary_nowait(int scomp, int ncomp, const IntVect &nghost, const Periodicity &period, bool cross=false)
Definition AMReX_FabArray.H:4134
const Vector< std::string > & tags() const noexcept
Memory-usage tags recorded for this FabArray's allocations.
Definition AMReX_FabArray.H:516
void CMD_local_setVal_gpu(value_type x, const CommMetaData &thecmd, int scomp, int ncomp)
Definition AMReX_FBI.H:693
MultiArray4< typename FabArray< FAB >::value_type const > arrays() const noexcept
Build, if needed, and return read-only Array4 views for local FABs.
Definition AMReX_FabArray.H:751
void FillBoundary_nowait(int scomp, int ncomp, bool cross=false)
Definition AMReX_FabArray.H:3919
value_type * singleChunkPtr() noexcept
Definition AMReX_FabArray.H:532
std::vector< BaseFab< GpuComplex< Real > > * > m_fabs_v
Definition AMReX_FabArray.H:1819
static void pack_send_buffer_gpu(FabArray< FAB > const &src, int scomp, int ncomp, Vector< char * > const &send_data, Vector< std::size_t > const &send_size, Vector< const CopyComTagsContainer * > const &send_cctc, std::uint64_t id)
void ParallelAdd(const FabArray< FAB > &src, int src_comp, int dest_comp, int num_comp, const IntVect &src_nghost, const IntVect &dst_nghost, const Periodicity &period=Periodicity::NonPeriodic())
Definition AMReX_FabArray.H:1115
bool define_function_called
Definition AMReX_FabArray.H:1815
void ParallelAdd(const FabArray< FAB > &src, int src_comp, int dest_comp, int num_comp, const Periodicity &period=Periodicity::NonPeriodic())
This function copies data from src to this FabArray. Each FAB in src is intersected with all FABs in ...
Definition AMReX_FabArray.H:1044
FabArray() noexcept
Constructs an empty FabArray<FAB>.
Definition AMReX_FabArray.H:2386
void FillBoundary_test()
Definition AMReX_FabArrayCommI.H:1064
void plus(value_type val, int comp, int num_comp, int nghost=0)
bool defined(int K) const noexcept
Definition AMReX_FabArray.H:2092
FAB * fabPtr(int K) noexcept
Definition AMReX_FabArray.H:2138
void ParallelAdd(const FabArray< FAB > &src, int src_comp, int dest_comp, int num_comp, int src_nghost, int dst_nghost, const Periodicity &period=Periodicity::NonPeriodic())
Similar to the above function, except that source and destination are grown by src_nghost and dst_ngh...
Definition AMReX_FabArray.H:1106
const FAB & atLocalIdx(int L) const noexcept
Definition AMReX_FabArray.H:608
MultiArray4< typename FabArray< FAB >::value_type const > const_arrays() const noexcept
Read-only convenience wrapper equivalent to arrays() const.
Definition AMReX_FabArray.H:759
std::unique_ptr< FabFactory< BaseFab< GpuComplex< Real > > > > m_factory
Definition AMReX_FabArray.H:1804
void copy(const FabArray< FAB > &src, int src_comp, int dest_comp, int num_comp, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY)
Definition AMReX_FabArray.H:1059
void invert(value_type numerator, int comp, int num_comp, int nghost=0)
void copyTo(FAB &dest, int nghost=0) const
Copy the values contained in the intersection of the valid + nghost region of this FabArray with the ...
Definition AMReX_FabArray.H:3059
void ParallelAdd_nowait(const FabArray< FAB > &src, int src_comp, int dest_comp, int num_comp, int src_nghost, int dst_nghost, const Periodicity &period=Periodicity::NonPeriodic())
Asynchronous ParallelAdd with scalar ghost-cell growth.
Definition AMReX_FabArray.H:1156
Array4< typename FabArray< FAB >::value_type const > array(const MFIter &mfi) const noexcept
Read-only Array4 view for the FAB referenced by iterator mfi.
Definition AMReX_FabArray.H:649
void FillBoundary_nowait(int scomp, int ncomp, const Periodicity &period, bool cross=false)
Definition AMReX_FabArray.H:4126
const FAB & get(const MFIter &mfi) const noexcept
Return a constant reference to the FAB associated with mfi.
Definition AMReX_FabArray.H:586
void SumBoundary_finish()
Definition AMReX_FabArray.H:4083
std::size_t singleChunkSize() const noexcept
Definition AMReX_FabArray.H:544
void SumBoundary_nowait(int scomp, int ncomp, IntVect const &nghost, const Periodicity &period=Periodicity::NonPeriodic(), bool deterministic=false)
Definition AMReX_FabArray.H:4063
void FB_local_copy_cpu(const FB &TheFB, int scomp, int ncomp)
Definition AMReX_FBI.H:396
void abs(int comp, int ncomp, int nghost=0)
static void Saxpy(FabArray< BaseFab< GpuComplex< Real > > > &y, value_type a, FabArray< BaseFab< GpuComplex< Real > > > const &x, int xcomp, int ycomp, int ncomp, IntVect const &nghost)
MultiArray4< value_type > m_arrays
Definition AMReX_FabArray.H:1825
void FillBoundaryAndSync_nowait(const Periodicity &period=Periodicity::NonPeriodic())
Definition AMReX_FabArray.H:3949
void PC_local_gpu(const CPC &thecpc, FabArray< FAB > const &src, int scomp, int dcomp, int ncomp, CpOp op, bool deterministic)
Definition AMReX_PCI.H:98
void FillBoundaryAndSync_nowait(int scomp, int ncomp, const IntVect &nghost, const Periodicity &period)
Definition AMReX_FabArray.H:3956
void BuildMask(const Box &phys_domain, const Periodicity &period, value_type covered, value_type notcovered, value_type physbnd, value_type interior)
void * m_hp_arrays
Definition AMReX_FabArray.H:1824
static void Saypy_Saxpy(FabArray< BaseFab< GpuComplex< Real > > > &y1, value_type a1, FabArray< BaseFab< GpuComplex< Real > > > &y2, value_type a2, FabArray< BaseFab< GpuComplex< Real > > > const &x, int xcomp, int ycomp, int ncomp, IntVect const &nghost)
ShMem shmem
Definition AMReX_FabArray.H:1878
FabArray< FAB > & operator=(FabArray< FAB > &&rhs) noexcept
Definition AMReX_FabArray.H:2483
Array4< typename FabArray< FAB >::value_type > array(int K, int start_comp) noexcept
Definition AMReX_FabArray.H:716
MultiArray4< value_type const > m_const_arrays
Definition AMReX_FabArray.H:1826
void ParallelCopy(const FabArray< FAB > &src, int src_comp, int dest_comp, int num_comp, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY)
Definition AMReX_FabArray.H:1050
bool hasEBFabFactory() const noexcept
Return whether this FabArray uses an EBFArrayBoxFactory.
Definition AMReX_FabArray.H:521
TagVector< Array4CopyTag< value_type > > const * FB_get_local_copy_tag_vector(const FB &TheFB)
Definition AMReX_FBI.H:515
Array4< typename FabArray< FAB >::value_type const > array(int K, int start_comp) const noexcept
Definition AMReX_FabArray.H:708
bool isAllRegular() const noexcept
Return the EB factory's all-regular status, or true for other factories.
Definition AMReX_FabArray.H:549
Array4< typename FabArray< FAB >::value_type > array(int K) noexcept
Mutable Array4 view for the FAB identified by global index K.
Definition AMReX_FabArray.H:670
static void pack_send_buffer_cpu(FabArray< FAB > const &src, int scomp, int ncomp, Vector< char * > const &send_data, Vector< std::size_t > const &send_size, Vector< const CopyComTagsContainer * > const &send_cctc)
FAB * fabPtr(const MFIter &mfi) noexcept
Return pointer to FAB.
Definition AMReX_FabArray.H:2118
void copyTo(FAB &dest, int scomp, int dcomp, int ncomp, int nghost=0) const
Copy the values contained in the intersection of the num_comp component valid + nghost region of this...
Definition AMReX_FabArrayCommI.H:804
void OverrideSync_finish()
Definition AMReX_FabArray.H:4010
FAB * release(const MFIter &mfi)
Release ownership of the FAB. This function is not thread safe.
Definition AMReX_FabArray.H:2263
Array4< typename FabArray< FAB >::value_type const > const_array(const MFIter &mfi, int start_comp) const noexcept
Definition AMReX_FabArray.H:724
static void Xpay(FabArray< BaseFab< GpuComplex< Real > > > &y, value_type a, FabArray< BaseFab< GpuComplex< Real > > > const &x, int xcomp, int ycomp, int ncomp, IntVect const &nghost)
TagVector< CommRecvBufTag< value_type > > const * get_recv_copy_tag_vector(Vector< char * > const &recv_data, Vector< std::size_t > const &recv_size, Vector< CopyComTagsContainer const * > const &recv_cctc, int ncomp, std::uint64_t id)
FAB & atLocalIdx(int L) noexcept
Return a reference to the FAB associated with local index L.
Definition AMReX_FabArray.H:607
void define(const BoxArray &bxs, const DistributionMapping &dm, int nvar, const IntVect &ngrow, const MFInfo &info=MFInfo(), const FabFactory< FAB > &factory=DefaultFabFactory< FAB >())
Define this FabArray with a per-direction grow vector.
Definition AMReX_FabArray.H:2578
void ParallelCopy_nowait(const FabArray< FAB > &src, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY)
Start an asynchronous ParallelCopy that copies every component.
Definition AMReX_FabArray.H:1031
void LocalAdd(FabArray< BaseFab< GpuComplex< Real > > > const &src, int scomp, int dcomp, int ncomp, IntVect const &nghost)
Array4< typename FabArray< FAB >::value_type const > const_array(const MFIter &mfi) const noexcept
Synonym for array(const MFIter&) that highlights read-only semantics.
Definition AMReX_FabArray.H:677
const FAB & get(int K) const noexcept
Return a constant reference to the FAB associated with the Kth element.
Definition AMReX_FabArray.H:598
FAB const * fabPtr(const MFIter &mfi) const noexcept
Definition AMReX_FabArray.H:2128
void ParallelCopy_nowait(const FabArray< FAB > &src, int src_comp, int dest_comp, int num_comp, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY)
Start an asynchronous ParallelCopy for a component subset.
Definition AMReX_FabArray.H:1097
bool isDefined() const
Definition AMReX_FabArray.H:2559
value_type const * singleChunkPtr() const noexcept
Definition AMReX_FabArray.H:538
~FabArray()
The destructor – deletes all FABs in the array.
Definition AMReX_FabArray.H:2523
Array4< typename FabArray< FAB >::value_type const > array(const MFIter &mfi, int start_comp) const noexcept
Definition AMReX_FabArray.H:692
Array4< typename FabArray< FAB >::value_type const > const_array(int K, int start_comp) const noexcept
Definition AMReX_FabArray.H:732
F::value_type sum(int comp, IntVect const &nghost, bool local=false) const
Abstract factory interface for creating, aliasing, and destroying FAB objects.
Definition AMReX_FabFactory.H:73
virtual Long nBytes(const Box &box, int ncomps, int) const
Return the storage size for ncomps components in box, or -1 if unavailable.
Definition AMReX_FabFactory.H:106
virtual FabFactory< FAB > * clone() const =0
Return a heap-allocated copy of this factory.
virtual FAB * create(const Box &box, int ncomps, const FabInfo &info, int box_index) const =0
Create a FAB covering box with ncomps components.
Rectangular problem domain geometry.
Definition AMReX_Geometry.H:87
const Box & Domain() const noexcept
Returns our rectangular domain.
Definition AMReX_Geometry.H:246
bool isPeriodic(int dir) const noexcept
Is the domain periodic in the specified direction?
Definition AMReX_Geometry.H:399
GPU-compatible tuple.
Definition AMReX_Tuple.H:104
__host__ __device__ constexpr bool allGE(const IntVectND< dim > &rhs) const noexcept
Returns true if this is greater than or equal to argument for all components. NOTE: This is NOT a str...
Definition AMReX_IntVect.H:549
__host__ static __device__ constexpr IntVectND< dim > TheZeroVector() noexcept
Definition AMReX_IntVect.H:778
__host__ __device__ constexpr int max() const noexcept
maximum (no absolute values) value
Definition AMReX_IntVect.H:320
__host__ __device__ constexpr bool allLE(const IntVectND< dim > &rhs) const noexcept
Returns true if this is less than or equal to argument for all components. NOTE: This is NOT a strict...
Definition AMReX_IntVect.H:499
a one-thingy-per-box distributed object
Definition AMReX_LayoutData.H:13
Iterator for looping ever tiles and boxes of amrex::FabArray based containers.
Definition AMReX_MFIter.H:88
Box growntilebox(int ng=-1000000) const noexcept
Return the tile box at the current index grown to include ghost cells.
Definition AMReX_MFIter.cpp:457
const DistributionMapping & DistributionMap() const noexcept
Definition AMReX_MFIter.H:201
bool isValid() const noexcept
Is the iterator valid i.e. is it associated with a FAB?
Definition AMReX_MFIter.H:176
Box validbox() const noexcept
Return the valid Box in which the current tile resides.
Definition AMReX_MFIter.H:167
int index() const noexcept
The index into the underlying BoxArray of the current FAB.
Definition AMReX_MFIter.H:179
int LocalIndex() const noexcept
Return local index into the vector of fab pointers, m_fabs_v When AllBoxes is on, local_index_map is ...
Definition AMReX_MFIter.H:194
static int currentDepth()
Definition AMReX_MFIter.cpp:21
This provides length of period for periodic domains. 0 means it is not periodic in that direction....
Definition AMReX_Periodicity.H:17
static const Periodicity & NonPeriodic() noexcept
Definition AMReX_Periodicity.cpp:52
bool isAnyPeriodic() const noexcept
Definition AMReX_Periodicity.H:22
bool isPeriodic(int dir) const noexcept
Definition AMReX_Periodicity.H:26
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition AMReX_Vector.H:29
Checks if a type is derived from amrex::BaseFab.
Definition AMReX_Concepts.H:13
amrex_long Long
Definition AMReX_INT.H:30
ArrayND< T, 4, true > Array4
Convenient alias for a 4D ArrayND with three spatial dimensions plus a component dimension.
Definition AMReX_Array4.H:1352
@ covered
Every cell in the region is covered.
Definition AMReX_FabFactory.H:28
void EnforcePeriodicity(int scomp, int ncomp, const IntVect &nghost, const Periodicity &period)
Definition AMReX_FabArray.H:4113
void SumBoundary(int scomp, int ncomp, const Periodicity &period=Periodicity::NonPeriodic(), bool deterministic=false)
Definition AMReX_FabArray.H:4025
void SumBoundary(int scomp, int ncomp, IntVect const &nghost, const Periodicity &period=Periodicity::NonPeriodic(), bool deterministic=false)
Sum values in overlapped cells.
Definition AMReX_FabArray.H:4032
void OverrideSync(const Periodicity &period=Periodicity::NonPeriodic())
Synchronize nodal data.
Definition AMReX_FabArray.H:3973
void FillBoundary(bool cross=false)
Copy on intersection within a FabArray.
Definition AMReX_FabArray.H:3818
void OverrideSync(int scomp, int ncomp, const Periodicity &period)
Synchronize nodal data.
Definition AMReX_FabArray.H:3984
void FillBoundary(int scomp, int ncomp, const IntVect &nghost, const Periodicity &period, bool cross=false)
Definition AMReX_FabArray.H:3880
void FillBoundaryAndSync(int scomp, int ncomp, const IntVect &nghost, const Periodicity &period)
Fill ghost cells and synchronize nodal data.
Definition AMReX_FabArray.H:3937
void FillBoundary(const Periodicity &period, bool cross=false)
Definition AMReX_FabArray.H:3830
void FillBoundary(int scomp, int ncomp, bool cross=false)
Definition AMReX_FabArray.H:3856
void FillBoundaryAndSync(const Periodicity &period=Periodicity::NonPeriodic())
Fill ghost cells and synchronize nodal data.
Definition AMReX_FabArray.H:3926
void SumBoundary(const Periodicity &period=Periodicity::NonPeriodic(), bool deterministic=false)
Sum values in overlapped cells.
Definition AMReX_FabArray.H:4018
void SumBoundary(int scomp, int ncomp, IntVect const &src_nghost, IntVect const &dst_nghost, const Periodicity &period=Periodicity::NonPeriodic(), bool deterministic=false)
Sum values in overlapped cells.
Definition AMReX_FabArray.H:4039
F::value_type norminf(int comp, int ncomp, IntVect const &nghost, bool local=false, bool ignore_covered=false) const
void EnforcePeriodicity(const Periodicity &period)
Fill ghost cells with values from their corresponding cells across periodic boundaries,...
Definition AMReX_FabArray.H:4091
void FillBoundary(int scomp, int ncomp, const Periodicity &period, bool cross=false)
Definition AMReX_FabArray.H:3868
void EnforcePeriodicity(int scomp, int ncomp, const Periodicity &period)
Definition AMReX_FabArray.H:4102
__host__ __device__ BoxND< dim > convert(const BoxND< dim > &b, const IntVectND< dim > &typ) noexcept
Return a copy of b converted to the nodal flags typ.
Definition AMReX_Box.H:1632
Arena * The_Comms_Arena()
Definition AMReX_Arena.cpp:899
Arena * The_Pinned_Arena()
Definition AMReX_Arena.cpp:879
Arena * The_Arena()
Definition AMReX_Arena.cpp:839
int MyProc() noexcept
Definition AMReX_ParallelDescriptor.H:128
void Min(KeyValuePair< K, V > &vi, MPI_Comm comm)
Definition AMReX_ParallelReduce.H:161
void Sum(Gpu::DeviceVector< T > &v, MPI_Comm comm)
Definition AMReX_GpuParallelReduce.H:37
void Max(KeyValuePair< K, V > &vi, MPI_Comm comm)
Definition AMReX_ParallelReduce.H:133
void freeAsync(Arena *arena, void *mem) noexcept
Definition AMReX_GpuDevice.H:364
void streamSynchronize() noexcept
Definition AMReX_GpuDevice.H:329
bool inLaunchRegion() noexcept
Definition AMReX_GpuControl.H:88
int numGpuStreams() noexcept
Definition AMReX_GpuDevice.H:317
bool inNoSyncRegion() noexcept
Definition AMReX_GpuControl.H:148
void htod_memcpy_async(void *p_d, const void *p_h, const std::size_t sz) noexcept
Definition AMReX_GpuDevice.H:440
MPI_Comm CommunicatorSub() noexcept
sub-communicator for current frame
Definition AMReX_ParallelContext.H:70
int MyTeamLead() noexcept
Definition AMReX_ParallelDescriptor.H:325
int TeamSize() noexcept
Definition AMReX_ParallelDescriptor.H:310
const ProcessTeam & MyTeam() noexcept
Definition AMReX_ParallelDescriptor.H:365
int MPI_Comm
Definition AMReX_ccse-mpi.H:51
Definition AMReX_Amr.cpp:50
MakeType
Definition AMReX_MakeType.H:7
@ make_alias
Definition AMReX_MakeType.H:7
__host__ __device__ void ignore_unused(const Ts &...)
No-op helper that marks variables as intentionally unused.
Definition AMReX.H:273
void FillBoundary_finish(Vector< MF * > const &mf)
Wait for outstanding FillBoundary_nowait operations launched with the vector helper to complete.
Definition AMReX_FabArrayCommI.H:1171
void Copy(FabArray< DFAB > &dst, FabArray< SFAB > const &src, int srccomp, int dstcomp, int numcomp, int nghost)
Definition AMReX_FabArray.H:219
void OverrideSync_nowait(FabArray< FAB > &fa, FabArray< IFAB > const &msk, const Periodicity &period)
Start the masked OverrideSync operation; call OverrideSync_finish() to complete it.
Definition AMReX_FabArrayUtility.H:1668
void Add(FabArray< FAB > &dst, FabArray< FAB > const &src, int srccomp, int dstcomp, int numcomp, int nghost)
Definition AMReX_FabArray.H:278
ReduceData< Ts... >::Type ParReduce(TypeList< Ops... > operation_list, TypeList< Ts... > type_list, FabArray< FAB > const &fa, IntVect const &nghost, F &&f)
Parallel reduce for MultiFab/FabArray. The reduce result is local and it's the user's responsibility ...
Definition AMReX_ParReduce.H:48
std::unique_ptr< char, TheFaArenaDeleter > TheFaArenaPointer
Definition AMReX_FabArray.H:118
Long nBytesOwned(T const &) noexcept
Definition AMReX_FabArray.H:65
BoxArray const & boxArray(FabArrayBase const &fa)
Convenience wrapper that forwards to fa.boxArray().
Definition AMReX_FabArrayBase.cpp:3038
DistributionMapping const & DistributionMap(FabArrayBase const &fa)
Convenience wrapper that forwards to fa.DistributionMap().
Definition AMReX_FabArrayBase.cpp:3043
void FillBoundary_nowait(Vector< MF * > const &mf, Vector< int > const &scomp, Vector< int > const &ncomp, Vector< IntVect > const &nghost, Vector< Periodicity > const &period, Vector< int > const &cross={})
Launch FillBoundary_nowait across a vector of FabArrays.
Definition AMReX_FabArrayCommI.H:1125
void FillBoundaryAndSync_nowait(Vector< MF * > const &mf, Vector< int > const &scomp, Vector< int > const &ncomp, Vector< IntVect > const &nghost, Vector< Periodicity > const &period)
Launch FillBoundaryAndSync_nowait across a vector of FabArrays.
Definition AMReX_FabArrayCommI.H:1194
void ParallelFor(TypeList< CTOs... > ctos, std::array< int, sizeof...(CTOs)> const &runtime_options, T N, F &&f)
Definition AMReX_CTOParallelForImpl.H:202
FabArray< BaseFab< GpuComplex< Real > > > cMultiFab
Definition AMReX_FabArray.H:4449
BoxND< 3 > Box
Box is an alias for amrex::BoxND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:35
bool isMFIterSafe(const FabArrayBase &x, const FabArrayBase &y)
Definition AMReX_MFIter.H:256
int nComp(FabArrayBase const &fa)
Convenience wrapper that forwards to fa.nComp().
Definition AMReX_FabArrayBase.cpp:3028
@ interior
Definition AMReX_LO_BCTYPES.H:28
@ y
Definition AMReX_Orientation.H:14
@ x
Definition AMReX_Orientation.H:14
__host__ __device__ T abs(const GpuComplex< T > &a_z) noexcept
Return the absolute value of a complex number.
Definition AMReX_GpuComplex.H:361
constexpr bool IsBaseFab_v
Definition AMReX_TypeTraits.H:25
void OverrideSync_finish(FabArray< FAB > &fa)
Finish the current masked OverrideSync_nowait() operation.
Definition AMReX_FabArrayUtility.H:1718
IntVect nGrowVect(FabArrayBase const &fa)
Convenience wrapper that forwards to fa.nGrowVect().
Definition AMReX_FabArrayBase.cpp:3033
bool TilingIfNotGPU() noexcept
Definition AMReX_MFIter.H:12
IntVectND< 3 > IntVect
IntVect is an alias for amrex::IntVectND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:38
void setBndry(MF &dst, typename MF::value_type val, int scomp, int ncomp)
dst = val in ghost cells.
Definition AMReX_FabArrayUtility.H:2181
void Abort(const std::string &msg)
Print a fatal-error message to stderr and abort execution.
Definition AMReX.cpp:239
void update_fab_stats(Long n, Long s, size_t szt) noexcept
Definition AMReX_BaseFab.cpp:146
void FillBoundaryAndSync_finish(Vector< MF * > const &mf)
Wait for outstanding FillBoundaryAndSync_nowait operations launched with the vector helper to complet...
Definition AMReX_FabArrayCommI.H:1236
void setVal(MF &dst, typename MF::value_type val)
dst = val
Definition AMReX_FabArrayUtility.H:2174
__host__ __device__ constexpr int get(IntVectND< dim > const &iv) noexcept
Get the Idx-th element of IntVectND<dim>.
Definition AMReX_IntVect.H:1367
Definition AMReX_TagParallelFor.H:58
Lightweight allocator/deallocator backed by an Arena.
Definition AMReX_DataAllocator.H:21
Definition AMReX_FabArray.H:146
const FabArrayBase::FB * fb
Definition AMReX_FabArray.H:148
FBOnArrivalMeta< FAB > * oa_meta
Definition AMReX_FabArray.H:167
bool deterministic
Definition AMReX_FabArray.H:165
char * the_recv_data
Definition AMReX_FabArray.H:153
Vector< MPI_Request > recv_reqs
Definition AMReX_FabArray.H:158
Vector< char * > recv_data
Definition AMReX_FabArray.H:156
Vector< MPI_Status > recv_stat
Definition AMReX_FabArray.H:159
int scomp
Definition AMReX_FabArray.H:149
Vector< int > recv_from
Definition AMReX_FabArray.H:155
char * the_send_data
Definition AMReX_FabArray.H:154
Vector< MPI_Request > send_reqs
Definition AMReX_FabArray.H:162
Vector< char * > send_data
Definition AMReX_FabArray.H:161
Vector< std::size_t > recv_size
Definition AMReX_FabArray.H:157
int ncomp
Definition AMReX_FabArray.H:150
int tag
Definition AMReX_FabArray.H:163
One message per neighbor rank, all in one device buffer.
Definition AMReX_FabArray.H:130
Vector< std::size_t > size
Definition AMReX_FabArray.H:132
Vector< int > rank
Definition AMReX_FabArray.H:131
Vector< std::size_t > offset
Definition AMReX_FabArray.H:133
std::size_t nbytes
Definition AMReX_FabArray.H:134
Definition AMReX_FabArray.H:126
Layout rcv
Definition AMReX_FabArray.H:136
typename FAB::value_type value_type
Definition AMReX_FabArray.H:127
TagVector< CommSendBufTag< value_type > > send_tv
Definition AMReX_FabArray.H:138
Layout snd
Definition AMReX_FabArray.H:136
Vector< int > rcv_warp_begin
Definition AMReX_FabArray.H:140
TagVector< CommRecvBufTag< value_type > > recv_tv
Definition AMReX_FabArray.H:139
parallel copy or add
Definition AMReX_FabArrayBase.H:651
Definition AMReX_FabArrayBase.H:563
std::unique_ptr< MapOfCopyComTagContainers > m_RcvTags
Definition AMReX_FabArrayBase.H:569
std::unique_ptr< CopyComTagsContainer > m_LocTags
Definition AMReX_FabArrayBase.H:567
Used by a bunch of routines when communicating via MPI.
Definition AMReX_FabArrayBase.H:211
Box dbox
Definition AMReX_FabArrayBase.H:212
int dstIndex
Definition AMReX_FabArrayBase.H:214
FillBoundary.
Definition AMReX_FabArrayBase.H:588
Definition AMReX_FabArray.H:386
FAB value_type
Definition AMReX_FabArray.H:387
ShMem(ShMem &&rhs) noexcept
Definition AMReX_FabArray.H:1845
ShMem() noexcept=default
Long n_values
Definition AMReX_FabArray.H:1872
Long n_points
Definition AMReX_FabArray.H:1873
bool alloc
Definition AMReX_FabArray.H:1871
ShMem & operator=(ShMem &&rhs) noexcept
Definition AMReX_FabArray.H:1856
ShMem(const ShMem &)=delete
Construction hints passed to FabFactory::create.
Definition AMReX_FabFactory.H:41
FabInfo & SetShared(bool s) noexcept
Set the shared flag; returns *this for chaining.
Definition AMReX_FabFactory.H:53
FabInfo & SetAlloc(bool a) noexcept
Set the alloc flag; returns *this for chaining.
Definition AMReX_FabFactory.H:47
FabInfo & SetArena(Arena *ar) noexcept
Set the arena pointer; returns *this for chaining.
Definition AMReX_FabFactory.H:59
FabArray memory allocation information.
Definition AMReX_FabArray.H:73
MFInfo & SetTag(T &&t, Ts &&... ts) noexcept
Append one or more memory-usage tags.
Definition AMReX_FabArray.H:106
MFInfo & SetTag(const std::string &t) noexcept
Append memory-usage tag t.
Definition AMReX_FabArray.H:99
Arena * arena
Definition AMReX_FabArray.H:77
MFInfo & SetArena(Arena *ar) noexcept
Select the Arena used for FAB storage.
Definition AMReX_FabArray.H:87
MFInfo & SetTag() noexcept
Terminate a variadic SetTag() call.
Definition AMReX_FabArray.H:90
bool alloc
Definition AMReX_FabArray.H:75
MFInfo & SetAlloc(bool a) noexcept
Control whether FAB storage is allocated when the FabArray is defined.
Definition AMReX_FabArray.H:81
bool alloc_single_chunk
Definition AMReX_FabArray.H:76
MFInfo & SetAllocSingleChunk(bool a) noexcept
Control whether eligible FABs use one contiguous allocation.
Definition AMReX_FabArray.H:84
Vector< std::string > tags
Definition AMReX_FabArray.H:78
MFInfo & SetTag(const char *t) noexcept
Append memory-usage tag t.
Definition AMReX_FabArray.H:93
Definition AMReX_MFIter.H:20
MFItInfo & DisableDeviceSync() noexcept
Definition AMReX_MFIter.H:47
Definition AMReX_FabArray.H:195
Array4< T > const *__restrict__ hp
Definition AMReX_FabArray.H:211
__host__ __device__ Array4< T > const & operator[](int li) const noexcept
Definition AMReX_FabArray.H:197
Array4< T > const *__restrict__ dp
Definition AMReX_FabArray.H:209
Definition AMReX_FabArray.H:173
int actual_n_rcvs
Definition AMReX_FabArray.H:179
Vector< std::size_t > recv_size
Definition AMReX_FabArray.H:186
int DC
Definition AMReX_FabArray.H:180
Vector< MPI_Request > send_reqs
Definition AMReX_FabArray.H:188
int tag
Definition AMReX_FabArray.H:178
const FabArray< FAB > * src
Definition AMReX_FabArray.H:176
char * the_recv_data
Definition AMReX_FabArray.H:182
FabArrayBase::CpOp op
Definition AMReX_FabArray.H:177
Vector< MPI_Request > recv_reqs
Definition AMReX_FabArray.H:187
char * the_send_data
Definition AMReX_FabArray.H:183
const FabArrayBase::CPC * cpc
Definition AMReX_FabArray.H:175
Vector< int > recv_from
Definition AMReX_FabArray.H:184
bool deterministic
Definition AMReX_FabArray.H:189
int NC
Definition AMReX_FabArray.H:180
int SC
Definition AMReX_FabArray.H:180
Vector< char * > recv_data
Definition AMReX_FabArray.H:185
void MemoryBarrier() const
memory fence
Definition AMReX_ParallelDescriptor.H:161
const team_t & get() const
Definition AMReX_ParallelDescriptor.H:189
Definition AMReX_TagParallelFor.H:156
Definition AMReX_FabArray.H:112
char * pointer
Definition AMReX_FabArray.H:113
void operator()(pointer p) const noexcept
Definition AMReX_FabArray.H:114
Struct for holding types.
Definition AMReX_TypeList.H:13