2 #ifndef BL_FACOPYDESCRIPTOR_H_
3 #define BL_FACOPYDESCRIPTOR_H_
4 #include <AMReX_Config.H>
90 localFabSource(nullptr)
97 if (cacheDataAllocated) {
98 delete localFabSource;
109 using FCDMap = std::multimap<int,FabCopyDescriptor<FAB>*>;
129 const Box& destFabBox,
133 const Box& destFabBox,
142 const Box& destFabBox,
148 bool bUseValidBox =
true);
178 const Box& destFabBox,
179 BoxList* returnedUnfilledBoxes,
206 BL_ASSERT(fabArrays.size() == fabCopyDescList.size());
210 fabArrays.push_back(fabarray);
212 fabCopyDescList.push_back(
FCDMap());
220 const Box& destFabBox,
221 BoxList* returnedUnfilledBoxes,
254 +
":: _in AddBoxDoIt: nProcs remoteProc = "
256 +
" " + std::to_string(remoteProc) +
"\n");
258 fcd->fillBoxId = nextFillBoxId;
260 fcd->myProc = myProc;
261 fcd->copyFromProc = remoteProc;
262 fcd->copyFromIndex = faindex;
263 fcd->srcComp = srccomp;
264 fcd->destComp = destcomp;
265 fcd->nComp = numcomp;
273 fcd->localFabSource = &(*fabArray)[faindex];
282 dataAvailable =
false;
285 fcd->cacheDataAllocated =
true;
294 fabComTag.
nComp = numcomp;
298 fabComTagList.push_back(fabComTag);
303 if (returnedUnfilledBoxes !=
nullptr)
313 const Box& destFabBox,
314 BoxList* returnedUnfilledBoxes,
321 if (returnedUnfilledBoxes !=
nullptr)
323 unfilledBoxDomain.
add(destFabBox);
326 std::vector< std::pair<int,Box> > isects;
328 fabArrays[fabarrayid.
Id()]->boxArray().intersections(destFabBox,isects);
330 for (
auto & isect : isects)
332 AddBoxDoIt(fabarrayid,
334 returnedUnfilledBoxes,
343 if (returnedUnfilledBoxes !=
nullptr)
345 returnedUnfilledBoxes->
clear();
346 (*returnedUnfilledBoxes) = unfilledBoxDomain.
boxList();
349 return FillBoxId(nextFillBoxId++, destFabBox);
355 const Box& destFabBox,
356 BoxList* returnedUnfilledBoxes,
365 if (returnedUnfilledBoxes !=
nullptr)
367 unfilledBoxDomain.
add(destFabBox);
370 AddBoxDoIt(fabarrayid,
372 returnedUnfilledBoxes,
380 if (returnedUnfilledBoxes !=
nullptr)
382 returnedUnfilledBoxes->
clear();
383 (*returnedUnfilledBoxes) = unfilledBoxDomain.
boxList();
386 return FillBoxId(nextFillBoxId++, destFabBox);
392 const Box& destFabBox,
393 BoxList* returnedUnfilledBoxes)
395 return AddBox(fabarrayid,
397 returnedUnfilledBoxes,
400 fabArrays[fabarrayid.
Id()]->nComp(),
414 for (
unsigned int i = 0, N = fabCopyDescList.size(); i < N; ++i)
416 for (
auto fmi = fabCopyDescList[i].
begin(), End = fabCopyDescList[i].
end();
420 delete (*fmi).second;
425 fabCopyDescList.clear();
426 fabComTagList.clear();
429 dataAvailable =
false;
436 dataAvailable =
true;
441 using value_type =
typename FAB::value_type;
443 BL_PROFILE(
"FabArrayCopyDescriptor::CollectData()");
448 int Total_Rcvs_Size = 0;
452 std::map< int,FabComTagIterContainer > RcvTags;
454 std::map<int,int> Snds, Rcvs, Npts;
458 for (
auto it = fabComTagList.begin(),
459 End = fabComTagList.end();
467 const int Who = it->procThatHasData;
468 const auto Cnt =
static_cast<int>((it->box.numPts())*(it->nComp));
470 RcvTags[Who].emplace_back(it);
472 Total_Rcvs_Size += Cnt;
474 if (Rcvs.count(Who) > 0)
483 if (Npts.count(Who) > 0)
494 BL_ASSERT((Total_Rcvs_Size*
sizeof(value_type))
502 for (
auto const& Rcv : Rcvs)
504 RcvsArray[Rcv.first] = Rcv.second;
510 BLProfiler::BeforeCall());
512 BL_MPI_REQUIRE( MPI_Alltoall(RcvsArray.
dataPtr(),
521 BLProfiler::AfterCall());
527 for (
int i = 0; i <
NProcs; i++) {
528 if (SndsArray[i] > 0) {
529 Snds[i] = SndsArray[i];
540 const auto N_snds =
static_cast<int>(Snds.size());
541 const auto N_rcvs =
static_cast<int>(Rcvs.size());
543 if ( N_snds == 0 && N_rcvs == 0 ) {
return; }
545 const int Nints = 4 + 3*AMREX_SPACEDIM;
548 Vector<int> md_sender, md_offset, md_icnts, md_bcnts;
549 int* md_recv_data =
nullptr;
555 value_type* recv_data =
nullptr;
564 for (
auto const& Snd : Snds)
566 md_sender.push_back(Snd.first);
567 md_bcnts.push_back(Snd.second);
568 int cnt = Snd.second * Nints;
569 md_icnts.push_back(cnt);
570 md_offset.push_back(N);
576 for (
int i = 0; i < N_snds; ++i)
579 md_icnts[i], md_sender[i],
587 for (
auto const& Rcv : Rcvs)
589 int rank = Rcv.first;
590 int Nmds = Rcv.second;
591 int cnt = Nmds * Nints;
594 md_send_data.push_back(p);
600 for (
int i = 0; i < Nmds; ++i, md += Nints)
602 md[0] = tags[i]->fabArrayId;
603 md[1] = tags[i]->fabIndex;
604 md[2] = tags[i]->srcComp;
605 md[3] = tags[i]->nComp;
606 const int* lo = tags[i]->box.loVect();
607 const int* hi = tags[i]->box.hiVect();
608 const IntVect& bxtyp = tags[i]->box.type();
609 const int* tp = bxtyp.
getVect();
614 md[5+ AMREX_SPACEDIM] = hi[1],
615 md[6+ AMREX_SPACEDIM] = hi[2]);
617 md[5+2*AMREX_SPACEDIM] = tp[1],
618 md[6+2*AMREX_SPACEDIM] = tp[2]);
631 for (
auto & Npt : Npts)
634 int Cnt = Npt.second;
637 data_sender.push_back(Who);
639 Cnt,Who,SeqNum_data).req());
640 data_offset.push_back(Idx);
648 int send_counter = 0;
649 while (send_counter++ < N_snds)
655 int rank = status.MPI_SOURCE;
659 const int* p = &md_recv_data[md_offset[index]];
660 int numboxes = md_bcnts[index];
669 for (
int i = 0; i < numboxes; ++i, md += Nints)
676 IntVect(&md[4+AMREX_SPACEDIM]),
677 IntVect(&md[4+AMREX_SPACEDIM*2])));
678 npts[i] =
static_cast<int>(bxs.back().numPts()*ncomp[i]);
685 value_type* dptr = data;
686 send_data.push_back(data);
688 for (
int i = 0; i < numboxes; ++i)
690 (*fabArrays[faid[i]])[fidx[i]].
template copyToMem<RunOn::Host>(bxs[i],scomp[i],ncomp[i],dptr);
706 for (
int i = 0; i < N_rcvs; ++i) {
712 std::pair<FCDMapIter,FCDMapIter>
match;
713 std::map< int,FabComTagIterContainer >::const_iterator found;
715 for (
int k = 0; k < N_rcvs; k++)
717 const int Who = data_sender[k];
718 const value_type* dptr = &recv_data[data_offset[k]];
722 found = RcvTags.find(Who);
728 for (
auto const& it : tags)
736 for (
auto fmi =
match.first; fmi !=
match.second; ++fmi)
764 for (
int i = 0; i < N_snds; ++i) {
781 std::pair<FCDMapIter,FCDMapIter>
match = fabCopyDescList[faid.
Id()].equal_range(fillboxid.
Id());
783 for (
auto fmi =
match.first; fmi !=
match.second; ++fmi)
789 destFab.template copy<RunOn::Host>
808 FCDMapIter fmi = fabCopyDescList[faid.
Id()].lower_bound(fillboxid.
Id());
810 BL_ASSERT(fmi != fabCopyDescList[faid.
Id()].end());
825 BL_ASSERT(++fmi == fabCopyDescList[faid.
Id()].upper_bound(fillboxid.
Id()));
836 <<
": Parallel stats for FabArrayCopyDescriptor:" <<
'\n';
838 for (
int fa = 0; fa < fabArrays.size(); ++fa)
842 <<
"]->boxArray() = "
843 << fabArrays[fa]->boxArray()
#define BL_PROFILE(a)
Definition: AMReX_BLProfiler.H:551
#define BL_COMM_PROFILE(cft, size, pid, tag)
Definition: AMReX_BLProfiler.H:587
#define BL_PROFILE_VAR_STOP(vname)
Definition: AMReX_BLProfiler.H:563
#define BL_PROFILE_VAR(fname, vname)
Definition: AMReX_BLProfiler.H:560
#define BL_ASSERT(EX)
Definition: AMReX_BLassert.H:39
#define AMREX_D_EXPR(a, b, c)
Definition: AMReX_SPACE.H:81
Print on all processors of the default communicator.
Definition: AMReX_Print.H:117
virtual void free(void *pt)=0
A pure virtual function for deleting the arena pointed to by pt.
virtual void * alloc(std::size_t sz)=0
bool ok() const
Return true if Box is valid and they all have the same IndexType. Is true by default if the BoxArray ...
A List of Disjoint Boxes.
Definition: AMReX_BoxDomain.H:67
BoxDomain & rmBox(const Box &b)
Remove a box from the domain.
Definition: AMReX_BoxDomain.cpp:159
const BoxList & boxList() const
Return a const reference to the underlying BoxList of this BoxDomain.
Definition: AMReX_BoxDomain.cpp:82
void add(const Box &b)
Add a Box to the domain.
Definition: AMReX_BoxDomain.cpp:121
A class for managing a List of Boxes that share a common IndexType. This class implements operations ...
Definition: AMReX_BoxList.H:52
void clear()
Remove all Boxes from this BoxList.
AMREX_GPU_HOST_DEVICE IndexTypeND< dim > ixType() const noexcept
Returns the indexing type.
Definition: AMReX_Box.H:127
AMREX_GPU_HOST_DEVICE bool sameSize(const BoxND &b) const noexcept
Returns true is Boxes same size, ie translates of each other,. It is an error if they have different ...
Definition: AMReX_Box.H:279
AMREX_GPU_HOST_DEVICE Long numPts() const noexcept
Returns the number of points contained in the BoxND.
Definition: AMReX_Box.H:346
const DistributionMapping & DistributionMap() const noexcept
Return constant reference to associated DistributionMapping.
Definition: AMReX_FabArrayBase.H:130
Box box(int K) const noexcept
Return the Kth Box in the BoxArray. That is, the valid region of the Kth grid.
Definition: AMReX_FabArrayBase.H:100
Box fabbox(int K) const noexcept
Return the Kth FABs Box in the FabArray. That is, the region the Kth fab is actually defined on.
This class orchestrates filling a destination fab of size destFabBox from fabarray on the local proce...
Definition: AMReX_FACopyDescriptor.H:108
void PrintStats() const
Definition: AMReX_FACopyDescriptor.H:830
std::vector< FabArray< FAB > * > fabArrays
Definition: AMReX_FACopyDescriptor.H:195
int nFabCopyDescs() const
Definition: AMReX_FACopyDescriptor.H:171
bool dataAvailable
Definition: AMReX_FACopyDescriptor.H:199
FabArrayCopyDescriptor(const FabArrayCopyDescriptor< FAB > &)=delete
FabArrayCopyDescriptor(FabArrayCopyDescriptor< FAB > &&)=delete
int nFabComTags() const
Definition: AMReX_FACopyDescriptor.H:169
void clear()
Definition: AMReX_FACopyDescriptor.H:412
void FillFab(FabArrayId faid, const FillBoxId &fillboxid, FAB &destFab)
Definition: AMReX_FACopyDescriptor.H:775
typename FCDMap::const_iterator FCDMapConstIter
Definition: AMReX_FACopyDescriptor.H:113
int nextFillBoxId
Definition: AMReX_FACopyDescriptor.H:198
FabArrayCopyDescriptor()=default
void CollectData()
Definition: AMReX_FACopyDescriptor.H:434
FillBoxId AddBox(FabArrayId fabarrayid, const Box &destFabBox, BoxList *unfilledBoxes)
Definition: AMReX_FACopyDescriptor.H:391
FabComTagContainer fabComTagList
Definition: AMReX_FACopyDescriptor.H:197
FabArrayId RegisterFabArray(FabArray< FAB > *fabarray)
Definition: AMReX_FACopyDescriptor.H:204
int CurrentNFabArrays() const
Definition: AMReX_FACopyDescriptor.H:167
FabArrayCopyDescriptor< FAB > & operator=(const FabArrayCopyDescriptor< FAB > &)=delete
FillBoxId AddBox(FabArrayId fabarrayid, const Box &destFabBox, BoxList *unfilledBoxes, int srccomp, int destcomp, int numcomp)
Definition: AMReX_FACopyDescriptor.H:312
~FabArrayCopyDescriptor()
Definition: AMReX_FACopyDescriptor.H:405
bool DataAvailable() const
Definition: AMReX_FACopyDescriptor.H:163
std::vector< FabArrayBase::FabComTag > FabComTagContainer
Definition: AMReX_FACopyDescriptor.H:189
void AddBoxDoIt(FabArrayId fabarrayid, const Box &destFabBox, BoxList *returnedUnfilledBoxes, int faindex, int srccomp, int destcomp, int numcomp, bool bUseValidBox, BoxDomain &unfilledBoxDomain)
Definition: AMReX_FACopyDescriptor.H:219
FillBoxId AddBox(FabArrayId fabarrayid, const Box &destFabBox, BoxList *unfilledBoxes, int fabarrayindex, int srccomp, int destcomp, int numcomp, bool bUseValidBox=true)
Definition: AMReX_FACopyDescriptor.H:354
typename FCDMap::iterator FCDMapIter
Definition: AMReX_FACopyDescriptor.H:112
typename FCDMap::value_type FCDMapValueType
Definition: AMReX_FACopyDescriptor.H:111
void FillFab(FabArrayId faid, const FillBoxId &fillboxid, FAB &destFab, const Box &destBox)
Definition: AMReX_FACopyDescriptor.H:801
std::multimap< int, FabCopyDescriptor< FAB > * > FCDMap
Definition: AMReX_FACopyDescriptor.H:109
std::vector< FCDMap > fabCopyDescList
Definition: AMReX_FACopyDescriptor.H:196
std::vector< FabComTagContainer::const_iterator > FabComTagIterContainer
Definition: AMReX_FACopyDescriptor.H:191
Definition: AMReX_FACopyDescriptor.H:43
int Id() const
Definition: AMReX_FACopyDescriptor.H:50
int fabArrayId
Definition: AMReX_FACopyDescriptor.H:59
bool operator==(const FabArrayId &rhs) const
Definition: AMReX_FACopyDescriptor.H:52
FabArrayId(int newid=-1)
Definition: AMReX_FACopyDescriptor.H:46
An Array of FortranArrayBox(FAB)-like Objects.
Definition: AMReX_FabArray.H:344
Definition: AMReX_FACopyDescriptor.H:19
int Id() const
Definition: AMReX_FACopyDescriptor.H:30
int FabIndex() const
Definition: AMReX_FACopyDescriptor.H:31
FillBoxId(int newid, const Box &fillbox)
Definition: AMReX_FACopyDescriptor.H:24
Box m_fillBox
Definition: AMReX_FACopyDescriptor.H:37
int m_fabIndex
Definition: AMReX_FACopyDescriptor.H:39
int m_fillBoxId
Definition: AMReX_FACopyDescriptor.H:38
const Box & box() const
Definition: AMReX_FACopyDescriptor.H:33
void FabIndex(int fabindex)
Definition: AMReX_FACopyDescriptor.H:32
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE const int * getVect() const &noexcept
Returns a const pointer to an array of coordinates of the IntVectND. Useful for arguments to FORTRAN ...
Definition: AMReX_IntVect.H:284
MPI_Request req() const
Definition: AMReX_ParallelDescriptor.H:74
T * dataPtr() noexcept
get access to the underlying data pointer
Definition: AMReX_Vector.H:46
@ FAB
Definition: AMReX_AmrvisConstants.H:86
AMREX_GPU_HOST_DEVICE Long size(T const &b) noexcept
integer version
Definition: AMReX_GpuRange.H:26
int NProcs()
Process ID in MPI_COMM_WORLD.
Definition: AMReX_MPMD.cpp:122
int MyProc()
Definition: AMReX_MPMD.cpp:117
bool sameTeam(int rank) noexcept
Definition: AMReX_ParallelDescriptor.H:329
Message Asend(const T *, size_t n, int pid, int tag)
Definition: AMReX_ParallelDescriptor.H:1088
MPI_Comm Communicator() noexcept
Definition: AMReX_ParallelDescriptor.H:210
void Waitany(Vector< MPI_Request > &, int &, MPI_Status &)
Definition: AMReX_ParallelDescriptor.cpp:1299
int MyProc() noexcept
return the rank number local to the current Parallel Context
Definition: AMReX_ParallelDescriptor.H:125
void Waitall(Vector< MPI_Request > &, Vector< MPI_Status > &)
Definition: AMReX_ParallelDescriptor.cpp:1295
int SeqNum() noexcept
Returns sequential message sequence numbers, usually used as tags for send/recv.
Definition: AMReX_ParallelDescriptor.H:613
int NProcs() noexcept
return the number of MPI ranks local to the current Parallel Context
Definition: AMReX_ParallelDescriptor.H:243
Message Arecv(T *, size_t n, int pid, int tag)
Definition: AMReX_ParallelDescriptor.H:1130
@ max
Definition: AMReX_ParallelReduce.H:17
Definition: AMReX_Amr.cpp:49
BoxND< AMREX_SPACEDIM > Box
Definition: AMReX_BaseFwd.H:27
BoxArray intersect(const BoxArray &ba, const Box &b, int ng=0)
Make a BoxArray from the intersection of Box b and BoxArray(+ghostcells).
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Dim3 end(BoxND< dim > const &box) noexcept
Definition: AMReX_Box.H:1890
IntVectND< AMREX_SPACEDIM > IntVect
Definition: AMReX_BaseFwd.H:30
bool match(const BoxArray &x, const BoxArray &y)
Note that two BoxArrays that match are not necessarily equal.
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void ignore_unused(const Ts &...)
This shuts up the compiler about unused variables.
Definition: AMReX.H:111
FillType
Definition: AMReX_FACopyDescriptor.H:16
@ FillLocally
Definition: AMReX_FACopyDescriptor.H:16
@ Unfillable
Definition: AMReX_FACopyDescriptor.H:16
@ FillRemotely
Definition: AMReX_FACopyDescriptor.H:16
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Dim3 begin(BoxND< dim > const &box) noexcept
Definition: AMReX_Box.H:1881
void Abort(const std::string &msg)
Print out message to cerr and exit via abort().
Definition: AMReX.cpp:225
Arena * The_Arena()
Definition: AMReX_Arena.cpp:609
Definition: AMReX_ccse-mpi.H:51
Used for collecting information used in communicating FABs.
Definition: AMReX_FabArrayBase.H:271
int srcComp
Definition: AMReX_FabArrayBase.H:276
int fabArrayId
Definition: AMReX_FabArrayBase.H:280
int procThatHasData
Definition: AMReX_FabArrayBase.H:283
int procThatNeedsData
Definition: AMReX_FabArrayBase.H:282
int destComp
Definition: AMReX_FabArrayBase.H:277
Box box
Definition: AMReX_FabArrayBase.H:284
int fabIndex
Definition: AMReX_FabArrayBase.H:274
int nComp
Definition: AMReX_FabArrayBase.H:278
int fillBoxId
Definition: AMReX_FabArrayBase.H:281
Definition: AMReX_FACopyDescriptor.H:64
int destComp
Definition: AMReX_FACopyDescriptor.H:81
FabCopyDescriptor(const FabCopyDescriptor &)=delete
int copyFromIndex
Definition: AMReX_FACopyDescriptor.H:78
FabCopyDescriptor()
Definition: AMReX_FACopyDescriptor.H:88
int myProc
Definition: AMReX_FACopyDescriptor.H:76
bool cacheDataAllocated
Definition: AMReX_FACopyDescriptor.H:84
FAB * localFabSource
Definition: AMReX_FACopyDescriptor.H:74
~FabCopyDescriptor()
Definition: AMReX_FACopyDescriptor.H:95
Box subBox
Definition: AMReX_FACopyDescriptor.H:75
int nComp
Definition: AMReX_FACopyDescriptor.H:82
int fillBoxId
Definition: AMReX_FACopyDescriptor.H:79
int srcComp
Definition: AMReX_FACopyDescriptor.H:80
FillType fillType
Definition: AMReX_FACopyDescriptor.H:83
FabCopyDescriptor(FabCopyDescriptor &&)=delete
int copyFromProc
Definition: AMReX_FACopyDescriptor.H:77
FabCopyDescriptor & operator=(const FabCopyDescriptor &)=delete
Communication datatype (note: this structure also works without MPI)
Definition: AMReX_ccse-mpi.H:68