Block-Structured AMR Software Framework
AMReX_FACopyDescriptor.H
Go to the documentation of this file.
1 
2 #ifndef BL_FACOPYDESCRIPTOR_H_
3 #define BL_FACOPYDESCRIPTOR_H_
4 #include <AMReX_Config.H>
5 
6 #include <AMReX_FabArray.H>
7 #include <string>
8 
9 namespace amrex {
10 
15 
17 
18 class FillBoxId
19 {
20  public:
21 
22  FillBoxId () = default;
23 
24  FillBoxId (int newid, const Box& fillbox)
25  :
26  m_fillBox(fillbox),
27  m_fillBoxId(newid)
28  {}
29 
30  [[nodiscard]] int Id () const { return m_fillBoxId; }
31  [[nodiscard]] int FabIndex () const { return m_fabIndex; }
32  void FabIndex (int fabindex) { m_fabIndex = fabindex; }
33  [[nodiscard]] const Box& box () const { return m_fillBox; }
34 
35 private:
36 
38  int m_fillBoxId{-1};
39  int m_fabIndex{-1};
40 };
41 
43 {
44 public:
45 
46  explicit FabArrayId (int newid = -1)
47  :
48  fabArrayId(newid) {}
49 
50  [[nodiscard]] int Id () const { return fabArrayId; }
51 
52  bool operator== (const FabArrayId& rhs) const
53  {
54  return fabArrayId == rhs.fabArrayId;
55  }
56 
57 private:
58 
60 };
61 
62 template <class FAB>
64 {
66 
68 
73 
76  int myProc{-1};
77  int copyFromProc{-1};
78  int copyFromIndex{-1};
79  int fillBoxId{-1};
80  int srcComp{-1};
81  int destComp{-1};
82  int nComp{-1};
84  bool cacheDataAllocated{false};
85 };
86 
87 template <class FAB>
89  :
90  localFabSource(nullptr)
91 
92 {}
93 
94 template <class FAB>
96 {
97  if (cacheDataAllocated) {
98  delete localFabSource;
99  }
100 }
101 
106 template <class FAB>
108 {
109  using FCDMap = std::multimap<int,FabCopyDescriptor<FAB>*>;
110 
111  using FCDMapValueType = typename FCDMap::value_type;
112  using FCDMapIter = typename FCDMap::iterator;
113  using FCDMapConstIter = typename FCDMap::const_iterator;
114 
115 public:
116 
118 
120 
125 
127 
129  const Box& destFabBox,
130  BoxList* unfilledBoxes);
131 
133  const Box& destFabBox,
134  BoxList* unfilledBoxes,
135  int srccomp,
136  int destcomp,
137  int numcomp);
142  const Box& destFabBox,
143  BoxList* unfilledBoxes,
144  int fabarrayindex,
145  int srccomp,
146  int destcomp,
147  int numcomp,
148  bool bUseValidBox = true);
149 
150  void CollectData ();
151 
152  void FillFab (FabArrayId faid,
153  const FillBoxId& fillboxid,
154  FAB& destFab);
155 
156  void FillFab (FabArrayId faid,
157  const FillBoxId& fillboxid,
158  FAB& destFab,
159  const Box& destBox);
160 
161  void PrintStats () const;
162 
163  [[nodiscard]] bool DataAvailable () const { return dataAvailable; }
164 
165  void clear ();
166 
167  [[nodiscard]] int CurrentNFabArrays () const { return fabArrays.size(); }
168 
169  [[nodiscard]] int nFabComTags () const { return fabComTagList.size(); }
170 
171  [[nodiscard]] int nFabCopyDescs () const { return fabCopyDescList.size(); }
172 
173 private:
177  void AddBoxDoIt (FabArrayId fabarrayid,
178  const Box& destFabBox,
179  BoxList* returnedUnfilledBoxes,
180  int faindex,
181  int srccomp,
182  int destcomp,
183  int numcomp,
184  bool bUseValidBox,
185  BoxDomain& unfilledBoxDomain);
189  using FabComTagContainer = std::vector<FabArrayBase::FabComTag>;
190 
191  using FabComTagIterContainer = std::vector<FabComTagContainer::const_iterator>;
195  std::vector<FabArray<FAB>*> fabArrays;
196  std::vector<FCDMap> fabCopyDescList;
199  bool dataAvailable{false};
200 };
201 
202 template <class FAB>
203 FabArrayId
205 {
206  BL_ASSERT(fabArrays.size() == fabCopyDescList.size());
207 
208  FabArrayId result(fabArrays.size());
209 
210  fabArrays.push_back(fabarray); /* Bump size() by one */
211 
212  fabCopyDescList.push_back(FCDMap());
213 
214  return result;
215 }
216 
217 template <class FAB>
218 void
220  const Box& destFabBox,
221  BoxList* returnedUnfilledBoxes,
222  int faindex,
223  int srccomp,
224  int destcomp,
225  int numcomp,
226  bool bUseValidBox,
227  BoxDomain& unfilledBoxDomain)
228 {
229  const int myProc = ParallelDescriptor::MyProc();
230 
231  FabArray<FAB>* fabArray = fabArrays[fabarrayid.Id()];
232 
233  BL_ASSERT(faindex >= 0 && faindex < fabArray->size());
234 
235  Box intersect = destFabBox;
236 
237  if (bUseValidBox)
238  {
239  intersect &= fabArray->box(faindex);
240  }
241  else
242  {
243  intersect &= fabArray->fabbox(faindex);
244  }
245 
246  if (intersect.ok())
247  {
248  auto* fcd = new FabCopyDescriptor<FAB>;
249 
250  int remoteProc = fabArray->DistributionMap()[faindex];
251  if(remoteProc >= ParallelDescriptor::NProcs()) {
252  amrex::Abort("Bad remoteProc: "
253  + std::to_string(ParallelDescriptor::MyProc())
254  + ":: _in AddBoxDoIt: nProcs remoteProc = "
255  + std::to_string(ParallelDescriptor::NProcs())
256  + " " + std::to_string(remoteProc) + "\n");
257  }
258  fcd->fillBoxId = nextFillBoxId;
259  fcd->subBox = intersect;
260  fcd->myProc = myProc;
261  fcd->copyFromProc = remoteProc;
262  fcd->copyFromIndex = faindex;
263  fcd->srcComp = srccomp;
264  fcd->destComp = destcomp;
265  fcd->nComp = numcomp;
266 
267  if (ParallelDescriptor::sameTeam(remoteProc))
268  {
269  //
270  // Data is local.
271  //
272  fcd->fillType = FillLocally;
273  fcd->localFabSource = &(*fabArray)[faindex];
274  }
275  else
276  {
277  //
278  // Data is remote.
279  //
280  FabArrayBase::FabComTag fabComTag;
281 
282  dataAvailable = false;
283  fcd->fillType = FillRemotely;
284  fcd->localFabSource = new FAB(intersect, numcomp);
285  fcd->cacheDataAllocated = true;
286  fabComTag.fabArrayId = fabarrayid.Id();
287  fabComTag.fillBoxId = nextFillBoxId;
288  fabComTag.fabIndex = faindex;
289  fabComTag.procThatNeedsData = myProc;
290  fabComTag.procThatHasData = remoteProc;
291  fabComTag.box = intersect;
292  fabComTag.srcComp = srccomp;
293  fabComTag.destComp = destcomp;
294  fabComTag.nComp = numcomp;
295  //
296  // Do not send the data yet.
297  //
298  fabComTagList.push_back(fabComTag);
299  }
300 
301  fabCopyDescList[fabarrayid.Id()].insert(FCDMapValueType(fcd->fillBoxId,fcd));
302 
303  if (returnedUnfilledBoxes != nullptr)
304  {
305  unfilledBoxDomain.rmBox(intersect);
306  }
307  }
308 }
309 
310 template <class FAB>
311 FillBoxId
313  const Box& destFabBox,
314  BoxList* returnedUnfilledBoxes,
315  int srccomp,
316  int destcomp,
317  int numcomp)
318 {
319  BoxDomain unfilledBoxDomain(destFabBox.ixType());
320 
321  if (returnedUnfilledBoxes != nullptr)
322  {
323  unfilledBoxDomain.add(destFabBox);
324  }
325 
326  std::vector< std::pair<int,Box> > isects;
327 
328  fabArrays[fabarrayid.Id()]->boxArray().intersections(destFabBox,isects);
329 
330  for (auto & isect : isects)
331  {
332  AddBoxDoIt(fabarrayid,
333  destFabBox,
334  returnedUnfilledBoxes,
335  isect.first,
336  srccomp,
337  destcomp,
338  numcomp,
339  true,
340  unfilledBoxDomain);
341  }
342 
343  if (returnedUnfilledBoxes != nullptr)
344  {
345  returnedUnfilledBoxes->clear();
346  (*returnedUnfilledBoxes) = unfilledBoxDomain.boxList();
347  }
348 
349  return FillBoxId(nextFillBoxId++, destFabBox);
350 }
351 
352 template <class FAB>
353 FillBoxId
355  const Box& destFabBox,
356  BoxList* returnedUnfilledBoxes,
357  int fabarrayindex,
358  int srccomp,
359  int destcomp,
360  int numcomp,
361  bool bUseValidBox)
362 {
363  BoxDomain unfilledBoxDomain(destFabBox.ixType());
364 
365  if (returnedUnfilledBoxes != nullptr)
366  {
367  unfilledBoxDomain.add(destFabBox);
368  }
369 
370  AddBoxDoIt(fabarrayid,
371  destFabBox,
372  returnedUnfilledBoxes,
373  fabarrayindex,
374  srccomp,
375  destcomp,
376  numcomp,
377  bUseValidBox,
378  unfilledBoxDomain);
379 
380  if (returnedUnfilledBoxes != nullptr)
381  {
382  returnedUnfilledBoxes->clear();
383  (*returnedUnfilledBoxes) = unfilledBoxDomain.boxList();
384  }
385 
386  return FillBoxId(nextFillBoxId++, destFabBox);
387 }
388 
389 template <class FAB>
390 FillBoxId
392  const Box& destFabBox,
393  BoxList* returnedUnfilledBoxes)
394 {
395  return AddBox(fabarrayid,
396  destFabBox,
397  returnedUnfilledBoxes,
398  0,
399  0,
400  fabArrays[fabarrayid.Id()]->nComp(),
401  true);
402 }
403 
404 template <class FAB>
406 {
407  clear();
408 }
409 
410 template <class FAB>
411 void
413 {
414  for (unsigned int i = 0, N = fabCopyDescList.size(); i < N; ++i)
415  {
416  for (auto fmi = fabCopyDescList[i].begin(), End = fabCopyDescList[i].end();
417  fmi != End;
418  ++fmi)
419  {
420  delete (*fmi).second;
421  }
422  }
423 
424  fabArrays.clear();
425  fabCopyDescList.clear();
426  fabComTagList.clear();
427 
428  nextFillBoxId = 0;
429  dataAvailable = false;
430 }
431 
432 template <class FAB>
433 void
435 {
436  dataAvailable = true;
437 
438  if (ParallelDescriptor::NProcs() == 1) { return; }
439 
440 #ifdef BL_USE_MPI
441  using value_type = typename FAB::value_type;
442 
443  BL_PROFILE("FabArrayCopyDescriptor::CollectData()");
444 
445  const int MyProc = ParallelDescriptor::MyProc();
447 
448  int Total_Rcvs_Size = 0;
449  //
450  // We use this to make finding matching FabComTags more efficient.
451  //
452  std::map< int,FabComTagIterContainer > RcvTags;
453 
454  std::map<int,int> Snds, Rcvs, Npts;
455  //
456  // Set Rcvs[i] to # of blocks needed from CPU i
457  //
458  for (auto it = fabComTagList.begin(),
459  End = fabComTagList.end();
460  it != End;
461  ++it)
462  {
463  BL_ASSERT(it->box.ok());
464  BL_ASSERT(it->procThatNeedsData == MyProc);
465  BL_ASSERT(it->procThatHasData != MyProc);
466 
467  const int Who = it->procThatHasData;
468  const auto Cnt = static_cast<int>((it->box.numPts())*(it->nComp));
469 
470  RcvTags[Who].emplace_back(it);
471 
472  Total_Rcvs_Size += Cnt;
473 
474  if (Rcvs.count(Who) > 0)
475  {
476  Rcvs[Who] += 1;
477  }
478  else
479  {
480  Rcvs[Who] = 1;
481  }
482 
483  if (Npts.count(Who) > 0)
484  {
485  Npts[Who] += Cnt;
486  }
487  else
488  {
489  Npts[Who] = Cnt;
490  }
491  }
492  BL_ASSERT(Rcvs.count(MyProc) == 0);
493 
494  BL_ASSERT((Total_Rcvs_Size*sizeof(value_type))
495  < static_cast<std::size_t>(std::numeric_limits<int>::max()));
496 
497  const int NProcs = ParallelDescriptor::NProcs();
498 
499  {
500  Vector<int> SndsArray(NProcs,0), RcvsArray(NProcs,0);
501 
502  for (auto const& Rcv : Rcvs)
503  {
504  RcvsArray[Rcv.first] = Rcv.second;
505  }
506 
507  {
508  BL_PROFILE_VAR("CollectData_Alltoall()", blpvCDATA);
509  BL_COMM_PROFILE(BLProfiler::Alltoall, sizeof(int), ParallelDescriptor::MyProc(),
510  BLProfiler::BeforeCall());
511 
512  BL_MPI_REQUIRE( MPI_Alltoall(RcvsArray.dataPtr(),
513  1,
515  SndsArray.dataPtr(),
516  1,
519 
520  BL_COMM_PROFILE(BLProfiler::Alltoall, sizeof(int), ParallelDescriptor::MyProc(),
521  BLProfiler::AfterCall());
522 
523  BL_PROFILE_VAR_STOP(blpvCDATA);
524  }
525  BL_ASSERT(SndsArray[MyProc] == 0);
526 
527  for (int i = 0; i < NProcs; i++) {
528  if (SndsArray[i] > 0) {
529  Snds[i] = SndsArray[i];
530  }
531  }
532  }
533 
534  // There are two rounds of send and recv.
535  // First, the data receivers need to send the data senders meta-data (e.g., boxes).
536  // Then, the senders know what data to send and perform send.
537  const int SeqNum_md = ParallelDescriptor::SeqNum();
538  const int SeqNum_data = ParallelDescriptor::SeqNum();
539 
540  const auto N_snds = static_cast<int>(Snds.size());
541  const auto N_rcvs = static_cast<int>(Rcvs.size());
542 
543  if ( N_snds == 0 && N_rcvs == 0 ) { return; }
544 
545  const int Nints = 4 + 3*AMREX_SPACEDIM; // # of ints in a meta-data
546 
547  // for meta-data
548  Vector<int> md_sender, md_offset, md_icnts, md_bcnts;
549  int* md_recv_data = nullptr;
550  Vector<int*> md_send_data;
551  Vector<MPI_Request> md_recv_reqs, md_send_reqs;
552 
553  // for data
554  Vector<int> data_sender, data_offset;
555  value_type* recv_data = nullptr;
556  Vector<value_type*> send_data;
557  Vector<MPI_Request> data_recv_reqs, data_send_reqs;
558 
559  if (N_snds > 0)
560  {
561  // Recv meta-data
562 
563  int N = 0;
564  for (auto const& Snd : Snds)
565  {
566  md_sender.push_back(Snd.first);
567  md_bcnts.push_back(Snd.second);
568  int cnt = Snd.second * Nints;
569  md_icnts.push_back(cnt);
570  md_offset.push_back(N);
571  N += cnt;
572  }
573 
574  md_recv_data = static_cast<int*>(amrex::The_Arena()->alloc(N*sizeof(int)));
575 
576  for (int i = 0; i < N_snds; ++i)
577  {
578  md_recv_reqs.push_back(ParallelDescriptor::Arecv(&md_recv_data[md_offset[i]],
579  md_icnts[i], md_sender[i],
580  SeqNum_md).req());
581  }
582  }
583 
584  if (N_rcvs > 0)
585  {
586  // Send meta-data
587  for (auto const& Rcv : Rcvs)
588  {
589  int rank = Rcv.first;
590  int Nmds = Rcv.second;
591  int cnt = Nmds * Nints;
592 
593  int* p = static_cast<int*>(amrex::The_Arena()->alloc(cnt*sizeof(int)));
594  md_send_data.push_back(p);
595 
596  const FabComTagIterContainer& tags = RcvTags[rank];
597 
598  // initialized the data
599  int * md = p;
600  for (int i = 0; i < Nmds; ++i, md += Nints)
601  {
602  md[0] = tags[i]->fabArrayId;
603  md[1] = tags[i]->fabIndex;
604  md[2] = tags[i]->srcComp;
605  md[3] = tags[i]->nComp;
606  const int* lo = tags[i]->box.loVect();
607  const int* hi = tags[i]->box.hiVect();
608  const IntVect& bxtyp = tags[i]->box.type();
609  const int* tp = bxtyp.getVect();
610  AMREX_D_EXPR(md[4] = lo[0],
611  md[5] = lo[1],
612  md[6] = lo[2]);
613  AMREX_D_EXPR(md[4+ AMREX_SPACEDIM] = hi[0],
614  md[5+ AMREX_SPACEDIM] = hi[1],
615  md[6+ AMREX_SPACEDIM] = hi[2]);
616  AMREX_D_EXPR(md[4+2*AMREX_SPACEDIM] = tp[0],
617  md[5+2*AMREX_SPACEDIM] = tp[1],
618  md[6+2*AMREX_SPACEDIM] = tp[2]);
619  }
620 
621  md_send_reqs.push_back(ParallelDescriptor::Asend(p,cnt,rank,SeqNum_md).req());
622  }
623  }
624 
625  if (N_rcvs > 0)
626  {
627  recv_data = static_cast<value_type*>(amrex::The_Arena()->alloc(Total_Rcvs_Size*sizeof(value_type)));
628 
629  // Post receives for data
630  int Idx = 0;
631  for (auto & Npt : Npts)
632  {
633  int Who = Npt.first;
634  int Cnt = Npt.second;
635  BL_ASSERT(Cnt > 0);
637  data_sender.push_back(Who);
638  data_recv_reqs.push_back(ParallelDescriptor::Arecv(&recv_data[Idx],
639  Cnt,Who,SeqNum_data).req());
640  data_offset.push_back(Idx);
641  Idx += Cnt;
642  }
643  }
644 
645  // Wait on meta-data and do send
646  if (N_snds > 0)
647  {
648  int send_counter = 0;
649  while (send_counter++ < N_snds)
650  {
651  MPI_Status status;
652  int index;
653  ParallelDescriptor::Waitany(md_recv_reqs, index, status);
654 
655  int rank = status.MPI_SOURCE;
656  BL_ASSERT(status.MPI_TAG == SeqNum_md);
657  BL_ASSERT(rank == md_sender[index]);
658 
659  const int* p = &md_recv_data[md_offset[index]];
660  int numboxes = md_bcnts[index];
661  Vector<int> faid(numboxes);
662  Vector<int> fidx(numboxes);
663  Vector<int> scomp(numboxes);
664  Vector<int> ncomp(numboxes);
665  Vector<int> npts(numboxes);
666  Vector<Box> bxs;
667  int N = 0;
668  const int * md = p;
669  for (int i = 0; i < numboxes; ++i, md += Nints)
670  {
671  faid[i] = md[0];
672  fidx[i] = md[1];
673  scomp[i] = md[2];
674  ncomp[i] = md[3];
675  bxs.push_back(Box(IntVect(&md[4]),
676  IntVect(&md[4+AMREX_SPACEDIM]),
677  IntVect(&md[4+AMREX_SPACEDIM*2])));
678  npts[i] = static_cast<int>(bxs.back().numPts()*ncomp[i]);
679  N += npts[i];
680  }
681 
683 
684  auto* data = static_cast<value_type*>(amrex::The_Arena()->alloc(N*sizeof(value_type)));
685  value_type* dptr = data;
686  send_data.push_back(data);
687 
688  for (int i = 0; i < numboxes; ++i)
689  {
690  (*fabArrays[faid[i]])[fidx[i]].template copyToMem<RunOn::Host>(bxs[i],scomp[i],ncomp[i],dptr);
691  dptr += npts[i];
692  }
693 
694  data_send_reqs.push_back(ParallelDescriptor::Asend(data,N,rank,SeqNum_data).req());
695  }
696 
697  amrex::The_Arena()->free(md_recv_data);
698  }
699 
700  // Wait and unpack data
701  if (N_rcvs > 0)
702  {
703  Vector<MPI_Status> stats(N_rcvs);
704 
705  ParallelDescriptor::Waitall(md_send_reqs, stats);
706  for (int i = 0; i < N_rcvs; ++i) {
707  amrex::The_Arena()->free(md_send_data[i]);
708  }
709 
710  ParallelDescriptor::Waitall(data_recv_reqs, stats);
711 
712  std::pair<FCDMapIter,FCDMapIter> match;
713  std::map< int,FabComTagIterContainer >::const_iterator found;
714 
715  for (int k = 0; k < N_rcvs; k++)
716  {
717  const int Who = data_sender[k];
718  const value_type* dptr = &recv_data[data_offset[k]];
719 
720  BL_ASSERT(dptr != nullptr);
721 
722  found = RcvTags.find(Who);
723 
724  BL_ASSERT(found != RcvTags.end());
725 
726  const FabComTagIterContainer& tags = found->second;
727 
728  for (auto const& it : tags)
729  {
730  const FabArrayBase::FabComTag& tag = *it;
731 
732  BL_ASSERT(tag.procThatHasData == Who);
733 
734  match = fabCopyDescList[tag.fabArrayId].equal_range(tag.fillBoxId);
735 
736  for (auto fmi = match.first; fmi != match.second; ++fmi)
737  {
738  FabCopyDescriptor<FAB>* fcdp = (*fmi).second;
739 
740  BL_ASSERT(fcdp->fillBoxId == tag.fillBoxId);
741 
742  if (fcdp->subBox == tag.box)
743  {
744  BL_ASSERT(fcdp->localFabSource->dataPtr() != nullptr);
745  BL_ASSERT(fcdp->localFabSource->box() == tag.box);
746  auto Cnt = static_cast<int>(tag.box.numPts()*tag.nComp);
747  fcdp->localFabSource->template copyFromMem<RunOn::Host>(tag.box,0,tag.nComp,dptr);
748  dptr += Cnt;
749  break;
750  }
751  }
752  }
753  }
754 
755  amrex::The_Arena()->free(recv_data);
756  }
757 
758  // Finished send
759  if (N_snds > 0)
760  {
761  Vector<MPI_Status> stats(N_snds);
762  ParallelDescriptor::Waitall(data_send_reqs, stats);
763 
764  for (int i = 0; i < N_snds; ++i) {
765  amrex::The_Arena()->free(send_data[i]);
766  }
767  }
768 
769 #endif /*BL_USE_MPI*/
770 }
771 
772 
773 template <class FAB>
774 void
776  const FillBoxId& fillboxid,
777  FAB& destFab)
778 {
779  BL_ASSERT(dataAvailable);
780 
781  std::pair<FCDMapIter,FCDMapIter> match = fabCopyDescList[faid.Id()].equal_range(fillboxid.Id());
782 
783  for (auto fmi = match.first; fmi != match.second; ++fmi)
784  {
785  FabCopyDescriptor<FAB>* fcdp = (*fmi).second;
786 
787  BL_ASSERT(fcdp->fillBoxId == fillboxid.Id());
788 
789  destFab.template copy<RunOn::Host>
790  (*fcdp->localFabSource,
791  fcdp->subBox,
792  fcdp->fillType == FillLocally ? fcdp->srcComp : 0,
793  fcdp->subBox,
794  fcdp->destComp,
795  fcdp->nComp);
796  }
797 }
798 
799 template <class FAB>
800 void
802  const FillBoxId& fillboxid,
803  FAB& destFab,
804  const Box& destBox)
805 {
806  BL_ASSERT(dataAvailable);
807 
808  FCDMapIter fmi = fabCopyDescList[faid.Id()].lower_bound(fillboxid.Id());
809 
810  BL_ASSERT(fmi != fabCopyDescList[faid.Id()].end());
811 
812  FabCopyDescriptor<FAB>* fcdp = (*fmi).second;
813 
814  BL_ASSERT(fcdp->fillBoxId == fillboxid.Id());
815 
816  BL_ASSERT(fcdp->subBox.sameSize(destBox));
817 
818  destFab.ParallelCopy(*fcdp->localFabSource,
819  fcdp->subBox,
820  fcdp->fillType == FillLocally ? fcdp->srcComp : 0,
821  destBox,
822  fcdp->destComp,
823  fcdp->nComp);
824 
825  BL_ASSERT(++fmi == fabCopyDescList[faid.Id()].upper_bound(fillboxid.Id()));
826 }
827 
828 template <class FAB>
829 void
831 {
832  const int MyProc = ParallelDescriptor::MyProc();
833 
834  amrex::AllPrint() << "----- "
835  << MyProc
836  << ": Parallel stats for FabArrayCopyDescriptor:" << '\n';
837 
838  for (int fa = 0; fa < fabArrays.size(); ++fa)
839  {
840  amrex::AllPrint() << "fabArrays["
841  << fa
842  << "]->boxArray() = "
843  << fabArrays[fa]->boxArray()
844  << '\n';
845  }
846 }
847 
848 }
849 
850 #endif
#define BL_PROFILE(a)
Definition: AMReX_BLProfiler.H:551
#define BL_COMM_PROFILE(cft, size, pid, tag)
Definition: AMReX_BLProfiler.H:587
#define BL_PROFILE_VAR_STOP(vname)
Definition: AMReX_BLProfiler.H:563
#define BL_PROFILE_VAR(fname, vname)
Definition: AMReX_BLProfiler.H:560
#define BL_ASSERT(EX)
Definition: AMReX_BLassert.H:39
#define AMREX_D_EXPR(a, b, c)
Definition: AMReX_SPACE.H:81
Print on all processors of the default communicator.
Definition: AMReX_Print.H:117
virtual void free(void *pt)=0
A pure virtual function for deleting the arena pointed to by pt.
virtual void * alloc(std::size_t sz)=0
bool ok() const
Return true if Box is valid and they all have the same IndexType. Is true by default if the BoxArray ...
A List of Disjoint Boxes.
Definition: AMReX_BoxDomain.H:67
BoxDomain & rmBox(const Box &b)
Remove a box from the domain.
Definition: AMReX_BoxDomain.cpp:159
const BoxList & boxList() const
Return a const reference to the underlying BoxList of this BoxDomain.
Definition: AMReX_BoxDomain.cpp:82
void add(const Box &b)
Add a Box to the domain.
Definition: AMReX_BoxDomain.cpp:121
A class for managing a List of Boxes that share a common IndexType. This class implements operations ...
Definition: AMReX_BoxList.H:52
void clear()
Remove all Boxes from this BoxList.
AMREX_GPU_HOST_DEVICE IndexTypeND< dim > ixType() const noexcept
Returns the indexing type.
Definition: AMReX_Box.H:127
AMREX_GPU_HOST_DEVICE bool sameSize(const BoxND &b) const noexcept
Returns true is Boxes same size, ie translates of each other,. It is an error if they have different ...
Definition: AMReX_Box.H:279
AMREX_GPU_HOST_DEVICE Long numPts() const noexcept
Returns the number of points contained in the BoxND.
Definition: AMReX_Box.H:346
const DistributionMapping & DistributionMap() const noexcept
Return constant reference to associated DistributionMapping.
Definition: AMReX_FabArrayBase.H:130
Box box(int K) const noexcept
Return the Kth Box in the BoxArray. That is, the valid region of the Kth grid.
Definition: AMReX_FabArrayBase.H:100
Box fabbox(int K) const noexcept
Return the Kth FABs Box in the FabArray. That is, the region the Kth fab is actually defined on.
This class orchestrates filling a destination fab of size destFabBox from fabarray on the local proce...
Definition: AMReX_FACopyDescriptor.H:108
void PrintStats() const
Definition: AMReX_FACopyDescriptor.H:830
std::vector< FabArray< FAB > * > fabArrays
Definition: AMReX_FACopyDescriptor.H:195
int nFabCopyDescs() const
Definition: AMReX_FACopyDescriptor.H:171
bool dataAvailable
Definition: AMReX_FACopyDescriptor.H:199
FabArrayCopyDescriptor(const FabArrayCopyDescriptor< FAB > &)=delete
FabArrayCopyDescriptor(FabArrayCopyDescriptor< FAB > &&)=delete
int nFabComTags() const
Definition: AMReX_FACopyDescriptor.H:169
void clear()
Definition: AMReX_FACopyDescriptor.H:412
void FillFab(FabArrayId faid, const FillBoxId &fillboxid, FAB &destFab)
Definition: AMReX_FACopyDescriptor.H:775
typename FCDMap::const_iterator FCDMapConstIter
Definition: AMReX_FACopyDescriptor.H:113
int nextFillBoxId
Definition: AMReX_FACopyDescriptor.H:198
void CollectData()
Definition: AMReX_FACopyDescriptor.H:434
FillBoxId AddBox(FabArrayId fabarrayid, const Box &destFabBox, BoxList *unfilledBoxes)
Definition: AMReX_FACopyDescriptor.H:391
FabComTagContainer fabComTagList
Definition: AMReX_FACopyDescriptor.H:197
FabArrayId RegisterFabArray(FabArray< FAB > *fabarray)
Definition: AMReX_FACopyDescriptor.H:204
int CurrentNFabArrays() const
Definition: AMReX_FACopyDescriptor.H:167
FabArrayCopyDescriptor< FAB > & operator=(const FabArrayCopyDescriptor< FAB > &)=delete
FillBoxId AddBox(FabArrayId fabarrayid, const Box &destFabBox, BoxList *unfilledBoxes, int srccomp, int destcomp, int numcomp)
Definition: AMReX_FACopyDescriptor.H:312
~FabArrayCopyDescriptor()
Definition: AMReX_FACopyDescriptor.H:405
bool DataAvailable() const
Definition: AMReX_FACopyDescriptor.H:163
std::vector< FabArrayBase::FabComTag > FabComTagContainer
Definition: AMReX_FACopyDescriptor.H:189
void AddBoxDoIt(FabArrayId fabarrayid, const Box &destFabBox, BoxList *returnedUnfilledBoxes, int faindex, int srccomp, int destcomp, int numcomp, bool bUseValidBox, BoxDomain &unfilledBoxDomain)
Definition: AMReX_FACopyDescriptor.H:219
FillBoxId AddBox(FabArrayId fabarrayid, const Box &destFabBox, BoxList *unfilledBoxes, int fabarrayindex, int srccomp, int destcomp, int numcomp, bool bUseValidBox=true)
Definition: AMReX_FACopyDescriptor.H:354
typename FCDMap::iterator FCDMapIter
Definition: AMReX_FACopyDescriptor.H:112
typename FCDMap::value_type FCDMapValueType
Definition: AMReX_FACopyDescriptor.H:111
void FillFab(FabArrayId faid, const FillBoxId &fillboxid, FAB &destFab, const Box &destBox)
Definition: AMReX_FACopyDescriptor.H:801
std::multimap< int, FabCopyDescriptor< FAB > * > FCDMap
Definition: AMReX_FACopyDescriptor.H:109
std::vector< FCDMap > fabCopyDescList
Definition: AMReX_FACopyDescriptor.H:196
std::vector< FabComTagContainer::const_iterator > FabComTagIterContainer
Definition: AMReX_FACopyDescriptor.H:191
Definition: AMReX_FACopyDescriptor.H:43
int Id() const
Definition: AMReX_FACopyDescriptor.H:50
int fabArrayId
Definition: AMReX_FACopyDescriptor.H:59
bool operator==(const FabArrayId &rhs) const
Definition: AMReX_FACopyDescriptor.H:52
FabArrayId(int newid=-1)
Definition: AMReX_FACopyDescriptor.H:46
An Array of FortranArrayBox(FAB)-like Objects.
Definition: AMReX_FabArray.H:344
Definition: AMReX_FACopyDescriptor.H:19
FillBoxId()=default
int Id() const
Definition: AMReX_FACopyDescriptor.H:30
int FabIndex() const
Definition: AMReX_FACopyDescriptor.H:31
FillBoxId(int newid, const Box &fillbox)
Definition: AMReX_FACopyDescriptor.H:24
Box m_fillBox
Definition: AMReX_FACopyDescriptor.H:37
int m_fabIndex
Definition: AMReX_FACopyDescriptor.H:39
int m_fillBoxId
Definition: AMReX_FACopyDescriptor.H:38
const Box & box() const
Definition: AMReX_FACopyDescriptor.H:33
void FabIndex(int fabindex)
Definition: AMReX_FACopyDescriptor.H:32
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE const int * getVect() const &noexcept
Returns a const pointer to an array of coordinates of the IntVectND. Useful for arguments to FORTRAN ...
Definition: AMReX_IntVect.H:284
MPI_Request req() const
Definition: AMReX_ParallelDescriptor.H:74
T * dataPtr() noexcept
get access to the underlying data pointer
Definition: AMReX_Vector.H:46
@ FAB
Definition: AMReX_AmrvisConstants.H:86
AMREX_GPU_HOST_DEVICE Long size(T const &b) noexcept
integer version
Definition: AMReX_GpuRange.H:26
int NProcs()
Process ID in MPI_COMM_WORLD.
Definition: AMReX_MPMD.cpp:122
int MyProc()
Definition: AMReX_MPMD.cpp:117
bool sameTeam(int rank) noexcept
Definition: AMReX_ParallelDescriptor.H:329
Message Asend(const T *, size_t n, int pid, int tag)
Definition: AMReX_ParallelDescriptor.H:1088
MPI_Comm Communicator() noexcept
Definition: AMReX_ParallelDescriptor.H:210
void Waitany(Vector< MPI_Request > &, int &, MPI_Status &)
Definition: AMReX_ParallelDescriptor.cpp:1299
int MyProc() noexcept
return the rank number local to the current Parallel Context
Definition: AMReX_ParallelDescriptor.H:125
void Waitall(Vector< MPI_Request > &, Vector< MPI_Status > &)
Definition: AMReX_ParallelDescriptor.cpp:1295
int SeqNum() noexcept
Returns sequential message sequence numbers, usually used as tags for send/recv.
Definition: AMReX_ParallelDescriptor.H:613
int NProcs() noexcept
return the number of MPI ranks local to the current Parallel Context
Definition: AMReX_ParallelDescriptor.H:243
Message Arecv(T *, size_t n, int pid, int tag)
Definition: AMReX_ParallelDescriptor.H:1130
@ max
Definition: AMReX_ParallelReduce.H:17
Definition: AMReX_Amr.cpp:49
BoxND< AMREX_SPACEDIM > Box
Definition: AMReX_BaseFwd.H:27
BoxArray intersect(const BoxArray &ba, const Box &b, int ng=0)
Make a BoxArray from the intersection of Box b and BoxArray(+ghostcells).
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Dim3 end(BoxND< dim > const &box) noexcept
Definition: AMReX_Box.H:1890
IntVectND< AMREX_SPACEDIM > IntVect
Definition: AMReX_BaseFwd.H:30
bool match(const BoxArray &x, const BoxArray &y)
Note that two BoxArrays that match are not necessarily equal.
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void ignore_unused(const Ts &...)
This shuts up the compiler about unused variables.
Definition: AMReX.H:111
FillType
Definition: AMReX_FACopyDescriptor.H:16
@ FillLocally
Definition: AMReX_FACopyDescriptor.H:16
@ Unfillable
Definition: AMReX_FACopyDescriptor.H:16
@ FillRemotely
Definition: AMReX_FACopyDescriptor.H:16
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Dim3 begin(BoxND< dim > const &box) noexcept
Definition: AMReX_Box.H:1881
void Abort(const std::string &msg)
Print out message to cerr and exit via abort().
Definition: AMReX.cpp:225
Arena * The_Arena()
Definition: AMReX_Arena.cpp:609
Definition: AMReX_ccse-mpi.H:51
Used for collecting information used in communicating FABs.
Definition: AMReX_FabArrayBase.H:271
int srcComp
Definition: AMReX_FabArrayBase.H:276
int fabArrayId
Definition: AMReX_FabArrayBase.H:280
int procThatHasData
Definition: AMReX_FabArrayBase.H:283
int procThatNeedsData
Definition: AMReX_FabArrayBase.H:282
int destComp
Definition: AMReX_FabArrayBase.H:277
Box box
Definition: AMReX_FabArrayBase.H:284
int fabIndex
Definition: AMReX_FabArrayBase.H:274
int nComp
Definition: AMReX_FabArrayBase.H:278
int fillBoxId
Definition: AMReX_FabArrayBase.H:281
Definition: AMReX_FACopyDescriptor.H:64
int destComp
Definition: AMReX_FACopyDescriptor.H:81
FabCopyDescriptor(const FabCopyDescriptor &)=delete
int copyFromIndex
Definition: AMReX_FACopyDescriptor.H:78
FabCopyDescriptor()
Definition: AMReX_FACopyDescriptor.H:88
int myProc
Definition: AMReX_FACopyDescriptor.H:76
bool cacheDataAllocated
Definition: AMReX_FACopyDescriptor.H:84
FAB * localFabSource
Definition: AMReX_FACopyDescriptor.H:74
~FabCopyDescriptor()
Definition: AMReX_FACopyDescriptor.H:95
Box subBox
Definition: AMReX_FACopyDescriptor.H:75
int nComp
Definition: AMReX_FACopyDescriptor.H:82
int fillBoxId
Definition: AMReX_FACopyDescriptor.H:79
int srcComp
Definition: AMReX_FACopyDescriptor.H:80
FillType fillType
Definition: AMReX_FACopyDescriptor.H:83
FabCopyDescriptor(FabCopyDescriptor &&)=delete
int copyFromProc
Definition: AMReX_FACopyDescriptor.H:77
FabCopyDescriptor & operator=(const FabCopyDescriptor &)=delete
Communication datatype (note: this structure also works without MPI)
Definition: AMReX_ccse-mpi.H:68