Block-Structured AMR Software Framework
AMReX_DistributionMapping.H
Go to the documentation of this file.
1 #ifndef BL_DISTRIBUTIONMAPPING_H
2 #define BL_DISTRIBUTIONMAPPING_H
3 #include <AMReX_Config.H>
4 
5 #include <AMReX.H>
6 #include <AMReX_Array.H>
7 #include <AMReX_Vector.H>
8 #include <AMReX_Box.H>
9 #include <AMReX_REAL.H>
11 
12 #include <map>
13 #include <limits>
14 #include <memory>
15 #include <cstddef>
16 #include <iosfwd>
17 
18 namespace amrex {
19 
20 class BoxArray;
21 class MultiFab;
22 template <typename T> class FabArray;
23 template <typename T> class LayoutData;
24 class FabArrayBase;
25 
41 {
42  public:
43 
44  template <typename T> friend class FabArray;
45  friend class FabArrayBase;
46 
49 
50  struct Ref
51  {
53  Ref () = default;
54 
55  explicit Ref (int len) : m_pmap(len) {}
56 
57  explicit Ref (const Vector<int>& pmap) : m_pmap(pmap) {}
58 
59  explicit Ref (Vector<int>&& pmap) noexcept : m_pmap(std::move(pmap)) {}
60 
62 
63  void clear () { m_pmap.clear(); m_index_array.clear(); m_ownership.clear(); }
64 
67  std::vector<bool> m_ownership;
68  };
69 
71  DistributionMapping () noexcept;
72 
74  DistributionMapping (const DistributionMapping& rhs) = default;
75 
77  DistributionMapping& operator= (const DistributionMapping& rhs) = default;
78 
80  DistributionMapping (DistributionMapping&& rhs) noexcept = default;
81 
83  DistributionMapping& operator= (DistributionMapping&& rhs) noexcept = default;
84 
86  ~DistributionMapping() noexcept = default;
87 
91  explicit DistributionMapping (const Vector<int>& pmap);
92  explicit DistributionMapping (Vector<int>&& pmap) noexcept;
94  explicit DistributionMapping (const BoxArray& boxes,
95  int nprocs = ParallelDescriptor::NProcs());
96 
97  explicit DistributionMapping (std::shared_ptr<Ref> a_ref);
98 
104  const DistributionMapping& d2);
105 
111  void define (const BoxArray& boxes, int nprocs = ParallelDescriptor::NProcs());
116  void define (const Vector<int>& pmap);
117  void define (Vector<int>&& pmap) noexcept;
124  [[nodiscard]] const Vector<int>& ProcessorMap () const noexcept;
125 
127  [[nodiscard]] Long size () const noexcept { return Long(m_ref->m_pmap.size()); }
128  [[nodiscard]] Long capacity () const noexcept { return Long(m_ref->m_pmap.capacity()); }
129  [[nodiscard]] bool empty () const noexcept { return m_ref->m_pmap.empty(); }
130 
132  [[nodiscard]] Long linkCount () const noexcept { return m_ref.use_count(); }
133 
135  [[nodiscard]] int operator[] (int index) const noexcept { return m_ref->m_pmap[index]; }
136 
137  std::istream& readFrom (std::istream& is);
138 
139  std::ostream& writeOn (std::ostream& os) const;
140 
142  static void strategy (Strategy how);
143 
144  static Strategy strategy ();
145 
147  static void SFC_Threshold (int n);
148 
149  static int SFC_Threshold ();
150 
152  bool operator== (const DistributionMapping& rhs) const noexcept;
153 
155  bool operator!= (const DistributionMapping& rhs) const noexcept;
156 
157  void SFCProcessorMap (const BoxArray& boxes, const std::vector<Long>& wgts, int nprocs,
158  bool sort=true);
159  void SFCProcessorMap (const BoxArray& boxes, const std::vector<Long>& wgts, int nprocs,
160  Real& efficiency, bool sort=true);
161  void KnapSackProcessorMap (const std::vector<Long>& wgts, int nprocs,
162  Real* efficiency=nullptr,
163  bool do_full_knapsack=true,
165  bool sort=true);
166  void KnapSackProcessorMap (const DistributionMapping& olddm,
167  const std::vector<Long>& wgts, Real keep_ratio,
168  Real& old_efficiency, Real& new_efficiency,
169  int nmax=std::numeric_limits<int>::max());
170  void RoundRobinProcessorMap (int nboxes, int nprocs, bool sort=true);
171  void RoundRobinProcessorMap (const std::vector<Long>& wgts, int nprocs, bool sort=true);
172 
183  static void Initialize ();
184 
185  static void Finalize ();
186 
187  static bool SameRefs (const DistributionMapping& lhs,
188  const DistributionMapping& rhs)
189  { return lhs.m_ref == rhs.m_ref; }
190 
191  static DistributionMapping makeKnapSack (const MultiFab& weight,
192  int nmax=std::numeric_limits<int>::max());
193  static DistributionMapping makeKnapSack (const MultiFab& weight, Real& eff,
194  int nmax=std::numeric_limits<int>::max());
195  static DistributionMapping makeKnapSack (const Vector<Real>& rcost,
196  int nmax=std::numeric_limits<int>::max());
197  static DistributionMapping makeKnapSack (const Vector<Real>& rcost, Real& eff,
199  bool sort=true);
200 
225  static DistributionMapping makeKnapSack (const LayoutData<Real>& rcost_local,
226  Real& currentEfficiency, Real& proposedEfficiency,
228  bool broadcastToAll=true,
230  Real keep_ratio = Real(0.0));
231 
232  static DistributionMapping makeRoundRobin (const MultiFab& weight);
233  static DistributionMapping makeSFC (const MultiFab& weight, bool sort=true);
234  static DistributionMapping makeSFC (const MultiFab& weight, Real& eff, bool sort=true);
235  static DistributionMapping makeSFC (const Vector<Real>& rcost,
236  const BoxArray& ba, bool sort=true);
237  static DistributionMapping makeSFC (const Vector<Real>& rcost,
238  const BoxArray& ba, Real& eff, bool sort=true);
239 
260  static DistributionMapping makeSFC (const LayoutData<Real>& rcost_local,
261  Real& currentEfficiency, Real& proposedEfficiency,
262  bool broadcastToAll=true,
264 
269  static std::vector<std::vector<int> > makeSFC (const BoxArray& ba,
270  bool use_box_vol=true,
271  int nprocs=ParallelContext::NProcsSub() );
272 
280  template <typename T>
282  const std::vector<T>& cost,
283  Real* efficiency);
284 
285  [[nodiscard]] std::weak_ptr<Ref> getWeakRef () const;
286 
287 private:
288 
289  const Vector<int>& getIndexArray ();
290  const std::vector<bool>& getOwnerShip ();
291 
293  void RoundRobinProcessorMap (const BoxArray& boxes, int nprocs);
294  void KnapSackProcessorMap (const BoxArray& boxes, int nprocs);
295  void SFCProcessorMap (const BoxArray& boxes, int nprocs);
296  void RRSFCProcessorMap (const BoxArray& boxes, int nprocs);
297 
298  using LIpair = std::pair<Long,int>;
299 
300  struct LIpairLT
301  {
302  bool operator () (const LIpair& lhs,
303  const LIpair& rhs) const noexcept
304  {
305  return lhs.first < rhs.first;
306  }
307  };
308 
309  struct LIpairGT
310  {
311  bool operator () (const LIpair& lhs,
312  const LIpair& rhs) const noexcept
313  {
314  return lhs.first > rhs.first;
315  }
316  };
317 
318  static void Sort (std::vector<LIpair>& vec, bool reverse);
319 
320  void RoundRobinDoIt (int nboxes,
321  int nprocs,
322  std::vector<LIpair>* LIpairV = nullptr,
323  bool sort = true);
324 
325  void KnapSackDoIt (const std::vector<Long>& wgts,
326  int nprocs,
327  Real& efficiency,
328  bool do_full_knapsack,
330  bool sort=true);
331 
332  void SFCProcessorMapDoIt (const BoxArray& boxes,
333  const std::vector<Long>& wgts,
334  int nprocs,
335  bool sort=true,
336  Real* efficiency=nullptr);
337 
338  void RRSFCDoIt (const BoxArray& boxes,
339  int nprocs);
340 
342  static void LeastUsedCPUs (int nprocs, Vector<int>& result);
347  static void LeastUsedTeams (Vector<int>& rteam, Vector<Vector<int> >& rworker, int nteams, int nworkers);
348 
350  using PVMF = void (DistributionMapping::*)(const BoxArray &, int);
351 
358  static PVMF m_BuildMap;
359 
360  //
362  std::shared_ptr<Ref> m_ref;
363 
364 public:
365  struct RefID {
366  constexpr RefID () noexcept {} // =default does not work due to a clang bug // NOLINT
367  explicit RefID (Ref* data_) noexcept : data(data_) {}
368  bool operator< (const RefID& rhs) const noexcept { return std::less<>{}(data,rhs.data); }
369  bool operator== (const RefID& rhs) const noexcept { return data == rhs.data; }
370  bool operator!= (const RefID& rhs) const noexcept { return data != rhs.data; }
371  [[nodiscard]] const Ref *dataPtr() const noexcept { return data; }
372  void PrintPtr(std::ostream &os) const { os << data << '\n'; }
373  friend std::ostream& operator<< (std::ostream& os, const RefID& id);
374  private:
375  Ref* data = nullptr;
376  };
377 
379  [[nodiscard]] RefID getRefID () const noexcept { return RefID { m_ref.get() }; }
380 };
381 
383 
385 std::ostream& operator<< (std::ostream& os, const DistributionMapping& pmap);
386 
387 std::ostream& operator<< (std::ostream& os, const DistributionMapping::RefID& id);
388 
402 DistributionMapping MakeSimilarDM (const BoxArray& ba, const MultiFab& mf, const IntVect& ng);
403 
418 DistributionMapping MakeSimilarDM (const BoxArray& ba, const BoxArray& src_ba,
419  const DistributionMapping& src_dm, const IntVect& ng);
420 
421 template <typename T>
423  const DistributionMapping& dm, const std::vector<T>& cost, Real* efficiency)
424 {
425  const int nprocs = ParallelDescriptor::NProcs();
426  Vector<T> wgts(nprocs, T(0));
427 
428  const auto nboxes = int(dm.size());
429  for (int ibox = 0; ibox < nboxes; ++ibox) {
430  wgts[dm[ibox]] += cost[ibox];
431  }
432 
433  T max_weight = 0;
434  T sum_weight = 0;
435  for (auto const& w : wgts) {
436  max_weight = std::max(w, max_weight);
437  sum_weight += w;
438  }
439 
440  *efficiency = static_cast<Real>(sum_weight) /
441  (static_cast<Real>(nprocs) * static_cast<Real>(max_weight));
442 }
443 
444 }
445 
446 #endif /*BL_DISTRIBUTIONMAPPING_H*/
A collection of Boxes stored in an Array.
Definition: AMReX_BoxArray.H:550
Calculates the distribution of FABs to MPI processes.
Definition: AMReX_DistributionMapping.H:41
static void Initialize()
Initializes distribution strategy from ParmParse.
Definition: AMReX_DistributionMapping.cpp:107
int operator[](int index) const noexcept
Equivalent to ProcessorMap()[index].
Definition: AMReX_DistributionMapping.H:135
static DistributionMapping makeKnapSack(const MultiFab &weight, int nmax=std::numeric_limits< int >::max())
Definition: AMReX_DistributionMapping.cpp:1728
Long linkCount() const noexcept
Number of references to this DistributionMapping.
Definition: AMReX_DistributionMapping.H:132
void RRSFCProcessorMap(const BoxArray &boxes, int nprocs)
Definition: AMReX_DistributionMapping.cpp:1571
Long size() const noexcept
Length of the underlying processor map.
Definition: AMReX_DistributionMapping.H:127
void SFCProcessorMapDoIt(const BoxArray &boxes, const std::vector< Long > &wgts, int nprocs, bool sort=true, Real *efficiency=nullptr)
Definition: AMReX_DistributionMapping.cpp:1267
void define(const BoxArray &boxes, int nprocs=ParallelDescriptor::NProcs())
Build mapping out of BoxArray over nprocs processors. You need to call this if you built your Distrib...
Definition: AMReX_DistributionMapping.cpp:345
static bool SameRefs(const DistributionMapping &lhs, const DistributionMapping &rhs)
Definition: AMReX_DistributionMapping.H:187
void KnapSackDoIt(const std::vector< Long > &wgts, int nprocs, Real &efficiency, bool do_full_knapsack, int nmax=std::numeric_limits< int >::max(), bool sort=true)
Definition: AMReX_DistributionMapping.cpp:724
void RRSFCDoIt(const BoxArray &boxes, int nprocs)
Definition: AMReX_DistributionMapping.cpp:1538
static int SFC_Threshold()
Definition: AMReX_DistributionMapping.cpp:89
bool empty() const noexcept
Definition: AMReX_DistributionMapping.H:129
bool operator!=(const DistributionMapping &rhs) const noexcept
Are the distributions different?
Definition: AMReX_DistributionMapping.cpp:101
std::ostream & writeOn(std::ostream &os) const
Definition: AMReX_DistributionMapping.cpp:2021
static void LeastUsedTeams(Vector< int > &rteam, Vector< Vector< int > > &rworker, int nteams, int nworkers)
rteam: Least used ordering of Teams rworker[i]: Least used ordering of team workers for Team i
Definition: AMReX_DistributionMapping.cpp:239
static Strategy m_Strategy
Everyone uses the same Strategy – defaults to SFC.
Definition: AMReX_DistributionMapping.H:353
DistributionMapping() noexcept
The default constructor.
Definition: AMReX_DistributionMapping.cpp:303
static void Sort(std::vector< LIpair > &vec, bool reverse)
Definition: AMReX_DistributionMapping.cpp:177
std::shared_ptr< Ref > m_ref
The data – a reference-counted pointer to a Ref.
Definition: AMReX_DistributionMapping.H:362
static DistributionMapping makeRoundRobin(const MultiFab &weight)
Definition: AMReX_DistributionMapping.cpp:1751
void KnapSackProcessorMap(const std::vector< Long > &wgts, int nprocs, Real *efficiency=nullptr, bool do_full_knapsack=true, int nmax=std::numeric_limits< int >::max(), bool sort=true)
Definition: AMReX_DistributionMapping.cpp:850
std::pair< Long, int > LIpair
Definition: AMReX_DistributionMapping.H:298
const std::vector< bool > & getOwnerShip()
Definition: AMReX_DistributionMapping.cpp:1948
bool operator==(const DistributionMapping &rhs) const noexcept
Are the distributions equal?
Definition: AMReX_DistributionMapping.cpp:95
static DistributionMapping makeSFC(const MultiFab &weight, bool sort=true)
Definition: AMReX_DistributionMapping.cpp:1762
void RoundRobinProcessorMap(int nboxes, int nprocs, bool sort=true)
Definition: AMReX_DistributionMapping.cpp:455
const Vector< int > & getIndexArray()
Definition: AMReX_DistributionMapping.cpp:1929
void SFCProcessorMap(const BoxArray &boxes, const std::vector< Long > &wgts, int nprocs, bool sort=true)
Definition: AMReX_DistributionMapping.cpp:1493
const Vector< int > & ProcessorMap() const noexcept
Returns a constant reference to the mapping of boxes in the underlying BoxArray to the CPU that holds...
Definition: AMReX_DistributionMapping.cpp:47
RefID getRefID() const noexcept
This gives a unique ID of the reference, which is different from dmID above.
Definition: AMReX_DistributionMapping.H:379
static Strategy strategy()
Definition: AMReX_DistributionMapping.cpp:53
void RoundRobinDoIt(int nboxes, int nprocs, std::vector< LIpair > *LIpairV=nullptr, bool sort=true)
Definition: AMReX_DistributionMapping.cpp:371
static void Finalize()
Definition: AMReX_DistributionMapping.cpp:167
static void LeastUsedCPUs(int nprocs, Vector< int > &result)
Least used ordering of CPUs (by # of bytes of FAB data).
Definition: AMReX_DistributionMapping.cpp:192
std::weak_ptr< Ref > getWeakRef() const
Definition: AMReX_DistributionMapping.cpp:1967
Strategy
The distribution strategies.
Definition: AMReX_DistributionMapping.H:48
@ UNDEFINED
Definition: AMReX_DistributionMapping.H:48
@ KNAPSACK
Definition: AMReX_DistributionMapping.H:48
@ RRSFC
Definition: AMReX_DistributionMapping.H:48
@ ROUNDROBIN
Definition: AMReX_DistributionMapping.H:48
@ SFC
Definition: AMReX_DistributionMapping.H:48
void(DistributionMapping::*)(const BoxArray &, int) PVMF
A useful typedef.
Definition: AMReX_DistributionMapping.H:350
Long capacity() const noexcept
Definition: AMReX_DistributionMapping.H:128
static PVMF m_BuildMap
Pointer to one of the CreateProcessorMap() functions. Corresponds to the one specified by m_Strategy.
Definition: AMReX_DistributionMapping.H:358
std::istream & readFrom(std::istream &is)
Definition: AMReX_DistributionMapping.cpp:2000
static void ComputeDistributionMappingEfficiency(const DistributionMapping &dm, const std::vector< T > &cost, Real *efficiency)
Computes the average cost per MPI rank given a distribution mapping global cost vector.
Definition: AMReX_DistributionMapping.H:422
Base class for FabArray.
Definition: AMReX_FabArrayBase.H:41
An Array of FortranArrayBox(FAB)-like Objects.
Definition: AMReX_FabArray.H:344
a one-thingy-per-box distributed object
Definition: AMReX_LayoutData.H:13
A collection (stored as an array) of FArrayBox objects.
Definition: AMReX_MultiFab.H:38
int NProcs()
Process ID in MPI_COMM_WORLD.
Definition: AMReX_MPMD.cpp:122
int NProcsSub() noexcept
number of ranks in current frame
Definition: AMReX_ParallelContext.H:74
int NProcs() noexcept
return the number of MPI ranks local to the current Parallel Context
Definition: AMReX_ParallelDescriptor.H:243
int IOProcessorNumber() noexcept
Definition: AMReX_ParallelDescriptor.H:266
@ max
Definition: AMReX_ParallelReduce.H:17
Definition: AMReX_Amr.cpp:49
DistributionMapping MakeSimilarDM(const BoxArray &ba, const MultiFab &mf, const IntVect &ng)
Function that creates a DistributionMapping "similar" to that of a MultiFab.
Definition: AMReX_DistributionMapping.cpp:2034
const int[]
Definition: AMReX_BLProfiler.cpp:1664
std::ostream & operator<<(std::ostream &os, AmrMesh const &amr_mesh)
Definition: AMReX_AmrMesh.cpp:1236
Definition: AMReX_DistributionMapping.H:310
bool operator()(const LIpair &lhs, const LIpair &rhs) const noexcept
Definition: AMReX_DistributionMapping.H:311
Definition: AMReX_DistributionMapping.H:301
bool operator()(const LIpair &lhs, const LIpair &rhs) const noexcept
Definition: AMReX_DistributionMapping.H:302
Definition: AMReX_DistributionMapping.H:365
constexpr RefID() noexcept
Definition: AMReX_DistributionMapping.H:366
friend std::ostream & operator<<(std::ostream &os, const RefID &id)
Definition: AMReX_DistributionMapping.cpp:1993
bool operator==(const RefID &rhs) const noexcept
Definition: AMReX_DistributionMapping.H:369
void PrintPtr(std::ostream &os) const
Definition: AMReX_DistributionMapping.H:372
Ref * data
Definition: AMReX_DistributionMapping.H:375
const Ref * dataPtr() const noexcept
Definition: AMReX_DistributionMapping.H:371
bool operator!=(const RefID &rhs) const noexcept
Definition: AMReX_DistributionMapping.H:370
bool operator<(const RefID &rhs) const noexcept
Definition: AMReX_DistributionMapping.H:368
RefID(Ref *data_) noexcept
Definition: AMReX_DistributionMapping.H:367
Definition: AMReX_DistributionMapping.H:51
Ref()=default
Constructors to match those in DistributionMapping ....
std::vector< bool > m_ownership
true ownership
Definition: AMReX_DistributionMapping.H:67
Ref(const Vector< int > &pmap)
Definition: AMReX_DistributionMapping.H:57
Vector< int > m_pmap
index array for all boxes
Definition: AMReX_DistributionMapping.H:65
Vector< int > m_index_array
index array for local boxes owned by the team
Definition: AMReX_DistributionMapping.H:66
Ref(int len)
Definition: AMReX_DistributionMapping.H:55
void clear()
dtor, copy-ctor, copy-op=, move-ctor, and move-op= are compiler generated.
Definition: AMReX_DistributionMapping.H:63
Ref(Vector< int > &&pmap) noexcept
Definition: AMReX_DistributionMapping.H:59