1 #ifndef BL_DISTRIBUTIONMAPPING_H
2 #define BL_DISTRIBUTIONMAPPING_H
3 #include <AMReX_Config.H>
22 template <
typename T>
class FabArray;
23 template <
typename T>
class LayoutData;
75 int nprocs = ParallelDescriptor::
NProcs());
104 [[nodiscard]] Long
size () const noexcept {
return Long(
m_ref->m_pmap.size()); }
105 [[nodiscard]] Long
capacity () const noexcept {
return Long(
m_ref->m_pmap.capacity()); }
106 [[nodiscard]]
bool empty () const noexcept {
return m_ref->m_pmap.empty(); }
112 [[nodiscard]]
int operator[] (
int index)
const noexcept {
return m_ref->m_pmap[index]; }
114 std::istream&
readFrom (std::istream& is);
116 std::ostream&
writeOn (std::ostream& os)
const;
137 Real& efficiency,
bool sort=
true);
139 Real* efficiency=
nullptr,
140 bool do_full_knapsack=
true,
144 const std::vector<Long>& wgts, Real keep_ratio,
145 Real& old_efficiency, Real& new_efficiency,
203 Real& currentEfficiency, Real& proposedEfficiency,
205 bool broadcastToAll=
true,
207 Real keep_ratio = Real(0.0));
213 const BoxArray& ba,
bool sort=
true);
215 const BoxArray& ba, Real& eff,
bool sort=
true);
238 Real& currentEfficiency, Real& proposedEfficiency,
239 bool broadcastToAll=
true,
247 bool use_box_vol=
true,
257 template <
typename T>
259 const std::vector<T>& cost,
278 const LIpair& rhs)
const noexcept
280 return lhs.first < rhs.first;
287 const LIpair& rhs)
const noexcept
289 return lhs.first > rhs.first;
293 static void Sort (std::vector<LIpair>& vec,
bool reverse);
297 std::vector<LIpair>* LIpairV =
nullptr,
303 bool do_full_knapsack,
308 const std::vector<Long>& wgts,
311 Real* efficiency=
nullptr);
379 std::ostream&
operator<< (std::ostream& os,
const DistributionMapping& pmap);
381 std::ostream&
operator<< (std::ostream& os,
const DistributionMapping::RefID&
id);
412 DistributionMapping
MakeSimilarDM (
const BoxArray& ba,
const BoxArray& src_ba,
413 const DistributionMapping& src_dm,
const IntVect& ng);
415 template <
typename T>
422 const auto nboxes =
int(dm.
size());
423 for (
int ibox = 0; ibox < nboxes; ++ibox) {
424 wgts[dm[ibox]] += cost[ibox];
429 for (
auto const& w : wgts) {
430 max_weight =
std::max(w, max_weight);
434 *efficiency =
static_cast<Real
>(sum_weight) /
435 (
static_cast<Real
>(nprocs) *
static_cast<Real
>(max_weight));
A collection of Boxes stored in an Array.
Definition: AMReX_BoxArray.H:549
Calculates the distribution of FABs to MPI processes.
Definition: AMReX_DistributionMapping.H:41
static void Initialize()
Initializes distribution strategy from ParmParse.
Definition: AMReX_DistributionMapping.cpp:107
int operator[](int index) const noexcept
Equivalent to ProcessorMap()[index].
Definition: AMReX_DistributionMapping.H:112
static DistributionMapping makeKnapSack(const MultiFab &weight, int nmax=std::numeric_limits< int >::max())
Definition: AMReX_DistributionMapping.cpp:1723
Long linkCount() const noexcept
Number of references to this DistributionMapping.
Definition: AMReX_DistributionMapping.H:109
void RRSFCProcessorMap(const BoxArray &boxes, int nprocs)
Definition: AMReX_DistributionMapping.cpp:1566
Long size() const noexcept
Length of the underlying processor map.
Definition: AMReX_DistributionMapping.H:104
void SFCProcessorMapDoIt(const BoxArray &boxes, const std::vector< Long > &wgts, int nprocs, bool sort=true, Real *efficiency=nullptr)
Definition: AMReX_DistributionMapping.cpp:1262
void define(const BoxArray &boxes, int nprocs=ParallelDescriptor::NProcs())
Build mapping out of BoxArray over nprocs processors. You need to call this if you built your Distrib...
Definition: AMReX_DistributionMapping.cpp:340
static bool SameRefs(const DistributionMapping &lhs, const DistributionMapping &rhs)
Definition: AMReX_DistributionMapping.H:164
void KnapSackDoIt(const std::vector< Long > &wgts, int nprocs, Real &efficiency, bool do_full_knapsack, int nmax=std::numeric_limits< int >::max(), bool sort=true)
Definition: AMReX_DistributionMapping.cpp:719
void RRSFCDoIt(const BoxArray &boxes, int nprocs)
Definition: AMReX_DistributionMapping.cpp:1533
static int SFC_Threshold()
Definition: AMReX_DistributionMapping.cpp:89
bool empty() const noexcept
Definition: AMReX_DistributionMapping.H:106
bool operator!=(const DistributionMapping &rhs) const noexcept
Are the distributions different?
Definition: AMReX_DistributionMapping.cpp:101
std::ostream & writeOn(std::ostream &os) const
Definition: AMReX_DistributionMapping.cpp:2010
static void LeastUsedTeams(Vector< int > &rteam, Vector< Vector< int > > &rworker, int nteams, int nworkers)
rteam: Least used ordering of Teams rworker[i]: Least used ordering of team workers for Team i
Definition: AMReX_DistributionMapping.cpp:239
static Strategy m_Strategy
Everyone uses the same Strategy – defaults to SFC.
Definition: AMReX_DistributionMapping.H:328
DistributionMapping() noexcept
The default constructor.
Definition: AMReX_DistributionMapping.cpp:303
static void Sort(std::vector< LIpair > &vec, bool reverse)
Definition: AMReX_DistributionMapping.cpp:177
std::shared_ptr< Ref > m_ref
The data – a reference-counted pointer to a Ref.
Definition: AMReX_DistributionMapping.H:358
static DistributionMapping makeRoundRobin(const MultiFab &weight)
Definition: AMReX_DistributionMapping.cpp:1746
void KnapSackProcessorMap(const std::vector< Long > &wgts, int nprocs, Real *efficiency=nullptr, bool do_full_knapsack=true, int nmax=std::numeric_limits< int >::max(), bool sort=true)
Definition: AMReX_DistributionMapping.cpp:845
std::pair< Long, int > LIpair
Definition: AMReX_DistributionMapping.H:273
const std::vector< bool > & getOwnerShip()
Definition: AMReX_DistributionMapping.cpp:1943
bool operator==(const DistributionMapping &rhs) const noexcept
Are the distributions equal?
Definition: AMReX_DistributionMapping.cpp:95
static DistributionMapping makeSFC(const MultiFab &weight, bool sort=true)
Definition: AMReX_DistributionMapping.cpp:1757
void RoundRobinProcessorMap(int nboxes, int nprocs, bool sort=true)
Definition: AMReX_DistributionMapping.cpp:450
const Vector< int > & getIndexArray()
Definition: AMReX_DistributionMapping.cpp:1924
void SFCProcessorMap(const BoxArray &boxes, const std::vector< Long > &wgts, int nprocs, bool sort=true)
Definition: AMReX_DistributionMapping.cpp:1488
const Vector< int > & ProcessorMap() const noexcept
Returns a constant reference to the mapping of boxes in the underlying BoxArray to the CPU that holds...
Definition: AMReX_DistributionMapping.cpp:47
RefID getRefID() const noexcept
This gives a unique ID of the reference, which is different from dmID above.
Definition: AMReX_DistributionMapping.H:375
static Strategy strategy()
Definition: AMReX_DistributionMapping.cpp:53
void RoundRobinDoIt(int nboxes, int nprocs, std::vector< LIpair > *LIpairV=nullptr, bool sort=true)
Definition: AMReX_DistributionMapping.cpp:366
static void Finalize()
Definition: AMReX_DistributionMapping.cpp:167
static void LeastUsedCPUs(int nprocs, Vector< int > &result)
Least used ordering of CPUs (by # of bytes of FAB data).
Definition: AMReX_DistributionMapping.cpp:192
Strategy
The distribution strategies.
Definition: AMReX_DistributionMapping.H:48
@ UNDEFINED
Definition: AMReX_DistributionMapping.H:48
@ KNAPSACK
Definition: AMReX_DistributionMapping.H:48
@ RRSFC
Definition: AMReX_DistributionMapping.H:48
@ ROUNDROBIN
Definition: AMReX_DistributionMapping.H:48
@ SFC
Definition: AMReX_DistributionMapping.H:48
void(DistributionMapping::*)(const BoxArray &, int) PVMF
A useful typedef.
Definition: AMReX_DistributionMapping.H:325
Long capacity() const noexcept
Definition: AMReX_DistributionMapping.H:105
static PVMF m_BuildMap
Pointer to one of the CreateProcessorMap() functions. Corresponds to the one specified by m_Strategy.
Definition: AMReX_DistributionMapping.H:333
std::istream & readFrom(std::istream &is)
Definition: AMReX_DistributionMapping.cpp:1989
static void ComputeDistributionMappingEfficiency(const DistributionMapping &dm, const std::vector< T > &cost, Real *efficiency)
Computes the average cost per MPI rank given a distribution mapping global cost vector.
Definition: AMReX_DistributionMapping.H:416
Base class for FabArray.
Definition: AMReX_FabArrayBase.H:41
An Array of FortranArrayBox(FAB)-like Objects.
Definition: AMReX_FabArray.H:344
a one-thingy-per-box distributed object
Definition: AMReX_LayoutData.H:13
A collection (stored as an array) of FArrayBox objects.
Definition: AMReX_MultiFab.H:38
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition: AMReX_Vector.H:27
int NProcs()
Process ID in MPI_COMM_WORLD.
Definition: AMReX_MPMD.cpp:122
int NProcsSub() noexcept
number of ranks in current frame
Definition: AMReX_ParallelContext.H:74
int NProcs() noexcept
return the number of MPI ranks local to the current Parallel Context
Definition: AMReX_ParallelDescriptor.H:243
int IOProcessorNumber() noexcept
Definition: AMReX_ParallelDescriptor.H:266
@ max
Definition: AMReX_ParallelReduce.H:17
Definition: AMReX_Amr.cpp:49
IntVectND< AMREX_SPACEDIM > IntVect
Definition: AMReX_BaseFwd.H:30
DistributionMapping MakeSimilarDM(const BoxArray &ba, const MultiFab &mf, const IntVect &ng)
Function that creates a DistributionMapping "similar" to that of a MultiFab.
Definition: AMReX_DistributionMapping.cpp:2023
const int[]
Definition: AMReX_BLProfiler.cpp:1664
std::ostream & operator<<(std::ostream &os, AmrMesh const &amr_mesh)
Definition: AMReX_AmrMesh.cpp:1236
Definition: AMReX_DistributionMapping.H:285
bool operator()(const LIpair &lhs, const LIpair &rhs) const noexcept
Definition: AMReX_DistributionMapping.H:286
Definition: AMReX_DistributionMapping.H:276
bool operator()(const LIpair &lhs, const LIpair &rhs) const noexcept
Definition: AMReX_DistributionMapping.H:277
Definition: AMReX_DistributionMapping.H:361
constexpr RefID() noexcept
Definition: AMReX_DistributionMapping.H:362
friend std::ostream & operator<<(std::ostream &os, const RefID &id)
Definition: AMReX_DistributionMapping.cpp:1982
bool operator==(const RefID &rhs) const noexcept
Definition: AMReX_DistributionMapping.H:365
void PrintPtr(std::ostream &os) const
Definition: AMReX_DistributionMapping.H:368
Ref * data
Definition: AMReX_DistributionMapping.H:371
const Ref * dataPtr() const noexcept
Definition: AMReX_DistributionMapping.H:367
bool operator!=(const RefID &rhs) const noexcept
Definition: AMReX_DistributionMapping.H:366
bool operator<(const RefID &rhs) const noexcept
Definition: AMReX_DistributionMapping.H:364
RefID(Ref *data_) noexcept
Definition: AMReX_DistributionMapping.H:363
Definition: AMReX_DistributionMapping.H:336
Ref()=default
Constructors to match those in DistributionMapping ....
std::vector< bool > m_ownership
true ownership
Definition: AMReX_DistributionMapping.H:354
Ref(const Vector< int > &pmap)
Definition: AMReX_DistributionMapping.H:344
Vector< int > m_pmap
index array for all boxes
Definition: AMReX_DistributionMapping.H:352
Vector< int > m_index_array
index array for local boxes owned by the team
Definition: AMReX_DistributionMapping.H:353
Ref(int len)
Definition: AMReX_DistributionMapping.H:342
void clear()
dtor, copy-ctor, copy-op=, move-ctor, and move-op= are compiler generated.
Definition: AMReX_DistributionMapping.H:350
Ref(Vector< int > &&pmap) noexcept
Definition: AMReX_DistributionMapping.H:346