Block-Structured AMR Software Framework
amrex::DistributionMapping Class Reference

Calculates the distribution of FABs to MPI processes. More...

#include <AMReX_DistributionMapping.H>

Classes

struct  LIpairGT
 
struct  LIpairLT
 
struct  Ref
 
struct  RefID
 

Public Types

enum  Strategy {
  UNDEFINED = -1 , ROUNDROBIN , KNAPSACK , SFC ,
  RRSFC
}
 The distribution strategies. More...
 

Public Member Functions

 DistributionMapping () noexcept
 The default constructor. More...
 
 DistributionMapping (const DistributionMapping &rhs)=default
 The copy constructor. More...
 
DistributionMappingoperator= (const DistributionMapping &rhs)=default
 The copy assignment operator. More...
 
 DistributionMapping (DistributionMapping &&rhs) noexcept=default
 The move constructor. More...
 
DistributionMappingoperator= (DistributionMapping &&rhs) noexcept=default
 The move assignment operator. More...
 
 ~DistributionMapping () noexcept=default
 The destructor. More...
 
 DistributionMapping (const Vector< int > &pmap)
 Create an object with the specified mapping. More...
 
 DistributionMapping (Vector< int > &&pmap) noexcept
 
 DistributionMapping (const BoxArray &boxes, int nprocs=ParallelDescriptor::NProcs())
 Build mapping out of BoxArray over nprocs processors. More...
 
 DistributionMapping (const DistributionMapping &d1, const DistributionMapping &d2)
 This is a very specialized distribution map. Do NOT use it unless you really understand what it does. More...
 
void define (const BoxArray &boxes, int nprocs=ParallelDescriptor::NProcs())
 Build mapping out of BoxArray over nprocs processors. You need to call this if you built your DistributionMapping with the default constructor. More...
 
void define (const Vector< int > &pmap)
 Build mapping out of an Array of ints. You need to call this if you built your DistributionMapping with the default constructor. More...
 
void define (Vector< int > &&pmap) noexcept
 
const Vector< int > & ProcessorMap () const noexcept
 Returns a constant reference to the mapping of boxes in the underlying BoxArray to the CPU that holds the FAB on that Box. ProcessorMap()[i] is an integer in the interval [0, NCPU) where NCPU is the number of CPUs being used. More...
 
Long size () const noexcept
 Length of the underlying processor map. More...
 
Long capacity () const noexcept
 
bool empty () const noexcept
 
Long linkCount () const noexcept
 Number of references to this DistributionMapping. More...
 
int operator[] (int index) const noexcept
 Equivalent to ProcessorMap()[index]. More...
 
std::istream & readFrom (std::istream &is)
 
std::ostream & writeOn (std::ostream &os) const
 
bool operator== (const DistributionMapping &rhs) const noexcept
 Are the distributions equal? More...
 
bool operator!= (const DistributionMapping &rhs) const noexcept
 Are the distributions different? More...
 
void SFCProcessorMap (const BoxArray &boxes, const std::vector< Long > &wgts, int nprocs, bool sort=true)
 
void SFCProcessorMap (const BoxArray &boxes, const std::vector< Long > &wgts, int nprocs, Real &efficiency, bool sort=true)
 
void KnapSackProcessorMap (const std::vector< Long > &wgts, int nprocs, Real *efficiency=nullptr, bool do_full_knapsack=true, int nmax=std::numeric_limits< int >::max(), bool sort=true)
 
void KnapSackProcessorMap (const DistributionMapping &olddm, const std::vector< Long > &wgts, Real keep_ratio, Real &old_efficiency, Real &new_efficiency, int nmax=std::numeric_limits< int >::max())
 
void RoundRobinProcessorMap (int nboxes, int nprocs, bool sort=true)
 
void RoundRobinProcessorMap (const std::vector< Long > &wgts, int nprocs, bool sort=true)
 
RefID getRefID () const noexcept
 This gives a unique ID of the reference, which is different from dmID above. More...
 

Static Public Member Functions

static void strategy (Strategy how)
 Set/get the distribution strategy. More...
 
static Strategy strategy ()
 
static void SFC_Threshold (int n)
 Set/get the space filling curve threshold. More...
 
static int SFC_Threshold ()
 
static void Initialize ()
 Initializes distribution strategy from ParmParse. More...
 
static void Finalize ()
 
static bool SameRefs (const DistributionMapping &lhs, const DistributionMapping &rhs)
 
static DistributionMapping makeKnapSack (const MultiFab &weight, int nmax=std::numeric_limits< int >::max())
 
static DistributionMapping makeKnapSack (const MultiFab &weight, Real &eff, int nmax=std::numeric_limits< int >::max())
 
static DistributionMapping makeKnapSack (const Vector< Real > &rcost, int nmax=std::numeric_limits< int >::max())
 
static DistributionMapping makeKnapSack (const Vector< Real > &rcost, Real &eff, int nmax=std::numeric_limits< int >::max(), bool sort=true)
 
static DistributionMapping makeKnapSack (const LayoutData< Real > &rcost_local, Real &currentEfficiency, Real &proposedEfficiency, int nmax=std::numeric_limits< int >::max(), bool broadcastToAll=true, int root=ParallelDescriptor::IOProcessorNumber(), Real keep_ratio=Real(0.0))
 Computes a new distribution mapping by distributing input costs according to the knapsack algorithm. More...
 
static DistributionMapping makeRoundRobin (const MultiFab &weight)
 
static DistributionMapping makeSFC (const MultiFab &weight, bool sort=true)
 
static DistributionMapping makeSFC (const MultiFab &weight, Real &eff, bool sort=true)
 
static DistributionMapping makeSFC (const Vector< Real > &rcost, const BoxArray &ba, bool sort=true)
 
static DistributionMapping makeSFC (const Vector< Real > &rcost, const BoxArray &ba, Real &eff, bool sort=true)
 
static DistributionMapping makeSFC (const LayoutData< Real > &rcost_local, Real &currentEfficiency, Real &proposedEfficiency, bool broadcastToAll=true, int root=ParallelDescriptor::IOProcessorNumber())
 Computes a new distribution mapping by distributing input costs according to a space filling curve (SFC) algorithm. More...
 
static std::vector< std::vector< int > > makeSFC (const BoxArray &ba, bool use_box_vol=true, int nprocs=ParallelContext::NProcsSub())
 
template<typename T >
static void ComputeDistributionMappingEfficiency (const DistributionMapping &dm, const std::vector< T > &cost, Real *efficiency)
 Computes the average cost per MPI rank given a distribution mapping global cost vector. More...
 

Private Types

using LIpair = std::pair< Long, int >
 
using PVMF = void(DistributionMapping::*)(const BoxArray &, int)
 A useful typedef. More...
 

Private Member Functions

const Vector< int > & getIndexArray ()
 
const std::vector< bool > & getOwnerShip ()
 
void RoundRobinProcessorMap (const BoxArray &boxes, int nprocs)
 Ways to create the processor map. More...
 
void KnapSackProcessorMap (const BoxArray &boxes, int nprocs)
 
void SFCProcessorMap (const BoxArray &boxes, int nprocs)
 
void RRSFCProcessorMap (const BoxArray &boxes, int nprocs)
 
void RoundRobinDoIt (int nboxes, int nprocs, std::vector< LIpair > *LIpairV=nullptr, bool sort=true)
 
void KnapSackDoIt (const std::vector< Long > &wgts, int nprocs, Real &efficiency, bool do_full_knapsack, int nmax=std::numeric_limits< int >::max(), bool sort=true)
 
void SFCProcessorMapDoIt (const BoxArray &boxes, const std::vector< Long > &wgts, int nprocs, bool sort=true, Real *efficiency=nullptr)
 
void RRSFCDoIt (const BoxArray &boxes, int nprocs)
 

Static Private Member Functions

static void Sort (std::vector< LIpair > &vec, bool reverse)
 
static void LeastUsedCPUs (int nprocs, Vector< int > &result)
 Least used ordering of CPUs (by # of bytes of FAB data). More...
 
static void LeastUsedTeams (Vector< int > &rteam, Vector< Vector< int > > &rworker, int nteams, int nworkers)
 rteam: Least used ordering of Teams rworker[i]: Least used ordering of team workers for Team i More...
 

Private Attributes

std::shared_ptr< Refm_ref
 The data – a reference-counted pointer to a Ref. More...
 

Static Private Attributes

static Strategy m_Strategy = DistributionMapping::SFC
 Everyone uses the same Strategy – defaults to SFC. More...
 
static PVMF m_BuildMap = nullptr
 Pointer to one of the CreateProcessorMap() functions. Corresponds to the one specified by m_Strategy. More...
 

Friends

template<typename T >
class FabArray
 
class FabArrayBase
 

Detailed Description

Calculates the distribution of FABs to MPI processes.

This class calculates the distribution of FABs to MPI processes in a FabArray in a multi-processor environment. By distribution is meant what MPI process in the multi-processor environment owns what FAB. Only the BoxArray on which the FabArray is built is used in determining the distribution. The three types of distributions supported are round-robin, knapsack, and SFC. In the round-robin distribution FAB i is owned by CPU iN where N is total number of CPUs. In the knapsack distribution the FABs are partitioned across CPUs such that the total volume of the Boxes in the underlying BoxArray are as equal across CPUs as is possible. The SFC distribution is based on a space filling curve.

Member Typedef Documentation

◆ LIpair

using amrex::DistributionMapping::LIpair = std::pair<Long,int>
private

◆ PVMF

using amrex::DistributionMapping::PVMF = void (DistributionMapping::*)(const BoxArray &, int)
private

A useful typedef.

Member Enumeration Documentation

◆ Strategy

The distribution strategies.

Enumerator
UNDEFINED 
ROUNDROBIN 
KNAPSACK 
SFC 
RRSFC 

Constructor & Destructor Documentation

◆ DistributionMapping() [1/7]

amrex::DistributionMapping::DistributionMapping ( )
noexcept

The default constructor.

◆ DistributionMapping() [2/7]

amrex::DistributionMapping::DistributionMapping ( const DistributionMapping rhs)
default

The copy constructor.

◆ DistributionMapping() [3/7]

amrex::DistributionMapping::DistributionMapping ( DistributionMapping &&  rhs)
defaultnoexcept

The move constructor.

◆ ~DistributionMapping()

amrex::DistributionMapping::~DistributionMapping ( )
defaultnoexcept

The destructor.

◆ DistributionMapping() [4/7]

amrex::DistributionMapping::DistributionMapping ( const Vector< int > &  pmap)
explicit

Create an object with the specified mapping.

◆ DistributionMapping() [5/7]

amrex::DistributionMapping::DistributionMapping ( Vector< int > &&  pmap)
explicitnoexcept

◆ DistributionMapping() [6/7]

amrex::DistributionMapping::DistributionMapping ( const BoxArray boxes,
int  nprocs = ParallelDescriptor::NProcs() 
)
explicit

Build mapping out of BoxArray over nprocs processors.

◆ DistributionMapping() [7/7]

amrex::DistributionMapping::DistributionMapping ( const DistributionMapping d1,
const DistributionMapping d2 
)

This is a very specialized distribution map. Do NOT use it unless you really understand what it does.

Member Function Documentation

◆ capacity()

Long amrex::DistributionMapping::capacity ( ) const
inlinenoexcept

◆ ComputeDistributionMappingEfficiency()

template<typename T >
void amrex::DistributionMapping::ComputeDistributionMappingEfficiency ( const DistributionMapping dm,
const std::vector< T > &  cost,
Real *  efficiency 
)
static

Computes the average cost per MPI rank given a distribution mapping global cost vector.

Parameters
[in]dmdistribution mapping (mapping from FAB to MPI processes)
[in]costvector giving mapping from FAB to the corresponding cost
[in,out]efficiencyaverage cost per MPI process, as computed from the given distribution mapping and cost

◆ define() [1/3]

void amrex::DistributionMapping::define ( const BoxArray boxes,
int  nprocs = ParallelDescriptor::NProcs() 
)

Build mapping out of BoxArray over nprocs processors. You need to call this if you built your DistributionMapping with the default constructor.

◆ define() [2/3]

void amrex::DistributionMapping::define ( const Vector< int > &  pmap)

Build mapping out of an Array of ints. You need to call this if you built your DistributionMapping with the default constructor.

◆ define() [3/3]

void amrex::DistributionMapping::define ( Vector< int > &&  pmap)
noexcept

◆ empty()

bool amrex::DistributionMapping::empty ( ) const
inlinenoexcept

◆ Finalize()

void amrex::DistributionMapping::Finalize ( )
static

◆ getIndexArray()

const Vector< int > & amrex::DistributionMapping::getIndexArray ( )
private

◆ getOwnerShip()

const std::vector< bool > & amrex::DistributionMapping::getOwnerShip ( )
private

◆ getRefID()

RefID amrex::DistributionMapping::getRefID ( ) const
inlinenoexcept

This gives a unique ID of the reference, which is different from dmID above.

◆ Initialize()

void amrex::DistributionMapping::Initialize ( )
static

Initializes distribution strategy from ParmParse.

ParmParse options are:

DistributionMapping.strategy = ROUNDROBIN DistributionMapping.strategy = KNAPSACK DistributionMapping.strategy = SFC DistributionMapping.strategy = RRFC

◆ KnapSackDoIt()

void amrex::DistributionMapping::KnapSackDoIt ( const std::vector< Long > &  wgts,
int  nprocs,
Real &  efficiency,
bool  do_full_knapsack,
int  nmax = std::numeric_limits<int>::max(),
bool  sort = true 
)
private

◆ KnapSackProcessorMap() [1/3]

void amrex::DistributionMapping::KnapSackProcessorMap ( const BoxArray boxes,
int  nprocs 
)
private

◆ KnapSackProcessorMap() [2/3]

void amrex::DistributionMapping::KnapSackProcessorMap ( const DistributionMapping olddm,
const std::vector< Long > &  wgts,
Real  keep_ratio,
Real &  old_efficiency,
Real &  new_efficiency,
int  nmax = std::numeric_limits<int>::max() 
)

◆ KnapSackProcessorMap() [3/3]

void amrex::DistributionMapping::KnapSackProcessorMap ( const std::vector< Long > &  wgts,
int  nprocs,
Real *  efficiency = nullptr,
bool  do_full_knapsack = true,
int  nmax = std::numeric_limits<int>::max(),
bool  sort = true 
)

◆ LeastUsedCPUs()

void amrex::DistributionMapping::LeastUsedCPUs ( int  nprocs,
Vector< int > &  result 
)
staticprivate

Least used ordering of CPUs (by # of bytes of FAB data).

◆ LeastUsedTeams()

void amrex::DistributionMapping::LeastUsedTeams ( Vector< int > &  rteam,
Vector< Vector< int > > &  rworker,
int  nteams,
int  nworkers 
)
staticprivate

rteam: Least used ordering of Teams rworker[i]: Least used ordering of team workers for Team i

◆ linkCount()

Long amrex::DistributionMapping::linkCount ( ) const
inlinenoexcept

Number of references to this DistributionMapping.

◆ makeKnapSack() [1/5]

DistributionMapping amrex::DistributionMapping::makeKnapSack ( const LayoutData< Real > &  rcost_local,
Real &  currentEfficiency,
Real &  proposedEfficiency,
int  nmax = std::numeric_limits<int>::max(),
bool  broadcastToAll = true,
int  root = ParallelDescriptor::IOProcessorNumber(),
Real  keep_ratio = Real(0.0) 
)
static

Computes a new distribution mapping by distributing input costs according to the knapsack algorithm.

Parameters
[in]rcost_localLayoutData of costs; contains, e.g., costs for the local boxes in the FAB array, corresponding indices in the global indices in the FAB array, and the distribution mapping
[in,out]currentEfficiencywrites the efficiency (i.e., mean cost over all MPI ranks, normalized to the max cost) given the current distribution mapping
[in,out]proposedEfficiencywrites the efficiency for the proposed distribution mapping
[in]nmaxthe maximum number of boxes that can be assigned to any MPI rank by the knapsack algorithm
[in]broadcastToAllcontrols whether to transmit the proposed distribution mapping to all other processes; setting this to false allows to, e.g., test whether the proposed distribution mapping is an improvement relative to the current distribution mapping, before deciding to broadcast the proposed distribution mapping
[in]rootwhich process to collect the local costs from others and compute the proposed distribution mapping
[in]keep_ratiocontrols the fraction of load that should be kept on the original process.
Returns
the proposed load-balanced distribution mapping

◆ makeKnapSack() [2/5]

DistributionMapping amrex::DistributionMapping::makeKnapSack ( const MultiFab weight,
int  nmax = std::numeric_limits<int>::max() 
)
static

◆ makeKnapSack() [3/5]

DistributionMapping amrex::DistributionMapping::makeKnapSack ( const MultiFab weight,
Real &  eff,
int  nmax = std::numeric_limits<int>::max() 
)
static

◆ makeKnapSack() [4/5]

DistributionMapping amrex::DistributionMapping::makeKnapSack ( const Vector< Real > &  rcost,
int  nmax = std::numeric_limits<int>::max() 
)
static

◆ makeKnapSack() [5/5]

DistributionMapping amrex::DistributionMapping::makeKnapSack ( const Vector< Real > &  rcost,
Real &  eff,
int  nmax = std::numeric_limits<int>::max(),
bool  sort = true 
)
static

◆ makeRoundRobin()

DistributionMapping amrex::DistributionMapping::makeRoundRobin ( const MultiFab weight)
static

◆ makeSFC() [1/6]

std::vector< std::vector< int > > amrex::DistributionMapping::makeSFC ( const BoxArray ba,
bool  use_box_vol = true,
int  nprocs = ParallelContext::NProcsSub() 
)
static

if use_box_vol is true, weight boxes by their volume in Distribute otherwise, all boxes will be treated with equal weight

◆ makeSFC() [2/6]

DistributionMapping amrex::DistributionMapping::makeSFC ( const LayoutData< Real > &  rcost_local,
Real &  currentEfficiency,
Real &  proposedEfficiency,
bool  broadcastToAll = true,
int  root = ParallelDescriptor::IOProcessorNumber() 
)
static

Computes a new distribution mapping by distributing input costs according to a space filling curve (SFC) algorithm.

Parameters
[in]rcost_localLayoutData of costs; contains, e.g., costs for the local boxes in the FAB array, corresponding indices in the global indices in the FAB array, and the distribution mapping
[in,out]currentEfficiencywrites the efficiency (i.e., mean cost over all MPI ranks, normalized to the max cost) given the current distribution mapping
[in,out]proposedEfficiencywrites the efficiency for the proposed distribution mapping
[in]broadcastToAllcontrols whether to transmit the proposed distribution mapping to all other processes; setting this to false allows to, e.g., test whether the proposed distribution mapping is an improvement relative to the current distribution mapping, before deciding to broadcast the proposed distribution mapping
[in]rootwhich process to collect the local costs from others and compute the proposed distribution mapping
Returns
the proposed load-balanced distribution mapping

◆ makeSFC() [3/6]

DistributionMapping amrex::DistributionMapping::makeSFC ( const MultiFab weight,
bool  sort = true 
)
static

◆ makeSFC() [4/6]

DistributionMapping amrex::DistributionMapping::makeSFC ( const MultiFab weight,
Real &  eff,
bool  sort = true 
)
static

◆ makeSFC() [5/6]

DistributionMapping amrex::DistributionMapping::makeSFC ( const Vector< Real > &  rcost,
const BoxArray ba,
bool  sort = true 
)
static

◆ makeSFC() [6/6]

DistributionMapping amrex::DistributionMapping::makeSFC ( const Vector< Real > &  rcost,
const BoxArray ba,
Real &  eff,
bool  sort = true 
)
static

◆ operator!=()

bool amrex::DistributionMapping::operator!= ( const DistributionMapping rhs) const
noexcept

Are the distributions different?

◆ operator=() [1/2]

DistributionMapping& amrex::DistributionMapping::operator= ( const DistributionMapping rhs)
default

The copy assignment operator.

◆ operator=() [2/2]

DistributionMapping& amrex::DistributionMapping::operator= ( DistributionMapping &&  rhs)
defaultnoexcept

The move assignment operator.

◆ operator==()

bool amrex::DistributionMapping::operator== ( const DistributionMapping rhs) const
noexcept

Are the distributions equal?

◆ operator[]()

int amrex::DistributionMapping::operator[] ( int  index) const
inlinenoexcept

Equivalent to ProcessorMap()[index].

◆ ProcessorMap()

const Vector< int > & amrex::DistributionMapping::ProcessorMap ( ) const
noexcept

Returns a constant reference to the mapping of boxes in the underlying BoxArray to the CPU that holds the FAB on that Box. ProcessorMap()[i] is an integer in the interval [0, NCPU) where NCPU is the number of CPUs being used.

◆ readFrom()

std::istream & amrex::DistributionMapping::readFrom ( std::istream &  is)

◆ RoundRobinDoIt()

void amrex::DistributionMapping::RoundRobinDoIt ( int  nboxes,
int  nprocs,
std::vector< LIpair > *  LIpairV = nullptr,
bool  sort = true 
)
private

◆ RoundRobinProcessorMap() [1/3]

void amrex::DistributionMapping::RoundRobinProcessorMap ( const BoxArray boxes,
int  nprocs 
)
private

Ways to create the processor map.

◆ RoundRobinProcessorMap() [2/3]

void amrex::DistributionMapping::RoundRobinProcessorMap ( const std::vector< Long > &  wgts,
int  nprocs,
bool  sort = true 
)

◆ RoundRobinProcessorMap() [3/3]

void amrex::DistributionMapping::RoundRobinProcessorMap ( int  nboxes,
int  nprocs,
bool  sort = true 
)

◆ RRSFCDoIt()

void amrex::DistributionMapping::RRSFCDoIt ( const BoxArray boxes,
int  nprocs 
)
private

◆ RRSFCProcessorMap()

void amrex::DistributionMapping::RRSFCProcessorMap ( const BoxArray boxes,
int  nprocs 
)
private

◆ SameRefs()

static bool amrex::DistributionMapping::SameRefs ( const DistributionMapping lhs,
const DistributionMapping rhs 
)
inlinestatic

◆ SFC_Threshold() [1/2]

int amrex::DistributionMapping::SFC_Threshold ( )
static

◆ SFC_Threshold() [2/2]

void amrex::DistributionMapping::SFC_Threshold ( int  n)
static

Set/get the space filling curve threshold.

◆ SFCProcessorMap() [1/3]

void amrex::DistributionMapping::SFCProcessorMap ( const BoxArray boxes,
const std::vector< Long > &  wgts,
int  nprocs,
bool  sort = true 
)

◆ SFCProcessorMap() [2/3]

void amrex::DistributionMapping::SFCProcessorMap ( const BoxArray boxes,
const std::vector< Long > &  wgts,
int  nprocs,
Real &  efficiency,
bool  sort = true 
)

◆ SFCProcessorMap() [3/3]

void amrex::DistributionMapping::SFCProcessorMap ( const BoxArray boxes,
int  nprocs 
)
private

◆ SFCProcessorMapDoIt()

void amrex::DistributionMapping::SFCProcessorMapDoIt ( const BoxArray boxes,
const std::vector< Long > &  wgts,
int  nprocs,
bool  sort = true,
Real *  efficiency = nullptr 
)
private

◆ size()

Long amrex::DistributionMapping::size ( ) const
inlinenoexcept

Length of the underlying processor map.

◆ Sort()

void amrex::DistributionMapping::Sort ( std::vector< LIpair > &  vec,
bool  reverse 
)
staticprivate

◆ strategy() [1/2]

DistributionMapping::Strategy amrex::DistributionMapping::strategy ( )
static

◆ strategy() [2/2]

void amrex::DistributionMapping::strategy ( DistributionMapping::Strategy  how)
static

Set/get the distribution strategy.

◆ writeOn()

std::ostream & amrex::DistributionMapping::writeOn ( std::ostream &  os) const

Friends And Related Function Documentation

◆ FabArray

template<typename T >
friend class FabArray
friend

◆ FabArrayBase

friend class FabArrayBase
friend

Member Data Documentation

◆ m_BuildMap

DistributionMapping::PVMF amrex::DistributionMapping::m_BuildMap = nullptr
staticprivate

Pointer to one of the CreateProcessorMap() functions. Corresponds to the one specified by m_Strategy.

◆ m_ref

std::shared_ptr<Ref> amrex::DistributionMapping::m_ref
private

The data – a reference-counted pointer to a Ref.

◆ m_Strategy

DistributionMapping::Strategy amrex::DistributionMapping::m_Strategy = DistributionMapping::SFC
staticprivate

Everyone uses the same Strategy – defaults to SFC.


The documentation for this class was generated from the following files: