AMReX_DistributionMapping.H

Go to the documentation of this file.

#ifndef BL_DISTRIBUTIONMAPPING_H
#define BL_DISTRIBUTIONMAPPING_H
#include <AMReX_Config.H>
 
#include <AMReX.H>
#include <AMReX_Array.H>
#include <AMReX_Vector.H>
#include <AMReX_Box.H>
#include <AMReX_REAL.H>
#include <AMReX_ParallelDescriptor.H>
 
#include <map>
#include <limits>
#include <memory>
#include <cstddef>
#include <iosfwd>
 
namespace amrex {
 
class BoxArray;
class MultiFab;
template <typename T> class FabArray;
template <typename T> class LayoutData;
class FabArrayBase;
 
class DistributionMapping
{
  public:
 
    template <typename T> friend class FabArray;
    friend class FabArrayBase;
 
    enum Strategy { UNDEFINED = -1, ROUNDROBIN, KNAPSACK, SFC, RRSFC };
 
    struct Ref
    {
        Ref () = default;
 
        explicit Ref (int len) : m_pmap(len) {}
 
        explicit Ref (const Vector<int>& pmap) : m_pmap(pmap) {}
 
        explicit Ref (Vector<int>&& pmap) noexcept : m_pmap(std::move(pmap)) {}
 
 
        void clear () { m_pmap.clear();  m_index_array.clear();   m_ownership.clear(); }
 
        Vector<int> m_pmap; 
        Vector<int> m_index_array;  
        std::vector<bool> m_ownership; 
    };
 
    DistributionMapping () noexcept;
 
    DistributionMapping (const DistributionMapping& rhs) = default;
 
    DistributionMapping& operator= (const DistributionMapping& rhs) = default;
 
    DistributionMapping (DistributionMapping&& rhs) noexcept = default;
 
    DistributionMapping& operator= (DistributionMapping&& rhs) noexcept = default;
 
    ~DistributionMapping() noexcept = default;
 
    explicit DistributionMapping (const Vector<int>& pmap);
    explicit DistributionMapping (Vector<int>&& pmap) noexcept;
    explicit DistributionMapping (const BoxArray& boxes,
                                  int nprocs = ParallelDescriptor::NProcs());
 
    explicit DistributionMapping (std::shared_ptr<Ref> a_ref);
 
    DistributionMapping (const DistributionMapping& d1,
                         const DistributionMapping& d2);
 
    void define (const BoxArray& boxes, int nprocs = ParallelDescriptor::NProcs());
    void define (const Vector<int>& pmap);
    void define (Vector<int>&& pmap) noexcept;
    [[nodiscard]] const Vector<int>& ProcessorMap () const noexcept;
 
    [[nodiscard]] Long size () const noexcept { return Long(m_ref->m_pmap.size()); }
    [[nodiscard]] Long capacity () const noexcept { return Long(m_ref->m_pmap.capacity()); }
    [[nodiscard]] bool empty () const noexcept { return m_ref->m_pmap.empty(); }
 
    [[nodiscard]] Long linkCount () const noexcept { return m_ref.use_count(); }
 
    [[nodiscard]] int operator[] (int index) const noexcept { return m_ref->m_pmap[index]; }
 
    std::istream& readFrom (std::istream& is);
 
    std::ostream& writeOn (std::ostream& os) const;
 
    static void strategy (Strategy how);
 
    static Strategy strategy ();
 
    static void SFC_Threshold (int n);
 
    static int SFC_Threshold ();
 
    bool operator== (const DistributionMapping& rhs) const noexcept;
 
    bool operator!= (const DistributionMapping& rhs) const noexcept;
 
    void SFCProcessorMap (const BoxArray& boxes, const std::vector<Long>& wgts, int nprocs,
                          bool sort=true);
    void SFCProcessorMap (const BoxArray& boxes, const std::vector<Long>& wgts, int nprocs,
                          Real& efficiency, bool sort=true);
    void KnapSackProcessorMap (const std::vector<Long>& wgts, int nprocs,
                               Real* efficiency=nullptr,
                               bool do_full_knapsack=true,
                               int nmax=std::numeric_limits<int>::max(),
                               bool sort=true);
    void KnapSackProcessorMap (const DistributionMapping& olddm,
                               const std::vector<Long>& wgts, Real keep_ratio,
                               Real& old_efficiency, Real& new_efficiency,
                               int nmax=std::numeric_limits<int>::max());
    void RoundRobinProcessorMap (int nboxes, int nprocs, bool sort=true);
    void RoundRobinProcessorMap (const std::vector<Long>& wgts, int nprocs, bool sort=true);
 
    static void Initialize ();
 
    static void Finalize ();
 
    static bool SameRefs (const DistributionMapping& lhs,
                          const DistributionMapping& rhs)
          { return lhs.m_ref == rhs.m_ref; }
 
    static DistributionMapping makeKnapSack (const MultiFab& weight,
                                             int nmax=std::numeric_limits<int>::max());
    static DistributionMapping makeKnapSack (const MultiFab& weight, Real& eff,
                                             int nmax=std::numeric_limits<int>::max());
    static DistributionMapping makeKnapSack (const Vector<Real>& rcost,
                                             int nmax=std::numeric_limits<int>::max());
    static DistributionMapping makeKnapSack (const Vector<Real>& rcost, Real& eff,
                                             int nmax=std::numeric_limits<int>::max(),
                                             bool sort=true);
 
    static DistributionMapping makeKnapSack (const LayoutData<Real>& rcost_local,
                                             Real& currentEfficiency, Real& proposedEfficiency,
                                             int nmax=std::numeric_limits<int>::max(),
                                             bool broadcastToAll=true,
                                             int root=ParallelDescriptor::IOProcessorNumber(),
                                             Real keep_ratio = Real(0.0));
 
    static DistributionMapping makeRoundRobin (const MultiFab& weight);
    static DistributionMapping makeSFC (const MultiFab& weight, bool sort=true);
    static DistributionMapping makeSFC (const MultiFab& weight, Real& eff, bool sort=true);
    static DistributionMapping makeSFC (const Vector<Real>& rcost,
                                        const BoxArray& ba, bool sort=true);
    static DistributionMapping makeSFC (const Vector<Real>& rcost,
                                        const BoxArray& ba, Real& eff, bool sort=true);
 
    static DistributionMapping makeSFC (const LayoutData<Real>& rcost_local,
                                        Real& currentEfficiency, Real& proposedEfficiency,
                                        bool broadcastToAll=true,
                                        int root=ParallelDescriptor::IOProcessorNumber());
 
    static std::vector<std::vector<int> > makeSFC (const BoxArray& ba,
                                                   bool use_box_vol=true,
                                                   int nprocs=ParallelContext::NProcsSub() );
 
    template <typename T>
    static void ComputeDistributionMappingEfficiency (const DistributionMapping& dm,
                                                      const std::vector<T>& cost,
                                                      Real* efficiency);
 
    static Vector<Long> ConvertCostRealToLong (const Vector<Real>& rcost);
 
    [[nodiscard]] std::weak_ptr<Ref> getWeakRef () const;
 
private:
 
    const Vector<int>& getIndexArray ();
    const std::vector<bool>& getOwnerShip ();
 
    void RoundRobinProcessorMap (const BoxArray& boxes, int nprocs);
    void KnapSackProcessorMap   (const BoxArray& boxes, int nprocs);
    void SFCProcessorMap        (const BoxArray& boxes, int nprocs);
    void RRSFCProcessorMap      (const BoxArray& boxes, int nprocs);
 
    using LIpair = std::pair<Long,int>;
 
    struct LIpairLT
    {
        bool operator () (const LIpair& lhs,
                          const LIpair& rhs) const noexcept
            {
                return lhs.first < rhs.first;
            }
    };
 
    struct LIpairGT
    {
        bool operator () (const LIpair& lhs,
                          const LIpair& rhs) const noexcept
            {
                return lhs.first > rhs.first;
            }
    };
 
    static void Sort (std::vector<LIpair>& vec, bool reverse);
 
    void RoundRobinDoIt (int                  nboxes,
                         int                  nprocs,
                         std::vector<LIpair>* LIpairV = nullptr,
                         bool                 sort = true);
 
    void KnapSackDoIt (const std::vector<Long>& wgts,
                       int                      nprocs,
                       Real&                    efficiency,
                       bool                     do_full_knapsack,
                       int                      nmax=std::numeric_limits<int>::max(),
                       bool                     sort=true);
 
    void SFCProcessorMapDoIt (const BoxArray&          boxes,
                              const std::vector<Long>& wgts,
                              int                      nprocs,
                              bool                     sort=true,
                              Real*                    efficiency=nullptr);
 
    void RRSFCDoIt           (const BoxArray&          boxes,
                              int                      nprocs);
 
    static void LeastUsedCPUs (int nprocs, Vector<int>& result);
    static void LeastUsedTeams (Vector<int>& rteam, Vector<Vector<int> >& rworker, int nteams, int nworkers);
 
    using PVMF = void (DistributionMapping::*)(const BoxArray &, int);
 
    static Strategy m_Strategy;
    static PVMF m_BuildMap;
 
    //
    std::shared_ptr<Ref> m_ref;
 
public:
    struct RefID {
        constexpr RefID () noexcept {} // =default does not work due to a clang bug // NOLINT
        explicit RefID (Ref* data_) noexcept : data(data_) {}
        bool operator<  (const RefID& rhs) const noexcept { return std::less<>{}(data,rhs.data); }
        bool operator== (const RefID& rhs) const noexcept { return data == rhs.data; }
        bool operator!= (const RefID& rhs) const noexcept { return data != rhs.data; }
        [[nodiscard]] const Ref *dataPtr() const noexcept { return data; }
        void PrintPtr(std::ostream &os) const { os << data << '\n'; }
        friend std::ostream& operator<< (std::ostream& os, const RefID& id);
   private:
        Ref* data = nullptr;
    };
 
    [[nodiscard]] RefID getRefID () const noexcept { return RefID { m_ref.get() }; }
};
 
using DMRef = DistributionMapping::Ref;
 
std::ostream& operator<< (std::ostream& os, const DistributionMapping& pmap);
 
std::ostream& operator<< (std::ostream& os, const DistributionMapping::RefID& id);
 
DistributionMapping MakeSimilarDM (const BoxArray& ba, const MultiFab& mf, const IntVect& ng);
 
DistributionMapping MakeSimilarDM (const BoxArray& ba, const BoxArray& src_ba,
                                   const DistributionMapping& src_dm, const IntVect& ng);
 
template <typename T>
void DistributionMapping::ComputeDistributionMappingEfficiency (
    const DistributionMapping& dm, const std::vector<T>& cost, Real* efficiency)
{
    const int nprocs = ParallelDescriptor::NProcs();
    Vector<T> wgts(nprocs, T(0));
 
    const auto nboxes = int(dm.size());
    for (int ibox = 0; ibox < nboxes; ++ibox) {
        wgts[dm[ibox]] += cost[ibox];
    }
 
    T max_weight = 0;
    T sum_weight = 0;
    for (auto const& w : wgts) {
        max_weight = std::max(w, max_weight);
        sum_weight += w;
    }
 
    AMREX_ASSERT(nprocs > 0 && max_weight > T(0));
 
    *efficiency = static_cast<Real>(sum_weight) /
        (static_cast<Real>(nprocs) * static_cast<Real>(max_weight));
}
 
}
 
#endif /*BL_DISTRIBUTIONMAPPING_H*/