AMReX_NonLocalBC.H

Go to the documentation of this file.

#ifndef AMREX_NONLOCAL_BC_H_
#define AMREX_NONLOCAL_BC_H_
#include <AMReX_Config.H>
#include <AMReX_Concepts.H>
#include <AMReX_TypeTraits.H>
#include <AMReX_FabArray.H>
#include <AMReX_FArrayBox.H>
 
namespace amrex::NonLocalBC {

//                                                            [concept.IndexMapping]
//

template <typename T, typename... Args>
using Inverse_t = decltype(std::declval<T>().Inverse(std::declval<Args>()...));

template <typename T>
struct HasInverseMemFn : IsDetectedExact<Dim3, Inverse_t, T, Dim3> {};

template <class IndexMap>
struct IsIndexMapping
    : Conjunction<IsCallableR<Dim3, IndexMap&, const Dim3&>,
                  HasInverseMemFn<const IndexMap&>> {};

//                                             [IndexMapping.MultiBlockIndexMapping]
//

struct MultiBlockIndexMapping {
    IntVect permutation{AMREX_D_DECL(0, 1, 2)};
    IntVect offset{AMREX_D_DECL(0, 0, 0)};
    IntVect sign{AMREX_D_DECL(1, 1, 1)};

    [[nodiscard]] AMREX_GPU_HOST_DEVICE Dim3 operator()(Dim3 i) const noexcept {
        int iv[3]{i.x, i.y, i.z};
        int iv_new[3]{};
        for (int d = 0; d < AMREX_SPACEDIM; ++d) {
            iv_new[d] = sign[d] * (iv[permutation[d]] - offset[d]);
        }
        return Dim3{.x = iv_new[0], .y = iv_new[1], .z = iv_new[2]};
    }

    [[nodiscard]] AMREX_GPU_HOST_DEVICE Dim3 Inverse(Dim3 i) const noexcept {
        int iv_new[3]{i.x, i.y, i.z};
        int iv[3]{};
        for (int d = 0; d < AMREX_SPACEDIM; ++d) {
            AMREX_ASSERT(sign[d] == 1 || sign[d] == -1);
            iv[permutation[d]] = iv_new[d] * sign[d] + offset[d];
        }
        return Dim3{.x = iv[0], .y = iv[1], .z = iv[2]};
    }
 
    [[nodiscard]] IndexType operator()(IndexType it) const noexcept {
        return IndexType{IntVect{AMREX_D_DECL(it[permutation[0]], it[permutation[1]], it[permutation[2]])}};
    }
 
    [[nodiscard]] IndexType Inverse(IndexType it) const noexcept {
        IntVect inverse_permutation;
        for (int i = 0; i < AMREX_SPACEDIM; ++i) {
            inverse_permutation[permutation[i]] = i;
        }
        return IndexType{IntVect{AMREX_D_DECL(it[inverse_permutation[0]], it[inverse_permutation[1]], it[inverse_permutation[2]])}};
    }
};

template <typename DTOS>
requires (IsCallableR<Dim3, DTOS, Dim3>::value)
IntVect
Apply(DTOS const& dtos, const IntVect& iv)
{
    Dim3 i = dtos(iv.dim3());
    return IntVect{AMREX_D_DECL(i.x, i.y, i.z)};
}

template <typename DTOS>
requires (IsCallableR<Dim3, DTOS, Dim3>::value && !IsCallableR<IndexType, DTOS, IndexType>::value)
Box
Image (DTOS const& dtos, const Box& box)
{
    IntVect mapped_smallEnd = Apply(dtos, box.smallEnd());
    IntVect mapped_bigEnd = Apply(dtos, box.bigEnd());
    IntVect smallEnd;
    IntVect bigEnd;
    for (int d = 0; d < AMREX_SPACEDIM; ++d) {
        smallEnd[d] = std::min(mapped_smallEnd[d], mapped_bigEnd[d]);
        bigEnd[d] = std::max(mapped_smallEnd[d], mapped_bigEnd[d]);
    }
    return Box{smallEnd, bigEnd, box.ixType()};
}

template <typename DTOS>
requires (IsCallableR<Dim3, DTOS, Dim3>::value && IsCallableR<IndexType, DTOS, IndexType>::value)
Box
Image (DTOS const& dtos, const Box& box)
{
    // "Forget" the index type mapping and invoke Image without changing the index type.
    Box srcbox = Image([&dtos](Dim3 d) { return dtos(d); }, box);
    // Fix the index type of the resulting box
    srcbox.setType(dtos(box.ixType()));
    return srcbox;
}

template <typename DTOS>
requires (HasInverseMemFn<DTOS>::value)
IntVect
ApplyInverse(DTOS const& dtos, const IntVect& iv)
{
    return Apply([&dtos](Dim3 i) { return dtos.Inverse(i); }, iv);
}

template <typename DTOS>
requires (HasInverseMemFn<DTOS>::value && !IsCallableR<IndexType, DTOS, IndexType>::value)
Box
InverseImage (DTOS const& dtos, const Box& box)
{
    return Image([&dtos](Dim3 i) { return dtos.Inverse(i); }, box);
}

template <typename DTOS>
requires (HasInverseMemFn<DTOS>::value && IsCallableR<IndexType, DTOS, IndexType>::value)
Box
InverseImage (DTOS const& dtos, const Box& box)
{
    return Image([&dtos](auto&& i) { return dtos.Inverse(i); }, box);
}

static_assert(IsIndexMapping<MultiBlockIndexMapping>(), // NOLINT(bugprone-throw-keyword-missing)
              "MultiBlockIndexMapping is expected to satisfy IndexMapping");

//                                                    [class.MultiBlockCommMetaData]
//

struct MultiBlockCommMetaData : FabArrayBase::CommMetaData {

    template <typename DTOS>
    requires (IsIndexMapping<DTOS>::value)
    MultiBlockCommMetaData(const FabArrayBase& dst, const Box& dstbox, const FabArrayBase& src,
                           const IntVect& ngrow, DTOS const& dtos);

    template <typename DTOS>
    requires (IsIndexMapping<DTOS>::value)
    MultiBlockCommMetaData(const BoxArray& dstba, const DistributionMapping& dstdm,
                           const Box& dstbox, const BoxArray& srcba,
                           const DistributionMapping& srcdm, const IntVect& ngrow, DTOS const& dtos);


    template <typename DTOS>
    requires (IsIndexMapping<DTOS>::value)
    void define(const BoxArray& dstba, const DistributionMapping& dstdm, const Box& dstbox,
                const BoxArray& srcba, const DistributionMapping& srcdm, const IntVect& ngrow,
                DTOS const& dtos);
};

//                                                           [concept.FabProjection]
//

template <typename P, BaseFabType FAB>
struct IsFabProjection
    : IsCallableR<typename FAB::value_type, P, Array4<const typename FAB::value_type>, Dim3, int>
{};

//                                                          [FabProjection.Identity]
//                                                           [IndexMapping.Identity]

struct Identity  {
    constexpr Dim3 operator()(Dim3 i) const noexcept { return i; }
    [[nodiscard]] static constexpr Dim3 Inverse(Dim3 i) noexcept { return i; }

    template <typename T>
    constexpr T operator()(Array4<const T> array, Dim3 i, int comp = 0) const
        noexcept(noexcept(array(i.x, i.y, i.z, comp))) {
        return array(i.x, i.y, i.z, comp);
    }

    constexpr int operator()(int i) const noexcept { return i; }
};
static constexpr Identity identity{};
 
static_assert(sizeof(Identity) == 1 );
static_assert(std::is_trivially_default_constructible_v<Identity> );
static_assert(std::is_trivially_copy_assignable_v<Identity> );
static_assert(std::is_trivially_copy_constructible_v<Identity> );
static_assert(IsIndexMapping<Identity>() ); // NOLINT(bugprone-throw-keyword-missing)
static_assert(IsFabProjection<Identity, FArrayBox>() ); // NOLINT(bugprone-throw-keyword-missing)

//                                                     [FabProjection.MapComponents]

template <typename Base, typename Map = Identity> struct MapComponents {
    static_assert(IsCallable<Base, Array4<const Real>, Dim3, int>::value,
                  "Base needs to be a callable function: (Array4<const T>, Dim3, i) -> auto.");
 
    static_assert(IsCallableR<int, Map, int>::value,
                  "Map needs to be a callable function: int -> int.");
 
    Base base;
    Map map;
 
    template <typename T>
    requires (IsCallable<Base, Array4<const T>, Dim3, int>::value && IsCallableR<int, Map, int>::value)
    constexpr decltype(auto) operator()(Array4<const T> array, Dim3 i, int comp) const
        noexcept(noexcept(base(array, i, map(comp))))
    {
        return base(array, i, map(comp));
    }
};
 
static_assert(std::is_trivially_copy_assignable<MapComponents<Identity>>() ); // NOLINT(bugprone-throw-keyword-missing)
static_assert(std::is_trivially_copy_constructible<MapComponents<Identity>>() ); // NOLINT(bugprone-throw-keyword-missing)
static_assert(IsFabProjection<MapComponents<Identity>, FArrayBox>() ); // NOLINT(bugprone-throw-keyword-missing)

//                                      [FabProjection.MapComponents.SwapComponents]

template <int CompI, int CompJ> struct SwapComponents {
    static_assert(CompI >= 0 && CompJ >= 0, "CompI >= 0 && CompJ >= 0");

    constexpr int operator()(int i) const noexcept {
        const int map[2] = {CompI, CompJ};
        return i == CompI || i == CompJ ? map[std::size_t(i == CompI)] : i;
    }
};
 
template <int CompI> struct SwapComponents<CompI, -1> {
    static_assert(CompI >= 0, "CompI >= 0");
 
    int J;
    constexpr int operator()(int i) const noexcept {
        const int map[2] = {CompI, J};
        return i == CompI || i == J ? map[std::size_t(i == CompI)] : i;
    }
};
 
template <int CompJ> struct SwapComponents<-1, CompJ> {
    static_assert(CompJ >= 0, "CompJ >= 0");
 
    int I;
    constexpr int operator()(int i) const noexcept {
        const int map[2] = {I, CompJ};
        return i == I || i == CompJ ? map[std::size_t(i == I)] : i;
    }
};
 
template <> struct SwapComponents<-1, -1> {
    int I;
    int J;
    constexpr int operator()(int i) const noexcept {
        const int map[2] = {I, J};
        return i == I || i == J ? map[std::size_t(i == I)] : i;
    }
};
 
using DynamicSwapComponents = SwapComponents<-1, -1>;
 
template <int CompI, int CompJ> static constexpr SwapComponents<CompI, CompJ> swap_indices{};
 
static_assert(sizeof(SwapComponents<0, 1>) == 1 );
static_assert(sizeof(DynamicSwapComponents) == 2 * sizeof(int) );
static_assert(sizeof(SwapComponents<0, -1>) == sizeof(int) );
static_assert(sizeof(SwapComponents<-1, 1>) == sizeof(int) );
static_assert(std::is_trivially_default_constructible<MapComponents<Identity, SwapComponents<0, 1>>>() ); // NOLINT(bugprone-throw-keyword-missing)
static_assert(std::is_trivially_copy_assignable<MapComponents<Identity, SwapComponents<0, 1>>>() ); // NOLINT(bugprone-throw-keyword-missing)
static_assert(std::is_trivially_copy_constructible<MapComponents<Identity, SwapComponents<0, 1>>>() ); // NOLINT(bugprone-throw-keyword-missing)
static_assert(IsFabProjection<MapComponents<Identity, SwapComponents<0, 1>>, FArrayBox>() ); // NOLINT(bugprone-throw-keyword-missing)
 
static_assert(swap_indices<0, 1>(0) == 1 );
static_assert(swap_indices<0, 1>(1) == 0 );
static_assert(swap_indices<0, 1>(2) == 2 );
static_assert(DynamicSwapComponents{0, 1}(0) == 1 );
static_assert(DynamicSwapComponents{0, 1}(1) == 0 );
static_assert(DynamicSwapComponents{0, 1}(2) == 2 );

//                                                                  [class.CommData]
//                                                               [class.CommHandler]

struct CommData {
#ifdef AMREX_USE_MPI
    TheFaArenaPointer the_data = nullptr;
    Vector<int> rank{};
    Vector<char*> data{};
    Vector<std::size_t> offset{};
    Vector<std::size_t> size{};
    Vector<MPI_Request> request{};
    Vector<MPI_Status> stats{};
    Vector<const FabArrayBase::CopyComTagsContainer*> cctc{};
    Vector<int> comm_rank{};
    Vector<int> counts{};
    Vector<int> displs{};
    std::size_t total_bytes = 0;
    std::uint64_t id = std::numeric_limits<std::uint64_t>::max();
#endif
};
 
#ifdef AMREX_USE_MPI
void PrepareCommBuffers(CommData& comm, const FabArrayBase::MapOfCopyComTagContainers& cctc,
                        int n_components, std::size_t object_size, std::size_t align);

void PostRecvs(CommData& recv, int mpi_tag);

void PostSends(CommData& send, int mpi_tag);

void TranslateRanks(CommData& comm, MPI_Comm c);

void PrepareAlltoallvCounts(CommData& comm, MPI_Comm c);

void Alltoallv(CommData const& send, CommData& recv, MPI_Comm c);

void PrepareCommBuffersUniform(CommData& comm, const FabArrayBase::MapOfCopyComTagContainers& cctc,
                               int n_components, std::size_t object_size, std::size_t block_size,
                               MPI_Comm c);

void Alltoall(CommData const& send, CommData& recv, std::size_t block_size, MPI_Comm c);

void ReleaseCommBuffer(CommData& comm);

void AcquireCommBuffer(CommData& comm);
#endif

struct CommHandler {
#ifdef AMREX_USE_MPI
    int mpi_tag{};
    CommData recv{};
    CommData send{};
#endif
};

//                                                             [concept.DataPacking]
//
template <typename... Args>
using PrepareSendBuffers_t = decltype(PrepareSendBuffers(std::declval<Args>()...));
 
template <typename... Args>
using PrepareRecvBuffers_t = decltype(PrepareRecvBuffers(std::declval<Args>()...));
 
template <typename... Args>
using PackSendBuffers_t = decltype(PackSendBuffers(std::declval<Args>()...));
 
template <typename... Args>
using UnpackRecvBuffers_t = decltype(UnpackRecvBuffers(std::declval<Args>()...));
 
template <typename... Args>
using LocalCopy_t = decltype(LocalCopy(std::declval<Args>()...));

#if defined(AMREX_USE_CUDA) && defined(_WIN32)
template <typename DP, BaseFabType FAB> struct IsDataPacking : std::true_type {};
#else
template <typename DP, BaseFabType FAB>
struct IsDataPacking :
    Conjunction<
        IsDetected<LocalCopy_t, DP&, FabArray<FAB>&, const FabArray<FAB>&, const FabArrayBase::CopyComTagsContainer&>
#ifdef AMREX_USE_MPI
       ,IsDetected<PrepareSendBuffers_t, DP&, FabArray<FAB>&, const FabArray<FAB>&, CommData&, const FabArrayBase::MapOfCopyComTagContainers&>,
        IsDetected<PrepareRecvBuffers_t, DP&, FabArray<FAB>&, const FabArray<FAB>&, CommData&, const FabArrayBase::MapOfCopyComTagContainers&>,
        IsDetected<PackSendBuffers_t, DP&, const FabArray<FAB>&, CommData&>,
        IsDetected<UnpackRecvBuffers_t, DP&, FabArray<FAB>&, CommData&>
#endif
    > {};
#endif
 
template <BaseFabType FAB, class DTOS = Identity, class Proj = Identity>
requires (IsCallableR<Dim3, DTOS, Dim3>::value && IsFabProjection<Proj, FAB>::value)
void
local_copy_cpu (FabArray<FAB>& dest, const FabArray<FAB>& src, int dcomp, int scomp, int ncomp,
                FabArrayBase::CopyComTagsContainer const& local_tags, DTOS const& dtos = DTOS{},
                Proj const& proj = Proj{}) noexcept;
 
 
template <BaseFabType FAB, class DTOS = Identity, class Proj = Identity>
requires (IsCallableR<Dim3, DTOS, Dim3>::value && IsFabProjection<Proj, FAB>::value)
void
unpack_recv_buffer_cpu (FabArray<FAB>& mf, int dcomp, int ncomp, Vector<char*> const& recv_data,
                        Vector<std::size_t> const& recv_size,
                        Vector<FabArrayBase::CopyComTagsContainer const*> const& recv_cctc,
                        DTOS const& dtos = DTOS{}, Proj const& proj = Proj{}) noexcept;
 
 
#ifdef AMREX_USE_GPU
template <BaseFabType FAB, class DTOS = Identity, class Proj = Identity>
requires (IsCallableR<Dim3, DTOS, Dim3>::value && IsFabProjection<Proj, FAB>::value)
void
local_copy_gpu (FabArray<FAB>& dest, const FabArray<FAB>& src, int dcomp, int scomp, int ncomp,
                FabArrayBase::CopyComTagsContainer const& local_tags, DTOS const& dtos = DTOS{},
                Proj const& proj = Proj{}) noexcept;
 
 
template <BaseFabType FAB, class DTOS = Identity, class Proj = Identity>
requires (IsCallableR<Dim3, DTOS, Dim3>::value && IsFabProjection<Proj, FAB>::value)
void
unpack_recv_buffer_gpu (FabArray<FAB>& mf, int scomp, int ncomp,
                        Vector<char*> const& recv_data,
                        Vector<std::size_t> const& recv_size,
                        Vector<FabArrayBase::CopyComTagsContainer const*> const& recv_cctc,
                        DTOS const& dtos = DTOS{}, Proj const& proj = Proj{});
#endif

//                                                      [DataPacking.PackComponents]
//
// PackComponents is the simplest data packing policy.
// This provides us with sane default behaviour that we can use when defining new
// data packing policies.

struct PackComponents {
    int dest_component{0};
    int src_component{0};
    int n_components{1};
};

template <BaseFabType FAB>
void
LocalCopy (const PackComponents& components, FabArray<FAB>& dest, const FabArray<FAB>& src,
           const FabArrayBase::CopyComTagsContainer& local_tags)
{
#ifdef AMREX_USE_GPU
    if (Gpu::inLaunchRegion()) {
        local_copy_gpu(dest, src, components.dest_component, components.src_component,
                       components.n_components, local_tags);
    } else
#endif
    {
        local_copy_cpu(dest, src, components.dest_component, components.src_component,
                       components.n_components, local_tags);
    }
}
 
#ifdef AMREX_USE_MPI
template <BaseFabType FAB>
void
PrepareSendBuffers (const PackComponents& components, FabArray<FAB>& dest, const FabArray<FAB>& src,
                    CommData& comm, const FabArrayBase::MapOfCopyComTagContainers& cctc)
{
    using T = typename FAB::value_type;
    ignore_unused(dest, src);
    PrepareCommBuffers(comm, cctc, components.n_components, sizeof(T), alignof(T));
}

template <BaseFabType FAB>
void
PrepareRecvBuffers (const PackComponents& components, FabArray<FAB>& dest, const FabArray<FAB>& src,
                    CommData& comm, const FabArrayBase::MapOfCopyComTagContainers& cctc)
{
    using T = typename FAB::value_type;
    ignore_unused(dest, src);
    PrepareCommBuffers(comm, cctc, components.n_components, sizeof(T), alignof(T));
}

template <BaseFabType FAB>
void
PackSendBuffers (const PackComponents& components, const FabArray<FAB>& src, CommData& send)
{
#ifdef AMREX_USE_GPU
    if (Gpu::inLaunchRegion()) {
        FabArray<FAB>::pack_send_buffer_gpu(src, components.src_component, components.n_components,
                                            send.data, send.size, send.cctc, send.id);
    } else
#endif // AMREX_USE_GPU
    {
        FabArray<FAB>::pack_send_buffer_cpu(src, components.src_component, components.n_components,
                                            send.data, send.size, send.cctc);
    }
}

template <BaseFabType FAB>
void
UnpackRecvBuffers (const PackComponents& components, FabArray<FAB>& dest, const CommData& recv)
{
#ifdef AMREX_USE_GPU
    if (Gpu::inLaunchRegion()) {
        unpack_recv_buffer_gpu(dest, components.dest_component, components.n_components, recv.data,
                               recv.size, recv.cctc);
    } else
#endif // AMREX_USE_GPU
    {
        unpack_recv_buffer_cpu(dest, components.dest_component, components.n_components, recv.data,
                               recv.size, recv.cctc);
    }
}
#endif // AMREX_USE_MPI
 
static_assert(IsDataPacking<PackComponents, FArrayBox>(), // NOLINT(bugprone-throw-keyword-missing)
              "PackComponents is expected to satisfy the concept DataPacking.");

//                                    [DataPacking.ApplyDtosAndProjectionOnReciever]
//
template <typename DTOS = Identity, typename FabProj = Identity>
struct ApplyDtosAndProjectionOnReciever : PackComponents {
    constexpr ApplyDtosAndProjectionOnReciever() = default;
    constexpr ApplyDtosAndProjectionOnReciever(const PackComponents& components, DTOS dtos_ = DTOS{}, FabProj proj_ = FabProj{})
        : PackComponents(components), dtos(std::move(dtos_)), proj(std::move(proj_)) {}
 
    DTOS dtos;
    FabProj proj;
 
    static_assert(IsCallableR<Dim3, DTOS, Dim3>::value, "DTOS needs to be a callable: Dim3 -> Dim3");
    static_assert(IsFabProjection<FabProj, FArrayBox>(), "FabProj needs to be at least a projection on FArrayBox.");
};

template <BaseFabType FAB, typename DTOS, typename FabProj>
void
LocalCopy (const ApplyDtosAndProjectionOnReciever<DTOS, FabProj>& packing, FabArray<FAB>& dest,
           const FabArray<FAB>& src, const FabArrayBase::CopyComTagsContainer& local_tags)
{
    static_assert(IsFabProjection<FabProj, FAB>(), "FabProj needs to be a projection for given FAB type.");
#ifdef AMREX_USE_GPU
    if (Gpu::inLaunchRegion()) {
        local_copy_gpu(dest, src, packing.dest_component, packing.src_component,
                       packing.n_components, local_tags, packing.dtos, packing.proj);
    } else
#endif
    {
        local_copy_cpu(dest, src, packing.dest_component, packing.src_component,
                       packing.n_components, local_tags, packing.dtos, packing.proj);
    }
}
 
#ifdef AMREX_USE_MPI
template <BaseFabType FAB, typename DTOS, typename FabProj>
void
UnpackRecvBuffers (const ApplyDtosAndProjectionOnReciever<DTOS, FabProj>& packing,
                   FabArray<FAB>& dest, const CommData& recv)
{
    // If FAB is not FArrayBox we have not checked for the correct types yet.
    static_assert(IsFabProjection<FabProj, FAB>(), "FabProj needs to be a projection for given FAB type.");
#ifdef AMREX_USE_GPU
    if (Gpu::inLaunchRegion()) {
        unpack_recv_buffer_gpu(dest, packing.dest_component, packing.n_components, recv.data,
                               recv.size, recv.cctc, packing.dtos, packing.proj);
    } else
#endif // AMREX_USE_GPU
    {
        unpack_recv_buffer_cpu(dest, packing.dest_component, packing.n_components, recv.data,
                               recv.size, recv.cctc, packing.dtos, packing.proj);
    }
}
#endif // AMREX_USE_MPI
 
static_assert(IsDataPacking<ApplyDtosAndProjectionOnReciever<>, FArrayBox>(), // NOLINT(bugprone-throw-keyword-missing)
              "ApplyDtosAndProjectionOnReciever<> is expected to satisfy the DataPacking concept.");

//                                                             [ParallelCopy_nowait]
 
static constexpr struct NoLocalCopy {} no_local_copy{};
static constexpr struct DoLocalCopy {} do_local_copy{};

template <BaseFabType FAB, typename DataPacking>
requires (IsDataPacking<DataPacking, FAB>::value)
#ifdef AMREX_USE_MPI
[[nodiscard]] CommHandler
ParallelCopy_nowait (NoLocalCopy, FabArray<FAB>& dest, const FabArray<FAB>& src,
                     const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing)
{
    CommHandler handler{};
    if (ParallelContext::NProcsSub() == 1) {
        return handler;
    }
    //
    // Do this before prematurely exiting if running in parallel.
    // Otherwise sequence numbers will not match across MPI processes.
    //
    handler.mpi_tag = ParallelDescriptor::SeqNum();
 
    if (cmd.m_RcvTags && !(cmd.m_RcvTags->empty())) {
        PrepareRecvBuffers(data_packing, dest, src, handler.recv, *cmd.m_RcvTags);
        PostRecvs(handler.recv, handler.mpi_tag);
    }
 
    if (cmd.m_SndTags && !(cmd.m_SndTags->empty())) {
        PrepareSendBuffers(data_packing, dest, src, handler.send, *cmd.m_SndTags);
        PackSendBuffers(data_packing, src, handler.send);
        PostSends(handler.send, handler.mpi_tag);
    }
    return handler;
}
#else
CommHandler ParallelCopy_nowait (NoLocalCopy, FabArray<FAB>&, const FabArray<FAB>&,
                                 const FabArrayBase::CommMetaData&, const DataPacking&) {
    return CommHandler{};
}
#endif

template <BaseFabType FAB, typename DataPacking>
requires (IsDataPacking<DataPacking, FAB>::value)
#ifdef AMREX_USE_MPI
[[nodiscard]]
#endif
CommHandler
ParallelCopy_nowait (FabArray<FAB>& dest, const FabArray<FAB>& src,
                     const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing)
{
    CommHandler comm = ParallelCopy_nowait(no_local_copy, dest, src, cmd, data_packing);
    // Eagerly do the local work and hope for some overlap with communication
    if (cmd.m_LocTags && !cmd.m_LocTags->empty()) {
        LocalCopy(data_packing, dest, src, *cmd.m_LocTags);
    }
    return comm;
}

//                                                             [ParallelCopy_finish]

template <BaseFabType FAB, typename DataPacking>
requires (IsDataPacking<DataPacking, FAB>::value)
void
#ifdef AMREX_USE_MPI
ParallelCopy_finish (FabArray<FAB>& dest, CommHandler handler,
                     const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing)
{
    // If any FabArray is empty we have nothing to do.
    if (dest.empty()) {
        return;
    }
    // Return if nothing do
    if (ParallelContext::NProcsSub() == 1) {
        return;
    }
    // Unpack receives
    if (cmd.m_RcvTags && !(cmd.m_RcvTags->empty())) {
        ParallelDescriptor::Waitall(handler.recv.request, handler.recv.stats);
#ifdef AMREX_DEBUG
        if (!CheckRcvStats(handler.recv.stats, handler.recv.size, handler.mpi_tag)) {
            amrex::Abort("NonLocalPC::ParallelCopy_finish failed with wrong message size");
        }
#endif
        UnpackRecvBuffers(data_packing, dest, handler.recv);
    }
 
    // Wait for all sends to be done
    if (cmd.m_SndTags && !(cmd.m_SndTags->empty())) {
        ParallelDescriptor::Waitall(handler.send.request, handler.send.stats);
    }
}
#else
ParallelCopy_finish (FabArray<FAB>&, CommHandler, const FabArrayBase::CommMetaData&, const DataPacking&) {}
#endif

template <BaseFabType FAB, typename DataPacking>
requires (IsDataPacking<DataPacking, FAB>::value)
void
ParallelCopy_finish (DoLocalCopy, FabArray<FAB>& dest, const FabArray<FAB>& src, CommHandler handler,
                     const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing)
{
    // Eagerly do the local work and hope for some overlap with communication
    if (cmd.m_LocTags && !cmd.m_LocTags->empty()) {
        LocalCopy(data_packing, dest, src, *cmd.m_LocTags);
    }
    ParallelCopy_finish(dest, std::move(handler), cmd, data_packing); // NOLINT
}

//                                                          [ParallelCopy_alltoallv]

template <BaseFabType FAB, typename DataPacking>
requires (IsDataPacking<DataPacking, FAB>::value)
void
ParallelCopy_alltoallv (FabArray<FAB>& dest, const FabArray<FAB>& src,
                        const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing,
                        CommHandler* cached = nullptr, MPI_Comm comm = MPI_COMM_NULL)
{
    if (dest.empty()) { return; }
 
#ifdef AMREX_USE_MPI
    if (comm == MPI_COMM_NULL) { comm = ParallelContext::CommunicatorSub(); }
    int nprocs_comm;
    MPI_Comm_size(comm, &nprocs_comm);
    if (nprocs_comm > 1)
    {
        CommHandler local_handler{};
        CommHandler& handler = cached ? *cached : local_handler;
 
        static const FabArrayBase::MapOfCopyComTagContainers no_tags{};
        auto const& rcv_tags = cmd.m_RcvTags ? *cmd.m_RcvTags : no_tags;
        auto const& snd_tags = cmd.m_SndTags ? *cmd.m_SndTags : no_tags;
 
        // PrepareCommBuffers sets the id even when there is nothing to communicate.
        if (handler.send.id == std::numeric_limits<std::uint64_t>::max()) {
            PrepareRecvBuffers(data_packing, dest, src, handler.recv, rcv_tags);
            PrepareSendBuffers(data_packing, dest, src, handler.send, snd_tags);
            TranslateRanks(handler.recv, comm);
            TranslateRanks(handler.send, comm);
            PrepareAlltoallvCounts(handler.recv, comm);
            PrepareAlltoallvCounts(handler.send, comm);
        } else {
            AcquireCommBuffer(handler.recv);
            AcquireCommBuffer(handler.send);
        }
 
        if (!snd_tags.empty()) {
            PackSendBuffers(data_packing, src, handler.send);
        }
 
        // The local copy comes first. MPI_Ialltoallv would allow overlapping
        // it, but it was slower than the blocking call on the systems tested.
        if (cmd.m_LocTags && !cmd.m_LocTags->empty()) {
            LocalCopy(data_packing, dest, src, *cmd.m_LocTags);
        }
 
        Alltoallv(handler.send, handler.recv, comm);
 
        if (!rcv_tags.empty()) {
            UnpackRecvBuffers(data_packing, dest, handler.recv);
        }
        if (cached) {
            ReleaseCommBuffer(handler.send);
            ReleaseCommBuffer(handler.recv);
        }
        return;
    }
#else
    amrex::ignore_unused(cached, comm);
#endif
 
    if (cmd.m_LocTags && !cmd.m_LocTags->empty()) {
        LocalCopy(data_packing, dest, src, *cmd.m_LocTags);
    }
}

template <BaseFabType FAB, typename DTOS = Identity, typename Proj = Identity>
requires (IsCallableR<Dim3, DTOS, Dim3>::value && IsFabProjection<Proj, FAB>::value)
void
ParallelCopy_alltoallv (FabArray<FAB>& dest, const FabArray<FAB>& src,
                        const FabArrayBase::CommMetaData& cmd,
                        int srccomp, int destcomp, int numcomp,
                        DTOS const& dtos = DTOS{}, Proj const& proj = Proj{},
                        CommHandler* cached = nullptr, MPI_Comm comm = MPI_COMM_NULL)
{
    PackComponents components{};
    components.dest_component = destcomp;
    components.src_component = srccomp;
    components.n_components = numcomp;
    ApplyDtosAndProjectionOnReciever<DTOS, Proj> packing{components, dtos, proj};
    ParallelCopy_alltoallv(dest, src, cmd, packing, cached, comm);
}

//                                                           [ParallelCopy_alltoall]

template <BaseFabType FAB, typename DataPacking>
requires (IsDataPacking<DataPacking, FAB>::value)
void
ParallelCopy_alltoall (FabArray<FAB>& dest, const FabArray<FAB>& src,
                       const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing,
                       std::size_t block_size, CommHandler* cached = nullptr,
                       MPI_Comm comm = MPI_COMM_NULL)
{
    if (dest.empty()) { return; }
 
#ifdef AMREX_USE_MPI
    if (comm == MPI_COMM_NULL) { comm = ParallelContext::CommunicatorSub(); }
    int nprocs_comm;
    MPI_Comm_size(comm, &nprocs_comm);
    if (nprocs_comm > 1)
    {
        using T = typename FAB::value_type;
 
        CommHandler local_handler{};
        CommHandler& handler = cached ? *cached : local_handler;
 
        static const FabArrayBase::MapOfCopyComTagContainers no_tags{};
        auto const& rcv_tags = cmd.m_RcvTags ? *cmd.m_RcvTags : no_tags;
        auto const& snd_tags = cmd.m_SndTags ? *cmd.m_SndTags : no_tags;
 
        if (handler.send.id == std::numeric_limits<std::uint64_t>::max()) {
            PrepareCommBuffersUniform(handler.recv, rcv_tags, data_packing.n_components,
                                      sizeof(T), block_size, comm);
            PrepareCommBuffersUniform(handler.send, snd_tags, data_packing.n_components,
                                      sizeof(T), block_size, comm);
        } else {
            AcquireCommBuffer(handler.recv);
            AcquireCommBuffer(handler.send);
        }
 
        if (!snd_tags.empty()) {
            PackSendBuffers(data_packing, src, handler.send);
        }
 
        if (cmd.m_LocTags && !cmd.m_LocTags->empty()) {
            LocalCopy(data_packing, dest, src, *cmd.m_LocTags);
        }
 
        Alltoall(handler.send, handler.recv, block_size, comm);
 
        if (!rcv_tags.empty()) {
            UnpackRecvBuffers(data_packing, dest, handler.recv);
        }
        if (cached) {
            ReleaseCommBuffer(handler.send);
            ReleaseCommBuffer(handler.recv);
        }
        return;
    }
#else
    amrex::ignore_unused(block_size, cached, comm);
#endif
 
    if (cmd.m_LocTags && !cmd.m_LocTags->empty()) {
        LocalCopy(data_packing, dest, src, *cmd.m_LocTags);
    }
}

template <BaseFabType FAB, typename DTOS = Identity, typename Proj = Identity>
requires (IsCallableR<Dim3, DTOS, Dim3>::value && IsFabProjection<Proj, FAB>::value)
void
ParallelCopy_alltoall (FabArray<FAB>& dest, const FabArray<FAB>& src,
                       const FabArrayBase::CommMetaData& cmd,
                       int srccomp, int destcomp, int numcomp,
                       DTOS const& dtos, Proj const& proj,
                       std::size_t block_size, CommHandler* cached = nullptr,
                       MPI_Comm comm = MPI_COMM_NULL)
{
    PackComponents components{};
    components.dest_component = destcomp;
    components.src_component = srccomp;
    components.n_components = numcomp;
    ApplyDtosAndProjectionOnReciever<DTOS, Proj> packing{components, dtos, proj};
    ParallelCopy_alltoall(dest, src, cmd, packing, block_size, cached, comm);
}

template <BaseFabType FAB, typename DTOS = Identity, typename Proj = Identity>
requires (IsCallableR<Dim3, DTOS, Dim3>::value && IsFabProjection<Proj, FAB>::value)
void
ParallelCopy (FabArray<FAB>& dest, const FabArray<FAB>& src, const FabArrayBase::CommMetaData& cmd,
              SrcComp srccomp, DestComp destcomp, NumComps numcomp, DTOS const& dtos = DTOS{}, Proj const& proj = Proj{})
{
    PackComponents components{};
    components.dest_component = destcomp.i;
    components.src_component = srccomp.i;
    components.n_components = numcomp.n;
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
    ApplyDtosAndProjectionOnReciever<DTOS, Proj> packing{components, dtos, proj};
    CommHandler handler = ParallelCopy_nowait(dest, src, cmd, packing);
    ParallelCopy_finish(dest, std::move(handler), cmd, packing); // NOLINT
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic pop
#endif
}

template <BaseFabType FAB, typename DTOS = Identity, typename Proj = Identity>
requires (IsCallableR<Dim3, DTOS, Dim3>::value && IsFabProjection<Proj, FAB>::value)
void
ParallelCopy (FabArray<FAB>& dest, const FabArray<FAB>& src, const FabArrayBase::CommMetaData& cmd,
              int srccomp, int destcomp, int numcomp, DTOS const& dtos = DTOS{}, Proj const& proj = Proj{})
{
    ParallelCopy(dest, src, cmd, SrcComp(srccomp), DestComp(destcomp), NumComps(numcomp), dtos, proj);
}

template <BaseFabType FAB, typename DTOS = Identity, typename Proj = Identity>
requires (IsIndexMapping<DTOS>::value && IsFabProjection<Proj, FAB>::value)
MultiBlockCommMetaData
ParallelCopy (FabArray<FAB>& dest, const Box& destbox, const FabArray<FAB>& src, SrcComp srccomp,
              DestComp destcomp, NumComps numcomp, const IntVect& ngrow, DTOS const& dtos = DTOS{}, Proj const& proj = Proj{})
{
    MultiBlockCommMetaData cmd(dest, destbox, src, ngrow, dtos);
    ParallelCopy(dest, src, cmd, srccomp, destcomp, numcomp, dtos, proj);
    return cmd;
}

template <BaseFabType FAB, typename DTOS = Identity, typename Proj = Identity>
requires (IsIndexMapping<DTOS>::value && IsFabProjection<Proj, FAB>::value)
MultiBlockCommMetaData
ParallelCopy (FabArray<FAB>& dest, const Box& destbox, const FabArray<FAB>& src, int srccomp,
              int destcomp, int numcomp, const IntVect& ngrow, DTOS const& dtos = DTOS{}, Proj const& proj = Proj{})
{
    return ParallelCopy(dest, destbox, src, SrcComp(srccomp), DestComp(destcomp), NumComps(numcomp), ngrow, dtos, proj);
}
 
// Rotate90 fills the lo-x and lo-y boundary regions by rotating the data
// around (x=0,y=0) by 90 degrees in either direction.  It also fills the
// corner of lo-x and lo-y boundary region by rotating the data by 180
// degrees.
 
template <BaseFabType FAB>
void
Rotate90 (FabArray<FAB>& mf, int scomp, int ncomp, IntVect const& nghost, Box const& domain);
 
template <BaseFabType FAB>
void
Rotate90 (FabArray<FAB>& mf, Box const& domain);
 
// Rotate180 fills the lo-x boundary by rotating the data around
// (x=0,y=L_y/2) by 180 degrees.
 
template <BaseFabType FAB>
void
Rotate180 (FabArray<FAB>& mf, int scomp, int ncomp, IntVect const& nghost, Box const& domain);
 
template <BaseFabType FAB>
void
Rotate180 (FabArray<FAB>& mf, Box const& domain);
 
// Fill the polar boundaries of the spherical coordinates (theta, phi, r).
// The lo-x boundary is filled with f(-x,y) = f(x,mod(y+pi,2*pi)), and
// the hi-x boundary is filled with f(pi+x,y) = f(pi-x,mod(y+pi,2*pi)).
 
template <BaseFabType FAB>
void
FillPolar (FabArray<FAB>& mf, int scomp, int ncomp, IntVect const& nghost, Box const& domain);
 
template <BaseFabType FAB>
void
FillPolar (FabArray<FAB>& mf, Box const& domain);

template <BaseFabType FAB, typename DTOS, typename Proj = Identity>
requires (IsCallableR<Dim3,DTOS,Dim3>::value && IsFabProjection<Proj,FAB>::value)
[[nodiscard]]
CommHandler
FillBoundary_nowait (FabArray<FAB>& mf, const FabArrayBase::CommMetaData& cmd,
                     int scomp, int ncomp, DTOS const& dtos,
                     Proj const& proj = Proj{});

template <BaseFabType FAB, typename DTOS, typename Proj = Identity>
requires (IsCallableR<Dim3,DTOS,Dim3>::value && IsFabProjection<Proj,FAB>::value)
void
FillBoundary_finish (CommHandler handler,
                     FabArray<FAB>& mf, const FabArrayBase::CommMetaData& cmd,
                     int scomp, int ncomp, DTOS const& dtos,
                     Proj const& proj = Proj{});

template <BaseFabType FAB, typename DTOS, typename Proj = Identity>
requires (IsCallableR<Dim3,DTOS,Dim3>::value && IsFabProjection<Proj,FAB>::value)
void
FillBoundary (FabArray<FAB>& mf, const FabArrayBase::CommMetaData& cmd,
              int scomp, int ncomp, DTOS const& dtos, Proj const& proj = Proj{})
{
    BL_PROFILE("FillBoundary(cmd)");
    auto handler = FillBoundary_nowait(mf, cmd, scomp, ncomp, dtos, proj);
    FillBoundary_finish(std::move(handler), mf, cmd, scomp, ncomp, dtos, proj);
}

template <BaseFabType FAB, typename DTOS>
requires (IsCallableR<Dim3,DTOS,Dim3>::value)
[[nodiscard]]
FabArrayBase::CommMetaData
makeFillBoundaryMetaData (FabArray<FAB>& mf, IntVect const& nghost,
                          Geometry const& geom, DTOS const& dtos);
 
}
 
#include <AMReX_NonLocalBCImpl.H>
 
namespace amrex {
    using NonLocalBC::FillBoundary;
    using NonLocalBC::FillBoundary_nowait;
    using NonLocalBC::FillBoundary_finish;
    using NonLocalBC::makeFillBoundaryMetaData;
    using NonLocalBC::SphThetaPhiRIndexMapping;
    using NonLocalBC::SphThetaPhiRComponentMapping;
    using NonLocalBC::ParallelCopy;
    using NonLocalBC::ParallelCopy_nowait;
    using NonLocalBC::ParallelCopy_finish;
    using NonLocalBC::ParallelCopy_alltoallv;
    using NonLocalBC::ParallelCopy_alltoall;
    using NonLocalBC::MultiBlockIndexMapping;
    using NonLocalBC::MultiBlockCommMetaData;
    using NonLocalBC::CommHandler;
}
 
#endif