AMReX_NonLocalBC.H

Go to the documentation of this file.

#ifndef AMREX_NONLOCAL_BC_H_
#define AMREX_NONLOCAL_BC_H_
#include <AMReX_Config.H>
#include <AMReX_TypeTraits.H>
#include <AMReX_FabArray.H>
#include <AMReX_FArrayBox.H>
 
namespace amrex::NonLocalBC {
 
//                                                            [concept.IndexMapping]
//
 
template <typename T, typename... Args>
using Inverse_t = decltype(std::declval<T>().Inverse(std::declval<Args>()...));
 
template <typename T>
struct HasInverseMemFn : IsDetectedExact<Dim3, Inverse_t, T, Dim3> {};
 
template <class IndexMap>
struct IsIndexMapping
    : Conjunction<IsCallableR<Dim3, IndexMap&, const Dim3&>,
                  HasInverseMemFn<const IndexMap&>> {};
 
//                                             [IndexMapping.MultiBlockIndexMapping]
//
 
struct MultiBlockIndexMapping {
    IntVect permutation{AMREX_D_DECL(0, 1, 2)};
    IntVect offset{AMREX_D_DECL(0, 0, 0)};
    IntVect sign{AMREX_D_DECL(1, 1, 1)};
 
    [[nodiscard]] AMREX_GPU_HOST_DEVICE Dim3 operator()(Dim3 i) const noexcept {
        int iv[3]{i.x, i.y, i.z};
        int iv_new[3]{};
        for (int d = 0; d < AMREX_SPACEDIM; ++d) {
            iv_new[d] = sign[d] * (iv[permutation[d]] - offset[d]);
        }
        return {iv_new[0], iv_new[1], iv_new[2]};
    }
 
    [[nodiscard]] AMREX_GPU_HOST_DEVICE Dim3 Inverse(Dim3 i) const noexcept {
        int iv_new[3]{i.x, i.y, i.z};
        int iv[3]{};
        for (int d = 0; d < AMREX_SPACEDIM; ++d) {
            AMREX_ASSERT(sign[d] == 1 || sign[d] == -1);
            iv[permutation[d]] = iv_new[d] * sign[d] + offset[d];
        }
        return {iv[0], iv[1], iv[2]};
    }
 
    [[nodiscard]] IndexType operator()(IndexType it) const noexcept {
        return IndexType{IntVect{AMREX_D_DECL(it[permutation[0]], it[permutation[1]], it[permutation[2]])}};
    }
 
    [[nodiscard]] IndexType Inverse(IndexType it) const noexcept {
        IntVect inverse_permutation;
        for (int i = 0; i < AMREX_SPACEDIM; ++i) {
            inverse_permutation[permutation[i]] = i;
        }
        return IndexType{IntVect{AMREX_D_DECL(it[inverse_permutation[0]], it[inverse_permutation[1]], it[inverse_permutation[2]])}};
    }
};
 
template <typename DTOS>
std::enable_if_t<IsCallableR<Dim3, DTOS, Dim3>::value, IntVect>
Apply(DTOS const& dtos, const IntVect& iv)
{
    Dim3 i = dtos(iv.dim3());
    return IntVect{AMREX_D_DECL(i.x, i.y, i.z)};
}
 
template <typename DTOS>
std::enable_if_t<IsCallableR<Dim3, DTOS, Dim3>::value && !IsCallableR<IndexType, DTOS, IndexType>::value, Box>
Image (DTOS const& dtos, const Box& box)
{
    IntVect mapped_smallEnd = Apply(dtos, box.smallEnd());
    IntVect mapped_bigEnd = Apply(dtos, box.bigEnd());
    IntVect smallEnd;
    IntVect bigEnd;
    for (int d = 0; d < AMREX_SPACEDIM; ++d) {
        smallEnd[d] = std::min(mapped_smallEnd[d], mapped_bigEnd[d]);
        bigEnd[d] = std::max(mapped_smallEnd[d], mapped_bigEnd[d]);
    }
    return Box{smallEnd, bigEnd, box.ixType()};
}
 
template <typename DTOS>
std::enable_if_t<IsCallableR<Dim3, DTOS, Dim3>::value && IsCallableR<IndexType, DTOS, IndexType>::value, Box>
Image (DTOS const& dtos, const Box& box)
{
    // "Forget" the index type mapping and invoke Image without changing the index type.
    Box srcbox = Image([&dtos](Dim3 d) { return dtos(d); }, box);
    // Fix the index type of the resulting box
    srcbox.setType(dtos(box.ixType()));
    return srcbox;
}
 
template <typename DTOS>
std::enable_if_t<HasInverseMemFn<DTOS>::value, IntVect>
ApplyInverse(DTOS const& dtos, const IntVect& iv)
{
    return Apply([&dtos](Dim3 i) { return dtos.Inverse(i); }, iv);
}
 
template <typename DTOS>
std::enable_if_t<HasInverseMemFn<DTOS>::value && !IsCallableR<IndexType, DTOS, IndexType>::value, Box>
InverseImage (DTOS const& dtos, const Box& box)
{
    return Image([&dtos](Dim3 i) { return dtos.Inverse(i); }, box);
}
 
template <typename DTOS>
std::enable_if_t<HasInverseMemFn<DTOS>::value && IsCallableR<IndexType, DTOS, IndexType>::value, Box>
InverseImage (DTOS const& dtos, const Box& box)
{
    return Image([&dtos](auto&& i) { return dtos.Inverse(i); }, box);
}
 
static_assert(IsIndexMapping<MultiBlockIndexMapping>(), // NOLINT(bugprone-throw-keyword-missing)
              "MultiBlockIndexMapping is expected to satisfy IndexMapping");
 
//                                                    [class.MultiBlockCommMetaData]
//
 
struct MultiBlockCommMetaData : FabArrayBase::CommMetaData {
 
    template <typename DTOS,
              typename = std::enable_if_t<IsIndexMapping<DTOS>::value>>
    MultiBlockCommMetaData(const FabArrayBase& dst, const Box& dstbox, const FabArrayBase& src,
                           const IntVect& ngrow, DTOS const& dtos);
 
    template <typename DTOS,
              typename = std::enable_if_t<IsIndexMapping<DTOS>::value>>
    MultiBlockCommMetaData(const BoxArray& dstba, const DistributionMapping& dstdm,
                           const Box& dstbox, const BoxArray& srcba,
                           const DistributionMapping& srcdm, const IntVect& ngrow, DTOS const& dtos);
 
 
    template <typename DTOS>
    std::enable_if_t<IsIndexMapping<DTOS>::value>
    define(const BoxArray& dstba, const DistributionMapping& dstdm, const Box& dstbox,
           const BoxArray& srcba, const DistributionMapping& srcdm, const IntVect& ngrow,
           DTOS const& dtos);
};
 
//                                                           [concept.FabProjection]
//
 
template <typename P, typename FAB>
struct IsFabProjection
    : IsCallableR<typename FAB::value_type, P, Array4<const typename FAB::value_type>, Dim3, int>
{};
 
//                                                          [FabProjection.Identity]
//                                                           [IndexMapping.Identity]
 
struct Identity  {
    constexpr Dim3 operator()(Dim3 i) const noexcept { return i; }
    [[nodiscard]] static constexpr Dim3 Inverse(Dim3 i) noexcept { return i; }
 
    template <typename T>
    constexpr T operator()(Array4<const T> array, Dim3 i, int comp = 0) const
        noexcept(noexcept(array(i.x, i.y, i.z, comp))) {
        return array(i.x, i.y, i.z, comp);
    }
 
    constexpr int operator()(int i) const noexcept { return i; }
};
static constexpr Identity identity{};
 
static_assert(sizeof(Identity) == 1 );
static_assert(std::is_trivially_default_constructible_v<Identity> );
static_assert(std::is_trivially_copy_assignable_v<Identity> );
static_assert(std::is_trivially_copy_constructible_v<Identity> );
static_assert(IsIndexMapping<Identity>() ); // NOLINT(bugprone-throw-keyword-missing)
static_assert(IsFabProjection<Identity, FArrayBox>() ); // NOLINT(bugprone-throw-keyword-missing)
 
//                                                     [FabProjection.MapComponents]
 
template <typename Base, typename Map = Identity> struct MapComponents {
    static_assert(IsCallable<Base, Array4<const Real>, Dim3, int>::value,
                  "Base needs to be a callable function: (Array4<const T>, Dim3, i) -> auto.");
 
    static_assert(IsCallableR<int, Map, int>::value,
                  "Map needs to be a callable function: int -> int.");
 
    Base base;
    Map map;
 
    template <typename T,
              typename = std::enable_if_t<IsCallable<Base, Array4<const T>, Dim3, int>::value>,
              typename = std::enable_if_t<IsCallableR<int, Map, int>::value>>
    constexpr decltype(auto) operator()(Array4<const T> array, Dim3 i, int comp) const
        noexcept(noexcept(base(array, i, map(comp)))) {
        return base(array, i, map(comp));
    }
};
 
static_assert(std::is_trivially_copy_assignable<MapComponents<Identity>>() ); // NOLINT(bugprone-throw-keyword-missing)
static_assert(std::is_trivially_copy_constructible<MapComponents<Identity>>() ); // NOLINT(bugprone-throw-keyword-missing)
static_assert(IsFabProjection<MapComponents<Identity>, FArrayBox>() ); // NOLINT(bugprone-throw-keyword-missing)
 
//                                      [FabProjection.MapComponents.SwapComponents]
 
template <int I, int J> struct SwapComponents {
    static_assert(I >= 0 && J >= 0, "I >= 0 && J >= 0");
 
    constexpr int operator()(int i) const noexcept {
        const int map[2] = {I, J};
        return i == I || i == J ? map[std::size_t(i == I)] : i;
    }
};
 
template <int I> struct SwapComponents<I, -1> {
    static_assert(I >= 0, "I >= 0");
 
    int J;
    constexpr int operator()(int i) const noexcept {
        const int map[2] = {I, J};
        return i == I || i == J ? map[std::size_t(i == I)] : i;
    }
};
 
template <int J> struct SwapComponents<-1, J> {
    static_assert(J >= 0, "J >= 0");
 
    int I;
    constexpr int operator()(int i) const noexcept {
        const int map[2] = {I, J};
        return i == I || i == J ? map[std::size_t(i == I)] : i;
    }
};
 
template <> struct SwapComponents<-1, -1> {
    int I;
    int J;
    constexpr int operator()(int i) const noexcept {
        const int map[2] = {I, J};
        return i == I || i == J ? map[std::size_t(i == I)] : i;
    }
};
 
using DynamicSwapComponents = SwapComponents<-1, -1>;
 
template <int I, int J> static constexpr SwapComponents<I, J> swap_indices{};
 
static_assert(sizeof(SwapComponents<0, 1>) == 1 );
static_assert(sizeof(DynamicSwapComponents) == 2 * sizeof(int) );
static_assert(sizeof(SwapComponents<0, -1>) == sizeof(int) );
static_assert(sizeof(SwapComponents<-1, 1>) == sizeof(int) );
static_assert(std::is_trivially_default_constructible<MapComponents<Identity, SwapComponents<0, 1>>>() ); // NOLINT(bugprone-throw-keyword-missing)
static_assert(std::is_trivially_copy_assignable<MapComponents<Identity, SwapComponents<0, 1>>>() ); // NOLINT(bugprone-throw-keyword-missing)
static_assert(std::is_trivially_copy_constructible<MapComponents<Identity, SwapComponents<0, 1>>>() ); // NOLINT(bugprone-throw-keyword-missing)
static_assert(IsFabProjection<MapComponents<Identity, SwapComponents<0, 1>>, FArrayBox>() ); // NOLINT(bugprone-throw-keyword-missing)
 
static_assert(swap_indices<0, 1>(0) == 1 );
static_assert(swap_indices<0, 1>(1) == 0 );
static_assert(swap_indices<0, 1>(2) == 2 );
static_assert(DynamicSwapComponents{0, 1}(0) == 1 );
static_assert(DynamicSwapComponents{0, 1}(1) == 0 );
static_assert(DynamicSwapComponents{0, 1}(2) == 2 );
 
//                                                                  [class.CommData]
//                                                               [class.CommHandler]
 
struct CommData {
#ifdef AMREX_USE_MPI
    TheFaArenaPointer the_data = nullptr;
    Vector<int> rank{};
    Vector<char*> data{};
    Vector<std::size_t> offset{};
    Vector<std::size_t> size{};
    Vector<MPI_Request> request{};
    Vector<MPI_Status> stats{};
    Vector<const FabArrayBase::CopyComTagsContainer*> cctc{};
    std::uint64_t id = std::numeric_limits<std::uint64_t>::max();
#endif
};
 
#ifdef AMREX_USE_MPI
void PrepareCommBuffers(CommData& comm, const FabArrayBase::MapOfCopyComTagContainers& cctc,
                        int n_components, std::size_t object_size, std::size_t align);
 
void PostRecvs(CommData& recv, int mpi_tag);
 
void PostSends(CommData& send, int mpi_tag);
#endif
 
 
struct CommHandler {
#ifdef AMREX_USE_MPI
    int mpi_tag{};
    CommData recv{};
    CommData send{};
#endif
};
 
//                                                             [concept.DataPacking]
//
template <typename... Args>
using PrepareSendBuffers_t = decltype(PrepareSendBuffers(std::declval<Args>()...));
 
template <typename... Args>
using PrepareRecvBuffers_t = decltype(PrepareRecvBuffers(std::declval<Args>()...));
 
template <typename... Args>
using PackSendBuffers_t = decltype(PackSendBuffers(std::declval<Args>()...));
 
template <typename... Args>
using UnpackRecvBuffers_t = decltype(UnpackRecvBuffers(std::declval<Args>()...));
 
template <typename... Args>
using LocalCopy_t = decltype(LocalCopy(std::declval<Args>()...));
 
#if defined(AMREX_USE_CUDA) && defined(_WIN32)
template <typename DP, typename FAB> struct IsDataPacking : std::true_type {};
#else
template <typename DP, typename FAB>
struct IsDataPacking :
    Conjunction<
        IsDetected<LocalCopy_t, DP&, FabArray<FAB>&, const FabArray<FAB>&, const FabArrayBase::CopyComTagsContainer&>
#ifdef AMREX_USE_MPI
       ,IsDetected<PrepareSendBuffers_t, DP&, FabArray<FAB>&, const FabArray<FAB>&, CommData&, const FabArrayBase::MapOfCopyComTagContainers&>,
        IsDetected<PrepareRecvBuffers_t, DP&, FabArray<FAB>&, const FabArray<FAB>&, CommData&, const FabArrayBase::MapOfCopyComTagContainers&>,
        IsDetected<PackSendBuffers_t, DP&, const FabArray<FAB>&, CommData&>,
        IsDetected<UnpackRecvBuffers_t, DP&, FabArray<FAB>&, CommData&>
#endif
    > {};
#endif
 
template <class FAB, class DTOS = Identity, class Proj = Identity>
std::enable_if_t<IsBaseFab<FAB>() && IsCallableR<Dim3, DTOS, Dim3>() && IsFabProjection<Proj, FAB>()>
local_copy_cpu (FabArray<FAB>& dest, const FabArray<FAB>& src, int dcomp, int scomp, int ncomp,
                FabArrayBase::CopyComTagsContainer const& local_tags, DTOS const& dtos = DTOS{},
                Proj const& proj = Proj{}) noexcept;
 
template <class FAB, class DTOS = Identity, class Proj = Identity>
std::enable_if_t<IsBaseFab<FAB>() && IsCallableR<Dim3, DTOS, Dim3>() && IsFabProjection<Proj, FAB>()>
unpack_recv_buffer_cpu (FabArray<FAB>& mf, int dcomp, int ncomp, Vector<char*> const& recv_data,
                        Vector<std::size_t> const& recv_size,
                        Vector<FabArrayBase::CopyComTagsContainer const*> const& recv_cctc,
                        DTOS const& dtos = DTOS{}, Proj const& proj = Proj{}) noexcept;
 
#ifdef AMREX_USE_GPU
template <class FAB, class DTOS = Identity, class Proj = Identity>
std::enable_if_t<IsBaseFab<FAB>() && IsCallableR<Dim3, DTOS, Dim3>() && IsFabProjection<Proj, FAB>()>
local_copy_gpu (FabArray<FAB>& dest, const FabArray<FAB>& src, int dcomp, int scomp, int ncomp,
                FabArrayBase::CopyComTagsContainer const& local_tags, DTOS const& dtos = DTOS{},
                Proj const& proj = Proj{}) noexcept;
 
template <class FAB, class DTOS = Identity, class Proj = Identity>
std::enable_if_t<IsBaseFab<FAB>() && IsCallableR<Dim3, DTOS, Dim3>() && IsFabProjection<Proj, FAB>()>
unpack_recv_buffer_gpu (FabArray<FAB>& mf, int scomp, int ncomp,
                        Vector<char*> const& recv_data,
                        Vector<std::size_t> const& recv_size,
                        Vector<FabArrayBase::CopyComTagsContainer const*> const& recv_cctc,
                        DTOS const& dtos = DTOS{}, Proj const& proj = Proj{});
#endif
 
//                                                      [DataPacking.PackComponents]
//
// PackComponents is the simplest data packing policy.
// This provides us with sane default behaviour that we can use when defining new
// data packing policies.
 
struct PackComponents {
    int dest_component{0};
    int src_component{0};
    int n_components{1};
};
 
template <typename FAB>
std::enable_if_t<IsBaseFab<FAB>::value>
LocalCopy (const PackComponents& components, FabArray<FAB>& dest, const FabArray<FAB>& src,
           const FabArrayBase::CopyComTagsContainer& local_tags) {
#ifdef AMREX_USE_GPU
    if (Gpu::inLaunchRegion()) {
        local_copy_gpu(dest, src, components.dest_component, components.src_component,
                       components.n_components, local_tags);
    } else
#endif
    {
        local_copy_cpu(dest, src, components.dest_component, components.src_component,
                       components.n_components, local_tags);
    }
}
 
#ifdef AMREX_USE_MPI
template <typename FAB>
std::enable_if_t<IsBaseFab<FAB>::value>
PrepareSendBuffers (const PackComponents& components, FabArray<FAB>& dest, const FabArray<FAB>& src,
                    CommData& comm, const FabArrayBase::MapOfCopyComTagContainers& cctc) {
    using T = typename FAB::value_type;
    ignore_unused(dest, src);
    PrepareCommBuffers(comm, cctc, components.n_components, sizeof(T), alignof(T));
}
 
template <typename FAB>
std::enable_if_t<IsBaseFab<FAB>::value>
PrepareRecvBuffers (const PackComponents& components, FabArray<FAB>& dest, const FabArray<FAB>& src,
                    CommData& comm, const FabArrayBase::MapOfCopyComTagContainers& cctc) {
    using T = typename FAB::value_type;
    ignore_unused(dest, src);
    PrepareCommBuffers(comm, cctc, components.n_components, sizeof(T), alignof(T));
}
 
template <typename FAB>
std::enable_if_t<IsBaseFab<FAB>::value>
PackSendBuffers (const PackComponents& components, const FabArray<FAB>& src, CommData& send) {
#ifdef AMREX_USE_GPU
    if (Gpu::inLaunchRegion()) {
        FabArray<FAB>::pack_send_buffer_gpu(src, components.src_component, components.n_components,
                                            send.data, send.size, send.cctc, send.id);
    } else
#endif // AMREX_USE_GPU
    {
        FabArray<FAB>::pack_send_buffer_cpu(src, components.src_component, components.n_components,
                                            send.data, send.size, send.cctc);
    }
}
 
template <typename FAB>
std::enable_if_t<IsBaseFab<FAB>::value>
UnpackRecvBuffers (const PackComponents& components, FabArray<FAB>& dest, const CommData& recv) {
#ifdef AMREX_USE_GPU
    if (Gpu::inLaunchRegion()) {
        unpack_recv_buffer_gpu(dest, components.dest_component, components.n_components, recv.data,
                               recv.size, recv.cctc);
    } else
#endif // AMREX_USE_GPU
    {
        unpack_recv_buffer_cpu(dest, components.dest_component, components.n_components, recv.data,
                               recv.size, recv.cctc);
    }
}
#endif // AMREX_USE_MPI
 
static_assert(IsDataPacking<PackComponents, FArrayBox>(), // NOLINT(bugprone-throw-keyword-missing)
              "PackComponents is expected to satisfy the concept DataPacking.");
 
//                                    [DataPacking.ApplyDtosAndProjectionOnReciever]
//
template <typename DTOS = Identity, typename FabProj = Identity>
struct ApplyDtosAndProjectionOnReciever : PackComponents {
    constexpr ApplyDtosAndProjectionOnReciever() = default;
    constexpr ApplyDtosAndProjectionOnReciever(const PackComponents& components, DTOS dtos_ = DTOS{}, FabProj proj_ = FabProj{})
        : PackComponents(components), dtos(std::move(dtos_)), proj(std::move(proj_)) {}
 
    DTOS dtos;
    FabProj proj;
 
    static_assert(IsCallableR<Dim3, DTOS, Dim3>(), "DTOS needs to be a callable: Dim3 -> Dim3");
    static_assert(IsFabProjection<FabProj, FArrayBox>(), "FabProj needs to be at least a projection on FArrayBox.");
};
 
template <typename FAB, typename DTOS, typename FabProj>
std::enable_if_t<IsBaseFab<FAB>::value>
LocalCopy (const ApplyDtosAndProjectionOnReciever<DTOS, FabProj>& packing, FabArray<FAB>& dest,
           const FabArray<FAB>& src, const FabArrayBase::CopyComTagsContainer& local_tags) {
    static_assert(IsFabProjection<FabProj, FAB>(), "FabProj needs to be a projection for given FAB type.");
#ifdef AMREX_USE_GPU
    if (Gpu::inLaunchRegion()) {
        local_copy_gpu(dest, src, packing.dest_component, packing.src_component,
                       packing.n_components, local_tags, packing.dtos, packing.proj);
    } else
#endif
    {
        local_copy_cpu(dest, src, packing.dest_component, packing.src_component,
                       packing.n_components, local_tags, packing.dtos, packing.proj);
    }
}
 
#ifdef AMREX_USE_MPI
template <typename FAB, typename DTOS, typename FabProj>
std::enable_if_t<IsBaseFab<FAB>::value>
UnpackRecvBuffers (const ApplyDtosAndProjectionOnReciever<DTOS, FabProj>& packing,
                   FabArray<FAB>& dest, const CommData& recv) {
    // If FAB is not FArrayBox we have not checked for the correct types yet.
    static_assert(IsFabProjection<FabProj, FAB>(), "FabProj needs to be a projection for given FAB type.");
#ifdef AMREX_USE_GPU
    if (Gpu::inLaunchRegion()) {
        unpack_recv_buffer_gpu(dest, packing.dest_component, packing.n_components, recv.data,
                               recv.size, recv.cctc, packing.dtos, packing.proj);
    } else
#endif // AMREX_USE_GPU
    {
        unpack_recv_buffer_cpu(dest, packing.dest_component, packing.n_components, recv.data,
                               recv.size, recv.cctc, packing.dtos, packing.proj);
    }
}
#endif // AMREX_USE_MPI
 
static_assert(IsDataPacking<ApplyDtosAndProjectionOnReciever<>, FArrayBox>(), // NOLINT(bugprone-throw-keyword-missing)
              "ApplyDtosAndProjectionOnReciever<> is expected to satisfy the DataPacking concept.");
 
//                                                             [ParallelCopy_nowait]
 
static constexpr struct NoLocalCopy {} no_local_copy{};
static constexpr struct DoLocalCopy {} do_local_copy{};
 
template <typename FAB, typename DataPacking,
          typename = std::enable_if_t<IsBaseFab<FAB>::value>,
          typename = std::enable_if_t<IsDataPacking<DataPacking, FAB>::value>>
#ifdef AMREX_USE_MPI
AMREX_NODISCARD CommHandler
ParallelCopy_nowait (NoLocalCopy, FabArray<FAB>& dest, const FabArray<FAB>& src,
                     const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing) {
    CommHandler handler{};
    if (ParallelContext::NProcsSub() == 1) {
        return handler;
    }
    //
    // Do this before prematurely exiting if running in parallel.
    // Otherwise sequence numbers will not match across MPI processes.
    //
    handler.mpi_tag = ParallelDescriptor::SeqNum();
 
    if (cmd.m_RcvTags && !(cmd.m_RcvTags->empty())) {
        PrepareRecvBuffers(data_packing, dest, src, handler.recv, *cmd.m_RcvTags);
        PostRecvs(handler.recv, handler.mpi_tag);
    }
 
    if (cmd.m_SndTags && !(cmd.m_SndTags->empty())) {
        PrepareSendBuffers(data_packing, dest, src, handler.send, *cmd.m_SndTags);
        PackSendBuffers(data_packing, src, handler.send);
        PostSends(handler.send, handler.mpi_tag);
    }
    return handler;
}
#else
CommHandler ParallelCopy_nowait (NoLocalCopy, FabArray<FAB>&, const FabArray<FAB>&,
                                 const FabArrayBase::CommMetaData&, const DataPacking&) {
    return CommHandler{};
}
#endif
 
template <typename FAB, typename DataPacking,
          typename = std::enable_if_t<IsBaseFab<FAB>::value>,
          typename = std::enable_if_t<IsDataPacking<DataPacking, FAB>::value>>
#ifdef AMREX_USE_MPI
AMREX_NODISCARD
#endif
CommHandler
ParallelCopy_nowait (FabArray<FAB>& dest, const FabArray<FAB>& src,
                     const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing) {
    CommHandler comm = ParallelCopy_nowait(no_local_copy, dest, src, cmd, data_packing);
    // Eagerly do the local work and hope for some overlap with communication
    if (cmd.m_LocTags && !cmd.m_LocTags->empty()) {
        LocalCopy(data_packing, dest, src, *cmd.m_LocTags);
    }
    return comm;
}
//                                                             [ParallelCopy_finish]
 
template <typename FAB, typename DataPacking>
std::enable_if_t<IsBaseFab<FAB>() && IsDataPacking<DataPacking, FAB>()>
#ifdef AMREX_USE_MPI
ParallelCopy_finish (FabArray<FAB>& dest, CommHandler handler,
                     const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing) {
    // If any FabArray is empty we have nothing to do.
    if (dest.empty()) {
        return;
    }
    // Return if nothing do
    if (ParallelContext::NProcsSub() == 1) {
        return;
    }
    // Unpack receives
    if (cmd.m_RcvTags && !(cmd.m_RcvTags->empty())) {
        ParallelDescriptor::Waitall(handler.recv.request, handler.recv.stats);
#ifdef AMREX_DEBUG
        if (!CheckRcvStats(handler.recv.stats, handler.recv.size, handler.mpi_tag)) {
            amrex::Abort("NonLocalPC::ParallelCopy_finish failed with wrong message size");
        }
#endif
        UnpackRecvBuffers(data_packing, dest, handler.recv);
    }
 
    // Wait for all sends to be done
    if (cmd.m_SndTags && !(cmd.m_SndTags->empty())) {
        ParallelDescriptor::Waitall(handler.send.request, handler.send.stats);
    }
}
#else
ParallelCopy_finish (FabArray<FAB>&, CommHandler, const FabArrayBase::CommMetaData&, const DataPacking&) {}
#endif
 
template <typename FAB, typename DataPacking>
std::enable_if_t<IsBaseFab<FAB>() && IsDataPacking<DataPacking, FAB>()>
ParallelCopy_finish (DoLocalCopy, FabArray<FAB>& dest, const FabArray<FAB>& src, CommHandler handler,
                     const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing) {
    // Eagerly do the local work and hope for some overlap with communication
    if (cmd.m_LocTags && !cmd.m_LocTags->empty()) {
        LocalCopy(data_packing, dest, src, *cmd.m_LocTags);
    }
    ParallelCopy_finish(dest, std::move(handler), cmd, data_packing); // NOLINT
}
 
template <typename FAB, typename DTOS = Identity, typename Proj = Identity>
std::enable_if_t<IsBaseFab<FAB>() && IsCallableR<Dim3, DTOS, Dim3>() && IsFabProjection<Proj, FAB>()>
ParallelCopy (FabArray<FAB>& dest, const FabArray<FAB>& src, const FabArrayBase::CommMetaData& cmd,
              SrcComp srccomp, DestComp destcomp, NumComps numcomp, DTOS const& dtos = DTOS{}, Proj const& proj = Proj{}) {
    PackComponents components{};
    components.dest_component = destcomp.i;
    components.src_component = srccomp.i;
    components.n_components = numcomp.n;
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
    ApplyDtosAndProjectionOnReciever<DTOS, Proj> packing{components, dtos, proj};
    CommHandler handler = ParallelCopy_nowait(dest, src, cmd, packing);
    ParallelCopy_finish(dest, std::move(handler), cmd, packing); // NOLINT
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic pop
#endif
}
 
template <typename FAB, typename DTOS = Identity, typename Proj = Identity>
std::enable_if_t<IsBaseFab<FAB>() && IsCallableR<Dim3, DTOS, Dim3>() && IsFabProjection<Proj, FAB>()>
ParallelCopy (FabArray<FAB>& dest, const FabArray<FAB>& src, const FabArrayBase::CommMetaData& cmd,
              int srccomp, int destcomp, int numcomp, DTOS const& dtos = DTOS{}, Proj const& proj = Proj{}) {
    ParallelCopy(dest, src, cmd, SrcComp(srccomp), DestComp(destcomp), NumComps(numcomp), dtos, proj);
}
 
template <typename FAB, typename DTOS = Identity, typename Proj = Identity>
std::enable_if_t<IsBaseFab<FAB>() && IsIndexMapping<DTOS>() && IsFabProjection<Proj, FAB>(),
MultiBlockCommMetaData>
ParallelCopy (FabArray<FAB>& dest, const Box& destbox, const FabArray<FAB>& src, SrcComp srccomp,
              DestComp destcomp, NumComps numcomp, const IntVect& ngrow, DTOS const& dtos = DTOS{}, Proj const& proj = Proj{}) {
    MultiBlockCommMetaData cmd(dest, destbox, src, ngrow, dtos);
    ParallelCopy(dest, src, cmd, srccomp, destcomp, numcomp, dtos, proj);
    return cmd;
}
 
template <typename FAB, typename DTOS = Identity, typename Proj = Identity>
std::enable_if_t<IsBaseFab<FAB>() && IsIndexMapping<DTOS>() && IsFabProjection<Proj, FAB>(),
MultiBlockCommMetaData>
ParallelCopy (FabArray<FAB>& dest, const Box& destbox, const FabArray<FAB>& src, int srccomp,
              int destcomp, int numcomp, const IntVect& ngrow, DTOS const& dtos = DTOS{}, Proj const& proj = Proj{}) {
    return ParallelCopy(dest, destbox, src, SrcComp(srccomp), DestComp(destcomp), NumComps(numcomp), ngrow, dtos, proj);
}
 
// Rotate90 fills the lo-x and lo-y boundary regions by rotating the data
// around (x=0,y=0) by 90 degrees in either direction.  It also fills the
// corner of lo-x and lo-y boundary region by rotating the data by 180
// degrees.
 
template <class FAB>
std::enable_if_t<IsBaseFab<FAB>::value>
Rotate90 (FabArray<FAB>& mf, int scomp, int ncomp, IntVect const& nghost, Box const& domain);
 
template <class FAB>
std::enable_if_t<IsBaseFab<FAB>::value>
Rotate90 (FabArray<FAB>& mf, Box const& domain);
 
// Rotate180 fills the lo-x boundary by rotating the data around
// (x=0,y=L_y/2) by 180 degrees.
 
template <class FAB>
std::enable_if_t<IsBaseFab<FAB>::value>
Rotate180 (FabArray<FAB>& mf, int scomp, int ncomp, IntVect const& nghost, Box const& domain);
 
template <class FAB>
std::enable_if_t<IsBaseFab<FAB>::value>
Rotate180 (FabArray<FAB>& mf, Box const& domain);
 
// Fill the polar boundaries of the spherical coordinates (theta, phi, r).
// The lo-x boundary is filled with f(-x,y) = f(x,mod(y+pi,2*pi)), and
// the hi-x boundary is filled with f(pi+x,y) = f(pi-x,mod(y+pi,2*pi)).
 
template <class FAB>
std::enable_if_t<IsBaseFab<FAB>::value>
FillPolar (FabArray<FAB>& mf, int scomp, int ncomp, IntVect const& nghost, Box const& domain);
 
template <class FAB>
std::enable_if_t<IsBaseFab<FAB>::value>
FillPolar (FabArray<FAB>& mf, Box const& domain);
 
template <typename FAB, typename DTOS, typename Proj = Identity>
[[nodiscard]]
std::enable_if_t<IsBaseFab<FAB>() &&
                 IsCallableR<Dim3,DTOS,Dim3>() &&
                 IsFabProjection<Proj,FAB>(),
                 CommHandler>
FillBoundary_nowait (FabArray<FAB>& mf, const FabArrayBase::CommMetaData& cmd,
                     int scomp, int ncomp, DTOS const& dtos,
                     Proj const& proj = Proj{});
 
template <typename FAB, typename DTOS, typename Proj = Identity>
std::enable_if_t<IsBaseFab<FAB>() &&
                 IsCallableR<Dim3,DTOS,Dim3>() &&
                 IsFabProjection<Proj,FAB>()>
FillBoundary_finish (CommHandler handler,
                     FabArray<FAB>& mf, const FabArrayBase::CommMetaData& cmd,
                     int scomp, int ncomp, DTOS const& dtos,
                     Proj const& proj = Proj{});
 
template <typename FAB, typename DTOS, typename Proj = Identity>
std::enable_if_t<IsBaseFab<FAB>() &&
                 IsCallableR<Dim3,DTOS,Dim3>() &&
                 IsFabProjection<Proj,FAB>()>
FillBoundary (FabArray<FAB>& mf, const FabArrayBase::CommMetaData& cmd,
              int scomp, int ncomp, DTOS const& dtos, Proj const& proj = Proj{})
{
    BL_PROFILE("FillBoundary(cmd)");
    auto handler = FillBoundary_nowait(mf, cmd, scomp, ncomp, dtos, proj);
    FillBoundary_finish(std::move(handler), mf, cmd, scomp, ncomp, dtos, proj);
}
 
template <typename FAB, typename DTOS>
[[nodiscard]]
std::enable_if_t<IsBaseFab<FAB>() && IsCallableR<Dim3,DTOS,Dim3>(),
                 FabArrayBase::CommMetaData>
makeFillBoundaryMetaData (FabArray<FAB>& mf, IntVect const& nghost,
                          Geometry const& geom, DTOS const& dtos);
 
}
 
#include <AMReX_NonLocalBCImpl.H>
 
namespace amrex {
    using NonLocalBC::FillBoundary;
    using NonLocalBC::FillBoundary_nowait;
    using NonLocalBC::FillBoundary_finish;
    using NonLocalBC::makeFillBoundaryMetaData;
    using NonLocalBC::SphThetaPhiRIndexMapping;
    using NonLocalBC::SphThetaPhiRComponentMapping;
    using NonLocalBC::ParallelCopy;
    using NonLocalBC::ParallelCopy_nowait;
    using NonLocalBC::ParallelCopy_finish;
    using NonLocalBC::MultiBlockIndexMapping;
    using NonLocalBC::MultiBlockCommMetaData;
    using NonLocalBC::CommHandler;
}
 
#endif