1 #ifndef AMREX_FFT_R2C_H_
2 #define AMREX_FFT_R2C_H_
3 #include <AMReX_Config.H>
14 template <
typename T>
class OpenBCSolver;
36 using MF = std::conditional_t<std::is_same_v<T,Real>,
77 std::enable_if_t<DIR == Direction::both, int> = 0>
120 template <Direction DIR=D, std::enable_if_t<DIR == Direction::both,
int> = 0>
164 template <
typename F>
222 template <
typename T, Direction D, DomainStrategy S>
224 : m_real_domain(domain),
229 #
if (AMREX_SPACEDIM >= 2)
234 #
if (AMREX_SPACEDIM == 3)
245 static_assert(std::is_same_v<float,T> || std::is_same_v<double,T>);
247 #if (AMREX_SPACEDIM == 3)
257 #if (AMREX_SPACEDIM == 3)
266 m_rx.define(bax, dmx, 1, 0,
MFInfo().SetAlloc(
false));
270 for (
auto &
b : bl) {
286 #if (AMREX_SPACEDIM >= 2)
291 if (cbay.size() == dmx.size()) {
300 #if (AMREX_SPACEDIM == 3)
305 {
false,
true,
true},
true);
307 if (cbaz.size() == dmx.size()) {
309 }
else if (cbaz.size() == cdmy.
size()) {
330 #if (AMREX_SPACEDIM >= 2)
333 m_cmd_x2y = std::make_unique<MultiBlockCommMetaData>
335 m_cmd_y2x = std::make_unique<MultiBlockCommMetaData>
339 #if (AMREX_SPACEDIM == 3)
343 m_cmd_x2z = std::make_unique<MultiBlockCommMetaData>
345 m_cmd_z2x = std::make_unique<MultiBlockCommMetaData>
349 m_cmd_y2z = std::make_unique<MultiBlockCommMetaData>
351 m_cmd_z2y = std::make_unique<MultiBlockCommMetaData>
361 if (myproc <
m_rx.size())
363 Box const& box =
m_rx.box(myproc);
364 auto* pr =
m_rx[myproc].dataPtr();
366 #ifdef AMREX_USE_SYCL
379 #if (AMREX_SPACEDIM >= 2)
384 #if (AMREX_SPACEDIM == 3)
396 auto* pr = (
void*)
m_rx[0].dataPtr();
397 auto* pc = (
void*)
m_cx[0].dataPtr();
398 #ifdef AMREX_USE_SYCL
399 m_fft_fwd_x.template init_r2c<Direction::forward>(len, pr, pc,
false);
403 m_fft_fwd_x.template init_r2c<Direction::forward>(len, pr, pc,
false);
406 m_fft_bwd_x.template init_r2c<Direction::backward>(len, pr, pc,
false);
412 template <
typename T, Direction D, DomainStrategy S>
415 if (m_fft_bwd_x.plan != m_fft_fwd_x.plan) {
416 m_fft_bwd_x.destroy();
418 if (m_fft_bwd_y.plan != m_fft_fwd_y.plan) {
419 m_fft_bwd_y.destroy();
421 if (m_fft_bwd_z.plan != m_fft_fwd_z.plan) {
422 m_fft_bwd_z.destroy();
424 m_fft_fwd_x.destroy();
425 m_fft_fwd_y.destroy();
426 m_fft_fwd_z.destroy();
427 if (m_fft_bwd_x_half.plan != m_fft_fwd_x_half.plan) {
428 m_fft_bwd_x_half.destroy();
430 m_fft_fwd_x_half.destroy();
433 template <
typename T, Direction D, DomainStrategy S>
436 #if (AMREX_SPACEDIM == 3)
437 if (m_do_alld_fft) {
return; }
442 Box bottom_half = m_real_domain;
443 bottom_half.
growHi(2,-m_real_domain.length(2)/2);
444 Box box = fab->box() & bottom_half;
446 auto* pr = fab->dataPtr();
449 #ifdef AMREX_USE_SYCL
450 m_fft_fwd_x_half.template init_r2c<Direction::forward>
451 (box, pr, pc, m_slab_decomp);
452 m_fft_bwd_x_half = m_fft_fwd_x_half;
455 m_fft_fwd_x_half.template init_r2c<Direction::forward>
456 (box, pr, pc, m_slab_decomp);
459 m_fft_bwd_x_half.template init_r2c<Direction::backward>
460 (box, pr, pc, m_slab_decomp);
467 if (m_cmd_x2z && ! m_cmd_x2z_half) {
468 Box bottom_half = m_spectral_domain_z;
471 bottom_half.
growHi(0,-m_spectral_domain_z.length(0)/2);
472 m_cmd_x2z_half = std::make_unique<MultiBlockCommMetaData>
473 (m_cz, bottom_half, m_cx,
IntVect(0), m_dtos_x2z);
476 if (m_cmd_z2x && ! m_cmd_z2x_half) {
477 Box bottom_half = m_spectral_domain_x;
478 bottom_half.
growHi(2,-m_spectral_domain_x.length(2)/2);
479 m_cmd_z2x_half = std::make_unique<MultiBlockCommMetaData>
480 (m_cx, bottom_half, m_cz,
IntVect(0), m_dtos_z2x);
485 template <
typename T, Direction D, DomainStrategy S>
492 if (&m_rx != &inmf) {
493 m_rx.ParallelCopy(inmf, 0, 0, 1);
497 m_fft_fwd_x.template compute_r2c<Direction::forward>();
501 auto& fft_x = m_openbc_half ? m_fft_fwd_x_half : m_fft_fwd_x;
502 fft_x.template compute_r2c<Direction::forward>();
505 ParallelCopy(m_cy, m_cx, *m_cmd_x2y, 0, 0, 1, m_dtos_x2y);
507 m_fft_fwd_y.template compute_c2c<Direction::forward>();
510 ParallelCopy(m_cz, m_cy, *m_cmd_y2z, 0, 0, 1, m_dtos_y2z);
512 #if (AMREX_SPACEDIM == 3)
513 else if ( m_cmd_x2z) {
519 Box upper_half = m_spectral_domain_z;
522 upper_half.
growLo (0,-m_spectral_domain_z.length(0)/2);
523 m_cz.setVal(0, upper_half, 0, 1);
527 ParallelCopy(m_cz, m_cx, *m_cmd_x2z, 0, 0, 1, m_dtos_x2z);
531 m_fft_fwd_z.template compute_c2c<Direction::forward>();
534 template <
typename T, Direction D, DomainStrategy S>
535 template <Direction DIR, std::enable_if_t<DIR == Direction::both,
int> >
538 backward_doit(outmf);
541 template <
typename T, Direction D, DomainStrategy S>
547 m_fft_bwd_x.template compute_r2c<Direction::backward>();
548 outmf.ParallelCopy(m_rx, 0, 0, 1,
IntVect(0), ngout);
552 m_fft_bwd_z.template compute_c2c<Direction::backward>();
554 ParallelCopy(m_cy, m_cz, *m_cmd_z2y, 0, 0, 1, m_dtos_z2y);
556 #if (AMREX_SPACEDIM == 3)
557 else if ( m_cmd_z2x) {
558 auto const& cmd = m_openbc_half ? m_cmd_z2x_half : m_cmd_z2x;
563 m_fft_bwd_y.template compute_c2c<Direction::backward>();
565 ParallelCopy(m_cx, m_cy, *m_cmd_y2x, 0, 0, 1, m_dtos_y2x);
568 auto& fft_x = m_openbc_half ? m_fft_bwd_x_half : m_fft_bwd_x;
569 fft_x.template compute_r2c<Direction::backward>();
570 outmf.ParallelCopy(m_rx, 0, 0, 1,
IntVect(0), ngout);
573 template <
typename T, Direction D, DomainStrategy S>
581 if (!fab) {
return {fwd, bwd};}
583 Box const& box = fab->box();
586 #ifdef AMREX_USE_SYCL
587 fwd.template init_c2c<Direction::forward>(box, pio);
591 fwd.template init_c2c<Direction::forward>(box, pio);
594 bwd.template init_c2c<Direction::backward>(box, pio);
601 template <
typename T, Direction D, DomainStrategy S>
602 template <
typename F>
605 if (m_info.batch_mode) {
608 if ( ! m_cz.empty()) {
611 auto const& a = spectral_fab->array();
615 post_forward(jx,ky,
iz,a(
iz,jx,ky));
618 }
else if ( ! m_cy.empty()) {
621 auto const& a = spectral_fab->array();
625 post_forward(jx,
iy,k,a(
iy,jx,k));
631 auto const& a = spectral_fab->array();
635 post_forward(i,j,k,a(i,j,k));
642 template <
typename T, Direction D, DomainStrategy S>
645 #if (AMREX_SPACEDIM == 3)
646 if (m_info.batch_mode) {
647 return T(1)/T(Long(m_real_domain.length(0)) *
648 Long(m_real_domain.length(1)));
652 return T(1)/T(m_real_domain.numPts());
656 template <
typename T, Direction D, DomainStrategy S>
659 std::pair<typename R2C<T,D,S>::cMF *,
IntVect>
664 }
else if (!m_cy.empty()) {
671 template <
typename T, Direction D, DomainStrategy S>
682 (outmf, m_spectral_domain_x, m_cz,
IntVect(0), dtos);
684 }
else if (!m_cy.empty()) {
686 (outmf, m_spectral_domain_x, m_cy,
IntVect(0), m_dtos_y2x);
693 template <
typename T, Direction D, DomainStrategy S>
703 (m_cz, m_spectral_domain_z, inmf,
IntVect(0), dtos);
705 }
else if (!m_cy.empty()) {
707 (m_cy, m_spectral_domain_y, inmf,
IntVect(0), m_dtos_x2y);
710 m_cx.ParallelCopy(inmf, 0, 0, 1);
712 backward_doit(outmf);
715 template <
typename T, Direction D, DomainStrategy S>
716 std::pair<BoxArray,DistributionMapping>
719 #if (AMREX_SPACEDIM == 3)
721 BoxList bl = m_cz.boxArray().boxList();
723 auto lo =
b.smallEnd();
724 auto hi =
b.bigEnd();
732 return std::make_pair(
BoxArray(std::move(bl)), m_cz.DistributionMap());
735 #if (AMREX_SPACEDIM >= 2)
737 BoxList bl = m_cy.boxArray().boxList();
739 auto lo =
b.smallEnd();
740 auto hi =
b.bigEnd();
746 return std::make_pair(
BoxArray(std::move(bl)), m_cy.DistributionMap());
750 return std::make_pair(m_cx.boxArray(), m_cx.DistributionMap());
#define BL_PROFILE(a)
Definition: AMReX_BLProfiler.H:551
#define AMREX_ALWAYS_ASSERT(EX)
Definition: AMReX_BLassert.H:50
#define AMREX_GPU_DEVICE
Definition: AMReX_GpuQualifiers.H:18
#define AMREX_D_DECL(a, b, c)
Definition: AMReX_SPACE.H:104
if(!(yy_init))
Definition: amrex_iparser.lex.nolint.H:935
A collection of Boxes stored in an Array.
Definition: AMReX_BoxArray.H:550
A class for managing a List of Boxes that share a common IndexType. This class implements operations ...
Definition: AMReX_BoxList.H:52
AMREX_GPU_HOST_DEVICE const IntVectND< dim > & smallEnd() const &noexcept
Get the smallend of the BoxND.
Definition: AMReX_Box.H:105
AMREX_GPU_HOST_DEVICE const IntVectND< dim > & bigEnd() const &noexcept
Get the bigend.
Definition: AMReX_Box.H:116
AMREX_GPU_HOST_DEVICE IntVectND< dim > length() const noexcept
Return the length of the BoxND.
Definition: AMReX_Box.H:146
AMREX_GPU_HOST_DEVICE BoxND & growHi(int idir, int n_cell=1) noexcept
Grow the BoxND on the high end by n_cell cells in direction idir. NOTE: n_cell negative shrinks the B...
Definition: AMReX_Box.H:659
AMREX_GPU_HOST_DEVICE BoxND & growLo(int idir, int n_cell=1) noexcept
Grow the BoxND on the low end by n_cell cells in direction idir. NOTE: n_cell negative shrinks the Bo...
Definition: AMReX_Box.H:648
AMREX_GPU_HOST_DEVICE bool ok() const noexcept
Checks if it is a proper BoxND (including a valid type).
Definition: AMReX_Box.H:200
Calculates the distribution of FABs to MPI processes.
Definition: AMReX_DistributionMapping.H:41
Long size() const noexcept
Length of the underlying processor map.
Definition: AMReX_DistributionMapping.H:127
Definition: AMReX_FFT_OpenBCSolver.H:11
Parallel Discrete Fourier Transform.
Definition: AMReX_FFT_R2C.H:34
T scalingFactor() const
Definition: AMReX_FFT_R2C.H:643
void prepare_openbc()
Definition: AMReX_FFT_R2C.H:434
std::pair< cMF *, IntVect > getSpectralData()
Get the internal spectral data.
RotateFwd m_dtos_x2z
Definition: AMReX_FFT_R2C.H:199
std::unique_ptr< MultiBlockCommMetaData > m_cmd_x2z_half
Definition: AMReX_FFT_R2C.H:193
Swap01 m_dtos_x2y
Definition: AMReX_FFT_R2C.H:195
Plan< T > m_fft_bwd_y
Definition: AMReX_FFT_R2C.H:178
static std::pair< Plan< T >, Plan< T > > make_c2c_plans(cMF &inout)
Definition: AMReX_FFT_R2C.H:575
Box m_real_domain
Definition: AMReX_FFT_R2C.H:210
Plan< T > m_fft_fwd_x_half
Definition: AMReX_FFT_R2C.H:181
Plan< T > m_fft_fwd_x
Definition: AMReX_FFT_R2C.H:175
bool m_do_alld_fft
Definition: AMReX_FFT_R2C.H:217
Swap02 m_dtos_y2z
Definition: AMReX_FFT_R2C.H:197
std::unique_ptr< MultiBlockCommMetaData > m_cmd_x2y
Definition: AMReX_FFT_R2C.H:187
std::unique_ptr< MultiBlockCommMetaData > m_cmd_z2y
Definition: AMReX_FFT_R2C.H:190
Plan< T > m_fft_bwd_x
Definition: AMReX_FFT_R2C.H:176
void forward(MF const &inmf, cMF &outmf)
Forward transform.
Definition: AMReX_FFT_R2C.H:674
cMF m_cz
Definition: AMReX_FFT_R2C.H:205
std::unique_ptr< MultiBlockCommMetaData > m_cmd_z2x_half
Definition: AMReX_FFT_R2C.H:194
Plan< T > m_fft_bwd_z
Definition: AMReX_FFT_R2C.H:180
bool m_slab_decomp
Definition: AMReX_FFT_R2C.H:218
Plan< T > m_fft_bwd_x_half
Definition: AMReX_FFT_R2C.H:182
cMF m_cx
Definition: AMReX_FFT_R2C.H:203
void backward(MF &outmf)
Backward transform.
Definition: AMReX_FFT_R2C.H:536
Box m_spectral_domain_y
Definition: AMReX_FFT_R2C.H:212
RotateBwd m_dtos_z2x
Definition: AMReX_FFT_R2C.H:200
std::pair< BoxArray, DistributionMapping > getSpectralDataLayout() const
Get BoxArray and DistributionMapping for spectral data.
Definition: AMReX_FFT_R2C.H:717
Info m_info
Definition: AMReX_FFT_R2C.H:215
std::unique_ptr< MultiBlockCommMetaData > m_cmd_y2z
Definition: AMReX_FFT_R2C.H:189
Box m_spectral_domain_z
Definition: AMReX_FFT_R2C.H:213
std::unique_ptr< MultiBlockCommMetaData > m_cmd_z2x
Definition: AMReX_FFT_R2C.H:192
std::conditional_t< std::is_same_v< T, Real >, MultiFab, FabArray< BaseFab< T > > > MF
Definition: AMReX_FFT_R2C.H:37
Plan< T > m_fft_fwd_y
Definition: AMReX_FFT_R2C.H:177
R2C(Box const &domain, Info const &info=Info{})
Constructor.
Definition: AMReX_FFT_R2C.H:223
std::unique_ptr< MultiBlockCommMetaData > m_cmd_y2x
Definition: AMReX_FFT_R2C.H:188
bool m_openbc_half
Definition: AMReX_FFT_R2C.H:219
void post_forward_doit(F const &post_forward)
Definition: AMReX_FFT_R2C.H:603
Plan< T > m_fft_fwd_z
Definition: AMReX_FFT_R2C.H:179
std::unique_ptr< char, DataDeleter > m_data_2
Definition: AMReX_FFT_R2C.H:208
Swap02 m_dtos_z2y
Definition: AMReX_FFT_R2C.H:198
R2C & operator=(R2C const &)=delete
MF m_rx
Definition: AMReX_FFT_R2C.H:202
void backward_doit(MF &outmf, IntVect const &ngout=IntVect(0))
Definition: AMReX_FFT_R2C.H:542
Box m_spectral_domain_x
Definition: AMReX_FFT_R2C.H:211
Swap01 m_dtos_y2x
Definition: AMReX_FFT_R2C.H:196
std::unique_ptr< char, DataDeleter > m_data_1
Definition: AMReX_FFT_R2C.H:207
cMF m_cy
Definition: AMReX_FFT_R2C.H:204
~R2C()
Definition: AMReX_FFT_R2C.H:413
void backward(cMF const &inmf, MF &outmf)
Backward transform.
Definition: AMReX_FFT_R2C.H:696
void forward(MF const &inmf)
Forward transform.
Definition: AMReX_FFT_R2C.H:488
std::unique_ptr< MultiBlockCommMetaData > m_cmd_x2z
Definition: AMReX_FFT_R2C.H:191
void forwardThenBackward(MF const &inmf, MF &outmf, F const &post_forward)
Forward and then backward transform.
Definition: AMReX_FFT_R2C.H:78
bool empty() const noexcept
Definition: AMReX_FabArrayBase.H:88
An Array of FortranArrayBox(FAB)-like Objects.
Definition: AMReX_FabArray.H:344
void ParallelCopy(const FabArray< FAB > &src, const Periodicity &period=Periodicity::NonPeriodic(), CpOp op=FabArrayBase::COPY)
Definition: AMReX_FabArray.H:778
void define(const BoxArray &bxs, const DistributionMapping &dm, int nvar, int ngrow, const MFInfo &info=MFInfo(), const FabFactory< FAB > &factory=DefaultFabFactory< FAB >())
Define this FabArray identically to that performed by the constructor having an analogous function si...
Definition: AMReX_FabArray.H:2027
A collection (stored as an array) of FArrayBox objects.
Definition: AMReX_MultiFab.H:38
FA::FABType::value_type * get_fab(FA &fa)
Definition: AMReX_FFT_Helper.H:1303
std::unique_ptr< char, DataDeleter > make_mfs_share(FA1 &fa1, FA2 &fa2)
Definition: AMReX_FFT_Helper.H:1314
DistributionMapping make_iota_distromap(Long n)
Definition: AMReX_FFT.cpp:88
Definition: AMReX_FFT.cpp:7
Direction
Definition: AMReX_FFT_Helper.H:46
DomainStrategy
Definition: AMReX_FFT_Helper.H:48
std::enable_if_t< IsBaseFab< FAB >) &&IsDataPacking< DataPacking, FAB >)> ParallelCopy_finish(FabArray< FAB > &dest, CommHandler handler, const FabArrayBase::CommMetaData &cmd, const DataPacking &data_packing)
Definition: AMReX_NonLocalBC.H:793
AMREX_NODISCARD CommHandler ParallelCopy_nowait(NoLocalCopy, FabArray< FAB > &dest, const FabArray< FAB > &src, const FabArrayBase::CommMetaData &cmd, const DataPacking &data_packing)
Definition: AMReX_NonLocalBC.H:701
int MyProcSub() noexcept
my sub-rank in current frame
Definition: AMReX_ParallelContext.H:76
int NProcsSub() noexcept
number of ranks in current frame
Definition: AMReX_ParallelContext.H:74
int NProcs() noexcept
return the number of MPI ranks local to the current Parallel Context
Definition: AMReX_ParallelDescriptor.H:243
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void swap(T &a, T &b) noexcept
Definition: AMReX_algoim_K.H:113
@ min
Definition: AMReX_ParallelReduce.H:18
constexpr int iz
Definition: AMReX_Interp_3D_C.H:37
constexpr int iy
Definition: AMReX_Interp_2D_C.H:33
std::enable_if_t< std::is_integral_v< T > > ParallelFor(TypeList< CTOs... > ctos, std::array< int, sizeof...(CTOs)> const &runtime_options, T N, F &&f)
Definition: AMReX_CTOParallelForImpl.H:200
IntVectND< AMREX_SPACEDIM > IntVect
Definition: AMReX_BaseFwd.H:30
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Dim3 length(Array4< T > const &a) noexcept
Definition: AMReX_Array4.H:322
BoxArray decompose(Box const &domain, int nboxes, Array< bool, AMREX_SPACEDIM > const &decomp={AMREX_D_DECL(true, true, true)}, bool no_overlap=false)
Decompose domain box into BoxArray.
void Abort(const std::string &msg)
Print out message to cerr and exit via abort().
Definition: AMReX.cpp:225
void ParallelCopy(MF &dst, MF const &src, int scomp, int dcomp, int ncomp, IntVect const &ng_src=IntVect(0), IntVect const &ng_dst=IntVect(0), Periodicity const &period=Periodicity::NonPeriodic())
dst = src w/ MPI communication
Definition: AMReX_FabArrayUtility.H:1672
Definition: AMReX_FFT_Helper.H:56
bool batch_mode
Definition: AMReX_FFT_Helper.H:60
int nprocs
Max number of processes to use.
Definition: AMReX_FFT_Helper.H:63
Definition: AMReX_FFT_Helper.H:111
std::conditional_t< std::is_same_v< float, T >, cuComplex, cuDoubleComplex > VendorComplex
Definition: AMReX_FFT_Helper.H:115
Definition: AMReX_FFT_Helper.H:1426
Definition: AMReX_FFT_Helper.H:1401
Definition: AMReX_FFT_Helper.H:1355
Definition: AMReX_FFT_Helper.H:1378
FabArray memory allocation information.
Definition: AMReX_FabArray.H:66
This class specializes behaviour on local copies and unpacking receive buffers.
Definition: AMReX_NonLocalBC.H:615
Contains information about which components take part of the data transaction.
Definition: AMReX_NonLocalBC.H:528