1 #ifndef AMREX_FFT_R2X_H_
2 #define AMREX_FFT_R2X_H_
3 #include <AMReX_Config.H>
20 template <
typename T = Real>
24 using MF = std::conditional_t<std::is_same_v<T,Real>,
29 Array<std::pair<Boundary,Boundary>,AMREX_SPACEDIM>
const& bc,
45 template <
int dim,
typename FAB,
typename F>
96 Array<std::pair<Boundary,Boundary>,AMREX_SPACEDIM>
const& bc,
104 static_assert(std::is_same_v<float,T> || std::is_same_v<double,T>);
108 #if (AMREX_SPACEDIM == 3)
111 for (
int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
113 if (bc[idim].first == Boundary::periodic ||
114 bc[idim].
second == Boundary::periodic) {
129 m_rx.define(bax, dmx, 1, 0,
MFInfo().SetAlloc(
false));
132 if (bc[0].first == Boundary::periodic) {
138 for (
auto &
b : bl) {
145 #if (AMREX_SPACEDIM >= 2)
146 if (domain.
length(1) > 1) {
171 if (ba.size() ==
m_rx.size()) {
172 dm =
m_rx.DistributionMap();
176 m_ry.define(ba, dm, 1, 0,
MFInfo().SetAlloc(
false));
181 if (bc[1].first == Boundary::periodic) {
186 for (
auto &
b : bl) {
196 #if (AMREX_SPACEDIM == 3)
197 if (domain.
length(2) > 1) {
222 if (ba.size() ==
m_ry.size()) {
223 dm =
m_ry.DistributionMap();
227 m_rz.define(ba, dm, 1, 0,
MFInfo().SetAlloc(
false));
232 if (bc[2].first == Boundary::periodic) {
237 for (
auto &
b : bl) {
287 #if (AMREX_SPACEDIM >= 2)
288 if (domain.
length(1) > 1) {
291 m_cmd_cx2cy = std::make_unique<MultiBlockCommMetaData>
293 m_cmd_cy2cx = std::make_unique<MultiBlockCommMetaData>
297 m_cmd_rx2ry = std::make_unique<MultiBlockCommMetaData>
299 m_cmd_ry2rx = std::make_unique<MultiBlockCommMetaData>
305 #if (AMREX_SPACEDIM == 3)
306 if (domain.
length(2) > 1) {
309 m_cmd_cy2cz = std::make_unique<MultiBlockCommMetaData>
311 m_cmd_cz2cy = std::make_unique<MultiBlockCommMetaData>
315 m_cmd_ry2rz = std::make_unique<MultiBlockCommMetaData>
317 m_cmd_rz2ry = std::make_unique<MultiBlockCommMetaData>
329 if (myproc <
m_rx.size())
331 Box const& box =
m_rx.box(myproc);
332 auto* pf =
m_rx[myproc].dataPtr();
333 if (bc[0].first == Boundary::periodic) {
334 auto* pb = (VendorComplex*)
m_cx[myproc].dataPtr();
335 m_fft_fwd_x.template init_r2c<Direction::forward>(box, pf, pb);
336 #if defined(AMREX_USE_SYCL)
339 m_fft_bwd_x.template init_r2c<Direction::backward>(box, pf, pb);
342 m_fft_fwd_x.template init_r2r<Direction::forward>(box, pf, bc[0]);
343 #if defined(AMREX_USE_GPU)
344 if ((bc[0].first == Boundary::even && bc[0].
second == Boundary::odd) ||
345 (bc[0].first == Boundary::odd && bc[0].
second == Boundary::even)) {
350 m_fft_bwd_x.template init_r2r<Direction::backward>(box, pf, bc[0]);
355 #if (AMREX_SPACEDIM >= 2)
356 if (
m_ry.empty() &&
m_bc[1].first == Boundary::periodic) {
359 auto* p = (VendorComplex *)
m_cy[myproc].dataPtr();
360 m_fft_fwd_y.template init_c2c<Direction::forward>(box, p);
361 #if defined(AMREX_USE_SYCL)
364 m_fft_bwd_y.template init_c2c<Direction::backward>(box, p);
367 }
else if (!
m_ry.empty() &&
m_bc[1].first == Boundary::periodic) {
368 if (myproc <
m_ry.size()) {
369 Box const& box =
m_ry.box(myproc);
370 auto* pr =
m_ry[myproc].dataPtr();
371 auto* pc = (VendorComplex*)
m_cy[myproc].dataPtr();
372 m_fft_fwd_y.template init_r2c<Direction::forward>(box, pr, pc);
373 #if defined(AMREX_USE_SYCL)
376 m_fft_bwd_y.template init_r2c<Direction::backward>(box, pr, pc);
382 auto* p = (VendorComplex*)
m_cy[myproc].dataPtr();
383 m_fft_fwd_y.template init_r2r<Direction::forward>(box, p, bc[1]);
384 #if defined(AMREX_USE_GPU)
385 if ((bc[1].first == Boundary::even && bc[1].
second == Boundary::odd) ||
386 (bc[1].first == Boundary::odd && bc[1].
second == Boundary::even)) {
391 m_fft_bwd_y.template init_r2r<Direction::backward>(box, p, bc[1]);
395 if (myproc <
m_ry.size()) {
396 Box const& box =
m_ry.box(myproc);
397 auto* p =
m_ry[myproc].dataPtr();
398 m_fft_fwd_y.template init_r2r<Direction::forward>(box, p, bc[1]);
399 #if defined(AMREX_USE_GPU)
400 if ((bc[1].first == Boundary::even && bc[1].
second == Boundary::odd) ||
401 (bc[1].first == Boundary::odd && bc[1].
second == Boundary::even)) {
406 m_fft_bwd_y.template init_r2r<Direction::backward>(box, p, bc[1]);
412 #if (AMREX_SPACEDIM == 3)
413 if (
m_rz.empty() &&
m_bc[2].first == Boundary::periodic) {
416 auto* p = (VendorComplex*)
m_cz[myproc].dataPtr();
417 m_fft_fwd_z.template init_c2c<Direction::forward>(box, p);
418 #if defined(AMREX_USE_SYCL)
421 m_fft_bwd_z.template init_c2c<Direction::backward>(box, p);
424 }
else if (!
m_rz.empty() &&
m_bc[2].first == Boundary::periodic) {
425 if (myproc <
m_rz.size()) {
426 Box const& box =
m_rz.box(myproc);
427 auto* pr =
m_rz[myproc].dataPtr();
428 auto* pc = (VendorComplex*)
m_cz[myproc].dataPtr();
429 m_fft_fwd_z.template init_r2c<Direction::forward>(box, pr, pc);
430 #if defined(AMREX_USE_SYCL)
433 m_fft_bwd_z.template init_r2c<Direction::backward>(box, pr, pc);
439 auto* p = (VendorComplex*)
m_cz[myproc].dataPtr();
440 m_fft_fwd_z.template init_r2r<Direction::forward>(box, p, bc[2]);
441 #if defined(AMREX_USE_GPU)
442 if ((bc[2].first == Boundary::even && bc[2].
second == Boundary::odd) ||
443 (bc[2].first == Boundary::odd && bc[2].
second == Boundary::even)) {
448 m_fft_bwd_z.template init_r2r<Direction::backward>(box, p, bc[2]);
452 if (myproc <
m_rz.size()) {
453 Box const& box =
m_rz.box(myproc);
454 auto* p =
m_rz[myproc].dataPtr();
455 m_fft_fwd_z.template init_r2r<Direction::forward>(box, p, bc[2]);
456 #if defined(AMREX_USE_GPU)
457 if ((bc[2].first == Boundary::even && bc[2].
second == Boundary::odd) ||
458 (bc[2].first == Boundary::odd && bc[2].
second == Boundary::even)) {
463 m_fft_bwd_z.template init_r2r<Direction::backward>(box, p, bc[2]);
470 template <
typename T>
473 if (m_fft_bwd_x.plan != m_fft_fwd_x.plan) {
474 m_fft_bwd_x.destroy();
476 if (m_fft_bwd_y.plan != m_fft_fwd_y.plan) {
477 m_fft_bwd_y.destroy();
479 if (m_fft_bwd_z.plan != m_fft_fwd_z.plan) {
480 m_fft_bwd_z.destroy();
482 m_fft_fwd_x.destroy();
483 m_fft_fwd_y.destroy();
484 m_fft_fwd_z.destroy();
487 template <
typename T>
490 auto r = m_dom_0.numPts();
491 for (
int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
492 if (m_bc[idim].first != Boundary::periodic
493 && m_dom_0.length(idim) > 1)
501 template <
typename T>
502 template <
typename F>
509 m_rx.ParallelCopy(inmf, 0, 0, 1);
510 if (m_bc[0].first == Boundary::periodic) {
511 m_fft_fwd_x.template compute_r2c<Direction::forward>();
513 m_fft_fwd_x.template compute_r2r<Direction::forward>();
516 #if (AMREX_SPACEDIM >= 2)
518 ParallelCopy(m_cy, m_cx, *m_cmd_cx2cy, 0, 0, 1, m_dtos_x2y);
519 }
else if ( m_cmd_rx2ry) {
520 ParallelCopy(m_ry, m_rx, *m_cmd_rx2ry, 0, 0, 1, m_dtos_x2y);
522 if (m_bc[1].first != Boundary::periodic)
524 m_fft_fwd_y.template compute_r2r<Direction::forward>();
526 else if (m_bc[0].first == Boundary::periodic)
528 m_fft_fwd_y.template compute_c2c<Direction::forward>();
532 m_fft_fwd_y.template compute_r2c<Direction::forward>();
536 #if (AMREX_SPACEDIM == 3)
538 ParallelCopy(m_cz, m_cy, *m_cmd_cy2cz, 0, 0, 1, m_dtos_y2z);
539 }
else if ( m_cmd_ry2rz) {
540 ParallelCopy(m_rz, m_ry, *m_cmd_ry2rz, 0, 0, 1, m_dtos_y2z);
542 if (m_bc[2].first != Boundary::periodic)
544 m_fft_fwd_z.template compute_r2r<Direction::forward>();
546 else if (m_bc[0].first == Boundary::periodic ||
547 m_bc[1].first == Boundary::periodic)
549 m_fft_fwd_z.template compute_c2c<Direction::forward>();
553 m_fft_fwd_z.template compute_r2c<Direction::forward>();
559 int actual_dim = AMREX_SPACEDIM;
560 #if (AMREX_SPACEDIM >= 2)
561 if (m_dom_0.length(1) == 1) { actual_dim = 1; }
563 #if (AMREX_SPACEDIM == 3)
564 if ((m_dom_0.length(2) == 1) && (m_dom_0.length(1) > 1)) { actual_dim = 2; }
567 if (actual_dim == 1) {
574 #if (AMREX_SPACEDIM >= 2)
575 else if (actual_dim == 2) {
583 #if (AMREX_SPACEDIM == 3)
584 else if (actual_dim == 3) {
595 #if (AMREX_SPACEDIM == 3)
596 if (m_bc[2].first != Boundary::periodic)
598 m_fft_bwd_z.template compute_r2r<Direction::backward>();
600 else if (m_bc[0].first == Boundary::periodic ||
601 m_bc[1].first == Boundary::periodic)
603 m_fft_bwd_z.template compute_c2c<Direction::backward>();
607 m_fft_bwd_z.template compute_r2c<Direction::backward>();
610 ParallelCopy(m_cy, m_cz, *m_cmd_cz2cy, 0, 0, 1, m_dtos_z2y);
611 }
else if ( m_cmd_rz2ry) {
612 ParallelCopy(m_ry, m_rz, *m_cmd_rz2ry, 0, 0, 1, m_dtos_z2y);
616 #if (AMREX_SPACEDIM >= 2)
617 if (m_bc[1].first != Boundary::periodic)
619 m_fft_bwd_y.template compute_r2r<Direction::backward>();
621 else if (m_bc[0].first == Boundary::periodic)
623 m_fft_bwd_y.template compute_c2c<Direction::backward>();
627 m_fft_bwd_y.template compute_r2c<Direction::backward>();
630 ParallelCopy(m_cx, m_cy, *m_cmd_cy2cx, 0, 0, 1, m_dtos_y2x);
631 }
else if ( m_cmd_ry2rx) {
632 ParallelCopy(m_rx, m_ry, *m_cmd_ry2rx, 0, 0, 1, m_dtos_y2x);
636 if (m_bc[0].first == Boundary::periodic) {
637 m_fft_bwd_x.template compute_r2c<Direction::backward>();
639 m_fft_bwd_x.template compute_r2r<Direction::backward>();
641 outmf.ParallelCopy(m_rx, 0, 0, 1);
644 template <
typename T>
645 template <
int dim,
typename FAB,
typename F>
649 auto const& a = fab->array();
653 if constexpr (dim == 0) {
655 }
else if constexpr (dim == 1) {
#define BL_PROFILE(a)
Definition: AMReX_BLProfiler.H:551
#define AMREX_ALWAYS_ASSERT(EX)
Definition: AMReX_BLassert.H:50
#define AMREX_GPU_DEVICE
Definition: AMReX_GpuQualifiers.H:18
#define AMREX_D_DECL(a, b, c)
Definition: AMReX_SPACE.H:104
A collection of Boxes stored in an Array.
Definition: AMReX_BoxArray.H:549
A class for managing a List of Boxes that share a common IndexType. This class implements operations ...
Definition: AMReX_BoxList.H:52
AMREX_GPU_HOST_DEVICE const IntVectND< dim > & smallEnd() const &noexcept
Get the smallend of the BoxND.
Definition: AMReX_Box.H:105
AMREX_GPU_HOST_DEVICE const IntVectND< dim > & bigEnd() const &noexcept
Get the bigend.
Definition: AMReX_Box.H:116
AMREX_GPU_HOST_DEVICE IntVectND< dim > length() const noexcept
Return the length of the BoxND.
Definition: AMReX_Box.H:146
AMREX_GPU_HOST_DEVICE bool cellCentered() const noexcept
Returns true if BoxND is cell-centered in all indexing directions.
Definition: AMReX_Box.H:319
Calculates the distribution of FABs to MPI processes.
Definition: AMReX_DistributionMapping.H:41
Discrete Fourier Transform.
Definition: AMReX_FFT_R2X.H:22
MF m_rz
Definition: AMReX_FFT_R2X.H:76
Array< std::pair< Boundary, Boundary >, AMREX_SPACEDIM > m_bc
Definition: AMReX_FFT_R2X.H:50
void post_forward_doit(FAB *fab, F const &f)
Definition: AMReX_FFT_R2X.H:646
Swap01 m_dtos_x2y
Definition: AMReX_FFT_R2X.H:69
std::conditional_t< std::is_same_v< T, Real >, MultiFab, FabArray< BaseFab< T > > > MF
Definition: AMReX_FFT_R2X.H:25
std::unique_ptr< MultiBlockCommMetaData > m_cmd_rz2ry
Definition: AMReX_FFT_R2X.H:67
MF m_ry
Definition: AMReX_FFT_R2X.H:75
Box m_dom_cz
Definition: AMReX_FFT_R2X.H:89
Swap02 m_dtos_y2z
Definition: AMReX_FFT_R2X.H:71
~R2X()
Definition: AMReX_FFT_R2X.H:471
void forwardThenBackward(MF const &inmf, MF &outmf, F const &post_forward)
Definition: AMReX_FFT_R2X.H:503
std::unique_ptr< MultiBlockCommMetaData > m_cmd_ry2rx
Definition: AMReX_FFT_R2X.H:65
Info m_info
Definition: AMReX_FFT_R2X.H:91
std::unique_ptr< MultiBlockCommMetaData > m_cmd_cz2cy
Definition: AMReX_FFT_R2X.H:66
std::unique_ptr< MultiBlockCommMetaData > m_cmd_cy2cz
Definition: AMReX_FFT_R2X.H:61
Plan< T > m_fft_fwd_y
Definition: AMReX_FFT_R2X.H:54
MF m_rx
Definition: AMReX_FFT_R2X.H:74
std::unique_ptr< MultiBlockCommMetaData > m_cmd_ry2rz
Definition: AMReX_FFT_R2X.H:62
Box m_dom_rx
Definition: AMReX_FFT_R2X.H:84
std::unique_ptr< MultiBlockCommMetaData > m_cmd_rx2ry
Definition: AMReX_FFT_R2X.H:60
Swap01 m_dtos_y2x
Definition: AMReX_FFT_R2X.H:70
Box m_dom_cy
Definition: AMReX_FFT_R2X.H:88
T scalingFactor() const
Definition: AMReX_FFT_R2X.H:488
Box m_dom_ry
Definition: AMReX_FFT_R2X.H:85
Plan< T > m_fft_bwd_z
Definition: AMReX_FFT_R2X.H:57
std::unique_ptr< char, DataDeleter > m_data_1
Definition: AMReX_FFT_R2X.H:81
Plan< T > m_fft_fwd_x
Definition: AMReX_FFT_R2X.H:52
R2X(Box const &domain, Array< std::pair< Boundary, Boundary >, AMREX_SPACEDIM > const &bc, Info const &info=Info{})
Definition: AMReX_FFT_R2X.H:95
std::unique_ptr< MultiBlockCommMetaData > m_cmd_cy2cx
Definition: AMReX_FFT_R2X.H:64
Box m_dom_0
Definition: AMReX_FFT_R2X.H:49
std::unique_ptr< MultiBlockCommMetaData > m_cmd_cx2cy
Definition: AMReX_FFT_R2X.H:59
std::unique_ptr< char, DataDeleter > m_data_2
Definition: AMReX_FFT_R2X.H:82
Plan< T > m_fft_fwd_z
Definition: AMReX_FFT_R2X.H:56
Plan< T > m_fft_bwd_y
Definition: AMReX_FFT_R2X.H:55
cMF m_cx
Definition: AMReX_FFT_R2X.H:77
cMF m_cz
Definition: AMReX_FFT_R2X.H:79
R2X & operator=(R2X const &)=delete
Swap02 m_dtos_z2y
Definition: AMReX_FFT_R2X.H:72
cMF m_cy
Definition: AMReX_FFT_R2X.H:78
Box m_dom_cx
Definition: AMReX_FFT_R2X.H:87
Plan< T > m_fft_bwd_x
Definition: AMReX_FFT_R2X.H:53
Box m_dom_rz
Definition: AMReX_FFT_R2X.H:86
int size() const noexcept
Return the number of FABs in the FabArray.
Definition: AMReX_FabArrayBase.H:109
bool empty() const noexcept
Definition: AMReX_FabArrayBase.H:88
const DistributionMapping & DistributionMap() const noexcept
Return constant reference to associated DistributionMapping.
Definition: AMReX_FabArrayBase.H:130
Box box(int K) const noexcept
Return the Kth Box in the BoxArray. That is, the valid region of the Kth grid.
Definition: AMReX_FabArrayBase.H:100
An Array of FortranArrayBox(FAB)-like Objects.
Definition: AMReX_FabArray.H:344
void define(const BoxArray &bxs, const DistributionMapping &dm, int nvar, int ngrow, const MFInfo &info=MFInfo(), const FabFactory< FAB > &factory=DefaultFabFactory< FAB >())
Define this FabArray identically to that performed by the constructor having an analogous function si...
Definition: AMReX_FabArray.H:2027
A collection (stored as an array) of FArrayBox objects.
Definition: AMReX_MultiFab.H:38
@ FAB
Definition: AMReX_AmrvisConstants.H:86
FA::FABType::value_type * get_fab(FA &fa)
Definition: AMReX_FFT_Helper.H:1303
std::unique_ptr< char, DataDeleter > make_mfs_share(FA1 &fa1, FA2 &fa2)
Definition: AMReX_FFT_Helper.H:1314
DistributionMapping make_iota_distromap(Long n)
Definition: AMReX_FFT.cpp:88
Definition: AMReX_FFT.cpp:7
int MyProcSub() noexcept
my sub-rank in current frame
Definition: AMReX_ParallelContext.H:76
int NProcsSub() noexcept
number of ranks in current frame
Definition: AMReX_ParallelContext.H:74
static int f(amrex::Real t, N_Vector y_data, N_Vector y_rhs, void *user_data)
Definition: AMReX_SundialsIntegrator.H:44
@ min
Definition: AMReX_ParallelReduce.H:18
std::enable_if_t< std::is_integral_v< T > > ParallelFor(TypeList< CTOs... > ctos, std::array< int, sizeof...(CTOs)> const &runtime_options, T N, F &&f)
Definition: AMReX_CTOParallelForImpl.H:200
BoxND< AMREX_SPACEDIM > Box
Definition: AMReX_BaseFwd.H:27
double second() noexcept
Definition: AMReX_Utility.cpp:922
IntVectND< AMREX_SPACEDIM > IntVect
Definition: AMReX_BaseFwd.H:30
BoxArray decompose(Box const &domain, int nboxes, Array< bool, AMREX_SPACEDIM > const &decomp={AMREX_D_DECL(true, true, true)}, bool no_overlap=false)
Decompose domain box into BoxArray.
void ParallelCopy(MF &dst, MF const &src, int scomp, int dcomp, int ncomp, IntVect const &ng_src=IntVect(0), IntVect const &ng_dst=IntVect(0), Periodicity const &period=Periodicity::NonPeriodic())
dst = src w/ MPI communication
Definition: AMReX_FabArrayUtility.H:1672
std::array< T, N > Array
Definition: AMReX_Array.H:24
Definition: AMReX_FFT_Helper.H:56
int nprocs
Max number of processes to use.
Definition: AMReX_FFT_Helper.H:63
Definition: AMReX_FFT_Helper.H:111
std::conditional_t< std::is_same_v< float, T >, cuComplex, cuDoubleComplex > VendorComplex
Definition: AMReX_FFT_Helper.H:115
Definition: AMReX_FFT_Helper.H:1355
Definition: AMReX_FFT_Helper.H:1378
FabArray memory allocation information.
Definition: AMReX_FabArray.H:66