1#ifndef AMREX_PAR_REDUCE_H_
2#define AMREX_PAR_REDUCE_H_
3#include <AMReX_Config.H>
46template <
typename... Ops,
typename... Ts,
BaseFabType FAB,
typename F>
54 reduce_op.eval(fa, nghost, reduce_data, std::forward<F>(f));
55 return reduce_data.value(reduce_op);
96template <
typename Op,
typename T, BaseFabType FAB,
typename F>
104 reduce_op.
eval(fa, nghost, reduce_data, std::forward<F>(f));
105 auto const& hv = reduce_data.
value(reduce_op);
147template <
typename... Ops,
typename... Ts, BaseFabType FAB,
typename F>
148typename ReduceData<Ts...>::Type
155 reduce_op.eval(fa, nghost, ncomp, reduce_data, std::forward<F>(f));
156 return reduce_data.value(reduce_op);
194template <
typename Op,
typename T, BaseFabType FAB,
typename F>
202 reduce_op.
eval(fa, nghost, ncomp, reduce_data, std::forward<F>(f));
203 auto const& hv = reduce_data.
value(reduce_op);
241template <
typename... Ops,
typename... Ts, BaseFabType FAB,
typename F>
242typename ReduceData<Ts...>::Type
246 return ParReduce(operation_list, type_list, fa,
IntVect(0), std::forward<F>(f));
285template <
typename Op,
typename T, BaseFabType FAB,
typename F>
290 return ParReduce(operation_list, type_list, fa,
IntVect(0), std::forward<F>(f));
294namespace Reduce::detail {
299template <
bool PerBox,
typename... Ops>
302 template <
typename RT, BaseFabType FAB,
typename F>
304 eval (FabArray<FAB>
const& fa, IntVect
const& nghost,
int ncomp, F
const& f)
306 const int nlocal = fa.local_size();
307 const int nkeys = PerBox ? nlocal*ncomp : ncomp;
308 Vector<RT> r(std::max(nkeys,0));
309 for (
auto& x : r) { for_each_init<0, RT, Ops...>(x); }
310 if (nlocal == 0 || ncomp <= 0) {
return r; }
316 constexpr int MT = AMREX_GPU_MAX_THREADS;
317 const int maxblocks =
int(Gpu::Device::maxBlocksPerLaunch());
319 for (
int L = 0; L < nlocal; ++L) {
320 Box b = fa.box(fa.IndexArray()[L]);
324 const Long cells_per_block = std::max(
Long(MT)*4, (ntot*ncomp + maxblocks - 1) / maxblocks);
325 Gpu::PinnedVector<int> hoff(nlocal+1, 0);
326 for (
int L = 0; L < nlocal; ++L) {
327 Box b = fa.box(fa.IndexArray()[L]);
329 Long const nb = (npts + cells_per_block - 1) / cells_per_block;
331 hoff[L+1] = hoff[L] +
int(nb);
333 const int nb1 = hoff[nlocal];
335 const int nblocks = nb1*ncomp;
336 if (nblocks == 0) {
return r; }
338 auto const& stream = Gpu::gpuStream();
339 Gpu::DeviceVector<int> doff(nlocal+1);
340 Gpu::copyAsync(Gpu::hostToDevice, hoff.begin(), hoff.end(), doff.begin());
341 Gpu::DeviceVector<RT> dpart(nblocks);
342 Gpu::DeviceVector<RT> dr(nkeys);
343 int const* poff = doff.data();
344 RT* ppart = dpart.data();
346 BoxIndexer const* pbx = fa.getParForInfo(nghost).getBoxes();
347 const Math::FastDivmodU32 fdm_nb1(nb1);
350 constexpr std::size_t shared_mem_bytes =
sizeof(RT)
351 * std::max(Gpu::Device::warp_size, MT/Gpu::Device::warp_size);
355 Dim1 blockIdx {gh.blockIdx()};
356 Dim1 threadIdx{gh.threadIdx()};
357 Dim1 gridDim {gh.gridDim()};
363 for (
int b =
int(blockIdx.x); b < nblocks; b +=
int(gridDim.x)) {
364 std::uint32_t n32, bb32;
365 fdm_nb1(n32, bb32, std::uint32_t(b));
366 const int n =
int(n32);
367 const int bb =
int(bb32);
369 const int nb = poff[L+1] - poff[L];
372 for_each_init<0, RT, Ops...>(t);
373 for (
auto icell = BoxIndexer::UInt(bb-poff[L])*MT + BoxIndexer::UInt(threadIdx.x);
374 icell < indexer.numPts(); icell += BoxIndexer::UInt(nb)*MT)
376 auto [i, j, k] = indexer(icell);
377 for_each_local<0, RT, Ops...>(t, f(L,i,j,k,n));
380 if (threadIdx.x == 0) { for_each_init<0, RT, Ops...>(dst); }
382 for_each_parallel<0, RT, Ops...>(dst, t, gh);
384 for_each_parallel<0, RT, Ops...>(dst, t);
394 Dim1 blockIdx {gh.blockIdx()};
395 Dim1 threadIdx{gh.threadIdx()};
396 Dim1 gridDim {gh.gridDim()};
402 for (
int key =
int(blockIdx.x); key < nkeys; key +=
int(gridDim.x)) {
406 const int L = key / ncomp;
407 const int n = key - L*ncomp;
408 begin = n*nb1 + poff[L];
409 end = n*nb1 + poff[L+1];
415 for_each_init<0, RT, Ops...>(t);
416 for (
int ib = begin +
int(threadIdx.x); ib < end; ib += MT) {
417 for_each_local<0, RT, Ops...>(t, ppart[ib]);
420 if (threadIdx.x == 0) { for_each_init<0, RT, Ops...>(dst); }
422 for_each_parallel<0, RT, Ops...>(dst, t, gh);
424 for_each_parallel<0, RT, Ops...>(dst, t);
429 Gpu::dtoh_memcpy_async(r.data(), pr,
sizeof(RT)*nkeys);
430 Gpu::streamSynchronize();
432 if constexpr (PerBox) {
437 constexpr int d = AMREX_SPACEDIM-1;
438 const int nthreads = OpenMP::in_parallel() ? 1 : OpenMP::get_max_threads();
439 const bool split = nthreads > 1 && nkeys < 4*nthreads;
440 Vector<Box> bx(nlocal);
441 Vector<int> thick(nlocal), nchunk(nlocal), ioff(nlocal+1, 0);
442 for (
int L = 0; L < nlocal; ++L) {
443 bx[L] = fa.box(fa.IndexArray()[L]);
444 if (bx[L].ok()) { bx[L].grow(nghost); }
446 const int len = bx[L].ok() ? bx[L].length(d) : 1;
447 thick[L] = (
split && bx[L].ok()) ?
int(std::max(
Long(1),
Long(16384)/(bx[L].numPts()/len))) : len;
448 nchunk[L] = (len + thick[L] - 1) / thick[L];
449 ioff[L+1] = ioff[L] + nchunk[L]*ncomp;
452 const int nitems = ioff[nlocal];
453 Vector<RT> part(split ? nitems : 0);
454 RT* pr =
split ? part.data() : r.data();
456#pragma omp parallel for schedule(dynamic) if (!system::regtest_reduction)
458 for (
int item = 0; item < nitems; ++item) {
460 const int m = item - ioff[L];
461 const int n = m / nchunk[L];
462 const int c = m - n*nchunk[L];
464 b.
setSmall(d, bx[L].smallEnd(d) + c*thick[L]);
465 b.
setBig(d, std::min(b.smallEnd(d)+thick[L]-1, bx[L].bigEnd(d)));
469 for_each_init<0, RT, Ops...>(rem);
471 lanes_set<0>(acc, rem);
472 for (
int k = lo.z; k <= hi.z; ++k) {
473 for (
int j = lo.y; j <= hi.y; ++j) {
474 reduce_row<Ops...>(lo.x, hi.x, acc, rem,
475 [&] (
int i) {
return f(L,i,j,k,n); });
478 for_each_init<0, RT, Ops...>(
x);
479 lanes_fold<0, RT, Ops...>(acc,
x);
480 for_each_local<0, RT, Ops...>(
x, rem);
483 for (
int L = 0; L < nlocal; ++L) {
484 for (
int m = 0; m < nchunk[L]*ncomp; ++m) {
485 for_each_local<0, RT, Ops...>(r[L*ncomp+m/nchunk[L]], part[ioff[L]+m]);
490 const int nthreads = OpenMP::in_parallel() ? 1 : OpenMP::get_max_threads();
491 Vector<RT> rt(std::size_t(nthreads)*ncomp);
492 for (
auto& x : rt) { for_each_init<0, RT, Ops...>(
x); }
494#pragma omp parallel if (nthreads > 1 && !system::regtest_reduction)
497 RT* rr = rt.data() + std::size_t(nthreads > 1 ? OpenMP::get_thread_num() : 0)*ncomp;
499 for (
int n = 0; n < ncomp; ++n) {
501 for_each_init<0, RT, Ops...>(rem);
503 lanes_set<0>(acc, rem);
504 for (MFIter mfi(fa,
true); mfi.isValid(); ++mfi) {
505 Box const& b = mfi.growntilebox(nghost);
506 const int L = mfi.LocalIndex();
509 for (
int k = lo.z; k <= hi.z; ++k) {
510 for (
int j = lo.y; j <= hi.y; ++j) {
511 reduce_row<Ops...>(lo.x, hi.x, acc, rem,
512 [&] (
int i) {
return f(L,i,j,k,n); });
515 lanes_fold<0, RT, Ops...>(acc, rr[n]);
516 for_each_local<0, RT, Ops...>(rr[n], rem);
519 for (
int it = 0; it < nthreads; ++it) {
520 for (
int n = 0; n < ncomp; ++n) {
521 for_each_local<0, RT, Ops...>(r[n], rt[std::size_t(it)*ncomp+n]);
564template <
typename... Ops,
typename... Ts, BaseFabType FAB,
typename F>
565Vector<
typename ReduceData<Ts...>::Type>
570 return Reduce::detail::ParReduceComp<
false,Ops...>::template
571 eval<
typename ReduceData<Ts...>::Type>(fa, nghost, ncomp, std::forward<F>(f));
579template <
typename Op,
typename T, BaseFabType FAB,
typename F>
585 auto const& rt = Reduce::detail::ParReduceComp<false,Op>::template
586 eval<typename ReduceData<T>::Type>(fa, nghost, ncomp, std::forward<F>(f));
588 for (
int i = 0, N =
int(rt.size()); i < N; ++i) { r[i] =
amrex::get<0>(rt[i]); }
601template <
typename... Ops,
typename... Ts, BaseFabType FAB,
typename F>
602Vector<
typename ReduceData<Ts...>::Type>
607 return Reduce::detail::ParReduceComp<
true,Ops...>::template
608 eval<
typename ReduceData<Ts...>::Type>(fa, nghost, ncomp, std::forward<F>(f));
616template <
typename Op,
typename T, BaseFabType FAB,
typename F>
622 auto const& rt = Reduce::detail::ParReduceComp<true,Op>::template
623 eval<typename ReduceData<T>::Type>(fa, nghost, ncomp, std::forward<F>(f));
625 for (
int i = 0, N =
int(rt.size()); i < N; ++i) { r[i] =
amrex::get<0>(rt[i]); }
#define AMREX_ALWAYS_ASSERT(EX)
Definition AMReX_BLassert.H:50
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
__host__ __device__ BoxND & setBig(const IntVectND< dim > &bg) noexcept
Redefine the big end of the BoxND.
Definition AMReX_Box.H:516
__host__ __device__ bool ok() const noexcept
Return true if high bounds are >= low bounds and the index type is valid.
Definition AMReX_Box.H:229
__host__ __device__ BoxND & setSmall(const IntVectND< dim > &sm) noexcept
Definition AMReX_Box.H:508
An Array of FortranArrayBox(FAB)-like Objects.
Definition AMReX_FabArray.H:383
Definition AMReX_Reduce.H:556
Type value()
Definition AMReX_Reduce.H:591
Definition AMReX_Reduce.H:715
void eval(MF const &mf, IntVect const &nghost, D &reduce_data, F &&f)
Definition AMReX_Reduce.H:855
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition AMReX_Vector.H:29
Checks if a type is derived from amrex::BaseFab.
Definition AMReX_Concepts.H:13
amrex_long Long
Definition AMReX_INT.H:30
__host__ __device__ Dim3 ubound(Array4< T > const &a) noexcept
Return the inclusive upper bounds of an Array4 in Dim3 form.
Definition AMReX_Array4.H:1377
__host__ __device__ Dim3 lbound(Array4< T > const &a) noexcept
Return the inclusive lower bounds of an Array4 in Dim3 form.
Definition AMReX_Array4.H:1363
__host__ __device__ BoxND< dim > grow(const BoxND< dim > &b, int i) noexcept
Return a copy of b grown uniformly by i cells in every direction.
Definition AMReX_Box.H:1328
__host__ __device__ T bisect(T lo, T hi, F f, T tol=1e-12, int max_iter=100)
Find a root of a scalar function on a bracketing interval using bisection.
Definition AMReX_Algorithm.H:170
Definition AMReX_Amr.cpp:50
__host__ __device__ void ignore_unused(const Ts &...)
No-op helper that marks variables as intentionally unused.
Definition AMReX.H:273
@ F
Definition AMReX_Order.H:7
ReduceData< Ts... >::Type ParReduce(TypeList< Ops... > operation_list, TypeList< Ts... > type_list, FabArray< FAB > const &fa, IntVect const &nghost, F &&f)
Parallel reduce for MultiFab/FabArray. The reduce result is local and it's the user's responsibility ...
Definition AMReX_ParReduce.H:48
Vector< typename ReduceData< Ts... >::Type > ParReducePerComp(TypeList< Ops... > operation_list, TypeList< Ts... > type_list, FabArray< FAB > const &fa, IntVect const &nghost, int ncomp, F &&f)
Parallel reduce for MultiFab/FabArray with a separate result for each component. The reduce result is...
Definition AMReX_ParReduce.H:566
__host__ __device__ Dim3 begin(BoxND< dim > const &box) noexcept
Return the iterator begin coordinate of box as Dim3.
Definition AMReX_Box.H:2241
void launch(T const &n, L const &f) noexcept
Definition AMReX_GpuLaunchFunctsC.H:122
BoxND< 3 > Box
Box is an alias for amrex::BoxND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:35
@ x
Definition AMReX_Orientation.H:14
IntVectND< 3 > IntVect
IntVect is an alias for amrex::IntVectND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:38
BoxIndexerND< 3 > BoxIndexer
Definition AMReX_Box.H:2641
const int[]
Definition AMReX_BLProfiler.cpp:1665
Vector< typename ReduceData< Ts... >::Type > ParReducePerBoxComp(TypeList< Ops... > operation_list, TypeList< Ts... > type_list, FabArray< FAB > const &fa, IntVect const &nghost, int ncomp, F &&f)
Parallel reduce for MultiFab/FabArray with a separate result for each local box and component....
Definition AMReX_ParReduce.H:603
__host__ __device__ Dim3 end(BoxND< dim > const &box) noexcept
Return the iterator end coordinate of box as Dim3.
Definition AMReX_Box.H:2259
std::vector< std::string > split(std::string const &s, std::string const &sep)
Split a string using given tokens in sep.
Definition AMReX_String.cpp:44
__host__ __device__ constexpr int get(IntVectND< dim > const &iv) noexcept
Get the Idx-th element of IntVectND<dim>.
Definition AMReX_IntVect.H:1367
Struct for holding types.
Definition AMReX_TypeList.H:13