Block-Structured AMR Software Framework
Loading...
Searching...
No Matches
AMReX_ParReduce.H
Go to the documentation of this file.
1#ifndef AMREX_PAR_REDUCE_H_
2#define AMREX_PAR_REDUCE_H_
3#include <AMReX_Config.H>
4
5#include <AMReX_Concepts.H>
6#include <AMReX_Reduce.H>
7
8namespace amrex {
9
46template <typename... Ops, typename... Ts, BaseFabType FAB, typename F>
47typename ReduceData<Ts...>::Type
48ParReduce (TypeList<Ops...> operation_list, TypeList<Ts...> type_list,
49 FabArray<FAB> const& fa, IntVect const& nghost, F&& f)
50{
51 amrex::ignore_unused(operation_list,type_list);
52 ReduceOps<Ops...> reduce_op;
53 ReduceData<Ts...> reduce_data(reduce_op);
54 reduce_op.eval(fa, nghost, reduce_data, std::forward<F>(f));
55 return reduce_data.value(reduce_op);
56}
57
96template <typename Op, typename T, BaseFabType FAB, typename F>
97T
98ParReduce (TypeList<Op> operation_list, TypeList<T> type_list,
99 FabArray<FAB> const& fa, IntVect const& nghost, F&& f)
100{
101 amrex::ignore_unused(operation_list,type_list);
102 ReduceOps<Op> reduce_op;
103 ReduceData<T> reduce_data(reduce_op);
104 reduce_op.eval(fa, nghost, reduce_data, std::forward<F>(f));
105 auto const& hv = reduce_data.value(reduce_op);
106 return amrex::get<0>(hv);
107}
108
147template <typename... Ops, typename... Ts, BaseFabType FAB, typename F>
148typename ReduceData<Ts...>::Type
149ParReduce (TypeList<Ops...> operation_list, TypeList<Ts...> type_list,
150 FabArray<FAB> const& fa, IntVect const& nghost, int ncomp, F&& f)
151{
152 amrex::ignore_unused(operation_list,type_list);
153 ReduceOps<Ops...> reduce_op;
154 ReduceData<Ts...> reduce_data(reduce_op);
155 reduce_op.eval(fa, nghost, ncomp, reduce_data, std::forward<F>(f));
156 return reduce_data.value(reduce_op);
157}
158
194template <typename Op, typename T, BaseFabType FAB, typename F>
195T
196ParReduce (TypeList<Op> operation_list, TypeList<T> type_list,
197 FabArray<FAB> const& fa, IntVect const& nghost, int ncomp, F&& f)
198{
199 amrex::ignore_unused(operation_list,type_list);
200 ReduceOps<Op> reduce_op;
201 ReduceData<T> reduce_data(reduce_op);
202 reduce_op.eval(fa, nghost, ncomp, reduce_data, std::forward<F>(f));
203 auto const& hv = reduce_data.value(reduce_op);
204 return amrex::get<0>(hv);
205}
206
241template <typename... Ops, typename... Ts, BaseFabType FAB, typename F>
242typename ReduceData<Ts...>::Type
243ParReduce (TypeList<Ops...> operation_list, TypeList<Ts...> type_list,
244 FabArray<FAB> const& fa, F&& f)
245{
246 return ParReduce(operation_list, type_list, fa, IntVect(0), std::forward<F>(f));
247}
248
285template <typename Op, typename T, BaseFabType FAB, typename F>
286T
287ParReduce (TypeList<Op> operation_list, TypeList<T> type_list,
288 FabArray<FAB> const& fa, F&& f)
289{
290 return ParReduce(operation_list, type_list, fa, IntVect(0), std::forward<F>(f));
291}
292
294namespace Reduce::detail {
295
296// Reduce f(box_no,i,j,k,n) separately for each component n (PerBox false) or
297// for each local box and component (PerBox true, result index box_no*ncomp+n).
298// A class template, so that nvcc can name the function of the device lambdas.
299template <bool PerBox, typename... Ops>
300struct ParReduceComp
301{
302 template <typename RT, BaseFabType FAB, typename F>
303 static Vector<RT>
304 eval (FabArray<FAB> const& fa, IntVect const& nghost, int ncomp, F const& f)
305 {
306 const int nlocal = fa.local_size();
307 const int nkeys = PerBox ? nlocal*ncomp : ncomp;
308 Vector<RT> r(std::max(nkeys,0));
309 for (auto& x : r) { for_each_init<0, RT, Ops...>(x); }
310 if (nlocal == 0 || ncomp <= 0) { return r; }
311
312#ifdef AMREX_USE_GPU
313 // Block b works on component n = b/nb1 of local box L and stores its
314 // partial result in slot b, so each key's slots are contiguous. Box L
315 // owns blocks [hoff[L],hoff[L+1]) of every component.
316 constexpr int MT = AMREX_GPU_MAX_THREADS;
317 const int maxblocks = int(Gpu::Device::maxBlocksPerLaunch());
318 Long ntot = 0;
319 for (int L = 0; L < nlocal; ++L) {
320 Box b = fa.box(fa.IndexArray()[L]);
321 ntot += b.ok() ? amrex::grow(b,nghost).numPts() : Long(0);
322 }
323 // About one launch worth of blocks, with at least 4 cells per thread.
324 const Long cells_per_block = std::max(Long(MT)*4, (ntot*ncomp + maxblocks - 1) / maxblocks);
325 Gpu::PinnedVector<int> hoff(nlocal+1, 0);
326 for (int L = 0; L < nlocal; ++L) {
327 Box b = fa.box(fa.IndexArray()[L]);
328 Long const npts = b.ok() ? amrex::grow(b,nghost).numPts() : Long(0);
329 Long const nb = (npts + cells_per_block - 1) / cells_per_block;
330 AMREX_ALWAYS_ASSERT(Long(hoff[L]) + nb < Long(std::numeric_limits<int>::max()));
331 hoff[L+1] = hoff[L] + int(nb);
332 }
333 const int nb1 = hoff[nlocal];
334 AMREX_ALWAYS_ASSERT(Long(nb1)*ncomp < Long(std::numeric_limits<int>::max()));
335 const int nblocks = nb1*ncomp;
336 if (nblocks == 0) { return r; }
337
338 auto const& stream = Gpu::gpuStream();
339 Gpu::DeviceVector<int> doff(nlocal+1);
340 Gpu::copyAsync(Gpu::hostToDevice, hoff.begin(), hoff.end(), doff.begin());
341 Gpu::DeviceVector<RT> dpart(nblocks);
342 Gpu::DeviceVector<RT> dr(nkeys);
343 int const* poff = doff.data();
344 RT* ppart = dpart.data();
345 RT* pr = dr.data();
346 BoxIndexer const* pbx = fa.getParForInfo(nghost).getBoxes();
347 const Math::FastDivmodU32 fdm_nb1(nb1);
348
349#ifdef AMREX_USE_SYCL
350 constexpr std::size_t shared_mem_bytes = sizeof(RT)
351 * std::max(Gpu::Device::warp_size, MT/Gpu::Device::warp_size);
352 amrex::launch<MT>(std::min(nblocks,maxblocks), shared_mem_bytes, stream,
353 [=] AMREX_GPU_DEVICE (Gpu::Handler const& gh) noexcept
354 {
355 Dim1 blockIdx {gh.blockIdx()};
356 Dim1 threadIdx{gh.threadIdx()};
357 Dim1 gridDim {gh.gridDim()};
358#else
359 amrex::launch<MT>(std::min(nblocks,maxblocks), 0, stream,
360 [=] AMREX_GPU_DEVICE () noexcept
361 {
362#endif
363 for (int b = int(blockIdx.x); b < nblocks; b += int(gridDim.x)) {
364 std::uint32_t n32, bb32;
365 fdm_nb1(n32, bb32, std::uint32_t(b));
366 const int n = int(n32);
367 const int bb = int(bb32);
368 const int L = amrex::bisect(poff, 0, nlocal, bb);
369 const int nb = poff[L+1] - poff[L];
370 BoxIndexer const& indexer = pbx[L];
371 RT t;
372 for_each_init<0, RT, Ops...>(t);
373 for (auto icell = BoxIndexer::UInt(bb-poff[L])*MT + BoxIndexer::UInt(threadIdx.x);
374 icell < indexer.numPts(); icell += BoxIndexer::UInt(nb)*MT)
375 {
376 auto [i, j, k] = indexer(icell);
377 for_each_local<0, RT, Ops...>(t, f(L,i,j,k,n));
378 }
379 RT& dst = ppart[b];
380 if (threadIdx.x == 0) { for_each_init<0, RT, Ops...>(dst); }
381#ifdef AMREX_USE_SYCL
382 for_each_parallel<0, RT, Ops...>(dst, t, gh);
383#else
384 for_each_parallel<0, RT, Ops...>(dst, t);
385#endif
386 }
387 });
388
389 // Combine each key's partial results in a fixed order.
390#ifdef AMREX_USE_SYCL
391 amrex::launch<MT>(std::min(nkeys,maxblocks), shared_mem_bytes, stream,
392 [=] AMREX_GPU_DEVICE (Gpu::Handler const& gh) noexcept
393 {
394 Dim1 blockIdx {gh.blockIdx()};
395 Dim1 threadIdx{gh.threadIdx()};
396 Dim1 gridDim {gh.gridDim()};
397#else
398 amrex::launch<MT>(std::min(nkeys,maxblocks), 0, stream,
399 [=] AMREX_GPU_DEVICE () noexcept
400 {
401#endif
402 for (int key = int(blockIdx.x); key < nkeys; key += int(gridDim.x)) {
403 int begin = 0;
404 int end = 0;
405 if (PerBox) { // not constexpr: nvcc cannot first-capture ncomp there
406 const int L = key / ncomp;
407 const int n = key - L*ncomp;
408 begin = n*nb1 + poff[L];
409 end = n*nb1 + poff[L+1];
410 } else {
411 begin = key*nb1;
412 end = begin + nb1;
413 }
414 RT t;
415 for_each_init<0, RT, Ops...>(t);
416 for (int ib = begin + int(threadIdx.x); ib < end; ib += MT) {
417 for_each_local<0, RT, Ops...>(t, ppart[ib]);
418 }
419 RT& dst = pr[key];
420 if (threadIdx.x == 0) { for_each_init<0, RT, Ops...>(dst); }
421#ifdef AMREX_USE_SYCL
422 for_each_parallel<0, RT, Ops...>(dst, t, gh);
423#else
424 for_each_parallel<0, RT, Ops...>(dst, t);
425#endif
426 }
427 });
428
429 Gpu::dtoh_memcpy_async(r.data(), pr, sizeof(RT)*nkeys);
430 Gpu::streamSynchronize();
431#else
432 if constexpr (PerBox) {
433 // With too few (box, component) pairs for the threads, split each
434 // box along its last dimension into contiguous chunks of about 16K
435 // cells. Items are ordered by box, component and chunk, which is
436 // the memory order, and their partial results are combined in order.
437 constexpr int d = AMREX_SPACEDIM-1;
438 const int nthreads = OpenMP::in_parallel() ? 1 : OpenMP::get_max_threads();
439 const bool split = nthreads > 1 && nkeys < 4*nthreads;
440 Vector<Box> bx(nlocal);
441 Vector<int> thick(nlocal), nchunk(nlocal), ioff(nlocal+1, 0);
442 for (int L = 0; L < nlocal; ++L) {
443 bx[L] = fa.box(fa.IndexArray()[L]);
444 if (bx[L].ok()) { bx[L].grow(nghost); }
445 // An empty box is one empty chunk.
446 const int len = bx[L].ok() ? bx[L].length(d) : 1;
447 thick[L] = (split && bx[L].ok()) ? int(std::max(Long(1), Long(16384)/(bx[L].numPts()/len))) : len;
448 nchunk[L] = (len + thick[L] - 1) / thick[L];
449 ioff[L+1] = ioff[L] + nchunk[L]*ncomp;
450 }
451 // Without splitting, item is the result index.
452 const int nitems = ioff[nlocal];
453 Vector<RT> part(split ? nitems : 0);
454 RT* pr = split ? part.data() : r.data();
455#ifdef AMREX_USE_OMP
456#pragma omp parallel for schedule(dynamic) if (!system::regtest_reduction)
457#endif
458 for (int item = 0; item < nitems; ++item) {
459 const int L = split ? amrex::bisect(ioff.data(), 0, nlocal, item) : item/ncomp;
460 const int m = item - ioff[L];
461 const int n = m / nchunk[L];
462 const int c = m - n*nchunk[L];
463 Box b = bx[L];
464 b.setSmall(d, bx[L].smallEnd(d) + c*thick[L]);
465 b.setBig(d, std::min(b.smallEnd(d)+thick[L]-1, bx[L].bigEnd(d)));
466 const auto lo = amrex::lbound(b);
467 const auto hi = amrex::ubound(b);
468 RT rem;
469 for_each_init<0, RT, Ops...>(rem);
470 Lanes<RT> acc;
471 lanes_set<0>(acc, rem);
472 for (int k = lo.z; k <= hi.z; ++k) {
473 for (int j = lo.y; j <= hi.y; ++j) {
474 reduce_row<Ops...>(lo.x, hi.x, acc, rem,
475 [&] (int i) { return f(L,i,j,k,n); });
476 }}
477 RT& x = pr[item];
478 for_each_init<0, RT, Ops...>(x);
479 lanes_fold<0, RT, Ops...>(acc, x);
480 for_each_local<0, RT, Ops...>(x, rem);
481 }
482 if (split) {
483 for (int L = 0; L < nlocal; ++L) {
484 for (int m = 0; m < nchunk[L]*ncomp; ++m) {
485 for_each_local<0, RT, Ops...>(r[L*ncomp+m/nchunk[L]], part[ioff[L]+m]);
486 }
487 }
488 }
489 } else {
490 const int nthreads = OpenMP::in_parallel() ? 1 : OpenMP::get_max_threads();
491 Vector<RT> rt(std::size_t(nthreads)*ncomp);
492 for (auto& x : rt) { for_each_init<0, RT, Ops...>(x); }
493#ifdef AMREX_USE_OMP
494#pragma omp parallel if (nthreads > 1 && !system::regtest_reduction)
495#endif
496 {
497 RT* rr = rt.data() + std::size_t(nthreads > 1 ? OpenMP::get_thread_num() : 0)*ncomp;
498 // Component loop outermost, so that each pass streams one component.
499 for (int n = 0; n < ncomp; ++n) {
500 RT rem;
501 for_each_init<0, RT, Ops...>(rem);
502 Lanes<RT> acc;
503 lanes_set<0>(acc, rem);
504 for (MFIter mfi(fa,true); mfi.isValid(); ++mfi) {
505 Box const& b = mfi.growntilebox(nghost);
506 const int L = mfi.LocalIndex();
507 const auto lo = amrex::lbound(b);
508 const auto hi = amrex::ubound(b);
509 for (int k = lo.z; k <= hi.z; ++k) {
510 for (int j = lo.y; j <= hi.y; ++j) {
511 reduce_row<Ops...>(lo.x, hi.x, acc, rem,
512 [&] (int i) { return f(L,i,j,k,n); });
513 }}
514 }
515 lanes_fold<0, RT, Ops...>(acc, rr[n]);
516 for_each_local<0, RT, Ops...>(rr[n], rem);
517 }
518 }
519 for (int it = 0; it < nthreads; ++it) {
520 for (int n = 0; n < ncomp; ++n) {
521 for_each_local<0, RT, Ops...>(r[n], rt[std::size_t(it)*ncomp+n]);
522 }
523 }
524 }
525#endif
526 return r;
527 }
528};
529
530}
532
564template <typename... Ops, typename... Ts, BaseFabType FAB, typename F>
565Vector<typename ReduceData<Ts...>::Type>
567 FabArray<FAB> const& fa, IntVect const& nghost, int ncomp, F&& f)
568{
569 amrex::ignore_unused(operation_list,type_list);
570 return Reduce::detail::ParReduceComp<false,Ops...>::template
571 eval<typename ReduceData<Ts...>::Type>(fa, nghost, ncomp, std::forward<F>(f));
572}
573
579template <typename Op, typename T, BaseFabType FAB, typename F>
580Vector<T>
581ParReducePerComp (TypeList<Op> operation_list, TypeList<T> type_list,
582 FabArray<FAB> const& fa, IntVect const& nghost, int ncomp, F&& f)
583{
584 amrex::ignore_unused(operation_list,type_list);
585 auto const& rt = Reduce::detail::ParReduceComp<false,Op>::template
586 eval<typename ReduceData<T>::Type>(fa, nghost, ncomp, std::forward<F>(f));
587 Vector<T> r(rt.size());
588 for (int i = 0, N = int(rt.size()); i < N; ++i) { r[i] = amrex::get<0>(rt[i]); }
589 return r;
590}
591
601template <typename... Ops, typename... Ts, BaseFabType FAB, typename F>
602Vector<typename ReduceData<Ts...>::Type>
604 FabArray<FAB> const& fa, IntVect const& nghost, int ncomp, F&& f)
605{
606 amrex::ignore_unused(operation_list,type_list);
607 return Reduce::detail::ParReduceComp<true,Ops...>::template
608 eval<typename ReduceData<Ts...>::Type>(fa, nghost, ncomp, std::forward<F>(f));
609}
610
616template <typename Op, typename T, BaseFabType FAB, typename F>
617Vector<T>
619 FabArray<FAB> const& fa, IntVect const& nghost, int ncomp, F&& f)
620{
621 amrex::ignore_unused(operation_list,type_list);
622 auto const& rt = Reduce::detail::ParReduceComp<true,Op>::template
623 eval<typename ReduceData<T>::Type>(fa, nghost, ncomp, std::forward<F>(f));
624 Vector<T> r(rt.size());
625 for (int i = 0, N = int(rt.size()); i < N; ++i) { r[i] = amrex::get<0>(rt[i]); }
626 return r;
627}
628
629}
630#endif
#define AMREX_ALWAYS_ASSERT(EX)
Definition AMReX_BLassert.H:50
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
__host__ __device__ BoxND & setBig(const IntVectND< dim > &bg) noexcept
Redefine the big end of the BoxND.
Definition AMReX_Box.H:516
__host__ __device__ bool ok() const noexcept
Return true if high bounds are >= low bounds and the index type is valid.
Definition AMReX_Box.H:229
__host__ __device__ BoxND & setSmall(const IntVectND< dim > &sm) noexcept
Definition AMReX_Box.H:508
An Array of FortranArrayBox(FAB)-like Objects.
Definition AMReX_FabArray.H:383
Definition AMReX_Reduce.H:556
Type value()
Definition AMReX_Reduce.H:591
Definition AMReX_Reduce.H:715
void eval(MF const &mf, IntVect const &nghost, D &reduce_data, F &&f)
Definition AMReX_Reduce.H:855
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition AMReX_Vector.H:29
Checks if a type is derived from amrex::BaseFab.
Definition AMReX_Concepts.H:13
amrex_long Long
Definition AMReX_INT.H:30
__host__ __device__ Dim3 ubound(Array4< T > const &a) noexcept
Return the inclusive upper bounds of an Array4 in Dim3 form.
Definition AMReX_Array4.H:1377
__host__ __device__ Dim3 lbound(Array4< T > const &a) noexcept
Return the inclusive lower bounds of an Array4 in Dim3 form.
Definition AMReX_Array4.H:1363
__host__ __device__ BoxND< dim > grow(const BoxND< dim > &b, int i) noexcept
Return a copy of b grown uniformly by i cells in every direction.
Definition AMReX_Box.H:1328
__host__ __device__ T bisect(T lo, T hi, F f, T tol=1e-12, int max_iter=100)
Find a root of a scalar function on a bracketing interval using bisection.
Definition AMReX_Algorithm.H:170
Definition AMReX_Amr.cpp:50
__host__ __device__ void ignore_unused(const Ts &...)
No-op helper that marks variables as intentionally unused.
Definition AMReX.H:273
@ F
Definition AMReX_Order.H:7
ReduceData< Ts... >::Type ParReduce(TypeList< Ops... > operation_list, TypeList< Ts... > type_list, FabArray< FAB > const &fa, IntVect const &nghost, F &&f)
Parallel reduce for MultiFab/FabArray. The reduce result is local and it's the user's responsibility ...
Definition AMReX_ParReduce.H:48
Vector< typename ReduceData< Ts... >::Type > ParReducePerComp(TypeList< Ops... > operation_list, TypeList< Ts... > type_list, FabArray< FAB > const &fa, IntVect const &nghost, int ncomp, F &&f)
Parallel reduce for MultiFab/FabArray with a separate result for each component. The reduce result is...
Definition AMReX_ParReduce.H:566
__host__ __device__ Dim3 begin(BoxND< dim > const &box) noexcept
Return the iterator begin coordinate of box as Dim3.
Definition AMReX_Box.H:2241
void launch(T const &n, L const &f) noexcept
Definition AMReX_GpuLaunchFunctsC.H:122
BoxND< 3 > Box
Box is an alias for amrex::BoxND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:35
@ x
Definition AMReX_Orientation.H:14
IntVectND< 3 > IntVect
IntVect is an alias for amrex::IntVectND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:38
BoxIndexerND< 3 > BoxIndexer
Definition AMReX_Box.H:2641
const int[]
Definition AMReX_BLProfiler.cpp:1665
Vector< typename ReduceData< Ts... >::Type > ParReducePerBoxComp(TypeList< Ops... > operation_list, TypeList< Ts... > type_list, FabArray< FAB > const &fa, IntVect const &nghost, int ncomp, F &&f)
Parallel reduce for MultiFab/FabArray with a separate result for each local box and component....
Definition AMReX_ParReduce.H:603
__host__ __device__ Dim3 end(BoxND< dim > const &box) noexcept
Return the iterator end coordinate of box as Dim3.
Definition AMReX_Box.H:2259
std::vector< std::string > split(std::string const &s, std::string const &sep)
Split a string using given tokens in sep.
Definition AMReX_String.cpp:44
__host__ __device__ constexpr int get(IntVectND< dim > const &iv) noexcept
Get the Idx-th element of IntVectND<dim>.
Definition AMReX_IntVect.H:1367
Struct for holding types.
Definition AMReX_TypeList.H:13