Block-Structured AMR Software Framework
Loading...
Searching...
No Matches
AMReX_Reduce.H
Go to the documentation of this file.
1#ifndef AMREX_REDUCE_H_
2#define AMREX_REDUCE_H_
3#include <AMReX_Config.H>
4#include <AMReX.H>
5
6#include <AMReX_Concepts.H>
7#include <AMReX_Gpu.H>
8#include <AMReX_Arena.H>
9#include <AMReX_OpenMP.H>
10#include <AMReX_MFIter.H>
11#include <AMReX_TypeList.H>
12#include <AMReX_ValLocPair.H>
13
14#include <algorithm>
15#include <concepts>
16#include <functional>
17#include <limits>
18#include <utility>
19
20namespace amrex {
21
22namespace Reduce {
23
24// The declaration of these functions are here to work around doxygen issues.
25
40template <typename T, std::integral N>
41T Sum (N n, T const* v, T init_val = 0);
42
59template <typename T, std::integral N, typename F>
60requires (!std::same_as<T*,std::decay_t<F>>)
61T Sum (N n, F const& f, T init_val = 0);
62
77template <typename T, std::integral N>
78T Min (N n, T const* v, T init_val = std::numeric_limits<T>::max());
79
96template <typename T, std::integral N, typename F>
97requires (!std::same_as<T*,std::decay_t<F>>)
98T Min (N n, F const& f, T init_val = std::numeric_limits<T>::max());
99
114template <typename T, std::integral N>
115T Max (N n, T const* v, T init_val = std::numeric_limits<T>::lowest());
116
133template <typename T, std::integral N, typename F>
134requires (!std::same_as<T*,std::decay_t<F>>)
135T Max (N n, F const& f, T init_val = std::numeric_limits<T>::lowest());
136
150template <typename T, std::integral N>
151std::pair<T,T> MinMax (N n, T const* v);
152
168template <typename T, std::integral N, typename F>
169requires (!std::same_as<T*,std::decay_t<F>>)
170std::pair<T,T> MinMax (N n, F const& f);
171
187template <typename T, std::integral N, typename P>
188bool AnyOf (N n, T const* v, P const& pred);
189
203template <typename P, int dim>
204bool AnyOf (BoxND<dim> const& box, P const& pred);
205
206}
207
209namespace Reduce::detail {
210
211// CPU reductions keep nacc partial accumulators so that the inner loop
212// vectorizes.
213inline constexpr int nacc = 8;
214
215// Pragmas for the loop over the nacc accumulators. With GCC, `unroll 4`
216// keeps it vectorizable and novector plus a full unroll keeps it scalar on
217// purpose. Clang needs neither, and nvcc does not know them.
218#if defined(AMREX_CXX_GCC) && !defined(__CUDACC__)
219#define AMREX_PRAGMA_NACC_VECTORIZE _Pragma("GCC unroll 4")
220#if __GNUC__ >= 14
221#define AMREX_PRAGMA_NACC_NOVECTOR _Pragma("GCC novector")
222#else
223#define AMREX_PRAGMA_NACC_NOVECTOR
224#endif
225#define AMREX_PRAGMA_NACC_SCALAR AMREX_PRAGMA_NACC_NOVECTOR _Pragma("GCC unroll 8")
226#else
227#define AMREX_PRAGMA_NACC_VECTORIZE
228#define AMREX_PRAGMA_NACC_NOVECTOR
229#define AMREX_PRAGMA_NACC_SCALAR
230#endif
231static_assert(nacc == 8, "The AMREX_PRAGMA_NACC unroll factors assume nacc == 8");
232
233// Split a 1D range into chunks of a multiple of nacc, enough for all threads.
234template <std::integral N>
235std::pair<N,N> chunks_1d (N n)
236{
237 auto const nthreads = OpenMP::get_max_threads();
238 N const chunk = std::max(static_cast<N>(nacc), static_cast<N>((n / (4*nthreads) / nacc) * nacc));
239 N const nchunks = n / chunk + ((n % chunk != 0) ? 1 : 0);
240 return {chunk, nchunks};
241}
242
243#ifdef AMREX_USE_GPU
244#ifdef AMREX_USE_SYCL
245 template <std::size_t Idx, typename T, typename P>
247 void for_each_parallel (T& d, T const& s, Gpu::Handler const& h)
248 {
249 P().parallel_update(amrex::get<Idx>(d), amrex::get<Idx>(s), h);
250 }
251
252 template <std::size_t Idx, typename T, typename P, typename P1, typename... Ps>
254 void for_each_parallel (T& d, T const& s, Gpu::Handler const& h)
255 {
256 P().parallel_update(amrex::get<Idx>(d), amrex::get<Idx>(s), h);
257 for_each_parallel<Idx+1,T,P1,Ps...>(d, s, h);
258 }
259#else
260 template <std::size_t Idx, typename T, typename P>
262 void for_each_parallel (T& d, T const& s)
263 {
264 P().parallel_update(amrex::get<Idx>(d), amrex::get<Idx>(s));
265 }
266
267 template <std::size_t Idx, typename T, typename P, typename P1, typename... Ps>
269 void for_each_parallel (T& d, T const& s)
270 {
271 P().parallel_update(amrex::get<Idx>(d), amrex::get<Idx>(s));
272 for_each_parallel<Idx+1,T,P1,Ps...>(d, s);
273 }
274#endif
275#endif
276
277 template <std::size_t Idx, typename T, typename P>
279 void for_each_local (T& d, T const& s)
280 {
281 P().local_update(amrex::get<Idx>(d), amrex::get<Idx>(s));
282 }
283
284 template <std::size_t Idx, typename T, typename P, typename P1, typename... Ps>
286 void for_each_local (T& d, T const& s)
287 {
288 P().local_update(amrex::get<Idx>(d), amrex::get<Idx>(s));
289 for_each_local<Idx+1,T,P1,Ps...>(d, s);
290 }
291
292 template <std::size_t Idx, typename T, typename P>
294 constexpr void for_each_init (T& t)
295 {
296 P().init(amrex::get<Idx>(t));
297 }
298
299 template <std::size_t Idx, typename T, typename P, typename P1, typename... Ps>
301 constexpr void for_each_init (T& t)
302 {
303 P().init(amrex::get<Idx>(t));
304 for_each_init<Idx+1,T,P1,Ps...>(t);
305 }
306}
308
311{
312
313#ifdef AMREX_USE_GPU
314#ifdef AMREX_USE_SYCL
315 template <typename T>
317 void parallel_update (T& d, T const& s, Gpu::Handler const& h) const noexcept {
318 T r = Gpu::blockReduceSum(s,h);
319 if (h.threadIdx() == 0) { d += r; }
320 }
321#else
322 template <typename T, int MT=AMREX_GPU_MAX_THREADS>
324 void parallel_update (T& d, T const& s) const noexcept {
326 if (threadIdx.x == 0) { d += r; }
327 }
328#endif
329#endif
330
331 template <typename T>
333 void local_update (T& d, T const& s) const noexcept { d += s; }
334
335 template <typename T>
336 constexpr void init (T& t) const noexcept { t = 0; }
337};
338
341{
342#ifdef AMREX_USE_GPU
343#ifdef AMREX_USE_SYCL
344 template <typename T>
346 void parallel_update (T& d, T const& s, Gpu::Handler const& h) const noexcept {
347 T r = Gpu::blockReduceMin(s,h);
348 if (h.threadIdx() == 0) { d = amrex::min(d,r); }
349 }
350#else
351 template <typename T, int MT=AMREX_GPU_MAX_THREADS>
353 void parallel_update (T& d, T const& s) const noexcept {
355 if (threadIdx.x == 0) { d = amrex::min(d,r); }
356 }
357#endif
358#endif
359
360 template <typename T>
362 void local_update (T& d, T const& s) const noexcept { d = amrex::min(d,s); }
363
364 template <typename T>
365 requires (std::numeric_limits<T>::is_specialized)
366 constexpr void init (T& t) const noexcept { t = std::numeric_limits<T>::max(); }
367
368 template <typename T>
369 requires (!std::numeric_limits<T>::is_specialized)
370 constexpr void init (T& t) const noexcept { t = T::max(); }
371};
372
375{
376#ifdef AMREX_USE_GPU
377#ifdef AMREX_USE_SYCL
378 template <typename T>
380 void parallel_update (T& d, T const& s, Gpu::Handler const& h) const noexcept {
381 T r = Gpu::blockReduceMax(s,h);
382 if (h.threadIdx() == 0) { d = amrex::max(d,r); }
383 }
384#else
385 template <typename T, int MT=AMREX_GPU_MAX_THREADS>
387 void parallel_update (T& d, T const& s) const noexcept {
389 if (threadIdx.x == 0) { d = amrex::max(d,r); }
390 }
391#endif
392#endif
393
394 template <typename T>
396 void local_update (T& d, T const& s) const noexcept { d = amrex::max(d,s); }
397
398 template <typename T>
399 requires (std::numeric_limits<T>::is_specialized)
400 constexpr void init (T& t) const noexcept { t = std::numeric_limits<T>::lowest(); }
401
402 template <typename T>
403 requires (!std::numeric_limits<T>::is_specialized)
404 constexpr void init (T& t) const noexcept { t = T::lowest(); }
405};
406
409{
410#ifdef AMREX_USE_GPU
411#ifdef AMREX_USE_SYCL
412 template <std::integral T>
414 void parallel_update (T& d, T s, Gpu::Handler const& h) const noexcept {
416 if (h.threadIdx() == 0) { d = d && r; }
417 }
418#else
419 template <std::integral T, int MT=AMREX_GPU_MAX_THREADS>
421 void parallel_update (T& d, T s) const noexcept {
423 if (threadIdx.x == 0) { d = d && r; }
424 }
425#endif
426#endif
427
428 template <std::integral T>
430 void local_update (T& d, T s) const noexcept { d = d && s; }
431
432 template <std::integral T>
433 constexpr void init (T& t) const noexcept { t = true; }
434};
435
438{
439#ifdef AMREX_USE_GPU
440#ifdef AMREX_USE_SYCL
441 template <std::integral T>
443 void parallel_update (T& d, T s, Gpu::Handler const& h) const noexcept {
444 T r = Gpu::blockReduceLogicalOr(s,h);
445 if (h.threadIdx() == 0) { d = d || r; }
446 }
447#else
448 template <std::integral T, int MT=AMREX_GPU_MAX_THREADS>
450 void parallel_update (T& d, T s) const noexcept {
452 if (threadIdx.x == 0) { d = d || r; }
453 }
454#endif
455#endif
456
457 template <std::integral T>
459 void local_update (T& d, T s) const noexcept { d = d || s; }
460
461 template <std::integral T>
462 constexpr void init (T& t) const noexcept { t = false; }
463};
464
466namespace Reduce::detail {
467// Whether the accumulator loop of a CPU reduction over Ps is vectorized.
468// A lone min or max stays scalar with GCC 14 or newer on x86 without
469// SSE4.1, where the vector form is slower. Older GCC has no novector.
470template <typename... Ps>
471inline constexpr bool vectorize_nacc_loop =
472#if defined(AMREX_CXX_GCC) && !defined(__CUDACC__) && (__GNUC__ >= 14) && \
473 (defined(__x86_64__) || defined(__i386__)) && !defined(__SSE4_1__)
474 sizeof...(Ps) != 1 ||
475 !((std::is_same_v<Ps,ReduceOpMin> || std::is_same_v<Ps,ReduceOpMax>) && ...);
476#else
477 true;
478#endif
479
480// The nacc partial accumulators of a CPU reduction. GCC keeps them in
481// vector registers only when each tuple member has its own array, clang
482// only when each lane holds a whole tuple.
483template <typename Tup> struct Lanes;
484
485template <typename... Ts>
486struct Lanes<GpuTuple<Ts...>>
487{
488#ifdef AMREX_CXX_GCC
489 GpuTuple<GpuArray<Ts,nacc>...> data;
490 template <int K> auto& get (int lane) noexcept { return amrex::get<K>(data)[lane]; }
491 template <int K> [[nodiscard]] auto const& get (int lane) const noexcept { return amrex::get<K>(data)[lane]; }
492#else
493 GpuTuple<Ts...> data[nacc];
494 template <int K> auto& get (int lane) noexcept { return amrex::get<K>(data[lane]); }
495 template <int K> [[nodiscard]] auto const& get (int lane) const noexcept { return amrex::get<K>(data[lane]); }
496#endif
497};
498
499template <int K, typename Tup>
501void lanes_set (Lanes<Tup>& acc, Tup const& v)
502{
503 for (int lane = 0; lane < nacc; ++lane) { acc.template get<K>(lane) = amrex::get<K>(v); }
504 if constexpr (K+1 < GpuTupleSize<Tup>::value) { lanes_set<K+1>(acc, v); }
505}
506
507template <int K, typename Tup, typename P, typename... Ps>
509void lanes_update (Lanes<Tup>& acc, int lane, Tup const& s)
510{
511 P().local_update(acc.template get<K>(lane), amrex::get<K>(s));
512 if constexpr (sizeof...(Ps) > 0) { lanes_update<K+1,Tup,Ps...>(acc, lane, s); }
513}
514
515template <int K, typename Tup, typename P, typename... Ps>
517void lanes_fold (Lanes<Tup> const& acc, Tup& r)
518{
519 for (int lane = 0; lane < nacc; ++lane) { P().local_update(amrex::get<K>(r), acc.template get<K>(lane)); }
520 if constexpr (sizeof...(Ps) > 0) { lanes_fold<K+1,Tup,Ps...>(acc, r); }
521}
522
523// One row of a CPU reduction over the ops Ps: full blocks of nacc cells go
524// into the lanes, the rest into rem. value(i) is the tuple for cell i.
525template <typename... Ps, typename Tup, std::integral N, typename F>
527void reduce_row (N ilo, N ihi, Lanes<Tup>& acc, Tup& rem, F const& value)
528{
529 N i = ilo;
530 AMREX_PRAGMA_NACC_NOVECTOR
531 for (; i + (nacc-1) <= ihi; i += nacc) {
532 if constexpr (vectorize_nacc_loop<Ps...>) {
533 AMREX_PRAGMA_NACC_VECTORIZE
534 for (int lane = 0; lane < nacc; ++lane) { lanes_update<0,Tup,Ps...>(acc, lane, value(static_cast<N>(i+lane))); }
535 } else {
536 AMREX_PRAGMA_NACC_SCALAR
537 for (int lane = 0; lane < nacc; ++lane) { lanes_update<0,Tup,Ps...>(acc, lane, value(static_cast<N>(i+lane))); }
538 }
539 }
540 AMREX_PRAGMA_NACC_NOVECTOR
541 for (; i <= ihi; ++i) { for_each_local<0,Tup,Ps...>(rem, value(i)); }
542}
543}
544#undef AMREX_PRAGMA_NACC_VECTORIZE
545#undef AMREX_PRAGMA_NACC_NOVECTOR
546#undef AMREX_PRAGMA_NACC_SCALAR
548
549template <typename... Ps> class ReduceOps;
550
551#ifdef AMREX_USE_GPU
552
554template <typename... Ts>
556{
557public:
558 using Type = GpuTuple<Ts...>;
559
560 template <typename... Ps>
561 explicit ReduceData (ReduceOps<Ps...>& reduce_op)
562 : m_max_blocks(Gpu::Device::maxBlocksPerLaunch()),
563 m_host_tuple((Type*)(The_Pinned_Arena()->alloc(sizeof(Type)))),
564 m_device_tuple((Type*)(The_Arena()->alloc((AMREX_GPU_MAX_STREAMS)
565 * m_max_blocks * sizeof(Type)))),
566 m_fn_value([&reduce_op,this] () -> Type { return this->value(reduce_op); })
567 {
568 reduce_op.resetResultReadiness();
569 static_assert(std::is_trivially_copyable<Type>(),
570 "ReduceData::Type must be trivially copyable");
571 static_assert(std::is_trivially_destructible<Type>(),
572 "ReduceData::Type must be trivially destructible");
573
574 new (m_host_tuple) Type();
575 m_nblocks.fill(0);
576 }
577
580 !m_used_external_stream || m_value_called,
581 "ReduceData used on an external GPU stream must call value() before destruction.");
582 The_Pinned_Arena()->free(m_host_tuple);
583 The_Arena()->free(m_device_tuple);
584 }
585
586 ReduceData (ReduceData<Ts...> const&) = delete;
588 void operator= (ReduceData<Ts...> const&) = delete;
589 void operator= (ReduceData<Ts...> &&) = delete;
590
592 {
593 Type r = m_fn_value();
594 m_value_called = true;
595 return r;
596 }
597
598 template <typename... Ps>
600 {
601 Type r = reduce_op.value(*this);
602 m_value_called = true;
603 return r;
604 }
605
606 Type* devicePtr () { return m_device_tuple; }
608 return m_device_tuple+streamIndexChecked(s)*m_max_blocks;
609 }
610
611 Type* hostPtr () { return m_host_tuple; }
612
614 int& nBlocks (gpuStream_t const& s) { return m_nblocks[streamIndexChecked(s)]; }
615
616 int maxBlocks () const { return m_max_blocks; }
617
618 int maxStreamIndex () const { return m_max_stream_index; }
620 m_max_stream_index = std::max(m_max_stream_index,streamIndexChecked(s));
621 }
622
623 void markValueCalled () noexcept { m_value_called = true; }
624
625private:
626 int streamIndexChecked (gpuStream_t const& s)
627 {
628 int const idx = Gpu::Device::streamIndex(s);
629 m_used_external_stream = m_used_external_stream || Gpu::Device::usingExternalStream();
630 if (idx == 0) {
631 if (m_stream_index_zero_set) {
632 AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_stream_index_zero == s,
633 "ReduceData cannot be reused across different external GPU streams "
634 "or between an external GPU stream and AMReX stream 0.");
635 } else {
636 m_stream_index_zero = s;
637 m_stream_index_zero_set = true;
638 }
639 }
640 return idx;
641 }
642
643 int m_max_blocks;
644 int m_max_stream_index = 0;
645 Type* m_host_tuple = nullptr;
646 Type* m_device_tuple = nullptr;
648 gpuStream_t m_stream_index_zero{};
649 bool m_stream_index_zero_set = false;
650 bool m_used_external_stream = false;
651 bool m_value_called = false;
652 std::function<Type()> m_fn_value;
653};
654
656namespace Reduce::detail {
657
658 // call_f_intvect_box
659
660 template <typename F, int dim>
662 auto call_f_intvect_box (F const& f, IntVectND<dim> iv, IndexTypeND<dim>) noexcept ->
663 decltype(amrex::detail::call_f_intvect_inner(std::make_index_sequence<dim>(), f, iv))
664 {
665 return amrex::detail::call_f_intvect_inner(std::make_index_sequence<dim>(), f, iv);
666 }
667
668 template <typename F, int dim>
670 auto call_f_intvect_box (F const& f, IntVectND<dim> iv, IndexTypeND<dim> t) noexcept ->
671 decltype(f(BoxND<dim>(iv, iv, t)))
672 {
673 return f(BoxND<dim>(iv, iv, t));
674 }
675
676 // call_f_intvect_n
677 template <typename F, typename T, int dim>
679 auto call_f_intvect_n (F const& f, IntVectND<dim> iv, T n) noexcept ->
680 decltype(amrex::detail::call_f_intvect_inner(std::make_index_sequence<dim>(), f, iv, n))
681 {
682 return amrex::detail::call_f_intvect_inner(std::make_index_sequence<dim>(), f, iv, n);
683 }
684
685 // mf_call_f
686
687 struct iterate_box {};
688 struct iterate_box_comp {};
689
690 template <typename Tag, typename F, typename T, typename... Ps>
691 requires (std::same_as<iterate_box,Tag>)
693 void mf_call_f (F const& f, int ibox, int i, int j, int k, int, T& r) noexcept
694 {
695 auto const& pr = f(ibox,i,j,k);
696 Reduce::detail::for_each_local<0, T, Ps...>(r, pr);
697 }
698
699 template <typename Tag, typename F, typename T, typename... Ps>
700 requires (std::same_as<iterate_box_comp,Tag>)
702 void mf_call_f (F const& f, int ibox, int i, int j, int k, int ncomp, T& r) noexcept
703 {
704 for (int n = 0; n < ncomp; ++n) {
705 auto const& pr = f(ibox,i,j,k,n);
706 Reduce::detail::for_each_local<0, T, Ps...>(r, pr);
707 }
708 }
709}
711
713template <typename... Ps>
715{
716public:
717
719
720 // This is public for CUDA
721 template <typename Tag, typename MF, typename D, typename F>
722 void eval_mf (Tag, MF const& mf, IntVect const& nghost, int ncomp, D& reduce_data, F const& f)
723 {
724 using ReduceTuple = typename D::Type;
725 const int nboxes = mf.local_size();
726 if (nboxes > 0) {
727 auto const& parforinfo = mf.getParForInfo(nghost);
728 auto nblocks_per_box = parforinfo.getNBlocksPerBox(AMREX_GPU_MAX_THREADS);
729 AMREX_ASSERT(Long(nblocks_per_box)*Long(nboxes) < Long(std::numeric_limits<int>::max()));
730 const int nblocks = nblocks_per_box * nboxes;
731 const BoxIndexer* dp_boxes = parforinfo.getBoxes();
732 const Math::FastDivmodU32 fdm_box(nblocks_per_box);
733
734 auto const& stream = Gpu::gpuStream();
735 auto pdst = reduce_data.devicePtr(stream);
736 int nblocks_ec = std::min(nblocks, reduce_data.maxBlocks());
737 AMREX_ASSERT(Long(nblocks_ec)*2 <= Long(std::numeric_limits<int>::max()));
738 int& nblocks_ref = reduce_data.nBlocks(stream);
739 auto old_nblocks = static_cast<unsigned int>(nblocks_ref);
740 nblocks_ref = amrex::max(nblocks_ref, nblocks_ec);
741 reduce_data.updateMaxStreamIndex(stream);
742
743#ifdef AMREX_USE_SYCL
744 // device reduce needs local(i.e., shared) memory
745 constexpr std::size_t shared_mem_bytes = sizeof(ReduceTuple)
746 * std::max(Gpu::Device::warp_size, AMREX_GPU_MAX_THREADS/Gpu::Device::warp_size);
747 amrex::launch<AMREX_GPU_MAX_THREADS>(nblocks_ec, shared_mem_bytes, stream,
748 [=] AMREX_GPU_DEVICE (Gpu::Handler const& gh) noexcept
749 {
750 Dim1 blockIdx {gh.blockIdx()};
751 Dim1 threadIdx{gh.threadIdx()};
752#else
754 <<<nblocks_ec, AMREX_GPU_MAX_THREADS, 0, stream>>>
755 ([=] AMREX_GPU_DEVICE () noexcept
756 {
757#endif
758 ReduceTuple r;
759 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(r);
760 ReduceTuple& dst = pdst[blockIdx.x];
761 if (threadIdx.x == 0 && blockIdx.x >= old_nblocks) {
762 dst = r;
763 }
764 for (int iblock = blockIdx.x; iblock < nblocks; iblock += nblocks_ec) {
765 std::uint32_t ibox, ib;
766 fdm_box(ibox, ib, std::uint32_t(iblock));
767 auto icell = BoxIndexer::UInt(ib)*AMREX_GPU_MAX_THREADS + BoxIndexer::UInt(threadIdx.x);
768
769 BoxIndexer const& indexer = dp_boxes[ibox];
770 if (icell < indexer.numPts()) {
771 auto [i, j, k] = indexer(icell);
772 Reduce::detail::mf_call_f<Tag, F, ReduceTuple, Ps...>
773 (f, int(ibox), i, j, k, ncomp, r);
774 }
775 }
776#ifdef AMREX_USE_SYCL
777 Reduce::detail::for_each_parallel<0, ReduceTuple, Ps...>(dst, r, gh);
778#else
779 Reduce::detail::for_each_parallel<0, ReduceTuple, Ps...>(dst, r);
780#endif
781 });
782 }
783 }
784
785 // This is public for CUDA
786 template <typename Tag, int dim, typename D, typename F>
787 void eval_box (Tag, BoxND<dim> const& box, int ncomp, D& reduce_data, F const& f)
788 {
789 if (box.isEmpty()) { return; }
790 using ReduceTuple = typename D::Type;
791 auto const& stream = Gpu::gpuStream();
792 auto dp = reduce_data.devicePtr(stream);
793 int& nblocks = reduce_data.nBlocks(stream);
794 const BoxIndexerND<dim> indexer(box);
795 using UInt = typename BoxIndexerND<dim>::UInt;
796 IndexTypeND<dim> ixtype = box.ixType();
797 constexpr int nitems_per_thread = 4;
798 Long nblocks_ec = (box.numPts() + nitems_per_thread*AMREX_GPU_MAX_THREADS-1)
799 / (nitems_per_thread*AMREX_GPU_MAX_THREADS);
800 nblocks_ec = std::min<Long>(nblocks_ec, reduce_data.maxBlocks());
801 reduce_data.updateMaxStreamIndex(stream);
802#ifdef AMREX_USE_SYCL
803 // device reduce needs local(i.e., shared) memory
804 constexpr std::size_t shared_mem_bytes = sizeof(ReduceTuple)
805 * std::max(Gpu::Device::warp_size, AMREX_GPU_MAX_THREADS/Gpu::Device::warp_size);
806 amrex::launch<AMREX_GPU_MAX_THREADS>(nblocks_ec, shared_mem_bytes, stream,
807 [=] AMREX_GPU_DEVICE (Gpu::Handler const& gh) noexcept
808 {
809 Dim1 blockIdx {gh.blockIdx()};
810 Dim1 threadIdx{gh.threadIdx()};
811 Dim1 gridDim {gh.gridDim()};
812#else
813 amrex::launch<AMREX_GPU_MAX_THREADS>(nblocks_ec, 0, stream,
814 [=] AMREX_GPU_DEVICE () noexcept
815 {
816#endif
817 ReduceTuple r;
818 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(r);
819 ReduceTuple& dst = *(dp+blockIdx.x);
820 if (threadIdx.x == 0 && static_cast<int>(blockIdx.x) >= nblocks) {
821 dst = r;
822 }
823 for (UInt icell = UInt(AMREX_GPU_MAX_THREADS)*UInt(blockIdx.x)+UInt(threadIdx.x),
824 stride = UInt(AMREX_GPU_MAX_THREADS)*UInt(gridDim.x);
825 icell < indexer.numPts();
826 icell += stride)
827 {
828 auto iv = indexer.intVect(icell);
829 amrex::ignore_unused(f,ncomp,ixtype); // work around first-capture
830 if constexpr (std::is_same_v<Reduce::detail::iterate_box,Tag>) {
831 auto pr = Reduce::detail::call_f_intvect_box(f, iv, ixtype);
832 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(r, pr);
833 } else {
834 for (int n = 0; n < ncomp; ++n) {
835 auto pr = Reduce::detail::call_f_intvect_n(f, iv, n);
836 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(r, pr);
837 }
838 }
839 }
840#ifdef AMREX_USE_SYCL
841 Reduce::detail::for_each_parallel<0, ReduceTuple, Ps...>(dst, r, gh);
842#else
843 Reduce::detail::for_each_parallel<0, ReduceTuple, Ps...>(dst, r);
844#endif
845 });
846 nblocks = std::max(nblocks, static_cast<int>(nblocks_ec));
847 }
848
850
851 template <FabArrayType MF, typename D, typename F>
852#ifndef AMREX_USE_CUDA
854#endif
855 void eval (MF const& mf, IntVect const& nghost, D& reduce_data, F&& f)
856 {
857 using ReduceTuple = typename D::Type;
858 const int nboxes = mf.local_size();
859 if (nboxes == 0) {
860 return;
861 } else if (!mf.isFusingCandidate()) {
862 for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
863 Box const& b = amrex::grow(mfi.validbox(), nghost);
864 const int li = mfi.LocalIndex();
865 this->eval(b, reduce_data,
866 [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept -> ReduceTuple
867 {
868 return f(li, i, j, k);
869 });
870 }
871 } else {
872 eval_mf(Reduce::detail::iterate_box{},
873 mf, nghost, 0, reduce_data, std::forward<F>(f));
874 }
875 }
876
877 template <FabArrayType MF, typename D, typename F>
878#ifndef AMREX_USE_CUDA
880#endif
881 void eval (MF const& mf, IntVect const& nghost, int ncomp, D& reduce_data, F&& f)
882 {
883 using ReduceTuple = typename D::Type;
884
885 const int nboxes = mf.local_size();
886
887 if (nboxes == 0) {
888 return;
889 } else if (!mf.isFusingCandidate()) {
890 for (MFIter mfi(mf); mfi.isValid(); ++mfi) {
891 Box const& b = amrex::grow(mfi.validbox(), nghost);
892 const int li = mfi.LocalIndex();
893 this->eval(b, ncomp, reduce_data,
894 [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) noexcept -> ReduceTuple
895 {
896 return f(li, i, j, k, n);
897 });
898 }
899 } else if (ncomp == 1) {
900 // Without the loop over components, the kernel reads memory faster.
901 eval_mf(Reduce::detail::iterate_box{}, mf, nghost, 0, reduce_data,
902 [=] AMREX_GPU_DEVICE (int box_no, int i, int j, int k) noexcept -> ReduceTuple
903 {
904 return f(box_no, i, j, k, 0);
905 });
906 } else {
907 eval_mf(Reduce::detail::iterate_box_comp{},
908 mf, nghost, ncomp, reduce_data, std::forward<F>(f));
909 }
910 }
911
912 template <typename D, typename F, int dim>
913 void eval (BoxND<dim> const& box, D & reduce_data, F const& f)
914 {
915 eval_box(Reduce::detail::iterate_box{}, box, 0, reduce_data, f);
916 }
917
918 template <std::integral N, typename D, typename F, int dim>
919 void eval (BoxND<dim> const& box, N ncomp, D & reduce_data, F const& f)
920 {
921 if (ncomp == 1) {
922 // Without the loop over components, the kernel reads memory faster.
923 eval_box(Reduce::detail::iterate_box{}, box, 0, reduce_data,
924 [=] AMREX_GPU_DEVICE (IntVectND<dim> const& iv) noexcept
925 {
926 return Reduce::detail::call_f_intvect_n(f, iv, 0);
927 });
928 } else {
929 eval_box(Reduce::detail::iterate_box_comp{}, box, ncomp, reduce_data, f);
930 }
931 }
932
933 template <std::integral N, typename D, typename F>
934 void eval (N n, D & reduce_data, F const& f)
935 {
936 if (n <= 0) { return; }
937 using ReduceTuple = typename D::Type;
938 auto const& stream = Gpu::gpuStream();
939 auto dp = reduce_data.devicePtr(stream);
940 int& nblocks = reduce_data.nBlocks(stream);
941 constexpr int nitems_per_thread = 4;
942 Long nblocks_ec_long = (Long(n) + nitems_per_thread*AMREX_GPU_MAX_THREADS-1)
943 / (nitems_per_thread*AMREX_GPU_MAX_THREADS);
944 int nblocks_ec = static_cast<int>(std::min<Long>(nblocks_ec_long,
945 reduce_data.maxBlocks()));
946 reduce_data.updateMaxStreamIndex(stream);
947#ifdef AMREX_USE_SYCL
948 // device reduce needs local(i.e., shared) memory
949 constexpr std::size_t shared_mem_bytes = sizeof(ReduceTuple)
950 * std::max(Gpu::Device::warp_size, AMREX_GPU_MAX_THREADS/Gpu::Device::warp_size);
951 amrex::launch<AMREX_GPU_MAX_THREADS>(nblocks_ec, shared_mem_bytes, stream,
952 [=] AMREX_GPU_DEVICE (Gpu::Handler const& gh) noexcept
953 {
954 Dim1 blockIdx {gh.blockIdx()};
955 Dim1 threadIdx{gh.threadIdx()};
956 Dim1 gridDim {gh.gridDim()};
957#else
958 amrex::launch<AMREX_GPU_MAX_THREADS>(nblocks_ec, 0, stream,
959 [=] AMREX_GPU_DEVICE () noexcept
960 {
961#endif
962 ReduceTuple r;
963 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(r);
964 ReduceTuple& dst = *(dp+blockIdx.x);
965 if (threadIdx.x == 0 && static_cast<int>(blockIdx.x) >= nblocks) {
966 dst = r;
967 }
968 for (Long i = Long(AMREX_GPU_MAX_THREADS)*blockIdx.x+threadIdx.x,
969 stride = Long(AMREX_GPU_MAX_THREADS)*gridDim.x;
970 i < Long(n);
971 i += stride)
972 {
973 auto pr = f(N(i));
974 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(r,pr);
975 }
976#ifdef AMREX_USE_SYCL
977 Reduce::detail::for_each_parallel<0, ReduceTuple, Ps...>(dst, r, gh);
978#else
979 Reduce::detail::for_each_parallel<0, ReduceTuple, Ps...>(dst, r);
980#endif
981 });
982 nblocks = amrex::max(nblocks, nblocks_ec);
983 }
984
985 template <typename D>
986 typename D::Type value (D & reduce_data)
987 {
988 auto hp = reduce_data.hostPtr();
989
990 if (m_result_is_ready) {
991 reduce_data.markValueCalled();
992 return *hp;
993 }
994
995 using ReduceTuple = typename D::Type;
996 auto const& stream = Gpu::gpuStream();
997 auto dp = reduce_data.devicePtr();
998 auto const& nblocks = reduce_data.nBlocks();
999#if defined(AMREX_USE_SYCL)
1000 if (reduce_data.maxStreamIndex() == 0 && nblocks[0] <= 4096) {
1001 const int N = nblocks[0];
1002 if (N == 0) {
1003 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(*hp);
1004 } else {
1006 Gpu::dtoh_memcpy_async(tmp.data(), dp, sizeof(ReduceTuple)*N);
1008 for (int i = 1; i < N; ++i) {
1009 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(tmp[0], tmp[i]);
1010 }
1011 *hp = tmp[0];
1012 }
1013 } else
1014#endif
1015 {
1016 int maxblocks = reduce_data.maxBlocks();
1017#ifdef AMREX_USE_SYCL
1018 // device reduce needs local(i.e., shared) memory
1019 constexpr std::size_t shared_mem_bytes = sizeof(ReduceTuple)
1020 * std::max(Gpu::Device::warp_size, AMREX_GPU_MAX_THREADS/Gpu::Device::warp_size);
1021#ifndef AMREX_NO_SYCL_REDUCE_WORKAROUND
1022 // xxxxx SYCL todo: reduce bug workaround
1024 auto presult = dtmp.data();
1025#else
1026 auto presult = hp;
1027#endif
1028 amrex::launch<AMREX_GPU_MAX_THREADS>(1, shared_mem_bytes, stream,
1029 [=] AMREX_GPU_DEVICE (Gpu::Handler const& gh) noexcept
1030 {
1031 ReduceTuple r;
1032 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(r);
1033 ReduceTuple dst = r;
1034 for (int istream = 0, nstreams = nblocks.size(); istream < nstreams; ++istream) {
1035 auto dp_stream = dp+istream*maxblocks;
1036 for (int i = gh.item->get_global_id(0), stride = gh.item->get_global_range(0);
1037 i < nblocks[istream]; i += stride) {
1038 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(r, dp_stream[i]);
1039 }
1040 }
1041 Reduce::detail::for_each_parallel<0, ReduceTuple, Ps...>(dst, r, gh);
1042 if (gh.threadIdx() == 0) { *presult = dst; }
1043 });
1044#ifndef AMREX_NO_SYCL_REDUCE_WORKAROUND
1045 Gpu::dtoh_memcpy_async(hp, dtmp.data(), sizeof(ReduceTuple));
1046#endif
1047#else
1049 [=] AMREX_GPU_DEVICE () noexcept
1050 {
1051 ReduceTuple r;
1052 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(r);
1053 ReduceTuple dst = r;
1054 for (int istream = 0, nstreams = nblocks.size(); istream < nstreams; ++istream) {
1055 auto dp_stream = dp+istream*maxblocks;
1056 for (int i = AMREX_GPU_MAX_THREADS*blockIdx.x+threadIdx.x, stride = AMREX_GPU_MAX_THREADS*gridDim.x;
1057 i < nblocks[istream]; i += stride) {
1058 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(r, dp_stream[i]);
1059 }
1060 }
1061 Reduce::detail::for_each_parallel<0, ReduceTuple, Ps...>(dst, r);
1062 if (threadIdx.x == 0) { *hp = dst; }
1063 });
1064#endif
1066 }
1067
1068 m_result_is_ready = true;
1069 reduce_data.markValueCalled();
1070 return *hp;
1071 }
1072
1073private:
1074 template <typename... T> friend class ReduceData;
1075 bool m_result_is_ready = false;
1076 void resetResultReadiness () { m_result_is_ready = false; }
1077};
1078
1079namespace Reduce {
1080
1081template <typename T, std::integral N>
1082T Sum (N n, T const* v, T init_val)
1083{
1084 ReduceOps<ReduceOpSum> reduce_op;
1085 ReduceData<T> reduce_data(reduce_op);
1086 using ReduceTuple = typename decltype(reduce_data)::Type;
1087 reduce_op.eval(n, reduce_data, [=] AMREX_GPU_DEVICE (N i) -> ReduceTuple { return {v[i]}; });
1088 ReduceTuple hv = reduce_data.value(reduce_op);
1089 return amrex::get<0>(hv) + init_val;
1090}
1091
1092template <typename T, std::integral N, typename F>
1093requires (!std::same_as<T*,std::decay_t<F>>)
1094T Sum (N n, F const& f, T init_val)
1095{
1096 ReduceOps<ReduceOpSum> reduce_op;
1097 ReduceData<T> reduce_data(reduce_op);
1098 using ReduceTuple = typename decltype(reduce_data)::Type;
1099 reduce_op.eval(n, reduce_data, [=] AMREX_GPU_DEVICE (N i) -> ReduceTuple { return {f(i)}; });
1100 ReduceTuple hv = reduce_data.value(reduce_op);
1101 return amrex::get<0>(hv) + init_val;
1102}
1103
1104template <typename T, std::integral N>
1105T Min (N n, T const* v, T init_val)
1106{
1107 ReduceOps<ReduceOpMin> reduce_op;
1108 ReduceData<T> reduce_data(reduce_op);
1109 using ReduceTuple = typename decltype(reduce_data)::Type;
1110 reduce_op.eval(n, reduce_data, [=] AMREX_GPU_DEVICE (N i) -> ReduceTuple { return {v[i]}; });
1111 ReduceTuple hv = reduce_data.value(reduce_op);
1112 return std::min(amrex::get<0>(hv),init_val);
1113}
1114
1115template <typename T, std::integral N, typename F>
1116requires (!std::same_as<T*,std::decay_t<F>>)
1117T Min (N n, F const& f, T init_val)
1118{
1119 ReduceOps<ReduceOpMin> reduce_op;
1120 ReduceData<T> reduce_data(reduce_op);
1121 using ReduceTuple = typename decltype(reduce_data)::Type;
1122 reduce_op.eval(n, reduce_data, [=] AMREX_GPU_DEVICE (N i) -> ReduceTuple { return {f(i)}; });
1123 ReduceTuple hv = reduce_data.value(reduce_op);
1124 return std::min(amrex::get<0>(hv),init_val);
1125}
1126
1127template <typename T, std::integral N>
1128T Max (N n, T const* v, T init_val)
1129{
1130 ReduceOps<ReduceOpMax> reduce_op;
1131 ReduceData<T> reduce_data(reduce_op);
1132 using ReduceTuple = typename decltype(reduce_data)::Type;
1133 reduce_op.eval(n, reduce_data, [=] AMREX_GPU_DEVICE (N i) -> ReduceTuple { return {v[i]}; });
1134 ReduceTuple hv = reduce_data.value(reduce_op);
1135 return std::max(amrex::get<0>(hv),init_val);
1136}
1137
1138template <typename T, std::integral N, typename F>
1139requires (!std::same_as<T*,std::decay_t<F>>)
1140T Max (N n, F const& f, T init_val)
1141{
1142 ReduceOps<ReduceOpMax> reduce_op;
1143 ReduceData<T> reduce_data(reduce_op);
1144 using ReduceTuple = typename decltype(reduce_data)::Type;
1145 reduce_op.eval(n, reduce_data, [=] AMREX_GPU_DEVICE (N i) -> ReduceTuple { return {f(i)}; });
1146 ReduceTuple hv = reduce_data.value(reduce_op);
1147 return std::max(amrex::get<0>(hv),init_val);
1148}
1149
1150template <typename T, std::integral N>
1151std::pair<T,T> MinMax (N n, T const* v)
1152{
1154 ReduceData<T,T> reduce_data(reduce_op);
1155 using ReduceTuple = typename decltype(reduce_data)::Type;
1156 reduce_op.eval(n, reduce_data, [=] AMREX_GPU_DEVICE (N i) -> ReduceTuple {
1157 return {v[i],v[i]};
1158 });
1159 auto hv = reduce_data.value(reduce_op);
1160 return std::make_pair(amrex::get<0>(hv), amrex::get<1>(hv));
1161}
1162
1163template <typename T, std::integral N, typename F>
1164requires (!std::same_as<T*,std::decay_t<F>>)
1165std::pair<T,T> MinMax (N n, F const& f)
1166{
1168 ReduceData<T,T> reduce_data(reduce_op);
1169 using ReduceTuple = typename decltype(reduce_data)::Type;
1170 reduce_op.eval(n, reduce_data, [=] AMREX_GPU_DEVICE (N i) -> ReduceTuple {
1171 T tmp = f(i);
1172 return {tmp,tmp};
1173 });
1174 auto hv = reduce_data.value(reduce_op);
1175 return std::make_pair(amrex::get<0>(hv), amrex::get<1>(hv));
1176}
1177
1178template <typename T, std::integral N, typename P>
1179bool AnyOf (N n, T const* v, P const& pred)
1180{
1181 Gpu::LaunchSafeGuard lsg(true);
1183 int* dp = ds.dataPtr();
1184 auto ec = Gpu::ExecutionConfig(n);
1185 ec.numBlocks.x = std::min(ec.numBlocks.x, Gpu::Device::maxBlocksPerLaunch());
1186
1187#ifdef AMREX_USE_SYCL
1188 const int num_ints = std::max(Gpu::Device::warp_size, int(ec.numThreads.x)/Gpu::Device::warp_size) + 1;
1189 const std::size_t shared_mem_bytes = num_ints*sizeof(int);
1190 amrex::launch<AMREX_GPU_MAX_THREADS>(ec.numBlocks.x, shared_mem_bytes, Gpu::gpuStream(),
1191 [=] AMREX_GPU_DEVICE (Gpu::Handler const& gh) noexcept {
1192 int* has_any = &(static_cast<int*>(gh.sharedMemory())[num_ints-1]);
1193 if (gh.threadIdx() == 0) { *has_any = *dp; }
1194 gh.syncThreads();
1195
1196 if (!(*has_any))
1197 {
1198 int r = false;
1199 for (Long i = Long(AMREX_GPU_MAX_THREADS)*gh.blockIdx()+gh.threadIdx(),
1200 stride = Long(AMREX_GPU_MAX_THREADS)*gh.gridDim();
1201 i < Long(n) && !r; i += stride)
1202 {
1203 r = pred(v[i]) ? 1 : 0;
1204 }
1205
1208 if (gh.threadIdx() == 0 && r) { *dp = 1; }
1209 }
1210 });
1211#else
1213 [=] AMREX_GPU_DEVICE () noexcept {
1214 __shared__ int has_any;
1215 if (threadIdx.x == 0) { has_any = *dp; }
1216 __syncthreads();
1217
1218 if (!has_any)
1219 {
1220 int r = false;
1221 for (Long i = Long(AMREX_GPU_MAX_THREADS)*blockIdx.x+threadIdx.x,
1222 stride = Long(AMREX_GPU_MAX_THREADS)*gridDim.x;
1223 i < Long(n) && !r; i += stride)
1224 {
1225 r = pred(v[i]) ? 1 : 0;
1226 }
1229 if (threadIdx.x == 0 && r) *dp = 1;
1230 }
1231 });
1232#endif
1233 return ds.dataValue();
1234}
1235
1236template <typename P, int dim>
1237bool AnyOf (BoxND<dim> const& box, P const& pred)
1238{
1239 Gpu::LaunchSafeGuard lsg(true);
1241 int* dp = ds.dataPtr();
1242 const BoxIndexerND<dim> indexer(box);
1243 using UInt = typename BoxIndexerND<dim>::UInt;
1244 auto ec = Gpu::ExecutionConfig(box.numPts());
1245 ec.numBlocks.x = std::min(ec.numBlocks.x, Gpu::Device::maxBlocksPerLaunch());
1246
1247#ifdef AMREX_USE_SYCL
1248 const int num_ints = std::max(Gpu::Device::warp_size, int(ec.numThreads.x)/Gpu::Device::warp_size) + 1;
1249 const std::size_t shared_mem_bytes = num_ints*sizeof(int);
1250 amrex::launch<AMREX_GPU_MAX_THREADS>(ec.numBlocks.x, shared_mem_bytes, Gpu::gpuStream(),
1251 [=] AMREX_GPU_DEVICE (Gpu::Handler const& gh) noexcept {
1252 int* has_any = &(static_cast<int*>(gh.sharedMemory())[num_ints-1]);
1253 if (gh.threadIdx() == 0) { *has_any = *dp; }
1254 gh.syncThreads();
1255
1256 if (!(*has_any))
1257 {
1258 int r = false;
1259 for (UInt icell = UInt(AMREX_GPU_MAX_THREADS)*UInt(gh.blockIdx())+UInt(gh.threadIdx()),
1260 stride = UInt(AMREX_GPU_MAX_THREADS)*UInt(gh.gridDim());
1261 icell < indexer.numPts() && !r;
1262 icell += stride)
1263 {
1264 auto iv = indexer.intVect(icell);
1265 r = amrex::detail::call_f_intvect(pred, iv) ? 1 : 0;
1266 }
1269 if (gh.threadIdx() == 0 && r) { *dp = 1; }
1270 }
1271 });
1272#else
1273 AMREX_LAUNCH_KERNEL(AMREX_GPU_MAX_THREADS, ec.numBlocks, ec.numThreads, 0,
1275 [=] AMREX_GPU_DEVICE () noexcept {
1276 __shared__ int has_any;
1277 if (threadIdx.x == 0) { has_any = *dp; }
1278 __syncthreads();
1279
1280 if (!has_any)
1281 {
1282 int r = false;
1283 for (UInt icell = UInt(AMREX_GPU_MAX_THREADS)*UInt(blockIdx.x)+UInt(threadIdx.x),
1284 stride = UInt(AMREX_GPU_MAX_THREADS)*UInt(gridDim.x);
1285 icell < indexer.numPts() && !r;
1286 icell += stride)
1287 {
1288 auto iv = indexer.intVect(icell);
1289 r = amrex::detail::call_f_intvect(pred, iv) ? 1 : 0;
1290 }
1293 if (threadIdx.x == 0 && r) *dp = 1;
1294 }
1295 });
1296#endif
1297 return ds.dataValue();
1298}
1299
1300}
1301
1302#else
1303
1304template <typename... Ts>
1305class ReduceData
1306{
1307public:
1308 using Type = GpuTuple<Ts...>;
1309
1310 template <typename... Ps>
1311 explicit ReduceData (ReduceOps<Ps...>& reduce_op)
1312 : m_tuple(OpenMP::in_parallel() ? 1 : OpenMP::get_max_threads()),
1313 m_fn_value([&reduce_op,this] () -> Type { return this->value(reduce_op); })
1314 {
1315 reduce_op.resetResultReadiness();
1316 for (auto& t : m_tuple) {
1317 Reduce::detail::for_each_init<0, Type, Ps...>(t);
1318 }
1319 }
1320
1321 ~ReduceData () = default;
1322 ReduceData (ReduceData<Ts...> const&) = delete;
1323 ReduceData (ReduceData<Ts...> &&) = delete;
1324 void operator= (ReduceData<Ts...> const&) = delete;
1325 void operator= (ReduceData<Ts...> &&) = delete;
1326
1327 Type value () { return m_fn_value(); }
1328
1329 template <typename... Ps>
1330 Type value (ReduceOps<Ps...>& reduce_op)
1331 {
1332 return reduce_op.value(*this);
1333 }
1334
1335 Vector<Type>& reference () { return m_tuple; }
1336
1337 Type& reference (int tid)
1338 {
1339 if (m_tuple.size() == 1) {
1340 // No OpenMP or already inside OpenMP parallel when reduce_data is constructed
1341 return m_tuple[0];
1342 } else {
1343 return m_tuple[tid];
1344 }
1345 }
1346
1347private:
1348 Vector<Type> m_tuple;
1349 std::function<Type()> m_fn_value;
1350};
1351
1352namespace Reduce::detail {
1353
1354 // call_f_intvect
1355
1356 template <typename F, int dim>
1358 auto call_f_intvect (F const& f, IntVectND<dim> iv) noexcept ->
1359 decltype(amrex::detail::call_f_intvect_inner(std::make_index_sequence<dim>(), f, iv))
1360 {
1361 return amrex::detail::call_f_intvect_inner(std::make_index_sequence<dim>(), f, iv);
1362 }
1363
1364 // call_f_intvect_n
1365
1366 template <typename F, typename T, int dim>
1368 auto call_f_intvect_n (F const& f, IntVectND<dim> iv, T n) noexcept ->
1369 decltype(amrex::detail::call_f_intvect_inner(std::make_index_sequence<dim>(), f, iv, n))
1370 {
1371 return amrex::detail::call_f_intvect_inner(std::make_index_sequence<dim>(), f, iv, n);
1372 }
1373}
1374
1375template <typename... Ps>
1376class ReduceOps
1377{
1378private:
1379
1380 // call_f_box
1381
1382 template <typename D, typename F, int dim>
1383 requires (std::same_as<std::decay_t<decltype(
1384 Reduce::detail::call_f_intvect(std::declval<F const&>(), IntVectND<dim>()))>,
1385 typename D::Type>)
1387 static void call_f_box (BoxND<dim> const& box, typename D::Type & r, F const& f) noexcept
1388 {
1389 using ReduceTuple = typename D::Type;
1390 ReduceTuple rem;
1391 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(rem);
1392 Reduce::detail::Lanes<ReduceTuple> acc;
1393 Reduce::detail::lanes_set<0>(acc, rem);
1394 int const ilo = box.smallEnd(0);
1395 int const ihi = box.bigEnd(0);
1396 BoxND<dim> rows = box;
1397 rows.setBig(0, ilo);
1398 For(rows, [&] (IntVectND<dim> iv) {
1399 Reduce::detail::reduce_row<Ps...>(ilo, ihi, acc, rem, [&] (int i) {
1400 iv[0] = i;
1401 return Reduce::detail::call_f_intvect(f, iv);
1402 });
1403 });
1404 Reduce::detail::lanes_fold<0, ReduceTuple, Ps...>(acc, r);
1405 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(r, rem);
1406 }
1407
1408 template <typename D, typename F, int dim>
1409 requires (std::same_as<std::decay_t<decltype(
1410 std::declval<F const&>()(std::declval<BoxND<dim> const&>()))>,
1411 typename D::Type>)
1413 static void call_f_box (BoxND<dim> const& box, typename D::Type & r, F const& f) noexcept
1414 {
1415 using ReduceTuple = typename D::Type;
1416 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(r, f(box));
1417 }
1418
1419public:
1420
1421 template <FabArrayType MF, typename D, typename F>
1422 requires (IsCallable<F, int, int, int, int>::value)
1423 void eval (MF const& mf, IntVect const& nghost, D & reduce_data, F const& f)
1424 {
1425 using ReduceTuple = typename D::Type;
1426#ifdef AMREX_USE_OMP
1427#pragma omp parallel if (!system::regtest_reduction)
1428#endif
1429 {
1430 // The partial accumulators break the dependency chain so that
1431 // the i loop can vectorize. They live across the MFIter loop so
1432 // that they stay in vector registers.
1433 ReduceTuple rem;
1434 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(rem);
1435 Reduce::detail::Lanes<ReduceTuple> acc;
1436 Reduce::detail::lanes_set<0>(acc, rem);
1437 for (MFIter mfi(mf,true); mfi.isValid(); ++mfi) {
1438 Box const& b = mfi.growntilebox(nghost);
1439 const int li = mfi.LocalIndex();
1440 const auto lo = amrex::lbound(b);
1441 const auto hi = amrex::ubound(b);
1442 for (int k = lo.z; k <= hi.z; ++k) {
1443 for (int j = lo.y; j <= hi.y; ++j) {
1444 Reduce::detail::reduce_row<Ps...>(lo.x, hi.x, acc, rem,
1445 [&] (int i) { return f(li,i,j,k); });
1446 }}
1447 }
1448 auto& rr = reduce_data.reference(OpenMP::get_thread_num());
1449 Reduce::detail::lanes_fold<0, ReduceTuple, Ps...>(acc, rr);
1450 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(rr, rem);
1451 }
1452 }
1453
1454 template <FabArrayType MF, typename D, typename F>
1455 requires (IsCallable<F, int, int, int, int, int>::value)
1456 void eval (MF const& mf, IntVect const& nghost, int ncomp, D & reduce_data, F const& f)
1457 {
1458 using ReduceTuple = typename D::Type;
1459#ifdef AMREX_USE_OMP
1460#pragma omp parallel if (!system::regtest_reduction)
1461#endif
1462 {
1463 ReduceTuple rem;
1464 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(rem);
1465 Reduce::detail::Lanes<ReduceTuple> acc;
1466 Reduce::detail::lanes_set<0>(acc, rem);
1467 for (MFIter mfi(mf,true); mfi.isValid(); ++mfi) {
1468 Box const& b = mfi.growntilebox(nghost);
1469 const int li = mfi.LocalIndex();
1470 const auto lo = amrex::lbound(b);
1471 const auto hi = amrex::ubound(b);
1472 for (int n = 0; n < ncomp; ++n) {
1473 for (int k = lo.z; k <= hi.z; ++k) {
1474 for (int j = lo.y; j <= hi.y; ++j) {
1475 Reduce::detail::reduce_row<Ps...>(lo.x, hi.x, acc, rem,
1476 [&] (int i) { return f(li,i,j,k,n); });
1477 }}}
1478 }
1479 auto& rr = reduce_data.reference(OpenMP::get_thread_num());
1480 Reduce::detail::lanes_fold<0, ReduceTuple, Ps...>(acc, rr);
1481 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(rr, rem);
1482 }
1483 }
1484
1485 template <typename D, typename F, int dim>
1486 void eval (BoxND<dim> const& box, D & reduce_data, F&& f)
1487 {
1488 using ReduceTuple = typename D::Type;
1489 ReduceTuple rr;
1490 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(rr);
1491 call_f_box<D>(box, rr, std::forward<F>(f));
1492 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(
1493 reduce_data.reference(OpenMP::get_thread_num()), rr);
1494 }
1495
1496 template <std::integral N, typename D, typename F, int dim>
1497 void eval (BoxND<dim> const& box, N ncomp, D & reduce_data, F const& f)
1498 {
1499 using ReduceTuple = typename D::Type;
1500 ReduceTuple rem;
1501 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(rem);
1502 Reduce::detail::Lanes<ReduceTuple> acc;
1503 Reduce::detail::lanes_set<0>(acc, rem);
1504 int const ilo = box.smallEnd(0);
1505 int const ihi = box.bigEnd(0);
1506 BoxND<dim> rows = box;
1507 rows.setBig(0, ilo);
1508 For(rows, ncomp, [&] (IntVectND<dim> iv, int n) {
1509 Reduce::detail::reduce_row<Ps...>(ilo, ihi, acc, rem, [&] (int i) {
1510 iv[0] = i;
1511 return Reduce::detail::call_f_intvect_n(f, iv, n);
1512 });
1513 });
1514 auto& rr = reduce_data.reference(OpenMP::get_thread_num());
1515 Reduce::detail::lanes_fold<0, ReduceTuple, Ps...>(acc, rr);
1516 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(rr, rem);
1517 }
1518
1519 template <std::integral N, typename D, typename F>
1520 void eval (N n, D & reduce_data, F const& f)
1521 {
1522 using ReduceTuple = typename D::Type;
1523 ReduceTuple rem;
1524 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(rem);
1525 Reduce::detail::Lanes<ReduceTuple> acc;
1526 Reduce::detail::lanes_set<0>(acc, rem);
1527 if (n > 0) {
1528 Reduce::detail::reduce_row<Ps...>(N(0), static_cast<N>(n-1), acc, rem,
1529 [&] (N i) { return f(i); });
1530 }
1531 auto& rr = reduce_data.reference(OpenMP::get_thread_num());
1532 Reduce::detail::lanes_fold<0, ReduceTuple, Ps...>(acc, rr);
1533 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(rr, rem);
1534 }
1535
1536 template <typename D>
1537 typename D::Type value (D & reduce_data)
1538 {
1539 auto& rrv = reduce_data.reference();
1540 if (! m_result_is_ready) {
1541 using ReduceTuple = typename D::Type;
1542 if (rrv.size() > 1) {
1543 for (int i = 1, N = rrv.size(); i < N; ++i) {
1544 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(rrv[0], rrv[i]);
1545 }
1546 }
1547 m_result_is_ready = true;
1548 }
1549 return rrv[0];
1550 }
1551
1552private:
1553 template <typename... T> friend class ReduceData;
1554 bool m_result_is_ready = false;
1555 void resetResultReadiness () { m_result_is_ready = false; }
1556};
1557
1558namespace Reduce {
1559
1560template <typename T, std::integral N, typename F>
1561requires (!std::same_as<T*,std::decay_t<F>>)
1562T Sum (N n, F const& f, T init_val)
1563{
1564 auto const cn = Reduce::detail::chunks_1d(n);
1565 N const chunk = cn.first;
1566 N const nchunks = cn.second;
1567 T r = init_val;
1568#ifdef AMREX_USE_OMP
1569#pragma omp parallel for reduction(+:r)
1570#endif
1571 for (N ic = 0; ic < nchunks; ++ic) {
1572 N const ib = ic*chunk;
1573 N const ie = (n - ib < chunk) ? n : ib + chunk;
1574 using Tup = GpuTuple<T>;
1575 Tup rem{};
1576 Reduce::detail::Lanes<Tup> acc;
1577 Reduce::detail::lanes_set<0>(acc, rem);
1578 Reduce::detail::reduce_row<ReduceOpSum>(ib, static_cast<N>(ie-1), acc, rem,
1579 [&] (N i) { return Tup{f(i)}; });
1580 Reduce::detail::lanes_fold<0,Tup,ReduceOpSum>(acc, rem);
1581 r += amrex::get<0>(rem);
1582 }
1583 return r;
1584}
1585
1586template <typename T, std::integral N>
1587T Sum (N n, T const* v, T init_val)
1588{
1589 return Sum(n, [=] (N i) -> T { return v[i]; }, init_val);
1590}
1591
1592template <typename T, std::integral N, typename F>
1593requires (!std::same_as<T*,std::decay_t<F>>)
1594T Min (N n, F const& f, T init_val)
1595{
1596 auto const cn = Reduce::detail::chunks_1d(n);
1597 N const chunk = cn.first;
1598 N const nchunks = cn.second;
1599 T r = init_val;
1600#ifdef AMREX_USE_OMP
1601#pragma omp parallel for reduction(min:r)
1602#endif
1603 for (N ic = 0; ic < nchunks; ++ic) {
1604 N const ib = ic*chunk;
1605 N const ie = (n - ib < chunk) ? n : ib + chunk;
1606 using Tup = GpuTuple<T>;
1607 Tup rem{init_val};
1608 Reduce::detail::Lanes<Tup> acc;
1609 Reduce::detail::lanes_set<0>(acc, rem);
1610 Reduce::detail::reduce_row<ReduceOpMin>(ib, static_cast<N>(ie-1), acc, rem,
1611 [&] (N i) { return Tup{f(i)}; });
1612 Reduce::detail::lanes_fold<0,Tup,ReduceOpMin>(acc, rem);
1613 r = std::min(r, amrex::get<0>(rem));
1614 }
1615 return r;
1616}
1617
1618template <typename T, std::integral N>
1619T Min (N n, T const* v, T init_val)
1620{
1621 return Reduce::Min(n, [=] (N i) -> T { return v[i]; }, init_val);
1622}
1623
1624template <typename T, std::integral N, typename F>
1625requires (!std::same_as<T*,std::decay_t<F>>)
1626T Max (N n, F const& f, T init_val)
1627{
1628 auto const cn = Reduce::detail::chunks_1d(n);
1629 N const chunk = cn.first;
1630 N const nchunks = cn.second;
1631 T r = init_val;
1632#ifdef AMREX_USE_OMP
1633#pragma omp parallel for reduction(max:r)
1634#endif
1635 for (N ic = 0; ic < nchunks; ++ic) {
1636 N const ib = ic*chunk;
1637 N const ie = (n - ib < chunk) ? n : ib + chunk;
1638 using Tup = GpuTuple<T>;
1639 Tup rem{init_val};
1640 Reduce::detail::Lanes<Tup> acc;
1641 Reduce::detail::lanes_set<0>(acc, rem);
1642 Reduce::detail::reduce_row<ReduceOpMax>(ib, static_cast<N>(ie-1), acc, rem,
1643 [&] (N i) { return Tup{f(i)}; });
1644 Reduce::detail::lanes_fold<0,Tup,ReduceOpMax>(acc, rem);
1645 r = std::max(r, amrex::get<0>(rem));
1646 }
1647 return r;
1648}
1649
1650template <typename T, std::integral N>
1651T Max (N n, T const* v, T init_val)
1652{
1653 return Reduce::Max(n, [=] (N i) -> T { return v[i]; }, init_val);
1654}
1655
1656template <typename T, std::integral N, typename F>
1657requires (!std::same_as<T*,std::decay_t<F>>)
1658std::pair<T,T> MinMax (N n, F const& f)
1659{
1660 auto const cn = Reduce::detail::chunks_1d(n);
1661 N const chunk = cn.first;
1662 N const nchunks = cn.second;
1663 T r_min = std::numeric_limits<T>::max();
1664 T r_max = std::numeric_limits<T>::lowest();
1665#ifdef AMREX_USE_OMP
1666#pragma omp parallel for reduction(min:r_min) reduction(max:r_max)
1667#endif
1668 for (N ic = 0; ic < nchunks; ++ic) {
1669 N const ib = ic*chunk;
1670 N const ie = (n - ib < chunk) ? n : ib + chunk;
1671 using Tup = GpuTuple<T,T>;
1672 Tup rem{std::numeric_limits<T>::max(), std::numeric_limits<T>::lowest()};
1673 Reduce::detail::Lanes<Tup> acc;
1674 Reduce::detail::lanes_set<0>(acc, rem);
1675 Reduce::detail::reduce_row<ReduceOpMin,ReduceOpMax>(ib, static_cast<N>(ie-1), acc, rem,
1676 [&] (N i) { T const tmp = f(i); return Tup{tmp,tmp}; });
1677 Reduce::detail::lanes_fold<0,Tup,ReduceOpMin,ReduceOpMax>(acc, rem);
1678 r_min = std::min(r_min, amrex::get<0>(rem));
1679 r_max = std::max(r_max, amrex::get<1>(rem));
1680 }
1681 return std::make_pair(r_min,r_max);
1682}
1683
1684template <typename T, std::integral N>
1685std::pair<T,T> MinMax (N n, T const* v)
1686{
1687 return Reduce::MinMax<T>(n, [=] (N i) -> T { return v[i]; });
1688}
1689
1690template <typename T, std::integral N, typename P>
1691bool AnyOf (N n, T const* v, P const& pred)
1692{
1693 return std::any_of(v, v+n, pred);
1694}
1695
1696template <typename P, int dim>
1697bool AnyOf (BoxND<dim> const& box, P const& pred)
1698{
1699 for (auto iv : box.iterator()) { // NOLINT(readability-use-anyofallof)
1700 if (Reduce::detail::call_f_intvect(pred, iv)) { return true; }
1701 }
1702 return false;
1703}
1704
1705}
1706
1707#endif
1708
1713template <typename... Ts, typename... Ps>
1715constexpr GpuTuple<Ts...>
1717{
1718 GpuTuple<Ts...> r{};
1719 Reduce::detail::for_each_init<0, decltype(r), Ps...>(r);
1720 return r;
1721}
1722
1727template <typename... Ts, typename... Ps>
1729constexpr GpuTuple<Ts...>
1731{
1732 GpuTuple<Ts...> r{};
1733 Reduce::detail::for_each_init<0, decltype(r), Ps...>(r);
1734 return r;
1735}
1736
1738template <typename Ops, typename Ts>
1739class ReducerImpl;
1740
1741template <typename... Ops, typename... Ts>
1742class ReducerImpl<TypeList<Ops...>, TypeList<Ts...>>
1743{
1744public:
1745 static_assert(sizeof...(Ops) > 0);
1746 static_assert(sizeof...(Ts) > 0);
1747 static_assert(sizeof...(Ops) == sizeof...(Ts));
1748
1749 ReducerImpl ()
1750 : m_reduce_data(m_reduce_op)
1751 {}
1752
1753protected:
1754 using Result_t = GpuTuple<Ts...>;
1755 ReduceOps<Ops...> m_reduce_op;
1756 ReduceData<Ts...> m_reduce_data;
1757};
1759
1824template <typename Ops, typename Ts>
1826 : public ReducerImpl<ToTypeList_t<Ops>, ToTypeList_t<Ts>>
1827{
1828 using Base = ReducerImpl<ToTypeList_t<Ops>, ToTypeList_t<Ts>>;
1829public:
1830
1831 using Result_t = typename Base::Result_t;
1832 static constexpr int size = GpuTupleSize<Result_t>::value;
1833
1834 Reducer () = default;
1835 ~Reducer () = default;
1836
1838 Reducer (Reducer const&) = delete;
1839 Reducer (Reducer &&) = delete;
1840 void operator= (Reducer const&) = delete;
1841 void operator= (Reducer &&) = delete;
1843
1857 template <typename F, int dim>
1860 void eval (BoxND<dim> const& box, F&& f)
1861 {
1862 this->m_reduce_op.eval(box, this->m_reduce_data, std::forward<F>(f));
1863 }
1864
1880 template <typename F, int dim>
1882 IsCallable<F, IntVectND<dim>, int>::value)
1883 void eval (BoxND<dim> const& box, int ncomp, F&& f)
1884 {
1885 this->m_reduce_op.eval(box, ncomp, this->m_reduce_data, std::forward<F>(f));
1886 }
1887
1907 template <FabArrayType MF, typename F>
1909 void eval (MF const& mf, IntVect const& nghost, F && f)
1910 {
1911 this->m_reduce_op.eval(mf, nghost, this->m_reduce_data, std::forward<F>(f));
1912 }
1913
1936 template <FabArrayType MF, typename F>
1938 void eval (MF const& mf, IntVect const& nghost, int ncomp, F && f)
1939 {
1940 this->m_reduce_op.eval(mf, nghost, ncomp, this->m_reduce_data, std::forward<F>(f));
1941 }
1942
1955 template <typename N, typename F>
1956 requires (IsCallable<F, N>::value)
1957 void eval (N n, F && f)
1958 {
1959 this->m_reduce_op.eval(n, this->m_reduce_data, std::forward<F>(f));
1960 }
1961
1972 [[nodiscard]] Result_t getResult ()
1973 {
1974 return this->m_reduce_data.value(this->m_reduce_op);
1975 }
1976};
1977
1978}
1979
1980#endif
Runtime initialization/finalization helpers and global diagnostics.
Memory arena base class and global arena accessors.
#define AMREX_ALWAYS_ASSERT_WITH_MESSAGE(EX, MSG)
Definition AMReX_BLassert.H:49
#define AMREX_ASSERT(EX)
Definition AMReX_BLassert.H:38
#define AMREX_FORCE_INLINE
Definition AMReX_Extension.H:123
#define AMREX_GPU_MAX_STREAMS
Definition AMReX_GpuDevice.H:21
#define AMREX_LAUNCH_KERNEL(MT, blocks, threads, sharedMem, stream,...)
Definition AMReX_GpuLaunch.H:37
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
#define AMREX_GPU_HOST_DEVICE
Definition AMReX_GpuQualifiers.H:20
Convenience header for the core AMReX GPU facilities.
virtual void free(void *pt)=0
Free a previously allocated block pointed to by pt.
A Rectangular Domain on an Integer Lattice.
Definition AMReX_Box.H:54
__host__ __device__ bool isEmpty() const noexcept
Checks if it is an empty BoxND.
Definition AMReX_Box.H:223
__host__ __device__ Long numPts() const noexcept
Return the number of points contained in the BoxND.
Definition AMReX_Box.H:385
__host__ __device__ IndexTypeND< dim > ixType() const noexcept
Return the indexing type.
Definition AMReX_Box.H:148
GPU-compatible tuple.
Definition AMReX_Tuple.H:104
static int streamIndex(gpuStream_t s=gpuStream()) noexcept
Definition AMReX_GpuDevice.cpp:846
static unsigned int maxBlocksPerLaunch() noexcept
Definition AMReX_GpuDevice.H:242
static constexpr int warp_size
Definition AMReX_GpuDevice.H:239
static bool usingExternalStream() noexcept
Definition AMReX_GpuDevice.cpp:973
Cell-Based or Node-Based Indices.
Definition AMReX_IndexType.H:36
An Integer Vector in dim-Dimensional Space.
Definition AMReX_IntVect.H:156
Iterator for looping ever tiles and boxes of amrex::FabArray based containers.
Definition AMReX_MFIter.H:88
bool isValid() const noexcept
Is the iterator valid i.e. is it associated with a FAB?
Definition AMReX_MFIter.H:176
T * data() noexcept
Definition AMReX_PODVector.H:672
Definition AMReX_Reduce.H:556
~ReduceData()
Definition AMReX_Reduce.H:578
int maxStreamIndex() const
Definition AMReX_Reduce.H:618
Type value()
Definition AMReX_Reduce.H:591
void updateMaxStreamIndex(gpuStream_t const &s)
Definition AMReX_Reduce.H:619
int & nBlocks(gpuStream_t const &s)
Definition AMReX_Reduce.H:614
GpuTuple< Ts... > Type
Definition AMReX_Reduce.H:558
ReduceData(ReduceOps< Ps... > &reduce_op)
Definition AMReX_Reduce.H:561
void markValueCalled() noexcept
Definition AMReX_Reduce.H:623
Type * devicePtr(gpuStream_t const &s)
Definition AMReX_Reduce.H:607
Type value(ReduceOps< Ps... > &reduce_op)
Definition AMReX_Reduce.H:599
Type * devicePtr()
Definition AMReX_Reduce.H:606
GpuArray< int, 8 > & nBlocks()
Definition AMReX_Reduce.H:613
ReduceData(ReduceData< Ts... > const &)=delete
Type * hostPtr()
Definition AMReX_Reduce.H:611
int maxBlocks() const
Definition AMReX_Reduce.H:616
ReduceData(ReduceData< Ts... > &&)=delete
Definition AMReX_Reduce.H:715
void eval(BoxND< dim > const &box, N ncomp, D &reduce_data, F const &f)
Definition AMReX_Reduce.H:919
D::Type value(D &reduce_data)
Definition AMReX_Reduce.H:986
void eval(BoxND< dim > const &box, D &reduce_data, F const &f)
Definition AMReX_Reduce.H:913
void eval(MF const &mf, IntVect const &nghost, D &reduce_data, F &&f)
Definition AMReX_Reduce.H:855
friend class ReduceData
Definition AMReX_Reduce.H:1074
void eval(MF const &mf, IntVect const &nghost, int ncomp, D &reduce_data, F &&f)
Definition AMReX_Reduce.H:881
void eval(N n, D &reduce_data, F const &f)
Definition AMReX_Reduce.H:934
static constexpr int size
Number of reduction operators.
Definition AMReX_Reduce.H:1832
Reducer()=default
Result_t getResult()
Get the final reduction result.
Definition AMReX_Reduce.H:1972
typename Base::Result_t Result_t
Reduction result type, GpuTuple<U...>, where U... are the types in Ts.
Definition AMReX_Reduce.H:1831
~Reducer()=default
void eval(BoxND< dim > const &box, F &&f)
Reduction over a Box.
Definition AMReX_Reduce.H:1860
amrex_long Long
Definition AMReX_INT.H:30
T Min(N n, T const *v, T init_val=std::numeric_limits< T >::max())
Compute the minimum of an array of values.
Definition AMReX_Reduce.H:1105
bool AnyOf(N n, T const *v, P const &pred)
Test whether any element in an array satisfies a unary predicate.
Definition AMReX_Reduce.H:1179
std::pair< T, T > MinMax(N n, T const *v)
Compute the minimum and maximum of an array of values.
Definition AMReX_Reduce.H:1151
T Max(N n, T const *v, T init_val=std::numeric_limits< T >::lowest())
Compute the maximum of an array of values.
Definition AMReX_Reduce.H:1128
T Sum(N n, T const *v, T init_val=0)
Compute the sum of an array of values.
Definition AMReX_Reduce.H:1082
__host__ __device__ Dim3 ubound(Array4< T > const &a) noexcept
Return the inclusive upper bounds of an Array4 in Dim3 form.
Definition AMReX_Array4.H:1377
__host__ __device__ Dim3 lbound(Array4< T > const &a) noexcept
Return the inclusive lower bounds of an Array4 in Dim3 form.
Definition AMReX_Array4.H:1363
__host__ __device__ BoxND< dim > grow(const BoxND< dim > &b, int i) noexcept
Return a copy of b grown uniformly by i cells in every direction.
Definition AMReX_Box.H:1328
PODVector< T, PinnedArenaAllocator< T > > PinnedVector
A PODVector that uses the pinned memory arena.
Definition AMReX_GpuContainers.H:48
PODVector< T, ArenaAllocator< T > > DeviceVector
A PODVector that uses the standard memory Arena. Note that the memory might or might not be managed d...
Definition AMReX_GpuContainers.H:24
Arena * The_Pinned_Arena()
Definition AMReX_Arena.cpp:879
Arena * The_Arena()
Definition AMReX_Arena.cpp:839
void Sum(Gpu::DeviceVector< T > &v, MPI_Comm comm)
Definition AMReX_GpuParallelReduce.H:37
__host__ __device__ constexpr const T & min(const T &a, const T &b) noexcept
Definition AMReX_Algorithm.H:31
__host__ __device__ constexpr const T & max(const T &a, const T &b) noexcept
Definition AMReX_Algorithm.H:54
__host__ __device__ AMREX_FORCE_INLINE T Max(T *const m, T const value) noexcept
Definition AMReX_GpuAtomic.H:474
__host__ __device__ AMREX_FORCE_INLINE T Min(T *const m, T const value) noexcept
Definition AMReX_GpuAtomic.H:397
Definition AMReX_BaseFwd.H:60
__device__ T blockReduce(T x, WARPREDUCE &&warp_reduce, T x0)
Definition AMReX_GpuReduce.H:317
__device__ int blockReduceLogicalOr(int source) noexcept
Definition AMReX_GpuReduce.H:559
__device__ T blockReduceMax(T source) noexcept
Definition AMReX_GpuReduce.H:458
__device__ T blockReduceMin(T source) noexcept
Definition AMReX_GpuReduce.H:403
__device__ int blockReduceLogicalAnd(int source) noexcept
Definition AMReX_GpuReduce.H:511
void streamSynchronize() noexcept
Definition AMReX_GpuDevice.H:329
void dtoh_memcpy_async(void *p_h, const void *p_d, const std::size_t sz) noexcept
Definition AMReX_GpuDevice.H:454
gpuStream_t gpuStream() noexcept
Definition AMReX_GpuDevice.H:310
__device__ T blockReduceSum(T source) noexcept
Definition AMReX_GpuReduce.H:353
constexpr int get_max_threads()
Definition AMReX_OpenMP.H:36
Definition AMReX_Reduce.H:22
Definition AMReX_Amr.cpp:50
__host__ __device__ void ignore_unused(const Ts &...)
No-op helper that marks variables as intentionally unused.
Definition AMReX.H:273
@ F
Definition AMReX_Order.H:7
void For(TypeList< CTOs... > ctos, std::array< int, sizeof...(CTOs)> const &runtime_options, T N, F &&f)
Definition AMReX_CTOParallelForImpl.H:400
@ Device
Definition AMReX_GpuControl.H:65
__host__ __device__ constexpr GpuTuple< Ts... > IdentityTuple(GpuTuple< Ts... >, ReduceOps< Ps... >) noexcept
Return a GpuTuple containing the identity element for each operation in ReduceOps....
Definition AMReX_Reduce.H:1716
void launch(T const &n, L const &f) noexcept
Definition AMReX_GpuLaunchFunctsC.H:122
BoxND< 3 > Box
Box is an alias for amrex::BoxND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:35
typename ToTypeList< T >::type ToTypeList_t
Definition AMReX_TypeList.H:233
IntVectND< 3 > IntVect
IntVect is an alias for amrex::IntVectND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:38
cudaStream_t gpuStream_t
Definition AMReX_GpuControl.H:79
BoxIndexerND< 3 > BoxIndexer
Definition AMReX_Box.H:2641
const int[]
Definition AMReX_BLProfiler.cpp:1665
__global__ void launch_global(L f0, Lambdas... fs)
Definition AMReX_GpuLaunch.H:68
__host__ __device__ constexpr int get(IntVectND< dim > const &iv) noexcept
Get the Idx-th element of IntVectND<dim>.
Definition AMReX_IntVect.H:1367
Utility that maps flattened point indices back to IntVectND coordinates.
Definition AMReX_Box.H:2509
detail::BoxIndexerUInt UInt
Definition AMReX_Box.H:2511
__host__ __device__ IntVectND< dim > intVect(std::uint64_t icell) const
Convert flattened point index icell to its IntVectND coordinate.
Definition AMReX_Box.H:2541
__host__ __device__ UInt numPts() const
Return the number of points covered by the indexed box.
Definition AMReX_Box.H:2577
Fixed-size array that can be used on GPU.
Definition AMReX_Array.H:52
Definition AMReX_Tuple.H:133
Definition AMReX_GpuMemory.H:57
T * dataPtr()
Definition AMReX_GpuMemory.H:91
T dataValue() const
Definition AMReX_GpuMemory.H:93
Definition AMReX_GpuLaunch.H:121
Definition AMReX_GpuTypes.H:86
Definition AMReX_GpuControl.H:127
Definition AMReX_GpuReduce.H:290
Test if a given type T is callable with arguments of type Args...
Definition AMReX_TypeTraits.H:208
Fast division and modulus by a fixed 32-bit divisor, as FastDivmodU64, for dividends below 2^32-1.
Definition AMReX_Math.H:757
Function object that returns the sum of two values.
Definition AMReX_Functional.H:23
Definition AMReX_Reduce.H:409
__host__ __device__ void local_update(T &d, T s) const noexcept
Definition AMReX_Reduce.H:430
constexpr void init(T &t) const noexcept
Definition AMReX_Reduce.H:433
__device__ void parallel_update(T &d, T s) const noexcept
Definition AMReX_Reduce.H:421
Definition AMReX_Reduce.H:438
__device__ void parallel_update(T &d, T s) const noexcept
Definition AMReX_Reduce.H:450
__host__ __device__ void local_update(T &d, T s) const noexcept
Definition AMReX_Reduce.H:459
constexpr void init(T &t) const noexcept
Definition AMReX_Reduce.H:462
Definition AMReX_Reduce.H:375
constexpr void init(T &t) const noexcept
Definition AMReX_Reduce.H:400
__host__ __device__ void local_update(T &d, T const &s) const noexcept
Definition AMReX_Reduce.H:396
__device__ void parallel_update(T &d, T const &s) const noexcept
Definition AMReX_Reduce.H:387
Definition AMReX_Reduce.H:341
constexpr void init(T &t) const noexcept
Definition AMReX_Reduce.H:366
__device__ void parallel_update(T &d, T const &s) const noexcept
Definition AMReX_Reduce.H:353
__host__ __device__ void local_update(T &d, T const &s) const noexcept
Definition AMReX_Reduce.H:362
Definition AMReX_Reduce.H:311
__device__ void parallel_update(T &d, T const &s) const noexcept
Definition AMReX_Reduce.H:324
__host__ __device__ void local_update(T &d, T const &s) const noexcept
Definition AMReX_Reduce.H:333
constexpr void init(T &t) const noexcept
Definition AMReX_Reduce.H:336
Struct for holding types.
Definition AMReX_TypeList.H:13