3#include <AMReX_Config.H>
40template <
typename T, std::
integral N>
41T
Sum (N n, T
const* v, T init_val = 0);
59template <
typename T, std::
integral N,
typename F>
60requires (!std::same_as<T*,std::decay_t<F>>)
61T
Sum (N n,
F const& f, T init_val = 0);
77template <
typename T, std::
integral N>
78T
Min (N n, T
const* v, T init_val = std::numeric_limits<T>::max());
96template <
typename T, std::
integral N,
typename F>
97requires (!std::same_as<T*,std::decay_t<F>>)
98T
Min (N n,
F const& f, T init_val = std::numeric_limits<T>::max());
114template <
typename T, std::
integral N>
115T
Max (N n, T
const* v, T init_val = std::numeric_limits<T>::lowest());
133template <
typename T, std::
integral N,
typename F>
134requires (!std::same_as<T*,std::decay_t<F>>)
135T
Max (N n,
F const& f, T init_val = std::numeric_limits<T>::lowest());
150template <
typename T, std::
integral N>
151std::pair<T,T>
MinMax (N n, T
const* v);
168template <
typename T, std::
integral N,
typename F>
169requires (!std::same_as<T*,std::decay_t<F>>)
170std::pair<T,T>
MinMax (N n,
F const& f);
187template <
typename T, std::
integral N,
typename P>
188bool AnyOf (N n, T
const* v, P
const& pred);
203template <
typename P,
int dim>
209namespace Reduce::detail {
213inline constexpr int nacc = 8;
218#if defined(AMREX_CXX_GCC) && !defined(__CUDACC__)
219#define AMREX_PRAGMA_NACC_VECTORIZE _Pragma("GCC unroll 4")
221#define AMREX_PRAGMA_NACC_NOVECTOR _Pragma("GCC novector")
223#define AMREX_PRAGMA_NACC_NOVECTOR
225#define AMREX_PRAGMA_NACC_SCALAR AMREX_PRAGMA_NACC_NOVECTOR _Pragma("GCC unroll 8")
227#define AMREX_PRAGMA_NACC_VECTORIZE
228#define AMREX_PRAGMA_NACC_NOVECTOR
229#define AMREX_PRAGMA_NACC_SCALAR
231static_assert(nacc == 8,
"The AMREX_PRAGMA_NACC unroll factors assume nacc == 8");
234template <std::
integral N>
235std::pair<N,N> chunks_1d (N n)
238 N
const chunk = std::max(
static_cast<N
>(nacc),
static_cast<N
>((n / (4*nthreads) / nacc) * nacc));
239 N
const nchunks = n / chunk + ((n % chunk != 0) ? 1 : 0);
240 return {chunk, nchunks};
245 template <std::
size_t Idx,
typename T,
typename P>
247 void for_each_parallel (T& d, T
const& s, Gpu::Handler
const& h)
252 template <std::size_t Idx,
typename T,
typename P,
typename P1,
typename... Ps>
254 void for_each_parallel (T& d, T
const& s, Gpu::Handler
const& h)
257 for_each_parallel<Idx+1,T,P1,Ps...>(d, s, h);
260 template <std::
size_t Idx,
typename T,
typename P>
262 void for_each_parallel (T& d, T
const& s)
267 template <std::size_t Idx,
typename T,
typename P,
typename P1,
typename... Ps>
269 void for_each_parallel (T& d, T
const& s)
272 for_each_parallel<Idx+1,T,P1,Ps...>(d, s);
277 template <std::
size_t Idx,
typename T,
typename P>
279 void for_each_local (T& d, T
const& s)
284 template <std::size_t Idx,
typename T,
typename P,
typename P1,
typename... Ps>
286 void for_each_local (T& d, T
const& s)
289 for_each_local<Idx+1,T,P1,Ps...>(d, s);
292 template <std::
size_t Idx,
typename T,
typename P>
294 constexpr void for_each_init (T& t)
299 template <std::size_t Idx,
typename T,
typename P,
typename P1,
typename... Ps>
301 constexpr void for_each_init (T& t)
304 for_each_init<Idx+1,T,P1,Ps...>(t);
315 template <
typename T>
319 if (h.threadIdx() == 0) { d += r; }
322 template <
typename T,
int MT=AMREX_GPU_MAX_THREADS>
326 if (threadIdx.x == 0) { d += r; }
331 template <
typename T>
335 template <
typename T>
336 constexpr void init (T& t)
const noexcept { t = 0; }
344 template <
typename T>
348 if (h.threadIdx() == 0) { d =
amrex::min(d,r); }
351 template <
typename T,
int MT=AMREX_GPU_MAX_THREADS>
355 if (threadIdx.x == 0) { d =
amrex::min(d,r); }
360 template <
typename T>
364 template <
typename T>
365 requires (std::numeric_limits<T>::is_specialized)
366 constexpr void init (T& t)
const noexcept { t = std::numeric_limits<T>::max(); }
368 template <
typename T>
369 requires (!std::numeric_limits<T>::is_specialized)
370 constexpr void init (T& t)
const noexcept { t = T::max(); }
378 template <
typename T>
382 if (h.threadIdx() == 0) { d =
amrex::max(d,r); }
385 template <
typename T,
int MT=AMREX_GPU_MAX_THREADS>
389 if (threadIdx.x == 0) { d =
amrex::max(d,r); }
394 template <
typename T>
398 template <
typename T>
399 requires (std::numeric_limits<T>::is_specialized)
400 constexpr void init (T& t)
const noexcept { t = std::numeric_limits<T>::lowest(); }
402 template <
typename T>
403 requires (!std::numeric_limits<T>::is_specialized)
404 constexpr void init (T& t)
const noexcept { t = T::lowest(); }
412 template <std::
integral T>
416 if (h.threadIdx() == 0) { d = d && r; }
419 template <std::
integral T,
int MT=AMREX_GPU_MAX_THREADS>
423 if (threadIdx.x == 0) { d = d && r; }
428 template <std::
integral T>
432 template <std::
integral T>
433 constexpr void init (T& t)
const noexcept { t =
true; }
441 template <std::
integral T>
445 if (h.threadIdx() == 0) { d = d || r; }
448 template <std::
integral T,
int MT=AMREX_GPU_MAX_THREADS>
452 if (threadIdx.x == 0) { d = d || r; }
457 template <std::
integral T>
461 template <std::
integral T>
462 constexpr void init (T& t)
const noexcept { t =
false; }
466namespace Reduce::detail {
470template <
typename... Ps>
471inline constexpr bool vectorize_nacc_loop =
472#if defined(AMREX_CXX_GCC) && !defined(__CUDACC__) && (__GNUC__ >= 14) && \
473 (defined(__x86_64__) || defined(__i386__)) && !defined(__SSE4_1__)
474 sizeof...(Ps) != 1 ||
475 !((std::is_same_v<Ps,ReduceOpMin> || std::is_same_v<Ps,ReduceOpMax>) && ...);
483template <
typename Tup>
struct Lanes;
485template <
typename... Ts>
486struct Lanes<GpuTuple<Ts...>>
489 GpuTuple<GpuArray<Ts,nacc>...> data;
490 template <
int K>
auto& get (
int lane)
noexcept {
return amrex::get<K>(data)[lane]; }
491 template <
int K> [[nodiscard]]
auto const&
get (
int lane)
const noexcept {
return amrex::get<K>(data)[lane]; }
493 GpuTuple<Ts...> data[nacc];
494 template <
int K>
auto&
get (
int lane)
noexcept {
return amrex::get<K>(data[lane]); }
495 template <
int K> [[nodiscard]]
auto const&
get (
int lane)
const noexcept {
return amrex::get<K>(data[lane]); }
499template <
int K,
typename Tup>
501void lanes_set (Lanes<Tup>& acc, Tup
const& v)
503 for (
int lane = 0; lane < nacc; ++lane) { acc.template get<K>(lane) =
amrex::get<K>(v); }
504 if constexpr (K+1 < GpuTupleSize<Tup>::value) { lanes_set<K+1>(acc, v); }
507template <
int K,
typename Tup,
typename P,
typename... Ps>
509void lanes_update (Lanes<Tup>& acc,
int lane, Tup
const& s)
511 P().local_update(acc.template get<K>(lane),
amrex::get<K>(s));
512 if constexpr (
sizeof...(Ps) > 0) { lanes_update<K+1,Tup,Ps...>(acc, lane, s); }
515template <
int K,
typename Tup,
typename P,
typename... Ps>
517void lanes_fold (Lanes<Tup>
const& acc, Tup& r)
519 for (
int lane = 0; lane < nacc; ++lane) { P().local_update(
amrex::get<K>(r), acc.template get<K>(lane)); }
520 if constexpr (
sizeof...(Ps) > 0) { lanes_fold<K+1,Tup,Ps...>(acc, r); }
525template <
typename... Ps,
typename Tup, std::integral N,
typename F>
527void reduce_row (N ilo, N ihi, Lanes<Tup>& acc, Tup& rem, F
const& value)
530 AMREX_PRAGMA_NACC_NOVECTOR
531 for (; i + (nacc-1) <= ihi; i += nacc) {
532 if constexpr (vectorize_nacc_loop<Ps...>) {
533 AMREX_PRAGMA_NACC_VECTORIZE
534 for (
int lane = 0; lane < nacc; ++lane) { lanes_update<0,Tup,Ps...>(acc, lane, value(
static_cast<N
>(i+lane))); }
536 AMREX_PRAGMA_NACC_SCALAR
537 for (
int lane = 0; lane < nacc; ++lane) { lanes_update<0,Tup,Ps...>(acc, lane, value(
static_cast<N
>(i+lane))); }
540 AMREX_PRAGMA_NACC_NOVECTOR
541 for (; i <= ihi; ++i) { for_each_local<0,Tup,Ps...>(rem, value(i)); }
544#undef AMREX_PRAGMA_NACC_VECTORIZE
545#undef AMREX_PRAGMA_NACC_NOVECTOR
546#undef AMREX_PRAGMA_NACC_SCALAR
549template <
typename... Ps>
class ReduceOps;
554template <
typename... Ts>
560 template <
typename... Ps>
562 : m_max_blocks(
Gpu::
Device::maxBlocksPerLaunch()),
565 * m_max_blocks * sizeof(
Type)))),
566 m_fn_value([&reduce_op,this] () ->
Type { return this->
value(reduce_op); })
568 reduce_op.resetResultReadiness();
569 static_assert(std::is_trivially_copyable<Type>(),
570 "ReduceData::Type must be trivially copyable");
571 static_assert(std::is_trivially_destructible<Type>(),
572 "ReduceData::Type must be trivially destructible");
574 new (m_host_tuple) Type();
580 !m_used_external_stream || m_value_called,
581 "ReduceData used on an external GPU stream must call value() before destruction.");
593 Type r = m_fn_value();
594 m_value_called =
true;
598 template <
typename... Ps>
602 m_value_called =
true;
608 return m_device_tuple+streamIndexChecked(s)*m_max_blocks;
620 m_max_stream_index = std::max(m_max_stream_index,streamIndexChecked(s));
631 if (m_stream_index_zero_set) {
633 "ReduceData cannot be reused across different external GPU streams "
634 "or between an external GPU stream and AMReX stream 0.");
636 m_stream_index_zero = s;
637 m_stream_index_zero_set =
true;
644 int m_max_stream_index = 0;
645 Type* m_host_tuple =
nullptr;
646 Type* m_device_tuple =
nullptr;
649 bool m_stream_index_zero_set =
false;
650 bool m_used_external_stream =
false;
651 bool m_value_called =
false;
652 std::function<Type()> m_fn_value;
656namespace Reduce::detail {
660 template <
typename F,
int dim>
662 auto call_f_intvect_box (F
const& f, IntVectND<dim> iv, IndexTypeND<dim>)
noexcept ->
663 decltype(amrex::detail::call_f_intvect_inner(std::make_index_sequence<dim>(), f, iv))
665 return amrex::detail::call_f_intvect_inner(std::make_index_sequence<dim>(), f, iv);
668 template <
typename F,
int dim>
670 auto call_f_intvect_box (F
const& f, IntVectND<dim> iv, IndexTypeND<dim> t)
noexcept ->
671 decltype(f(BoxND<dim>(iv, iv, t)))
673 return f(BoxND<dim>(iv, iv, t));
677 template <
typename F,
typename T,
int dim>
679 auto call_f_intvect_n (F
const& f, IntVectND<dim> iv, T n)
noexcept ->
680 decltype(amrex::detail::call_f_intvect_inner(std::make_index_sequence<dim>(), f, iv, n))
682 return amrex::detail::call_f_intvect_inner(std::make_index_sequence<dim>(), f, iv, n);
687 struct iterate_box {};
688 struct iterate_box_comp {};
690 template <
typename Tag,
typename F,
typename T,
typename... Ps>
691 requires (std::same_as<iterate_box,Tag>)
693 void mf_call_f (F
const& f,
int ibox,
int i,
int j,
int k,
int, T& r)
noexcept
695 auto const& pr = f(ibox,i,j,k);
696 Reduce::detail::for_each_local<0, T, Ps...>(r, pr);
699 template <
typename Tag,
typename F,
typename T,
typename... Ps>
700 requires (std::same_as<iterate_box_comp,Tag>)
702 void mf_call_f (F
const& f,
int ibox,
int i,
int j,
int k,
int ncomp, T& r)
noexcept
704 for (
int n = 0; n < ncomp; ++n) {
705 auto const& pr = f(ibox,i,j,k,n);
706 Reduce::detail::for_each_local<0, T, Ps...>(r, pr);
713template <
typename... Ps>
721 template <
typename Tag,
typename MF,
typename D,
typename F>
722 void eval_mf (Tag, MF
const& mf,
IntVect const& nghost,
int ncomp, D& reduce_data,
F const& f)
724 using ReduceTuple =
typename D::Type;
725 const int nboxes = mf.local_size();
727 auto const& parforinfo = mf.getParForInfo(nghost);
728 auto nblocks_per_box = parforinfo.getNBlocksPerBox(AMREX_GPU_MAX_THREADS);
730 const int nblocks = nblocks_per_box * nboxes;
731 const BoxIndexer* dp_boxes = parforinfo.getBoxes();
735 auto pdst = reduce_data.devicePtr(stream);
736 int nblocks_ec = std::min(nblocks, reduce_data.maxBlocks());
738 int& nblocks_ref = reduce_data.nBlocks(stream);
739 auto old_nblocks =
static_cast<unsigned int>(nblocks_ref);
740 nblocks_ref =
amrex::max(nblocks_ref, nblocks_ec);
741 reduce_data.updateMaxStreamIndex(stream);
745 constexpr std::size_t shared_mem_bytes =
sizeof(ReduceTuple)
750 Dim1 blockIdx {gh.blockIdx()};
751 Dim1 threadIdx{gh.threadIdx()};
754 <<<nblocks_ec, AMREX_GPU_MAX_THREADS, 0, stream>>>
759 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(r);
760 ReduceTuple& dst = pdst[blockIdx.x];
761 if (threadIdx.x == 0 && blockIdx.x >= old_nblocks) {
764 for (
int iblock = blockIdx.x; iblock < nblocks; iblock += nblocks_ec) {
765 std::uint32_t ibox, ib;
766 fdm_box(ibox, ib, std::uint32_t(iblock));
770 if (icell < indexer.
numPts()) {
771 auto [i, j, k] = indexer(icell);
772 Reduce::detail::mf_call_f<Tag,
F, ReduceTuple, Ps...>
773 (f,
int(ibox), i, j, k, ncomp, r);
777 Reduce::detail::for_each_parallel<0, ReduceTuple, Ps...>(dst, r, gh);
779 Reduce::detail::for_each_parallel<0, ReduceTuple, Ps...>(dst, r);
786 template <
typename Tag,
int dim,
typename D,
typename F>
787 void eval_box (Tag,
BoxND<dim> const& box,
int ncomp, D& reduce_data,
F const& f)
790 using ReduceTuple =
typename D::Type;
792 auto dp = reduce_data.devicePtr(stream);
793 int& nblocks = reduce_data.nBlocks(stream);
797 constexpr int nitems_per_thread = 4;
798 Long nblocks_ec = (box.
numPts() + nitems_per_thread*AMREX_GPU_MAX_THREADS-1)
799 / (nitems_per_thread*AMREX_GPU_MAX_THREADS);
800 nblocks_ec = std::min<Long>(nblocks_ec, reduce_data.maxBlocks());
801 reduce_data.updateMaxStreamIndex(stream);
804 constexpr std::size_t shared_mem_bytes =
sizeof(ReduceTuple)
809 Dim1 blockIdx {gh.blockIdx()};
810 Dim1 threadIdx{gh.threadIdx()};
811 Dim1 gridDim {gh.gridDim()};
818 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(r);
819 ReduceTuple& dst = *(dp+blockIdx.x);
820 if (threadIdx.x == 0 &&
static_cast<int>(blockIdx.x) >= nblocks) {
823 for (UInt icell = UInt(AMREX_GPU_MAX_THREADS)*UInt(blockIdx.x)+UInt(threadIdx.x),
824 stride = UInt(AMREX_GPU_MAX_THREADS)*UInt(gridDim.x);
828 auto iv = indexer.
intVect(icell);
830 if constexpr (std::is_same_v<Reduce::detail::iterate_box,Tag>) {
831 auto pr = Reduce::detail::call_f_intvect_box(f, iv, ixtype);
832 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(r, pr);
834 for (
int n = 0; n < ncomp; ++n) {
835 auto pr = Reduce::detail::call_f_intvect_n(f, iv, n);
836 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(r, pr);
841 Reduce::detail::for_each_parallel<0, ReduceTuple, Ps...>(dst, r, gh);
843 Reduce::detail::for_each_parallel<0, ReduceTuple, Ps...>(dst, r);
846 nblocks = std::max(nblocks,
static_cast<int>(nblocks_ec));
851 template <FabArrayType MF,
typename D,
typename F>
852#ifndef AMREX_USE_CUDA
855 void eval (MF
const& mf,
IntVect const& nghost, D& reduce_data, F&& f)
857 using ReduceTuple =
typename D::Type;
858 const int nboxes = mf.local_size();
861 }
else if (!mf.isFusingCandidate()) {
864 const int li = mfi.LocalIndex();
865 this->
eval(b, reduce_data,
868 return f(li, i, j, k);
872 eval_mf(Reduce::detail::iterate_box{},
873 mf, nghost, 0, reduce_data, std::forward<F>(f));
877 template <FabArrayType MF,
typename D,
typename F>
878#ifndef AMREX_USE_CUDA
881 void eval (MF
const& mf,
IntVect const& nghost,
int ncomp, D& reduce_data, F&& f)
883 using ReduceTuple =
typename D::Type;
885 const int nboxes = mf.local_size();
889 }
else if (!mf.isFusingCandidate()) {
892 const int li = mfi.LocalIndex();
893 this->
eval(b, ncomp, reduce_data,
896 return f(li, i, j, k, n);
899 }
else if (ncomp == 1) {
901 eval_mf(Reduce::detail::iterate_box{}, mf, nghost, 0, reduce_data,
902 [=]
AMREX_GPU_DEVICE (
int box_no,
int i,
int j,
int k)
noexcept -> ReduceTuple
904 return f(box_no, i, j, k, 0);
907 eval_mf(Reduce::detail::iterate_box_comp{},
908 mf, nghost, ncomp, reduce_data, std::forward<F>(f));
912 template <
typename D,
typename F,
int dim>
915 eval_box(Reduce::detail::iterate_box{}, box, 0, reduce_data, f);
918 template <std::
integral N,
typename D,
typename F,
int dim>
923 eval_box(Reduce::detail::iterate_box{}, box, 0, reduce_data,
926 return Reduce::detail::call_f_intvect_n(f, iv, 0);
929 eval_box(Reduce::detail::iterate_box_comp{}, box, ncomp, reduce_data, f);
933 template <std::
integral N,
typename D,
typename F>
934 void eval (N n, D & reduce_data, F
const& f)
936 if (n <= 0) {
return; }
937 using ReduceTuple =
typename D::Type;
939 auto dp = reduce_data.devicePtr(stream);
940 int& nblocks = reduce_data.nBlocks(stream);
941 constexpr int nitems_per_thread = 4;
942 Long nblocks_ec_long = (
Long(n) + nitems_per_thread*AMREX_GPU_MAX_THREADS-1)
943 / (nitems_per_thread*AMREX_GPU_MAX_THREADS);
944 int nblocks_ec =
static_cast<int>(std::min<Long>(nblocks_ec_long,
945 reduce_data.maxBlocks()));
946 reduce_data.updateMaxStreamIndex(stream);
949 constexpr std::size_t shared_mem_bytes =
sizeof(ReduceTuple)
954 Dim1 blockIdx {gh.blockIdx()};
955 Dim1 threadIdx{gh.threadIdx()};
956 Dim1 gridDim {gh.gridDim()};
963 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(r);
964 ReduceTuple& dst = *(dp+blockIdx.x);
965 if (threadIdx.x == 0 &&
static_cast<int>(blockIdx.x) >= nblocks) {
968 for (
Long i =
Long(AMREX_GPU_MAX_THREADS)*blockIdx.x+threadIdx.x,
969 stride =
Long(AMREX_GPU_MAX_THREADS)*gridDim.x;
974 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(r,pr);
977 Reduce::detail::for_each_parallel<0, ReduceTuple, Ps...>(dst, r, gh);
979 Reduce::detail::for_each_parallel<0, ReduceTuple, Ps...>(dst, r);
985 template <
typename D>
986 typename D::Type
value (D & reduce_data)
988 auto hp = reduce_data.hostPtr();
990 if (m_result_is_ready) {
991 reduce_data.markValueCalled();
995 using ReduceTuple =
typename D::Type;
997 auto dp = reduce_data.devicePtr();
998 auto const& nblocks = reduce_data.nBlocks();
999#if defined(AMREX_USE_SYCL)
1000 if (reduce_data.maxStreamIndex() == 0 && nblocks[0] <= 4096) {
1001 const int N = nblocks[0];
1003 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(*hp);
1008 for (
int i = 1; i < N; ++i) {
1009 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(tmp[0], tmp[i]);
1016 int maxblocks = reduce_data.maxBlocks();
1017#ifdef AMREX_USE_SYCL
1019 constexpr std::size_t shared_mem_bytes =
sizeof(ReduceTuple)
1021#ifndef AMREX_NO_SYCL_REDUCE_WORKAROUND
1024 auto presult = dtmp.
data();
1032 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(r);
1033 ReduceTuple dst = r;
1034 for (
int istream = 0, nstreams = nblocks.size(); istream < nstreams; ++istream) {
1035 auto dp_stream = dp+istream*maxblocks;
1036 for (
int i = gh.item->get_global_id(0), stride = gh.item->get_global_range(0);
1037 i < nblocks[istream]; i += stride) {
1038 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(r, dp_stream[i]);
1041 Reduce::detail::for_each_parallel<0, ReduceTuple, Ps...>(dst, r, gh);
1042 if (gh.threadIdx() == 0) { *presult = dst; }
1044#ifndef AMREX_NO_SYCL_REDUCE_WORKAROUND
1052 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(r);
1053 ReduceTuple dst = r;
1054 for (
int istream = 0, nstreams = nblocks.size(); istream < nstreams; ++istream) {
1055 auto dp_stream = dp+istream*maxblocks;
1056 for (
int i = AMREX_GPU_MAX_THREADS*blockIdx.x+threadIdx.x, stride = AMREX_GPU_MAX_THREADS*gridDim.x;
1057 i < nblocks[istream]; i += stride) {
1058 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(r, dp_stream[i]);
1061 Reduce::detail::for_each_parallel<0, ReduceTuple, Ps...>(dst, r);
1062 if (threadIdx.x == 0) { *hp = dst; }
1068 m_result_is_ready =
true;
1069 reduce_data.markValueCalled();
1075 bool m_result_is_ready =
false;
1076 void resetResultReadiness () { m_result_is_ready =
false; }
1081template <
typename T, std::
integral N>
1082T
Sum (N n, T
const* v, T init_val)
1086 using ReduceTuple =
typename decltype(reduce_data)::Type;
1088 ReduceTuple hv = reduce_data.
value(reduce_op);
1092template <
typename T, std::
integral N,
typename F>
1093requires (!std::same_as<T*,std::decay_t<F>>)
1094T
Sum (N n, F
const& f, T init_val)
1098 using ReduceTuple =
typename decltype(reduce_data)::Type;
1100 ReduceTuple hv = reduce_data.
value(reduce_op);
1104template <
typename T, std::
integral N>
1105T
Min (N n, T
const* v, T init_val)
1109 using ReduceTuple =
typename decltype(reduce_data)::Type;
1111 ReduceTuple hv = reduce_data.
value(reduce_op);
1115template <
typename T, std::
integral N,
typename F>
1116requires (!std::same_as<T*,std::decay_t<F>>)
1117T
Min (N n, F
const& f, T init_val)
1121 using ReduceTuple =
typename decltype(reduce_data)::Type;
1123 ReduceTuple hv = reduce_data.
value(reduce_op);
1127template <
typename T, std::
integral N>
1128T
Max (N n, T
const* v, T init_val)
1132 using ReduceTuple =
typename decltype(reduce_data)::Type;
1134 ReduceTuple hv = reduce_data.
value(reduce_op);
1138template <
typename T, std::
integral N,
typename F>
1139requires (!std::same_as<T*,std::decay_t<F>>)
1140T
Max (N n, F
const& f, T init_val)
1144 using ReduceTuple =
typename decltype(reduce_data)::Type;
1146 ReduceTuple hv = reduce_data.
value(reduce_op);
1150template <
typename T, std::
integral N>
1155 using ReduceTuple =
typename decltype(reduce_data)::Type;
1159 auto hv = reduce_data.
value(reduce_op);
1163template <
typename T, std::
integral N,
typename F>
1164requires (!std::same_as<T*,std::decay_t<F>>)
1169 using ReduceTuple =
typename decltype(reduce_data)::Type;
1174 auto hv = reduce_data.
value(reduce_op);
1178template <
typename T, std::
integral N,
typename P>
1179bool AnyOf (N n, T
const* v, P
const& pred)
1187#ifdef AMREX_USE_SYCL
1189 const std::size_t shared_mem_bytes = num_ints*
sizeof(
int);
1192 int* has_any = &(
static_cast<int*
>(gh.sharedMemory())[num_ints-1]);
1193 if (gh.threadIdx() == 0) { *has_any = *dp; }
1199 for (
Long i =
Long(AMREX_GPU_MAX_THREADS)*gh.blockIdx()+gh.threadIdx(),
1200 stride =
Long(AMREX_GPU_MAX_THREADS)*gh.gridDim();
1201 i <
Long(n) && !r; i += stride)
1203 r = pred(v[i]) ? 1 : 0;
1208 if (gh.threadIdx() == 0 && r) { *dp = 1; }
1214 __shared__
int has_any;
1215 if (threadIdx.x == 0) { has_any = *dp; }
1221 for (
Long i =
Long(AMREX_GPU_MAX_THREADS)*blockIdx.x+threadIdx.x,
1222 stride =
Long(AMREX_GPU_MAX_THREADS)*gridDim.x;
1223 i <
Long(n) && !r; i += stride)
1225 r = pred(v[i]) ? 1 : 0;
1229 if (threadIdx.x == 0 && r) *dp = 1;
1236template <
typename P,
int dim>
1247#ifdef AMREX_USE_SYCL
1249 const std::size_t shared_mem_bytes = num_ints*
sizeof(
int);
1252 int* has_any = &(
static_cast<int*
>(gh.sharedMemory())[num_ints-1]);
1253 if (gh.threadIdx() == 0) { *has_any = *dp; }
1259 for (UInt icell = UInt(AMREX_GPU_MAX_THREADS)*UInt(gh.blockIdx())+UInt(gh.threadIdx()),
1260 stride = UInt(AMREX_GPU_MAX_THREADS)*UInt(gh.gridDim());
1261 icell < indexer.
numPts() && !r;
1264 auto iv = indexer.
intVect(icell);
1265 r = amrex::detail::call_f_intvect(pred, iv) ? 1 : 0;
1269 if (gh.threadIdx() == 0 && r) { *dp = 1; }
1276 __shared__
int has_any;
1277 if (threadIdx.x == 0) { has_any = *dp; }
1283 for (UInt icell = UInt(AMREX_GPU_MAX_THREADS)*UInt(blockIdx.x)+UInt(threadIdx.x),
1284 stride = UInt(AMREX_GPU_MAX_THREADS)*UInt(gridDim.x);
1285 icell < indexer.
numPts() && !r;
1288 auto iv = indexer.
intVect(icell);
1289 r = amrex::detail::call_f_intvect(pred, iv) ? 1 : 0;
1293 if (threadIdx.x == 0 && r) *dp = 1;
1304template <
typename... Ts>
1308 using Type = GpuTuple<Ts...>;
1310 template <
typename... Ps>
1311 explicit ReduceData (ReduceOps<Ps...>& reduce_op)
1312 : m_tuple(OpenMP::in_parallel() ? 1 : OpenMP::get_max_threads()),
1313 m_fn_value([&reduce_op,this] () -> Type { return this->value(reduce_op); })
1315 reduce_op.resetResultReadiness();
1316 for (
auto& t : m_tuple) {
1317 Reduce::detail::for_each_init<0, Type, Ps...>(t);
1321 ~ReduceData () =
default;
1322 ReduceData (ReduceData<Ts...>
const&) =
delete;
1323 ReduceData (ReduceData<Ts...> &&) =
delete;
1324 void operator= (ReduceData<Ts...>
const&) =
delete;
1325 void operator= (ReduceData<Ts...> &&) =
delete;
1327 Type value () {
return m_fn_value(); }
1329 template <
typename... Ps>
1330 Type value (ReduceOps<Ps...>& reduce_op)
1332 return reduce_op.value(*
this);
1335 Vector<Type>& reference () {
return m_tuple; }
1337 Type& reference (
int tid)
1339 if (m_tuple.size() == 1) {
1343 return m_tuple[tid];
1348 Vector<Type> m_tuple;
1349 std::function<Type()> m_fn_value;
1352namespace Reduce::detail {
1356 template <
typename F,
int dim>
1358 auto call_f_intvect (F
const& f, IntVectND<dim> iv)
noexcept ->
1359 decltype(amrex::detail::call_f_intvect_inner(std::make_index_sequence<dim>(), f, iv))
1361 return amrex::detail::call_f_intvect_inner(std::make_index_sequence<dim>(), f, iv);
1366 template <
typename F,
typename T,
int dim>
1368 auto call_f_intvect_n (F
const& f, IntVectND<dim> iv, T n)
noexcept ->
1369 decltype(amrex::detail::call_f_intvect_inner(std::make_index_sequence<dim>(), f, iv, n))
1371 return amrex::detail::call_f_intvect_inner(std::make_index_sequence<dim>(), f, iv, n);
1375template <
typename... Ps>
1382 template <
typename D,
typename F,
int dim>
1383 requires (std::same_as<std::decay_t<
decltype(
1384 Reduce::detail::call_f_intvect(std::declval<F const&>(), IntVectND<dim>()))>,
1387 static void call_f_box (BoxND<dim>
const& box,
typename D::Type & r, F
const& f)
noexcept
1389 using ReduceTuple =
typename D::Type;
1391 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(rem);
1392 Reduce::detail::Lanes<ReduceTuple> acc;
1393 Reduce::detail::lanes_set<0>(acc, rem);
1394 int const ilo = box.smallEnd(0);
1395 int const ihi = box.bigEnd(0);
1396 BoxND<dim> rows = box;
1397 rows.setBig(0, ilo);
1398 For(rows, [&] (IntVectND<dim> iv) {
1399 Reduce::detail::reduce_row<Ps...>(ilo, ihi, acc, rem, [&] (
int i) {
1401 return Reduce::detail::call_f_intvect(f, iv);
1404 Reduce::detail::lanes_fold<0, ReduceTuple, Ps...>(acc, r);
1405 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(r, rem);
1408 template <
typename D,
typename F,
int dim>
1409 requires (std::same_as<std::decay_t<
decltype(
1410 std::declval<F const&>()(std::declval<BoxND<dim>
const&>()))>,
1413 static void call_f_box (BoxND<dim>
const& box,
typename D::Type & r, F
const& f)
noexcept
1415 using ReduceTuple =
typename D::Type;
1416 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(r, f(box));
1421 template <FabArrayType MF,
typename D,
typename F>
1422 requires (IsCallable<F, int, int, int, int>::value)
1423 void eval (MF
const& mf, IntVect
const& nghost, D & reduce_data, F
const& f)
1425 using ReduceTuple =
typename D::Type;
1427#pragma omp parallel if (!system::regtest_reduction)
1434 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(rem);
1435 Reduce::detail::Lanes<ReduceTuple> acc;
1436 Reduce::detail::lanes_set<0>(acc, rem);
1437 for (MFIter mfi(mf,
true); mfi.isValid(); ++mfi) {
1438 Box const& b = mfi.growntilebox(nghost);
1439 const int li = mfi.LocalIndex();
1442 for (
int k = lo.z; k <= hi.z; ++k) {
1443 for (
int j = lo.y; j <= hi.y; ++j) {
1444 Reduce::detail::reduce_row<Ps...>(lo.x, hi.x, acc, rem,
1445 [&] (
int i) {
return f(li,i,j,k); });
1448 auto& rr = reduce_data.reference(OpenMP::get_thread_num());
1449 Reduce::detail::lanes_fold<0, ReduceTuple, Ps...>(acc, rr);
1450 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(rr, rem);
1454 template <FabArrayType MF,
typename D,
typename F>
1455 requires (IsCallable<F, int, int, int, int, int>::value)
1456 void eval (MF
const& mf, IntVect
const& nghost,
int ncomp, D & reduce_data, F
const& f)
1458 using ReduceTuple =
typename D::Type;
1460#pragma omp parallel if (!system::regtest_reduction)
1464 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(rem);
1465 Reduce::detail::Lanes<ReduceTuple> acc;
1466 Reduce::detail::lanes_set<0>(acc, rem);
1467 for (MFIter mfi(mf,
true); mfi.isValid(); ++mfi) {
1468 Box const& b = mfi.growntilebox(nghost);
1469 const int li = mfi.LocalIndex();
1472 for (
int n = 0; n < ncomp; ++n) {
1473 for (
int k = lo.z; k <= hi.z; ++k) {
1474 for (
int j = lo.y; j <= hi.y; ++j) {
1475 Reduce::detail::reduce_row<Ps...>(lo.x, hi.x, acc, rem,
1476 [&] (
int i) {
return f(li,i,j,k,n); });
1479 auto& rr = reduce_data.reference(OpenMP::get_thread_num());
1480 Reduce::detail::lanes_fold<0, ReduceTuple, Ps...>(acc, rr);
1481 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(rr, rem);
1485 template <
typename D,
typename F,
int dim>
1486 void eval (BoxND<dim>
const& box, D & reduce_data, F&& f)
1488 using ReduceTuple =
typename D::Type;
1490 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(rr);
1491 call_f_box<D>(box, rr, std::forward<F>(f));
1492 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(
1493 reduce_data.reference(OpenMP::get_thread_num()), rr);
1496 template <std::
integral N,
typename D,
typename F,
int dim>
1497 void eval (BoxND<dim>
const& box, N ncomp, D & reduce_data, F
const& f)
1499 using ReduceTuple =
typename D::Type;
1501 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(rem);
1502 Reduce::detail::Lanes<ReduceTuple> acc;
1503 Reduce::detail::lanes_set<0>(acc, rem);
1504 int const ilo = box.smallEnd(0);
1505 int const ihi = box.bigEnd(0);
1506 BoxND<dim> rows = box;
1507 rows.setBig(0, ilo);
1508 For(rows, ncomp, [&] (IntVectND<dim> iv,
int n) {
1509 Reduce::detail::reduce_row<Ps...>(ilo, ihi, acc, rem, [&] (
int i) {
1511 return Reduce::detail::call_f_intvect_n(f, iv, n);
1514 auto& rr = reduce_data.reference(OpenMP::get_thread_num());
1515 Reduce::detail::lanes_fold<0, ReduceTuple, Ps...>(acc, rr);
1516 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(rr, rem);
1519 template <std::
integral N,
typename D,
typename F>
1520 void eval (N n, D & reduce_data, F
const& f)
1522 using ReduceTuple =
typename D::Type;
1524 Reduce::detail::for_each_init<0, ReduceTuple, Ps...>(rem);
1525 Reduce::detail::Lanes<ReduceTuple> acc;
1526 Reduce::detail::lanes_set<0>(acc, rem);
1528 Reduce::detail::reduce_row<Ps...>(N(0),
static_cast<N
>(n-1), acc, rem,
1529 [&] (N i) {
return f(i); });
1531 auto& rr = reduce_data.reference(OpenMP::get_thread_num());
1532 Reduce::detail::lanes_fold<0, ReduceTuple, Ps...>(acc, rr);
1533 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(rr, rem);
1536 template <
typename D>
1537 typename D::Type value (D & reduce_data)
1539 auto& rrv = reduce_data.reference();
1540 if (! m_result_is_ready) {
1541 using ReduceTuple =
typename D::Type;
1542 if (rrv.size() > 1) {
1543 for (
int i = 1, N = rrv.size(); i < N; ++i) {
1544 Reduce::detail::for_each_local<0, ReduceTuple, Ps...>(rrv[0], rrv[i]);
1547 m_result_is_ready =
true;
1553 template <
typename... T>
friend class ReduceData;
1554 bool m_result_is_ready =
false;
1555 void resetResultReadiness () { m_result_is_ready =
false; }
1560template <
typename T, std::
integral N,
typename F>
1561requires (!std::same_as<T*,std::decay_t<F>>)
1562T Sum (N n, F
const& f, T init_val)
1564 auto const cn = Reduce::detail::chunks_1d(n);
1565 N
const chunk = cn.first;
1566 N
const nchunks = cn.second;
1569#pragma omp parallel for reduction(+:r)
1571 for (N ic = 0; ic < nchunks; ++ic) {
1572 N
const ib = ic*chunk;
1573 N
const ie = (n - ib < chunk) ? n : ib + chunk;
1574 using Tup = GpuTuple<T>;
1576 Reduce::detail::Lanes<Tup> acc;
1577 Reduce::detail::lanes_set<0>(acc, rem);
1578 Reduce::detail::reduce_row<ReduceOpSum>(ib,
static_cast<N
>(ie-1), acc, rem,
1579 [&] (N i) {
return Tup{f(i)}; });
1580 Reduce::detail::lanes_fold<0,Tup,ReduceOpSum>(acc, rem);
1586template <
typename T, std::
integral N>
1587T
Sum (N n, T
const* v, T init_val)
1589 return Sum(n, [=] (N i) -> T {
return v[i]; }, init_val);
1592template <
typename T, std::
integral N,
typename F>
1593requires (!std::same_as<T*,std::decay_t<F>>)
1594T Min (N n, F
const& f, T init_val)
1596 auto const cn = Reduce::detail::chunks_1d(n);
1597 N
const chunk = cn.first;
1598 N
const nchunks = cn.second;
1601#pragma omp parallel for reduction(min:r)
1603 for (N ic = 0; ic < nchunks; ++ic) {
1604 N
const ib = ic*chunk;
1605 N
const ie = (n - ib < chunk) ? n : ib + chunk;
1606 using Tup = GpuTuple<T>;
1608 Reduce::detail::Lanes<Tup> acc;
1609 Reduce::detail::lanes_set<0>(acc, rem);
1610 Reduce::detail::reduce_row<ReduceOpMin>(ib,
static_cast<N
>(ie-1), acc, rem,
1611 [&] (N i) {
return Tup{f(i)}; });
1612 Reduce::detail::lanes_fold<0,Tup,ReduceOpMin>(acc, rem);
1618template <
typename T, std::
integral N>
1619T
Min (N n, T
const* v, T init_val)
1621 return Reduce::Min(n, [=] (N i) -> T {
return v[i]; }, init_val);
1624template <
typename T, std::
integral N,
typename F>
1625requires (!std::same_as<T*,std::decay_t<F>>)
1626T Max (N n, F
const& f, T init_val)
1628 auto const cn = Reduce::detail::chunks_1d(n);
1629 N
const chunk = cn.first;
1630 N
const nchunks = cn.second;
1633#pragma omp parallel for reduction(max:r)
1635 for (N ic = 0; ic < nchunks; ++ic) {
1636 N
const ib = ic*chunk;
1637 N
const ie = (n - ib < chunk) ? n : ib + chunk;
1638 using Tup = GpuTuple<T>;
1640 Reduce::detail::Lanes<Tup> acc;
1641 Reduce::detail::lanes_set<0>(acc, rem);
1642 Reduce::detail::reduce_row<ReduceOpMax>(ib,
static_cast<N
>(ie-1), acc, rem,
1643 [&] (N i) {
return Tup{f(i)}; });
1644 Reduce::detail::lanes_fold<0,Tup,ReduceOpMax>(acc, rem);
1650template <
typename T, std::
integral N>
1651T
Max (N n, T
const* v, T init_val)
1653 return Reduce::Max(n, [=] (N i) -> T {
return v[i]; }, init_val);
1656template <
typename T, std::
integral N,
typename F>
1657requires (!std::same_as<T*,std::decay_t<F>>)
1658std::pair<T,T> MinMax (N n, F
const& f)
1660 auto const cn = Reduce::detail::chunks_1d(n);
1661 N
const chunk = cn.first;
1662 N
const nchunks = cn.second;
1663 T r_min = std::numeric_limits<T>::max();
1664 T r_max = std::numeric_limits<T>::lowest();
1666#pragma omp parallel for reduction(min:r_min) reduction(max:r_max)
1668 for (N ic = 0; ic < nchunks; ++ic) {
1669 N
const ib = ic*chunk;
1670 N
const ie = (n - ib < chunk) ? n : ib + chunk;
1671 using Tup = GpuTuple<T,T>;
1672 Tup rem{std::numeric_limits<T>::max(), std::numeric_limits<T>::lowest()};
1673 Reduce::detail::Lanes<Tup> acc;
1674 Reduce::detail::lanes_set<0>(acc, rem);
1675 Reduce::detail::reduce_row<ReduceOpMin,ReduceOpMax>(ib,
static_cast<N
>(ie-1), acc, rem,
1676 [&] (N i) { T
const tmp = f(i);
return Tup{tmp,tmp}; });
1677 Reduce::detail::lanes_fold<0,Tup,ReduceOpMin,ReduceOpMax>(acc, rem);
1681 return std::make_pair(r_min,r_max);
1684template <
typename T, std::
integral N>
1685std::pair<T,T>
MinMax (N n, T
const* v)
1687 return Reduce::MinMax<T>(n, [=] (N i) -> T {
return v[i]; });
1690template <
typename T, std::
integral N,
typename P>
1691bool AnyOf (N n, T
const* v, P
const& pred)
1693 return std::any_of(v, v+n, pred);
1696template <
typename P,
int dim>
1697bool AnyOf (BoxND<dim>
const& box, P
const& pred)
1699 for (
auto iv : box.iterator()) {
1700 if (Reduce::detail::call_f_intvect(pred, iv)) {
return true; }
1713template <
typename... Ts,
typename... Ps>
1715constexpr GpuTuple<Ts...>
1719 Reduce::detail::for_each_init<0,
decltype(r), Ps...>(r);
1727template <
typename... Ts,
typename... Ps>
1729constexpr GpuTuple<Ts...>
1733 Reduce::detail::for_each_init<0,
decltype(r), Ps...>(r);
1738template <
typename Ops,
typename Ts>
1741template <
typename... Ops,
typename... Ts>
1742class ReducerImpl<TypeList<Ops...>, TypeList<Ts...>>
1745 static_assert(
sizeof...(Ops) > 0);
1746 static_assert(
sizeof...(Ts) > 0);
1747 static_assert(
sizeof...(Ops) ==
sizeof...(Ts));
1750 : m_reduce_data(m_reduce_op)
1754 using Result_t = GpuTuple<Ts...>;
1755 ReduceOps<Ops...> m_reduce_op;
1756 ReduceData<Ts...> m_reduce_data;
1824template <
typename Ops,
typename Ts>
1826 :
public ReducerImpl<ToTypeList_t<Ops>, ToTypeList_t<Ts>>
1840 void operator= (
Reducer const&) =
delete;
1841 void operator= (
Reducer &&) =
delete;
1857 template <
typename F,
int dim>
1862 this->m_reduce_op.eval(box, this->m_reduce_data, std::forward<F>(f));
1880 template <
typename F,
int dim>
1885 this->m_reduce_op.eval(box, ncomp, this->m_reduce_data, std::forward<F>(f));
1907 template <FabArrayType MF,
typename F>
1911 this->m_reduce_op.eval(mf, nghost, this->m_reduce_data, std::forward<F>(f));
1936 template <FabArrayType MF,
typename F>
1940 this->m_reduce_op.eval(mf, nghost, ncomp, this->m_reduce_data, std::forward<F>(f));
1955 template <
typename N,
typename F>
1959 this->m_reduce_op.eval(n, this->m_reduce_data, std::forward<F>(f));
1974 return this->m_reduce_data.value(this->m_reduce_op);
Runtime initialization/finalization helpers and global diagnostics.
Memory arena base class and global arena accessors.
#define AMREX_ALWAYS_ASSERT_WITH_MESSAGE(EX, MSG)
Definition AMReX_BLassert.H:49
#define AMREX_ASSERT(EX)
Definition AMReX_BLassert.H:38
#define AMREX_FORCE_INLINE
Definition AMReX_Extension.H:123
#define AMREX_GPU_MAX_STREAMS
Definition AMReX_GpuDevice.H:21
#define AMREX_LAUNCH_KERNEL(MT, blocks, threads, sharedMem, stream,...)
Definition AMReX_GpuLaunch.H:37
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
#define AMREX_GPU_HOST_DEVICE
Definition AMReX_GpuQualifiers.H:20
Convenience header for the core AMReX GPU facilities.
virtual void free(void *pt)=0
Free a previously allocated block pointed to by pt.
A Rectangular Domain on an Integer Lattice.
Definition AMReX_Box.H:54
__host__ __device__ bool isEmpty() const noexcept
Checks if it is an empty BoxND.
Definition AMReX_Box.H:223
__host__ __device__ Long numPts() const noexcept
Return the number of points contained in the BoxND.
Definition AMReX_Box.H:385
__host__ __device__ IndexTypeND< dim > ixType() const noexcept
Return the indexing type.
Definition AMReX_Box.H:148
GPU-compatible tuple.
Definition AMReX_Tuple.H:104
static int streamIndex(gpuStream_t s=gpuStream()) noexcept
Definition AMReX_GpuDevice.cpp:846
static unsigned int maxBlocksPerLaunch() noexcept
Definition AMReX_GpuDevice.H:242
static constexpr int warp_size
Definition AMReX_GpuDevice.H:239
static bool usingExternalStream() noexcept
Definition AMReX_GpuDevice.cpp:973
Cell-Based or Node-Based Indices.
Definition AMReX_IndexType.H:36
An Integer Vector in dim-Dimensional Space.
Definition AMReX_IntVect.H:156
Iterator for looping ever tiles and boxes of amrex::FabArray based containers.
Definition AMReX_MFIter.H:88
bool isValid() const noexcept
Is the iterator valid i.e. is it associated with a FAB?
Definition AMReX_MFIter.H:176
T * data() noexcept
Definition AMReX_PODVector.H:672
Definition AMReX_Reduce.H:556
~ReduceData()
Definition AMReX_Reduce.H:578
int maxStreamIndex() const
Definition AMReX_Reduce.H:618
Type value()
Definition AMReX_Reduce.H:591
void updateMaxStreamIndex(gpuStream_t const &s)
Definition AMReX_Reduce.H:619
int & nBlocks(gpuStream_t const &s)
Definition AMReX_Reduce.H:614
GpuTuple< Ts... > Type
Definition AMReX_Reduce.H:558
ReduceData(ReduceOps< Ps... > &reduce_op)
Definition AMReX_Reduce.H:561
void markValueCalled() noexcept
Definition AMReX_Reduce.H:623
Type * devicePtr(gpuStream_t const &s)
Definition AMReX_Reduce.H:607
Type value(ReduceOps< Ps... > &reduce_op)
Definition AMReX_Reduce.H:599
Type * devicePtr()
Definition AMReX_Reduce.H:606
GpuArray< int, 8 > & nBlocks()
Definition AMReX_Reduce.H:613
ReduceData(ReduceData< Ts... > const &)=delete
Type * hostPtr()
Definition AMReX_Reduce.H:611
int maxBlocks() const
Definition AMReX_Reduce.H:616
ReduceData(ReduceData< Ts... > &&)=delete
Definition AMReX_Reduce.H:715
void eval(BoxND< dim > const &box, N ncomp, D &reduce_data, F const &f)
Definition AMReX_Reduce.H:919
D::Type value(D &reduce_data)
Definition AMReX_Reduce.H:986
void eval(BoxND< dim > const &box, D &reduce_data, F const &f)
Definition AMReX_Reduce.H:913
void eval(MF const &mf, IntVect const &nghost, D &reduce_data, F &&f)
Definition AMReX_Reduce.H:855
friend class ReduceData
Definition AMReX_Reduce.H:1074
void eval(MF const &mf, IntVect const &nghost, int ncomp, D &reduce_data, F &&f)
Definition AMReX_Reduce.H:881
void eval(N n, D &reduce_data, F const &f)
Definition AMReX_Reduce.H:934
static constexpr int size
Number of reduction operators.
Definition AMReX_Reduce.H:1832
Result_t getResult()
Get the final reduction result.
Definition AMReX_Reduce.H:1972
typename Base::Result_t Result_t
Reduction result type, GpuTuple<U...>, where U... are the types in Ts.
Definition AMReX_Reduce.H:1831
void eval(BoxND< dim > const &box, F &&f)
Reduction over a Box.
Definition AMReX_Reduce.H:1860
amrex_long Long
Definition AMReX_INT.H:30
T Min(N n, T const *v, T init_val=std::numeric_limits< T >::max())
Compute the minimum of an array of values.
Definition AMReX_Reduce.H:1105
bool AnyOf(N n, T const *v, P const &pred)
Test whether any element in an array satisfies a unary predicate.
Definition AMReX_Reduce.H:1179
std::pair< T, T > MinMax(N n, T const *v)
Compute the minimum and maximum of an array of values.
Definition AMReX_Reduce.H:1151
T Max(N n, T const *v, T init_val=std::numeric_limits< T >::lowest())
Compute the maximum of an array of values.
Definition AMReX_Reduce.H:1128
T Sum(N n, T const *v, T init_val=0)
Compute the sum of an array of values.
Definition AMReX_Reduce.H:1082
__host__ __device__ Dim3 ubound(Array4< T > const &a) noexcept
Return the inclusive upper bounds of an Array4 in Dim3 form.
Definition AMReX_Array4.H:1377
__host__ __device__ Dim3 lbound(Array4< T > const &a) noexcept
Return the inclusive lower bounds of an Array4 in Dim3 form.
Definition AMReX_Array4.H:1363
__host__ __device__ BoxND< dim > grow(const BoxND< dim > &b, int i) noexcept
Return a copy of b grown uniformly by i cells in every direction.
Definition AMReX_Box.H:1328
PODVector< T, PinnedArenaAllocator< T > > PinnedVector
A PODVector that uses the pinned memory arena.
Definition AMReX_GpuContainers.H:48
PODVector< T, ArenaAllocator< T > > DeviceVector
A PODVector that uses the standard memory Arena. Note that the memory might or might not be managed d...
Definition AMReX_GpuContainers.H:24
Arena * The_Pinned_Arena()
Definition AMReX_Arena.cpp:879
Arena * The_Arena()
Definition AMReX_Arena.cpp:839
void Sum(Gpu::DeviceVector< T > &v, MPI_Comm comm)
Definition AMReX_GpuParallelReduce.H:37
__host__ __device__ constexpr const T & min(const T &a, const T &b) noexcept
Definition AMReX_Algorithm.H:31
__host__ __device__ constexpr const T & max(const T &a, const T &b) noexcept
Definition AMReX_Algorithm.H:54
__host__ __device__ AMREX_FORCE_INLINE T Max(T *const m, T const value) noexcept
Definition AMReX_GpuAtomic.H:474
__host__ __device__ AMREX_FORCE_INLINE T Min(T *const m, T const value) noexcept
Definition AMReX_GpuAtomic.H:397
Definition AMReX_BaseFwd.H:60
__device__ T blockReduce(T x, WARPREDUCE &&warp_reduce, T x0)
Definition AMReX_GpuReduce.H:317
__device__ int blockReduceLogicalOr(int source) noexcept
Definition AMReX_GpuReduce.H:559
__device__ T blockReduceMax(T source) noexcept
Definition AMReX_GpuReduce.H:458
__device__ T blockReduceMin(T source) noexcept
Definition AMReX_GpuReduce.H:403
__device__ int blockReduceLogicalAnd(int source) noexcept
Definition AMReX_GpuReduce.H:511
void streamSynchronize() noexcept
Definition AMReX_GpuDevice.H:329
void dtoh_memcpy_async(void *p_h, const void *p_d, const std::size_t sz) noexcept
Definition AMReX_GpuDevice.H:454
gpuStream_t gpuStream() noexcept
Definition AMReX_GpuDevice.H:310
__device__ T blockReduceSum(T source) noexcept
Definition AMReX_GpuReduce.H:353
constexpr int get_max_threads()
Definition AMReX_OpenMP.H:36
Definition AMReX_Reduce.H:22
Definition AMReX_Amr.cpp:50
__host__ __device__ void ignore_unused(const Ts &...)
No-op helper that marks variables as intentionally unused.
Definition AMReX.H:273
@ F
Definition AMReX_Order.H:7
void For(TypeList< CTOs... > ctos, std::array< int, sizeof...(CTOs)> const &runtime_options, T N, F &&f)
Definition AMReX_CTOParallelForImpl.H:400
@ Device
Definition AMReX_GpuControl.H:65
__host__ __device__ constexpr GpuTuple< Ts... > IdentityTuple(GpuTuple< Ts... >, ReduceOps< Ps... >) noexcept
Return a GpuTuple containing the identity element for each operation in ReduceOps....
Definition AMReX_Reduce.H:1716
void launch(T const &n, L const &f) noexcept
Definition AMReX_GpuLaunchFunctsC.H:122
BoxND< 3 > Box
Box is an alias for amrex::BoxND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:35
typename ToTypeList< T >::type ToTypeList_t
Definition AMReX_TypeList.H:233
IntVectND< 3 > IntVect
IntVect is an alias for amrex::IntVectND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:38
cudaStream_t gpuStream_t
Definition AMReX_GpuControl.H:79
BoxIndexerND< 3 > BoxIndexer
Definition AMReX_Box.H:2641
const int[]
Definition AMReX_BLProfiler.cpp:1665
__global__ void launch_global(L f0, Lambdas... fs)
Definition AMReX_GpuLaunch.H:68
__host__ __device__ constexpr int get(IntVectND< dim > const &iv) noexcept
Get the Idx-th element of IntVectND<dim>.
Definition AMReX_IntVect.H:1367
Utility that maps flattened point indices back to IntVectND coordinates.
Definition AMReX_Box.H:2509
detail::BoxIndexerUInt UInt
Definition AMReX_Box.H:2511
__host__ __device__ IntVectND< dim > intVect(std::uint64_t icell) const
Convert flattened point index icell to its IntVectND coordinate.
Definition AMReX_Box.H:2541
__host__ __device__ UInt numPts() const
Return the number of points covered by the indexed box.
Definition AMReX_Box.H:2577
Fixed-size array that can be used on GPU.
Definition AMReX_Array.H:52
Definition AMReX_Tuple.H:133
Definition AMReX_GpuMemory.H:57
T * dataPtr()
Definition AMReX_GpuMemory.H:91
T dataValue() const
Definition AMReX_GpuMemory.H:93
Definition AMReX_GpuLaunch.H:121
Definition AMReX_GpuTypes.H:86
Definition AMReX_GpuControl.H:127
Definition AMReX_GpuReduce.H:290
Test if a given type T is callable with arguments of type Args...
Definition AMReX_TypeTraits.H:208
Fast division and modulus by a fixed 32-bit divisor, as FastDivmodU64, for dividends below 2^32-1.
Definition AMReX_Math.H:757
Function object that returns the sum of two values.
Definition AMReX_Functional.H:23
Definition AMReX_Reduce.H:409
__host__ __device__ void local_update(T &d, T s) const noexcept
Definition AMReX_Reduce.H:430
constexpr void init(T &t) const noexcept
Definition AMReX_Reduce.H:433
__device__ void parallel_update(T &d, T s) const noexcept
Definition AMReX_Reduce.H:421
Definition AMReX_Reduce.H:438
__device__ void parallel_update(T &d, T s) const noexcept
Definition AMReX_Reduce.H:450
__host__ __device__ void local_update(T &d, T s) const noexcept
Definition AMReX_Reduce.H:459
constexpr void init(T &t) const noexcept
Definition AMReX_Reduce.H:462
Definition AMReX_Reduce.H:375
constexpr void init(T &t) const noexcept
Definition AMReX_Reduce.H:400
__host__ __device__ void local_update(T &d, T const &s) const noexcept
Definition AMReX_Reduce.H:396
__device__ void parallel_update(T &d, T const &s) const noexcept
Definition AMReX_Reduce.H:387
Definition AMReX_Reduce.H:341
constexpr void init(T &t) const noexcept
Definition AMReX_Reduce.H:366
__device__ void parallel_update(T &d, T const &s) const noexcept
Definition AMReX_Reduce.H:353
__host__ __device__ void local_update(T &d, T const &s) const noexcept
Definition AMReX_Reduce.H:362
Definition AMReX_Reduce.H:311
__device__ void parallel_update(T &d, T const &s) const noexcept
Definition AMReX_Reduce.H:324
__host__ __device__ void local_update(T &d, T const &s) const noexcept
Definition AMReX_Reduce.H:333
constexpr void init(T &t) const noexcept
Definition AMReX_Reduce.H:336
Struct for holding types.
Definition AMReX_TypeList.H:13