1#ifndef AMREX_MF_PARALLEL_FOR_G_H_
2#define AMREX_MF_PARALLEL_FOR_G_H_
3#include <AMReX_Config.H>
14namespace amrex::detail {
18void build_par_for_boxes (
char*& hp,
BoxIndexer*& pboxes, Vector<Box>
const& boxes,
19 Vector<int>
const& extra = {})
21 if (boxes.empty()) {
return; }
23 const int nboxes = boxes.size();
24 const std::size_t nbytes_boxes = nboxes*
sizeof(
BoxIndexer);
25 const std::size_t nbytes = nbytes_boxes + extra.size()*
sizeof(
int);
28 for (
int i = 0; i < nboxes; ++i) {
31 auto* hp_extra = (
int*)(hp + nbytes_boxes);
32 for (
int i = 0; i <
int(extra.size()); ++i) {
33 hp_extra[i] = extra[i];
42void destroy_par_for_boxes (
char* hp,
char* dp)
54namespace parfor_mf_detail {
57 auto call_f (F
const& f,
int b,
int i,
int j,
int k,
int)
noexcept
58 ->
decltype(f(0,0,0,0))
65 auto call_f (F
const& f,
int b,
int i,
int j,
int k,
int ncomp)
noexcept
66 ->
decltype(f(0,0,0,0,0))
68 for (
int n = 0; n < ncomp; ++n) {
74template <
int MT, FabArrayType MF,
typename F>
76ParallelFor_doit (MF
const& mf,
IntVect const& nghost,
int ncomp,
IntVect const&,
bool,
F const& f)
78 const auto& index_array = mf.IndexArray();
79 const int nboxes = index_array.size();
83 }
else if (nboxes == 1) {
87 parfor_mf_detail::call_f(f, 0, i, j, k, ncomp);
90 auto const& parforinfo = mf.getParForInfo(nghost);
91 auto nblocks_per_box = parforinfo.getNBlocksPerBox(MT);
93 const int nblocks = nblocks_per_box * nboxes;
94 const BoxIndexer* dp_boxes = parforinfo.getBoxes();
96 const Math::FastDivmodU32 fdm_box(nblocks_per_box);
98#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
104 std::uint32_t ibox, iblock;
105 fdm_box(ibox, iblock, blockIdx.x);
108#elif defined(AMREX_USE_SYCL)
113 auto blockIdxx = std::uint32_t(item.get_group_linear_id());
114 int threadIdxx = item.get_local_linear_id();
115 std::uint32_t ibox, iblock;
116 fdm_box(ibox, iblock, blockIdxx);
120 if (icell < indexer.numPts()) {
121 auto [i, j, k] = indexer(icell);
122 parfor_mf_detail::call_f(f,
int(ibox), i, j, k, ncomp);
129template <FabArrayType MF,
typename F>
131ParallelFor_doit (MF
const& mf,
IntVect const& nghost,
int ncomp,
IntVect const& ts,
bool dynamic,
F&& f)
134 constexpr int MT = 128;
136 constexpr int MT = AMREX_GPU_MAX_THREADS;
138 ParallelFor_doit<MT>(mf, nghost, ncomp, ts, dynamic, std::forward<F>(f));
141template <FabArrayType MF,
typename F>
143ParallelFor_doit (MF
const& mf,
IntVect const& nghost,
IntVect const& ts,
bool dynamic,
F&& f)
146 constexpr int MT = 128;
148 constexpr int MT = AMREX_GPU_MAX_THREADS;
150 ParallelFor_doit<MT>(mf, nghost, 1, ts, dynamic, std::forward<F>(f));
155template <
int MT, FabArrayType MF,
typename F>
157ParallelForStrided_doit (MF
const& mf,
IntVect const& stride,
IntVect const& offset,
F const& f)
159 const int nboxes = mf.IndexArray().size();
160 if (nboxes == 0) {
return; }
162 auto const& parforinfo = mf.getParForInfo(stride, offset);
163 const auto nblocks_per_box = parforinfo.getNBlocksPerBox(MT);
164 if (nblocks_per_box == 0) {
return; }
166 const int nblocks = nblocks_per_box * nboxes;
167 const BoxIndexer* dp_boxes = parforinfo.getBoxes();
168 const Math::FastDivmodU32 fdm_box(nblocks_per_box);
170#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
176 std::uint32_t ibox, iblock;
177 fdm_box(ibox, iblock, blockIdx.x);
180#elif defined(AMREX_USE_SYCL)
185 auto blockIdxx = std::uint32_t(item.get_group_linear_id());
186 int threadIdxx = item.get_local_linear_id();
187 std::uint32_t ibox, iblock;
188 fdm_box(ibox, iblock, blockIdxx);
192 if (icell < indexer.numPts()) {
193 IntVect iv = indexer.intVect(icell);
194 for (
int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
195 iv[idim] = indexer.lo[idim] + (iv[idim]-indexer.lo[idim])*stride[idim];
197 auto const d3 = iv.
dim3();
198 f(
int(ibox), d3.x, d3.y, d3.z);
206template <
int MT, FabArrayType MF,
typename F>
208ParallelForRedBlack_doit (MF
const& mf,
int redblack,
F const& f)
210 const int nboxes = mf.IndexArray().size();
211 if (nboxes == 0) {
return; }
213 auto const& parforinfo = mf.getParForInfoRedBlack();
214 const auto nblocks_per_box = parforinfo.getNBlocksPerBox(MT);
215 if (nblocks_per_box == 0) {
return; }
217 const int nblocks = nblocks_per_box * nboxes;
218 const BoxIndexer* dp_boxes = parforinfo.getBoxes();
219 const int* dp_xbounds = parforinfo.getXBounds();
220 const Math::FastDivmodU32 fdm_box(nblocks_per_box);
222#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
228 std::uint32_t ibox, iblock;
229 fdm_box(ibox, iblock, blockIdx.x);
232#elif defined(AMREX_USE_SYCL)
237 auto blockIdxx = std::uint32_t(item.get_group_linear_id());
238 int threadIdxx = item.get_local_linear_id();
239 std::uint32_t ibox, iblock;
240 fdm_box(ibox, iblock, blockIdxx);
245 if (icell < indexer.numPts()) {
246 auto [ic, j, k] = indexer(icell);
247 const int i = 2*ic + ((j+k+redblack) & 1);
248 if (i >= dp_xbounds[2*ibox] && i <= dp_xbounds[2*ibox+1]) {
249 f(
int(ibox), i, j, k);
256template <FabArrayType MF,
typename F>
258ParallelForStrided_doit (MF
const& mf,
IntVect const& stride,
IntVect const& offset,
F const& f)
260 const auto& index_array = mf.IndexArray();
261 if (index_array.size() == 1) {
262 Box const& b = strided_box(mf.box(index_array[0]), stride, offset);
263 const IntVect lo = b.smallEnd();
267 j = lo[1] + (j-lo[1])*stride[1];,
268 k = lo[2] + (k-lo[2])*stride[2];)
273 constexpr int MT = 128;
275 constexpr int MT = AMREX_GPU_MAX_THREADS;
277 ParallelForStrided_doit<MT>(mf, stride, offset, f);
281template <FabArrayType MF,
typename F>
283ParallelForRedBlack_doit (MF
const& mf,
int redblack,
F const& f)
285 const auto& index_array = mf.IndexArray();
286 if (index_array.size() == 1) {
288 Box b = mf.box(index_array[0]);
290 const int xhi = b.bigEnd(0);
295 const int i = 2*ic + ((j+k+redblack) & 1);
296 if (i >= xlo && i <= xhi) {
302 constexpr int MT = 128;
304 constexpr int MT = AMREX_GPU_MAX_THREADS;
306 ParallelForRedBlack_doit<MT>(mf, redblack, f);
#define AMREX_ASSERT(EX)
Definition AMReX_BLassert.H:38
#define AMREX_GPU_ERROR_CHECK()
Definition AMReX_GpuError.H:151
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
#define AMREX_D_TERM(a, b, c)
Definition AMReX_SPACE.H:172
virtual void * alloc(std::size_t sz)=0
Allocate sz bytes from this arena.
__host__ __device__ const IntVectND< dim > & smallEnd() const &noexcept
Return the inclusive lower bound of the box.
Definition AMReX_Box.H:124
__host__ __device__ constexpr Dim3 dim3() const noexcept
Definition AMReX_IntVect.H:269
amrex_long Long
Definition AMReX_INT.H:30
__host__ __device__ BoxND< dim > coarsen(const BoxND< dim > &b, int ref_ratio) noexcept
Return a copy of b coarsened by the isotropic ratio ref_ratio.
Definition AMReX_Box.H:1471
__host__ __device__ BoxND< dim > grow(const BoxND< dim > &b, int i) noexcept
Return a copy of b grown uniformly by i cells in every direction.
Definition AMReX_Box.H:1328
Arena * The_Pinned_Arena()
Definition AMReX_Arena.cpp:879
Arena * The_Arena()
Definition AMReX_Arena.cpp:839
void freeAsync(Arena *arena, void *mem) noexcept
Definition AMReX_GpuDevice.H:364
void htod_memcpy_async(void *p_d, const void *p_h, const std::size_t sz) noexcept
Definition AMReX_GpuDevice.H:440
gpuStream_t gpuStream() noexcept
Definition AMReX_GpuDevice.H:310
@ F
Definition AMReX_Order.H:7
void launch(T const &n, L const &f) noexcept
Definition AMReX_GpuLaunchFunctsC.H:122
void ParallelFor(TypeList< CTOs... > ctos, std::array< int, sizeof...(CTOs)> const &runtime_options, T N, F &&f)
Definition AMReX_CTOParallelForImpl.H:202
BoxND< 3 > Box
Box is an alias for amrex::BoxND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:35
IntVectND< 3 > IntVect
IntVect is an alias for amrex::IntVectND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:38
BoxIndexerND< 3 > BoxIndexer
Definition AMReX_Box.H:2641
const int[]
Definition AMReX_BLProfiler.cpp:1665
__global__ void launch_global(L f0, Lambdas... fs)
Definition AMReX_GpuLaunch.H:68
detail::BoxIndexerUInt UInt
Definition AMReX_Box.H:2511