Block-Structured AMR Software Framework
Loading...
Searching...
No Matches
AMReX_MFParallelForG.H
Go to the documentation of this file.
1#ifndef AMREX_MF_PARALLEL_FOR_G_H_
2#define AMREX_MF_PARALLEL_FOR_G_H_
3#include <AMReX_Config.H>
4
5#ifdef AMREX_USE_GPU
6
7#include <AMReX_Concepts.H>
8#include <AMReX_GpuDevice.H>
9#include <algorithm>
10#include <cmath>
11#include <limits>
12
14namespace amrex::detail {
15
16// Copies the BoxIndexers of boxes, followed by the ints in extra, to the device.
17inline
18void build_par_for_boxes (char*& hp, BoxIndexer*& pboxes, Vector<Box> const& boxes,
19 Vector<int> const& extra = {})
20{
21 if (boxes.empty()) { return; }
22 static_assert(sizeof(BoxIndexer) % alignof(int) == 0);
23 const int nboxes = boxes.size();
24 const std::size_t nbytes_boxes = nboxes*sizeof(BoxIndexer);
25 const std::size_t nbytes = nbytes_boxes + extra.size()*sizeof(int);
26 hp = (char*)The_Pinned_Arena()->alloc(nbytes);
27 auto* hp_boxes = (BoxIndexer*)hp;
28 for (int i = 0; i < nboxes; ++i) {
29 new (hp_boxes+i) BoxIndexer(boxes[i]);
30 }
31 auto* hp_extra = (int*)(hp + nbytes_boxes);
32 for (int i = 0; i < int(extra.size()); ++i) {
33 hp_extra[i] = extra[i];
34 }
35
36 auto dp = (char*) The_Arena()->alloc(nbytes);
37 Gpu::htod_memcpy_async(dp, hp, nbytes);
38 pboxes = (BoxIndexer*)dp;
39}
40
41inline
42void destroy_par_for_boxes (char* hp, char* dp)
43{
44 // The host-to-device copy in build_par_for_boxes() is asynchronous. In a
45 // NoSync region it may still be pending when the owning ParForInfo is
46 // destroyed (e.g., when a temporary FabArray with a unique BoxArray dies).
47 // Use stream-ordered frees so that the buffers are not reused too early.
48 if (hp) {
51 }
52}
53
54namespace parfor_mf_detail {
55 template <typename F>
57 auto call_f (F const& f, int b, int i, int j, int k, int) noexcept
58 -> decltype(f(0,0,0,0))
59 {
60 f(b,i,j,k);
61 }
62
63 template <typename F>
65 auto call_f (F const& f, int b, int i, int j, int k, int ncomp) noexcept
66 -> decltype(f(0,0,0,0,0))
67 {
68 for (int n = 0; n < ncomp; ++n) {
69 f(b,i,j,k,n);
70 }
71 }
72}
73
74template <int MT, FabArrayType MF, typename F>
75void
76ParallelFor_doit (MF const& mf, IntVect const& nghost, int ncomp, IntVect const&, bool, F const& f)
77{
78 const auto& index_array = mf.IndexArray();
79 const int nboxes = index_array.size();
80
81 if (nboxes == 0) {
82 return;
83 } else if (nboxes == 1) {
84 Box const& b = amrex::grow(mf.box(index_array[0]), nghost);
85 amrex::ParallelFor(b, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
86 {
87 parfor_mf_detail::call_f(f, 0, i, j, k, ncomp);
88 });
89 } else {
90 auto const& parforinfo = mf.getParForInfo(nghost);
91 auto nblocks_per_box = parforinfo.getNBlocksPerBox(MT);
92 AMREX_ASSERT(Long(nblocks_per_box)*Long(nboxes) < Long(std::numeric_limits<int>::max()));
93 const int nblocks = nblocks_per_box * nboxes;
94 const BoxIndexer* dp_boxes = parforinfo.getBoxes();
95 // Box and block within the box from the block index, without a division.
96 const Math::FastDivmodU32 fdm_box(nblocks_per_box);
97
98#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
99
101 <<<nblocks, MT, 0, Gpu::gpuStream()>>>
102 ([=] AMREX_GPU_DEVICE () noexcept
103 {
104 std::uint32_t ibox, iblock;
105 fdm_box(ibox, iblock, blockIdx.x);
106 auto icell = BoxIndexer::UInt(iblock)*MT + threadIdx.x;
107
108#elif defined(AMREX_USE_SYCL)
109
111 [=] AMREX_GPU_DEVICE (sycl::nd_item<1> const& item) noexcept
112 {
113 auto blockIdxx = std::uint32_t(item.get_group_linear_id());
114 int threadIdxx = item.get_local_linear_id();
115 std::uint32_t ibox, iblock;
116 fdm_box(ibox, iblock, blockIdxx);
117 auto icell = BoxIndexer::UInt(iblock)*MT + threadIdxx;
118#endif
119 BoxIndexer const& indexer = dp_boxes[ibox];
120 if (icell < indexer.numPts()) {
121 auto [i, j, k] = indexer(icell);
122 parfor_mf_detail::call_f(f, int(ibox), i, j, k, ncomp);
123 }
124 });
125 }
127}
128
129template <FabArrayType MF, typename F>
130void
131ParallelFor_doit (MF const& mf, IntVect const& nghost, int ncomp, IntVect const& ts, bool dynamic, F&& f)
132{
133#ifdef AMREX_USE_CUDA
134 constexpr int MT = 128;
135#else
136 constexpr int MT = AMREX_GPU_MAX_THREADS;
137#endif
138 ParallelFor_doit<MT>(mf, nghost, ncomp, ts, dynamic, std::forward<F>(f));
139}
140
141template <FabArrayType MF, typename F>
142void
143ParallelFor_doit (MF const& mf, IntVect const& nghost, IntVect const& ts, bool dynamic, F&& f)
144{
145#ifdef AMREX_USE_CUDA
146 constexpr int MT = 128;
147#else
148 constexpr int MT = AMREX_GPU_MAX_THREADS;
149#endif
150 ParallelFor_doit<MT>(mf, nghost, 1, ts, dynamic, std::forward<F>(f));
151}
152
153// One launch over the valid points of all boxes whose index equals offset
154// modulo stride.
155template <int MT, FabArrayType MF, typename F>
156void
157ParallelForStrided_doit (MF const& mf, IntVect const& stride, IntVect const& offset, F const& f)
158{
159 const int nboxes = mf.IndexArray().size();
160 if (nboxes == 0) { return; }
161
162 auto const& parforinfo = mf.getParForInfo(stride, offset);
163 const auto nblocks_per_box = parforinfo.getNBlocksPerBox(MT);
164 if (nblocks_per_box == 0) { return; }
165 AMREX_ASSERT(Long(nblocks_per_box)*Long(nboxes) < Long(std::numeric_limits<int>::max()));
166 const int nblocks = nblocks_per_box * nboxes;
167 const BoxIndexer* dp_boxes = parforinfo.getBoxes();
168 const Math::FastDivmodU32 fdm_box(nblocks_per_box);
169
170#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
171
173 <<<nblocks, MT, 0, Gpu::gpuStream()>>>
174 ([=] AMREX_GPU_DEVICE () noexcept
175 {
176 std::uint32_t ibox, iblock;
177 fdm_box(ibox, iblock, blockIdx.x);
178 auto icell = BoxIndexer::UInt(iblock)*MT + threadIdx.x;
179
180#elif defined(AMREX_USE_SYCL)
181
183 [=] AMREX_GPU_DEVICE (sycl::nd_item<1> const& item) noexcept
184 {
185 auto blockIdxx = std::uint32_t(item.get_group_linear_id());
186 int threadIdxx = item.get_local_linear_id();
187 std::uint32_t ibox, iblock;
188 fdm_box(ibox, iblock, blockIdxx);
189 auto icell = BoxIndexer::UInt(iblock)*MT + threadIdxx;
190#endif
191 BoxIndexer const& indexer = dp_boxes[ibox];
192 if (icell < indexer.numPts()) {
193 IntVect iv = indexer.intVect(icell);
194 for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
195 iv[idim] = indexer.lo[idim] + (iv[idim]-indexer.lo[idim])*stride[idim];
196 }
197 auto const d3 = iv.dim3();
198 f(int(ibox), d3.x, d3.y, d3.z);
199 }
200 });
202}
203
204// One launch over the red or black points of all boxes, as in the single-box
205// path: thread ic of a row takes i = 2*ic or 2*ic+1, whichever has the color.
206template <int MT, FabArrayType MF, typename F>
207void
208ParallelForRedBlack_doit (MF const& mf, int redblack, F const& f)
209{
210 const int nboxes = mf.IndexArray().size();
211 if (nboxes == 0) { return; }
212
213 auto const& parforinfo = mf.getParForInfoRedBlack();
214 const auto nblocks_per_box = parforinfo.getNBlocksPerBox(MT);
215 if (nblocks_per_box == 0) { return; }
216 AMREX_ASSERT(Long(nblocks_per_box)*Long(nboxes) < Long(std::numeric_limits<int>::max()));
217 const int nblocks = nblocks_per_box * nboxes;
218 const BoxIndexer* dp_boxes = parforinfo.getBoxes();
219 const int* dp_xbounds = parforinfo.getXBounds();
220 const Math::FastDivmodU32 fdm_box(nblocks_per_box);
221
222#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
223
225 <<<nblocks, MT, 0, Gpu::gpuStream()>>>
226 ([=] AMREX_GPU_DEVICE () noexcept
227 {
228 std::uint32_t ibox, iblock;
229 fdm_box(ibox, iblock, blockIdx.x);
230 auto icell = BoxIndexer::UInt(iblock)*MT + threadIdx.x;
231
232#elif defined(AMREX_USE_SYCL)
233
235 [=] AMREX_GPU_DEVICE (sycl::nd_item<1> const& item) noexcept
236 {
237 auto blockIdxx = std::uint32_t(item.get_group_linear_id());
238 int threadIdxx = item.get_local_linear_id();
239 std::uint32_t ibox, iblock;
240 fdm_box(ibox, iblock, blockIdxx);
241 auto icell = BoxIndexer::UInt(iblock)*MT + threadIdxx;
242#endif
243 // The box coarsened by 2 in x, and the box's x bounds.
244 BoxIndexer const& indexer = dp_boxes[ibox];
245 if (icell < indexer.numPts()) {
246 auto [ic, j, k] = indexer(icell);
247 const int i = 2*ic + ((j+k+redblack) & 1);
248 if (i >= dp_xbounds[2*ibox] && i <= dp_xbounds[2*ibox+1]) {
249 f(int(ibox), i, j, k);
250 }
251 }
252 });
254}
255
256template <FabArrayType MF, typename F>
257void
258ParallelForStrided_doit (MF const& mf, IntVect const& stride, IntVect const& offset, F const& f)
259{
260 const auto& index_array = mf.IndexArray();
261 if (index_array.size() == 1) {
262 Box const& b = strided_box(mf.box(index_array[0]), stride, offset);
263 const IntVect lo = b.smallEnd();
264 amrex::ParallelFor(b, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
265 {
266 AMREX_D_TERM(i = lo[0] + (i-lo[0])*stride[0];,
267 j = lo[1] + (j-lo[1])*stride[1];,
268 k = lo[2] + (k-lo[2])*stride[2];)
269 f(0, i, j, k);
270 });
271 } else {
272#ifdef AMREX_USE_CUDA
273 constexpr int MT = 128;
274#else
275 constexpr int MT = AMREX_GPU_MAX_THREADS;
276#endif
277 ParallelForStrided_doit<MT>(mf, stride, offset, f);
278 }
279}
280
281template <FabArrayType MF, typename F>
282void
283ParallelForRedBlack_doit (MF const& mf, int redblack, F const& f)
284{
285 const auto& index_array = mf.IndexArray();
286 if (index_array.size() == 1) {
287 // Checkerboard: i = 2*ic + ((j+k+redblack)&1)
288 Box b = mf.box(index_array[0]);
289 const int xlo = b.smallEnd(0);
290 const int xhi = b.bigEnd(0);
291 b.setSmall(0, amrex::coarsen(xlo,2));
292 b.setBig (0, amrex::coarsen(xhi,2));
293 amrex::ParallelFor(b, [=] AMREX_GPU_DEVICE (int ic, int j, int k) noexcept
294 {
295 const int i = 2*ic + ((j+k+redblack) & 1);
296 if (i >= xlo && i <= xhi) {
297 f(0, i, j, k);
298 }
299 });
300 } else {
301#ifdef AMREX_USE_CUDA
302 constexpr int MT = 128;
303#else
304 constexpr int MT = AMREX_GPU_MAX_THREADS;
305#endif
306 ParallelForRedBlack_doit<MT>(mf, redblack, f);
307 }
308}
309
310}
312
313#endif
314#endif
#define AMREX_ASSERT(EX)
Definition AMReX_BLassert.H:38
#define AMREX_GPU_ERROR_CHECK()
Definition AMReX_GpuError.H:151
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
#define AMREX_D_TERM(a, b, c)
Definition AMReX_SPACE.H:172
virtual void * alloc(std::size_t sz)=0
Allocate sz bytes from this arena.
__host__ __device__ const IntVectND< dim > & smallEnd() const &noexcept
Return the inclusive lower bound of the box.
Definition AMReX_Box.H:124
__host__ __device__ constexpr Dim3 dim3() const noexcept
Definition AMReX_IntVect.H:269
amrex_long Long
Definition AMReX_INT.H:30
__host__ __device__ BoxND< dim > coarsen(const BoxND< dim > &b, int ref_ratio) noexcept
Return a copy of b coarsened by the isotropic ratio ref_ratio.
Definition AMReX_Box.H:1471
__host__ __device__ BoxND< dim > grow(const BoxND< dim > &b, int i) noexcept
Return a copy of b grown uniformly by i cells in every direction.
Definition AMReX_Box.H:1328
Arena * The_Pinned_Arena()
Definition AMReX_Arena.cpp:879
Arena * The_Arena()
Definition AMReX_Arena.cpp:839
void freeAsync(Arena *arena, void *mem) noexcept
Definition AMReX_GpuDevice.H:364
void htod_memcpy_async(void *p_d, const void *p_h, const std::size_t sz) noexcept
Definition AMReX_GpuDevice.H:440
gpuStream_t gpuStream() noexcept
Definition AMReX_GpuDevice.H:310
@ F
Definition AMReX_Order.H:7
void launch(T const &n, L const &f) noexcept
Definition AMReX_GpuLaunchFunctsC.H:122
void ParallelFor(TypeList< CTOs... > ctos, std::array< int, sizeof...(CTOs)> const &runtime_options, T N, F &&f)
Definition AMReX_CTOParallelForImpl.H:202
BoxND< 3 > Box
Box is an alias for amrex::BoxND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:35
IntVectND< 3 > IntVect
IntVect is an alias for amrex::IntVectND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:38
BoxIndexerND< 3 > BoxIndexer
Definition AMReX_Box.H:2641
const int[]
Definition AMReX_BLProfiler.cpp:1665
__global__ void launch_global(L f0, Lambdas... fs)
Definition AMReX_GpuLaunch.H:68
detail::BoxIndexerUInt UInt
Definition AMReX_Box.H:2511