1#ifndef AMREX_BASEFAB_UTILITY_H_
2#define AMREX_BASEFAB_UTILITY_H_
3#include <AMReX_Config.H>
27template <
class Tto,
class Tfrom>
33 auto const& tdata = tofab.array();
34 auto const& fdata = fromfab.const_array();
37 tdata(i,j,k,n+dcomp.i) = static_cast<Tto>(fdata(i,j,k,n+scomp.i));
54template <
typename STRUCT,
typename F>
55requires ((
sizeof(STRUCT)<=36*8) &&
57 std::is_trivially_destructible_v<STRUCT>)
60 Box const& box = aos_fab.
box();
61 auto const& aos = aos_fab.
array();
62 using T =
typename STRUCT::value_type;
63 constexpr int STRUCTSIZE =
sizeof(STRUCT)/
sizeof(T);
64 static_assert(
sizeof(STRUCT) ==
sizeof(T)*STRUCTSIZE,
65 "amrex::fill: sizeof(STRUCT) != sizeof(T)*STRUCTSIZE");
69 const auto ntotcells = std::uint64_t(box.
numPts());
70 constexpr int nthreads_per_block = (STRUCTSIZE <= 8) ? 256 : 128;
71 std::uint64_t nblocks_long = (ntotcells+nthreads_per_block-1)/nthreads_per_block;
72 AMREX_ASSERT(nblocks_long <= std::uint64_t(std::numeric_limits<int>::max()));
73 auto nblocks =
int(nblocks_long);
74 std::size_t shared_mem_bytes = nthreads_per_block *
sizeof(STRUCT);
81 auto const blockDimx = UInt(handler.blockDim());
82 auto const threadIdxx = UInt(handler.threadIdx());
83 auto const block_start = blockDimx*UInt(handler.blockIdx());
84 auto const icell = block_start + threadIdxx;
85 auto const shared = (T*)handler.sharedMemory();
86 if (icell < indexer.
numPts()) {
87 auto ga =
new(shared+threadIdxx*STRUCTSIZE) STRUCT;
88 auto [i, j, k] = indexer(icell);
91 handler.syncThreads();
92 T*
const pb =
p + std::uint64_t(block_start)*STRUCTSIZE;
93 for (UInt m = threadIdxx,
95 m < mend; m += blockDimx) {
104 auto const block_start = UInt(blockDim.x)*blockIdx.x;
105 auto const icell = block_start + threadIdx.x;
107 T*
const shared = gsm.
dataPtr();
108 if (icell < indexer.
numPts()) {
109 auto ga =
new(shared+threadIdx.x*STRUCTSIZE) STRUCT;
110 auto [i, j, k] = indexer(icell);
114 T*
const pb =
p + std::uint64_t(block_start)*STRUCTSIZE;
115 for (UInt m = threadIdx.x,
116 mend =
amrex::min(UInt(blockDim.x), indexer.
numPts()-block_start) * STRUCTSIZE;
117 m < mend; m += blockDim.x) {
127 f(aos(i,j,k), i, j, k);
151#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
153 constexpr int tile_dim = 32;
154 constexpr int block_rows = 16;
155 constexpr int nthreads = tile_dim*block_rows;
161 dim3 block{unsigned(tile_dim), unsigned(block_rows), 1};
162 dim3 grid{unsigned((nx+tile_dim-1)/tile_dim),
163 unsigned((nz+tile_dim-1)/tile_dim),
164 unsigned(std::min(ny, 65535))};
169 __shared__ T tile[tile_dim][tile_dim+1];
171 for (
unsigned int j = blockIdx.z; j <
unsigned(ny); j += gridDim.z) {
173 int k = blockIdx.y * tile_dim + threadIdx.x;
174 int i = blockIdx.x * tile_dim + threadIdx.y;
177 for (
int it = 0; it < tile_dim; it += block_rows, i += block_rows) {
180 tile[threadIdx.y+it][threadIdx.x] = pi[k + (j+i*std::size_t(ny))*nz];
187 i = blockIdx.x * tile_dim + threadIdx.x;
188 k = blockIdx.y * tile_dim + threadIdx.y;
191 for (
int it = 0; it < tile_dim; it += block_rows, k += block_rows) {
193 po[i + (j+k*std::size_t(ny))*nx] = tile[threadIdx.x][threadIdx.y+it];
203#elif defined(AMREX_USE_SYCL)
205 constexpr int tile_dim = 32;
206 constexpr int block_rows = 8;
211 sycl::range<3> block{std::size_t(1), std::size_t(block_rows), std::size_t(tile_dim)};
212 sycl::range<3> grid{std::size_t(ny), std::size_t((nz+tile_dim-1)/tile_dim),
213 std::size_t((nx+tile_dim-1)/tile_dim)};
214 sycl::range<3> global_size{grid[0]*block[0],
220 q.submit([&] (sycl::handler& h)
222 auto tile = sycl::local_accessor<T,2>(sycl::range<2>(tile_dim,tile_dim+1),h);
224 h.parallel_for(sycl::nd_range<3>(global_size, block),
225 [=] (sycl::nd_item<3> item)
227 auto group = item.get_group();
228 dim3 blockIdx{unsigned(group.get_group_id(2)),
229 unsigned(group.get_group_id(1)),
230 unsigned(group.get_group_id(0))};
231 dim3 threadIdx{unsigned(item.get_local_id(2)),
232 unsigned(item.get_local_id(1)),
233 unsigned(item.get_local_id(0))};
235 int k = blockIdx.y * tile_dim + threadIdx.x;
236 int i = blockIdx.x * tile_dim + threadIdx.y;
241 for (
int it = 0; it < tile_dim; it += block_rows, i += block_rows) {
244 tile[threadIdx.y+it][threadIdx.x] = pi[k + (j+i*std::size_t(ny))*nz];
249 sycl::group_barrier(group);
251 i = blockIdx.x * tile_dim + threadIdx.x;
252 k = blockIdx.y * tile_dim + threadIdx.y;
255 for (
int it = 0; it < tile_dim; it += block_rows, k += block_rows) {
257 po[i + (j+k*std::size_t(ny))*nx] = tile[threadIdx.x][threadIdx.y+it];
263 }
catch (sycl::exception
const& ex) {
264 amrex::Abort(std::string(
"transposeCtoF: ")+ex.what()+
"!!!!!");
269 constexpr int bx = 32;
270 constexpr int bz = 32;
272 std::size_t nxy = std::size_t(nx) * ny;
273 std::size_t nyz = std::size_t(ny) * nz;
276#pragma omp parallel for collapse(3)
278 for (
int j = 0; j < ny; ++j) {
279 for (
int k0 = 0; k0 < nz; k0 += bz) {
280 for (
int i0 = 0; i0 < nx; i0 += bx) {
281 int imax = std::min(i0+bx, nx);
282 int kmax = std::min(k0+bz, nz);
285 for (
int i = i0; i < imax; ++i) {
287 for (
int k = k0; k < kmax; ++k) {
288 pdst[i + k*nxy] = psrc[k + i*nyz];
#define AMREX_ASSERT(EX)
Definition AMReX_BLassert.H:38
#define AMREX_ALWAYS_ASSERT(EX)
Definition AMReX_BLassert.H:50
BaseFab container template providing box-based field storage.
#define AMREX_PRAGMA_SIMD
Definition AMReX_Extension.H:84
#define AMREX_RESTRICT
Definition AMReX_Extension.H:37
#define AMREX_GPU_ERROR_CHECK()
Definition AMReX_GpuError.H:151
#define AMREX_LAUNCH_KERNEL(MT, blocks, threads, sharedMem, stream,...)
Definition AMReX_GpuLaunch.H:37
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
#define AMREX_GPU_HOST_DEVICE
Definition AMReX_GpuQualifiers.H:20
while parser_exe_t p
Definition AMReX_Parser_Exe_Body.H:33
#define AMREX_IS_TRIVIALLY_COPYABLE(T)
Definition AMReX_TypeTraits.H:10
A FortranArrayBox(FAB)-like object.
Definition AMReX_BaseFab.H:243
const Box & box() const noexcept
Returns the domain (box) where the array is defined.
Definition AMReX_BaseFab.H:378
Array4< T const > array() const noexcept
Create an Array4 view over all components.
Definition AMReX_BaseFab.H:496
T * dataPtr(int n=0) noexcept
Returns a pointer to an object of type T that is the value of the Nth component associated with the c...
Definition AMReX_BaseFab.H:439
__host__ __device__ Long numPts() const noexcept
Return the number of points contained in the BoxND.
Definition AMReX_Box.H:385
__host__ __device__ constexpr const T & min(const T &a, const T &b) noexcept
Definition AMReX_Algorithm.H:31
bool inLaunchRegion() noexcept
Definition AMReX_GpuControl.H:88
gpuStream_t gpuStream() noexcept
Definition AMReX_GpuDevice.H:310
Definition AMReX_Amr.cpp:50
void transposeCtoF(T const *pi, T *po, int nx, int ny, int nz)
Transpose a 3D array of shape (nx, ny, nz) from C-order to Fortran-order storage.
Definition AMReX_BaseFabUtility.H:147
@ F
Definition AMReX_Order.H:7
__host__ __device__ void cast(BaseFab< Tto > &tofab, BaseFab< Tfrom > const &fromfab, Box const &bx, SrcComp scomp, DestComp dcomp, NumComps ncomp) noexcept
Cast components from one BaseFab to another over a region.
Definition AMReX_BaseFabUtility.H:30
void launch(T const &n, L const &f) noexcept
Definition AMReX_GpuLaunchFunctsC.H:122
BoxND< 3 > Box
Box is an alias for amrex::BoxND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:35
BoxIndexerND< 3 > BoxIndexer
Definition AMReX_Box.H:2641
void Abort(const std::string &msg)
Print a fatal-error message to stderr and abort execution.
Definition AMReX.cpp:239
const int[]
Definition AMReX_BLProfiler.cpp:1665
void LoopOnCpu(Dim3 lo, Dim3 hi, F const &f) noexcept
Definition AMReX_Loop.H:365
void fill(BaseFab< STRUCT > &aos_fab, F const &f)
Fill an array-of-structs BaseFab by invoking a functor per cell.
Definition AMReX_BaseFabUtility.H:58
__host__ __device__ void LoopConcurrent(Dim3 lo, Dim3 hi, F const &f) noexcept
Definition AMReX_Loop.H:152
detail::BoxIndexerUInt UInt
Definition AMReX_Box.H:2511
__host__ __device__ UInt numPts() const
Return the number of points covered by the indexed box.
Definition AMReX_Box.H:2577
Destination-component descriptor.
Definition AMReX_BaseFab.H:105
Definition AMReX_GpuTypes.H:86
Definition AMReX_GpuMemory.H:126
__device__ T * dataPtr() noexcept
Definition AMReX_GpuMemory.H:127
Number-of-components descriptor.
Definition AMReX_BaseFab.H:112
Source-component descriptor.
Definition AMReX_BaseFab.H:98