1#ifndef AMREX_GPU_ATOMIC_H_
2#define AMREX_GPU_ATOMIC_H_
3#include <AMReX_Config.H>
12#if defined(AMREX_USE_CUDA) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 600)
13#error "AMReX requires CUDA compute capability 6.0 or newer"
36 template <
typename R,
typename Int,
typename F>
38 R atomic_op (R*
const address, R
const val,
F const f)
noexcept
40#if defined(__SYCL_DEVICE_ONLY__)
41 constexpr auto mo = sycl::memory_order::relaxed;
42 constexpr auto ms = sycl::memory_scope::device;
43 constexpr auto as = sycl::access::address_space::global_space;
44 static_assert(
sizeof(R) ==
sizeof(Int),
"sizeof R != sizeof Int");
45 Int*
const add_as_I =
reinterpret_cast<Int*
>(address);
46 sycl::atomic_ref<Int,mo,ms,as> a{*add_as_I};
47 Int old_I = a.load(), new_I;
49 R
const new_R = f(*(
reinterpret_cast<R const*
>(&old_I)), val);
50 new_I = *(
reinterpret_cast<Int const*
>(&new_R));
51 }
while (! a.compare_exchange_strong(old_I, new_I));
52 return *(
reinterpret_cast<R const*
>(&old_I));
55 *address = f(*address, val);
60 template <
typename R,
typename Int,
typename Op,
typename Cond>
62 bool atomic_op_if (R*
const address, R
const val, Op&& op, Cond&& cond)
noexcept
64#if defined(__SYCL_DEVICE_ONLY__)
65 constexpr auto mo = sycl::memory_order::relaxed;
66 constexpr auto ms = sycl::memory_scope::device;
67 constexpr auto as = sycl::access::address_space::global_space;
68 static_assert(
sizeof(R) ==
sizeof(Int),
"sizeof R != sizeof Int");
69 Int*
const add_as_I =
reinterpret_cast<Int*
>(address);
70 sycl::atomic_ref<Int,mo,ms,as> a{*add_as_I};
71 Int old_I = a.load(), new_I;
74 R
const tmp = op(*(
reinterpret_cast<R const*
>(&old_I)), val);
75 new_I = *(
reinterpret_cast<Int const*
>(&tmp));
76 test_success = cond(tmp);
77 }
while (test_success && ! a.compare_exchange_strong(old_I, new_I));
81 R
const tmp = op(*(
reinterpret_cast<R const*
>(&old)), val);
91#elif defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
96 template <
bool IS_MIN,
typename T>
98 T atomic_fp_min_max (T*
const m, T
const value)
noexcept
100 using I = std::conditional_t<
sizeof(T) == 4,
int,
long long>;
101 using U = std::make_unsigned_t<I>;
103 if constexpr (
sizeof(T) == 4) {
104 iv = __float_as_int(value);
106 iv = __double_as_longlong(value);
109 auto* im =
reinterpret_cast<I*
>(m);
110 if constexpr (IS_MIN) {
111 old = atomicMin(im, iv);
113 old = atomicMax(im, iv);
116 auto* um =
reinterpret_cast<U*
>(m);
117 auto const uv =
static_cast<U
>(iv);
118 if constexpr (IS_MIN) {
119 old =
static_cast<I
>(atomicMax(um, uv));
121 old =
static_cast<I
>(atomicMin(um, uv));
124 if constexpr (
sizeof(T) == 4) {
125 return __int_as_float(old);
127 return __longlong_as_double(old);
131 template <
typename R,
typename Int,
typename F>
133 R atomic_op (R*
const address, R
const val,
F const f)
noexcept
135 static_assert(
sizeof(R) ==
sizeof(Int),
"sizeof R != sizeof Int");
136 Int*
const add_as_I =
reinterpret_cast<Int*
>(address);
137 Int old_I = *add_as_I, assumed_I;
140 R
const new_R = f(*(
reinterpret_cast<R const*
>(&assumed_I)), val);
141 old_I = atomicCAS(add_as_I, assumed_I, *(
reinterpret_cast<Int const*
>(&new_R)));
142 }
while (assumed_I != old_I);
143 return *(
reinterpret_cast<R const*
>(&old_I));
146 template <
typename R,
typename Int,
typename Op,
typename Cond>
148 bool atomic_op_if (R*
const address, R
const val, Op&& op, Cond&& cond)
noexcept
150 static_assert(
sizeof(R) ==
sizeof(Int),
"sizeof R != sizeof Int");
151 Int*
const add_as_I =
reinterpret_cast<Int*
>(address);
152 Int old_I = *add_as_I, assumed_I;
156 R
const new_R = op(*(
reinterpret_cast<R const*
>(&assumed_I)), val);
157 test_success = cond(new_R);
159 old_I = atomicCAS(add_as_I, assumed_I, *(
reinterpret_cast<Int const*
>(&new_R)));
161 }
while (test_success && assumed_I != old_I);
177 template<
class T, sycl::access::address_space AS = sycl::access::address_space::global_space>
184#if defined(__SYCL_DEVICE_ONLY__)
185 constexpr auto mo = sycl::memory_order::relaxed;
186 constexpr auto ms = sycl::memory_scope::device;
187 sycl::atomic_ref<T,mo,ms,AS> a{*sum};
188 return a.fetch_add(value);
198#if defined(AMREX_USE_HIP) && defined(__gfx90a__)
201 float Add_device (
float*
const sum,
float const value)
noexcept
203 return unsafeAtomicAdd(sum, value);
207 double Add_device (
double*
const sum,
double const value)
noexcept
209 return unsafeAtomicAdd(sum, value);
213#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
219 return __hip_atomic_fetch_add(sum, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
223 static_assert(
sizeof(
Long) ==
sizeof(
unsigned long long));
224 return static_cast<Long>(atomicAdd(
reinterpret_cast<unsigned long long*
>(sum),
225 static_cast<unsigned long long>(value)));
234 template<
class T, sycl::access::address_space AS = sycl::access::address_space::global_space>
239 T
Add (T* sum, T value)
noexcept
257#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP) || defined(AMREX_USE_SYCL)
258 template <
typename T,
typename Op,
typename Cond>
259 requires (
sizeof(T) ==
sizeof(
unsigned int))
261 bool If_device (T*
const sum, T
const value, Op&& op, Cond&& cond)
noexcept
263 return detail::atomic_op_if<T, unsigned int>(sum, value,
264 std::forward<Op>(op), std::forward<Cond>(cond));
267 template <
typename T,
typename Op,
typename Cond>
268 requires (
sizeof(T) ==
sizeof(
unsigned long long))
270 bool If_device (T*
const sum, T
const value, Op&& op, Cond&& cond)
noexcept
272 return detail::atomic_op_if<T, unsigned long long>(sum, value,
273 std::forward<Op>(op), std::forward<Cond>(cond));
293 template<
class T,
class Op,
class Cond>
295 bool If (T*
const add, T
const value, Op&& op, Cond&& cond)
noexcept
298 return If_device(add, value, std::forward<Op>(op), std::forward<Cond>(cond));
302 T
const tmp = std::forward<Op>(op)(old, value);
303 if (std::forward<Cond>(cond)(tmp)) {
317 template<
class T, sycl::access::address_space AS = sycl::access::address_space::global_space>
324#if defined(__SYCL_DEVICE_ONLY__)
342#if defined(__SYCL_DEVICE_ONLY__)
343 constexpr auto mo = sycl::memory_order::relaxed;
344 constexpr auto ms = sycl::memory_scope::device;
345 constexpr auto as = sycl::access::address_space::global_space;
346 sycl::atomic_ref<T,mo,ms,as> a{*m};
347 return a.fetch_min(value);
358 float Min_device (
float*
const m,
float const value)
noexcept
360#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
361 return detail::atomic_fp_min_max<true>(m, value);
368 double Min_device (
double*
const m,
double const value)
noexcept
370#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
371 return detail::atomic_fp_min_max<true>(m, value);
377#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
383 return __hip_atomic_fetch_min(m, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
385 static_assert(
sizeof(
Long) ==
sizeof(
long long));
386 return static_cast<Long>(atomicMin(
reinterpret_cast<long long*
>(m),
387 static_cast<long long>(value)));
397 T
Min (T*
const m, T
const value)
noexcept
404 *m = (*m) < value ? (*m) : value;
419#if defined(__SYCL_DEVICE_ONLY__)
420 constexpr auto mo = sycl::memory_order::relaxed;
421 constexpr auto ms = sycl::memory_scope::device;
422 constexpr auto as = sycl::access::address_space::global_space;
423 sycl::atomic_ref<T,mo,ms,as> a{*m};
424 return a.fetch_max(value);
435 float Max_device (
float*
const m,
float const value)
noexcept
437#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
438 return detail::atomic_fp_min_max<false>(m, value);
445 double Max_device (
double*
const m,
double const value)
noexcept
447#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
448 return detail::atomic_fp_min_max<false>(m, value);
454#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
460 return __hip_atomic_fetch_max(m, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
462 static_assert(
sizeof(
Long) ==
sizeof(
long long));
463 return static_cast<Long>(atomicMax(
reinterpret_cast<long long*
>(m),
464 static_cast<long long>(value)));
474 T
Max (T*
const m, T
const value)
noexcept
481 *m = (*m) > value ? (*m) : value;
493#if defined(__SYCL_DEVICE_ONLY__)
494 constexpr auto mo = sycl::memory_order::relaxed;
495 constexpr auto ms = sycl::memory_scope::device;
496 constexpr auto as = sycl::access::address_space::global_space;
497 sycl::atomic_ref<int,mo,ms,as> a{*m};
498 return a.fetch_or(value);
501 return atomicOr(m, value);
518#if defined(__SYCL_DEVICE_ONLY__)
519 constexpr auto mo = sycl::memory_order::relaxed;
520 constexpr auto ms = sycl::memory_scope::device;
521 constexpr auto as = sycl::access::address_space::global_space;
522 sycl::atomic_ref<int,mo,ms,as> a{*m};
523 return a.fetch_and(value ? ~0x0 : 0);
526 return atomicAnd(m, value ? ~0x0 : 0);
540 template <
typename T>
542 T
Exch (T* address, T val)
noexcept
544#if defined(__SYCL_DEVICE_ONLY__)
545 constexpr auto mo = sycl::memory_order::relaxed;
546 constexpr auto ms = sycl::memory_scope::device;
547 constexpr auto as = sycl::access::address_space::global_space;
548 sycl::atomic_ref<T,mo,ms,as> a{*address};
549 return a.exchange(val);
552 return atomicExch(address, val);
555 auto const old = *address;
566 template <
typename T>
568 T
CAS (T*
const address, T compare, T
const val)
noexcept
570#if defined(__SYCL_DEVICE_ONLY__)
571 constexpr auto mo = sycl::memory_order::relaxed;
572 constexpr auto ms = sycl::memory_scope::device;
573 constexpr auto as = sycl::access::address_space::global_space;
574 sycl::atomic_ref<T,mo,ms,as> a{*address};
575 a.compare_exchange_strong(compare, val);
579 return atomicCAS(address, compare, val);
582 auto const old = *address;
583 *address = (old == compare ? val : old);
595 template <
typename T>
596 requires (
sizeof(T) ==
sizeof(
int))
603 template <
typename T>
604 requires (
sizeof(T) ==
sizeof(
unsigned long long))
621 auto const old = *prod;
633 template <
typename T>
634 requires (
sizeof(T) ==
sizeof(
int))
641 template <
typename T>
642 requires (
sizeof(T) ==
sizeof(
unsigned long long))
653 T
Divide (T*
const quot, T
const value)
noexcept
659 auto const old = *quot;
670 void Add_Host (T*
const sum, T
const value)
noexcept
673#pragma omp atomic update
684#pragma omp atomic capture
695 void Add (T*
const sum, T
const value)
noexcept
714 template <
typename T>
722 template <
typename T>
730 template <
typename T>
738 template <
typename T>
746 template <
typename T>
#define AMREX_FORCE_INLINE
Definition AMReX_Extension.H:123
Function object (functor) types for use with reductions and transforms.
#define AMREX_IF_ON_DEVICE(CODE)
Definition AMReX_GpuQualifiers.H:56
#define AMREX_IF_ON_HOST(CODE)
Definition AMReX_GpuQualifiers.H:58
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
#define AMREX_GPU_HOST_DEVICE
Definition AMReX_GpuQualifiers.H:20
amrex_long Long
Definition AMReX_INT.H:30
Definition AMReX_GpuAtomic.H:18
__host__ __device__ AMREX_FORCE_INLINE T Exch(T *address, T val) noexcept
Definition AMReX_GpuAtomic.H:542
__host__ __device__ AMREX_FORCE_INLINE int LogicalAnd(int *const m, int const value) noexcept
Definition AMReX_GpuAtomic.H:516
__host__ __device__ AMREX_FORCE_INLINE T CAS(T *const address, T compare, T const val) noexcept
Definition AMReX_GpuAtomic.H:568
__host__ __device__ AMREX_FORCE_INLINE T Multiply(T *const prod, T const value) noexcept
Definition AMReX_GpuAtomic.H:615
__host__ __device__ AMREX_FORCE_INLINE int LogicalOr(int *const m, int const value) noexcept
Definition AMReX_GpuAtomic.H:491
__host__ __device__ AMREX_FORCE_INLINE bool If(T *const add, T const value, Op &&op, Cond &&cond) noexcept
Conditionally perform an atomic operation.
Definition AMReX_GpuAtomic.H:295
__device__ AMREX_FORCE_INLINE T Max_device(T *const m, T const value) noexcept
Definition AMReX_GpuAtomic.H:417
__host__ __device__ AMREX_FORCE_INLINE void AddNoRet(T *sum, T value) noexcept
Definition AMReX_GpuAtomic.H:322
__device__ AMREX_FORCE_INLINE T Divide_device(T *const quot, T const value) noexcept
Definition AMReX_GpuAtomic.H:636
__device__ AMREX_FORCE_INLINE bool If_device(T *const sum, T const value, Op &&op, Cond &&cond) noexcept
Definition AMReX_GpuAtomic.H:261
__device__ AMREX_FORCE_INLINE T Min_device(T *const m, T const value) noexcept
Definition AMReX_GpuAtomic.H:340
__host__ __device__ AMREX_FORCE_INLINE T Add(T *sum, T value) noexcept
Definition AMReX_GpuAtomic.H:239
__device__ AMREX_FORCE_INLINE T Add_device(T *const sum, T const value) noexcept
Definition AMReX_GpuAtomic.H:182
__device__ AMREX_FORCE_INLINE T Multiply_device(T *const prod, T const value) noexcept
Definition AMReX_GpuAtomic.H:598
__host__ __device__ AMREX_FORCE_INLINE T Max(T *const m, T const value) noexcept
Definition AMReX_GpuAtomic.H:474
__host__ __device__ AMREX_FORCE_INLINE T Min(T *const m, T const value) noexcept
Definition AMReX_GpuAtomic.H:397
__host__ __device__ AMREX_FORCE_INLINE T Divide(T *const quot, T const value) noexcept
Definition AMReX_GpuAtomic.H:653
Definition AMReX_BaseFwd.H:60
Definition AMReX_GpuAtomic.H:666
AMREX_FORCE_INLINE T FetchAdd_Host(T *const sum, T const value) noexcept
Definition AMReX_GpuAtomic.H:680
__host__ __device__ AMREX_FORCE_INLINE void Add(T *const sum, T const value) noexcept
Definition AMReX_GpuAtomic.H:695
AMREX_FORCE_INLINE void Add_Host(T *const sum, T const value) noexcept
Definition AMReX_GpuAtomic.H:670
__host__ __device__ AMREX_FORCE_INLINE T FetchAdd(T *const sum, T const value) noexcept
Definition AMReX_GpuAtomic.H:703
Definition AMReX_Amr.cpp:50
__host__ __device__ void ignore_unused(const Ts &...)
No-op helper that marks variables as intentionally unused.
Definition AMReX.H:273
@ F
Definition AMReX_Order.H:7
const int[]
Definition AMReX_BLProfiler.cpp:1665
Function object that returns the quotient of two values.
Definition AMReX_Functional.H:121
Definition AMReX_GpuAtomic.H:716
__device__ void operator()(T *const dest, T const source) noexcept
Definition AMReX_GpuAtomic.H:717
Definition AMReX_GpuAtomic.H:740
__device__ void operator()(T *const dest, T const source) noexcept
Definition AMReX_GpuAtomic.H:741
Definition AMReX_GpuAtomic.H:748
__device__ void operator()(T *const dest, T const source) noexcept
Definition AMReX_GpuAtomic.H:749
Definition AMReX_GpuAtomic.H:732
__device__ void operator()(T *const dest, T const source) noexcept
Definition AMReX_GpuAtomic.H:733
Definition AMReX_GpuAtomic.H:724
__device__ void operator()(T *const dest, T const source) noexcept
Definition AMReX_GpuAtomic.H:725
Function object that returns the larger of two values.
Definition AMReX_Functional.H:65
Function object that returns the smaller of two values.
Definition AMReX_Functional.H:51
Function object that returns the product of two values.
Definition AMReX_Functional.H:107