Block-Structured AMR Software Framework
Loading...
Searching...
No Matches
AMReX_GpuAtomic.H
Go to the documentation of this file.
1#ifndef AMREX_GPU_ATOMIC_H_
2#define AMREX_GPU_ATOMIC_H_
3#include <AMReX_Config.H>
4
6#include <AMReX_Functional.H>
7#include <AMReX_INT.H>
8
9#include <type_traits>
10#include <utility>
11
12#if defined(AMREX_USE_CUDA) && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ < 600)
13#error "AMReX requires CUDA compute capability 6.0 or newer"
14#endif
15
16namespace amrex {
17
18namespace Gpu::Atomic {
19
20// For Add, Min and Max, we support int, unsigned int, long, unsigned long long, float and double.
21// The result of Min and Max is unspecified if any input is NaN, and so is
22// the sign of a zero result.
23// For Multiply and Divide, we support generic types provided they are the same size as int or unsigned long long
24// and have *= and /= operators.
25// For LogicalOr and LogicalAnd, the data type is int.
26// For Exch and CAS, the data type is generic.
27// All these functions are non-atomic in host code!!!
28// If one needs them to be atomic in host code, use HostDevice::Atomic::*.
29// If we add more types for atomicAdd, we also need to update HasAtomicAdd in AMReX_TypeTraits.H.
30
32namespace detail {
33
34#ifdef AMREX_USE_SYCL
35
36 template <typename R, typename Int, typename F>
38 R atomic_op (R* const address, R const val, F const f) noexcept
39 {
40#if defined(__SYCL_DEVICE_ONLY__)
41 constexpr auto mo = sycl::memory_order::relaxed;
42 constexpr auto ms = sycl::memory_scope::device;
43 constexpr auto as = sycl::access::address_space::global_space;
44 static_assert(sizeof(R) == sizeof(Int), "sizeof R != sizeof Int");
45 Int* const add_as_I = reinterpret_cast<Int*>(address);
46 sycl::atomic_ref<Int,mo,ms,as> a{*add_as_I};
47 Int old_I = a.load(), new_I;
48 do {
49 R const new_R = f(*(reinterpret_cast<R const*>(&old_I)), val);
50 new_I = *(reinterpret_cast<Int const*>(&new_R));
51 } while (! a.compare_exchange_strong(old_I, new_I));
52 return *(reinterpret_cast<R const*>(&old_I));
53#else
54 R old = *address;
55 *address = f(*address, val);
56 return old;
57#endif
58 }
59
60 template <typename R, typename Int, typename Op, typename Cond>
62 bool atomic_op_if (R* const address, R const val, Op&& op, Cond&& cond) noexcept
63 {
64#if defined(__SYCL_DEVICE_ONLY__)
65 constexpr auto mo = sycl::memory_order::relaxed;
66 constexpr auto ms = sycl::memory_scope::device;
67 constexpr auto as = sycl::access::address_space::global_space;
68 static_assert(sizeof(R) == sizeof(Int), "sizeof R != sizeof Int");
69 Int* const add_as_I = reinterpret_cast<Int*>(address);
70 sycl::atomic_ref<Int,mo,ms,as> a{*add_as_I};
71 Int old_I = a.load(), new_I;
72 bool test_success;
73 do {
74 R const tmp = op(*(reinterpret_cast<R const*>(&old_I)), val);
75 new_I = *(reinterpret_cast<Int const*>(&tmp));
76 test_success = cond(tmp);
77 } while (test_success && ! a.compare_exchange_strong(old_I, new_I));
78 return test_success;
79#else
80 R old = *address;
81 R const tmp = op(*(reinterpret_cast<R const*>(&old)), val);
82 if (cond(tmp)) {
83 *address = tmp;
84 return true;
85 } else {
86 return false;
87 }
88#endif
89 }
90
91#elif defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
92
93 // Min (IS_MIN) or Max of float or double with one integer atomic.
94 // Sign bit clear: the values order like signed ints. Sign bit set: the
95 // reverse order of unsigned ints. NaNs are ordered by their bit pattern.
96 template <bool IS_MIN, typename T>
98 T atomic_fp_min_max (T* const m, T const value) noexcept
99 {
100 using I = std::conditional_t<sizeof(T) == 4, int, long long>;
101 using U = std::make_unsigned_t<I>;
102 I iv, old;
103 if constexpr (sizeof(T) == 4) {
104 iv = __float_as_int(value);
105 } else {
106 iv = __double_as_longlong(value);
107 }
108 if (iv >= 0) {
109 auto* im = reinterpret_cast<I*>(m);
110 if constexpr (IS_MIN) {
111 old = atomicMin(im, iv);
112 } else {
113 old = atomicMax(im, iv);
114 }
115 } else {
116 auto* um = reinterpret_cast<U*>(m);
117 auto const uv = static_cast<U>(iv);
118 if constexpr (IS_MIN) {
119 old = static_cast<I>(atomicMax(um, uv));
120 } else {
121 old = static_cast<I>(atomicMin(um, uv));
122 }
123 }
124 if constexpr (sizeof(T) == 4) {
125 return __int_as_float(old);
126 } else {
127 return __longlong_as_double(old);
128 }
129 }
130
131 template <typename R, typename Int, typename F>
133 R atomic_op (R* const address, R const val, F const f) noexcept
134 {
135 static_assert(sizeof(R) == sizeof(Int), "sizeof R != sizeof Int");
136 Int* const add_as_I = reinterpret_cast<Int*>(address);
137 Int old_I = *add_as_I, assumed_I;
138 do {
139 assumed_I = old_I;
140 R const new_R = f(*(reinterpret_cast<R const*>(&assumed_I)), val);
141 old_I = atomicCAS(add_as_I, assumed_I, *(reinterpret_cast<Int const*>(&new_R)));
142 } while (assumed_I != old_I);
143 return *(reinterpret_cast<R const*>(&old_I));
144 }
145
146 template <typename R, typename Int, typename Op, typename Cond>
148 bool atomic_op_if (R* const address, R const val, Op&& op, Cond&& cond) noexcept
149 {
150 static_assert(sizeof(R) == sizeof(Int), "sizeof R != sizeof Int");
151 Int* const add_as_I = reinterpret_cast<Int*>(address);
152 Int old_I = *add_as_I, assumed_I;
153 bool test_success;
154 do {
155 assumed_I = old_I;
156 R const new_R = op(*(reinterpret_cast<R const*>(&assumed_I)), val);
157 test_success = cond(new_R);
158 if (test_success) {
159 old_I = atomicCAS(add_as_I, assumed_I, *(reinterpret_cast<Int const*>(&new_R)));
160 }
161 } while (test_success && assumed_I != old_I);
162 return test_success;
163 }
164
165#endif
166
167}
169
171// Add
173
174#ifdef AMREX_USE_GPU
175
176#ifdef AMREX_USE_SYCL
177 template<class T, sycl::access::address_space AS = sycl::access::address_space::global_space>
178#else
179 template<class T>
180#endif
182 T Add_device (T* const sum, T const value) noexcept
183 {
184#if defined(__SYCL_DEVICE_ONLY__)
185 constexpr auto mo = sycl::memory_order::relaxed;
186 constexpr auto ms = sycl::memory_scope::device;
187 sycl::atomic_ref<T,mo,ms,AS> a{*sum};
188 return a.fetch_add(value);
189#else
190 AMREX_IF_ON_DEVICE(( return atomicAdd(sum, value); ))
192 amrex::ignore_unused(sum, value);
193 return T(); // should never get here, but have to return something
194 ))
195#endif
196 }
197
198#if defined(AMREX_USE_HIP) && defined(__gfx90a__)
199 // https://github.com/ROCm-Developer-Tools/hipamd/blob/rocm-4.5.x/include/hip/amd_detail/amd_hip_unsafe_atomics.h
201 float Add_device (float* const sum, float const value) noexcept
202 {
203 return unsafeAtomicAdd(sum, value);
204 }
205
207 double Add_device (double* const sum, double const value) noexcept
208 {
209 return unsafeAtomicAdd(sum, value);
210 }
211#endif
212
213#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
214
216 Long Add_device (Long* const sum, Long const value) noexcept
217 {
218#ifdef AMREX_USE_HIP
219 return __hip_atomic_fetch_add(sum, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
220#else
221 // No 64-bit signed overload. Two's complement makes the unsigned add
222 // bitwise identical.
223 static_assert(sizeof(Long) == sizeof(unsigned long long));
224 return static_cast<Long>(atomicAdd(reinterpret_cast<unsigned long long*>(sum),
225 static_cast<unsigned long long>(value)));
226#endif
227 }
228
229#endif
230
231#endif
232
233#ifdef AMREX_USE_SYCL
234 template<class T, sycl::access::address_space AS = sycl::access::address_space::global_space>
235#else
236 template<class T>
237#endif
239 T Add (T* sum, T value) noexcept
240 {
241#ifdef AMREX_USE_SYCL
242 AMREX_IF_ON_DEVICE((return Add_device<T,AS>(sum, value);))
243#else
244 AMREX_IF_ON_DEVICE((return Add_device(sum, value);))
245#endif
247 auto old = *sum;
248 *sum += value;
249 return old;
250 ))
251 }
252
254// If
256
257#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP) || defined(AMREX_USE_SYCL)
258 template <typename T, typename Op, typename Cond>
259 requires (sizeof(T) == sizeof(unsigned int))
261 bool If_device (T* const sum, T const value, Op&& op, Cond&& cond) noexcept
262 {
263 return detail::atomic_op_if<T, unsigned int>(sum, value,
264 std::forward<Op>(op), std::forward<Cond>(cond));
265 }
266
267 template <typename T, typename Op, typename Cond>
268 requires (sizeof(T) == sizeof(unsigned long long))
270 bool If_device (T* const sum, T const value, Op&& op, Cond&& cond) noexcept
271 {
272 return detail::atomic_op_if<T, unsigned long long>(sum, value,
273 std::forward<Op>(op), std::forward<Cond>(cond));
274 }
275#endif
276
293 template<class T, class Op, class Cond>
295 bool If (T* const add, T const value, Op&& op, Cond&& cond) noexcept
296 {
298 return If_device(add, value, std::forward<Op>(op), std::forward<Cond>(cond));
299 ))
301 T old = *add;
302 T const tmp = std::forward<Op>(op)(old, value);
303 if (std::forward<Cond>(cond)(tmp)) {
304 *add = tmp;
305 return true;
306 } else {
307 return false;
308 }
309 ))
310 }
311
313// AddNoRet
315
316#ifdef AMREX_USE_SYCL
317 template<class T, sycl::access::address_space AS = sycl::access::address_space::global_space>
318#else
319 template<class T>
320#endif
322 void AddNoRet (T* sum, T value) noexcept
323 {
324#if defined(__SYCL_DEVICE_ONLY__)
325 Add_device<T,AS>(sum, value);
326#else
327 AMREX_IF_ON_DEVICE((Add_device(sum, value);))
328 AMREX_IF_ON_HOST((*sum += value;))
329#endif
330 }
331
333// Min
335
336#ifdef AMREX_USE_GPU
337
338 template<class T>
340 T Min_device (T* const m, T const value) noexcept
341 {
342#if defined(__SYCL_DEVICE_ONLY__)
343 constexpr auto mo = sycl::memory_order::relaxed;
344 constexpr auto ms = sycl::memory_scope::device;
345 constexpr auto as = sycl::access::address_space::global_space;
346 sycl::atomic_ref<T,mo,ms,as> a{*m};
347 return a.fetch_min(value);
348#else
349 AMREX_IF_ON_DEVICE(( return atomicMin(m, value); ))
351 amrex::ignore_unused(m,value);
352 return T(); // should never get here, but have to return something
353 ))
354#endif
355 }
356
358 float Min_device (float* const m, float const value) noexcept
359 {
360#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
361 return detail::atomic_fp_min_max<true>(m, value);
362#else
363 return detail::atomic_op<float,int>(m,value,amrex::Minimum<float>());
364#endif
365 }
366
368 double Min_device (double* const m, double const value) noexcept
369 {
370#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
371 return detail::atomic_fp_min_max<true>(m, value);
372#else
373 return detail::atomic_op<double,unsigned long long int>(m,value,amrex::Minimum<double>());
374#endif
375 }
376
377#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
378
380 Long Min_device (Long* const m, Long const value) noexcept
381 {
382#ifdef AMREX_USE_HIP
383 return __hip_atomic_fetch_min(m, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
384#else
385 static_assert(sizeof(Long) == sizeof(long long));
386 return static_cast<Long>(atomicMin(reinterpret_cast<long long*>(m),
387 static_cast<long long>(value)));
388#endif
389 }
390
391#endif
392
393#endif
394
395 template<class T>
397 T Min (T* const m, T const value) noexcept
398 {
400 return Min_device(m, value);
401 ))
403 auto const old = *m;
404 *m = (*m) < value ? (*m) : value;
405 return old;
406 ))
407 }
408
410// Max
412
413#ifdef AMREX_USE_GPU
414
415 template<class T>
417 T Max_device (T* const m, T const value) noexcept
418 {
419#if defined(__SYCL_DEVICE_ONLY__)
420 constexpr auto mo = sycl::memory_order::relaxed;
421 constexpr auto ms = sycl::memory_scope::device;
422 constexpr auto as = sycl::access::address_space::global_space;
423 sycl::atomic_ref<T,mo,ms,as> a{*m};
424 return a.fetch_max(value);
425#else
426 AMREX_IF_ON_DEVICE(( return atomicMax(m, value); ))
428 amrex::ignore_unused(m,value);
429 return T(); // should never get here, but have to return something
430 ))
431#endif
432 }
433
435 float Max_device (float* const m, float const value) noexcept
436 {
437#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
438 return detail::atomic_fp_min_max<false>(m, value);
439#else
440 return detail::atomic_op<float,int>(m,value,amrex::Maximum<float>());
441#endif
442 }
443
445 double Max_device (double* const m, double const value) noexcept
446 {
447#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
448 return detail::atomic_fp_min_max<false>(m, value);
449#else
450 return detail::atomic_op<double,unsigned long long int>(m,value,amrex::Maximum<double>());
451#endif
452 }
453
454#if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP)
455
457 Long Max_device (Long* const m, Long const value) noexcept
458 {
459#ifdef AMREX_USE_HIP
460 return __hip_atomic_fetch_max(m, value, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
461#else
462 static_assert(sizeof(Long) == sizeof(long long));
463 return static_cast<Long>(atomicMax(reinterpret_cast<long long*>(m),
464 static_cast<long long>(value)));
465#endif
466 }
467
468#endif
469
470#endif
471
472 template<class T>
474 T Max (T* const m, T const value) noexcept
475 {
477 return Max_device(m, value);
478 ))
480 auto const old = *m;
481 *m = (*m) > value ? (*m) : value;
482 return old;
483 ))
484 }
485
487// LogicalOr
489
491 int LogicalOr (int* const m, int const value) noexcept
492 {
493#if defined(__SYCL_DEVICE_ONLY__)
494 constexpr auto mo = sycl::memory_order::relaxed;
495 constexpr auto ms = sycl::memory_scope::device;
496 constexpr auto as = sycl::access::address_space::global_space;
497 sycl::atomic_ref<int,mo,ms,as> a{*m};
498 return a.fetch_or(value);
499#else
501 return atomicOr(m, value);
502 ))
504 int const old = *m;
505 *m = (*m) || value;
506 return old;
507 ))
508#endif
509 }
510
512// LogicalAnd
514
516 int LogicalAnd (int* const m, int const value) noexcept
517 {
518#if defined(__SYCL_DEVICE_ONLY__)
519 constexpr auto mo = sycl::memory_order::relaxed;
520 constexpr auto ms = sycl::memory_scope::device;
521 constexpr auto as = sycl::access::address_space::global_space;
522 sycl::atomic_ref<int,mo,ms,as> a{*m};
523 return a.fetch_and(value ? ~0x0 : 0);
524#else
526 return atomicAnd(m, value ? ~0x0 : 0);
527 ))
529 int const old = *m;
530 *m = (*m) && value;
531 return old;
532 ))
533#endif
534 }
535
537// Exch
539
540 template <typename T>
542 T Exch (T* address, T val) noexcept
543 {
544#if defined(__SYCL_DEVICE_ONLY__)
545 constexpr auto mo = sycl::memory_order::relaxed;
546 constexpr auto ms = sycl::memory_scope::device;
547 constexpr auto as = sycl::access::address_space::global_space;
548 sycl::atomic_ref<T,mo,ms,as> a{*address};
549 return a.exchange(val);
550#else
552 return atomicExch(address, val);
553 ))
555 auto const old = *address;
556 *address = val;
557 return old;
558 ))
559#endif
560 }
561
563// CAS
565
566 template <typename T>
568 T CAS (T* const address, T compare, T const val) noexcept
569 { // cannot be T const compare because of compare_exchange_strong
570#if defined(__SYCL_DEVICE_ONLY__)
571 constexpr auto mo = sycl::memory_order::relaxed;
572 constexpr auto ms = sycl::memory_scope::device;
573 constexpr auto as = sycl::access::address_space::global_space;
574 sycl::atomic_ref<T,mo,ms,as> a{*address};
575 a.compare_exchange_strong(compare, val);
576 return compare;
577#else
579 return atomicCAS(address, compare, val);
580 ))
582 auto const old = *address;
583 *address = (old == compare ? val : old);
584 return old;
585 ))
586#endif
587 }
588
590// Multiply
592
593#ifdef AMREX_USE_GPU
594
595 template <typename T>
596 requires (sizeof(T) == sizeof(int))
598 T Multiply_device (T* const prod, T const value) noexcept
599 {
600 return detail::atomic_op<T, int>(prod,value,amrex::Multiplies<T>());
601 }
602
603 template <typename T>
604 requires (sizeof(T) == sizeof(unsigned long long))
606 T Multiply_device (T* const prod, T const value) noexcept
607 {
608 return detail::atomic_op<T, unsigned long long>(prod,value,amrex::Multiplies<T>());
609 }
610
611#endif
612
613 template<class T>
615 T Multiply (T* const prod, T const value) noexcept
616 {
618 return Multiply_device(prod, value);
619 ))
621 auto const old = *prod;
622 *prod *= value;
623 return old;
624 ))
625 }
626
628// Divide
630
631#ifdef AMREX_USE_GPU
632
633 template <typename T>
634 requires (sizeof(T) == sizeof(int))
636 T Divide_device (T* const quot, T const value) noexcept
637 {
638 return detail::atomic_op<T, int>(quot,value,amrex::Divides<T>());
639 }
640
641 template <typename T>
642 requires (sizeof(T) == sizeof(unsigned long long))
644 T Divide_device (T* const quot, T const value) noexcept
645 {
646 return detail::atomic_op<T, unsigned long long>(quot,value,amrex::Divides<T>());
647 }
648
649#endif
650
651 template<class T>
653 T Divide (T* const quot, T const value) noexcept
654 {
656 return Divide_device(quot, value);
657 ))
659 auto const old = *quot;
660 *quot /= value;
661 return old;
662 ))
663 }
664}
665
667
668 template <class T>
670 void Add_Host (T* const sum, T const value) noexcept
671 {
672#ifdef AMREX_USE_OMP
673#pragma omp atomic update
674#endif
675 *sum += value;
676 }
677
678 template <class T>
680 T FetchAdd_Host (T* const sum, T const value) noexcept
681 {
682 T old;
683#ifdef AMREX_USE_OMP
684#pragma omp atomic capture
685#endif
686 {
687 old = *sum;
688 *sum += value;
689 }
690 return old;
691 }
692
693 template <class T>
695 void Add (T* const sum, T const value) noexcept
696 {
698 AMREX_IF_ON_HOST((Add_Host(sum,value);))
699 }
700
701 template <class T>
703 T FetchAdd (T* const sum, T const value) noexcept
704 {
705 AMREX_IF_ON_DEVICE((return Gpu::Atomic::Add(sum,value);))
706 AMREX_IF_ON_HOST((return FetchAdd_Host(sum,value);))
707 }
708
709}
710
711#ifdef AMREX_USE_GPU
712// functors
713namespace Gpu {
714 template <typename T>
716 {
717 AMREX_GPU_DEVICE void operator() (T* const dest, T const source) noexcept {
718 Gpu::Atomic::AddNoRet(dest, source);
719 }
720 };
721
722 template <typename T>
724 {
725 AMREX_GPU_DEVICE void operator() (T* const dest, T const source) noexcept {
726 Gpu::Atomic::Min(dest, source);
727 }
728 };
729
730 template <typename T>
732 {
733 AMREX_GPU_DEVICE void operator() (T* const dest, T const source) noexcept {
734 Gpu::Atomic::Max(dest, source);
735 }
736 };
737
738 template <typename T>
740 {
741 AMREX_GPU_DEVICE void operator() (T* const dest, T const source) noexcept {
742 Gpu::Atomic::LogicalAnd(dest, source);
743 }
744 };
745
746 template <typename T>
748 {
749 AMREX_GPU_DEVICE void operator() (T* const dest, T const source) noexcept {
750 Gpu::Atomic::LogicalOr(dest, source);
751 }
752 };
753}
754#endif
755
756}
757#endif
#define AMREX_FORCE_INLINE
Definition AMReX_Extension.H:123
Function object (functor) types for use with reductions and transforms.
#define AMREX_IF_ON_DEVICE(CODE)
Definition AMReX_GpuQualifiers.H:56
#define AMREX_IF_ON_HOST(CODE)
Definition AMReX_GpuQualifiers.H:58
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
#define AMREX_GPU_HOST_DEVICE
Definition AMReX_GpuQualifiers.H:20
amrex_long Long
Definition AMReX_INT.H:30
Definition AMReX_GpuAtomic.H:18
__host__ __device__ AMREX_FORCE_INLINE T Exch(T *address, T val) noexcept
Definition AMReX_GpuAtomic.H:542
__host__ __device__ AMREX_FORCE_INLINE int LogicalAnd(int *const m, int const value) noexcept
Definition AMReX_GpuAtomic.H:516
__host__ __device__ AMREX_FORCE_INLINE T CAS(T *const address, T compare, T const val) noexcept
Definition AMReX_GpuAtomic.H:568
__host__ __device__ AMREX_FORCE_INLINE T Multiply(T *const prod, T const value) noexcept
Definition AMReX_GpuAtomic.H:615
__host__ __device__ AMREX_FORCE_INLINE int LogicalOr(int *const m, int const value) noexcept
Definition AMReX_GpuAtomic.H:491
__host__ __device__ AMREX_FORCE_INLINE bool If(T *const add, T const value, Op &&op, Cond &&cond) noexcept
Conditionally perform an atomic operation.
Definition AMReX_GpuAtomic.H:295
__device__ AMREX_FORCE_INLINE T Max_device(T *const m, T const value) noexcept
Definition AMReX_GpuAtomic.H:417
__host__ __device__ AMREX_FORCE_INLINE void AddNoRet(T *sum, T value) noexcept
Definition AMReX_GpuAtomic.H:322
__device__ AMREX_FORCE_INLINE T Divide_device(T *const quot, T const value) noexcept
Definition AMReX_GpuAtomic.H:636
__device__ AMREX_FORCE_INLINE bool If_device(T *const sum, T const value, Op &&op, Cond &&cond) noexcept
Definition AMReX_GpuAtomic.H:261
__device__ AMREX_FORCE_INLINE T Min_device(T *const m, T const value) noexcept
Definition AMReX_GpuAtomic.H:340
__host__ __device__ AMREX_FORCE_INLINE T Add(T *sum, T value) noexcept
Definition AMReX_GpuAtomic.H:239
__device__ AMREX_FORCE_INLINE T Add_device(T *const sum, T const value) noexcept
Definition AMReX_GpuAtomic.H:182
__device__ AMREX_FORCE_INLINE T Multiply_device(T *const prod, T const value) noexcept
Definition AMReX_GpuAtomic.H:598
__host__ __device__ AMREX_FORCE_INLINE T Max(T *const m, T const value) noexcept
Definition AMReX_GpuAtomic.H:474
__host__ __device__ AMREX_FORCE_INLINE T Min(T *const m, T const value) noexcept
Definition AMReX_GpuAtomic.H:397
__host__ __device__ AMREX_FORCE_INLINE T Divide(T *const quot, T const value) noexcept
Definition AMReX_GpuAtomic.H:653
Definition AMReX_BaseFwd.H:60
Definition AMReX_GpuAtomic.H:666
AMREX_FORCE_INLINE T FetchAdd_Host(T *const sum, T const value) noexcept
Definition AMReX_GpuAtomic.H:680
__host__ __device__ AMREX_FORCE_INLINE void Add(T *const sum, T const value) noexcept
Definition AMReX_GpuAtomic.H:695
AMREX_FORCE_INLINE void Add_Host(T *const sum, T const value) noexcept
Definition AMReX_GpuAtomic.H:670
__host__ __device__ AMREX_FORCE_INLINE T FetchAdd(T *const sum, T const value) noexcept
Definition AMReX_GpuAtomic.H:703
Definition AMReX_Amr.cpp:50
__host__ __device__ void ignore_unused(const Ts &...)
No-op helper that marks variables as intentionally unused.
Definition AMReX.H:273
@ F
Definition AMReX_Order.H:7
const int[]
Definition AMReX_BLProfiler.cpp:1665
Function object that returns the quotient of two values.
Definition AMReX_Functional.H:121
Definition AMReX_GpuAtomic.H:716
__device__ void operator()(T *const dest, T const source) noexcept
Definition AMReX_GpuAtomic.H:717
Definition AMReX_GpuAtomic.H:740
__device__ void operator()(T *const dest, T const source) noexcept
Definition AMReX_GpuAtomic.H:741
Definition AMReX_GpuAtomic.H:748
__device__ void operator()(T *const dest, T const source) noexcept
Definition AMReX_GpuAtomic.H:749
Definition AMReX_GpuAtomic.H:732
__device__ void operator()(T *const dest, T const source) noexcept
Definition AMReX_GpuAtomic.H:733
Definition AMReX_GpuAtomic.H:724
__device__ void operator()(T *const dest, T const source) noexcept
Definition AMReX_GpuAtomic.H:725
Function object that returns the larger of two values.
Definition AMReX_Functional.H:65
Function object that returns the smaller of two values.
Definition AMReX_Functional.H:51
Function object that returns the product of two values.
Definition AMReX_Functional.H:107