Block-Structured AMR Software Framework
Loading...
Searching...
No Matches
AMReX_ParticleTile.H
Go to the documentation of this file.
1#ifndef AMREX_PARTICLETILE_H_
2#define AMREX_PARTICLETILE_H_
3#include <AMReX_Config.H>
4
5#include <AMReX_Extension.H>
6#include <AMReX_Particle.H>
9#include <AMReX_Vector.H>
10#include <AMReX_REAL.H>
11#include <AMReX_RealVect.H>
12
13#include <array>
14#include <atomic>
15#include <string>
16#include <type_traits>
17#include <vector>
18
19
20namespace amrex {
21
22// Forward Declaration
23template <int NArrayReal, int NArrayInt>
24struct ConstSoAParticle;
25template <int NArrayReal, int NArrayInt>
26struct SoAParticle;
27
28template <typename T_ParticleType, int NArrayReal, int NArrayInt>
30
32template <typename T_ParticleType, int NArrayReal, int NArrayInt>
34{
35 static constexpr int NAR = NArrayReal;
36 static constexpr int NAI = NArrayInt;
37
38 using ParticleType = T_ParticleType;
39 using ParticleRefType = T_ParticleType&;
42 using IntType = int;
43
44 static constexpr int NStructReal = ParticleType::NReal;
45 static constexpr int NStructInt = ParticleType::NInt;
46
48
49 static constexpr bool is_particle_tile_data = true;
50
52
53 using AOS_PTR = std::conditional_t<T_ParticleType::is_soa_particle,
56
57 uint64_t* m_idcpu;
60
65
67 decltype(auto) pos (const int dir, const int index) const &
68 {
69 if constexpr(!ParticleType::is_soa_particle) {
70 return this->m_aos[index].pos(dir);
71 } else {
72 return this->m_rdata[dir][index];
73 }
74 }
75
77 decltype(auto) id (const int index) const &
78 {
79 if constexpr(!ParticleType::is_soa_particle) {
80 return this->m_aos[index].id();
81 } else {
82 return ParticleIDWrapper(this->m_idcpu[index]);
83 }
84 }
85
87 decltype(auto) cpu (const int index) const &
88 {
89 if constexpr(!ParticleType::is_soa_particle) {
90 return this->m_aos[index].cpu();
91 } else {
92 return ParticleCPUWrapper(this->m_idcpu[index]);
93 }
94 }
95
97 decltype(auto) idcpu (const int index) const &
98 {
99 if constexpr(ParticleType::is_soa_particle) {
100 return this->m_idcpu[index];
101 } else {
102 return this->m_aos[index].idcpu();
103 }
104 }
105
107 ParticleReal * rdata (const int attribute_index) const
108 {
109 return this->m_rdata[attribute_index];
110 }
111
113 int * idata (const int attribute_index) const
114 {
115 return this->m_idata[attribute_index];
116 }
117
119 decltype(auto) operator[] (const int index) const
120 {
121 if constexpr (!ParticleType::is_soa_particle) {
122 return m_aos[index];
123 } else {
124 return SoAParticle<NAR, NAI>(*this, index);
125 }
126 }
127
129 void packParticleData (char* buffer, int src_index, std::size_t dst_offset,
130 const int* comm_real, const int * comm_int) const noexcept
131 {
132 AMREX_ASSERT(src_index < m_size);
133 auto* dst = buffer + dst_offset;
134 if constexpr (!ParticleType::is_soa_particle) {
135 memcpy(dst, m_aos + src_index, sizeof(ParticleType));
136 dst += sizeof(ParticleType);
137 } else {
138 memcpy(dst, m_idcpu + src_index, sizeof(uint64_t));
139 dst += sizeof(uint64_t);
140 }
141 int array_start_index = 0;
142 if constexpr (!ParticleType::is_soa_particle) {
143 array_start_index = AMREX_SPACEDIM + NStructReal;
144 }
145 for (int i = 0; i < NAR; ++i)
146 {
147 if (comm_real[array_start_index + i])
148 {
149 memcpy(dst, m_rdata[i] + src_index, sizeof(ParticleReal));
150 dst += sizeof(ParticleReal);
151 }
152 }
153 int runtime_start_index = array_start_index + NAR;
154 for (int i = 0; i < m_num_runtime_real; ++i)
155 {
156 if (comm_real[runtime_start_index + i])
157 {
158 memcpy(dst, m_runtime_rdata[i] + src_index, sizeof(ParticleReal));
159 dst += sizeof(ParticleReal);
160 }
161 }
162 array_start_index = 2 + NStructInt;
163 for (int i = 0; i < NAI; ++i)
164 {
165 if (comm_int[array_start_index + i])
166 {
167 memcpy(dst, m_idata[i] + src_index, sizeof(int));
168 dst += sizeof(int);
169 }
170 }
171 runtime_start_index = 2 + NStructInt + NAI;
172 for (int i = 0; i < m_num_runtime_int; ++i)
173 {
174 if (comm_int[runtime_start_index + i])
175 {
176 memcpy(dst, m_runtime_idata[i] + src_index, sizeof(int));
177 dst += sizeof(int);
178 }
179 }
180 }
181
183 void unpackParticleData (const char* buffer, Long src_offset, int dst_index,
184 const int* comm_real, const int* comm_int) const noexcept
185 {
186 AMREX_ASSERT(dst_index < m_size);
187 const auto* src = buffer + src_offset;
188 if constexpr (!ParticleType::is_soa_particle) {
189 memcpy(m_aos + dst_index, src, sizeof(ParticleType));
190 src += sizeof(ParticleType);
191 } else {
192 memcpy(m_idcpu + dst_index, src, sizeof(uint64_t));
193 src += sizeof(uint64_t);
194 }
195 int array_start_index = 0;
196 if constexpr (!ParticleType::is_soa_particle) {
197 array_start_index = AMREX_SPACEDIM + NStructReal;
198 }
199 if constexpr (NAR > 0) {
200 for (int i = 0; i < NAR; ++i)
201 {
202 if (comm_real[array_start_index + i])
203 {
204 memcpy(m_rdata[i] + dst_index, src, sizeof(ParticleReal));
205 src += sizeof(ParticleReal);
206 }
207 }
208 }
209 int runtime_start_index = array_start_index + NAR;
210 for (int i = 0; i < m_num_runtime_real; ++i)
211 {
212 if (comm_real[runtime_start_index + i])
213 {
214 memcpy(m_runtime_rdata[i] + dst_index, src, sizeof(ParticleReal));
215 src += sizeof(ParticleReal);
216 }
217 }
218 array_start_index = 2 + NStructInt;
219 if constexpr (NAI > 0) {
220 for (int i = 0; i < NAI; ++i)
221 {
222 if (comm_int[array_start_index + i])
223 {
224 memcpy(m_idata[i] + dst_index, src, sizeof(int));
225 src += sizeof(int);
226 }
227 }
228 }
229 runtime_start_index = 2 + NStructInt + NAI;
230 for (int i = 0; i < m_num_runtime_int; ++i)
231 {
232 if (comm_int[runtime_start_index + i])
233 {
234 memcpy(m_runtime_idata[i] + dst_index, src, sizeof(int));
235 src += sizeof(int);
236 }
237 }
238 }
239
241 SuperParticleType getSuperParticle (int index) const noexcept
242 requires (!ParticleType::is_soa_particle)
243 {
244 AMREX_ASSERT(index < m_size);
246 for (int i = 0; i < AMREX_SPACEDIM; ++i) {
247 sp.pos(i) = m_aos[index].pos(i);
248 }
249 for (int i = 0; i < NStructReal; ++i) {
250 sp.rdata(i) = m_aos[index].rdata(i);
251 }
252 if constexpr (NAR >0) {
253 for (int i = 0; i < NAR; ++i) {
254 sp.rdata(NStructReal+i) = m_rdata[i][index];
255 }
256 }
257 sp.id() = m_aos[index].id();
258 sp.cpu() = m_aos[index].cpu();
259 for (int i = 0; i < NStructInt; ++i) {
260 sp.idata(i) = m_aos[index].idata(i);
261 }
262 if constexpr (NAI > 0) {
263 for (int i = 0; i < NAI; ++i) {
264 sp.idata(NStructInt+i) = m_idata[i][index];
265 }
266 }
267 return sp;
268 }
269
271 SuperParticleType getSuperParticle (int index) const noexcept
272 requires (ParticleType::is_soa_particle)
273 {
274 AMREX_ASSERT(index < m_size);
276 sp.m_idcpu = m_idcpu[index];
277 for (int i = 0; i < AMREX_SPACEDIM; ++i) {sp.pos(i) = m_rdata[i][index];}
278 for (int i = 0; i < NAR; ++i) {
279 sp.rdata(i) = m_rdata[i][index];
280 }
281 for (int i = 0; i < NAI; ++i) {
282 sp.idata(i) = m_idata[i][index];
283 }
284 return sp;
285 }
286
288 void setSuperParticle (const SuperParticleType& sp, int index) const noexcept
289 requires (!ParticleType::is_soa_particle)
290 {
291 for (int i = 0; i < AMREX_SPACEDIM; ++i) {
292 m_aos[index].pos(i) = sp.pos(i);
293 }
294 for (int i = 0; i < NStructReal; ++i) {
295 m_aos[index].rdata(i) = sp.rdata(i);
296 }
297 for (int i = 0; i < NAR; ++i) {
298 m_rdata[i][index] = sp.rdata(NStructReal+i);
299 }
300 m_aos[index].id() = sp.id();
301 m_aos[index].cpu() = sp.cpu();
302 for (int i = 0; i < NStructInt; ++i) {
303 m_aos[index].idata(i) = sp.idata(i);
304 }
305 for (int i = 0; i < NAI; ++i) {
306 m_idata[i][index] = sp.idata(NStructInt+i);
307 }
308 }
309
311 void setSuperParticle (const SuperParticleType& sp, int index) const noexcept
312 requires (ParticleType::is_soa_particle)
313 {
314 m_idcpu[index] = sp.m_idcpu;
315 for (int i = 0; i < NAR; ++i) {
316 m_rdata[i][index] = sp.rdata(i);
317 }
318 for (int i = 0; i < NAI; ++i) {
319 m_idata[i][index] = sp.idata(i);
320 }
321 }
322};
323
324// SOA Particle Structure
325template <int T_NArrayReal, int T_NArrayInt>
326struct alignas(sizeof(double)) ConstSoAParticle : SoAParticleBase
327{
328 static constexpr int NArrayReal = T_NArrayReal;
329 static constexpr int NArrayInt = T_NArrayInt;
332 static constexpr bool is_soa_particle = true;
333 static constexpr bool is_rtsoa_particle = false;
334 static constexpr bool is_constsoa_particle = true;
335
337 using IntType = int;
338
340 ConstSoAParticle (ConstPTD const& ptd, long i) : // Note: should this be int instead?
341 m_constparticle_tile_data(ptd), m_index(int(i))
342 {
343 }
344
345 //static Long the_next_id;
346
347 //functions to get id and cpu in the SOA data
348
350 ConstParticleCPUWrapper cpu () const { return this->m_constparticle_tile_data.m_idcpu[m_index]; }
351
353 ConstParticleIDWrapper id () const { return this->m_constparticle_tile_data.m_idcpu[m_index]; }
354
355 //functions to get positions of the particle in the SOA data
356
358 RealVect pos () const & {return RealVect(AMREX_D_DECL(this->m_constparticle_tile_data.m_rdata[0][m_index], this->m_constparticle_tile_data.m_rdata[1][m_index], this->m_constparticle_tile_data.m_rdata[2][m_index]));}
359
361 const RealType& pos (int position_index) const &
362 {
363 AMREX_ASSERT(position_index < AMREX_SPACEDIM);
364 return this->m_constparticle_tile_data.m_rdata[position_index][m_index];
365 }
366
367 static Long NextID ();
368
373
379 static void NextID (Long nextid);
380
381 private :
382
383 static_assert(std::is_trivially_copyable<ConstPTD>(), "ParticleTileData is not trivially copyable");
384
385 ConstPTD m_constparticle_tile_data;
386 int m_index;
387};
388
389template <int T_NArrayReal, int T_NArrayInt>
390struct alignas(sizeof(double)) SoAParticle : SoAParticleBase
391{
392 static constexpr int NArrayReal = T_NArrayReal;
393 static constexpr int NArrayInt = T_NArrayInt;
396 static constexpr bool is_soa_particle = true;
397 static constexpr bool is_rtsoa_particle = false;
398 static constexpr bool is_constsoa_particle = false;
399
402 using IntType = int;
403
405 SoAParticle (PTD const& ptd, long i) : // Note: should this be int instead?
406 m_particle_tile_data(ptd), m_index(int(i))
407 {
408 }
409
411
412 //functions to get id and cpu in the SOA data
413
415 ParticleCPUWrapper cpu () & { return this->m_particle_tile_data.m_idcpu[m_index]; }
416
418 ParticleIDWrapper<> id () & { return this->m_particle_tile_data.m_idcpu[m_index]; }
419
421 uint64_t& idcpu () & { return this->m_particle_tile_data.m_idcpu[m_index]; }
422
424 ConstParticleCPUWrapper cpu () const & { return this->m_particle_tile_data.m_idcpu[m_index]; }
425
427 ConstParticleIDWrapper id () const & { return this->m_particle_tile_data.m_idcpu[m_index]; }
428
430 const uint64_t& idcpu () const & { return this->m_particle_tile_data.m_idcpu[m_index]; }
431
432 //functions to get positions of the particle in the SOA data
433
435 RealVect pos () const & {return RealVect(AMREX_D_DECL(this->m_particle_tile_data.m_rdata[0][m_index], this->m_particle_tile_data.m_rdata[1][m_index], this->m_particle_tile_data.m_rdata[2][m_index]));}
436
438 RealType& pos (int position_index) &
439 {
440 AMREX_ASSERT(position_index < AMREX_SPACEDIM);
441 return this->m_particle_tile_data.m_rdata[position_index][m_index];
442 }
443
445 RealType pos (int position_index) const &
446 {
447 AMREX_ASSERT(position_index < AMREX_SPACEDIM);
448 return this->m_particle_tile_data.m_rdata[position_index][m_index];
449 }
450
451 static Long NextID ();
452
456 static Long UnprotectedNextID ();
457
463 static void NextID (Long nextid);
464
465private :
466
467 static_assert(std::is_trivially_copyable<PTD>(), "ParticleTileData is not trivially copyable");
468
469 PTD m_particle_tile_data;
470 int m_index;
471};
472
473//template <int NArrayReal, int NArrayInt> Long ConstSoAParticle<NArrayReal, NArrayInt>::the_next_id = 1;
474template <int NArrayReal, int NArrayInt> Long SoAParticle<NArrayReal, NArrayInt>::the_next_id = 1;
475
476template <int NArrayReal, int NArrayInt>
477Long
479{
480 Long next;
481// we should be able to test on _OPENMP < 201107 for capture (version 3.1)
482// but we must work around a bug in gcc < 4.9
483#if defined(AMREX_USE_OMP) && defined(_OPENMP) && _OPENMP < 201307
484#pragma omp critical (amrex_particle_nextid)
485#elif defined(AMREX_USE_OMP)
486#pragma omp atomic capture
487#endif
488 next = the_next_id++;
489
491 amrex::Abort("SoAParticle<NArrayReal, NArrayInt>::NextID() -- too many particles");
492 }
493
494 return next;
495}
496
497template <int NArrayReal, int NArrayInt>
498Long
500{
501 Long next = the_next_id++;
503 amrex::Abort("SoAParticle<NArrayReal, NArrayInt>::NextID() -- too many particles");
504 }
505 return next;
506}
507
508template <int NArrayReal, int NArrayInt>
509void
514
515template <typename T_ParticleType, int NArrayReal, int NArrayInt>
517{
518 static constexpr int NAR = NArrayReal;
519 static constexpr int NAI = NArrayInt;
520 using ParticleType = T_ParticleType;
521 using ParticleRefType = T_ParticleType const&;
523 using IntType = int;
524
525 static constexpr int NStructReal = ParticleType::NReal;
526 static constexpr int NStructInt = ParticleType::NInt;
527
529
530 static constexpr bool is_particle_tile_data = true;
531
533
534 using AOS_PTR = std::conditional_t<T_ParticleType::is_soa_particle,
535 void const * AMREX_RESTRICT, ParticleType const * AMREX_RESTRICT>;
537
538 const uint64_t* m_idcpu;
541
543 decltype(auto) pos (const int dir, const int index) const &
544 {
545 if constexpr(!ParticleType::is_soa_particle) {
546 return this->m_aos[index].pos(dir);
547 } else {
548 return this->m_rdata[dir][index];
549 }
550 }
551
553 decltype(auto) id (const int index) const &
554 {
555 if constexpr(!ParticleType::is_soa_particle) {
556 return this->m_aos[index].id();
557 } else {
558 return ConstParticleIDWrapper(this->m_idcpu[index]);
559 }
560 }
561
563 decltype(auto) cpu (const int index) const &
564 {
565 if constexpr(!ParticleType::is_soa_particle) {
566 return this->m_aos[index].cpu();
567 } else {
568 return ConstParticleCPUWrapper(this->m_idcpu[index]);
569 }
570 }
571
573 decltype(auto) idcpu (const int index) const &
574 {
575 if constexpr(ParticleType::is_soa_particle) {
576 return this->m_idcpu[index];
577 } else {
578 return this->m_aos[index].idcpu();
579 }
580 }
581
583 const ParticleReal * rdata (const int attribute_index) const
584 {
585 return this->m_rdata[attribute_index];
586 }
587
589 const int * idata (const int attribute_index) const
590 {
591 return this->m_idata[attribute_index];
592 }
593
595 decltype(auto) operator[] (const int index) const
596 {
597 if constexpr (!ParticleType::is_soa_particle) {
598 return m_aos[index];
599 } else {
600 return ConstSoAParticle<NAR, NAI>(*this, index);
601 }
602 }
603
608
610 void packParticleData(char* buffer, int src_index, Long dst_offset,
611 const int* comm_real, const int * comm_int) const noexcept
612 {
613 AMREX_ASSERT(src_index < m_size);
614 auto* dst = buffer + dst_offset;
615 if constexpr (!ParticleType::is_soa_particle) {
616 memcpy(dst, m_aos + src_index, sizeof(ParticleType));
617 dst += sizeof(ParticleType);
618 } else {
619 memcpy(dst, m_idcpu + src_index, sizeof(uint64_t));
620 dst += sizeof(uint64_t);
621 }
622 int array_start_index = 0;
623 if constexpr (!ParticleType::is_soa_particle) {
624 array_start_index = AMREX_SPACEDIM + NStructReal;
625 }
626 if constexpr (NArrayReal > 0) {
627 for (int i = 0; i < NArrayReal; ++i)
628 {
629 if (comm_real[array_start_index + i])
630 {
631 memcpy(dst, m_rdata[i] + src_index, sizeof(ParticleReal));
632 dst += sizeof(ParticleReal);
633 }
634 }
635 }
636 int runtime_start_index = array_start_index + NArrayReal;
637 for (int i = 0; i < m_num_runtime_real; ++i)
638 {
639 if (comm_real[runtime_start_index + i])
640 {
641 memcpy(dst, m_runtime_rdata[i] + src_index, sizeof(ParticleReal));
642 dst += sizeof(ParticleReal);
643 }
644 }
645 array_start_index = 2 + NStructInt;
646 if constexpr (NArrayInt > 0) {
647 for (int i = 0; i < NArrayInt; ++i)
648 {
649 if (comm_int[array_start_index + i])
650 {
651 memcpy(dst, m_idata[i] + src_index, sizeof(int));
652 dst += sizeof(int);
653 }
654 }
655 }
656 runtime_start_index = 2 + NStructInt + NArrayInt;
657 for (int i = 0; i < m_num_runtime_int; ++i)
658 {
659 if (comm_int[runtime_start_index + i])
660 {
661 memcpy(dst, m_runtime_idata[i] + src_index, sizeof(int));
662 dst += sizeof(int);
663 }
664 }
665 }
666
668 SuperParticleType getSuperParticle (int index) const noexcept
669 requires (!ParticleType::is_soa_particle)
670 {
671 AMREX_ASSERT(index < m_size);
673 for (int i = 0; i < AMREX_SPACEDIM; ++i) {
674 sp.pos(i) = m_aos[index].pos(i);
675 }
676 for (int i = 0; i < NStructReal; ++i) {
677 sp.rdata(i) = m_aos[index].rdata(i);
678 }
679 if constexpr(NArrayReal > 0) {
680 for (int i = 0; i < NArrayReal; ++i) {
681 sp.rdata(NStructReal+i) = m_rdata[i][index];
682 }
683 }
684 sp.id() = m_aos[index].id();
685 sp.cpu() = m_aos[index].cpu();
686 for (int i = 0; i < NStructInt; ++i) {
687 sp.idata(i) = m_aos[index].idata(i);
688 }
689 if constexpr(NArrayInt > 0) {
690 for (int i = 0; i < NArrayInt; ++i) {
691 sp.idata(NStructInt+i) = m_idata[i][index];
692 }
693 }
694 return sp;
695 }
696
698 SuperParticleType getSuperParticle (int index) const noexcept
699 requires (ParticleType::is_soa_particle)
700 {
701 AMREX_ASSERT(index < m_size);
703 for (int i = 0; i < AMREX_SPACEDIM; ++i) {sp.pos(i) = m_rdata[i][index];}
704 sp.m_idcpu = m_idcpu[index];
705 for (int i = 0; i < NAR; ++i) {
706 sp.rdata(i) = m_rdata[i][index];
707 }
708 for (int i = 0; i < NAI; ++i) {
709 sp.idata(i) = m_idata[i][index];
710 }
711 return sp;
712 }
713};
714
716
720
722{
723 RuntimePtrCacheDirtyFlag () noexcept = default;
724
726
728 : m_value(true)
729 {}
730
732 : m_value(true)
733 {}
734
736 {
737 m_value.store(true, std::memory_order_relaxed);
738 return *this;
739 }
740
742 {
743 m_value.store(true, std::memory_order_relaxed);
744 return *this;
745 }
746
747 void store (bool value, std::memory_order order = std::memory_order_seq_cst) noexcept
748 {
749 m_value.store(value, order);
750 }
751
752 [[nodiscard]] bool load (std::memory_order order = std::memory_order_seq_cst) const noexcept
753 {
754 return m_value.load(order);
755 }
756
757private:
758 std::atomic<bool> m_value {true};
759};
760
761template <typename T_ParticleType, int NArrayReal, int NArrayInt,
762 template<class> class Allocator=DefaultAllocator>
764{
765 template <typename T>
766 using AllocatorType = Allocator<T>;
767
768 using ParticleType = T_ParticleType;
769 static constexpr int NAR = NArrayReal;
770 static constexpr int NAI = NArrayInt;
771 using RealType = typename ParticleType::RealType;
772
773 static constexpr int NStructReal = ParticleType::NReal;
774 static constexpr int NStructInt = ParticleType::NInt;
775
777
778 using AoS = std::conditional_t<
779 ParticleType::is_soa_particle,
782 using ParticleVector = typename AoS::ParticleVector;
783
784 using SoA = std::conditional_t<
785 ParticleType::is_soa_particle,
788 using RealVector = typename SoA::RealVector;
789 using IntVector = typename SoA::IntVector;
790 using StorageParticleType = typename ParticleType::StorageParticleType;
791
794
795 static constexpr bool has_polymorphic_allocator =
797
798 ParticleTile () = default;
799
800#ifndef _WIN32 // workaround windows compiler bug
801 ~ParticleTile () = default;
802
803 ParticleTile (ParticleTile const&) = delete;
804 ParticleTile (ParticleTile &&) noexcept = default;
805
806 ParticleTile& operator= (ParticleTile const&) = delete;
807 ParticleTile& operator= (ParticleTile &&) noexcept = default;
808#endif
809
810 void define (
811 int a_num_runtime_real,
812 int a_num_runtime_int,
813 std::vector<std::string>* soa_rdata_names=nullptr,
814 std::vector<std::string>* soa_idata_names=nullptr,
815 Arena* a_arena=nullptr
816 )
817 {
818 GetStructOfArrays().define(a_num_runtime_real, a_num_runtime_int, soa_rdata_names, soa_idata_names);
819
820 if constexpr (has_polymorphic_allocator) {
821 if (m_defined) {
822 // it is not allowed to change the arena after the tile has been defined
823 if constexpr (ParticleType::is_soa_particle) {
825 a_arena == GetStructOfArrays().GetIdCPUData().arena(),
826 "ParticleTile with PolymorphicArenaAllocator redefined with "
827 "different memory arena");
828 } else {
830 a_arena == m_aos_tile().arena(),
831 "ParticleTile with PolymorphicArenaAllocator redefined with "
832 "different memory arena");
833 }
834 }
835
836 AMREX_ALWAYS_ASSERT_WITH_MESSAGE(a_arena != nullptr,
837 "ParticleTile with PolymorphicArenaAllocator defined with no memory arena! "
838 "Make sure to call setArena() on the ParticleContainer before initialization or "
839 "to pass an Arena to ParticleTile::define()");
840
841 // A tile that was filled before it was defined (e.g. created with
842 // GetParticles(lev)[index]) holds memory from the allocator's default
843 // arena. Move that data to a_arena instead of only switching the
844 // allocator, so the memory is freed through the arena it came from.
845 auto set_arena = [a_arena] (auto& vec) {
846 using Vec = std::remove_reference_t<decltype(vec)>;
847 if (vec.arena() == a_arena) {
848 vec.setArena(a_arena);
849 return;
850 }
851 Vec tmp{typename Vec::allocator_type(a_arena)};
852 if (!vec.empty()) {
853 tmp.resize(vec.size(), GrowthStrategy::Exact);
854 // refer to the allocators as base classes, without copying them
855 using Alloc = typename Vec::allocator_type;
856 amrex::detail::memCopyImpl(tmp.data(), vec.data(),
857 vec.size()*sizeof(typename Vec::value_type),
858 static_cast<Alloc const&>(tmp),
859 static_cast<Alloc const&>(vec));
860 }
861 vec.swap(tmp); // tmp now frees the old memory with the old arena
862 };
863
864 if constexpr (ParticleType::is_soa_particle) {
865 set_arena(GetStructOfArrays().GetIdCPUData());
866 } else {
867 set_arena(m_aos_tile());
868 }
869 for (int j = 0; j < NumRealComps(); ++j) {
870 set_arena(GetStructOfArrays().GetRealData(j));
871 }
872 for (int j = 0; j < NumIntComps(); ++j) {
873 set_arena(GetStructOfArrays().GetIntData(j));
874 }
875
876 set_arena(m_runtime_r_ptrs);
877 set_arena(m_runtime_i_ptrs);
878 set_arena(m_runtime_r_cptrs);
879 set_arena(m_runtime_i_cptrs);
880 }
881
882 m_runtime_r_ptrs.resize(a_num_runtime_real);
883 m_runtime_i_ptrs.resize(a_num_runtime_int);
884 m_runtime_r_cptrs.resize(a_num_runtime_real);
885 m_runtime_i_cptrs.resize(a_num_runtime_int);
886
887 m_defined = true;
888 invalidateRuntimePtrCaches();
889 }
890
891 // Get id data
892 decltype(auto) id (int index) & {
893 if constexpr (!ParticleType::is_soa_particle) {
894 return m_aos_tile[index].id();
895 } else {
896 return ParticleIDWrapper(m_soa_tile.GetIdCPUData()[index]);
897 }
898 }
899
900 // const
901 decltype(auto) id (int index) const & {
902 if constexpr (!ParticleType::is_soa_particle) {
903 return m_aos_tile[index].id();
904 } else {
905 return ConstParticleIDWrapper(m_soa_tile.GetIdCPUData()[index]);
906 }
907 }
908
909 // Get cpu data
910 decltype(auto) cpu (int index) & {
911 if constexpr (!ParticleType::is_soa_particle) {
912 return m_aos_tile[index].cpu();
913 } else {
914 return ParticleCPUWrapper(m_soa_tile.GetIdCPUData()[index]);
915 }
916 }
917
918 // const
919 decltype(auto) cpu (int index) const & {
920 if constexpr (!ParticleType::is_soa_particle) {
921 return m_aos_tile[index].cpu();
922 } else {
923 return ConstParticleCPUWrapper(m_soa_tile.GetIdCPUData()[index]);
924 }
925 }
926
927 // Get positions data
928 RealType& pos (int index, int position_index) & {
929 if constexpr (!ParticleType::is_soa_particle) {
930 return m_aos_tile[index].pos(position_index);
931 } else {
932 static_assert(NArrayReal == ParticleType::PTD::NAR, "ParticleTile mismatch in R");
933 static_assert(NArrayInt == ParticleType::PTD::NAI, "ParticleTile mismatch in I");
934 static_assert(0 == ParticleType::StorageParticleType::NReal, "ParticleTile 2 mismatch in R");
935 static_assert(0 == ParticleType::StorageParticleType::NInt, "ParticleTile 2 mismatch in I");
936
937 return m_soa_tile.GetRealData(position_index)[index];
938 }
939 }
940
941 // const
942 RealType pos (int index, int position_index) const &
943 {
944 if constexpr (!ParticleType::is_soa_particle) {
945 return m_aos_tile[index].pos(position_index);
946 } else {
947 return m_soa_tile.GetRealData(position_index)[index];
948 }
949 }
950
951 AoS& GetArrayOfStructs () { invalidateRuntimePtrCaches(); return m_aos_tile; }
952 const AoS& GetArrayOfStructs () const { return m_aos_tile; }
953
954 SoA& GetStructOfArrays () { invalidateRuntimePtrCaches(); return m_soa_tile; }
955 const SoA& GetStructOfArrays () const { return m_soa_tile; }
956
957 bool empty () const { return size() == 0; }
958
963 std::size_t size () const
964 {
965 if constexpr (!ParticleType::is_soa_particle) {
966 return m_aos_tile.size();
967 } else {
968 return m_soa_tile.size();
969 }
970 }
971
976 int numParticles () const
977 {
978 if constexpr (!ParticleType::is_soa_particle) {
979 return m_aos_tile.numParticles();
980 } else {
981 return m_soa_tile.numParticles();
982 }
983 }
984
989 int numRealParticles () const
990 {
991 if constexpr (!ParticleType::is_soa_particle) {
992 return m_aos_tile.numRealParticles();
993 } else {
994 return m_soa_tile.numRealParticles();
995 }
996 }
997
1003 {
1004 if constexpr (!ParticleType::is_soa_particle) {
1005 return m_aos_tile.numNeighborParticles();
1006 } else {
1007 return m_soa_tile.numNeighborParticles();
1008 }
1009 }
1010
1016 {
1017 if constexpr (!ParticleType::is_soa_particle) {
1018 return m_aos_tile.numTotalParticles();
1019 } else {
1020 return m_soa_tile.numTotalParticles();
1021 }
1022 }
1023
1024 void setNumNeighbors (int num_neighbors)
1025 {
1026 invalidateRuntimePtrCaches();
1027 if constexpr(!ParticleType::is_soa_particle) {
1028 m_aos_tile.setNumNeighbors(num_neighbors);
1029 }
1030 m_soa_tile.setNumNeighbors(num_neighbors);
1031 }
1032
1033 int getNumNeighbors () const
1034 {
1035 if constexpr (!ParticleType::is_soa_particle) {
1036 AMREX_ASSERT( m_soa_tile.getNumNeighbors() == m_aos_tile.getNumNeighbors() );
1037 return m_aos_tile.getNumNeighbors();
1038 } else {
1039 return m_soa_tile.getNumNeighbors();
1040 }
1041 }
1042
1043 void resize (std::size_t count, GrowthStrategy strategy = GrowthStrategy::Poisson)
1044 {
1045 invalidateRuntimePtrCaches();
1046 if constexpr (!ParticleType::is_soa_particle) {
1047 m_aos_tile.resize(count, strategy);
1048 }
1049 m_soa_tile.resize(count, strategy);
1050 }
1051
1053 {
1054 invalidateRuntimePtrCaches();
1055 if constexpr (!ParticleType::is_soa_particle) {
1056 m_aos_tile.reserve(capacity, strategy);
1057 }
1058 m_soa_tile.reserve(capacity, strategy);
1059 }
1060
1065 requires (!ParticleType::is_soa_particle)
1066 {
1067 invalidateRuntimePtrCaches();
1068 m_aos_tile().push_back(p);
1069 }
1070
1075 requires (NArrayReal != 0 || NArrayInt != 0)
1076 {
1077 invalidateRuntimePtrCaches();
1078 auto np = numParticles();
1079
1080 if constexpr (!ParticleType::is_soa_particle) {
1081 m_aos_tile.resize(np+1);
1082 for (int i = 0; i < AMREX_SPACEDIM; ++i) {
1083 m_aos_tile[np].pos(i) = sp.pos(i);
1084 }
1085 for (int i = 0; i < NStructReal; ++i) {
1086 m_aos_tile[np].rdata(i) = sp.rdata(i);
1087 }
1088 m_aos_tile[np].id() = sp.id();
1089 m_aos_tile[np].cpu() = sp.cpu();
1090 for (int i = 0; i < NStructInt; ++i) {
1091 m_aos_tile[np].idata(i) = sp.idata(i);
1092 }
1093 }
1094
1095 m_soa_tile.resize(np+1);
1096 if constexpr (ParticleType::is_soa_particle) {
1097 m_soa_tile.GetIdCPUData()[np] = sp.m_idcpu;
1098 }
1099 auto& arr_rdata = m_soa_tile.GetRealData();
1100 auto& arr_idata = m_soa_tile.GetIntData();
1101 for (int i = 0; i < NArrayReal; ++i) {
1102 arr_rdata[i][np] = sp.rdata(NStructReal+i);
1103 }
1104 for (int i = 0; i < NArrayInt; ++i) {
1105 arr_idata[i][np] = sp.idata(NStructInt+i);
1106 }
1107 }
1108
1113 void push_back_real (int comp, ParticleReal v) {
1114 invalidateRuntimePtrCaches();
1115 m_soa_tile.GetRealData(comp).push_back(v);
1116 }
1117
1122 void push_back_real (const std::array<ParticleReal, NArrayReal>& v) {
1123 for (int i = 0; i < NArrayReal; ++i) {
1124 m_soa_tile.GetRealData(i).push_back(v[i]);
1125 }
1126 }
1127
1132 void push_back_real (int comp, const ParticleReal* beg, const ParticleReal* end) {
1133 invalidateRuntimePtrCaches();
1134 auto it = m_soa_tile.GetRealData(comp).end();
1135 m_soa_tile.GetRealData(comp).insert(it, beg, end);
1136 }
1137
1142 void push_back_real (int comp, amrex::Vector<amrex::ParticleReal>::const_iterator beg, amrex::Vector<amrex::ParticleReal>::const_iterator end) {
1143 push_back_real(comp, &(*beg), &(*end));
1144 }
1145
1151 push_back_real(comp, vec.cbegin(), vec.cend());
1152 }
1153
1158 void push_back_real (int comp, std::size_t npar, ParticleReal v) {
1159 invalidateRuntimePtrCaches();
1160 auto new_size = m_soa_tile.GetRealData(comp).size() + npar;
1161 m_soa_tile.GetRealData(comp).resize(new_size, v);
1162 }
1163
1168 void push_back_int (int comp, int v) {
1169 invalidateRuntimePtrCaches();
1170 m_soa_tile.GetIntData(comp).push_back(v);
1171 }
1172
1177 void push_back_int (const std::array<int, NArrayInt>& v) {
1178 for (int i = 0; i < NArrayInt; ++i) {
1179 m_soa_tile.GetIntData(i).push_back(v[i]);
1180 }
1181 }
1182
1187 void push_back_int (int comp, const int* beg, const int* end) {
1188 invalidateRuntimePtrCaches();
1189 auto it = m_soa_tile.GetIntData(comp).end();
1190 m_soa_tile.GetIntData(comp).insert(it, beg, end);
1191 }
1192
1197 void push_back_int (int comp, amrex::Vector<int>::const_iterator beg, amrex::Vector<int>::const_iterator end) {
1198 push_back_int(comp, &(*beg), &(*end));
1199 }
1200
1205 void push_back_int (int comp, amrex::Vector<int> const & vec) {
1206 push_back_int(comp, vec.cbegin(), vec.cend());
1207 }
1208
1213 void push_back_int (int comp, std::size_t npar, int v) {
1214 invalidateRuntimePtrCaches();
1215 auto new_size = m_soa_tile.GetIntData(comp).size() + npar;
1216 m_soa_tile.GetIntData(comp).resize(new_size, v);
1217 }
1218
1219 int NumRealComps () const noexcept { return m_soa_tile.NumRealComps(); }
1220
1221 int NumIntComps () const noexcept { return m_soa_tile.NumIntComps(); }
1222
1223 int NumRuntimeRealComps () const noexcept { return m_runtime_r_ptrs.size(); }
1224
1225 int NumRuntimeIntComps () const noexcept { return m_runtime_i_ptrs.size(); }
1226
1228 {
1229 invalidateRuntimePtrCaches();
1230 if constexpr (ParticleType::is_soa_particle) {
1231 GetStructOfArrays().GetIdCPUData().shrink_to_fit();
1232 } else {
1233 m_aos_tile().shrink_to_fit();
1234 }
1235 for (int j = 0; j < NumRealComps(); ++j)
1236 {
1237 auto& rdata = GetStructOfArrays().GetRealData(j);
1238 rdata.shrink_to_fit();
1239 }
1240
1241 for (int j = 0; j < NumIntComps(); ++j)
1242 {
1243 auto& idata = GetStructOfArrays().GetIntData(j);
1244 idata.shrink_to_fit();
1245 }
1246 }
1247
1249 {
1250 Long nbytes = 0;
1251 if constexpr (ParticleType::is_soa_particle) {
1252 nbytes += GetStructOfArrays().GetIdCPUData().capacity() * sizeof(uint64_t);
1253 } else {
1254 nbytes += m_aos_tile().capacity() * sizeof(ParticleType);
1255 }
1256 for (int j = 0; j < NumRealComps(); ++j)
1257 {
1258 auto& rdata = GetStructOfArrays().GetRealData(j);
1259 nbytes += rdata.capacity() * sizeof(ParticleReal);
1260 }
1261
1262 for (int j = 0; j < NumIntComps(); ++j)
1263 {
1264 auto& idata = GetStructOfArrays().GetIntData(j);
1265 nbytes += idata.capacity()*sizeof(int);
1266 }
1267 return nbytes;
1268 }
1269
1271 {
1272 if constexpr (ParticleType::is_soa_particle) {
1273 GetStructOfArrays().GetIdCPUData().swap(other.GetStructOfArrays().GetIdCPUData());
1274 } else {
1275 m_aos_tile().swap(other.GetArrayOfStructs()());
1276 }
1277 for (int j = 0; j < NumRealComps(); ++j)
1278 {
1279 auto& rdata = GetStructOfArrays().GetRealData(j);
1280 rdata.swap(other.GetStructOfArrays().GetRealData(j));
1281 }
1282
1283 for (int j = 0; j < NumIntComps(); ++j)
1284 {
1285 auto& idata = GetStructOfArrays().GetIntData(j);
1286 idata.swap(other.GetStructOfArrays().GetIntData(j));
1287 }
1288 invalidateRuntimePtrCaches();
1289 other.invalidateRuntimePtrCaches();
1290 }
1291
1293 {
1294 refreshRuntimePtrCaches();
1295
1297 if constexpr (!ParticleType::is_soa_particle) {
1298 ptd.m_aos = m_aos_tile().dataPtr();
1299 } else {
1300 ptd.m_aos = nullptr;
1301 }
1302 if constexpr (ParticleType::is_soa_particle) {
1303 ptd.m_idcpu = m_soa_tile.GetIdCPUData().dataPtr();
1304 } else {
1305 ptd.m_idcpu = nullptr;
1306 }
1307 if constexpr(NArrayReal > 0) {
1308 for (int i = 0; i < NArrayReal; ++i) {
1309 ptd.m_rdata[i] = m_soa_tile.GetRealData(i).dataPtr();
1310 }
1311 }
1312 if constexpr(NArrayInt > 0) {
1313 for (int i = 0; i < NArrayInt; ++i) {
1314 ptd.m_idata[i] = m_soa_tile.GetIntData(i).dataPtr();
1315 }
1316 }
1317 ptd.m_size = size();
1318 ptd.m_num_runtime_real = m_runtime_r_ptrs.size();
1319 ptd.m_num_runtime_int = m_runtime_i_ptrs.size();
1320 ptd.m_runtime_rdata = m_runtime_r_ptrs.dataPtr();
1321 ptd.m_runtime_idata = m_runtime_i_ptrs.dataPtr();
1322
1323 return ptd;
1324 }
1325
1327 {
1328 refreshConstRuntimePtrCaches();
1329
1331 if constexpr (!ParticleType::is_soa_particle) {
1332 ptd.m_aos = m_aos_tile().dataPtr();
1333 } else {
1334 ptd.m_aos = nullptr;
1335 }
1336 if constexpr (ParticleType::is_soa_particle) {
1337 ptd.m_idcpu = m_soa_tile.GetIdCPUData().dataPtr();
1338 } else {
1339 ptd.m_idcpu = nullptr;
1340 }
1341 if constexpr(NArrayReal > 0) {
1342 for (int i = 0; i < NArrayReal; ++i) {
1343 ptd.m_rdata[i] = m_soa_tile.GetRealData(i).dataPtr();
1344 }
1345 }
1346 if constexpr(NArrayInt > 0) {
1347 for (int i = 0; i < NArrayInt; ++i) {
1348 ptd.m_idata[i] = m_soa_tile.GetIntData(i).dataPtr();
1349 }
1350 }
1351 ptd.m_size = size();
1352 ptd.m_num_runtime_real = m_runtime_r_cptrs.size();
1353 ptd.m_num_runtime_int = m_runtime_i_cptrs.size();
1354 ptd.m_runtime_rdata = m_runtime_r_cptrs.dataPtr();
1355 ptd.m_runtime_idata = m_runtime_i_cptrs.dataPtr();
1356
1357 return ptd;
1358 }
1359
1363 {
1364 invalidateRuntimePtrCaches();
1365 if constexpr (!ParticleType::is_soa_particle) {
1366 m_aos_tile.collectVectors(pv);
1367 } else {
1369 }
1370 m_soa_tile.collectVectors(idcpuv, rv, iv);
1371 }
1372
1373 static void reserve (std::map<ParticleTile<T_ParticleType,NArrayReal,NArrayInt,Allocator>*, int> const& addsizes)
1374 {
1375 if constexpr (!IsArenaAllocator<Allocator<int>>::value) {
1376 for (auto [p,s] : addsizes) {
1377 p->reserve(p->size()+s);
1378 }
1379 } else {
1380 using PV = std::conditional_t<ParticleType::is_soa_particle,
1386 for (auto [p,s] : addsizes) {
1387 if (s > 0) {
1392 p->collectVectors(pv, idcpuv, rv, iv);
1393 if constexpr (!ParticleType::is_soa_particle) {
1394 for (auto* v : pv) {
1395 pvs.emplace_back(v, s);
1396 }
1397 }
1398 for (auto* v : idcpuv) {
1399 ids.emplace_back(v, s);
1400 }
1401 for (auto* v : rv) {
1402 rvs.emplace_back(v, s);
1403 }
1404 for (auto* v : iv) {
1405 ivs.emplace_back(v, s);
1406 }
1407 }
1408 }
1409
1410 std::ranges::sort(pvs, [] (auto const& a, auto const& b) {
1411 return (a.first->size() + a.second) >
1412 (b.first->size() + b.second);
1413 });
1414 std::ranges::sort(ids, [] (auto const& a, auto const& b) {
1415 return (a.first->size() + a.second) >
1416 (b.first->size() + b.second);
1417 });
1418 std::ranges::sort(rvs, [] (auto const& a, auto const& b) {
1419 return (a.first->size() + a.second) >
1420 (b.first->size() + b.second);
1421 });
1422 std::ranges::sort(ivs, [] (auto const& a, auto const& b) {
1423 return (a.first->size() + a.second) >
1424 (b.first->size() + b.second);
1425 });
1426
1427 // Handle big vectors first
1428 {
1429 int i_pvs = 0, i_ids = 0, i_rvs = 0, i_ivs = 0;
1430 auto n_pvs = int(pvs.size());
1431 auto n_ids = int(ids.size());
1432 auto n_rvs = int(rvs.size());
1433 auto n_ivs = int(ivs.size());
1434 while ((i_pvs < n_pvs) || (i_ids < n_ids) || (i_rvs < n_rvs) ||
1435 (i_ivs < n_ivs)) {
1436 std::size_t nbytes = 0;
1437 int ii = -1;
1438 if (i_pvs < n_pvs) {
1439 std::size_t my_bytes = (pvs[i_pvs].first->size()
1440 + pvs[i_pvs].second) * sizeof(typename PV::value_type);
1441 if (my_bytes > nbytes) {
1442 nbytes = my_bytes;
1443 ii = 0;
1444 }
1445 }
1446 if (i_ids < n_ids) {
1447 std::size_t my_bytes = (ids[i_ids].first->size()
1448 + ids[i_ids].second) * sizeof(typename SoA::IdCPU::value_type);
1449 if (my_bytes > nbytes) {
1450 nbytes = my_bytes;
1451 ii = 1;
1452 }
1453 }
1454 if (i_rvs < n_rvs) {
1455 std::size_t my_bytes = (rvs[i_rvs].first->size()
1456 + rvs[i_rvs].second) * sizeof(typename RealVector::value_type);
1457 if (my_bytes > nbytes) {
1458 nbytes = my_bytes;
1459 ii = 2;
1460 }
1461 }
1462 if (i_ivs < n_ivs) {
1463 std::size_t my_bytes = (ivs[i_ivs].first->size()
1464 + ivs[i_ivs].second) * sizeof(typename IntVector::value_type);
1465 if (my_bytes > nbytes) {
1466 nbytes = my_bytes;
1467 ii = 3;
1468 }
1469 }
1470 if (ii == 0) {
1471 auto [p,s] = pvs[i_pvs++];
1472 p->reserve(p->size() + s);
1473 } else if (ii == 1) {
1474 auto [p,s] = ids[i_ids++];
1475 p->reserve(p->size() + s);
1476 } else if (ii == 2) {
1477 auto [p,s] = rvs[i_rvs++];
1478 p->reserve(p->size() + s);
1479 } else {
1480 auto [p,s] = ivs[i_ivs++];
1481 p->reserve(p->size() + s);
1482 }
1483 }
1484 }
1485 }
1486 }
1487
1488private:
1489
1490 void invalidateRuntimePtrCaches () noexcept
1491 {
1492 m_runtime_ptrs_dirty.store(true, std::memory_order_release);
1493 m_runtime_cptrs_dirty.store(true, std::memory_order_release);
1494 }
1495
1496 void refreshRuntimePtrCaches ()
1497 {
1498 if (!m_runtime_ptrs_dirty.load(std::memory_order_acquire)) {
1499 return;
1500 }
1501#if defined(AMREX_USE_OMP)
1502#pragma omp critical (amrex_particle_tile_data_cache)
1503#endif
1504 {
1505 if (m_runtime_ptrs_dirty.load(std::memory_order_relaxed)) {
1506 m_runtime_r_ptrs.resize(m_soa_tile.NumRealComps() - NArrayReal);
1507 m_runtime_i_ptrs.resize(m_soa_tile.NumIntComps() - NArrayInt);
1508#ifdef AMREX_USE_GPU
1509 bool copy_real = false;
1510 m_h_runtime_r_ptrs.resize(m_soa_tile.NumRealComps() - NArrayReal, nullptr);
1511 for (std::size_t i = 0; i < m_h_runtime_r_ptrs.size(); ++i) {
1512 if (m_h_runtime_r_ptrs[i] != m_soa_tile.GetRealData(i + NArrayReal).dataPtr()) {
1513 m_h_runtime_r_ptrs[i] = m_soa_tile.GetRealData(i + NArrayReal).dataPtr();
1514 copy_real = true;
1515 }
1516 }
1517 if (copy_real) {
1518 Gpu::htod_memcpy_async(m_runtime_r_ptrs.data(), m_h_runtime_r_ptrs.data(),
1519 m_h_runtime_r_ptrs.size()*sizeof(ParticleReal*));
1520 }
1521
1522 bool copy_int = false;
1523 m_h_runtime_i_ptrs.resize(m_soa_tile.NumIntComps() - NArrayInt, nullptr);
1524 for (std::size_t i = 0; i < m_h_runtime_i_ptrs.size(); ++i) {
1525 if (m_h_runtime_i_ptrs[i] != m_soa_tile.GetIntData(i + NArrayInt).dataPtr()) {
1526 m_h_runtime_i_ptrs[i] = m_soa_tile.GetIntData(i + NArrayInt).dataPtr();
1527 copy_int = true;
1528 }
1529 }
1530 if (copy_int) {
1531 Gpu::htod_memcpy_async(m_runtime_i_ptrs.data(), m_h_runtime_i_ptrs.data(),
1532 m_h_runtime_i_ptrs.size()*sizeof(int*));
1533 }
1534
1535 if (copy_real || copy_int) {
1537 }
1538#else
1539 for (std::size_t i = 0; i < m_runtime_r_ptrs.size(); ++i) {
1540 m_runtime_r_ptrs[i] = m_soa_tile.GetRealData(i + NArrayReal).dataPtr();
1541 }
1542
1543 for (std::size_t i = 0; i < m_runtime_i_ptrs.size(); ++i) {
1544 m_runtime_i_ptrs[i] = m_soa_tile.GetIntData(i + NArrayInt).dataPtr();
1545 }
1546#endif
1547 m_runtime_ptrs_dirty.store(false, std::memory_order_release);
1548 }
1549 }
1550 }
1551
1552 void refreshConstRuntimePtrCaches () const
1553 {
1554 if (!m_runtime_cptrs_dirty.load(std::memory_order_acquire)) {
1555 return;
1556 }
1557#if defined(AMREX_USE_OMP)
1558#pragma omp critical (amrex_const_particle_tile_data_cache)
1559#endif
1560 {
1561 if (m_runtime_cptrs_dirty.load(std::memory_order_relaxed)) {
1562 m_runtime_r_cptrs.resize(m_soa_tile.NumRealComps() - NArrayReal);
1563 m_runtime_i_cptrs.resize(m_soa_tile.NumIntComps() - NArrayInt);
1564#ifdef AMREX_USE_GPU
1565 bool copy_real = false;
1566 m_h_runtime_r_cptrs.resize(m_soa_tile.NumRealComps() - NArrayReal, nullptr);
1567 for (std::size_t i = 0; i < m_h_runtime_r_cptrs.size(); ++i) {
1568 if (m_h_runtime_r_cptrs[i] != m_soa_tile.GetRealData(i + NArrayReal).dataPtr()) {
1569 m_h_runtime_r_cptrs[i] = m_soa_tile.GetRealData(i + NArrayReal).dataPtr();
1570 copy_real = true;
1571 }
1572 }
1573 if (copy_real) {
1574 Gpu::htod_memcpy_async(m_runtime_r_cptrs.data(), m_h_runtime_r_cptrs.data(),
1575 m_h_runtime_r_cptrs.size()*sizeof(ParticleReal*));
1576 }
1577
1578 bool copy_int = false;
1579 m_h_runtime_i_cptrs.resize(m_soa_tile.NumIntComps() - NArrayInt, nullptr);
1580 for (std::size_t i = 0; i < m_h_runtime_i_cptrs.size(); ++i) {
1581 if (m_h_runtime_i_cptrs[i] != m_soa_tile.GetIntData(i + NArrayInt).dataPtr()) {
1582 m_h_runtime_i_cptrs[i] = m_soa_tile.GetIntData(i + NArrayInt).dataPtr();
1583 copy_int = true;
1584 }
1585 }
1586 if (copy_int) {
1587 Gpu::htod_memcpy_async(m_runtime_i_cptrs.data(), m_h_runtime_i_cptrs.data(),
1588 m_h_runtime_i_cptrs.size()*sizeof(int*));
1589 }
1590
1591 if (copy_real || copy_int) {
1593 }
1594#else
1595 for (std::size_t i = 0; i < m_runtime_r_cptrs.size(); ++i) {
1596 m_runtime_r_cptrs[i] = m_soa_tile.GetRealData(i + NArrayReal).dataPtr();
1597 }
1598
1599 for (std::size_t i = 0; i < m_runtime_i_cptrs.size(); ++i) {
1600 m_runtime_i_cptrs[i] = m_soa_tile.GetIntData(i + NArrayInt).dataPtr();
1601 }
1602#endif
1603 m_runtime_cptrs_dirty.store(false, std::memory_order_release);
1604 }
1605 }
1606 }
1607
1608 AoS m_aos_tile;
1609 SoA m_soa_tile;
1610
1611 bool m_defined = false;
1612
1613 amrex::PODVector<ParticleReal*, Allocator<ParticleReal*> > m_runtime_r_ptrs;
1614 amrex::PODVector<int*, Allocator<int*> > m_runtime_i_ptrs;
1615
1616 mutable amrex::PODVector<const ParticleReal*, Allocator<const ParticleReal*> > m_runtime_r_cptrs;
1617 mutable amrex::PODVector<const int*, Allocator<const int*> >m_runtime_i_cptrs;
1618
1619 amrex::Gpu::HostVector<ParticleReal*> m_h_runtime_r_ptrs;
1620 amrex::Gpu::HostVector<int*> m_h_runtime_i_ptrs;
1621
1622 mutable amrex::Gpu::HostVector<const ParticleReal*> m_h_runtime_r_cptrs;
1623 mutable amrex::Gpu::HostVector<const int*> m_h_runtime_i_cptrs;
1624
1625 RuntimePtrCacheDirtyFlag m_runtime_ptrs_dirty;
1626 mutable RuntimePtrCacheDirtyFlag m_runtime_cptrs_dirty;
1627};
1628
1629} // namespace amrex
1630
1631#endif // AMREX_PARTICLETILE_H_
#define AMREX_ALWAYS_ASSERT_WITH_MESSAGE(EX, MSG)
Definition AMReX_BLassert.H:49
#define AMREX_ASSERT(EX)
Definition AMReX_BLassert.H:38
Compiler- and backend-specific extension macros (e.g., restrict, SIMD, inline).
#define AMREX_FORCE_INLINE
Definition AMReX_Extension.H:123
#define AMREX_RESTRICT
Definition AMReX_Extension.H:37
#define AMREX_GPU_HOST_DEVICE
Definition AMReX_GpuQualifiers.H:20
while parser_exe_t p
Definition AMReX_Parser_Exe_Body.H:33
#define AMREX_D_DECL(a, b, c)
Definition AMReX_SPACE.H:171
Abstract base class for memory arenas.
Definition AMReX_Arena.H:154
Definition AMReX_ArrayOfStructs.H:13
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition AMReX_Vector.H:29
Long size() const noexcept
Definition AMReX_Vector.H:54
amrex_particle_real ParticleReal
Floating Point Type for Particles.
Definition AMReX_REAL.H:91
amrex_long Long
Definition AMReX_INT.H:30
PinnedVector< T > HostVector
A PODVector that uses pinned host memory. Same as PinnedVector. For a vector class that uses std::all...
Definition AMReX_GpuContainers.H:67
void streamSynchronize() noexcept
Definition AMReX_GpuDevice.H:329
void htod_memcpy_async(void *p_d, const void *p_h, const std::size_t sz) noexcept
Definition AMReX_GpuDevice.H:440
constexpr Long LastParticleID
Definition AMReX_Particle.H:21
Definition AMReX_Amr.cpp:50
__host__ __device__ void ignore_unused(const Ts &...)
No-op helper that marks variables as intentionally unused.
Definition AMReX.H:273
amrex::ArenaAllocator< T > DefaultAllocator
Definition AMReX_GpuAllocators.H:205
GrowthStrategy
Definition AMReX_PODVector.H:250
@ Exact
Definition AMReX_PODVector.H:250
@ Poisson
Definition AMReX_PODVector.H:250
double second() noexcept
Definition AMReX_Utility.cpp:874
void Abort(const std::string &msg)
Print a fatal-error message to stderr and abort execution.
Definition AMReX.cpp:239
const int[]
Definition AMReX_BLProfiler.cpp:1665
RealVectND< 3 > RealVect
Definition AMReX_ParmParse.H:40
__host__ __device__ Dim3 end(BoxND< dim > const &box) noexcept
Return the iterator end coordinate of box as Dim3.
Definition AMReX_Box.H:2259
Definition AMReX_Particle.H:327
Definition AMReX_Particle.H:301
Definition AMReX_ParticleTile.H:517
GpuArray< const int *, NArrayInt > m_idata
Definition AMReX_ParticleTile.H:540
std::conditional_t< typename ParticleType::StorageParticleType::is_soa_particle, void const *__restrict__, ParticleType const *__restrict__ > AOS_PTR
Definition AMReX_ParticleTile.H:534
Particle< NStructReal+NArrayReal, NStructInt+NArrayInt > SuperParticleType
Definition AMReX_ParticleTile.H:528
__host__ __device__ decltype(auto) idcpu(const int index) const &
Definition AMReX_ParticleTile.H:573
__host__ __device__ const int * idata(const int attribute_index) const
Definition AMReX_ParticleTile.H:589
typename ParticleType::StorageParticleType ParticleType
Definition AMReX_ParticleTile.H:520
__host__ __device__ SuperParticleType getSuperParticle(int index) const noexcept
Definition AMReX_ParticleTile.H:698
__host__ __device__ void packParticleData(char *buffer, int src_index, Long dst_offset, const int *comm_real, const int *comm_int) const noexcept
Definition AMReX_ParticleTile.H:610
const int *__restrict__ *__restrict__ m_runtime_idata
Definition AMReX_ParticleTile.H:607
__host__ __device__ SuperParticleType getSuperParticle(int index) const noexcept
Definition AMReX_ParticleTile.H:668
__host__ __device__ decltype(auto) pos(const int dir, const int index) const &
Definition AMReX_ParticleTile.H:543
__host__ __device__ decltype(auto) cpu(const int index) const &
Definition AMReX_ParticleTile.H:563
typename ParticleType::StorageParticleType const & ParticleRefType
Definition AMReX_ParticleTile.H:521
__host__ __device__ const ParticleReal * rdata(const int attribute_index) const
Definition AMReX_ParticleTile.H:583
GpuArray< const ParticleReal *, NArrayReal > m_rdata
Definition AMReX_ParticleTile.H:539
const ParticleReal *__restrict__ *__restrict__ m_runtime_rdata
Definition AMReX_ParticleTile.H:606
Definition AMReX_ParticleTile.H:327
__host__ __device__ ConstParticleIDWrapper id() const
Definition AMReX_ParticleTile.H:353
static constexpr int NArrayReal
Definition AMReX_ParticleTile.H:328
ParticleReal RealType
Definition AMReX_ParticleTile.H:336
__host__ __device__ ConstSoAParticle(ConstPTD const &ptd, long i)
Definition AMReX_ParticleTile.H:340
SoAParticleBase StorageParticleType
Definition AMReX_ParticleTile.H:330
static constexpr int NArrayInt
Definition AMReX_ParticleTile.H:329
static constexpr bool is_rtsoa_particle
Definition AMReX_ParticleTile.H:333
static void NextID(Long nextid)
Reset on restart.
static constexpr bool is_constsoa_particle
Definition AMReX_ParticleTile.H:334
ConstParticleTileData< SoAParticleBase, NArrayReal, NArrayInt > ConstPTD
Definition AMReX_ParticleTile.H:331
static Long UnprotectedNextID()
This version can only be used inside omp critical.
__host__ __device__ const RealType & pos(int position_index) const &
Definition AMReX_ParticleTile.H:361
static constexpr bool is_soa_particle
Definition AMReX_ParticleTile.H:332
__host__ __device__ RealVect pos() const &
Definition AMReX_ParticleTile.H:358
__host__ __device__ ConstParticleCPUWrapper cpu() const
Definition AMReX_ParticleTile.H:350
int IntType
Definition AMReX_ParticleTile.H:337
Fixed-size array that can be used on GPU.
Definition AMReX_Array.H:52
Definition AMReX_GpuAllocators.H:173
Definition AMReX_GpuAllocators.H:184
uint64_t m_idcpu
Definition AMReX_Particle.H:359
Definition AMReX_Particle.H:259
Definition AMReX_Particle.H:154
Definition AMReX_ParticleTile.H:34
std::conditional_t< typename ParticleType::StorageParticleType::is_soa_particle, void *__restrict__, ParticleType *__restrict__ > AOS_PTR
Definition AMReX_ParticleTile.H:53
__host__ __device__ SuperParticleType getSuperParticle(int index) const noexcept
Definition AMReX_ParticleTile.H:241
typename ParticleType::StorageParticleType & ParticleRefType
Definition AMReX_ParticleTile.H:39
__host__ __device__ decltype(auto) cpu(const int index) const &
Definition AMReX_ParticleTile.H:87
GpuArray< ParticleReal *, NAR > m_rdata
Definition AMReX_ParticleTile.H:58
__host__ __device__ void unpackParticleData(const char *buffer, Long src_offset, int dst_index, const int *comm_real, const int *comm_int) const noexcept
Definition AMReX_ParticleTile.H:183
ParticleTileData< ParticleType, NAR, NAI > Self
Definition AMReX_ParticleTile.H:40
__host__ __device__ decltype(auto) idcpu(const int index) const &
Definition AMReX_ParticleTile.H:97
__host__ __device__ ParticleReal * rdata(const int attribute_index) const
Definition AMReX_ParticleTile.H:107
__host__ __device__ void packParticleData(char *buffer, int src_index, std::size_t dst_offset, const int *comm_real, const int *comm_int) const noexcept
Definition AMReX_ParticleTile.H:129
__host__ __device__ void setSuperParticle(const SuperParticleType &sp, int index) const noexcept
Definition AMReX_ParticleTile.H:311
__host__ __device__ decltype(auto) pos(const int dir, const int index) const &
Definition AMReX_ParticleTile.H:67
typename ParticleType::StorageParticleType ParticleType
Definition AMReX_ParticleTile.H:38
__host__ __device__ SuperParticleType getSuperParticle(int index) const noexcept
Definition AMReX_ParticleTile.H:271
__host__ __device__ void setSuperParticle(const SuperParticleType &sp, int index) const noexcept
Definition AMReX_ParticleTile.H:288
Particle< NStructReal+NAR, NStructInt+NAI > SuperParticleType
Definition AMReX_ParticleTile.H:47
ParticleReal *__restrict__ *__restrict__ m_runtime_rdata
Definition AMReX_ParticleTile.H:63
static constexpr bool is_particle_tile_data
Definition AMReX_ParticleTile.H:49
__host__ __device__ int * idata(const int attribute_index) const
Definition AMReX_ParticleTile.H:113
int *__restrict__ *__restrict__ m_runtime_idata
Definition AMReX_ParticleTile.H:64
typename ParticleType::StorageParticleType StorageParticleType
Definition AMReX_ParticleTile.H:790
int NumRealComps() const noexcept
Definition AMReX_ParticleTile.H:1219
void push_back_real(int comp, amrex::Vector< amrex::ParticleReal > const &vec)
Definition AMReX_ParticleTile.H:1150
int NumIntComps() const noexcept
Definition AMReX_ParticleTile.H:1221
typename ParticleType::RealType RealType
Definition AMReX_ParticleTile.H:771
ParticleTile()=default
ParticleTileData< StorageParticleType, NArrayReal, NArrayInt > ParticleTileDataType
Definition AMReX_ParticleTile.H:792
int getNumNeighbors() const
Definition AMReX_ParticleTile.H:1033
typename AoS::ParticleVector ParticleVector
Definition AMReX_ParticleTile.H:782
ParticleTile(ParticleTile const &)=delete
const SoA & GetStructOfArrays() const
Definition AMReX_ParticleTile.H:955
static constexpr int NAI
Definition AMReX_ParticleTile.H:770
ParticleTile(ParticleTile &&) noexcept=default
void push_back_real(int comp, const ParticleReal *beg, const ParticleReal *end)
Definition AMReX_ParticleTile.H:1132
static constexpr int NStructInt
Definition AMReX_ParticleTile.H:774
std::conditional_t< ParticleType::is_soa_particle, StructOfArrays< NArrayReal, NArrayInt, Allocator, true >, StructOfArrays< NArrayReal, NArrayInt, Allocator, false > > SoA
Definition AMReX_ParticleTile.H:784
void setNumNeighbors(int num_neighbors)
Definition AMReX_ParticleTile.H:1024
Long capacity() const
Definition AMReX_ParticleTile.H:1248
int numTotalParticles() const
Returns the total number of particles, real and neighbor.
Definition AMReX_ParticleTile.H:1015
ConstParticleTileData< StorageParticleType, NArrayReal, NArrayInt > ConstParticleTileDataType
Definition AMReX_ParticleTile.H:793
std::size_t size() const
Returns the total number of particles (real and neighbor).
Definition AMReX_ParticleTile.H:963
static constexpr bool has_polymorphic_allocator
Definition AMReX_ParticleTile.H:795
ParticleTileDataType getParticleTileData()
Definition AMReX_ParticleTile.H:1292
void reserve(std::size_t capacity, GrowthStrategy strategy=GrowthStrategy::Poisson)
Definition AMReX_ParticleTile.H:1052
void push_back(const ParticleType &p)
Definition AMReX_ParticleTile.H:1064
int numNeighborParticles() const
Returns the number of neighbor particles (excluding reals).
Definition AMReX_ParticleTile.H:1002
~ParticleTile()=default
const AoS & GetArrayOfStructs() const
Definition AMReX_ParticleTile.H:952
static constexpr int NAR
Definition AMReX_ParticleTile.H:769
ConstParticleTileDataType getConstParticleTileData() const
Definition AMReX_ParticleTile.H:1326
void shrink_to_fit()
Definition AMReX_ParticleTile.H:1227
T_ParticleType ParticleType
Definition AMReX_ParticleTile.H:768
int numParticles() const
Returns the number of real particles (excluding neighbors).
Definition AMReX_ParticleTile.H:976
void push_back_int(int comp, amrex::Vector< int > const &vec)
Definition AMReX_ParticleTile.H:1205
void resize(std::size_t count, GrowthStrategy strategy=GrowthStrategy::Poisson)
Definition AMReX_ParticleTile.H:1043
void swap(ParticleTile< ParticleType, NArrayReal, NArrayInt, Allocator > &other) noexcept
Definition AMReX_ParticleTile.H:1270
void push_back_int(int comp, amrex::Vector< int >::const_iterator beg, amrex::Vector< int >::const_iterator end)
Definition AMReX_ParticleTile.H:1197
void push_back_real(const std::array< ParticleReal, NArrayReal > &v)
Definition AMReX_ParticleTile.H:1122
int NumRuntimeRealComps() const noexcept
Definition AMReX_ParticleTile.H:1223
std::conditional_t< ParticleType::is_soa_particle, ThisParticleTileHasNoAoS, ArrayOfStructs< ParticleType, Allocator > > AoS
Definition AMReX_ParticleTile.H:778
void collectVectors(Vector< ParticleVector * > &pv, Vector< typename SoA::IdCPU * > &idcpuv, Vector< RealVector * > &rv, Vector< IntVector * > &iv)
Definition AMReX_ParticleTile.H:1360
AoS & GetArrayOfStructs()
Definition AMReX_ParticleTile.H:951
RealType & pos(int index, int position_index) &
Definition AMReX_ParticleTile.H:928
typename SoA::IntVector IntVector
Definition AMReX_ParticleTile.H:789
void push_back_int(const std::array< int, NArrayInt > &v)
Definition AMReX_ParticleTile.H:1177
void push_back_real(int comp, amrex::Vector< amrex::ParticleReal >::const_iterator beg, amrex::Vector< amrex::ParticleReal >::const_iterator end)
Definition AMReX_ParticleTile.H:1142
Allocator< T > AllocatorType
Definition AMReX_ParticleTile.H:766
int NumRuntimeIntComps() const noexcept
Definition AMReX_ParticleTile.H:1225
bool empty() const
Definition AMReX_ParticleTile.H:957
void push_back_int(int comp, int v)
Definition AMReX_ParticleTile.H:1168
RealType pos(int index, int position_index) const &
Definition AMReX_ParticleTile.H:942
decltype(auto) cpu(int index) &
Definition AMReX_ParticleTile.H:910
Particle< NStructReal+NArrayReal, NStructInt+NArrayInt > SuperParticleType
Definition AMReX_ParticleTile.H:776
void define(int a_num_runtime_real, int a_num_runtime_int, std::vector< std::string > *soa_rdata_names=nullptr, std::vector< std::string > *soa_idata_names=nullptr, Arena *a_arena=nullptr)
Definition AMReX_ParticleTile.H:810
SoA & GetStructOfArrays()
Definition AMReX_ParticleTile.H:954
void push_back_int(int comp, std::size_t npar, int v)
Definition AMReX_ParticleTile.H:1213
void push_back_real(int comp, std::size_t npar, ParticleReal v)
Definition AMReX_ParticleTile.H:1158
void push_back_int(int comp, const int *beg, const int *end)
Definition AMReX_ParticleTile.H:1187
typename SoA::RealVector RealVector
Definition AMReX_ParticleTile.H:788
int numRealParticles() const
Returns the number of real particles (excluding neighbors).
Definition AMReX_ParticleTile.H:989
void push_back(const SuperParticleType &sp)
Definition AMReX_ParticleTile.H:1074
static constexpr int NStructReal
Definition AMReX_ParticleTile.H:773
void push_back_real(int comp, ParticleReal v)
Definition AMReX_ParticleTile.H:1113
static void reserve(std::map< ParticleTile< T_ParticleType, NArrayReal, NArrayInt, Allocator > *, int > const &addsizes)
Definition AMReX_ParticleTile.H:1373
The struct used to store particles.
Definition AMReX_Particle.H:405
__host__ __device__ RealVect pos() const &
Definition AMReX_Particle.H:456
__host__ __device__ int & idata(int index) &
Definition AMReX_Particle.H:538
__host__ __device__ ParticleCPUWrapper cpu() &
Definition AMReX_Particle.H:424
__host__ __device__ RealType & rdata(int index) &
Definition AMReX_Particle.H:473
__host__ __device__ ParticleIDWrapper id() &
Definition AMReX_Particle.H:427
RuntimePtrCacheDirtyFlag(RuntimePtrCacheDirtyFlag &&) noexcept
Definition AMReX_ParticleTile.H:731
RuntimePtrCacheDirtyFlag() noexcept=default
void store(bool value, std::memory_order order=std::memory_order_seq_cst) noexcept
Definition AMReX_ParticleTile.H:747
bool load(std::memory_order order=std::memory_order_seq_cst) const noexcept
Definition AMReX_ParticleTile.H:752
RuntimePtrCacheDirtyFlag & operator=(RuntimePtrCacheDirtyFlag const &) noexcept
Definition AMReX_ParticleTile.H:735
Definition AMReX_Particle.H:388
Definition AMReX_ParticleTile.H:391
static constexpr int NArrayReal
Definition AMReX_ParticleTile.H:392
ParticleTileData< SoAParticleBase, NArrayReal, NArrayInt > PTD
Definition AMReX_ParticleTile.H:395
__host__ __device__ uint64_t & idcpu() &
Definition AMReX_ParticleTile.H:421
static Long UnprotectedNextID()
This version can only be used inside omp critical.
Definition AMReX_ParticleTile.H:499
__host__ __device__ ConstParticleCPUWrapper cpu() const &
Definition AMReX_ParticleTile.H:424
static Long the_next_id
Definition AMReX_ParticleTile.H:410
static constexpr int NArrayInt
Definition AMReX_ParticleTile.H:393
__host__ __device__ const uint64_t & idcpu() const &
Definition AMReX_ParticleTile.H:430
__host__ __device__ RealVect pos() const &
Definition AMReX_ParticleTile.H:435
ParticleReal RealType
Definition AMReX_ParticleTile.H:401
__host__ __device__ ParticleCPUWrapper cpu() &
Definition AMReX_ParticleTile.H:415
__host__ __device__ ParticleIDWrapper id() &
Definition AMReX_ParticleTile.H:418
static constexpr bool is_soa_particle
Definition AMReX_ParticleTile.H:396
static constexpr bool is_constsoa_particle
Definition AMReX_ParticleTile.H:398
__host__ __device__ RealType & pos(int position_index) &
Definition AMReX_ParticleTile.H:438
static constexpr bool is_rtsoa_particle
Definition AMReX_ParticleTile.H:397
int IntType
Definition AMReX_ParticleTile.H:402
__host__ __device__ SoAParticle(PTD const &ptd, long i)
Definition AMReX_ParticleTile.H:405
__host__ __device__ ConstParticleIDWrapper id() const &
Definition AMReX_ParticleTile.H:427
SoAParticleBase StorageParticleType
Definition AMReX_ParticleTile.H:394
static Long NextID()
Definition AMReX_ParticleTile.H:478
ConstSoAParticle< T_NArrayReal, T_NArrayInt > ConstType
Definition AMReX_ParticleTile.H:400
__host__ __device__ RealType pos(int position_index) const &
Definition AMReX_ParticleTile.H:445
Definition AMReX_StructOfArrays.H:20
Definition AMReX_ParticleTile.H:717
ThisParticleTileHasNoParticleVector ParticleVector
Definition AMReX_ParticleTile.H:718
Definition AMReX_ParticleTile.H:715