Block-Structured AMR Software Framework
Loading...
Searching...
No Matches
AMReX_ParticleCommunication.H
Go to the documentation of this file.
1#ifndef AMREX_PARTICLECOMMUNICATION_H_
2#define AMREX_PARTICLECOMMUNICATION_H_
3#include <AMReX_Config.H>
4
5#include <AMReX_Gpu.H>
7#include <AMReX_IntVect.H>
9#include <AMReX_MFIter.H>
10#include <AMReX_OpenMP.H>
11#include <AMReX_Scan.H>
12#include <AMReX_TypeTraits.H>
13#include <AMReX_MakeParticle.H>
14#include <AMReX_ParmParse.H>
15
16#include <algorithm>
17#include <iterator>
18#include <map>
19#include <numeric>
20#include <utility>
21
22namespace amrex {
23
25
27{
28 template <class PTile>
29 void resizeTiles (std::vector<PTile*>& tiles, const std::vector<int>& sizes, std::vector<int>& offsets) const
30 {
31 for(int i = 0; i < std::ssize(sizes); ++i)
32 {
33 int offset = tiles[i]->numTotalParticles();
34 int nn = tiles[i]->getNumNeighbors();
35 tiles[i]->setNumNeighbors(nn + sizes[i]);
36 offsets.push_back(offset);
37 }
38 }
39};
40
42{
43 template <class PTile>
44 void resizeTiles (std::vector<PTile*>& tiles, const std::vector<int>& sizes, std::vector<int>& offsets) const
45 {
46 int N = static_cast<int>(sizes.size());
47
48 std::map<PTile*, int> tile_sizes;
49 for(int i = 0; i < N; ++i) {
50 tile_sizes[tiles[i]] = tiles[i]->numParticles();
51 }
52
53 for(int i = 0; i < N; ++i)
54 {
55 offsets.push_back(tile_sizes[tiles[i]]);
56 tile_sizes[tiles[i]] += sizes[i];
57 }
58
59 for (auto& kv : tile_sizes) {
60 kv.first->resize(kv.second);
61 }
62 }
63};
64
66{
67 using TileKey = std::pair<int, int>;
68
74
75 void clear ();
76
77 void setNumLevels (int num_levels);
78
79 void resize (int gid, int tid, int lev, int size);
80
81 [[nodiscard]] int numCopies (TileKey const& index, int lev) const
82 {
83 if (m_boxes.size() <= lev) { return 0; }
84 auto mit = m_boxes[lev].find(index);
85 return mit == m_boxes[lev].end() ? 0 : int(mit->second.size());
86 }
87
88 [[nodiscard]] int numLevels () const { return int(m_boxes.size()); }
89};
90
92{
93 using TileKey = std::pair<int, int>;
94
98
99#if defined(AMREX_USE_OMP) && !defined(AMREX_USE_GPU)
100 struct OmpCopyWork
101 {
102 const int* boxes = nullptr;
103 const int* levs = nullptr;
104 const int* tiles = nullptr;
105 int* dst_indices = nullptr;
106 int num_copies = 0;
107 };
108#endif
109
111 {
112 explicit BuildWorkspace (int a_num_buckets)
113 : num_buckets(a_num_buckets)
114#if defined(AMREX_USE_OMP) && !defined(AMREX_USE_GPU)
115 , omp_copy_offsets(1, 0)
116#endif
117 {}
118
119 int num_buckets = 0;
121
122#if defined(AMREX_USE_OMP) && !defined(AMREX_USE_GPU)
123 Vector<unsigned int> omp_thread_box_counts;
124 Vector<OmpCopyWork> omp_copy_work;
125 Vector<Long> omp_copy_offsets;
126#endif
127 };
128
129 template <class PC, class F>
130 void forEachCopyBatch (const PC& pc, const ParticleCopyOp& op, F&& f)
131 {
132 const int num_levels = op.numLevels();
133 m_dst_indices.resize(num_levels);
134 auto&& batch_fn = std::forward<F>(f);
135
136 for (int lev = 0; lev < num_levels; ++lev)
137 {
138 for (const auto& kv : pc.GetParticles(lev))
139 {
140 auto index = kv.first;
141 int num_copies = op.numCopies(index, lev);
142 if (num_copies == 0) { continue; }
143
144 auto& dst_indices = m_dst_indices[lev][index];
145 dst_indices.resize(num_copies);
146
147 batch_fn(lev, index, num_copies, dst_indices);
148 }
149 }
150 }
151
152 template <class PC, class GetBucket>
153 void buildCopies (const PC& pc, const ParticleCopyOp& op,
154 StableOrderedAlgorithm, BuildWorkspace& workspace, GetBucket const& getBucket)
155 {
156 BL_PROFILE("ParticleCopyPlan::buildCopiesStableOrdered");
157
158 forEachCopyBatch(pc, op,
159 [&] (int lev, TileKey const& index, int num_copies, Gpu::DeviceVector<int>& dst_indices)
160 {
161#ifdef AMREX_USE_GPU
162 const Gpu::DeviceVector<int>& d_boxes = op.m_boxes[lev].at(index);
163 Gpu::HostVector<int> h_boxes(d_boxes.size());
164 Gpu::copy(Gpu::deviceToHost, d_boxes.begin(), d_boxes.end(), h_boxes.begin());
165
166 const Gpu::DeviceVector<int>& d_levs = op.m_levels[lev].at(index);
167 Gpu::HostVector<int> h_levs(d_levs.size());
168 Gpu::copy(Gpu::deviceToHost, d_levs.begin(), d_levs.end(), h_levs.begin());
169
170 const Gpu::DeviceVector<int>& d_tiles = op.m_tiles[lev].at(index);
171 Gpu::HostVector<int> h_tiles(d_tiles.size());
172 Gpu::copy(Gpu::deviceToHost, d_tiles.begin(), d_tiles.end(), h_tiles.begin());
173
174 Gpu::HostVector<int> h_dst_indices(num_copies);
175#else
176 const Gpu::DeviceVector<int>& h_boxes = op.m_boxes[lev].at(index);
177 const Gpu::DeviceVector<int>& h_levs = op.m_levels[lev].at(index);
178 const Gpu::DeviceVector<int>& h_tiles = op.m_tiles[lev].at(index);
179
180 Gpu::DeviceVector<int>& h_dst_indices = dst_indices;
181#endif
182 for (int i = 0; i < num_copies; ++i) {
183 int dst_box = h_boxes[i];
184 if (dst_box >= 0) {
185 int dst_tile = h_tiles[i];
186 int dst_lev = h_levs[i];
187 int dst_index = static_cast<int>(workspace.h_box_counts[getBucket(dst_lev, dst_box, dst_tile)]++);
188 h_dst_indices[i] = dst_index;
189 }
190 }
191
192#ifdef AMREX_USE_GPU
194 h_dst_indices.begin(), h_dst_indices.end(),
195 dst_indices.begin());
196#endif
197 });
198 }
199
200#if defined(AMREX_USE_OMP) && !defined(AMREX_USE_GPU)
201 template <class PC, class GetBucket>
202 void buildCopies (const PC& pc, const ParticleCopyOp& op,
203 TwoPassHostAlgorithm, BuildWorkspace& workspace, GetBucket const& getBucket)
204 {
205 BL_PROFILE("ParticleCopyPlan::buildCopiesTwoPassHost");
206
207 workspace.omp_thread_box_counts.resize(OpenMP::get_max_threads()*workspace.num_buckets, 0U);
208
209 forEachCopyBatch(pc, op,
210 [&op, &workspace] (int lev, TileKey const& index, int num_copies, Gpu::DeviceVector<int>& dst_indices)
211 {
212 const auto* p_boxes = op.m_boxes[lev].at(index).dataPtr();
213 const auto* p_levs = op.m_levels[lev].at(index).dataPtr();
214 const auto* p_tiles = op.m_tiles[lev].at(index).dataPtr();
215 auto* p_dst_indices = dst_indices.dataPtr();
216
217 workspace.omp_copy_work.push_back({p_boxes, p_levs, p_tiles, p_dst_indices, num_copies});
218 workspace.omp_copy_offsets.push_back(workspace.omp_copy_offsets.back() + num_copies);
219 });
220
221 if (workspace.omp_copy_work.empty()) { return; }
222
223 std::fill(workspace.omp_thread_box_counts.begin(), workspace.omp_thread_box_counts.end(), 0U);
224 auto* p_omp_thread_box_counts = workspace.omp_thread_box_counts.data();
225 const auto* p_omp_copy_work = workspace.omp_copy_work.data();
226 const auto* p_omp_copy_offsets = workspace.omp_copy_offsets.data();
227 const Long total_num_copies = workspace.omp_copy_offsets.back();
228
229#pragma omp parallel
230 {
231 int thread_num = OpenMP::get_thread_num();
232 int num_threads = OpenMP::get_num_threads();
233 Long ibegin = thread_num*total_num_copies/num_threads;
234 Long iend = (thread_num+1)*total_num_copies/num_threads;
235 auto* p_thread_box_counts = p_omp_thread_box_counts + thread_num*workspace.num_buckets;
236
237 if (ibegin < iend)
238 {
239 int iwork = static_cast<int>(std::upper_bound(workspace.omp_copy_offsets.begin(),
240 workspace.omp_copy_offsets.end(),
241 ibegin)
242 - workspace.omp_copy_offsets.begin()) - 1;
243 while (iwork < static_cast<int>(workspace.omp_copy_work.size()) &&
244 p_omp_copy_offsets[iwork] < iend)
245 {
246 auto const& work = p_omp_copy_work[iwork];
247 Long global_begin = std::max(ibegin, p_omp_copy_offsets[iwork]);
248 Long global_end = std::min(iend, p_omp_copy_offsets[iwork+1]);
249 int local_begin = static_cast<int>(global_begin - p_omp_copy_offsets[iwork]);
250 int local_end = static_cast<int>(global_end - p_omp_copy_offsets[iwork]);
251 for (int i = local_begin; i < local_end; ++i)
252 {
253 int dst_box = work.boxes[i];
254 if (dst_box >= 0)
255 {
256 int dst_tile = work.tiles[i];
257 int dst_lev = work.levs[i];
258 ++p_thread_box_counts[getBucket(dst_lev, dst_box, dst_tile)];
259 }
260 }
261 ++iwork;
262 }
263 }
264
265#pragma omp barrier
266#pragma omp for
267 for (int ibucket = 0; ibucket < workspace.num_buckets; ++ibucket)
268 {
269 unsigned int offset = workspace.h_box_counts[ibucket];
270 for (int tid = 0; tid < num_threads; ++tid)
271 {
272 auto& count = p_omp_thread_box_counts[tid*workspace.num_buckets + ibucket];
273 unsigned int total = count;
274 count = offset;
275 offset += total;
276 }
277 workspace.h_box_counts[ibucket] = offset;
278 }
279
280 if (ibegin < iend)
281 {
282 int iwork = static_cast<int>(std::upper_bound(workspace.omp_copy_offsets.begin(),
283 workspace.omp_copy_offsets.end(),
284 ibegin)
285 - workspace.omp_copy_offsets.begin()) - 1;
286 while (iwork < static_cast<int>(workspace.omp_copy_work.size()) &&
287 p_omp_copy_offsets[iwork] < iend)
288 {
289 auto const& work = p_omp_copy_work[iwork];
290 Long global_begin = std::max(ibegin, p_omp_copy_offsets[iwork]);
291 Long global_end = std::min(iend, p_omp_copy_offsets[iwork+1]);
292 int local_begin = static_cast<int>(global_begin - p_omp_copy_offsets[iwork]);
293 int local_end = static_cast<int>(global_end - p_omp_copy_offsets[iwork]);
294 for (int i = local_begin; i < local_end; ++i)
295 {
296 int dst_box = work.boxes[i];
297 if (dst_box >= 0)
298 {
299 int dst_tile = work.tiles[i];
300 int dst_lev = work.levs[i];
301 int bucket = getBucket(dst_lev, dst_box, dst_tile);
302 work.dst_indices[i] = static_cast<int>(p_thread_box_counts[bucket]++);
303 }
304 }
305 ++iwork;
306 }
307 }
308 }
309 }
310#endif
311
312 template <class PC, class GetBucket>
313 void buildCopies (const PC& pc, const ParticleCopyOp& op,
315 {
316 BL_PROFILE("ParticleCopyPlan::buildCopiesAtomicScatter");
317
318 auto* p_dst_box_counts = m_box_counts_d.dataPtr();
319
320 forEachCopyBatch(pc, op,
321 [&op, &getBucket, p_dst_box_counts] (int lev, TileKey const& index,
322 int num_copies, Gpu::DeviceVector<int>& dst_indices)
323 {
324 const auto* p_boxes = op.m_boxes[lev].at(index).dataPtr();
325 const auto* p_levs = op.m_levels[lev].at(index).dataPtr();
326 const auto* p_tiles = op.m_tiles[lev].at(index).dataPtr();
327 auto* p_dst_indices = dst_indices.dataPtr();
328
329 amrex::ParallelFor(num_copies, [=] AMREX_GPU_DEVICE (int i)
330 {
331 int dst_box = p_boxes[i];
332 if (dst_box >= 0)
333 {
334 int dst_tile = p_tiles[i];
335 int dst_lev = p_levs[i];
336 int dst_index = static_cast<int>(HostDevice::Atomic::FetchAdd(
337 &p_dst_box_counts[getBucket(dst_lev, dst_box, dst_tile)], 1U));
338 p_dst_indices[i] = dst_index;
339 }
340 });
341 });
342 }
343
344 void finalizeBuildBoxCounts (BuildWorkspace const& workspace, bool use_host_box_counters)
345 {
346 if (use_host_box_counters) {
348 workspace.h_box_counts.begin(), workspace.h_box_counts.end(),
349 m_box_counts_d.begin());
350 }
351
353 m_box_offsets.begin());
354 }
355
356public:
357
359
363
370
372 int m_nrcvs = 0;
375
378
381
384
386
392
395
398
401
404
406
407 template <class PC>
409 void build (const PC& pc,
410 const ParticleCopyOp& op,
411 const Vector<int>& int_comp_mask,
412 const Vector<int>& real_comp_mask,
413 bool local, IntVect max_cells_moved)
414 {
415 BL_PROFILE("ParticleCopyPlan::build");
416
417 ParmParse pp("particles");
418 pp.query("do_one_sided_comms", m_do_one_sided_comms);
419 const int num_buckets = pc.BufferMap().numBuckets();
420
421 AMREX_ASSERT(!local || max_cells_moved.allGE(0));
422 m_local = local;
423 if (m_local)
424 {
425 m_neighbor_procs = pc.NeighborProcs(max_cells_moved);
426 }
427 else
428 {
430 std::iota(m_neighbor_procs.begin(), m_neighbor_procs.end(), 0);
431 }
432
433 m_box_counts_d.resize(0);
434 m_box_counts_d.resize(num_buckets+1, 0);
435 m_box_offsets.resize(num_buckets+1);
436
437#if defined(AMREX_USE_OMP) && !defined(AMREX_USE_GPU)
438 constexpr bool use_host_bucket_counters = true;
439#else
440 constexpr bool use_host_bucket_counters = false;
441#endif
442 BuildWorkspace workspace(num_buckets);
443 bool use_host_box_counters = pc.stableRedistribute() || use_host_bucket_counters;
444 if (use_host_box_counters) {
445 workspace.h_box_counts.resize(m_box_counts_d.size(), 0);
446 }
447
448 if (pc.stableRedistribute())
449 {
450 buildCopies(pc, op, StableOrderedAlgorithm{}, workspace, pc.BufferMap().getHostBucketFunctor());
451 }
452 else
453 {
454#if defined(AMREX_USE_OMP) && !defined(AMREX_USE_GPU)
455 buildCopies(pc, op, TwoPassHostAlgorithm{}, workspace, pc.BufferMap().getBucketFunctor());
456#else
457 buildCopies(pc, op, AtomicScatterAlgorithm{}, workspace, pc.BufferMap().getBucketFunctor());
458#endif
459 }
460
461 finalizeBuildBoxCounts(workspace, use_host_box_counters);
462
463 m_box_counts_h.resize(m_box_counts_d.size());
465 m_box_counts_h.begin());
466
467 m_snd_pad_correction_h.resize(0);
469
472 m_snd_pad_correction_d.begin());
473
474 d_int_comp_mask.resize(int_comp_mask.size());
475 Gpu::copyAsync(Gpu::hostToDevice, int_comp_mask.begin(), int_comp_mask.end(),
476 d_int_comp_mask.begin());
477 d_real_comp_mask.resize(real_comp_mask.size());
478 Gpu::copyAsync(Gpu::hostToDevice, real_comp_mask.begin(), real_comp_mask.end(),
479 d_real_comp_mask.begin());
480
482
483 int NStructReal = PC::ParticleContainerType::NStructReal;
484 int NStructInt = PC::ParticleContainerType::NStructInt;
485
486 int num_real_comm_comp = 0;
487 int comm_comps_start = 0;
488 if constexpr (!PC::ParticleType::is_soa_particle) {
489 comm_comps_start += AMREX_SPACEDIM + NStructReal;
490 }
491 for (int i = comm_comps_start; i < std::ssize(real_comp_mask); ++i) {
492 if (real_comp_mask[i]) {++num_real_comm_comp;}
493 }
494
495 int num_int_comm_comp = 0;
496 for (int i = 2 + NStructInt; i < std::ssize(int_comp_mask); ++i) {
497 if (int_comp_mask[i]) {++num_int_comm_comp;}
498 }
499
500 if constexpr (PC::ParticleType::is_soa_particle) {
501 m_superparticle_size = sizeof(uint64_t); // idcpu
502 } else {
503 m_superparticle_size = sizeof(typename PC::ParticleType);
504 }
505 m_superparticle_size += num_real_comm_comp * sizeof(typename PC::ParticleType::RealType)
506 + num_int_comm_comp * sizeof(int);
507
508 buildMPIStart(pc, pc.BufferMap(), m_superparticle_size);
509 }
510
511 void clear ();
512
513 void buildMPIFinish (const ParticleBufferMap& map);
514
515private:
516
517 void buildMPIStart (const ParticleContainerBase& pc, const ParticleBufferMap& map, Long psize);
518
519 //
520 // Snds - a Vector with the number of bytes that is process will send to each proc.
521 // Rcvs - a Vector that, after calling this method, will contain the
522 // number of bytes this process will receive from each proc.
523 //
524 void doHandShake (const ParticleContainerBase& pc, const Vector<Long>& Snds, Vector<Long>& Rcvs) const;
525
526 //
527 // In the local version of this method, each proc knows which other
528 // procs it could possibly receive messages from, meaning we can do
529 // this purely with point-to-point communication.
530 //
531 void doHandShakeLocal (const Vector<Long>& Snds, Vector<Long>& Rcvs) const;
532
533 //
534 // In the global version, we don't know who we'll receive from, so we
535 // need to do some collective communication first.
536 //
537 static void doHandShakeReduceScatter (const Vector<Long>& Snds, Vector<Long>& Rcvs);
538
539 //
540 // Another version of the global handshake implemented with MPI-3
541 // one-sided communication.
542 //
543 static void doHandShakeOneSided (const ParticleContainerBase& pc,
544 const Vector<Long>& Snds, Vector<Long>& Rcvs);
545
546 //
547 // Another version of the above that is implemented using MPI All-to-All
548 //
549 static void doHandShakeAllToAll (const Vector<Long>& Snds, Vector<Long>& Rcvs);
550
551 bool m_local = false;
552 int m_do_one_sided_comms = 0;
553};
554
556{
557 const unsigned int* m_box_offsets;
558 const std::size_t* m_pad_correction;
559
562
564 : m_box_offsets(plan.m_box_offsets.dataPtr()),
565 m_pad_correction(plan.m_snd_pad_correction_d.dataPtr()),
566 m_get_pid(map.getPIDFunctor()),
567 m_get_bucket(map.getBucketFunctor())
568 {}
569
571 Long operator() (int dst_box, int dst_tile, int dst_lev, std::size_t psize, int i) const
572 {
573 int dst_pid = m_get_pid(dst_lev, dst_box, dst_tile);
574 Long dst_offset = Long(psize)*(m_box_offsets[m_get_bucket(dst_lev, dst_box, dst_tile)] + i);
575 dst_offset += Long(m_pad_correction[dst_pid]);
576 return dst_offset;
577 }
578};
579
580template <class PC, class Buffer>
581requires (IsParticleContainer<PC>::value &&
582 std::is_base_of_v<PolymorphicArenaAllocator<typename Buffer::value_type>, Buffer>)
583void packBuffer (const PC& pc, const ParticleCopyOp& op, const ParticleCopyPlan& plan,
584 Buffer& snd_buffer)
585{
586 BL_PROFILE("amrex::packBuffer");
587
589
590 int num_levels = op.numLevels();
591 int num_buckets = pc.BufferMap().numBuckets();
592
593 std::size_t total_buffer_size = 0;
594 if (plan.m_snd_offsets.empty())
595 {
596 unsigned int np = 0;
597 Gpu::copy(Gpu::deviceToHost, plan.m_box_offsets.begin() + num_buckets,
598 plan.m_box_offsets.begin() + num_buckets + 1, &np);
599 total_buffer_size = np*psize;
600 }
601 else
602 {
603 total_buffer_size = plan.m_snd_offsets.back();
604 }
605
606 if (snd_buffer.capacity() < total_buffer_size &&
607 ! snd_buffer.arena()->hasFreeDeviceMemory(total_buffer_size)) {
608 // Free the old memory with its own arena before switching arenas.
609 snd_buffer.free_async();
610 snd_buffer.setArena(The_Pinned_Arena());
611 }
612 // The old contents are not needed; clear first so that growing the
613 // buffer does not copy stale data into the new allocation.
614 snd_buffer.clear();
615 snd_buffer.resize(total_buffer_size);
616
617 const auto* p_comm_real = plan.d_real_comp_mask.dataPtr();
618 const auto* p_comm_int = plan.d_int_comp_mask.dataPtr();
619
620 const auto plo = pc.Geom(0).ProbLoArray();
621 const auto phi = pc.Geom(0).ProbHiArray();
622 const auto is_per = pc.Geom(0).isPeriodicArray();
623#if defined(AMREX_USE_OMP) && !defined(AMREX_USE_GPU)
624 struct OmpPackWork
625 {
626 typename PC::ParticleTileType const* src_tile = nullptr;
627 const int* boxes = nullptr;
628 const int* levels = nullptr;
629 const int* tiles = nullptr;
630 const int* src_indices = nullptr;
631 const IntVect* periodic_shift = nullptr;
632 const int* dst_indices = nullptr;
633 int num_copies = 0;
634 };
635 Vector<OmpPackWork> omp_pack_work;
636 Vector<Long> omp_pack_offsets(1, 0);
637#endif
638 for (int lev = 0; lev < num_levels; ++lev)
639 {
640 auto& plev = pc.GetParticles(lev);
641 for (auto& kv : plev)
642 {
643 auto index = kv.first;
644 auto& src_tile = plev.at(index);
645 int num_copies = op.numCopies(index, lev);
646 if (num_copies == 0) { continue; }
647
648 const auto* p_boxes = op.m_boxes[lev].at(index).dataPtr();
649 const auto* p_levels = op.m_levels[lev].at(index).dataPtr();
650 const auto* p_tiles = op.m_tiles[lev].at(index).dataPtr();
651 const auto* p_src_indices = op.m_src_indices[lev].at(index).dataPtr();
652 const auto* p_periodic_shift = op.m_periodic_shift[lev].at(index).dataPtr();
653 const auto* p_dst_indices = plan.m_dst_indices[lev].at(index).dataPtr();
654#if defined(AMREX_USE_OMP) && !defined(AMREX_USE_GPU)
655 omp_pack_work.push_back({&src_tile, p_boxes, p_levels, p_tiles,
656 p_src_indices, p_periodic_shift, p_dst_indices, num_copies});
657 omp_pack_offsets.push_back(omp_pack_offsets.back() + num_copies);
658#else
659 const auto& ptd = src_tile.getConstParticleTileData();
660 auto* p_snd_buffer = snd_buffer.dataPtr();
661 GetSendBufferOffset get_offset(plan, pc.BufferMap());
662 amrex::ParallelFor(num_copies, [=] AMREX_GPU_DEVICE (int i)
663 {
664 int dst_box = p_boxes[i];
665 if (dst_box >= 0)
666 {
667 int dst_tile = p_tiles[i];
668 int dst_lev = p_levels[i];
669 auto dst_offset = get_offset(dst_box, dst_tile, dst_lev, psize, p_dst_indices[i]);
670 int src_index = p_src_indices[i];
671 ptd.packParticleData(p_snd_buffer, src_index, dst_offset, p_comm_real, p_comm_int);
672
673 const IntVect& pshift = p_periodic_shift[i];
674 bool do_periodic_shift =
675 AMREX_D_TERM( (is_per[0] && pshift[0] != 0),
676 || (is_per[1] && pshift[1] != 0),
677 || (is_per[2] && pshift[2] != 0) );
678
679 if (do_periodic_shift)
680 {
681 ParticleReal pos[AMREX_SPACEDIM];
682 Long pos_offset = dst_offset;
683 // for pure SoA positions come after idcpu
684 if constexpr (PC::ParticleType::is_soa_particle) {
685 pos_offset += sizeof(uint64_t);
686 }
687 amrex::Gpu::memcpy(&pos[0], &p_snd_buffer[pos_offset],
688 AMREX_SPACEDIM*sizeof(ParticleReal));
689 for (int idim = 0; idim < AMREX_SPACEDIM; ++idim)
690 {
691 if (! is_per[idim]) { continue; }
692 if (pshift[idim] > 0) {
693 pos[idim] = static_cast<ParticleReal>(pos[idim] + (phi[idim] - plo[idim]));
694 } else if (pshift[idim] < 0) {
695 pos[idim] = static_cast<ParticleReal>(pos[idim] - (phi[idim] - plo[idim]));
696 }
697 }
698 amrex::Gpu::memcpy(&p_snd_buffer[pos_offset], &pos[0],
699 AMREX_SPACEDIM*sizeof(ParticleReal));
700 }
701 }
702 });
703#endif
704 }
705 }
706
707#if defined(AMREX_USE_OMP) && !defined(AMREX_USE_GPU)
708 if (!omp_pack_work.empty())
709 {
710 auto* p_snd_buffer = snd_buffer.dataPtr();
711 GetSendBufferOffset get_offset(plan, pc.BufferMap());
712 const Long total_num_copies = omp_pack_offsets.back();
713
714#pragma omp parallel
715 {
716 int thread_num = OpenMP::get_thread_num();
717 int num_threads = OpenMP::get_num_threads();
718 Long ibegin = thread_num*total_num_copies/num_threads;
719 Long iend = (thread_num+1)*total_num_copies/num_threads;
720
721 if (ibegin < iend)
722 {
723 int iwork = static_cast<int>(std::upper_bound(omp_pack_offsets.begin(),
724 omp_pack_offsets.end(),
725 ibegin)
726 - omp_pack_offsets.begin()) - 1;
727 while (iwork < static_cast<int>(omp_pack_work.size()) &&
728 omp_pack_offsets[iwork] < iend)
729 {
730 auto const& work = omp_pack_work[iwork];
731 auto const& ptd = work.src_tile->getConstParticleTileData();
732 Long global_begin = std::max(ibegin, omp_pack_offsets[iwork]);
733 Long global_end = std::min(iend, omp_pack_offsets[iwork+1]);
734 int local_begin = static_cast<int>(global_begin - omp_pack_offsets[iwork]);
735 int local_end = static_cast<int>(global_end - omp_pack_offsets[iwork]);
736 for (int i = local_begin; i < local_end; ++i)
737 {
738 int dst_box = work.boxes[i];
739 if (dst_box >= 0)
740 {
741 int dst_tile = work.tiles[i];
742 int dst_lev = work.levels[i];
743 auto dst_offset = get_offset(dst_box, dst_tile, dst_lev, psize,
744 work.dst_indices[i]);
745 int src_index = work.src_indices[i];
746 ptd.packParticleData(p_snd_buffer, src_index, dst_offset,
747 p_comm_real, p_comm_int);
748
749 const IntVect& pshift = work.periodic_shift[i];
750 bool do_periodic_shift =
751 AMREX_D_TERM( (is_per[0] && pshift[0] != 0),
752 || (is_per[1] && pshift[1] != 0),
753 || (is_per[2] && pshift[2] != 0) );
754
755 if (do_periodic_shift)
756 {
757 ParticleReal pos[AMREX_SPACEDIM];
758 Long pos_offset = dst_offset;
759 if constexpr (PC::ParticleType::is_soa_particle) {
760 pos_offset += sizeof(uint64_t);
761 }
762 amrex::Gpu::memcpy(&pos[0], &p_snd_buffer[pos_offset],
763 AMREX_SPACEDIM*sizeof(ParticleReal));
764 for (int idim = 0; idim < AMREX_SPACEDIM; ++idim)
765 {
766 if (! is_per[idim]) { continue; }
767 if (pshift[idim] > 0) {
768 pos[idim] = static_cast<ParticleReal>(pos[idim] + (phi[idim] - plo[idim]));
769 } else if (pshift[idim] < 0) {
770 pos[idim] = static_cast<ParticleReal>(pos[idim] - (phi[idim] - plo[idim]));
771 }
772 }
773 amrex::Gpu::memcpy(&p_snd_buffer[pos_offset], &pos[0],
774 AMREX_SPACEDIM*sizeof(ParticleReal));
775 }
776 }
777 }
778 ++iwork;
779 }
780 }
781 }
782 }
783#endif
784}
785
786template <class PC, class Buffer, class UnpackPolicy>
787requires (IsParticleContainer<PC>::value)
788void unpackBuffer (PC& pc, const ParticleCopyPlan& plan, const Buffer& snd_buffer, UnpackPolicy const& policy)
789{
790 BL_PROFILE("amrex::unpackBuffer");
791
792 using PTile = typename PC::ParticleTileType;
793
794 int num_levels = pc.BufferMap().numLevels();
796
797 // count how many particles we have to add to each tile
798 std::vector<int> sizes;
799 std::vector<PTile*> tiles;
800 for (int lev = 0; lev < num_levels; ++lev)
801 {
802 for(MFIter mfi = pc.MakeMFIter(lev); mfi.isValid(); ++mfi)
803 {
804 int gid = mfi.index();
805 int tid = mfi.LocalTileIndex();
806 auto& tile = pc.DefineAndReturnParticleTile(lev, gid, tid);
807 int num_copies = plan.m_box_counts_h[pc.BufferMap().gridAndTileAndLevToBucket(gid, tid, lev)];
808 sizes.push_back(num_copies);
809 tiles.push_back(&tile);
810 }
811 }
812
813 // resize the tiles and compute offsets
814 std::vector<int> offsets;
815 policy.resizeTiles(tiles, sizes, offsets);
816
817 const auto* p_comm_real = plan.d_real_comp_mask.dataPtr();
818 const auto* p_comm_int = plan.d_int_comp_mask.dataPtr();
819
820#if defined(AMREX_USE_OMP) && !defined(AMREX_USE_GPU)
821 struct OmpUnpackWork
822 {
823 PTile* tile = nullptr;
824 int gid = 0;
825 int tid = 0;
826 int lev = 0;
827 int offset = 0;
828 int size = 0;
829 };
830 Vector<OmpUnpackWork> omp_unpack_work;
831 Vector<Long> omp_unpack_offsets(1, 0);
832#endif
833
834 // local unpack
835 int uindex = 0;
836 for (int lev = 0; lev < num_levels; ++lev)
837 {
838 auto& plev = pc.GetParticles(lev);
839 for(MFIter mfi = pc.MakeMFIter(lev); mfi.isValid(); ++mfi)
840 {
841 int gid = mfi.index();
842 int tid = mfi.LocalTileIndex();
843 auto index = std::make_pair(gid, tid);
844
845 auto& tile = plev[index];
846
847 GetSendBufferOffset get_offset(plan, pc.BufferMap());
848
849 int offset = offsets[uindex];
850 int size = sizes[uindex];
851 ++uindex;
852
853#if defined(AMREX_USE_OMP) && !defined(AMREX_USE_GPU)
854 omp_unpack_work.push_back({&tile, gid, tid, lev, offset, size});
855 omp_unpack_offsets.push_back(omp_unpack_offsets.back() + size);
856#else
857 auto p_snd_buffer = snd_buffer.dataPtr();
858 auto ptd = tile.getParticleTileData();
859 amrex::ParallelFor(size, [=] AMREX_GPU_DEVICE (int i)
860 {
861 auto src_offset = get_offset(gid, tid, lev, psize, i);
862 int dst_index = offset + i;
863 ptd.unpackParticleData(p_snd_buffer, src_offset, dst_index, p_comm_real, p_comm_int);
864 });
865#endif
866 }
867 }
868
869#if defined(AMREX_USE_OMP) && !defined(AMREX_USE_GPU)
870 if (!omp_unpack_work.empty())
871 {
872 GetSendBufferOffset get_offset(plan, pc.BufferMap());
873 auto p_snd_buffer = snd_buffer.dataPtr();
874 const Long total_num_copies = omp_unpack_offsets.back();
875
876#pragma omp parallel
877 {
878 int thread_num = OpenMP::get_thread_num();
879 int num_threads = OpenMP::get_num_threads();
880 Long ibegin = thread_num*total_num_copies/num_threads;
881 Long iend = (thread_num+1)*total_num_copies/num_threads;
882
883 if (ibegin < iend)
884 {
885 int iwork = static_cast<int>(std::upper_bound(omp_unpack_offsets.begin(),
886 omp_unpack_offsets.end(),
887 ibegin)
888 - omp_unpack_offsets.begin()) - 1;
889 while (iwork < static_cast<int>(omp_unpack_work.size()) &&
890 omp_unpack_offsets[iwork] < iend)
891 {
892 auto const& work = omp_unpack_work[iwork];
893 auto ptd = work.tile->getParticleTileData();
894 Long global_begin = std::max(ibegin, omp_unpack_offsets[iwork]);
895 Long global_end = std::min(iend, omp_unpack_offsets[iwork+1]);
896 int local_begin = static_cast<int>(global_begin - omp_unpack_offsets[iwork]);
897 int local_end = static_cast<int>(global_end - omp_unpack_offsets[iwork]);
898 for (int i = local_begin; i < local_end; ++i)
899 {
900 auto src_offset = get_offset(work.gid, work.tid, work.lev, psize, i);
901 int dst_index = work.offset + i;
902 ptd.unpackParticleData(p_snd_buffer, src_offset, dst_index,
903 p_comm_real, p_comm_int);
904 }
905 ++iwork;
906 }
907 }
908 }
909 }
910#endif
911}
912
913template <class PC, class SndBuffer, class RcvBuffer>
914requires (IsParticleContainer<PC>::value)
915void communicateParticlesStart (const PC& pc, ParticleCopyPlan& plan, const SndBuffer& snd_buffer, RcvBuffer& rcv_buffer)
916{
917 BL_PROFILE("amrex::communicateParticlesStart");
918
919#ifdef AMREX_USE_MPI
921 const int NProcs = ParallelContext::NProcsSub();
922 const int MyProc = ParallelContext::MyProcSub();
923
924 if (NProcs == 1) { return; }
925
926 Vector<int> RcvProc;
927 Vector<Long> rOffset;
928
929 plan.m_rcv_pad_correction_h.resize(0);
930 plan.m_rcv_pad_correction_h.push_back(0);
931
932 Long TotRcvBytes = 0;
933 for (int i = 0; i < NProcs; ++i) {
934 if (plan.m_rcv_num_particles[i] > 0) {
935 RcvProc.push_back(i);
936 Long nbytes = plan.m_rcv_num_particles[i]*psize;
937 std::size_t acd = ParallelDescriptor::sizeof_selected_comm_data_type(nbytes);
938 TotRcvBytes = Long(amrex::aligned_size(acd, TotRcvBytes));
939 rOffset.push_back(TotRcvBytes);
940 TotRcvBytes += Long(amrex::aligned_size(acd, nbytes));
941 plan.m_rcv_pad_correction_h.push_back(plan.m_rcv_pad_correction_h.back() + nbytes);
942 }
943 }
944
945 for (int i = 0; i < plan.m_nrcvs; ++i)
946 {
947 plan.m_rcv_pad_correction_h[i] = rOffset[i] - plan.m_rcv_pad_correction_h[i];
948 }
949
952 plan.m_rcv_pad_correction_d.begin());
953
954 // The old contents are not needed; clear first so that growing the
955 // buffer does not copy stale data into the new allocation.
956 rcv_buffer.clear();
957 rcv_buffer.resize(TotRcvBytes);
958
959 plan.m_nrcvs = int(RcvProc.size());
960
961 plan.m_particle_rstats.resize(0);
962 plan.m_particle_rstats.resize(plan.m_nrcvs);
963
964 plan.m_particle_rreqs.resize(0);
965 plan.m_particle_rreqs.resize(plan.m_nrcvs);
966
967 plan.m_particle_sstats.resize(0);
968 plan.m_particle_sreqs.resize(0);
969
970 const int SeqNum = ParallelDescriptor::SeqNum();
971
972 // Post receives.
973 for (int i = 0; i < plan.m_nrcvs; ++i) {
974 const auto Who = RcvProc[i];
975 const auto offset = rOffset[i];
976 Long nbytes = plan.m_rcv_num_particles[Who]*psize;
977 std::size_t acd = ParallelDescriptor::sizeof_selected_comm_data_type(nbytes);
978 const auto Cnt = amrex::aligned_size(acd, nbytes);
979
980 AMREX_ASSERT(Cnt > 0);
981 AMREX_ASSERT(Who >= 0 && Who < NProcs);
982 AMREX_ASSERT(amrex::aligned_size(acd, nbytes) % acd == 0);
983
984 plan.m_particle_rreqs[i] =
985 ParallelDescriptor::Arecv((char*) (rcv_buffer.dataPtr() + offset), Cnt, Who, SeqNum, ParallelContext::CommunicatorSub()).req();
986 }
987
988 if (plan.m_NumSnds == 0) { return; }
989
990 // Send.
991 for (int i = 0; i < NProcs; ++i)
992 {
993 if (i == MyProc) { continue; }
994 const auto Who = i;
995 const auto Cnt = plan.m_snd_counts[i];
996 if (Cnt == 0) { continue; }
997
998 auto snd_offset = plan.m_snd_offsets[i];
999 AMREX_ASSERT(plan.m_snd_counts[i] % ParallelDescriptor::sizeof_selected_comm_data_type(plan.m_snd_num_particles[i]*psize) == 0);
1000 AMREX_ASSERT(Who >= 0 && Who < NProcs);
1001
1002 plan.m_particle_sreqs.push_back(ParallelDescriptor::Asend((char const*)(snd_buffer.dataPtr()+snd_offset), Cnt, Who, SeqNum,
1004 }
1005
1006 plan.m_particle_sstats.resize(plan.m_particle_sreqs.size());
1007
1009#else
1010 amrex::ignore_unused(pc,plan,snd_buffer,rcv_buffer);
1011#endif // MPI
1012}
1013
1014void communicateParticlesFinish (const ParticleCopyPlan& plan);
1015
1016template <class PC, class Buffer, class UnpackPolicy>
1017requires (IsParticleContainer<PC>::value)
1018void unpackRemotes (PC& pc, const ParticleCopyPlan& plan, Buffer& rcv_buffer, UnpackPolicy const& policy)
1019{
1020 BL_PROFILE("amrex::unpackRemotes");
1021
1022#ifdef AMREX_USE_MPI
1023 const int NProcs = ParallelContext::NProcsSub();
1024 if (NProcs == 1) { return; }
1025
1026 const int MyProc = ParallelContext::MyProcSub();
1027 amrex::ignore_unused(MyProc);
1028 using PTile = typename PC::ParticleTileType;
1029
1030 if (plan.m_nrcvs > 0)
1031 {
1032 const auto* p_comm_real = plan.d_real_comp_mask.dataPtr();
1033 const auto* p_comm_int = plan.d_int_comp_mask.dataPtr();
1034 auto* p_rcv_buffer = rcv_buffer.dataPtr();
1035
1036 std::vector<int> sizes;
1037 std::vector<PTile*> tiles;
1038 for (int i = 0; i < std::ssize(plan.m_rcv_box_counts); ++i)
1039 {
1040 int copy_size = plan.m_rcv_box_counts[i];
1041 int lev = plan.m_rcv_box_levs[i];
1042 int gid = plan.m_rcv_box_ids[i];
1043 int tid = plan.m_rcv_box_tids[i];
1044 auto& tile = pc.DefineAndReturnParticleTile(lev, gid, tid);
1045 sizes.push_back(copy_size);
1046 tiles.push_back(&tile);
1047 }
1048
1049 Vector<int> offsets;
1050 policy.resizeTiles(tiles, sizes, offsets);
1052 int uindex = 0;
1053 int procindex = 0, rproc = plan.m_rcv_box_pids[0];
1054 for (int i = 0; i < std::ssize(plan.m_rcv_box_counts); ++i)
1055 {
1056 int lev = plan.m_rcv_box_levs[i];
1057 int gid = plan.m_rcv_box_ids[i];
1058 int tid = plan.m_rcv_box_tids[i];
1059 auto offset = plan.m_rcv_box_offsets[i];
1060 procindex = (rproc == plan.m_rcv_box_pids[i]) ? procindex : procindex+1;
1061 rproc = plan.m_rcv_box_pids[i];
1062
1063 auto& tile = pc.DefineAndReturnParticleTile(lev, gid, tid);
1064 auto ptd = tile.getParticleTileData();
1065
1066 AMREX_ASSERT(MyProc ==
1067 ParallelContext::global_to_local_rank(pc.ParticleDistributionMap(lev)[gid]));
1068
1069 int dst_offset = offsets[uindex];
1070 int size = sizes[uindex];
1071 ++uindex;
1072
1073 Long psize = plan.superParticleSize();
1074 const auto* p_pad_adjust = plan.m_rcv_pad_correction_d.dataPtr();
1075
1076 amrex::ParallelForOMP(size, [=] AMREX_GPU_DEVICE (int ip) {
1077 Long src_offset = psize * static_cast<Long>(offset + ip)
1078 + static_cast<Long>(p_pad_adjust[procindex]);
1079 int dst_index = dst_offset + ip;
1080 ptd.unpackParticleData(p_rcv_buffer, src_offset, dst_index,
1081 p_comm_real, p_comm_int);
1082 });
1083 }
1084 }
1085#else
1086 amrex::ignore_unused(pc,plan,rcv_buffer,policy);
1087#endif // MPI
1088}
1089
1090} // namespace amrex
1091
1092#endif // AMREX_PARTICLECOMMUNICATION_H_
#define BL_PROFILE(a)
Definition AMReX_BLProfiler.H:593
#define AMREX_ASSERT(EX)
Definition AMReX_BLassert.H:38
#define AMREX_FORCE_INLINE
Definition AMReX_Extension.H:123
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
Convenience header for the core AMReX GPU facilities.
int psize
Definition AMReX_Parser_Exe_Body.H:12
#define AMREX_D_TERM(a, b, c)
Definition AMReX_SPACE.H:172
__host__ __device__ constexpr bool allGE(const IntVectND< dim > &rhs) const noexcept
Returns true if this is greater than or equal to argument for all components. NOTE: This is NOT a str...
Definition AMReX_IntVect.H:549
Iterator for looping ever tiles and boxes of amrex::FabArray based containers.
Definition AMReX_MFIter.H:88
bool isValid() const noexcept
Is the iterator valid i.e. is it associated with a FAB?
Definition AMReX_MFIter.H:176
size_type size() const noexcept
Definition AMReX_PODVector.H:654
iterator begin() noexcept
Definition AMReX_PODVector.H:680
iterator end() noexcept
Definition AMReX_PODVector.H:684
T * dataPtr() noexcept
Definition AMReX_PODVector.H:676
MPI_Request req() const
Definition AMReX_ParallelDescriptor.H:74
Parse Parameters From Command Line and Input Files.
Definition AMReX_ParmParse.H:355
int query(std::string_view name, bool &ref, int ival=FIRST) const
Same as querykth() but searches for the last occurrence of name.
Definition AMReX_ParmParse.cpp:2032
Definition AMReX_ParticleBufferMap.H:59
Definition AMReX_ParticleContainerBase.H:43
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition AMReX_Vector.H:29
T * dataPtr() noexcept
get access to the underlying data pointer
Definition AMReX_Vector.H:50
Long size() const noexcept
Definition AMReX_Vector.H:54
amrex_particle_real ParticleReal
Floating Point Type for Particles.
Definition AMReX_REAL.H:91
amrex_long Long
Definition AMReX_INT.H:30
OutIter exclusive_scan(InIter begin, InIter end, OutIter result)
Definition AMReX_Scan.H:1181
void ParallelForOMP(T n, L const &f) noexcept
Performance-portable kernel launch function with optional OpenMP threading.
Definition AMReX_GpuLaunch.H:328
PinnedVector< T > HostVector
A PODVector that uses pinned host memory. Same as PinnedVector. For a vector class that uses std::all...
Definition AMReX_GpuContainers.H:67
PODVector< T, ArenaAllocator< T > > DeviceVector
A PODVector that uses the standard memory Arena. Note that the memory might or might not be managed d...
Definition AMReX_GpuContainers.H:24
Arena * The_Pinned_Arena()
Definition AMReX_Arena.cpp:879
std::size_t aligned_size(std::size_t align_requirement, std::size_t size) noexcept
Return the smallest multiple of align_requirement that is >= size.
Definition AMReX_Arena.H:39
void copy(HostToDevice, InIter begin, InIter end, OutIter result) noexcept
A host-to-device copy routine. Note this is just a wrapper around memcpy, so it assumes contiguous st...
Definition AMReX_GpuContainers.H:128
void copyAsync(HostToDevice, InIter begin, InIter end, OutIter result) noexcept
A host-to-device copy routine. Note this is just a wrapper around memcpy, so it assumes contiguous st...
Definition AMReX_GpuContainers.H:228
static constexpr DeviceToHost deviceToHost
Definition AMReX_GpuContainers.H:106
static constexpr HostToDevice hostToDevice
Definition AMReX_GpuContainers.H:105
void streamSynchronize() noexcept
Definition AMReX_GpuDevice.H:329
__host__ __device__ void * memcpy(void *dest, const void *src, std::size_t count)
Definition AMReX_GpuUtility.H:203
__host__ __device__ AMREX_FORCE_INLINE T FetchAdd(T *const sum, T const value) noexcept
Definition AMReX_GpuAtomic.H:703
constexpr int get_thread_num()
Definition AMReX_OpenMP.H:37
constexpr int get_num_threads()
Definition AMReX_OpenMP.H:35
constexpr int get_max_threads()
Definition AMReX_OpenMP.H:36
MPI_Comm CommunicatorSub() noexcept
sub-communicator for current frame
Definition AMReX_ParallelContext.H:70
int MyProcSub() noexcept
my sub-rank in current frame
Definition AMReX_ParallelContext.H:76
int global_to_local_rank(int rank) noexcept
Definition AMReX_ParallelContext.H:101
int NProcsSub() noexcept
number of ranks in current frame
Definition AMReX_ParallelContext.H:74
Message Asend(const T *, size_t n, int pid, int tag)
Definition AMReX_ParallelDescriptor.H:1163
int SeqNum() noexcept
Returns sequential message sequence numbers, usually used as tags for send/recv.
Definition AMReX_ParallelDescriptor.H:678
Message Arecv(T *, size_t n, int pid, int tag)
Definition AMReX_ParallelDescriptor.H:1205
Definition AMReX_Amr.cpp:50
__host__ __device__ void ignore_unused(const Ts &...)
No-op helper that marks variables as intentionally unused.
Definition AMReX.H:273
@ F
Definition AMReX_Order.H:7
Long doHandShake(const std::map< int, Vector< char > > &not_ours, Vector< Long > &Snds, Vector< Long > &Rcvs)
Definition AMReX_ParticleMPIUtil.cpp:25
void communicateParticlesStart(const PC &pc, ParticleCopyPlan &plan, const SndBuffer &snd_buffer, RcvBuffer &rcv_buffer)
Definition AMReX_ParticleCommunication.H:915
Long doHandShakeLocal(const std::map< int, Vector< char > > &not_ours, const Vector< int > &neighbor_procs, Vector< Long > &Snds, Vector< Long > &Rcvs)
Definition AMReX_ParticleMPIUtil.cpp:50
void ParallelFor(TypeList< CTOs... > ctos, std::array< int, sizeof...(CTOs)> const &runtime_options, T N, F &&f)
Definition AMReX_CTOParallelForImpl.H:202
void unpackBuffer(PC &pc, const ParticleCopyPlan &plan, const Buffer &snd_buffer, UnpackPolicy const &policy)
Definition AMReX_ParticleCommunication.H:788
void packBuffer(const PC &pc, const ParticleCopyOp &op, const ParticleCopyPlan &plan, Buffer &snd_buffer)
Definition AMReX_ParticleCommunication.H:583
void communicateParticlesFinish(const ParticleCopyPlan &plan)
Definition AMReX_ParticleCommunication.cpp:445
void unpackRemotes(PC &pc, const ParticleCopyPlan &plan, Buffer &rcv_buffer, UnpackPolicy const &policy)
Definition AMReX_ParticleCommunication.H:1018
IntVectND< 3 > IntVect
IntVect is an alias for amrex::IntVectND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:38
const int[]
Definition AMReX_BLProfiler.cpp:1665
Definition AMReX_ParticleBufferMap.H:38
Definition AMReX_ParticleBufferMap.H:14
Definition AMReX_ParticleCommunication.H:556
const unsigned int * m_box_offsets
Definition AMReX_ParticleCommunication.H:557
GetPID m_get_pid
Definition AMReX_ParticleCommunication.H:560
GetBucket m_get_bucket
Definition AMReX_ParticleCommunication.H:561
const std::size_t * m_pad_correction
Definition AMReX_ParticleCommunication.H:558
GetSendBufferOffset(const ParticleCopyPlan &plan, const ParticleBufferMap &map)
Definition AMReX_ParticleCommunication.H:563
__device__ Long operator()(int dst_box, int dst_tile, int dst_lev, std::size_t psize, int i) const
Definition AMReX_ParticleCommunication.H:571
Definition AMReX_TypeTraits.H:90
Definition AMReX_ParticleCommunication.H:27
void resizeTiles(std::vector< PTile * > &tiles, const std::vector< int > &sizes, std::vector< int > &offsets) const
Definition AMReX_ParticleCommunication.H:29
Definition AMReX_ParticleCommunication.H:66
void resize(int gid, int tid, int lev, int size)
Definition AMReX_ParticleCommunication.cpp:27
void setNumLevels(int num_levels)
Definition AMReX_ParticleCommunication.cpp:18
Vector< std::map< TileKey, Gpu::DeviceVector< IntVect > > > m_periodic_shift
Definition AMReX_ParticleCommunication.H:73
int numLevels() const
Definition AMReX_ParticleCommunication.H:88
Vector< std::map< TileKey, Gpu::DeviceVector< int > > > m_levels
Definition AMReX_ParticleCommunication.H:70
Vector< std::map< TileKey, Gpu::DeviceVector< int > > > m_boxes
Definition AMReX_ParticleCommunication.H:69
int numCopies(TileKey const &index, int lev) const
Definition AMReX_ParticleCommunication.H:81
std::pair< int, int > TileKey
Definition AMReX_ParticleCommunication.H:67
Vector< std::map< TileKey, Gpu::DeviceVector< int > > > m_tiles
Definition AMReX_ParticleCommunication.H:71
Vector< std::map< TileKey, Gpu::DeviceVector< int > > > m_src_indices
Definition AMReX_ParticleCommunication.H:72
void clear()
Definition AMReX_ParticleCommunication.cpp:9
Definition AMReX_ParticleCommunication.H:97
Definition AMReX_ParticleCommunication.H:111
BuildWorkspace(int a_num_buckets)
Definition AMReX_ParticleCommunication.H:112
Gpu::HostVector< unsigned int > h_box_counts
Definition AMReX_ParticleCommunication.H:120
int num_buckets
Definition AMReX_ParticleCommunication.H:119
Definition AMReX_ParticleCommunication.H:95
Definition AMReX_ParticleCommunication.H:96
Definition AMReX_ParticleCommunication.H:92
Vector< int > m_rcv_box_ids
Definition AMReX_ParticleCommunication.H:366
Vector< std::size_t > m_snd_offsets
Definition AMReX_ParticleCommunication.H:393
Vector< int > m_rcv_box_counts
Definition AMReX_ParticleCommunication.H:364
Vector< std::size_t > m_snd_counts
Definition AMReX_ParticleCommunication.H:394
void finalizeBuildBoxCounts(BuildWorkspace const &workspace, bool use_host_box_counters)
Definition AMReX_ParticleCommunication.H:344
Long m_NumSnds
Definition AMReX_ParticleCommunication.H:371
void buildMPIFinish(const ParticleBufferMap &map)
Definition AMReX_ParticleCommunication.cpp:223
Vector< int > m_neighbor_procs
Definition AMReX_ParticleCommunication.H:385
void clear()
Definition AMReX_ParticleCommunication.cpp:41
void build(const PC &pc, const ParticleCopyOp &op, const Vector< int > &int_comp_mask, const Vector< int > &real_comp_mask, bool local, IntVect max_cells_moved)
Definition AMReX_ParticleCommunication.H:409
Vector< int > m_rcv_box_pids
Definition AMReX_ParticleCommunication.H:368
void buildCopies(const PC &pc, const ParticleCopyOp &op, AtomicScatterAlgorithm, BuildWorkspace &, GetBucket const &getBucket)
Definition AMReX_ParticleCommunication.H:313
Vector< int > m_rcv_box_levs
Definition AMReX_ParticleCommunication.H:369
Gpu::DeviceVector< int > d_real_comp_mask
Definition AMReX_ParticleCommunication.H:402
std::pair< int, int > TileKey
Definition AMReX_ParticleCommunication.H:93
Gpu::DeviceVector< std::size_t > m_snd_pad_correction_d
Definition AMReX_ParticleCommunication.H:397
void forEachCopyBatch(const PC &pc, const ParticleCopyOp &op, F &&f)
Definition AMReX_ParticleCommunication.H:130
Long m_superparticle_size
Definition AMReX_ParticleCommunication.H:403
Vector< int > m_rcv_box_tids
Definition AMReX_ParticleCommunication.H:367
Vector< Long > m_Snds
Definition AMReX_ParticleCommunication.H:387
Vector< MPI_Request > m_particle_sreqs
Definition AMReX_ParticleCommunication.H:380
Vector< std::map< TileKey, Gpu::DeviceVector< int > > > m_dst_indices
Definition AMReX_ParticleCommunication.H:358
Gpu::DeviceVector< unsigned int > m_box_counts_d
Definition AMReX_ParticleCommunication.H:360
Vector< Long > m_Rcvs
Definition AMReX_ParticleCommunication.H:388
Vector< int > m_rcv_box_offsets
Definition AMReX_ParticleCommunication.H:365
Vector< std::size_t > m_snd_pad_correction_h
Definition AMReX_ParticleCommunication.H:396
Vector< MPI_Status > m_particle_sstats
Definition AMReX_ParticleCommunication.H:379
Gpu::DeviceVector< unsigned int > m_box_offsets
Definition AMReX_ParticleCommunication.H:362
Gpu::DeviceVector< std::size_t > m_rcv_pad_correction_d
Definition AMReX_ParticleCommunication.H:400
Vector< std::size_t > m_rOffset
Definition AMReX_ParticleCommunication.H:390
Vector< MPI_Status > m_particle_rstats
Definition AMReX_ParticleCommunication.H:376
Vector< Long > m_snd_num_particles
Definition AMReX_ParticleCommunication.H:382
Vector< MPI_Request > m_particle_rreqs
Definition AMReX_ParticleCommunication.H:377
Long superParticleSize() const
Definition AMReX_ParticleCommunication.H:405
Gpu::HostVector< int > m_rcv_data
Definition AMReX_ParticleCommunication.H:391
void buildCopies(const PC &pc, const ParticleCopyOp &op, StableOrderedAlgorithm, BuildWorkspace &workspace, GetBucket const &getBucket)
Definition AMReX_ParticleCommunication.H:153
Vector< MPI_Status > m_build_stats
Definition AMReX_ParticleCommunication.H:373
Vector< int > m_RcvProc
Definition AMReX_ParticleCommunication.H:389
Vector< std::size_t > m_rcv_pad_correction_h
Definition AMReX_ParticleCommunication.H:399
int m_nrcvs
Definition AMReX_ParticleCommunication.H:372
Gpu::HostVector< unsigned int > m_box_counts_h
Definition AMReX_ParticleCommunication.H:361
Vector< MPI_Request > m_build_rreqs
Definition AMReX_ParticleCommunication.H:374
Vector< Long > m_rcv_num_particles
Definition AMReX_ParticleCommunication.H:383
Gpu::DeviceVector< int > d_int_comp_mask
Definition AMReX_ParticleCommunication.H:402
Definition AMReX_ParticleCommunication.H:42
void resizeTiles(std::vector< PTile * > &tiles, const std::vector< int > &sizes, std::vector< int > &offsets) const
Definition AMReX_ParticleCommunication.H:44