Block-Structured AMR Software Framework
Loading...
Searching...
No Matches
AMReX_WriteBinaryParticleData.H
Go to the documentation of this file.
1#ifndef AMREX_WRITE_BINARY_PARTICLE_DATA_H
2#define AMREX_WRITE_BINARY_PARTICLE_DATA_H
3#include <AMReX_Config.H>
4
5#include <AMReX_TypeTraits.H>
7#include <AMReX_GpuDevice.H>
8#include <AMReX_VisMF.H>
9
10namespace amrex {
11
13{
14 template <typename SrcData>
16 int operator() (const SrcData& src, int i) const noexcept
17 {
18 return (src.id(i).is_valid());
19 }
20};
21
23namespace particle_detail {
24
25template <typename ParticleReal>
26std::size_t PSizeInFile (const Vector<int>& wrc, const Vector<int>& wic)
27{
28 std::size_t rsize = sizeof(ParticleReal)*std::accumulate(wrc.begin(), wrc.end(), 0);
29 std::size_t isize = sizeof(int)*std::accumulate(wic.begin(), wic.end(), 0);
30 return rsize + isize + AMREX_SPACEDIM*sizeof(ParticleReal) + 2*sizeof(int);
31}
32
33template <class Container,
34 class PTile,
35 class F>
36void
37fillFlagsGpu (Container& pflags, const PTile& ptile, F const& f)
38{
39 const auto& ptd = ptile.getConstParticleTileData();
40 const auto np = ptile.numParticles();
41 pflags.resize(np, 0);
42 auto flag_ptr = pflags.data();
44 [=] AMREX_GPU_DEVICE (int k, amrex::RandomEngine const& engine) noexcept
45 {
46 const auto p = ptd.getSuperParticle(k);
47 amrex::ignore_unused(flag_ptr, f, engine);
48 if constexpr (IsCallable<F,decltype(p),RandomEngine>::value) {
49 flag_ptr[k] = f(p,engine);
50 } else if constexpr (IsCallable<F,decltype(p)>::value) {
51 flag_ptr[k] = f(p);
52 } else if constexpr (IsCallable<F,decltype(ptd),int,RandomEngine>::value) {
53 flag_ptr[k] = f(ptd,k,engine);
54 } else {
55 flag_ptr[k] = f(ptd,k);
56 }
57 });
58}
59
60template <class Container,
61 class PTile,
62 class F>
63void
64fillFlagsCpu (Container& pflags, const PTile& ptile, F const& f)
65{
66 const auto& ptd = ptile.getConstParticleTileData();
67 const auto np = ptile.numParticles();
68 pflags.resize(np, 0);
69 auto flag_ptr = pflags.data();
70 for (int k = 0; k < np; ++k) {
71 const auto p = ptd.getSuperParticle(k);
72 if constexpr (IsCallable<F,decltype(p),RandomEngine>::value) {
73 flag_ptr[k] = f(p,getInvalidRandomEngine());
74 } else if constexpr (IsCallable<F,decltype(p)>::value) {
75 flag_ptr[k] = f(p);
76 } else if constexpr (IsCallable<F,decltype(ptd),int,RandomEngine>::value) {
77 flag_ptr[k] = f(ptd,k,getInvalidRandomEngine());
78 } else {
79 flag_ptr[k] = f(ptd,k);
80 }
81 }
82}
83
84template <template <class, class> class Container,
85 class Allocator,
86 class PTile,
87 class F>
88void
89fillFlags (Container<int, Allocator>& pflags, const PTile& ptile, F const& f)
90{
91 if constexpr (IsPolymorphicArenaAllocator<Allocator>::value) {
92 if (pflags.arena()->isManaged() || pflags.arena()->isDevice()) {
93 fillFlagsGpu(pflags, ptile, f);
94 } else {
95 fillFlagsCpu(pflags, ptile, f);
96 }
97 } else {
98 if constexpr (RunOnGpu<Allocator>::value) {
99 fillFlagsGpu(pflags, ptile, f);
100 } else {
101 fillFlagsCpu(pflags, ptile, f);
102 }
103 }
104}
105
106template <class Container, class PC>
108countFlagsGpu (const Vector<std::map<std::pair<int,int>,Container>>& particle_io_flags, const PC& pc)
109{
110 ReduceOps<ReduceOpSum> reduce_op;
111 ReduceData<Long> reduce_data(reduce_op);
112 using ReduceTuple = typename decltype(reduce_data)::Type;
113
114 for (int lev = 0; lev < pc.GetParticles().size(); lev++)
115 {
116 const auto& pmap = pc.GetParticles(lev);
117 for (const auto& kv : pmap)
118 {
119 const auto& pflags = particle_io_flags[lev].at(kv.first);
120 const auto flag_ptr = pflags.data();
121 reduce_op.eval(pflags.size(), reduce_data,
122 [=] AMREX_GPU_DEVICE (const int i) -> ReduceTuple
123 {
124 return flag_ptr[i] ? 1 : 0;
125 });
126 }
127 }
128 ReduceTuple hv = reduce_data.value(reduce_op);
129 return amrex::get<0>(hv);
130}
131
132template <class Container>
134countFlagsGpu (const Container& pflags)
135{
136 ReduceOps<ReduceOpSum> reduce_op;
137 ReduceData<Long> reduce_data(reduce_op);
138 using ReduceTuple = typename decltype(reduce_data)::Type;
139
140 const auto flag_ptr = pflags.data();
141 reduce_op.eval(pflags.size(), reduce_data,
142 [=] AMREX_GPU_DEVICE (const amrex::Long i) -> ReduceTuple
143 {
144 return flag_ptr[i] ? 1 : 0;
145 });
146 ReduceTuple hv = reduce_data.value(reduce_op);
147 return amrex::get<0>(hv);
148}
149
150template <class Container, class PC>
152countFlagsCpu (const Vector<std::map<std::pair<int,int>,Container>>& particle_io_flags, const PC& pc)
153{
154 amrex::Long nparticles = 0;
155 for (int lev = 0; lev < pc.GetParticles().size(); lev++)
156 {
157 const auto& pmap = pc.GetParticles(lev);
158 for (const auto& kv : pmap)
159 {
160 const auto& pflags = particle_io_flags[lev].at(kv.first);
161 for (int k = 0; k < kv.second.numParticles(); ++k)
162 {
163 if (pflags[k]) { nparticles++; }
164 }
165 }
166 }
167 return nparticles;
168}
169
170template <class Container>
172countFlagsCpu (const Container& pflags)
173{
174 amrex::Long nparticles = 0;
175 for (std::size_t k = 0; k < pflags.size(); ++k)
176 {
177 if (pflags[k]) { nparticles++; }
178 }
179 return nparticles;
180}
181
182template <template <class, class> class Container, class Allocator, class PC>
184countFlags (const Vector<std::map<std::pair<int,int>,Container<int,Allocator>>>& particle_io_flags, const PC& pc)
185{
186 if constexpr (IsPolymorphicArenaAllocator<Allocator>::value) {
187 if (pc.arena()->isManaged() || pc.arena()->isDevice()) {
188 return countFlagsGpu(particle_io_flags, pc);
189 } else {
190 return countFlagsCpu(particle_io_flags, pc);
191 }
192 } else {
193 if constexpr (RunOnGpu<Allocator>::value) {
194 return countFlagsGpu(particle_io_flags, pc);
195 } else {
196 return countFlagsCpu(particle_io_flags, pc);
197 }
198 }
199}
200
201template <template <class, class> class Container, class Allocator>
203countFlags (const Container<int,Allocator>& pflags)
204{
205 if constexpr (IsPolymorphicArenaAllocator<Allocator>::value) {
206 if (pflags.arena()->isManaged() || pflags.arena()->isDevice()) {
207 return countFlagsGpu(pflags);
208 } else {
209 return countFlagsCpu(pflags);
210 }
211 } else {
212 if constexpr (RunOnGpu<Allocator>::value) {
213 return countFlagsGpu(pflags);
214 } else {
215 return countFlagsCpu(pflags);
216 }
217 }
218}
219
220template <typename Int>
222void packParticleIDs (Int* idata, const std::uint64_t idcpu, bool is_checkpoint) noexcept
223{
224 if (is_checkpoint) {
225 std::int32_t xi, yi;
226 std::uint32_t xu, yu;
227 xu = (std::uint32_t)((idcpu & 0xFFFFFFFF00000000LL) >> 32);
228 yu = (std::uint32_t)( idcpu & 0xFFFFFFFFLL);
229 amrex::Gpu::memcpy(&xi, &xu, sizeof(xu));
230 amrex::Gpu::memcpy(&yi, &yu, sizeof(yu));
231 idata[0] = xi;
232 idata[1] = yi;
233 } else {
234 idata[0] = ConstParticleIDWrapper{idcpu};
235 idata[1] = ConstParticleCPUWrapper{idcpu};
236 }
237}
238
239template<class PTD>
241rPackParticleData (const PTD& ptd, int idx, std::remove_const_t<typename PTD::RealType> * rdata_ptr,
242 const int * write_real_comp)
243{
244 std::size_t rout_index = 0;
245
246 if constexpr (!PTD::ParticleType::is_rtsoa_particle) {
247 for (int j = 0; j < AMREX_SPACEDIM; ++j) {
248 rdata_ptr[rout_index] = ptd.pos(j, idx);
249 rout_index++;
250 }
251
252 if constexpr (!PTD::ParticleType::is_soa_particle) {
253 const auto& p = ptd[idx];
254
255 for (int j = 0; j < PTD::ParticleType::NReal; ++j) {
256 if (write_real_comp[j]) {
257 rdata_ptr[rout_index] = p.rdata(j);
258 rout_index++;
259 }
260 }
261 }
262
263 constexpr int real_start_offset = PTD::ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0;
264
265 if constexpr (PTD::NAR > 0) {
266 for (int j = real_start_offset; j < PTD::NAR; ++j) {
267 if (write_real_comp[PTD::ParticleType::NReal + j - real_start_offset]) {
268 rdata_ptr[rout_index] = ptd.rdata(j)[idx];
269 rout_index++;
270 }
271 }
272 }
273
274 for (int j = 0; j < ptd.m_num_runtime_real; ++j) {
275 if (write_real_comp[PTD::ParticleType::NReal + PTD::NAR + j - real_start_offset]) {
276 rdata_ptr[rout_index] = ptd.m_runtime_rdata[j][idx];
277 rout_index++;
278 }
279 }
280 } else {
281 for (int j = 0; j < ptd.m_n_real; ++j) {
282 if ((j < AMREX_SPACEDIM) || write_real_comp[j - AMREX_SPACEDIM]) {
283 rdata_ptr[rout_index] = ptd.rdata(j)[idx];
284 rout_index++;
285 }
286 }
287 }
288}
289
290template<class PTD>
292iPackParticleData (const PTD& ptd, int idx, std::remove_const_t<typename PTD::IntType> * idata_ptr,
293 const int * write_int_comp, bool is_checkpoint)
294{
295 std::size_t iout_index = 0;
296
297 packParticleIDs(&idata_ptr[iout_index], ptd.idcpu(idx), is_checkpoint);
298 iout_index += 2;
299
300 if constexpr (!PTD::ParticleType::is_rtsoa_particle) {
301 if constexpr (!PTD::ParticleType::is_soa_particle) {
302 const auto& p = ptd[idx];
303
304 for (int j = 0; j < PTD::ParticleType::NInt; ++j) {
305 if (write_int_comp[j]) {
306 idata_ptr[iout_index] = p.idata(j);
307 iout_index++;
308 }
309 }
310 }
311
312 if constexpr (PTD::NAI > 0) {
313 for (int j = 0; j < PTD::NAI; ++j) {
314 if (write_int_comp[PTD::ParticleType::NInt + j]) {
315 idata_ptr[iout_index] = ptd.idata(j)[idx];
316 iout_index++;
317 }
318 }
319 }
320
321 for (int j = 0; j < ptd.m_num_runtime_int; ++j) {
322 if (write_int_comp[PTD::ParticleType::NInt + PTD::NAI + j]) {
323 idata_ptr[iout_index] = ptd.m_runtime_idata[j][idx];
324 iout_index++;
325 }
326 }
327 } else {
328 for (int j = 0; j < ptd.m_n_int; ++j) {
329 if (write_int_comp[j]) {
330 idata_ptr[iout_index] = ptd.idata(j)[idx];
331 iout_index++;
332 }
333 }
334 }
335}
336
337template <class PC>
338void
339packIODataGpu (Vector<int>& idata, Vector<ParticleReal>& rdata, const PC& pc, int lev, int grid,
340 const Vector<int>& write_real_comp, const Vector<int>& write_int_comp,
341 const Vector<std::map<std::pair<int, int>, typename PC::FlagsVector>>& particle_io_flags,
342 const Vector<int>& tiles, int np, bool is_checkpoint)
343{
344 int num_output_int = 0;
345 for (int i = 0; i < pc.NumIntComps() + PC::NStructInt; ++i) {
346 if (write_int_comp[i]) { ++num_output_int; }
347 }
348
349 const Long iChunkSize = 2 + num_output_int;
350 idata.resize(np*iChunkSize);
351
352 int num_output_real = 0;
353 for (int i : write_real_comp) {
354 if (i) { ++num_output_real; }
355 }
356
357 const Long rChunkSize = AMREX_SPACEDIM + num_output_real;
358 rdata.resize(np*rChunkSize);
359
360 Gpu::DeviceVector<int> write_int_comp_d(write_int_comp.size());
361 Gpu::DeviceVector<int> write_real_comp_d(write_real_comp.size());
362 Gpu::copyAsync(Gpu::hostToDevice, write_int_comp.begin(), write_int_comp.end(),
363 write_int_comp_d.begin());
364 Gpu::copyAsync(Gpu::hostToDevice, write_real_comp.begin(), write_real_comp.end(),
365 write_real_comp_d.begin());
366
367 const auto* write_int_comp_d_ptr = write_int_comp_d.data();
368 const auto* write_real_comp_d_ptr = write_real_comp_d.data();
369
370 std::size_t poffset = 0;
371 for (int tile : tiles) {
372 const auto& ptile = pc.ParticlesAt(lev, grid, tile);
373 const auto& pflags = particle_io_flags[lev].at(std::make_pair(grid, tile));
374 int np_tile = ptile.numParticles();
375 Gpu::DeviceVector<int> offsets(np_tile);
376 int num_copies = Scan::ExclusiveSum(np_tile, pflags.begin(), offsets.begin(), Scan::retSum);
377
378 Gpu::DeviceVector<int> idata_d(num_copies*iChunkSize);
379 Gpu::DeviceVector<ParticleReal> rdata_d(num_copies*rChunkSize);
380
381 const auto* flag_ptr = pflags.data();
382 const auto* offset_ptr = offsets.data();
383
384 auto* idata_d_ptr = idata_d.data();
385 auto* rdata_d_ptr = rdata_d.data();
386
387 const auto& ptd = ptile.getConstParticleTileData();
388 amrex::ParallelFor(ptile.numParticles(),
389 [=] AMREX_GPU_DEVICE (int pindex) noexcept
390 {
391 if (flag_ptr[pindex]) {
392 const int out_indx = offset_ptr[pindex];
393 iPackParticleData(ptd, pindex, idata_d_ptr + out_indx * iChunkSize,
394 write_int_comp_d_ptr, is_checkpoint);
395
396 rPackParticleData(ptd, pindex, rdata_d_ptr + out_indx * rChunkSize,
397 write_real_comp_d_ptr);
398 }
399 });
400
401 Gpu::copyAsync(Gpu::deviceToHost, idata_d.begin(), idata_d.end(),
402 idata.begin() + static_cast<Long>(poffset * iChunkSize));
403 Gpu::copyAsync(Gpu::deviceToHost, rdata_d.begin(), rdata_d.end(),
404 rdata.begin() + static_cast<Long>(poffset * rChunkSize));
405 Gpu::Device::streamSynchronize();
406
407 poffset += num_copies;
408 }
409}
410
411template <class PC>
412void
413packIODataCpu (Vector<int>& idata, Vector<ParticleReal>& rdata, const PC& pc, int lev, int grid,
414 const Vector<int>& write_real_comp, const Vector<int>& write_int_comp,
415 const Vector<std::map<std::pair<int, int>, typename PC::FlagsVector>>& particle_io_flags,
416 const Vector<int>& tiles, int np, bool is_checkpoint)
417{
418 int num_output_int = 0;
419 for (int i = 0; i < pc.NumIntComps() + PC::NStructInt; ++i) {
420 if (write_int_comp[i]) { ++num_output_int; }
421 }
422
423 const Long iChunkSize = 2 + num_output_int;
424 idata.resize(np*iChunkSize);
425
426 int num_output_real = 0;
427 for (int i : write_real_comp) {
428 if (i) { ++num_output_real; }
429 }
430
431 const Long rChunkSize = AMREX_SPACEDIM + num_output_real;
432 rdata.resize(np*rChunkSize);
433
434 int* iptr = idata.dataPtr();
435 ParticleReal* rptr = rdata.dataPtr();
436 for (int tile : tiles) {
437 const auto& ptile = pc.ParticlesAt(lev, grid, tile);
438 const auto& pflags = particle_io_flags[lev].at(std::make_pair(grid, tile));
439 const auto& ptd = ptile.getConstParticleTileData();
440
441 for (int pindex = 0; pindex < ptile.numParticles(); ++pindex) {
442 if (pflags[pindex]) {
443 iPackParticleData(ptd, pindex, iptr,
444 write_int_comp.dataPtr(), is_checkpoint);
445 iptr += iChunkSize;
446
447 rPackParticleData(ptd, pindex, rptr,
448 write_real_comp.dataPtr());
449 rptr += rChunkSize;
450 }
451 }
452 }
453}
454
455template <class PC>
456void
457packIOData (Vector<int>& idata, Vector<ParticleReal>& rdata, const PC& pc, int lev, int grid,
458 const Vector<int>& write_real_comp, const Vector<int>& write_int_comp,
459 const Vector<std::map<std::pair<int, int>, typename PC::FlagsVector>>& particle_io_flags,
460 const Vector<int>& tiles, int np, bool is_checkpoint)
461{
462 if constexpr (IsPolymorphicArenaAllocator<typename PC::FlagsVector::allocator_type>::value) {
463 if (pc.arena()->isManaged() || pc.arena()->isDevice()) {
464 packIODataGpu(idata, rdata, pc, lev, grid, write_real_comp, write_int_comp,
465 particle_io_flags, tiles, np, is_checkpoint);
466 } else {
467 packIODataCpu(idata, rdata, pc, lev, grid, write_real_comp, write_int_comp,
468 particle_io_flags, tiles, np, is_checkpoint);
469 }
470 } else {
471 if constexpr (RunOnGpu<typename PC::FlagsVector::allocator_type>::value) {
472 packIODataGpu(idata, rdata, pc, lev, grid, write_real_comp, write_int_comp,
473 particle_io_flags, tiles, np, is_checkpoint);
474 } else {
475 packIODataCpu(idata, rdata, pc, lev, grid, write_real_comp, write_int_comp,
476 particle_io_flags, tiles, np, is_checkpoint);
477 }
478 }
479}
480
481}
483
484template <class PC, class F>
487 const std::string& dir, const std::string& name,
488 const Vector<int>& write_real_comp,
489 const Vector<int>& write_int_comp,
490 const Vector<std::string>& real_comp_names,
491 const Vector<std::string>& int_comp_names,
492 F const& f, bool is_checkpoint)
493{
494 BL_PROFILE("WriteBinaryParticleData()");
495 // Particles are written under the grids that hold them, without mapping their positions
496 // to cells, so pc.OK() is not required: positions may be in a frame other than the
497 // geometry's, e.g. physical coordinates on a mapped grid.
498
499 AMREX_ASSERT(sizeof(typename PC::ParticleType::RealType) == 4 ||
500 sizeof(typename PC::ParticleType::RealType) == 8);
501
502 constexpr int NStructReal = PC::NStructReal;
503 constexpr int NStructInt = PC::NStructInt;
504
505 const int NProcs = ParallelDescriptor::NProcs();
506 const int IOProcNumber = ParallelDescriptor::IOProcessorNumber();
507
508 if constexpr(PC::ParticleType::is_soa_particle) {
509 AMREX_ALWAYS_ASSERT(real_comp_names.size() == pc.NumRealComps() + NStructReal - AMREX_SPACEDIM); // pure SoA: skip positions
510 } else {
511 AMREX_ALWAYS_ASSERT(real_comp_names.size() == pc.NumRealComps() + NStructReal);
512 }
513 AMREX_ALWAYS_ASSERT( int_comp_names.size() == pc.NumIntComps() + NStructInt);
514
515 std::string pdir = dir;
516 if ( ! pdir.empty() && pdir[pdir.size()-1] != '/') { pdir += '/'; }
517 pdir += name;
518
519 if ( ! pc.GetLevelDirectoriesCreated()) {
521 {
522 if ( ! amrex::UtilCreateDirectory(pdir, 0755))
523 {
525 }
526 }
528 }
529
530 std::ofstream HdrFile;
531
532 Long nparticles = 0;
533 Long maxnextid;
534
535 // evaluate f for every particle to determine which ones to output
536 Vector<std::map<std::pair<int, int>, typename PC::FlagsVector > >
537 particle_io_flags(pc.GetParticles().size());
538
539 for (int lev = 0; lev < pc.GetParticles().size(); lev++)
540 {
541 const auto& pmap = pc.GetParticles(lev);
542 for (const auto& kv : pmap)
543 {
544 auto& flags = particle_io_flags[lev][kv.first];
545 if constexpr (PC::has_polymorphic_allocator) {
546 flags.setArena(pc.arena());
547 }
548 particle_detail::fillFlags(flags, kv.second, f);
549 }
550 }
551
553
554 if(pc.GetUsePrePost())
555 {
556 nparticles = pc.GetNParticlesPrePost();
557 maxnextid = pc.GetMaxNextIDPrePost();
558 }
559 else
560 {
561 nparticles = particle_detail::countFlags(particle_io_flags, pc);
562 maxnextid = PC::ParticleType::NextID();
563 ParallelDescriptor::ReduceLongSum(nparticles, IOProcNumber);
564 PC::ParticleType::NextID(maxnextid);
565 ParallelDescriptor::ReduceLongMax(maxnextid, IOProcNumber);
566 }
567
569 {
570 std::string HdrFileName = pdir;
571
572 if ( ! HdrFileName.empty() && HdrFileName[HdrFileName.size()-1] != '/') {
573 HdrFileName += '/';
574 }
575
576 HdrFileName += "Header";
577 pc.HdrFileNamePrePost = HdrFileName;
578
579 HdrFile.open(HdrFileName.c_str(), std::ios::out|std::ios::trunc);
580
581 if ( ! HdrFile.good()) { amrex::FileOpenFailed(HdrFileName); }
582
583 //
584 // First thing written is our version string.
585 // We append "_single" or "_double" to the version string indicating
586 // whether we're using "float" or "double" floating point data.
587 //
588 std::string version_string = is_checkpoint ? PC::CheckpointVersion() : PC::PlotfileVersion();
589 if (sizeof(typename PC::ParticleType::RealType) == 4)
590 {
591 HdrFile << version_string << "_single" << '\n';
592 }
593 else
594 {
595 HdrFile << version_string << "_double" << '\n';
596 }
597
598 int num_output_real = 0;
599 for (int i : write_real_comp) {
600 if (i) { ++num_output_real; }
601 }
602
603 int num_output_int = 0;
604 for (int i = 0; i < pc.NumIntComps() + NStructInt; ++i) {
605 if (write_int_comp[i]) { ++num_output_int; }
606 }
607
608 // AMREX_SPACEDIM and N for sanity checking.
609 HdrFile << AMREX_SPACEDIM << '\n';
610
611 // The number of extra real parameters
612 HdrFile << num_output_real << '\n';
613
614 // Real component names
615 for (int i = 0; i < (int) real_comp_names.size(); ++i ) {
616 if (write_real_comp[i]) { HdrFile << real_comp_names[i] << '\n'; }
617 }
618
619 // The number of extra int parameters
620 HdrFile << num_output_int << '\n';
621
622 // int component names
623 for (int i = 0; i < NStructInt + pc.NumIntComps(); ++i ) {
624 if (write_int_comp[i]) { HdrFile << int_comp_names[i] << '\n'; }
625 }
626
627 bool is_checkpoint_legacy = true; // legacy
628 HdrFile << is_checkpoint_legacy << '\n';
629
630 // The total number of particles.
631 HdrFile << nparticles << '\n';
632
633 // The value of nextid that we need to restore on restart.
634 HdrFile << maxnextid << '\n';
635
636 // Then the finest level of the AMR hierarchy.
637 HdrFile << pc.finestLevel() << '\n';
638
639 // Then the number of grids at each level.
640 for (int lev = 0; lev <= pc.finestLevel(); lev++) {
641 HdrFile << pc.ParticleBoxArray(lev).size() << '\n';
642 }
643 }
644
645 // We want to write the data out in parallel.
646 // We'll allow up to nOutFiles active writers at a time.
647 int nOutFiles(256);
648
649 ParmParse pp("particles");
650 pp.queryAdd("particles_nfiles",nOutFiles);
651 if(nOutFiles == -1) { nOutFiles = NProcs; }
652 nOutFiles = std::max(1, std::min(nOutFiles,NProcs));
653 pc.nOutFilesPrePost = nOutFiles;
654
655 for (int lev = 0; lev <= pc.finestLevel(); lev++)
656 {
657 bool gotsome;
658 if(pc.usePrePost)
659 {
660 gotsome = (pc.nParticlesAtLevelPrePost[lev] > 0);
661 }
662 else
663 {
664 gotsome = (pc.NumberOfParticlesAtLevel(lev) > 0);
665 }
666
667 // We store the particles at each level in their own subdirectory.
668 std::string LevelDir = pdir;
669
670 if (gotsome)
671 {
672 if ( ! LevelDir.empty() && LevelDir[LevelDir.size()-1] != '/') { LevelDir += '/'; }
673
674 LevelDir = amrex::Concatenate(LevelDir.append("Level_"), lev, 1);
675
676 if ( ! pc.GetLevelDirectoriesCreated())
677 {
679 if ( ! amrex::UtilCreateDirectory(LevelDir, 0755)) {
681 }
682 }
684 }
685 }
686
687 // Write out the header for each particle
688 if (gotsome && ParallelDescriptor::IOProcessor()) {
689 std::string HeaderFileName = LevelDir;
690 HeaderFileName += "/Particle_H";
691 std::ofstream ParticleHeader(HeaderFileName);
692
693 pc.ParticleBoxArray(lev).writeOn(ParticleHeader);
694 ParticleHeader << '\n';
695
696 ParticleHeader.flush();
697 ParticleHeader.close();
698 }
699
700 MFInfo info;
701 info.SetAlloc(false);
702 MultiFab state(pc.ParticleBoxArray(lev),
703 pc.ParticleDistributionMap(lev),
704 1,0,info);
705
706 // We eventually want to write out the file name and the offset
707 // into that file into which each grid of particles is written.
708 Vector<int> which(state.size(),0);
709 Vector<int > count(state.size(),0);
710 Vector<Long> where(state.size(),0);
711
712 std::string filePrefix(LevelDir);
713 filePrefix += '/';
714 filePrefix += PC::DataPrefix();
715 if(pc.usePrePost) {
716 pc.filePrefixPrePost[lev] = filePrefix;
717 }
718 bool groupSets(false), setBuf(true);
719
720 if (gotsome)
721 {
722 for(NFilesIter nfi(nOutFiles, filePrefix, groupSets, setBuf); nfi.ReadyToWrite(); ++nfi)
723 {
724 auto& myStream = (std::ofstream&) nfi.Stream();
725 pc.WriteParticles(lev, myStream, nfi.FileNumber(), which, count, where,
726 write_real_comp, write_int_comp, particle_io_flags, is_checkpoint);
727 }
728
729 if(pc.usePrePost) {
730 pc.whichPrePost[lev] = which;
731 pc.countPrePost[lev] = count;
732 pc.wherePrePost[lev] = where;
733 } else {
734 ParallelDescriptor::ReduceIntSum (which.dataPtr(), static_cast<int>(which.size()), IOProcNumber);
735 ParallelDescriptor::ReduceIntSum (count.dataPtr(), static_cast<int>(count.size()), IOProcNumber);
736 ParallelDescriptor::ReduceLongSum(where.dataPtr(), static_cast<int>(where.size()), IOProcNumber);
737 }
738 }
739
741 {
742 if(pc.GetUsePrePost()) {
743 // ---- write to the header and unlink in CheckpointPost
744 } else {
745 for (int j = 0; j < state.size(); j++)
746 {
747 HdrFile << which[j] << ' ' << count[j] << ' ' << where[j] << '\n';
748 }
749
750 if (gotsome && pc.doUnlink)
751 {
752 // Unlink any zero-length data files.
753 Vector<Long> cnt(nOutFiles,0);
754
755 for (int i = 0, N=static_cast<int>(count.size()); i < N; i++) {
756 cnt[which[i]] += count[i];
757 }
758
759 for (int i = 0, N=static_cast<int>(cnt.size()); i < N; i++)
760 {
761 if (cnt[i] == 0)
762 {
763 std::string FullFileName = NFilesIter::FileName(i, filePrefix);
764 FileSystem::Remove(FullFileName);
765 }
766 }
767 }
768 }
769 }
772 }
773 }
774
776 {
777 HdrFile.flush();
778 HdrFile.close();
779 if ( ! HdrFile.good())
780 {
781 amrex::Abort("amrex::WriteBinaryParticleDataSync(): problem writing HdrFile");
782 }
783 }
784}
785
786template <class PC>
787requires (IsParticleContainer<PC>::value)
789 const std::string& dir, const std::string& name,
790 const Vector<int>& write_real_comp,
791 const Vector<int>& write_int_comp,
792 const Vector<std::string>& real_comp_names,
793 const Vector<std::string>& int_comp_names, bool is_checkpoint)
794{
795 BL_PROFILE("WriteBinaryParticleDataAsync");
796 // Particles are written under the grids that hold them, without mapping their positions
797 // to cells, so pc.OK() is not required: positions may be in a frame other than the
798 // geometry's, e.g. physical coordinates on a mapped grid.
799
800 AMREX_ASSERT(sizeof(typename PC::ParticleType::RealType) == 4 ||
801 sizeof(typename PC::ParticleType::RealType) == 8);
802
803 constexpr int NStructReal = PC::NStructReal;
804 constexpr int NStructInt = PC::NStructInt;
805 constexpr int NArrayReal = PC::NArrayReal;
806 constexpr int NArrayInt = PC::NArrayInt;
807
808 const int MyProc = ParallelDescriptor::MyProc();
809 const int NProcs = ParallelDescriptor::NProcs();
810 const int IOProcNumber = NProcs - 1;
811
812 if constexpr(PC::ParticleType::is_soa_particle) {
813 AMREX_ALWAYS_ASSERT(real_comp_names.size() == pc.NumRealComps() + NStructReal - AMREX_SPACEDIM); // pure SoA: skip positions
814 } else {
815 AMREX_ALWAYS_ASSERT(real_comp_names.size() == pc.NumRealComps() + NStructReal);
816 }
817 AMREX_ALWAYS_ASSERT( int_comp_names.size() == pc.NumIntComps() + NStructInt);
818
819 Vector<LayoutData<Long> > np_per_grid_local(pc.finestLevel()+1);
820 for (int lev = 0; lev <= pc.finestLevel(); lev++)
821 {
822 np_per_grid_local[lev].define(pc.ParticleBoxArray(lev), pc.ParticleDistributionMap(lev));
823 using ParIter = typename PC::ParConstIterType;
824 for (ParIter pti(pc, lev); pti.isValid(); ++pti)
825 {
826 int gid = pti.index();
827 const auto& ptile = pc.ParticlesAt(lev, pti);
828 const auto& ptd = ptile.getConstParticleTileData();
829 const int np = ptile.numParticles();
830
831 ReduceOps<ReduceOpSum> reduce_op;
832 ReduceData<int> reduce_data(reduce_op);
833 using ReduceTuple = typename decltype(reduce_data)::Type;
834
835 reduce_op.eval(np, reduce_data,
836 [=] AMREX_GPU_DEVICE (int i) -> ReduceTuple
837 {
838 return (ptd.id(i).is_valid()) ? 1 : 0;
839 });
840
841 int np_valid = amrex::get<0>(reduce_data.value(reduce_op));
842 np_per_grid_local[lev][gid] += np_valid;
843 }
844 }
845
846 Vector<Vector<Long> > np_per_grid_global(pc.finestLevel()+1);
847 Long total_np = 0;
848 Vector<Long> np_per_level(pc.finestLevel()+1);
849 for (int lev = 0; lev <= pc.finestLevel(); lev++)
850 {
851 np_per_grid_global[lev].resize(np_per_grid_local[lev].size());
853 np_per_grid_global[lev],
854 IOProcNumber);
855 np_per_level[lev] = std::accumulate(np_per_grid_global[lev].begin(),
856 np_per_grid_global[lev].end(), 0L);
857 total_np += np_per_level[lev];
858 }
859
860 std::string pdir = dir;
861 if ( ! pdir.empty() && pdir[pdir.size()-1] != '/') { pdir += '/'; }
862 pdir += name;
863
864 if (MyProc == IOProcNumber)
865 {
866 if ( ! pc.GetLevelDirectoriesCreated())
867 {
868 if ( ! amrex::UtilCreateDirectory(pdir, 0755))
869 {
871 }
872 }
873
874 for (int lev = 0; lev <= pc.finestLevel(); lev++)
875 {
876 std::string LevelDir = pdir;
877 bool gotsome = np_per_level[lev];
878
879 if (gotsome)
880 {
881 if ( ! LevelDir.empty() && LevelDir[LevelDir.size()-1] != '/') { LevelDir += '/'; }
882
883 LevelDir = amrex::Concatenate(LevelDir.append("Level_"), lev, 1);
884
885 if ( ! pc.GetLevelDirectoriesCreated())
886 {
887 if ( ! amrex::UtilCreateDirectory(LevelDir, 0755))
888 {
890 }
891 }
892
893 std::string HeaderFileName = LevelDir;
894 HeaderFileName += "/Particle_H";
895 std::ofstream ParticleHeader(HeaderFileName);
896
897 pc.ParticleBoxArray(lev).writeOn(ParticleHeader);
898 ParticleHeader << '\n';
899
900 ParticleHeader.flush();
901 ParticleHeader.close();
902 }
903 }
904 }
906
907 Long maxnextid = PC::ParticleType::NextID();
908 ParallelDescriptor::ReduceLongMax(maxnextid, IOProcNumber);
909
910 Vector<Long> np_on_rank(NProcs, 0L);
911 std::size_t psize = particle_detail::PSizeInFile<ParticleReal>(write_real_comp, write_int_comp);
912 Vector<int64_t> rank_start_offset(NProcs);
913 if (MyProc == IOProcNumber)
914 {
915 for (int lev = 0; lev <= pc.finestLevel(); lev++)
916 {
917 for (int k = 0; k < pc.ParticleBoxArray(lev).size(); ++k)
918 {
919 int rank = pc.ParticleDistributionMap(lev)[k];
920 np_on_rank[rank] += np_per_grid_global[lev][k];
921 }
922 }
923
924 for (int ip = 0; ip < NProcs; ++ip)
925 {
926 auto info = AsyncOut::GetWriteInfo(ip);
927 rank_start_offset[ip] = (info.ispot == 0) ? 0 : static_cast<int64_t>(rank_start_offset[ip-1] + np_on_rank[ip-1]*psize);
928 }
929 }
930
931 // make tmp particle tiles in pinned memory to write
932 using PinnedPTile = std::conditional_t<
933 PC::is_rtsoa_pc,
934 typename PC::ParticleTileType,
936 >;
937 auto myptiles = std::make_shared<Vector<std::map<std::pair<int, int>,PinnedPTile> > >();
938 myptiles->resize(pc.finestLevel()+1);
939 for (int lev = 0; lev <= pc.finestLevel(); lev++)
940 {
941 for (MFIter mfi = pc.MakeMFIter(lev); mfi.isValid(); ++mfi)
942 {
943 auto& new_ptile = (*myptiles)[lev][std::make_pair(mfi.index(),
944 mfi.LocalTileIndex())];
945
946 if (np_per_grid_local[lev][mfi.index()] > 0)
947 {
948 const auto& ptile = pc.ParticlesAt(lev, mfi);
949
950 const auto np = np_per_grid_local[lev][mfi.index()];
951
952 const auto runtime_real_comps = ptile.NumRuntimeRealComps();
953 const auto runtime_int_comps = ptile.NumRuntimeIntComps();
954
955 new_ptile.define(runtime_real_comps, runtime_int_comps,
956 nullptr, nullptr, The_Pinned_Arena());
957
958 new_ptile.resize(np);
959
960 amrex::filterParticles(new_ptile, ptile, KeepValidFilter());
961 }
962 }
963 }
964
965 int finest_level = pc.finestLevel();
968 for (int lev = 0; lev <= pc.finestLevel(); lev++)
969 {
970 bas.push_back(pc.ParticleBoxArray(lev));
971 dms.push_back(pc.ParticleDistributionMap(lev));
972 }
973
974 int nic = pc.NumIntComps();
975 int rnames_size = (int) real_comp_names.size();
976
977 auto RD = pc.ParticleRealDescriptor;
978
979 AsyncOut::Submit([=] ()
980#if defined(__GNUC__) && (__GNUC__ == 8) && (__GNUC_MINOR__ == 1)
981 mutable // workaround for bug in gcc 8.1
982#endif
983 {
984 if (MyProc == IOProcNumber)
985 {
986 std::string HdrFileName = pdir;
987 std::ofstream HdrFile;
988
989 if ( ! HdrFileName.empty() && HdrFileName[HdrFileName.size()-1] != '/') {
990 HdrFileName += '/';
991 }
992
993 HdrFileName += "Header";
994
995 HdrFile.open(HdrFileName.c_str(), std::ios::out|std::ios::trunc);
996
997 if ( ! HdrFile.good()) { amrex::FileOpenFailed(HdrFileName); }
998
999 std::string version_string = is_checkpoint ? PC::CheckpointVersion() : PC::PlotfileVersion();
1000 if (sizeof(typename PC::ParticleType::RealType) == 4)
1001 {
1002 HdrFile << version_string << "_single" << '\n';
1003 }
1004 else
1005 {
1006 HdrFile << version_string << "_double" << '\n';
1007 }
1008
1009 int num_output_real = 0;
1010 for (int i = 0; i < rnames_size; ++i) {
1011 if (write_real_comp[i]) { ++num_output_real; }
1012 }
1013
1014 int num_output_int = 0;
1015 for (int i = 0; i < nic + NStructInt; ++i) {
1016 if (write_int_comp[i]) { ++num_output_int; }
1017 }
1018
1019 // AMREX_SPACEDIM and N for sanity checking.
1020 HdrFile << AMREX_SPACEDIM << '\n';
1021
1022 // The number of extra real parameters
1023 HdrFile << num_output_real << '\n';
1024
1025 // Real component names
1026 for (int i = 0; i < rnames_size; ++i ) {
1027 if (write_real_comp[i]) { HdrFile << real_comp_names[i] << '\n'; }
1028 }
1029
1030 // The number of extra int parameters
1031 HdrFile << num_output_int << '\n';
1032
1033 // int component names
1034 for (int i = 0; i < NStructInt + nic; ++i ) {
1035 if (write_int_comp[i]) { HdrFile << int_comp_names[i] << '\n'; }
1036 }
1037
1038 bool is_checkpoint_legacy = true; // legacy
1039 HdrFile << is_checkpoint_legacy << '\n';
1040
1041 // The total number of particles.
1042 HdrFile << total_np << '\n';
1043
1044 // The value of nextid that we need to restore on restart.
1045 HdrFile << maxnextid << '\n';
1046
1047 // Then the finest level of the AMR hierarchy.
1048 HdrFile << finest_level << '\n';
1049
1050 // Then the number of grids at each level.
1051 for (int lev = 0; lev <= finest_level; lev++) {
1052 HdrFile << dms[lev].size() << '\n';
1053 }
1054
1055 for (int lev = 0; lev <= finest_level; lev++)
1056 {
1057 Vector<int64_t> grid_offset(NProcs, 0);
1058 for (int k = 0; k < bas[lev].size(); ++k)
1059 {
1060 int rank = dms[lev][k];
1061 auto info = AsyncOut::GetWriteInfo(rank);
1062 HdrFile << info.ifile << ' '
1063 << np_per_grid_global[lev][k] << ' '
1064 << grid_offset[rank] + rank_start_offset[rank] << '\n';
1065 grid_offset[rank] += static_cast<int64_t>(np_per_grid_global[lev][k]*psize);
1066 }
1067 }
1068
1069 HdrFile.flush();
1070 HdrFile.close();
1071 if ( ! HdrFile.good())
1072 {
1073 amrex::Abort("amrex::WriteBinaryParticleDataAsync(): problem writing HdrFile");
1074 }
1075 }
1076
1077 AsyncOut::Wait(); // Wait for my turn
1078
1079 for (int lev = 0; lev <= finest_level; lev++)
1080 {
1081 // For a each grid, the tiles it contains
1082 std::map<int, Vector<int> > tile_map;
1083
1084 for (const auto& kv : (*myptiles)[lev])
1085 {
1086 const int grid = kv.first.first;
1087 const int tile = kv.first.second;
1088 tile_map[grid].push_back(tile);
1089 }
1090
1091 std::string LevelDir = pdir;
1092 if ( ! LevelDir.empty() && LevelDir[LevelDir.size()-1] != '/') { LevelDir += '/'; }
1093 LevelDir = amrex::Concatenate(LevelDir.append("Level_"), lev, 1);
1094 std::string filePrefix(LevelDir);
1095 filePrefix += '/';
1096 filePrefix += PC::DataPrefix();
1097 auto info = AsyncOut::GetWriteInfo(MyProc);
1098 std::string file_name = amrex::Concatenate(filePrefix, info.ifile, 5);
1099 std::ofstream ofs;
1100 ofs.open(file_name.c_str(), (info.ispot == 0) ? (std::ios::binary | std::ios::trunc)
1101 : (std::ios::binary | std::ios::app));
1102
1103 for (int k = 0; k < bas[lev].size(); ++k)
1104 {
1105 int rank = dms[lev][k];
1106 if (rank != MyProc) { continue; }
1107 const int grid = k;
1108 if (np_per_grid_local[lev][grid] == 0) { continue; }
1109
1110 // First write out the integer data in binary.
1111 int num_output_int = 0;
1112 for (int i = 0; i < nic + NStructInt; ++i) {
1113 if (write_int_comp[i]) { ++num_output_int; }
1114 }
1115
1116 const Long iChunkSize = 2 + num_output_int;
1117 Vector<int> istuff(np_per_grid_local[lev][grid]*iChunkSize);
1118 int* iptr = istuff.dataPtr();
1119
1120 for (unsigned i = 0; i < tile_map[grid].size(); i++) {
1121 auto ptile_index = std::make_pair(grid, tile_map[grid][i]);
1122 const auto& pbox = (*myptiles)[lev][ptile_index];
1123 const auto& ptd = pbox.getConstParticleTileData();
1124 for (int pindex = 0; pindex < pbox.numParticles(); ++pindex)
1125 {
1126 if (!ptd.id(pindex).is_valid()) { continue; }
1127
1128 particle_detail::iPackParticleData(ptd, pindex, iptr,
1129 write_int_comp.dataPtr(), is_checkpoint);
1130 iptr += iChunkSize;
1131 }
1132 }
1133
1134 writeIntData(istuff.dataPtr(), istuff.size(), ofs);
1135 ofs.flush(); // Some systems require this flush() (probably due to a bug)
1136
1137 // Write the Real data in binary.
1138 int num_output_real = 0;
1139 for (int i = 0; i < rnames_size; ++i) {
1140 if (write_real_comp[i]) { ++num_output_real; }
1141 }
1142
1143 const Long rChunkSize = AMREX_SPACEDIM + num_output_real;
1144 Vector<typename PC::ParticleType::RealType> rstuff(np_per_grid_local[lev][grid]*rChunkSize);
1145 typename PC::ParticleType::RealType* rptr = rstuff.dataPtr();
1146
1147 for (unsigned i = 0; i < tile_map[grid].size(); i++) {
1148 auto ptile_index = std::make_pair(grid, tile_map[grid][i]);
1149 const auto& pbox = (*myptiles)[lev][ptile_index];
1150 const auto& ptd = pbox.getConstParticleTileData();
1151 for (int pindex = 0; pindex < pbox.numParticles(); ++pindex)
1152 {
1153 if (!ptd.id(pindex).is_valid()) { continue; }
1154
1155 particle_detail::rPackParticleData(ptd, pindex, rptr,
1156 write_real_comp.dataPtr());
1157 rptr += rChunkSize;
1158 }
1159 }
1160
1161 if (sizeof(typename PC::ParticleType::RealType) == 4) {
1162 writeFloatData((float*) rstuff.dataPtr(), rstuff.size(), ofs, RD);
1163 }
1164 else if (sizeof(typename PC::ParticleType::RealType) == 8) {
1165 writeDoubleData((double*) rstuff.dataPtr(), rstuff.size(), ofs, RD);
1166 }
1167
1168 ofs.flush(); // Some systems require this flush() (probably due to a bug)
1169 }
1170 }
1171 AsyncOut::Notify(); // Notify others I am done
1172 });
1173}
1174
1175}
1176
1177#ifdef AMREX_USE_HDF5
1179#endif
1180
1181#endif /*AMREX_WRITE_BINARY_PARTICLE_DATA_H*/
#define BL_PROFILE(a)
Definition AMReX_BLProfiler.H:593
#define AMREX_ASSERT(EX)
Definition AMReX_BLassert.H:38
#define AMREX_ALWAYS_ASSERT(EX)
Definition AMReX_BLassert.H:50
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
#define AMREX_GPU_HOST_DEVICE
Definition AMReX_GpuQualifiers.H:20
int psize
Definition AMReX_Parser_Exe_Body.H:12
while parser_exe_t p
Definition AMReX_Parser_Exe_Body.H:33
int size() const noexcept
Return the number of FABs in the FabArray.
Definition AMReX_FabArrayBase.H:115
static void streamSynchronize() noexcept
Definition AMReX_GpuDevice.cpp:992
Iterator for looping ever tiles and boxes of amrex::FabArray based containers.
Definition AMReX_MFIter.H:88
bool isValid() const noexcept
Is the iterator valid i.e. is it associated with a FAB?
Definition AMReX_MFIter.H:176
A collection (stored as an array) of FArrayBox objects.
Definition AMReX_MultiFab.H:40
This class encapsulates writing to nfiles.
Definition AMReX_NFiles.H:27
bool ReadyToWrite(bool appendFirst=false)
if appendFirst is true, the first set for this iterator will open the files in append mode
Definition AMReX_NFiles.cpp:211
const std::string & FileName() const
Definition AMReX_NFiles.H:167
Parse Parameters From Command Line and Input Files.
Definition AMReX_ParmParse.H:355
int queryAdd(std::string_view name, T &ref)
If name is found, the value in the ParmParse database will be stored in the ref argument....
Definition AMReX_ParmParse.H:1062
Definition AMReX_Reduce.H:556
Type value()
Definition AMReX_Reduce.H:591
Definition AMReX_Reduce.H:715
void eval(MF const &mf, IntVect const &nghost, D &reduce_data, F &&f)
Definition AMReX_Reduce.H:855
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition AMReX_Vector.H:29
T * dataPtr() noexcept
get access to the underlying data pointer
Definition AMReX_Vector.H:50
Long size() const noexcept
Definition AMReX_Vector.H:54
static bool GetBarrierAfterLevel()
Definition AMReX_VisMF.H:287
amrex_particle_real ParticleReal
Floating Point Type for Particles.
Definition AMReX_REAL.H:91
amrex_long Long
Definition AMReX_INT.H:30
WriteInfo GetWriteInfo(int rank)
Compute this rank's write slot information for multi-file output.
Definition AMReX_AsyncOut.cpp:72
void Wait()
Called inside a submitted user job function to wait for this rank's turn to access the file.
Definition AMReX_AsyncOut.cpp:112
void Notify()
Called inside a submitted user job function to notify the next rank sharing the file that it may proc...
Definition AMReX_AsyncOut.cpp:127
void Submit(std::function< void()> &&a_f)
Submit a job functor to the asynchronous writer thread.
Definition AMReX_AsyncOut.cpp:95
Arena * The_Pinned_Arena()
Definition AMReX_Arena.cpp:879
int MyProc() noexcept
Definition AMReX_ParallelDescriptor.H:128
void ReduceIntSum(int &)
Definition AMReX_ParallelDescriptor.cpp:1269
int NProcs() noexcept
Definition AMReX_ParallelDescriptor.H:255
void ReduceLongSum(Long &)
Definition AMReX_ParallelDescriptor.cpp:1240
int IOProcessorNumber() noexcept
The MPI rank number of the I/O Processor (probably rank 0). This rank is usually used to write to std...
Definition AMReX_ParallelDescriptor.H:279
void ReduceLongMax(Long &)
Definition AMReX_ParallelDescriptor.cpp:1241
void Barrier(const std::string &)
Definition AMReX_ParallelDescriptor.cpp:1219
bool IOProcessor() noexcept
Is this CPU the I/O Processor? To get the rank number, call IOProcessorNumber().
Definition AMReX_ParallelDescriptor.H:289
bool Remove(std::string const &filename)
Remove a file, symbolic link, or empty directory.
Definition AMReX_FileSystem.cpp:41
__host__ __device__ void * memcpy(void *dest, const void *src, std::size_t count)
Definition AMReX_GpuUtility.H:203
void GatherLayoutDataToVector(const LayoutData< T > &sendbuf, Vector< T > &recvbuf, int root)
Gather LayoutData values to a vector on root.
Definition AMReX_ParallelDescriptor.H:1286
Definition AMReX_Amr.cpp:50
void writeIntData(const From *data, std::size_t size, std::ostream &os, const amrex::IntDescriptor &id)
Definition AMReX_IntConv.H:23
__host__ __device__ void ignore_unused(const Ts &...)
No-op helper that marks variables as intentionally unused.
Definition AMReX.H:273
void FileOpenFailed(const std::string &file)
Output a message and abort when couldn't open the file.
Definition AMReX_Utility.cpp:62
@ F
Definition AMReX_Order.H:7
void WriteBinaryParticleDataAsync(PC const &pc, const std::string &dir, const std::string &name, const Vector< int > &write_real_comp, const Vector< int > &write_int_comp, const Vector< std::string > &real_comp_names, const Vector< std::string > &int_comp_names, bool is_checkpoint)
Definition AMReX_WriteBinaryParticleData.H:788
__host__ __device__ Dim3 begin(BoxND< dim > const &box) noexcept
Return the iterator begin coordinate of box as Dim3.
Definition AMReX_Box.H:2241
void writeFloatData(const float *data, std::size_t size, std::ostream &os, const RealDescriptor &rd=FPC::Native32RealDescriptor())
Definition AMReX_VectorIO.cpp:130
void ParallelFor(TypeList< CTOs... > ctos, std::array< int, sizeof...(CTOs)> const &runtime_options, T N, F &&f)
Definition AMReX_CTOParallelForImpl.H:202
void WriteBinaryParticleDataSync(PC const &pc, const std::string &dir, const std::string &name, const Vector< int > &write_real_comp, const Vector< int > &write_int_comp, const Vector< std::string > &real_comp_names, const Vector< std::string > &int_comp_names, F const &f, bool is_checkpoint)
Definition AMReX_WriteBinaryParticleData.H:486
std::string Concatenate(const std::string &root, int num, int mindigits)
Returns rootNNNN where NNNN == num.
Definition AMReX_String.cpp:35
void CreateDirectoryFailed(const std::string &dir)
Output a message and abort when couldn't create the directory.
Definition AMReX_Utility.cpp:54
RandomEngine getInvalidRandomEngine()
Definition AMReX_RandomEngine.H:86
bool UtilCreateDirectory(const std::string &path, mode_t mode, bool verbose=false)
Creates the specified directories. path may be either a full pathname or a relative pathname....
Definition AMReX_Utility.cpp:41
void writeDoubleData(const double *data, std::size_t size, std::ostream &os, const RealDescriptor &rd=FPC::Native64RealDescriptor())
Definition AMReX_VectorIO.cpp:142
AMREX_ATTRIBUTE_FLATTEN_FOR void ParallelForRNG(T n, L const &f) noexcept
Definition AMReX_GpuLaunchFunctsC.H:1157
ParIter_impl< Particle< T_NStructReal, T_NStructInt >, T_NArrayReal, T_NArrayInt, Allocator, CellAssignor > ParIter
Definition AMReX_ParIter.H:239
void Abort(const std::string &msg)
Print a fatal-error message to stderr and abort execution.
Definition AMReX.cpp:239
const int[]
Definition AMReX_BLProfiler.cpp:1665
__host__ __device__ Dim3 end(BoxND< dim > const &box) noexcept
Return the iterator end coordinate of box as Dim3.
Definition AMReX_Box.H:2259
Index filterParticles(DstTile &dst, const SrcTile &src, const Index *mask) noexcept
Conditionally copy particles from src to dst based on the value of mask.
Definition AMReX_ParticleTransformation.H:389
__host__ __device__ constexpr int get(IntVectND< dim > const &iv) noexcept
Get the Idx-th element of IntVectND<dim>.
Definition AMReX_IntVect.H:1367
Definition AMReX_TypeTraits.H:90
Definition AMReX_WriteBinaryParticleData.H:13
__host__ __device__ int operator()(const SrcData &src, int i) const noexcept
Definition AMReX_WriteBinaryParticleData.H:16
FabArray memory allocation information.
Definition AMReX_FabArray.H:73
MFInfo & SetAlloc(bool a) noexcept
Control whether FAB storage is allocated when the FabArray is defined.
Definition AMReX_FabArray.H:81
Metadata prefix of an AMReX particle "Header" file.
Definition AMReX_ParticleHeader.H:38
Definition AMReX_ParticleTile.H:764
void resize(std::size_t count, GrowthStrategy strategy=GrowthStrategy::Poisson)
Definition AMReX_ParticleTile.H:1043