Block-Structured AMR Software Framework
AMReX_VisMF.H
Go to the documentation of this file.
1 
2 #ifndef AMREX_VISMF_H_
3 #define AMREX_VISMF_H_
4 #include <AMReX_Config.H>
5 
6 #include <AMReX_AsyncOut.H>
7 #include <AMReX_BLProfiler.H>
8 #include <AMReX_FabArray.H>
9 #include <AMReX_FArrayBox.H>
10 #include <AMReX_FabConv.H>
11 #include <AMReX_NFiles.H>
13 #include <AMReX_VisMFBuffer.H>
14 
15 #include <fstream>
16 #include <iostream>
17 #include <sstream>
18 #include <deque>
19 #include <map>
20 #include <numeric>
21 #include <string>
22 #include <type_traits>
23 
24 namespace amrex {
25 
26 class IArrayBox;
27 
32 class VisMF
33  : public VisMFBuffer
34 {
35 public:
48  explicit VisMF (std::string fafab_name);
49  ~VisMF () = default;
50  VisMF (const VisMF&) = delete;
51  VisMF (VisMF&&) = delete;
52  VisMF& operator= (const VisMF&) = delete;
53  VisMF& operator= (VisMF&&) = delete;
55  struct FabOnDisk
56  {
58  FabOnDisk () = default;
60  FabOnDisk (std::string name, Long offset);
62  std::string m_name;
63  Long m_head = 0;
64  };
66  struct Header
67  {
69  enum Version {
71  Version_v1 = 1,
79  };
81  Header ();
83  Header (const FabArray<FArrayBox>& mf, VisMF::How how, Version version = Version_v1,
84  bool calcMinMax = true, MPI_Comm = ParallelDescriptor::Communicator());
85 
86  ~Header () = default;
87  Header (Header&& rhs) noexcept = default;
88  Header (Header const&) = delete;
89  Header& operator= (Header const&) = delete;
90  Header& operator= (Header &&) = delete;
91 
93  void CalculateMinMax(const FabArray<FArrayBox>& mf,
94  int procToWrite = ParallelDescriptor::IOProcessorNumber(),
96  //
97  // The data.
98  //
101  int m_ncomp;
105  //
106  // These are not defined if they are not in the header
107  //
113  };
114 
116  struct FabReadLink
117  {
118  int rankToRead{-1};
119  int faIndex{-1};
120  Long fileOffset{-1};
122 
123  FabReadLink() = default;
124  FabReadLink(int ranktoread, int faindex, Long fileoffset, const Box &b);
125  };
126 
129  {
130  std::ifstream *pstr{nullptr};
131  std::streampos currentPosition{0};
132  bool isOpen{false};
134 
135  PersistentIFStream () = default;
141  };
142 
148  static std::ifstream *OpenStream(const std::string &fileName);
149  static void CloseStream(const std::string &fileName, bool forceClose = false);
150  static void DeleteStream(const std::string &fileName);
151  static void CloseAllStreams();
152  static bool NoFabHeader(const VisMF::Header &hdr);
153 
155  [[nodiscard]] int nComp () const;
157  [[nodiscard]] int nGrow () const;
158  [[nodiscard]] IntVect nGrowVect () const;
160  [[nodiscard]] int size () const;
162  [[nodiscard]] const BoxArray& boxArray () const;
164  [[nodiscard]] Real min (int fabIndex, int nComp) const;
166  [[nodiscard]] Real min (int nComp) const;
168  [[nodiscard]] Real max (int fabIndex, int nComp) const;
170  [[nodiscard]] Real max (int nComp) const;
171 
177  [[nodiscard]] const FArrayBox& GetFab (int fabIndex, int compIndex) const;
179  void clear (int fabIndex, int compIndex);
181  void clear (int fabIndex);
183  void clear ();
191  static Long Write (const FabArray<FArrayBox> &mf,
192  const std::string& name,
193  VisMF::How how = NFiles,
194  bool set_ghost = false);
195 
196  static void AsyncWrite (const FabArray<FArrayBox>& mf, const std::string& mf_name,
197  bool valid_cells_only = false);
198  static void AsyncWrite (FabArray<FArrayBox>&& mf, const std::string& mf_name,
199  bool valid_cells_only = false);
200 
208  static Long WriteOnlyHeader (const FabArray<FArrayBox> & mf,
209  const std::string & mf_name,
210  VisMF::How how = NFiles);
212  static void RemoveFiles(const std::string &name, bool verbose = false);
213 
223  static void Read (FabArray<FArrayBox> &mf,
224  const std::string &name,
225  const char *faHeader = nullptr,
226  int coordinatorProc = ParallelDescriptor::IOProcessorNumber(),
227  int allow_empty_mf = 0);
228 
230  static bool Exist (const std::string &name);
231 
233  static void ReadFAHeader (const std::string &fafabName,
234  Vector<char> &header);
235 
237  static bool Check (const std::string &name);
239  static Long FileOffset (std::ostream& os);
241  FArrayBox* readFAB (int idx, const std::string& mf_name);
243  FArrayBox* readFAB (int idx, int icomp);
244 
245  static int GetNOutFiles ();
246  static void SetNOutFiles (int noutfiles, MPI_Comm comm = ParallelDescriptor::Communicator());
247 
248  static int GetMFFileInStreams () { return nMFFileInStreams; }
249  static void SetMFFileInStreams (int nstreams, MPI_Comm comm = ParallelDescriptor::Communicator());
250 
251  static int GetVerbose () { return verbose; }
252  static void SetVerbose (int v) { verbose = v; }
253 
256  { currentVersion = version; }
257 
258  static bool GetGroupSets () { return groupSets; }
259  static void SetGroupSets (bool groupsets) { groupSets = groupsets; }
260 
261  static bool GetSetBuf () { return setBuf; }
262  static void SetSetBuf (bool setbuf) { setBuf = setbuf; }
263 
264  static bool GetUseSingleRead () { return useSingleRead; }
265  static void SetUseSingleRead (bool usesingleread) { useSingleRead = usesingleread; }
266 
267  static bool GetUseSingleWrite () { return useSingleWrite; }
268  static void SetUseSingleWrite (bool usesinglewrite) { useSingleWrite = usesinglewrite; }
269 
270  static bool GetCheckFilePositions () { return checkFilePositions; }
271  static void SetCheckFilePositions (bool cfp) { checkFilePositions = cfp; }
272 
274  static void SetUsePersistentIFStreams (bool usepifs) { usePersistentIFStreams = usepifs; }
275 
276  static bool GetUseSynchronousReads () { return useSynchronousReads; }
277  static void SetUseSynchronousReads (bool usepsr) { useSynchronousReads = usepsr; }
278 
280  static void SetUseDynamicSetSelection (bool usedss) { useDynamicSetSelection = usedss; }
281 
282  static std::string DirName (const std::string& filename);
283  static std::string BaseName (const std::string& filename);
284 
285  static void Initialize ();
286  static void Finalize ();
287 
288 private:
289  static FabOnDisk Write (const FArrayBox& fab,
290  const std::string& filename,
291  std::ostream& os,
292  Long& bytes);
293 
294  static Long WriteHeaderDoit (const std::string &mf_name,
295  VisMF::Header const &hdr);
296 
297  static Long WriteHeader (const std::string &mf_name,
298  VisMF::Header &hdr,
299  int procToWrite = ParallelDescriptor::IOProcessorNumber(),
301 
303  static void FindOffsets (const FabArray<FArrayBox> &mf,
304  const std::string &filePrefix,
305  VisMF::Header &hdr,
306  VisMF::Header::Version whichVersion,
307  NFilesIter &nfi,
316  static FArrayBox *readFAB (int idx,
317  const std::string &mf_name,
318  const Header &hdr,
319  int whichComp = -1);
321  static void readFAB (FabArray<FArrayBox> &mf,
322  int idx,
323  const std::string &mf_name,
324  const Header& hdr);
325 
326  static void AsyncWriteDoit (const FabArray<FArrayBox>& mf, const std::string& mf_name,
327  bool is_rvalue, bool valid_cells_only);
328 
330  std::string m_fafabname;
340  static AMREX_EXPORT std::map<std::string, VisMF::PersistentIFStream> persistentIFStreams;
344 
345  static AMREX_EXPORT int verbose;
347  static AMREX_EXPORT bool groupSets;
348  static AMREX_EXPORT bool setBuf;
356 };
357 
359 std::ostream& operator<< (std::ostream& os, const VisMF::FabOnDisk& fod);
361 std::istream& operator>> (std::istream& is, VisMF::FabOnDisk& fod);
363 std::ostream& operator<< (std::ostream& os, const Vector<VisMF::FabOnDisk>& fa);
365 std::istream& operator>> (std::istream& is, Vector<VisMF::FabOnDisk>& fa);
367 std::ostream& operator<< (std::ostream& os, const VisMF::Header& hd);
369 std::istream& operator>> (std::istream& is, VisMF::Header& hd);
370 
380 template <typename FAB>
381 // This function does work for MultiFab, but let's disable it to avoid confusion.
382 std::enable_if_t<std::is_same_v<FAB,IArrayBox>>
383 Write (const FabArray<FAB>& fa, const std::string& name)
384 {
385  BL_PROFILE("Write(FabArray)");
386  AMREX_ASSERT(name.back() != '/');
387 
388  auto data_descriptor = FAB::getDataDescriptor();
389  int data_bytes = data_descriptor->numBytes();
390 
391  bool useSparseFPP = false;
392  const Vector<int> &pmap = fa.DistributionMap().ProcessorMap();
393  std::set<int> procsWithData;
394  Vector<int> procsWithDataVector;
395  for(int i : pmap) {
396  procsWithData.insert(i);
397  }
398  const int nOutFiles = VisMF::GetNOutFiles();
399  if (static_cast<int>(procsWithData.size()) < nOutFiles) {
400  useSparseFPP = true;
401  for (auto i : procsWithData) {
402  procsWithDataVector.push_back(i);
403  }
404  }
405 
406  std::string filePrefix = name + "_D_";
407 
408  NFilesIter nfi(nOutFiles, filePrefix, VisMF::GetGroupSets(), VisMF::GetSetBuf());
409 
410  if (useSparseFPP) {
411  nfi.SetSparseFPP(procsWithDataVector);
412  } else {
413  nfi.SetDynamic();
414  }
415 
416  const auto &fio = FAB::getFABio();
417 
418  for ( ; nfi.ReadyToWrite(); ++nfi) {
419  for(MFIter mfi(fa); mfi.isValid(); ++mfi) {
420  const FAB &fab = fa[mfi];
421  {
422  std::stringstream hss;
423  fio.write_header(hss, fab, fab.nComp());
424  auto hLength = static_cast<std::streamoff>(hss.tellp());
425  auto tstr = hss.str();
426  nfi.Stream().write(tstr.c_str(), hLength);
427  nfi.Stream().flush();
428  }
429  auto const* fabdata = fab.dataPtr();
430 #ifdef AMREX_USE_GPU
431  std::unique_ptr<FAB> hostfab;
432  if (fab.arena()->isManaged() || fab.arena()->isDevice()) {
433  hostfab = std::make_unique<FAB>(fab.box(), fab.nComp(), The_Pinned_Arena());
434  Gpu::dtoh_memcpy_async(hostfab->dataPtr(), fab.dataPtr(),
435  fab.size()*sizeof(typename FAB::value_type));
437  fabdata = hostfab->dataPtr();
438  }
439 #endif
440  Long writeDataItems = fab.box().numPts() * fa.nComp();
441  Long writeDataSize = writeDataItems * data_bytes;
442  nfi.Stream().write((char *) fabdata, writeDataSize);
443  nfi.Stream().flush();
444  }
445  }
446 
447  int coordinatorProc = ParallelDescriptor::IOProcessorNumber();
448  if (nfi.GetDynamic()) {
449  coordinatorProc = nfi.CoordinatorProc();
450  }
451 
452  if (ParallelDescriptor::MyProc() == coordinatorProc) {
453  std::string header_file_name = name + "_H";
455  std::ofstream ofs;
456  ofs.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
457  ofs.open(header_file_name.c_str(), std::ios::out | std::ios::trunc);
458  if (!ofs.good()) {
459  amrex::FileOpenFailed(header_file_name);
460  }
461 
462  ofs << "amrex::FabArray<" << FAB::getClassName() << "> v1.0\n";
463  ofs << fa.nComp() << '\n';
464  ofs << fa.nGrowVect() << '\n';
465  fa.boxArray().writeOn(ofs);
466  ofs << '\n';
467 
468  const DistributionMapping& dm = fa.DistributionMap();
469  int nfabs = fa.boxArray().size();
470  int nFiles = NFilesIter::ActualNFiles(nOutFiles);
471  int nprocs = ParallelDescriptor::NProcs();
472 
473  Vector<Long> fabBytes(nfabs, 0);
474  std::map<int, Vector<int> > rankBoxOrder;
475  for (int i = 0; i < nfabs; ++i) {
476  std::stringstream hss;
477  FAB tmp(fa.fabbox(i), fa.nComp(), false);
478  fio.write_header(hss, tmp, tmp.nComp());
479  // Size includes header and data
480  fabBytes[i] = static_cast<std::streamoff>(hss.tellp()) + tmp.size() * data_bytes;
481  rankBoxOrder[dm[i]].push_back(i);
482  }
483 
484  Vector<int> fileNumbers;
485  if (nfi.GetDynamic()) {
486  fileNumbers = nfi.FileNumbersWritten();
487  } else if (nfi.GetSparseFPP()) {
488  // if sparse, write to (file number = rank)
489  fileNumbers.resize(nprocs);
490  std::iota(fileNumbers.begin(), fileNumbers.end(), 0);
491  } else {
492  fileNumbers.resize(nprocs);
493  for (int i = 0; i < nprocs; ++i) {
494  fileNumbers[i] = NFilesIter::FileNumber(nFiles, i, VisMF::GetGroupSets());
495  }
496  }
497 
498  Vector<VisMF::FabOnDisk> fod(nfabs);
499 
500  const Vector< Vector<int> > &fileNumbersWriteOrder = nfi.FileNumbersWriteOrder();
501  for (auto const& rv : fileNumbersWriteOrder) {
502  Long currentOffset = 0;
503  for (auto rank : rv) {
504  auto rbo_it = rankBoxOrder.find(rank);
505  if (rbo_it != rankBoxOrder.end()) {
506  Vector<int> const& index = rbo_it->second;
507  int whichFileNumber = fileNumbers[rank];
508  std::string const& whichFileName =
509  VisMF::BaseName(NFilesIter::FileName(whichFileNumber, filePrefix));
510  for (int i : index) {
511  fod[i].m_name = whichFileName;
512  fod[i].m_head = currentOffset;
513  currentOffset += fabBytes[i];
514  }
515  }
516  }
517  }
518  ofs << fod;
519  }
520 }
521 
522 namespace detail {
523 template <typename FAB>
524 void read_fab (FAB& fab, VisMF::FabOnDisk const& fod, std::string const& name)
525 {
526  std::string fullname = VisMF::DirName(name);
527  fullname += fod.m_name;
529  std::ifstream ifs;
530  ifs.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
531  ifs.open(fullname.c_str(), std::ios::in | std::ios::binary);
532  if (!ifs.good()) {
533  amrex::FileOpenFailed(fullname);
534  }
535  ifs.seekg(fod.m_head, std::ios::beg);
536  fab.readFrom(ifs);
537 }
538 }
539 
557 template <typename FAB>
558 // This function does work for MultiFab, but let's disable it to avoid confusion.
559 std::enable_if_t<std::is_same_v<FAB,IArrayBox>>
560 Read (FabArray<FAB>& fa, const std::string& name)
561 {
562  BL_PROFILE("Read(FabArray)");
563  AMREX_ASSERT(name.back() != '/');
564 
565  BoxArray ba;
566  int ncomp;
567  IntVect ngrow;
569  {
570  std::string header_file_name = name + "_H";
571  Vector<char> header_file_chars;
572  ParallelDescriptor::ReadAndBcastFile(header_file_name, header_file_chars);
573  std::string header_file_string(header_file_chars.data());
574  std::stringstream ifs(header_file_string, std::istringstream::in);
575 
576  std::string type, version;
577  ifs >> type >> version;
578  AMREX_ASSERT(type == "amrex::FabArray<amrex::IArrayBox>" ||
579  type == "amrex::FabArray<amrex::FArrayBox>");
580  ifs >> ncomp;
581  ifs >> ngrow;
582  ba.readFrom(ifs);
583  ifs >> fod;
584  }
585 
586  if (fa.empty()) {
587  fa.define(ba, DistributionMapping{ba}, ncomp, ngrow);
588  } else {
590  }
591 
592 #ifdef AMREX_USE_MPI
593  const int nopensperfile = VisMF::GetMFFileInStreams(); // # of concurrent readers per file
594  const int myproc = ParallelDescriptor::MyProc();
595  const int coordproc = ParallelDescriptor::IOProcessorNumber();
596 
597  int nreqs = 0;
598  int allreadsindex = 0;
599  std::map<std::string, int> filenames; // <filename, allreadsindex>
600 
601  const int nboxes = fa.size();
602  const auto& dm = fa.DistributionMap();
603  for (int i = 0; i < nboxes; ++i) {
604  if (myproc == dm[i]) {
605  ++nreqs;
606  }
607  if (myproc == coordproc) {
608  std::string const& fname = fod[i].m_name;
609  auto r =filenames.insert(std::make_pair(fname, allreadsindex));
610  if (r.second) {
611  ++allreadsindex;
612  }
613  }
614  }
615 
616  const int readtag = ParallelDescriptor::SeqNum();
617  const int donetag = ParallelDescriptor::SeqNum();
618 
619  if (myproc == coordproc) {
620  std::multiset<int> availablefiles; // [whichFile] supports multiple reads/file
621  Vector<std::map<int,std::map<Long,int> > > allreads; // [file]<proc,<seek,index>>
622 
623  const auto nfiles = static_cast<int>(filenames.size());
624  for (int i = 0; i < nfiles; ++i) {
625  for (int j = 0; j < nopensperfile; ++j) {
626  availablefiles.insert(i);
627  }
628  }
629  allreads.resize(nfiles);
630  for (int i = 0; i < nboxes; ++i) {
631  const auto whichproc = dm[i];
632  const auto iseekpos = fod[i].m_head;
633  std::string const& fname = fod[i].m_name;
634  auto filenamesiter = filenames.find(fname);
635  if (filenamesiter != filenames.end()) {
636  const int fi = filenamesiter->second;
637  allreads[fi][whichproc].insert(std::make_pair(iseekpos,i));
638  } else {
639  amrex::Error("Error in amrex::Read: filename not found "+fname);
640  }
641  }
642 
643  int totalioreqs = nboxes;
644  int reqspending = 0;
645  int iopfileindex;
646  std::deque<int> iopreads;
647  std::set<int> busyprocs;
648  while (totalioreqs > 0) {
649  auto afilesiter = availablefiles.begin();
650  while (afilesiter != availablefiles.end()) {
651  const int fi = *afilesiter;
652  if (allreads[fi].empty()) {
653  availablefiles.erase(fi);
654  afilesiter = availablefiles.begin();
655  continue;
656  }
657  auto whichread = allreads[fi].begin();
658  for ( ; whichread != allreads[fi].end(); ++whichread) {
659  const int tryproc = whichread->first;
660  if (busyprocs.find(tryproc) == busyprocs.end()) { // not busy
661  busyprocs.insert(tryproc);
662  Vector<int> vreads;
663  vreads.reserve(whichread->second.size());
664  for (auto const& kv : whichread->second) {
665  vreads.push_back(kv.second);
666  }
667  if (tryproc == coordproc) {
668  iopfileindex = fi;
669  for (auto x : vreads) {
670  iopreads.push_back(x);
671  }
672  } else {
673  ParallelDescriptor::Send(vreads, tryproc, readtag);
674  ++reqspending;
675  }
676  availablefiles.erase(afilesiter);
677  afilesiter = availablefiles.begin();
678  break;
679  }
680  }
681  if (whichread == allreads[fi].end()) {
682  ++afilesiter;
683  } else {
684  allreads[fi].erase(whichread);
685  }
686  }
687 
688  while (!iopreads.empty()) {
689  int i = iopreads.front();
690  detail::read_fab(fa[i], fod[i], name);
691  --totalioreqs;
692  iopreads.pop_front();
693  if (iopreads.empty()) {
694  availablefiles.insert(iopfileindex);
695  busyprocs.erase(coordproc);
696  }
697  int doneflag;
698  MPI_Status status;
699  ParallelDescriptor::IProbe(MPI_ANY_SOURCE, donetag, doneflag, status);
700  if (doneflag) {
701  break;
702  }
703  }
704 
705  if (reqspending > 0) {
706  Vector<int> idone(2);
707  ParallelDescriptor::Message rmess = ParallelDescriptor::Recv(idone, MPI_ANY_SOURCE,
708  donetag);
709  const int i = idone[0];
710  const int procdone = rmess.pid();
711  totalioreqs -= idone[1];
712  --reqspending;
713  busyprocs.erase(procdone);
714  std::string const& fname = fod[i].m_name;
715  const int fi = filenames.find(fname)->second;
716  availablefiles.insert(fi);
717  }
718  }
719  } else {
720  Vector<int> recreads(nreqs, -1);
721  Vector<int> idone(2);
722  while (nreqs > 0) {
723  ParallelDescriptor::Message rmess = ParallelDescriptor::Recv(recreads, coordproc,
724  readtag);
725  const auto nrmess = static_cast<int>(rmess.count());
726  for (int ir = 0; ir < nrmess; ++ir) {
727  int i = recreads[ir];
728  detail::read_fab(fa[i], fod[i], name);
729  }
730  nreqs -= nrmess;
731  idone[0] = recreads[0];
732  idone[1] = nrmess;
733  ParallelDescriptor::Send(idone, coordproc, donetag);
734  }
735  }
736 #else
737  for (MFIter mfi(fa); mfi.isValid(); ++mfi) {
738  detail::read_fab(fa[mfi], fod[mfi.index()], name);
739  }
740 #endif
741 }
742 
743 }
744 
745 #endif /*BL_VISMF_H*/
#define BL_PROFILE(a)
Definition: AMReX_BLProfiler.H:551
#define AMREX_ASSERT(EX)
Definition: AMReX_BLassert.H:38
#define AMREX_EXPORT
Definition: AMReX_Extension.H:191
Array4< int const > offset
Definition: AMReX_HypreMLABecLap.cpp:1089
int MPI_Comm
Definition: AMReX_ccse-mpi.H:47
A collection of Boxes stored in an Array.
Definition: AMReX_BoxArray.H:549
int readFrom(std::istream &is)
Initialize the BoxArray from the supplied istream. It is an error if the BoxArray has already been in...
std::ostream & writeOn(std::ostream &) const
Output this BoxArray to a checkpoint file.
Long size() const noexcept
Return the number of boxes in the BoxArray.
Definition: AMReX_BoxArray.H:596
Calculates the distribution of FABs to MPI processes.
Definition: AMReX_DistributionMapping.H:41
const Vector< int > & ProcessorMap() const noexcept
Returns a constant reference to the mapping of boxes in the underlying BoxArray to the CPU that holds...
Definition: AMReX_DistributionMapping.cpp:47
A Fortran Array of REALs.
Definition: AMReX_FArrayBox.H:229
IntVect nGrowVect() const noexcept
Definition: AMReX_FabArrayBase.H:79
const BoxArray & boxArray() const noexcept
Return a constant reference to the BoxArray that defines the valid region associated with this FabArr...
Definition: AMReX_FabArrayBase.H:94
int size() const noexcept
Return the number of FABs in the FabArray.
Definition: AMReX_FabArrayBase.H:109
bool empty() const noexcept
Definition: AMReX_FabArrayBase.H:88
const DistributionMapping & DistributionMap() const noexcept
Return constant reference to associated DistributionMapping.
Definition: AMReX_FabArrayBase.H:130
Box fabbox(int K) const noexcept
Return the Kth FABs Box in the FabArray. That is, the region the Kth fab is actually defined on.
int nComp() const noexcept
Return number of variables (aka components) associated with each point.
Definition: AMReX_FabArrayBase.H:82
void define(const BoxArray &bxs, const DistributionMapping &dm, int nvar, int ngrow, const MFInfo &info=MFInfo(), const FabFactory< FAB > &factory=DefaultFabFactory< FAB >())
Define this FabArray identically to that performed by the constructor having an analogous function si...
Definition: AMReX_FabArray.H:2027
Definition: AMReX_MFIter.H:57
bool isValid() const noexcept
Is the iterator valid i.e. is it associated with a FAB?
Definition: AMReX_MFIter.H:141
This class encapsulates writing to nfiles.
Definition: AMReX_NFiles.H:27
Vector< int > FileNumbersWritten()
these are the file numbers to which each rank wrote [rank] a rank only writes to one file
Definition: AMReX_NFiles.cpp:517
static int ActualNFiles(int nOutFiles)
this returns the actual number of files used the range [1, nProcs] is enforced
Definition: AMReX_NFiles.H:130
bool GetDynamic() const
Definition: AMReX_NFiles.H:54
bool GetSparseFPP() const
Definition: AMReX_NFiles.H:65
void SetSparseFPP(const Vector< int > &ranksToWrite)
call this to use a file per process for a sparse set of writers. ranksToWrite.size() will set noutfil...
Definition: AMReX_NFiles.cpp:114
std::fstream & Stream()
Definition: AMReX_NFiles.H:98
bool ReadyToWrite(bool appendFirst=false)
if appendFirst is true, the first set for this iterator will open the files in append mode
Definition: AMReX_NFiles.cpp:204
int FileNumber() const
Definition: AMReX_NFiles.H:161
const std::string & FileName() const
Definition: AMReX_NFiles.H:160
int CoordinatorProc() const
this is the processor coordinating dynamic set selection
Definition: AMReX_NFiles.H:184
const Vector< Vector< int > > & FileNumbersWriteOrder() const
these are the order of ranks which wrote to each file [filenumber][ranks in order they wrote to filen...
Definition: AMReX_NFiles.H:198
void SetDynamic(int deciderproc=-1)
call this to use dynamic set selection deciderproc defaults to nprocs - 1 if < 0
Definition: AMReX_NFiles.cpp:69
Hold the description and status of communication data.
Definition: AMReX_ParallelDescriptor.H:57
A Descriptor of the Real Type.
Definition: AMReX_FabConv.H:105
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition: AMReX_Vector.H:27
Long size() const noexcept
Definition: AMReX_Vector.H:50
T * dataPtr() noexcept
get access to the underlying data pointer
Definition: AMReX_Vector.H:46
Definition: AMReX_VisMFBuffer.H:13
static Long GetIOBufferSize()
Definition: AMReX_VisMFBuffer.H:26
File I/O for FabArray<FArrayBox>. Wrapper class for reading/writing FabArray<FArrayBox> objects to di...
Definition: AMReX_VisMF.H:34
static AMREX_EXPORT bool useSingleWrite
Definition: AMReX_VisMF.H:350
IntVect nGrowVect() const
Definition: AMReX_VisMF.cpp:449
static AMREX_EXPORT bool setBuf
Definition: AMReX_VisMF.H:348
static AMREX_EXPORT bool useSynchronousReads
Definition: AMReX_VisMF.H:353
static void SetGroupSets(bool groupsets)
Definition: AMReX_VisMF.H:259
Real max(int fabIndex, int nComp) const
The max of the FAB (in valid region) at specified index and component.
Definition: AMReX_VisMF.cpp:492
static void SetNOutFiles(int noutfiles, MPI_Comm comm=ParallelDescriptor::Communicator())
Definition: AMReX_VisMF.cpp:110
static AMREX_EXPORT bool checkFilePositions
Definition: AMReX_VisMF.H:351
static Long FileOffset(std::ostream &os)
The file offset of the passed ostream.
Definition: AMReX_VisMF.cpp:533
static std::ifstream * OpenStream(const std::string &fileName)
Open the stream if it is not already open Close the stream if not persistent or forced Close all open...
Definition: AMReX_VisMF.cpp:2166
static bool GetUseDynamicSetSelection()
Definition: AMReX_VisMF.H:279
static bool GetSetBuf()
Definition: AMReX_VisMF.H:261
const BoxArray & boxArray() const
The BoxArray of the on-disk FabArray<FArrayBox>.
Definition: AMReX_VisMF.cpp:461
static void SetUseSingleWrite(bool usesinglewrite)
Definition: AMReX_VisMF.H:268
static void SetHeaderVersion(VisMF::Header::Version version)
Definition: AMReX_VisMF.H:255
static void Read(FabArray< FArrayBox > &mf, const std::string &name, const char *faHeader=nullptr, int coordinatorProc=ParallelDescriptor::IOProcessorNumber(), int allow_empty_mf=0)
Read a FabArray<FArrayBox> from disk written using VisMF::Write(). If the FabArray<FArrayBox> fafab h...
Definition: AMReX_VisMF.cpp:1494
static Long WriteHeader(const std::string &mf_name, VisMF::Header &hdr, int procToWrite=ParallelDescriptor::IOProcessorNumber(), MPI_Comm comm=ParallelDescriptor::Communicator())
Definition: AMReX_VisMF.cpp:908
std::string m_fafabname
Name of the FabArray<FArrayBox>.
Definition: AMReX_VisMF.H:330
static void CloseAllStreams()
Definition: AMReX_VisMF.cpp:2216
static void AsyncWriteDoit(const FabArray< FArrayBox > &mf, const std::string &mf_name, bool is_rvalue, bool valid_cells_only)
Definition: AMReX_VisMF.cpp:2255
int size() const
Definition: AMReX_VisMF.cpp:455
static AMREX_EXPORT int verbose
Definition: AMReX_VisMF.H:345
static int GetVerbose()
Definition: AMReX_VisMF.H:251
static Long WriteHeaderDoit(const std::string &mf_name, VisMF::Header const &hdr)
Definition: AMReX_VisMF.cpp:876
static AMREX_EXPORT bool groupSets
Definition: AMReX_VisMF.H:347
static VisMF::Header::Version GetHeaderVersion()
Definition: AMReX_VisMF.H:254
static void SetCheckFilePositions(bool cfp)
Definition: AMReX_VisMF.H:271
static void SetUseSingleRead(bool usesingleread)
Definition: AMReX_VisMF.H:265
FArrayBox * readFAB(int idx, const std::string &mf_name)
Read the entire fab (all components).
Definition: AMReX_VisMF.cpp:549
static bool GetCheckFilePositions()
Definition: AMReX_VisMF.H:270
const FArrayBox & GetFab(int fabIndex, int compIndex) const
The FAB at the specified index and component. Reads it from disk if necessary. This reads only the sp...
Definition: AMReX_VisMF.cpp:517
static void DeleteStream(const std::string &fileName)
Definition: AMReX_VisMF.cpp:2205
static Long Write(const FabArray< FArrayBox > &mf, const std::string &name, VisMF::How how=NFiles, bool set_ghost=false)
Write a FabArray<FArrayBox> to disk in a "smart" way. Returns the total number of bytes written on th...
Definition: AMReX_VisMF.cpp:933
Vector< Vector< FArrayBox * > > m_pa
We manage the FABs individually.
Definition: AMReX_VisMF.H:334
static AMREX_EXPORT bool useSingleRead
Definition: AMReX_VisMF.H:349
static bool GetUseSynchronousReads()
Definition: AMReX_VisMF.H:276
static int GetMFFileInStreams()
Definition: AMReX_VisMF.H:248
static int GetNOutFiles()
Definition: AMReX_VisMF.cpp:122
static AMREX_EXPORT VisMF::Header::Version currentVersion
Definition: AMReX_VisMF.H:346
static AMREX_EXPORT bool usePersistentIFStreams
Definition: AMReX_VisMF.H:352
static bool GetGroupSets()
Definition: AMReX_VisMF.H:258
static std::string DirName(const std::string &filename)
Definition: AMReX_VisMF.cpp:579
static Long WriteOnlyHeader(const FabArray< FArrayBox > &mf, const std::string &mf_name, VisMF::How how=NFiles)
Write only the header-file corresponding to FabArray<FArrayBox> to disk without the corresponding FAB...
Definition: AMReX_VisMF.cpp:1134
VisMF(VisMF &&)=delete
int nGrow() const
The grow factor of the on-disk FabArray<FArrayBox>.
Definition: AMReX_VisMF.cpp:443
Header m_hdr
The VisMF header as read from disk.
Definition: AMReX_VisMF.H:332
static AMREX_EXPORT bool useDynamicSetSelection
Definition: AMReX_VisMF.H:354
static bool GetUsePersistentIFStreams()
Definition: AMReX_VisMF.H:273
static void SetUseSynchronousReads(bool usepsr)
Definition: AMReX_VisMF.H:277
static void SetVerbose(int v)
Definition: AMReX_VisMF.H:252
static void CloseStream(const std::string &fileName, bool forceClose=false)
Definition: AMReX_VisMF.cpp:2188
How
How we write out FabArray<FArrayBox>s. These are deprecated, we always use NFiles....
Definition: AMReX_VisMF.H:41
@ NFiles
Definition: AMReX_VisMF.H:41
@ OneFilePerCPU
Definition: AMReX_VisMF.H:41
static void AsyncWrite(const FabArray< FArrayBox > &mf, const std::string &mf_name, bool valid_cells_only=false)
Definition: AMReX_VisMF.cpp:2222
static void Finalize()
Definition: AMReX_VisMF.cpp:104
void clear()
Delete()s all the FABs.
Definition: AMReX_VisMF.cpp:2134
static bool Exist(const std::string &name)
Does FabArray exist?
Definition: AMReX_VisMF.cpp:1996
static void SetMFFileInStreams(int nstreams, MPI_Comm comm=ParallelDescriptor::Communicator())
Definition: AMReX_VisMF.cpp:116
static void ReadFAHeader(const std::string &fafabName, Vector< char > &header)
Read only the header of a FabArray, header will be resized here.
Definition: AMReX_VisMF.cpp:2010
~VisMF()=default
VisMF & operator=(const VisMF &)=delete
static AMREX_EXPORT int nMFFileInStreams
Definition: AMReX_VisMF.H:343
VisMF(std::string fafab_name)
Construct by reading in the on-disk VisMF of the specified name. The FABs in the on-disk FabArray are...
Definition: AMReX_VisMF.cpp:1356
static void Initialize()
Definition: AMReX_VisMF.cpp:66
static bool GetUseSingleRead()
Definition: AMReX_VisMF.H:264
static AMREX_EXPORT std::map< std::string, VisMF::PersistentIFStream > persistentIFStreams
Persistent streams. These open on demand and should be closed when not needed with CloseAllStreams....
Definition: AMReX_VisMF.H:340
static std::string BaseName(const std::string &filename)
Definition: AMReX_VisMF.cpp:561
static void RemoveFiles(const std::string &name, bool verbose=false)
this will remove nfiles associated with name and the header
Definition: AMReX_VisMF.cpp:1325
static AMREX_EXPORT bool allowSparseWrites
Definition: AMReX_VisMF.H:355
static AMREX_EXPORT int nOutFiles
The number of files to write for a FabArray<FArrayBox>.
Definition: AMReX_VisMF.H:342
int nComp() const
The number of components in the on-disk FabArray<FArrayBox>.
Definition: AMReX_VisMF.cpp:437
Real min(int fabIndex, int nComp) const
The min of the FAB (in valid region) at specified index and component.
Definition: AMReX_VisMF.cpp:467
static bool Check(const std::string &name)
Check if the multifab is ok, false is returned if not ok.
Definition: AMReX_VisMF.cpp:2021
static void SetUsePersistentIFStreams(bool usepifs)
Definition: AMReX_VisMF.H:274
static void SetUseDynamicSetSelection(bool usedss)
Definition: AMReX_VisMF.H:280
static bool GetUseSingleWrite()
Definition: AMReX_VisMF.H:267
static void FindOffsets(const FabArray< FArrayBox > &mf, const std::string &filePrefix, VisMF::Header &hdr, VisMF::Header::Version whichVersion, NFilesIter &nfi, MPI_Comm comm=ParallelDescriptor::Communicator())
fileNumbers must be passed in for dynamic set selection [proc]
Definition: AMReX_VisMF.cpp:1168
VisMF(const VisMF &)=delete
static void SetSetBuf(bool setbuf)
Definition: AMReX_VisMF.H:262
static bool NoFabHeader(const VisMF::Header &hdr)
Definition: AMReX_VisMF.cpp:2144
@ FAB
Definition: AMReX_AmrvisConstants.H:86
void streamSynchronize() noexcept
Definition: AMReX_GpuDevice.H:237
void dtoh_memcpy_async(void *p_h, const void *p_d, const std::size_t sz) noexcept
Definition: AMReX_GpuDevice.H:265
MPI_Comm Communicator() noexcept
Definition: AMReX_ParallelDescriptor.H:210
int MyProc() noexcept
return the rank number local to the current Parallel Context
Definition: AMReX_ParallelDescriptor.H:125
void ReadAndBcastFile(const std::string &filename, Vector< char > &charBuf, bool bExitOnError, const MPI_Comm &comm)
Definition: AMReX_ParallelDescriptor.cpp:1459
Message Send(const T *buf, size_t n, int dst_pid, int tag)
Definition: AMReX_ParallelDescriptor.H:1109
int SeqNum() noexcept
Returns sequential message sequence numbers, usually used as tags for send/recv.
Definition: AMReX_ParallelDescriptor.H:613
int NProcs() noexcept
return the number of MPI ranks local to the current Parallel Context
Definition: AMReX_ParallelDescriptor.H:243
void IProbe(int, int, int &, MPI_Status &)
Definition: AMReX_ParallelDescriptor.cpp:1209
int IOProcessorNumber() noexcept
Definition: AMReX_ParallelDescriptor.H:266
Message Recv(T *, size_t n, int pid, int tag)
Definition: AMReX_ParallelDescriptor.H:1151
static int fi(amrex::Real t, N_Vector y_data, N_Vector y_rhs, void *user_data)
Definition: AMReX_SundialsIntegrator.H:49
void read_fab(FAB &fab, VisMF::FabOnDisk const &fod, std::string const &name)
Definition: AMReX_VisMF.H:524
Definition: AMReX_Amr.cpp:49
void FileOpenFailed(const std::string &file)
Output a message and abort when couldn't open the file.
Definition: AMReX_Utility.cpp:131
std::enable_if_t< std::is_same_v< FAB, IArrayBox > > Write(const FabArray< FAB > &fa, const std::string &name)
Write iMultiFab/FabArray<IArrayBox>
Definition: AMReX_VisMF.H:383
std::enable_if_t< std::is_same_v< FAB, IArrayBox > > Read(FabArray< FAB > &fa, const std::string &name)
Read iMultiFab/FabArray<IArrayBox>
Definition: AMReX_VisMF.H:560
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Dim3 end(BoxND< dim > const &box) noexcept
Definition: AMReX_Box.H:1890
bool match(const BoxArray &x, const BoxArray &y)
Note that two BoxArrays that match are not necessarily equal.
void Error(const std::string &msg)
Print out message to cerr and exit via amrex::Abort().
Definition: AMReX.cpp:219
Arena * The_Pinned_Arena()
Definition: AMReX_Arena.cpp:649
std::ostream & operator<<(std::ostream &os, AmrMesh const &amr_mesh)
Definition: AMReX_AmrMesh.cpp:1236
std::istream & operator>>(std::istream &is, BoxND< dim > &bx)
Read from istream.
Definition: AMReX_Box.H:1700
Definition: AMReX_FabArrayCommI.H:841
Definition: AMReX_ccse-mpi.H:51
A structure containing info regarding an on-disk FAB.
Definition: AMReX_VisMF.H:56
Long m_head
Offset to start of FAB in file.
Definition: AMReX_VisMF.H:63
std::string m_name
The two data values in a FabOnDisk structure.
Definition: AMReX_VisMF.H:62
FabOnDisk()=default
The default constructor – null out all fields.
An on-disk FabArray<FArrayBox> contains this info in a header file.
Definition: AMReX_VisMF.H:67
Vector< Vector< Real > > m_min
The min()s of each component of FABs. [findex][comp].
Definition: AMReX_VisMF.H:108
Header(Header const &)=delete
Vector< Real > m_famax
The max()s of each component of the FabArray. [comp].
Definition: AMReX_VisMF.H:111
void CalculateMinMax(const FabArray< FArrayBox > &mf, int procToWrite=ParallelDescriptor::IOProcessorNumber(), MPI_Comm=ParallelDescriptor::Communicator())
Calculate the min and max arrays.
Definition: AMReX_VisMF.cpp:699
Version
The versions of the FabArray<FArrayBox> Header code.
Definition: AMReX_VisMF.H:69
@ NoFabHeader_v1
-— no fab headers, no fab mins or maxes
Definition: AMReX_VisMF.H:74
@ NoFabHeaderMinMax_v1
Definition: AMReX_VisMF.H:75
@ Undefined_v1
-— undefined
Definition: AMReX_VisMF.H:70
@ Version_v1
Definition: AMReX_VisMF.H:71
@ NoFabHeaderFAMinMax_v1
Definition: AMReX_VisMF.H:77
int m_vers
The version of the Header.
Definition: AMReX_VisMF.H:99
BoxArray m_ba
The BoxArray of the MF.
Definition: AMReX_VisMF.H:103
Vector< Vector< Real > > m_max
The max()s of each component of FABs. [findex][comp].
Definition: AMReX_VisMF.H:109
Header()
The default constructor.
How m_how
How the MF was written to disk.
Definition: AMReX_VisMF.H:100
Header(Header &&rhs) noexcept=default
Header & operator=(Header const &)=delete
Vector< FabOnDisk > m_fod
FabOnDisk info for contained FABs.
Definition: AMReX_VisMF.H:104
int m_ncomp
Number of components in MF.
Definition: AMReX_VisMF.H:101
RealDescriptor m_writtenRD
Definition: AMReX_VisMF.H:112
IntVect m_ngrow
The number of ghost cells in MF.
Definition: AMReX_VisMF.H:102
Vector< Real > m_famin
The min()s of each component of the FabArray. [comp].
Definition: AMReX_VisMF.H:110
This structure is used to store file ifstreams that remain open.
Definition: AMReX_VisMF.H:129
~PersistentIFStream()
Definition: AMReX_VisMF.cpp:2155
VisMF::IO_Buffer ioBuffer
Definition: AMReX_VisMF.H:133
PersistentIFStream(PersistentIFStream &&)=delete
std::streampos currentPosition
Definition: AMReX_VisMF.H:131
std::ifstream * pstr
Definition: AMReX_VisMF.H:130
bool isOpen
Definition: AMReX_VisMF.H:132
PersistentIFStream(PersistentIFStream const &)=delete
PersistentIFStream & operator=(PersistentIFStream const &)=delete