Block-Structured AMR Software Framework
Loading...
Searching...
No Matches
AMReX_InterpBndryData.H
Go to the documentation of this file.
1
2#ifndef AMREX_INTERPBNDRYDATA_H_
3#define AMREX_INTERPBNDRYDATA_H_
4#include <AMReX_Config.H>
5
6#include <AMReX_BCRec.H>
7#include <AMReX_BndryData.H>
8#include <AMReX_InterpBndryData_K.H>
9
14
15namespace amrex {
16
18namespace detail {
19 // One boundary face of one box, for the fused GPU launch in setBndryValues.
20 template <typename T>
21 struct InterpBndryTag {
22 Array4<T> bdry;
23 Array4<T const> src; // coarse data, or fine data on a physical boundary
24 Array4<int const> mask;
25 Box bx;
26 int dir; // direction of a coarse/fine face; -1 on a physical boundary
27
29 Box const& box () const noexcept { return bx; }
30 };
31}
33
58template <typename MF>
60 :
61 public BndryDataT<MF>
62{
63public:
64
65 using value_type = typename MF::value_type;
66
70 InterpBndryDataT () noexcept = default;
71
80 InterpBndryDataT (const BoxArray& _grids,
81 const DistributionMapping& _dmap,
82 int _ncomp,
83 const Geometry& geom);
84
85 ~InterpBndryDataT () = default;
86
87 InterpBndryDataT (const InterpBndryDataT<MF>& rhs) = delete;
88 InterpBndryDataT (InterpBndryDataT<MF>&& rhs) = delete;
89 InterpBndryDataT<MF>& operator= (const InterpBndryDataT<MF>& rhs) = delete;
90 InterpBndryDataT<MF>& operator= (InterpBndryDataT<MF>&& rhs) = delete;
91
100 void setPhysBndryValues (const MF& mf, int mf_start, int bnd_start, int num_comp);
101
121 void setBndryValues (BndryRegisterT<MF> const& crse, int c_start, const MF& fine, int f_start,
122 int bnd_start, int num_comp, const IntVect& ratio,
123 int max_order = IBD_max_order_DEF, int max_width = 2);
124
136 void updateBndryValues (BndryRegisterT<MF>& crse, int c_start, int bnd_start, int num_comp,
137 const IntVect& ratio, int max_order = IBD_max_order_DEF, int max_width = 2);
138
141
142 // For sliding parabolic interp in bdfuncs
143 static constexpr int IBD_max_order_DEF = 3;
144
145};
146
147template <typename MF>
149 const DistributionMapping& _dmap,
150 int _ncomp,
151 const Geometry& _geom)
152 :
153 BndryDataT<MF>(_grids,_dmap,_ncomp,_geom)
154{}
155
156template <typename MF>
157void
158InterpBndryDataT<MF>::setPhysBndryValues (const MF& mf, int mf_start, int bnd_start, int num_comp)
159{
160 AMREX_ASSERT(this->grids == mf.boxArray());
161
162 const Box& fine_domain = this->geom.Domain();
163
164#ifdef AMREX_USE_OMP
165#pragma omp parallel if (Gpu::notInLaunchRegion())
166#endif
167 for (MFIter mfi(mf,MFItInfo().SetDynamic(true)); mfi.isValid(); ++mfi)
168 {
169 const Box& bx = mfi.validbox();
170 for (OrientationIter fi; fi; ++fi) {
171 const Orientation face = fi();
172 if (bx[face] == fine_domain[face] && !(this->geom.isPeriodic(face.coordDir())))
173 {
174 // Physical bndry, copy from grid.
175 auto & bnd_fab = this->bndry[face][mfi];
176 auto const& src_fab = mf[mfi];
177 auto const& bnd_array = bnd_fab.array();
178 auto const& src_array = src_fab.const_array();
179 const Box& b = src_fab.box() & bnd_fab.box();
180 AMREX_HOST_DEVICE_PARALLEL_FOR_4D ( b, num_comp, i, j, k, n,
181 {
182 bnd_array(i,j,k,n+bnd_start) = src_array(i,j,k,n+mf_start);
183 });
184 }
185 }
186 }
187}
188
189template <typename MF>
190void
191InterpBndryDataT<MF>::setBndryValues (BndryRegisterT<MF> const& crse, int c_start, const MF& fine,
192 int f_start, int bnd_start, int num_comp, const IntVect& ratio,
193 int max_order, int max_width)
194{
195 AMREX_ASSERT(this->grids == fine.boxArray());
196
197 const Box& fine_domain = this->geom.Domain();
198
199 // Order-3 interpolation reads the boundary mask at tangential offsets of
200 // ±ratio. The masks are built with a fixed tangential half-width
201 // (BndryDataT::NTangHalfWidth == 5, i.e. ref_ratio + 1), so ratios above 4
202 // would walk past the allocated mask halo. Order-1 does not touch the mask.
203 if (max_order == 3) {
205 "InterpBndryDataT<MF>::setBndryValues: max_order=3 requires refinement ratio <= 4");
206 }
207
208#ifdef AMREX_USE_GPU
209 if ((max_order==3 || max_order==1) && Gpu::inLaunchRegion())
210 {
211 // One launch for all faces of all boxes.
212 using Tag = detail::InterpBndryTag<value_type>;
213 Vector<Tag> tags;
214 tags.reserve(std::size_t(fine.local_size())*std::size_t(2*AMREX_SPACEDIM));
215 for (MFIter mfi(fine,MFItInfo().DisableDeviceSync()); mfi.isValid(); ++mfi)
216 {
217 const Box& fine_bx = mfi.validbox();
218 for (OrientationIter fi; fi; ++fi) {
219 const Orientation face = fi();
220 const int dir = face.coordDir();
221 if (fine_bx[face] != fine_domain[face] || this->geom.isPeriodic(dir))
222 {
223 tags.push_back(Tag{.bdry = this->bndry[face].array(mfi),
224 .src = crse[face].const_array(mfi),
225 .mask = (max_order == 3) ? this->masks[face].const_array(mfi)
227 .bx = this->bndry[face][mfi].box(),
228 .dir = dir});
229 }
230 else if (fine.defined(mfi)) {
231 auto & bnd_fab = this->bndry[face][mfi];
232 auto const& src_fab = fine[mfi];
233 tags.push_back(Tag{.bdry = bnd_fab.array(),
234 .src = src_fab.const_array(),
235 .mask = Array4<int const>{},
236 .bx = bnd_fab.box() & src_fab.box(),
237 .dir = -1});
238 }
239 }
240 }
241
242 const auto rr = ratio.dim3();
243 const int is_not_covered = BndryData::not_covered;
244 ParallelFor(tags, num_comp,
245 [=] AMREX_GPU_DEVICE (int i, int j, int k, int n, Tag const& tag) noexcept
246 {
247 if (tag.dir < 0) {
248 tag.bdry(i,j,k,n+bnd_start) = tag.src(i,j,k,n+f_start);
249 } else if (max_order == 1) {
250 interpbndrydata_o1(i,j,k,n,tag.bdry,bnd_start,tag.src,c_start,rr);
251 } else if (tag.dir == 0) {
252 interpbndrydata_x_o3(i,j,k,n,tag.bdry,bnd_start,tag.src,c_start,rr,
253 tag.mask, is_not_covered, max_width);
254 }
255#if (AMREX_SPACEDIM >= 2)
256 else if (tag.dir == 1) {
257 interpbndrydata_y_o3(i,j,k,n,tag.bdry,bnd_start,tag.src,c_start,rr,
258 tag.mask, is_not_covered, max_width);
259 }
260#if (AMREX_SPACEDIM == 3)
261 else {
262 interpbndrydata_z_o3(i,j,k,n,tag.bdry,bnd_start,tag.src,c_start,rr,
263 tag.mask, is_not_covered, max_width);
264 }
265#endif
266#endif
267 });
268 if (!Gpu::inNoSyncRegion()) {
270 }
271 }
272 else
273#endif
274 if (max_order==3 || max_order==1)
275 {
276#ifdef AMREX_USE_OMP
277#pragma omp parallel
278#endif
279 for (MFIter mfi(fine,MFItInfo().SetDynamic(true)); mfi.isValid(); ++mfi)
280 {
281 const Box& fine_bx = mfi.validbox();
282 for (OrientationIter fi; fi; ++fi) {
283 const Orientation face = fi();
284 const int dir = face.coordDir();
285 if (fine_bx[face] != fine_domain[face] || this->geom.isPeriodic(dir))
286 { // coarse/fine boundary: interpolate from coarse data stored in BndryRegister crse
287 auto const& crse_array = crse[face].const_array(mfi);
288 auto const& bdry_array = this->bndry[face].array(mfi);
289 Box const& b = this->bndry[face][mfi].box();
290 const auto rr = ratio.dim3();
291
292 if (max_order == 1)
293 {
294 amrex::LoopConcurrentOnCpu(b, num_comp, [&] (int it, int jt, int kt, int n)
295 {
296 interpbndrydata_o1(it,jt,kt,n,bdry_array,bnd_start,
297 crse_array,c_start,rr);
298 });
299 }
300 else
301 {
302 auto const& mask_array = this->masks[face].const_array(mfi);
303 int is_not_covered = BndryData::not_covered;
304 switch (dir) {
305 case 0:
306 {
307 amrex::LoopConcurrentOnCpu(b, num_comp, [&] (int it, int jt, int kt, int n)
308 {
309 interpbndrydata_x_o3(it,jt,kt,n,bdry_array,bnd_start,
310 crse_array,c_start,rr,
311 mask_array, is_not_covered, max_width);
312 });
313 break;
314 }
315#if (AMREX_SPACEDIM >= 2)
316 case 1:
317 {
318 amrex::LoopConcurrentOnCpu(b, num_comp, [&] (int it, int jt, int kt, int n)
319 {
320 interpbndrydata_y_o3(it,jt,kt,n,bdry_array,bnd_start,
321 crse_array,c_start,rr,
322 mask_array, is_not_covered, max_width);
323 });
324 break;
325 }
326#if (AMREX_SPACEDIM == 3)
327 case 2:
328 {
329 amrex::LoopConcurrentOnCpu(b, num_comp, [&] (int it, int jt, int kt, int n)
330 {
331 interpbndrydata_z_o3(it,jt,kt,n,bdry_array,bnd_start,
332 crse_array,c_start,rr,
333 mask_array, is_not_covered, max_width);
334 });
335 break;
336 }
337#endif
338#endif
339 default: {}
340 }
341 }
342 }
343 else if (fine.defined(mfi)) {
344 // Physical bndry
345 auto & bnd_fab = this->bndry[face][mfi];
346 auto const& src_fab = fine[mfi];
347 auto const& bnd_array = bnd_fab.array();
348 auto const& src_array = src_fab.const_array();
349 const Box& b = bnd_fab.box() & src_fab.box();
350 amrex::LoopConcurrentOnCpu(b, num_comp, [&] (int ii, int jj, int kk, int nn)
351 {
352 bnd_array(ii,jj,kk,nn+bnd_start) = src_array(ii,jj,kk,nn+f_start);
353 });
354 }
355 }
356 }
357 }
358 else
359 {
360 amrex::Abort("InterpBndryDataT<MF>::setBndryValues supports only max_order=1 or 3");
361 }
362}
363
364template <typename MF>
365void
367 int num_comp, const IntVect& ratio, int max_order, int max_width)
368{
369 MF foo(this->grids, this->bndry[0].DistributionMap(), 1, num_comp, MFInfo().SetAlloc(false));
370 setBndryValues(crse, c_start, foo, 0, bnd_start, num_comp, ratio, max_order, max_width);
371}
372
373template <typename MF>
374void
379
382
383}
384
385#endif /*AMREX_INTERPBNDRYDATA_H_*/
Boundary condition descriptors used throughout AMReX.
#define AMREX_ALWAYS_ASSERT_WITH_MESSAGE(EX, MSG)
Definition AMReX_BLassert.H:49
#define AMREX_ASSERT(EX)
Definition AMReX_BLassert.H:38
Boundary data container that owns masks, boundary values, and metadata.
#define AMREX_FORCE_INLINE
Definition AMReX_Extension.H:123
#define AMREX_HOST_DEVICE_PARALLEL_FOR_4D(...)
Definition AMReX_GpuLaunchMacrosC.nolint.H:111
#define AMREX_GPU_DEVICE
Definition AMReX_GpuQualifiers.H:18
#define AMREX_GPU_HOST_DEVICE
Definition AMReX_GpuQualifiers.H:20
BndryDataT() noexcept=default
Default constructor.
Vector< MultiMask > masks
Boundary condition mask.
Definition AMReX_BndryData.H:171
@ not_covered
Definition AMReX_BndryData.H:50
Geometry geom
Definition AMReX_BndryData.H:173
A BndryRegister organizes FabSets bounding each grid in a BoxArray. A FabSet is maintained for each b...
Definition AMReX_BndryRegister.H:46
BoxArray grids
Definition AMReX_BndryRegister.H:173
const DistributionMapping & DistributionMap() const noexcept
Returns the DistributionMapping describing which MPI ranks own each boundary FAB.
Definition AMReX_BndryRegister.H:143
FabSetT< MF > bndry[2 *3]
The data.
Definition AMReX_BndryRegister.H:172
Reference-counted collection of Boxes.
Definition AMReX_BoxArray.H:681
Calculates the distribution of FABs to MPI processes.
Definition AMReX_DistributionMapping.H:55
Rectangular problem domain geometry.
Definition AMReX_Geometry.H:87
__host__ __device__ constexpr Dim3 dim3() const noexcept
Definition AMReX_IntVect.H:269
__host__ __device__ constexpr bool allLE(const IntVectND< dim > &rhs) const noexcept
Returns true if this is less than or equal to argument for all components. NOTE: This is NOT a strict...
Definition AMReX_IntVect.H:499
An InterpBndryData object adds to a BndryData object the ability to manipulate and set the data store...
Definition AMReX_InterpBndryData.H:62
void setPhysBndryValues(const MultiFab &mf, int mf_start, int bnd_start, int num_comp)
static constexpr int IBD_max_order_DEF
Definition AMReX_InterpBndryData.H:143
void updateBndryValues(BndryRegisterT< MultiFab > &crse, int c_start, int bnd_start, int num_comp, const IntVect &ratio, int max_order=IBD_max_order_DEF, int max_width=2)
InterpBndryDataT() noexcept=default
default constructor
void setBndryValues(BndryRegisterT< MultiFab > const &crse, int c_start, const MultiFab &fine, int f_start, int bnd_start, int num_comp, const IntVect &ratio, int max_order=IBD_max_order_DEF, int max_width=2)
typename MF::value_type value_type
Definition AMReX_InterpBndryData.H:65
Iterator for looping ever tiles and boxes of amrex::FabArray based containers.
Definition AMReX_MFIter.H:88
bool isValid() const noexcept
Is the iterator valid i.e. is it associated with a FAB?
Definition AMReX_MFIter.H:176
An Iterator over the Orientation of Faces of a Box.
Definition AMReX_Orientation.H:135
Encapsulation of the Orientation of the Faces of a Box.
Definition AMReX_Orientation.H:29
__host__ __device__ int coordDir() const noexcept
Returns the coordinate direction.
Definition AMReX_Orientation.H:83
This class is a thin wrapper around std::vector. Unlike vector, Vector::operator[] provides bound che...
Definition AMReX_Vector.H:29
ArrayND< T, 4, true > Array4
Convenient alias for a 4D ArrayND with three spatial dimensions plus a component dimension.
Definition AMReX_Array4.H:1352
void streamSynchronize() noexcept
Definition AMReX_GpuDevice.H:329
bool inLaunchRegion() noexcept
Definition AMReX_GpuControl.H:88
bool inNoSyncRegion() noexcept
Definition AMReX_GpuControl.H:148
Definition AMReX_Amr.cpp:50
InterpBndryDataT< MultiFab > InterpBndryData
Definition AMReX_InterpBndryData.H:380
void ParallelFor(TypeList< CTOs... > ctos, std::array< int, sizeof...(CTOs)> const &runtime_options, T N, F &&f)
Definition AMReX_CTOParallelForImpl.H:202
BoxND< 3 > Box
Box is an alias for amrex::BoxND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:35
InterpBndryDataT< fMultiFab > fInterpBndryData
Definition AMReX_InterpBndryData.H:381
IntVectND< 3 > IntVect
IntVect is an alias for amrex::IntVectND instantiated with AMREX_SPACEDIM.
Definition AMReX_BaseFwd.H:38
void LoopConcurrentOnCpu(Dim3 lo, Dim3 hi, F const &f) noexcept
Definition AMReX_Loop.H:388
void Abort(const std::string &msg)
Print a fatal-error message to stderr and abort execution.
Definition AMReX.cpp:239
void setVal(MF &dst, typename MF::value_type val)
dst = val
Definition AMReX_FabArrayUtility.H:2174
FabArray memory allocation information.
Definition AMReX_FabArray.H:73
Definition AMReX_MFIter.H:20