Block-Structured AMR Software Framework
AMReX_MLNodeLap_3D_K.H
Go to the documentation of this file.
1 #ifndef AMREX_MLNODELAP_3D_K_H_
2 #define AMREX_MLNODELAP_3D_K_H_
3 #include <AMReX_Config.H>
4 
5 namespace amrex {
6 
7 //
8 // masks
9 //
10 
11 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
12 void mlndlap_set_nodal_mask (int i, int j, int k, Array4<int> const& nmsk,
13  Array4<int const> const& cmsk) noexcept
14 {
15  using namespace nodelap_detail;
16 
17  int s = cmsk(i-1,j-1,k-1) + cmsk(i ,j-1,k-1)
18  + cmsk(i-1,j ,k-1) + cmsk(i ,j ,k-1)
19  + cmsk(i-1,j-1,k ) + cmsk(i ,j-1,k )
20  + cmsk(i-1,j ,k ) + cmsk(i ,j ,k );
21  if (s == 8*crse_cell) {
22  nmsk(i,j,k) = crse_node;
23  }
24  else if (s == 8*fine_cell) {
25  nmsk(i,j,k) = fine_node;
26  } else {
27  nmsk(i,j,k) = crse_fine_node;
28  }
29 }
30 
31 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
32 void mlndlap_set_dirichlet_mask (Box const& bx, Array4<int> const& dmsk,
33  Array4<int const> const& omsk, Box const& dom,
34  GpuArray<LinOpBCType, AMREX_SPACEDIM> const& bclo,
35  GpuArray<LinOpBCType, AMREX_SPACEDIM> const& bchi) noexcept
36 {
37  const auto lo = amrex::lbound(bx);
38  const auto hi = amrex::ubound(bx);
39  for (int k = lo.z; k <= hi.z; ++k) {
40  for (int j = lo.y; j <= hi.y; ++j) {
41  AMREX_PRAGMA_SIMD
42  for (int i = lo.x; i <= hi.x; ++i) {
43  if (!dmsk(i,j,k)) {
44  dmsk(i,j,k) = (omsk(i-1,j-1,k-1) == 1 || omsk(i,j-1,k-1) == 1 ||
45  omsk(i-1,j ,k-1) == 1 || omsk(i,j ,k-1) == 1 ||
46  omsk(i-1,j-1,k ) == 1 || omsk(i,j-1,k ) == 1 ||
47  omsk(i-1,j ,k ) == 1 || omsk(i,j ,k ) == 1);
48  }
49  }}}
50 
51  const auto domlo = amrex::lbound(dom);
52  const auto domhi = amrex::ubound(dom);
53 
54  if (bclo[0] == LinOpBCType::Dirichlet && lo.x == domlo.x) {
55  for (int k = lo.z; k <= hi.z; ++k) {
56  for (int j = lo.y; j <= hi.y; ++j) {
57  dmsk(lo.x,j,k) = 1;
58  }}
59  }
60 
61  if (bchi[0] == LinOpBCType::Dirichlet && hi.x == domhi.x) {
62  for (int k = lo.z; k <= hi.z; ++k) {
63  for (int j = lo.y; j <= hi.y; ++j) {
64  dmsk(hi.x,j,k) = 1;
65  }}
66  }
67 
68  if (bclo[1] == LinOpBCType::Dirichlet && lo.y == domlo.y) {
69  for (int k = lo.z; k <= hi.z; ++k) {
70  AMREX_PRAGMA_SIMD
71  for (int i = lo.x; i <= hi.x; ++i) {
72  dmsk(i,lo.y,k) = 1;
73  }}
74  }
75 
76  if (bchi[1] == LinOpBCType::Dirichlet && hi.y == domhi.y) {
77  for (int k = lo.z; k <= hi.z; ++k) {
78  AMREX_PRAGMA_SIMD
79  for (int i = lo.x; i <= hi.x; ++i) {
80  dmsk(i,hi.y,k) = 1;
81  }}
82  }
83 
84  if (bclo[2] == LinOpBCType::Dirichlet && lo.z == domlo.z) {
85  for (int j = lo.y; j <= hi.y; ++j) {
86  AMREX_PRAGMA_SIMD
87  for (int i = lo.x; i <= hi.x; ++i) {
88  dmsk(i,j,lo.z) = 1;
89  }}
90  }
91 
92  if (bchi[2] == LinOpBCType::Dirichlet && hi.z == domhi.z) {
93  for (int j = lo.y; j <= hi.y; ++j) {
94  AMREX_PRAGMA_SIMD
95  for (int i = lo.x; i <= hi.x; ++i) {
96  dmsk(i,j,hi.z) = 1;
97  }}
98  }
99 }
100 
101 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
102 void mlndlap_set_dot_mask (Box const& bx, Array4<Real> const& dmsk,
103  Array4<int const> const& omsk, Box const& dom,
104  GpuArray<LinOpBCType, AMREX_SPACEDIM> const& bclo,
105  GpuArray<LinOpBCType, AMREX_SPACEDIM> const& bchi) noexcept
106 {
107  const auto lo = amrex::lbound(bx);
108  const auto hi = amrex::ubound(bx);
109  for (int k = lo.z; k <= hi.z; ++k) {
110  for (int j = lo.y; j <= hi.y; ++j) {
111  AMREX_PRAGMA_SIMD
112  for (int i = lo.x; i <= hi.x; ++i) {
113  dmsk(i,j,k) = static_cast<Real>(omsk(i,j,k));
114  }}}
115 
116  const auto domlo = amrex::lbound(dom);
117  const auto domhi = amrex::ubound(dom);
118 
119  if ((bclo[0] == LinOpBCType::Neumann || bclo[0] == LinOpBCType::inflow)
120  && lo.x == domlo.x)
121  {
122  for (int k = lo.z; k <= hi.z; ++k) {
123  for (int j = lo.y; j <= hi.y; ++j) {
124  dmsk(lo.x,j,k) *= Real(0.5);
125  }}
126  }
127 
128  if ((bchi[0] == LinOpBCType::Neumann || bchi[0] == LinOpBCType::inflow)
129  && hi.x == domhi.x)
130  {
131  for (int k = lo.z; k <= hi.z; ++k) {
132  for (int j = lo.y; j <= hi.y; ++j) {
133  dmsk(hi.x,j,k) *= Real(0.5);
134  }}
135  }
136 
137  if ((bclo[1] == LinOpBCType::Neumann || bclo[1] == LinOpBCType::inflow)
138  && lo.y == domlo.y)
139  {
140  for (int k = lo.z; k <= hi.z; ++k) {
141  AMREX_PRAGMA_SIMD
142  for (int i = lo.x; i <= hi.x; ++i) {
143  dmsk(i,lo.y,k) *= Real(0.5);
144  }}
145  }
146 
147  if ((bchi[1] == LinOpBCType::Neumann || bchi[1] == LinOpBCType::inflow)
148  && hi.y == domhi.y)
149  {
150  for (int k = lo.z; k <= hi.z; ++k) {
151  AMREX_PRAGMA_SIMD
152  for (int i = lo.x; i <= hi.x; ++i) {
153  dmsk(i,hi.y,k) *= Real(0.5);
154  }}
155  }
156 
157  if ((bclo[2] == LinOpBCType::Neumann || bclo[2] == LinOpBCType::inflow)
158  && lo.z == domlo.z)
159  {
160  for (int j = lo.y; j <= hi.y; ++j) {
161  AMREX_PRAGMA_SIMD
162  for (int i = lo.x; i <= hi.x; ++i) {
163  dmsk(i,j,lo.z) *= Real(0.5);
164  }}
165  }
166 
167  if ((bchi[2] == LinOpBCType::Neumann || bchi[2] == LinOpBCType::inflow)
168  && hi.z == domhi.z)
169  {
170  for (int j = lo.y; j <= hi.y; ++j) {
171  AMREX_PRAGMA_SIMD
172  for (int i = lo.x; i <= hi.x; ++i) {
173  dmsk(i,j,hi.z) *= Real(0.5);
174  }}
175  }
176 }
177 
178 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
179 void mlndlap_zero_fine (int i, int j, int k, Array4<Real> const& phi,
180  Array4<int const> const& msk, int fine_flag) noexcept
181 {
182  // Testing if the node is covered by a fine level in computing
183  // coarse sync residual
184  if (msk(i-1,j-1,k-1) == fine_flag &&
185  msk(i ,j-1,k-1) == fine_flag &&
186  msk(i-1,j ,k-1) == fine_flag &&
187  msk(i ,j ,k-1) == fine_flag &&
188  msk(i-1,j-1,k ) == fine_flag &&
189  msk(i ,j-1,k ) == fine_flag &&
190  msk(i-1,j ,k ) == fine_flag &&
191  msk(i ,j ,k ) == fine_flag)
192  {
193  phi(i,j,k) = Real(0.0);
194  }
195 }
196 
197 //
198 // coeffs
199 //
200 
201 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
202 void mlndlap_avgdown_coeff_x (int i, int j, int k, Array4<Real> const& crse,
203  Array4<Real const> const& fine) noexcept
204 {
205  Real cl = fine(2*i ,2*j,2*k )+fine(2*i ,2*j+1,2*k )+
206  fine(2*i ,2*j,2*k+1)+fine(2*i ,2*j+1,2*k+1);
207  Real cr = fine(2*i+1,2*j,2*k )+fine(2*i+1,2*j+1,2*k )+
208  fine(2*i+1,2*j,2*k+1)+fine(2*i+1,2*j+1,2*k+1);
209  crse(i,j,k) = Real(0.5)*cl*cr/(cl+cr);
210 }
211 
212 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
213 void mlndlap_avgdown_coeff_y (int i, int j, int k, Array4<Real> const& crse,
214  Array4<Real const> const& fine) noexcept
215 {
216  Real cl = fine(2*i,2*j ,2*k )+fine(2*i+1,2*j ,2*k )+
217  fine(2*i,2*j ,2*k+1)+fine(2*i+1,2*j ,2*k+1);
218  Real cr = fine(2*i,2*j+1,2*k )+fine(2*i+1,2*j+1,2*k )+
219  fine(2*i,2*j+1,2*k+1)+fine(2*i+1,2*j+1,2*k+1);
220  crse(i,j,k) = Real(0.5)*cl*cr/(cl+cr);
221 }
222 
223 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
224 void mlndlap_avgdown_coeff_z (int i, int j, int k, Array4<Real> const& crse,
225  Array4<Real const> const& fine) noexcept
226 {
227  Real cl = fine(2*i,2*j ,2*k )+fine(2*i+1,2*j ,2*k )+
228  fine(2*i,2*j+1,2*k )+fine(2*i+1,2*j+1,2*k );
229  Real cr = fine(2*i,2*j ,2*k+1)+fine(2*i+1,2*j ,2*k+1)+
230  fine(2*i,2*j+1,2*k+1)+fine(2*i+1,2*j+1,2*k+1);
231  crse(i,j,k) = Real(0.5)*cl*cr/(cl+cr);
232 }
233 
234 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
235 void mlndlap_semi_avgdown_coeff (int i, int j, int k, Array4<Real> const& crse,
236  Array4<Real const> const& fine, int idir) noexcept
237 {
238  if (idir == 2) {
239  Real cl = fine(2*i ,2*j,k) + fine(2*i ,2*j+1,k);
240  Real cr = fine(2*i+1,2*j,k) + fine(2*i+1,2*j+1,k);
241  crse(i,j,k) = cl*cr/(cl+cr);
242  } else if (idir == 1) {
243  Real cl = fine(2*i ,j,2*k) + fine(2*i ,j,2*k+1);
244  Real cr = fine(2*i+1,j,2*k) + fine(2*i+1,j,2*k+1);
245  crse(i,j,k) = cl*cr/(cl+cr);
246  } else {
247  Real cl = fine(i,2*j ,2*k) + fine(i,2*j ,2*k+1);
248  Real cr = fine(i,2*j+1,2*k) + fine(i,2*j+1,2*k+1);
249  crse(i,j,k) = cl*cr/(cl+cr);
250  }
251 }
252 //
253 // bc
254 //
255 
256 template <typename T>
257 inline void mlndlap_bc_doit (Box const& vbx, Array4<T> const& a, Box const& domain,
258  GpuArray<bool,AMREX_SPACEDIM> const& bflo,
259  GpuArray<bool,AMREX_SPACEDIM> const& bfhi) noexcept
260 {
261  Box gdomain = domain;
262  for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
263  if (! bflo[idim]) { gdomain.growLo(idim,1); }
264  if (! bfhi[idim]) { gdomain.growHi(idim,1); }
265  }
266 
267  if (gdomain.strictly_contains(vbx)) { return; }
268 
269  const int offset = domain.cellCentered() ? 0 : 1;
270 
271  const auto dlo = amrex::lbound(domain);
272  const auto dhi = amrex::ubound(domain);
273 
274  Box const& sbox = amrex::grow(vbx,1);
275  AMREX_HOST_DEVICE_FOR_3D(sbox, i, j, k,
276  {
277  if (! gdomain.contains(IntVect(i,j,k))) {
278  // xlo & ylo & zlo
279  if (i == dlo.x-1 && j == dlo.y-1 && k == dlo.z-1 && (bflo[0] || bflo[1] || bflo[2]))
280  {
281  if (bflo[0] && bflo[1] && bflo[2])
282  {
283  a(i,j,k) = a(i+1+offset, j+1+offset, k+1+offset);
284  }
285  else if (bflo[0] && bflo[1])
286  {
287  a(i,j,k) = a(i+1+offset, j+1+offset, k);
288  }
289  else if (bflo[0] && bflo[2])
290  {
291  a(i,j,k) = a(i+1+offset, j, k+1+offset);
292  }
293  else if (bflo[1] && bflo[2])
294  {
295  a(i,j,k) = a(i, j+1+offset, k+1+offset);
296  }
297  else if (bflo[0])
298  {
299  a(i,j,k) = a(i+1+offset, j, k);
300  }
301  else if (bflo[1])
302  {
303  a(i,j,k) = a(i, j+1+offset, k);
304  }
305  else if (bflo[2])
306  {
307  a(i,j,k) = a(i, j, k+1+offset);
308  }
309  }
310  // xhi & ylo & zlo
311  else if (i == dhi.x+1 && j == dlo.y-1 && k == dlo.z-1 && (bfhi[0] || bflo[1] || bflo[2]))
312  {
313  if (bfhi[0] && bflo[1] && bflo[2])
314  {
315  a(i,j,k) = a(i-1-offset, j+1+offset, k+1+offset);
316  }
317  else if (bfhi[0] && bflo[1])
318  {
319  a(i,j,k) = a(i-1-offset, j+1+offset, k);
320  }
321  else if (bfhi[0] && bflo[2])
322  {
323  a(i,j,k) = a(i-1-offset, j, k+1+offset);
324  }
325  else if (bflo[1] && bflo[2])
326  {
327  a(i,j,k) = a(i, j+1+offset, k+1+offset);
328  }
329  else if (bfhi[0])
330  {
331  a(i,j,k) = a(i-1-offset, j, k);
332  }
333  else if (bflo[1])
334  {
335  a(i,j,k) = a(i, j+1+offset, k);
336  }
337  else if (bflo[2])
338  {
339  a(i,j,k) = a(i, j, k+1+offset);
340  }
341  }
342  // xlo & yhi & zlo
343  else if (i == dlo.x-1 && j == dhi.y+1 && k == dlo.z-1 && (bflo[0] || bfhi[1] || bflo[2]))
344  {
345  if (bflo[0] && bfhi[1] && bflo[2])
346  {
347  a(i,j,k) = a(i+1+offset, j-1-offset, k+1+offset);
348  }
349  else if (bflo[0] && bfhi[1])
350  {
351  a(i,j,k) = a(i+1+offset, j-1-offset, k);
352  }
353  else if (bflo[0] && bflo[2])
354  {
355  a(i,j,k) = a(i+1+offset, j, k+1+offset);
356  }
357  else if (bfhi[1] && bflo[2])
358  {
359  a(i,j,k) = a(i, j-1-offset, k+1+offset);
360  }
361  else if (bflo[0])
362  {
363  a(i,j,k) = a(i+1+offset, j, k);
364  }
365  else if (bfhi[1])
366  {
367  a(i,j,k) = a(i, j-1-offset, k);
368  }
369  else if (bflo[2])
370  {
371  a(i,j,k) = a(i, j, k+1+offset);
372  }
373  }
374  // xhi & yhi & zlo
375  else if (i == dhi.x+1 && j == dhi.y+1 && k == dlo.z-1 && (bfhi[0] || bfhi[1] || bflo[2]))
376  {
377  if (bfhi[0] && bfhi[1] && bflo[2])
378  {
379  a(i,j,k) = a(i-1-offset, j-1-offset, k+1+offset);
380  }
381  else if (bfhi[0] && bfhi[1])
382  {
383  a(i,j,k) = a(i-1-offset, j-1-offset, k);
384  }
385  else if (bfhi[0] && bflo[2])
386  {
387  a(i,j,k) = a(i-1-offset, j, k+1+offset);
388  }
389  else if (bfhi[1] && bflo[2])
390  {
391  a(i,j,k) = a(i, j-1-offset, k+1+offset);
392  }
393  else if (bfhi[0])
394  {
395  a(i,j,k) = a(i-1-offset, j, k);
396  }
397  else if (bfhi[1])
398  {
399  a(i,j,k) = a(i, j-1-offset, k);
400  }
401  else if (bflo[2])
402  {
403  a(i,j,k) = a(i, j, k+1+offset);
404  }
405  }
406  // xlo & ylo & zhi
407  else if (i == dlo.x-1 && j == dlo.y-1 && k == dhi.z+1 && (bflo[0] || bflo[1] || bfhi[2]))
408  {
409  if (bflo[0] && bflo[1] && bfhi[2])
410  {
411  a(i,j,k) = a(i+1+offset, j+1+offset, k-1-offset);
412  }
413  else if (bflo[0] && bflo[1])
414  {
415  a(i,j,k) = a(i+1+offset, j+1+offset, k);
416  }
417  else if (bflo[0] && bfhi[2])
418  {
419  a(i,j,k) = a(i+1+offset, j, k-1-offset);
420  }
421  else if (bflo[1] && bfhi[2])
422  {
423  a(i,j,k) = a(i, j+1+offset, k-1-offset);
424  }
425  else if (bflo[0])
426  {
427  a(i,j,k) = a(i+1+offset, j, k);
428  }
429  else if (bflo[1])
430  {
431  a(i,j,k) = a(i, j+1+offset, k);
432  }
433  else if (bfhi[2])
434  {
435  a(i,j,k) = a(i, j, k-1-offset);
436  }
437  }
438  // xhi & ylo & zhi
439  else if (i == dhi.x+1 && j == dlo.y-1 && k == dhi.z+1 && (bfhi[0] || bflo[1] || bfhi[2]))
440  {
441  if (bfhi[0] && bflo[1] && bfhi[2])
442  {
443  a(i,j,k) = a(i-1-offset, j+1+offset, k-1-offset);
444  }
445  else if (bfhi[0] && bflo[1])
446  {
447  a(i,j,k) = a(i-1-offset, j+1+offset, k);
448  }
449  else if (bfhi[0] && bfhi[2])
450  {
451  a(i,j,k) = a(i-1-offset, j, k-1-offset);
452  }
453  else if (bflo[1] && bfhi[2])
454  {
455  a(i,j,k) = a(i, j+1+offset, k-1-offset);
456  }
457  else if (bfhi[0])
458  {
459  a(i,j,k) = a(i-1-offset, j, k);
460  }
461  else if (bflo[1])
462  {
463  a(i,j,k) = a(i, j+1+offset, k);
464  }
465  else if (bfhi[2])
466  {
467  a(i,j,k) = a(i, j, k-1-offset);
468  }
469  }
470  // xlo & yhi & zhi
471  else if (i == dlo.x-1 && j == dhi.y+1 && k == dhi.z+1 && (bflo[0] || bfhi[1] || bfhi[2]))
472  {
473  if (bflo[0] && bfhi[1] && bfhi[2])
474  {
475  a(i,j,k) = a(i+1+offset, j-1-offset, k-1-offset);
476  }
477  else if (bflo[0] && bfhi[1])
478  {
479  a(i,j,k) = a(i+1+offset, j-1-offset, k);
480  }
481  else if (bflo[0] && bfhi[2])
482  {
483  a(i,j,k) = a(i+1+offset, j, k-1-offset);
484  }
485  else if (bfhi[1] && bfhi[2])
486  {
487  a(i,j,k) = a(i, j-1-offset, k-1-offset);
488  }
489  else if (bflo[0])
490  {
491  a(i,j,k) = a(i+1+offset, j, k);
492  }
493  else if (bfhi[1])
494  {
495  a(i,j,k) = a(i, j-1-offset, k);
496  }
497  else if (bfhi[2])
498  {
499  a(i,j,k) = a(i, j, k-1-offset);
500  }
501  }
502  // xhi & yhi & zhi
503  else if (i == dhi.x+1 && j == dhi.y+1 && k == dhi.z+1 && (bfhi[0] || bfhi[1] || bfhi[2]))
504  {
505  if (bfhi[0] && bfhi[1] && bfhi[2])
506  {
507  a(i,j,k) = a(i-1-offset, j-1-offset, k-1-offset);
508  }
509  else if (bfhi[0] && bfhi[1])
510  {
511  a(i,j,k) = a(i-1-offset, j-1-offset, k);
512  }
513  else if (bfhi[0] && bfhi[2])
514  {
515  a(i,j,k) = a(i-1-offset, j, k-1-offset);
516  }
517  else if (bfhi[1] && bfhi[2])
518  {
519  a(i,j,k) = a(i, j-1-offset, k-1-offset);
520  }
521  else if (bfhi[0])
522  {
523  a(i,j,k) = a(i-1-offset, j, k);
524  }
525  else if (bfhi[1])
526  {
527  a(i,j,k) = a(i, j-1-offset, k);
528  }
529  else if (bfhi[2])
530  {
531  a(i,j,k) = a(i, j, k-1-offset);
532  }
533  }
534  // xlo & ylo
535  else if (i == dlo.x-1 && j == dlo.y-1 && (bflo[0] || bflo[1]))
536  {
537  if (bflo[0] && bflo[1])
538  {
539  a(i,j,k) = a(i+1+offset, j+1+offset, k);
540  }
541  else if (bflo[0])
542  {
543  a(i,j,k) = a(i+1+offset, j, k);
544  }
545  else if (bflo[1])
546  {
547  a(i,j,k) = a(i, j+1+offset, k);
548  }
549  }
550  // xhi & ylo
551  else if (i == dhi.x+1 && j == dlo.y-1 && (bfhi[0] || bflo[1]))
552  {
553  if (bfhi[0] && bflo[1])
554  {
555  a(i,j,k) = a(i-1-offset, j+1+offset, k);
556  }
557  else if (bfhi[0])
558  {
559  a(i,j,k) = a(i-1-offset, j, k);
560  }
561  else if (bflo[1])
562  {
563  a(i,j,k) = a(i, j+1+offset, k);
564  }
565  }
566  // xlo & yhi
567  else if (i == dlo.x-1 && j == dhi.y+1 && (bflo[0] || bfhi[1]))
568  {
569  if (bflo[0] && bfhi[1])
570  {
571  a(i,j,k) = a(i+1+offset, j-1-offset, k);
572  }
573  else if (bflo[0])
574  {
575  a(i,j,k) = a(i+1+offset, j, k);
576  }
577  else if (bfhi[1])
578  {
579  a(i,j,k) = a(i, j-1-offset, k);
580  }
581  }
582  // xhi & yhi
583  else if (i == dhi.x+1 && j == dhi.y+1 && (bfhi[0] || bfhi[1]))
584  {
585  if (bfhi[0] && bfhi[1])
586  {
587  a(i,j,k) = a(i-1-offset, j-1-offset, k);
588  }
589  else if (bfhi[0])
590  {
591  a(i,j,k) = a(i-1-offset, j, k);
592  }
593  else if (bfhi[1])
594  {
595  a(i,j,k) = a(i, j-1-offset, k);
596  }
597  }
598  // xlo & zlo
599  else if (i == dlo.x-1 && k == dlo.z-1 && (bflo[0] || bflo[2]))
600  {
601  if (bflo[0] && bflo[2])
602  {
603  a(i,j,k) = a(i+1+offset, j, k+1+offset);
604  }
605  else if (bflo[0])
606  {
607  a(i,j,k) = a(i+1+offset, j, k);
608  }
609  else if (bflo[2])
610  {
611  a(i,j,k) = a(i, j, k+1+offset);
612  }
613  }
614  // xhi & zlo
615  else if (i == dhi.x+1 && k == dlo.z-1 && (bfhi[0] || bflo[2]))
616  {
617  if (bfhi[0] && bflo[2])
618  {
619  a(i,j,k) = a(i-1-offset, j, k+1+offset);
620  }
621  else if (bfhi[0])
622  {
623  a(i,j,k) = a(i-1-offset, j, k);
624  }
625  else if (bflo[2])
626  {
627  a(i,j,k) = a(i, j, k+1+offset);
628  }
629  }
630  // xlo & zhi
631  else if (i == dlo.x-1 && k == dhi.z+1 && (bflo[0] || bfhi[2]))
632  {
633  if (bflo[0] && bfhi[2])
634  {
635  a(i,j,k) = a(i+1+offset, j, k-1-offset);
636  }
637  else if (bflo[0])
638  {
639  a(i,j,k) = a(i+1+offset, j, k);
640  }
641  else if (bfhi[2])
642  {
643  a(i,j,k) = a(i, j, k-1-offset);
644  }
645  }
646  // xhi & zhi
647  else if (i == dhi.x+1 && k == dhi.z+1 && (bfhi[0] || bfhi[2]))
648  {
649  if (bfhi[0] && bfhi[2])
650  {
651  a(i,j,k) = a(i-1-offset, j, k-1-offset);
652  }
653  else if (bfhi[0])
654  {
655  a(i,j,k) = a(i-1-offset, j, k);
656  }
657  else if (bfhi[2])
658  {
659  a(i,j,k) = a(i, j, k-1-offset);
660  }
661  }
662  // ylo & zlo
663  else if (j == dlo.y-1 && k == dlo.z-1 && (bflo[1] || bflo[2]))
664  {
665  if (bflo[1] && bflo[2])
666  {
667  a(i,j,k) = a(i, j+1+offset, k+1+offset);
668  }
669  else if (bflo[1])
670  {
671  a(i,j,k) = a(i, j+1+offset, k);
672  }
673  else if (bflo[2])
674  {
675  a(i,j,k) = a(i, j, k+1+offset);
676  }
677  }
678  // yhi & zlo
679  else if (j == dhi.y+1 && k == dlo.z-1 && (bfhi[1] || bflo[2]))
680  {
681  if (bfhi[1] && bflo[2])
682  {
683  a(i,j,k) = a(i, j-1-offset, k+1+offset);
684  }
685  else if (bfhi[1])
686  {
687  a(i,j,k) = a(i, j-1-offset, k);
688  }
689  else if (bflo[2])
690  {
691  a(i,j,k) = a(i, j, k+1+offset);
692  }
693  }
694  // ylo & zhi
695  else if (j == dlo.y-1 && k == dhi.z+1 && (bflo[1] || bfhi[2]))
696  {
697  if (bflo[1] && bfhi[2])
698  {
699  a(i,j,k) = a(i, j+1+offset, k-1-offset);
700  }
701  else if (bflo[1])
702  {
703  a(i,j,k) = a(i, j+1+offset, k);
704  }
705  else if (bfhi[2])
706  {
707  a(i,j,k) = a(i, j, k-1-offset);
708  }
709  }
710  // yhi & zhi
711  else if (j == dhi.y+1 && k == dhi.z+1 && (bfhi[1] || bfhi[2]))
712  {
713  if (bfhi[1] && bfhi[2])
714  {
715  a(i,j,k) = a(i, j-1-offset, k-1-offset);
716  }
717  else if (bfhi[1])
718  {
719  a(i,j,k) = a(i, j-1-offset, k);
720  }
721  else if (bfhi[2])
722  {
723  a(i,j,k) = a(i, j, k-1-offset);
724  }
725  }
726  else if (i == dlo.x-1 && bflo[0])
727  {
728  a(i,j,k) = a(i+1+offset, j, k);
729  }
730  else if (i == dhi.x+1 && bfhi[0])
731  {
732  a(i,j,k) = a(i-1-offset, j, k);
733  }
734  else if (j == dlo.y-1 && bflo[1])
735  {
736  a(i,j,k) = a(i, j+1+offset, k);
737  }
738  else if (j == dhi.y+1 && bfhi[1])
739  {
740  a(i,j,k) = a(i, j-1-offset, k);
741  }
742  else if (k == dlo.z-1 && bflo[2])
743  {
744  a(i,j,k) = a(i, j, k+1+offset);
745  }
746  else if (k == dhi.z+1 && bfhi[2])
747  {
748  a(i,j,k) = a(i, j, k-1-offset);
749  }
750  }
751  });
752 }
753 
754 //
755 // operator
756 //
757 
758 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
759 Real mlndlap_adotx_ha (int i, int j, int k, Array4<Real const> const& x,
760  Array4<Real const> const& sx, Array4<Real const> const& sy,
761  Array4<Real const> const& sz, Array4<int const> const& msk,
762  GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
763 {
764  if (msk(i,j,k)) {
765  return Real(0.0);
766  } else {
767  Real facx = Real(1./36.)*dxinv[0]*dxinv[0];
768  Real facy = Real(1./36.)*dxinv[1]*dxinv[1];
769  Real facz = Real(1./36.)*dxinv[2]*dxinv[2];
770  Real y = x(i,j,k)*Real(-4.0)*(facx*(sx(i-1,j-1,k-1)+sx(i,j-1,k-1)+sx(i-1,j,k-1)+sx(i,j,k-1)
771  +sx(i-1,j-1,k )+sx(i,j-1,k )+sx(i-1,j,k )+sx(i,j,k ))
772  +facy*(sy(i-1,j-1,k-1)+sy(i,j-1,k-1)+sy(i-1,j,k-1)+sy(i,j,k-1)
773  +sy(i-1,j-1,k )+sy(i,j-1,k )+sy(i-1,j,k )+sy(i,j,k ))
774  +facz*(sz(i-1,j-1,k-1)+sz(i,j-1,k-1)+sz(i-1,j,k-1)+sz(i,j,k-1)
775  +sz(i-1,j-1,k )+sz(i,j-1,k )+sz(i-1,j,k )+sz(i,j,k )));
776  y += x(i-1,j-1,k-1)*(facx*sx(i-1,j-1,k-1)
777  +facy*sy(i-1,j-1,k-1)
778  +facz*sz(i-1,j-1,k-1))
779  + x(i+1,j-1,k-1)*(facx*sx(i ,j-1,k-1)
780  +facy*sy(i ,j-1,k-1)
781  +facz*sz(i ,j-1,k-1))
782  + x(i-1,j+1,k-1)*(facx*sx(i-1,j ,k-1)
783  +facy*sy(i-1,j ,k-1)
784  +facz*sz(i-1,j ,k-1))
785  + x(i+1,j+1,k-1)*(facx*sx(i ,j ,k-1)
786  +facy*sy(i ,j ,k-1)
787  +facz*sz(i ,j ,k-1))
788  + x(i-1,j-1,k+1)*(facx*sx(i-1,j-1,k )
789  +facy*sy(i-1,j-1,k )
790  +facz*sz(i-1,j-1,k ))
791  + x(i+1,j-1,k+1)*(facx*sx(i ,j-1,k )
792  +facy*sy(i ,j-1,k )
793  +facz*sz(i ,j-1,k ))
794  + x(i-1,j+1,k+1)*(facx*sx(i-1,j ,k )
795  +facy*sy(i-1,j ,k )
796  +facz*sz(i-1,j ,k ))
797  + x(i+1,j+1,k+1)*(facx*sx(i ,j ,k )
798  +facy*sy(i ,j ,k )
799  +facz*sz(i ,j ,k ));
800  y += x(i ,j-1,k-1)*( -facx*(sx(i-1,j-1,k-1)+sx(i,j-1,k-1))
801  +Real(2.0)*facy*(sy(i-1,j-1,k-1)+sy(i,j-1,k-1))
802  +Real(2.0)*facz*(sz(i-1,j-1,k-1)+sz(i,j-1,k-1)))
803  + x(i ,j+1,k-1)*( -facx*(sx(i-1,j ,k-1)+sx(i,j ,k-1))
804  +Real(2.0)*facy*(sy(i-1,j ,k-1)+sy(i,j ,k-1))
805  +Real(2.0)*facz*(sz(i-1,j ,k-1)+sz(i,j ,k-1)))
806  + x(i ,j-1,k+1)*( -facx*(sx(i-1,j-1,k )+sx(i,j-1,k ))
807  +Real(2.0)*facy*(sy(i-1,j-1,k )+sy(i,j-1,k ))
808  +Real(2.0)*facz*(sz(i-1,j-1,k )+sz(i,j-1,k )))
809  + x(i ,j+1,k+1)*( -facx*(sx(i-1,j ,k )+sx(i,j ,k ))
810  +Real(2.0)*facy*(sy(i-1,j ,k )+sy(i,j ,k ))
811  +Real(2.0)*facz*(sz(i-1,j ,k )+sz(i,j ,k )))
812  + x(i-1,j ,k-1)*( Real(2.0)*facx*(sx(i-1,j-1,k-1)+sx(i-1,j,k-1))
813  -facy*(sy(i-1,j-1,k-1)+sy(i-1,j,k-1))
814  +Real(2.0)*facz*(sz(i-1,j-1,k-1)+sz(i-1,j,k-1)))
815  + x(i+1,j ,k-1)*( Real(2.0)*facx*(sx(i ,j-1,k-1)+sx(i ,j,k-1))
816  -facy*(sy(i ,j-1,k-1)+sy(i ,j,k-1))
817  +Real(2.0)*facz*(sz(i ,j-1,k-1)+sz(i ,j,k-1)))
818  + x(i-1,j ,k+1)*( Real(2.0)*facx*(sx(i-1,j-1,k )+sx(i-1,j,k ))
819  -facy*(sy(i-1,j-1,k )+sy(i-1,j,k ))
820  +Real(2.0)*facz*(sz(i-1,j-1,k )+sz(i-1,j,k )))
821  + x(i+1,j ,k+1)*( Real(2.0)*facx*(sx(i ,j-1,k )+sx(i ,j,k ))
822  -facy*(sy(i ,j-1,k )+sy(i ,j,k ))
823  +Real(2.0)*facz*(sz(i ,j-1,k )+sz(i ,j,k )))
824  + x(i-1,j-1,k )*( Real(2.0)*facx*(sx(i-1,j-1,k-1)+sx(i-1,j-1,k))
825  +Real(2.0)*facy*(sy(i-1,j-1,k-1)+sy(i-1,j-1,k))
826  -facz*(sz(i-1,j-1,k-1)+sz(i-1,j-1,k)))
827  + x(i+1,j-1,k )*( Real(2.0)*facx*(sx(i ,j-1,k-1)+sx(i ,j-1,k))
828  +Real(2.0)*facy*(sy(i ,j-1,k-1)+sy(i ,j-1,k))
829  -facz*(sz(i ,j-1,k-1)+sz(i ,j-1,k)))
830  + x(i-1,j+1,k )*( Real(2.0)*facx*(sx(i-1,j ,k-1)+sx(i-1,j ,k))
831  +Real(2.0)*facy*(sy(i-1,j ,k-1)+sy(i-1,j ,k))
832  -facz*(sz(i-1,j ,k-1)+sz(i-1,j ,k)))
833  + x(i+1,j+1,k )*( Real(2.0)*facx*(sx(i ,j ,k-1)+sx(i ,j ,k))
834  +Real(2.0)*facy*(sy(i ,j ,k-1)+sy(i ,j ,k))
835  -facz*(sz(i ,j ,k-1)+sz(i ,j ,k)));
836  y += Real(2.0)*x(i-1,j,k)*( Real(2.0)*facx*(sx(i-1,j-1,k-1)+sx(i-1,j,k-1)+sx(i-1,j-1,k)+sx(i-1,j,k))
837  -facy*(sy(i-1,j-1,k-1)+sy(i-1,j,k-1)+sy(i-1,j-1,k)+sy(i-1,j,k))
838  -facz*(sz(i-1,j-1,k-1)+sz(i-1,j,k-1)+sz(i-1,j-1,k)+sz(i-1,j,k)))
839  + Real(2.0)*x(i+1,j,k)*( Real(2.0)*facx*(sx(i ,j-1,k-1)+sx(i ,j,k-1)+sx(i ,j-1,k)+sx(i ,j,k))
840  -facy*(sy(i ,j-1,k-1)+sy(i ,j,k-1)+sy(i ,j-1,k)+sy(i ,j,k))
841  -facz*(sz(i ,j-1,k-1)+sz(i ,j,k-1)+sz(i ,j-1,k)+sz(i ,j,k)))
842  + Real(2.0)*x(i,j-1,k)*( -facx*(sx(i-1,j-1,k-1)+sx(i,j-1,k-1)+sx(i-1,j-1,k)+sx(i,j-1,k))
843  +Real(2.0)*facy*(sy(i-1,j-1,k-1)+sy(i,j-1,k-1)+sy(i-1,j-1,k)+sy(i,j-1,k))
844  -facz*(sz(i-1,j-1,k-1)+sz(i,j-1,k-1)+sz(i-1,j-1,k)+sz(i,j-1,k)))
845  + Real(2.0)*x(i,j+1,k)*( -facx*(sx(i-1,j ,k-1)+sx(i,j ,k-1)+sx(i-1,j ,k)+sx(i,j ,k))
846  +Real(2.0)*facy*(sy(i-1,j ,k-1)+sy(i,j ,k-1)+sy(i-1,j ,k)+sy(i,j ,k))
847  -facz*(sz(i-1,j ,k-1)+sz(i,j ,k-1)+sz(i-1,j ,k)+sz(i,j ,k)))
848  + Real(2.0)*x(i,j,k-1)*( -facx*(sx(i-1,j-1,k-1)+sx(i,j-1,k-1)+sx(i-1,j,k-1)+sx(i,j,k-1))
849  -facy*(sy(i-1,j-1,k-1)+sy(i,j-1,k-1)+sy(i-1,j,k-1)+sy(i,j,k-1))
850  +Real(2.0)*facz*(sz(i-1,j-1,k-1)+sz(i,j-1,k-1)+sz(i-1,j,k-1)+sz(i,j,k-1)))
851  + Real(2.0)*x(i,j,k+1)*( -facx*(sx(i-1,j-1,k )+sx(i,j-1,k )+sx(i-1,j,k )+sx(i,j,k ))
852  -facy*(sy(i-1,j-1,k )+sy(i,j-1,k )+sy(i-1,j,k )+sy(i,j,k ))
853  +Real(2.0)*facz*(sz(i-1,j-1,k )+sz(i,j-1,k )+sz(i-1,j,k )+sz(i,j,k )));
854  return y;
855  }
856 }
857 
858 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
859 Real mlndlap_adotx_aa (int i, int j, int k, Array4<Real const> const& x,
860  Array4<Real const> const& sig, Array4<int const> const& msk,
861  GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
862 {
863  if (msk(i,j,k)) {
864  return Real(0.0);
865  } else {
866  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0];
867  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1];
868  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2];
869  Real fxyz = facx + facy + facz;
870  Real fmx2y2z = -facx + Real(2.0)*facy + Real(2.0)*facz;
871  Real f2xmy2z = Real(2.0)*facx - facy + Real(2.0)*facz;
872  Real f2x2ymz = Real(2.0)*facx + Real(2.0)*facy - facz;
873  Real f4xm2ym2z = Real(4.0)*facx - Real(2.0)*facy - Real(2.0)*facz;
874  Real fm2x4ym2z = -Real(2.0)*facx + Real(4.0)*facy - Real(2.0)*facz;
875  Real fm2xm2y4z = -Real(2.0)*facx - Real(2.0)*facy + Real(4.0)*facz;
876  return x(i,j,k)*Real(-4.0)*fxyz*
877  (sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j,k-1)+sig(i,j,k-1)
878  +sig(i-1,j-1,k )+sig(i,j-1,k )+sig(i-1,j,k )+sig(i,j,k ))
879  + fxyz*(x(i-1,j-1,k-1)*sig(i-1,j-1,k-1)
880  + x(i+1,j-1,k-1)*sig(i ,j-1,k-1)
881  + x(i-1,j+1,k-1)*sig(i-1,j ,k-1)
882  + x(i+1,j+1,k-1)*sig(i ,j ,k-1)
883  + x(i-1,j-1,k+1)*sig(i-1,j-1,k )
884  + x(i+1,j-1,k+1)*sig(i ,j-1,k )
885  + x(i-1,j+1,k+1)*sig(i-1,j ,k )
886  + x(i+1,j+1,k+1)*sig(i ,j ,k ))
887  + fmx2y2z*(x(i ,j-1,k-1)*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1))
888  + x(i ,j+1,k-1)*(sig(i-1,j ,k-1)+sig(i,j ,k-1))
889  + x(i ,j-1,k+1)*(sig(i-1,j-1,k )+sig(i,j-1,k ))
890  + x(i ,j+1,k+1)*(sig(i-1,j ,k )+sig(i,j ,k )))
891  + f2xmy2z*(x(i-1,j ,k-1)*(sig(i-1,j-1,k-1)+sig(i-1,j,k-1))
892  + x(i+1,j ,k-1)*(sig(i ,j-1,k-1)+sig(i ,j,k-1))
893  + x(i-1,j ,k+1)*(sig(i-1,j-1,k )+sig(i-1,j,k ))
894  + x(i+1,j ,k+1)*(sig(i ,j-1,k )+sig(i ,j,k )))
895  + f2x2ymz*(x(i-1,j-1,k )*(sig(i-1,j-1,k-1)+sig(i-1,j-1,k))
896  + x(i+1,j-1,k )*(sig(i ,j-1,k-1)+sig(i ,j-1,k))
897  + x(i-1,j+1,k )*(sig(i-1,j ,k-1)+sig(i-1,j ,k))
898  + x(i+1,j+1,k )*(sig(i ,j ,k-1)+sig(i ,j ,k)))
899  + f4xm2ym2z*(x(i-1,j,k)*(sig(i-1,j-1,k-1)+sig(i-1,j,k-1)+sig(i-1,j-1,k)+sig(i-1,j,k))
900  + x(i+1,j,k)*(sig(i ,j-1,k-1)+sig(i ,j,k-1)+sig(i ,j-1,k)+sig(i ,j,k)))
901  + fm2x4ym2z*(x(i,j-1,k)*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j-1,k)+sig(i,j-1,k))
902  + x(i,j+1,k)*(sig(i-1,j ,k-1)+sig(i,j ,k-1)+sig(i-1,j ,k)+sig(i,j ,k)))
903  + fm2xm2y4z*(x(i,j,k-1)*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j,k-1)+sig(i,j,k-1))
904  + x(i,j,k+1)*(sig(i-1,j-1,k )+sig(i,j-1,k )+sig(i-1,j,k )+sig(i,j,k )));
905  }
906 }
907 
908 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
909 Real mlndlap_adotx_c (int i, int j, int k, Array4<Real const> const& x,
910  Real sig, Array4<int const> const& msk,
911  GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
912 {
913  if (msk(i,j,k)) {
914  return Real(0.0);
915  } else {
916  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0];
917  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1];
918  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2];
919  Real fxyz = facx + facy + facz;
920  Real fmx2y2z = -facx + Real(2.0)*facy + Real(2.0)*facz;
921  Real f2xmy2z = Real(2.0)*facx - facy + Real(2.0)*facz;
922  Real f2x2ymz = Real(2.0)*facx + Real(2.0)*facy - facz;
923  Real f4xm2ym2z = Real(4.0)*facx - Real(2.0)*facy - Real(2.0)*facz;
924  Real fm2x4ym2z = -Real(2.0)*facx + Real(4.0)*facy - Real(2.0)*facz;
925  Real fm2xm2y4z = -Real(2.0)*facx - Real(2.0)*facy + Real(4.0)*facz;
926  return sig * (x(i,j,k)*Real(-4.0)*fxyz*Real(8.)
927  + fxyz*(x(i-1,j-1,k-1)
928  + x(i+1,j-1,k-1)
929  + x(i-1,j+1,k-1)
930  + x(i+1,j+1,k-1)
931  + x(i-1,j-1,k+1)
932  + x(i+1,j-1,k+1)
933  + x(i-1,j+1,k+1)
934  + x(i+1,j+1,k+1))
935  + fmx2y2z*(x(i ,j-1,k-1)*Real(2.)
936  + x(i ,j+1,k-1)*Real(2.)
937  + x(i ,j-1,k+1)*Real(2.)
938  + x(i ,j+1,k+1)*Real(2.))
939  + f2xmy2z*(x(i-1,j ,k-1)*Real(2.)
940  + x(i+1,j ,k-1)*Real(2.)
941  + x(i-1,j ,k+1)*Real(2.)
942  + x(i+1,j ,k+1)*Real(2.))
943  + f2x2ymz*(x(i-1,j-1,k )*Real(2.)
944  + x(i+1,j-1,k )*Real(2.)
945  + x(i-1,j+1,k )*Real(2.)
946  + x(i+1,j+1,k )*Real(2.))
947  + f4xm2ym2z*(x(i-1,j,k)*Real(4.)
948  + x(i+1,j,k)*Real(4.))
949  + fm2x4ym2z*(x(i,j-1,k)*Real(4.)
950  + x(i,j+1,k)*Real(4.))
951  + fm2xm2y4z*(x(i,j,k-1)*Real(4.)
952  + x(i,j,k+1)*Real(4.)));
953  }
954 }
955 
956 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
957 void mlndlap_normalize_ha (int i, int j, int k, Array4<Real> const& x, Array4<Real const> const& sx,
958  Array4<Real const> const& sy, Array4<Real const> const& sz,
959  Array4<int const> const& msk, GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
960 {
961  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0];
962  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1];
963  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2];
964 
965  if (!msk(i,j,k)) {
966  x(i,j,k) = x(i,j,k)/(Real(-4.0)*(facx*(sx(i-1,j-1,k-1)+sx(i,j-1,k-1)+sx(i-1,j,k-1)+sx(i,j,k-1)
967  +sx(i-1,j-1,k )+sx(i,j-1,k )+sx(i-1,j,k )+sx(i,j,k ))
968  +facy*(sy(i-1,j-1,k-1)+sy(i,j-1,k-1)+sy(i-1,j,k-1)+sy(i,j,k-1)
969  +sy(i-1,j-1,k )+sy(i,j-1,k )+sy(i-1,j,k )+sy(i,j,k ))
970  +facz*(sz(i-1,j-1,k-1)+sz(i,j-1,k-1)+sz(i-1,j,k-1)+sz(i,j,k-1)
971  +sz(i-1,j-1,k )+sz(i,j-1,k )+sz(i-1,j,k )+sz(i,j,k ))));
972  }
973 }
974 
975 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
976 void mlndlap_normalize_aa (int i, int j, int k, Array4<Real> const& x, Array4<Real const> const& sig,
977  Array4<int const> const& msk, GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
978 {
979  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0];
980  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1];
981  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2];
982  Real fxyz = facx + facy + facz;
983 
984  if (!msk(i,j,k)) {
985  x(i,j,k) = x(i,j,k) /
986  (Real(-4.0)*fxyz*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j,k-1)+sig(i,j,k-1)
987  +sig(i-1,j-1,k )+sig(i,j-1,k )+sig(i-1,j,k )+sig(i,j,k )));
988  }
989 }
990 
991 AMREX_GPU_DEVICE AMREX_FORCE_INLINE
992 void mlndlap_jacobi_ha (int i, int j, int k, Array4<Real> const& sol, Real Ax,
993  Array4<Real const> const& rhs, Array4<Real const> const& sx,
994  Array4<Real const> const& sy, Array4<Real const> const& sz,
995  Array4<int const> const& msk, GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
996 {
997  Real facx = Real(-4.0 / 36.0)*dxinv[0]*dxinv[0];
998  Real facy = Real(-4.0 / 36.0)*dxinv[1]*dxinv[1];
999  Real facz = Real(-4.0 / 36.0)*dxinv[2]*dxinv[2];
1000 
1001  if (msk(i,j,k)) {
1002  sol(i,j,k) = Real(0.0);
1003  } else {
1004  sol(i,j,k) += Real(2.0/3.0) * (rhs(i,j,k) - Ax)
1005  / (facx*(sx(i-1,j-1,k-1)+sx(i,j-1,k-1)+sx(i-1,j,k-1)+sx(i,j,k-1)
1006  +sx(i-1,j-1,k )+sx(i,j-1,k )+sx(i-1,j,k )+sx(i,j,k ))
1007  +facy*(sy(i-1,j-1,k-1)+sy(i,j-1,k-1)+sy(i-1,j,k-1)+sy(i,j,k-1)
1008  +sy(i-1,j-1,k )+sy(i,j-1,k )+sy(i-1,j,k )+sy(i,j,k ))
1009  +facz*(sz(i-1,j-1,k-1)+sz(i,j-1,k-1)+sz(i-1,j,k-1)+sz(i,j,k-1)
1010  +sz(i-1,j-1,k )+sz(i,j-1,k )+sz(i-1,j,k )+sz(i,j,k )));
1011  }
1012 }
1013 
1014 inline
1015 void mlndlap_jacobi_ha (Box const& bx, Array4<Real> const& sol, Array4<Real const> const& Ax,
1016  Array4<Real const> const& rhs, Array4<Real const> const& sx,
1017  Array4<Real const> const& sy, Array4<Real const> const& sz,
1018  Array4<int const> const& msk, GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
1019 {
1020  Real facx = Real(-4.0 / 36.0)*dxinv[0]*dxinv[0];
1021  Real facy = Real(-4.0 / 36.0)*dxinv[1]*dxinv[1];
1022  Real facz = Real(-4.0 / 36.0)*dxinv[2]*dxinv[2];
1023 
1024  amrex::LoopConcurrentOnCpu(bx, [&] (int i, int j, int k) noexcept
1025  {
1026  if (msk(i,j,k)) {
1027  sol(i,j,k) = Real(0.0);
1028  } else {
1029  sol(i,j,k) += Real(2.0/3.0) * (rhs(i,j,k) - Ax(i,j,k))
1030  / (facx*(sx(i-1,j-1,k-1)+sx(i,j-1,k-1)+sx(i-1,j,k-1)+sx(i,j,k-1)
1031  +sx(i-1,j-1,k )+sx(i,j-1,k )+sx(i-1,j,k )+sx(i,j,k ))
1032  +facy*(sy(i-1,j-1,k-1)+sy(i,j-1,k-1)+sy(i-1,j,k-1)+sy(i,j,k-1)
1033  +sy(i-1,j-1,k )+sy(i,j-1,k )+sy(i-1,j,k )+sy(i,j,k ))
1034  +facz*(sz(i-1,j-1,k-1)+sz(i,j-1,k-1)+sz(i-1,j,k-1)+sz(i,j,k-1)
1035  +sz(i-1,j-1,k )+sz(i,j-1,k )+sz(i-1,j,k )+sz(i,j,k )));
1036  }
1037  });
1038 }
1039 
1040 AMREX_GPU_DEVICE AMREX_FORCE_INLINE
1041 void mlndlap_jacobi_aa (int i, int j, int k, Array4<Real> const& sol, Real Ax,
1042  Array4<Real const> const& rhs, Array4<Real const> const& sig,
1043  Array4<int const> const& msk, GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
1044 {
1045  Real fxyz = Real(-4.0 / 36.0)*(dxinv[0]*dxinv[0] +
1046  dxinv[1]*dxinv[1] +
1047  dxinv[2]*dxinv[2]);
1048 
1049  if (msk(i,j,k)) {
1050  sol(i,j,k) = Real(0.0);
1051  } else {
1052  sol(i,j,k) += Real(2.0/3.0) * (rhs(i,j,k) - Ax)
1053  / (fxyz*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j,k-1)+sig(i,j,k-1)
1054  +sig(i-1,j-1,k )+sig(i,j-1,k )+sig(i-1,j,k )+sig(i,j,k )));
1055  }
1056 }
1057 
1058 AMREX_GPU_DEVICE AMREX_FORCE_INLINE
1059 void mlndlap_jacobi_c (int i, int j, int k, Array4<Real> const& sol, Real Ax,
1060  Array4<Real const> const& rhs, Real sig,
1061  Array4<int const> const& msk, GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
1062 {
1063  Real fxyz = Real(-4.0 / 36.0)*(dxinv[0]*dxinv[0] +
1064  dxinv[1]*dxinv[1] +
1065  dxinv[2]*dxinv[2]);
1066 
1067  if (msk(i,j,k)) {
1068  sol(i,j,k) = Real(0.0);
1069  } else {
1070  sol(i,j,k) += Real(2.0/3.0) * (rhs(i,j,k) - Ax)
1071  / (fxyz*Real(8.)*sig);
1072  }
1073 }
1074 
1075 inline
1076 void mlndlap_jacobi_aa (Box const& bx, Array4<Real> const& sol, Array4<Real const> const& Ax,
1077  Array4<Real const> const& rhs, Array4<Real const> const& sig,
1078  Array4<int const> const& msk, GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
1079 {
1080  Real fxyz = Real(-4.0 / 36.0)*(dxinv[0]*dxinv[0] +
1081  dxinv[1]*dxinv[1] +
1082  dxinv[2]*dxinv[2]);
1083 
1084  amrex::LoopConcurrentOnCpu(bx, [&] (int i, int j, int k) noexcept
1085  {
1086  if (msk(i,j,k)) {
1087  sol(i,j,k) = Real(0.0);
1088  } else {
1089  sol(i,j,k) += Real(2.0/3.0) * (rhs(i,j,k) - Ax(i,j,k))
1090  / (fxyz*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j,k-1)+sig(i,j,k-1)
1091  +sig(i-1,j-1,k )+sig(i,j-1,k )+sig(i-1,j,k )+sig(i,j,k )));
1092  }
1093  });
1094 }
1095 
1096 inline
1097 void mlndlap_jacobi_c (Box const& bx, Array4<Real> const& sol, Array4<Real const> const& Ax,
1098  Array4<Real const> const& rhs, Real sig,
1099  Array4<int const> const& msk, GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
1100 {
1101  Real fxyz = Real(-4.0 / 36.0)*(dxinv[0]*dxinv[0] +
1102  dxinv[1]*dxinv[1] +
1103  dxinv[2]*dxinv[2]);
1104 
1105  amrex::LoopConcurrentOnCpu(bx, [&] (int i, int j, int k) noexcept
1106  {
1107  if (msk(i,j,k)) {
1108  sol(i,j,k) = Real(0.0);
1109  } else {
1110  sol(i,j,k) += Real(2.0/3.0) * (rhs(i,j,k) - Ax(i,j,k))
1111  / (fxyz*Real(8.)*sig);
1112  }
1113  });
1114 }
1115 
1116 inline
1117 void mlndlap_gauss_seidel_ha (Box const& bx, Array4<Real> const& sol,
1118  Array4<Real const> const& rhs, Array4<Real const> const& sx,
1119  Array4<Real const> const& sy, Array4<Real const> const& sz,
1120  Array4<int const> const& msk,
1121  GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
1122 {
1123  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0];
1124  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1];
1125  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2];
1126 
1127  amrex::LoopOnCpu(bx, [&] (int i, int j, int k) noexcept
1128  {
1129  if (msk(i,j,k)) {
1130  sol(i,j,k) = Real(0.0);
1131  } else {
1132  Real s0 = Real(-4.0)*(facx*(sx(i-1,j-1,k-1)+sx(i,j-1,k-1)+sx(i-1,j,k-1)+sx(i,j,k-1)
1133  +sx(i-1,j-1,k )+sx(i,j-1,k )+sx(i-1,j,k )+sx(i,j,k ))
1134  +facy*(sy(i-1,j-1,k-1)+sy(i,j-1,k-1)+sy(i-1,j,k-1)+sy(i,j,k-1)
1135  +sy(i-1,j-1,k )+sy(i,j-1,k )+sy(i-1,j,k )+sy(i,j,k ))
1136  +facz*(sz(i-1,j-1,k-1)+sz(i,j-1,k-1)+sz(i-1,j,k-1)+sz(i,j,k-1)
1137  +sz(i-1,j-1,k )+sz(i,j-1,k )+sz(i-1,j,k )+sz(i,j,k )));
1138  Real Ax = sol(i,j,k)*s0
1139  + sol(i-1,j-1,k-1)*(facx*sx(i-1,j-1,k-1)
1140  +facy*sy(i-1,j-1,k-1)
1141  +facz*sz(i-1,j-1,k-1))
1142  + sol(i+1,j-1,k-1)*(facx*sx(i ,j-1,k-1)
1143  +facy*sy(i ,j-1,k-1)
1144  +facz*sz(i ,j-1,k-1))
1145  + sol(i-1,j+1,k-1)*(facx*sx(i-1,j ,k-1)
1146  +facy*sy(i-1,j ,k-1)
1147  +facz*sz(i-1,j ,k-1))
1148  + sol(i+1,j+1,k-1)*(facx*sx(i ,j ,k-1)
1149  +facy*sy(i ,j ,k-1)
1150  +facz*sz(i ,j ,k-1))
1151  + sol(i-1,j-1,k+1)*(facx*sx(i-1,j-1,k )
1152  +facy*sy(i-1,j-1,k )
1153  +facz*sz(i-1,j-1,k ))
1154  + sol(i+1,j-1,k+1)*(facx*sx(i ,j-1,k )
1155  +facy*sy(i ,j-1,k )
1156  +facz*sz(i ,j-1,k ))
1157  + sol(i-1,j+1,k+1)*(facx*sx(i-1,j ,k )
1158  +facy*sy(i-1,j ,k )
1159  +facz*sz(i-1,j ,k ))
1160  + sol(i+1,j+1,k+1)*(facx*sx(i ,j ,k )
1161  +facy*sy(i ,j ,k )
1162  +facz*sz(i ,j ,k ))
1163  +sol(i ,j-1,k-1)*( -facx*(sx(i-1,j-1,k-1)+sx(i,j-1,k-1))
1164  +Real(2.0)*facy*(sy(i-1,j-1,k-1)+sy(i,j-1,k-1))
1165  +Real(2.0)*facz*(sz(i-1,j-1,k-1)+sz(i,j-1,k-1)))
1166  +sol(i ,j+1,k-1)*( -facx*(sx(i-1,j ,k-1)+sx(i,j ,k-1))
1167  +Real(2.0)*facy*(sy(i-1,j ,k-1)+sy(i,j ,k-1))
1168  +Real(2.0)*facz*(sz(i-1,j ,k-1)+sz(i,j ,k-1)))
1169  +sol(i ,j-1,k+1)*( -facx*(sx(i-1,j-1,k )+sx(i,j-1,k ))
1170  +Real(2.0)*facy*(sy(i-1,j-1,k )+sy(i,j-1,k ))
1171  +Real(2.0)*facz*(sz(i-1,j-1,k )+sz(i,j-1,k )))
1172  +sol(i ,j+1,k+1)*( -facx*(sx(i-1,j ,k )+sx(i,j ,k ))
1173  +Real(2.0)*facy*(sy(i-1,j ,k )+sy(i,j ,k ))
1174  +Real(2.0)*facz*(sz(i-1,j ,k )+sz(i,j ,k )))
1175  +sol(i-1,j ,k-1)*( Real(2.0)*facx*(sx(i-1,j-1,k-1)+sx(i-1,j,k-1))
1176  -facy*(sy(i-1,j-1,k-1)+sy(i-1,j,k-1))
1177  +Real(2.0)*facz*(sz(i-1,j-1,k-1)+sz(i-1,j,k-1)))
1178  +sol(i+1,j ,k-1)*( Real(2.0)*facx*(sx(i ,j-1,k-1)+sx(i ,j,k-1))
1179  -facy*(sy(i ,j-1,k-1)+sy(i ,j,k-1))
1180  +Real(2.0)*facz*(sz(i ,j-1,k-1)+sz(i ,j,k-1)))
1181  +sol(i-1,j ,k+1)*( Real(2.0)*facx*(sx(i-1,j-1,k )+sx(i-1,j,k ))
1182  -facy*(sy(i-1,j-1,k )+sy(i-1,j,k ))
1183  +Real(2.0)*facz*(sz(i-1,j-1,k )+sz(i-1,j,k )))
1184  +sol(i+1,j ,k+1)*( Real(2.0)*facx*(sx(i ,j-1,k )+sx(i ,j,k ))
1185  -facy*(sy(i ,j-1,k )+sy(i ,j,k ))
1186  +Real(2.0)*facz*(sz(i ,j-1,k )+sz(i ,j,k )))
1187  +sol(i-1,j-1,k )*( Real(2.0)*facx*(sx(i-1,j-1,k-1)+sx(i-1,j-1,k))
1188  +Real(2.0)*facy*(sy(i-1,j-1,k-1)+sy(i-1,j-1,k))
1189  -facz*(sz(i-1,j-1,k-1)+sz(i-1,j-1,k)))
1190  +sol(i+1,j-1,k )*( Real(2.0)*facx*(sx(i ,j-1,k-1)+sx(i ,j-1,k))
1191  +Real(2.0)*facy*(sy(i ,j-1,k-1)+sy(i ,j-1,k))
1192  -facz*(sz(i ,j-1,k-1)+sz(i ,j-1,k)))
1193  +sol(i-1,j+1,k )*( Real(2.0)*facx*(sx(i-1,j ,k-1)+sx(i-1,j ,k))
1194  +Real(2.0)*facy*(sy(i-1,j ,k-1)+sy(i-1,j ,k))
1195  -facz*(sz(i-1,j ,k-1)+sz(i-1,j ,k)))
1196  +sol(i+1,j+1,k )*( Real(2.0)*facx*(sx(i ,j ,k-1)+sx(i ,j ,k))
1197  +Real(2.0)*facy*(sy(i ,j ,k-1)+sy(i ,j ,k))
1198  -facz*(sz(i ,j ,k-1)+sz(i ,j ,k)))
1199  + Real(2.0)*sol(i-1,j,k)*(Real(2.0)*facx*(sx(i-1,j-1,k-1)+sx(i-1,j,k-1)+sx(i-1,j-1,k)+sx(i-1,j,k))
1200  -facy*(sy(i-1,j-1,k-1)+sy(i-1,j,k-1)+sy(i-1,j-1,k)+sy(i-1,j,k))
1201  -facz*(sz(i-1,j-1,k-1)+sz(i-1,j,k-1)+sz(i-1,j-1,k)+sz(i-1,j,k)))
1202  + Real(2.0)*sol(i+1,j,k)*(Real(2.0)*facx*(sx(i ,j-1,k-1)+sx(i ,j,k-1)+sx(i ,j-1,k)+sx(i ,j,k))
1203  -facy*(sy(i ,j-1,k-1)+sy(i ,j,k-1)+sy(i ,j-1,k)+sy(i ,j,k))
1204  -facz*(sz(i ,j-1,k-1)+sz(i ,j,k-1)+sz(i ,j-1,k)+sz(i ,j,k)))
1205  + Real(2.0)*sol(i,j-1,k)*( -facx*(sx(i-1,j-1,k-1)+sx(i,j-1,k-1)+sx(i-1,j-1,k)+sx(i,j-1,k))
1206  +Real(2.0)*facy*(sy(i-1,j-1,k-1)+sy(i,j-1,k-1)+sy(i-1,j-1,k)+sy(i,j-1,k))
1207  -facz*(sz(i-1,j-1,k-1)+sz(i,j-1,k-1)+sz(i-1,j-1,k)+sz(i,j-1,k)))
1208  + Real(2.0)*sol(i,j+1,k)*( -facx*(sx(i-1,j ,k-1)+sx(i,j ,k-1)+sx(i-1,j ,k)+sx(i,j ,k))
1209  +Real(2.0)*facy*(sy(i-1,j ,k-1)+sy(i,j ,k-1)+sy(i-1,j ,k)+sy(i,j ,k))
1210  -facz*(sz(i-1,j ,k-1)+sz(i,j ,k-1)+sz(i-1,j ,k)+sz(i,j ,k)))
1211  + Real(2.0)*sol(i,j,k-1)*( -facx*(sx(i-1,j-1,k-1)+sx(i,j-1,k-1)+sx(i-1,j,k-1)+sx(i,j,k-1))
1212  -facy*(sy(i-1,j-1,k-1)+sy(i,j-1,k-1)+sy(i-1,j,k-1)+sy(i,j,k-1))
1213  +Real(2.0)*facz*(sz(i-1,j-1,k-1)+sz(i,j-1,k-1)+sz(i-1,j,k-1)+sz(i,j,k-1)))
1214  + Real(2.0)*sol(i,j,k+1)*( -facx*(sx(i-1,j-1,k )+sx(i,j-1,k )+sx(i-1,j,k )+sx(i,j,k ))
1215  -facy*(sy(i-1,j-1,k )+sy(i,j-1,k )+sy(i-1,j,k )+sy(i,j,k ))
1216  +Real(2.0)*facz*(sz(i-1,j-1,k )+sz(i,j-1,k )+sz(i-1,j,k )+sz(i,j,k )));
1217 
1218  sol(i,j,k) += (rhs(i,j,k) - Ax) / s0;
1219  }
1220  });
1221 }
1222 
1223 inline
1224 void mlndlap_gauss_seidel_aa (Box const& bx, Array4<Real> const& sol,
1225  Array4<Real const> const& rhs, Array4<Real const> const& sig,
1226  Array4<int const> const& msk,
1227  GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
1228 {
1229  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0];
1230  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1];
1231  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2];
1232  Real fxyz = facx + facy + facz;
1233  Real fmx2y2z = -facx + Real(2.0)*facy + Real(2.0)*facz;
1234  Real f2xmy2z = Real(2.0)*facx - facy + Real(2.0)*facz;
1235  Real f2x2ymz = Real(2.0)*facx + Real(2.0)*facy - facz;
1236  Real f4xm2ym2z = Real(4.0)*facx - Real(2.0)*facy - Real(2.0)*facz;
1237  Real fm2x4ym2z = -Real(2.0)*facx + Real(4.0)*facy - Real(2.0)*facz;
1238  Real fm2xm2y4z = -Real(2.0)*facx - Real(2.0)*facy + Real(4.0)*facz;
1239 
1240  amrex::LoopOnCpu(bx, [&] (int i, int j, int k) noexcept
1241  {
1242  if (msk(i,j,k)) {
1243  sol(i,j,k) = Real(0.0);
1244  } else {
1245  Real s0 = Real(-4.0)*fxyz*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j,k-1)+sig(i,j,k-1)
1246  +sig(i-1,j-1,k )+sig(i,j-1,k )+sig(i-1,j,k )+sig(i,j,k ));
1247  Real Ax = sol(i,j,k)*s0
1248  + fxyz*(sol(i-1,j-1,k-1)*sig(i-1,j-1,k-1)
1249  + sol(i+1,j-1,k-1)*sig(i ,j-1,k-1)
1250  + sol(i-1,j+1,k-1)*sig(i-1,j ,k-1)
1251  + sol(i+1,j+1,k-1)*sig(i ,j ,k-1)
1252  + sol(i-1,j-1,k+1)*sig(i-1,j-1,k )
1253  + sol(i+1,j-1,k+1)*sig(i ,j-1,k )
1254  + sol(i-1,j+1,k+1)*sig(i-1,j ,k )
1255  + sol(i+1,j+1,k+1)*sig(i ,j ,k ))
1256  + fmx2y2z*(sol(i ,j-1,k-1)*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1))
1257  + sol(i ,j+1,k-1)*(sig(i-1,j ,k-1)+sig(i,j ,k-1))
1258  + sol(i ,j-1,k+1)*(sig(i-1,j-1,k )+sig(i,j-1,k ))
1259  + sol(i ,j+1,k+1)*(sig(i-1,j ,k )+sig(i,j ,k )))
1260  + f2xmy2z*(sol(i-1,j ,k-1)*(sig(i-1,j-1,k-1)+sig(i-1,j,k-1))
1261  + sol(i+1,j ,k-1)*(sig(i ,j-1,k-1)+sig(i ,j,k-1))
1262  + sol(i-1,j ,k+1)*(sig(i-1,j-1,k )+sig(i-1,j,k ))
1263  + sol(i+1,j ,k+1)*(sig(i ,j-1,k )+sig(i ,j,k )))
1264  + f2x2ymz*(sol(i-1,j-1,k )*(sig(i-1,j-1,k-1)+sig(i-1,j-1,k))
1265  + sol(i+1,j-1,k )*(sig(i ,j-1,k-1)+sig(i ,j-1,k))
1266  + sol(i-1,j+1,k )*(sig(i-1,j ,k-1)+sig(i-1,j ,k))
1267  + sol(i+1,j+1,k )*(sig(i ,j ,k-1)+sig(i ,j ,k)))
1268  + f4xm2ym2z*(sol(i-1,j,k)*(sig(i-1,j-1,k-1)+sig(i-1,j,k-1)+sig(i-1,j-1,k)+sig(i-1,j,k))
1269  + sol(i+1,j,k)*(sig(i ,j-1,k-1)+sig(i ,j,k-1)+sig(i ,j-1,k)+sig(i ,j,k)))
1270  + fm2x4ym2z*(sol(i,j-1,k)*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j-1,k)+sig(i,j-1,k))
1271  + sol(i,j+1,k)*(sig(i-1,j ,k-1)+sig(i,j ,k-1)+sig(i-1,j ,k)+sig(i,j ,k)))
1272  + fm2xm2y4z*(sol(i,j,k-1)*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j,k-1)+sig(i,j,k-1))
1273  + sol(i,j,k+1)*(sig(i-1,j-1,k )+sig(i,j-1,k )+sig(i-1,j,k )+sig(i,j,k )));
1274 
1275  sol(i,j,k) += (rhs(i,j,k) - Ax) / s0;
1276  }
1277  });
1278 }
1279 
1280 inline
1281 void mlndlap_gauss_seidel_c (Box const& bx, Array4<Real> const& sol,
1282  Array4<Real const> const& rhs, Real sig,
1283  Array4<int const> const& msk,
1284  GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
1285 {
1286  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0];
1287  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1];
1288  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2];
1289  Real fxyz = facx + facy + facz;
1290  Real fmx2y2z = -facx + Real(2.0)*facy + Real(2.0)*facz;
1291  Real f2xmy2z = Real(2.0)*facx - facy + Real(2.0)*facz;
1292  Real f2x2ymz = Real(2.0)*facx + Real(2.0)*facy - facz;
1293  Real f4xm2ym2z = Real(4.0)*facx - Real(2.0)*facy - Real(2.0)*facz;
1294  Real fm2x4ym2z = -Real(2.0)*facx + Real(4.0)*facy - Real(2.0)*facz;
1295  Real fm2xm2y4z = -Real(2.0)*facx - Real(2.0)*facy + Real(4.0)*facz;
1296 
1297  amrex::LoopOnCpu(bx, [&] (int i, int j, int k) noexcept
1298  {
1299  if (msk(i,j,k)) {
1300  sol(i,j,k) = Real(0.0);
1301  } else {
1302  Real s0 = Real(-4.0)*fxyz*Real(8.);
1303  Real Ax = sol(i,j,k)*s0
1304  + fxyz*(sol(i-1,j-1,k-1)
1305  + sol(i+1,j-1,k-1)
1306  + sol(i-1,j+1,k-1)
1307  + sol(i+1,j+1,k-1)
1308  + sol(i-1,j-1,k+1)
1309  + sol(i+1,j-1,k+1)
1310  + sol(i-1,j+1,k+1)
1311  + sol(i+1,j+1,k+1))
1312  + fmx2y2z*(sol(i ,j-1,k-1)*Real(2.)
1313  + sol(i ,j+1,k-1)*Real(2.)
1314  + sol(i ,j-1,k+1)*Real(2.)
1315  + sol(i ,j+1,k+1)*Real(2.))
1316  + f2xmy2z*(sol(i-1,j ,k-1)*Real(2.)
1317  + sol(i+1,j ,k-1)*Real(2.)
1318  + sol(i-1,j ,k+1)*Real(2.)
1319  + sol(i+1,j ,k+1)*Real(2.))
1320  + f2x2ymz*(sol(i-1,j-1,k )*Real(2.)
1321  + sol(i+1,j-1,k )*Real(2.)
1322  + sol(i-1,j+1,k )*Real(2.)
1323  + sol(i+1,j+1,k )*Real(2.))
1324  + f4xm2ym2z*(sol(i-1,j,k)*Real(4.)
1325  + sol(i+1,j,k)*Real(4.))
1326  + fm2x4ym2z*(sol(i,j-1,k)*Real(4.)
1327  + sol(i,j+1,k)*Real(4.))
1328  + fm2xm2y4z*(sol(i,j,k-1)*Real(4.)
1329  + sol(i,j,k+1)*Real(4.));
1330 
1331  sol(i,j,k) += (rhs(i,j,k) - Ax*sig) / (s0*sig);
1332  }
1333  });
1334 }
1335 
1336 AMREX_FORCE_INLINE
1337 void tridiagonal_solve (Array1D<Real,0,31>& a_ls, Array1D<Real,0,31>& b_ls, Array1D<Real,0,31>& c_ls,
1338  Array1D<Real,0,31>& r_ls, Array1D<Real,0,31>& u_ls, Array1D<Real,0,31>& gam,
1339  int ilen ) noexcept
1340 {
1341  Real bet = b_ls(0);
1342  u_ls(0) = r_ls(0) / bet;
1343 
1344  for (int i = 1; i <= ilen - 1; i++) {
1345  gam(i) = c_ls(i-1) / bet;
1346  bet = b_ls(i) - a_ls(i)*gam(i);
1347  if (bet == 0) { amrex::Abort(">>>TRIDIAG FAILED"); }
1348  u_ls(i) = (r_ls(i)-a_ls(i)*u_ls(i-1)) / bet;
1349  }
1350  for (int i = ilen-2; i >= 0; i--) {
1351  u_ls(i) = u_ls(i) - gam(i+1)*u_ls(i+1);
1352  }
1353 }
1354 
1355 inline
1356 void mlndlap_gauss_seidel_with_line_solve_aa (Box const& bx, Array4<Real> const& sol,
1357  Array4<Real const> const& rhs, Array4<Real const> const& sig,
1358  Array4<int const> const& msk,
1359  GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
1360 {
1361  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0];
1362  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1];
1363  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2];
1364  Real fxyz = facx + facy + facz;
1365  Real fmx2y2z = -facx + Real(2.0)*facy + Real(2.0)*facz;
1366  Real f2xmy2z = Real(2.0)*facx - facy + Real(2.0)*facz;
1367  Real f2x2ymz = Real(2.0)*facx + Real(2.0)*facy - facz;
1368  Real f4xm2ym2z = Real(4.0)*facx - Real(2.0)*facy - Real(2.0)*facz;
1369  Real fm2x4ym2z = -Real(2.0)*facx + Real(4.0)*facy - Real(2.0)*facz;
1370  Real fm2xm2y4z = -Real(2.0)*facx - Real(2.0)*facy + Real(4.0)*facz;
1371 
1372  const auto lo = amrex::lbound(bx);
1373  const auto hi = amrex::ubound(bx);
1374 
1375  int idir = -1;
1376  int ilen = 33;
1377 
1378  if ( (dxinv[0] <= dxinv[2]) && (dxinv[1] <= dxinv[2]) ) {
1379  idir = 2;
1380  ilen = hi.z - lo.z + 1;
1381  }
1382  if ( (dxinv[0] <= dxinv[1]) && (dxinv[2] <= dxinv[1]) ) {
1383  idir = 1;
1384  ilen = hi.y - lo.y + 1;
1385  }
1386  if ( (dxinv[1] <= dxinv[0]) && (dxinv[2] <= dxinv[0]) ) {
1387  idir = 0;
1388  ilen = hi.x - lo.x + 1;
1389  }
1390 
1391  if (ilen > 32) {
1392  amrex::Abort("mlndlap_gauss_seidel_with_line_solve_aa is hard-wired to be no longer than 32");
1393  }
1394 
1395  Array1D<Real,0,31> a_ls,b_ls,c_ls,u_ls,r_ls,gam;
1396 
1397 
1398  if ( idir == 2 )
1399  {
1400  for (int j = lo.y; j <= hi.y; ++j)
1401  {
1402  for (int i = lo.x; i <= hi.x; ++i)
1403  {
1404  for (int k = lo.z; k <= hi.z; ++k)
1405  {
1406  if (msk(i,j,k))
1407  {
1408  a_ls(k-lo.z) = Real(0.);
1409  b_ls(k-lo.z) = Real(1.);
1410  c_ls(k-lo.z) = Real(0.);
1411  u_ls(k-lo.z) = Real(0.);
1412  r_ls(k-lo.z) = Real(0.);
1413  }
1414  else
1415  {
1416  Real s0 = Real(-4.0)*fxyz*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j,k-1)+sig(i,j,k-1)
1417  + sig(i-1,j-1,k )+sig(i,j-1,k )+sig(i-1,j,k )+sig(i,j,k ));
1418 
1419  Real Ax = fxyz*(sol(i-1,j-1,k-1)*sig(i-1,j-1,k-1)
1420  + sol(i+1,j-1,k-1)*sig(i ,j-1,k-1)
1421  + sol(i-1,j+1,k-1)*sig(i-1,j ,k-1)
1422  + sol(i+1,j+1,k-1)*sig(i ,j ,k-1)
1423  + sol(i-1,j-1,k+1)*sig(i-1,j-1,k )
1424  + sol(i+1,j-1,k+1)*sig(i ,j-1,k )
1425  + sol(i-1,j+1,k+1)*sig(i-1,j ,k )
1426  + sol(i+1,j+1,k+1)*sig(i ,j ,k ))
1427  + fmx2y2z*(sol(i ,j-1,k-1)*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1))
1428  + sol(i ,j+1,k-1)*(sig(i-1,j ,k-1)+sig(i,j ,k-1))
1429  + sol(i ,j-1,k+1)*(sig(i-1,j-1,k )+sig(i,j-1,k ))
1430  + sol(i ,j+1,k+1)*(sig(i-1,j ,k )+sig(i,j ,k )))
1431  + f2xmy2z*(sol(i-1,j ,k-1)*(sig(i-1,j-1,k-1)+sig(i-1,j,k-1))
1432  + sol(i+1,j ,k-1)*(sig(i ,j-1,k-1)+sig(i ,j,k-1))
1433  + sol(i-1,j ,k+1)*(sig(i-1,j-1,k )+sig(i-1,j,k ))
1434  + sol(i+1,j ,k+1)*(sig(i ,j-1,k )+sig(i ,j,k )))
1435  + f2x2ymz*(sol(i-1,j-1,k )*(sig(i-1,j-1,k-1)+sig(i-1,j-1,k))
1436  + sol(i+1,j-1,k )*(sig(i ,j-1,k-1)+sig(i ,j-1,k))
1437  + sol(i-1,j+1,k )*(sig(i-1,j ,k-1)+sig(i-1,j ,k))
1438  + sol(i+1,j+1,k )*(sig(i ,j ,k-1)+sig(i ,j ,k)))
1439  + f4xm2ym2z*(sol(i-1,j,k)*(sig(i-1,j-1,k-1)+sig(i-1,j,k-1)+sig(i-1,j-1,k)+sig(i-1,j,k))
1440  + sol(i+1,j,k)*(sig(i ,j-1,k-1)+sig(i ,j,k-1)+sig(i ,j-1,k)+sig(i ,j,k)))
1441  + fm2x4ym2z*(sol(i,j-1,k)*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j-1,k)+sig(i,j-1,k))
1442  + sol(i,j+1,k)*(sig(i-1,j ,k-1)+sig(i,j ,k-1)+sig(i-1,j ,k)+sig(i,j ,k)));
1443 
1444  a_ls(k-lo.z) = fm2xm2y4z*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j,k-1)+sig(i,j,k-1));
1445  b_ls(k-lo.z) = s0;
1446  c_ls(k-lo.z) = fm2xm2y4z*(sig(i-1,j-1,k )+sig(i,j-1,k )+sig(i-1,j,k )+sig(i,j,k ));
1447  u_ls(k-lo.z) = Real(0.);
1448  r_ls(k-lo.z) = rhs(i,j,k) - Ax;
1449  }
1450  }
1451  tridiagonal_solve(a_ls, b_ls, c_ls, r_ls, u_ls, gam, ilen);
1452 
1453  for (int k = lo.z; k <= hi.z; ++k)
1454  {
1455  sol(i,j,k) = u_ls(k-lo.z);
1456  }
1457  }
1458  }
1459  }
1460  else if (idir == 1)
1461  {
1462  for (int k = lo.z; k <= hi.z; ++k)
1463  {
1464  for (int i = lo.x; i <= hi.x; ++i)
1465  {
1466  for (int j = lo.y; j <= hi.y; ++j)
1467  {
1468  if (msk(i,j,k)) {
1469  a_ls(j-lo.y) = Real(0.);
1470  b_ls(j-lo.y) = Real(1.);
1471  c_ls(j-lo.y) = Real(0.);
1472  u_ls(j-lo.y) = Real(0.);
1473  r_ls(j-lo.y) = Real(0.);
1474  }
1475  else
1476  {
1477  Real s0 = Real(-4.0)*fxyz*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j,k-1)+sig(i,j,k-1)
1478  + sig(i-1,j-1,k )+sig(i,j-1,k )+sig(i-1,j,k )+sig(i,j,k ));
1479 
1480  Real Ax = fxyz*(sol(i-1,j-1,k-1)*sig(i-1,j-1,k-1)
1481  + sol(i+1,j-1,k-1)*sig(i ,j-1,k-1)
1482  + sol(i-1,j+1,k-1)*sig(i-1,j ,k-1)
1483  + sol(i+1,j+1,k-1)*sig(i ,j ,k-1)
1484  + sol(i-1,j-1,k+1)*sig(i-1,j-1,k )
1485  + sol(i+1,j-1,k+1)*sig(i ,j-1,k )
1486  + sol(i-1,j+1,k+1)*sig(i-1,j ,k )
1487  + sol(i+1,j+1,k+1)*sig(i ,j ,k ))
1488  + fmx2y2z*(sol(i ,j-1,k-1)*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1))
1489  + sol(i ,j+1,k-1)*(sig(i-1,j ,k-1)+sig(i,j ,k-1))
1490  + sol(i ,j-1,k+1)*(sig(i-1,j-1,k )+sig(i,j-1,k ))
1491  + sol(i ,j+1,k+1)*(sig(i-1,j ,k )+sig(i,j ,k )))
1492  + f2xmy2z*(sol(i-1,j ,k-1)*(sig(i-1,j-1,k-1)+sig(i-1,j,k-1))
1493  + sol(i+1,j ,k-1)*(sig(i ,j-1,k-1)+sig(i ,j,k-1))
1494  + sol(i-1,j ,k+1)*(sig(i-1,j-1,k )+sig(i-1,j,k ))
1495  + sol(i+1,j ,k+1)*(sig(i ,j-1,k )+sig(i ,j,k )))
1496  + f2x2ymz*(sol(i-1,j-1,k )*(sig(i-1,j-1,k-1)+sig(i-1,j-1,k))
1497  + sol(i+1,j-1,k )*(sig(i ,j-1,k-1)+sig(i ,j-1,k))
1498  + sol(i-1,j+1,k )*(sig(i-1,j ,k-1)+sig(i-1,j ,k))
1499  + sol(i+1,j+1,k )*(sig(i ,j ,k-1)+sig(i ,j ,k)))
1500  + f4xm2ym2z*(sol(i-1,j,k)*(sig(i-1,j-1,k-1)+sig(i-1,j,k-1)+sig(i-1,j-1,k)+sig(i-1,j,k))
1501  + sol(i+1,j,k)*(sig(i ,j-1,k-1)+sig(i ,j,k-1)+sig(i ,j-1,k)+sig(i ,j,k)))
1502  + fm2xm2y4z*(sol(i,j,k-1)*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j,k-1)+sig(i,j,k-1))
1503  + sol(i,j,k+1)*(sig(i-1,j-1,k )+sig(i,j-1,k )+sig(i-1,j,k )+sig(i,j,k )));
1504 
1505  a_ls(j-lo.y) = fm2x4ym2z*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j-1,k)+sig(i,j-1,k));
1506  b_ls(j-lo.y) = s0;
1507  c_ls(j-lo.y) = fm2x4ym2z*(sig(i-1,j ,k-1)+sig(i,j ,k-1)+sig(i-1,j ,k)+sig(i,j ,k));
1508  u_ls(j-lo.y) = Real(0.);
1509  r_ls(j-lo.y) = rhs(i,j,k) - Ax;
1510 
1511  }
1512  }
1513  tridiagonal_solve(a_ls, b_ls, c_ls, r_ls, u_ls, gam, ilen);
1514 
1515  for (int j = lo.y; j <= hi.y; ++j)
1516  {
1517  sol(i,j,k) = u_ls(j-lo.y);
1518  }
1519  }
1520  }
1521  }
1522  else if (idir == 0)
1523  {
1524  for (int j = lo.y; j <= hi.y; ++j)
1525  {
1526  for (int k = lo.z; k <= hi.z; ++k)
1527  {
1528  for (int i = lo.x; i <= hi.x; ++i)
1529  {
1530  if (msk(i,j,k))
1531  {
1532  a_ls(i-lo.x) = Real(0.);
1533  b_ls(i-lo.x) = Real(1.);
1534  c_ls(i-lo.x) = Real(0.);
1535  u_ls(i-lo.x) = Real(0.);
1536  r_ls(i-lo.x) = Real(0.);
1537  }
1538  else
1539  {
1540  Real s0 = Real(-4.0)*fxyz*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j,k-1)+sig(i,j,k-1)
1541  + sig(i-1,j-1,k )+sig(i,j-1,k )+sig(i-1,j,k )+sig(i,j,k ));
1542 
1543  Real Ax = fxyz*(sol(i-1,j-1,k-1)*sig(i-1,j-1,k-1)
1544  + sol(i+1,j-1,k-1)*sig(i ,j-1,k-1)
1545  + sol(i-1,j+1,k-1)*sig(i-1,j ,k-1)
1546  + sol(i+1,j+1,k-1)*sig(i ,j ,k-1)
1547  + sol(i-1,j-1,k+1)*sig(i-1,j-1,k )
1548  + sol(i+1,j-1,k+1)*sig(i ,j-1,k )
1549  + sol(i-1,j+1,k+1)*sig(i-1,j ,k )
1550  + sol(i+1,j+1,k+1)*sig(i ,j ,k ))
1551  + fmx2y2z*(sol(i ,j-1,k-1)*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1))
1552  + sol(i ,j+1,k-1)*(sig(i-1,j ,k-1)+sig(i,j ,k-1))
1553  + sol(i ,j-1,k+1)*(sig(i-1,j-1,k )+sig(i,j-1,k ))
1554  + sol(i ,j+1,k+1)*(sig(i-1,j ,k )+sig(i,j ,k )))
1555  + f2xmy2z*(sol(i-1,j ,k-1)*(sig(i-1,j-1,k-1)+sig(i-1,j,k-1))
1556  + sol(i+1,j ,k-1)*(sig(i ,j-1,k-1)+sig(i ,j,k-1))
1557  + sol(i-1,j ,k+1)*(sig(i-1,j-1,k )+sig(i-1,j,k ))
1558  + sol(i+1,j ,k+1)*(sig(i ,j-1,k )+sig(i ,j,k )))
1559  + f2x2ymz*(sol(i-1,j-1,k )*(sig(i-1,j-1,k-1)+sig(i-1,j-1,k))
1560  + sol(i+1,j-1,k )*(sig(i ,j-1,k-1)+sig(i ,j-1,k))
1561  + sol(i-1,j+1,k )*(sig(i-1,j ,k-1)+sig(i-1,j ,k))
1562  + sol(i+1,j+1,k )*(sig(i ,j ,k-1)+sig(i ,j ,k)))
1563  + fm2x4ym2z*(sol(i,j-1,k)*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j-1,k)+sig(i,j-1,k))
1564  + sol(i,j+1,k)*(sig(i-1,j ,k-1)+sig(i,j ,k-1)+sig(i-1,j ,k)+sig(i,j ,k)))
1565  + fm2xm2y4z*(sol(i,j,k-1)*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j,k-1)+sig(i,j,k-1))
1566  + sol(i,j,k+1)*(sig(i-1,j-1,k )+sig(i,j-1,k )+sig(i-1,j,k )+sig(i,j,k )));
1567 
1568  a_ls(i-lo.x) = f4xm2ym2z*(sig(i-1,j-1,k-1)+sig(i-1,j,k-1)+sig(i-1,j-1,k)+sig(i-1,j,k));
1569  b_ls(i-lo.x) = s0;
1570  c_ls(i-lo.x) = f4xm2ym2z*(sig(i ,j-1,k-1)+sig(i ,j,k-1)+sig(i ,j-1,k)+sig(i ,j,k));
1571  u_ls(i-lo.x) = Real(0.);
1572  r_ls(i-lo.x) = rhs(i,j,k) - Ax;
1573  }
1574  }
1575  tridiagonal_solve(a_ls, b_ls, c_ls, r_ls, u_ls, gam, ilen);
1576 
1577  for (int i = lo.x; i <= hi.x; ++i)
1578  {
1579  sol(i,j,k) = u_ls(i-lo.x);
1580  }
1581  }
1582  }
1583  }
1584  else
1585  {
1586  amrex::Abort("mlndlap_gauss_seidel_with_line_solve_aa is wrong direction.");
1587  }
1588 }
1589 
1590 //
1591 // restriction
1592 //
1593 
1594 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
1595 void mlndlap_restriction (int i, int j, int k, Array4<Real> const& crse,
1596  Array4<Real const> const& fine, Array4<int const> const& msk) noexcept
1597 {
1598  int ii = i*2;
1599  int jj = j*2;
1600  int kk = k*2;
1601  if (msk(ii,jj,kk)) {
1602  crse(i,j,k) = Real(0.0);
1603  } else {
1604  crse(i,j,k) = Real(1./64.)*(fine(ii-1,jj-1,kk-1)+fine(ii+1,jj-1,kk-1)
1605  +fine(ii-1,jj+1,kk-1)+fine(ii+1,jj+1,kk-1)
1606  +fine(ii-1,jj-1,kk+1)+fine(ii+1,jj-1,kk+1)
1607  +fine(ii-1,jj+1,kk+1)+fine(ii+1,jj+1,kk+1))
1608  + Real(1./32.)*(fine(ii ,jj-1,kk-1)+fine(ii ,jj+1,kk-1)
1609  +fine(ii ,jj-1,kk+1)+fine(ii ,jj+1,kk+1)
1610  +fine(ii-1,jj ,kk-1)+fine(ii+1,jj ,kk-1)
1611  +fine(ii-1,jj ,kk+1)+fine(ii+1,jj ,kk+1)
1612  +fine(ii-1,jj-1,kk )+fine(ii+1,jj-1,kk )
1613  +fine(ii-1,jj+1,kk )+fine(ii+1,jj+1,kk ))
1614  + Real(1./16.)*(fine(ii-1,jj,kk)+fine(ii+1,jj,kk)
1615  +fine(ii,jj-1,kk)+fine(ii,jj+1,kk)
1616  +fine(ii,jj,kk-1)+fine(ii,jj,kk+1))
1617  + Real(1./8.)*fine(ii,jj,kk);
1618  }
1619 }
1620 
1621 template <int rr>
1622 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
1623 void mlndlap_restriction (int i, int j, int k, Array4<Real> const& crse,
1624  Array4<Real const> const& fine, Array4<int const> const& msk,
1625  Box const& fdom,
1626  GpuArray<LinOpBCType, AMREX_SPACEDIM> const& bclo,
1627  GpuArray<LinOpBCType, AMREX_SPACEDIM> const& bchi) noexcept
1628 {
1629  const int ii = i*rr;
1630  const int jj = j*rr;
1631  const int kk = k*rr;
1632  if (msk(ii,jj,kk)) {
1633  crse(i,j,k) = Real(0.0);
1634  } else {
1635  const auto ndlo = amrex::lbound(fdom);
1636  const auto ndhi = amrex::ubound(fdom);
1637  Real tmp = Real(0.0);
1638  for (int koff = -rr+1; koff <= rr-1; ++koff) {
1639  Real wz = rr - std::abs(koff);
1640  for (int joff = -rr+1; joff <= rr-1; ++joff) {
1641  Real wy = rr - std::abs(joff);
1642  for (int ioff = -rr+1; ioff <= rr-1; ++ioff) {
1643  Real wx = rr - std::abs(ioff);
1644  int itmp = ii + ioff;
1645  int jtmp = jj + joff;
1646  int ktmp = kk + koff;
1647  if ((itmp < ndlo.x && (bclo[0] == LinOpBCType::Neumann ||
1648  bclo[0] == LinOpBCType::inflow)) ||
1649  (itmp > ndhi.x && (bchi[0] == LinOpBCType::Neumann ||
1650  bchi[0] == LinOpBCType::inflow))) {
1651  itmp = ii - ioff;
1652  }
1653  if ((jtmp < ndlo.y && (bclo[1] == LinOpBCType::Neumann ||
1654  bclo[1] == LinOpBCType::inflow)) ||
1655  (jtmp > ndhi.y && (bchi[1] == LinOpBCType::Neumann ||
1656  bchi[1] == LinOpBCType::inflow))) {
1657  jtmp = jj - joff;
1658  }
1659  if ((ktmp < ndlo.z && (bclo[2] == LinOpBCType::Neumann ||
1660  bclo[2] == LinOpBCType::inflow)) ||
1661  (ktmp > ndhi.z && (bchi[2] == LinOpBCType::Neumann ||
1662  bchi[2] == LinOpBCType::inflow))) {
1663  ktmp = kk - koff;
1664  }
1665  tmp += wx*wy*wz*fine(itmp,jtmp,ktmp);
1666  }
1667  }
1668  }
1669  crse(i,j,k) = tmp/Real(rr*rr*rr*rr*rr*rr);
1670  }
1671 }
1672 
1673 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
1674 void mlndlap_semi_restriction (int i, int j, int k, Array4<Real> const& crse,
1675  Array4<Real const> const& fine, Array4<int const> const& msk, int idir) noexcept
1676 {
1677  if (idir == 2)
1678  {
1679  int ii = i*2;
1680  int jj = j*2;
1681  int kk = k;
1682  if (msk(ii,jj,kk)) {
1683  crse(i,j,k) = Real(0.0);
1684  } else { // use 2-D formula
1685  crse(i,j,k) = Real(1./16.)*(fine(ii-1,jj-1,kk) + Real(2.)*fine(ii ,jj-1,kk) + fine(ii+1,jj-1,kk)
1686  + Real(2.)*fine(ii-1,jj ,kk) + Real(4.)*fine(ii ,jj ,kk) + Real(2.)*fine(ii+1,jj ,kk)
1687  + fine(ii-1,jj+1,kk) + Real(2.)*fine(ii ,jj+1,kk) + fine(ii+1,jj+1,kk));
1688  }
1689  }
1690  else if (idir == 1)
1691  {
1692  int ii = i*2;
1693  int jj = j;
1694  int kk = k*2;
1695  if (msk(ii,jj,kk)) {
1696  crse(i,j,k) = Real(0.0);
1697  } else { // use 2-D formula
1698  crse(i,j,k) = Real(1./16.)*(fine(ii-1,jj,kk-1) + Real(2.)*fine(ii ,jj,kk-1) + fine(ii+1,jj,kk-1)
1699  + Real(2.)*fine(ii-1,jj ,kk) + Real(4.)*fine(ii ,jj,kk ) + Real(2.)*fine(ii+1,jj,kk )
1700  + fine(ii-1,jj,kk+1) + Real(2.)*fine(ii ,jj,kk+1) + fine(ii+1,jj,kk+1));
1701  }
1702  }
1703  else if (idir == 0)
1704  {
1705  int ii = i;
1706  int jj = j*2;
1707  int kk = k*2;
1708  if (msk(ii,jj,kk)) {
1709  crse(i,j,k) = Real(0.0);
1710  } else { // use 2-D formula
1711  crse(i,j,k) = Real(1./16.)*(fine(ii,jj-1,kk-1) + Real(2.)*fine(ii ,jj,kk-1) + fine(ii,jj+1,kk-1)
1712  + Real(2.)*fine(ii,jj-1 ,kk) + Real(4.)*fine(ii ,jj,kk ) + Real(2.)*fine(ii,jj+1,kk )
1713  + fine(ii,jj-1,kk+1) + Real(2.)*fine(ii ,jj,kk+1) + fine(ii,jj+1,kk+1));
1714  }
1715  }
1716  else
1717  {
1718  amrex::Abort("mlndlap_semi_restriction semi direction wrong semi-direction. ");
1719  }
1720 }
1721 
1722 //
1723 // interpolation
1724 //
1725 
1726  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
1727  Real aa_interp_line_x (Array4<Real const> const& crse, Array4<Real const> const& sig,
1728  int i, int j, int k, int ic, int jc, int kc) noexcept
1729  {
1730  Real w1 = sig(i-1,j-1,k-1) + sig(i-1,j,k-1) + sig(i-1,j-1,k) + sig(i-1,j,k);
1731  Real w2 = sig(i ,j-1,k-1) + sig(i ,j,k-1) + sig(i ,j-1,k) + sig(i ,j,k);
1732  return (w1*crse(ic,jc,kc)+w2*crse(ic+1,jc,kc))/(w1+w2);
1733  }
1734 
1735  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
1736  Real aa_interp_line_y (Array4<Real const> const& crse, Array4<Real const> const& sig,
1737  int i, int j, int k, int ic, int jc, int kc) noexcept
1738  {
1739  Real w1 = sig(i-1,j-1,k-1) + sig(i,j-1,k-1) + sig(i-1,j-1,k) + sig(i,j-1,k);
1740  Real w2 = sig(i-1,j ,k-1) + sig(i,j ,k-1) + sig(i-1,j ,k) + sig(i,j ,k);
1741  return (w1*crse(ic,jc,kc)+w2*crse(ic,jc+1,kc))/(w1+w2);
1742  }
1743 
1744  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
1745  Real aa_interp_line_z (Array4<Real const> const& crse, Array4<Real const> const& sig,
1746  int i, int j, int k, int ic, int jc, int kc) noexcept
1747  {
1748  Real w1 = sig(i-1,j-1,k-1) + sig(i,j-1,k-1) + sig(i-1,j,k-1) + sig(i,j,k-1);
1749  Real w2 = sig(i-1,j-1,k ) + sig(i,j-1,k ) + sig(i-1,j,k ) + sig(i,j,k );
1750  return (w1*crse(ic,jc,kc)+w2*crse(ic,jc,kc+1))/(w1+w2);
1751  }
1752 
1753  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
1754  Real aa_interp_face_xy (Array4<Real const> const& crse, Array4<Real const> const& sig,
1755  int i, int j, int k, int ic, int jc, int kc) noexcept
1756  {
1757  Real w1 = sig(i-1,j-1,k-1) + sig(i-1,j,k-1) + sig(i-1,j-1,k) + sig(i-1,j,k);
1758  Real w2 = sig(i ,j-1,k-1) + sig(i ,j,k-1) + sig(i ,j-1,k) + sig(i ,j,k);
1759  Real w3 = sig(i-1,j-1,k-1) + sig(i,j-1,k-1) + sig(i-1,j-1,k) + sig(i,j-1,k);
1760  Real w4 = sig(i-1,j ,k-1) + sig(i,j ,k-1) + sig(i-1,j ,k) + sig(i,j ,k);
1761  return (w1 * aa_interp_line_y(crse,sig,i-1,j ,k,ic ,jc ,kc) +
1762  w2 * aa_interp_line_y(crse,sig,i+1,j ,k,ic+1,jc ,kc) +
1763  w3 * aa_interp_line_x(crse,sig,i ,j-1,k,ic ,jc ,kc) +
1764  w4 * aa_interp_line_x(crse,sig,i ,j+1,k,ic ,jc+1,kc)) / (w1+w2+w3+w4);
1765  }
1766 
1767  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
1768  Real aa_interp_face_xz (Array4<Real const> const& crse, Array4<Real const> const& sig,
1769  int i, int j, int k, int ic, int jc, int kc) noexcept
1770  {
1771  Real w1 = sig(i-1,j-1,k-1) + sig(i-1,j,k-1) + sig(i-1,j-1,k) + sig(i-1,j,k);
1772  Real w2 = sig(i ,j-1,k-1) + sig(i ,j,k-1) + sig(i ,j-1,k) + sig(i ,j,k);
1773  Real w3 = sig(i-1,j-1,k-1) + sig(i,j-1,k-1) + sig(i-1,j,k-1) + sig(i,j,k-1);
1774  Real w4 = sig(i-1,j-1,k ) + sig(i,j-1,k ) + sig(i-1,j,k ) + sig(i,j,k );
1775  return (w1 * aa_interp_line_z(crse,sig,i-1,j,k ,ic ,jc,kc ) +
1776  w2 * aa_interp_line_z(crse,sig,i+1,j,k ,ic+1,jc,kc ) +
1777  w3 * aa_interp_line_x(crse,sig,i ,j,k-1,ic ,jc,kc ) +
1778  w4 * aa_interp_line_x(crse,sig,i ,j,k+1,ic ,jc,kc+1)) / (w1+w2+w3+w4);
1779  }
1780 
1781  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
1782  Real aa_interp_face_yz (Array4<Real const> const& crse, Array4<Real const> const& sig,
1783  int i, int j, int k, int ic, int jc, int kc) noexcept
1784  {
1785  Real w1 = sig(i-1,j-1,k-1) + sig(i,j-1,k-1) + sig(i-1,j-1,k) + sig(i,j-1,k);
1786  Real w2 = sig(i-1,j ,k-1) + sig(i,j ,k-1) + sig(i-1,j ,k) + sig(i,j ,k);
1787  Real w3 = sig(i-1,j-1,k-1) + sig(i,j-1,k-1) + sig(i-1,j,k-1) + sig(i,j,k-1);
1788  Real w4 = sig(i-1,j-1,k ) + sig(i,j-1,k ) + sig(i-1,j,k ) + sig(i,j,k );
1789  return (w1 * aa_interp_line_z(crse,sig,i,j-1,k ,ic,jc ,kc ) +
1790  w2 * aa_interp_line_z(crse,sig,i,j+1,k ,ic,jc+1,kc ) +
1791  w3 * aa_interp_line_y(crse,sig,i,j ,k-1,ic,jc ,kc ) +
1792  w4 * aa_interp_line_y(crse,sig,i,j ,k+1,ic,jc ,kc+1)) / (w1+w2+w3+w4);
1793  }
1794 
1795 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
1796 void mlndlap_interpadd_c (int i, int j, int k, Array4<Real> const& fine,
1797  Array4<Real const> const& crse,
1798  Array4<int const> const& msk) noexcept
1799 {
1800  if (!msk(i,j,k)) {
1801  int ic = amrex::coarsen(i,2);
1802  int jc = amrex::coarsen(j,2);
1803  int kc = amrex::coarsen(k,2);
1804  bool i_is_odd = (ic*2 != i);
1805  bool j_is_odd = (jc*2 != j);
1806  bool k_is_odd = (kc*2 != k);
1807  if (i_is_odd && j_is_odd && k_is_odd) {
1808  // Fine node at center of cell
1809  fine(i,j,k) += Real(0.125) *
1810  (crse(ic,jc ,kc ) + crse(ic+1,jc ,kc ) +
1811  crse(ic,jc+1,kc ) + crse(ic+1,jc+1,kc ) +
1812  crse(ic,jc ,kc+1) + crse(ic+1,jc ,kc+1) +
1813  crse(ic,jc+1,kc+1) + crse(ic+1,jc+1,kc+1));
1814  } else if (j_is_odd && k_is_odd) {
1815  // Node on a Y-Z face
1816  fine(i,j,k) += Real(0.25) *
1817  (crse(ic,jc,kc ) + crse(ic,jc+1,kc ) +
1818  crse(ic,jc,kc+1) + crse(ic,jc+1,kc+1));
1819  } else if (i_is_odd && k_is_odd) {
1820  // Node on a Z-X face
1821  fine(i,j,k) += Real(0.25) *
1822  (crse(ic,jc,kc ) + crse(ic+1,jc,kc ) +
1823  crse(ic,jc,kc+1) + crse(ic+1,jc,kc+1));
1824  } else if (i_is_odd && j_is_odd) {
1825  // Node on a X-Y face
1826  fine(i,j,k) += Real(0.25) *
1827  (crse(ic,jc ,kc) + crse(ic+1,jc ,kc) +
1828  crse(ic,jc+1,kc) + crse(ic+1,jc+1,kc));
1829  } else if (i_is_odd) {
1830  // Node on X line
1831  fine(i,j,k) += Real(0.5)*(crse(ic,jc,kc)+crse(ic+1,jc,kc));
1832  } else if (j_is_odd) {
1833  // Node on Y line
1834  fine(i,j,k) += Real(0.5)*(crse(ic,jc,kc)+crse(ic,jc+1,kc));
1835  } else if (k_is_odd) {
1836  // Node on Z line
1837  fine(i,j,k) += Real(0.5)*(crse(ic,jc,kc)+crse(ic,jc,kc+1));
1838  } else {
1839  // Node coincident with coarse node
1840  fine(i,j,k) += crse(ic,jc,kc);
1841  }
1842  }
1843 }
1844 
1845 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
1846 void mlndlap_interpadd_aa (int i, int j, int k, Array4<Real> const& fine,
1847  Array4<Real const> const& crse, Array4<Real const> const& sig,
1848  Array4<int const> const& msk) noexcept
1849 {
1850  if (!msk(i,j,k)) {
1851  int ic = amrex::coarsen(i,2);
1852  int jc = amrex::coarsen(j,2);
1853  int kc = amrex::coarsen(k,2);
1854  bool i_is_odd = (ic*2 != i);
1855  bool j_is_odd = (jc*2 != j);
1856  bool k_is_odd = (kc*2 != k);
1857  if (i_is_odd && j_is_odd && k_is_odd) {
1858  // Fine node at center of cell
1859  Real w1 = sig(i-1,j-1,k-1) + sig(i-1,j,k-1) + sig(i-1,j-1,k) + sig(i-1,j,k);
1860  Real w2 = sig(i ,j-1,k-1) + sig(i ,j,k-1) + sig(i ,j-1,k) + sig(i ,j,k);
1861  Real w3 = sig(i-1,j-1,k-1) + sig(i,j-1,k-1) + sig(i-1,j-1,k) + sig(i,j-1,k);
1862  Real w4 = sig(i-1,j ,k-1) + sig(i,j ,k-1) + sig(i-1,j ,k) + sig(i,j ,k);
1863  Real w5 = sig(i-1,j-1,k-1) + sig(i,j-1,k-1) + sig(i-1,j,k-1) + sig(i,j,k-1);
1864  Real w6 = sig(i-1,j-1,k ) + sig(i,j-1,k ) + sig(i-1,j,k ) + sig(i,j,k );
1865  fine(i,j,k) += (w1 * aa_interp_face_yz(crse,sig,i-1,j ,k ,ic ,jc ,kc ) +
1866  w2 * aa_interp_face_yz(crse,sig,i+1,j ,k ,ic+1,jc ,kc ) +
1867  w3 * aa_interp_face_xz(crse,sig,i ,j-1,k ,ic ,jc ,kc ) +
1868  w4 * aa_interp_face_xz(crse,sig,i ,j+1,k ,ic ,jc+1,kc ) +
1869  w5 * aa_interp_face_xy(crse,sig,i ,j ,k-1,ic ,jc ,kc ) +
1870  w6 * aa_interp_face_xy(crse,sig,i ,j ,k+1,ic ,jc ,kc+1))
1871  / (w1+w2+w3+w4+w5+w6);
1872  } else if (j_is_odd && k_is_odd) {
1873  // Node on a Y-Z face
1874  fine(i,j,k) += aa_interp_face_yz(crse,sig,i,j,k,ic,jc,kc);
1875  } else if (i_is_odd && k_is_odd) {
1876  // Node on a Z-X face
1877  fine(i,j,k) += aa_interp_face_xz(crse,sig,i,j,k,ic,jc,kc);
1878  } else if (i_is_odd && j_is_odd) {
1879  // Node on a X-Y face
1880  fine(i,j,k) += aa_interp_face_xy(crse,sig,i,j,k,ic,jc,kc);
1881  } else if (i_is_odd) {
1882  // Node on X line
1883  fine(i,j,k) += aa_interp_line_x(crse,sig,i,j,k,ic,jc,kc);
1884  } else if (j_is_odd) {
1885  // Node on Y line
1886  fine(i,j,k) += aa_interp_line_y(crse,sig,i,j,k,ic,jc,kc);
1887  } else if (k_is_odd) {
1888  // Node on Z line
1889  fine(i,j,k) += aa_interp_line_z(crse,sig,i,j,k,ic,jc,kc);
1890  } else {
1891  // Node coincident with coarse node
1892  fine(i,j,k) += crse(ic,jc,kc);
1893  }
1894  }
1895 }
1896 
1897 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
1898 void mlndlap_semi_interpadd_aa (int i, int j, int k, Array4<Real> const& fine,
1899  Array4<Real const> const& crse, Array4<Real const> const& sig,
1900  Array4<int const> const& msk, int idir) noexcept
1901 {
1902  if (idir == 2 )
1903  {
1904  if (!msk(i,j,k)) {
1905  int ic = amrex::coarsen(i,2);
1906  int jc = amrex::coarsen(j,2);
1907  int kc = k;
1908  bool i_is_odd = (ic*2 != i);
1909  bool j_is_odd = (jc*2 != j);
1910 
1911  if (i_is_odd && j_is_odd) {
1912  // Node on a X-Y face
1913  fine(i,j,k) += aa_interp_face_xy(crse,sig,i,j,k,ic,jc,kc);
1914  } else if (i_is_odd) {
1915  // Node on X line
1916  fine(i,j,k) += aa_interp_line_x(crse,sig,i,j,k,ic,jc,kc);
1917  } else if (j_is_odd) {
1918  // Node on Y line
1919  fine(i,j,k) += aa_interp_line_y(crse,sig,i,j,k,ic,jc,kc);
1920  } else {
1921  // Node coincident with coarse node
1922  fine(i,j,k) += crse(ic,jc,kc);
1923  }
1924  }
1925  } else if (idir ==1 ){
1926  if (!msk(i,j,k)) {
1927  int ic = amrex::coarsen(i,2);
1928  int jc = j;
1929  int kc = amrex::coarsen(k,2);
1930  bool i_is_odd = (ic*2 != i);
1931  bool k_is_odd = (kc*2 != k);
1932 
1933  if (i_is_odd && k_is_odd) {
1934  // Node on a X-Z face
1935  fine(i,j,k) += aa_interp_face_xz(crse,sig,i,j,k,ic,jc,kc);
1936  } else if (i_is_odd) {
1937  // Node on X line
1938  fine(i,j,k) += aa_interp_line_x(crse,sig,i,j,k,ic,jc,kc);
1939  } else if (k_is_odd) {
1940  // Node on Z line
1941  fine(i,j,k) += aa_interp_line_z(crse,sig,i,j,k,ic,jc,kc);
1942  } else {
1943  // Node coincident with coarse node
1944  fine(i,j,k) += crse(ic,jc,kc);
1945  }
1946  }
1947  } else if (idir == 0 ) {
1948  if (!msk(i,j,k)) {
1949  int ic = i;
1950  int jc = amrex::coarsen(j,2);
1951  int kc = amrex::coarsen(k,2);
1952  bool j_is_odd = (jc*2 != j);
1953  bool k_is_odd = (kc*2 != k);
1954 
1955  if (j_is_odd && k_is_odd) {
1956  // Node on a Y-Z face
1957  fine(i,j,k) += aa_interp_face_yz(crse,sig,i,j,k,ic,jc,kc);
1958  } else if (j_is_odd) {
1959  // Node on Y line
1960  fine(i,j,k) += aa_interp_line_y(crse,sig,i,j,k,ic,jc,kc);
1961  } else if (k_is_odd) {
1962  // Node on Z line
1963  fine(i,j,k) += aa_interp_line_z(crse,sig,i,j,k,ic,jc,kc);
1964  } else {
1965  // Node coincident with coarse node
1966  fine(i,j,k) += crse(ic,jc,kc);
1967  }
1968  }
1969  } else {
1970  amrex::Abort("mlndlap_semi_interpolation semi direction wrong semi-direction. ");
1971  }
1972 }
1973 
1974 
1975  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
1976  Real ha_interp_face_xy (Array4<Real const> const& crse,
1977  Array4<Real const> const& sigx, Array4<Real const> const& sigy,
1978  int i, int j, int k, int ic, int jc, int kc) noexcept
1979  {
1980  Real w1 = sigx(i-1,j-1,k-1) + sigx(i-1,j,k-1) + sigx(i-1,j-1,k) + sigx(i-1,j,k);
1981  Real w2 = sigx(i ,j-1,k-1) + sigx(i ,j,k-1) + sigx(i ,j-1,k) + sigx(i ,j,k);
1982  Real w3 = sigy(i-1,j-1,k-1) + sigy(i,j-1,k-1) + sigy(i-1,j-1,k) + sigy(i,j-1,k);
1983  Real w4 = sigy(i-1,j ,k-1) + sigy(i,j ,k-1) + sigy(i-1,j ,k) + sigy(i,j ,k);
1984  return (w1 * aa_interp_line_y(crse,sigy,i-1,j ,k,ic ,jc ,kc) +
1985  w2 * aa_interp_line_y(crse,sigy,i+1,j ,k,ic+1,jc ,kc) +
1986  w3 * aa_interp_line_x(crse,sigx,i ,j-1,k,ic ,jc ,kc) +
1987  w4 * aa_interp_line_x(crse,sigx,i ,j+1,k,ic ,jc+1,kc)) / (w1+w2+w3+w4);
1988  }
1989 
1990  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
1991  Real ha_interp_face_xz (Array4<Real const> const& crse,
1992  Array4<Real const> const& sigx, Array4<Real const> const& sigz,
1993  int i, int j, int k, int ic, int jc, int kc) noexcept
1994  {
1995  Real w1 = sigx(i-1,j-1,k-1) + sigx(i-1,j,k-1) + sigx(i-1,j-1,k) + sigx(i-1,j,k);
1996  Real w2 = sigx(i ,j-1,k-1) + sigx(i ,j,k-1) + sigx(i ,j-1,k) + sigx(i ,j,k);
1997  Real w3 = sigz(i-1,j-1,k-1) + sigz(i,j-1,k-1) + sigz(i-1,j,k-1) + sigz(i,j,k-1);
1998  Real w4 = sigz(i-1,j-1,k ) + sigz(i,j-1,k ) + sigz(i-1,j,k ) + sigz(i,j,k );
1999  return (w1 * aa_interp_line_z(crse,sigz,i-1,j,k ,ic ,jc,kc ) +
2000  w2 * aa_interp_line_z(crse,sigz,i+1,j,k ,ic+1,jc,kc ) +
2001  w3 * aa_interp_line_x(crse,sigx,i ,j,k-1,ic ,jc,kc ) +
2002  w4 * aa_interp_line_x(crse,sigx,i ,j,k+1,ic ,jc,kc+1)) / (w1+w2+w3+w4);
2003  }
2004 
2005  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2006  Real ha_interp_face_yz (Array4<Real const> const& crse,
2007  Array4<Real const> const& sigy, Array4<Real const> const& sigz,
2008  int i, int j, int k, int ic, int jc, int kc) noexcept
2009  {
2010  Real w1 = sigy(i-1,j-1,k-1) + sigy(i,j-1,k-1) + sigy(i-1,j-1,k) + sigy(i,j-1,k);
2011  Real w2 = sigy(i-1,j ,k-1) + sigy(i,j ,k-1) + sigy(i-1,j ,k) + sigy(i,j ,k);
2012  Real w3 = sigz(i-1,j-1,k-1) + sigz(i,j-1,k-1) + sigz(i-1,j,k-1) + sigz(i,j,k-1);
2013  Real w4 = sigz(i-1,j-1,k ) + sigz(i,j-1,k ) + sigz(i-1,j,k ) + sigz(i,j,k );
2014  return (w1 * aa_interp_line_z(crse,sigz,i,j-1,k ,ic,jc ,kc ) +
2015  w2 * aa_interp_line_z(crse,sigz,i,j+1,k ,ic,jc+1,kc ) +
2016  w3 * aa_interp_line_y(crse,sigy,i,j ,k-1,ic,jc ,kc ) +
2017  w4 * aa_interp_line_y(crse,sigy,i,j ,k+1,ic,jc ,kc+1)) / (w1+w2+w3+w4);
2018  }
2019 
2020 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2021 void mlndlap_interpadd_ha (int i, int j, int k, Array4<Real> const& fine,
2022  Array4<Real const> const& crse, Array4<Real const> const& sigx,
2023  Array4<Real const> const& sigy, Array4<Real const> const& sigz,
2024  Array4<int const> const& msk) noexcept
2025 {
2026  if (!msk(i,j,k)) {
2027  int ic = amrex::coarsen(i,2);
2028  int jc = amrex::coarsen(j,2);
2029  int kc = amrex::coarsen(k,2);
2030  bool i_is_odd = (ic*2 != i);
2031  bool j_is_odd = (jc*2 != j);
2032  bool k_is_odd = (kc*2 != k);
2033  if (i_is_odd && j_is_odd && k_is_odd) {
2034  // Fine node at center of cell
2035  Real w1 = sigx(i-1,j-1,k-1) + sigx(i-1,j,k-1) + sigx(i-1,j-1,k) + sigx(i-1,j,k);
2036  Real w2 = sigx(i ,j-1,k-1) + sigx(i ,j,k-1) + sigx(i ,j-1,k) + sigx(i ,j,k);
2037  Real w3 = sigy(i-1,j-1,k-1) + sigy(i,j-1,k-1) + sigy(i-1,j-1,k) + sigy(i,j-1,k);
2038  Real w4 = sigy(i-1,j ,k-1) + sigy(i,j ,k-1) + sigy(i-1,j ,k) + sigy(i,j ,k);
2039  Real w5 = sigz(i-1,j-1,k-1) + sigz(i,j-1,k-1) + sigz(i-1,j,k-1) + sigz(i,j,k-1);
2040  Real w6 = sigz(i-1,j-1,k ) + sigz(i,j-1,k ) + sigz(i-1,j,k ) + sigz(i,j,k );
2041  fine(i,j,k) += (w1 * ha_interp_face_yz(crse,sigy,sigz,i-1,j ,k ,ic ,jc ,kc ) +
2042  w2 * ha_interp_face_yz(crse,sigy,sigz,i+1,j ,k ,ic+1,jc ,kc ) +
2043  w3 * ha_interp_face_xz(crse,sigx,sigz,i ,j-1,k ,ic ,jc ,kc ) +
2044  w4 * ha_interp_face_xz(crse,sigx,sigz,i ,j+1,k ,ic ,jc+1,kc ) +
2045  w5 * ha_interp_face_xy(crse,sigx,sigy,i ,j ,k-1,ic ,jc ,kc ) +
2046  w6 * ha_interp_face_xy(crse,sigx,sigy,i ,j ,k+1,ic ,jc ,kc+1))
2047  / (w1+w2+w3+w4+w5+w6);
2048  } else if (j_is_odd && k_is_odd) {
2049  // Node on a Y-Z face
2050  fine(i,j,k) += ha_interp_face_yz(crse,sigy,sigz,i,j,k,ic,jc,kc);
2051  } else if (i_is_odd && k_is_odd) {
2052  // Node on a Z-X face
2053  fine(i,j,k) += ha_interp_face_xz(crse,sigx,sigz,i,j,k,ic,jc,kc);
2054  } else if (i_is_odd && j_is_odd) {
2055  // Node on a X-Y face
2056  fine(i,j,k) += ha_interp_face_xy(crse,sigx,sigy,i,j,k,ic,jc,kc);
2057  } else if (i_is_odd) {
2058  // Node on X line
2059  fine(i,j,k) += aa_interp_line_x(crse,sigx,i,j,k,ic,jc,kc);
2060  } else if (j_is_odd) {
2061  // Node on Y line
2062  fine(i,j,k) += aa_interp_line_y(crse,sigy,i,j,k,ic,jc,kc);
2063  } else if (k_is_odd) {
2064  // Node on Z line
2065  fine(i,j,k) += aa_interp_line_z(crse,sigz,i,j,k,ic,jc,kc);
2066  } else {
2067  // Node coincident with coarse node
2068  fine(i,j,k) += crse(ic,jc,kc);
2069  }
2070  }
2071 }
2072 
2073 //
2074 // rhs & u
2075 //
2076 
2077 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2078 void mlndlap_divu (int i, int j, int k, Array4<Real> const& rhs, Array4<Real const> const& vel,
2079  Array4<int const> const& msk,
2080  GpuArray<Real,AMREX_SPACEDIM> const& dxinv,
2081  Box const& nodal_domain,
2082  GpuArray<LinOpBCType, AMREX_SPACEDIM> const& bclo,
2083  GpuArray<LinOpBCType, AMREX_SPACEDIM> const& bchi) noexcept
2084 {
2085  Real facx = Real(0.25)*dxinv[0];
2086  Real facy = Real(0.25)*dxinv[1];
2087  Real facz = Real(0.25)*dxinv[2];
2088 
2089  const auto domlo = amrex::lbound(nodal_domain);
2090  const auto domhi = amrex::ubound(nodal_domain);
2091 
2092  if (msk(i,j,k)) {
2093  rhs(i,j,k) = Real(0.0);
2094  } else {
2095 
2096  Real zero_ilo = Real(1.0);
2097  Real zero_ihi = Real(1.0);
2098  Real zero_jlo = Real(1.0);
2099  Real zero_jhi = Real(1.0);
2100  Real zero_klo = Real(1.0);
2101  Real zero_khi = Real(1.0);
2102 
2103  // The nodal divergence operator should not see the tangential velocity
2104  // at an inflow face
2105  if ((bclo[0] == LinOpBCType::Neumann || bclo[0] == LinOpBCType::inflow)
2106  && i == domlo.x)
2107  {
2108  zero_ilo = Real(0.0);
2109  }
2110  if ((bchi[0] == LinOpBCType::Neumann || bchi[0] == LinOpBCType::inflow)
2111  && i == domhi.x)
2112  {
2113  zero_ihi = Real(0.0);
2114  }
2115  if ((bclo[1] == LinOpBCType::Neumann || bclo[1] == LinOpBCType::inflow)
2116  && j == domlo.y)
2117  {
2118  zero_jlo = Real(0.0);
2119  }
2120  if ((bchi[1] == LinOpBCType::Neumann || bchi[1] == LinOpBCType::inflow)
2121  && j == domhi.y)
2122  {
2123  zero_jhi = Real(0.0);
2124  }
2125  if ((bclo[2] == LinOpBCType::Neumann || bclo[2] == LinOpBCType::inflow)
2126  && k == domlo.z)
2127  {
2128  zero_klo = Real(0.0);
2129  }
2130  if ((bchi[2] == LinOpBCType::Neumann || bchi[2] == LinOpBCType::inflow)
2131  && k == domhi.z)
2132  {
2133  zero_khi = Real(0.0);
2134  }
2135 
2136  rhs(i,j,k) = facx*(-vel(i-1,j-1,k-1,0)*zero_jlo*zero_klo+vel(i,j-1,k-1,0)*zero_jlo*zero_klo
2137  -vel(i-1,j ,k-1,0)*zero_jhi*zero_klo+vel(i,j ,k-1,0)*zero_jhi*zero_klo
2138  -vel(i-1,j-1,k ,0)*zero_jlo*zero_khi+vel(i,j-1,k ,0)*zero_jlo*zero_khi
2139  -vel(i-1,j ,k ,0)*zero_jhi*zero_khi+vel(i,j ,k ,0)*zero_jhi*zero_khi)
2140 
2141  + facy*(-vel(i-1,j-1,k-1,1)*zero_ilo*zero_klo-vel(i,j-1,k-1,1)*zero_ihi*zero_klo
2142  +vel(i-1,j ,k-1,1)*zero_ilo*zero_klo+vel(i,j ,k-1,1)*zero_ihi*zero_klo
2143  -vel(i-1,j-1,k ,1)*zero_ilo*zero_khi-vel(i,j-1,k ,1)*zero_ihi*zero_khi
2144  +vel(i-1,j ,k ,1)*zero_ilo*zero_khi+vel(i,j ,k ,1)*zero_ihi*zero_khi)
2145 
2146  + facz*(-vel(i-1,j-1,k-1,2)*zero_ilo*zero_jlo-vel(i,j-1,k-1,2)*zero_ihi*zero_jlo
2147  -vel(i-1,j ,k-1,2)*zero_ilo*zero_jhi-vel(i,j ,k-1,2)*zero_ihi*zero_jhi
2148  +vel(i-1,j-1,k ,2)*zero_ilo*zero_jlo+vel(i,j-1,k ,2)*zero_ihi*zero_jlo
2149  +vel(i-1,j ,k ,2)*zero_ilo*zero_jhi+vel(i,j ,k ,2)*zero_ihi*zero_jhi);
2150  }
2151 }
2152 
2153 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2154 Real mlndlap_rhcc (int i, int j, int k, Array4<Real const> const& rhcc,
2155  Array4<int const> const& msk) noexcept
2156 {
2157  Real r;
2158  if (msk(i,j,k)) {
2159  r = Real(0.0);
2160  } else {
2161  r = Real(0.125) * (rhcc(i-1,j-1,k-1)+rhcc(i,j-1,k-1)+rhcc(i-1,j,k-1)+rhcc(i,j,k-1) +
2162  rhcc(i-1,j-1,k )+rhcc(i,j-1,k )+rhcc(i-1,j,k )+rhcc(i,j,k ));
2163  }
2164  return r;
2165 }
2166 
2167 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2168 void mlndlap_mknewu (int i, int j, int k, Array4<Real> const& u, Array4<Real const> const& p,
2169  Array4<Real const> const& sig, GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
2170 {
2171  Real facx = Real(0.25)*dxinv[0];
2172  Real facy = Real(0.25)*dxinv[1];
2173  Real facz = Real(0.25)*dxinv[2];
2174  u(i,j,k,0) -= sig(i,j,k)*facx
2175  * (-p(i,j,k )+p(i+1,j,k )-p(i,j+1,k )+p(i+1,j+1,k )
2176  -p(i,j,k+1)+p(i+1,j,k+1)-p(i,j+1,k+1)+p(i+1,j+1,k+1));
2177  u(i,j,k,1) -= sig(i,j,k)*facy
2178  * (-p(i,j,k )-p(i+1,j,k )+p(i,j+1,k )+p(i+1,j+1,k )
2179  -p(i,j,k+1)-p(i+1,j,k+1)+p(i,j+1,k+1)+p(i+1,j+1,k+1));
2180  u(i,j,k,2) -= sig(i,j,k)*facz
2181  * (-p(i,j,k )-p(i+1,j,k )-p(i,j+1,k )-p(i+1,j+1,k )
2182  +p(i,j,k+1)+p(i+1,j,k+1)+p(i,j+1,k+1)+p(i+1,j+1,k+1));
2183 }
2184 
2185 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2186 void mlndlap_mknewu_c (int i, int j, int k, Array4<Real> const& u, Array4<Real const> const& p,
2187  Real sig, GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
2188 {
2189  Real facx = Real(0.25)*dxinv[0];
2190  Real facy = Real(0.25)*dxinv[1];
2191  Real facz = Real(0.25)*dxinv[2];
2192  u(i,j,k,0) -= sig*facx
2193  * (-p(i,j,k )+p(i+1,j,k )-p(i,j+1,k )+p(i+1,j+1,k )
2194  -p(i,j,k+1)+p(i+1,j,k+1)-p(i,j+1,k+1)+p(i+1,j+1,k+1));
2195  u(i,j,k,1) -= sig*facy
2196  * (-p(i,j,k )-p(i+1,j,k )+p(i,j+1,k )+p(i+1,j+1,k )
2197  -p(i,j,k+1)-p(i+1,j,k+1)+p(i,j+1,k+1)+p(i+1,j+1,k+1));
2198  u(i,j,k,2) -= sig*facz
2199  * (-p(i,j,k )-p(i+1,j,k )-p(i,j+1,k )-p(i+1,j+1,k )
2200  +p(i,j,k+1)+p(i+1,j,k+1)+p(i,j+1,k+1)+p(i+1,j+1,k+1));
2201 }
2202 
2203  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2204  Real mlndlap_sum_Df (int ii, int jj, int kk, Real facx, Real facy, Real facz,
2205  Array4<Real const> const& vel, Box const& velbx) noexcept
2206  {
2207  Real Df = Real(0.0);
2208  if (velbx.contains(ii-1,jj-1,kk-1)) {
2209  Df += -facx*vel(ii-1,jj-1,kk-1,0) - facy*vel(ii-1,jj-1,kk-1,1) - facz*vel(ii-1,jj-1,kk-1,2);
2210  }
2211  if (velbx.contains(ii,jj-1,kk-1)) {
2212  Df += facx*vel(ii,jj-1,kk-1,0) - facy*vel(ii,jj-1,kk-1,1) - facz*vel(ii,jj-1,kk-1,2);
2213  }
2214  if (velbx.contains(ii-1,jj,kk-1)) {
2215  Df += -facx*vel(ii-1,jj,kk-1,0) + facy*vel(ii-1,jj,kk-1,1) - facz*vel(ii-1,jj,kk-1,2);
2216  }
2217  if (velbx.contains(ii,jj,kk-1)) {
2218  Df += facx*vel(ii,jj,kk-1,0) + facy*vel(ii,jj,kk-1,1) - facz*vel(ii,jj,kk-1,2);
2219  }
2220  if (velbx.contains(ii-1,jj-1,kk)) {
2221  Df += -facx*vel(ii-1,jj-1,kk,0) - facy*vel(ii-1,jj-1,kk,1) + facz*vel(ii-1,jj-1,kk,2);
2222  }
2223  if (velbx.contains(ii,jj-1,kk)) {
2224  Df += facx*vel(ii,jj-1,kk,0) - facy*vel(ii,jj-1,kk,1) + facz*vel(ii,jj-1,kk,2);
2225  }
2226  if (velbx.contains(ii-1,jj,kk)) {
2227  Df += -facx*vel(ii-1,jj,kk,0) + facy*vel(ii-1,jj,kk,1) + facz*vel(ii-1,jj,kk,2);
2228  }
2229  if (velbx.contains(ii,jj,kk)) {
2230  Df += facx*vel(ii,jj,kk,0) + facy*vel(ii,jj,kk,1) + facz*vel(ii,jj,kk,2);
2231  }
2232  return Df;
2233  }
2234 
2235 template <int rr>
2236 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2237 void mlndlap_divu_fine_contrib (int i, int j, int k, Box const& fvbx, Box const& velbx,
2238  Array4<Real> const& rhs, Array4<Real const> const& vel,
2239  Array4<Real const> const& frhs, Array4<int const> const& msk,
2240  GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
2241 {
2242  const int ii = rr*i;
2243  const int jj = rr*j;
2244  const int kk = rr*k;
2245  if (msk(ii,jj,kk)) {
2246  const Real facx = Real(0.25)*dxinv[0];
2247  const Real facy = Real(0.25)*dxinv[1];
2248  const Real facz = Real(0.25)*dxinv[2];
2249 
2250  Real Df = Real(0.0);
2251 
2252  const int ilo = amrex::max(ii-rr+1, fvbx.smallEnd(0));
2253  const int ihi = amrex::min(ii+rr-1, fvbx.bigEnd (0));
2254  const int jlo = amrex::max(jj-rr+1, fvbx.smallEnd(1));
2255  const int jhi = amrex::min(jj+rr-1, fvbx.bigEnd (1));
2256  const int klo = amrex::max(kk-rr+1, fvbx.smallEnd(2));
2257  const int khi = amrex::min(kk+rr-1, fvbx.bigEnd (2));
2258 
2259  for (int koff = klo; koff <= khi; ++koff) {
2260  for (int joff = jlo; joff <= jhi; ++joff) {
2261  for (int ioff = ilo; ioff <= ihi; ++ioff) {
2262  Real scale = static_cast<Real>((rr-std::abs(ii-ioff)) *
2263  (rr-std::abs(jj-joff)) *
2264  (rr-std::abs(kk-koff)));
2265  if (fvbx.strictly_contains(ioff,joff,koff)) {
2266  Df += scale * frhs(ioff,joff,koff);
2267  } else {
2268  Df += scale * mlndlap_sum_Df(ioff, joff, koff, facx, facy, facz, vel, velbx);
2269  }
2270  }}}
2271 
2272  rhs(i,j,k) = Df * (Real(1.0)/static_cast<Real>(rr*rr*rr*rr*rr*rr));
2273  } else {
2274  rhs(i,j,k) = Real(0.0);
2275  }
2276 }
2277 
2278 template<int rr>
2279 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2280 void mlndlap_rhcc_fine_contrib (int i, int j, int k, Box const& ccbx,
2281  Array4<Real> const& rhs, Array4<Real const> const& cc,
2282  Array4<int const> const& msk) noexcept
2283 {
2284  const int ii = rr*i;
2285  const int jj = rr*j;
2286  const int kk = rr*k;
2287  if (msk(ii,jj,kk)) {
2288  Real tmp = Real(0.0);
2289 
2290  const int ilo = amrex::max(ii-rr , ccbx.smallEnd(0));
2291  const int ihi = amrex::min(ii+rr-1, ccbx.bigEnd (0));
2292  const int jlo = amrex::max(jj-rr , ccbx.smallEnd(1));
2293  const int jhi = amrex::min(jj+rr-1, ccbx.bigEnd (1));
2294  const int klo = amrex::max(kk-rr , ccbx.smallEnd(2));
2295  const int khi = amrex::min(kk+rr-1, ccbx.bigEnd (2));
2296 
2297  for (int koff = klo; koff <= khi; ++koff) {
2298  for (int joff = jlo; joff <= jhi; ++joff) {
2299  for (int ioff = ilo; ioff <= ihi; ++ioff) {
2300  Real scale = (static_cast<Real>(rr)-std::abs(static_cast<Real>(ioff-ii)+Real(0.5)))
2301  * (static_cast<Real>(rr)-std::abs(static_cast<Real>(joff-jj)+Real(0.5)))
2302  * (static_cast<Real>(rr)-std::abs(static_cast<Real>(koff-kk)+Real(0.5)));
2303  tmp += cc(ioff,joff,koff) * scale;
2304  }}}
2305 
2306  rhs(i,j,k) += tmp * (Real(1.0)/Real(rr*rr*rr*rr*rr*rr));
2307  }
2308 }
2309 
2310  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2311  Real neumann_scale (int i, int j, int k, Box const& nddom,
2312  GpuArray<LinOpBCType,AMREX_SPACEDIM> const& bclo,
2313  GpuArray<LinOpBCType,AMREX_SPACEDIM> const& bchi) noexcept
2314  {
2315  Real val = Real(1.0);
2316 
2317  const auto ndlo = amrex::lbound(nddom);
2318  const auto ndhi = amrex::ubound(nddom);
2319 
2320  if ((i == ndlo.x && ( bclo[0] == LinOpBCType::Neumann ||
2321  bclo[0] == LinOpBCType::inflow)) ||
2322  (i == ndhi.x && ( bchi[0] == LinOpBCType::Neumann ||
2323  bchi[0] == LinOpBCType::inflow))) {
2324  val *= Real(2.);
2325  }
2326 
2327  if ((j == ndlo.y && ( bclo[1] == LinOpBCType::Neumann ||
2328  bclo[1] == LinOpBCType::inflow)) ||
2329  (j == ndhi.y && ( bchi[1] == LinOpBCType::Neumann ||
2330  bchi[1] == LinOpBCType::inflow))) {
2331  val *= Real(2.);
2332  }
2333 
2334  if ((k == ndlo.z && ( bclo[2] == LinOpBCType::Neumann ||
2335  bclo[2] == LinOpBCType::inflow)) ||
2336  (k == ndhi.z && ( bchi[2] == LinOpBCType::Neumann ||
2337  bchi[2] == LinOpBCType::inflow))) {
2338  val *= Real(2.);
2339  }
2340 
2341  return val;
2342  }
2343 
2344 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2345 void mlndlap_divu_cf_contrib (int i, int j, int k, Array4<Real> const& rhs,
2346  Array4<Real const> const& vel, Array4<Real const> const& fc,
2347  Array4<Real const> const& rhcc, Array4<int const> const& dmsk,
2348  Array4<int const> const& ndmsk, Array4<int const> const& ccmsk,
2349  GpuArray<Real,AMREX_SPACEDIM> const& dxinv,
2350  Box const& ccdom_p, Box const& veldom, Box const& nddom,
2351  GpuArray<LinOpBCType,AMREX_SPACEDIM> const& bclo,
2352  GpuArray<LinOpBCType,AMREX_SPACEDIM> const& bchi) noexcept
2353 {
2354  using namespace nodelap_detail;
2355 
2356  if (!dmsk(i,j,k) && ndmsk(i,j,k) == crse_fine_node) {
2357  Real facx = Real(0.25) * dxinv[0];
2358  Real facy = Real(0.25) * dxinv[1];
2359  Real facz = Real(0.25) * dxinv[2];
2360  Real tmp = fc(i,j,k);
2361 
2362  // Where there is inflow, veldom there is bigger than ccdom_p by one cell.
2363  // ccdom_p is cc domain grown at periodic boundaries.
2364 
2365  if (ccmsk(i-1,j-1,k-1) == crse_cell && veldom.contains(i-1,j-1,k-1)) {
2366  tmp += -facx*vel(i-1,j-1,k-1,0) - facy*vel(i-1,j-1,k-1,1) - facz*vel(i-1,j-1,k-1,2);
2367  if (rhcc && ccdom_p.contains(i-1,j-1,k-1)) {
2368  tmp += Real(0.125) * rhcc(i-1,j-1,k-1);
2369  }
2370  }
2371 
2372  if (ccmsk(i,j-1,k-1) == crse_cell && veldom.contains(i,j-1,k-1)) {
2373  tmp += facx*vel(i,j-1,k-1,0) - facy*vel(i,j-1,k-1,1) - facz*vel(i,j-1,k-1,2);
2374  if (rhcc && ccdom_p.contains(i,j-1,k-1)) {
2375  tmp += Real(0.125) * rhcc(i,j-1,k-1);
2376  }
2377  }
2378 
2379  if (ccmsk(i-1,j,k-1) == crse_cell && veldom.contains(i-1,j,k-1)) {
2380  tmp += -facx*vel(i-1,j,k-1,0) + facy*vel(i-1,j,k-1,1) - facz*vel(i-1,j,k-1,2);
2381  if (rhcc && ccdom_p.contains(i-1,j,k-1)) {
2382  tmp += Real(0.125) * rhcc(i-1,j,k-1);
2383  }
2384  }
2385 
2386  if (ccmsk(i,j,k-1) == crse_cell && veldom.contains(i,j,k-1)) {
2387  tmp += facx*vel(i,j,k-1,0) + facy*vel(i,j,k-1,1) - facz*vel(i,j,k-1,2);
2388  if (rhcc && ccdom_p.contains(i,j,k-1)) {
2389  tmp += Real(0.125) * rhcc(i,j,k-1);
2390  }
2391  }
2392 
2393  if (ccmsk(i-1,j-1,k) == crse_cell && veldom.contains(i-1,j-1,k)) {
2394  tmp += -facx*vel(i-1,j-1,k,0) - facy*vel(i-1,j-1,k,1) + facz*vel(i-1,j-1,k,2);
2395  if (rhcc && ccdom_p.contains(i-1,j-1,k)) {
2396  tmp += Real(0.125) * rhcc(i-1,j-1,k);
2397  }
2398  }
2399 
2400  if (ccmsk(i,j-1,k) == crse_cell && veldom.contains(i,j-1,k)) {
2401  tmp += facx*vel(i,j-1,k,0) - facy*vel(i,j-1,k,1) + facz*vel(i,j-1,k,2);
2402  if (rhcc && ccdom_p.contains(i,j-1,k)) {
2403  tmp += Real(0.125) * rhcc(i,j-1,k);
2404  }
2405  }
2406 
2407  if (ccmsk(i-1,j,k) == crse_cell && veldom.contains(i-1,j,k)) {
2408  tmp += -facx*vel(i-1,j,k,0) + facy*vel(i-1,j,k,1) + facz*vel(i-1,j,k,2);
2409  if (rhcc && ccdom_p.contains(i-1,j,k)) {
2410  tmp += Real(0.125) * rhcc(i-1,j,k);
2411  }
2412  }
2413 
2414  if (ccmsk(i,j,k) == crse_cell && veldom.contains(i,j,k)) {
2415  tmp += facx*vel(i,j,k,0) + facy*vel(i,j,k,1) + facz*vel(i,j,k,2);
2416  if (rhcc && ccdom_p.contains(i,j,k)) {
2417  tmp += Real(0.125) * rhcc(i,j,k);
2418  }
2419  }
2420 
2421  rhs(i,j,k) = tmp * neumann_scale(i, j, k, nddom, bclo, bchi);
2422  }
2423 }
2424 
2425 //
2426 // residual
2427 //
2428 
2429 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2430 void mlndlap_crse_resid (int i, int j, int k, Array4<Real> const& resid,
2431  Array4<Real const> const& rhs, Array4<int const> const& msk,
2432  Box const& nddom, GpuArray<LinOpBCType,AMREX_SPACEDIM> const& bclo,
2433  GpuArray<LinOpBCType,AMREX_SPACEDIM> const& bchi,
2434  bool neumann_doubling) noexcept
2435 {
2436  if ( msk(i-1,j-1,k-1) == 0 ||
2437  msk(i ,j-1,k-1) == 0 ||
2438  msk(i-1,j ,k-1) == 0 ||
2439  msk(i ,j ,k-1) == 0 ||
2440  msk(i-1,j-1,k ) == 0 ||
2441  msk(i ,j-1,k ) == 0 ||
2442  msk(i-1,j ,k ) == 0 ||
2443  msk(i ,j ,k ) == 0 )
2444  {
2445  Real fac = Real(1.0);
2446  if (neumann_doubling) {
2447  const auto ndlo = amrex::lbound(nddom);
2448  const auto ndhi = amrex::ubound(nddom);
2449  if ((i == ndlo.x && ( bclo[0] == LinOpBCType::Neumann ||
2450  bclo[0] == LinOpBCType::inflow)) ||
2451  (i == ndhi.x && ( bchi[0] == LinOpBCType::Neumann ||
2452  bchi[0] == LinOpBCType::inflow))) {
2453  fac *= Real(2.);
2454  }
2455  if ((j == ndlo.y && ( bclo[1] == LinOpBCType::Neumann ||
2456  bclo[1] == LinOpBCType::inflow)) ||
2457  (j == ndhi.y && ( bchi[1] == LinOpBCType::Neumann ||
2458  bchi[1] == LinOpBCType::inflow))) {
2459  fac *= Real(2.);
2460  }
2461  if ((k == ndlo.z && ( bclo[2] == LinOpBCType::Neumann ||
2462  bclo[2] == LinOpBCType::inflow)) ||
2463  (k == ndhi.z && ( bchi[2] == LinOpBCType::Neumann ||
2464  bchi[2] == LinOpBCType::inflow))) {
2465  fac *= Real(2.);
2466  }
2467  }
2468  resid(i,j,k) = (rhs(i,j,k) - resid(i,j,k)) * fac;
2469  } else {
2470  resid(i,j,k) = Real(0.);
2471  }
2472 }
2473 
2474 //
2475 // sync residual
2476 //
2477 
2478  template <typename P, typename S>
2479  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2480  Real mlndlap_sum_Ax (P const& pred, S const& sig,
2481  int i, int j, int k, Real facx, Real facy, Real facz,
2482  Array4<Real const> const& phi) noexcept
2483  {
2484  Real Ax = Real(0.0);
2485  if (pred(i-1,j-1,k-1)) {
2486  Ax += sig(i-1,j-1,k-1)*(facx*(Real(4.)*(phi(i-1,j ,k )-phi(i ,j ,k ))
2487  +Real(2.)*(phi(i-1,j-1,k )-phi(i ,j-1,k ))
2488  +Real(2.)*(phi(i-1,j ,k-1)-phi(i ,j ,k-1))
2489  + (phi(i-1,j-1,k-1)-phi(i ,j-1,k-1)))
2490  + facy*(Real(4.)*(phi(i ,j-1,k )-phi(i ,j ,k ))
2491  +Real(2.)*(phi(i-1,j-1,k )-phi(i-1,j ,k ))
2492  +Real(2.)*(phi(i ,j-1,k-1)-phi(i ,j ,k-1))
2493  + (phi(i-1,j-1,k-1)-phi(i-1,j ,k-1)))
2494  + facz*(Real(4.)*(phi(i ,j ,k-1)-phi(i ,j ,k ))
2495  +Real(2.)*(phi(i-1,j ,k-1)-phi(i-1,j ,k ))
2496  +Real(2.)*(phi(i ,j-1,k-1)-phi(i ,j-1,k ))
2497  + (phi(i-1,j-1,k-1)-phi(i-1,j-1,k ))));
2498  }
2499  if (pred(i,j-1,k-1)) {
2500  Ax += sig(i,j-1,k-1)*(facx*(Real(4.)*(phi(i+1,j ,k )-phi(i ,j ,k ))
2501  +Real(2.)*(phi(i+1,j-1,k )-phi(i ,j-1,k ))
2502  +Real(2.)*(phi(i+1,j ,k-1)-phi(i ,j ,k-1))
2503  + (phi(i+1,j-1,k-1)-phi(i ,j-1,k-1)))
2504  + facy*(Real(4.)*(phi(i ,j-1,k )-phi(i ,j ,k ))
2505  +Real(2.)*(phi(i+1,j-1,k )-phi(i+1,j ,k ))
2506  +Real(2.)*(phi(i ,j-1,k-1)-phi(i ,j ,k-1))
2507  + (phi(i+1,j-1,k-1)-phi(i+1,j ,k-1)))
2508  + facz*(Real(4.)*(phi(i ,j ,k-1)-phi(i ,j ,k ))
2509  +Real(2.)*(phi(i+1,j ,k-1)-phi(i+1,j ,k ))
2510  +Real(2.)*(phi(i ,j-1,k-1)-phi(i ,j-1,k ))
2511  + (phi(i+1,j-1,k-1)-phi(i+1,j-1,k ))));
2512  }
2513  if (pred(i-1,j,k-1)) {
2514  Ax += sig(i-1,j,k-1)*(facx*(Real(4.)*(phi(i-1,j ,k )-phi(i ,j ,k ))
2515  +Real(2.)*(phi(i-1,j+1,k )-phi(i ,j+1,k ))
2516  +Real(2.)*(phi(i-1,j ,k-1)-phi(i ,j ,k-1))
2517  + (phi(i-1,j+1,k-1)-phi(i ,j+1,k-1)))
2518  + facy*(Real(4.)*(phi(i ,j+1,k )-phi(i ,j ,k ))
2519  +Real(2.)*(phi(i-1,j+1,k )-phi(i-1,j ,k ))
2520  +Real(2.)*(phi(i ,j+1,k-1)-phi(i ,j ,k-1))
2521  + (phi(i-1,j+1,k-1)-phi(i-1,j ,k-1)))
2522  + facz*(Real(4.)*(phi(i ,j ,k-1)-phi(i ,j ,k ))
2523  +Real(2.)*(phi(i-1,j ,k-1)-phi(i-1,j ,k ))
2524  +Real(2.)*(phi(i ,j+1,k-1)-phi(i ,j+1,k ))
2525  + (phi(i-1,j+1,k-1)-phi(i-1,j+1,k ))));
2526  }
2527  if (pred(i,j,k-1)) {
2528  Ax += sig(i,j,k-1)*(facx*(Real(4.)*(phi(i+1,j ,k )-phi(i ,j ,k ))
2529  +Real(2.)*(phi(i+1,j+1,k )-phi(i ,j+1,k ))
2530  +Real(2.)*(phi(i+1,j ,k-1)-phi(i ,j ,k-1))
2531  + (phi(i+1,j+1,k-1)-phi(i ,j+1,k-1)))
2532  + facy*(Real(4.)*(phi(i ,j+1,k )-phi(i ,j ,k ))
2533  +Real(2.)*(phi(i+1,j+1,k )-phi(i+1,j ,k ))
2534  +Real(2.)*(phi(i ,j+1,k-1)-phi(i ,j ,k-1))
2535  + (phi(i+1,j+1,k-1)-phi(i+1,j ,k-1)))
2536  + facz*(Real(4.)*(phi(i ,j ,k-1)-phi(i ,j ,k ))
2537  +Real(2.)*(phi(i+1,j ,k-1)-phi(i+1,j ,k ))
2538  +Real(2.)*(phi(i ,j+1,k-1)-phi(i ,j+1,k ))
2539  + (phi(i+1,j+1,k-1)-phi(i+1,j+1,k ))));
2540  }
2541  if (pred(i-1,j-1,k)) {
2542  Ax += sig(i-1,j-1,k)*(facx*(Real(4.)*(phi(i-1,j ,k )-phi(i ,j ,k ))
2543  +Real(2.)*(phi(i-1,j-1,k )-phi(i ,j-1,k ))
2544  +Real(2.)*(phi(i-1,j ,k+1)-phi(i ,j ,k+1))
2545  + (phi(i-1,j-1,k+1)-phi(i ,j-1,k+1)))
2546  + facy*(Real(4.)*(phi(i ,j-1,k )-phi(i ,j ,k ))
2547  +Real(2.)*(phi(i-1,j-1,k )-phi(i-1,j ,k ))
2548  +Real(2.)*(phi(i ,j-1,k+1)-phi(i ,j ,k+1))
2549  + (phi(i-1,j-1,k+1)-phi(i-1,j ,k+1)))
2550  + facz*(Real(4.)*(phi(i ,j ,k+1)-phi(i ,j ,k ))
2551  +Real(2.)*(phi(i-1,j ,k+1)-phi(i-1,j ,k ))
2552  +Real(2.)*(phi(i ,j-1,k+1)-phi(i ,j-1,k ))
2553  + (phi(i-1,j-1,k+1)-phi(i-1,j-1,k ))));
2554  }
2555  if (pred(i,j-1,k)) {
2556  Ax += sig(i,j-1,k)*(facx*(Real(4.)*(phi(i+1,j ,k )-phi(i ,j ,k ))
2557  +Real(2.)*(phi(i+1,j-1,k )-phi(i ,j-1,k ))
2558  +Real(2.)*(phi(i+1,j ,k+1)-phi(i ,j ,k+1))
2559  + (phi(i+1,j-1,k+1)-phi(i ,j-1,k+1)))
2560  + facy*(Real(4.)*(phi(i ,j-1,k )-phi(i ,j ,k ))
2561  +Real(2.)*(phi(i+1,j-1,k )-phi(i+1,j ,k ))
2562  +Real(2.)*(phi(i ,j-1,k+1)-phi(i ,j ,k+1))
2563  + (phi(i+1,j-1,k+1)-phi(i+1,j ,k+1)))
2564  + facz*(Real(4.)*(phi(i ,j ,k+1)-phi(i ,j ,k ))
2565  +Real(2.)*(phi(i+1,j ,k+1)-phi(i+1,j ,k ))
2566  +Real(2.)*(phi(i ,j-1,k+1)-phi(i ,j-1,k ))
2567  + (phi(i+1,j-1,k+1)-phi(i+1,j-1,k ))));
2568  }
2569  if (pred(i-1,j,k)) {
2570  Ax += sig(i-1,j,k)*(facx*(Real(4.)*(phi(i-1,j ,k )-phi(i ,j ,k ))
2571  +Real(2.)*(phi(i-1,j+1,k )-phi(i ,j+1,k ))
2572  +Real(2.)*(phi(i-1,j ,k+1)-phi(i ,j ,k+1))
2573  + (phi(i-1,j+1,k+1)-phi(i ,j+1,k+1)))
2574  + facy*(Real(4.)*(phi(i ,j+1,k )-phi(i ,j ,k ))
2575  +Real(2.)*(phi(i-1,j+1,k )-phi(i-1,j ,k ))
2576  +Real(2.)*(phi(i ,j+1,k+1)-phi(i ,j ,k+1))
2577  + (phi(i-1,j+1,k+1)-phi(i-1,j ,k+1)))
2578  + facz*(Real(4.)*(phi(i ,j ,k+1)-phi(i ,j ,k ))
2579  +Real(2.)*(phi(i-1,j ,k+1)-phi(i-1,j ,k ))
2580  +Real(2.)*(phi(i ,j+1,k+1)-phi(i ,j+1,k ))
2581  + (phi(i-1,j+1,k+1)-phi(i-1,j+1,k ))));
2582  }
2583  if (pred(i,j,k)) {
2584  Ax += sig(i,j,k)*(facx*(Real(4.)*(phi(i+1,j ,k )-phi(i ,j ,k ))
2585  +Real(2.)*(phi(i+1,j+1,k )-phi(i ,j+1,k ))
2586  +Real(2.)*(phi(i+1,j ,k+1)-phi(i ,j ,k+1))
2587  + (phi(i+1,j+1,k+1)-phi(i ,j+1,k+1)))
2588  + facy*(Real(4.)*(phi(i ,j+1,k )-phi(i ,j ,k ))
2589  +Real(2.)*(phi(i+1,j+1,k )-phi(i+1,j ,k ))
2590  +Real(2.)*(phi(i ,j+1,k+1)-phi(i ,j ,k+1))
2591  + (phi(i+1,j+1,k+1)-phi(i+1,j ,k+1)))
2592  + facz*(Real(4.)*(phi(i ,j ,k+1)-phi(i ,j ,k ))
2593  +Real(2.)*(phi(i+1,j ,k+1)-phi(i+1,j ,k ))
2594  +Real(2.)*(phi(i ,j+1,k+1)-phi(i ,j+1,k ))
2595  + (phi(i+1,j+1,k+1)-phi(i+1,j+1,k ))));
2596  }
2597  return Ax;
2598  }
2599 
2600  template <int rr, typename S>
2601  AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2602  void mlndlap_Ax_fine_contrib_doit (S const& sig,
2603  int i, int j, int k, Box const& ndbx, Box const& ccbx,
2604  Array4<Real> const& f, Array4<Real const> const& res,
2605  Array4<Real const> const& rhs,
2606  Array4<Real const> const& phi,
2607  Array4<int const> const& msk,
2608  GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
2609  {
2610  const int ii = rr*i;
2611  const int jj = rr*j;
2612  const int kk = rr*k;
2613  if (msk(ii,jj,kk)) {
2614  Real facx = Real(1./36.)*dxinv[0]*dxinv[0];
2615  Real facy = Real(1./36.)*dxinv[1]*dxinv[1];
2616  Real facz = Real(1./36.)*dxinv[2]*dxinv[2];
2617 
2618  auto is_fine = [&ccbx] (int ix, int iy, int iz) -> bool {
2619  return ccbx.contains(ix,iy,iz);
2620  };
2621 
2622  Real Df = Real(0.0);
2623 
2624  const int ilo = amrex::max(ii-rr+1, ndbx.smallEnd(0));
2625  const int ihi = amrex::min(ii+rr-1, ndbx.bigEnd (0));
2626  const int jlo = amrex::max(jj-rr+1, ndbx.smallEnd(1));
2627  const int jhi = amrex::min(jj+rr-1, ndbx.bigEnd (1));
2628  const int klo = amrex::max(kk-rr+1, ndbx.smallEnd(2));
2629  const int khi = amrex::min(kk+rr-1, ndbx.bigEnd (2));
2630 
2631  for (int koff = klo; koff <= khi; ++koff) {
2632  for (int joff = jlo; joff <= jhi; ++joff) {
2633  for (int ioff = ilo; ioff <= ihi; ++ioff) {
2634  Real scale = static_cast<Real>((rr-std::abs(ii-ioff)) *
2635  (rr-std::abs(jj-joff)) *
2636  (rr-std::abs(kk-koff)));
2637  if (ndbx.strictly_contains(ioff,joff,koff)) {
2638  Df += scale * (rhs(ioff,joff,koff)-res(ioff,joff,koff));
2639  } else {
2640  Df += scale * mlndlap_sum_Ax
2641  (is_fine, sig, ioff, joff, koff, facx, facy, facz, phi);
2642  }
2643  }}}
2644 
2645  f(i,j,k) = Df * (Real(1.0)/static_cast<Real>(rr*rr*rr*rr*rr*rr));
2646  } else {
2647  f(i,j,k) = Real(0.0);
2648  }
2649  }
2650 
2651 template <int rr>
2652 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2653 void mlndlap_Ax_fine_contrib (int i, int j, int k, Box const& ndbx, Box const& ccbx,
2654  Array4<Real> const& f, Array4<Real const> const& res,
2655  Array4<Real const> const& rhs, Array4<Real const> const& phi,
2656  Array4<Real const> const& sig, Array4<int const> const& msk,
2657  GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
2658 {
2659  mlndlap_Ax_fine_contrib_doit<rr>
2660  ([&sig] (int ix, int iy, int iz) -> Real const& { return sig(ix,iy,iz); },
2661  i,j,k,ndbx,ccbx,f,res,rhs,phi,msk,dxinv);
2662 }
2663 
2664 template <int rr>
2665 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2666 void mlndlap_Ax_fine_contrib_cs (int i, int j, int k, Box const& ndbx, Box const& ccbx,
2667  Array4<Real> const& f, Array4<Real const> const& res,
2668  Array4<Real const> const& rhs, Array4<Real const> const& phi,
2669  Real const sig, Array4<int const> const& msk,
2670  GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
2671 {
2672  mlndlap_Ax_fine_contrib_doit<rr>
2673  ([=] (int, int, int) -> Real { return sig; },
2674  i,j,k,ndbx,ccbx,f,res,rhs,phi,msk,dxinv);
2675 }
2676 
2677 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2678 void mlndlap_res_cf_contrib (int i, int j, int k, Array4<Real> const& res,
2679  Array4<Real const> const& phi, Array4<Real const> const& rhs,
2680  Array4<Real const> const& sig, Array4<int const> const& dmsk,
2681  Array4<int const> const& ndmsk, Array4<int const> const& ccmsk,
2682  Array4<Real const> const& fc,
2683  GpuArray<Real,AMREX_SPACEDIM> const& dxinv,
2684  Box const& ccdom_p, Box const& nddom,
2685  GpuArray<LinOpBCType, AMREX_SPACEDIM> const& bclo,
2686  GpuArray<LinOpBCType, AMREX_SPACEDIM> const& bchi,
2687  bool neumann_doubling) noexcept
2688 {
2689  using namespace nodelap_detail;
2690 
2691  if (!dmsk(i,j,k) && ndmsk(i,j,k) == crse_fine_node) {
2692  Real facx = Real(1./36.)*dxinv[0]*dxinv[0];
2693  Real facy = Real(1./36.)*dxinv[1]*dxinv[1];
2694  Real facz = Real(1./36.)*dxinv[2]*dxinv[2];
2695 
2696  Real Ax = mlndlap_sum_Ax([&ccmsk, &ccdom_p] (int ix, int iy, int iz) -> bool
2697  {
2698  return ccdom_p.contains(ix,iy,iz)
2699  && (ccmsk(ix,iy,iz) == crse_cell);
2700  },
2701  [&sig] (int ix, int iy, int iz) -> Real const&
2702  {
2703  return sig(ix,iy,iz);
2704  },
2705  i, j, k, facx, facy, facz, phi);
2706  Ax += fc(i,j,k);
2707  Real const ns = (neumann_doubling) ? neumann_scale(i,j,k,nddom,bclo,bchi) : Real(1.0);
2708  res(i,j,k) = rhs(i,j,k) - Ax*ns;
2709  }
2710 }
2711 
2712 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2713 void mlndlap_res_cf_contrib_cs (int i, int j, int k, Array4<Real> const& res,
2714  Array4<Real const> const& phi, Array4<Real const> const& rhs,
2715  Real const sig, Array4<int const> const& dmsk,
2716  Array4<int const> const& ndmsk, Array4<int const> const& ccmsk,
2717  Array4<Real const> const& fc,
2718  GpuArray<Real,AMREX_SPACEDIM> const& dxinv,
2719  Box const& ccdom_p, Box const& nddom,
2720  GpuArray<LinOpBCType, AMREX_SPACEDIM> const& bclo,
2721  GpuArray<LinOpBCType, AMREX_SPACEDIM> const& bchi,
2722  bool neumann_doubling) noexcept
2723 {
2724  using namespace nodelap_detail;
2725 
2726  if (!dmsk(i,j,k) && ndmsk(i,j,k) == crse_fine_node) {
2727  Real facx = Real(1./36.)*dxinv[0]*dxinv[0];
2728  Real facy = Real(1./36.)*dxinv[1]*dxinv[1];
2729  Real facz = Real(1./36.)*dxinv[2]*dxinv[2];
2730 
2731  Real Ax = mlndlap_sum_Ax([&ccmsk, &ccdom_p] (int ix, int iy, int iz) -> bool
2732  {
2733  return ccdom_p.contains(ix,iy,iz)
2734  && (ccmsk(ix,iy,iz) == crse_cell);
2735  },
2736  [=] (int, int, int) -> Real
2737  {
2738  return sig;
2739  },
2740  i, j, k, facx, facy, facz, phi);
2741  Ax += fc(i,j,k);
2742  Real const ns = (neumann_doubling) ? neumann_scale(i,j,k,nddom,bclo,bchi) : Real(1.0);
2743  res(i,j,k) = rhs(i,j,k) - Ax*ns;
2744  }
2745 }
2746 
2747 //
2748 // RAP
2749 //
2750 
2751 namespace nodelap_detail {
2752 
2753  constexpr int ist_000 = 0;
2754  constexpr int ist_p00 = 1;
2755  constexpr int ist_0p0 = 2;
2756  constexpr int ist_00p = 3;
2757  constexpr int ist_pp0 = 4;
2758  constexpr int ist_p0p = 5;
2759  constexpr int ist_0pp = 6;
2760  constexpr int ist_ppp = 7;
2761  constexpr int ist_inv = 8;
2762  constexpr int n_sten = 9;
2763 }
2764 
2765 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2766 void mlndlap_set_stencil (Box const& bx, Array4<Real> const& sten,
2767  Array4<Real const> const& sig,
2768  GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
2769 {
2770  using namespace nodelap_detail;
2771 
2772  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0];
2773  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1];
2774  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2];
2775  Real fxyz = facx + facy + facz;
2776  Real fmx2y2z = -facx + Real(2.0)*facy + Real(2.0)*facz;
2777  Real f2xmy2z = Real(2.0)*facx - facy + Real(2.0)*facz;
2778  Real f2x2ymz = Real(2.0)*facx + Real(2.0)*facy - facz;
2779  Real f4xm2ym2z = Real(4.0)*facx - Real(2.0)*facy - Real(2.0)*facz;
2780  Real fm2x4ym2z = -Real(2.0)*facx + Real(4.0)*facy - Real(2.0)*facz;
2781  Real fm2xm2y4z = -Real(2.0)*facx - Real(2.0)*facy + Real(4.0)*facz;
2782 
2783  amrex::LoopConcurrent(bx, [=] (int i, int j, int k) noexcept
2784  {
2785  // i+1,j,k
2786  sten(i,j,k,ist_p00) = f4xm2ym2z * (sig(i,j-1,k-1)+sig(i,j,k-1)+sig(i,j-1,k)+sig(i,j,k));
2787  // i-1,j,k: sten(i-1,j,k,ist_p00)
2788 
2789  // i,j+1,k
2790  sten(i,j,k,ist_0p0) = fm2x4ym2z * (sig(i-1,j,k-1)+sig(i,j,k-1)+sig(i-1,j,k)+sig(i,j,k));
2791  // i,j-1,k: sten(i,j-1,k,ist_0p0)
2792 
2793  // i,j,k+1
2794  sten(i,j,k,ist_00p) = fm2xm2y4z * (sig(i-1,j-1,k)+sig(i,j-1,k)+sig(i-1,j,k)+sig(i,j,k));
2795  // i,j,k-1: sten(i,j,k-1,ist_00p)
2796 
2797  // i+1,j+1,k
2798  sten(i,j,k,ist_pp0) = f2x2ymz * (sig(i,j,k-1)+sig(i,j,k));
2799  // i-1,j-1,k: sten(i-1,j-1,k,ist_pp0)
2800  // i+1,j-1,k: sten(i ,j-1,k,ist_pp0)
2801  // i-1,j+1,k: sten(i-1,j ,k,ist_pp0)
2802 
2803  // i+1,j,k+1
2804  sten(i,j,k,ist_p0p) = f2xmy2z * (sig(i,j-1,k)+sig(i,j,k));
2805  // i-1,j,k-1: sten(i-1,j,k-1,ist_p0p)
2806  // i+1,j,k-1: sten(i ,j,k-1,ist_p0p)
2807  // i-1,j,k+1: sten(i-1,j,k ,ist_p0p)
2808 
2809  // i,j+1,k+1
2810  sten(i,j,k,ist_0pp) = fmx2y2z * (sig(i-1,j,k)+sig(i,j,k));
2811  // i,j-1,k-1: sten(i,j-1,k-1,ist_0pp)
2812  // i,j+1,k-1: sten(i,j ,k-1,ist_0pp)
2813  // i,j-1,k+1: sten(i,j-1,k ,ist_0pp)
2814 
2815  // i+1,j+1,k+1
2816  sten(i,j,k,ist_ppp) = fxyz * sig(i,j,k);
2817  // i-1,j-1,k-1: sten(i-1,j-1,k-1,ist_ppp)
2818  // i+1,j-1,k-1: sten(i ,j-1,k-1,ist_ppp)
2819  // i-1,j+1,k-1: sten(i-1,j ,k-1,ist_ppp)
2820  // i+1,j+1,k-1: sten(i ,j ,k-1,ist_ppp)
2821  // i-1,j-1,k+1: sten(i-1,j-1,k ,ist_ppp)
2822  // i+1,j-1,k+1: sten(i ,j-1,k ,ist_ppp)
2823  // i-1,j+1,k+1: sten(i-1,j ,k ,ist_ppp)
2824  });
2825 }
2826 
2827 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2828 void mlndlap_set_stencil_s0 (int i, int j, int k, Array4<Real> const& sten) noexcept
2829 {
2830  using namespace nodelap_detail;
2831 
2832  sten(i,j,k,ist_000) = -(sten(i-1,j,k,ist_p00) + sten(i,j,k,ist_p00)
2833  + sten(i,j-1,k,ist_0p0) + sten(i,j,k,ist_0p0)
2834  + sten(i,j,k-1,ist_00p) + sten(i,j,k,ist_00p)
2835  + sten(i-1,j-1,k,ist_pp0) + sten(i,j-1,k,ist_pp0)
2836  + sten(i-1,j,k,ist_pp0) + sten(i,j,k,ist_pp0)
2837  + sten(i-1,j,k-1,ist_p0p) + sten(i,j,k-1,ist_p0p)
2838  + sten(i-1,j,k,ist_p0p) + sten(i,j,k,ist_p0p)
2839  + sten(i,j-1,k-1,ist_0pp) + sten(i,j,k-1,ist_0pp)
2840  + sten(i,j-1,k,ist_0pp) + sten(i,j,k,ist_0pp)
2841  + sten(i-1,j-1,k-1,ist_ppp) + sten(i,j-1,k-1,ist_ppp)
2842  + sten(i-1,j,k-1,ist_ppp) + sten(i,j,k-1,ist_ppp)
2843  + sten(i-1,j-1,k,ist_ppp) + sten(i,j-1,k,ist_ppp)
2844  + sten(i-1,j,k,ist_ppp) + sten(i,j,k,ist_ppp));
2845  sten(i,j,k,ist_inv) = Real(1.0) /
2846  ( std::abs(sten(i-1,j,k,ist_p00)) + std::abs(sten(i,j,k,ist_p00))
2847  + std::abs(sten(i,j-1,k,ist_0p0)) + std::abs(sten(i,j,k,ist_0p0))
2848  + std::abs(sten(i,j,k-1,ist_00p)) + std::abs(sten(i,j,k,ist_00p))
2849  + std::abs(sten(i-1,j-1,k,ist_pp0)) + std::abs(sten(i,j-1,k,ist_pp0))
2850  + std::abs(sten(i-1,j,k,ist_pp0)) + std::abs(sten(i,j,k,ist_pp0))
2851  + std::abs(sten(i-1,j,k-1,ist_p0p)) + std::abs(sten(i,j,k-1,ist_p0p))
2852  + std::abs(sten(i-1,j,k,ist_p0p)) + std::abs(sten(i,j,k,ist_p0p))
2853  + std::abs(sten(i,j-1,k-1,ist_0pp)) + std::abs(sten(i,j,k-1,ist_0pp))
2854  + std::abs(sten(i,j-1,k,ist_0pp)) + std::abs(sten(i,j,k,ist_0pp))
2855  + std::abs(sten(i-1,j-1,k-1,ist_ppp)) + std::abs(sten(i,j-1,k-1,ist_ppp))
2856  + std::abs(sten(i-1,j,k-1,ist_ppp)) + std::abs(sten(i,j,k-1,ist_ppp))
2857  + std::abs(sten(i-1,j-1,k,ist_ppp)) + std::abs(sten(i,j-1,k,ist_ppp))
2858  + std::abs(sten(i-1,j,k,ist_ppp)) + std::abs(sten(i,j,k,ist_ppp)) + eps);
2859 }
2860 
2861 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
2862 void mlndlap_stencil_rap (int i, int j, int k, Array4<Real> const& csten,
2863  Array4<Real const> const& fsten) noexcept
2864 {
2865  using namespace nodelap_detail;
2866 
2867  auto interp_from_mmm_to = [&fsten] (int i_, int j_, int k_) -> Real {
2868  Real p = Real(1.);
2869  p += std::abs(fsten(i_-1,j_ ,k_ ,ist_p00)) /
2870  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
2871  + std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp))
2872  + std::abs(fsten(i_-1,j_-1,k_ ,ist_ppp))
2873  + std::abs(fsten(i_-1,j_ ,k_ ,ist_ppp)) + eps);
2874  p += std::abs(fsten(i_ ,j_-1,k_ ,ist_0p0)) /
2875  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
2876  + std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp))
2877  + std::abs(fsten(i_-1,j_-1,k_ ,ist_ppp))
2878  + std::abs(fsten(i_ ,j_-1,k_ ,ist_ppp)) + eps);
2879  p += std::abs(fsten(i_ ,j_ ,k_-1,ist_00p)) /
2880  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
2881  + std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp))
2882  + std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp))
2883  + std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp)) + eps);
2884  p += std::abs(fsten(i_-1,j_-1,k_ ,ist_pp0)) /
2885  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
2886  + std::abs(fsten(i_-1,j_-1,k_ ,ist_ppp)) + eps);
2887  p += std::abs(fsten(i_-1,j_ ,k_-1,ist_p0p)) /
2888  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
2889  + std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp)) + eps);
2890  p += std::abs(fsten(i_ ,j_-1,k_-1,ist_0pp)) /
2891  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
2892  + std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp)) + eps);
2893  p *= std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp)) * fsten(i_,j_,k_,ist_inv);
2894  return p;
2895  };
2896  amrex::ignore_unused(interp_from_mmm_to);
2897 
2898  auto interp_from_pmm_to = [&fsten] (int i_, int j_, int k_) -> Real {
2899  Real p = Real(1.);
2900  p += std::abs(fsten(i_ ,j_ ,k_ ,ist_p00)) /
2901  ( std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp))
2902  + std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp))
2903  + std::abs(fsten(i_ ,j_-1,k_ ,ist_ppp))
2904  + std::abs(fsten(i_ ,j_ ,k_ ,ist_ppp)) + eps);
2905  p += std::abs(fsten(i_ ,j_-1,k_ ,ist_0p0)) /
2906  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
2907  + std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp))
2908  + std::abs(fsten(i_-1,j_-1,k_ ,ist_ppp))
2909  + std::abs(fsten(i_ ,j_-1,k_ ,ist_ppp)) + eps);
2910  p += std::abs(fsten(i_ ,j_ ,k_-1,ist_00p)) /
2911  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
2912  + std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp))
2913  + std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp))
2914  + std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp)) + eps);
2915  p += std::abs(fsten(i_ ,j_-1,k_ ,ist_pp0)) /
2916  ( std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp))
2917  + std::abs(fsten(i_ ,j_-1,k_ ,ist_ppp)) + eps);
2918  p += std::abs(fsten(i_ ,j_ ,k_-1,ist_p0p)) /
2919  ( std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp))
2920  + std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp)) + eps);
2921  p += std::abs(fsten(i_ ,j_-1,k_-1,ist_0pp)) /
2922  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
2923  + std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp)) + eps);
2924  p *= std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp)) * fsten(i_,j_,k_,ist_inv);
2925  return p;
2926  };
2927 
2928  auto interp_from_mpm_to = [&fsten] (int i_, int j_, int k_) -> Real {
2929  Real p = Real(1.);
2930  p += std::abs(fsten(i_-1,j_ ,k_ ,ist_p00)) /
2931  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
2932  + std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp))
2933  + std::abs(fsten(i_-1,j_-1,k_ ,ist_ppp))
2934  + std::abs(fsten(i_-1,j_ ,k_ ,ist_ppp)) + eps);
2935  p += std::abs(fsten(i_ ,j_ ,k_ ,ist_0p0)) /
2936  ( std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp))
2937  + std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp))
2938  + std::abs(fsten(i_-1,j_ ,k_ ,ist_ppp))
2939  + std::abs(fsten(i_ ,j_ ,k_ ,ist_ppp)) + eps);
2940  p += std::abs(fsten(i_ ,j_ ,k_-1,ist_00p)) /
2941  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
2942  + std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp))
2943  + std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp))
2944  + std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp)) + eps);
2945  p += std::abs(fsten(i_-1,j_ ,k_ ,ist_pp0)) /
2946  ( std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp))
2947  + std::abs(fsten(i_-1,j_ ,k_ ,ist_ppp)) + eps);
2948  p += std::abs(fsten(i_-1,j_ ,k_-1,ist_p0p)) /
2949  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
2950  + std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp)) + eps);
2951  p += std::abs(fsten(i_ ,j_ ,k_-1,ist_0pp)) /
2952  ( std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp))
2953  + std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp)) + eps);
2954  p *= std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp)) * fsten(i_,j_,k_,ist_inv);
2955  return p;
2956  };
2957 
2958  auto interp_from_ppm_to = [&fsten] (int i_, int j_, int k_) -> Real {
2959  Real p = Real(1.);
2960  p += std::abs(fsten(i_ ,j_ ,k_ ,ist_p00)) /
2961  ( std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp))
2962  + std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp))
2963  + std::abs(fsten(i_ ,j_-1,k_ ,ist_ppp))
2964  + std::abs(fsten(i_ ,j_ ,k_ ,ist_ppp)) + eps);
2965  p += std::abs(fsten(i_ ,j_ ,k_ ,ist_0p0)) /
2966  ( std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp))
2967  + std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp))
2968  + std::abs(fsten(i_-1,j_ ,k_ ,ist_ppp))
2969  + std::abs(fsten(i_ ,j_ ,k_ ,ist_ppp)) + eps);
2970  p += std::abs(fsten(i_ ,j_ ,k_-1,ist_00p)) /
2971  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
2972  + std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp))
2973  + std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp))
2974  + std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp)) + eps);
2975  p += std::abs(fsten(i_ ,j_ ,k_ ,ist_pp0)) /
2976  ( std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp))
2977  + std::abs(fsten(i_ ,j_ ,k_ ,ist_ppp)) + eps);
2978  p += std::abs(fsten(i_ ,j_ ,k_-1,ist_p0p)) /
2979  ( std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp))
2980  + std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp)) + eps);
2981  p += std::abs(fsten(i_ ,j_ ,k_-1,ist_0pp)) /
2982  ( std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp))
2983  + std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp)) + eps);
2984  p *= std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp)) * fsten(i_,j_,k_,ist_inv);
2985  return p;
2986  };
2987 
2988  auto interp_from_mmp_to = [&fsten] (int i_, int j_, int k_) -> Real {
2989  Real p = Real(1.);
2990  p += std::abs(fsten(i_-1,j_ ,k_ ,ist_p00)) /
2991  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
2992  + std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp))
2993  + std::abs(fsten(i_-1,j_-1,k_ ,ist_ppp))
2994  + std::abs(fsten(i_-1,j_ ,k_ ,ist_ppp)) + eps);
2995  p += std::abs(fsten(i_ ,j_-1,k_ ,ist_0p0)) /
2996  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
2997  + std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp))
2998  + std::abs(fsten(i_-1,j_-1,k_ ,ist_ppp))
2999  + std::abs(fsten(i_ ,j_-1,k_ ,ist_ppp)) + eps);
3000  p += std::abs(fsten(i_ ,j_ ,k_ ,ist_00p)) /
3001  ( std::abs(fsten(i_-1,j_-1,k_,ist_ppp))
3002  + std::abs(fsten(i_ ,j_-1,k_,ist_ppp))
3003  + std::abs(fsten(i_-1,j_ ,k_,ist_ppp))
3004  + std::abs(fsten(i_ ,j_ ,k_,ist_ppp)) + eps);
3005  p += std::abs(fsten(i_-1,j_-1,k_,ist_pp0)) /
3006  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
3007  + std::abs(fsten(i_-1,j_-1,k_ ,ist_ppp)) + eps);
3008  p += std::abs(fsten(i_-1,j_ ,k_,ist_p0p)) /
3009  ( std::abs(fsten(i_-1,j_-1,k_,ist_ppp))
3010  + std::abs(fsten(i_-1,j_ ,k_,ist_ppp)) + eps);
3011  p += std::abs(fsten(i_ ,j_-1,k_,ist_0pp)) /
3012  ( std::abs(fsten(i_-1,j_-1,k_,ist_ppp))
3013  + std::abs(fsten(i_ ,j_-1,k_,ist_ppp)) + eps);
3014  p *= std::abs(fsten(i_-1,j_-1,k_ ,ist_ppp)) * fsten(i_,j_,k_,ist_inv);
3015  return p;
3016  };
3017 
3018  auto interp_from_pmp_to = [&fsten] (int i_, int j_, int k_) -> Real {
3019  Real p = Real(1.);
3020  p += std::abs(fsten(i_ ,j_ ,k_,ist_p00)) /
3021  ( std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp))
3022  + std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp))
3023  + std::abs(fsten(i_ ,j_-1,k_ ,ist_ppp))
3024  + std::abs(fsten(i_ ,j_ ,k_ ,ist_ppp)) + eps);
3025  p += std::abs(fsten(i_ ,j_-1,k_,ist_0p0)) /
3026  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
3027  + std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp))
3028  + std::abs(fsten(i_-1,j_-1,k_ ,ist_ppp))
3029  + std::abs(fsten(i_ ,j_-1,k_ ,ist_ppp)) + eps);
3030  p += std::abs(fsten(i_ ,j_ ,k_,ist_00p)) /
3031  ( std::abs(fsten(i_-1,j_-1,k_,ist_ppp))
3032  + std::abs(fsten(i_ ,j_-1,k_,ist_ppp))
3033  + std::abs(fsten(i_-1,j_ ,k_,ist_ppp))
3034  + std::abs(fsten(i_ ,j_ ,k_,ist_ppp)) + eps);
3035  p += std::abs(fsten(i_ ,j_-1,k_,ist_pp0)) /
3036  ( std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp))
3037  + std::abs(fsten(i_ ,j_-1,k_ ,ist_ppp)) + eps);
3038  p += std::abs(fsten(i_ ,j_ ,k_,ist_p0p)) /
3039  ( std::abs(fsten(i_ ,j_-1,k_,ist_ppp))
3040  + std::abs(fsten(i_ ,j_ ,k_,ist_ppp)) + eps);
3041  p += std::abs(fsten(i_ ,j_-1,k_,ist_0pp)) /
3042  ( std::abs(fsten(i_-1,j_-1,k_,ist_ppp))
3043  + std::abs(fsten(i_ ,j_-1,k_,ist_ppp)) + eps);
3044  p *= std::abs(fsten(i_ ,j_-1,k_ ,ist_ppp)) * fsten(i_,j_ ,k_,ist_inv);
3045  return p;
3046  };
3047 
3048  auto interp_from_mpp_to = [&fsten] (int i_, int j_, int k_) -> Real {
3049  Real p = Real(1.);
3050  p += std::abs(fsten(i_-1,j_ ,k_ ,ist_p00)) /
3051  ( std::abs(fsten(i_-1,j_-1,k_-1,ist_ppp))
3052  + std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp))
3053  + std::abs(fsten(i_-1,j_-1,k_ ,ist_ppp))
3054  + std::abs(fsten(i_-1,j_ ,k_ ,ist_ppp)) + eps);
3055  p += std::abs(fsten(i_ ,j_ ,k_ ,ist_0p0)) /
3056  ( std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp))
3057  + std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp))
3058  + std::abs(fsten(i_-1,j_ ,k_ ,ist_ppp))
3059  + std::abs(fsten(i_ ,j_ ,k_ ,ist_ppp)) + eps);
3060  p += std::abs(fsten(i_ ,j_ ,k_ ,ist_00p)) /
3061  ( std::abs(fsten(i_-1,j_-1,k_ ,ist_ppp))
3062  + std::abs(fsten(i_ ,j_-1,k_ ,ist_ppp))
3063  + std::abs(fsten(i_-1,j_ ,k_ ,ist_ppp))
3064  + std::abs(fsten(i_ ,j_ ,k_ ,ist_ppp)) + eps);
3065  p += std::abs(fsten(i_-1,j_ ,k_ ,ist_pp0)) /
3066  ( std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp))
3067  + std::abs(fsten(i_-1,j_ ,k_ ,ist_ppp)) + eps);
3068  p += std::abs(fsten(i_-1,j_ ,k_ ,ist_p0p)) /
3069  ( std::abs(fsten(i_-1,j_-1,k_ ,ist_ppp))
3070  + std::abs(fsten(i_-1,j_ ,k_ ,ist_ppp)) + eps);
3071  p += std::abs(fsten(i_ ,j_ ,k_ ,ist_0pp)) /
3072  ( std::abs(fsten(i_-1,j_ ,k_ ,ist_ppp))
3073  + std::abs(fsten(i_ ,j_ ,k_ ,ist_ppp)) + eps);
3074  p *= std::abs(fsten(i_-1,j_ ,k_ ,ist_ppp)) * fsten(i_,j_,k_,ist_inv);
3075  return p;
3076  };
3077 
3078  auto interp_from_ppp_to = [&fsten] (int i_, int j_, int k_) -> Real {
3079  Real p = Real(1.);
3080  p += std::abs(fsten(i_ ,j_ ,k_ ,ist_p00)) /
3081  ( std::abs(fsten(i_ ,j_-1,k_-1,ist_ppp))
3082  + std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp))
3083  + std::abs(fsten(i_ ,j_-1,k_ ,ist_ppp))
3084  + std::abs(fsten(i_ ,j_ ,k_ ,ist_ppp)) + eps);
3085  p += std::abs(fsten(i_ ,j_ ,k_ ,ist_0p0)) /
3086  ( std::abs(fsten(i_-1,j_ ,k_-1,ist_ppp))
3087  + std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp))
3088  + std::abs(fsten(i_-1,j_ ,k_ ,ist_ppp))
3089  + std::abs(fsten(i_ ,j_ ,k_ ,ist_ppp)) + eps);
3090  p += std::abs(fsten(i_ ,j_ ,k_ ,ist_00p)) /
3091  ( std::abs(fsten(i_-1,j_-1,k_ ,ist_ppp))
3092  + std::abs(fsten(i_ ,j_-1,k_ ,ist_ppp))
3093  + std::abs(fsten(i_-1,j_ ,k_ ,ist_ppp))
3094  + std::abs(fsten(i_ ,j_ ,k_ ,ist_ppp)) + eps);
3095  p += std::abs(fsten(i_ ,j_ ,k_ ,ist_pp0)) /
3096  ( std::abs(fsten(i_ ,j_ ,k_-1,ist_ppp))
3097  + std::abs(fsten(i_ ,j_ ,k_ ,ist_ppp)) + eps);
3098  p += std::abs(fsten(i_ ,j_ ,k_ ,ist_p0p)) /
3099  ( std::abs(fsten(i_ ,j_-1,k_ ,ist_ppp))
3100  + std::abs(fsten(i_ ,j_ ,k_ ,ist_ppp)) + eps);
3101  p += std::abs(fsten(i_ ,j_ ,k_ ,ist_0pp)) /
3102  ( std::abs(fsten(i_-1,j_ ,k_ ,ist_ppp))
3103  + std::abs(fsten(i_ ,j_ ,k_ ,ist_ppp)) + eps);
3104  p *= std::abs(fsten(i_ ,j_ ,k_,ist_ppp)) * fsten(i_,j_,k_,ist_inv);
3105  return p;
3106  };
3107 
3108  auto interp_from_0mm_to = [&fsten] (int i_, int j_, int k_) -> Real {
3109  Real w1m = std::abs(fsten(i_ ,j_-1,k_ ,ist_0p0)) / (std::abs(fsten(i_ ,j_-1,k_-1,ist_0pp))
3110  +std::abs(fsten(i_ ,j_-1,k_ ,ist_0pp)) + eps);
3111  Real w1p = std::abs(fsten(i_ ,j_ ,k_ ,ist_0p0)) / (std::abs(fsten(i_ ,j_ ,k_-1,ist_0pp))
3112  +std::abs(fsten(i_ ,j_ ,k_ ,ist_0pp)) + eps);
3113  Real w2m = std::abs(fsten(i_ ,j_ ,k_-1,ist_00p)) / (std::abs(fsten(i_ ,j_-1,k_-1,ist_0pp))
3114  +std::abs(fsten(i_ ,j_ ,k_-1,ist_0pp)) + eps);
3115  Real w2p = std::abs(fsten(i_ ,j_ ,k_ ,ist_00p)) / (std::abs(fsten(i_ ,j_-1,k_ ,ist_0pp))
3116  +std::abs(fsten(i_ ,j_ ,k_ ,ist_0pp)) + eps);
3117  Real wmm = std::abs(fsten(i_ ,j_-1,k_-1,ist_0pp)) * (Real(1.) + w1m + w2m);
3118  Real wpm = std::abs(fsten(i_ ,j_ ,k_-1,ist_0pp)) * (Real(1.) + w1p + w2m);
3119  Real wmp = std::abs(fsten(i_ ,j_-1,k_ ,ist_0pp)) * (Real(1.) + w1m + w2p);
3120  Real wpp = std::abs(fsten(i_ ,j_ ,k_ ,ist_0pp)) * (Real(1.) + w1p + w2p);
3121  return wmm / (wmm+wpm+wmp+wpp+eps);
3122  };
3123 
3124  auto interp_from_0mp_to = [&fsten] (int i_, int j_, int k_) -> Real {
3125  Real w1m = std::abs(fsten(i_ ,j_-1,k_ ,ist_0p0)) / (std::abs(fsten(i_ ,j_-1,k_-1,ist_0pp))
3126  +std::abs(fsten(i_ ,j_-1,k_ ,ist_0pp)) + eps);
3127  Real w1p = std::abs(fsten(i_ ,j_ ,k_ ,ist_0p0)) / (std::abs(fsten(i_ ,j_ ,k_-1,ist_0pp))
3128  +std::abs(fsten(i_ ,j_ ,k_ ,ist_0pp)) + eps);
3129  Real w2m = std::abs(fsten(i_ ,j_ ,k_-1,ist_00p)) / (std::abs(fsten(i_ ,j_-1,k_-1,ist_0pp))
3130  +std::abs(fsten(i_ ,j_ ,k_-1,ist_0pp)) + eps);
3131  Real w2p = std::abs(fsten(i_ ,j_ ,k_ ,ist_00p)) / (std::abs(fsten(i_ ,j_-1,k_ ,ist_0pp))
3132  +std::abs(fsten(i_ ,j_ ,k_ ,ist_0pp)) + eps);
3133  Real wmm = std::abs(fsten(i_ ,j_-1,k_-1,ist_0pp)) * (Real(1.) + w1m + w2m);
3134  Real wpm = std::abs(fsten(i_ ,j_ ,k_-1,ist_0pp)) * (Real(1.) + w1p + w2m);
3135  Real wmp = std::abs(fsten(i_ ,j_-1,k_ ,ist_0pp)) * (Real(1.) + w1m + w2p);
3136  Real wpp = std::abs(fsten(i_ ,j_ ,k_ ,ist_0pp)) * (Real(1.) + w1p + w2p);
3137  return wmp / (wmm+wpm+wmp+wpp+eps);
3138  };
3139 
3140  auto interp_from_0pm_to = [&fsten] (int i_, int j_, int k_) -> Real {
3141  Real w1m = std::abs(fsten(i_ ,j_-1,k_ ,ist_0p0)) / (std::abs(fsten(i_ ,j_-1,k_-1,ist_0pp))
3142  +std::abs(fsten(i_ ,j_-1,k_ ,ist_0pp)) + eps);
3143  Real w1p = std::abs(fsten(i_ ,j_ ,k_ ,ist_0p0)) / (std::abs(fsten(i_ ,j_ ,k_-1,ist_0pp))
3144  +std::abs(fsten(i_ ,j_ ,k_ ,ist_0pp)) + eps);
3145  Real w2m = std::abs(fsten(i_ ,j_ ,k_-1,ist_00p)) / (std::abs(fsten(i_ ,j_-1,k_-1,ist_0pp))
3146  +std::abs(fsten(i_ ,j_ ,k_-1,ist_0pp)) + eps);
3147  Real w2p = std::abs(fsten(i_ ,j_ ,k_ ,ist_00p)) / (std::abs(fsten(i_ ,j_-1,k_ ,ist_0pp))
3148  +std::abs(fsten(i_ ,j_ ,k_ ,ist_0pp)) + eps);
3149  Real wmm = std::abs(fsten(i_ ,j_-1,k_-1,ist_0pp)) * (Real(1.) + w1m + w2m);
3150  Real wpm = std::abs(fsten(i_ ,j_ ,k_-1,ist_0pp)) * (Real(1.) + w1p + w2m);
3151  Real wmp = std::abs(fsten(i_ ,j_-1,k_ ,ist_0pp)) * (Real(1.) + w1m + w2p);
3152  Real wpp = std::abs(fsten(i_ ,j_ ,k_ ,ist_0pp)) * (Real(1.) + w1p + w2p);
3153  return wpm / (wmm+wpm+wmp+wpp+eps);
3154  };
3155 
3156  auto interp_from_0pp_to = [&fsten] (int i_, int j_, int k_) -> Real {
3157  Real w1m = std::abs(fsten(i_ ,j_-1,k_ ,ist_0p0)) / (std::abs(fsten(i_ ,j_-1,k_-1,ist_0pp))
3158  +std::abs(fsten(i_ ,j_-1,k_ ,ist_0pp)) + eps);
3159  Real w1p = std::abs(fsten(i_ ,j_ ,k_ ,ist_0p0)) / (std::abs(fsten(i_ ,j_ ,k_-1,ist_0pp))
3160  +std::abs(fsten(i_ ,j_ ,k_ ,ist_0pp)) + eps);
3161  Real w2m = std::abs(fsten(i_ ,j_ ,k_-1,ist_00p)) / (std::abs(fsten(i_ ,j_-1,k_-1,ist_0pp))
3162  +std::abs(fsten(i_ ,j_ ,k_-1,ist_0pp)) + eps);
3163  Real w2p = std::abs(fsten(i_ ,j_ ,k_ ,ist_00p)) / (std::abs(fsten(i_ ,j_-1,k_ ,ist_0pp))
3164  +std::abs(fsten(i_ ,j_ ,k_ ,ist_0pp)) + eps);
3165  Real wmm = std::abs(fsten(i_ ,j_-1,k_-1,ist_0pp)) * (Real(1.) + w1m + w2m);
3166  Real wpm = std::abs(fsten(i_ ,j_ ,k_-1,ist_0pp)) * (Real(1.) + w1p + w2m);
3167  Real wmp = std::abs(fsten(i_ ,j_-1,k_ ,ist_0pp)) * (Real(1.) + w1m + w2p);
3168  Real wpp = std::abs(fsten(i_ ,j_ ,k_ ,ist_0pp)) * (Real(1.) + w1p + w2p);
3169  return wpp / (wmm+wpm+wmp+wpp+eps);
3170  };
3171 
3172  auto interp_from_m0m_to = [&fsten] (int i_, int j_, int k_) -> Real {
3173  Real w1m = std::abs(fsten(i_-1,j_ ,k_ ,ist_p00)) / (std::abs(fsten(i_-1,j_ ,k_-1,ist_p0p))
3174  +std::abs(fsten(i_-1,j_ ,k_ ,ist_p0p)) + eps);
3175  Real w1p = std::abs(fsten(i_ ,j_ ,k_ ,ist_p00)) / (std::abs(fsten(i_ ,j_ ,k_-1,ist_p0p))
3176  +std::abs(fsten(i_ ,j_ ,k_ ,ist_p0p)) + eps);
3177  Real w2m = std::abs(fsten(i_ ,j_ ,k_-1,ist_00p)) / (std::abs(fsten(i_-1,j_ ,k_-1,ist_p0p))
3178  +std::abs(fsten(i_ ,j_ ,k_-1,ist_p0p)) + eps);
3179  Real w2p = std::abs(fsten(i_ ,j_ ,k_ ,ist_00p)) / (std::abs(fsten(i_-1,j_ ,k_ ,ist_p0p))
3180  +std::abs(fsten(i_ ,j_ ,k_ ,ist_p0p)) + eps);
3181  Real wmm = std::abs(fsten(i_-1,j_ ,k_-1,ist_p0p)) * (Real(1.) + w1m + w2m);
3182  Real wpm = std::abs(fsten(i_ ,j_ ,k_-1,ist_p0p)) * (Real(1.) + w1p + w2m);
3183  Real wmp = std::abs(fsten(i_-1,j_ ,k_ ,ist_p0p)) * (Real(1.) + w1m + w2p);
3184  Real wpp = std::abs(fsten(i_ ,j_ ,k_ ,ist_p0p)) * (Real(1.) + w1p + w2p);
3185  return wmm / (wmm+wpm+wmp+wpp+eps);
3186  };
3187 
3188  auto interp_from_p0m_to = [&fsten] (int i_, int j_, int k_) -> Real {
3189  Real w1m = std::abs(fsten(i_-1,j_ ,k_ ,ist_p00)) / (std::abs(fsten(i_-1,j_ ,k_-1,ist_p0p))
3190  +std::abs(fsten(i_-1,j_ ,k_ ,ist_p0p)) + eps);
3191  Real w1p = std::abs(fsten(i_ ,j_ ,k_ ,ist_p00)) / (std::abs(fsten(i_ ,j_ ,k_-1,ist_p0p))
3192  +std::abs(fsten(i_ ,j_ ,k_ ,ist_p0p)) + eps);
3193  Real w2m = std::abs(fsten(i_ ,j_ ,k_-1,ist_00p)) / (std::abs(fsten(i_-1,j_ ,k_-1,ist_p0p))
3194  +std::abs(fsten(i_ ,j_ ,k_-1,ist_p0p)) + eps);
3195  Real w2p = std::abs(fsten(i_ ,j_ ,k_ ,ist_00p)) / (std::abs(fsten(i_-1,j_ ,k_ ,ist_p0p))
3196  +std::abs(fsten(i_ ,j_ ,k_ ,ist_p0p)) + eps);
3197  Real wmm = std::abs(fsten(i_-1,j_ ,k_-1,ist_p0p)) * (Real(1.) + w1m + w2m);
3198  Real wpm = std::abs(fsten(i_ ,j_ ,k_-1,ist_p0p)) * (Real(1.) + w1p + w2m);
3199  Real wmp = std::abs(fsten(i_-1,j_ ,k_ ,ist_p0p)) * (Real(1.) + w1m + w2p);
3200  Real wpp = std::abs(fsten(i_ ,j_ ,k_ ,ist_p0p)) * (Real(1.) + w1p + w2p);
3201  return wpm / (wmm+wpm+wmp+wpp+eps);
3202  };
3203 
3204  auto interp_from_m0p_to = [&fsten] (int i_, int j_, int k_) -> Real {
3205  Real w1m = std::abs(fsten(i_-1,j_ ,k_ ,ist_p00)) / (std::abs(fsten(i_-1,j_ ,k_-1,ist_p0p))
3206  +std::abs(fsten(i_-1,j_ ,k_ ,ist_p0p)) + eps);
3207  Real w1p = std::abs(fsten(i_ ,j_ ,k_ ,ist_p00)) / (std::abs(fsten(i_ ,j_ ,k_-1,ist_p0p))
3208  +std::abs(fsten(i_ ,j_ ,k_ ,ist_p0p)) + eps);
3209  Real w2m = std::abs(fsten(i_ ,j_ ,k_-1,ist_00p)) / (std::abs(fsten(i_-1,j_ ,k_-1,ist_p0p))
3210  +std::abs(fsten(i_ ,j_ ,k_-1,ist_p0p)) + eps);
3211  Real w2p = std::abs(fsten(i_ ,j_ ,k_ ,ist_00p)) / (std::abs(fsten(i_-1,j_ ,k_ ,ist_p0p))
3212  +std::abs(fsten(i_ ,j_ ,k_ ,ist_p0p)) + eps);
3213  Real wmm = std::abs(fsten(i_-1,j_ ,k_-1,ist_p0p)) * (Real(1.) + w1m + w2m);
3214  Real wpm = std::abs(fsten(i_ ,j_ ,k_-1,ist_p0p)) * (Real(1.) + w1p + w2m);
3215  Real wmp = std::abs(fsten(i_-1,j_ ,k_ ,ist_p0p)) * (Real(1.) + w1m + w2p);
3216  Real wpp = std::abs(fsten(i_ ,j_ ,k_ ,ist_p0p)) * (Real(1.) + w1p + w2p);
3217  return wmp / (wmm+wpm+wmp+wpp+eps);
3218  };
3219 
3220  auto interp_from_p0p_to = [&fsten] (int i_, int j_, int k_) -> Real {
3221  Real w1m = std::abs(fsten(i_-1,j_ ,k_ ,ist_p00)) / (std::abs(fsten(i_-1,j_ ,k_-1,ist_p0p))
3222  +std::abs(fsten(i_-1,j_ ,k_ ,ist_p0p)) + eps);
3223  Real w1p = std::abs(fsten(i_ ,j_ ,k_ ,ist_p00)) / (std::abs(fsten(i_ ,j_ ,k_-1,ist_p0p))
3224  +std::abs(fsten(i_ ,j_ ,k_ ,ist_p0p)) + eps);
3225  Real w2m = std::abs(fsten(i_ ,j_ ,k_-1,ist_00p)) / (std::abs(fsten(i_-1,j_ ,k_-1,ist_p0p))
3226  +std::abs(fsten(i_ ,j_ ,k_-1,ist_p0p)) + eps);
3227  Real w2p = std::abs(fsten(i_ ,j_ ,k_ ,ist_00p)) / (std::abs(fsten(i_-1,j_ ,k_ ,ist_p0p))
3228  +std::abs(fsten(i_ ,j_ ,k_ ,ist_p0p)) + eps);
3229  Real wmm = std::abs(fsten(i_-1,j_ ,k_-1,ist_p0p)) * (Real(1.) + w1m + w2m);
3230  Real wpm = std::abs(fsten(i_ ,j_ ,k_-1,ist_p0p)) * (Real(1.) + w1p + w2m);
3231  Real wmp = std::abs(fsten(i_-1,j_ ,k_ ,ist_p0p)) * (Real(1.) + w1m + w2p);
3232  Real wpp = std::abs(fsten(i_ ,j_ ,k_ ,ist_p0p)) * (Real(1.) + w1p + w2p);
3233  return wpp / (wmm+wpm+wmp+wpp+eps);
3234  };
3235 
3236  auto interp_from_mm0_to = [&fsten] (int i_, int j_, int k_) -> Real {
3237  Real w1m = std::abs(fsten(i_-1,j_ ,k_ ,ist_p00)) / (std::abs(fsten(i_-1,j_-1,k_ ,ist_pp0))
3238  +std::abs(fsten(i_-1,j_ ,k_ ,ist_pp0)) + eps);
3239  Real w1p = std::abs(fsten(i_ ,j_ ,k_ ,ist_p00)) / (std::abs(fsten(i_ ,j_-1,k_ ,ist_pp0))
3240  +std::abs(fsten(i_ ,j_ ,k_ ,ist_pp0)) + eps);
3241  Real w2m = std::abs(fsten(i_ ,j_-1,k_ ,ist_0p0)) / (std::abs(fsten(i_-1,j_-1,k_ ,ist_pp0))
3242  +std::abs(fsten(i_ ,j_-1,k_ ,ist_pp0)) + eps);
3243  Real w2p = std::abs(fsten(i_ ,j_ ,k_ ,ist_0p0)) / (std::abs(fsten(i_-1,j_ ,k_ ,ist_pp0))
3244  +std::abs(fsten(i_ ,j_ ,k_ ,ist_pp0)) + eps);
3245  Real wmm = std::abs(fsten(i_-1,j_-1,k_ ,ist_pp0)) * (Real(1.) + w1m + w2m);
3246  Real wpm = std::abs(fsten(i_ ,j_-1,k_ ,ist_pp0)) * (Real(1.) + w1p + w2m);
3247  Real wmp = std::abs(fsten(i_-1,j_ ,k_ ,ist_pp0)) * (Real(1.) + w1m + w2p);
3248  Real wpp = std::abs(fsten(i_ ,j_ ,k_ ,ist_pp0)) * (Real(1.) + w1p + w2p);
3249  return wmm / (wmm+wpm+wmp+wpp+eps);
3250  };
3251 
3252  auto interp_from_mp0_to = [&fsten] (int i_, int j_, int k_) -> Real {
3253  Real w1m = std::abs(fsten(i_-1,j_ ,k_ ,ist_p00)) / (std::abs(fsten(i_-1,j_-1,k_ ,ist_pp0))
3254  +std::abs(fsten(i_-1,j_ ,k_ ,ist_pp0)) + eps);
3255  Real w1p = std::abs(fsten(i_ ,j_ ,k_ ,ist_p00)) / (std::abs(fsten(i_ ,j_-1,k_ ,ist_pp0))
3256  +std::abs(fsten(i_ ,j_ ,k_ ,ist_pp0)) + eps);
3257  Real w2m = std::abs(fsten(i_ ,j_-1,k_ ,ist_0p0)) / (std::abs(fsten(i_-1,j_-1,k_ ,ist_pp0))
3258  +std::abs(fsten(i_ ,j_-1,k_ ,ist_pp0)) + eps);
3259  Real w2p = std::abs(fsten(i_ ,j_ ,k_ ,ist_0p0)) / (std::abs(fsten(i_-1,j_ ,k_ ,ist_pp0))
3260  +std::abs(fsten(i_ ,j_ ,k_ ,ist_pp0)) + eps);
3261  Real wmm = std::abs(fsten(i_-1,j_-1,k_ ,ist_pp0)) * (Real(1.) + w1m + w2m);
3262  Real wpm = std::abs(fsten(i_ ,j_-1,k_ ,ist_pp0)) * (Real(1.) + w1p + w2m);
3263  Real wmp = std::abs(fsten(i_-1,j_ ,k_ ,ist_pp0)) * (Real(1.) + w1m + w2p);
3264  Real wpp = std::abs(fsten(i_ ,j_ ,k_ ,ist_pp0)) * (Real(1.) + w1p + w2p);
3265  return wmp / (wmm+wpm+wmp+wpp+eps);
3266  };
3267 
3268  auto interp_from_pm0_to = [&fsten] (int i_, int j_, int k_) -> Real {
3269  Real w1m = std::abs(fsten(i_-1,j_ ,k_ ,ist_p00)) / (std::abs(fsten(i_-1,j_-1,k_ ,ist_pp0))
3270  +std::abs(fsten(i_-1,j_ ,k_ ,ist_pp0)) + eps);
3271  Real w1p = std::abs(fsten(i_ ,j_ ,k_ ,ist_p00)) / (std::abs(fsten(i_ ,j_-1,k_ ,ist_pp0))
3272  +std::abs(fsten(i_ ,j_ ,k_ ,ist_pp0)) + eps);
3273  Real w2m = std::abs(fsten(i_ ,j_-1,k_ ,ist_0p0)) / (std::abs(fsten(i_-1,j_-1,k_ ,ist_pp0))
3274  +std::abs(fsten(i_ ,j_-1,k_ ,ist_pp0)) + eps);
3275  Real w2p = std::abs(fsten(i_ ,j_ ,k_ ,ist_0p0)) / (std::abs(fsten(i_-1,j_ ,k_ ,ist_pp0))
3276  +std::abs(fsten(i_ ,j_ ,k_ ,ist_pp0)) + eps);
3277  Real wmm = std::abs(fsten(i_-1,j_-1,k_ ,ist_pp0)) * (Real(1.) + w1m + w2m);
3278  Real wpm = std::abs(fsten(i_ ,j_-1,k_ ,ist_pp0)) * (Real(1.) + w1p + w2m);
3279  Real wmp = std::abs(fsten(i_-1,j_ ,k_ ,ist_pp0)) * (Real(1.) + w1m + w2p);
3280  Real wpp = std::abs(fsten(i_ ,j_ ,k_ ,ist_pp0)) * (Real(1.) + w1p + w2p);
3281  return wpm / (wmm+wpm+wmp+wpp+eps);
3282  };
3283 
3284  auto interp_from_pp0_to = [&fsten] (int i_, int j_, int k_) -> Real {
3285  Real w1m = std::abs(fsten(i_-1,j_ ,k_ ,ist_p00)) / (std::abs(fsten(i_-1,j_-1,k_ ,ist_pp0))
3286  +std::abs(fsten(i_-1,j_ ,k_ ,ist_pp0)) + eps);
3287  Real w1p = std::abs(fsten(i_ ,j_ ,k_ ,ist_p00)) / (std::abs(fsten(i_ ,j_-1,k_ ,ist_pp0))
3288  +std::abs(fsten(i_ ,j_ ,k_ ,ist_pp0)) + eps);
3289  Real w2m = std::abs(fsten(i_ ,j_-1,k_ ,ist_0p0)) / (std::abs(fsten(i_-1,j_-1,k_ ,ist_pp0))
3290  +std::abs(fsten(i_ ,j_-1,k_ ,ist_pp0)) + eps);
3291  Real w2p = std::abs(fsten(i_ ,j_ ,k_ ,ist_0p0)) / (std::abs(fsten(i_-1,j_ ,k_ ,ist_pp0))
3292  +std::abs(fsten(i_ ,j_ ,k_ ,ist_pp0)) + eps);
3293  Real wmm = std::abs(fsten(i_-1,j_-1,k_ ,ist_pp0)) * (Real(1.) + w1m + w2m);
3294  Real wpm = std::abs(fsten(i_ ,j_-1,k_ ,ist_pp0)) * (Real(1.) + w1p + w2m);
3295  Real wmp = std::abs(fsten(i_-1,j_ ,k_ ,ist_pp0)) * (Real(1.) + w1m + w2p);
3296  Real wpp = std::abs(fsten(i_ ,j_ ,k_ ,ist_pp0)) * (Real(1.) + w1p + w2p);
3297  return wpp / (wmm+wpm+wmp+wpp+eps);
3298  };
3299 
3300  auto interp_from_00m_to = [&fsten] (int i_, int j_, int k_) -> Real {
3301  Real w1 = std::abs(fsten(i_ ,j_ ,k_-1,ist_00p));
3302  Real w2 = std::abs(fsten(i_ ,j_ ,k_ ,ist_00p));
3303  if (w1 == Real(0.) && w2 == Real(0.)) {
3304  return Real(0.5);
3305  } else {
3306  return w1 / (w1+w2);
3307  }
3308  };
3309 
3310  auto interp_from_00p_to = [&fsten] (int i_, int j_, int k_) -> Real {
3311  Real w1 = std::abs(fsten(i_ ,j_ ,k_-1,ist_00p));
3312  Real w2 = std::abs(fsten(i_ ,j_ ,k_ ,ist_00p));
3313  if (w1 == Real(0.) && w2 == Real(0.)) {
3314  return Real(0.5);
3315  } else {
3316  return w2 / (w1+w2);
3317  }
3318  };
3319 
3320  auto interp_from_0m0_to = [&fsten] (int i_, int j_, int k_) -> Real {
3321  Real w1 = std::abs(fsten(i_ ,j_-1,k_ ,ist_0p0));
3322  Real w2 = std::abs(fsten(i_ ,j_ ,k_ ,ist_0p0));
3323  if (w1 == Real(0.) && w2 == Real(0.)) {
3324  return Real(0.5);
3325  } else {
3326  return w1 / (w1+w2);
3327  }
3328  };
3329 
3330  auto interp_from_0p0_to = [&fsten] (int i_, int j_, int k_) -> Real {
3331  Real w1 = std::abs(fsten(i_ ,j_-1,k_ ,ist_0p0));
3332  Real w2 = std::abs(fsten(i_ ,j_ ,k_ ,ist_0p0));
3333  if (w1 == Real(0.) && w2 == Real(0.)) {
3334  return Real(0.5);
3335  } else {
3336  return w2 / (w1+w2);
3337  }
3338  };
3339 
3340  auto interp_from_m00_to = [&fsten] (int i_, int j_, int k_) -> Real {
3341  Real w1 = std::abs(fsten(i_-1,j_ ,k_ ,ist_p00));
3342  Real w2 = std::abs(fsten(i_ ,j_ ,k_ ,ist_p00));
3343  if (w1 == Real(0.) && w2 == Real(0.)) {
3344  return Real(0.5);
3345  } else {
3346  return w1 / (w1+w2);
3347  }
3348  };
3349 
3350  auto interp_from_p00_to = [&fsten] (int i_, int j_, int k_) -> Real {
3351  Real w1 = std::abs(fsten(i_-1,j_ ,k_ ,ist_p00));
3352  Real w2 = std::abs(fsten(i_ ,j_ ,k_ ,ist_p00));
3353  if (w1 == Real(0.) && w2 == Real(0.)) {
3354  return Real(0.5);
3355  } else {
3356  return w2 / (w1+w2);
3357  }
3358  };
3359 
3360  auto Ammm = [&fsten] (int i_, int j_, int k_) -> Real {
3361  return fsten(i_-1,j_-1,k_-1,ist_ppp);
3362  };
3363  amrex::ignore_unused(Ammm);
3364 
3365  auto A0mm = [&fsten] (int i_, int j_, int k_) -> Real {
3366  return fsten(i_ ,j_-1,k_-1,ist_0pp);
3367  };
3368 
3369  auto Apmm = [&fsten] (int i_, int j_, int k_) -> Real {
3370  return fsten(i_ ,j_-1,k_-1,ist_ppp);
3371  };
3372 
3373  auto Am0m = [&fsten] (int i_, int j_, int k_) -> Real {
3374  return fsten(i_-1,j_ ,k_-1,ist_p0p);
3375  };
3376 
3377  auto A00m = [&fsten] (int i_, int j_, int k_) -> Real {
3378  return fsten(i_ ,j_ ,k_-1,ist_00p);
3379  };
3380 
3381  auto Ap0m = [&fsten] (int i_, int j_, int k_) -> Real {
3382  return fsten(i_ ,j_ ,k_-1,ist_p0p);
3383  };
3384 
3385  auto Ampm = [&fsten] (int i_, int j_, int k_) -> Real {
3386  return fsten(i_-1,j_ ,k_-1,ist_ppp);
3387  };
3388 
3389  auto A0pm = [&fsten] (int i_, int j_, int k_) -> Real {
3390  return fsten(i_ ,j_ ,k_-1,ist_0pp);
3391  };
3392 
3393  auto Appm = [&fsten] (int i_, int j_, int k_) -> Real {
3394  return fsten(i_ ,j_ ,k_-1,ist_ppp);
3395  };
3396 
3397  auto Amm0 = [&fsten] (int i_, int j_, int k_) -> Real {
3398  return fsten(i_-1,j_-1,k_ ,ist_pp0);
3399  };
3400 
3401  auto A0m0 = [&fsten] (int i_, int j_, int k_) -> Real {
3402  return fsten(i_ ,j_-1,k_ ,ist_0p0);
3403  };
3404 
3405  auto Apm0 = [&fsten] (int i_, int j_, int k_) -> Real {
3406  return fsten(i_ ,j_-1,k_ ,ist_pp0);
3407  };
3408 
3409  auto Am00 = [&fsten] (int i_, int j_, int k_) -> Real {
3410  return fsten(i_-1,j_ ,k_ ,ist_p00);
3411  };
3412 
3413  auto A000 = [&fsten] (int i_, int j_, int k_) -> Real {
3414  return fsten(i_ ,j_ ,k_ ,ist_000);
3415  };
3416 
3417  auto Ap00 = [&fsten] (int i_, int j_, int k_) -> Real {
3418  return fsten(i_ ,j_ ,k_ ,ist_p00);
3419  };
3420 
3421  auto Amp0 = [&fsten] (int i_, int j_, int k_) -> Real {
3422  return fsten(i_-1,j_ ,k_ ,ist_pp0);
3423  };
3424 
3425  auto A0p0 = [&fsten] (int i_, int j_, int k_) -> Real {
3426  return fsten(i_ ,j_ ,k_ ,ist_0p0);
3427  };
3428 
3429  auto App0 = [&fsten] (int i_, int j_, int k_) -> Real {
3430  return fsten(i_ ,j_ ,k_ ,ist_pp0);
3431  };
3432 
3433  auto Ammp = [&fsten] (int i_, int j_, int k_) -> Real {
3434  return fsten(i_-1,j_-1,k_ ,ist_ppp);
3435  };
3436 
3437  auto A0mp = [&fsten] (int i_, int j_, int k_) -> Real {
3438  return fsten(i_ ,j_-1,k_ ,ist_0pp);
3439  };
3440 
3441  auto Apmp = [&fsten] (int i_, int j_, int k_) -> Real {
3442  return fsten(i_ ,j_-1,k_ ,ist_ppp);
3443  };
3444 
3445  auto Am0p = [&fsten] (int i_, int j_, int k_) -> Real {
3446  return fsten(i_-1,j_ ,k_ ,ist_p0p);
3447  };
3448 
3449  auto A00p = [&fsten] (int i_, int j_, int k_) -> Real {
3450  return fsten(i_ ,j_ ,k_ ,ist_00p);
3451  };
3452 
3453  auto Ap0p = [&fsten] (int i_, int j_, int k_) -> Real {
3454  return fsten(i_ ,j_ ,k_ ,ist_p0p);
3455  };
3456 
3457  auto Ampp = [&fsten] (int i_, int j_, int k_) -> Real {
3458  return fsten(i_-1,j_ ,k_ ,ist_ppp);
3459  };
3460 
3461  auto A0pp = [&fsten] (int i_, int j_, int k_) -> Real {
3462  return fsten(i_ ,j_ ,k_ ,ist_0pp);
3463  };
3464 
3465  auto Appp = [&fsten] (int i_, int j_, int k_) -> Real {
3466  return fsten(i_ ,j_ ,k_ ,ist_ppp);
3467  };
3468 
3469  auto restrict_from_mmm_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3470  Real r = Real(1.);
3471  r += std::abs(fsten(ii_-1,jj_-1,kk_-1,ist_p00)) /
3472  ( std::abs(fsten(ii_-1,jj_-2,kk_-2,ist_ppp))
3473  + std::abs(fsten(ii_-1,jj_-1,kk_-2,ist_ppp))
3474  + std::abs(fsten(ii_-1,jj_-2,kk_-1,ist_ppp))
3475  + std::abs(fsten(ii_-1,jj_-1,kk_-1,ist_ppp)) + eps);
3476  r += std::abs(fsten(ii_-1,jj_-1,kk_-1,ist_0p0)) /
3477  ( std::abs(fsten(ii_-2,jj_-1,kk_-2,ist_ppp))
3478  + std::abs(fsten(ii_-1,jj_-1,kk_-2,ist_ppp))
3479  + std::abs(fsten(ii_-2,jj_-1,kk_-1,ist_ppp))
3480  + std::abs(fsten(ii_-1,jj_-1,kk_-1,ist_ppp)) + eps);
3481  r += std::abs(fsten(ii_-1,jj_-1,kk_-1,ist_00p)) /
3482  ( std::abs(fsten(ii_-2,jj_-2,kk_-1,ist_ppp))
3483  + std::abs(fsten(ii_-1,jj_-2,kk_-1,ist_ppp))
3484  + std::abs(fsten(ii_-2,jj_-1,kk_-1,ist_ppp))
3485  + std::abs(fsten(ii_-1,jj_-1,kk_-1,ist_ppp)) + eps);
3486  r += std::abs(fsten(ii_-1,jj_-1,kk_-1,ist_pp0)) /
3487  ( std::abs(fsten(ii_-1,jj_-1,kk_-2,ist_ppp))
3488  + std::abs(fsten(ii_-1,jj_-1,kk_-1,ist_ppp)) + eps);
3489  r += std::abs(fsten(ii_-1,jj_-1,kk_-1,ist_p0p)) /
3490  ( std::abs(fsten(ii_-1,jj_-2,kk_-1,ist_ppp))
3491  + std::abs(fsten(ii_-1,jj_-1,kk_-1,ist_ppp)) + eps);
3492  r += std::abs(fsten(ii_-1,jj_-1,kk_-1,ist_0pp)) /
3493  ( std::abs(fsten(ii_-2,jj_-1,kk_-1,ist_ppp))
3494  + std::abs(fsten(ii_-1,jj_-1,kk_-1,ist_ppp)) + eps);
3495  r *= std::abs(fsten(ii_-1,jj_-1,kk_-1,ist_ppp)) * fsten(ii_-1,jj_-1,kk_-1,ist_inv);
3496  return r;
3497  };
3498  amrex::ignore_unused(restrict_from_mmm_to);
3499 
3500  auto restrict_from_0mm_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3501  Real w1m = std::abs(fsten(ii_,jj_-2,kk_-1,ist_0p0)) / (std::abs(fsten(ii_,jj_-2,kk_-2,ist_0pp))
3502  +std::abs(fsten(ii_,jj_-2,kk_-1,ist_0pp)) + eps);
3503  Real w1p = std::abs(fsten(ii_,jj_-1,kk_-1,ist_0p0)) / (std::abs(fsten(ii_,jj_-1,kk_-2,ist_0pp))
3504  +std::abs(fsten(ii_,jj_-1,kk_-1,ist_0pp)) + eps);
3505  Real w2m = std::abs(fsten(ii_,jj_-1,kk_-2,ist_00p)) / (std::abs(fsten(ii_,jj_-2,kk_-2,ist_0pp))
3506  +std::abs(fsten(ii_,jj_-1,kk_-2,ist_0pp)) + eps);
3507  Real w2p = std::abs(fsten(ii_,jj_-1,kk_-1,ist_00p)) / (std::abs(fsten(ii_,jj_-2,kk_-1,ist_0pp))
3508  +std::abs(fsten(ii_,jj_-1,kk_-1,ist_0pp)) + eps);
3509  Real wmm = std::abs(fsten(ii_,jj_-2,kk_-2,ist_0pp)) * (Real(1.) + w1m + w2m);
3510  Real wpm = std::abs(fsten(ii_,jj_-1,kk_-2,ist_0pp)) * (Real(1.) + w1p + w2m);
3511  Real wmp = std::abs(fsten(ii_,jj_-2,kk_-1,ist_0pp)) * (Real(1.) + w1m + w2p);
3512  Real wpp = std::abs(fsten(ii_,jj_-1,kk_-1,ist_0pp)) * (Real(1.) + w1p + w2p);
3513  return wpp / (wmm+wpm+wmp+wpp+eps);
3514  };
3515 
3516  auto restrict_from_pmm_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3517  Real r = Real(1.);
3518  r += std::abs(fsten(ii_ ,jj_-1,kk_-1,ist_p00)) /
3519  ( std::abs(fsten(ii_ ,jj_-2,kk_-2,ist_ppp))
3520  + std::abs(fsten(ii_ ,jj_-1,kk_-2,ist_ppp))
3521  + std::abs(fsten(ii_ ,jj_-2,kk_-1,ist_ppp))
3522  + std::abs(fsten(ii_ ,jj_-1,kk_-1,ist_ppp)) + eps);
3523  r += std::abs(fsten(ii_+1,jj_-1,kk_-1,ist_0p0)) /
3524  ( std::abs(fsten(ii_ ,jj_-1,kk_-2,ist_ppp))
3525  + std::abs(fsten(ii_+1,jj_-1,kk_-2,ist_ppp))
3526  + std::abs(fsten(ii_ ,jj_-1,kk_-1,ist_ppp))
3527  + std::abs(fsten(ii_+1,jj_-1,kk_-1,ist_ppp)) + eps);
3528  r += std::abs(fsten(ii_+1,jj_-1,kk_-1,ist_00p)) /
3529  ( std::abs(fsten(ii_ ,jj_-2,kk_-1,ist_ppp))
3530  + std::abs(fsten(ii_+1,jj_-2,kk_-1,ist_ppp))
3531  + std::abs(fsten(ii_ ,jj_-1,kk_-1,ist_ppp))
3532  + std::abs(fsten(ii_+1,jj_-1,kk_-1,ist_ppp)) + eps);
3533  r += std::abs(fsten(ii_ ,jj_-1,kk_-1,ist_pp0)) /
3534  ( std::abs(fsten(ii_ ,jj_-1,kk_-2,ist_ppp))
3535  + std::abs(fsten(ii_ ,jj_-1,kk_-1,ist_ppp)) + eps);
3536  r += std::abs(fsten(ii_ ,jj_-1,kk_-1,ist_p0p)) /
3537  ( std::abs(fsten(ii_ ,jj_-2,kk_-1,ist_ppp))
3538  + std::abs(fsten(ii_ ,jj_-1,kk_-1,ist_ppp)) + eps);
3539  r += std::abs(fsten(ii_+1,jj_-1,kk_-1,ist_0pp)) /
3540  ( std::abs(fsten(ii_ ,jj_-1,kk_-1,ist_ppp))
3541  + std::abs(fsten(ii_+1,jj_-1,kk_-1,ist_ppp)) + eps);
3542  r *= std::abs(fsten(ii_ ,jj_-1,kk_-1,ist_ppp)) * fsten(ii_+1,jj_-1,kk_-1,ist_inv);
3543  return r;
3544  };
3545 
3546  auto restrict_from_m0m_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3547  Real w1m = std::abs(fsten(ii_-2,jj_,kk_-1,ist_p00)) / (std::abs(fsten(ii_-2,jj_,kk_-2,ist_p0p))
3548  +std::abs(fsten(ii_-2,jj_,kk_-1,ist_p0p)) + eps);
3549  Real w1p = std::abs(fsten(ii_-1,jj_,kk_-1,ist_p00)) / (std::abs(fsten(ii_-1,jj_,kk_-2,ist_p0p))
3550  +std::abs(fsten(ii_-1,jj_,kk_-1,ist_p0p)) + eps);
3551  Real w2m = std::abs(fsten(ii_-1,jj_,kk_-2,ist_00p)) / (std::abs(fsten(ii_-2,jj_,kk_-2,ist_p0p))
3552  +std::abs(fsten(ii_-1,jj_,kk_-2,ist_p0p)) + eps);
3553  Real w2p = std::abs(fsten(ii_-1,jj_,kk_-1,ist_00p)) / (std::abs(fsten(ii_-2,jj_,kk_-1,ist_p0p))
3554  +std::abs(fsten(ii_-1,jj_,kk_-1,ist_p0p)) + eps);
3555  Real wmm = std::abs(fsten(ii_-2,jj_,kk_-2,ist_p0p)) * (Real(1.) + w1m + w2m);
3556  Real wpm = std::abs(fsten(ii_-1,jj_,kk_-2,ist_p0p)) * (Real(1.) + w1p + w2m);
3557  Real wmp = std::abs(fsten(ii_-2,jj_,kk_-1,ist_p0p)) * (Real(1.) + w1m + w2p);
3558  Real wpp = std::abs(fsten(ii_-1,jj_,kk_-1,ist_p0p)) * (Real(1.) + w1p + w2p);
3559  return wpp / (wmm+wpm+wmp+wpp+eps);
3560  };
3561 
3562  auto restrict_from_00m_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3563  Real w1 = std::abs(fsten(ii_,jj_,kk_-2,ist_00p));
3564  Real w2 = std::abs(fsten(ii_,jj_,kk_-1,ist_00p));
3565  if (w1 == Real(0.) && w2 == Real(0.)) {
3566  return Real(0.5);
3567  } else {
3568  return w2 / (w1+w2);
3569  }
3570  };
3571 
3572  auto restrict_from_p0m_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3573  Real w1m = std::abs(fsten(ii_ ,jj_,kk_-1,ist_p00)) / (std::abs(fsten(ii_ ,jj_,kk_-2,ist_p0p))
3574  +std::abs(fsten(ii_ ,jj_,kk_-1,ist_p0p)) + eps);
3575  Real w1p = std::abs(fsten(ii_+1,jj_,kk_-1,ist_p00)) / (std::abs(fsten(ii_+1,jj_,kk_-2,ist_p0p))
3576  +std::abs(fsten(ii_+1,jj_,kk_-1,ist_p0p)) + eps);
3577  Real w2m = std::abs(fsten(ii_+1,jj_,kk_-2,ist_00p)) / (std::abs(fsten(ii_ ,jj_,kk_-2,ist_p0p))
3578  +std::abs(fsten(ii_+1,jj_,kk_-2,ist_p0p)) + eps);
3579  Real w2p = std::abs(fsten(ii_+1,jj_,kk_-1,ist_00p)) / (std::abs(fsten(ii_ ,jj_,kk_-1,ist_p0p))
3580  +std::abs(fsten(ii_+1,jj_,kk_-1,ist_p0p)) + eps);
3581  Real wmm = std::abs(fsten(ii_ ,jj_,kk_-2,ist_p0p)) * (Real(1.) + w1m + w2m);
3582  Real wpm = std::abs(fsten(ii_+1,jj_,kk_-2,ist_p0p)) * (Real(1.) + w1p + w2m);
3583  Real wmp = std::abs(fsten(ii_ ,jj_,kk_-1,ist_p0p)) * (Real(1.) + w1m + w2p);
3584  Real wpp = std::abs(fsten(ii_+1,jj_,kk_-1,ist_p0p)) * (Real(1.) + w1p + w2p);
3585  return wmp / (wmm+wpm+wmp+wpp+eps);
3586  };
3587 
3588  auto restrict_from_mpm_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3589  Real r = Real(1.);
3590  r += std::abs(fsten(ii_-1,jj_+1,kk_-1,ist_p00)) /
3591  ( std::abs(fsten(ii_-1,jj_ ,kk_-2,ist_ppp))
3592  + std::abs(fsten(ii_-1,jj_+1,kk_-2,ist_ppp))
3593  + std::abs(fsten(ii_-1,jj_ ,kk_-1,ist_ppp))
3594  + std::abs(fsten(ii_-1,jj_+1,kk_-1,ist_ppp)) + eps);
3595  r += std::abs(fsten(ii_-1,jj_ ,kk_-1,ist_0p0)) /
3596  ( std::abs(fsten(ii_-2,jj_ ,kk_-2,ist_ppp))
3597  + std::abs(fsten(ii_-1,jj_ ,kk_-2,ist_ppp))
3598  + std::abs(fsten(ii_-2,jj_ ,kk_-1,ist_ppp))
3599  + std::abs(fsten(ii_-1,jj_ ,kk_-1,ist_ppp)) + eps);
3600  r += std::abs(fsten(ii_-1,jj_+1,kk_-1,ist_00p)) /
3601  ( std::abs(fsten(ii_-2,jj_ ,kk_-1,ist_ppp))
3602  + std::abs(fsten(ii_-1,jj_ ,kk_-1,ist_ppp))
3603  + std::abs(fsten(ii_-2,jj_+1,kk_-1,ist_ppp))
3604  + std::abs(fsten(ii_-1,jj_+1,kk_-1,ist_ppp)) + eps);
3605  r += std::abs(fsten(ii_-1,jj_ ,kk_-1,ist_pp0)) /
3606  ( std::abs(fsten(ii_-1,jj_ ,kk_-2,ist_ppp))
3607  + std::abs(fsten(ii_-1,jj_ ,kk_-1,ist_ppp)) + eps);
3608  r += std::abs(fsten(ii_-1,jj_+1,kk_-1,ist_p0p)) /
3609  ( std::abs(fsten(ii_-1,jj_ ,kk_-1,ist_ppp))
3610  + std::abs(fsten(ii_-1,jj_+1,kk_-1,ist_ppp)) + eps);
3611  r += std::abs(fsten(ii_-1,jj_ ,kk_-1,ist_0pp)) /
3612  ( std::abs(fsten(ii_-2,jj_ ,kk_-1,ist_ppp))
3613  + std::abs(fsten(ii_-1,jj_ ,kk_-1,ist_ppp)) + eps);
3614  r *= std::abs(fsten(ii_-1,jj_ ,kk_-1,ist_ppp)) * fsten(ii_-1,jj_+1,kk_-1,ist_inv);
3615  return r;
3616  };
3617 
3618  auto restrict_from_0pm_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3619  Real w1m = std::abs(fsten(ii_,jj_ ,kk_-1,ist_0p0)) / (std::abs(fsten(ii_,jj_ ,kk_-2,ist_0pp))
3620  +std::abs(fsten(ii_,jj_ ,kk_-1,ist_0pp)) + eps);
3621  Real w1p = std::abs(fsten(ii_,jj_+1,kk_-1,ist_0p0)) / (std::abs(fsten(ii_,jj_+1,kk_-2,ist_0pp))
3622  +std::abs(fsten(ii_,jj_+1,kk_-1,ist_0pp)) + eps);
3623  Real w2m = std::abs(fsten(ii_,jj_+1,kk_-2,ist_00p)) / (std::abs(fsten(ii_,jj_ ,kk_-2,ist_0pp))
3624  +std::abs(fsten(ii_,jj_+1,kk_-2,ist_0pp)) + eps);
3625  Real w2p = std::abs(fsten(ii_,jj_+1,kk_-1,ist_00p)) / (std::abs(fsten(ii_,jj_ ,kk_-1,ist_0pp))
3626  +std::abs(fsten(ii_,jj_+1,kk_-1,ist_0pp)) + eps);
3627  Real wmm = std::abs(fsten(ii_,jj_ ,kk_-2,ist_0pp)) * (Real(1.) + w1m + w2m);
3628  Real wpm = std::abs(fsten(ii_,jj_+1,kk_-2,ist_0pp)) * (Real(1.) + w1p + w2m);
3629  Real wmp = std::abs(fsten(ii_,jj_ ,kk_-1,ist_0pp)) * (Real(1.) + w1m + w2p);
3630  Real wpp = std::abs(fsten(ii_,jj_+1,kk_-1,ist_0pp)) * (Real(1.) + w1p + w2p);
3631  return wmp / (wmm+wpm+wmp+wpp+eps);
3632  };
3633 
3634  auto restrict_from_ppm_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3635  Real r = Real(1.);
3636  r += std::abs(fsten(ii_ ,jj_+1,kk_-1,ist_p00)) /
3637  ( std::abs(fsten(ii_ ,jj_ ,kk_-2,ist_ppp))
3638  + std::abs(fsten(ii_ ,jj_+1,kk_-2,ist_ppp))
3639  + std::abs(fsten(ii_ ,jj_ ,kk_-1,ist_ppp))
3640  + std::abs(fsten(ii_ ,jj_+1,kk_-1,ist_ppp)) + eps);
3641  r += std::abs(fsten(ii_+1,jj_ ,kk_-1,ist_0p0)) /
3642  ( std::abs(fsten(ii_ ,jj_ ,kk_-2,ist_ppp))
3643  + std::abs(fsten(ii_+1,jj_ ,kk_-2,ist_ppp))
3644  + std::abs(fsten(ii_ ,jj_ ,kk_-1,ist_ppp))
3645  + std::abs(fsten(ii_+1,jj_ ,kk_-1,ist_ppp)) + eps);
3646  r += std::abs(fsten(ii_+1,jj_+1,kk_-1,ist_00p)) /
3647  ( std::abs(fsten(ii_ ,jj_ ,kk_-1,ist_ppp))
3648  + std::abs(fsten(ii_+1,jj_ ,kk_-1,ist_ppp))
3649  + std::abs(fsten(ii_ ,jj_+1,kk_-1,ist_ppp))
3650  + std::abs(fsten(ii_+1,jj_+1,kk_-1,ist_ppp)) + eps);
3651  r += std::abs(fsten(ii_ ,jj_ ,kk_-1,ist_pp0)) /
3652  ( std::abs(fsten(ii_ ,jj_ ,kk_-2,ist_ppp))
3653  + std::abs(fsten(ii_ ,jj_ ,kk_-1,ist_ppp)) + eps);
3654  r += std::abs(fsten(ii_ ,jj_+1,kk_-1,ist_p0p)) /
3655  ( std::abs(fsten(ii_ ,jj_ ,kk_-1,ist_ppp))
3656  + std::abs(fsten(ii_ ,jj_+1,kk_-1,ist_ppp)) + eps);
3657  r += std::abs(fsten(ii_+1,jj_ ,kk_-1,ist_0pp)) /
3658  ( std::abs(fsten(ii_ ,jj_ ,kk_-1,ist_ppp))
3659  + std::abs(fsten(ii_+1,jj_ ,kk_-1,ist_ppp)) + eps);
3660  r *= std::abs(fsten(ii_ ,jj_ ,kk_-1,ist_ppp)) * fsten(ii_+1,jj_+1,kk_-1,ist_inv);
3661  return r;
3662  };
3663 
3664  auto restrict_from_mm0_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3665  Real w1m = std::abs(fsten(ii_-2,jj_-1,kk_,ist_p00)) / (std::abs(fsten(ii_-2,jj_-2,kk_,ist_pp0))
3666  +std::abs(fsten(ii_-2,jj_-1,kk_,ist_pp0)) + eps);
3667  Real w1p = std::abs(fsten(ii_-1,jj_-1,kk_,ist_p00)) / (std::abs(fsten(ii_-1,jj_-2,kk_,ist_pp0))
3668  +std::abs(fsten(ii_-1,jj_-1,kk_,ist_pp0)) + eps);
3669  Real w2m = std::abs(fsten(ii_-1,jj_-2,kk_,ist_0p0)) / (std::abs(fsten(ii_-2,jj_-2,kk_,ist_pp0))
3670  +std::abs(fsten(ii_-1,jj_-2,kk_,ist_pp0)) + eps);
3671  Real w2p = std::abs(fsten(ii_-1,jj_-1,kk_,ist_0p0)) / (std::abs(fsten(ii_-2,jj_-1,kk_,ist_pp0))
3672  +std::abs(fsten(ii_-1,jj_-1,kk_,ist_pp0)) + eps);
3673  Real wmm = std::abs(fsten(ii_-2,jj_-2,kk_,ist_pp0)) * (Real(1.) + w1m + w2m);
3674  Real wpm = std::abs(fsten(ii_-1,jj_-2,kk_,ist_pp0)) * (Real(1.) + w1p + w2m);
3675  Real wmp = std::abs(fsten(ii_-2,jj_-1,kk_,ist_pp0)) * (Real(1.) + w1m + w2p);
3676  Real wpp = std::abs(fsten(ii_-1,jj_-1,kk_,ist_pp0)) * (Real(1.) + w1p + w2p);
3677  return wpp / (wmm+wpm+wmp+wpp+eps);
3678  };
3679 
3680  auto restrict_from_0m0_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3681  Real w1 = std::abs(fsten(ii_,jj_-2,kk_,ist_0p0));
3682  Real w2 = std::abs(fsten(ii_,jj_-1,kk_,ist_0p0));
3683  if (w1 == Real(0.) && w2 == Real(0.)) {
3684  return Real(0.5);
3685  } else {
3686  return w2 / (w1+w2);
3687  }
3688  };
3689 
3690  auto restrict_from_pm0_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3691  Real w1m = std::abs(fsten(ii_ ,jj_-1,kk_,ist_p00)) / (std::abs(fsten(ii_ ,jj_-2,kk_,ist_pp0))
3692  +std::abs(fsten(ii_ ,jj_-1,kk_,ist_pp0)) + eps);
3693  Real w1p = std::abs(fsten(ii_+1,jj_-1,kk_,ist_p00)) / (std::abs(fsten(ii_+1,jj_-2,kk_,ist_pp0))
3694  +std::abs(fsten(ii_+1,jj_-1,kk_,ist_pp0)) + eps);
3695  Real w2m = std::abs(fsten(ii_+1,jj_-2,kk_,ist_0p0)) / (std::abs(fsten(ii_ ,jj_-2,kk_,ist_pp0))
3696  +std::abs(fsten(ii_+1,jj_-2,kk_,ist_pp0)) + eps);
3697  Real w2p = std::abs(fsten(ii_+1,jj_-1,kk_,ist_0p0)) / (std::abs(fsten(ii_ ,jj_-1,kk_,ist_pp0))
3698  +std::abs(fsten(ii_+1,jj_-1,kk_,ist_pp0)) + eps);
3699  Real wmm = std::abs(fsten(ii_ ,jj_-2,kk_,ist_pp0)) * (Real(1.) + w1m + w2m);
3700  Real wpm = std::abs(fsten(ii_+1,jj_-2,kk_,ist_pp0)) * (Real(1.) + w1p + w2m);
3701  Real wmp = std::abs(fsten(ii_ ,jj_-1,kk_,ist_pp0)) * (Real(1.) + w1m + w2p);
3702  Real wpp = std::abs(fsten(ii_+1,jj_-1,kk_,ist_pp0)) * (Real(1.) + w1p + w2p);
3703  return wmp / (wmm+wpm+wmp+wpp+eps);
3704  };
3705 
3706  auto restrict_from_m00_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3707  Real w1 = std::abs(fsten(ii_-2,jj_,kk_,ist_p00));
3708  Real w2 = std::abs(fsten(ii_-1,jj_,kk_,ist_p00));
3709  if (w1 == Real(0.) && w2 == Real(0.)) {
3710  return Real(0.5);
3711  } else {
3712  return w2 / (w1+w2);
3713  }
3714  };
3715 
3716  auto restrict_from_000_to = [] (int /*ii_*/, int /*jj_*/, int /*kk_*/) -> Real {
3717  return Real(1.);
3718  };
3719 
3720  auto restrict_from_p00_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3721  Real w1 = std::abs(fsten(ii_ ,jj_,kk_,ist_p00));
3722  Real w2 = std::abs(fsten(ii_+1,jj_,kk_,ist_p00));
3723  if (w1 == Real(0.) && w2 == Real(0.)) {
3724  return Real(0.5);
3725  } else {
3726  return w1 / (w1+w2);
3727  }
3728  };
3729 
3730  auto restrict_from_mp0_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3731  Real w1m = std::abs(fsten(ii_-2,jj_+1,kk_,ist_p00)) / (std::abs(fsten(ii_-2,jj_ ,kk_,ist_pp0))
3732  +std::abs(fsten(ii_-2,jj_+1,kk_,ist_pp0)) + eps);
3733  Real w1p = std::abs(fsten(ii_-1,jj_+1,kk_,ist_p00)) / (std::abs(fsten(ii_-1,jj_ ,kk_,ist_pp0))
3734  +std::abs(fsten(ii_-1,jj_+1,kk_,ist_pp0)) + eps);
3735  Real w2m = std::abs(fsten(ii_-1,jj_ ,kk_,ist_0p0)) / (std::abs(fsten(ii_-2,jj_ ,kk_,ist_pp0))
3736  +std::abs(fsten(ii_-1,jj_ ,kk_,ist_pp0)) + eps);
3737  Real w2p = std::abs(fsten(ii_-1,jj_+1,kk_,ist_0p0)) / (std::abs(fsten(ii_-2,jj_+1,kk_,ist_pp0))
3738  +std::abs(fsten(ii_-1,jj_+1,kk_,ist_pp0)) + eps);
3739  Real wmm = std::abs(fsten(ii_-2,jj_ ,kk_,ist_pp0)) * (Real(1.) + w1m + w2m);
3740  Real wpm = std::abs(fsten(ii_-1,jj_ ,kk_,ist_pp0)) * (Real(1.) + w1p + w2m);
3741  Real wmp = std::abs(fsten(ii_-2,jj_+1,kk_,ist_pp0)) * (Real(1.) + w1m + w2p);
3742  Real wpp = std::abs(fsten(ii_-1,jj_+1,kk_,ist_pp0)) * (Real(1.) + w1p + w2p);
3743  return wpm / (wmm+wpm+wmp+wpp+eps);
3744  };
3745 
3746  auto restrict_from_0p0_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3747  Real w1 = std::abs(fsten(ii_,jj_ ,kk_,ist_0p0));
3748  Real w2 = std::abs(fsten(ii_,jj_+1,kk_,ist_0p0));
3749  if (w1 == Real(0.) && w2 == Real(0.)) {
3750  return Real(0.5);
3751  } else {
3752  return w1 / (w1+w2);
3753  }
3754  };
3755 
3756  auto restrict_from_pp0_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3757  Real w1m = std::abs(fsten(ii_ ,jj_+1,kk_,ist_p00)) / (std::abs(fsten(ii_ ,jj_ ,kk_,ist_pp0))
3758  +std::abs(fsten(ii_ ,jj_+1,kk_,ist_pp0)) + eps);
3759  Real w1p = std::abs(fsten(ii_+1,jj_+1,kk_,ist_p00)) / (std::abs(fsten(ii_+1,jj_ ,kk_,ist_pp0))
3760  +std::abs(fsten(ii_+1,jj_+1,kk_,ist_pp0)) + eps);
3761  Real w2m = std::abs(fsten(ii_+1,jj_ ,kk_,ist_0p0)) / (std::abs(fsten(ii_ ,jj_ ,kk_,ist_pp0))
3762  +std::abs(fsten(ii_+1,jj_ ,kk_,ist_pp0)) + eps);
3763  Real w2p = std::abs(fsten(ii_+1,jj_+1,kk_,ist_0p0)) / (std::abs(fsten(ii_ ,jj_+1,kk_,ist_pp0))
3764  +std::abs(fsten(ii_+1,jj_+1,kk_,ist_pp0)) + eps);
3765  Real wmm = std::abs(fsten(ii_ ,jj_ ,kk_,ist_pp0)) * (Real(1.) + w1m + w2m);
3766  Real wpm = std::abs(fsten(ii_+1,jj_ ,kk_,ist_pp0)) * (Real(1.) + w1p + w2m);
3767  Real wmp = std::abs(fsten(ii_ ,jj_+1,kk_,ist_pp0)) * (Real(1.) + w1m + w2p);
3768  Real wpp = std::abs(fsten(ii_+1,jj_+1,kk_,ist_pp0)) * (Real(1.) + w1p + w2p);
3769  return wmm / (wmm+wpm+wmp+wpp+eps);
3770  };
3771 
3772  auto restrict_from_mmp_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3773  Real r = Real(1.);
3774  r += std::abs(fsten(ii_-1,jj_-1,kk_+1,ist_p00)) /
3775  ( std::abs(fsten(ii_-1,jj_-2,kk_ ,ist_ppp))
3776  + std::abs(fsten(ii_-1,jj_-1,kk_ ,ist_ppp))
3777  + std::abs(fsten(ii_-1,jj_-2,kk_+1,ist_ppp))
3778  + std::abs(fsten(ii_-1,jj_-1,kk_+1,ist_ppp)) + eps);
3779  r += std::abs(fsten(ii_-1,jj_-1,kk_+1,ist_0p0)) /
3780  ( std::abs(fsten(ii_-2,jj_-1,kk_ ,ist_ppp))
3781  + std::abs(fsten(ii_-1,jj_-1,kk_ ,ist_ppp))
3782  + std::abs(fsten(ii_-2,jj_-1,kk_+1,ist_ppp))
3783  + std::abs(fsten(ii_-1,jj_-1,kk_+1,ist_ppp)) + eps);
3784  r += std::abs(fsten(ii_-1,jj_-1,kk_ ,ist_00p)) /
3785  ( std::abs(fsten(ii_-2,jj_-2,kk_ ,ist_ppp))
3786  + std::abs(fsten(ii_-1,jj_-2,kk_ ,ist_ppp))
3787  + std::abs(fsten(ii_-2,jj_-1,kk_ ,ist_ppp))
3788  + std::abs(fsten(ii_-1,jj_-1,kk_ ,ist_ppp)) + eps);
3789  r += std::abs(fsten(ii_-1,jj_-1,kk_+1,ist_pp0)) /
3790  ( std::abs(fsten(ii_-1,jj_-1,kk_ ,ist_ppp))
3791  + std::abs(fsten(ii_-1,jj_-1,kk_+1,ist_ppp)) + eps);
3792  r += std::abs(fsten(ii_-1,jj_-1,kk_ ,ist_p0p)) /
3793  ( std::abs(fsten(ii_-1,jj_-2,kk_ ,ist_ppp))
3794  + std::abs(fsten(ii_-1,jj_-1,kk_ ,ist_ppp)) + eps);
3795  r += std::abs(fsten(ii_-1,jj_-1,kk_ ,ist_0pp)) /
3796  ( std::abs(fsten(ii_-2,jj_-1,kk_ ,ist_ppp))
3797  + std::abs(fsten(ii_-1,jj_-1,kk_ ,ist_ppp)) + eps);
3798  r *= std::abs(fsten(ii_-1,jj_-1,kk_ ,ist_ppp)) * fsten(ii_-1,jj_-1,kk_+1,ist_inv);
3799  return r;
3800  };
3801 
3802  auto restrict_from_0mp_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3803  Real w1m = std::abs(fsten(ii_,jj_-2,kk_+1,ist_0p0)) / (std::abs(fsten(ii_,jj_-2,kk_ ,ist_0pp))
3804  +std::abs(fsten(ii_,jj_-2,kk_+1,ist_0pp)) + eps);
3805  Real w1p = std::abs(fsten(ii_,jj_-1,kk_+1,ist_0p0)) / (std::abs(fsten(ii_,jj_-1,kk_ ,ist_0pp))
3806  +std::abs(fsten(ii_,jj_-1,kk_+1,ist_0pp)) + eps);
3807  Real w2m = std::abs(fsten(ii_,jj_-1,kk_ ,ist_00p)) / (std::abs(fsten(ii_,jj_-2,kk_ ,ist_0pp))
3808  +std::abs(fsten(ii_,jj_-1,kk_ ,ist_0pp)) + eps);
3809  Real w2p = std::abs(fsten(ii_,jj_-1,kk_+1,ist_00p)) / (std::abs(fsten(ii_,jj_-2,kk_+1,ist_0pp))
3810  +std::abs(fsten(ii_,jj_-1,kk_+1,ist_0pp)) + eps);
3811  Real wmm = std::abs(fsten(ii_,jj_-2,kk_ ,ist_0pp)) * (Real(1.) + w1m + w2m);
3812  Real wpm = std::abs(fsten(ii_,jj_-1,kk_ ,ist_0pp)) * (Real(1.) + w1p + w2m);
3813  Real wmp = std::abs(fsten(ii_,jj_-2,kk_+1,ist_0pp)) * (Real(1.) + w1m + w2p);
3814  Real wpp = std::abs(fsten(ii_,jj_-1,kk_+1,ist_0pp)) * (Real(1.) + w1p + w2p);
3815  return wpm / (wmm+wpm+wmp+wpp+eps);
3816  };
3817 
3818  auto restrict_from_pmp_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3819  Real r = Real(1.);
3820  r += std::abs(fsten(ii_ ,jj_-1,kk_+1,ist_p00)) /
3821  ( std::abs(fsten(ii_ ,jj_-2,kk_ ,ist_ppp))
3822  + std::abs(fsten(ii_ ,jj_-1,kk_ ,ist_ppp))
3823  + std::abs(fsten(ii_ ,jj_-2,kk_+1,ist_ppp))
3824  + std::abs(fsten(ii_ ,jj_-1,kk_+1,ist_ppp)) + eps);
3825  r += std::abs(fsten(ii_+1,jj_-1,kk_+1,ist_0p0)) /
3826  ( std::abs(fsten(ii_ ,jj_-1,kk_ ,ist_ppp))
3827  + std::abs(fsten(ii_+1,jj_-1,kk_ ,ist_ppp))
3828  + std::abs(fsten(ii_ ,jj_-1,kk_+1,ist_ppp))
3829  + std::abs(fsten(ii_+1,jj_-1,kk_+1,ist_ppp)) + eps);
3830  r += std::abs(fsten(ii_+1,jj_-1,kk_ ,ist_00p)) /
3831  ( std::abs(fsten(ii_ ,jj_-2,kk_ ,ist_ppp))
3832  + std::abs(fsten(ii_+1,jj_-2,kk_ ,ist_ppp))
3833  + std::abs(fsten(ii_ ,jj_-1,kk_ ,ist_ppp))
3834  + std::abs(fsten(ii_+1,jj_-1,kk_ ,ist_ppp)) + eps);
3835  r += std::abs(fsten(ii_ ,jj_-1,kk_+1,ist_pp0)) /
3836  ( std::abs(fsten(ii_ ,jj_-1,kk_ ,ist_ppp))
3837  + std::abs(fsten(ii_ ,jj_-1,kk_+1,ist_ppp)) + eps);
3838  r += std::abs(fsten(ii_ ,jj_-1,kk_ ,ist_p0p)) /
3839  ( std::abs(fsten(ii_ ,jj_-2,kk_ ,ist_ppp))
3840  + std::abs(fsten(ii_ ,jj_-1,kk_ ,ist_ppp)) + eps);
3841  r += std::abs(fsten(ii_+1,jj_-1,kk_ ,ist_0pp)) /
3842  ( std::abs(fsten(ii_ ,jj_-1,kk_ ,ist_ppp))
3843  + std::abs(fsten(ii_+1,jj_-1,kk_ ,ist_ppp)) + eps);
3844  r *= std::abs(fsten(ii_ ,jj_-1,kk_ ,ist_ppp)) * fsten(ii_+1,jj_-1,kk_+1,ist_inv);
3845  return r;
3846  };
3847 
3848  auto restrict_from_m0p_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3849  Real w1m = std::abs(fsten(ii_-2,jj_,kk_+1,ist_p00)) / (std::abs(fsten(ii_-2,jj_,kk_ ,ist_p0p))
3850  +std::abs(fsten(ii_-2,jj_,kk_+1,ist_p0p)) + eps);
3851  Real w1p = std::abs(fsten(ii_-1,jj_,kk_+1,ist_p00)) / (std::abs(fsten(ii_-1,jj_,kk_ ,ist_p0p))
3852  +std::abs(fsten(ii_-1,jj_,kk_+1,ist_p0p)) + eps);
3853  Real w2m = std::abs(fsten(ii_-1,jj_,kk_ ,ist_00p)) / (std::abs(fsten(ii_-2,jj_,kk_ ,ist_p0p))
3854  +std::abs(fsten(ii_-1,jj_,kk_ ,ist_p0p)) + eps);
3855  Real w2p = std::abs(fsten(ii_-1,jj_,kk_+1,ist_00p)) / (std::abs(fsten(ii_-2,jj_,kk_+1,ist_p0p))
3856  +std::abs(fsten(ii_-1,jj_,kk_+1,ist_p0p)) + eps);
3857  Real wmm = std::abs(fsten(ii_-2,jj_,kk_ ,ist_p0p)) * (Real(1.) + w1m + w2m);
3858  Real wpm = std::abs(fsten(ii_-1,jj_,kk_ ,ist_p0p)) * (Real(1.) + w1p + w2m);
3859  Real wmp = std::abs(fsten(ii_-2,jj_,kk_+1,ist_p0p)) * (Real(1.) + w1m + w2p);
3860  Real wpp = std::abs(fsten(ii_-1,jj_,kk_+1,ist_p0p)) * (Real(1.) + w1p + w2p);
3861  return wpm / (wmm+wpm+wmp+wpp+eps);
3862  };
3863 
3864  auto restrict_from_00p_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3865  Real w1 = std::abs(fsten(ii_,jj_,kk_ ,ist_00p));
3866  Real w2 = std::abs(fsten(ii_,jj_,kk_+1,ist_00p));
3867  if (w1 == Real(0.) && w2 == Real(0.)) {
3868  return Real(0.5);
3869  } else {
3870  return w1 / (w1+w2);
3871  }
3872  };
3873 
3874  auto restrict_from_p0p_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3875  Real w1m = std::abs(fsten(ii_ ,jj_,kk_+1,ist_p00)) / (std::abs(fsten(ii_ ,jj_,kk_ ,ist_p0p))
3876  +std::abs(fsten(ii_ ,jj_,kk_+1,ist_p0p)) + eps);
3877  Real w1p = std::abs(fsten(ii_+1,jj_,kk_+1,ist_p00)) / (std::abs(fsten(ii_+1,jj_,kk_ ,ist_p0p))
3878  +std::abs(fsten(ii_+1,jj_,kk_+1,ist_p0p)) + eps);
3879  Real w2m = std::abs(fsten(ii_+1,jj_,kk_ ,ist_00p)) / (std::abs(fsten(ii_ ,jj_,kk_ ,ist_p0p))
3880  +std::abs(fsten(ii_+1,jj_,kk_ ,ist_p0p)) + eps);
3881  Real w2p = std::abs(fsten(ii_+1,jj_,kk_+1,ist_00p)) / (std::abs(fsten(ii_ ,jj_,kk_+1,ist_p0p))
3882  +std::abs(fsten(ii_+1,jj_,kk_+1,ist_p0p)) + eps);
3883  Real wmm = std::abs(fsten(ii_ ,jj_,kk_ ,ist_p0p)) * (Real(1.) + w1m + w2m);
3884  Real wpm = std::abs(fsten(ii_+1,jj_,kk_ ,ist_p0p)) * (Real(1.) + w1p + w2m);
3885  Real wmp = std::abs(fsten(ii_ ,jj_,kk_+1,ist_p0p)) * (Real(1.) + w1m + w2p);
3886  Real wpp = std::abs(fsten(ii_+1,jj_,kk_+1,ist_p0p)) * (Real(1.) + w1p + w2p);
3887  return wmm / (wmm+wpm+wmp+wpp+eps);
3888  };
3889 
3890  auto restrict_from_mpp_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3891  Real r = Real(1.);
3892  r += std::abs(fsten(ii_-1,jj_+1,kk_+1,ist_p00)) /
3893  ( std::abs(fsten(ii_-1,jj_ ,kk_ ,ist_ppp))
3894  + std::abs(fsten(ii_-1,jj_+1,kk_ ,ist_ppp))
3895  + std::abs(fsten(ii_-1,jj_ ,kk_+1,ist_ppp))
3896  + std::abs(fsten(ii_-1,jj_+1,kk_+1,ist_ppp)) + eps);
3897  r += std::abs(fsten(ii_-1,jj_ ,kk_+1,ist_0p0)) /
3898  ( std::abs(fsten(ii_-2,jj_ ,kk_ ,ist_ppp))
3899  + std::abs(fsten(ii_-1,jj_ ,kk_ ,ist_ppp))
3900  + std::abs(fsten(ii_-2,jj_ ,kk_+1,ist_ppp))
3901  + std::abs(fsten(ii_-1,jj_ ,kk_+1,ist_ppp)) + eps);
3902  r += std::abs(fsten(ii_-1,jj_+1,kk_ ,ist_00p)) /
3903  ( std::abs(fsten(ii_-2,jj_ ,kk_ ,ist_ppp))
3904  + std::abs(fsten(ii_-1,jj_ ,kk_ ,ist_ppp))
3905  + std::abs(fsten(ii_-2,jj_+1,kk_ ,ist_ppp))
3906  + std::abs(fsten(ii_-1,jj_+1,kk_ ,ist_ppp)) + eps);
3907  r += std::abs(fsten(ii_-1,jj_ ,kk_+1,ist_pp0)) /
3908  ( std::abs(fsten(ii_-1,jj_ ,kk_ ,ist_ppp))
3909  + std::abs(fsten(ii_-1,jj_ ,kk_+1,ist_ppp)) + eps);
3910  r += std::abs(fsten(ii_-1,jj_+1,kk_ ,ist_p0p)) /
3911  ( std::abs(fsten(ii_-1,jj_ ,kk_ ,ist_ppp))
3912  + std::abs(fsten(ii_-1,jj_+1,kk_ ,ist_ppp)) + eps);
3913  r += std::abs(fsten(ii_-1,jj_ ,kk_ ,ist_0pp)) /
3914  ( std::abs(fsten(ii_-2,jj_ ,kk_ ,ist_ppp))
3915  + std::abs(fsten(ii_-1,jj_ ,kk_ ,ist_ppp)) + eps);
3916  r *= std::abs(fsten(ii_-1,jj_ ,kk_ ,ist_ppp)) * fsten(ii_-1,jj_+1,kk_+1,ist_inv);
3917  return r;
3918  };
3919 
3920  auto restrict_from_0pp_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3921  Real w1m = std::abs(fsten(ii_,jj_ ,kk_+1,ist_0p0)) / (std::abs(fsten(ii_,jj_ ,kk_ ,ist_0pp))
3922  +std::abs(fsten(ii_,jj_ ,kk_+1,ist_0pp)) + eps);
3923  Real w1p = std::abs(fsten(ii_,jj_+1,kk_+1,ist_0p0)) / (std::abs(fsten(ii_,jj_+1,kk_ ,ist_0pp))
3924  +std::abs(fsten(ii_,jj_+1,kk_+1,ist_0pp)) + eps);
3925  Real w2m = std::abs(fsten(ii_,jj_+1,kk_ ,ist_00p)) / (std::abs(fsten(ii_,jj_ ,kk_ ,ist_0pp))
3926  +std::abs(fsten(ii_,jj_+1,kk_ ,ist_0pp)) + eps);
3927  Real w2p = std::abs(fsten(ii_,jj_+1,kk_+1,ist_00p)) / (std::abs(fsten(ii_,jj_ ,kk_+1,ist_0pp))
3928  +std::abs(fsten(ii_,jj_+1,kk_+1,ist_0pp)) + eps);
3929  Real wmm = std::abs(fsten(ii_,jj_ ,kk_ ,ist_0pp)) * (Real(1.) + w1m + w2m);
3930  Real wpm = std::abs(fsten(ii_,jj_+1,kk_ ,ist_0pp)) * (Real(1.) + w1p + w2m);
3931  Real wmp = std::abs(fsten(ii_,jj_ ,kk_+1,ist_0pp)) * (Real(1.) + w1m + w2p);
3932  Real wpp = std::abs(fsten(ii_,jj_+1,kk_+1,ist_0pp)) * (Real(1.) + w1p + w2p);
3933  return wmm / (wmm+wpm+wmp+wpp+eps);
3934  };
3935 
3936  auto restrict_from_ppp_to = [&fsten] (int ii_, int jj_, int kk_) -> Real {
3937  Real r = Real(1.);
3938  r += std::abs(fsten(ii_ ,jj_+1,kk_+1,ist_p00)) /
3939  ( std::abs(fsten(ii_ ,jj_ ,kk_ ,ist_ppp))
3940  + std::abs(fsten(ii_ ,jj_+1,kk_ ,ist_ppp))
3941  + std::abs(fsten(ii_ ,jj_ ,kk_+1,ist_ppp))
3942  + std::abs(fsten(ii_ ,jj_+1,kk_+1,ist_ppp)) + eps);
3943  r += std::abs(fsten(ii_+1,jj_ ,kk_+1,ist_0p0)) /
3944  ( std::abs(fsten(ii_ ,jj_ ,kk_ ,ist_ppp))
3945  + std::abs(fsten(ii_+1,jj_ ,kk_ ,ist_ppp))
3946  + std::abs(fsten(ii_ ,jj_ ,kk_+1,ist_ppp))
3947  + std::abs(fsten(ii_+1,jj_ ,kk_+1,ist_ppp)) + eps);
3948  r += std::abs(fsten(ii_+1,jj_+1,kk_ ,ist_00p)) /
3949  ( std::abs(fsten(ii_ ,jj_ ,kk_ ,ist_ppp))
3950  + std::abs(fsten(ii_+1,jj_ ,kk_ ,ist_ppp))
3951  + std::abs(fsten(ii_ ,jj_+1,kk_ ,ist_ppp))
3952  + std::abs(fsten(ii_+1,jj_+1,kk_ ,ist_ppp)) + eps);
3953  r += std::abs(fsten(ii_ ,jj_ ,kk_+1,ist_pp0)) /
3954  ( std::abs(fsten(ii_ ,jj_ ,kk_ ,ist_ppp))
3955  + std::abs(fsten(ii_ ,jj_ ,kk_+1,ist_ppp)) + eps);
3956  r += std::abs(fsten(ii_ ,jj_+1,kk_ ,ist_p0p)) /
3957  ( std::abs(fsten(ii_ ,jj_ ,kk_ ,ist_ppp))
3958  + std::abs(fsten(ii_ ,jj_+1,kk_ ,ist_ppp)) + eps);
3959  r += std::abs(fsten(ii_+1,jj_ ,kk_ ,ist_0pp)) /
3960  ( std::abs(fsten(ii_ ,jj_ ,kk_ ,ist_ppp))
3961  + std::abs(fsten(ii_+1,jj_ ,kk_ ,ist_ppp)) + eps);
3962  r *= std::abs(fsten(ii_ ,jj_ ,kk_ ,ist_ppp)) * fsten(ii_+1,jj_+1,kk_+1,ist_inv);
3963  return r;
3964  };
3965 
3966  int ii = 2*i;
3967  int jj = 2*j;
3968  int kk = 2*k;
3969  Array3D<Real,-1,1,-1,1,-1,1> p;
3970  Array3D<Real,-1,1,-1,1,-1,1> ap;
3971  Real cs1, cs2, cs3, cs4;
3972 
3973  // csten(i,j,k,ist_p00)
3974  int iii = ii;
3975  int jjj = jj;
3976  int kkk = kk;;
3977  p(-1,-1,-1) = interp_from_ppp_to(iii+1,jjj-1,kkk-1);
3978  p( 0,-1,-1) = interp_from_0pp_to(iii+2,jjj-1,kkk-1);
3979  p(-1, 0,-1) = interp_from_p0p_to(iii+1,jjj ,kkk-1);
3980  p( 0, 0,-1) = interp_from_00p_to(iii+2,jjj ,kkk-1);
3981  p(-1,+1,-1) = interp_from_pmp_to(iii+1,jjj+1,kkk-1);
3982  p( 0,+1,-1) = interp_from_0mp_to(iii+2,jjj+1,kkk-1);
3983  p(-1,-1, 0) = interp_from_pp0_to(iii+1,jjj-1,kkk );
3984  p( 0,-1, 0) = interp_from_0p0_to(iii+2,jjj-1,kkk );
3985  p(-1, 0, 0) = interp_from_p00_to(iii+1,jjj ,kkk );
3986  p( 0, 0, 0) = Real(1.);
3987  p(-1,+1, 0) = interp_from_pm0_to(iii+1,jjj+1,kkk );
3988  p( 0,+1, 0) = interp_from_0m0_to(iii+2,jjj+1,kkk );
3989  p(-1,-1,+1) = interp_from_ppm_to(iii+1,jjj-1,kkk+1);
3990  p( 0,-1,+1) = interp_from_0pm_to(iii+2,jjj-1,kkk+1);
3991  p(-1, 0,+1) = interp_from_p0m_to(iii+1,jjj ,kkk+1);
3992  p( 0, 0,+1) = interp_from_00m_to(iii+2,jjj ,kkk+1);
3993  p(-1,+1,+1) = interp_from_pmm_to(iii+1,jjj+1,kkk+1);
3994  p( 0,+1,+1) = interp_from_0mm_to(iii+2,jjj+1,kkk+1);
3995  ap(0,-1,-1) =
3996  Ap00(iii,jjj-1,kkk-1) * p(-1,-1,-1)
3997  + App0(iii,jjj-1,kkk-1) * p(-1, 0,-1)
3998  + Ap0p(iii,jjj-1,kkk-1) * p(-1,-1, 0)
3999  + Appp(iii,jjj-1,kkk-1) * p(-1, 0, 0);
4000  ap(1,-1,-1) =
4001  A000(iii+1,jjj-1,kkk-1) * p(-1,-1,-1)
4002  + Ap00(iii+1,jjj-1,kkk-1) * p( 0,-1,-1)
4003  + A0p0(iii+1,jjj-1,kkk-1) * p(-1, 0,-1)
4004  + App0(iii+1,jjj-1,kkk-1) * p( 0, 0,-1)
4005  + A00p(iii+1,jjj-1,kkk-1) * p(-1,-1, 0)
4006  + Ap0p(iii+1,jjj-1,kkk-1) * p( 0,-1, 0)
4007  + A0pp(iii+1,jjj-1,kkk-1) * p(-1, 0, 0)
4008  + Appp(iii+1,jjj-1,kkk-1) * p( 0, 0, 0);
4009  ap(0,0,-1) =
4010  Apm0(iii,jjj,kkk-1) * p(-1,-1,-1)
4011  + Ap00(iii,jjj,kkk-1) * p(-1, 0,-1)
4012  + App0(iii,jjj,kkk-1) * p(-1,+1,-1)
4013  + Apmp(iii,jjj,kkk-1) * p(-1,-1, 0)
4014  + Ap0p(iii,jjj,kkk-1) * p(-1, 0, 0)
4015  + Appp(iii,jjj,kkk-1) * p(-1,+1, 0);
4016  ap(1,0,-1) =
4017  A0m0(iii+1,jjj,kkk-1) * p(-1,-1,-1)
4018  + Apm0(iii+1,jjj,kkk-1) * p( 0,-1,-1)
4019  + A000(iii+1,jjj,kkk-1) * p(-1, 0,-1)
4020  + Ap00(iii+1,jjj,kkk-1) * p( 0, 0,-1)
4021  + A0p0(iii+1,jjj,kkk-1) * p(-1,+1,-1)
4022  + App0(iii+1,jjj,kkk-1) * p( 0,+1,-1)
4023  + A0mp(iii+1,jjj,kkk-1) * p(-1,-1, 0)
4024  + Apmp(iii+1,jjj,kkk-1) * p( 0,-1, 0)
4025  + A00p(iii+1,jjj,kkk-1) * p(-1, 0, 0)
4026  + Ap0p(iii+1,jjj,kkk-1) * p( 0, 0, 0)
4027  + A0pp(iii+1,jjj,kkk-1) * p(-1,+1, 0)
4028  + Appp(iii+1,jjj,kkk-1) * p( 0,+1, 0);
4029  ap(0,1,-1) =
4030  Apm0(iii,jjj+1,kkk-1) * p(-1, 0,-1)
4031  + Ap00(iii,jjj+1,kkk-1) * p(-1,+1,-1)
4032  + Apmp(iii,jjj+1,kkk-1) * p(-1, 0, 0)
4033  + Ap0p(iii,jjj+1,kkk-1) * p(-1,+1, 0);
4034  ap(1,1,-1) =
4035  A0m0(iii+1,jjj+1,kkk-1) * p(-1, 0,-1)
4036  + Apm0(iii+1,jjj+1,kkk-1) * p( 0, 0,-1)
4037  + A000(iii+1,jjj+1,kkk-1) * p(-1,+1,-1)
4038  + Ap00(iii+1,jjj+1,kkk-1) * p( 0,+1,-1)
4039  + A0mp(iii+1,jjj+1,kkk-1) * p(-1, 0, 0)
4040  + Apmp(iii+1,jjj+1,kkk-1) * p( 0, 0, 0)
4041  + A00p(iii+1,jjj+1,kkk-1) * p(-1,+1, 0)
4042  + Ap0p(iii+1,jjj+1,kkk-1) * p( 0,+1, 0);
4043  ap(0,-1,0) =
4044  Ap0m(iii,jjj-1,kkk) * p(-1,-1,-1)
4045  + Appm(iii,jjj-1,kkk) * p(-1, 0,-1)
4046  + Ap00(iii,jjj-1,kkk) * p(-1,-1, 0)
4047  + App0(iii,jjj-1,kkk) * p(-1, 0, 0)
4048  + Ap0p(iii,jjj-1,kkk) * p(-1,-1,+1)
4049  + Appp(iii,jjj-1,kkk) * p(-1, 0,+1);
4050  ap(1,-1,0) =
4051  A00m(iii+1,jjj-1,kkk) * p(-1,-1,-1)
4052  + Ap0m(iii+1,jjj-1,kkk) * p( 0,-1,-1)
4053  + A0pm(iii+1,jjj-1,kkk) * p(-1, 0,-1)
4054  + Appm(iii+1,jjj-1,kkk) * p( 0, 0,-1)
4055  + A000(iii+1,jjj-1,kkk) * p(-1,-1, 0)
4056  + Ap00(iii+1,jjj-1,kkk) * p( 0,-1, 0)
4057  + A0p0(iii+1,jjj-1,kkk) * p(-1, 0, 0)
4058  + App0(iii+1,jjj-1,kkk) * p( 0, 0, 0)
4059  + A00p(iii+1,jjj-1,kkk) * p(-1,-1,+1)
4060  + Ap0p(iii+1,jjj-1,kkk) * p( 0,-1,+1)
4061  + A0pp(iii+1,jjj-1,kkk) * p(-1, 0,+1)
4062  + Appp(iii+1,jjj-1,kkk) * p( 0, 0,+1);
4063  ap(0,0,0) =
4064  Apmm(iii,jjj,kkk) * p(-1,-1,-1)
4065  + Ap0m(iii,jjj,kkk) * p(-1, 0,-1)
4066  + Appm(iii,jjj,kkk) * p(-1,+1,-1)
4067  + Apm0(iii,jjj,kkk) * p(-1,-1, 0)
4068  + Ap00(iii,jjj,kkk) * p(-1, 0, 0)
4069  + App0(iii,jjj,kkk) * p(-1,+1, 0)
4070  + Apmp(iii,jjj,kkk) * p(-1,-1,+1)
4071  + Ap0p(iii,jjj,kkk) * p(-1, 0,+1)
4072  + Appp(iii,jjj,kkk) * p(-1,+1,+1);
4073  ap(1,0,0) =
4074  A0mm(iii+1,jjj,kkk) * p(-1,-1,-1)
4075  + Apmm(iii+1,jjj,kkk) * p( 0,-1,-1)
4076  + A00m(iii+1,jjj,kkk) * p(-1, 0,-1)
4077  + Ap0m(iii+1,jjj,kkk) * p( 0, 0,-1)
4078  + A0pm(iii+1,jjj,kkk) * p(-1,+1,-1)
4079  + Appm(iii+1,jjj,kkk) * p( 0,+1,-1)
4080  + A0m0(iii+1,jjj,kkk) * p(-1,-1, 0)
4081  + Apm0(iii+1,jjj,kkk) * p( 0,-1, 0)
4082  + A000(iii+1,jjj,kkk) * p(-1, 0, 0)
4083  + Ap00(iii+1,jjj,kkk) * p( 0, 0, 0)
4084  + A0p0(iii+1,jjj,kkk) * p(-1,+1, 0)
4085  + App0(iii+1,jjj,kkk) * p( 0,+1, 0)
4086  + A0mp(iii+1,jjj,kkk) * p(-1,-1,+1)
4087  + Apmp(iii+1,jjj,kkk) * p( 0,-1,+1)
4088  + A00p(iii+1,jjj,kkk) * p(-1, 0,+1)
4089  + Ap0p(iii+1,jjj,kkk) * p( 0, 0,+1)
4090  + A0pp(iii+1,jjj,kkk) * p(-1,+1,+1)
4091  + Appp(iii+1,jjj,kkk) * p( 0,+1,+1);
4092  ap(0,1,0) =
4093  Apmm(iii,jjj+1,kkk) * p(-1, 0,-1)
4094  + Ap0m(iii,jjj+1,kkk) * p(-1,+1,-1)
4095  + Apm0(iii,jjj+1,kkk) * p(-1, 0, 0)
4096  + Ap00(iii,jjj+1,kkk) * p(-1,+1, 0)
4097  + Apmp(iii,jjj+1,kkk) * p(-1, 0,+1)
4098  + Ap0p(iii,jjj+1,kkk) * p(-1,+1,+1);
4099  ap(1,1,0) =
4100  A0mm(iii+1,jjj+1,kkk) * p(-1, 0,-1)
4101  + Apmm(iii+1,jjj+1,kkk) * p( 0, 0,-1)
4102  + A00m(iii+1,jjj+1,kkk) * p(-1,+1,-1)
4103  + Ap0m(iii+1,jjj+1,kkk) * p( 0,+1,-1)
4104  + A0m0(iii+1,jjj+1,kkk) * p(-1, 0, 0)
4105  + Apm0(iii+1,jjj+1,kkk) * p( 0, 0, 0)
4106  + A000(iii+1,jjj+1,kkk) * p(-1,+1, 0)
4107  + Ap00(iii+1,jjj+1,kkk) * p( 0,+1, 0)
4108  + A0mp(iii+1,jjj+1,kkk) * p(-1, 0,+1)
4109  + Apmp(iii+1,jjj+1,kkk) * p( 0, 0,+1)
4110  + A00p(iii+1,jjj+1,kkk) * p(-1,+1,+1)
4111  + Ap0p(iii+1,jjj+1,kkk) * p( 0,+1,+1);
4112  ap(0,-1,1) =
4113  Ap0m(iii,jjj-1,kkk+1) * p(-1,-1, 0)
4114  + Appm(iii,jjj-1,kkk+1) * p(-1, 0, 0)
4115  + Ap00(iii,jjj-1,kkk+1) * p(-1,-1,+1)
4116  + App0(iii,jjj-1,kkk+1) * p(-1, 0,+1);
4117  ap(1,-1,1) =
4118  A00m(iii+1,jjj-1,kkk+1) * p(-1,-1, 0)
4119  + Ap0m(iii+1,jjj-1,kkk+1) * p( 0,-1, 0)
4120  + A0pm(iii+1,jjj-1,kkk+1) * p(-1, 0, 0)
4121  + Appm(iii+1,jjj-1,kkk+1) * p( 0, 0, 0)
4122  + A000(iii+1,jjj-1,kkk+1) * p(-1,-1,+1)
4123  + Ap00(iii+1,jjj-1,kkk+1) * p( 0,-1,+1)
4124  + A0p0(iii+1,jjj-1,kkk+1) * p(-1, 0,+1)
4125  + App0(iii+1,jjj-1,kkk+1) * p( 0, 0,+1);
4126  ap(0,0,1) =
4127  Apmm(iii,jjj,kkk+1) * p(-1,-1, 0)
4128  + Ap0m(iii,jjj,kkk+1) * p(-1, 0, 0)
4129  + Appm(iii,jjj,kkk+1) * p(-1,+1, 0)
4130  + Apm0(iii,jjj,kkk+1) * p(-1,-1,+1)
4131  + Ap00(iii,jjj,kkk+1) * p(-1, 0,+1)
4132  + App0(iii,jjj,kkk+1) * p(-1,+1,+1);
4133  ap(1,0,1) =
4134  A0mm(iii+1,jjj,kkk+1) * p(-1,-1, 0)
4135  + Apmm(iii+1,jjj,kkk+1) * p( 0,-1, 0)
4136  + A00m(iii+1,jjj,kkk+1) * p(-1, 0, 0)
4137  + Ap0m(iii+1,jjj,kkk+1) * p( 0, 0, 0)
4138  + A0pm(iii+1,jjj,kkk+1) * p(-1,+1, 0)
4139  + Appm(iii+1,jjj,kkk+1) * p( 0,+1, 0)
4140  + A0m0(iii+1,jjj,kkk+1) * p(-1,-1,+1)
4141  + Apm0(iii+1,jjj,kkk+1) * p( 0,-1,+1)
4142  + A000(iii+1,jjj,kkk+1) * p(-1, 0,+1)
4143  + Ap00(iii+1,jjj,kkk+1) * p( 0, 0,+1)
4144  + A0p0(iii+1,jjj,kkk+1) * p(-1,+1,+1)
4145  + App0(iii+1,jjj,kkk+1) * p( 0,+1,+1);
4146  ap(0,1,1) =
4147  Apmm(iii,jjj+1,kkk+1) * p(-1, 0, 0)
4148  + Ap0m(iii,jjj+1,kkk+1) * p(-1,+1, 0)
4149  + Apm0(iii,jjj+1,kkk+1) * p(-1, 0,+1)
4150  + Ap00(iii,jjj+1,kkk+1) * p(-1,+1,+1);
4151  ap(1,1,1) =
4152  A0mm(iii+1,jjj+1,kkk+1) * p(-1, 0, 0)
4153  + Apmm(iii+1,jjj+1,kkk+1) * p( 0, 0, 0)
4154  + A00m(iii+1,jjj+1,kkk+1) * p(-1,+1, 0)
4155  + Ap0m(iii+1,jjj+1,kkk+1) * p( 0,+1, 0)
4156  + A0m0(iii+1,jjj+1,kkk+1) * p(-1, 0,+1)
4157  + Apm0(iii+1,jjj+1,kkk+1) * p( 0, 0,+1)
4158  + A000(iii+1,jjj+1,kkk+1) * p(-1,+1,+1)
4159  + Ap00(iii+1,jjj+1,kkk+1) * p( 0,+1,+1);
4160  csten(i,j,k,ist_p00) = Real(0.125) *
4161  ( restrict_from_0mm_to(iii,jjj,kkk) * ap( 0,-1,-1)
4162  + restrict_from_pmm_to(iii,jjj,kkk) * ap(+1,-1,-1)
4163  + restrict_from_00m_to(iii,jjj,kkk) * ap( 0, 0,-1)
4164  + restrict_from_p0m_to(iii,jjj,kkk) * ap(+1, 0,-1)
4165  + restrict_from_0pm_to(iii,jjj,kkk) * ap( 0,+1,-1)
4166  + restrict_from_ppm_to(iii,jjj,kkk) * ap(+1,+1,-1)
4167  + restrict_from_0m0_to(iii,jjj,kkk) * ap( 0,-1, 0)
4168  + restrict_from_pm0_to(iii,jjj,kkk) * ap(+1,-1, 0)
4169  + restrict_from_000_to(iii,jjj,kkk) * ap( 0, 0, 0)
4170  + restrict_from_p00_to(iii,jjj,kkk) * ap(+1, 0, 0)
4171  + restrict_from_0p0_to(iii,jjj,kkk) * ap( 0,+1, 0)
4172  + restrict_from_pp0_to(iii,jjj,kkk) * ap(+1,+1, 0)
4173  + restrict_from_0mp_to(iii,jjj,kkk) * ap( 0,-1,+1)
4174  + restrict_from_pmp_to(iii,jjj,kkk) * ap(+1,-1,+1)
4175  + restrict_from_00p_to(iii,jjj,kkk) * ap( 0, 0,+1)
4176  + restrict_from_p0p_to(iii,jjj,kkk) * ap(+1, 0,+1)
4177  + restrict_from_0pp_to(iii,jjj,kkk) * ap( 0,+1,+1)
4178  + restrict_from_ppp_to(iii,jjj,kkk) * ap(+1,+1,+1));
4179 
4180  // csten(i,j,k,ist_0p0)
4181  iii = ii;
4182  jjj = jj;
4183  kkk = kk;
4184  p(-1,-1,-1) = interp_from_ppp_to(iii-1,jjj+1,kkk-1);
4185  p( 0,-1,-1) = interp_from_0pp_to(iii ,jjj+1,kkk-1);
4186  p(+1,-1,-1) = interp_from_mpp_to(iii+1,jjj+1,kkk-1);
4187  p(-1, 0,-1) = interp_from_p0p_to(iii-1,jjj+2,kkk-1);
4188  p( 0, 0,-1) = interp_from_00p_to(iii ,jjj+2,kkk-1);
4189  p(+1, 0,-1) = interp_from_m0p_to(iii+1,jjj+2,kkk-1);
4190  p(-1,-1, 0) = interp_from_pp0_to(iii-1,jjj+1,kkk );
4191  p( 0,-1, 0) = interp_from_0p0_to(iii ,jjj+1,kkk );
4192  p(+1,-1, 0) = interp_from_mp0_to(iii+1,jjj+1,kkk );
4193  p(-1, 0, 0) = interp_from_p00_to(iii-1,jjj+2,kkk );
4194  p( 0, 0, 0) = Real(1.);
4195  p(+1, 0, 0) = interp_from_m00_to(iii+1,jjj+2,kkk );
4196  p(-1,-1,+1) = interp_from_ppm_to(iii-1,jjj+1,kkk+1);
4197  p( 0,-1,+1) = interp_from_0pm_to(iii ,jjj+1,kkk+1);
4198  p(+1,-1,+1) = interp_from_mpm_to(iii+1,jjj+1,kkk+1);
4199  p(-1, 0,+1) = interp_from_p0m_to(iii-1,jjj+2,kkk+1);
4200  p( 0, 0,+1) = interp_from_00m_to(iii ,jjj+2,kkk+1);
4201  p(+1, 0,+1) = interp_from_m0m_to(iii+1,jjj+2,kkk+1);
4202  ap(-1,0,-1) =
4203  A0p0(iii-1,jjj,kkk-1) * p(-1,-1,-1)
4204  + App0(iii-1,jjj,kkk-1) * p( 0,-1,-1)
4205  + A0pp(iii-1,jjj,kkk-1) * p(-1,-1, 0)
4206  + Appp(iii-1,jjj,kkk-1) * p( 0,-1, 0);
4207  ap(0,0,-1) =
4208  Amp0(iii,jjj,kkk-1) * p(-1,-1,-1)
4209  + A0p0(iii,jjj,kkk-1) * p( 0,-1,-1)
4210  + App0(iii,jjj,kkk-1) * p(+1,-1,-1)
4211  + Ampp(iii,jjj,kkk-1) * p(-1,-1, 0)
4212  + A0pp(iii,jjj,kkk-1) * p( 0,-1, 0)
4213  + Appp(iii,jjj,kkk-1) * p(+1,-1, 0);
4214  ap(1,0,-1) =
4215  Amp0(iii+1,jjj,kkk-1) * p( 0,-1,-1)
4216  + A0p0(iii+1,jjj,kkk-1) * p(+1,-1,-1)
4217  + Ampp(iii+1,jjj,kkk-1) * p( 0,-1, 0)
4218  + A0pp(iii+1,jjj,kkk-1) * p(+1,-1, 0);
4219  ap(-1,1,-1) =
4220  A000(iii-1,jjj+1,kkk-1) * p(-1,-1,-1)
4221  + Ap00(iii-1,jjj+1,kkk-1) * p( 0,-1,-1)
4222  + A0p0(iii-1,jjj+1,kkk-1) * p(-1, 0,-1)
4223  + App0(iii-1,jjj+1,kkk-1) * p( 0, 0,-1)
4224  + A00p(iii-1,jjj+1,kkk-1) * p(-1,-1, 0)
4225  + Ap0p(iii-1,jjj+1,kkk-1) * p( 0,-1, 0)
4226  + A0pp(iii-1,jjj+1,kkk-1) * p(-1, 0, 0)
4227  + Appp(iii-1,jjj+1,kkk-1) * p( 0, 0, 0);
4228  ap(0,1,-1) =
4229  Am00(iii,jjj+1,kkk-1) * p(-1,-1,-1)
4230  + A000(iii,jjj+1,kkk-1) * p( 0,-1,-1)
4231  + Ap00(iii,jjj+1,kkk-1) * p(+1,-1,-1)
4232  + Amp0(iii,jjj+1,kkk-1) * p(-1, 0,-1)
4233  + A0p0(iii,jjj+1,kkk-1) * p( 0, 0,-1)
4234  + App0(iii,jjj+1,kkk-1) * p(+1, 0,-1)
4235  + Am0p(iii,jjj+1,kkk-1) * p(-1,-1, 0)
4236  + A00p(iii,jjj+1,kkk-1) * p( 0,-1, 0)
4237  + Ap0p(iii,jjj+1,kkk-1) * p(+1,-1, 0)
4238  + Ampp(iii,jjj+1,kkk-1) * p(-1, 0, 0)
4239  + A0pp(iii,jjj+1,kkk-1) * p( 0, 0, 0)
4240  + Appp(iii,jjj+1,kkk-1) * p(+1, 0, 0);
4241  ap(1,1,-1) =
4242  Am00(iii+1,jjj+1,kkk-1) * p( 0,-1,-1)
4243  + A000(iii+1,jjj+1,kkk-1) * p(+1,-1,-1)
4244  + Amp0(iii+1,jjj+1,kkk-1) * p( 0, 0,-1)
4245  + A0p0(iii+1,jjj+1,kkk-1) * p(+1, 0,-1)
4246  + Am0p(iii+1,jjj+1,kkk-1) * p( 0,-1, 0)
4247  + A00p(iii+1,jjj+1,kkk-1) * p(+1,-1, 0)
4248  + Ampp(iii+1,jjj+1,kkk-1) * p( 0, 0, 0)
4249  + A0pp(iii+1,jjj+1,kkk-1) * p(+1, 0, 0);
4250  ap(-1,0,0) =
4251  A0pm(iii-1,jjj,kkk) * p(-1,-1,-1)
4252  + Appm(iii-1,jjj,kkk) * p( 0,-1,-1)
4253  + A0p0(iii-1,jjj,kkk) * p(-1,-1, 0)
4254  + App0(iii-1,jjj,kkk) * p( 0,-1, 0)
4255  + A0pp(iii-1,jjj,kkk) * p(-1,-1,+1)
4256  + Appp(iii-1,jjj,kkk) * p( 0,-1,+1);
4257  ap(0,0,0) =
4258  Ampm(iii,jjj,kkk) * p(-1,-1,-1)
4259  + A0pm(iii,jjj,kkk) * p( 0,-1,-1)
4260  + Appm(iii,jjj,kkk) * p(+1,-1,-1)
4261  + Amp0(iii,jjj,kkk) * p(-1,-1, 0)
4262  + A0p0(iii,jjj,kkk) * p( 0,-1, 0)
4263  + App0(iii,jjj,kkk) * p(+1,-1, 0)
4264  + Ampp(iii,jjj,kkk) * p(-1,-1,+1)
4265  + A0pp(iii,jjj,kkk) * p( 0,-1,+1)
4266  + Appp(iii,jjj,kkk) * p(+1,-1,+1);
4267  ap(1,0,0) =
4268  Ampm(iii+1,jjj,kkk) * p( 0,-1,-1)
4269  + A0pm(iii+1,jjj,kkk) * p(+1,-1,-1)
4270  + Amp0(iii+1,jjj,kkk) * p( 0,-1, 0)
4271  + A0p0(iii+1,jjj,kkk) * p(+1,-1, 0)
4272  + Ampp(iii+1,jjj,kkk) * p( 0,-1,+1)
4273  + A0pp(iii+1,jjj,kkk) * p(+1,-1,+1);
4274  ap(-1,1,0) =
4275  A00m(iii-1,jjj+1,kkk) * p(-1,-1,-1)
4276  + Ap0m(iii-1,jjj+1,kkk) * p( 0,-1,-1)
4277  + A0pm(iii-1,jjj+1,kkk) * p(-1, 0,-1)
4278  + Appm(iii-1,jjj+1,kkk) * p( 0, 0,-1)
4279  + A000(iii-1,jjj+1,kkk) * p(-1,-1, 0)
4280  + Ap00(iii-1,jjj+1,kkk) * p( 0,-1, 0)
4281  + A0p0(iii-1,jjj+1,kkk) * p(-1, 0, 0)
4282  + App0(iii-1,jjj+1,kkk) * p( 0, 0, 0)
4283  + A00p(iii-1,jjj+1,kkk) * p(-1,-1,+1)
4284  + Ap0p(iii-1,jjj+1,kkk) * p( 0,-1,+1)
4285  + A0pp(iii-1,jjj+1,kkk) * p(-1, 0,+1)
4286  + Appp(iii-1,jjj+1,kkk) * p( 0, 0,+1);
4287  ap(0,1,0) =
4288  Am0m(iii,jjj+1,kkk) * p(-1,-1,-1)
4289  + A00m(iii,jjj+1,kkk) * p( 0,-1,-1)
4290  + Ap0m(iii,jjj+1,kkk) * p(+1,-1,-1)
4291  + Ampm(iii,jjj+1,kkk) * p(-1, 0,-1)
4292  + A0pm(iii,jjj+1,kkk) * p( 0, 0,-1)
4293  + Appm(iii,jjj+1,kkk) * p(+1, 0,-1)
4294  + Am00(iii,jjj+1,kkk) * p(-1,-1, 0)
4295  + A000(iii,jjj+1,kkk) * p( 0,-1, 0)
4296  + Ap00(iii,jjj+1,kkk) * p(+1,-1, 0)
4297  + Amp0(iii,jjj+1,kkk) * p(-1, 0, 0)
4298  + A0p0(iii,jjj+1,kkk) * p( 0, 0, 0)
4299  + App0(iii,jjj+1,kkk) * p(+1, 0, 0)
4300  + Am0p(iii,jjj+1,kkk) * p(-1,-1,+1)
4301  + A00p(iii,jjj+1,kkk) * p( 0,-1,+1)
4302  + Ap0p(iii,jjj+1,kkk) * p(+1,-1,+1)
4303  + Ampp(iii,jjj+1,kkk) * p(-1, 0,+1)
4304  + A0pp(iii,jjj+1,kkk) * p( 0, 0,+1)
4305  + Appp(iii,jjj+1,kkk) * p(+1, 0,+1);
4306  ap(1,1,0) =
4307  Am0m(iii+1,jjj+1,kkk) * p( 0,-1,-1)
4308  + A00m(iii+1,jjj+1,kkk) * p(+1,-1,-1)
4309  + Ampm(iii+1,jjj+1,kkk) * p( 0, 0,-1)
4310  + A0pm(iii+1,jjj+1,kkk) * p(+1, 0,-1)
4311  + Am00(iii+1,jjj+1,kkk) * p( 0,-1, 0)
4312  + A000(iii+1,jjj+1,kkk) * p(+1,-1, 0)
4313  + Amp0(iii+1,jjj+1,kkk) * p( 0, 0, 0)
4314  + A0p0(iii+1,jjj+1,kkk) * p(+1, 0, 0)
4315  + Am0p(iii+1,jjj+1,kkk) * p( 0,-1,+1)
4316  + A00p(iii+1,jjj+1,kkk) * p(+1,-1,+1)
4317  + Ampp(iii+1,jjj+1,kkk) * p( 0, 0,+1)
4318  + A0pp(iii+1,jjj+1,kkk) * p(+1, 0,+1);
4319  ap(-1,0,1) =
4320  A0pm(iii-1,jjj,kkk+1) * p(-1,-1, 0)
4321  + Appm(iii-1,jjj,kkk+1) * p( 0,-1, 0)
4322  + A0p0(iii-1,jjj,kkk+1) * p(-1,-1,+1)
4323  + App0(iii-1,jjj,kkk+1) * p( 0,-1,+1);
4324  ap(0,0,1) =
4325  Ampm(iii,jjj,kkk+1) * p(-1,-1, 0)
4326  + A0pm(iii,jjj,kkk+1) * p( 0,-1, 0)
4327  + Appm(iii,jjj,kkk+1) * p(+1,-1, 0)
4328  + Amp0(iii,jjj,kkk+1) * p(-1,-1,+1)
4329  + A0p0(iii,jjj,kkk+1) * p( 0,-1,+1)
4330  + App0(iii,jjj,kkk+1) * p(+1,-1,+1);
4331  ap(1,0,1) =
4332  Ampm(iii+1,jjj,kkk+1) * p( 0,-1, 0)
4333  + A0pm(iii+1,jjj,kkk+1) * p(+1,-1, 0)
4334  + Amp0(iii+1,jjj,kkk+1) * p( 0,-1,+1)
4335  + A0p0(iii+1,jjj,kkk+1) * p(+1,-1,+1);
4336  ap(-1,1,1) =
4337  A00m(iii-1,jjj+1,kkk+1) * p(-1,-1, 0)
4338  + Ap0m(iii-1,jjj+1,kkk+1) * p( 0,-1, 0)
4339  + A0pm(iii-1,jjj+1,kkk+1) * p(-1, 0, 0)
4340  + Appm(iii-1,jjj+1,kkk+1) * p( 0, 0, 0)
4341  + A000(iii-1,jjj+1,kkk+1) * p(-1,-1,+1)
4342  + Ap00(iii-1,jjj+1,kkk+1) * p( 0,-1,+1)
4343  + A0p0(iii-1,jjj+1,kkk+1) * p(-1, 0,+1)
4344  + App0(iii-1,jjj+1,kkk+1) * p( 0, 0,+1);
4345  ap(0,1,1) =
4346  Am0m(iii,jjj+1,kkk+1) * p(-1,-1, 0)
4347  + A00m(iii,jjj+1,kkk+1) * p( 0,-1, 0)
4348  + Ap0m(iii,jjj+1,kkk+1) * p(+1,-1, 0)
4349  + Ampm(iii,jjj+1,kkk+1) * p(-1, 0, 0)
4350  + A0pm(iii,jjj+1,kkk+1) * p( 0, 0, 0)
4351  + Appm(iii,jjj+1,kkk+1) * p(+1, 0, 0)
4352  + Am00(iii,jjj+1,kkk+1) * p(-1,-1,+1)
4353  + A000(iii,jjj+1,kkk+1) * p( 0,-1,+1)
4354  + Ap00(iii,jjj+1,kkk+1) * p(+1,-1,+1)
4355  + Amp0(iii,jjj+1,kkk+1) * p(-1, 0,+1)
4356  + A0p0(iii,jjj+1,kkk+1) * p( 0, 0,+1)
4357  + App0(iii,jjj+1,kkk+1) * p(+1, 0,+1);
4358  ap(1,1,1) =
4359  Am0m(iii+1,jjj+1,kkk+1) * p( 0,-1, 0)
4360  + A00m(iii+1,jjj+1,kkk+1) * p(+1,-1, 0)
4361  + Ampm(iii+1,jjj+1,kkk+1) * p( 0, 0, 0)
4362  + A0pm(iii+1,jjj+1,kkk+1) * p(+1, 0, 0)
4363  + Am00(iii+1,jjj+1,kkk+1) * p( 0,-1,+1)
4364  + A000(iii+1,jjj+1,kkk+1) * p(+1,-1,+1)
4365  + Amp0(iii+1,jjj+1,kkk+1) * p( 0, 0,+1)
4366  + A0p0(iii+1,jjj+1,kkk+1) * p(+1, 0,+1);
4367  csten(i,j,k,ist_0p0) = Real(0.125) *
4368  ( restrict_from_m0m_to(iii,jjj,kkk) * ap(-1, 0,-1)
4369  + restrict_from_00m_to(iii,jjj,kkk) * ap( 0, 0,-1)
4370  + restrict_from_p0m_to(iii,jjj,kkk) * ap(+1, 0,-1)
4371  + restrict_from_mpm_to(iii,jjj,kkk) * ap(-1,+1,-1)
4372  + restrict_from_0pm_to(iii,jjj,kkk) * ap( 0,+1,-1)
4373  + restrict_from_ppm_to(iii,jjj,kkk) * ap(+1,+1,-1)
4374  + restrict_from_m00_to(iii,jjj,kkk) * ap(-1, 0, 0)
4375  + restrict_from_000_to(iii,jjj,kkk) * ap( 0, 0, 0)
4376  + restrict_from_p00_to(iii,jjj,kkk) * ap(+1, 0, 0)
4377  + restrict_from_mp0_to(iii,jjj,kkk) * ap(-1,+1, 0)
4378  + restrict_from_0p0_to(iii,jjj,kkk) * ap( 0,+1, 0)
4379  + restrict_from_pp0_to(iii,jjj,kkk) * ap(+1,+1, 0)
4380  + restrict_from_m0p_to(iii,jjj,kkk) * ap(-1, 0,+1)
4381  + restrict_from_00p_to(iii,jjj,kkk) * ap( 0, 0,+1)
4382  + restrict_from_p0p_to(iii,jjj,kkk) * ap(+1, 0,+1)
4383  + restrict_from_mpp_to(iii,jjj,kkk) * ap(-1,+1,+1)
4384  + restrict_from_0pp_to(iii,jjj,kkk) * ap( 0,+1,+1)
4385  + restrict_from_ppp_to(iii,jjj,kkk) * ap(+1,+1,+1));
4386 
4387  // csten(i,j,k,ist_00p)
4388  iii = ii;
4389  jjj = jj;
4390  kkk = kk;
4391  p(-1,-1,-1) = interp_from_ppp_to(iii-1,jjj-1,kkk+1);
4392  p( 0,-1,-1) = interp_from_0pp_to(iii ,jjj-1,kkk+1);
4393  p(+1,-1,-1) = interp_from_mpp_to(iii+1,jjj-1,kkk+1);
4394  p(-1, 0,-1) = interp_from_p0p_to(iii-1,jjj ,kkk+1);
4395  p( 0, 0,-1) = interp_from_00p_to(iii ,jjj ,kkk+1);
4396  p(+1, 0,-1) = interp_from_m0p_to(iii+1,jjj ,kkk+1);
4397  p(-1,+1,-1) = interp_from_pmp_to(iii-1,jjj+1,kkk+1);
4398  p( 0,+1,-1) = interp_from_0mp_to(iii ,jjj+1,kkk+1);
4399  p(+1,+1,-1) = interp_from_mmp_to(iii+1,jjj+1,kkk+1);
4400  p(-1,-1, 0) = interp_from_pp0_to(iii-1,jjj-1,kkk+2);
4401  p( 0,-1, 0) = interp_from_0p0_to(iii ,jjj-1,kkk+2);
4402  p(+1,-1, 0) = interp_from_mp0_to(iii+1,jjj-1,kkk+2);
4403  p(-1, 0, 0) = interp_from_p00_to(iii-1,jjj ,kkk+2);
4404  p( 0, 0, 0) = Real(1.);
4405  p(+1, 0, 0) = interp_from_m00_to(iii+1,jjj ,kkk+2);
4406  p(-1,+1, 0) = interp_from_pm0_to(iii-1,jjj+1,kkk+2);
4407  p( 0,+1, 0) = interp_from_0m0_to(iii ,jjj+1,kkk+2);
4408  p(+1,+1, 0) = interp_from_mm0_to(iii+1,jjj+1,kkk+2);
4409  ap(-1,-1,0) =
4410  A00p(iii-1,jjj-1,kkk) * p(-1,-1,-1)
4411  + Ap0p(iii-1,jjj-1,kkk) * p( 0,-1,-1)
4412  + A0pp(iii-1,jjj-1,kkk) * p(-1, 0,-1)
4413  + Appp(iii-1,jjj-1,kkk) * p( 0, 0,-1);
4414  ap(0,-1,0) =
4415  Am0p(iii,jjj-1,kkk) * p(-1,-1,-1)
4416  + A00p(iii,jjj-1,kkk) * p( 0,-1,-1)
4417  + Ap0p(iii,jjj-1,kkk) * p(+1,-1,-1)
4418  + Ampp(iii,jjj-1,kkk) * p(-1, 0,-1)
4419  + A0pp(iii,jjj-1,kkk) * p( 0, 0,-1)
4420  + Appp(iii,jjj-1,kkk) * p(+1, 0,-1);
4421  ap(1,-1,0) =
4422  Am0p(iii+1,jjj-1,kkk) * p( 0,-1,-1)
4423  + A00p(iii+1,jjj-1,kkk) * p(+1,-1,-1)
4424  + Ampp(iii+1,jjj-1,kkk) * p( 0, 0,-1)
4425  + A0pp(iii+1,jjj-1,kkk) * p(+1, 0,-1);
4426  ap(-1,0,0) =
4427  A0mp(iii-1,jjj,kkk) * p(-1,-1,-1)
4428  + Apmp(iii-1,jjj,kkk) * p( 0,-1,-1)
4429  + A00p(iii-1,jjj,kkk) * p(-1, 0,-1)
4430  + Ap0p(iii-1,jjj,kkk) * p( 0, 0,-1)
4431  + A0pp(iii-1,jjj,kkk) * p(-1,+1,-1)
4432  + Appp(iii-1,jjj,kkk) * p( 0,+1,-1);
4433  ap(0,0,0) =
4434  Ammp(iii,jjj,kkk) * p(-1,-1,-1)
4435  + A0mp(iii,jjj,kkk) * p( 0,-1,-1)
4436  + Apmp(iii,jjj,kkk) * p(+1,-1,-1)
4437  + Am0p(iii,jjj,kkk) * p(-1, 0,-1)
4438  + A00p(iii,jjj,kkk) * p( 0, 0,-1)
4439  + Ap0p(iii,jjj,kkk) * p(+1, 0,-1)
4440  + Ampp(iii,jjj,kkk) * p(-1,+1,-1)
4441  + A0pp(iii,jjj,kkk) * p( 0,+1,-1)
4442  + Appp(iii,jjj,kkk) * p(+1,+1,-1);
4443  ap(1,0,0) =
4444  Ammp(iii+1,jjj,kkk) * p( 0,-1,-1)
4445  + A0mp(iii+1,jjj,kkk) * p(+1,-1,-1)
4446  + Am0p(iii+1,jjj,kkk) * p( 0, 0,-1)
4447  + A00p(iii+1,jjj,kkk) * p(+1, 0,-1)
4448  + Ampp(iii+1,jjj,kkk) * p( 0,+1,-1)
4449  + A0pp(iii+1,jjj,kkk) * p(+1,+1,-1);
4450  ap(-1,1,0) =
4451  A0mp(iii-1,jjj+1,kkk) * p(-1, 0,-1)
4452  + Apmp(iii-1,jjj+1,kkk) * p( 0, 0,-1)
4453  + A00p(iii-1,jjj+1,kkk) * p(-1,+1,-1)
4454  + Ap0p(iii-1,jjj+1,kkk) * p( 0,+1,-1);
4455  ap(0,1,0) =
4456  Ammp(iii,jjj+1,kkk) * p(-1, 0,-1)
4457  + A0mp(iii,jjj+1,kkk) * p( 0, 0,-1)
4458  + Apmp(iii,jjj+1,kkk) * p(+1, 0,-1)
4459  + Am0p(iii,jjj+1,kkk) * p(-1,+1,-1)
4460  + A00p(iii,jjj+1,kkk) * p( 0,+1,-1)
4461  + Ap0p(iii,jjj+1,kkk) * p(+1,+1,-1);
4462  ap(1,1,0) =
4463  Ammp(iii+1,jjj+1,kkk) * p( 0, 0,-1)
4464  + A0mp(iii+1,jjj+1,kkk) * p(+1, 0,-1)
4465  + Am0p(iii+1,jjj+1,kkk) * p( 0,+1,-1)
4466  + A00p(iii+1,jjj+1,kkk) * p(+1,+1,-1);
4467  ap(-1,-1,1) =
4468  A000(iii-1,jjj-1,kkk+1) * p(-1,-1,-1)
4469  + Ap00(iii-1,jjj-1,kkk+1) * p( 0,-1,-1)
4470  + A0p0(iii-1,jjj-1,kkk+1) * p(-1, 0,-1)
4471  + App0(iii-1,jjj-1,kkk+1) * p( 0, 0,-1)
4472  + A00p(iii-1,jjj-1,kkk+1) * p(-1,-1, 0)
4473  + Ap0p(iii-1,jjj-1,kkk+1) * p( 0,-1, 0)
4474  + A0pp(iii-1,jjj-1,kkk+1) * p(-1, 0, 0)
4475  + Appp(iii-1,jjj-1,kkk+1) * p( 0, 0, 0);
4476  ap(0,-1,1) =
4477  Am00(iii,jjj-1,kkk+1) * p(-1,-1,-1)
4478  + A000(iii,jjj-1,kkk+1) * p( 0,-1,-1)
4479  + Ap00(iii,jjj-1,kkk+1) * p(+1,-1,-1)
4480  + Amp0(iii,jjj-1,kkk+1) * p(-1, 0,-1)
4481  + A0p0(iii,jjj-1,kkk+1) * p( 0, 0,-1)
4482  + App0(iii,jjj-1,kkk+1) * p(+1, 0,-1)
4483  + Am0p(iii,jjj-1,kkk+1) * p(-1,-1, 0)
4484  + A00p(iii,jjj-1,kkk+1) * p( 0,-1, 0)
4485  + Ap0p(iii,jjj-1,kkk+1) * p(+1,-1, 0)
4486  + Ampp(iii,jjj-1,kkk+1) * p(-1, 0, 0)
4487  + A0pp(iii,jjj-1,kkk+1) * p( 0, 0, 0)
4488  + Appp(iii,jjj-1,kkk+1) * p(+1, 0, 0);
4489  ap(1,-1,1) =
4490  Am00(iii+1,jjj-1,kkk+1) * p( 0,-1,-1)
4491  + A000(iii+1,jjj-1,kkk+1) * p(+1,-1,-1)
4492  + Amp0(iii+1,jjj-1,kkk+1) * p( 0, 0,-1)
4493  + A0p0(iii+1,jjj-1,kkk+1) * p(+1, 0,-1)
4494  + Am0p(iii+1,jjj-1,kkk+1) * p( 0,-1, 0)
4495  + A00p(iii+1,jjj-1,kkk+1) * p(+1,-1, 0)
4496  + Ampp(iii+1,jjj-1,kkk+1) * p( 0, 0, 0)
4497  + A0pp(iii+1,jjj-1,kkk+1) * p(+1, 0, 0);
4498  ap(-1,0,1) =
4499  A0m0(iii-1,jjj,kkk+1) * p(-1,-1,-1)
4500  + Apm0(iii-1,jjj,kkk+1) * p( 0,-1,-1)
4501  + A000(iii-1,jjj,kkk+1) * p(-1, 0,-1)
4502  + Ap00(iii-1,jjj,kkk+1) * p( 0, 0,-1)
4503  + A0p0(iii-1,jjj,kkk+1) * p(-1,+1,-1)
4504  + App0(iii-1,jjj,kkk+1) * p( 0,+1,-1)
4505  + A0mp(iii-1,jjj,kkk+1) * p(-1,-1, 0)
4506  + Apmp(iii-1,jjj,kkk+1) * p( 0,-1, 0)
4507  + A00p(iii-1,jjj,kkk+1) * p(-1, 0, 0)
4508  + Ap0p(iii-1,jjj,kkk+1) * p( 0, 0, 0)
4509  + A0pp(iii-1,jjj,kkk+1) * p(-1,+1, 0)
4510  + Appp(iii-1,jjj,kkk+1) * p( 0,+1, 0);
4511  ap(0,0,1) =
4512  Amm0(iii,jjj,kkk+1) * p(-1,-1,-1)
4513  + A0m0(iii,jjj,kkk+1) * p( 0,-1,-1)
4514  + Apm0(iii,jjj,kkk+1) * p(+1,-1,-1)
4515  + Am00(iii,jjj,kkk+1) * p(-1, 0,-1)
4516  + A000(iii,jjj,kkk+1) * p( 0, 0,-1)
4517  + Ap00(iii,jjj,kkk+1) * p(+1, 0,-1)
4518  + Amp0(iii,jjj,kkk+1) * p(-1,+1,-1)
4519  + A0p0(iii,jjj,kkk+1) * p( 0,+1,-1)
4520  + App0(iii,jjj,kkk+1) * p(+1,+1,-1)
4521  + Ammp(iii,jjj,kkk+1) * p(-1,-1, 0)
4522  + A0mp(iii,jjj,kkk+1) * p( 0,-1, 0)
4523  + Apmp(iii,jjj,kkk+1) * p(+1,-1, 0)
4524  + Am0p(iii,jjj,kkk+1) * p(-1, 0, 0)
4525  + A00p(iii,jjj,kkk+1) * p( 0, 0, 0)
4526  + Ap0p(iii,jjj,kkk+1) * p(+1, 0, 0)
4527  + Ampp(iii,jjj,kkk+1) * p(-1,+1, 0)
4528  + A0pp(iii,jjj,kkk+1) * p( 0,+1, 0)
4529  + Appp(iii,jjj,kkk+1) * p(+1,+1, 0);
4530  ap(1,0,1) =
4531  Amm0(iii+1,jjj,kkk+1) * p( 0,-1,-1)
4532  + A0m0(iii+1,jjj,kkk+1) * p(+1,-1,-1)
4533  + Am00(iii+1,jjj,kkk+1) * p( 0, 0,-1)
4534  + A000(iii+1,jjj,kkk+1) * p(+1, 0,-1)
4535  + Amp0(iii+1,jjj,kkk+1) * p( 0,+1,-1)
4536  + A0p0(iii+1,jjj,kkk+1) * p(+1,+1,-1)
4537  + Ammp(iii+1,jjj,kkk+1) * p( 0,-1, 0)
4538  + A0mp(iii+1,jjj,kkk+1) * p(+1,-1, 0)
4539  + Am0p(iii+1,jjj,kkk+1) * p( 0, 0, 0)
4540  + A00p(iii+1,jjj,kkk+1) * p(+1, 0, 0)
4541  + Ampp(iii+1,jjj,kkk+1) * p( 0,+1, 0)
4542  + A0pp(iii+1,jjj,kkk+1) * p(+1,+1, 0);
4543  ap(-1,1,1) =
4544  A0m0(iii-1,jjj+1,kkk+1) * p(-1, 0,-1)
4545  + Apm0(iii-1,jjj+1,kkk+1) * p( 0, 0,-1)
4546  + A000(iii-1,jjj+1,kkk+1) * p(-1,+1,-1)
4547  + Ap00(iii-1,jjj+1,kkk+1) * p( 0,+1,-1)
4548  + A0mp(iii-1,jjj+1,kkk+1) * p(-1, 0, 0)
4549  + Apmp(iii-1,jjj+1,kkk+1) * p( 0, 0, 0)
4550  + A00p(iii-1,jjj+1,kkk+1) * p(-1,+1, 0)
4551  + Ap0p(iii-1,jjj+1,kkk+1) * p( 0,+1, 0);
4552  ap(0,1,1) =
4553  Amm0(iii,jjj+1,kkk+1) * p(-1, 0,-1)
4554  + A0m0(iii,jjj+1,kkk+1) * p( 0, 0,-1)
4555  + Apm0(iii,jjj+1,kkk+1) * p(+1, 0,-1)
4556  + Am00(iii,jjj+1,kkk+1) * p(-1,+1,-1)
4557  + A000(iii,jjj+1,kkk+1) * p( 0,+1,-1)
4558  + Ap00(iii,jjj+1,kkk+1) * p(+1,+1,-1)
4559  + Ammp(iii,jjj+1,kkk+1) * p(-1, 0, 0)
4560  + A0mp(iii,jjj+1,kkk+1) * p( 0, 0, 0)
4561  + Apmp(iii,jjj+1,kkk+1) * p(+1, 0, 0)
4562  + Am0p(iii,jjj+1,kkk+1) * p(-1,+1, 0)
4563  + A00p(iii,jjj+1,kkk+1) * p( 0,+1, 0)
4564  + Ap0p(iii,jjj+1,kkk+1) * p(+1,+1, 0);
4565  ap(1,1,1) =
4566  Amm0(iii+1,jjj+1,kkk+1) * p( 0, 0,-1)
4567  + A0m0(iii+1,jjj+1,kkk+1) * p(+1, 0,-1)
4568  + Am00(iii+1,jjj+1,kkk+1) * p( 0,+1,-1)
4569  + A000(iii+1,jjj+1,kkk+1) * p(+1,+1,-1)
4570  + Ammp(iii+1,jjj+1,kkk+1) * p( 0, 0, 0)
4571  + A0mp(iii+1,jjj+1,kkk+1) * p(+1, 0, 0)
4572  + Am0p(iii+1,jjj+1,kkk+1) * p( 0,+1, 0)
4573  + A00p(iii+1,jjj+1,kkk+1) * p(+1,+1, 0);
4574  csten(i,j,k,ist_00p) = Real(0.125) *
4575  ( restrict_from_mm0_to(iii,jjj,kkk) * ap(-1,-1, 0)
4576  + restrict_from_0m0_to(iii,jjj,kkk) * ap( 0,-1, 0)
4577  + restrict_from_pm0_to(iii,jjj,kkk) * ap(+1,-1, 0)
4578  + restrict_from_m00_to(iii,jjj,kkk) * ap(-1, 0, 0)
4579  + restrict_from_000_to(iii,jjj,kkk) * ap( 0, 0, 0)
4580  + restrict_from_p00_to(iii,jjj,kkk) * ap(+1, 0, 0)
4581  + restrict_from_mp0_to(iii,jjj,kkk) * ap(-1,+1, 0)
4582  + restrict_from_0p0_to(iii,jjj,kkk) * ap( 0,+1, 0)
4583  + restrict_from_pp0_to(iii,jjj,kkk) * ap(+1,+1, 0)
4584  + restrict_from_mmp_to(iii,jjj,kkk) * ap(-1,-1,+1)
4585  + restrict_from_0mp_to(iii,jjj,kkk) * ap( 0,-1,+1)
4586  + restrict_from_pmp_to(iii,jjj,kkk) * ap(+1,-1,+1)
4587  + restrict_from_m0p_to(iii,jjj,kkk) * ap(-1, 0,+1)
4588  + restrict_from_00p_to(iii,jjj,kkk) * ap( 0, 0,+1)
4589  + restrict_from_p0p_to(iii,jjj,kkk) * ap(+1, 0,+1)
4590  + restrict_from_mpp_to(iii,jjj,kkk) * ap(-1,+1,+1)
4591  + restrict_from_0pp_to(iii,jjj,kkk) * ap( 0,+1,+1)
4592  + restrict_from_ppp_to(iii,jjj,kkk) * ap(+1,+1,+1));
4593 
4594  // csten(i,j,k,ist_pp0)
4595  iii = ii;
4596  jjj = jj;
4597  kkk = kk;
4598  p(-1,-1,-1) = interp_from_ppp_to(iii+1,jjj+1,kkk-1);
4599  p( 0,-1,-1) = interp_from_0pp_to(iii+2,jjj+1,kkk-1);
4600  p(-1, 0,-1) = interp_from_p0p_to(iii+1,jjj+2,kkk-1);
4601  p( 0, 0,-1) = interp_from_00p_to(iii+2,jjj+2,kkk-1);
4602  p(-1,-1, 0) = interp_from_pp0_to(iii+1,jjj+1,kkk );
4603  p( 0,-1, 0) = interp_from_0p0_to(iii+2,jjj+1,kkk );
4604  p(-1, 0, 0) = interp_from_p00_to(iii+1,jjj+2,kkk );
4605  p( 0, 0, 0) = Real(1.);
4606  p(-1,-1,+1) = interp_from_ppm_to(iii+1,jjj+1,kkk+1);
4607  p( 0,-1,+1) = interp_from_0pm_to(iii+2,jjj+1,kkk+1);
4608  p(-1, 0,+1) = interp_from_p0m_to(iii+1,jjj+2,kkk+1);
4609  p( 0, 0,+1) = interp_from_00m_to(iii+2,jjj+2,kkk+1);
4610  ap(0,0,-1) =
4611  App0(iii,jjj,kkk-1) * p(-1,-1,-1)
4612  + Appp(iii,jjj,kkk-1) * p(-1,-1, 0);
4613  ap(1,0,-1) =
4614  A0p0(iii+1,jjj,kkk-1) * p(-1,-1,-1)
4615  + App0(iii+1,jjj,kkk-1) * p( 0,-1,-1)
4616  + A0pp(iii+1,jjj,kkk-1) * p(-1,-1, 0)
4617  + Appp(iii+1,jjj,kkk-1) * p( 0,-1, 0);
4618  ap(0,1,-1) =
4619  Ap00(iii,jjj+1,kkk-1) * p(-1,-1,-1)
4620  + App0(iii,jjj+1,kkk-1) * p(-1, 0,-1)
4621  + Ap0p(iii,jjj+1,kkk-1) * p(-1,-1, 0)
4622  + Appp(iii,jjj+1,kkk-1) * p(-1, 0, 0);
4623  ap(1,1,-1) =
4624  A000(iii+1,jjj+1,kkk-1) * p(-1,-1,-1)
4625  + Ap00(iii+1,jjj+1,kkk-1) * p( 0,-1,-1)
4626  + A0p0(iii+1,jjj+1,kkk-1) * p(-1, 0,-1)
4627  + App0(iii+1,jjj+1,kkk-1) * p( 0, 0,-1)
4628  + A00p(iii+1,jjj+1,kkk-1) * p(-1,-1, 0)
4629  + Ap0p(iii+1,jjj+1,kkk-1) * p( 0,-1, 0)
4630  + A0pp(iii+1,jjj+1,kkk-1) * p(-1, 0, 0)
4631  + Appp(iii+1,jjj+1,kkk-1) * p( 0, 0, 0);
4632  ap(0,0,0) =
4633  Appm(iii,jjj,kkk) * p(-1,-1,-1)
4634  + App0(iii,jjj,kkk) * p(-1,-1, 0)
4635  + Appp(iii,jjj,kkk) * p(-1,-1,+1);
4636  ap(1,0,0) =
4637  A0pm(iii+1,jjj,kkk) * p(-1,-1,-1)
4638  + Appm(iii+1,jjj,kkk) * p( 0,-1,-1)
4639  + A0p0(iii+1,jjj,kkk) * p(-1,-1, 0)
4640  + App0(iii+1,jjj,kkk) * p( 0,-1, 0)
4641  + A0pp(iii+1,jjj,kkk) * p(-1,-1,+1)
4642  + Appp(iii+1,jjj,kkk) * p( 0,-1,+1);
4643  ap(0,1,0) =
4644  Ap0m(iii,jjj+1,kkk) * p(-1,-1,-1)
4645  + Appm(iii,jjj+1,kkk) * p(-1, 0,-1)
4646  + Ap00(iii,jjj+1,kkk) * p(-1,-1, 0)
4647  + App0(iii,jjj+1,kkk) * p(-1, 0, 0)
4648  + Ap0p(iii,jjj+1,kkk) * p(-1,-1,+1)
4649  + Appp(iii,jjj+1,kkk) * p(-1, 0,+1);
4650  ap(1,1,0) =
4651  A00m(iii+1,jjj+1,kkk) * p(-1,-1,-1)
4652  + Ap0m(iii+1,jjj+1,kkk) * p( 0,-1,-1)
4653  + A0pm(iii+1,jjj+1,kkk) * p(-1, 0,-1)
4654  + Appm(iii+1,jjj+1,kkk) * p( 0, 0,-1)
4655  + A000(iii+1,jjj+1,kkk) * p(-1,-1, 0)
4656  + Ap00(iii+1,jjj+1,kkk) * p( 0,-1, 0)
4657  + A0p0(iii+1,jjj+1,kkk) * p(-1, 0, 0)
4658  + App0(iii+1,jjj+1,kkk) * p( 0, 0, 0)
4659  + A00p(iii+1,jjj+1,kkk) * p(-1,-1,+1)
4660  + Ap0p(iii+1,jjj+1,kkk) * p( 0,-1,+1)
4661  + A0pp(iii+1,jjj+1,kkk) * p(-1, 0,+1)
4662  + Appp(iii+1,jjj+1,kkk) * p( 0, 0,+1);
4663  ap(0,0,1) =
4664  Appm(iii,jjj,kkk+1) * p(-1,-1, 0)
4665  + App0(iii,jjj,kkk+1) * p(-1,-1,+1);
4666  ap(1,0,1) =
4667  A0pm(iii+1,jjj,kkk+1) * p(-1,-1, 0)
4668  + Appm(iii+1,jjj,kkk+1) * p( 0,-1, 0)
4669  + A0p0(iii+1,jjj,kkk+1) * p(-1,-1,+1)
4670  + App0(iii+1,jjj,kkk+1) * p( 0,-1,+1);
4671  ap(0,1,1) =
4672  Ap0m(iii,jjj+1,kkk+1) * p(-1,-1, 0)
4673  + Appm(iii,jjj+1,kkk+1) * p(-1, 0, 0)
4674  + Ap00(iii,jjj+1,kkk+1) * p(-1,-1,+1)
4675  + App0(iii,jjj+1,kkk+1) * p(-1, 0,+1);
4676  ap(1,1,1) =
4677  A00m(iii+1,jjj+1,kkk+1) * p(-1,-1, 0)
4678  + Ap0m(iii+1,jjj+1,kkk+1) * p( 0,-1, 0)
4679  + A0pm(iii+1,jjj+1,kkk+1) * p(-1, 0, 0)
4680  + Appm(iii+1,jjj+1,kkk+1) * p( 0, 0, 0)
4681  + A000(iii+1,jjj+1,kkk+1) * p(-1,-1,+1)
4682  + Ap00(iii+1,jjj+1,kkk+1) * p( 0,-1,+1)
4683  + A0p0(iii+1,jjj+1,kkk+1) * p(-1, 0,+1)
4684  + App0(iii+1,jjj+1,kkk+1) * p( 0, 0,+1);
4685  cs1 = Real(0.125) *
4686  ( restrict_from_00m_to(iii,jjj,kkk) * ap( 0, 0,-1)
4687  + restrict_from_p0m_to(iii,jjj,kkk) * ap(+1, 0,-1)
4688  + restrict_from_0pm_to(iii,jjj,kkk) * ap( 0,+1,-1)
4689  + restrict_from_ppm_to(iii,jjj,kkk) * ap(+1,+1,-1)
4690  + restrict_from_000_to(iii,jjj,kkk) * ap( 0, 0, 0)
4691  + restrict_from_p00_to(iii,jjj,kkk) * ap(+1, 0, 0)
4692  + restrict_from_0p0_to(iii,jjj,kkk) * ap( 0,+1, 0)
4693  + restrict_from_pp0_to(iii,jjj,kkk) * ap(+1,+1, 0)
4694  + restrict_from_00p_to(iii,jjj,kkk) * ap( 0, 0,+1)
4695  + restrict_from_p0p_to(iii,jjj,kkk) * ap(+1, 0,+1)
4696  + restrict_from_0pp_to(iii,jjj,kkk) * ap( 0,+1,+1)
4697  + restrict_from_ppp_to(iii,jjj,kkk) * ap(+1,+1,+1));
4698 
4699  // alternative: csten(i+1,j,k,ist_mp0)
4700  iii = ii+2;
4701  jjj = jj;
4702  kkk = kk;
4703  p( 0,-1,-1) = interp_from_0pp_to(iii-2,jjj+1,kkk-1);
4704  p(+1,-1,-1) = interp_from_mpp_to(iii-1,jjj+1,kkk-1);
4705  p( 0, 0,-1) = interp_from_00p_to(iii-2,jjj+2,kkk-1);
4706  p(+1, 0,-1) = interp_from_m0p_to(iii-1,jjj+2,kkk-1);
4707  p( 0,-1, 0) = interp_from_0p0_to(iii-2,jjj+1,kkk );
4708  p(+1,-1, 0) = interp_from_mp0_to(iii-1,jjj+1,kkk );
4709  p( 0, 0, 0) = Real(1.);
4710  p(+1, 0, 0) = interp_from_m00_to(iii-1,jjj+2,kkk );
4711  p( 0,-1,+1) = interp_from_0pm_to(iii-2,jjj+1,kkk+1);
4712  p(+1,-1,+1) = interp_from_mpm_to(iii-1,jjj+1,kkk+1);
4713  p( 0, 0,+1) = interp_from_00m_to(iii-2,jjj+2,kkk+1);
4714  p(+1, 0,+1) = interp_from_m0m_to(iii-1,jjj+2,kkk+1);
4715  ap(-1,0,-1) =
4716  Amp0(iii-1,jjj,kkk-1) * p( 0,-1,-1)
4717  + A0p0(iii-1,jjj,kkk-1) * p(+1,-1,-1)
4718  + Ampp(iii-1,jjj,kkk-1) * p( 0,-1, 0)
4719  + A0pp(iii-1,jjj,kkk-1) * p(+1,-1, 0);
4720  ap(0,0,-1) =
4721  Amp0(iii,jjj,kkk-1) * p(+1,-1,-1)
4722  + Ampp(iii,jjj,kkk-1) * p(+1,-1, 0);
4723  ap(-1,1,-1) =
4724  Am00(iii-1,jjj+1,kkk-1) * p( 0,-1,-1)
4725  + A000(iii-1,jjj+1,kkk-1) * p(+1,-1,-1)
4726  + Amp0(iii-1,jjj+1,kkk-1) * p( 0, 0,-1)
4727  + A0p0(iii-1,jjj+1,kkk-1) * p(+1, 0,-1)
4728  + Am0p(iii-1,jjj+1,kkk-1) * p( 0,-1, 0)
4729  + A00p(iii-1,jjj+1,kkk-1) * p(+1,-1, 0)
4730  + Ampp(iii-1,jjj+1,kkk-1) * p( 0, 0, 0)
4731  + A0pp(iii-1,jjj+1,kkk-1) * p(+1, 0, 0);
4732  ap(0,1,-1) =
4733  Am00(iii,jjj+1,kkk-1) * p(+1,-1,-1)
4734  + Amp0(iii,jjj+1,kkk-1) * p(+1, 0,-1)
4735  + Am0p(iii,jjj+1,kkk-1) * p(+1,-1, 0)
4736  + Ampp(iii,jjj+1,kkk-1) * p(+1, 0, 0);
4737  ap(-1,0,0) =
4738  Ampm(iii-1,jjj,kkk) * p( 0,-1,-1)
4739  + A0pm(iii-1,jjj,kkk) * p(+1,-1,-1)
4740  + Amp0(iii-1,jjj,kkk) * p( 0,-1, 0)
4741  + A0p0(iii-1,jjj,kkk) * p(+1,-1, 0)
4742  + Ampp(iii-1,jjj,kkk) * p( 0,-1,+1)
4743  + A0pp(iii-1,jjj,kkk) * p(+1,-1,+1);
4744  ap(0,0,0) =
4745  Ampm(iii,jjj,kkk) * p(+1,-1,-1)
4746  + Amp0(iii,jjj,kkk) * p(+1,-1, 0)
4747  + Ampp(iii,jjj,kkk) * p(+1,-1,+1);
4748  ap(-1,1,0) =
4749  Am0m(iii-1,jjj+1,kkk) * p( 0,-1,-1)
4750  + A00m(iii-1,jjj+1,kkk) * p(+1,-1,-1)
4751  + Ampm(iii-1,jjj+1,kkk) * p( 0, 0,-1)
4752  + A0pm(iii-1,jjj+1,kkk) * p(+1, 0,-1)
4753  + Am00(iii-1,jjj+1,kkk) * p( 0,-1, 0)
4754  + A000(iii-1,jjj+1,kkk) * p(+1,-1, 0)
4755  + Amp0(iii-1,jjj+1,kkk) * p( 0, 0, 0)
4756  + A0p0(iii-1,jjj+1,kkk) * p(+1, 0, 0)
4757  + Am0p(iii-1,jjj+1,kkk) * p( 0,-1,+1)
4758  + A00p(iii-1,jjj+1,kkk) * p(+1,-1,+1)
4759  + Ampp(iii-1,jjj+1,kkk) * p( 0, 0,+1)
4760  + A0pp(iii-1,jjj+1,kkk) * p(+1, 0,+1);
4761  ap(0,1,0) =
4762  Am0m(iii,jjj+1,kkk) * p(+1,-1,-1)
4763  + Ampm(iii,jjj+1,kkk) * p(+1, 0,-1)
4764  + Am00(iii,jjj+1,kkk) * p(+1,-1, 0)
4765  + Amp0(iii,jjj+1,kkk) * p(+1, 0, 0)
4766  + Am0p(iii,jjj+1,kkk) * p(+1,-1,+1)
4767  + Ampp(iii,jjj+1,kkk) * p(+1, 0,+1);
4768  ap(-1,0,1) =
4769  Ampm(iii-1,jjj,kkk+1) * p( 0,-1, 0)
4770  + A0pm(iii-1,jjj,kkk+1) * p(+1,-1, 0)
4771  + Amp0(iii-1,jjj,kkk+1) * p( 0,-1,+1)
4772  + A0p0(iii-1,jjj,kkk+1) * p(+1,-1,+1);
4773  ap(0,0,1) =
4774  Ampm(iii,jjj,kkk+1) * p(+1,-1, 0)
4775  + Amp0(iii,jjj,kkk+1) * p(+1,-1,+1);
4776  ap(-1,1,1) =
4777  Am0m(iii-1,jjj+1,kkk+1) * p( 0,-1, 0)
4778  + A00m(iii-1,jjj+1,kkk+1) * p(+1,-1, 0)
4779  + Ampm(iii-1,jjj+1,kkk+1) * p( 0, 0, 0)
4780  + A0pm(iii-1,jjj+1,kkk+1) * p(+1, 0, 0)
4781  + Am00(iii-1,jjj+1,kkk+1) * p( 0,-1,+1)
4782  + A000(iii-1,jjj+1,kkk+1) * p(+1,-1,+1)
4783  + Amp0(iii-1,jjj+1,kkk+1) * p( 0, 0,+1)
4784  + A0p0(iii-1,jjj+1,kkk+1) * p(+1, 0,+1);
4785  ap(0,1,1) =
4786  Am0m(iii,jjj+1,kkk+1) * p(+1,-1, 0)
4787  + Ampm(iii,jjj+1,kkk+1) * p(+1, 0, 0)
4788  + Am00(iii,jjj+1,kkk+1) * p(+1,-1,+1)
4789  + Amp0(iii,jjj+1,kkk+1) * p(+1, 0,+1);
4790  cs2 = Real(0.125) *
4791  ( restrict_from_m0m_to(iii,jjj,kkk) * ap(-1, 0,-1)
4792  + restrict_from_00m_to(iii,jjj,kkk) * ap( 0, 0,-1)
4793  + restrict_from_mpm_to(iii,jjj,kkk) * ap(-1,+1,-1)
4794  + restrict_from_0pm_to(iii,jjj,kkk) * ap( 0,+1,-1)
4795  + restrict_from_m00_to(iii,jjj,kkk) * ap(-1, 0, 0)
4796  + restrict_from_000_to(iii,jjj,kkk) * ap( 0, 0, 0)
4797  + restrict_from_mp0_to(iii,jjj,kkk) * ap(-1,+1, 0)
4798  + restrict_from_0p0_to(iii,jjj,kkk) * ap( 0,+1, 0)
4799  + restrict_from_m0p_to(iii,jjj,kkk) * ap(-1, 0,+1)
4800  + restrict_from_00p_to(iii,jjj,kkk) * ap( 0, 0,+1)
4801  + restrict_from_mpp_to(iii,jjj,kkk) * ap(-1,+1,+1)
4802  + restrict_from_0pp_to(iii,jjj,kkk) * ap( 0,+1,+1));
4803 
4804  csten(i,j,k,ist_pp0) = Real(0.5)*(cs1 + cs2);
4805 
4806  // csten(i,j,k,ist_p0p)
4807  iii = ii;
4808  jjj = jj;
4809  kkk = kk;
4810  p(-1,-1,-1) = interp_from_ppp_to(iii+1,jjj-1,kkk+1);
4811  p( 0,-1,-1) = interp_from_0pp_to(iii+2,jjj-1,kkk+1);
4812  p(-1, 0,-1) = interp_from_p0p_to(iii+1,jjj ,kkk+1);
4813  p( 0, 0,-1) = interp_from_00p_to(iii+2,jjj ,kkk+1);
4814  p(-1,+1,-1) = interp_from_pmp_to(iii+1,jjj+1,kkk+1);
4815  p( 0,+1,-1) = interp_from_0mp_to(iii+2,jjj+1,kkk+1);
4816  p(-1,-1, 0) = interp_from_pp0_to(iii+1,jjj-1,kkk+2);
4817  p( 0,-1, 0) = interp_from_0p0_to(iii+2,jjj-1,kkk+2);
4818  p(-1, 0, 0) = interp_from_p00_to(iii+1,jjj ,kkk+2);
4819  p( 0, 0, 0) = Real(1.);
4820  p(-1,+1, 0) = interp_from_pm0_to(iii+1,jjj+1,kkk+2);
4821  p( 0,+1, 0) = interp_from_0m0_to(iii+2,jjj+1,kkk+2);
4822  ap(0,-1,0) =
4823  Ap0p(iii,jjj-1,kkk) * p(-1,-1,-1)
4824  + Appp(iii,jjj-1,kkk) * p(-1, 0,-1);
4825  ap(1,-1,0) =
4826  A00p(iii+1,jjj-1,kkk) * p(-1,-1,-1)
4827  + Ap0p(iii+1,jjj-1,kkk) * p( 0,-1,-1)
4828  + A0pp(iii+1,jjj-1,kkk) * p(-1, 0,-1)
4829  + Appp(iii+1,jjj-1,kkk) * p( 0, 0,-1);
4830  ap(0,0,0) =
4831  Apmp(iii,jjj,kkk) * p(-1,-1,-1)
4832  + Ap0p(iii,jjj,kkk) * p(-1, 0,-1)
4833  + Appp(iii,jjj,kkk) * p(-1,+1,-1);
4834  ap(1,0,0) =
4835  A0mp(iii+1,jjj,kkk) * p(-1,-1,-1)
4836  + Apmp(iii+1,jjj,kkk) * p( 0,-1,-1)
4837  + A00p(iii+1,jjj,kkk) * p(-1, 0,-1)
4838  + Ap0p(iii+1,jjj,kkk) * p( 0, 0,-1)
4839  + A0pp(iii+1,jjj,kkk) * p(-1,+1,-1)
4840  + Appp(iii+1,jjj,kkk) * p( 0,+1,-1);
4841  ap(0,1,0) =
4842  Apmp(iii,jjj+1,kkk) * p(-1, 0,-1)
4843  + Ap0p(iii,jjj+1,kkk) * p(-1,+1,-1);
4844  ap(1,1,0) =
4845  A0mp(iii+1,jjj+1,kkk) * p(-1, 0,-1)
4846  + Apmp(iii+1,jjj+1,kkk) * p( 0, 0,-1)
4847  + A00p(iii+1,jjj+1,kkk) * p(-1,+1,-1)
4848  + Ap0p(iii+1,jjj+1,kkk) * p( 0,+1,-1);
4849  ap(0,-1,1) =
4850  Ap00(iii,jjj-1,kkk+1) * p(-1,-1,-1)
4851  + App0(iii,jjj-1,kkk+1) * p(-1, 0,-1)
4852  + Ap0p(iii,jjj-1,kkk+1) * p(-1,-1, 0)
4853  + Appp(iii,jjj-1,kkk+1) * p(-1, 0, 0);
4854  ap(1,-1,1) =
4855  A000(iii+1,jjj-1,kkk+1) * p(-1,-1,-1)
4856  + Ap00(iii+1,jjj-1,kkk+1) * p( 0,-1,-1)
4857  + A0p0(iii+1,jjj-1,kkk+1) * p(-1, 0,-1)
4858  + App0(iii+1,jjj-1,kkk+1) * p( 0, 0,-1)
4859  + A00p(iii+1,jjj-1,kkk+1) * p(-1,-1, 0)
4860  + Ap0p(iii+1,jjj-1,kkk+1) * p( 0,-1, 0)
4861  + A0pp(iii+1,jjj-1,kkk+1) * p(-1, 0, 0)
4862  + Appp(iii+1,jjj-1,kkk+1) * p( 0, 0, 0);
4863  ap(0,0,1) =
4864  Apm0(iii,jjj,kkk+1) * p(-1,-1,-1)
4865  + Ap00(iii,jjj,kkk+1) * p(-1, 0,-1)
4866  + App0(iii,jjj,kkk+1) * p(-1,+1,-1)
4867  + Apmp(iii,jjj,kkk+1) * p(-1,-1, 0)
4868  + Ap0p(iii,jjj,kkk+1) * p(-1, 0, 0)
4869  + Appp(iii,jjj,kkk+1) * p(-1,+1, 0);
4870  ap(1,0,1) =
4871  A0m0(iii+1,jjj,kkk+1) * p(-1,-1,-1)
4872  + Apm0(iii+1,jjj,kkk+1) * p( 0,-1,-1)
4873  + A000(iii+1,jjj,kkk+1) * p(-1, 0,-1)
4874  + Ap00(iii+1,jjj,kkk+1) * p( 0, 0,-1)
4875  + A0p0(iii+1,jjj,kkk+1) * p(-1,+1,-1)
4876  + App0(iii+1,jjj,kkk+1) * p( 0,+1,-1)
4877  + A0mp(iii+1,jjj,kkk+1) * p(-1,-1, 0)
4878  + Apmp(iii+1,jjj,kkk+1) * p( 0,-1, 0)
4879  + A00p(iii+1,jjj,kkk+1) * p(-1, 0, 0)
4880  + Ap0p(iii+1,jjj,kkk+1) * p( 0, 0, 0)
4881  + A0pp(iii+1,jjj,kkk+1) * p(-1,+1, 0)
4882  + Appp(iii+1,jjj,kkk+1) * p( 0,+1, 0);
4883  ap(0,1,1) =
4884  Apm0(iii,jjj+1,kkk+1) * p(-1, 0,-1)
4885  + Ap00(iii,jjj+1,kkk+1) * p(-1,+1,-1)
4886  + Apmp(iii,jjj+1,kkk+1) * p(-1, 0, 0)
4887  + Ap0p(iii,jjj+1,kkk+1) * p(-1,+1, 0);
4888  ap(1,1,1) =
4889  A0m0(iii+1,jjj+1,kkk+1) * p(-1, 0,-1)
4890  + Apm0(iii+1,jjj+1,kkk+1) * p( 0, 0,-1)
4891  + A000(iii+1,jjj+1,kkk+1) * p(-1,+1,-1)
4892  + Ap00(iii+1,jjj+1,kkk+1) * p( 0,+1,-1)
4893  + A0mp(iii+1,jjj+1,kkk+1) * p(-1, 0, 0)
4894  + Apmp(iii+1,jjj+1,kkk+1) * p( 0, 0, 0)
4895  + A00p(iii+1,jjj+1,kkk+1) * p(-1,+1, 0)
4896  + Ap0p(iii+1,jjj+1,kkk+1) * p( 0,+1, 0);
4897  cs1 = Real(0.125) *
4898  ( restrict_from_0m0_to(iii,jjj,kkk) * ap( 0,-1, 0)
4899  + restrict_from_pm0_to(iii,jjj,kkk) * ap(+1,-1, 0)
4900  + restrict_from_000_to(iii,jjj,kkk) * ap( 0, 0, 0)
4901  + restrict_from_p00_to(iii,jjj,kkk) * ap(+1, 0, 0)
4902  + restrict_from_0p0_to(iii,jjj,kkk) * ap( 0,+1, 0)
4903  + restrict_from_pp0_to(iii,jjj,kkk) * ap(+1,+1, 0)
4904  + restrict_from_0mp_to(iii,jjj,kkk) * ap( 0,-1,+1)
4905  + restrict_from_pmp_to(iii,jjj,kkk) * ap(+1,-1,+1)
4906  + restrict_from_00p_to(iii,jjj,kkk) * ap( 0, 0,+1)
4907  + restrict_from_p0p_to(iii,jjj,kkk) * ap(+1, 0,+1)
4908  + restrict_from_0pp_to(iii,jjj,kkk) * ap( 0,+1,+1)
4909  + restrict_from_ppp_to(iii,jjj,kkk) * ap(+1,+1,+1));
4910 
4911  // alternative: csten(i+1,j,k,ist_m0p)
4912  iii = ii+2;
4913  jjj = jj;
4914  kkk = kk;
4915  p( 0,-1,-1) = interp_from_0pp_to(iii-2,jjj-1,kkk+1);
4916  p(+1,-1,-1) = interp_from_mpp_to(iii-1,jjj-1,kkk+1);
4917  p( 0, 0,-1) = interp_from_00p_to(iii-2,jjj ,kkk+1);
4918  p(+1, 0,-1) = interp_from_m0p_to(iii-1,jjj ,kkk+1);
4919  p( 0,+1,-1) = interp_from_0mp_to(iii-2,jjj+1,kkk+1);
4920  p(+1,+1,-1) = interp_from_mmp_to(iii-1,jjj+1,kkk+1);
4921  p( 0,-1, 0) = interp_from_0p0_to(iii-2,jjj-1,kkk+2);
4922  p(+1,-1, 0) = interp_from_mp0_to(iii-1,jjj-1,kkk+2);
4923  p( 0, 0, 0) = Real(1.);
4924  p(+1, 0, 0) = interp_from_m00_to(iii-1,jjj ,kkk+2);
4925  p( 0,+1, 0) = interp_from_0m0_to(iii-2,jjj+1,kkk+2);
4926  p(+1,+1, 0) = interp_from_mm0_to(iii-1,jjj+1,kkk+2);
4927 
4928  ap(-1,-1,0) =
4929  Am0p(iii-1,jjj-1,kkk) * p( 0,-1,-1)
4930  + A00p(iii-1,jjj-1,kkk) * p(+1,-1,-1)
4931  + Ampp(iii-1,jjj-1,kkk) * p( 0, 0,-1)
4932  + A0pp(iii-1,jjj-1,kkk) * p(+1, 0,-1);
4933  ap(0,-1,0) =
4934  Am0p(iii,jjj-1,kkk) * p(+1,-1,-1)
4935  + Ampp(iii,jjj-1,kkk) * p(+1, 0,-1);
4936  ap(-1,0,0) =
4937  Ammp(iii-1,jjj,kkk) * p( 0,-1,-1)
4938  + A0mp(iii-1,jjj,kkk) * p(+1,-1,-1)
4939  + Am0p(iii-1,jjj,kkk) * p( 0, 0,-1)
4940  + A00p(iii-1,jjj,kkk) * p(+1, 0,-1)
4941  + Ampp(iii-1,jjj,kkk) * p( 0,+1,-1)
4942  + A0pp(iii-1,jjj,kkk) * p(+1,+1,-1);
4943  ap(0,0,0) =
4944  Ammp(iii,jjj,kkk) * p(+1,-1,-1)
4945  + Am0p(iii,jjj,kkk) * p(+1, 0,-1)
4946  + Ampp(iii,jjj,kkk) * p(+1,+1,-1);
4947  ap(-1,1,0) =
4948  Ammp(iii-1,jjj+1,kkk) * p( 0, 0,-1)
4949  + A0mp(iii-1,jjj+1,kkk) * p(+1, 0,-1)
4950  + Am0p(iii-1,jjj+1,kkk) * p( 0,+1,-1)
4951  + A00p(iii-1,jjj+1,kkk) * p(+1,+1,-1);
4952  ap(0,1,0) =
4953  Ammp(iii,jjj+1,kkk) * p(+1, 0,-1)
4954  + Am0p(iii,jjj+1,kkk) * p(+1,+1,-1);
4955  ap(-1,-1,1) =
4956  Am00(iii-1,jjj-1,kkk+1) * p( 0,-1,-1)
4957  + A000(iii-1,jjj-1,kkk+1) * p(+1,-1,-1)
4958  + Amp0(iii-1,jjj-1,kkk+1) * p( 0, 0,-1)
4959  + A0p0(iii-1,jjj-1,kkk+1) * p(+1, 0,-1)
4960  + Am0p(iii-1,jjj-1,kkk+1) * p( 0,-1, 0)
4961  + A00p(iii-1,jjj-1,kkk+1) * p(+1,-1, 0)
4962  + Ampp(iii-1,jjj-1,kkk+1) * p( 0, 0, 0)
4963  + A0pp(iii-1,jjj-1,kkk+1) * p(+1, 0, 0);
4964  ap(0,-1,1) =
4965  Am00(iii,jjj-1,kkk+1) * p(+1,-1,-1)
4966  + Amp0(iii,jjj-1,kkk+1) * p(+1, 0,-1)
4967  + Am0p(iii,jjj-1,kkk+1) * p(+1,-1, 0)
4968  + Ampp(iii,jjj-1,kkk+1) * p(+1, 0, 0);
4969  ap(-1,0,1) =
4970  Amm0(iii-1,jjj,kkk+1) * p( 0,-1,-1)
4971  + A0m0(iii-1,jjj,kkk+1) * p(+1,-1,-1)
4972  + Am00(iii-1,jjj,kkk+1) * p( 0, 0,-1)
4973  + A000(iii-1,jjj,kkk+1) * p(+1, 0,-1)
4974  + Amp0(iii-1,jjj,kkk+1) * p( 0,+1,-1)
4975  + A0p0(iii-1,jjj,kkk+1) * p(+1,+1,-1)
4976  + Ammp(iii-1,jjj,kkk+1) * p( 0,-1, 0)
4977  + A0mp(iii-1,jjj,kkk+1) * p(+1,-1, 0)
4978  + Am0p(iii-1,jjj,kkk+1) * p( 0, 0, 0)
4979  + A00p(iii-1,jjj,kkk+1) * p(+1, 0, 0)
4980  + Ampp(iii-1,jjj,kkk+1) * p( 0,+1, 0)
4981  + A0pp(iii-1,jjj,kkk+1) * p(+1,+1, 0);
4982  ap(0,0,1) =
4983  Amm0(iii,jjj,kkk+1) * p(+1,-1,-1)
4984  + Am00(iii,jjj,kkk+1) * p(+1, 0,-1)
4985  + Amp0(iii,jjj,kkk+1) * p(+1,+1,-1)
4986  + Ammp(iii,jjj,kkk+1) * p(+1,-1, 0)
4987  + Am0p(iii,jjj,kkk+1) * p(+1, 0, 0)
4988  + Ampp(iii,jjj,kkk+1) * p(+1,+1, 0);
4989  ap(-1,1,1) =
4990  Amm0(iii-1,jjj+1,kkk+1) * p( 0, 0,-1)
4991  + A0m0(iii-1,jjj+1,kkk+1) * p(+1, 0,-1)
4992  + Am00(iii-1,jjj+1,kkk+1) * p( 0,+1,-1)
4993  + A000(iii-1,jjj+1,kkk+1) * p(+1,+1,-1)
4994  + Ammp(iii-1,jjj+1,kkk+1) * p( 0, 0, 0)
4995  + A0mp(iii-1,jjj+1,kkk+1) * p(+1, 0, 0)
4996  + Am0p(iii-1,jjj+1,kkk+1) * p( 0,+1, 0)
4997  + A00p(iii-1,jjj+1,kkk+1) * p(+1,+1, 0);
4998  ap(0,1,1) =
4999  Amm0(iii,jjj+1,kkk+1) * p(+1, 0,-1)
5000  + Am00(iii,jjj+1,kkk+1) * p(+1,+1,-1)
5001  + Ammp(iii,jjj+1,kkk+1) * p(+1, 0, 0)
5002  + Am0p(iii,jjj+1,kkk+1) * p(+1,+1, 0);
5003  cs2 = Real(0.125) *
5004  ( restrict_from_mm0_to(iii,jjj,kkk) * ap(-1,-1, 0)
5005  + restrict_from_0m0_to(iii,jjj,kkk) * ap( 0,-1, 0)
5006  + restrict_from_m00_to(iii,jjj,kkk) * ap(-1, 0, 0)
5007  + restrict_from_000_to(iii,jjj,kkk) * ap( 0, 0, 0)
5008  + restrict_from_mp0_to(iii,jjj,kkk) * ap(-1,+1, 0)
5009  + restrict_from_0p0_to(iii,jjj,kkk) * ap( 0,+1, 0)
5010  + restrict_from_mmp_to(iii,jjj,kkk) * ap(-1,-1,+1)
5011  + restrict_from_0mp_to(iii,jjj,kkk) * ap( 0,-1,+1)
5012  + restrict_from_m0p_to(iii,jjj,kkk) * ap(-1, 0,+1)
5013  + restrict_from_00p_to(iii,jjj,kkk) * ap( 0, 0,+1)
5014  + restrict_from_mpp_to(iii,jjj,kkk) * ap(-1,+1,+1)
5015  + restrict_from_0pp_to(iii,jjj,kkk) * ap( 0,+1,+1));
5016 
5017  csten(i,j,k,ist_p0p) = Real(0.5)*(cs1+cs2);
5018 
5019  // csten(i,j,k,ist_0pp)
5020  iii = ii;
5021  jjj = jj;
5022  kkk = kk;
5023  p(-1,-1,-1) = interp_from_ppp_to(iii-1,jjj+1,kkk+1);
5024  p( 0,-1,-1) = interp_from_0pp_to(iii ,jjj+1,kkk+1);
5025  p(+1,-1,-1) = interp_from_mpp_to(iii+1,jjj+1,kkk+1);
5026  p(-1, 0,-1) = interp_from_p0p_to(iii-1,jjj+2,kkk+1);
5027  p( 0, 0,-1) = interp_from_00p_to(iii ,jjj+2,kkk+1);
5028  p(+1, 0,-1) = interp_from_m0p_to(iii+1,jjj+2,kkk+1);
5029  p(-1,-1, 0) = interp_from_pp0_to(iii-1,jjj+1,kkk+2);
5030  p( 0,-1, 0) = interp_from_0p0_to(iii ,jjj+1,kkk+2);
5031  p(+1,-1, 0) = interp_from_mp0_to(iii+1,jjj+1,kkk+2);
5032  p(-1, 0, 0) = interp_from_p00_to(iii-1,jjj+2,kkk+2);
5033  p( 0, 0, 0) = Real(1.);
5034  p(+1, 0, 0) = interp_from_m00_to(iii+1,jjj+2,kkk+2);
5035  ap(-1,0,0) =
5036  A0pp(iii-1,jjj,kkk) * p(-1,-1,-1)
5037  + Appp(iii-1,jjj,kkk) * p( 0,-1,-1);
5038  ap(0,0,0) =
5039  Ampp(iii,jjj,kkk) * p(-1,-1,-1)
5040  + A0pp(iii,jjj,kkk) * p( 0,-1,-1)
5041  + Appp(iii,jjj,kkk) * p(+1,-1,-1);
5042  ap(1,0,0) =
5043  Ampp(iii+1,jjj,kkk) * p( 0,-1,-1)
5044  + A0pp(iii+1,jjj,kkk) * p(+1,-1,-1);
5045  ap(-1,1,0) =
5046  A00p(iii-1,jjj+1,kkk) * p(-1,-1,-1)
5047  + Ap0p(iii-1,jjj+1,kkk) * p( 0,-1,-1)
5048  + A0pp(iii-1,jjj+1,kkk) * p(-1, 0,-1)
5049  + Appp(iii-1,jjj+1,kkk) * p( 0, 0,-1);
5050  ap(0,1,0) =
5051  Am0p(iii,jjj+1,kkk) * p(-1,-1,-1)
5052  + A00p(iii,jjj+1,kkk) * p( 0,-1,-1)
5053  + Ap0p(iii,jjj+1,kkk) * p(+1,-1,-1)
5054  + Ampp(iii,jjj+1,kkk) * p(-1, 0,-1)
5055  + A0pp(iii,jjj+1,kkk) * p( 0, 0,-1)
5056  + Appp(iii,jjj+1,kkk) * p(+1, 0,-1);
5057  ap(1,1,0) =
5058  Am0p(iii+1,jjj+1,kkk) * p( 0,-1,-1)
5059  + A00p(iii+1,jjj+1,kkk) * p(+1,-1,-1)
5060  + Ampp(iii+1,jjj+1,kkk) * p( 0, 0,-1)
5061  + A0pp(iii+1,jjj+1,kkk) * p(+1, 0,-1);
5062  ap(-1,0,1) =
5063  A0p0(iii-1,jjj,kkk+1) * p(-1,-1,-1)
5064  + App0(iii-1,jjj,kkk+1) * p( 0,-1,-1)
5065  + A0pp(iii-1,jjj,kkk+1) * p(-1,-1, 0)
5066  + Appp(iii-1,jjj,kkk+1) * p( 0,-1, 0);
5067  ap(0,0,1) =
5068  Amp0(iii,jjj,kkk+1) * p(-1,-1,-1)
5069  + A0p0(iii,jjj,kkk+1) * p( 0,-1,-1)
5070  + App0(iii,jjj,kkk+1) * p(+1,-1,-1)
5071  + Ampp(iii,jjj,kkk+1) * p(-1,-1, 0)
5072  + A0pp(iii,jjj,kkk+1) * p( 0,-1, 0)
5073  + Appp(iii,jjj,kkk+1) * p(+1,-1, 0);
5074  ap(1,0,1) =
5075  Amp0(iii+1,jjj,kkk+1) * p( 0,-1,-1)
5076  + A0p0(iii+1,jjj,kkk+1) * p(+1,-1,-1)
5077  + Ampp(iii+1,jjj,kkk+1) * p( 0,-1, 0)
5078  + A0pp(iii+1,jjj,kkk+1) * p(+1,-1, 0);
5079  ap(-1,1,1) =
5080  A000(iii-1,jjj+1,kkk+1) * p(-1,-1,-1)
5081  + Ap00(iii-1,jjj+1,kkk+1) * p( 0,-1,-1)
5082  + A0p0(iii-1,jjj+1,kkk+1) * p(-1, 0,-1)
5083  + App0(iii-1,jjj+1,kkk+1) * p( 0, 0,-1)
5084  + A00p(iii-1,jjj+1,kkk+1) * p(-1,-1, 0)
5085  + Ap0p(iii-1,jjj+1,kkk+1) * p( 0,-1, 0)
5086  + A0pp(iii-1,jjj+1,kkk+1) * p(-1, 0, 0)
5087  + Appp(iii-1,jjj+1,kkk+1) * p( 0, 0, 0);
5088  ap(0,1,1) =
5089  Am00(iii,jjj+1,kkk+1) * p(-1,-1,-1)
5090  + A000(iii,jjj+1,kkk+1) * p( 0,-1,-1)
5091  + Ap00(iii,jjj+1,kkk+1) * p(+1,-1,-1)
5092  + Amp0(iii,jjj+1,kkk+1) * p(-1, 0,-1)
5093  + A0p0(iii,jjj+1,kkk+1) * p( 0, 0,-1)
5094  + App0(iii,jjj+1,kkk+1) * p(+1, 0,-1)
5095  + Am0p(iii,jjj+1,kkk+1) * p(-1,-1, 0)
5096  + A00p(iii,jjj+1,kkk+1) * p( 0,-1, 0)
5097  + Ap0p(iii,jjj+1,kkk+1) * p(+1,-1, 0)
5098  + Ampp(iii,jjj+1,kkk+1) * p(-1, 0, 0)
5099  + A0pp(iii,jjj+1,kkk+1) * p( 0, 0, 0)
5100  + Appp(iii,jjj+1,kkk+1) * p(+1, 0, 0);
5101  ap(1,1,1) =
5102  Am00(iii+1,jjj+1,kkk+1) * p( 0,-1,-1)
5103  + A000(iii+1,jjj+1,kkk+1) * p(+1,-1,-1)
5104  + Amp0(iii+1,jjj+1,kkk+1) * p( 0, 0,-1)
5105  + A0p0(iii+1,jjj+1,kkk+1) * p(+1, 0,-1)
5106  + Am0p(iii+1,jjj+1,kkk+1) * p( 0,-1, 0)
5107  + A00p(iii+1,jjj+1,kkk+1) * p(+1,-1, 0)
5108  + Ampp(iii+1,jjj+1,kkk+1) * p( 0, 0, 0)
5109  + A0pp(iii+1,jjj+1,kkk+1) * p(+1, 0, 0);
5110  cs1 = Real(0.125) *
5111  ( restrict_from_m00_to(iii,jjj,kkk) * ap(-1, 0, 0)
5112  + restrict_from_000_to(iii,jjj,kkk) * ap( 0, 0, 0)
5113  + restrict_from_p00_to(iii,jjj,kkk) * ap(+1, 0, 0)
5114  + restrict_from_mp0_to(iii,jjj,kkk) * ap(-1,+1, 0)
5115  + restrict_from_0p0_to(iii,jjj,kkk) * ap( 0,+1, 0)
5116  + restrict_from_pp0_to(iii,jjj,kkk) * ap(+1,+1, 0)
5117  + restrict_from_m0p_to(iii,jjj,kkk) * ap(-1, 0,+1)
5118  + restrict_from_00p_to(iii,jjj,kkk) * ap( 0, 0,+1)
5119  + restrict_from_p0p_to(iii,jjj,kkk) * ap(+1, 0,+1)
5120  + restrict_from_mpp_to(iii,jjj,kkk) * ap(-1,+1,+1)
5121  + restrict_from_0pp_to(iii,jjj,kkk) * ap( 0,+1,+1)
5122  + restrict_from_ppp_to(iii,jjj,kkk) * ap(+1,+1,+1));
5123 
5124  // alternative: csten(i,j+1,k,ist_0mp)
5125  iii = ii;
5126  jjj = jj+2;
5127  kkk = kk;
5128  p(-1, 0,-1) = interp_from_p0p_to(iii-1,jjj-2,kkk+1);
5129  p( 0, 0,-1) = interp_from_00p_to(iii ,jjj-2,kkk+1);
5130  p(+1, 0,-1) = interp_from_m0p_to(iii+1,jjj-2,kkk+1);
5131  p(-1,+1,-1) = interp_from_pmp_to(iii-1,jjj-1,kkk+1);
5132  p( 0,+1,-1) = interp_from_0mp_to(iii ,jjj-1,kkk+1);
5133  p(+1,+1,-1) = interp_from_mmp_to(iii+1,jjj-1,kkk+1);
5134  p(-1, 0, 0) = interp_from_p00_to(iii-1,jjj-2,kkk+2);
5135  p( 0, 0, 0) = Real(1.);
5136  p(+1, 0, 0) = interp_from_m00_to(iii+1,jjj-2,kkk+2);
5137  p(-1,+1, 0) = interp_from_pm0_to(iii-1,jjj-1,kkk+2);
5138  p( 0,+1, 0) = interp_from_0m0_to(iii ,jjj-1,kkk+2);
5139  p(+1,+1, 0) = interp_from_mm0_to(iii+1,jjj-1,kkk+2);
5140  ap(-1,-1,0) =
5141  A0mp(iii-1,jjj-1,kkk) * p(-1, 0,-1)
5142  + Apmp(iii-1,jjj-1,kkk) * p( 0, 0,-1)
5143  + A00p(iii-1,jjj-1,kkk) * p(-1,+1,-1)
5144  + Ap0p(iii-1,jjj-1,kkk) * p( 0,+1,-1);
5145  ap(0,-1,0) =
5146  Ammp(iii,jjj-1,kkk) * p(-1, 0,-1)
5147  + A0mp(iii,jjj-1,kkk) * p( 0, 0,-1)
5148  + Apmp(iii,jjj-1,kkk) * p(+1, 0,-1)
5149  + Am0p(iii,jjj-1,kkk) * p(-1,+1,-1)
5150  + A00p(iii,jjj-1,kkk) * p( 0,+1,-1)
5151  + Ap0p(iii,jjj-1,kkk) * p(+1,+1,-1);
5152  ap(1,-1,0) =
5153  Ammp(iii+1,jjj-1,kkk) * p( 0, 0,-1)
5154  + A0mp(iii+1,jjj-1,kkk) * p(+1, 0,-1)
5155  + Am0p(iii+1,jjj-1,kkk) * p( 0,+1,-1)
5156  + A00p(iii+1,jjj-1,kkk) * p(+1,+1,-1);
5157  ap(-1,0,0) =
5158  A0mp(iii-1,jjj,kkk) * p(-1,+1,-1)
5159  + Apmp(iii-1,jjj,kkk) * p( 0,+1,-1);
5160  ap(0,0,0) =
5161  Ammp(iii,jjj,kkk) * p(-1,+1,-1)
5162  + A0mp(iii,jjj,kkk) * p( 0,+1,-1)
5163  + Apmp(iii,jjj,kkk) * p(+1,+1,-1);
5164  ap(1,0,0) =
5165  Ammp(iii+1,jjj,kkk) * p( 0,+1,-1)
5166  + A0mp(iii+1,jjj,kkk) * p(+1,+1,-1);
5167  ap(-1,-1,1) =
5168  A0m0(iii-1,jjj-1,kkk+1) * p(-1, 0,-1)
5169  + Apm0(iii-1,jjj-1,kkk+1) * p( 0, 0,-1)
5170  + A000(iii-1,jjj-1,kkk+1) * p(-1,+1,-1)
5171  + Ap00(iii-1,jjj-1,kkk+1) * p( 0,+1,-1)
5172  + A0mp(iii-1,jjj-1,kkk+1) * p(-1, 0, 0)
5173  + Apmp(iii-1,jjj-1,kkk+1) * p( 0, 0, 0)
5174  + A00p(iii-1,jjj-1,kkk+1) * p(-1,+1, 0)
5175  + Ap0p(iii-1,jjj-1,kkk+1) * p( 0,+1, 0);
5176  ap(0,-1,1) =
5177  Amm0(iii,jjj-1,kkk+1) * p(-1, 0,-1)
5178  + A0m0(iii,jjj-1,kkk+1) * p( 0, 0,-1)
5179  + Apm0(iii,jjj-1,kkk+1) * p(+1, 0,-1)
5180  + Am00(iii,jjj-1,kkk+1) * p(-1,+1,-1)
5181  + A000(iii,jjj-1,kkk+1) * p( 0,+1,-1)
5182  + Ap00(iii,jjj-1,kkk+1) * p(+1,+1,-1)
5183  + Ammp(iii,jjj-1,kkk+1) * p(-1, 0, 0)
5184  + A0mp(iii,jjj-1,kkk+1) * p( 0, 0, 0)
5185  + Apmp(iii,jjj-1,kkk+1) * p(+1, 0, 0)
5186  + Am0p(iii,jjj-1,kkk+1) * p(-1,+1, 0)
5187  + A00p(iii,jjj-1,kkk+1) * p( 0,+1, 0)
5188  + Ap0p(iii,jjj-1,kkk+1) * p(+1,+1, 0);
5189  ap(1,-1,1) =
5190  Amm0(iii+1,jjj-1,kkk+1) * p( 0, 0,-1)
5191  + A0m0(iii+1,jjj-1,kkk+1) * p(+1, 0,-1)
5192  + Am00(iii+1,jjj-1,kkk+1) * p( 0,+1,-1)
5193  + A000(iii+1,jjj-1,kkk+1) * p(+1,+1,-1)
5194  + Ammp(iii+1,jjj-1,kkk+1) * p( 0, 0, 0)
5195  + A0mp(iii+1,jjj-1,kkk+1) * p(+1, 0, 0)
5196  + Am0p(iii+1,jjj-1,kkk+1) * p( 0,+1, 0)
5197  + A00p(iii+1,jjj-1,kkk+1) * p(+1,+1, 0);
5198  ap(-1,0,1) =
5199  A0m0(iii-1,jjj,kkk+1) * p(-1,+1,-1)
5200  + Apm0(iii-1,jjj,kkk+1) * p( 0,+1,-1)
5201  + A0mp(iii-1,jjj,kkk+1) * p(-1,+1, 0)
5202  + Apmp(iii-1,jjj,kkk+1) * p( 0,+1, 0);
5203  ap(0,0,1) =
5204  Amm0(iii,jjj,kkk+1) * p(-1,+1,-1)
5205  + A0m0(iii,jjj,kkk+1) * p( 0,+1,-1)
5206  + Apm0(iii,jjj,kkk+1) * p(+1,+1,-1)
5207  + Ammp(iii,jjj,kkk+1) * p(-1,+1, 0)
5208  + A0mp(iii,jjj,kkk+1) * p( 0,+1, 0)
5209  + Apmp(iii,jjj,kkk+1) * p(+1,+1, 0);
5210  ap(1,0,1) =
5211  Amm0(iii+1,jjj,kkk+1) * p( 0,+1,-1)
5212  + A0m0(iii+1,jjj,kkk+1) * p(+1,+1,-1)
5213  + Ammp(iii+1,jjj,kkk+1) * p( 0,+1, 0)
5214  + A0mp(iii+1,jjj,kkk+1) * p(+1,+1, 0);
5215  cs2 = Real(0.125) *
5216  ( restrict_from_mm0_to(iii,jjj,kkk) * ap(-1,-1, 0)
5217  + restrict_from_0m0_to(iii,jjj,kkk) * ap( 0,-1, 0)
5218  + restrict_from_pm0_to(iii,jjj,kkk) * ap(+1,-1, 0)
5219  + restrict_from_m00_to(iii,jjj,kkk) * ap(-1, 0, 0)
5220  + restrict_from_000_to(iii,jjj,kkk) * ap( 0, 0, 0)
5221  + restrict_from_p00_to(iii,jjj,kkk) * ap(+1, 0, 0)
5222  + restrict_from_mmp_to(iii,jjj,kkk) * ap(-1,-1,+1)
5223  + restrict_from_0mp_to(iii,jjj,kkk) * ap( 0,-1,+1)
5224  + restrict_from_pmp_to(iii,jjj,kkk) * ap(+1,-1,+1)
5225  + restrict_from_m0p_to(iii,jjj,kkk) * ap(-1, 0,+1)
5226  + restrict_from_00p_to(iii,jjj,kkk) * ap( 0, 0,+1)
5227  + restrict_from_p0p_to(iii,jjj,kkk) * ap(+1, 0,+1));
5228 
5229  csten(i,j,k,ist_0pp) = Real(0.5)*(cs1+cs2);
5230 
5231  // csten(i,j,k,ist_ppp)
5232  iii = ii;
5233  jjj = jj;
5234  kkk = kk;
5235  p(-1,-1,-1) = interp_from_ppp_to(iii+1,jjj+1,kkk+1);
5236  p( 0,-1,-1) = interp_from_0pp_to(iii+2,jjj+1,kkk+1);
5237  p(-1, 0,-1) = interp_from_p0p_to(iii+1,jjj+2,kkk+1);
5238  p( 0, 0,-1) = interp_from_00p_to(iii+2,jjj+2,kkk+1);
5239  p(-1,-1, 0) = interp_from_pp0_to(iii+1,jjj+1,kkk+2);
5240  p( 0,-1, 0) = interp_from_0p0_to(iii+2,jjj+1,kkk+2);
5241  p(-1, 0, 0) = interp_from_p00_to(iii+1,jjj+2,kkk+2);
5242  p( 0, 0, 0) = Real(1.);
5243  ap(0,0,0) =
5244  Appp(iii,jjj,kkk) * p(-1,-1,-1);
5245  ap(1,0,0) =
5246  A0pp(iii+1,jjj,kkk) * p(-1,-1,-1)
5247  + Appp(iii+1,jjj,kkk) * p( 0,-1,-1);
5248  ap(0,1,0) =
5249  Ap0p(iii,jjj+1,kkk) * p(-1,-1,-1)
5250  + Appp(iii,jjj+1,kkk) * p(-1, 0,-1);
5251  ap(1,1,0) =
5252  A00p(iii+1,jjj+1,kkk) * p(-1,-1,-1)
5253  + Ap0p(iii+1,jjj+1,kkk) * p( 0,-1,-1)
5254  + A0pp(iii+1,jjj+1,kkk) * p(-1, 0,-1)
5255  + Appp(iii+1,jjj+1,kkk) * p( 0, 0,-1);
5256  ap(0,0,1) =
5257  App0(iii,jjj,kkk+1) * p(-1,-1,-1)
5258  + Appp(iii,jjj,kkk+1) * p(-1,-1, 0);
5259  ap(1,0,1) =
5260  A0p0(iii+1,jjj,kkk+1) * p(-1,-1,-1)
5261  + App0(iii+1,jjj,kkk+1) * p( 0,-1,-1)
5262  + A0pp(iii+1,jjj,kkk+1) * p(-1,-1, 0)
5263  + Appp(iii+1,jjj,kkk+1) * p( 0,-1, 0);
5264  ap(0,1,1) =
5265  Ap00(iii,jjj+1,kkk+1) * p(-1,-1,-1)
5266  + App0(iii,jjj+1,kkk+1) * p(-1, 0,-1)
5267  + Ap0p(iii,jjj+1,kkk+1) * p(-1,-1, 0)
5268  + Appp(iii,jjj+1,kkk+1) * p(-1, 0, 0);
5269  ap(1,1,1) =
5270  A000(iii+1,jjj+1,kkk+1) * p(-1,-1,-1)
5271  + Ap00(iii+1,jjj+1,kkk+1) * p( 0,-1,-1)
5272  + A0p0(iii+1,jjj+1,kkk+1) * p(-1, 0,-1)
5273  + App0(iii+1,jjj+1,kkk+1) * p( 0, 0,-1)
5274  + A00p(iii+1,jjj+1,kkk+1) * p(-1,-1, 0)
5275  + Ap0p(iii+1,jjj+1,kkk+1) * p( 0,-1, 0)
5276  + A0pp(iii+1,jjj+1,kkk+1) * p(-1, 0, 0)
5277  + Appp(iii+1,jjj+1,kkk+1) * p( 0, 0, 0);
5278  cs1 = Real(0.125) *
5279  ( restrict_from_000_to(iii,jjj,kkk) * ap( 0, 0, 0)
5280  + restrict_from_p00_to(iii,jjj,kkk) * ap(+1, 0, 0)
5281  + restrict_from_0p0_to(iii,jjj,kkk) * ap( 0,+1, 0)
5282  + restrict_from_pp0_to(iii,jjj,kkk) * ap(+1,+1, 0)
5283  + restrict_from_00p_to(iii,jjj,kkk) * ap( 0, 0,+1)
5284  + restrict_from_p0p_to(iii,jjj,kkk) * ap(+1, 0,+1)
5285  + restrict_from_0pp_to(iii,jjj,kkk) * ap( 0,+1,+1)
5286  + restrict_from_ppp_to(iii,jjj,kkk) * ap(+1,+1,+1));
5287 
5288  // alternative: csten(i+1,j,k,ist_mpp)
5289  iii = ii+2;
5290  jjj = jj;
5291  kkk = kk;
5292  p( 0,-1,-1) = interp_from_0pp_to(iii-2,jjj+1,kkk+1);
5293  p(+1,-1,-1) = interp_from_mpp_to(iii-1,jjj+1,kkk+1);
5294  p( 0, 0,-1) = interp_from_00p_to(iii-2,jjj+2,kkk+1);
5295  p(+1, 0,-1) = interp_from_m0p_to(iii-1,jjj+2,kkk+1);
5296  p( 0,-1, 0) = interp_from_0p0_to(iii-2,jjj+1,kkk+2);
5297  p(+1,-1, 0) = interp_from_mp0_to(iii-1,jjj+1,kkk+2);
5298  p( 0, 0, 0) = Real(1.);
5299  p(+1, 0, 0) = interp_from_m00_to(iii-1,jjj+2,kkk+2);
5300  ap(-1,0,0) =
5301  Ampp(iii-1,jjj,kkk) * p( 0,-1,-1)
5302  + A0pp(iii-1,jjj,kkk) * p(+1,-1,-1);
5303  ap(0,0,0) =
5304  Ampp(iii,jjj,kkk) * p(+1,-1,-1);
5305  ap(-1,1,0) =
5306  Am0p(iii-1,jjj+1,kkk) * p( 0,-1,-1)
5307  + A00p(iii-1,jjj+1,kkk) * p(+1,-1,-1)
5308  + Ampp(iii-1,jjj+1,kkk) * p( 0, 0,-1)
5309  + A0pp(iii-1,jjj+1,kkk) * p(+1, 0,-1);
5310  ap(0,1,0) =
5311  Am0p(iii,jjj+1,kkk) * p(+1,-1,-1)
5312  + Ampp(iii,jjj+1,kkk) * p(+1, 0,-1);
5313  ap(-1,0,1) =
5314  Amp0(iii-1,jjj,kkk+1) * p( 0,-1,-1)
5315  + A0p0(iii-1,jjj,kkk+1) * p(+1,-1,-1)
5316  + Ampp(iii-1,jjj,kkk+1) * p( 0,-1, 0)
5317  + A0pp(iii-1,jjj,kkk+1) * p(+1,-1, 0);
5318  ap(0,0,1) =
5319  Amp0(iii,jjj,kkk+1) * p(+1,-1,-1)
5320  + Ampp(iii,jjj,kkk+1) * p(+1,-1, 0);
5321  ap(-1,1,1) =
5322  Am00(iii-1,jjj+1,kkk+1) * p( 0,-1,-1)
5323  + A000(iii-1,jjj+1,kkk+1) * p(+1,-1,-1)
5324  + Amp0(iii-1,jjj+1,kkk+1) * p( 0, 0,-1)
5325  + A0p0(iii-1,jjj+1,kkk+1) * p(+1, 0,-1)
5326  + Am0p(iii-1,jjj+1,kkk+1) * p( 0,-1, 0)
5327  + A00p(iii-1,jjj+1,kkk+1) * p(+1,-1, 0)
5328  + Ampp(iii-1,jjj+1,kkk+1) * p( 0, 0, 0)
5329  + A0pp(iii-1,jjj+1,kkk+1) * p(+1, 0, 0);
5330  ap(0,1,1) =
5331  Am00(iii,jjj+1,kkk+1) * p(+1,-1,-1)
5332  + Amp0(iii,jjj+1,kkk+1) * p(+1, 0,-1)
5333  + Am0p(iii,jjj+1,kkk+1) * p(+1,-1, 0)
5334  + Ampp(iii,jjj+1,kkk+1) * p(+1, 0, 0);
5335  cs2 = Real(0.125) *
5336  ( restrict_from_m00_to(iii,jjj,kkk) * ap(-1, 0, 0)
5337  + restrict_from_000_to(iii,jjj,kkk) * ap( 0, 0, 0)
5338  + restrict_from_mp0_to(iii,jjj,kkk) * ap(-1,+1, 0)
5339  + restrict_from_0p0_to(iii,jjj,kkk) * ap( 0,+1, 0)
5340  + restrict_from_m0p_to(iii,jjj,kkk) * ap(-1, 0,+1)
5341  + restrict_from_00p_to(iii,jjj,kkk) * ap( 0, 0,+1)
5342  + restrict_from_mpp_to(iii,jjj,kkk) * ap(-1,+1,+1)
5343  + restrict_from_0pp_to(iii,jjj,kkk) * ap( 0,+1,+1));
5344 
5345  // alternative: csten(i,j+1,k,ist_pmp)
5346  iii = ii;
5347  jjj = jj+2;
5348  kkk = kk;
5349  p(-1, 0,-1) = interp_from_p0p_to(iii+1,jjj-2,kkk+1);
5350  p( 0, 0,-1) = interp_from_00p_to(iii+2,jjj-2,kkk+1);
5351  p(-1,+1,-1) = interp_from_pmp_to(iii+1,jjj-1,kkk+1);
5352  p( 0,+1,-1) = interp_from_0mp_to(iii+2,jjj-1,kkk+1);
5353  p(-1, 0, 0) = interp_from_p00_to(iii+1,jjj-2,kkk+2);
5354  p( 0, 0, 0) = Real(1.);
5355  p(-1,+1, 0) = interp_from_pm0_to(iii+1,jjj-1,kkk+2);
5356  p( 0,+1, 0) = interp_from_0m0_to(iii+2,jjj-1,kkk+2);
5357  ap(0,-1,0) =
5358  Apmp(iii,jjj-1,kkk) * p(-1, 0,-1)
5359  + Ap0p(iii,jjj-1,kkk) * p(-1,+1,-1);
5360  ap(1,-1,0) =
5361  A0mp(iii+1,jjj-1,kkk) * p(-1, 0,-1)
5362  + Apmp(iii+1,jjj-1,kkk) * p( 0, 0,-1)
5363  + A00p(iii+1,jjj-1,kkk) * p(-1,+1,-1)
5364  + Ap0p(iii+1,jjj-1,kkk) * p( 0,+1,-1);
5365  ap(0,0,0) =
5366  Apmp(iii,jjj,kkk) * p(-1,+1,-1);
5367  ap(1,0,0) =
5368  A0mp(iii+1,jjj,kkk) * p(-1,+1,-1)
5369  + Apmp(iii+1,jjj,kkk) * p( 0,+1,-1);
5370  ap(0,-1,1) =
5371  Apm0(iii,jjj-1,kkk+1) * p(-1, 0,-1)
5372  + Ap00(iii,jjj-1,kkk+1) * p(-1,+1,-1)
5373  + Apmp(iii,jjj-1,kkk+1) * p(-1, 0, 0)
5374  + Ap0p(iii,jjj-1,kkk+1) * p(-1,+1, 0);
5375  ap(1,-1,1) =
5376  A0m0(iii+1,jjj-1,kkk+1) * p(-1, 0,-1)
5377  + Apm0(iii+1,jjj-1,kkk+1) * p( 0, 0,-1)
5378  + A000(iii+1,jjj-1,kkk+1) * p(-1,+1,-1)
5379  + Ap00(iii+1,jjj-1,kkk+1) * p( 0,+1,-1)
5380  + A0mp(iii+1,jjj-1,kkk+1) * p(-1, 0, 0)
5381  + Apmp(iii+1,jjj-1,kkk+1) * p( 0, 0, 0)
5382  + A00p(iii+1,jjj-1,kkk+1) * p(-1,+1, 0)
5383  + Ap0p(iii+1,jjj-1,kkk+1) * p( 0,+1, 0);
5384  ap(0,0,1) =
5385  Apm0(iii,jjj,kkk+1) * p(-1,+1,-1)
5386  + Apmp(iii,jjj,kkk+1) * p(-1,+1, 0);
5387  ap(1,0,1) =
5388  A0m0(iii+1,jjj,kkk+1) * p(-1,+1,-1)
5389  + Apm0(iii+1,jjj,kkk+1) * p( 0,+1,-1)
5390  + A0mp(iii+1,jjj,kkk+1) * p(-1,+1, 0)
5391  + Apmp(iii+1,jjj,kkk+1) * p( 0,+1, 0);
5392  cs3 = Real(0.125) *
5393  ( restrict_from_0m0_to(iii,jjj,kkk) * ap( 0,-1, 0)
5394  + restrict_from_pm0_to(iii,jjj,kkk) * ap(+1,-1, 0)
5395  + restrict_from_000_to(iii,jjj,kkk) * ap( 0, 0, 0)
5396  + restrict_from_p00_to(iii,jjj,kkk) * ap(+1, 0, 0)
5397  + restrict_from_0mp_to(iii,jjj,kkk) * ap( 0,-1,+1)
5398  + restrict_from_pmp_to(iii,jjj,kkk) * ap(+1,-1,+1)
5399  + restrict_from_00p_to(iii,jjj,kkk) * ap( 0, 0,+1)
5400  + restrict_from_p0p_to(iii,jjj,kkk) * ap(+1, 0,+1));
5401 
5402  // alternative: csten(i+1,j+1,k,ist_mmp)
5403  iii = ii+2;
5404  jjj = jj+2;
5405  kkk = kk;
5406  p( 0, 0,-1) = interp_from_00p_to(iii-2,jjj-2,kkk+1);
5407  p(+1, 0,-1) = interp_from_m0p_to(iii-1,jjj-2,kkk+1);
5408  p( 0,+1,-1) = interp_from_0mp_to(iii-2,jjj-1,kkk+1);
5409  p(+1,+1,-1) = interp_from_mmp_to(iii-1,jjj-1,kkk+1);
5410  p( 0, 0, 0) = Real(1.);
5411  p(+1, 0, 0) = interp_from_m00_to(iii-1,jjj-2,kkk+2);
5412  p( 0,+1, 0) = interp_from_0m0_to(iii-2,jjj-1,kkk+2);
5413  p(+1,+1, 0) = interp_from_mm0_to(iii-1,jjj-1,kkk+2);
5414  ap(-1,-1,0) =
5415  Ammp(iii-1,jjj-1,kkk) * p( 0, 0,-1)
5416  + A0mp(iii-1,jjj-1,kkk) * p(+1, 0,-1)
5417  + Am0p(iii-1,jjj-1,kkk) * p( 0,+1,-1)
5418  + A00p(iii-1,jjj-1,kkk) * p(+1,+1,-1);
5419  ap(0,-1,0) =
5420  Ammp(iii,jjj-1,kkk) * p(+1, 0,-1)
5421  + Am0p(iii,jjj-1,kkk) * p(+1,+1,-1);
5422  ap(-1,0,0) =
5423  Ammp(iii-1,jjj,kkk) * p( 0,+1,-1)
5424  + A0mp(iii-1,jjj,kkk) * p(+1,+1,-1);
5425  ap(0,0,0) =
5426  Ammp(iii,jjj,kkk) * p(+1,+1,-1);
5427  ap(-1,-1,1) =
5428  Amm0(iii-1,jjj-1,kkk+1) * p( 0, 0,-1)
5429  + A0m0(iii-1,jjj-1,kkk+1) * p(+1, 0,-1)
5430  + Am00(iii-1,jjj-1,kkk+1) * p( 0,+1,-1)
5431  + A000(iii-1,jjj-1,kkk+1) * p(+1,+1,-1)
5432  + Ammp(iii-1,jjj-1,kkk+1) * p( 0, 0, 0)
5433  + A0mp(iii-1,jjj-1,kkk+1) * p(+1, 0, 0)
5434  + Am0p(iii-1,jjj-1,kkk+1) * p( 0,+1, 0)
5435  + A00p(iii-1,jjj-1,kkk+1) * p(+1,+1, 0);
5436  ap(0,-1,1) =
5437  Amm0(iii,jjj-1,kkk+1) * p(+1, 0,-1)
5438  + Am00(iii,jjj-1,kkk+1) * p(+1,+1,-1)
5439  + Ammp(iii,jjj-1,kkk+1) * p(+1, 0, 0)
5440  + Am0p(iii,jjj-1,kkk+1) * p(+1,+1, 0);
5441  ap(-1,0,1) =
5442  Amm0(iii-1,jjj,kkk+1) * p( 0,+1,-1)
5443  + A0m0(iii-1,jjj,kkk+1) * p(+1,+1,-1)
5444  + Ammp(iii-1,jjj,kkk+1) * p( 0,+1, 0)
5445  + A0mp(iii-1,jjj,kkk+1) * p(+1,+1, 0);
5446  ap(0,0,1) =
5447  Amm0(iii,jjj,kkk+1) * p(+1,+1,-1)
5448  + Ammp(iii,jjj,kkk+1) * p(+1,+1, 0);
5449  cs4 = Real(0.125) *
5450  ( restrict_from_mm0_to(iii,jjj,kkk) * ap(-1,-1, 0)
5451  + restrict_from_0m0_to(iii,jjj,kkk) * ap( 0,-1, 0)
5452  + restrict_from_m00_to(iii,jjj,kkk) * ap(-1, 0, 0)
5453  + restrict_from_000_to(iii,jjj,kkk) * ap( 0, 0, 0)
5454  + restrict_from_mmp_to(iii,jjj,kkk) * ap(-1,-1,+1)
5455  + restrict_from_0mp_to(iii,jjj,kkk) * ap( 0,-1,+1)
5456  + restrict_from_m0p_to(iii,jjj,kkk) * ap(-1, 0,+1)
5457  + restrict_from_00p_to(iii,jjj,kkk) * ap( 0, 0,+1));
5458 
5459  csten(i,j,k,ist_ppp) = Real(0.25)*(cs1+cs2+cs3+cs4);
5460 }
5461 
5462 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
5463 Real mlndlap_adotx_sten_doit (int i, int j, int k, Array4<Real const> const& x,
5464  Array4<Real const> const& sten) noexcept
5465 {
5466  using namespace nodelap_detail;
5467 
5468  return x(i ,j ,k ) * sten(i ,j ,k ,ist_000)
5469  //
5470  + x(i-1,j ,k ) * sten(i-1,j ,k ,ist_p00)
5471  + x(i+1,j ,k ) * sten(i ,j ,k ,ist_p00)
5472  //
5473  + x(i ,j-1,k ) * sten(i ,j-1,k ,ist_0p0)
5474  + x(i ,j+1,k ) * sten(i ,j ,k ,ist_0p0)
5475  //
5476  + x(i ,j ,k-1) * sten(i ,j ,k-1,ist_00p)
5477  + x(i ,j ,k+1) * sten(i ,j ,k ,ist_00p)
5478  //
5479  + x(i-1,j-1,k ) * sten(i-1,j-1,k ,ist_pp0)
5480  + x(i+1,j-1,k ) * sten(i ,j-1,k ,ist_pp0)
5481  + x(i-1,j+1,k ) * sten(i-1,j ,k ,ist_pp0)
5482  + x(i+1,j+1,k ) * sten(i ,j ,k ,ist_pp0)
5483  //
5484  + x(i-1,j ,k-1) * sten(i-1,j ,k-1,ist_p0p)
5485  + x(i+1,j ,k-1) * sten(i ,j ,k-1,ist_p0p)
5486  + x(i-1,j ,k+1) * sten(i-1,j ,k ,ist_p0p)
5487  + x(i+1,j ,k+1) * sten(i ,j ,k ,ist_p0p)
5488  //
5489  + x(i ,j-1,k-1) * sten(i ,j-1,k-1,ist_0pp)
5490  + x(i ,j+1,k-1) * sten(i ,j ,k-1,ist_0pp)
5491  + x(i ,j-1,k+1) * sten(i ,j-1,k ,ist_0pp)
5492  + x(i ,j+1,k+1) * sten(i ,j ,k ,ist_0pp)
5493  //
5494  + x(i-1,j-1,k-1) * sten(i-1,j-1,k-1,ist_ppp)
5495  + x(i+1,j-1,k-1) * sten(i ,j-1,k-1,ist_ppp)
5496  + x(i-1,j+1,k-1) * sten(i-1,j ,k-1,ist_ppp)
5497  + x(i+1,j+1,k-1) * sten(i ,j ,k-1,ist_ppp)
5498  + x(i-1,j-1,k+1) * sten(i-1,j-1,k ,ist_ppp)
5499  + x(i+1,j-1,k+1) * sten(i ,j-1,k ,ist_ppp)
5500  + x(i-1,j+1,k+1) * sten(i-1,j ,k ,ist_ppp)
5501  + x(i+1,j+1,k+1) * sten(i ,j ,k ,ist_ppp);
5502 }
5503 
5504 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
5505 Real mlndlap_adotx_sten (int i, int j, int k, Array4<Real const> const& x,
5506  Array4<Real const> const& sten, Array4<int const> const& msk) noexcept
5507 {
5508  if (msk(i,j,k)) {
5509  return Real(0.0);
5510  } else {
5511  return mlndlap_adotx_sten_doit(i,j,k,x,sten);
5512  }
5513 }
5514 
5515 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
5516 void mlndlap_gauss_seidel_sten (int i, int j, int k, Array4<Real> const& sol,
5517  Array4<Real const> const& rhs,
5518  Array4<Real const> const& sten,
5519  Array4<int const> const& msk) noexcept
5520 {
5521  using namespace nodelap_detail;
5522 
5523  if (msk(i,j,k)) {
5524  sol(i,j,k) = Real(0.0);
5525  } else if (sten(i,j,k,ist_000) != Real(0.0)) {
5526  Real Ax = mlndlap_adotx_sten_doit(i,j,k,sol,sten);
5527  sol(i,j,k) += (rhs(i,j,k) - Ax) / sten(i,j,k,ist_000);
5528  }
5529 }
5530 
5531 inline
5532 void mlndlap_gauss_seidel_sten (Box const& bx, Array4<Real> const& sol,
5533  Array4<Real const> const& rhs,
5534  Array4<Real const> const& sten,
5535  Array4<int const> const& msk) noexcept
5536 {
5537  AMREX_LOOP_3D(bx, i, j, k,
5538  {
5539  mlndlap_gauss_seidel_sten(i,j,k,sol,rhs,sten,msk);
5540  });
5541 }
5542 
5543 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
5544 void mlndlap_interpadd_rap (int i, int j, int k, Array4<Real> const& fine,
5545  Array4<Real const> const& crse, Array4<Real const> const& sten,
5546  Array4<int const> const& msk) noexcept
5547 {
5548  using namespace nodelap_detail;
5549 
5550  if (!msk(i,j,k) && sten(i,j,k,ist_000) != Real(0.0)) {
5551  int ic = amrex::coarsen(i,2);
5552  int jc = amrex::coarsen(j,2);
5553  int kc = amrex::coarsen(k,2);
5554  bool ieven = ic*2 == i;
5555  bool jeven = jc*2 == j;
5556  bool keven = kc*2 == k;
5557  Real fv;
5558  if (ieven && jeven && keven) {
5559  fv = crse(ic,jc,kc);
5560  } else if (ieven && jeven) {
5561  Real w1 = std::abs(sten(i,j,k-1,ist_00p));
5562  Real w2 = std::abs(sten(i,j,k ,ist_00p));
5563  if (w1 == Real(0.0) && w2 == Real(0.0)) {
5564  fv = Real(0.5)*(crse(ic,jc,kc)+crse(ic,jc,kc+1));
5565  } else {
5566  fv = (w1*crse(ic,jc,kc) + w2*crse(ic,jc,kc+1)) / (w1+w2);
5567  }
5568  } else if (ieven && keven) {
5569  Real w1 = std::abs(sten(i,j-1,k,ist_0p0));
5570  Real w2 = std::abs(sten(i,j ,k,ist_0p0));
5571  if (w1 == Real(0.0) && w2 == Real(0.0)) {
5572  fv = Real(0.5)*(crse(ic,jc,kc)+crse(ic,jc+1,kc));
5573  } else {
5574  fv = (w1*crse(ic,jc,kc) + w2*crse(ic,jc+1,kc)) / (w1+w2);
5575  }
5576  } else if (jeven && keven) {
5577  Real w1 = std::abs(sten(i-1,j,k,ist_p00));
5578  Real w2 = std::abs(sten(i ,j,k,ist_p00));
5579  if (w1 == Real(0.0) && w2 == Real(0.0)) {
5580  fv = Real(0.5)*(crse(ic,jc,kc)+crse(ic+1,jc,kc));
5581  } else {
5582  fv = (w1*crse(ic,jc,kc) + w2*crse(ic+1,jc,kc)) / (w1+w2);
5583  }
5584  } else if (ieven) {
5585  Real w1m = std::abs(sten(i,j-1,k,ist_0p0)) / (std::abs(sten(i,j-1,k-1,ist_0pp))
5586  +std::abs(sten(i,j-1,k ,ist_0pp)) + eps);
5587  Real w1p = std::abs(sten(i,j ,k,ist_0p0)) / (std::abs(sten(i,j ,k-1,ist_0pp))
5588  +std::abs(sten(i,j ,k ,ist_0pp)) + eps);
5589  Real w2m = std::abs(sten(i,j,k-1,ist_00p)) / (std::abs(sten(i,j-1,k-1,ist_0pp))
5590  +std::abs(sten(i,j ,k-1,ist_0pp)) + eps);
5591  Real w2p = std::abs(sten(i,j,k ,ist_00p)) / (std::abs(sten(i,j-1,k ,ist_0pp))
5592  +std::abs(sten(i,j ,k ,ist_0pp)) + eps);
5593  Real wmm = std::abs(sten(i,j-1,k-1,ist_0pp)) * (Real(1.0) + w1m + w2m);
5594  Real wpm = std::abs(sten(i,j ,k-1,ist_0pp)) * (Real(1.0) + w1p + w2m);
5595  Real wmp = std::abs(sten(i,j-1,k ,ist_0pp)) * (Real(1.0) + w1m + w2p);
5596  Real wpp = std::abs(sten(i,j ,k ,ist_0pp)) * (Real(1.0) + w1p + w2p);
5597  fv = (wmm*crse(ic,jc,kc) + wpm*crse(ic,jc+1,kc)
5598  + wmp*crse(ic,jc,kc+1) + wpp*crse(ic,jc+1,kc+1))
5599  / (wmm+wpm+wmp+wpp+eps);
5600  } else if (jeven) {
5601  Real w1m = std::abs(sten(i-1,j,k,ist_p00)) / (std::abs(sten(i-1,j,k-1,ist_p0p))
5602  +std::abs(sten(i-1,j,k ,ist_p0p)) + eps);
5603  Real w1p = std::abs(sten(i ,j,k,ist_p00)) / (std::abs(sten(i ,j,k-1,ist_p0p))
5604  +std::abs(sten(i ,j,k ,ist_p0p)) + eps);
5605  Real w2m = std::abs(sten(i,j,k-1,ist_00p)) / (std::abs(sten(i-1,j,k-1,ist_p0p))
5606  +std::abs(sten(i ,j,k-1,ist_p0p)) + eps);
5607  Real w2p = std::abs(sten(i,j,k ,ist_00p)) / (std::abs(sten(i-1,j,k ,ist_p0p))
5608  +std::abs(sten(i ,j,k ,ist_p0p)) + eps);
5609  Real wmm = std::abs(sten(i-1,j,k-1,ist_p0p)) * (Real(1.0) + w1m + w2m);
5610  Real wpm = std::abs(sten(i ,j,k-1,ist_p0p)) * (Real(1.0) + w1p + w2m);
5611  Real wmp = std::abs(sten(i-1,j,k ,ist_p0p)) * (Real(1.0) + w1m + w2p);
5612  Real wpp = std::abs(sten(i ,j,k ,ist_p0p)) * (Real(1.0) + w1p + w2p);
5613  fv = (wmm*crse(ic,jc,kc) + wpm*crse(ic+1,jc,kc)
5614  + wmp*crse(ic,jc,kc+1) + wpp*crse(ic+1,jc,kc+1))
5615  / (wmm+wpm+wmp+wpp+eps);
5616  } else if (keven) {
5617  Real w1m = std::abs(sten(i-1,j,k,ist_p00)) / (std::abs(sten(i-1,j-1,k,ist_pp0))
5618  +std::abs(sten(i-1,j ,k,ist_pp0)) + eps);
5619  Real w1p = std::abs(sten(i ,j,k,ist_p00)) / (std::abs(sten(i ,j-1,k,ist_pp0))
5620  +std::abs(sten(i ,j ,k,ist_pp0)) + eps);
5621  Real w2m = std::abs(sten(i,j-1,k,ist_0p0)) / (std::abs(sten(i-1,j-1,k,ist_pp0))
5622  +std::abs(sten(i ,j-1,k,ist_pp0)) + eps);
5623  Real w2p = std::abs(sten(i,j ,k,ist_0p0)) / (std::abs(sten(i-1,j ,k,ist_pp0))
5624  +std::abs(sten(i ,j ,k,ist_pp0)) + eps);
5625  Real wmm = std::abs(sten(i-1,j-1,k,ist_pp0)) * (Real(1.0) + w1m + w2m);
5626  Real wpm = std::abs(sten(i ,j-1,k,ist_pp0)) * (Real(1.0) + w1p + w2m);
5627  Real wmp = std::abs(sten(i-1,j ,k,ist_pp0)) * (Real(1.0) + w1m + w2p);
5628  Real wpp = std::abs(sten(i ,j ,k,ist_pp0)) * (Real(1.0) + w1p + w2p);
5629  fv = (wmm*crse(ic,jc,kc) + wpm*crse(ic+1,jc,kc)
5630  + wmp*crse(ic,jc+1,kc) + wpp*crse(ic+1,jc+1,kc))
5631  / (wmm+wpm+wmp+wpp+eps);
5632  } else {
5633  Real wmmm = Real(1.0);
5634  Real wpmm = Real(1.0);
5635  Real wmpm = Real(1.0);
5636  Real wppm = Real(1.0);
5637  Real wmmp = Real(1.0);
5638  Real wpmp = Real(1.0);
5639  Real wmpp = Real(1.0);
5640  Real wppp = Real(1.0);
5641 
5642  Real wtmp = std::abs(sten(i-1,j,k,ist_p00)) /
5643  ( std::abs(sten(i-1,j-1,k-1,ist_ppp))
5644  + std::abs(sten(i-1,j ,k-1,ist_ppp))
5645  + std::abs(sten(i-1,j-1,k ,ist_ppp))
5646  + std::abs(sten(i-1,j ,k ,ist_ppp)) + eps);
5647  wmmm += wtmp;
5648  wmpm += wtmp;
5649  wmmp += wtmp;
5650  wmpp += wtmp;
5651 
5652  wtmp = std::abs(sten(i,j,k,ist_p00)) /
5653  ( std::abs(sten(i,j-1,k-1,ist_ppp))
5654  + std::abs(sten(i,j ,k-1,ist_ppp))
5655  + std::abs(sten(i,j-1,k ,ist_ppp))
5656  + std::abs(sten(i,j ,k ,ist_ppp)) + eps);
5657  wpmm += wtmp;
5658  wppm += wtmp;
5659  wpmp += wtmp;
5660  wppp += wtmp;
5661 
5662  wtmp = std::abs(sten(i,j-1,k,ist_0p0)) /
5663  ( std::abs(sten(i-1,j-1,k-1,ist_ppp))
5664  + std::abs(sten(i ,j-1,k-1,ist_ppp))
5665  + std::abs(sten(i-1,j-1,k ,ist_ppp))
5666  + std::abs(sten(i ,j-1,k ,ist_ppp)) + eps);
5667  wmmm += wtmp;
5668  wpmm += wtmp;
5669  wmmp += wtmp;
5670  wpmp += wtmp;
5671 
5672  wtmp = std::abs(sten(i,j,k,ist_0p0)) /
5673  ( std::abs(sten(i-1,j,k-1,ist_ppp))
5674  + std::abs(sten(i ,j,k-1,ist_ppp))
5675  + std::abs(sten(i-1,j,k ,ist_ppp))
5676  + std::abs(sten(i ,j,k ,ist_ppp)) + eps);
5677  wmpm += wtmp;
5678  wppm += wtmp;
5679  wmpp += wtmp;
5680  wppp += wtmp;
5681 
5682  wtmp = std::abs(sten(i,j,k-1,ist_00p)) /
5683  ( std::abs(sten(i-1,j-1,k-1,ist_ppp))
5684  + std::abs(sten(i ,j-1,k-1,ist_ppp))
5685  + std::abs(sten(i-1,j ,k-1,ist_ppp))
5686  + std::abs(sten(i ,j ,k-1,ist_ppp)) + eps);
5687  wmmm += wtmp;
5688  wpmm += wtmp;
5689  wmpm += wtmp;
5690  wppm += wtmp;
5691 
5692  wtmp = std::abs(sten(i,j,k,ist_00p)) /
5693  ( std::abs(sten(i-1,j-1,k,ist_ppp))
5694  + std::abs(sten(i ,j-1,k,ist_ppp))
5695  + std::abs(sten(i-1,j ,k,ist_ppp))
5696  + std::abs(sten(i ,j ,k,ist_ppp)) + eps);
5697  wmmp += wtmp;
5698  wpmp += wtmp;
5699  wmpp += wtmp;
5700  wppp += wtmp;
5701 
5702  wtmp = std::abs(sten(i-1,j-1,k,ist_pp0)) /
5703  ( std::abs(sten(i-1,j-1,k-1,ist_ppp))
5704  + std::abs(sten(i-1,j-1,k ,ist_ppp)) + eps);
5705  wmmm += wtmp;
5706  wmmp += wtmp;
5707 
5708  wtmp = std::abs(sten(i,j-1,k,ist_pp0)) /
5709  ( std::abs(sten(i,j-1,k-1,ist_ppp))
5710  + std::abs(sten(i,j-1,k ,ist_ppp)) + eps);
5711  wpmm += wtmp;
5712  wpmp += wtmp;
5713 
5714  wtmp = std::abs(sten(i-1,j,k,ist_pp0)) /
5715  ( std::abs(sten(i-1,j,k-1,ist_ppp))
5716  + std::abs(sten(i-1,j,k ,ist_ppp)) + eps);
5717  wmpm += wtmp;
5718  wmpp += wtmp;
5719 
5720  wtmp = std::abs(sten(i,j,k,ist_pp0)) /
5721  ( std::abs(sten(i,j,k-1,ist_ppp))
5722  + std::abs(sten(i,j,k ,ist_ppp)) + eps);
5723  wppm += wtmp;
5724  wppp += wtmp;
5725 
5726  wtmp = std::abs(sten(i-1,j,k-1,ist_p0p)) /
5727  ( std::abs(sten(i-1,j-1,k-1,ist_ppp))
5728  + std::abs(sten(i-1,j ,k-1,ist_ppp)) + eps);
5729  wmmm += wtmp;
5730  wmpm += wtmp;
5731 
5732  wtmp = std::abs(sten(i,j,k-1,ist_p0p)) /
5733  ( std::abs(sten(i,j-1,k-1,ist_ppp))
5734  + std::abs(sten(i,j ,k-1,ist_ppp)) + eps);
5735  wpmm += wtmp;
5736  wppm += wtmp;
5737 
5738  wtmp = std::abs(sten(i-1,j,k,ist_p0p)) /
5739  ( std::abs(sten(i-1,j-1,k,ist_ppp))
5740  + std::abs(sten(i-1,j ,k,ist_ppp)) + eps);
5741  wmmp += wtmp;
5742  wmpp += wtmp;
5743 
5744  wtmp = std::abs(sten(i,j,k,ist_p0p)) /
5745  ( std::abs(sten(i,j-1,k,ist_ppp))
5746  + std::abs(sten(i,j ,k,ist_ppp)) + eps);
5747  wpmp += wtmp;
5748  wppp += wtmp;
5749 
5750  wtmp = std::abs(sten(i,j-1,k-1,ist_0pp)) /
5751  ( std::abs(sten(i-1,j-1,k-1,ist_ppp))
5752  + std::abs(sten(i ,j-1,k-1,ist_ppp)) + eps);
5753  wmmm += wtmp;
5754  wpmm += wtmp;
5755 
5756  wtmp = std::abs(sten(i,j,k-1,ist_0pp)) /
5757  ( std::abs(sten(i-1,j,k-1,ist_ppp))
5758  + std::abs(sten(i ,j,k-1,ist_ppp)) + eps);
5759  wmpm += wtmp;
5760  wppm += wtmp;
5761 
5762  wtmp = std::abs(sten(i,j-1,k,ist_0pp)) /
5763  ( std::abs(sten(i-1,j-1,k,ist_ppp))
5764  + std::abs(sten(i ,j-1,k,ist_ppp)) + eps);
5765  wmmp += wtmp;
5766  wpmp += wtmp;
5767 
5768  wtmp = std::abs(sten(i,j,k,ist_0pp)) /
5769  ( std::abs(sten(i-1,j,k,ist_ppp))
5770  + std::abs(sten(i ,j,k,ist_ppp)) + eps);
5771  wmpp += wtmp;
5772  wppp += wtmp;
5773 
5774  wmmm *= std::abs(sten(i-1,j-1,k-1,ist_ppp));
5775  wpmm *= std::abs(sten(i ,j-1,k-1,ist_ppp));
5776  wmpm *= std::abs(sten(i-1,j ,k-1,ist_ppp));
5777  wppm *= std::abs(sten(i ,j ,k-1,ist_ppp));
5778  wmmp *= std::abs(sten(i-1,j-1,k ,ist_ppp));
5779  wpmp *= std::abs(sten(i ,j-1,k ,ist_ppp));
5780  wmpp *= std::abs(sten(i-1,j ,k ,ist_ppp));
5781  wppp *= std::abs(sten(i ,j ,k ,ist_ppp));
5782  fv = (wmmm*crse(ic,jc ,kc ) + wpmm*crse(ic+1,jc ,kc )
5783  + wmpm*crse(ic,jc+1,kc ) + wppm*crse(ic+1,jc+1,kc )
5784  + wmmp*crse(ic,jc ,kc+1) + wpmp*crse(ic+1,jc ,kc+1)
5785  + wmpp*crse(ic,jc+1,kc+1) + wppp*crse(ic+1,jc+1,kc+1))
5786  / (wmmm + wpmm + wmpm + wppm + wmmp + wpmp + wmpp + wppp + eps);
5787  }
5788 
5789  fine(i,j,k) += fv;
5790  }
5791 }
5792 
5793 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
5794 void mlndlap_restriction_rap (int i, int j, int k, Array4<Real> const& crse,
5795  Array4<Real const> const& fine, Array4<Real const> const& sten,
5796  Array4<int const> const& msk) noexcept
5797 {
5798  using namespace nodelap_detail;
5799 
5800  int ii = i*2;
5801  int jj = j*2;
5802  int kk = k*2;
5803  if (msk(ii,jj,kk)) {
5804  crse(i,j,k) = Real(0.0);
5805  } else {
5806 
5807  Real cv = fine(ii,jj,kk);
5808 
5809  // ************************************
5810  // Adding fine(ii-1,jj,kk)
5811  // ************************************
5812 
5813  Real sten_lo = std::abs(sten(ii-2,jj,kk,ist_p00));
5814  Real sten_hi = std::abs(sten(ii-1,jj,kk,ist_p00));
5815 
5816  if (sten_lo == Real(0.0) && sten_hi == Real(0.0)) {
5817  cv += Real(0.5)*fine(ii-1,jj,kk);
5818  } else {
5819  cv += fine(ii-1,jj,kk) * sten_hi / (sten_lo + sten_hi);
5820  }
5821 
5822  // ************************************
5823  // Adding fine(ii+1,jj,kk)
5824  // ************************************
5825 
5826  sten_lo = std::abs(sten(ii ,jj,kk,ist_p00));
5827  sten_hi = std::abs(sten(ii+1,jj,kk,ist_p00));
5828 
5829  if (sten_lo == Real(0.0) && sten_hi == Real(0.0)) {
5830  cv += Real(0.5)*fine(ii+1,jj,kk);
5831  } else {
5832  cv += fine(ii+1,jj,kk) * sten_lo / (sten_lo + sten_hi);
5833  }
5834 
5835  // ************************************
5836  // Adding fine(ii,jj-1,kk)
5837  // ************************************
5838 
5839  sten_lo = std::abs(sten(ii,jj-2,kk,ist_0p0));
5840  sten_hi = std::abs(sten(ii,jj-1,kk,ist_0p0));
5841 
5842  if (sten_lo == Real(0.0) && sten_hi == Real(0.0)) {
5843  cv += Real(0.5)*fine(ii,jj-1,kk);
5844  } else {
5845  cv += fine(ii,jj-1,kk) * sten_hi / (sten_lo + sten_hi);
5846  }
5847 
5848  // ************************************
5849  // Adding fine(ii,jj+1,kk)
5850  // ************************************
5851 
5852  sten_lo = std::abs(sten(ii,jj ,kk,ist_0p0));
5853  sten_hi = std::abs(sten(ii,jj+1,kk,ist_0p0));
5854 
5855  if (sten_lo == Real(0.0) && sten_hi == Real(0.0)) {
5856  cv += Real(0.5)*fine(ii,jj+1,kk);
5857  } else {
5858  cv += fine(ii,jj+1,kk) * sten_lo / (sten_lo + sten_hi);
5859  }
5860 
5861  // ************************************
5862  // Adding fine(ii,jj,kk-1)
5863  // ************************************
5864 
5865  sten_lo = std::abs(sten(ii,jj,kk-2,ist_00p));
5866  sten_hi = std::abs(sten(ii,jj,kk-1,ist_00p));
5867 
5868  if (sten_lo == Real(0.0) && sten_hi == Real(0.0)) {
5869  cv += Real(0.5)*fine(ii,jj,kk-1);
5870  } else {
5871  cv += fine(ii,jj,kk-1)*sten_hi / (sten_lo + sten_hi);
5872  }
5873 
5874  // ************************************
5875  // Adding fine(ii,jj,kk+1)
5876  // ************************************
5877 
5878  sten_lo = std::abs(sten(ii,jj,kk ,ist_00p));
5879  sten_hi = std::abs(sten(ii,jj,kk+1,ist_00p));
5880 
5881  if (sten_lo == Real(0.0) && sten_hi == Real(0.0)) {
5882  cv += Real(0.5)*fine(ii,jj,kk+1);
5883  } else {
5884  cv += fine(ii,jj,kk+1)*sten_lo / (sten_lo + sten_hi);
5885  }
5886 
5887  // ************************************
5888  // Adding fine(ii-1,jj-1,kk)
5889  // ************************************
5890 
5891  // keven
5892  Real w1m = std::abs(sten(ii-2,jj-1,kk,ist_p00))
5893  / ( std::abs(sten(ii-2,jj-2,kk,ist_pp0))
5894  +std::abs(sten(ii-2,jj-1,kk,ist_pp0)) + eps);
5895  Real w1p = std::abs(sten(ii-1,jj-1,kk,ist_p00))
5896  / ( std::abs(sten(ii-1,jj-2,kk,ist_pp0))
5897  +std::abs(sten(ii-1,jj-1,kk,ist_pp0)) + eps);
5898  Real w2m = std::abs(sten(ii-1,jj-2,kk,ist_0p0))
5899  / ( std::abs(sten(ii-2,jj-2,kk,ist_pp0))
5900  +std::abs(sten(ii-1,jj-2,kk,ist_pp0)) + eps);
5901  Real w2p = std::abs(sten(ii-1,jj-1,kk,ist_0p0))
5902  / ( std::abs(sten(ii-2,jj-1,kk,ist_pp0))
5903  +std::abs(sten(ii-1,jj-1,kk,ist_pp0)) + eps);
5904  Real wmm = std::abs(sten(ii-2,jj-2,kk,ist_pp0)) * (Real(1.0) + w1m + w2m);
5905  Real wpm = std::abs(sten(ii-1,jj-2,kk,ist_pp0)) * (Real(1.0) + w1p + w2m);
5906  Real wmp = std::abs(sten(ii-2,jj-1,kk,ist_pp0)) * (Real(1.0) + w1m + w2p);
5907  Real wpp = std::abs(sten(ii-1,jj-1,kk,ist_pp0)) * (Real(1.0) + w1p + w2p);
5908  cv += fine(ii-1,jj-1,kk)*wpp/(wmm+wpm+wmp+wpp+eps);
5909 
5910  // ************************************
5911  // Adding fine(ii+1,jj-1,kk)
5912  // ************************************
5913 
5914  w1m = std::abs(sten(ii ,jj-1,kk,ist_p00))
5915  / (std::abs(sten(ii ,jj-2,kk,ist_pp0))
5916  +std::abs(sten(ii ,jj-1,kk,ist_pp0)) + eps);
5917  w1p = std::abs(sten(ii+1,jj-1,kk,ist_p00))
5918  / (std::abs(sten(ii+1,jj-2,kk,ist_pp0))
5919  +std::abs(sten(ii+1,jj-1,kk,ist_pp0)) + eps);
5920  w2m = std::abs(sten(ii+1,jj-2,kk,ist_0p0))
5921  / (std::abs(sten(ii ,jj-2,kk,ist_pp0))
5922  +std::abs(sten(ii+1,jj-2,kk,ist_pp0)) + eps);
5923  w2p = std::abs(sten(ii+1,jj-1,kk,ist_0p0))
5924  / (std::abs(sten(ii ,jj-1,kk,ist_pp0))
5925  +std::abs(sten(ii+1,jj-1,kk,ist_pp0)) + eps);
5926  wmm = std::abs(sten(ii ,jj-2,kk,ist_pp0)) * (Real(1.0) + w1m + w2m);
5927  wpm = std::abs(sten(ii+1,jj-2,kk,ist_pp0)) * (Real(1.0) + w1p + w2m);
5928  wmp = std::abs(sten(ii ,jj-1,kk,ist_pp0)) * (Real(1.0) + w1m + w2p);
5929  wpp = std::abs(sten(ii+1,jj-1,kk,ist_pp0)) * (Real(1.0) + w1p + w2p);
5930  cv += fine(ii+1,jj-1,kk)*wmp/(wmm+wpm+wmp+wpp+eps);
5931 
5932  // ************************************
5933  // Adding fine(ii-1,jj+1,kk)
5934  // ************************************
5935 
5936  w1m = std::abs(sten(ii-2,jj+1,kk,ist_p00))
5937  / (std::abs(sten(ii-2,jj ,kk,ist_pp0))
5938  +std::abs(sten(ii-2,jj+1,kk,ist_pp0)) + eps);
5939  w1p = std::abs(sten(ii-1,jj+1,kk,ist_p00))
5940  / (std::abs(sten(ii-1,jj ,kk,ist_pp0))
5941  +std::abs(sten(ii-1,jj+1,kk,ist_pp0)) + eps);
5942  w2m = std::abs(sten(ii-1,jj ,kk,ist_0p0))
5943  / (std::abs(sten(ii-2,jj ,kk,ist_pp0))
5944  +std::abs(sten(ii-1,jj ,kk,ist_pp0)) + eps);
5945  w2p = std::abs(sten(ii-1,jj+1,kk,ist_0p0))
5946  / (std::abs(sten(ii-2,jj+1,kk,ist_pp0))
5947  +std::abs(sten(ii-1,jj+1,kk,ist_pp0)) + eps);
5948  wmm = std::abs(sten(ii-2,jj ,kk,ist_pp0)) * (Real(1.0) + w1m + w2m);
5949  wpm = std::abs(sten(ii-1,jj ,kk,ist_pp0)) * (Real(1.0) + w1p + w2m);
5950  wmp = std::abs(sten(ii-2,jj+1,kk,ist_pp0)) * (Real(1.0) + w1m + w2p);
5951  wpp = std::abs(sten(ii-1,jj+1,kk,ist_pp0)) * (Real(1.0) + w1p + w2p);
5952  cv += fine(ii-1,jj+1,kk)*wpm/(wmm+wpm+wmp+wpp+eps);
5953 
5954  // ************************************
5955  // Adding fine(ii+1,jj+1,kk)
5956  // ************************************
5957 
5958  w1m = std::abs(sten(ii ,jj+1,kk,ist_p00))
5959  / (std::abs(sten(ii ,jj+1,kk,ist_pp0))
5960  +std::abs(sten(ii ,jj ,kk,ist_pp0)) + eps);
5961  w1p = std::abs(sten(ii+1,jj+1,kk,ist_p00))
5962  / (std::abs(sten(ii+1,jj+1,kk,ist_pp0))
5963  +std::abs(sten(ii+1,jj ,kk,ist_pp0)) + eps);
5964  w2m = std::abs(sten(ii+1,jj ,kk,ist_0p0))
5965  / (std::abs(sten(ii+1,jj ,kk,ist_pp0))
5966  +std::abs(sten(ii ,jj ,kk,ist_pp0)) + eps);
5967  w2p = std::abs(sten(ii+1,jj+1,kk,ist_0p0))
5968  / (std::abs(sten(ii+1,jj+1,kk,ist_pp0))
5969  +std::abs(sten(ii ,jj+1,kk,ist_pp0)) + eps);
5970  wmm = std::abs(sten(ii ,jj ,kk,ist_pp0)) * (Real(1.0) + w1m + w2m);
5971  wpm = std::abs(sten(ii+1,jj ,kk,ist_pp0)) * (Real(1.0) + w1p + w2m);
5972  wmp = std::abs(sten(ii ,jj+1,kk,ist_pp0)) * (Real(1.0) + w1m + w2p);
5973  wpp = std::abs(sten(ii+1,jj+1,kk,ist_pp0)) * (Real(1.0) + w1p + w2p);
5974  cv += fine(ii+1,jj+1,kk)*wmm/(wmm+wpm+wmp+wpp+eps);
5975 
5976  // ************************************
5977  // Adding fine(ii-1,jj,kk-1)
5978  // ************************************
5979 
5980  // jeven
5981  w1m = std::abs(sten(ii-2,jj,kk-1,ist_p00))
5982  / (std::abs(sten(ii-2,jj,kk-2,ist_p0p))
5983  +std::abs(sten(ii-2,jj,kk-1,ist_p0p)) + eps);
5984  w1p = std::abs(sten(ii-1,jj,kk-1,ist_p00))
5985  / (std::abs(sten(ii-1,jj,kk-2,ist_p0p))
5986  +std::abs(sten(ii-1,jj,kk-1,ist_p0p)) + eps);
5987  w2m = std::abs(sten(ii-1,jj,kk-2,ist_00p))
5988  / (std::abs(sten(ii-2,jj,kk-2,ist_p0p))
5989  +std::abs(sten(ii-1,jj,kk-2,ist_p0p)) + eps);
5990  w2p = std::abs(sten(ii-1,jj,kk-1,ist_00p))
5991  / (std::abs(sten(ii-2,jj,kk-1,ist_p0p))
5992  +std::abs(sten(ii-1,jj,kk-1,ist_p0p)) + eps);
5993  wmm = std::abs(sten(ii-2,jj,kk-2,ist_p0p)) * (Real(1.0) + w1m + w2m);
5994  wpm = std::abs(sten(ii-1,jj,kk-2,ist_p0p)) * (Real(1.0) + w1p + w2m);
5995  wmp = std::abs(sten(ii-2,jj,kk-1,ist_p0p)) * (Real(1.0) + w1m + w2p);
5996  wpp = std::abs(sten(ii-1,jj,kk-1,ist_p0p)) * (Real(1.0) + w1p + w2p);
5997  cv += fine(ii-1,jj,kk-1)*wpp/(wmm+wpm+wmp+wpp+eps);
5998 
5999  // ************************************
6000  // Adding fine(ii+1,jj,kk-1)
6001  // ************************************
6002 
6003  w1m = std::abs(sten(ii ,jj,kk-1,ist_p00))
6004  / (std::abs(sten(ii ,jj,kk-2,ist_p0p))
6005  +std::abs(sten(ii ,jj,kk-1,ist_p0p)) + eps);
6006  w1p = std::abs(sten(ii+1,jj,kk-1,ist_p00))
6007  / (std::abs(sten(ii+1,jj,kk-2,ist_p0p))
6008  +std::abs(sten(ii+1,jj,kk-1,ist_p0p)) + eps);
6009  w2m = std::abs(sten(ii+1,jj,kk-2,ist_00p))
6010  / (std::abs(sten(ii+1,jj,kk-2,ist_p0p))
6011  +std::abs(sten(ii ,jj,kk-2,ist_p0p)) + eps);
6012  w2p = std::abs(sten(ii+1,jj,kk-1,ist_00p))
6013  / (std::abs(sten(ii+1,jj,kk-1,ist_p0p))
6014  +std::abs(sten(ii ,jj,kk-1,ist_p0p)) + eps);
6015  wmm = std::abs(sten(ii ,jj,kk-2,ist_p0p)) * (Real(1.0) + w1m + w2m);
6016  wpm = std::abs(sten(ii+1,jj,kk-2,ist_p0p)) * (Real(1.0) + w1p + w2m);
6017  wmp = std::abs(sten(ii ,jj,kk-1,ist_p0p)) * (Real(1.0) + w1m + w2p);
6018  wpp = std::abs(sten(ii+1,jj,kk-1,ist_p0p)) * (Real(1.0) + w1p + w2p);
6019  cv += fine(ii+1,jj,kk-1)*wmp/(wmm+wpm+wmp+wpp+eps);
6020 
6021  // ************************************
6022  // Adding fine(ii-1,jj,kk+1)
6023  // ************************************
6024 
6025  w1m = std::abs(sten(ii-2,jj,kk+1,ist_p00))
6026  / (std::abs(sten(ii-2,jj,kk+1,ist_p0p))
6027  +std::abs(sten(ii-2,jj,kk ,ist_p0p)) + eps);
6028  w1p = std::abs(sten(ii-1,jj,kk+1,ist_p00))
6029  / (std::abs(sten(ii-1,jj,kk+1,ist_p0p))
6030  +std::abs(sten(ii-1,jj,kk ,ist_p0p)) + eps);
6031  w2m = std::abs(sten(ii-1,jj,kk ,ist_00p))
6032  / (std::abs(sten(ii-2,jj,kk ,ist_p0p))
6033  +std::abs(sten(ii-1,jj,kk ,ist_p0p)) + eps);
6034  w2p = std::abs(sten(ii-1,jj,kk+1,ist_00p))
6035  / (std::abs(sten(ii-2,jj,kk+1,ist_p0p))
6036  +std::abs(sten(ii-1,jj,kk+1,ist_p0p)) + eps);
6037  wmm = std::abs(sten(ii-2,jj,kk ,ist_p0p)) * (Real(1.0) + w1m + w2m);
6038  wpm = std::abs(sten(ii-1,jj,kk ,ist_p0p)) * (Real(1.0) + w1p + w2m);
6039  wmp = std::abs(sten(ii-2,jj,kk+1,ist_p0p)) * (Real(1.0) + w1m + w2p);
6040  wpp = std::abs(sten(ii-1,jj,kk+1,ist_p0p)) * (Real(1.0) + w1p + w2p);
6041  cv += fine(ii-1,jj,kk+1)*wpm/(wmm+wpm+wmp+wpp+eps);
6042 
6043  // ************************************
6044  // Adding fine(ii+1,jj,kk+1)
6045  // ************************************
6046 
6047  w1m = std::abs(sten(ii ,jj,kk+1,ist_p00))
6048  / (std::abs(sten(ii ,jj,kk+1,ist_p0p))
6049  +std::abs(sten(ii ,jj,kk ,ist_p0p)) + eps);
6050  w1p = std::abs(sten(ii+1,jj,kk+1,ist_p00))
6051  / (std::abs(sten(ii+1,jj,kk+1,ist_p0p))
6052  +std::abs(sten(ii+1,jj,kk ,ist_p0p)) + eps);
6053  w2m = std::abs(sten(ii+1,jj,kk ,ist_00p))
6054  / (std::abs(sten(ii+1,jj,kk ,ist_p0p))
6055  +std::abs(sten(ii ,jj,kk ,ist_p0p)) + eps);
6056  w2p = std::abs(sten(ii+1,jj,kk+1,ist_00p))
6057  / (std::abs(sten(ii+1,jj,kk+1,ist_p0p))
6058  +std::abs(sten(ii ,jj,kk+1,ist_p0p)) + eps);
6059  wmm = std::abs(sten(ii ,jj,kk ,ist_p0p)) * (Real(1.0) + w1m + w2m);
6060  wpm = std::abs(sten(ii+1,jj,kk ,ist_p0p)) * (Real(1.0) + w1p + w2m);
6061  wmp = std::abs(sten(ii ,jj,kk+1,ist_p0p)) * (Real(1.0) + w1m + w2p);
6062  wpp = std::abs(sten(ii+1,jj,kk+1,ist_p0p)) * (Real(1.0) + w1p + w2p);
6063  cv += fine(ii+1,jj,kk+1)*wmm/(wmm+wpm+wmp+wpp+eps);
6064 
6065  // ************************************
6066  // Adding fine(ii,jj-1,kk-1)
6067  // ************************************
6068 
6069  // ieven
6070  w1m = std::abs(sten(ii,jj-2,kk-1,ist_0p0))
6071  / (std::abs(sten(ii,jj-2,kk-2,ist_0pp))
6072  +std::abs(sten(ii,jj-2,kk-1,ist_0pp)) + eps);
6073  w2m = std::abs(sten(ii,jj-1,kk-2,ist_00p))
6074  / (std::abs(sten(ii,jj-2,kk-2,ist_0pp))
6075  +std::abs(sten(ii,jj-1,kk-2,ist_0pp)) + eps);
6076  w1p = std::abs(sten(ii,jj-1,kk-1,ist_0p0))
6077  / (std::abs(sten(ii,jj-1,kk-2,ist_0pp))
6078  +std::abs(sten(ii,jj-1,kk-1,ist_0pp)) + eps);
6079  w2p = std::abs(sten(ii,jj-1,kk-1,ist_00p))
6080  / (std::abs(sten(ii,jj-2,kk-1,ist_0pp))
6081  +std::abs(sten(ii,jj-1,kk-1,ist_0pp)) + eps);
6082  wmm = std::abs(sten(ii,jj-2,kk-2,ist_0pp)) * (Real(1.0) + w1m + w2m);
6083  wpm = std::abs(sten(ii,jj-1,kk-2,ist_0pp)) * (Real(1.0) + w1p + w2m);
6084  wmp = std::abs(sten(ii,jj-2,kk-1,ist_0pp)) * (Real(1.0) + w1m + w2p);
6085  wpp = std::abs(sten(ii,jj-1,kk-1,ist_0pp)) * (Real(1.0) + w1p + w2p);
6086  cv += fine(ii,jj-1,kk-1)*wpp/(wmm+wpm+wmp+wpp+eps);
6087 
6088  // ************************************
6089  // Adding fine(ii,jj+1,kk-1)
6090  // ************************************
6091 
6092  w1m = std::abs(sten(ii,jj ,kk-1,ist_0p0))
6093  / (std::abs(sten(ii,jj ,kk-2,ist_0pp))
6094  +std::abs(sten(ii,jj ,kk-1,ist_0pp)) + eps);
6095  w1p = std::abs(sten(ii,jj+1,kk-1,ist_0p0))
6096  / (std::abs(sten(ii,jj+1,kk-2,ist_0pp))
6097  +std::abs(sten(ii,jj+1,kk-1,ist_0pp)) + eps);
6098  w2m = std::abs(sten(ii,jj+1,kk-2,ist_00p))
6099  / (std::abs(sten(ii,jj+1,kk-2,ist_0pp))
6100  +std::abs(sten(ii,jj ,kk-2,ist_0pp)) + eps);
6101  w2p = std::abs(sten(ii,jj+1,kk-1,ist_00p))
6102  / (std::abs(sten(ii,jj+1,kk-1,ist_0pp))
6103  +std::abs(sten(ii,jj ,kk-1,ist_0pp)) + eps);
6104  wmm = std::abs(sten(ii,jj ,kk-2,ist_0pp)) * (Real(1.0) + w1m + w2m);
6105  wpm = std::abs(sten(ii,jj+1,kk-2,ist_0pp)) * (Real(1.0) + w1p + w2m);
6106  wmp = std::abs(sten(ii,jj ,kk-1,ist_0pp)) * (Real(1.0) + w1m + w2p);
6107  wpp = std::abs(sten(ii,jj+1,kk-1,ist_0pp)) * (Real(1.0) + w1p + w2p);
6108  cv += fine(ii,jj+1,kk-1)*wmp/(wmm+wpm+wmp+wpp+eps);
6109 
6110  // ************************************
6111  // Adding fine(ii,jj-1,kk+1)
6112  // ************************************
6113 
6114  w1m = std::abs(sten(ii,jj-2,kk+1,ist_0p0))
6115  / (std::abs(sten(ii,jj-2,kk+1,ist_0pp))
6116  +std::abs(sten(ii,jj-2,kk ,ist_0pp)) + eps);
6117  w1p = std::abs(sten(ii,jj-1,kk+1,ist_0p0))
6118  / (std::abs(sten(ii,jj-1,kk+1,ist_0pp))
6119  +std::abs(sten(ii,jj-1,kk ,ist_0pp)) + eps);
6120  w2m = std::abs(sten(ii,jj-1,kk ,ist_00p))
6121  / (std::abs(sten(ii,jj-2,kk ,ist_0pp))
6122  +std::abs(sten(ii,jj-1,kk ,ist_0pp)) + eps);
6123  w2p = std::abs(sten(ii,jj-1,kk+1,ist_00p))
6124  / (std::abs(sten(ii,jj-2,kk+1,ist_0pp))
6125  +std::abs(sten(ii,jj-1,kk+1,ist_0pp)) + eps);
6126  wmm = std::abs(sten(ii,jj-2,kk ,ist_0pp)) * (Real(1.0) + w1m + w2m);
6127  wpm = std::abs(sten(ii,jj-1,kk ,ist_0pp)) * (Real(1.0) + w1p + w2m);
6128  wmp = std::abs(sten(ii,jj-2,kk+1,ist_0pp)) * (Real(1.0) + w1m + w2p);
6129  wpp = std::abs(sten(ii,jj-1,kk+1,ist_0pp)) * (Real(1.0) + w1p + w2p);
6130  cv += fine(ii,jj-1,kk+1)*wpm/(wmm+wpm+wmp+wpp+eps);
6131 
6132  // ************************************
6133  // Adding fine(ii,jj+1,kk+1)
6134  // ************************************
6135 
6136  w1m = std::abs(sten(ii,jj ,kk+1,ist_0p0))
6137  / (std::abs(sten(ii,jj ,kk+1,ist_0pp))
6138  +std::abs(sten(ii,jj ,kk ,ist_0pp)) + eps);
6139  w1p = std::abs(sten(ii,jj+1,kk+1,ist_0p0))
6140  / (std::abs(sten(ii,jj+1,kk+1,ist_0pp))
6141  +std::abs(sten(ii,jj+1,kk ,ist_0pp)) + eps);
6142  w2m = std::abs(sten(ii,jj+1,kk ,ist_00p))
6143  / (std::abs(sten(ii,jj+1,kk ,ist_0pp))
6144  +std::abs(sten(ii,jj ,kk ,ist_0pp)) + eps);
6145  w2p = std::abs(sten(ii,jj+1,kk+1,ist_00p))
6146  / (std::abs(sten(ii,jj+1,kk+1,ist_0pp))
6147  +std::abs(sten(ii,jj ,kk+1,ist_0pp)) + eps);
6148  wmm = std::abs(sten(ii,jj ,kk ,ist_0pp)) * (Real(1.0) + w1m + w2m);
6149  wpm = std::abs(sten(ii,jj+1,kk ,ist_0pp)) * (Real(1.0) + w1p + w2m);
6150  wmp = std::abs(sten(ii,jj ,kk+1,ist_0pp)) * (Real(1.0) + w1m + w2p);
6151  wpp = std::abs(sten(ii,jj+1,kk+1,ist_0pp)) * (Real(1.0) + w1p + w2p);
6152  cv += fine(ii,jj+1,kk+1)*wmm/(wmm+wpm+wmp+wpp+eps);
6153 
6154  // ************************************
6155  // Adding fine at corners
6156  // ************************************
6157 
6158  Real wmmm = Real(1.0)
6159  + std::abs(sten(ii ,jj+1,kk+1,ist_p00)) /
6160  ( std::abs(sten(ii ,jj ,kk ,ist_ppp))
6161  + std::abs(sten(ii ,jj+1,kk ,ist_ppp))
6162  + std::abs(sten(ii ,jj ,kk+1,ist_ppp))
6163  + std::abs(sten(ii ,jj+1,kk+1,ist_ppp)) + eps)
6164  + std::abs(sten(ii+1,jj ,kk+1,ist_0p0)) /
6165  ( std::abs(sten(ii ,jj ,kk ,ist_ppp))
6166  + std::abs(sten(ii+1,jj ,kk ,ist_ppp))
6167  + std::abs(sten(ii ,jj ,kk+1,ist_ppp))
6168  + std::abs(sten(ii+1,jj ,kk+1,ist_ppp)) + eps)
6169  + std::abs(sten(ii+1,jj+1,kk ,ist_00p)) /
6170  ( std::abs(sten(ii ,jj ,kk ,ist_ppp))
6171  + std::abs(sten(ii+1,jj ,kk ,ist_ppp))
6172  + std::abs(sten(ii ,jj+1,kk ,ist_ppp))
6173  + std::abs(sten(ii+1,jj+1,kk ,ist_ppp)) + eps)
6174  + std::abs(sten(ii ,jj ,kk+1,ist_pp0)) /
6175  ( std::abs(sten(ii ,jj ,kk ,ist_ppp))
6176  + std::abs(sten(ii ,jj ,kk+1,ist_ppp)) + eps)
6177  + std::abs(sten(ii ,jj+1,kk ,ist_p0p)) /
6178  ( std::abs(sten(ii ,jj ,kk ,ist_ppp))
6179  + std::abs(sten(ii ,jj+1,kk ,ist_ppp)) + eps)
6180  + std::abs(sten(ii+1,jj ,kk ,ist_0pp)) /
6181  ( std::abs(sten(ii ,jj ,kk ,ist_ppp))
6182  + std::abs(sten(ii+1,jj ,kk ,ist_ppp)) + eps);
6183  wmmm *= std::abs(sten(ii,jj,kk,ist_ppp));
6184  cv += wmmm*fine(ii+1,jj+1,kk+1)*sten(ii+1,jj+1,kk+1,ist_inv);
6185 
6186  Real wpmm = Real(1.0)
6187  + std::abs(sten(ii-1,jj+1,kk+1,ist_p00)) /
6188  ( std::abs(sten(ii-1,jj ,kk ,ist_ppp))
6189  + std::abs(sten(ii-1,jj+1,kk ,ist_ppp))
6190  + std::abs(sten(ii-1,jj ,kk+1,ist_ppp))
6191  + std::abs(sten(ii-1,jj+1,kk+1,ist_ppp)) + eps)
6192  + std::abs(sten(ii-1,jj ,kk+1,ist_0p0)) /
6193  ( std::abs(sten(ii-2,jj ,kk ,ist_ppp))
6194  + std::abs(sten(ii-1,jj ,kk ,ist_ppp))
6195  + std::abs(sten(ii-2,jj ,kk+1,ist_ppp))
6196  + std::abs(sten(ii-1,jj ,kk+1,ist_ppp)) + eps)
6197  + std::abs(sten(ii-1,jj+1,kk ,ist_00p)) /
6198  ( std::abs(sten(ii-2,jj ,kk ,ist_ppp))
6199  + std::abs(sten(ii-1,jj ,kk ,ist_ppp))
6200  + std::abs(sten(ii-2,jj+1,kk ,ist_ppp))
6201  + std::abs(sten(ii-1,jj+1,kk ,ist_ppp)) + eps)
6202  + std::abs(sten(ii-1,jj ,kk+1,ist_pp0)) /
6203  ( std::abs(sten(ii-1,jj ,kk ,ist_ppp))
6204  + std::abs(sten(ii-1,jj ,kk+1,ist_ppp)) + eps)
6205  + std::abs(sten(ii-1,jj+1,kk ,ist_p0p)) /
6206  ( std::abs(sten(ii-1,jj ,kk ,ist_ppp))
6207  + std::abs(sten(ii-1,jj+1,kk ,ist_ppp)) + eps)
6208  + std::abs(sten(ii-1,jj ,kk ,ist_0pp)) /
6209  ( std::abs(sten(ii-2,jj ,kk ,ist_ppp))
6210  + std::abs(sten(ii-1,jj ,kk ,ist_ppp)) + eps);
6211  wpmm *= std::abs(sten(ii-1,jj,kk,ist_ppp));
6212  cv += wpmm*fine(ii-1,jj+1,kk+1)*sten(ii-1,jj+1,kk+1,ist_inv);
6213 
6214  Real wmpm = Real(1.0)
6215  + std::abs(sten(ii ,jj-1,kk+1,ist_p00)) /
6216  ( std::abs(sten(ii ,jj-2,kk ,ist_ppp))
6217  + std::abs(sten(ii ,jj-1,kk ,ist_ppp))
6218  + std::abs(sten(ii ,jj-2,kk+1,ist_ppp))
6219  + std::abs(sten(ii ,jj-1,kk+1,ist_ppp)) + eps)
6220  + std::abs(sten(ii+1,jj-1,kk+1,ist_0p0)) /
6221  ( std::abs(sten(ii ,jj-1,kk ,ist_ppp))
6222  + std::abs(sten(ii+1,jj-1,kk ,ist_ppp))
6223  + std::abs(sten(ii ,jj-1,kk+1,ist_ppp))
6224  + std::abs(sten(ii+1,jj-1,kk+1,ist_ppp)) + eps)
6225  + std::abs(sten(ii+1,jj-1,kk ,ist_00p)) /
6226  ( std::abs(sten(ii ,jj-2,kk ,ist_ppp))
6227  + std::abs(sten(ii+1,jj-2,kk ,ist_ppp))
6228  + std::abs(sten(ii ,jj-1,kk ,ist_ppp))
6229  + std::abs(sten(ii+1,jj-1,kk ,ist_ppp)) + eps)
6230  + std::abs(sten(ii ,jj-1,kk+1,ist_pp0)) /
6231  ( std::abs(sten(ii ,jj-1,kk ,ist_ppp))
6232  + std::abs(sten(ii ,jj-1,kk+1,ist_ppp)) + eps)
6233  + std::abs(sten(ii ,jj-1,kk ,ist_p0p)) /
6234  ( std::abs(sten(ii ,jj-2,kk ,ist_ppp))
6235  + std::abs(sten(ii ,jj-1,kk ,ist_ppp)) + eps)
6236  + std::abs(sten(ii+1,jj-1,kk ,ist_0pp)) /
6237  ( std::abs(sten(ii ,jj-1,kk ,ist_ppp))
6238  + std::abs(sten(ii+1,jj-1,kk ,ist_ppp)) + eps);
6239  wmpm *= std::abs(sten(ii ,jj-1,kk ,ist_ppp));
6240  cv += wmpm*fine(ii+1,jj-1,kk+1)*sten(ii+1,jj-1,kk+1,ist_inv);
6241 
6242  Real wppm = Real(1.0)
6243  + std::abs(sten(ii-1,jj-1,kk+1,ist_p00)) /
6244  ( std::abs(sten(ii-1,jj-2,kk ,ist_ppp))
6245  + std::abs(sten(ii-1,jj-1,kk ,ist_ppp))
6246  + std::abs(sten(ii-1,jj-2,kk+1,ist_ppp))
6247  + std::abs(sten(ii-1,jj-1,kk+1,ist_ppp)) + eps)
6248  + std::abs(sten(ii-1,jj-1,kk+1,ist_0p0)) /
6249  ( std::abs(sten(ii-2,jj-1,kk ,ist_ppp))
6250  + std::abs(sten(ii-1,jj-1,kk ,ist_ppp))
6251  + std::abs(sten(ii-2,jj-1,kk+1,ist_ppp))
6252  + std::abs(sten(ii-1,jj-1,kk+1,ist_ppp)) + eps)
6253  + std::abs(sten(ii-1,jj-1,kk ,ist_00p)) /
6254  ( std::abs(sten(ii-2,jj-2,kk ,ist_ppp))
6255  + std::abs(sten(ii-1,jj-2,kk ,ist_ppp))
6256  + std::abs(sten(ii-2,jj-1,kk ,ist_ppp))
6257  + std::abs(sten(ii-1,jj-1,kk ,ist_ppp)) + eps)
6258  + std::abs(sten(ii-1,jj-1,kk+1,ist_pp0)) /
6259  ( std::abs(sten(ii-1,jj-1,kk ,ist_ppp))
6260  + std::abs(sten(ii-1,jj-1,kk+1,ist_ppp)) + eps)
6261  + std::abs(sten(ii-1,jj-1,kk ,ist_p0p)) /
6262  ( std::abs(sten(ii-1,jj-2,kk ,ist_ppp))
6263  + std::abs(sten(ii-1,jj-1,kk ,ist_ppp)) + eps)
6264  + std::abs(sten(ii-1,jj-1,kk ,ist_0pp)) /
6265  ( std::abs(sten(ii-2,jj-1,kk ,ist_ppp))
6266  + std::abs(sten(ii-1,jj-1,kk ,ist_ppp)) + eps);
6267  wppm *= std::abs(sten(ii-1,jj-1,kk ,ist_ppp));
6268  cv += wppm*fine(ii-1,jj-1,kk+1)*sten(ii-1,jj-1,kk+1,ist_inv);
6269 
6270  Real wmmp = Real(1.0)
6271  + std::abs(sten(ii ,jj+1,kk-1,ist_p00)) /
6272  ( std::abs(sten(ii ,jj ,kk-2,ist_ppp))
6273  + std::abs(sten(ii ,jj+1,kk-2,ist_ppp))
6274  + std::abs(sten(ii ,jj ,kk-1,ist_ppp))
6275  + std::abs(sten(ii ,jj+1,kk-1,ist_ppp)) + eps)
6276  + std::abs(sten(ii+1,jj ,kk-1,ist_0p0)) /
6277  ( std::abs(sten(ii ,jj ,kk-2,ist_ppp))
6278  + std::abs(sten(ii+1,jj ,kk-2,ist_ppp))
6279  + std::abs(sten(ii ,jj ,kk-1,ist_ppp))
6280  + std::abs(sten(ii+1,jj ,kk-1,ist_ppp)) + eps)
6281  + std::abs(sten(ii+1,jj+1,kk-1,ist_00p)) /
6282  ( std::abs(sten(ii ,jj ,kk-1,ist_ppp))
6283  + std::abs(sten(ii+1,jj ,kk-1,ist_ppp))
6284  + std::abs(sten(ii ,jj+1,kk-1,ist_ppp))
6285  + std::abs(sten(ii+1,jj+1,kk-1,ist_ppp)) + eps)
6286  + std::abs(sten(ii ,jj ,kk-1,ist_pp0)) /
6287  ( std::abs(sten(ii ,jj ,kk-2,ist_ppp))
6288  + std::abs(sten(ii ,jj ,kk-1,ist_ppp)) + eps)
6289  + std::abs(sten(ii ,jj+1,kk-1,ist_p0p)) /
6290  ( std::abs(sten(ii ,jj ,kk-1,ist_ppp))
6291  + std::abs(sten(ii ,jj+1,kk-1,ist_ppp)) + eps)
6292  + std::abs(sten(ii+1,jj ,kk-1,ist_0pp)) /
6293  ( std::abs(sten(ii ,jj ,kk-1,ist_ppp))
6294  + std::abs(sten(ii+1,jj ,kk-1,ist_ppp)) + eps);
6295  wmmp *= std::abs(sten(ii ,jj ,kk-1,ist_ppp));
6296  cv += wmmp*fine(ii+1,jj+1,kk-1)*sten(ii+1,jj+1,kk-1,ist_inv);
6297 
6298  Real wpmp = Real(1.0)
6299  + std::abs(sten(ii-1,jj+1,kk-1,ist_p00)) /
6300  ( std::abs(sten(ii-1,jj ,kk-2,ist_ppp))
6301  + std::abs(sten(ii-1,jj+1,kk-2,ist_ppp))
6302  + std::abs(sten(ii-1,jj ,kk-1,ist_ppp))
6303  + std::abs(sten(ii-1,jj+1,kk-1,ist_ppp)) + eps)
6304  + std::abs(sten(ii-1,jj ,kk-1,ist_0p0)) /
6305  ( std::abs(sten(ii-2,jj ,kk-2,ist_ppp))
6306  + std::abs(sten(ii-1,jj ,kk-2,ist_ppp))
6307  + std::abs(sten(ii-2,jj ,kk-1,ist_ppp))
6308  + std::abs(sten(ii-1,jj ,kk-1,ist_ppp)) + eps)
6309  + std::abs(sten(ii-1,jj+1,kk-1,ist_00p)) /
6310  ( std::abs(sten(ii-2,jj ,kk-1,ist_ppp))
6311  + std::abs(sten(ii-1,jj ,kk-1,ist_ppp))
6312  + std::abs(sten(ii-2,jj+1,kk-1,ist_ppp))
6313  + std::abs(sten(ii-1,jj+1,kk-1,ist_ppp)) + eps)
6314  + std::abs(sten(ii-1,jj ,kk-1,ist_pp0)) /
6315  ( std::abs(sten(ii-1,jj ,kk-2,ist_ppp))
6316  + std::abs(sten(ii-1,jj ,kk-1,ist_ppp)) + eps)
6317  + std::abs(sten(ii-1,jj+1,kk-1,ist_p0p)) /
6318  ( std::abs(sten(ii-1,jj ,kk-1,ist_ppp))
6319  + std::abs(sten(ii-1,jj+1,kk-1,ist_ppp)) + eps)
6320  + std::abs(sten(ii-1,jj ,kk-1,ist_0pp)) /
6321  ( std::abs(sten(ii-2,jj ,kk-1,ist_ppp))
6322  + std::abs(sten(ii-1,jj ,kk-1,ist_ppp)) + eps);
6323  wpmp *= std::abs(sten(ii-1,jj ,kk-1,ist_ppp));
6324  cv += wpmp*fine(ii-1,jj+1,kk-1)*sten(ii-1,jj+1,kk-1,ist_inv);
6325 
6326  Real wmpp = Real(1.0)
6327  + std::abs(sten(ii ,jj-1,kk-1,ist_p00)) /
6328  ( std::abs(sten(ii ,jj-2,kk-2,ist_ppp))
6329  + std::abs(sten(ii ,jj-1,kk-2,ist_ppp))
6330  + std::abs(sten(ii ,jj-2,kk-1,ist_ppp))
6331  + std::abs(sten(ii ,jj-1,kk-1,ist_ppp)) + eps)
6332  + std::abs(sten(ii+1,jj-1,kk-1,ist_0p0)) /
6333  ( std::abs(sten(ii ,jj-1,kk-2,ist_ppp))
6334  + std::abs(sten(ii+1,jj-1,kk-2,ist_ppp))
6335  + std::abs(sten(ii ,jj-1,kk-1,ist_ppp))
6336  + std::abs(sten(ii+1,jj-1,kk-1,ist_ppp)) + eps)
6337  + std::abs(sten(ii+1,jj-1,kk-1,ist_00p)) /
6338  ( std::abs(sten(ii ,jj-2,kk-1,ist_ppp))
6339  + std::abs(sten(ii+1,jj-2,kk-1,ist_ppp))
6340  + std::abs(sten(ii ,jj-1,kk-1,ist_ppp))
6341  + std::abs(sten(ii+1,jj-1,kk-1,ist_ppp)) + eps)
6342  + std::abs(sten(ii ,jj-1,kk-1,ist_pp0)) /
6343  ( std::abs(sten(ii ,jj-1,kk-2,ist_ppp))
6344  + std::abs(sten(ii ,jj-1,kk-1,ist_ppp)) + eps)
6345  + std::abs(sten(ii ,jj-1,kk-1,ist_p0p)) /
6346  ( std::abs(sten(ii ,jj-2,kk-1,ist_ppp))
6347  + std::abs(sten(ii ,jj-1,kk-1,ist_ppp)) + eps)
6348  + std::abs(sten(ii+1,jj-1,kk-1,ist_0pp)) /
6349  ( std::abs(sten(ii ,jj-1,kk-1,ist_ppp))
6350  + std::abs(sten(ii+1,jj-1,kk-1,ist_ppp)) + eps);
6351  wmpp *= std::abs(sten(ii ,jj-1,kk-1,ist_ppp));
6352  cv += wmpp*fine(ii+1,jj-1,kk-1)*sten(ii+1,jj-1,kk-1,ist_inv);
6353 
6354  Real wppp = Real(1.0)
6355  + std::abs(sten(ii-1,jj-1,kk-1,ist_p00)) /
6356  ( std::abs(sten(ii-1,jj-2,kk-2,ist_ppp))
6357  + std::abs(sten(ii-1,jj-1,kk-2,ist_ppp))
6358  + std::abs(sten(ii-1,jj-2,kk-1,ist_ppp))
6359  + std::abs(sten(ii-1,jj-1,kk-1,ist_ppp)) + eps)
6360  + std::abs(sten(ii-1,jj-1,kk-1,ist_0p0)) /
6361  ( std::abs(sten(ii-2,jj-1,kk-2,ist_ppp))
6362  + std::abs(sten(ii-1,jj-1,kk-2,ist_ppp))
6363  + std::abs(sten(ii-2,jj-1,kk-1,ist_ppp))
6364  + std::abs(sten(ii-1,jj-1,kk-1,ist_ppp)) + eps)
6365  + std::abs(sten(ii-1,jj-1,kk-1,ist_00p)) /
6366  ( std::abs(sten(ii-2,jj-2,kk-1,ist_ppp))
6367  + std::abs(sten(ii-1,jj-2,kk-1,ist_ppp))
6368  + std::abs(sten(ii-2,jj-1,kk-1,ist_ppp))
6369  + std::abs(sten(ii-1,jj-1,kk-1,ist_ppp)) + eps)
6370  + std::abs(sten(ii-1,jj-1,kk-1,ist_pp0)) /
6371  ( std::abs(sten(ii-1,jj-1,kk-2,ist_ppp))
6372  + std::abs(sten(ii-1,jj-1,kk-1,ist_ppp)) + eps)
6373  + std::abs(sten(ii-1,jj-1,kk-1,ist_p0p)) /
6374  ( std::abs(sten(ii-1,jj-2,kk-1,ist_ppp))
6375  + std::abs(sten(ii-1,jj-1,kk-1,ist_ppp)) + eps)
6376  + std::abs(sten(ii-1,jj-1,kk-1,ist_0pp)) /
6377  ( std::abs(sten(ii-2,jj-1,kk-1,ist_ppp))
6378  + std::abs(sten(ii-1,jj-1,kk-1,ist_ppp)) + eps);
6379  wppp *= std::abs(sten(ii-1,jj-1,kk-1,ist_ppp));
6380  cv += wppp*fine(ii-1,jj-1,kk-1)*sten(ii-1,jj-1,kk-1,ist_inv);
6381 
6382  crse(i,j,k) = cv * Real(0.125);
6383  }
6384 }
6385 
6386 #ifdef AMREX_USE_EB
6387 
6388 namespace nodelap_detail {
6389 
6390  constexpr int i_S_x = 0;
6391  constexpr int i_S_y = 1;
6392  constexpr int i_S_z = 2;
6393  constexpr int i_S_x2 = 3;
6394  constexpr int i_S_y2 = 4;
6395  constexpr int i_S_z2 = 5;
6396  constexpr int i_S_x_y = 6;
6397  constexpr int i_S_x_z = 7;
6398  constexpr int i_S_y_z = 8;
6399  constexpr int i_S_x2_y = 9;
6400  constexpr int i_S_x2_z = 10;
6401  constexpr int i_S_x_y2 = 11;
6402  constexpr int i_S_y2_z = 12;
6403  constexpr int i_S_x_z2 = 13;
6404  constexpr int i_S_y_z2 = 14;
6405  constexpr int i_S_x2_y2 = 15;
6406  constexpr int i_S_x2_z2 = 16;
6407  constexpr int i_S_y2_z2 = 17;
6408  constexpr int i_S_xyz = 18;
6409  constexpr int n_Sintg = 19;
6410 
6411  constexpr int i_c_xmym = 0;
6412  constexpr int i_c_xmyb = 1;
6413  constexpr int i_c_xmyp = 2;
6414  constexpr int i_c_xbym = 3;
6415  constexpr int i_c_xbyb = 4;
6416  constexpr int i_c_xbyp = 5;
6417  constexpr int i_c_xpym = 6;
6418  constexpr int i_c_xpyb = 7;
6419  constexpr int i_c_xpyp = 8;
6420  constexpr int i_c_xmzm = 9;
6421  constexpr int i_c_xmzb = 10;
6422  constexpr int i_c_xmzp = 11;
6423  constexpr int i_c_xbzm = 12;
6424  constexpr int i_c_xbzb = 13;
6425  constexpr int i_c_xbzp = 14;
6426  constexpr int i_c_xpzm = 15;
6427  constexpr int i_c_xpzb = 16;
6428  constexpr int i_c_xpzp = 17;
6429  constexpr int i_c_ymzm = 18;
6430  constexpr int i_c_ymzb = 19;
6431  constexpr int i_c_ymzp = 20;
6432  constexpr int i_c_ybzm = 21;
6433  constexpr int i_c_ybzb = 22;
6434  constexpr int i_c_ybzp = 23;
6435  constexpr int i_c_ypzm = 24;
6436  constexpr int i_c_ypzb = 25;
6437  constexpr int i_c_ypzp = 26;
6438  constexpr int n_conn = 27;
6439 
6440  constexpr int i_B_x = 0;
6441  constexpr int i_B_y = 1;
6442  constexpr int i_B_z = 2;
6443  constexpr int i_B_x_y = 3;
6444  constexpr int i_B_x_z = 4;
6445  constexpr int i_B_y_z = 5;
6446  constexpr int i_B_xyz = 6;
6447  constexpr int numSurfIntgs = 7;
6448 
6449 }
6450 
6451 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
6452 void mlndlap_set_connection (int i, int j, int k, Array4<Real> const& conn,
6453  Array4<Real const> const& intg, Array4<Real const> const& vol,
6454  Array4<EBCellFlag const> const& flag) noexcept
6455 {
6456  using namespace nodelap_detail;
6457 
6458  if (flag(i,j,k).isCovered()) {
6459  for (int n = 0; n < n_conn; ++n) { conn(i,j,k,n) = Real(0.); }
6460  } else if (flag(i,j,k).isRegular() || vol(i,j,k) >= almostone) {
6461  for (int n = 0; n < n_conn; ++n) { conn(i,j,k,n) = Real(1.); }
6462  } else {
6463  // Scaled by 9
6464  conn(i,j,k,i_c_xmym) = Real(0.5625)*vol(i,j,k)
6465  + Real(2.25)*(-intg(i,j,k,i_S_x ) - intg(i,j,k,i_S_y)
6466  +intg(i,j,k,i_S_x2) + intg(i,j,k,i_S_y2))
6467  + Real(9.)*( intg(i,j,k,i_S_x_y ) - intg(i,j,k,i_S_x2_y)
6468  -intg(i,j,k,i_S_x_y2) + intg(i,j,k,i_S_x2_y2));
6469 
6470  // Scaled by 18
6471  conn(i,j,k,i_c_xmyb) = Real(1.125)*vol(i,j,k)
6472  + Real(4.5)*(-intg(i,j,k,i_S_x) + intg(i,j,k,i_S_x2) - intg(i,j,k,i_S_y2))
6473  + Real(18.)*( intg(i,j,k,i_S_x_y2) - intg(i,j,k,i_S_x2_y2));
6474 
6475  // Scaled by 9
6476  conn(i,j,k,i_c_xmyp) = Real(0.5625)*vol(i,j,k)
6477  + Real(2.25)*(-intg(i,j,k,i_S_x ) + intg(i,j,k,i_S_y)
6478  +intg(i,j,k,i_S_x2) + intg(i,j,k,i_S_y2))
6479  + Real(9.)*(-intg(i,j,k,i_S_x_y ) + intg(i,j,k,i_S_x2_y)
6480  -intg(i,j,k,i_S_x_y2) + intg(i,j,k,i_S_x2_y2));
6481 
6482  // Scaled by 18
6483  conn(i,j,k,i_c_xbym) = Real(1.125)*vol(i,j,k)
6484  + Real(4.5)*(-intg(i,j,k,i_S_y) - intg(i,j,k,i_S_x2) + intg(i,j,k,i_S_y2))
6485  + Real(18.)*(intg(i,j,k,i_S_x2_y) - intg(i,j,k,i_S_x2_y2));
6486 
6487  // Scaled by 36
6488  conn(i,j,k,i_c_xbyb) = Real(2.25)*vol(i,j,k)
6489  + Real(9.)*(-intg(i,j,k,i_S_x2) - intg(i,j,k,i_S_y2))
6490  + Real(36.)*intg(i,j,k,i_S_x2_y2);
6491 
6492  // Scaled by 18
6493  conn(i,j,k,i_c_xbyp) = Real(1.125)*vol(i,j,k)
6494  + Real(4.5)*( intg(i,j,k,i_S_y) - intg(i,j,k,i_S_x2) + intg(i,j,k,i_S_y2))
6495  + Real(18.)*(-intg(i,j,k,i_S_x2_y) - intg(i,j,k,i_S_x2_y2));
6496 
6497  // Scaled by 9
6498  conn(i,j,k,i_c_xpym) = Real(0.5625)*vol(i,j,k)
6499  + Real(2.25)*( intg(i,j,k,i_S_x ) - intg(i,j,k,i_S_y)
6500  +intg(i,j,k,i_S_x2) + intg(i,j,k,i_S_y2))
6501  + Real(9.)*(-intg(i,j,k,i_S_x_y ) - intg(i,j,k,i_S_x2_y)
6502  +intg(i,j,k,i_S_x_y2) + intg(i,j,k,i_S_x2_y2));
6503 
6504  // Scaled by 18
6505  conn(i,j,k,i_c_xpyb) = Real(1.125)*vol(i,j,k)
6506  + Real(4.5)*( intg(i,j,k,i_S_x) + intg(i,j,k,i_S_x2) - intg(i,j,k,i_S_y2))
6507  + Real(18.)*(-intg(i,j,k,i_S_x_y2) - intg(i,j,k,i_S_x2_y2));
6508 
6509  // Scaled by 9
6510  conn(i,j,k,i_c_xpyp) = Real(0.5625)*vol(i,j,k)
6511  + Real(2.25)*( intg(i,j,k,i_S_x ) + intg(i,j,k,i_S_y)
6512  +intg(i,j,k,i_S_x2) + intg(i,j,k,i_S_y2))
6513  + Real(9.)*( intg(i,j,k,i_S_x_y ) + intg(i,j,k,i_S_x2_y)
6514  +intg(i,j,k,i_S_x_y2) + intg(i,j,k,i_S_x2_y2));
6515 
6516  // Scaled by 9
6517  conn(i,j,k,i_c_xmzm) = Real(0.5625)*vol(i,j,k)
6518  + Real(2.25)*(-intg(i,j,k,i_S_x) - intg(i,j,k,i_S_z)
6519  +intg(i,j,k,i_S_x2) + intg(i,j,k,i_S_z2))
6520  + Real(9.)*(intg(i,j,k,i_S_x_z) - intg(i,j,k,i_S_x2_z)
6521  -intg(i,j,k,i_S_x_z2) + intg(i,j,k,i_S_x2_z2));
6522 
6523  // Scaled by 18
6524  conn(i,j,k,i_c_xmzb) = Real(1.125)*vol(i,j,k)
6525  + Real(4.5)*(-intg(i,j,k,i_S_x) + intg(i,j,k,i_S_x2) - intg(i,j,k,i_S_z2))
6526  + Real(18.)*(intg(i,j,k,i_S_x_z2) - intg(i,j,k,i_S_x2_z2));
6527 
6528  // Scaled by 9
6529  conn(i,j,k,i_c_xmzp) = Real(0.5625)*vol(i,j,k)
6530  + Real(2.25)*(-intg(i,j,k,i_S_x ) + intg(i,j,k,i_S_z)
6531  +intg(i,j,k,i_S_x2) + intg(i,j,k,i_S_z2))
6532  + Real(9.)*(-intg(i,j,k,i_S_x_z ) + intg(i,j,k,i_S_x2_z)
6533  -intg(i,j,k,i_S_x_z2) + intg(i,j,k,i_S_x2_z2));
6534 
6535  // Scaled by 18
6536  conn(i,j,k,i_c_xbzm) = Real(1.125)*vol(i,j,k)
6537  + Real(4.5)*(-intg(i,j,k,i_S_z) - intg(i,j,k,i_S_x2) + intg(i,j,k,i_S_z2))
6538  + Real(18.)*(intg(i,j,k,i_S_x2_z) - intg(i,j,k,i_S_x2_z2));
6539 
6540  // Scaled by 18
6541  conn(i,j,k,i_c_xbzb) = Real(2.25)*vol(i,j,k)
6542  + Real(9.)*(-intg(i,j,k,i_S_x2) - intg(i,j,k,i_S_z2))
6543  + Real(36.)*intg(i,j,k,i_S_x2_z2);
6544 
6545  // Scaled by 18
6546  conn(i,j,k,i_c_xbzp) = Real(1.125)*vol(i,j,k)
6547  + Real(4.5)*( intg(i,j,k,i_S_z) - intg(i,j,k,i_S_x2) + intg(i,j,k,i_S_z2))
6548  + Real(18.)*(-intg(i,j,k,i_S_x2_z) - intg(i,j,k,i_S_x2_z2));
6549 
6550  // Scaled by 9
6551  conn(i,j,k,i_c_xpzm) = Real(0.5625)*vol(i,j,k)
6552  + Real(2.25)*( intg(i,j,k,i_S_x ) - intg(i,j,k,i_S_z)
6553  +intg(i,j,k,i_S_x2) + intg(i,j,k,i_S_z2))
6554  + Real(9.)*(-intg(i,j,k,i_S_x_z ) - intg(i,j,k,i_S_x2_z)
6555  +intg(i,j,k,i_S_x_z2) + intg(i,j,k,i_S_x2_z2));
6556 
6557  // Scaled by 18
6558  conn(i,j,k,i_c_xpzb) = Real(1.125)*vol(i,j,k)
6559  + Real(4.5)*( intg(i,j,k,i_S_x ) + intg(i,j,k,i_S_x2 ) - intg(i,j,k,i_S_z2))
6560  + Real(18.)*(-intg(i,j,k,i_S_x_z2) - intg(i,j,k,i_S_x2_z2));
6561 
6562  // Scaled by 9
6563  conn(i,j,k,i_c_xpzp) = Real(0.5625)*vol(i,j,k)
6564  + Real(2.25)*( intg(i,j,k,i_S_x ) + intg(i,j,k,i_S_z)
6565  +intg(i,j,k,i_S_x2) + intg(i,j,k,i_S_z2))
6566  + Real(9.)*( intg(i,j,k,i_S_x_z ) + intg(i,j,k,i_S_x2_z)
6567  +intg(i,j,k,i_S_x_z2) + intg(i,j,k,i_S_x2_z2));
6568 
6569  // Scaled by 9
6570  conn(i,j,k,i_c_ymzm) = Real(0.5625)*vol(i,j,k)
6571  + Real(2.25)*(-intg(i,j,k,i_S_y) - intg(i,j,k,i_S_z)
6572  +intg(i,j,k,i_S_y2) + intg(i,j,k,i_S_z2))
6573  + Real(9.)*(intg(i,j,k,i_S_y_z) - intg(i,j,k,i_S_y2_z)
6574  -intg(i,j,k,i_S_y_z2) + intg(i,j,k,i_S_y2_z2));
6575 
6576  // Scaled by 18
6577  conn(i,j,k,i_c_ymzb) = Real(1.125)*vol(i,j,k)
6578  + Real(4.5)*(-intg(i,j,k,i_S_y) + intg(i,j,k,i_S_y2) - intg(i,j,k,i_S_z2))
6579  + Real(18.)*(intg(i,j,k,i_S_y_z2) - intg(i,j,k,i_S_y2_z2));
6580 
6581  // Scaled by 9
6582  conn(i,j,k,i_c_ymzp) = Real(0.5625)*vol(i,j,k)
6583  + Real(2.25)*(-intg(i,j,k,i_S_y ) + intg(i,j,k,i_S_z)
6584  +intg(i,j,k,i_S_y2) + intg(i,j,k,i_S_z2))
6585  + Real(9.)*(-intg(i,j,k,i_S_y_z ) + intg(i,j,k,i_S_y2_z)
6586  -intg(i,j,k,i_S_y_z2) + intg(i,j,k,i_S_y2_z2));
6587 
6588  // Scaled by 18
6589  conn(i,j,k,i_c_ybzm) = Real(1.125)*vol(i,j,k)
6590  + Real(4.5)*(-intg(i,j,k,i_S_z) - intg(i,j,k,i_S_y2) + intg(i,j,k,i_S_z2))
6591  + Real(18.)*(intg(i,j,k,i_S_y2_z) - intg(i,j,k,i_S_y2_z2));
6592 
6593  // Scaled by 36
6594  conn(i,j,k,i_c_ybzb) = Real(2.25)*vol(i,j,k)
6595  + Real(9.)*(-intg(i,j,k,i_S_y2) - intg(i,j,k,i_S_z2))
6596  + Real(36.)*intg(i,j,k,i_S_y2_z2);
6597 
6598  // Scaled by 18
6599  conn(i,j,k,i_c_ybzp) = Real(1.125)*vol(i,j,k)
6600  + Real(4.5)*( intg(i,j,k,i_S_z) - intg(i,j,k,i_S_y2) + intg(i,j,k,i_S_z2))
6601  + Real(18.)*(-intg(i,j,k,i_S_y2_z) - intg(i,j,k,i_S_y2_z2));
6602 
6603  // Scaled by 9
6604  conn(i,j,k,i_c_ypzm) = Real(0.5625)*vol(i,j,k)
6605  + Real(2.25)*( intg(i,j,k,i_S_y ) - intg(i,j,k,i_S_z)
6606  +intg(i,j,k,i_S_y2) + intg(i,j,k,i_S_z2))
6607  + Real(9.)*(-intg(i,j,k,i_S_y_z ) - intg(i,j,k,i_S_y2_z)
6608  +intg(i,j,k,i_S_y_z2) + intg(i,j,k,i_S_y2_z2));
6609 
6610  // Scaled by 18
6611  conn(i,j,k,i_c_ypzb) = Real(1.125)*vol(i,j,k)
6612  + Real(4.5)*( intg(i,j,k,i_S_y ) + intg(i,j,k,i_S_y2) - intg(i,j,k,i_S_z2))
6613  + Real(18.)*(-intg(i,j,k,i_S_y_z2) - intg(i,j,k,i_S_y2_z2));
6614 
6615  // Scaled by 9
6616  conn(i,j,k,i_c_ypzp) = Real(0.5625)*vol(i,j,k)
6617  + Real(2.25)*( intg(i,j,k,i_S_y ) + intg(i,j,k,i_S_z)
6618  +intg(i,j,k,i_S_y2) + intg(i,j,k,i_S_z2))
6619  + Real(9.)*( intg(i,j,k,i_S_y_z ) + intg(i,j,k,i_S_y2_z)
6620  +intg(i,j,k,i_S_y_z2) + intg(i,j,k,i_S_y2_z2));
6621  }
6622 }
6623 
6624 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
6625 void mlndlap_set_stencil_eb (int i, int j, int k, Array4<Real> const& sten,
6626  Array4<Real const> const& sig, Array4<Real const> const& conn,
6627  GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
6628 {
6629  using namespace nodelap_detail;
6630 
6631  Real facx = Real(1./36.)*dxinv[0]*dxinv[0];
6632  Real facy = Real(1./36.)*dxinv[1]*dxinv[1];
6633  Real facz = Real(1./36.)*dxinv[2]*dxinv[2];
6634 
6635  // i+1,j,k
6636  sten(i,j,k,ist_p00) = (
6637  sig(i,j ,k )*(Real(4.)*facx*conn(i,j ,k ,i_c_ymzm) - Real(2.)*facy*conn(i,j ,k ,i_c_xbzm) - Real(2.)*facz*conn(i,j ,k ,i_c_xbym) ) +
6638  sig(i,j-1,k )*(Real(4.)*facx*conn(i,j-1,k ,i_c_ypzm) - Real(2.)*facy*conn(i,j-1,k ,i_c_xbzm) - Real(2.)*facz*conn(i,j-1,k ,i_c_xbyp) ) +
6639  sig(i,j ,k-1)*(Real(4.)*facx*conn(i,j ,k-1,i_c_ymzp) - Real(2.)*facy*conn(i,j ,k-1,i_c_xbzp) - Real(2.)*facz*conn(i,j ,k-1,i_c_xbym) ) +
6640  sig(i,j-1,k-1)*(Real(4.)*facx*conn(i,j-1,k-1,i_c_ypzp) - Real(2.)*facy*conn(i,j-1,k-1,i_c_xbzp) - Real(2.)*facz*conn(i,j-1,k-1,i_c_xbyp) ) );
6641 
6642  // i,j+1,k
6643  sten(i,j,k,ist_0p0) = (
6644  sig(i ,j,k )*(Real(-2.)*facx*conn(i ,j,k ,i_c_ybzm) + Real(4.)*facy*conn(i ,j,k ,i_c_xmzm) - Real(2.)*facz*conn(i ,j,k ,i_c_xmyb) ) +
6645  sig(i-1,j,k )*(Real(-2.)*facx*conn(i-1,j,k ,i_c_ybzm) + Real(4.)*facy*conn(i-1,j,k ,i_c_xpzm) - Real(2.)*facz*conn(i-1,j,k ,i_c_xpyb) ) +
6646  sig(i ,j,k-1)*(Real(-2.)*facx*conn(i ,j,k-1,i_c_ybzp) + Real(4.)*facy*conn(i ,j,k-1,i_c_xmzp) - Real(2.)*facz*conn(i ,j,k-1,i_c_xmyb) ) +
6647  sig(i-1,j,k-1)*(Real(-2.)*facx*conn(i-1,j,k-1,i_c_ybzp) + Real(4.)*facy*conn(i-1,j,k-1,i_c_xpzp) - Real(2.)*facz*conn(i-1,j,k-1,i_c_xpyb) ) );
6648 
6649  // i,j,k+1
6650  sten(i,j,k,ist_00p) = (
6651  sig(i ,j ,k)*(Real(-2.)*facx*conn(i ,j ,k,i_c_ymzb) - Real(2.)*facy*conn(i ,j ,k,i_c_xmzb) + Real(4.)*facz*conn(i ,j ,k,i_c_xmym) ) +
6652  sig(i-1,j ,k)*(Real(-2.)*facx*conn(i-1,j ,k,i_c_ymzb) - Real(2.)*facy*conn(i-1,j ,k,i_c_xpzb) + Real(4.)*facz*conn(i-1,j ,k,i_c_xpym) ) +
6653  sig(i ,j-1,k)*(Real(-2.)*facx*conn(i ,j-1,k,i_c_ypzb) - Real(2.)*facy*conn(i ,j-1,k,i_c_xmzb) + Real(4.)*facz*conn(i ,j-1,k,i_c_xmyp) ) +
6654  sig(i-1,j-1,k)*(Real(-2.)*facx*conn(i-1,j-1,k,i_c_ypzb) - Real(2.)*facy*conn(i-1,j-1,k,i_c_xpzb) + Real(4.)*facz*conn(i-1,j-1,k,i_c_xpyp) ) );
6655 
6656  // i+1,j+1,k
6657  sten(i,j,k,ist_pp0) = (
6658  sig(i,j,k )*(Real(2.)*facx*conn(i,j,k ,i_c_ybzm) + Real(2.)*facy*conn(i,j,k ,i_c_xbzm) - facz*conn(i,j,k ,i_c_xbyb) ) +
6659  sig(i,j,k-1)*(Real(2.)*facx*conn(i,j,k-1,i_c_ybzp) + Real(2.)*facy*conn(i,j,k-1,i_c_xbzp) - facz*conn(i,j,k-1,i_c_xbyb) ) );
6660 
6661  // i+1,j,k+1
6662  sten(i,j,k,ist_p0p) = (
6663  sig(i,j,k )*(Real(2.)*facx*conn(i,j,k ,i_c_ymzb) - facy*conn(i,j,k ,i_c_xbzb) + Real(2.)*facz*conn(i,j,k ,i_c_xbym) ) +
6664  sig(i,j-1,k)*(Real(2.)*facx*conn(i,j-1,k,i_c_ypzb) - facy*conn(i,j-1,k,i_c_xbzb) + Real(2.)*facz*conn(i,j-1,k,i_c_xbyp) ) );
6665 
6666  // i,j+1,k+1
6667  sten(i,j,k,ist_0pp) = (
6668  sig(i ,j,k)*(-facx*conn(i ,j,k,i_c_ybzb) + Real(2.)*facy*conn(i ,j,k,i_c_xmzb) + Real(2.)*facz*conn(i ,j,k,i_c_xmyb) ) +
6669  sig(i-1,j,k)*(-facx*conn(i-1,j,k,i_c_ybzb) + Real(2.)*facy*conn(i-1,j,k,i_c_xpzb) + Real(2.)*facz*conn(i-1,j,k,i_c_xpyb) ) );
6670 
6671  // i+1,j+1,k+1
6672  sten(i,j,k,ist_ppp) = sig(i,j,k) * (facx*conn(i,j,k,i_c_ybzb) + facy*conn(i,j,k,i_c_xbzb) + facz*conn(i,j,k,i_c_xbyb) );
6673 }
6674 
6675 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
6676 void mlndlap_divu_eb (int i, int j, int k, Array4<Real> const& rhs, Array4<Real const> const& vel,
6677  Array4<Real const> const& vfrac, Array4<Real const> const& intg,
6678  Array4<int const> const& msk, GpuArray<Real,AMREX_SPACEDIM> const& dxinv,
6679  Box const& nodal_domain,
6680  GpuArray<LinOpBCType, AMREX_SPACEDIM> const& bclo,
6681  GpuArray<LinOpBCType, AMREX_SPACEDIM> const& bchi) noexcept
6682 {
6683  using namespace nodelap_detail;
6684 
6685  Real facx = Real(0.25)*dxinv[0];
6686  Real facy = Real(0.25)*dxinv[1];
6687  Real facz = Real(0.25)*dxinv[2];
6688 
6689  const auto domlo = amrex::lbound(nodal_domain);
6690  const auto domhi = amrex::ubound(nodal_domain);
6691 
6692  if (!msk(i,j,k)) {
6693 
6694  Real zero_ilo = Real(1.0);
6695  Real zero_ihi = Real(1.0);
6696  Real zero_jlo = Real(1.0);
6697  Real zero_jhi = Real(1.0);
6698  Real zero_klo = Real(1.0);
6699  Real zero_khi = Real(1.0);
6700 
6701  // The nodal divergence operator should not see the tangential velocity
6702  // at an inflow face
6703  if ((bclo[0] == LinOpBCType::Neumann || bclo[0] == LinOpBCType::inflow)
6704  && i == domlo.x)
6705  {
6706  zero_ilo = Real(0.0);
6707  }
6708  if ((bchi[0] == LinOpBCType::Neumann || bchi[0] == LinOpBCType::inflow)
6709  && i == domhi.x)
6710  {
6711  zero_ihi = Real(0.0);
6712  }
6713  if ((bclo[1] == LinOpBCType::Neumann || bclo[1] == LinOpBCType::inflow)
6714  && j == domlo.y)
6715  {
6716  zero_jlo = Real(0.0);
6717  }
6718  if ((bchi[1] == LinOpBCType::Neumann || bchi[1] == LinOpBCType::inflow)
6719  && j == domhi.y)
6720  {
6721  zero_jhi = Real(0.0);
6722  }
6723  if ((bclo[2] == LinOpBCType::Neumann || bclo[2] == LinOpBCType::inflow)
6724  && k == domlo.z)
6725  {
6726  zero_klo = Real(0.0);
6727  }
6728  if ((bchi[2] == LinOpBCType::Neumann || bchi[2] == LinOpBCType::inflow)
6729  && k == domhi.z)
6730  {
6731  zero_khi = Real(0.0);
6732  }
6733 
6734  rhs(i,j,k) = facx*(
6735  vel(i-1,j-1,k ,0)*( -vfrac(i-1,j-1,k )
6736  -Real(2.)*intg(i-1,j-1,k ,i_S_y)
6737  +Real(2.)*intg(i-1,j-1,k ,i_S_z)
6738  +Real(4.)*intg(i-1,j-1,k ,i_S_y_z))*zero_jlo*zero_khi
6739  +vel(i ,j-1,k ,0)*( vfrac(i ,j-1,k )
6740  +Real(2.)*intg(i ,j-1,k ,i_S_y)
6741  -Real(2.)*intg(i ,j-1,k ,i_S_z)
6742  -Real(4.)*intg(i ,j-1,k ,i_S_y_z))*zero_jlo*zero_khi
6743  +vel(i-1,j ,k ,0)*( -vfrac(i-1,j ,k )
6744  +Real(2.)*intg(i-1,j ,k ,i_S_y)
6745  +Real(2.)*intg(i-1,j ,k ,i_S_z)
6746  -Real(4.)*intg(i-1,j ,k ,i_S_y_z))*zero_jhi*zero_khi
6747  +vel(i ,j ,k ,0)*( vfrac(i ,j ,k )
6748  -Real(2.)*intg(i ,j ,k ,i_S_y)
6749  -Real(2.)*intg(i ,j ,k ,i_S_z)
6750  +Real(4.)*intg(i ,j ,k ,i_S_y_z))*zero_jhi*zero_khi
6751  +vel(i-1,j-1,k-1,0)*( -vfrac(i-1,j-1,k-1)
6752  -Real(2.)*intg(i-1,j-1,k-1,i_S_y)
6753  -Real(2.)*intg(i-1,j-1,k-1,i_S_z)
6754  -Real(4.)*intg(i-1,j-1,k-1,i_S_y_z))*zero_jlo*zero_klo
6755  +vel(i ,j-1,k-1,0)*( vfrac(i ,j-1,k-1)
6756  +Real(2.)*intg(i ,j-1,k-1,i_S_y)
6757  +Real(2.)*intg(i ,j-1,k-1,i_S_z)
6758  +Real(4.)*intg(i ,j-1,k-1,i_S_y_z))*zero_jlo*zero_klo
6759  +vel(i-1,j ,k-1,0)*( -vfrac(i-1,j ,k-1)
6760  +Real(2.)*intg(i-1,j ,k-1,i_S_y)
6761  -Real(2.)*intg(i-1,j ,k-1,i_S_z)
6762  +Real(4.)*intg(i-1,j ,k-1,i_S_y_z))*zero_jhi*zero_klo
6763  +vel(i ,j ,k-1,0)*( vfrac(i ,j ,k-1)
6764  -Real(2.)*intg(i ,j ,k-1,i_S_y)
6765  +Real(2.)*intg(i ,j ,k-1,i_S_z)
6766  -Real(4.)*intg(i ,j ,k-1,i_S_y_z))*zero_jhi*zero_klo )
6767  + facy*(
6768  vel(i-1,j-1,k ,1)*( -vfrac(i-1,j-1,k )
6769  -Real(2.)*intg(i-1,j-1,k ,i_S_x)
6770  +Real(2.)*intg(i-1,j-1,k ,i_S_z)
6771  +Real(4.)*intg(i-1,j-1,k ,i_S_x_z))*zero_ilo*zero_khi
6772  +vel(i ,j-1,k ,1)*( -vfrac(i ,j-1,k )
6773  +Real(2.)*intg(i ,j-1,k ,i_S_x)
6774  +Real(2.)*intg(i ,j-1,k ,i_S_z)
6775  -Real(4.)*intg(i ,j-1,k ,i_S_x_z))*zero_ihi*zero_khi
6776  +vel(i-1,j ,k ,1)*( vfrac(i-1,j ,k )
6777  +Real(2.)*intg(i-1,j ,k ,i_S_x)
6778  -Real(2.)*intg(i-1,j ,k ,i_S_z)
6779  -Real(4.)*intg(i-1,j ,k ,i_S_x_z))*zero_ilo*zero_khi
6780  +vel(i ,j ,k ,1)*( vfrac(i ,j ,k )
6781  -Real(2.)*intg(i ,j ,k ,i_S_x)
6782  -Real(2.)*intg(i ,j ,k ,i_S_z)
6783  +Real(4.)*intg(i ,j ,k ,i_S_x_z))*zero_ihi*zero_khi
6784  +vel(i-1,j-1,k-1,1)*( -vfrac(i-1,j-1,k-1)
6785  -Real(2.)*intg(i-1,j-1,k-1,i_S_x)
6786  -Real(2.)*intg(i-1,j-1,k-1,i_S_z)
6787  -Real(4.)*intg(i-1,j-1,k-1,i_S_x_z))*zero_ilo*zero_klo
6788  +vel(i ,j-1,k-1,1)*( -vfrac(i ,j-1,k-1)
6789  +Real(2.)*intg(i ,j-1,k-1,i_S_x)
6790  -Real(2.)*intg(i ,j-1,k-1,i_S_z)
6791  +Real(4.)*intg(i ,j-1,k-1,i_S_x_z))*zero_ihi*zero_klo
6792  +vel(i-1,j ,k-1,1)*( vfrac(i-1,j ,k-1)
6793  +Real(2.)*intg(i-1,j ,k-1,i_S_x)
6794  +Real(2.)*intg(i-1,j ,k-1,i_S_z)
6795  +Real(4.)*intg(i-1,j ,k-1,i_S_x_z))*zero_ilo*zero_klo
6796  +vel(i ,j ,k-1,1)*( vfrac(i ,j ,k-1)
6797  -Real(2.)*intg(i ,j ,k-1,i_S_x)
6798  +Real(2.)*intg(i ,j ,k-1,i_S_z)
6799  -Real(4.)*intg(i ,j ,k-1,i_S_x_z))*zero_ihi*zero_klo )
6800  + facz*(
6801  vel(i-1,j-1,k ,2)*( vfrac(i-1,j-1,k )
6802  +Real(2.)*intg(i-1,j-1,k ,i_S_x)
6803  +Real(2.)*intg(i-1,j-1,k ,i_S_y)
6804  +Real(4.)*intg(i-1,j-1,k ,i_S_x_y))*zero_ilo*zero_jlo
6805  +vel(i ,j-1,k ,2)*( vfrac(i ,j-1,k )
6806  -Real(2.)*intg(i ,j-1,k ,i_S_x)
6807  +Real(2.)*intg(i ,j-1,k ,i_S_y)
6808  -Real(4.)*intg(i ,j-1,k ,i_S_x_y))*zero_ihi*zero_jlo
6809  +vel(i-1,j ,k ,2)*( vfrac(i-1,j ,k )
6810  +Real(2.)*intg(i-1,j ,k ,i_S_x)
6811  -Real(2.)*intg(i-1,j ,k ,i_S_y)
6812  -Real(4.)*intg(i-1,j ,k ,i_S_x_y))*zero_ilo*zero_jhi
6813  +vel(i ,j ,k ,2)*( vfrac(i ,j ,k )
6814  -Real(2.)*intg(i ,j ,k ,i_S_x)
6815  -Real(2.)*intg(i ,j ,k ,i_S_y)
6816  +Real(4.)*intg(i ,j ,k ,i_S_x_y))*zero_ihi*zero_jhi
6817  +vel(i-1,j-1,k-1,2)*( -vfrac(i-1,j-1,k-1)
6818  -Real(2.)*intg(i-1,j-1,k-1,i_S_x)
6819  -Real(2.)*intg(i-1,j-1,k-1,i_S_y)
6820  -Real(4.)*intg(i-1,j-1,k-1,i_S_x_y))*zero_ilo*zero_jlo
6821  +vel(i ,j-1,k-1,2)*( -vfrac(i ,j-1,k-1)
6822  +Real(2.)*intg(i ,j-1,k-1,i_S_x)
6823  -Real(2.)*intg(i ,j-1,k-1,i_S_y)
6824  +Real(4.)*intg(i ,j-1,k-1,i_S_x_y))*zero_ihi*zero_jlo
6825  +vel(i-1,j ,k-1,2)*( -vfrac(i-1,j ,k-1)
6826  -Real(2.)*intg(i-1,j ,k-1,i_S_x)
6827  +Real(2.)*intg(i-1,j ,k-1,i_S_y)
6828  +Real(4.)*intg(i-1,j ,k-1,i_S_x_y))*zero_ilo*zero_jhi
6829  +vel(i ,j ,k-1,2)*( -vfrac(i ,j ,k-1)
6830  +Real(2.)*intg(i ,j ,k-1,i_S_x)
6831  +Real(2.)*intg(i ,j ,k-1,i_S_y)
6832  -Real(4.)*intg(i ,j ,k-1,i_S_x_y))*zero_ihi*zero_jhi );
6833  } else {
6834  rhs(i,j,k) = Real(0.);
6835  }
6836 }
6837 
6838 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
6839 void add_eb_flow_contribution (int i, int j, int k, Array4<Real> const& rhs,
6840  Array4<int const> const& msk, GpuArray<Real,AMREX_SPACEDIM> const& dxinv,
6841  Array4<Real const> const& bareaarr,
6842  Array4<Real const> const& sintg,
6843  Array4<Real const> const& eb_vel_dot_n) noexcept
6844 {
6845  using namespace nodelap_detail;
6846 
6847  Real fac_eb = Real(0.125) * dxinv[0];
6848 
6849  if (!msk(i,j,k)) {
6850  rhs(i,j,k) += fac_eb*(
6851  eb_vel_dot_n(i-1,j-1,k )*( bareaarr(i-1,j-1,k)
6852  +Real(2.)*sintg(i-1,j-1,k ,i_B_x)
6853  +Real(2.)*sintg(i-1,j-1,k ,i_B_y)
6854  +Real(4.)*sintg(i-1,j-1,k ,i_B_x_y)
6855  -Real(2.)*sintg(i-1,j-1,k ,i_B_z)
6856  -Real(4.)*sintg(i-1,j-1,k ,i_B_y_z)
6857  -Real(4.)*sintg(i-1,j-1,k ,i_B_x_z)
6858  -Real(8.)*sintg(i-1,j-1,k ,i_B_xyz))
6859  +eb_vel_dot_n(i ,j-1,k )*( bareaarr(i ,j-1,k)
6860  -Real(2.)*sintg(i ,j-1,k ,i_B_x)
6861  +Real(2.)*sintg(i ,j-1,k ,i_B_y)
6862  -Real(4.)*sintg(i ,j-1,k ,i_B_x_y)
6863  -Real(2.)*sintg(i ,j-1,k ,i_B_z)
6864  -Real(4.)*sintg(i ,j-1,k ,i_B_y_z)
6865  +Real(4.)*sintg(i ,j-1,k ,i_B_x_z)
6866  +Real(8.)*sintg(i ,j-1,k ,i_B_xyz))
6867  +eb_vel_dot_n(i-1,j ,k )*( bareaarr(i-1,j ,k)
6868  +Real(2.)*sintg(i-1,j ,k ,i_B_x)
6869  -Real(2.)*sintg(i-1,j ,k ,i_B_y)
6870  -Real(4.)*sintg(i-1,j ,k ,i_B_x_y)
6871  -Real(2.)*sintg(i-1,j ,k ,i_B_z)
6872  +Real(4.)*sintg(i-1,j ,k ,i_B_y_z)
6873  -Real(4.)*sintg(i-1,j ,k ,i_B_x_z)
6874  +Real(8.)*sintg(i-1,j ,k ,i_B_xyz))
6875  +eb_vel_dot_n(i ,j ,k )*( bareaarr(i ,j ,k)
6876  -Real(2.)*sintg(i ,j ,k ,i_B_x)
6877  -Real(2.)*sintg(i ,j ,k ,i_B_y)
6878  +Real(4.)*sintg(i ,j ,k ,i_B_x_y)
6879  -Real(2.)*sintg(i ,j ,k ,i_B_z)
6880  +Real(4.)*sintg(i ,j ,k ,i_B_y_z)
6881  +Real(4.)*sintg(i ,j ,k ,i_B_x_z)
6882  -Real(8.)*sintg(i ,j ,k ,i_B_xyz))
6883  +eb_vel_dot_n(i-1,j-1,k-1)*( bareaarr(i-1,j-1,k-1)
6884  +Real(2.)*sintg(i-1,j-1,k-1,i_B_x)
6885  +Real(2.)*sintg(i-1,j-1,k-1,i_B_y)
6886  +Real(4.)*sintg(i-1,j-1,k-1,i_B_x_y)
6887  +Real(2.)*sintg(i-1,j-1,k-1,i_B_z)
6888  +Real(4.)*sintg(i-1,j-1,k-1,i_B_y_z)
6889  +Real(4.)*sintg(i-1,j-1,k-1,i_B_x_z)
6890  +Real(8.)*sintg(i-1,j-1,k-1,i_B_xyz))
6891  +eb_vel_dot_n(i ,j-1,k-1)*( bareaarr(i ,j-1,k-1)
6892  -Real(2.)*sintg(i ,j-1,k-1,i_B_x)
6893  +Real(2.)*sintg(i ,j-1,k-1,i_B_y)
6894  -Real(4.)*sintg(i ,j-1,k-1,i_B_x_y)
6895  +Real(2.)*sintg(i ,j-1,k-1,i_B_z)
6896  +Real(4.)*sintg(i ,j-1,k-1,i_B_y_z)
6897  -Real(4.)*sintg(i ,j-1,k-1,i_B_x_z)
6898  -Real(8.)*sintg(i ,j-1,k-1,i_B_xyz))
6899  +eb_vel_dot_n(i-1,j ,k-1)*( bareaarr(i-1,j ,k-1)
6900  +Real(2.)*sintg(i-1,j ,k-1,i_B_x)
6901  -Real(2.)*sintg(i-1,j ,k-1,i_B_y)
6902  -Real(4.)*sintg(i-1,j ,k-1,i_B_x_y)
6903  +Real(2.)*sintg(i-1,j ,k-1,i_B_z)
6904  -Real(4.)*sintg(i-1,j ,k-1,i_B_y_z)
6905  +Real(4.)*sintg(i-1,j ,k-1,i_B_x_z)
6906  -Real(8.)*sintg(i-1,j ,k-1,i_B_xyz))
6907  +eb_vel_dot_n(i ,j ,k-1)*( bareaarr(i ,j ,k-1)
6908  -Real(2.)*sintg(i ,j ,k-1,i_B_x)
6909  -Real(2.)*sintg(i ,j ,k-1,i_B_y)
6910  +Real(4.)*sintg(i ,j ,k-1,i_B_x_y)
6911  +Real(2.)*sintg(i ,j ,k-1,i_B_z)
6912  -Real(4.)*sintg(i ,j ,k-1,i_B_y_z)
6913  -Real(4.)*sintg(i ,j ,k-1,i_B_x_z)
6914  +Real(8.)*sintg(i ,j ,k-1,i_B_xyz)));
6915  }
6916 }
6917 
6918 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
6919 void mlndlap_mknewu_eb (int i, int j, int k, Array4<Real> const& u, Array4<Real const> const& p,
6920  Array4<Real const> const& sig, Array4<Real const> const& vfrac,
6921  Array4<Real const> const& intg, GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
6922 {
6923  using namespace nodelap_detail;
6924 
6925  if (vfrac(i,j,k) == Real(0.)) {
6926  u(i,j,k,0) = u(i,j,k,1) = u(i,j,k,2) = Real(0.);
6927  } else {
6928  Real dpdx = Real(0.25)*(-p(i,j,k )+p(i+1,j,k )-p(i,j+1,k )+p(i+1,j+1,k )
6929  -p(i,j,k+1)+p(i+1,j,k+1)-p(i,j+1,k+1)+p(i+1,j+1,k+1));
6930  Real dpdy = Real(0.25)*(-p(i,j,k )-p(i+1,j,k )+p(i,j+1,k )+p(i+1,j+1,k )
6931  -p(i,j,k+1)-p(i+1,j,k+1)+p(i,j+1,k+1)+p(i+1,j+1,k+1));
6932  Real dpdz = Real(0.25)*(-p(i,j,k )-p(i+1,j,k )-p(i,j+1,k )-p(i+1,j+1,k )
6933  +p(i,j,k+1)+p(i+1,j,k+1)+p(i,j+1,k+1)+p(i+1,j+1,k+1));
6934 
6935  Real dpp_xy = (p(i+1,j+1,k+1) - p(i,j+1,k+1) - p(i+1,j,k+1) + p(i,j,k+1)
6936  +p(i+1,j+1,k ) - p(i,j+1,k ) - p(i+1,j,k ) + p(i,j,k ) ) / vfrac(i,j,k);
6937 
6938  Real dpp_xz = (p(i+1,j+1,k+1) - p(i,j+1,k+1) + p(i+1,j,k+1) - p(i,j,k+1)
6939  -p(i+1,j+1,k ) + p(i,j+1,k ) - p(i+1,j,k ) + p(i,j,k ) ) / vfrac(i,j,k);
6940 
6941  Real dpp_yz = (p(i+1,j+1,k+1) + p(i,j+1,k+1) - p(i+1,j,k+1) - p(i,j,k+1)
6942  -p(i+1,j+1,k ) - p(i,j+1,k ) + p(i+1,j,k ) + p(i,j,k ) ) / vfrac(i,j,k);
6943 
6944  Real dpp_xyz = (p(i+1,j+1,k+1) - p(i,j+1,k+1) - p(i+1,j,k+1) + p(i,j,k+1)
6945  -p(i+1,j+1,k ) + p(i,j+1,k ) + p(i+1,j,k ) - p(i,j,k ) ) / vfrac(i,j,k);
6946 
6947  u(i,j,k,0) -= sig(i,j,k)*dxinv[0]*(dpdx + Real(0.5)*intg(i,j,k,i_S_y )*dpp_xy +
6948  Real(0.5)*intg(i,j,k,i_S_z )*dpp_xz +
6949  intg(i,j,k,i_S_y_z)*dpp_xyz );
6950  u(i,j,k,1) -= sig(i,j,k)*dxinv[1]*(dpdy + Real(0.5)*intg(i,j,k,i_S_x )*dpp_xy +
6951  Real(0.5)*intg(i,j,k,i_S_z )*dpp_yz +
6952  intg(i,j,k,i_S_x_z)*dpp_xyz );
6953  u(i,j,k,2) -= sig(i,j,k)*dxinv[2]*(dpdz + Real(0.5)*intg(i,j,k,i_S_x )*dpp_xz +
6954  Real(0.5)*intg(i,j,k,i_S_y )*dpp_yz +
6955  intg(i,j,k,i_S_x_y)*dpp_xyz );
6956  }
6957 }
6958 
6959 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
6960 void mlndlap_mknewu_eb_c (int i, int j, int k, Array4<Real> const& u, Array4<Real const> const& p,
6961  Real sig, Array4<Real const> const& vfrac,
6962  Array4<Real const> const& intg, GpuArray<Real,AMREX_SPACEDIM> const& dxinv) noexcept
6963 {
6964  using namespace nodelap_detail;
6965 
6966  if (vfrac(i,j,k) == Real(0.)) {
6967  u(i,j,k,0) = u(i,j,k,1) = u(i,j,k,2) = Real(0.);
6968  } else {
6969  Real dpdx = Real(0.25)*(-p(i,j,k )+p(i+1,j,k )-p(i,j+1,k )+p(i+1,j+1,k )
6970  -p(i,j,k+1)+p(i+1,j,k+1)-p(i,j+1,k+1)+p(i+1,j+1,k+1));
6971  Real dpdy = Real(0.25)*(-p(i,j,k )-p(i+1,j,k )+p(i,j+1,k )+p(i+1,j+1,k )
6972  -p(i,j,k+1)-p(i+1,j,k+1)+p(i,j+1,k+1)+p(i+1,j+1,k+1));
6973  Real dpdz = Real(0.25)*(-p(i,j,k )-p(i+1,j,k )-p(i,j+1,k )-p(i+1,j+1,k )
6974  +p(i,j,k+1)+p(i+1,j,k+1)+p(i,j+1,k+1)+p(i+1,j+1,k+1));
6975 
6976  Real dpp_xy = (p(i+1,j+1,k+1) - p(i,j+1,k+1) - p(i+1,j,k+1) + p(i,j,k+1)
6977  +p(i+1,j+1,k ) - p(i,j+1,k ) - p(i+1,j,k ) + p(i,j,k ) ) / vfrac(i,j,k);
6978 
6979  Real dpp_xz = (p(i+1,j+1,k+1) - p(i,j+1,k+1) + p(i+1,j,k+1) - p(i,j,k+1)
6980  -p(i+1,j+1,k ) + p(i,j+1,k ) - p(i+1,j,k ) + p(i,j,k ) ) / vfrac(i,j,k);
6981 
6982  Real dpp_yz = (p(i+1,j+1,k+1) + p(i,j+1,k+1) - p(i+1,j,k+1) - p(i,j,k+1)
6983  -p(i+1,j+1,k ) - p(i,j+1,k ) + p(i+1,j,k ) + p(i,j,k ) ) / vfrac(i,j,k);
6984 
6985  Real dpp_xyz = (p(i+1,j+1,k+1) - p(i,j+1,k+1) - p(i+1,j,k+1) + p(i,j,k+1)
6986  -p(i+1,j+1,k ) + p(i,j+1,k ) + p(i+1,j,k ) - p(i,j,k ) ) / vfrac(i,j,k);
6987 
6988  u(i,j,k,0) -= sig*dxinv[0]*(dpdx + Real(0.5)*intg(i,j,k,i_S_y )*dpp_xy +
6989  Real(0.5)*intg(i,j,k,i_S_z )*dpp_xz +
6990  intg(i,j,k,i_S_y_z)*dpp_xyz );
6991  u(i,j,k,1) -= sig*dxinv[1]*(dpdy + Real(0.5)*intg(i,j,k,i_S_x )*dpp_xy +
6992  Real(0.5)*intg(i,j,k,i_S_z )*dpp_yz +
6993  intg(i,j,k,i_S_x_z)*dpp_xyz );
6994  u(i,j,k,2) -= sig*dxinv[2]*(dpdz + Real(0.5)*intg(i,j,k,i_S_x )*dpp_xz +
6995  Real(0.5)*intg(i,j,k,i_S_y )*dpp_yz +
6996  intg(i,j,k,i_S_x_y)*dpp_xyz );
6997  }
6998 }
6999 
7000 AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
7001 Real mlndlap_rhcc_eb (int i, int j, int k, Array4<Real const> const& rhcc,
7002  Array4<Real const> const& vfrac, Array4<Real const> const& intg,
7003  Array4<int const> const& msk) noexcept
7004 {
7005  using namespace nodelap_detail;
7006 
7007  if (!msk(i,j,k)) {
7008  return
7009  rhcc(i ,j ,k ) *
7010  ( Real(0.125) * vfrac(i ,j ,k )
7011  + Real(0.25) * (-intg(i ,j ,k ,i_S_x)
7012  -intg(i ,j ,k ,i_S_y)
7013  -intg(i ,j ,k ,i_S_z))
7014  + Real(0.5) * ( intg(i ,j ,k ,i_S_x_y)
7015  +intg(i ,j ,k ,i_S_x_z)
7016  +intg(i ,j ,k ,i_S_y_z))
7017  + ( -intg(i ,j ,k ,i_S_xyz)))
7018  //
7019  + rhcc(i-1,j ,k ) *
7020  ( Real(0.125) * vfrac(i-1,j ,k )
7021  + Real(0.25) * ( intg(i-1,j ,k ,i_S_x)
7022  -intg(i-1,j ,k ,i_S_y)
7023  -intg(i-1,j ,k ,i_S_z))
7024  + Real(0.5) * ( -intg(i-1,j ,k ,i_S_x_y)
7025  -intg(i-1,j ,k ,i_S_x_z)
7026  +intg(i-1,j ,k ,i_S_y_z))
7027  + ( intg(i-1,j ,k ,i_S_xyz)))
7028  //
7029  + rhcc(i ,j-1,k ) *
7030  ( Real(0.125) * vfrac(i ,j-1,k )
7031  + Real(0.25) * (-intg(i ,j-1,k ,i_S_x)
7032  +intg(i ,j-1,k ,i_S_y)
7033  -intg(i ,j-1,k ,i_S_z))
7034  + Real(0.5) * ( -intg(i ,j-1,k ,i_S_x_y)
7035  +intg(i ,j-1,k ,i_S_x_z)
7036  -intg(i ,j-1,k ,i_S_y_z))
7037  + ( intg(i ,j-1,k ,i_S_xyz)))
7038  //
7039  + rhcc(i-1,j-1,k ) *
7040  ( Real(0.125) * vfrac(i-1,j-1,k )
7041  + Real(0.25) * ( intg(i-1,j-1,k ,i_S_x)
7042  +intg(i-1,j-1,k ,i_S_y)
7043  -intg(i-1,j-1,k ,i_S_z))
7044  + Real(0.5) * ( intg(i-1,j-1,k ,i_S_x_y)
7045  -intg(i-1,j-1,k ,i_S_x_z)
7046  -intg(i-1,j-1,k ,i_S_y_z))
7047  + ( -intg(i-1,j-1,k ,i_S_xyz)))
7048  //
7049  + rhcc(i ,j ,k-1) *
7050  ( Real(0.125) * vfrac(i ,j ,k-1)
7051  + Real(0.25) * (-intg(i ,j ,k-1,i_S_x)
7052  -intg(i ,j ,k-1,i_S_y)
7053  +intg(i ,j ,k-1,i_S_z))
7054  + Real(0.5) * ( intg(i ,j ,k-1,i_S_x_y)
7055  -intg(i ,j ,k-1,i_S_x_z)
7056  -intg(i ,j ,k-1,i_S_y_z))
7057  + ( intg(i ,j ,k-1,i_S_xyz)))
7058  //
7059  + rhcc(i-1,j ,k-1) *
7060  ( Real(0.125) * vfrac(i-1,j ,k-1)
7061  + Real(0.25) * ( intg(i-1,j ,k-1,i_S_x)
7062  -intg(i-1,j ,k-1,i_S_y)
7063  +intg(i-1,j ,k-1,i_S_z))
7064  + Real(0.5) * ( -intg(i-1,j ,k-1,i_S_x_y)
7065  +intg(i-1,j ,k-1,i_S_x_z)
7066  -intg(i-1,j ,k-1,i_S_y_z))
7067  + ( -intg(i-1,j ,k-1,i_S_xyz)))
7068  //
7069  + rhcc(i ,j-1,k-1) *
7070  ( Real(0.125) * vfrac(i ,j-1,k-1)
7071  + Real(0.25) * (-intg(i ,j-1,k-1,i_S_x)
7072  +intg(i ,j-1,k-1,i_S_y)
7073  +intg(i ,j-1,k-1,i_S_z))
7074  + Real(0.5) * ( -intg(i ,j-1,k-1,i_S_x_y)
7075  -intg(i ,j-1,k-1,i_S_x_z)
7076  +intg(i ,j-1,k-1,i_S_y_z))
7077  + ( -intg(i ,j-1,k-1,i_S_xyz)))
7078  //
7079  + rhcc(i-1,j-1,k-1) *
7080  ( Real(0.125) * vfrac(i-1,j-1,k-1)
7081  + Real(0.25) * ( intg(i-1,j-1,k-1,i_S_x)
7082  +intg(i-1,j-1,k-1,i_S_y)
7083  +intg(i-1,j-1,k-1,i_S_z))
7084  + Real(0.5) * ( intg(i-1,j-1,k-1,i_S_x_y)
7085  +intg(i-1,j-1,k-1,i_S_x_z)
7086  +intg(i-1,j-1,k-1,i_S_y_z))
7087  + ( intg(i-1,j-1,k-1,i_S_xyz)));
7088  } else {
7089  return Real(0.);
7090  }
7091 }
7092 
7093 #endif
7094 
7095 #if defined(AMREX_USE_HYPRE)
7096 
7097 template <typename HypreInt, typename AtomicInt>
7098 void mlndlap_fillijmat_sten_cpu (Box const& ndbx,
7099  Array4<AtomicInt const> const& gid,
7100  Array4<int const> const& lid,
7101  HypreInt* ncols, HypreInt* cols,
7102  Real* mat, // NOLINT(readability-non-const-parameter)
7103  Array4<Real const> const& sten) noexcept
7104 {
7105  constexpr int ist_000 = 1-1;
7106  constexpr int ist_p00 = 2-1;
7107  constexpr int ist_0p0 = 3-1;
7108  constexpr int ist_00p = 4-1;
7109  constexpr int ist_pp0 = 5-1;
7110  constexpr int ist_p0p = 6-1;
7111  constexpr int ist_0pp = 7-1;
7112  constexpr int ist_ppp = 8-1;
7113 
7114  constexpr auto gidmax = std::numeric_limits<AtomicInt>::max();
7115  HypreInt nelems = 0;
7116  amrex::LoopOnCpu(ndbx, [&] (int i, int j, int k) noexcept
7117  {
7118  if (lid(i,j,k) >= 0)
7119  {
7120  cols[nelems] = gid(i,j,k);
7121  mat[nelems] = sten(i,j,k,ist_000);
7122  HypreNodeLap::Int nelems_old = nelems;
7123  ++nelems;
7124 
7125  if (gid(i-1,j-1,k-1) < gidmax) {
7126  cols[nelems] = gid(i-1,j-1,k-1);
7127  mat[nelems] = sten(i-1,j-1,k-1,ist_ppp);
7128  ++nelems;
7129  }
7130 
7131  if (gid(i,j-1,k-1) < gidmax) {
7132  cols[nelems] = gid(i,j-1,k-1);
7133  mat[nelems] = sten(i,j-1,k-1,ist_0pp);
7134  ++nelems;
7135  }
7136 
7137  if (gid(i+1,j-1,k-1) < gidmax) {
7138  cols[nelems] = gid(i+1,j-1,k-1);
7139  mat[nelems] = sten(i,j-1,k-1,ist_ppp);
7140  ++nelems;
7141  }
7142 
7143  if (gid(i-1,j,k-1) < gidmax) {
7144  cols[nelems] = gid(i-1,j,k-1);
7145  mat[nelems] = sten(i-1,j,k-1,ist_p0p);
7146  ++nelems;
7147  }
7148 
7149  if (gid(i,j,k-1) < gidmax) {
7150  cols[nelems] = gid(i,j,k-1);
7151  mat[nelems] = sten(i,j,k-1,ist_00p);
7152  ++nelems;
7153  }
7154 
7155  if (gid(i+1,j,k-1) < gidmax) {
7156  cols[nelems] = gid(i+1,j,k-1);
7157  mat[nelems] = sten(i,j,k-1,ist_p0p);
7158  ++nelems;
7159  }
7160 
7161  if (gid(i-1,j+1,k-1) < gidmax) {
7162  cols[nelems] = gid(i-1,j+1,k-1);
7163  mat[nelems] = sten(i-1,j,k-1,ist_ppp);
7164  ++nelems;
7165  }
7166 
7167  if (gid(i,j+1,k-1) < gidmax) {
7168  cols[nelems] = gid(i,j+1,k-1);
7169  mat[nelems] = sten(i,j,k-1,ist_0pp);
7170  ++nelems;
7171  }
7172 
7173  if (gid(i+1,j+1,k-1) < gidmax) {
7174  cols[nelems] = gid(i+1,j+1,k-1);
7175  mat[nelems] = sten(i,j,k-1,ist_ppp);
7176  ++nelems;
7177  }
7178 
7179  if (gid(i-1,j-1,k) < gidmax) {
7180  cols[nelems] = gid(i-1,j-1,k);
7181  mat[nelems] = sten(i-1,j-1,k,ist_pp0);
7182  ++nelems;
7183  }
7184 
7185  if (gid(i,j-1,k) < gidmax) {
7186  cols[nelems] = gid(i,j-1,k);
7187  mat[nelems] = sten(i,j-1,k,ist_0p0);
7188  ++nelems;
7189  }
7190 
7191  if (gid(i+1,j-1,k) < gidmax) {
7192  cols[nelems] = gid(i+1,j-1,k);
7193  mat[nelems] = sten(i,j-1,k,ist_pp0);
7194  ++nelems;
7195  }
7196 
7197  if (gid(i-1,j,k) < gidmax) {
7198  cols[nelems] = gid(i-1,j,k);
7199  mat[nelems] = sten(i-1,j,k,ist_p00);
7200  ++nelems;
7201  }
7202 
7203  if (gid(i+1,j,k) < gidmax) {
7204  cols[nelems] = gid(i+1,j,k);
7205  mat[nelems] = sten(i,j,k,ist_p00);
7206  ++nelems;
7207  }
7208 
7209  if (gid(i-1,j+1,k) < gidmax) {
7210  cols[nelems] = gid(i-1,j+1,k);
7211  mat[nelems] = sten(i-1,j,k,ist_pp0);
7212  ++nelems;
7213  }
7214 
7215  if (gid(i,j+1,k) < gidmax) {
7216  cols[nelems] = gid(i,j+1,k);
7217  mat[nelems] = sten(i,j,k,ist_0p0);
7218  ++nelems;
7219  }
7220 
7221  if (gid(i+1,j+1,k) < gidmax) {
7222  cols[nelems] = gid(i+1,j+1,k);
7223  mat[nelems] = sten(i,j,k,ist_pp0);
7224  ++nelems;
7225  }
7226 
7227  if (gid(i-1,j-1,k+1) < gidmax) {
7228  cols[nelems] = gid(i-1,j-1,k+1);
7229  mat[nelems] = sten(i-1,j-1,k,ist_ppp);
7230  ++nelems;
7231  }
7232 
7233  if (gid(i,j-1,k+1) < gidmax) {
7234  cols[nelems] = gid(i,j-1,k+1);
7235  mat[nelems] = sten(i,j-1,k,ist_0pp);
7236  ++nelems;
7237  }
7238 
7239  if (gid(i+1,j-1,k+1) < gidmax) {
7240  cols[nelems] = gid(i+1,j-1,k+1);
7241  mat[nelems] = sten(i,j-1,k,ist_ppp);
7242  ++nelems;
7243  }
7244 
7245  if (gid(i-1,j,k+1) < gidmax) {
7246  cols[nelems] = gid(i-1,j,k+1);
7247  mat[nelems] = sten(i-1,j,k,ist_p0p);
7248  ++nelems;
7249  }
7250 
7251  if (gid(i,j,k+1) < gidmax) {
7252  cols[nelems] = gid(i,j,k+1);
7253  mat[nelems] = sten(i,j,k,ist_00p);
7254  ++nelems;
7255  }
7256 
7257  if (gid(i+1,j,k+1) < gidmax) {
7258  cols[nelems] = gid(i+1,j,k+1);
7259  mat[nelems] = sten(i,j,k,ist_p0p);
7260  ++nelems;
7261  }
7262 
7263  if (gid(i-1,j+1,k+1) < gidmax) {
7264  cols[nelems] = gid(i-1,j+1,k+1);
7265  mat[nelems] = sten(i-1,j,k,ist_ppp);
7266  ++nelems;
7267  }
7268 
7269  if (gid(i,j+1,k+1) < gidmax) {
7270  cols[nelems] = gid(i,j+1,k+1);
7271  mat[nelems] = sten(i,j,k,ist_0pp);
7272  ++nelems;
7273  }
7274 
7275  if (gid(i+1,j+1,k+1) < gidmax) {
7276  cols[nelems] = gid(i+1,j+1,k+1);
7277  mat[nelems] = sten(i,j,k,ist_ppp);
7278  ++nelems;
7279  }
7280 
7281  ncols[lid(i,j,k)] = nelems - nelems_old;
7282  }
7283  });
7284 }
7285 
7286 template <typename HypreInt, typename AtomicInt>
7287 void mlndlap_fillijmat_aa_cpu (Box const& ndbx,
7288  Array4<AtomicInt const> const& gid,
7289  Array4<int const> const& lid,
7290  HypreInt* ncols, HypreInt* cols,
7291  Real* mat, // NOLINT(readability-non-const-parameter)
7292  Array4<Real const> const& sig,
7293  GpuArray<Real,AMREX_SPACEDIM> const& dxinv,
7294  Box const& ccdom) noexcept
7295 {
7296  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0];
7297  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1];
7298  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2];
7299  Real fxyz = facx + facy + facz;
7300  Real fmx2y2z = -facx + Real(2.0)*facy + Real(2.0)*facz;
7301  Real f2xmy2z = Real(2.0)*facx - facy + Real(2.0)*facz;
7302  Real f2x2ymz = Real(2.0)*facx + Real(2.0)*facy - facz;
7303  Real f4xm2ym2z = Real(4.0)*facx - Real(2.0)*facy - Real(2.0)*facz;
7304  Real fm2x4ym2z = -Real(2.0)*facx + Real(4.0)*facy - Real(2.0)*facz;
7305  Real fm2xm2y4z = -Real(2.0)*facx - Real(2.0)*facy + Real(4.0)*facz;
7306 
7307  const Box& nddom = amrex::surroundingNodes(ccdom);
7308 
7309  constexpr auto gidmax = std::numeric_limits<AtomicInt>::max();
7310  HypreInt nelems = 0;
7311  amrex::LoopOnCpu(ndbx, [&] (int i, int j, int k) noexcept
7312  {
7313  if (lid(i,j,k) >= 0)
7314  {
7315  HypreInt nelems_old = nelems;
7316  cols[nelems_old] = gid(i,j,k);
7317  Real m0 = Real(0.);
7318  ++nelems;
7319 
7320  if (nddom.contains(i-1,j-1,k-1)) {
7321  Real tmp = sig(i-1,j-1,k-1) * fxyz;
7322  m0 -= tmp;
7323  if ( gid(i-1,j-1,k-1) < gidmax) {
7324  cols[nelems] = gid(i-1,j-1,k-1);
7325  mat[nelems] = tmp;
7326  ++nelems;
7327  }
7328  }
7329 
7330  if (nddom.contains(i,j-1,k-1)) {
7331  Real tmp = Real(0.);
7332  if (ccdom.contains(i-1,j-1,k-1)) {
7333  tmp += sig(i-1,j-1,k-1) * fmx2y2z;
7334  }
7335  if (ccdom.contains(i,j-1,k-1)) {
7336  tmp += sig(i,j-1,k-1) * fmx2y2z;
7337  }
7338  m0 -= tmp;
7339  if (gid(i,j-1,k-1) < gidmax) {
7340  cols[nelems] = gid(i,j-1,k-1);
7341  mat[nelems] = tmp;
7342  ++nelems;
7343  }
7344  }
7345 
7346  if (nddom.contains(i+1,j-1,k-1)) {
7347  Real tmp = sig(i ,j-1,k-1) * fxyz;
7348  m0 -= tmp;
7349  if (gid(i+1,j-1,k-1) < gidmax) {
7350  cols[nelems] = gid(i+1,j-1,k-1);
7351  mat[nelems] = tmp;
7352  ++nelems;
7353  }
7354  }
7355 
7356  if (nddom.contains(i-1,j,k-1)) {
7357  Real tmp = Real(0.);
7358  if (ccdom.contains(i-1,j-1,k-1)) {
7359  tmp += sig(i-1,j-1,k-1) * f2xmy2z;
7360  }
7361  if (ccdom.contains(i-1,j,k-1)) {
7362  tmp += sig(i-1,j,k-1) * f2xmy2z;
7363  }
7364  m0 -= tmp;
7365  if (gid(i-1,j,k-1) < gidmax) {
7366  cols[nelems] = gid(i-1,j,k-1);
7367  mat[nelems] = tmp;
7368  ++nelems;
7369  }
7370  }
7371 
7372  if (nddom.contains(i,j,k-1) && fm2xm2y4z != Real(0.0)) {
7373  Real tmp = Real(0.);
7374  if (ccdom.contains(i-1,j-1,k-1)) {
7375  tmp += sig(i-1,j-1,k-1) * fm2xm2y4z;
7376  }
7377  if (ccdom.contains(i,j-1,k-1)) {
7378  tmp += sig(i,j-1,k-1) * fm2xm2y4z;
7379  }
7380  if (ccdom.contains(i-1,j,k-1)) {
7381  tmp += sig(i-1,j,k-1) * fm2xm2y4z;
7382  }
7383  if (ccdom.contains(i,j,k-1)) {
7384  tmp += sig(i,j,k-1) * fm2xm2y4z;
7385  }
7386  m0 -= tmp;
7387  if (gid(i,j,k-1) < gidmax) {
7388  cols[nelems] = gid(i,j,k-1);
7389  mat[nelems] = tmp;
7390  ++nelems;
7391  }
7392  }
7393 
7394  if (nddom.contains(i+1,j,k-1)) {
7395  Real tmp = Real(0.);
7396  if (ccdom.contains(i ,j-1,k-1)) {
7397  tmp += sig(i ,j-1,k-1) * f2xmy2z;
7398  }
7399  if (ccdom.contains(i ,j,k-1)) {
7400  tmp += sig(i ,j,k-1) * f2xmy2z;
7401  }
7402  m0 -= tmp;
7403  if (gid(i+1,j,k-1) < gidmax) {
7404  cols[nelems] = gid(i+1,j,k-1);
7405  mat[nelems] = tmp;
7406  ++nelems;
7407  }
7408  }
7409 
7410  if (nddom.contains(i-1,j+1,k-1)) {
7411  Real tmp = sig(i-1,j ,k-1) * fxyz;
7412  m0 -= tmp;
7413  if (gid(i-1,j+1,k-1) < gidmax) {
7414  cols[nelems] = gid(i-1,j+1,k-1);
7415  mat[nelems] = tmp;
7416  ++nelems;
7417  }
7418  }
7419 
7420  if (nddom.contains(i,j+1,k-1)) {
7421  Real tmp = Real(0.);
7422  if (ccdom.contains(i-1,j ,k-1)) {
7423  tmp += sig(i-1,j ,k-1) * fmx2y2z;
7424  }
7425  if (ccdom.contains(i,j ,k-1)) {
7426  tmp += sig(i,j ,k-1) * fmx2y2z;
7427  }
7428  m0 -= tmp;
7429  if (gid(i,j+1,k-1) < gidmax) {
7430  cols[nelems] = gid(i,j+1,k-1);
7431  mat[nelems] = tmp;
7432  ++nelems;
7433  }
7434  }
7435 
7436  if (nddom.contains(i+1,j+1,k-1)) {
7437  Real tmp = sig(i ,j ,k-1) * fxyz;
7438  m0 -= tmp;
7439  if (gid(i+1,j+1,k-1) < gidmax) {
7440  cols[nelems] = gid(i+1,j+1,k-1);
7441  mat[nelems] = tmp;
7442  ++nelems;
7443  }
7444  }
7445 
7446  if (nddom.contains(i-1,j-1,k)) {
7447  Real tmp = Real(0.);
7448  if (ccdom.contains(i-1,j-1,k-1)) {
7449  tmp += sig(i-1,j-1,k-1) * f2x2ymz;
7450  }
7451  if (ccdom.contains(i-1,j-1,k)) {
7452  tmp += sig(i-1,j-1,k) * f2x2ymz;
7453  }
7454  m0 -= tmp;
7455  if (gid(i-1,j-1,k) < gidmax) {
7456  cols[nelems] = gid(i-1,j-1,k);
7457  mat[nelems] = tmp;
7458  ++nelems;
7459  }
7460  }
7461 
7462  if (nddom.contains(i,j-1,k) && fm2x4ym2z != Real(0.0)) {
7463  Real tmp = Real(0.);
7464  if (ccdom.contains(i-1,j-1,k-1)) {
7465  tmp += sig(i-1,j-1,k-1) * fm2x4ym2z;
7466  }
7467  if (ccdom.contains(i,j-1,k-1)) {
7468  tmp += sig(i,j-1,k-1) * fm2x4ym2z;
7469  }
7470  if (ccdom.contains(i-1,j-1,k)) {
7471  tmp += sig(i-1,j-1,k) * fm2x4ym2z;
7472  }
7473  if (ccdom.contains(i,j-1,k)) {
7474  tmp += sig(i,j-1,k) * fm2x4ym2z;
7475  }
7476  m0 -= tmp;
7477  if (gid(i,j-1,k) < gidmax) {
7478  cols[nelems] = gid(i,j-1,k);
7479  mat[nelems] = tmp;
7480  ++nelems;
7481  }
7482  }
7483 
7484  if (nddom.contains(i+1,j-1,k)) {
7485  Real tmp = Real(0.);
7486  if (ccdom.contains(i ,j-1,k-1)) {
7487  tmp += sig(i ,j-1,k-1) * f2x2ymz;
7488  }
7489  if (ccdom.contains(i ,j-1,k)) {
7490  tmp += sig(i ,j-1,k) * f2x2ymz;
7491  }
7492  m0 -= tmp;
7493  if (gid(i+1,j-1,k) < gidmax) {
7494  cols[nelems] = gid(i+1,j-1,k);
7495  mat[nelems] = tmp;
7496  ++nelems;
7497  }
7498  }
7499 
7500  if (nddom.contains(i-1,j,k) && f4xm2ym2z != Real(0.0)) {
7501  Real tmp = Real(0.);
7502  if (ccdom.contains(i-1,j-1,k-1)) {
7503  tmp += sig(i-1,j-1,k-1) * f4xm2ym2z;
7504  }
7505  if (ccdom.contains(i-1,j,k-1)) {
7506  tmp += sig(i-1,j,k-1) * f4xm2ym2z;
7507  }
7508  if (ccdom.contains(i-1,j-1,k)) {
7509  tmp += sig(i-1,j-1,k) * f4xm2ym2z;
7510  }
7511  if (ccdom.contains(i-1,j,k)) {
7512  tmp += sig(i-1,j,k) * f4xm2ym2z;
7513  }
7514  m0 -= tmp;
7515  if (gid(i-1,j,k) < gidmax) {
7516  cols[nelems] = gid(i-1,j,k);
7517  mat[nelems] = tmp;
7518  ++nelems;
7519  }
7520  }
7521 
7522  if (nddom.contains(i+1,j,k) && f4xm2ym2z != Real(0.0)) {
7523  Real tmp = Real(0.);
7524  if (ccdom.contains(i ,j-1,k-1)) {
7525  tmp += sig(i ,j-1,k-1) * f4xm2ym2z;
7526  }
7527  if (ccdom.contains(i ,j,k-1)) {
7528  tmp += sig(i ,j,k-1) * f4xm2ym2z;
7529  }
7530  if (ccdom.contains(i ,j-1,k)) {
7531  tmp += sig(i ,j-1,k) * f4xm2ym2z;
7532  }
7533  if (ccdom.contains(i ,j,k)) {
7534  tmp += sig(i ,j,k) * f4xm2ym2z;
7535  }
7536  m0 -= tmp;
7537  if (gid(i+1,j,k) < gidmax) {
7538  cols[nelems] = gid(i+1,j,k);
7539  mat[nelems] = tmp;
7540  ++nelems;
7541  }
7542  }
7543 
7544  if (nddom.contains(i-1,j+1,k)) {
7545  Real tmp = Real(0.);
7546  if (ccdom.contains(i-1,j ,k-1)) {
7547  tmp += sig(i-1,j ,k-1) * f2x2ymz;
7548  }
7549  if (ccdom.contains(i-1,j ,k)) {
7550  tmp += sig(i-1,j ,k) * f2x2ymz;
7551  }
7552  m0 -= tmp;
7553  if (gid(i-1,j+1,k) < gidmax) {
7554  cols[nelems] = gid(i-1,j+1,k);
7555  mat[nelems] = tmp;
7556  ++nelems;
7557  }
7558  }
7559 
7560  if (nddom.contains(i,j+1,k) && fm2x4ym2z != Real(0.0)) {
7561  Real tmp = Real(0.);
7562  if (ccdom.contains(i-1,j ,k-1)) {
7563  tmp += sig(i-1,j ,k-1) * fm2x4ym2z;
7564  }
7565  if (ccdom.contains(i,j ,k-1)) {
7566  tmp += sig(i,j ,k-1) * fm2x4ym2z;
7567  }
7568  if (ccdom.contains(i-1,j ,k)) {
7569  tmp += sig(i-1,j ,k) * fm2x4ym2z;
7570  }
7571  if (ccdom.contains(i,j ,k)) {
7572  tmp += sig(i,j ,k) * fm2x4ym2z;
7573  }
7574  m0 -= tmp;
7575  if (gid(i,j+1,k) < gidmax) {
7576  cols[nelems] = gid(i,j+1,k);
7577  mat[nelems] = tmp;
7578  ++nelems;
7579  }
7580  }
7581 
7582  if (nddom.contains(i+1,j+1,k)) {
7583  Real tmp = Real(0.);
7584  if (ccdom.contains(i ,j ,k-1)) {
7585  tmp += sig(i ,j ,k-1) * f2x2ymz;
7586  }
7587  if (ccdom.contains(i ,j ,k)) {
7588  tmp += sig(i ,j ,k) * f2x2ymz;
7589  }
7590  m0 -= tmp;
7591  if (gid(i+1,j+1,k) < gidmax) {
7592  cols[nelems] = gid(i+1,j+1,k);
7593  mat[nelems] = tmp;
7594  ++nelems;
7595  }
7596  }
7597 
7598  if (nddom.contains(i-1,j-1,k+1)) {
7599  Real tmp = sig(i-1,j-1,k ) * fxyz;
7600  m0 -= tmp;
7601  if (gid(i-1,j-1,k+1) < gidmax) {
7602  cols[nelems] = gid(i-1,j-1,k+1);
7603  mat[nelems] = tmp;
7604  ++nelems;
7605  }
7606  }
7607 
7608  if (nddom.contains(i,j-1,k+1)) {
7609  Real tmp = Real(0.);
7610  if (ccdom.contains(i-1,j-1,k )) {
7611  tmp += sig(i-1,j-1,k ) * fmx2y2z;
7612  }
7613  if (ccdom.contains(i,j-1,k )) {
7614  tmp += sig(i,j-1,k ) * fmx2y2z;
7615  }
7616  m0 -= tmp;
7617  if (gid(i,j-1,k+1) < gidmax) {
7618  cols[nelems] = gid(i,j-1,k+1);
7619  mat[nelems] = tmp;
7620  ++nelems;
7621  }
7622  }
7623 
7624  if (nddom.contains(i+1,j-1,k+1)) {
7625  Real tmp = sig(i ,j-1,k ) * fxyz;
7626  m0 -= tmp;
7627  if (gid(i+1,j-1,k+1) < gidmax) {
7628  cols[nelems] = gid(i+1,j-1,k+1);
7629  mat[nelems] = tmp;
7630  ++nelems;
7631  }
7632  }
7633 
7634  if (nddom.contains(i-1,j,k+1)) {
7635  Real tmp = Real(0.);
7636  if (ccdom.contains(i-1,j-1,k )) {
7637  tmp += sig(i-1,j-1,k ) * f2xmy2z;
7638  }
7639  if (ccdom.contains(i-1,j,k )) {
7640  tmp += sig(i-1,j,k ) * f2xmy2z;
7641  }
7642  m0 -= tmp;
7643  if (gid(i-1,j,k+1) < gidmax) {
7644  cols[nelems] = gid(i-1,j,k+1);
7645  mat[nelems] = tmp;
7646  ++nelems;
7647  }
7648  }
7649 
7650  if (nddom.contains(i,j,k+1) && fm2xm2y4z != Real(0.0)) {
7651  Real tmp = Real(0.);
7652  if (ccdom.contains(i-1,j-1,k )) {
7653  tmp += sig(i-1,j-1,k ) * fm2xm2y4z;
7654  }
7655  if (ccdom.contains(i,j-1,k )) {
7656  tmp += sig(i,j-1,k ) * fm2xm2y4z;
7657  }
7658  if (ccdom.contains(i-1,j,k )) {
7659  tmp += sig(i-1,j,k ) * fm2xm2y4z;
7660  }
7661  if (ccdom.contains(i,j,k )) {
7662  tmp += sig(i,j,k ) * fm2xm2y4z;
7663  }
7664  m0 -= tmp;
7665  if (gid(i,j,k+1) < gidmax) {
7666  cols[nelems] = gid(i,j,k+1);
7667  mat[nelems] = tmp;
7668  ++nelems;
7669  }
7670  }
7671 
7672  if (nddom.contains(i+1,j,k+1)) {
7673  Real tmp = Real(0.);
7674  if (ccdom.contains(i ,j-1,k )) {
7675  tmp += sig(i ,j-1,k ) * f2xmy2z;
7676  }
7677  if (ccdom.contains(i ,j,k )) {
7678  tmp += sig(i ,j,k ) * f2xmy2z;
7679  }
7680  m0 -= tmp;
7681  if (gid(i+1,j,k+1) < gidmax) {
7682  cols[nelems] = gid(i+1,j,k+1);
7683  mat[nelems] = tmp;
7684  ++nelems;
7685  }
7686  }
7687 
7688  if (nddom.contains(i-1,j+1,k+1)) {
7689  Real tmp = sig(i-1,j ,k ) * fxyz;
7690  m0 -= tmp;
7691  if (gid(i-1,j+1,k+1) < gidmax) {
7692  cols[nelems] = gid(i-1,j+1,k+1);
7693  mat[nelems] = tmp;
7694  ++nelems;
7695  }
7696  }
7697 
7698  if (nddom.contains(i,j+1,k+1)) {
7699  Real tmp = Real(0.);
7700  if (ccdom.contains(i-1,j ,k )) {
7701  tmp += sig(i-1,j ,k ) * fmx2y2z;
7702  }
7703  if (ccdom.contains(i,j ,k )) {
7704  tmp += sig(i,j ,k ) * fmx2y2z;
7705  }
7706  m0 -= tmp;
7707  if (gid(i,j+1,k+1) < gidmax) {
7708  cols[nelems] = gid(i,j+1,k+1);
7709  mat[nelems] = tmp;
7710  ++nelems;
7711  }
7712  }
7713 
7714  if (nddom.contains(i+1,j+1,k+1)) {
7715  Real tmp = sig(i ,j ,k ) * fxyz;
7716  m0 -= tmp;
7717  if (gid(i+1,j+1,k+1) < gidmax) {
7718  cols[nelems] = gid(i+1,j+1,k+1);
7719  mat[nelems] = tmp;
7720  ++nelems;
7721  }
7722  }
7723 
7724  mat[nelems_old] = m0;
7725  ncols[lid(i,j,k)] = nelems - nelems_old;
7726  }
7727  });
7728 }
7729 
7730 template <typename HypreInt, typename AtomicInt>
7731 void mlndlap_fillijmat_ha_cpu (Box const& ndbx,
7732  Array4<AtomicInt const> const& gid,
7733  Array4<int const> const& lid,
7734  HypreInt* ncols, HypreInt* cols,
7735  Real* mat, // NOLINT(readability-non-const-parameter)
7736  Array4<Real const> const& sx,
7737  Array4<Real const> const& sy,
7738  Array4<Real const> const& sz,
7739  GpuArray<Real,AMREX_SPACEDIM> const& dxinv,
7740  Box const& ccdom) noexcept
7741 {
7742  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0];
7743  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1];
7744  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2];
7745 
7746  const Box& nddom = amrex::surroundingNodes(ccdom);
7747 
7748  constexpr auto gidmax = std::numeric_limits<AtomicInt>::max();
7749  HypreInt nelems = 0;
7750  amrex::LoopOnCpu(ndbx, [&] (int i, int j, int k) noexcept
7751  {
7752  if (lid(i,j,k) >= 0)
7753  {
7754  HypreInt nelems_old = nelems;
7755  cols[nelems_old] = gid(i,j,k);
7756  Real m0 = Real(0.);
7757  ++nelems;
7758 
7759  if (nddom.contains(i-1,j-1,k-1)) {
7760  Real tmp = sx(i-1,j-1,k-1) * facx
7761  + sy(i-1,j-1,k-1) * facy
7762  + sz(i-1,j-1,k-1) * facz;
7763  m0 -= tmp;
7764  if ( gid(i-1,j-1,k-1) < gidmax) {
7765  cols[nelems] = gid(i-1,j-1,k-1);
7766  mat[nelems] = tmp;
7767  ++nelems;
7768  }
7769  }
7770 
7771  if (nddom.contains(i,j-1,k-1)) {
7772  Real tmp = Real(0.);
7773  if (ccdom.contains(i-1,j-1,k-1)) {
7774  tmp += - sx(i-1,j-1,k-1) * facx
7775  + sy(i-1,j-1,k-1) * facy * Real(2.0)
7776  + sz(i-1,j-1,k-1) * facz * Real(2.0);
7777  }
7778  if (ccdom.contains(i,j-1,k-1)) {
7779  tmp += - sx(i,j-1,k-1) * facx
7780  + sy(i,j-1,k-1) * facy * Real(2.0)
7781  + sz(i,j-1,k-1) * facz * Real(2.0);
7782  }
7783  m0 -= tmp;
7784  if (gid(i,j-1,k-1) < gidmax) {
7785  cols[nelems] = gid(i,j-1,k-1);
7786  mat[nelems] = tmp;
7787  ++nelems;
7788  }
7789  }
7790 
7791  if (nddom.contains(i+1,j-1,k-1)) {
7792  Real tmp = sx(i ,j-1,k-1) * facx
7793  + sy(i ,j-1,k-1) * facy
7794  + sz(i ,j-1,k-1) * facz;
7795  m0 -= tmp;
7796  if (gid(i+1,j-1,k-1) < gidmax) {
7797  cols[nelems] = gid(i+1,j-1,k-1);
7798  mat[nelems] = tmp;
7799  ++nelems;
7800  }
7801  }
7802 
7803  if (nddom.contains(i-1,j,k-1)) {
7804  Real tmp = Real(0.);
7805  if (ccdom.contains(i-1,j-1,k-1)) {
7806  tmp += sx(i-1,j-1,k-1) * facx * Real(2.0)
7807  - sy(i-1,j-1,k-1) * facy
7808  + sz(i-1,j-1,k-1) * facz * Real(2.0);
7809  }
7810  if (ccdom.contains(i-1,j,k-1)) {
7811  tmp += sx(i-1,j,k-1) * facx * Real(2.0)
7812  - sy(i-1,j,k-1) * facy
7813  + sz(i-1,j,k-1) * facz * Real(2.0);
7814  }
7815  m0 -= tmp;
7816  if (gid(i-1,j,k-1) < gidmax) {
7817  cols[nelems] = gid(i-1,j,k-1);
7818  mat[nelems] = tmp;
7819  ++nelems;
7820  }
7821  }
7822 
7823  if (nddom.contains(i,j,k-1)) {
7824  Real tmp = Real(0.);
7825  if (ccdom.contains(i-1,j-1,k-1)) {
7826  tmp += - sx(i-1,j-1,k-1) * facx * Real(2.0)
7827  - sy(i-1,j-1,k-1) * facy * Real(2.0)
7828  + sz(i-1,j-1,k-1) * facz * Real(4.0);
7829  }
7830  if (ccdom.contains(i,j-1,k-1)) {
7831  tmp += - sx(i,j-1,k-1) * facx * Real(2.0)
7832  - sy(i,j-1,k-1) * facy * Real(2.0)
7833  + sz(i,j-1,k-1) * facz * Real(4.0);
7834 
7835  }
7836  if (ccdom.contains(i-1,j,k-1)) {
7837  tmp += - sx(i-1,j,k-1) * facx * Real(2.0)
7838  - sy(i-1,j,k-1) * facy * Real(2.0)
7839  + sz(i-1,j,k-1) * facz * Real(4.0);
7840  }
7841  if (ccdom.contains(i,j,k-1)) {
7842  tmp += - sx(i,j,k-1) * facx * Real(2.0)
7843  - sy(i,j,k-1) * facy * Real(2.0)
7844  + sz(i,j,k-1) * facz * Real(4.0);
7845  }
7846  m0 -= tmp;
7847  if (gid(i,j,k-1) < gidmax && tmp != Real(0.0)) {
7848  cols[nelems] = gid(i,j,k-1);
7849  mat[nelems] = tmp;
7850  ++nelems;
7851  }
7852  }
7853 
7854  if (nddom.contains(i+1,j,k-1)) {
7855  Real tmp = Real(0.);
7856  if (ccdom.contains(i ,j-1,k-1)) {
7857  tmp += sx(i ,j-1,k-1) * facx * Real(2.0)
7858  - sy(i ,j-1,k-1) * facy
7859  + sz(i ,j-1,k-1) * facz * Real(2.0);
7860  }
7861  if (ccdom.contains(i ,j,k-1)) {
7862  tmp += sx(i ,j,k-1) * facx * Real(2.0)
7863  - sy(i ,j,k-1) * facy
7864  + sz(i ,j,k-1) * facz * Real(2.0);
7865  }
7866  m0 -= tmp;
7867  if (gid(i+1,j,k-1) < gidmax) {
7868  cols[nelems] = gid(i+1,j,k-1);
7869  mat[nelems] = tmp;
7870  ++nelems;
7871  }
7872  }
7873 
7874  if (nddom.contains(i-1,j+1,k-1)) {
7875  Real tmp = sx(i-1,j ,k-1) * facx
7876  + sy(i-1,j ,k-1) * facy
7877  + sz(i-1,j ,k-1) * facz;
7878  m0 -= tmp;
7879  if (gid(i-1,j+1,k-1) < gidmax) {
7880  cols[nelems] = gid(i-1,j+1,k-1);
7881  mat[nelems] = tmp;
7882  ++nelems;
7883  }
7884  }
7885 
7886  if (nddom.contains(i,j+1,k-1)) {
7887  Real tmp = Real(0.);
7888  if (ccdom.contains(i-1,j ,k-1)) {
7889  tmp += - sx(i-1,j ,k-1) * facx
7890  + sy(i-1,j ,k-1) * facy * Real(2.0)
7891  + sz(i-1,j ,k-1) * facz * Real(2.0);
7892  }
7893  if (ccdom.contains(i,j ,k-1)) {
7894  tmp += - sx(i,j ,k-1) * facx
7895  + sy(i,j ,k-1) * facy * Real(2.0)
7896  + sz(i,j ,k-1) * facz * Real(2.0);
7897  }
7898  m0 -= tmp;
7899  if (gid(i,j+1,k-1) < gidmax) {
7900  cols[nelems] = gid(i,j+1,k-1);
7901  mat[nelems] = tmp;
7902  ++nelems;
7903  }
7904  }
7905 
7906  if (nddom.contains(i+1,j+1,k-1)) {
7907  Real tmp = sx(i ,j ,k-1) * facx
7908  + sy(i ,j ,k-1) * facy
7909  + sz(i ,j ,k-1) * facz;
7910  m0 -= tmp;
7911  if (gid(i+1,j+1,k-1) < gidmax) {
7912  cols[nelems] = gid(i+1,j+1,k-1);
7913  mat[nelems] = tmp;
7914  ++nelems;
7915  }
7916  }
7917 
7918  if (nddom.contains(i-1,j-1,k)) {
7919  Real tmp = Real(0.);
7920  if (ccdom.contains(i-1,j-1,k-1)) {
7921  tmp += sx(i-1,j-1,k-1) * facx * Real(2.0)
7922  + sy(i-1,j-1,k-1) * facy * Real(2.0)
7923  - sz(i-1,j-1,k-1) * facz;
7924  }
7925  if (ccdom.contains(i-1,j-1,k)) {
7926  tmp += sx(i-1,j-1,k) * facx * Real(2.0)
7927  + sy(i-1,j-1,k) * facy * Real(2.0)
7928  - sz(i-1,j-1,k) * facz;
7929  }
7930  m0 -= tmp;
7931  if (gid(i-1,j-1,k) < gidmax) {
7932  cols[nelems] = gid(i-1,j-1,k);
7933  mat[nelems] = tmp;
7934  ++nelems;
7935  }
7936  }
7937 
7938  if (nddom.contains(i,j-1,k)) {
7939  Real tmp = Real(0.);
7940  if (ccdom.contains(i-1,j-1,k-1)) {
7941  tmp += - sx(i-1,j-1,k-1) * facx * Real(2.0)
7942  + sy(i-1,j-1,k-1) * facy * Real(4.0)
7943  - sz(i-1,j-1,k-1) * facz * Real(2.0);
7944  }
7945  if (ccdom.contains(i,j-1,k-1)) {
7946  tmp += - sx(i,j-1,k-1) * facx * Real(2.0)
7947  + sy(i,j-1,k-1) * facy * Real(4.0)
7948  - sz(i,j-1,k-1) * facz * Real(2.0);
7949  }
7950  if (ccdom.contains(i-1,j-1,k)) {
7951  tmp += - sx(i-1,j-1,k) * facx * Real(2.0)
7952  + sy(i-1,j-1,k) * facy * Real(4.0)
7953  - sz(i-1,j-1,k) * facz * Real(2.0);
7954  }
7955  if (ccdom.contains(i,j-1,k)) {
7956  tmp += - sx(i,j-1,k) * facx * Real(2.0)
7957  + sy(i,j-1,k) * facy * Real(4.0)
7958  - sz(i,j-1,k) * facz * Real(2.0);
7959  }
7960  m0 -= tmp;
7961  if (gid(i,j-1,k) < gidmax && tmp != Real(0.0)) {
7962  cols[nelems] = gid(i,j-1,k);
7963  mat[nelems] = tmp;
7964  ++nelems;
7965  }
7966  }
7967 
7968  if (nddom.contains(i+1,j-1,k)) {
7969  Real tmp = Real(0.);
7970  if (ccdom.contains(i ,j-1,k-1)) {
7971  tmp += sx(i ,j-1,k-1) * facx * Real(2.0)
7972  + sy(i ,j-1,k-1) * facy * Real(2.0)
7973  - sz(i ,j-1,k-1) * facz;
7974  }
7975  if (ccdom.contains(i ,j-1,k)) {
7976  tmp += sx(i ,j-1,k) * facx * Real(2.0)
7977  + sy(i ,j-1,k) * facy * Real(2.0)
7978  - sz(i ,j-1,k) * facz;
7979  }
7980  m0 -= tmp;
7981  if (gid(i+1,j-1,k) < gidmax) {
7982  cols[nelems] = gid(i+1,j-1,k);
7983  mat[nelems] = tmp;
7984  ++nelems;
7985  }
7986  }
7987 
7988  if (nddom.contains(i-1,j,k)) {
7989  Real tmp = Real(0.);
7990  if (ccdom.contains(i-1,j-1,k-1)) {
7991  tmp += sx(i-1,j-1,k-1) * facx * Real(4.0)
7992  - sy(i-1,j-1,k-1) * facy * Real(2.0)
7993  - sz(i-1,j-1,k-1) * facz * Real(2.0);
7994  }
7995  if (ccdom.contains(i-1,j,k-1)) {
7996  tmp += sx(i-1,j,k-1) * facx * Real(4.0)
7997  - sy(i-1,j,k-1) * facy * Real(2.0)
7998  - sz(i-1,j,k-1) * facz * Real(2.0);
7999  }
8000  if (ccdom.contains(i-1,j-1,k)) {
8001  tmp += sx(i-1,j-1,k) * facx * Real(4.0)
8002  - sy(i-1,j-1,k) * facy * Real(2.0)
8003  - sz(i-1,j-1,k) * facz * Real(2.0);
8004  }
8005  if (ccdom.contains(i-1,j,k)) {
8006  tmp += sx(i-1,j,k) * facx * Real(4.0)
8007  - sy(i-1,j,k) * facy * Real(2.0)
8008  - sz(i-1,j,k) * facz * Real(2.0);
8009  }
8010  m0 -= tmp;
8011  if (gid(i-1,j,k) < gidmax && tmp != Real(0.0)) {
8012  cols[nelems] = gid(i-1,j,k);
8013  mat[nelems] = tmp;
8014  ++nelems;
8015  }
8016  }
8017 
8018  if (nddom.contains(i+1,j,k)) {
8019  Real tmp = Real(0.);
8020  if (ccdom.contains(i ,j-1,k-1)) {
8021  tmp += sx(i ,j-1,k-1) * facx * Real(4.0)
8022  - sy(i ,j-1,k-1) * facy * Real(2.0)
8023  - sz(i ,j-1,k-1) * facz * Real(2.0);
8024  }
8025  if (ccdom.contains(i ,j,k-1)) {
8026  tmp += sx(i ,j,k-1) * facx * Real(4.0)
8027  - sy(i ,j,k-1) * facy * Real(2.0)
8028  - sz(i ,j,k-1) * facz * Real(2.0);
8029  }
8030  if (ccdom.contains(i ,j-1,k)) {
8031  tmp += sx(i ,j-1,k) * facx * Real(4.0)
8032  - sy(i ,j-1,k) * facy * Real(2.0)
8033  - sz(i ,j-1,k) * facz * Real(2.0);
8034  }
8035  if (ccdom.contains(i ,j,k)) {
8036  tmp += sx(i ,j,k) * facx * Real(4.0)
8037  - sy(i ,j,k) * facy * Real(2.0)
8038  - sz(i ,j,k) * facz * Real(2.0);
8039  }
8040  m0 -= tmp;
8041  if (gid(i+1,j,k) < gidmax && tmp != Real(0.0)) {
8042  cols[nelems] = gid(i+1,j,k);
8043  mat[nelems] = tmp;
8044  ++nelems;
8045  }
8046  }
8047 
8048  if (nddom.contains(i-1,j+1,k)) {
8049  Real tmp = Real(0.);
8050  if (ccdom.contains(i-1,j ,k-1)) {
8051  tmp += sx(i-1,j ,k-1) * facx * Real(2.0)
8052  + sy(i-1,j ,k-1) * facy * Real(2.0)
8053  - sz(i-1,j ,k-1) * facz;
8054  }
8055  if (ccdom.contains(i-1,j ,k)) {
8056  tmp += sx(i-1,j ,k) * facx * Real(2.0)
8057  + sy(i-1,j ,k) * facy * Real(2.0)
8058  - sz(i-1,j ,k) * facz;
8059  }
8060  m0 -= tmp;
8061  if (gid(i-1,j+1,k) < gidmax) {
8062  cols[nelems] = gid(i-1,j+1,k);
8063  mat[nelems] = tmp;
8064  ++nelems;
8065  }
8066  }
8067 
8068  if (nddom.contains(i,j+1,k)) {
8069  Real tmp = Real(0.);
8070  if (ccdom.contains(i-1,j ,k-1)) {
8071  tmp += - sx(i-1,j ,k-1) * facx * Real(2.0)
8072  + sy(i-1,j ,k-1) * facy * Real(4.0)
8073  - sz(i-1,j ,k-1) * facz * Real(2.0);
8074  }
8075  if (ccdom.contains(i,j ,k-1)) {
8076  tmp += - sx(i,j ,k-1) * facx * Real(2.0)
8077  + sy(i,j ,k-1) * facy * Real(4.0)
8078  - sz(i,j ,k-1) * facz * Real(2.0);
8079  }
8080  if (ccdom.contains(i-1,j ,k)) {
8081  tmp += - sx(i-1,j ,k) * facx * Real(2.0)
8082  + sy(i-1,j ,k) * facy * Real(4.0)
8083  - sz(i-1,j ,k) * facz * Real(2.0);
8084  }
8085  if (ccdom.contains(i,j ,k)) {
8086  tmp += - sx(i,j ,k) * facx * Real(2.0)
8087  + sy(i,j ,k) * facy * Real(4.0)
8088  - sz(i,j ,k) * facz * Real(2.0);
8089  }
8090  m0 -= tmp;
8091  if (gid(i,j+1,k) < gidmax && tmp != Real(0.0)) {
8092  cols[nelems] = gid(i,j+1,k);
8093  mat[nelems] = tmp;
8094  ++nelems;
8095  }
8096  }
8097 
8098  if (nddom.contains(i+1,j+1,k)) {
8099  Real tmp = Real(0.);
8100  if (ccdom.contains(i ,j ,k-1)) {
8101  tmp += sx(i ,j ,k-1) * facx * Real(2.0)
8102  + sy(i ,j ,k-1) * facy * Real(2.0)
8103  - sz(i ,j ,k-1) * facz;
8104  }
8105  if (ccdom.contains(i ,j ,k)) {
8106  tmp += sx(i ,j ,k) * facx * Real(2.0)
8107  + sy(i ,j ,k) * facy * Real(2.0)
8108  - sz(i ,j ,k) * facz;
8109  }
8110  m0 -= tmp;
8111  if (gid(i+1,j+1,k) < gidmax) {
8112  cols[nelems] = gid(i+1,j+1,k);
8113  mat[nelems] = tmp;
8114  ++nelems;
8115  }
8116  }
8117 
8118  if (nddom.contains(i-1,j-1,k+1)) {
8119  Real tmp = sx(i-1,j-1,k ) * facx
8120  + sy(i-1,j-1,k ) * facy
8121  + sz(i-1,j-1,k ) * facz;
8122  m0 -= tmp;
8123  if (gid(i-1,j-1,k+1) < gidmax) {
8124  cols[nelems] = gid(i-1,j-1,k+1);
8125  mat[nelems] = tmp;
8126  ++nelems;
8127  }
8128  }
8129 
8130  if (nddom.contains(i,j-1,k+1)) {
8131  Real tmp = Real(0.);
8132  if (ccdom.contains(i-1,j-1,k )) {
8133  tmp += - sx(i-1,j-1,k ) * facx
8134  + sy(i-1,j-1,k ) * facy * Real(2.0)
8135  + sz(i-1,j-1,k ) * facz * Real(2.0);
8136  }
8137  if (ccdom.contains(i,j-1,k )) {
8138  tmp += - sx(i,j-1,k ) * facx
8139  + sy(i,j-1,k ) * facy * Real(2.0)
8140  + sz(i,j-1,k ) * facz * Real(2.0);
8141  }
8142  m0 -= tmp;
8143  if (gid(i,j-1,k+1) < gidmax) {
8144  cols[nelems] = gid(i,j-1,k+1);
8145  mat[nelems] = tmp;
8146  ++nelems;
8147  }
8148  }
8149 
8150  if (nddom.contains(i+1,j-1,k+1)) {
8151  Real tmp = sx(i ,j-1,k ) * facx
8152  + sy(i ,j-1,k ) * facy
8153  + sz(i ,j-1,k ) * facz;
8154  m0 -= tmp;
8155  if (gid(i+1,j-1,k+1) < gidmax) {
8156  cols[nelems] = gid(i+1,j-1,k+1);
8157  mat[nelems] = tmp;
8158  ++nelems;
8159  }
8160  }
8161 
8162  if (nddom.contains(i-1,j,k+1)) {
8163  Real tmp = Real(0.);
8164  if (ccdom.contains(i-1,j-1,k )) {
8165  tmp += sx(i-1,j-1,k ) * facx * Real(2.0)
8166  - sy(i-1,j-1,k ) * facy
8167  + sz(i-1,j-1,k ) * facz * Real(2.0);
8168  }
8169  if (ccdom.contains(i-1,j,k )) {
8170  tmp += sx(i-1,j,k ) * facx * Real(2.0)
8171  - sy(i-1,j,k ) * facy
8172  + sz(i-1,j,k ) * facz * Real(2.0);
8173  }
8174  m0 -= tmp;
8175  if (gid(i-1,j,k+1) < gidmax) {
8176  cols[nelems] = gid(i-1,j,k+1);
8177  mat[nelems] = tmp;
8178  ++nelems;
8179  }
8180  }
8181 
8182  if (nddom.contains(i,j,k+1)) {
8183  Real tmp = Real(0.);
8184  if (ccdom.contains(i-1,j-1,k )) {
8185  tmp += - sx(i-1,j-1,k ) * facx * Real(2.0)
8186  - sy(i-1,j-1,k ) * facy * Real(2.0)
8187  + sz(i-1,j-1,k ) * facz * Real(4.0);
8188  }
8189  if (ccdom.contains(i,j-1,k )) {
8190  tmp += - sx(i,j-1,k ) * facx * Real(2.0)
8191  - sy(i,j-1,k ) * facy * Real(2.0)
8192  + sz(i,j-1,k ) * facz * Real(4.0);
8193  }
8194  if (ccdom.contains(i-1,j,k )) {
8195  tmp += - sx(i-1,j,k ) * facx * Real(2.0)
8196  - sy(i-1,j,k ) * facy * Real(2.0)
8197  + sz(i-1,j,k ) * facz * Real(4.0);
8198  }
8199  if (ccdom.contains(i,j,k )) {
8200  tmp += - sx(i,j,k ) * facx * Real(2.0)
8201  - sy(i,j,k ) * facy * Real(2.0)
8202  + sz(i,j,k ) * facz * Real(4.0);
8203  }
8204  m0 -= tmp;
8205  if (gid(i,j,k+1) < gidmax && tmp != Real(0.0)) {
8206  cols[nelems] = gid(i,j,k+1);
8207  mat[nelems] = tmp;
8208  ++nelems;
8209  }
8210  }
8211 
8212  if (nddom.contains(i+1,j,k+1)) {
8213  Real tmp = Real(0.);
8214  if (ccdom.contains(i ,j-1,k )) {
8215  tmp += sx(i ,j-1,k ) * facx * Real(2.0)
8216  - sy(i ,j-1,k ) * facy
8217  + sz(i ,j-1,k ) * facz * Real(2.0);
8218  }
8219  if (ccdom.contains(i ,j,k )) {
8220  tmp += sx(i ,j,k ) * facx * Real(2.0)
8221  - sy(i ,j,k ) * facy
8222  + sz(i ,j,k ) * facz * Real(2.0);
8223  }
8224  m0 -= tmp;
8225  if (gid(i+1,j,k+1) < gidmax) {
8226  cols[nelems] = gid(i+1,j,k+1);
8227  mat[nelems] = tmp;
8228  ++nelems;
8229  }
8230  }
8231 
8232  if (nddom.contains(i-1,j+1,k+1)) {
8233  Real tmp = sx(i-1,j ,k ) * facx
8234  + sy(i-1,j ,k ) * facy
8235  + sz(i-1,j ,k ) * facz;
8236  m0 -= tmp;
8237  if (gid(i-1,j+1,k+1) < gidmax) {
8238  cols[nelems] = gid(i-1,j+1,k+1);
8239  mat[nelems] = tmp;
8240  ++nelems;
8241  }
8242  }
8243 
8244  if (nddom.contains(i,j+1,k+1)) {
8245  Real tmp = Real(0.);
8246  if (ccdom.contains(i-1,j ,k )) {
8247  tmp += - sx(i-1,j ,k ) * facx
8248  + sy(i-1,j ,k ) * facy * Real(2.0)
8249  + sz(i-1,j ,k ) * facz * Real(2.0);
8250  }
8251  if (ccdom.contains(i,j ,k )) {
8252  tmp += - sx(i,j ,k ) * facx
8253  + sy(i,j ,k ) * facy * Real(2.0)
8254  + sz(i,j ,k ) * facz * Real(2.0);
8255  }
8256  m0 -= tmp;
8257  if (gid(i,j+1,k+1) < gidmax) {
8258  cols[nelems] = gid(i,j+1,k+1);
8259  mat[nelems] = tmp;
8260  ++nelems;
8261  }
8262  }
8263 
8264  if (nddom.contains(i+1,j+1,k+1)) {
8265  Real tmp = sx(i ,j ,k ) * facx
8266  + sy(i ,j ,k ) * facy
8267  + sz(i ,j ,k ) * facz;
8268  m0 -= tmp;
8269  if (gid(i+1,j+1,k+1) < gidmax) {
8270  cols[nelems] = gid(i+1,j+1,k+1);
8271  mat[nelems] = tmp;
8272  ++nelems;
8273  }
8274  }
8275 
8276  mat[nelems_old] = m0;
8277  ncols[lid(i,j,k)] = nelems - nelems_old;
8278  }
8279  });
8280 }
8281 
8282 template <typename HypreInt, typename AtomicInt>
8283 void mlndlap_fillijmat_cs_cpu (Box const& ndbx,
8284  Array4<AtomicInt const> const& gid,
8285  Array4<int const> const& lid,
8286  HypreInt* ncols, HypreInt* cols,
8287  Real* mat, // NOLINT(readability-non-const-parameter)
8288  Real sigma,
8289  GpuArray<Real,AMREX_SPACEDIM> const& dxinv,
8290  Box const& ccdom) noexcept
8291 {
8292  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0] * sigma;
8293  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1] * sigma;
8294  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2] * sigma;
8295  Real fxyz = facx + facy + facz;
8296  Real fmx2y2z = -facx + Real(2.0)*facy + Real(2.0)*facz;
8297  Real f2xmy2z = Real(2.0)*facx - facy + Real(2.0)*facz;
8298  Real f2x2ymz = Real(2.0)*facx + Real(2.0)*facy - facz;
8299  Real f4xm2ym2z = Real(4.0)*facx - Real(2.0)*facy - Real(2.0)*facz;
8300  Real fm2x4ym2z = -Real(2.0)*facx + Real(4.0)*facy - Real(2.0)*facz;
8301  Real fm2xm2y4z = -Real(2.0)*facx - Real(2.0)*facy + Real(4.0)*facz;
8302 
8303  const Box& nddom = amrex::surroundingNodes(ccdom);
8304 
8305  constexpr auto gidmax = std::numeric_limits<AtomicInt>::max();
8306  HypreInt nelems = 0;
8307  amrex::LoopOnCpu(ndbx, [&] (int i, int j, int k) noexcept
8308  {
8309  if (lid(i,j,k) >= 0)
8310  {
8311  HypreInt nelems_old = nelems;
8312  cols[nelems_old] = gid(i,j,k);
8313  Real m0 = Real(0.);
8314  ++nelems;
8315 
8316  if (nddom.contains(i-1,j-1,k-1)) {
8317  Real tmp = fxyz;
8318  m0 -= tmp;
8319  if ( gid(i-1,j-1,k-1) < gidmax) {
8320  cols[nelems] = gid(i-1,j-1,k-1);
8321  mat[nelems] = tmp;
8322  ++nelems;
8323  }
8324  }
8325 
8326  if (nddom.contains(i,j-1,k-1)) {
8327  Real tmp = Real(0.);
8328  if (ccdom.contains(i-1,j-1,k-1)) {
8329  tmp += fmx2y2z;
8330  }
8331  if (ccdom.contains(i,j-1,k-1)) {
8332  tmp += fmx2y2z;
8333  }
8334  m0 -= tmp;
8335  if (gid(i,j-1,k-1) < gidmax) {
8336  cols[nelems] = gid(i,j-1,k-1);
8337  mat[nelems] = tmp;
8338  ++nelems;
8339  }
8340  }
8341 
8342  if (nddom.contains(i+1,j-1,k-1)) {
8343  Real tmp = fxyz;
8344  m0 -= tmp;
8345  if (gid(i+1,j-1,k-1) < gidmax) {
8346  cols[nelems] = gid(i+1,j-1,k-1);
8347  mat[nelems] = tmp;
8348  ++nelems;
8349  }
8350  }
8351 
8352  if (nddom.contains(i-1,j,k-1)) {
8353  Real tmp = Real(0.);
8354  if (ccdom.contains(i-1,j-1,k-1)) {
8355  tmp += f2xmy2z;
8356  }
8357  if (ccdom.contains(i-1,j,k-1)) {
8358  tmp += f2xmy2z;
8359  }
8360  m0 -= tmp;
8361  if (gid(i-1,j,k-1) < gidmax) {
8362  cols[nelems] = gid(i-1,j,k-1);
8363  mat[nelems] = tmp;
8364  ++nelems;
8365  }
8366  }
8367 
8368  if (nddom.contains(i,j,k-1) && fm2xm2y4z != Real(0.0)) {
8369  Real tmp = Real(0.);
8370  if (ccdom.contains(i-1,j-1,k-1)) {
8371  tmp += fm2xm2y4z;
8372  }
8373  if (ccdom.contains(i,j-1,k-1)) {
8374  tmp += fm2xm2y4z;
8375  }
8376  if (ccdom.contains(i-1,j,k-1)) {
8377  tmp += fm2xm2y4z;
8378  }
8379  if (ccdom.contains(i,j,k-1)) {
8380  tmp += fm2xm2y4z;
8381  }
8382  m0 -= tmp;
8383  if (gid(i,j,k-1) < gidmax) {
8384  cols[nelems] = gid(i,j,k-1);
8385  mat[nelems] = tmp;
8386  ++nelems;
8387  }
8388  }
8389 
8390  if (nddom.contains(i+1,j,k-1)) {
8391  Real tmp = Real(0.);
8392  if (ccdom.contains(i ,j-1,k-1)) {
8393  tmp += f2xmy2z;
8394  }
8395  if (ccdom.contains(i ,j,k-1)) {
8396  tmp += f2xmy2z;
8397  }
8398  m0 -= tmp;
8399  if (gid(i+1,j,k-1) < gidmax) {
8400  cols[nelems] = gid(i+1,j,k-1);
8401  mat[nelems] = tmp;
8402  ++nelems;
8403  }
8404  }
8405 
8406  if (nddom.contains(i-1,j+1,k-1)) {
8407  Real tmp = fxyz;
8408  m0 -= tmp;
8409  if (gid(i-1,j+1,k-1) < gidmax) {
8410  cols[nelems] = gid(i-1,j+1,k-1);
8411  mat[nelems] = tmp;
8412  ++nelems;
8413  }
8414  }
8415 
8416  if (nddom.contains(i,j+1,k-1)) {
8417  Real tmp = Real(0.);
8418  if (ccdom.contains(i-1,j ,k-1)) {
8419  tmp += fmx2y2z;
8420  }
8421  if (ccdom.contains(i,j ,k-1)) {
8422  tmp += fmx2y2z;
8423  }
8424  m0 -= tmp;
8425  if (gid(i,j+1,k-1) < gidmax) {
8426  cols[nelems] = gid(i,j+1,k-1);
8427  mat[nelems] = tmp;
8428  ++nelems;
8429  }
8430  }
8431 
8432  if (nddom.contains(i+1,j+1,k-1)) {
8433  Real tmp = fxyz;
8434  m0 -= tmp;
8435  if (gid(i+1,j+1,k-1) < gidmax) {
8436  cols[nelems] = gid(i+1,j+1,k-1);
8437  mat[nelems] = tmp;
8438  ++nelems;
8439  }
8440  }
8441 
8442  if (nddom.contains(i-1,j-1,k)) {
8443  Real tmp = Real(0.);
8444  if (ccdom.contains(i-1,j-1,k-1)) {
8445  tmp += f2x2ymz;
8446  }
8447  if (ccdom.contains(i-1,j-1,k)) {
8448  tmp += f2x2ymz;
8449  }
8450  m0 -= tmp;
8451  if (gid(i-1,j-1,k) < gidmax) {
8452  cols[nelems] = gid(i-1,j-1,k);
8453  mat[nelems] = tmp;
8454  ++nelems;
8455  }
8456  }
8457 
8458  if (nddom.contains(i,j-1,k) && fm2x4ym2z != Real(0.0)) {
8459  Real tmp = Real(0.);
8460  if (ccdom.contains(i-1,j-1,k-1)) {
8461  tmp += fm2x4ym2z;
8462  }
8463  if (ccdom.contains(i,j-1,k-1)) {
8464  tmp += fm2x4ym2z;
8465  }
8466  if (ccdom.contains(i-1,j-1,k)) {
8467  tmp += fm2x4ym2z;
8468  }
8469  if (ccdom.contains(i,j-1,k)) {
8470  tmp += fm2x4ym2z;
8471  }
8472  m0 -= tmp;
8473  if (gid(i,j-1,k) < gidmax) {
8474  cols[nelems] = gid(i,j-1,k);
8475  mat[nelems] = tmp;
8476  ++nelems;
8477  }
8478  }
8479 
8480  if (nddom.contains(i+1,j-1,k)) {
8481  Real tmp = Real(0.);
8482  if (ccdom.contains(i ,j-1,k-1)) {
8483  tmp += f2x2ymz;
8484  }
8485  if (ccdom.contains(i ,j-1,k)) {
8486  tmp += f2x2ymz;
8487  }
8488  m0 -= tmp;
8489  if (gid(i+1,j-1,k) < gidmax) {
8490  cols[nelems] = gid(i+1,j-1,k);
8491  mat[nelems] = tmp;
8492  ++nelems;
8493  }
8494  }
8495 
8496  if (nddom.contains(i-1,j,k) && f4xm2ym2z != Real(0.0)) {
8497  Real tmp = Real(0.);
8498  if (ccdom.contains(i-1,j-1,k-1)) {
8499  tmp += f4xm2ym2z;
8500  }
8501  if (ccdom.contains(i-1,j,k-1)) {
8502  tmp += f4xm2ym2z;
8503  }
8504  if (ccdom.contains(i-1,j-1,k)) {
8505  tmp += f4xm2ym2z;
8506  }
8507  if (ccdom.contains(i-1,j,k)) {
8508  tmp += f4xm2ym2z;
8509  }
8510  m0 -= tmp;
8511  if (gid(i-1,j,k) < gidmax) {
8512  cols[nelems] = gid(i-1,j,k);
8513  mat[nelems] = tmp;
8514  ++nelems;
8515  }
8516  }
8517 
8518  if (nddom.contains(i+1,j,k) && f4xm2ym2z != Real(0.0)) {
8519  Real tmp = Real(0.);
8520  if (ccdom.contains(i ,j-1,k-1)) {
8521  tmp += f4xm2ym2z;
8522  }
8523  if (ccdom.contains(i ,j,k-1)) {
8524  tmp += f4xm2ym2z;
8525  }
8526  if (ccdom.contains(i ,j-1,k)) {
8527  tmp += f4xm2ym2z;
8528  }
8529  if (ccdom.contains(i ,j,k)) {
8530  tmp += f4xm2ym2z;
8531  }
8532  m0 -= tmp;
8533  if (gid(i+1,j,k) < gidmax) {
8534  cols[nelems] = gid(i+1,j,k);
8535  mat[nelems] = tmp;
8536  ++nelems;
8537  }
8538  }
8539 
8540  if (nddom.contains(i-1,j+1,k)) {
8541  Real tmp = Real(0.);
8542  if (ccdom.contains(i-1,j ,k-1)) {
8543  tmp += f2x2ymz;
8544  }
8545  if (ccdom.contains(i-1,j ,k)) {
8546  tmp += f2x2ymz;
8547  }
8548  m0 -= tmp;
8549  if (gid(i-1,j+1,k) < gidmax) {
8550  cols[nelems] = gid(i-1,j+1,k);
8551  mat[nelems] = tmp;
8552  ++nelems;
8553  }
8554  }
8555 
8556  if (nddom.contains(i,j+1,k) && fm2x4ym2z != Real(0.0)) {
8557  Real tmp = Real(0.);
8558  if (ccdom.contains(i-1,j ,k-1)) {
8559  tmp += fm2x4ym2z;
8560  }
8561  if (ccdom.contains(i,j ,k-1)) {
8562  tmp += fm2x4ym2z;
8563  }
8564  if (ccdom.contains(i-1,j ,k)) {
8565  tmp += fm2x4ym2z;
8566  }
8567  if (ccdom.contains(i,j ,k)) {
8568  tmp += fm2x4ym2z;
8569  }
8570  m0 -= tmp;
8571  if (gid(i,j+1,k) < gidmax) {
8572  cols[nelems] = gid(i,j+1,k);
8573  mat[nelems] = tmp;
8574  ++nelems;
8575  }
8576  }
8577 
8578  if (nddom.contains(i+1,j+1,k)) {
8579  Real tmp = Real(0.);
8580  if (ccdom.contains(i ,j ,k-1)) {
8581  tmp += f2x2ymz;
8582  }
8583  if (ccdom.contains(i ,j ,k)) {
8584  tmp += f2x2ymz;
8585  }
8586  m0 -= tmp;
8587  if (gid(i+1,j+1,k) < gidmax) {
8588  cols[nelems] = gid(i+1,j+1,k);
8589  mat[nelems] = tmp;
8590  ++nelems;
8591  }
8592  }
8593 
8594  if (nddom.contains(i-1,j-1,k+1)) {
8595  Real tmp = fxyz;
8596  m0 -= tmp;
8597  if (gid(i-1,j-1,k+1) < gidmax) {
8598  cols[nelems] = gid(i-1,j-1,k+1);
8599  mat[nelems] = tmp;
8600  ++nelems;
8601  }
8602  }
8603 
8604  if (nddom.contains(i,j-1,k+1)) {
8605  Real tmp = Real(0.);
8606  if (ccdom.contains(i-1,j-1,k )) {
8607  tmp += fmx2y2z;
8608  }
8609  if (ccdom.contains(i,j-1,k )) {
8610  tmp += fmx2y2z;
8611  }
8612  m0 -= tmp;
8613  if (gid(i,j-1,k+1) < gidmax) {
8614  cols[nelems] = gid(i,j-1,k+1);
8615  mat[nelems] = tmp;
8616  ++nelems;
8617  }
8618  }
8619 
8620  if (nddom.contains(i+1,j-1,k+1)) {
8621  Real tmp = fxyz;
8622  m0 -= tmp;
8623  if (gid(i+1,j-1,k+1) < gidmax) {
8624  cols[nelems] = gid(i+1,j-1,k+1);
8625  mat[nelems] = tmp;
8626  ++nelems;
8627  }
8628  }
8629 
8630  if (nddom.contains(i-1,j,k+1)) {
8631  Real tmp = Real(0.);
8632  if (ccdom.contains(i-1,j-1,k )) {
8633  tmp += f2xmy2z;
8634  }
8635  if (ccdom.contains(i-1,j,k )) {
8636  tmp += f2xmy2z;
8637  }
8638  m0 -= tmp;
8639  if (gid(i-1,j,k+1) < gidmax) {
8640  cols[nelems] = gid(i-1,j,k+1);
8641  mat[nelems] = tmp;
8642  ++nelems;
8643  }
8644  }
8645 
8646  if (nddom.contains(i,j,k+1) && fm2xm2y4z != Real(0.0)) {
8647  Real tmp = Real(0.);
8648  if (ccdom.contains(i-1,j-1,k )) {
8649  tmp += fm2xm2y4z;
8650  }
8651  if (ccdom.contains(i,j-1,k )) {
8652  tmp += fm2xm2y4z;
8653  }
8654  if (ccdom.contains(i-1,j,k )) {
8655  tmp += fm2xm2y4z;
8656  }
8657  if (ccdom.contains(i,j,k )) {
8658  tmp += fm2xm2y4z;
8659  }
8660  m0 -= tmp;
8661  if (gid(i,j,k+1) < gidmax) {
8662  cols[nelems] = gid(i,j,k+1);
8663  mat[nelems] = tmp;
8664  ++nelems;
8665  }
8666  }
8667 
8668  if (nddom.contains(i+1,j,k+1)) {
8669  Real tmp = Real(0.);
8670  if (ccdom.contains(i ,j-1,k )) {
8671  tmp += f2xmy2z;
8672  }
8673  if (ccdom.contains(i ,j,k )) {
8674  tmp += f2xmy2z;
8675  }
8676  m0 -= tmp;
8677  if (gid(i+1,j,k+1) < gidmax) {
8678  cols[nelems] = gid(i+1,j,k+1);
8679  mat[nelems] = tmp;
8680  ++nelems;
8681  }
8682  }
8683 
8684  if (nddom.contains(i-1,j+1,k+1)) {
8685  Real tmp = fxyz;
8686  m0 -= tmp;
8687  if (gid(i-1,j+1,k+1) < gidmax) {
8688  cols[nelems] = gid(i-1,j+1,k+1);
8689  mat[nelems] = tmp;
8690  ++nelems;
8691  }
8692  }
8693 
8694  if (nddom.contains(i,j+1,k+1)) {
8695  Real tmp = Real(0.);
8696  if (ccdom.contains(i-1,j ,k )) {
8697  tmp += fmx2y2z;
8698  }
8699  if (ccdom.contains(i,j ,k )) {
8700  tmp += fmx2y2z;
8701  }
8702  m0 -= tmp;
8703  if (gid(i,j+1,k+1) < gidmax) {
8704  cols[nelems] = gid(i,j+1,k+1);
8705  mat[nelems] = tmp;
8706  ++nelems;
8707  }
8708  }
8709 
8710  if (nddom.contains(i+1,j+1,k+1)) {
8711  Real tmp = fxyz;
8712  m0 -= tmp;
8713  if (gid(i+1,j+1,k+1) < gidmax) {
8714  cols[nelems] = gid(i+1,j+1,k+1);
8715  mat[nelems] = tmp;
8716  ++nelems;
8717  }
8718  }
8719 
8720  mat[nelems_old] = m0;
8721  ncols[lid(i,j,k)] = nelems - nelems_old;
8722  }
8723  });
8724 }
8725 
8726 #ifdef AMREX_USE_GPU
8727 
8728 template <typename HypreInt, typename AtomicInt>
8729 AMREX_GPU_DEVICE AMREX_FORCE_INLINE
8730 void mlndlap_fillijmat_sten_gpu (const int ps, const int i, const int j, const int k,
8731  const int offset,
8732  Array4<AtomicInt const> const& gid,
8733  Array4<int const> const& lid,
8734  HypreInt* ncols, HypreInt* cols,
8735  Real* mat, // NOLINT(readability-non-const-parameter)
8736  Array4<Real const> const& sten) noexcept
8737 {
8738  if (lid(i,j,k) >= 0)
8739  {
8740  constexpr int ist_000 = 1-1;
8741  constexpr int ist_p00 = 2-1;
8742  constexpr int ist_0p0 = 3-1;
8743  constexpr int ist_00p = 4-1;
8744  constexpr int ist_pp0 = 5-1;
8745  constexpr int ist_p0p = 6-1;
8746  constexpr int ist_0pp = 7-1;
8747  constexpr int ist_ppp = 8-1;
8748 
8749  constexpr auto gidmax = std::numeric_limits<AtomicInt>::max();
8750  int nelems = 0;
8751 
8752  if (offset == 1 || offset == 0) {
8753  if (gid(i-1,j-1,k-1) < gidmax) {
8754  if (offset != 0) {
8755  cols[ps] = gid(i-1,j-1,k-1);
8756  mat[ps] = sten(i-1,j-1,k-1,ist_ppp);
8757  }
8758  ++nelems;
8759  }
8760  if (offset != 0) { return; }
8761  }
8762 
8763  if (offset == 2 || offset == 0) {
8764  if (gid(i,j-1,k-1) < gidmax) {
8765  if (offset != 0) {
8766  cols[ps] = gid(i,j-1,k-1);
8767  mat[ps] = sten(i,j-1,k-1,ist_0pp);
8768  }
8769  ++nelems;
8770  }
8771  if (offset != 0) { return; }
8772  }
8773 
8774  if (offset == 3 || offset == 0) {
8775  if (gid(i+1,j-1,k-1) < gidmax) {
8776  if (offset != 0) {
8777  cols[ps] = gid(i+1,j-1,k-1);
8778  mat[ps] = sten(i,j-1,k-1,ist_ppp);
8779  }
8780  ++nelems;
8781  }
8782  if (offset != 0) { return; }
8783  }
8784 
8785  if (offset == 4 || offset == 0) {
8786  if (gid(i-1,j,k-1) < gidmax) {
8787  if (offset != 0) {
8788  cols[ps] = gid(i-1,j,k-1);
8789  mat[ps] = sten(i-1,j,k-1,ist_p0p);
8790  }
8791  ++nelems;
8792  }
8793  if (offset != 0) { return; }
8794  }
8795 
8796  if (offset == 5 || offset == 0) {
8797  if (gid(i,j,k-1) < gidmax) {
8798  if (offset != 0) {
8799  cols[ps] = gid(i,j,k-1);
8800  mat[ps] = sten(i,j,k-1,ist_00p);
8801  }
8802  ++nelems;
8803  }
8804  if (offset != 0) { return; }
8805  }
8806 
8807  if (offset == 6 || offset == 0) {
8808  if (gid(i+1,j,k-1) < gidmax) {
8809  if (offset != 0) {
8810  cols[ps] = gid(i+1,j,k-1);
8811  mat[ps] = sten(i,j,k-1,ist_p0p);
8812  }
8813  ++nelems;
8814  }
8815  if (offset != 0) { return; }
8816  }
8817 
8818  if (offset == 7 || offset == 0) {
8819  if (gid(i-1,j+1,k-1) < gidmax) {
8820  if (offset != 0) {
8821  cols[ps] = gid(i-1,j+1,k-1);
8822  mat[ps] = sten(i-1,j,k-1,ist_ppp);
8823  }
8824  ++nelems;
8825  }
8826  if (offset != 0) { return; }
8827  }
8828 
8829  if (offset == 8 || offset == 0) {
8830  if (gid(i,j+1,k-1) < gidmax) {
8831  if (offset != 0) {
8832  cols[ps] = gid(i,j+1,k-1);
8833  mat[ps] = sten(i,j,k-1,ist_0pp);
8834  }
8835  ++nelems;
8836  }
8837  if (offset != 0) { return; }
8838  }
8839 
8840  if (offset == 9 || offset == 0) {
8841  if (gid(i+1,j+1,k-1) < gidmax) {
8842  if (offset != 0) {
8843  cols[ps] = gid(i+1,j+1,k-1);
8844  mat[ps] = sten(i,j,k-1,ist_ppp);
8845  }
8846  ++nelems;
8847  }
8848  if (offset != 0) { return; }
8849  }
8850 
8851  if (offset == 10 || offset == 0) {
8852  if (gid(i-1,j-1,k) < gidmax) {
8853  if (offset != 0) {
8854  cols[ps] = gid(i-1,j-1,k);
8855  mat[ps] = sten(i-1,j-1,k,ist_pp0);
8856  }
8857  ++nelems;
8858  }
8859  if (offset != 0) { return; }
8860  }
8861 
8862  if (offset == 11 || offset == 0) {
8863  if (gid(i,j-1,k) < gidmax) {
8864  if (offset != 0) {
8865  cols[ps] = gid(i,j-1,k);
8866  mat[ps] = sten(i,j-1,k,ist_0p0);
8867  }
8868  ++nelems;
8869  }
8870  if (offset != 0) { return; }
8871  }
8872 
8873  if (offset == 12 || offset == 0) {
8874  if (gid(i+1,j-1,k) < gidmax) {
8875  if (offset != 0) {
8876  cols[ps] = gid(i+1,j-1,k);
8877  mat[ps] = sten(i,j-1,k,ist_pp0);
8878  }
8879  ++nelems;
8880  }
8881  if (offset != 0) { return; }
8882  }
8883 
8884  if (offset == 13 || offset == 0) {
8885  if (gid(i-1,j,k) < gidmax) {
8886  if (offset != 0) {
8887  cols[ps] = gid(i-1,j,k);
8888  mat[ps] = sten(i-1,j,k,ist_p00);
8889  }
8890  ++nelems;
8891  }
8892  if (offset != 0) { return; }
8893  }
8894 
8895  if (offset == 14 || offset == 0) {
8896  if (gid(i+1,j,k) < gidmax) {
8897  if (offset != 0) {
8898  cols[ps] = gid(i+1,j,k);
8899  mat[ps] = sten(i,j,k,ist_p00);
8900  }
8901  ++nelems;
8902  }
8903  if (offset != 0) { return; }
8904  }
8905 
8906  if (offset == 15 || offset == 0) {
8907  if (gid(i-1,j+1,k) < gidmax) {
8908  if (offset != 0) {
8909  cols[ps] = gid(i-1,j+1,k);
8910  mat[ps] = sten(i-1,j,k,ist_pp0);
8911  }
8912  ++nelems;
8913  }
8914  if (offset != 0) { return; }
8915  }
8916 
8917  if (offset == 16 || offset == 0) {
8918  if (gid(i,j+1,k) < gidmax) {
8919  if (offset != 0) {
8920  cols[ps] = gid(i,j+1,k);
8921  mat[ps] = sten(i,j,k,ist_0p0);
8922  }
8923  ++nelems;
8924  }
8925  if (offset != 0) { return; }
8926  }
8927 
8928  if (offset == 17 || offset == 0) {
8929  if (gid(i+1,j+1,k) < gidmax) {
8930  if (offset != 0) {
8931  cols[ps] = gid(i+1,j+1,k);
8932  mat[ps] = sten(i,j,k,ist_pp0);
8933  }
8934  ++nelems;
8935  }
8936  if (offset != 0) { return; }
8937  }
8938 
8939  if (offset == 18 || offset == 0) {
8940  if (gid(i-1,j-1,k+1) < gidmax) {
8941  if (offset != 0) {
8942  cols[ps] = gid(i-1,j-1,k+1);
8943  mat[ps] = sten(i-1,j-1,k,ist_ppp);
8944  }
8945  ++nelems;
8946  }
8947  if (offset != 0) { return; }
8948  }
8949 
8950  if (offset == 19 || offset == 0) {
8951  if (gid(i,j-1,k+1) < gidmax) {
8952  if (offset != 0) {
8953  cols[ps] = gid(i,j-1,k+1);
8954  mat[ps] = sten(i,j-1,k,ist_0pp);
8955  }
8956  ++nelems;
8957  }
8958  if (offset != 0) { return; }
8959  }
8960 
8961  if (offset == 20 || offset == 0) {
8962  if (gid(i+1,j-1,k+1) < gidmax) {
8963  if (offset != 0) {
8964  cols[ps] = gid(i+1,j-1,k+1);
8965  mat[ps] = sten(i,j-1,k,ist_ppp);
8966  }
8967  ++nelems;
8968  }
8969  if (offset != 0) { return; }
8970  }
8971 
8972  if (offset == 21 || offset == 0) {
8973  if (gid(i-1,j,k+1) < gidmax) {
8974  if (offset != 0) {
8975  cols[ps] = gid(i-1,j,k+1);
8976  mat[ps] = sten(i-1,j,k,ist_p0p);
8977  }
8978  ++nelems;
8979  }
8980  if (offset != 0) { return; }
8981  }
8982 
8983  if (offset == 22 || offset == 0) {
8984  if (gid(i,j,k+1) < gidmax) {
8985  if (offset != 0) {
8986  cols[ps] = gid(i,j,k+1);
8987  mat[ps] = sten(i,j,k,ist_00p);
8988  }
8989  ++nelems;
8990  }
8991  if (offset != 0) { return; }
8992  }
8993 
8994  if (offset == 23 || offset == 0) {
8995  if (gid(i+1,j,k+1) < gidmax) {
8996  if (offset != 0) {
8997  cols[ps] = gid(i+1,j,k+1);
8998  mat[ps] = sten(i,j,k,ist_p0p);
8999  }
9000  ++nelems;
9001  }
9002  if (offset != 0) { return; }
9003  }
9004 
9005  if (offset == 24 || offset == 0) {
9006  if (gid(i-1,j+1,k+1) < gidmax) {
9007  if (offset != 0) {
9008  cols[ps] = gid(i-1,j+1,k+1);
9009  mat[ps] = sten(i-1,j,k,ist_ppp);
9010  }
9011  ++nelems;
9012  }
9013  if (offset != 0) { return; }
9014  }
9015 
9016  if (offset == 25 || offset == 0) {
9017  if (gid(i,j+1,k+1) < gidmax) {
9018  if (offset != 0) {
9019  cols[ps] = gid(i,j+1,k+1);
9020  mat[ps] = sten(i,j,k,ist_0pp);
9021  }
9022  ++nelems;
9023  }
9024  if (offset != 0) { return; }
9025  }
9026 
9027  if (offset == 26 || offset == 0) {
9028  if (gid(i+1,j+1,k+1) < gidmax) {
9029  if (offset != 0) {
9030  cols[ps] = gid(i+1,j+1,k+1);
9031  mat[ps] = sten(i,j,k,ist_ppp);
9032  }
9033  ++nelems;
9034  }
9035  if (offset != 0) { return; }
9036  }
9037 
9038  // Only offset == 0 could get this far.
9039  cols[ps] = gid(i,j,k);
9040  mat[ps] = sten(i,j,k,ist_000);
9041  ncols[lid(i,j,k)] = nelems+1;
9042  }
9043 }
9044 
9045 template <typename HypreInt, typename AtomicInt>
9046 AMREX_GPU_DEVICE AMREX_FORCE_INLINE
9047 void mlndlap_fillijmat_aa_gpu (const int ps, const int i, const int j, const int k,
9048  const int offset,
9049  Box const& ndbx, Array4<AtomicInt const> const& gid,
9050  Array4<int const> const& lid,
9051  HypreInt* ncols, HypreInt* cols,
9052  Real* mat, // NOLINT(readability-non-const-parameter)
9053  Array4<Real const> const& sig,
9054  GpuArray<Real,AMREX_SPACEDIM> const& dxinv,
9055  Box const& ccdom) noexcept
9056 {
9057  if (lid(i,j,k) >= 0)
9058  {
9059  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0];
9060  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1];
9061  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2];
9062  Real fxyz = facx + facy + facz;
9063  Real fmx2y2z = -facx + Real(2.0)*facy + Real(2.0)*facz;
9064  Real f2xmy2z = Real(2.0)*facx - facy + Real(2.0)*facz;
9065  Real f2x2ymz = Real(2.0)*facx + Real(2.0)*facy - facz;
9066  Real f4xm2ym2z = Real(4.0)*facx - Real(2.0)*facy - Real(2.0)*facz;
9067  Real fm2x4ym2z = -Real(2.0)*facx + Real(4.0)*facy - Real(2.0)*facz;
9068  Real fm2xm2y4z = -Real(2.0)*facx - Real(2.0)*facy + Real(4.0)*facz;
9069 
9070  const Box& nddom = amrex::surroundingNodes(ccdom);
9071 
9072  constexpr auto gidmax = std::numeric_limits<AtomicInt>::max();
9073  int nelems = 0;
9074  Real m0 = Real(0.);
9075 
9076  if (offset == 1 || offset == 0) {
9077  if (nddom.contains(i-1,j-1,k-1)) {
9078  Real tmp = sig(i-1,j-1,k-1) * fxyz;
9079  m0 -= tmp;
9080  if ( gid(i-1,j-1,k-1) < gidmax) {
9081  if (offset != 0) {
9082  cols[ps] = gid(i-1,j-1,k-1);
9083  mat[ps] = tmp;
9084  }
9085  ++nelems;
9086  }
9087  }
9088  if (offset != 0) { return; }
9089  }
9090 
9091  if (offset == 2 || offset == 0) {
9092  if (nddom.contains(i,j-1,k-1)) {
9093  Real tmp = Real(0.);
9094  if (ccdom.contains(i-1,j-1,k-1)) {
9095  tmp += sig(i-1,j-1,k-1) * fmx2y2z;
9096  }
9097  if (ccdom.contains(i,j-1,k-1)) {
9098  tmp += sig(i,j-1,k-1) * fmx2y2z;
9099  }
9100  m0 -= tmp;
9101  if (gid(i,j-1,k-1) < gidmax) {
9102  if (offset != 0) {
9103  cols[ps] = gid(i,j-1,k-1);
9104  mat[ps] = tmp;
9105  }
9106  ++nelems;
9107  }
9108  }
9109  if (offset != 0) { return; }
9110  }
9111 
9112  if (offset == 3 || offset == 0) {
9113  if (nddom.contains(i+1,j-1,k-1)) {
9114  Real tmp = sig(i ,j-1,k-1) * fxyz;
9115  m0 -= tmp;
9116  if (gid(i+1,j-1,k-1) < gidmax) {
9117  if (offset != 0) {
9118  cols[ps] = gid(i+1,j-1,k-1);
9119  mat[ps] = tmp;
9120  }
9121  ++nelems;
9122  }
9123  }
9124  if (offset != 0) { return; }
9125  }
9126 
9127  if (offset == 4 || offset == 0) {
9128  if (nddom.contains(i-1,j,k-1)) {
9129  Real tmp = Real(0.);
9130  if (ccdom.contains(i-1,j-1,k-1)) {
9131  tmp += sig(i-1,j-1,k-1) * f2xmy2z;
9132  }
9133  if (ccdom.contains(i-1,j,k-1)) {
9134  tmp += sig(i-1,j,k-1) * f2xmy2z;
9135  }
9136  m0 -= tmp;
9137  if (gid(i-1,j,k-1) < gidmax) {
9138  if (offset != 0) {
9139  cols[ps] = gid(i-1,j,k-1);
9140  mat[ps] = tmp;
9141  }
9142  ++nelems;
9143  }
9144  }
9145  if (offset != 0) { return; }
9146  }
9147 
9148  if (offset == 5 || offset == 0) {
9149  if (nddom.contains(i,j,k-1)) {
9150  Real tmp = Real(0.);
9151  if (ccdom.contains(i-1,j-1,k-1)) {
9152  tmp += sig(i-1,j-1,k-1) * fm2xm2y4z;
9153  }
9154  if (ccdom.contains(i,j-1,k-1)) {
9155  tmp += sig(i,j-1,k-1) * fm2xm2y4z;
9156  }
9157  if (ccdom.contains(i-1,j,k-1)) {
9158  tmp += sig(i-1,j,k-1) * fm2xm2y4z;
9159  }
9160  if (ccdom.contains(i,j,k-1)) {
9161  tmp += sig(i,j,k-1) * fm2xm2y4z;
9162  }
9163  m0 -= tmp;
9164  if (gid(i,j,k-1) < gidmax) {
9165  if (offset != 0) {
9166  cols[ps] = gid(i,j,k-1);
9167  mat[ps] = tmp;
9168  }
9169  ++nelems;
9170  }
9171  }
9172  if (offset != 0) { return; }
9173  }
9174 
9175  if (offset == 6 || offset == 0) {
9176  if (nddom.contains(i+1,j,k-1)) {
9177  Real tmp = Real(0.);
9178  if (ccdom.contains(i ,j-1,k-1)) {
9179  tmp += sig(i ,j-1,k-1) * f2xmy2z;
9180  }
9181  if (ccdom.contains(i ,j,k-1)) {
9182  tmp += sig(i ,j,k-1) * f2xmy2z;
9183  }
9184  m0 -= tmp;
9185  if (gid(i+1,j,k-1) < gidmax) {
9186  if (offset != 0) {
9187  cols[ps] = gid(i+1,j,k-1);
9188  mat[ps] = tmp;
9189  }
9190  ++nelems;
9191  }
9192  }
9193  if (offset != 0) { return; }
9194  }
9195 
9196  if (offset == 7 || offset == 0) {
9197  if (nddom.contains(i-1,j+1,k-1)) {
9198  Real tmp = sig(i-1,j ,k-1) * fxyz;
9199  m0 -= tmp;
9200  if (gid(i-1,j+1,k-1) < gidmax) {
9201  if (offset != 0) {
9202  cols[ps] = gid(i-1,j+1,k-1);
9203  mat[ps] = tmp;
9204  }
9205  ++nelems;
9206  }
9207  }
9208  if (offset != 0) { return; }
9209  }
9210 
9211  if (offset == 8 || offset == 0) {
9212  if (nddom.contains(i,j+1,k-1)) {
9213  Real tmp = Real(0.);
9214  if (ccdom.contains(i-1,j ,k-1)) {
9215  tmp += sig(i-1,j ,k-1) * fmx2y2z;
9216  }
9217  if (ccdom.contains(i,j ,k-1)) {
9218  tmp += sig(i,j ,k-1) * fmx2y2z;
9219  }
9220  m0 -= tmp;
9221  if (gid(i,j+1,k-1) < gidmax) {
9222  if (offset != 0) {
9223  cols[ps] = gid(i,j+1,k-1);
9224  mat[ps] = tmp;
9225  }
9226  ++nelems;
9227  }
9228  }
9229  if (offset != 0) { return; }
9230  }
9231 
9232  if (offset == 9 || offset == 0) {
9233  if (nddom.contains(i+1,j+1,k-1)) {
9234  Real tmp = sig(i ,j ,k-1) * fxyz;
9235  m0 -= tmp;
9236  if (gid(i+1,j+1,k-1) < gidmax) {
9237  if (offset != 0) {
9238  cols[ps] = gid(i+1,j+1,k-1);
9239  mat[ps] = tmp;
9240  }
9241  ++nelems;
9242  }
9243  }
9244  if (offset != 0) { return; }
9245  }
9246 
9247  if (offset == 10 || offset == 0) {
9248  if (nddom.contains(i-1,j-1,k)) {
9249  Real tmp = Real(0.);
9250  if (ccdom.contains(i-1,j-1,k-1)) {
9251  tmp += sig(i-1,j-1,k-1) * f2x2ymz;
9252  }
9253  if (ccdom.contains(i-1,j-1,k)) {
9254  tmp += sig(i-1,j-1,k) * f2x2ymz;
9255  }
9256  m0 -= tmp;
9257  if (gid(i-1,j-1,k) < gidmax) {
9258  if (offset != 0) {
9259  cols[ps] = gid(i-1,j-1,k);
9260  mat[ps] = tmp;
9261  }
9262  ++nelems;
9263  }
9264  }
9265  if (offset != 0) { return; }
9266  }
9267 
9268  if (offset == 11 || offset == 0) {
9269  if (nddom.contains(i,j-1,k)) {
9270  Real tmp = Real(0.);
9271  if (ccdom.contains(i-1,j-1,k-1)) {
9272  tmp += sig(i-1,j-1,k-1) * fm2x4ym2z;
9273  }
9274  if (ccdom.contains(i,j-1,k-1)) {
9275  tmp += sig(i,j-1,k-1) * fm2x4ym2z;
9276  }
9277  if (ccdom.contains(i-1,j-1,k)) {
9278  tmp += sig(i-1,j-1,k) * fm2x4ym2z;
9279  }
9280  if (ccdom.contains(i,j-1,k)) {
9281  tmp += sig(i,j-1,k) * fm2x4ym2z;
9282  }
9283  m0 -= tmp;
9284  if (gid(i,j-1,k) < gidmax) {
9285  if (offset != 0) {
9286  cols[ps] = gid(i,j-1,k);
9287  mat[ps] = tmp;
9288  }
9289  ++nelems;
9290  }
9291  }
9292  if (offset != 0) { return; }
9293  }
9294 
9295  if (offset == 12 || offset == 0) {
9296  if (nddom.contains(i+1,j-1,k)) {
9297  Real tmp = Real(0.);
9298  if (ccdom.contains(i ,j-1,k-1)) {
9299  tmp += sig(i ,j-1,k-1) * f2x2ymz;
9300  }
9301  if (ccdom.contains(i ,j-1,k)) {
9302  tmp += sig(i ,j-1,k) * f2x2ymz;
9303  }
9304  m0 -= tmp;
9305  if (gid(i+1,j-1,k) < gidmax) {
9306  if (offset != 0) {
9307  cols[ps] = gid(i+1,j-1,k);
9308  mat[ps] = tmp;
9309  }
9310  ++nelems;
9311  }
9312  }
9313  if (offset != 0) { return; }
9314  }
9315 
9316  if (offset == 13 || offset == 0) {
9317  if (nddom.contains(i-1,j,k)) {
9318  Real tmp = Real(0.);
9319  if (ccdom.contains(i-1,j-1,k-1)) {
9320  tmp += sig(i-1,j-1,k-1) * f4xm2ym2z;
9321  }
9322  if (ccdom.contains(i-1,j,k-1)) {
9323  tmp += sig(i-1,j,k-1) * f4xm2ym2z;
9324  }
9325  if (ccdom.contains(i-1,j-1,k)) {
9326  tmp += sig(i-1,j-1,k) * f4xm2ym2z;
9327  }
9328  if (ccdom.contains(i-1,j,k)) {
9329  tmp += sig(i-1,j,k) * f4xm2ym2z;
9330  }
9331  m0 -= tmp;
9332  if (gid(i-1,j,k) < gidmax) {
9333  if (offset != 0) {
9334  cols[ps] = gid(i-1,j,k);
9335  mat[ps] = tmp;
9336  }
9337  ++nelems;
9338  }
9339  }
9340  if (offset != 0) { return; }
9341  }
9342 
9343  if (offset == 14 || offset == 0) {
9344  if (nddom.contains(i+1,j,k)) {
9345  Real tmp = Real(0.);
9346  if (ccdom.contains(i ,j-1,k-1)) {
9347  tmp += sig(i ,j-1,k-1) * f4xm2ym2z;
9348  }
9349  if (ccdom.contains(i ,j,k-1)) {
9350  tmp += sig(i ,j,k-1) * f4xm2ym2z;
9351  }
9352  if (ccdom.contains(i ,j-1,k)) {
9353  tmp += sig(i ,j-1,k) * f4xm2ym2z;
9354  }
9355  if (ccdom.contains(i ,j,k)) {
9356  tmp += sig(i ,j,k) * f4xm2ym2z;
9357  }
9358  m0 -= tmp;
9359  if (gid(i+1,j,k) < gidmax) {
9360  if (offset != 0) {
9361  cols[ps] = gid(i+1,j,k);
9362  mat[ps] = tmp;
9363  }
9364  ++nelems;
9365  }
9366  }
9367  if (offset != 0) { return; }
9368  }
9369 
9370  if (offset == 15 || offset == 0) {
9371  if (nddom.contains(i-1,j+1,k)) {
9372  Real tmp = Real(0.);
9373  if (ccdom.contains(i-1,j ,k-1)) {
9374  tmp += sig(i-1,j ,k-1) * f2x2ymz;
9375  }
9376  if (ccdom.contains(i-1,j ,k)) {
9377  tmp += sig(i-1,j ,k) * f2x2ymz;
9378  }
9379  m0 -= tmp;
9380  if (gid(i-1,j+1,k) < gidmax) {
9381  if (offset != 0) {
9382  cols[ps] = gid(i-1,j+1,k);
9383  mat[ps] = tmp;
9384  }
9385  ++nelems;
9386  }
9387  }
9388  if (offset != 0) { return; }
9389  }
9390 
9391  if (offset == 16 || offset == 0) {
9392  if (nddom.contains(i,j+1,k)) {
9393  Real tmp = Real(0.);
9394  if (ccdom.contains(i-1,j ,k-1)) {
9395  tmp += sig(i-1,j ,k-1) * fm2x4ym2z;
9396  }
9397  if (ccdom.contains(i,j ,k-1)) {
9398  tmp += sig(i,j ,k-1) * fm2x4ym2z;
9399  }
9400  if (ccdom.contains(i-1,j ,k)) {
9401  tmp += sig(i-1,j ,k) * fm2x4ym2z;
9402  }
9403  if (ccdom.contains(i,j ,k)) {
9404  tmp += sig(i,j ,k) * fm2x4ym2z;
9405  }
9406  m0 -= tmp;
9407  if (gid(i,j+1,k) < gidmax) {
9408  if (offset != 0) {
9409  cols[ps] = gid(i,j+1,k);
9410  mat[ps] = tmp;
9411  }
9412  ++nelems;
9413  }
9414  }
9415  if (offset != 0) { return; }
9416  }
9417 
9418  if (offset == 17 || offset == 0) {
9419  if (nddom.contains(i+1,j+1,k)) {
9420  Real tmp = Real(0.);
9421  if (ccdom.contains(i ,j ,k-1)) {
9422  tmp += sig(i ,j ,k-1) * f2x2ymz;
9423  }
9424  if (ccdom.contains(i ,j ,k)) {
9425  tmp += sig(i ,j ,k) * f2x2ymz;
9426  }
9427  m0 -= tmp;
9428  if (gid(i+1,j+1,k) < gidmax) {
9429  if (offset != 0) {
9430  cols[ps] = gid(i+1,j+1,k);
9431  mat[ps] = tmp;
9432  }
9433  ++nelems;
9434  }
9435  }
9436  if (offset != 0) { return; }
9437  }
9438 
9439  if (offset == 18 || offset == 0) {
9440  if (nddom.contains(i-1,j-1,k+1)) {
9441  Real tmp = sig(i-1,j-1,k ) * fxyz;
9442  m0 -= tmp;
9443  if (gid(i-1,j-1,k+1) < gidmax) {
9444  if (offset != 0) {
9445  cols[ps] = gid(i-1,j-1,k+1);
9446  mat[ps] = tmp;
9447  }
9448  ++nelems;
9449  }
9450  }
9451  if (offset != 0) { return; }
9452  }
9453 
9454  if (offset == 19 || offset == 0) {
9455  if (nddom.contains(i,j-1,k+1)) {
9456  Real tmp = Real(0.);
9457  if (ccdom.contains(i-1,j-1,k )) {
9458  tmp += sig(i-1,j-1,k ) * fmx2y2z;
9459  }
9460  if (ccdom.contains(i,j-1,k )) {
9461  tmp += sig(i,j-1,k ) * fmx2y2z;
9462  }
9463  m0 -= tmp;
9464  if (gid(i,j-1,k+1) < gidmax) {
9465  if (offset != 0) {
9466  cols[ps] = gid(i,j-1,k+1);
9467  mat[ps] = tmp;
9468  }
9469  ++nelems;
9470  }
9471  }
9472  if (offset != 0) { return; }
9473  }
9474 
9475  if (offset == 20 || offset == 0) {
9476  if (nddom.contains(i+1,j-1,k+1)) {
9477  Real tmp = sig(i ,j-1,k ) * fxyz;
9478  m0 -= tmp;
9479  if (gid(i+1,j-1,k+1) < gidmax) {
9480  if (offset != 0) {
9481  cols[ps] = gid(i+1,j-1,k+1);
9482  mat[ps] = tmp;
9483  }
9484  ++nelems;
9485  }
9486  }
9487  if (offset != 0) { return; }
9488  }
9489 
9490  if (offset == 21 || offset == 0) {
9491  if (nddom.contains(i-1,j,k+1)) {
9492  Real tmp = Real(0.);
9493  if (ccdom.contains(i-1,j-1,k )) {
9494  tmp += sig(i-1,j-1,k ) * f2xmy2z;
9495  }
9496  if (ccdom.contains(i-1,j,k )) {
9497  tmp += sig(i-1,j,k ) * f2xmy2z;
9498  }
9499  m0 -= tmp;
9500  if (gid(i-1,j,k+1) < gidmax) {
9501  if (offset != 0) {
9502  cols[ps] = gid(i-1,j,k+1);
9503  mat[ps] = tmp;
9504  }
9505  ++nelems;
9506  }
9507  }
9508  if (offset != 0) { return; }
9509  }
9510 
9511  if (offset == 22 || offset == 0) {
9512  if (nddom.contains(i,j,k+1)) {
9513  Real tmp = Real(0.);
9514  if (ccdom.contains(i-1,j-1,k )) {
9515  tmp += sig(i-1,j-1,k ) * fm2xm2y4z;
9516  }
9517  if (ccdom.contains(i,j-1,k )) {
9518  tmp += sig(i,j-1,k ) * fm2xm2y4z;
9519  }
9520  if (ccdom.contains(i-1,j,k )) {
9521  tmp += sig(i-1,j,k ) * fm2xm2y4z;
9522  }
9523  if (ccdom.contains(i,j,k )) {
9524  tmp += sig(i,j,k ) * fm2xm2y4z;
9525  }
9526  m0 -= tmp;
9527  if (gid(i,j,k+1) < gidmax) {
9528  if (offset != 0) {
9529  cols[ps] = gid(i,j,k+1);
9530  mat[ps] = tmp;
9531  }
9532  ++nelems;
9533  }
9534  }
9535  if (offset != 0) { return; }
9536  }
9537 
9538  if (offset == 23 || offset == 0) {
9539  if (nddom.contains(i+1,j,k+1)) {
9540  Real tmp = Real(0.);
9541  if (ccdom.contains(i ,j-1,k )) {
9542  tmp += sig(i ,j-1,k ) * f2xmy2z;
9543  }
9544  if (ccdom.contains(i ,j,k )) {
9545  tmp += sig(i ,j,k ) * f2xmy2z;
9546  }
9547  m0 -= tmp;
9548  if (gid(i+1,j,k+1) < gidmax) {
9549  if (offset != 0) {
9550  cols[ps] = gid(i+1,j,k+1);
9551  mat[ps] = tmp;
9552  }
9553  ++nelems;
9554  }
9555  }
9556  if (offset != 0) { return; }
9557  }
9558 
9559  if (offset == 24 || offset == 0) {
9560  if (nddom.contains(i-1,j+1,k+1)) {
9561  Real tmp = sig(i-1,j ,k ) * fxyz;
9562  m0 -= tmp;
9563  if (gid(i-1,j+1,k+1) < gidmax) {
9564  if (offset != 0) {
9565  cols[ps] = gid(i-1,j+1,k+1);
9566  mat[ps] = tmp;
9567  }
9568  ++nelems;
9569  }
9570  }
9571  if (offset != 0) { return; }
9572  }
9573 
9574  if (offset == 25 || offset == 0) {
9575  if (nddom.contains(i,j+1,k+1)) {
9576  Real tmp = Real(0.);
9577  if (ccdom.contains(i-1,j ,k )) {
9578  tmp += sig(i-1,j ,k ) * fmx2y2z;
9579  }
9580  if (ccdom.contains(i,j ,k )) {
9581  tmp += sig(i,j ,k ) * fmx2y2z;
9582  }
9583  m0 -= tmp;
9584  if (gid(i,j+1,k+1) < gidmax) {
9585  if (offset != 0) {
9586  cols[ps] = gid(i,j+1,k+1);
9587  mat[ps] = tmp;
9588  }
9589  ++nelems;
9590  }
9591  }
9592  if (offset != 0) { return; }
9593  }
9594 
9595  if (offset == 26 || offset == 0) {
9596  if (nddom.contains(i+1,j+1,k+1)) {
9597  Real tmp = sig(i ,j ,k ) * fxyz;
9598  m0 -= tmp;
9599  if (gid(i+1,j+1,k+1) < gidmax) {
9600  if (offset != 0) {
9601  cols[ps] = gid(i+1,j+1,k+1);
9602  mat[ps] = tmp;
9603  }
9604  ++nelems;
9605  }
9606  }
9607  if (offset != 0) { return; }
9608  }
9609 
9610  // Only offset == 0 could get this far.
9611  cols[ps] = gid(i,j,k);
9612  mat[ps] = m0;
9613  ncols[lid(i,j,k)] = nelems+1;
9614  }
9615 }
9616 
9617 template <typename HypreInt, typename AtomicInt>
9618 AMREX_GPU_DEVICE AMREX_FORCE_INLINE
9619 void mlndlap_fillijmat_ha_gpu (const int ps, const int i, const int j, const int k,
9620  const int offset,
9621  Box const& ndbx, Array4<AtomicInt const> const& gid,
9622  Array4<int const> const& lid,
9623  HypreInt* ncols, HypreInt* cols,
9624  Real* mat, // NOLINT(readability-non-const-parameter)
9625  Array4<Real const> const& sx,
9626  Array4<Real const> const& sy,
9627  Array4<Real const> const& sz,
9628  GpuArray<Real,AMREX_SPACEDIM> const& dxinv,
9629  Box const& ccdom) noexcept
9630 {
9631  if (lid(i,j,k) >= 0)
9632  {
9633  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0];
9634  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1];
9635  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2];
9636 
9637  const Box& nddom = amrex::surroundingNodes(ccdom);
9638 
9639  constexpr auto gidmax = std::numeric_limits<AtomicInt>::max();
9640  int nelems = 0;
9641  Real m0 = Real(0.);
9642 
9643  if (offset == 1 || offset == 0) {
9644  if (nddom.contains(i-1,j-1,k-1)) {
9645  Real tmp = sx(i-1,j-1,k-1) * facx
9646  + sy(i-1,j-1,k-1) * facy
9647  + sz(i-1,j-1,k-1) * facz;
9648  m0 -= tmp;
9649  if ( gid(i-1,j-1,k-1) < gidmax) {
9650  if (offset != 0) {
9651  cols[ps] = gid(i-1,j-1,k-1);
9652  mat[ps] = tmp;
9653  }
9654  ++nelems;
9655  }
9656  }
9657  if (offset != 0) { return; }
9658  }
9659 
9660  if (offset == 2 || offset == 0) {
9661  if (nddom.contains(i,j-1,k-1)) {
9662  Real tmp = Real(0.);
9663  if (ccdom.contains(i-1,j-1,k-1)) {
9664  tmp += - sx(i-1,j-1,k-1) * facx
9665  + sy(i-1,j-1,k-1) * facy * Real(2.0)
9666  + sz(i-1,j-1,k-1) * facz * Real(2.0);
9667  }
9668  if (ccdom.contains(i,j-1,k-1)) {
9669  tmp += - sx(i,j-1,k-1) * facx
9670  + sy(i,j-1,k-1) * facy * Real(2.0)
9671  + sz(i,j-1,k-1) * facz * Real(2.0);
9672  }
9673  m0 -= tmp;
9674  if (gid(i,j-1,k-1) < gidmax) {
9675  if (offset != 0) {
9676  cols[ps] = gid(i,j-1,k-1);
9677  mat[ps] = tmp;
9678  }
9679  ++nelems;
9680  }
9681  }
9682  if (offset != 0) { return; }
9683  }
9684 
9685  if (offset == 3 || offset == 0) {
9686  if (nddom.contains(i+1,j-1,k-1)) {
9687  Real tmp = sx(i ,j-1,k-1) * facx
9688  + sy(i ,j-1,k-1) * facy
9689  + sz(i ,j-1,k-1) * facz;
9690  m0 -= tmp;
9691  if (gid(i+1,j-1,k-1) < gidmax) {
9692  if (offset != 0) {
9693  cols[ps] = gid(i+1,j-1,k-1);
9694  mat[ps] = tmp;
9695  }
9696  ++nelems;
9697  }
9698  }
9699  if (offset != 0) { return; }
9700  }
9701 
9702  if (offset == 4 || offset == 0) {
9703  if (nddom.contains(i-1,j,k-1)) {
9704  Real tmp = Real(0.);
9705  if (ccdom.contains(i-1,j-1,k-1)) {
9706  tmp += sx(i-1,j-1,k-1) * facx * Real(2.0)
9707  - sy(i-1,j-1,k-1) * facy
9708  + sz(i-1,j-1,k-1) * facz * Real(2.0);
9709  }
9710  if (ccdom.contains(i-1,j,k-1)) {
9711  tmp += sx(i-1,j,k-1) * facx * Real(2.0)
9712  - sy(i-1,j,k-1) * facy
9713  + sz(i-1,j,k-1) * facz * Real(2.0);
9714  }
9715  m0 -= tmp;
9716  if (gid(i-1,j,k-1) < gidmax) {
9717  if (offset != 0) {
9718  cols[ps] = gid(i-1,j,k-1);
9719  mat[ps] = tmp;
9720  }
9721  ++nelems;
9722  }
9723  }
9724  if (offset != 0) { return; }
9725  }
9726 
9727  if (offset == 5 || offset == 0) {
9728  if (nddom.contains(i,j,k-1)) {
9729  Real tmp = Real(0.);
9730  if (ccdom.contains(i-1,j-1,k-1)) {
9731  tmp += - sx(i-1,j-1,k-1) * facx * Real(2.0)
9732  - sy(i-1,j-1,k-1) * facy * Real(2.0)
9733  + sz(i-1,j-1,k-1) * facz * Real(4.0);
9734  }
9735  if (ccdom.contains(i,j-1,k-1)) {
9736  tmp += - sx(i,j-1,k-1) * facx * Real(2.0)
9737  - sy(i,j-1,k-1) * facy * Real(2.0)
9738  + sz(i,j-1,k-1) * facz * Real(4.0);
9739 
9740  }
9741  if (ccdom.contains(i-1,j,k-1)) {
9742  tmp += - sx(i-1,j,k-1) * facx * Real(2.0)
9743  - sy(i-1,j,k-1) * facy * Real(2.0)
9744  + sz(i-1,j,k-1) * facz * Real(4.0);
9745  }
9746  if (ccdom.contains(i,j,k-1)) {
9747  tmp += - sx(i,j,k-1) * facx * Real(2.0)
9748  - sy(i,j,k-1) * facy * Real(2.0)
9749  + sz(i,j,k-1) * facz * Real(4.0);
9750  }
9751  m0 -= tmp;
9752  if (gid(i,j,k-1) < gidmax) {
9753  if (offset != 0) {
9754  cols[ps] = gid(i,j,k-1);
9755  mat[ps] = tmp;
9756  }
9757  ++nelems;
9758  }
9759  }
9760  if (offset != 0) { return; }
9761  }
9762 
9763  if (offset == 6 || offset == 0) {
9764  if (nddom.contains(i+1,j,k-1)) {
9765  Real tmp = Real(0.);
9766  if (ccdom.contains(i ,j-1,k-1)) {
9767  tmp += sx(i ,j-1,k-1) * facx * Real(2.0)
9768  - sy(i ,j-1,k-1) * facy
9769  + sz(i ,j-1,k-1) * facz * Real(2.0);
9770  }
9771  if (ccdom.contains(i ,j,k-1)) {
9772  tmp += sx(i ,j,k-1) * facx * Real(2.0)
9773  - sy(i ,j,k-1) * facy
9774  + sz(i ,j,k-1) * facz * Real(2.0);
9775  }
9776  m0 -= tmp;
9777  if (gid(i+1,j,k-1) < gidmax) {
9778  if (offset != 0) {
9779  cols[ps] = gid(i+1,j,k-1);
9780  mat[ps] = tmp;
9781  }
9782  ++nelems;
9783  }
9784  }
9785  if (offset != 0) { return; }
9786  }
9787 
9788  if (offset == 7 || offset == 0) {
9789  if (nddom.contains(i-1,j+1,k-1)) {
9790  Real tmp = sx(i-1,j ,k-1) * facx
9791  + sy(i-1,j ,k-1) * facy
9792  + sz(i-1,j ,k-1) * facz;
9793  m0 -= tmp;
9794  if (gid(i-1,j+1,k-1) < gidmax) {
9795  if (offset != 0) {
9796  cols[ps] = gid(i-1,j+1,k-1);
9797  mat[ps] = tmp;
9798  }
9799  ++nelems;
9800  }
9801  }
9802  if (offset != 0) { return; }
9803  }
9804 
9805  if (offset == 8 || offset == 0) {
9806  if (nddom.contains(i,j+1,k-1)) {
9807  Real tmp = Real(0.);
9808  if (ccdom.contains(i-1,j ,k-1)) {
9809  tmp += - sx(i-1,j ,k-1) * facx
9810  + sy(i-1,j ,k-1) * facy * Real(2.0)
9811  + sz(i-1,j ,k-1) * facz * Real(2.0);
9812  }
9813  if (ccdom.contains(i,j ,k-1)) {
9814  tmp += - sx(i,j ,k-1) * facx
9815  + sy(i,j ,k-1) * facy * Real(2.0)
9816  + sz(i,j ,k-1) * facz * Real(2.0);
9817  }
9818  m0 -= tmp;
9819  if (gid(i,j+1,k-1) < gidmax) {
9820  if (offset != 0) {
9821  cols[ps] = gid(i,j+1,k-1);
9822  mat[ps] = tmp;
9823  }
9824  ++nelems;
9825  }
9826  }
9827  if (offset != 0) { return; }
9828  }
9829 
9830  if (offset == 9 || offset == 0) {
9831  if (nddom.contains(i+1,j+1,k-1)) {
9832  Real tmp = sx(i ,j ,k-1) * facx
9833  + sy(i ,j ,k-1) * facy
9834  + sz(i ,j ,k-1) * facz;
9835  m0 -= tmp;
9836  if (gid(i+1,j+1,k-1) < gidmax) {
9837  if (offset != 0) {
9838  cols[ps] = gid(i+1,j+1,k-1);
9839  mat[ps] = tmp;
9840  }
9841  ++nelems;
9842  }
9843  }
9844  if (offset != 0) { return; }
9845  }
9846 
9847  if (offset == 10 || offset == 0) {
9848  if (nddom.contains(i-1,j-1,k)) {
9849  Real tmp = Real(0.);
9850  if (ccdom.contains(i-1,j-1,k-1)) {
9851  tmp += sx(i-1,j-1,k-1) * facx * Real(2.0)
9852  + sy(i-1,j-1,k-1) * facy * Real(2.0)
9853  - sz(i-1,j-1,k-1) * facz;
9854  }
9855  if (ccdom.contains(i-1,j-1,k)) {
9856  tmp += sx(i-1,j-1,k) * facx * Real(2.0)
9857  + sy(i-1,j-1,k) * facy * Real(2.0)
9858  - sz(i-1,j-1,k) * facz;
9859  }
9860  m0 -= tmp;
9861  if (gid(i-1,j-1,k) < gidmax) {
9862  if (offset != 0) {
9863  cols[ps] = gid(i-1,j-1,k);
9864  mat[ps] = tmp;
9865  }
9866  ++nelems;
9867  }
9868  }
9869  if (offset != 0) { return; }
9870  }
9871 
9872  if (offset == 11 || offset == 0) {
9873  if (nddom.contains(i,j-1,k)) {
9874  Real tmp = Real(0.);
9875  if (ccdom.contains(i-1,j-1,k-1)) {
9876  tmp += - sx(i-1,j-1,k-1) * facx * Real(2.0)
9877  + sy(i-1,j-1,k-1) * facy * Real(4.0)
9878  - sz(i-1,j-1,k-1) * facz * Real(2.0);
9879  }
9880  if (ccdom.contains(i,j-1,k-1)) {
9881  tmp += - sx(i,j-1,k-1) * facx * Real(2.0)
9882  + sy(i,j-1,k-1) * facy * Real(4.0)
9883  - sz(i,j-1,k-1) * facz * Real(2.0);
9884  }
9885  if (ccdom.contains(i-1,j-1,k)) {
9886  tmp += - sx(i-1,j-1,k) * facx * Real(2.0)
9887  + sy(i-1,j-1,k) * facy * Real(4.0)
9888  - sz(i-1,j-1,k) * facz * Real(2.0);
9889  }
9890  if (ccdom.contains(i,j-1,k)) {
9891  tmp += - sx(i,j-1,k) * facx * Real(2.0)
9892  + sy(i,j-1,k) * facy * Real(4.0)
9893  - sz(i,j-1,k) * facz * Real(2.0);
9894  }
9895  m0 -= tmp;
9896  if (gid(i,j-1,k) < gidmax) {
9897  if (offset != 0) {
9898  cols[ps] = gid(i,j-1,k);
9899  mat[ps] = tmp;
9900  }
9901  ++nelems;
9902  }
9903  }
9904  if (offset != 0) { return; }
9905  }
9906 
9907  if (offset == 12 || offset == 0) {
9908  if (nddom.contains(i+1,j-1,k)) {
9909  Real tmp = Real(0.);
9910  if (ccdom.contains(i ,j-1,k-1)) {
9911  tmp += sx(i ,j-1,k-1) * facx * Real(2.0)
9912  + sy(i ,j-1,k-1) * facy * Real(2.0)
9913  - sz(i ,j-1,k-1) * facz;
9914  }
9915  if (ccdom.contains(i ,j-1,k)) {
9916  tmp += sx(i ,j-1,k) * facx * Real(2.0)
9917  + sy(i ,j-1,k) * facy * Real(2.0)
9918  - sz(i ,j-1,k) * facz;
9919  }
9920  m0 -= tmp;
9921  if (gid(i+1,j-1,k) < gidmax) {
9922  if (offset != 0) {
9923  cols[ps] = gid(i+1,j-1,k);
9924  mat[ps] = tmp;
9925  }
9926  ++nelems;
9927  }
9928  }
9929  if (offset != 0) { return; }
9930  }
9931 
9932  if (offset == 13 || offset == 0) {
9933  if (nddom.contains(i-1,j,k)) {
9934  Real tmp = Real(0.);
9935  if (ccdom.contains(i-1,j-1,k-1)) {
9936  tmp += sx(i-1,j-1,k-1) * facx * Real(4.0)
9937  - sy(i-1,j-1,k-1) * facy * Real(2.0)
9938  - sz(i-1,j-1,k-1) * facz * Real(2.0);
9939  }
9940  if (ccdom.contains(i-1,j,k-1)) {
9941  tmp += sx(i-1,j,k-1) * facx * Real(4.0)
9942  - sy(i-1,j,k-1) * facy * Real(2.0)
9943  - sz(i-1,j,k-1) * facz * Real(2.0);
9944  }
9945  if (ccdom.contains(i-1,j-1,k)) {
9946  tmp += sx(i-1,j-1,k) * facx * Real(4.0)
9947  - sy(i-1,j-1,k) * facy * Real(2.0)
9948  - sz(i-1,j-1,k) * facz * Real(2.0);
9949  }
9950  if (ccdom.contains(i-1,j,k)) {
9951  tmp += sx(i-1,j,k) * facx * Real(4.0)
9952  - sy(i-1,j,k) * facy * Real(2.0)
9953  - sz(i-1,j,k) * facz * Real(2.0);
9954  }
9955  m0 -= tmp;
9956  if (gid(i-1,j,k) < gidmax) {
9957  if (offset != 0) {
9958  cols[ps] = gid(i-1,j,k);
9959  mat[ps] = tmp;
9960  }
9961  ++nelems;
9962  }
9963  }
9964  if (offset != 0) { return; }
9965  }
9966 
9967  if (offset == 14 || offset == 0) {
9968  if (nddom.contains(i+1,j,k)) {
9969  Real tmp = Real(0.);
9970  if (ccdom.contains(i ,j-1,k-1)) {
9971  tmp += sx(i ,j-1,k-1) * facx * Real(4.0)
9972  - sy(i ,j-1,k-1) * facy * Real(2.0)
9973  - sz(i ,j-1,k-1) * facz * Real(2.0);
9974  }
9975  if (ccdom.contains(i ,j,k-1)) {
9976  tmp += sx(i ,j,k-1) * facx * Real(4.0)
9977  - sy(i ,j,k-1) * facy * Real(2.0)
9978  - sz(i ,j,k-1) * facz * Real(2.0);
9979  }
9980  if (ccdom.contains(i ,j-1,k)) {
9981  tmp += sx(i ,j-1,k) * facx * Real(4.0)
9982  - sy(i ,j-1,k) * facy * Real(2.0)
9983  - sz(i ,j-1,k) * facz * Real(2.0);
9984  }
9985  if (ccdom.contains(i ,j,k)) {
9986  tmp += sx(i ,j,k) * facx * Real(4.0)
9987  - sy(i ,j,k) * facy * Real(2.0)
9988  - sz(i ,j,k) * facz * Real(2.0);
9989  }
9990  m0 -= tmp;
9991  if (gid(i+1,j,k) < gidmax) {
9992  if (offset != 0) {
9993  cols[ps] = gid(i+1,j,k);
9994  mat[ps] = tmp;
9995  }
9996  ++nelems;
9997  }
9998  }
9999  if (offset != 0) { return; }
10000  }
10001 
10002  if (offset == 15 || offset == 0) {
10003  if (nddom.contains(i-1,j+1,k)) {
10004  Real tmp = Real(0.);
10005  if (ccdom.contains(i-1,j ,k-1)) {
10006  tmp += sx(i-1,j ,k-1) * facx * Real(2.0)
10007  + sy(i-1,j ,k-1) * facy * Real(2.0)
10008  - sz(i-1,j ,k-1) * facz;
10009  }
10010  if (ccdom.contains(i-1,j ,k)) {
10011  tmp += sx(i-1,j ,k) * facx * Real(2.0)
10012  + sy(i-1,j ,k) * facy * Real(2.0)
10013  - sz(i-1,j ,k) * facz;
10014  }
10015  m0 -= tmp;
10016  if (gid(i-1,j+1,k) < gidmax) {
10017  if (offset != 0) {
10018  cols[ps] = gid(i-1,j+1,k);
10019  mat[ps] = tmp;
10020  }
10021  ++nelems;
10022  }
10023  }
10024  if (offset != 0) { return; }
10025  }
10026 
10027  if (offset == 16 || offset == 0) {
10028  if (nddom.contains(i,j+1,k)) {
10029  Real tmp = Real(0.);
10030  if (ccdom.contains(i-1,j ,k-1)) {
10031  tmp += - sx(i-1,j ,k-1) * facx * Real(2.0)
10032  + sy(i-1,j ,k-1) * facy * Real(4.0)
10033  - sz(i-1,j ,k-1) * facz * Real(2.0);
10034  }
10035  if (ccdom.contains(i,j ,k-1)) {
10036  tmp += - sx(i,j ,k-1) * facx * Real(2.0)
10037  + sy(i,j ,k-1) * facy * Real(4.0)
10038  - sz(i,j ,k-1) * facz * Real(2.0);
10039  }
10040  if (ccdom.contains(i-1,j ,k)) {
10041  tmp += - sx(i-1,j ,k) * facx * Real(2.0)
10042  + sy(i-1,j ,k) * facy * Real(4.0)
10043  - sz(i-1,j ,k) * facz * Real(2.0);
10044  }
10045  if (ccdom.contains(i,j ,k)) {
10046  tmp += - sx(i,j ,k) * facx * Real(2.0)
10047  + sy(i,j ,k) * facy * Real(4.0)
10048  - sz(i,j ,k) * facz * Real(2.0);
10049  }
10050  m0 -= tmp;
10051  if (gid(i,j+1,k) < gidmax) {
10052  if (offset != 0) {
10053  cols[ps] = gid(i,j+1,k);
10054  mat[ps] = tmp;
10055  }
10056  ++nelems;
10057  }
10058  }
10059  if (offset != 0) { return; }
10060  }
10061 
10062  if (offset == 17 || offset == 0) {
10063  if (nddom.contains(i+1,j+1,k)) {
10064  Real tmp = Real(0.);
10065  if (ccdom.contains(i ,j ,k-1)) {
10066  tmp += sx(i ,j ,k-1) * facx * Real(2.0)
10067  + sy(i ,j ,k-1) * facy * Real(2.0)
10068  - sz(i ,j ,k-1) * facz;
10069  }
10070  if (ccdom.contains(i ,j ,k)) {
10071  tmp += sx(i ,j ,k) * facx * Real(2.0)
10072  + sy(i ,j ,k) * facy * Real(2.0)
10073  - sz(i ,j ,k) * facz;
10074  }
10075  m0 -= tmp;
10076  if (gid(i+1,j+1,k) < gidmax) {
10077  if (offset != 0) {
10078  cols[ps] = gid(i+1,j+1,k);
10079  mat[ps] = tmp;
10080  }
10081  ++nelems;
10082  }
10083  }
10084  if (offset != 0) { return; }
10085  }
10086 
10087  if (offset == 18 || offset == 0) {
10088  if (nddom.contains(i-1,j-1,k+1)) {
10089  Real tmp = sx(i-1,j-1,k ) * facx
10090  + sy(i-1,j-1,k ) * facy
10091  + sz(i-1,j-1,k ) * facz;
10092  m0 -= tmp;
10093  if (gid(i-1,j-1,k+1) < gidmax) {
10094  if (offset != 0) {
10095  cols[ps] = gid(i-1,j-1,k+1);
10096  mat[ps] = tmp;
10097  }
10098  ++nelems;
10099  }
10100  }
10101  if (offset != 0) { return; }
10102  }
10103 
10104  if (offset == 19 || offset == 0) {
10105  if (nddom.contains(i,j-1,k+1)) {
10106  Real tmp = Real(0.);
10107  if (ccdom.contains(i-1,j-1,k )) {
10108  tmp += - sx(i-1,j-1,k ) * facx
10109  + sy(i-1,j-1,k ) * facy * Real(2.0)
10110  + sz(i-1,j-1,k ) * facz * Real(2.0);
10111  }
10112  if (ccdom.contains(i,j-1,k )) {
10113  tmp += - sx(i,j-1,k ) * facx
10114  + sy(i,j-1,k ) * facy * Real(2.0)
10115  + sz(i,j-1,k ) * facz * Real(2.0);
10116  }
10117  m0 -= tmp;
10118  if (gid(i,j-1,k+1) < gidmax) {
10119  if (offset != 0) {
10120  cols[ps] = gid(i,j-1,k+1);
10121  mat[ps] = tmp;
10122  }
10123  ++nelems;
10124  }
10125  }
10126  if (offset != 0) { return; }
10127  }
10128 
10129  if (offset == 20 || offset == 0) {
10130  if (nddom.contains(i+1,j-1,k+1)) {
10131  Real tmp = sx(i ,j-1,k ) * facx
10132  + sy(i ,j-1,k ) * facy
10133  + sz(i ,j-1,k ) * facz;
10134  m0 -= tmp;
10135  if (gid(i+1,j-1,k+1) < gidmax) {
10136  if (offset != 0) {
10137  cols[ps] = gid(i+1,j-1,k+1);
10138  mat[ps] = tmp;
10139  }
10140  ++nelems;
10141  }
10142  }
10143  if (offset != 0) { return; }
10144  }
10145 
10146  if (offset == 21 || offset == 0) {
10147  if (nddom.contains(i-1,j,k+1)) {
10148  Real tmp = Real(0.);
10149  if (ccdom.contains(i-1,j-1,k )) {
10150  tmp += sx(i-1,j-1,k ) * facx * Real(2.0)
10151  - sy(i-1,j-1,k ) * facy
10152  + sz(i-1,j-1,k ) * facz * Real(2.0);
10153  }
10154  if (ccdom.contains(i-1,j,k )) {
10155  tmp += sx(i-1,j,k ) * facx * Real(2.0)
10156  - sy(i-1,j,k ) * facy
10157  + sz(i-1,j,k ) * facz * Real(2.0);
10158  }
10159  m0 -= tmp;
10160  if (gid(i-1,j,k+1) < gidmax) {
10161  if (offset != 0) {
10162  cols[ps] = gid(i-1,j,k+1);
10163  mat[ps] = tmp;
10164  }
10165  ++nelems;
10166  }
10167  }
10168  if (offset != 0) { return; }
10169  }
10170 
10171  if (offset == 22 || offset == 0) {
10172  if (nddom.contains(i,j,k+1)) {
10173  Real tmp = Real(0.);
10174  if (ccdom.contains(i-1,j-1,k )) {
10175  tmp += - sx(i-1,j-1,k ) * facx * Real(2.0)
10176  - sy(i-1,j-1,k ) * facy * Real(2.0)
10177  + sz(i-1,j-1,k ) * facz * Real(4.0);
10178  }
10179  if (ccdom.contains(i,j-1,k )) {
10180  tmp += - sx(i,j-1,k ) * facx * Real(2.0)
10181  - sy(i,j-1,k ) * facy * Real(2.0)
10182  + sz(i,j-1,k ) * facz * Real(4.0);
10183  }
10184  if (ccdom.contains(i-1,j,k )) {
10185  tmp += - sx(i-1,j,k ) * facx * Real(2.0)
10186  - sy(i-1,j,k ) * facy * Real(2.0)
10187  + sz(i-1,j,k ) * facz * Real(4.0);
10188  }
10189  if (ccdom.contains(i,j,k )) {
10190  tmp += - sx(i,j,k ) * facx * Real(2.0)
10191  - sy(i,j,k ) * facy * Real(2.0)
10192  + sz(i,j,k ) * facz * Real(4.0);
10193  }
10194  m0 -= tmp;
10195  if (gid(i,j,k+1) < gidmax) {
10196  if (offset != 0) {
10197  cols[ps] = gid(i,j,k+1);
10198  mat[ps] = tmp;
10199  }
10200  ++nelems;
10201  }
10202  }
10203  if (offset != 0) { return; }
10204  }
10205 
10206  if (offset == 23 || offset == 0) {
10207  if (nddom.contains(i+1,j,k+1)) {
10208  Real tmp = Real(0.);
10209  if (ccdom.contains(i ,j-1,k )) {
10210  tmp += sx(i ,j-1,k ) * facx * Real(2.0)
10211  - sy(i ,j-1,k ) * facy
10212  + sz(i ,j-1,k ) * facz * Real(2.0);
10213  }
10214  if (ccdom.contains(i ,j,k )) {
10215  tmp += sx(i ,j,k ) * facx * Real(2.0)
10216  - sy(i ,j,k ) * facy
10217  + sz(i ,j,k ) * facz * Real(2.0);
10218  }
10219  m0 -= tmp;
10220  if (gid(i+1,j,k+1) < gidmax) {
10221  if (offset != 0) {
10222  cols[ps] = gid(i+1,j,k+1);
10223  mat[ps] = tmp;
10224  }
10225  ++nelems;
10226  }
10227  }
10228  if (offset != 0) { return; }
10229  }
10230 
10231  if (offset == 24 || offset == 0) {
10232  if (nddom.contains(i-1,j+1,k+1)) {
10233  Real tmp = sx(i-1,j ,k ) * facx
10234  + sy(i-1,j ,k ) * facy
10235  + sz(i-1,j ,k ) * facz;
10236  m0 -= tmp;
10237  if (gid(i-1,j+1,k+1) < gidmax) {
10238  if (offset != 0) {
10239  cols[ps] = gid(i-1,j+1,k+1);
10240  mat[ps] = tmp;
10241  }
10242  ++nelems;
10243  }
10244  }
10245  if (offset != 0) { return; }
10246  }
10247 
10248  if (offset == 25 || offset == 0) {
10249  if (nddom.contains(i,j+1,k+1)) {
10250  Real tmp = Real(0.);
10251  if (ccdom.contains(i-1,j ,k )) {
10252  tmp += - sx(i-1,j ,k ) * facx
10253  + sy(i-1,j ,k ) * facy * Real(2.0)
10254  + sz(i-1,j ,k ) * facz * Real(2.0);
10255  }
10256  if (ccdom.contains(i,j ,k )) {
10257  tmp += - sx(i,j ,k ) * facx
10258  + sy(i,j ,k ) * facy * Real(2.0)
10259  + sz(i,j ,k ) * facz * Real(2.0);
10260  }
10261  m0 -= tmp;
10262  if (gid(i,j+1,k+1) < gidmax) {
10263  if (offset != 0) {
10264  cols[ps] = gid(i,j+1,k+1);
10265  mat[ps] = tmp;
10266  }
10267  ++nelems;
10268  }
10269  }
10270  if (offset != 0) { return; }
10271  }
10272 
10273  if (offset == 26 || offset == 0) {
10274  if (nddom.contains(i+1,j+1,k+1)) {
10275  Real tmp = sx(i ,j ,k ) * facx
10276  + sy(i ,j ,k ) * facy
10277  + sz(i ,j ,k ) * facz;
10278  m0 -= tmp;
10279  if (gid(i+1,j+1,k+1) < gidmax) {
10280  if (offset != 0) {
10281  cols[ps] = gid(i+1,j+1,k+1);
10282  mat[ps] = tmp;
10283  }
10284  ++nelems;
10285  }
10286  }
10287  if (offset != 0) { return; }
10288  }
10289 
10290  // Only offset == 0 could get this far.
10291  cols[ps] = gid(i,j,k);
10292  mat[ps] = m0;
10293  ncols[lid(i,j,k)] = nelems+1;
10294  }
10295 }
10296 
10297 template <typename HypreInt, typename AtomicInt>
10298 AMREX_GPU_DEVICE AMREX_FORCE_INLINE
10299 void mlndlap_fillijmat_cs_gpu (const int ps, const int i, const int j, const int k,
10300  const int offset,
10301  Box const& ndbx, Array4<AtomicInt const> const& gid,
10302  Array4<int const> const& lid,
10303  HypreInt* ncols, HypreInt* cols,
10304  Real* mat, // NOLINT(readability-non-const-parameter)
10305  Real sigma, GpuArray<Real,AMREX_SPACEDIM> const& dxinv,
10306  Box const& ccdom) noexcept
10307 {
10308  if (lid(i,j,k) >= 0)
10309  {
10310  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0] * sigma;
10311  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1] * sigma;
10312  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2] * sigma;
10313  Real fxyz = facx + facy + facz;
10314  Real fmx2y2z = -facx + Real(2.0)*facy + Real(2.0)*facz;
10315  Real f2xmy2z = Real(2.0)*facx - facy + Real(2.0)*facz;
10316  Real f2x2ymz = Real(2.0)*facx + Real(2.0)*facy - facz;
10317  Real f4xm2ym2z = Real(4.0)*facx - Real(2.0)*facy - Real(2.0)*facz;
10318  Real fm2x4ym2z = -Real(2.0)*facx + Real(4.0)*facy - Real(2.0)*facz;
10319  Real fm2xm2y4z = -Real(2.0)*facx - Real(2.0)*facy + Real(4.0)*facz;
10320 
10321  const Box& nddom = amrex::surroundingNodes(ccdom);
10322 
10323  constexpr auto gidmax = std::numeric_limits<AtomicInt>::max();
10324  int nelems = 0;
10325  Real m0 = Real(0.);
10326 
10327  if (offset == 1 || offset == 0) {
10328  if (nddom.contains(i-1,j-1,k-1)) {
10329  Real tmp = fxyz;
10330  m0 -= tmp;
10331  if ( gid(i-1,j-1,k-1) < gidmax) {
10332  if (offset != 0) {
10333  cols[ps] = gid(i-1,j-1,k-1);
10334  mat[ps] = tmp;
10335  }
10336  ++nelems;
10337  }
10338  }
10339  if (offset != 0) { return; }
10340  }
10341 
10342  if (offset == 2 || offset == 0) {
10343  if (nddom.contains(i,j-1,k-1)) {
10344  Real tmp = Real(0.);
10345  if (ccdom.contains(i-1,j-1,k-1)) {
10346  tmp += fmx2y2z;
10347  }
10348  if (ccdom.contains(i,j-1,k-1)) {
10349  tmp += fmx2y2z;
10350  }
10351  m0 -= tmp;
10352  if (gid(i,j-1,k-1) < gidmax) {
10353  if (offset != 0) {
10354  cols[ps] = gid(i,j-1,k-1);
10355  mat[ps] = tmp;
10356  }
10357  ++nelems;
10358  }
10359  }
10360  if (offset != 0) { return; }
10361  }
10362 
10363  if (offset == 3 || offset == 0) {
10364  if (nddom.contains(i+1,j-1,k-1)) {
10365  Real tmp = fxyz;
10366  m0 -= tmp;
10367  if (gid(i+1,j-1,k-1) < gidmax) {
10368  if (offset != 0) {
10369  cols[ps] = gid(i+1,j-1,k-1);
10370  mat[ps] = tmp;
10371  }
10372  ++nelems;
10373  }
10374  }
10375  if (offset != 0) { return; }
10376  }
10377 
10378  if (offset == 4 || offset == 0) {
10379  if (nddom.contains(i-1,j,k-1)) {
10380  Real tmp = Real(0.);
10381  if (ccdom.contains(i-1,j-1,k-1)) {
10382  tmp += f2xmy2z;
10383  }
10384  if (ccdom.contains(i-1,j,k-1)) {
10385  tmp += f2xmy2z;
10386  }
10387  m0 -= tmp;
10388  if (gid(i-1,j,k-1) < gidmax) {
10389  if (offset != 0) {
10390  cols[ps] = gid(i-1,j,k-1);
10391  mat[ps] = tmp;
10392  }
10393  ++nelems;
10394  }
10395  }
10396  if (offset != 0) { return; }
10397  }
10398 
10399  if (offset == 5 || offset == 0) {
10400  if (nddom.contains(i,j,k-1)) {
10401  Real tmp = Real(0.);
10402  if (ccdom.contains(i-1,j-1,k-1)) {
10403  tmp += fm2xm2y4z;
10404  }
10405  if (ccdom.contains(i,j-1,k-1)) {
10406  tmp += fm2xm2y4z;
10407  }
10408  if (ccdom.contains(i-1,j,k-1)) {
10409  tmp += fm2xm2y4z;
10410  }
10411  if (ccdom.contains(i,j,k-1)) {
10412  tmp += fm2xm2y4z;
10413  }
10414  m0 -= tmp;
10415  if (gid(i,j,k-1) < gidmax) {
10416  if (offset != 0) {
10417  cols[ps] = gid(i,j,k-1);
10418  mat[ps] = tmp;
10419  }
10420  ++nelems;
10421  }
10422  }
10423  if (offset != 0) { return; }
10424  }
10425 
10426  if (offset == 6 || offset == 0) {
10427  if (nddom.contains(i+1,j,k-1)) {
10428  Real tmp = Real(0.);
10429  if (ccdom.contains(i ,j-1,k-1)) {
10430  tmp += f2xmy2z;
10431  }
10432  if (ccdom.contains(i ,j,k-1)) {
10433  tmp += f2xmy2z;
10434  }
10435  m0 -= tmp;
10436  if (gid(i+1,j,k-1) < gidmax) {
10437  if (offset != 0) {
10438  cols[ps] = gid(i+1,j,k-1);
10439  mat[ps] = tmp;
10440  }
10441  ++nelems;
10442  }
10443  }
10444  if (offset != 0) { return; }
10445  }
10446 
10447  if (offset == 7 || offset == 0) {
10448  if (nddom.contains(i-1,j+1,k-1)) {
10449  Real tmp = fxyz;
10450  m0 -= tmp;
10451  if (gid(i-1,j+1,k-1) < gidmax) {
10452  if (offset != 0) {
10453  cols[ps] = gid(i-1,j+1,k-1);
10454  mat[ps] = tmp;
10455  }
10456  ++nelems;
10457  }
10458  }
10459  if (offset != 0) { return; }
10460  }
10461 
10462  if (offset == 8 || offset == 0) {
10463  if (nddom.contains(i,j+1,k-1)) {
10464  Real tmp = Real(0.);
10465  if (ccdom.contains(i-1,j ,k-1)) {
10466  tmp += fmx2y2z;
10467  }
10468  if (ccdom.contains(i,j ,k-1)) {
10469  tmp += fmx2y2z;
10470  }
10471  m0 -= tmp;
10472  if (gid(i,j+1,k-1) < gidmax) {
10473  if (offset != 0) {
10474  cols[ps] = gid(i,j+1,k-1);
10475  mat[ps] = tmp;
10476  }
10477  ++nelems;
10478  }
10479  }
10480  if (offset != 0) { return; }
10481  }
10482 
10483  if (offset == 9 || offset == 0) {
10484  if (nddom.contains(i+1,j+1,k-1)) {
10485  Real tmp = fxyz;
10486  m0 -= tmp;
10487  if (gid(i+1,j+1,k-1) < gidmax) {
10488  if (offset != 0) {
10489  cols[ps] = gid(i+1,j+1,k-1);
10490  mat[ps] = tmp;
10491  }
10492  ++nelems;
10493  }
10494  }
10495  if (offset != 0) { return; }
10496  }
10497 
10498  if (offset == 10 || offset == 0) {
10499  if (nddom.contains(i-1,j-1,k)) {
10500  Real tmp = Real(0.);
10501  if (ccdom.contains(i-1,j-1,k-1)) {
10502  tmp += f2x2ymz;
10503  }
10504  if (ccdom.contains(i-1,j-1,k)) {
10505  tmp += f2x2ymz;
10506  }
10507  m0 -= tmp;
10508  if (gid(i-1,j-1,k) < gidmax) {
10509  if (offset != 0) {
10510  cols[ps] = gid(i-1,j-1,k);
10511  mat[ps] = tmp;
10512  }
10513  ++nelems;
10514  }
10515  }
10516  if (offset != 0) { return; }
10517  }
10518 
10519  if (offset == 11 || offset == 0) {
10520  if (nddom.contains(i,j-1,k)) {
10521  Real tmp = Real(0.);
10522  if (ccdom.contains(i-1,j-1,k-1)) {
10523  tmp += fm2x4ym2z;
10524  }
10525  if (ccdom.contains(i,j-1,k-1)) {
10526  tmp += fm2x4ym2z;
10527  }
10528  if (ccdom.contains(i-1,j-1,k)) {
10529  tmp += fm2x4ym2z;
10530  }
10531  if (ccdom.contains(i,j-1,k)) {
10532  tmp += fm2x4ym2z;
10533  }
10534  m0 -= tmp;
10535  if (gid(i,j-1,k) < gidmax) {
10536  if (offset != 0) {
10537  cols[ps] = gid(i,j-1,k);
10538  mat[ps] = tmp;
10539  }
10540  ++nelems;
10541  }
10542  }
10543  if (offset != 0) { return; }
10544  }
10545 
10546  if (offset == 12 || offset == 0) {
10547  if (nddom.contains(i+1,j-1,k)) {
10548  Real tmp = Real(0.);
10549  if (ccdom.contains(i ,j-1,k-1)) {
10550  tmp += f2x2ymz;
10551  }
10552  if (ccdom.contains(i ,j-1,k)) {
10553  tmp += f2x2ymz;
10554  }
10555  m0 -= tmp;
10556  if (gid(i+1,j-1,k) < gidmax) {
10557  if (offset != 0) {
10558  cols[ps] = gid(i+1,j-1,k);
10559  mat[ps] = tmp;
10560  }
10561  ++nelems;
10562  }
10563  }
10564  if (offset != 0) { return; }
10565  }
10566 
10567  if (offset == 13 || offset == 0) {
10568  if (nddom.contains(i-1,j,k)) {
10569  Real tmp = Real(0.);
10570  if (ccdom.contains(i-1,j-1,k-1)) {
10571  tmp += f4xm2ym2z;
10572  }
10573  if (ccdom.contains(i-1,j,k-1)) {
10574  tmp += f4xm2ym2z;
10575  }
10576  if (ccdom.contains(i-1,j-1,k)) {
10577  tmp += f4xm2ym2z;
10578  }
10579  if (ccdom.contains(i-1,j,k)) {
10580  tmp += f4xm2ym2z;
10581  }
10582  m0 -= tmp;
10583  if (gid(i-1,j,k) < gidmax) {
10584  if (offset != 0) {
10585  cols[ps] = gid(i-1,j,k);
10586  mat[ps] = tmp;
10587  }
10588  ++nelems;
10589  }
10590  }
10591  if (offset != 0) { return; }
10592  }
10593 
10594  if (offset == 14 || offset == 0) {
10595  if (nddom.contains(i+1,j,k)) {
10596  Real tmp = Real(0.);
10597  if (ccdom.contains(i ,j-1,k-1)) {
10598  tmp += f4xm2ym2z;
10599  }
10600  if (ccdom.contains(i ,j,k-1)) {
10601  tmp += f4xm2ym2z;
10602  }
10603  if (ccdom.contains(i ,j-1,k)) {
10604  tmp += f4xm2ym2z;
10605  }
10606  if (ccdom.contains(i ,j,k)) {
10607  tmp += f4xm2ym2z;
10608  }
10609  m0 -= tmp;
10610  if (gid(i+1,j,k) < gidmax) {
10611  if (offset != 0) {
10612  cols[ps] = gid(i+1,j,k);
10613  mat[ps] = tmp;
10614  }
10615  ++nelems;
10616  }
10617  }
10618  if (offset != 0) { return; }
10619  }
10620 
10621  if (offset == 15 || offset == 0) {
10622  if (nddom.contains(i-1,j+1,k)) {
10623  Real tmp = Real(0.);
10624  if (ccdom.contains(i-1,j ,k-1)) {
10625  tmp += f2x2ymz;
10626  }
10627  if (ccdom.contains(i-1,j ,k)) {
10628  tmp += f2x2ymz;
10629  }
10630  m0 -= tmp;
10631  if (gid(i-1,j+1,k) < gidmax) {
10632  if (offset != 0) {
10633  cols[ps] = gid(i-1,j+1,k);
10634  mat[ps] = tmp;
10635  }
10636  ++nelems;
10637  }
10638  }
10639  if (offset != 0) { return; }
10640  }
10641 
10642  if (offset == 16 || offset == 0) {
10643  if (nddom.contains(i,j+1,k)) {
10644  Real tmp = Real(0.);
10645  if (ccdom.contains(i-1,j ,k-1)) {
10646  tmp += fm2x4ym2z;
10647  }
10648  if (ccdom.contains(i,j ,k-1)) {
10649  tmp += fm2x4ym2z;
10650  }
10651  if (ccdom.contains(i-1,j ,k)) {
10652  tmp += fm2x4ym2z;
10653  }
10654  if (ccdom.contains(i,j ,k)) {
10655  tmp += fm2x4ym2z;
10656  }
10657  m0 -= tmp;
10658  if (gid(i,j+1,k) < gidmax) {
10659  if (offset != 0) {
10660  cols[ps] = gid(i,j+1,k);
10661  mat[ps] = tmp;
10662  }
10663  ++nelems;
10664  }
10665  }
10666  if (offset != 0) { return; }
10667  }
10668 
10669  if (offset == 17 || offset == 0) {
10670  if (nddom.contains(i+1,j+1,k)) {
10671  Real tmp = Real(0.);
10672  if (ccdom.contains(i ,j ,k-1)) {
10673  tmp += f2x2ymz;
10674  }
10675  if (ccdom.contains(i ,j ,k)) {
10676  tmp += f2x2ymz;
10677  }
10678  m0 -= tmp;
10679  if (gid(i+1,j+1,k) < gidmax) {
10680  if (offset != 0) {
10681  cols[ps] = gid(i+1,j+1,k);
10682  mat[ps] = tmp;
10683  }
10684  ++nelems;
10685  }
10686  }
10687  if (offset != 0) { return; }
10688  }
10689 
10690  if (offset == 18 || offset == 0) {
10691  if (nddom.contains(i-1,j-1,k+1)) {
10692  Real tmp = fxyz;
10693  m0 -= tmp;
10694  if (gid(i-1,j-1,k+1) < gidmax) {
10695  if (offset != 0) {
10696  cols[ps] = gid(i-1,j-1,k+1);
10697  mat[ps] = tmp;
10698  }
10699  ++nelems;
10700  }
10701  }
10702  if (offset != 0) { return; }
10703  }
10704 
10705  if (offset == 19 || offset == 0) {
10706  if (nddom.contains(i,j-1,k+1)) {
10707  Real tmp = Real(0.);
10708  if (ccdom.contains(i-1,j-1,k )) {
10709  tmp += fmx2y2z;
10710  }
10711  if (ccdom.contains(i,j-1,k )) {
10712  tmp += fmx2y2z;
10713  }
10714  m0 -= tmp;
10715  if (gid(i,j-1,k+1) < gidmax) {
10716  if (offset != 0) {
10717  cols[ps] = gid(i,j-1,k+1);
10718  mat[ps] = tmp;
10719  }
10720  ++nelems;
10721  }
10722  }
10723  if (offset != 0) { return; }
10724  }
10725 
10726  if (offset == 20 || offset == 0) {
10727  if (nddom.contains(i+1,j-1,k+1)) {
10728  Real tmp = fxyz;
10729  m0 -= tmp;
10730  if (gid(i+1,j-1,k+1) < gidmax) {
10731  if (offset != 0) {
10732  cols[ps] = gid(i+1,j-1,k+1);
10733  mat[ps] = tmp;
10734  }
10735  ++nelems;
10736  }
10737  }
10738  if (offset != 0) { return; }
10739  }
10740 
10741  if (offset == 21 || offset == 0) {
10742  if (nddom.contains(i-1,j,k+1)) {
10743  Real tmp = Real(0.);
10744  if (ccdom.contains(i-1,j-1,k )) {
10745  tmp += f2xmy2z;
10746  }
10747  if (ccdom.contains(i-1,j,k )) {
10748  tmp += f2xmy2z;
10749  }
10750  m0 -= tmp;
10751  if (gid(i-1,j,k+1) < gidmax) {
10752  if (offset != 0) {
10753  cols[ps] = gid(i-1,j,k+1);
10754  mat[ps] = tmp;
10755  }
10756  ++nelems;
10757  }
10758  }
10759  if (offset != 0) { return; }
10760  }
10761 
10762  if (offset == 22 || offset == 0) {
10763  if (nddom.contains(i,j,k+1)) {
10764  Real tmp = Real(0.);
10765  if (ccdom.contains(i-1,j-1,k )) {
10766  tmp += fm2xm2y4z;
10767  }
10768  if (ccdom.contains(i,j-1,k )) {
10769  tmp += fm2xm2y4z;
10770  }
10771  if (ccdom.contains(i-1,j,k )) {
10772  tmp += fm2xm2y4z;
10773  }
10774  if (ccdom.contains(i,j,k )) {
10775  tmp += fm2xm2y4z;
10776  }
10777  m0 -= tmp;
10778  if (gid(i,j,k+1) < gidmax) {
10779  if (offset != 0) {
10780  cols[ps] = gid(i,j,k+1);
10781  mat[ps] = tmp;
10782  }
10783  ++nelems;
10784  }
10785  }
10786  if (offset != 0) { return; }
10787  }
10788 
10789  if (offset == 23 || offset == 0) {
10790  if (nddom.contains(i+1,j,k+1)) {
10791  Real tmp = Real(0.);
10792  if (ccdom.contains(i ,j-1,k )) {
10793  tmp += f2xmy2z;
10794  }
10795  if (ccdom.contains(i ,j,k )) {
10796  tmp += f2xmy2z;
10797  }
10798  m0 -= tmp;
10799  if (gid(i+1,j,k+1) < gidmax) {
10800  if (offset != 0) {
10801  cols[ps] = gid(i+1,j,k+1);
10802  mat[ps] = tmp;
10803  }
10804  ++nelems;
10805  }
10806  }
10807  if (offset != 0) { return; }
10808  }
10809 
10810  if (offset == 24 || offset == 0) {
10811  if (nddom.contains(i-1,j+1,k+1)) {
10812  Real tmp = fxyz;
10813  m0 -= tmp;
10814  if (gid(i-1,j+1,k+1) < gidmax) {
10815  if (offset != 0) {
10816  cols[ps] = gid(i-1,j+1,k+1);
10817  mat[ps] = tmp;
10818  }
10819  ++nelems;
10820  }
10821  }
10822  if (offset != 0) { return; }
10823  }
10824 
10825  if (offset == 25 || offset == 0) {
10826  if (nddom.contains(i,j+1,k+1)) {
10827  Real tmp = Real(0.);
10828  if (ccdom.contains(i-1,j ,k )) {
10829  tmp += fmx2y2z;
10830  }
10831  if (ccdom.contains(i,j ,k )) {
10832  tmp += fmx2y2z;
10833  }
10834  m0 -= tmp;
10835  if (gid(i,j+1,k+1) < gidmax) {
10836  if (offset != 0) {
10837  cols[ps] = gid(i,j+1,k+1);
10838  mat[ps] = tmp;
10839  }
10840  ++nelems;
10841  }
10842  }
10843  if (offset != 0) { return; }
10844  }
10845 
10846  if (offset == 26 || offset == 0) {
10847  if (nddom.contains(i+1,j+1,k+1)) {
10848  Real tmp = fxyz;
10849  m0 -= tmp;
10850  if (gid(i+1,j+1,k+1) < gidmax) {
10851  if (offset != 0) {
10852  cols[ps] = gid(i+1,j+1,k+1);
10853  mat[ps] = tmp;
10854  }
10855  ++nelems;
10856  }
10857  }
10858  if (offset != 0) { return; }
10859  }
10860 
10861  // Only offset == 0 could get this far.
10862  cols[ps] = gid(i,j,k);
10863  mat[ps] = m0;
10864  ncols[lid(i,j,k)] = nelems+1;
10865  }
10866 }
10867 
10868 #endif
10869 
10870 #endif
10871 
10872 AMREX_GPU_DEVICE AMREX_FORCE_INLINE
10873 int mlndlap_color (int i, int j, int k)
10874 {
10875  return (i%2) + (j%2)*2 + (k%2)*4;
10876 }
10877 
10878 AMREX_GPU_DEVICE AMREX_FORCE_INLINE
10879 void mlndlap_gscolor_ha (int i, int j, int k, Array4<Real> const& sol,
10880  Array4<Real const> const& rhs, Array4<Real const> const& sx,
10881  Array4<Real const> const& sy, Array4<Real const> const& sz,
10882  Array4<int const> const& msk,
10883  GpuArray<Real,AMREX_SPACEDIM> const& dxinv, int color) noexcept
10884 {
10885  if (mlndlap_color(i,j,k) == color) {
10886  if (msk(i,j,k)) {
10887  sol(i,j,k) = Real(0.0);
10888  } else {
10889  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0];
10890  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1];
10891  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2];
10892 
10893  Real s0 = Real(-4.0)*(facx*(sx(i-1,j-1,k-1)+sx(i,j-1,k-1)+sx(i-1,j,k-1)+sx(i,j,k-1)
10894  +sx(i-1,j-1,k )+sx(i,j-1,k )+sx(i-1,j,k )+sx(i,j,k ))
10895  +facy*(sy(i-1,j-1,k-1)+sy(i,j-1,k-1)+sy(i-1,j,k-1)+sy(i,j,k-1)
10896  +sy(i-1,j-1,k )+sy(i,j-1,k )+sy(i-1,j,k )+sy(i,j,k ))
10897  +facz*(sz(i-1,j-1,k-1)+sz(i,j-1,k-1)+sz(i-1,j,k-1)+sz(i,j,k-1)
10898  +sz(i-1,j-1,k )+sz(i,j-1,k )+sz(i-1,j,k )+sz(i,j,k )));
10899  Real Ax = sol(i,j,k)*s0
10900  + sol(i-1,j-1,k-1)*(facx*sx(i-1,j-1,k-1)
10901  +facy*sy(i-1,j-1,k-1)
10902  +facz*sz(i-1,j-1,k-1))
10903  + sol(i+1,j-1,k-1)*(facx*sx(i ,j-1,k-1)
10904  +facy*sy(i ,j-1,k-1)
10905  +facz*sz(i ,j-1,k-1))
10906  + sol(i-1,j+1,k-1)*(facx*sx(i-1,j ,k-1)
10907  +facy*sy(i-1,j ,k-1)
10908  +facz*sz(i-1,j ,k-1))
10909  + sol(i+1,j+1,k-1)*(facx*sx(i ,j ,k-1)
10910  +facy*sy(i ,j ,k-1)
10911  +facz*sz(i ,j ,k-1))
10912  + sol(i-1,j-1,k+1)*(facx*sx(i-1,j-1,k )
10913  +facy*sy(i-1,j-1,k )
10914  +facz*sz(i-1,j-1,k ))
10915  + sol(i+1,j-1,k+1)*(facx*sx(i ,j-1,k )
10916  +facy*sy(i ,j-1,k )
10917  +facz*sz(i ,j-1,k ))
10918  + sol(i-1,j+1,k+1)*(facx*sx(i-1,j ,k )
10919  +facy*sy(i-1,j ,k )
10920  +facz*sz(i-1,j ,k ))
10921  + sol(i+1,j+1,k+1)*(facx*sx(i ,j ,k )
10922  +facy*sy(i ,j ,k )
10923  +facz*sz(i ,j ,k ))
10924  +sol(i ,j-1,k-1)*( -facx*(sx(i-1,j-1,k-1)+sx(i,j-1,k-1))
10925  +Real(2.0)*facy*(sy(i-1,j-1,k-1)+sy(i,j-1,k-1))
10926  +Real(2.0)*facz*(sz(i-1,j-1,k-1)+sz(i,j-1,k-1)))
10927  +sol(i ,j+1,k-1)*( -facx*(sx(i-1,j ,k-1)+sx(i,j ,k-1))
10928  +Real(2.0)*facy*(sy(i-1,j ,k-1)+sy(i,j ,k-1))
10929  +Real(2.0)*facz*(sz(i-1,j ,k-1)+sz(i,j ,k-1)))
10930  +sol(i ,j-1,k+1)*( -facx*(sx(i-1,j-1,k )+sx(i,j-1,k ))
10931  +Real(2.0)*facy*(sy(i-1,j-1,k )+sy(i,j-1,k ))
10932  +Real(2.0)*facz*(sz(i-1,j-1,k )+sz(i,j-1,k )))
10933  +sol(i ,j+1,k+1)*( -facx*(sx(i-1,j ,k )+sx(i,j ,k ))
10934  +Real(2.0)*facy*(sy(i-1,j ,k )+sy(i,j ,k ))
10935  +Real(2.0)*facz*(sz(i-1,j ,k )+sz(i,j ,k )))
10936  +sol(i-1,j ,k-1)*( Real(2.0)*facx*(sx(i-1,j-1,k-1)+sx(i-1,j,k-1))
10937  -facy*(sy(i-1,j-1,k-1)+sy(i-1,j,k-1))
10938  +Real(2.0)*facz*(sz(i-1,j-1,k-1)+sz(i-1,j,k-1)))
10939  +sol(i+1,j ,k-1)*( Real(2.0)*facx*(sx(i ,j-1,k-1)+sx(i ,j,k-1))
10940  -facy*(sy(i ,j-1,k-1)+sy(i ,j,k-1))
10941  +Real(2.0)*facz*(sz(i ,j-1,k-1)+sz(i ,j,k-1)))
10942  +sol(i-1,j ,k+1)*( Real(2.0)*facx*(sx(i-1,j-1,k )+sx(i-1,j,k ))
10943  -facy*(sy(i-1,j-1,k )+sy(i-1,j,k ))
10944  +Real(2.0)*facz*(sz(i-1,j-1,k )+sz(i-1,j,k )))
10945  +sol(i+1,j ,k+1)*( Real(2.0)*facx*(sx(i ,j-1,k )+sx(i ,j,k ))
10946  -facy*(sy(i ,j-1,k )+sy(i ,j,k ))
10947  +Real(2.0)*facz*(sz(i ,j-1,k )+sz(i ,j,k )))
10948  +sol(i-1,j-1,k )*( Real(2.0)*facx*(sx(i-1,j-1,k-1)+sx(i-1,j-1,k))
10949  +Real(2.0)*facy*(sy(i-1,j-1,k-1)+sy(i-1,j-1,k))
10950  -facz*(sz(i-1,j-1,k-1)+sz(i-1,j-1,k)))
10951  +sol(i+1,j-1,k )*( Real(2.0)*facx*(sx(i ,j-1,k-1)+sx(i ,j-1,k))
10952  +Real(2.0)*facy*(sy(i ,j-1,k-1)+sy(i ,j-1,k))
10953  -facz*(sz(i ,j-1,k-1)+sz(i ,j-1,k)))
10954  +sol(i-1,j+1,k )*( Real(2.0)*facx*(sx(i-1,j ,k-1)+sx(i-1,j ,k))
10955  +Real(2.0)*facy*(sy(i-1,j ,k-1)+sy(i-1,j ,k))
10956  -facz*(sz(i-1,j ,k-1)+sz(i-1,j ,k)))
10957  +sol(i+1,j+1,k )*( Real(2.0)*facx*(sx(i ,j ,k-1)+sx(i ,j ,k))
10958  +Real(2.0)*facy*(sy(i ,j ,k-1)+sy(i ,j ,k))
10959  -facz*(sz(i ,j ,k-1)+sz(i ,j ,k)))
10960  + Real(2.0)*sol(i-1,j,k)*(Real(2.0)*facx*(sx(i-1,j-1,k-1)+sx(i-1,j,k-1)+sx(i-1,j-1,k)+sx(i-1,j,k))
10961  -facy*(sy(i-1,j-1,k-1)+sy(i-1,j,k-1)+sy(i-1,j-1,k)+sy(i-1,j,k))
10962  -facz*(sz(i-1,j-1,k-1)+sz(i-1,j,k-1)+sz(i-1,j-1,k)+sz(i-1,j,k)))
10963  + Real(2.0)*sol(i+1,j,k)*(Real(2.0)*facx*(sx(i ,j-1,k-1)+sx(i ,j,k-1)+sx(i ,j-1,k)+sx(i ,j,k))
10964  -facy*(sy(i ,j-1,k-1)+sy(i ,j,k-1)+sy(i ,j-1,k)+sy(i ,j,k))
10965  -facz*(sz(i ,j-1,k-1)+sz(i ,j,k-1)+sz(i ,j-1,k)+sz(i ,j,k)))
10966  + Real(2.0)*sol(i,j-1,k)*( -facx*(sx(i-1,j-1,k-1)+sx(i,j-1,k-1)+sx(i-1,j-1,k)+sx(i,j-1,k))
10967  +Real(2.0)*facy*(sy(i-1,j-1,k-1)+sy(i,j-1,k-1)+sy(i-1,j-1,k)+sy(i,j-1,k))
10968  -facz*(sz(i-1,j-1,k-1)+sz(i,j-1,k-1)+sz(i-1,j-1,k)+sz(i,j-1,k)))
10969  + Real(2.0)*sol(i,j+1,k)*( -facx*(sx(i-1,j ,k-1)+sx(i,j ,k-1)+sx(i-1,j ,k)+sx(i,j ,k))
10970  +Real(2.0)*facy*(sy(i-1,j ,k-1)+sy(i,j ,k-1)+sy(i-1,j ,k)+sy(i,j ,k))
10971  -facz*(sz(i-1,j ,k-1)+sz(i,j ,k-1)+sz(i-1,j ,k)+sz(i,j ,k)))
10972  + Real(2.0)*sol(i,j,k-1)*( -facx*(sx(i-1,j-1,k-1)+sx(i,j-1,k-1)+sx(i-1,j,k-1)+sx(i,j,k-1))
10973  -facy*(sy(i-1,j-1,k-1)+sy(i,j-1,k-1)+sy(i-1,j,k-1)+sy(i,j,k-1))
10974  +Real(2.0)*facz*(sz(i-1,j-1,k-1)+sz(i,j-1,k-1)+sz(i-1,j,k-1)+sz(i,j,k-1)))
10975  + Real(2.0)*sol(i,j,k+1)*( -facx*(sx(i-1,j-1,k )+sx(i,j-1,k )+sx(i-1,j,k )+sx(i,j,k ))
10976  -facy*(sy(i-1,j-1,k )+sy(i,j-1,k )+sy(i-1,j,k )+sy(i,j,k ))
10977  +Real(2.0)*facz*(sz(i-1,j-1,k )+sz(i,j-1,k )+sz(i-1,j,k )+sz(i,j,k )));
10978 
10979  sol(i,j,k) += (rhs(i,j,k) - Ax) / s0;
10980  }
10981  }
10982 }
10983 
10984 AMREX_GPU_DEVICE AMREX_FORCE_INLINE
10985 void mlndlap_gscolor_aa (int i, int j, int k, Array4<Real> const& sol,
10986  Array4<Real const> const& rhs, Array4<Real const> const& sig,
10987  Array4<int const> const& msk,
10988  GpuArray<Real,AMREX_SPACEDIM> const& dxinv, int color) noexcept
10989 {
10990  if (mlndlap_color(i,j,k) == color) {
10991  if (msk(i,j,k)) {
10992  sol(i,j,k) = Real(0.0);
10993  } else {
10994  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0];
10995  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1];
10996  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2];
10997  Real fxyz = facx + facy + facz;
10998  Real fmx2y2z = -facx + Real(2.0)*facy + Real(2.0)*facz;
10999  Real f2xmy2z = Real(2.0)*facx - facy + Real(2.0)*facz;
11000  Real f2x2ymz = Real(2.0)*facx + Real(2.0)*facy - facz;
11001  Real f4xm2ym2z = Real(4.0)*facx - Real(2.0)*facy - Real(2.0)*facz;
11002  Real fm2x4ym2z = -Real(2.0)*facx + Real(4.0)*facy - Real(2.0)*facz;
11003  Real fm2xm2y4z = -Real(2.0)*facx - Real(2.0)*facy + Real(4.0)*facz;
11004 
11005  Real s0 = Real(-4.0)*fxyz*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j,k-1)+sig(i,j,k-1)
11006  +sig(i-1,j-1,k )+sig(i,j-1,k )+sig(i-1,j,k )+sig(i,j,k ));
11007  Real Ax = sol(i,j,k)*s0
11008  + fxyz*(sol(i-1,j-1,k-1)*sig(i-1,j-1,k-1)
11009  + sol(i+1,j-1,k-1)*sig(i ,j-1,k-1)
11010  + sol(i-1,j+1,k-1)*sig(i-1,j ,k-1)
11011  + sol(i+1,j+1,k-1)*sig(i ,j ,k-1)
11012  + sol(i-1,j-1,k+1)*sig(i-1,j-1,k )
11013  + sol(i+1,j-1,k+1)*sig(i ,j-1,k )
11014  + sol(i-1,j+1,k+1)*sig(i-1,j ,k )
11015  + sol(i+1,j+1,k+1)*sig(i ,j ,k ))
11016  + fmx2y2z*(sol(i ,j-1,k-1)*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1))
11017  + sol(i ,j+1,k-1)*(sig(i-1,j ,k-1)+sig(i,j ,k-1))
11018  + sol(i ,j-1,k+1)*(sig(i-1,j-1,k )+sig(i,j-1,k ))
11019  + sol(i ,j+1,k+1)*(sig(i-1,j ,k )+sig(i,j ,k )))
11020  + f2xmy2z*(sol(i-1,j ,k-1)*(sig(i-1,j-1,k-1)+sig(i-1,j,k-1))
11021  + sol(i+1,j ,k-1)*(sig(i ,j-1,k-1)+sig(i ,j,k-1))
11022  + sol(i-1,j ,k+1)*(sig(i-1,j-1,k )+sig(i-1,j,k ))
11023  + sol(i+1,j ,k+1)*(sig(i ,j-1,k )+sig(i ,j,k )))
11024  + f2x2ymz*(sol(i-1,j-1,k )*(sig(i-1,j-1,k-1)+sig(i-1,j-1,k))
11025  + sol(i+1,j-1,k )*(sig(i ,j-1,k-1)+sig(i ,j-1,k))
11026  + sol(i-1,j+1,k )*(sig(i-1,j ,k-1)+sig(i-1,j ,k))
11027  + sol(i+1,j+1,k )*(sig(i ,j ,k-1)+sig(i ,j ,k)))
11028  + f4xm2ym2z*(sol(i-1,j,k)*(sig(i-1,j-1,k-1)+sig(i-1,j,k-1)+sig(i-1,j-1,k)+sig(i-1,j,k))
11029  + sol(i+1,j,k)*(sig(i ,j-1,k-1)+sig(i ,j,k-1)+sig(i ,j-1,k)+sig(i ,j,k)))
11030  + fm2x4ym2z*(sol(i,j-1,k)*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j-1,k)+sig(i,j-1,k))
11031  + sol(i,j+1,k)*(sig(i-1,j ,k-1)+sig(i,j ,k-1)+sig(i-1,j ,k)+sig(i,j ,k)))
11032  + fm2xm2y4z*(sol(i,j,k-1)*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j,k-1)+sig(i,j,k-1))
11033  + sol(i,j,k+1)*(sig(i-1,j-1,k )+sig(i,j-1,k )+sig(i-1,j,k )+sig(i,j,k )));
11034 
11035  sol(i,j,k) += (rhs(i,j,k) - Ax) / s0;
11036  }
11037  }
11038 }
11039 
11040 AMREX_GPU_DEVICE AMREX_FORCE_INLINE
11041 void mlndlap_gscolor_c (int i, int j, int k, Array4<Real> const& sol,
11042  Array4<Real const> const& rhs, Real sig,
11043  Array4<int const> const& msk,
11044  GpuArray<Real,AMREX_SPACEDIM> const& dxinv, int color) noexcept
11045 {
11046  if (mlndlap_color(i,j,k) == color) {
11047  if (msk(i,j,k)) {
11048  sol(i,j,k) = Real(0.0);
11049  } else {
11050  Real facx = Real(1.0/36.0)*dxinv[0]*dxinv[0];
11051  Real facy = Real(1.0/36.0)*dxinv[1]*dxinv[1];
11052  Real facz = Real(1.0/36.0)*dxinv[2]*dxinv[2];
11053  Real fxyz = facx + facy + facz;
11054  Real fmx2y2z = -facx + Real(2.0)*facy + Real(2.0)*facz;
11055  Real f2xmy2z = Real(2.0)*facx - facy + Real(2.0)*facz;
11056  Real f2x2ymz = Real(2.0)*facx + Real(2.0)*facy - facz;
11057  Real f4xm2ym2z = Real(4.0)*facx - Real(2.0)*facy - Real(2.0)*facz;
11058  Real fm2x4ym2z = -Real(2.0)*facx + Real(4.0)*facy - Real(2.0)*facz;
11059  Real fm2xm2y4z = -Real(2.0)*facx - Real(2.0)*facy + Real(4.0)*facz;
11060 
11061  Real s0 = Real(-4.0)*fxyz*Real(8.);
11062  Real Ax = sol(i,j,k)*s0
11063  + fxyz*(sol(i-1,j-1,k-1)
11064  + sol(i+1,j-1,k-1)
11065  + sol(i-1,j+1,k-1)
11066  + sol(i+1,j+1,k-1)
11067  + sol(i-1,j-1,k+1)
11068  + sol(i+1,j-1,k+1)
11069  + sol(i-1,j+1,k+1)
11070  + sol(i+1,j+1,k+1))
11071  + fmx2y2z*(sol(i ,j-1,k-1)*Real(2.)
11072  + sol(i ,j+1,k-1)*Real(2.)
11073  + sol(i ,j-1,k+1)*Real(2.)
11074  + sol(i ,j+1,k+1)*Real(2.))
11075  + f2xmy2z*(sol(i-1,j ,k-1)*Real(2.)
11076  + sol(i+1,j ,k-1)*Real(2.)
11077  + sol(i-1,j ,k+1)*Real(2.)
11078  + sol(i+1,j ,k+1)*Real(2.))
11079  + f2x2ymz*(sol(i-1,j-1,k )*Real(2.)
11080  + sol(i+1,j-1,k )*Real(2.)
11081  + sol(i-1,j+1,k )*Real(2.)
11082  + sol(i+1,j+1,k )*Real(2.))
11083  + f4xm2ym2z*(sol(i-1,j,k)*Real(4.)
11084  + sol(i+1,j,k)*Real(4.))
11085  + fm2x4ym2z*(sol(i,j-1,k)*Real(4.)
11086  + sol(i,j+1,k)*Real(4.))
11087  + fm2xm2y4z*(sol(i,j,k-1)*Real(4.)
11088  + sol(i,j,k+1)*Real(4.));
11089 
11090  sol(i,j,k) += (rhs(i,j,k) - Ax*sig) / (s0*sig);
11091  }
11092  }
11093 }
11094 
11095 AMREX_GPU_DEVICE AMREX_FORCE_INLINE
11096 void mlndlap_gscolor_sten (int i, int j, int k, Array4<Real> const& sol,
11097  Array4<Real const> const& rhs,
11098  Array4<Real const> const& sten,
11099  Array4<int const> const& msk, int color) noexcept
11100 {
11101  if (mlndlap_color(i,j,k) == color) {
11102  mlndlap_gauss_seidel_sten(i,j,k,sol,rhs,sten,msk);
11103  }
11104 }
11105 
11106 }
11107 #endif
AMREX_PRAGMA_SIMD
#define AMREX_PRAGMA_SIMD
Definition: AMReX_Extension.H:80
AMREX_FORCE_INLINE
#define AMREX_FORCE_INLINE
Definition: AMReX_Extension.H:119
AMREX_HOST_DEVICE_FOR_3D
#define AMREX_HOST_DEVICE_FOR_3D(...)
Definition: AMReX_GpuLaunch.nolint.H:50
AMREX_GPU_DEVICE
#define AMREX_GPU_DEVICE
Definition: AMReX_GpuQualifiers.H:18
AMREX_GPU_HOST_DEVICE
#define AMREX_GPU_HOST_DEVICE
Definition: AMReX_GpuQualifiers.H:20
idir
int idir
Definition: AMReX_HypreMLABecLap.cpp:1093
offset
Array4< int const > offset
Definition: AMReX_HypreMLABecLap.cpp:1089
fine
Array4< Real > fine
Definition: AMReX_InterpFaceRegister.cpp:90
crse
Array4< Real const > crse
Definition: AMReX_InterpFaceRegister.cpp:92
AMREX_LOOP_3D
#define AMREX_LOOP_3D(bx, i, j, k, block)
Definition: AMReX_Loop.nolint.H:4
if
if(!(yy_init))
Definition: amrex_iparser.lex.nolint.H:935
amrex::BoxND::growHi
AMREX_GPU_HOST_DEVICE BoxND & growHi(int idir, int n_cell=1) noexcept
Grow the BoxND on the high end by n_cell cells in direction idir. NOTE: n_cell negative shrinks the B...
Definition: AMReX_Box.H:659
amrex::BoxND::growLo
AMREX_GPU_HOST_DEVICE BoxND & growLo(int idir, int n_cell=1) noexcept
Grow the BoxND on the low end by n_cell cells in direction idir. NOTE: n_cell negative shrinks the Bo...
Definition: AMReX_Box.H:648
amrex::HypreNodeLap::Int
HYPRE_Int Int
Definition: AMReX_HypreNodeLap.H:36
abs
#define abs(x)
Definition: complex-type.h:85
amrex::SundialsUserFun::f
static int f(amrex::Real t, N_Vector y_data, N_Vector y_rhs, void *user_data)
Definition: AMReX_SundialsIntegrator.H:44
amrex::algoim::i_B_xyz
static constexpr int i_B_xyz
Definition: AMReX_algoim.H:42
amrex::algoim::i_S_y_z2
static constexpr int i_S_y_z2
Definition: AMReX_algoim.H:26
amrex::algoim::i_S_y_z
static constexpr int i_S_y_z
Definition: AMReX_algoim.H:20
amrex::algoim::i_S_x2
static constexpr int i_S_x2
Definition: AMReX_algoim.H:15
amrex::algoim::numSurfIntgs
static constexpr int numSurfIntgs
Definition: AMReX_algoim.H:43
amrex::algoim::i_S_x_y
static constexpr int i_S_x_y
Definition: AMReX_algoim.H:18
amrex::algoim::i_S_y
static constexpr int i_S_y
Definition: AMReX_algoim.H:13
amrex::algoim::i_S_x_z
static constexpr int i_S_x_z
Definition: AMReX_algoim.H:19
amrex::algoim::i_B_z
static constexpr int i_B_z
Definition: AMReX_algoim.H:38
amrex::algoim::i_S_x_y2
static constexpr int i_S_x_y2
Definition: AMReX_algoim.H:23
amrex::algoim::i_S_xyz
static constexpr int i_S_xyz
Definition: AMReX_algoim.H:30
amrex::algoim::i_S_y2_z2
static constexpr int i_S_y2_z2
Definition: AMReX_algoim.H:29
amrex::algoim::i_B_x_y
static constexpr int i_B_x_y
Definition: AMReX_algoim.H:39
amrex::algoim::i_B_x_z
static constexpr int i_B_x_z
Definition: AMReX_algoim.H:40
amrex::algoim::i_S_x2_y2
static constexpr int i_S_x2_y2
Definition: AMReX_algoim.H:27
amrex::algoim::i_S_x2_z2
static constexpr int i_S_x2_z2
Definition: AMReX_algoim.H:28
amrex::algoim::i_S_x
static constexpr int i_S_x
Definition: AMReX_algoim.H:12
amrex::algoim::i_S_y2_z
static constexpr int i_S_y2_z
Definition: AMReX_algoim.H:24
amrex::algoim::i_S_x2_y
static constexpr int i_S_x2_y
Definition: AMReX_algoim.H:21
amrex::algoim::i_B_y_z
static constexpr int i_B_y_z
Definition: AMReX_algoim.H:41
amrex::algoim::i_B_x
static constexpr int i_B_x
Definition: AMReX_algoim.H:36
amrex::algoim::i_S_z
static constexpr int i_S_z
Definition: AMReX_algoim.H:14
amrex::algoim::i_S_z2
static constexpr int i_S_z2
Definition: AMReX_algoim.H:17
amrex::algoim::i_S_x_z2
static constexpr int i_S_x_z2
Definition: AMReX_algoim.H:25
amrex::algoim::i_S_x2_z
static constexpr int i_S_x2_z
Definition: AMReX_algoim.H:22
amrex::algoim::i_B_y
static constexpr int i_B_y
Definition: AMReX_algoim.H:37
amrex::algoim::i_S_y2
static constexpr int i_S_y2
Definition: AMReX_algoim.H:16
amrex::detail::max
@ max
Definition: AMReX_ParallelReduce.H:17
amrex::interp_detail::iz
constexpr int iz
Definition: AMReX_Interp_3D_C.H:37
amrex::interp_detail::iy
constexpr int iy
Definition: AMReX_Interp_2D_C.H:33
amrex::interp_detail::ix
constexpr int ix
Definition: AMReX_Interp_2D_C.H:32
amrex::nodelap_detail::ist_000
constexpr int ist_000
Definition: AMReX_MLNodeLap_3D_K.H:2753
amrex::nodelap_detail::ist_pp0
constexpr int ist_pp0
Definition: AMReX_MLNodeLap_3D_K.H:2757
amrex::nodelap_detail::fine_cell
constexpr int fine_cell
Definition: AMReX_MLNodeLap_K.H:57
amrex::nodelap_detail::ist_p00
constexpr int ist_p00
Definition: AMReX_MLNodeLap_3D_K.H:2754
amrex::nodelap_detail::fine_node
constexpr int fine_node
Definition: AMReX_MLNodeLap_K.H:60
amrex::nodelap_detail::ist_ppp
constexpr int ist_ppp
Definition: AMReX_MLNodeLap_3D_K.H:2760
amrex::nodelap_detail::ist_00p
constexpr int ist_00p
Definition: AMReX_MLNodeLap_3D_K.H:2756
amrex::nodelap_detail::crse_node
constexpr int crse_node
Definition: AMReX_MLNodeLap_K.H:58
amrex::nodelap_detail::crse_cell
constexpr int crse_cell
Definition: AMReX_MLNodeLap_K.H:56
amrex::nodelap_detail::ist_p0p
constexpr int ist_p0p
Definition: AMReX_MLNodeLap_3D_K.H:2758
amrex::nodelap_detail::ist_inv
constexpr int ist_inv
Definition: AMReX_MLNodeLap_3D_K.H:2761
amrex::nodelap_detail::ist_0p0
constexpr int ist_0p0
Definition: AMReX_MLNodeLap_3D_K.H:2755
amrex::nodelap_detail::crse_fine_node
constexpr int crse_fine_node
Definition: AMReX_MLNodeLap_K.H:59
amrex::nodelap_detail::eps
constexpr double eps
Definition: AMReX_MLNodeLap_K.H:64
amrex::nodelap_detail::n_sten
constexpr int n_sten
Definition: AMReX_MLNodeLap_3D_K.H:2762
amrex::nodelap_detail::ist_0pp
constexpr int ist_0pp
Definition: AMReX_MLNodeLap_3D_K.H:2759
amrex::nodelap_detail::almostone
constexpr Real almostone
Definition: AMReX_MLNodeLap_K.H:66
amrex::openbc::P
static constexpr int P
Definition: AMReX_OpenBC.H:14
amrex
Definition: AMReX_Amr.cpp:49
amrex::mlndlap_adotx_ha
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real mlndlap_adotx_ha(int i, int, int, Array4< Real const > const &x, Array4< Real const > const &sx, Array4< int const > const &msk, GpuArray< Real, AMREX_SPACEDIM > const &dxinv) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:158
amrex::mlndlap_gscolor_ha
AMREX_GPU_DEVICE AMREX_FORCE_INLINE void mlndlap_gscolor_ha(int i, int j, int k, Array4< Real > const &sol, Array4< Real const > const &rhs, Array4< Real const > const &sx, Array4< int const > const &msk, GpuArray< Real, AMREX_SPACEDIM > const &dxinv, int color) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:585
amrex::ha_interp_face_xz
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real ha_interp_face_xz(Array4< Real const > const &crse, Array4< Real const > const &sigx, Array4< Real const > const &sigz, int i, int j, int k, int ic, int jc, int kc) noexcept
Definition: AMReX_MLNodeLap_3D_K.H:1991
amrex::mlndlap_bc_doit
void mlndlap_bc_doit(Box const &vbx, Array4< T > const &a, Box const &domain, GpuArray< bool, AMREX_SPACEDIM > const &bflo, GpuArray< bool, AMREX_SPACEDIM > const &bfhi) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:110
amrex::LoopOnCpu
AMREX_ATTRIBUTE_FLATTEN_FOR void LoopOnCpu(Dim3 lo, Dim3 hi, F const &f) noexcept
Definition: AMReX_Loop.H:354
amrex::mlndlap_set_nodal_mask
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_set_nodal_mask(int i, int, int, Array4< int > const &nmsk, Array4< int const > const &cmsk) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:8
amrex::mlndlap_Ax_fine_contrib_cs
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_Ax_fine_contrib_cs(int i, int j, int, Box const &ndbx, Box const &ccbx, Array4< Real > const &f, Array4< Real const > const &res, Array4< Real const > const &rhs, Array4< Real const > const &phi, Real const sig, Array4< int const > const &msk, bool is_rz, GpuArray< Real, AMREX_SPACEDIM > const &dxinv) noexcept
Definition: AMReX_MLNodeLap_2D_K.H:1473
amrex::mlndlap_divu_fine_contrib
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_divu_fine_contrib(int i, int j, int, Box const &fvbx, Box const &velbx, Array4< Real > const &rhs, Array4< Real const > const &vel, Array4< Real const > const &frhs, Array4< int const > const &msk, GpuArray< Real, AMREX_SPACEDIM > const &dxinv, bool is_rz) noexcept
Definition: AMReX_MLNodeLap_2D_K.H:1183
amrex::mlndlap_avgdown_coeff_z
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_avgdown_coeff_z(int i, int j, int k, Array4< Real > const &crse, Array4< Real const > const &fine) noexcept
Definition: AMReX_MLNodeLap_3D_K.H:224
amrex::Box
BoxND< AMREX_SPACEDIM > Box
Definition: AMReX_BaseFwd.H:27
amrex::LoopConcurrentOnCpu
AMREX_ATTRIBUTE_FLATTEN_FOR void LoopConcurrentOnCpu(Dim3 lo, Dim3 hi, F const &f) noexcept
Definition: AMReX_Loop.H:377
amrex::aa_interp_line_z
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real aa_interp_line_z(Array4< Real const > const &crse, Array4< Real const > const &sig, int i, int j, int k, int ic, int jc, int kc) noexcept
Definition: AMReX_MLNodeLap_3D_K.H:1745
amrex::mlndlap_gauss_seidel_with_line_solve_aa
void mlndlap_gauss_seidel_with_line_solve_aa(Box const &, Array4< Real > const &, Array4< Real const > const &, Array4< Real const > const &, Array4< int const > const &, GpuArray< Real, AMREX_SPACEDIM > const &) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:331
amrex::mlndlap_jacobi_ha
AMREX_GPU_DEVICE AMREX_FORCE_INLINE void mlndlap_jacobi_ha(int i, int, int, Array4< Real > const &sol, Real Ax, Array4< Real const > const &rhs, Array4< Real const > const &sx, Array4< int const > const &msk, GpuArray< Real, AMREX_SPACEDIM > const &dxinv) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:198
amrex::mlndlap_semi_avgdown_coeff
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_semi_avgdown_coeff(int i, int j, int k, Array4< Real > const &crse, Array4< Real const > const &fine, int) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:103
amrex::ha_interp_face_yz
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real ha_interp_face_yz(Array4< Real const > const &crse, Array4< Real const > const &sigy, Array4< Real const > const &sigz, int i, int j, int k, int ic, int jc, int kc) noexcept
Definition: AMReX_MLNodeLap_3D_K.H:2006
amrex::mlndlap_interpadd_rap
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_interpadd_rap(int, int, int, Array4< Real > const &, Array4< Real const > const &, Array4< Real const > const &, Array4< int const > const &) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:567
amrex::mlndlap_avgdown_coeff_y
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_avgdown_coeff_y(int i, int j, int k, Array4< Real > const &crse, Array4< Real const > const &fine) noexcept
Definition: AMReX_MLNodeLap_2D_K.H:157
amrex::mlndlap_gauss_seidel_aa
void mlndlap_gauss_seidel_aa(Box const &bx, Array4< Real > const &sol, Array4< Real const > const &rhs, Array4< Real const > const &sx, Array4< int const > const &msk, GpuArray< Real, AMREX_SPACEDIM > const &dxinv) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:299
amrex::max
AMREX_GPU_HOST_DEVICE constexpr AMREX_FORCE_INLINE const T & max(const T &a, const T &b) noexcept
Definition: AMReX_Algorithm.H:35
amrex::mlndlap_adotx_aa
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real mlndlap_adotx_aa(int i, int j, int k, Array4< Real const > const &x, Array4< Real const > const &sx, Array4< int const > const &msk, GpuArray< Real, AMREX_SPACEDIM > const &dxinv) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:172
amrex::mlndlap_jacobi_c
AMREX_GPU_DEVICE AMREX_FORCE_INLINE void mlndlap_jacobi_c(int i, int, int, Array4< Real > const &sol, Real Ax, Array4< Real const > const &rhs, Real sig, Array4< int const > const &msk, GpuArray< Real, AMREX_SPACEDIM > const &dxinv) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:247
amrex::mlndlap_jacobi_aa
AMREX_GPU_DEVICE AMREX_FORCE_INLINE void mlndlap_jacobi_aa(int i, int j, int k, Array4< Real > const &sol, Real Ax, Array4< Real const > const &rhs, Array4< Real const > const &sig, Array4< int const > const &msk, GpuArray< Real, AMREX_SPACEDIM > const &dxinv) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:231
amrex::grow
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE BoxND< dim > grow(const BoxND< dim > &b, int i) noexcept
Grow BoxND in all directions by given amount.
Definition: AMReX_Box.H:1211
amrex::mlndlap_adotx_sten
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real mlndlap_adotx_sten(int, int, int, Array4< Real const > const &, Array4< Real const > const &, Array4< int const > const &) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:555
amrex::min
AMREX_GPU_HOST_DEVICE constexpr AMREX_FORCE_INLINE const T & min(const T &a, const T &b) noexcept
Definition: AMReX_Algorithm.H:21
amrex::mlndlap_interpadd_c
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_interpadd_c(int i, int, int, Array4< Real > const &fine, Array4< Real const > const &crse, Array4< int const > const &msk) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:392
amrex::mlndlap_set_stencil_s0
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_set_stencil_s0(int, int, int, Array4< Real > const &) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:546
amrex::mlndlap_gauss_seidel_ha
void mlndlap_gauss_seidel_ha(Box const &bx, Array4< Real > const &sol, Array4< Real const > const &rhs, Array4< Real const > const &sx, Array4< int const > const &msk, GpuArray< Real, AMREX_SPACEDIM > const &dxinv) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:276
amrex::mlndlap_res_cf_contrib
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_res_cf_contrib(int, int, int, Array4< Real > const &, Array4< Real const > const &, Array4< Real const > const &, Array4< Real const > const &, Array4< int const > const &, Array4< int const > const &, Array4< int const > const &, Array4< Real const > const &, GpuArray< Real, AMREX_SPACEDIM > const &, Box const &, GpuArray< LinOpBCType, AMREX_SPACEDIM > const &, GpuArray< LinOpBCType, AMREX_SPACEDIM > const &, bool) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:516
amrex::mlndlap_normalize_aa
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_normalize_aa(int i, int j, int k, Array4< Real > const &x, Array4< Real const > const &sx, Array4< int const > const &msk, GpuArray< Real, AMREX_SPACEDIM > const &dxinv) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:190
amrex::mlndlap_sum_Ax
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real mlndlap_sum_Ax(P const &pred, S const &sig, int i, int j, Real facx, Real facy, Array4< Real const > const &phi, bool is_rz) noexcept
Definition: AMReX_MLNodeLap_2D_K.H:1373
amrex::mlndlap_restriction
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_restriction(int i, int, int, Array4< Real > const &crse, Array4< Real const > const &fine, Array4< int const > const &msk) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:340
amrex::ubound
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Dim3 ubound(Array4< T > const &a) noexcept
Definition: AMReX_Array4.H:315
amrex::mlndlap_rhcc_fine_contrib
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_rhcc_fine_contrib(int, int, int, Box const &, Array4< Real > const &, Array4< Real const > const &, Array4< int const > const &) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:491
amrex::mlndlap_semi_interpadd_aa
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_semi_interpadd_aa(int, int, int, Array4< Real > const &, Array4< Real const > const &, Array4< Real const > const &, Array4< int const > const &, int) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:429
amrex::lbound
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Dim3 lbound(Array4< T > const &a) noexcept
Definition: AMReX_Array4.H:308
amrex::mlndlap_semi_restriction
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_semi_restriction(int, int, int, Array4< Real > const &, Array4< Real const > const &, Array4< int const > const &, int) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:385
amrex::mlndlap_set_stencil
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_set_stencil(Box const &, Array4< Real > const &, Array4< Real const > const &, GpuArray< Real, AMREX_SPACEDIM > const &) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:540
amrex::aa_interp_line_y
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real aa_interp_line_y(Array4< Real const > const &crse, Array4< Real const > const &sig, int i, int j, int ic, int jc) noexcept
Definition: AMReX_MLNodeLap_2D_K.H:898
amrex::mlndlap_zero_fine
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_zero_fine(int i, int, int, Array4< Real > const &phi, Array4< int const > const &msk, int fine_flag) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:81
amrex::mlndlap_adotx_c
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real mlndlap_adotx_c(int i, int, int, Array4< Real const > const &x, Real sigma, Array4< int const > const &msk, GpuArray< Real, AMREX_SPACEDIM > const &dxinv) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:143
amrex::ha_interp_face_xy
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real ha_interp_face_xy(Array4< Real const > const &crse, Array4< Real const > const &sigx, Array4< Real const > const &sigy, int i, int j, int ic, int jc) noexcept
Definition: AMReX_MLNodeLap_2D_K.H:1010
amrex::mlndlap_interpadd_aa
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_interpadd_aa(int i, int, int, Array4< Real > const &fine, Array4< Real const > const &crse, Array4< Real const > const &sig, Array4< int const > const &msk) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:410
amrex::IntVect
IntVectND< AMREX_SPACEDIM > IntVect
Definition: AMReX_BaseFwd.H:30
amrex::mlndlap_res_cf_contrib_cs
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_res_cf_contrib_cs(int, int, int, Array4< Real > const &, Array4< Real const > const &, Array4< Real const > const &, Real, Array4< int const > const &, Array4< int const > const &, Array4< int const > const &, Array4< Real const > const &, GpuArray< Real, AMREX_SPACEDIM > const &, Box const &, GpuArray< LinOpBCType, AMREX_SPACEDIM > const &, GpuArray< LinOpBCType, AMREX_SPACEDIM > const &, bool) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:528
amrex::mlndlap_divu
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_divu(int i, int, int, Array4< Real > const &rhs, Array4< Real const > const &vel, Array4< int const > const &msk, GpuArray< Real, AMREX_SPACEDIM > const &dxinv, Box const &, GpuArray< LinOpBCType, AMREX_SPACEDIM > const &, GpuArray< LinOpBCType, AMREX_SPACEDIM > const &) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:443
amrex::mlndlap_divu_cf_contrib
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_divu_cf_contrib(int, int, int, Array4< Real > const &, Array4< Real const > const &, Array4< Real const > const &, Array4< Real const > const &, Array4< int const > const &, Array4< int const > const &, Array4< int const > const &, GpuArray< Real, AMREX_SPACEDIM > const &, Box const &, GpuArray< LinOpBCType, AMREX_SPACEDIM > const &, GpuArray< LinOpBCType, AMREX_SPACEDIM > const &, bool) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:497
amrex::ignore_unused
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void ignore_unused(const Ts &...)
This shuts up the compiler about unused variables.
Definition: AMReX.H:111
amrex::mlndlap_gauss_seidel_sten
void mlndlap_gauss_seidel_sten(Box const &, Array4< Real > const &, Array4< Real const > const &, Array4< Real const > const &, Array4< int const > const &) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:560
amrex::mlndlap_gscolor_sten
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_gscolor_sten(int, int, int, Array4< Real > const &, Array4< Real const > const &, Array4< Real const > const &, Array4< int const > const &, int) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:638
amrex::surroundingNodes
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE BoxND< dim > surroundingNodes(const BoxND< dim > &b, int dir) noexcept
Returns a BoxND with NODE based coordinates in direction dir that encloses BoxND b....
Definition: AMReX_Box.H:1399
amrex::scale
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE IntVectND< dim > scale(const IntVectND< dim > &p, int s) noexcept
Returns a IntVectND obtained by multiplying each of the components of this IntVectND by s.
Definition: AMReX_IntVect.H:1006
amrex::mlndlap_gscolor_aa
AMREX_GPU_DEVICE AMREX_FORCE_INLINE void mlndlap_gscolor_aa(int i, int j, int k, Array4< Real > const &sol, Array4< Real const > const &rhs, Array4< Real const > const &sx, Array4< int const > const &msk, GpuArray< Real, AMREX_SPACEDIM > const &dxinv, int color) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:607
amrex::mlndlap_mknewu
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_mknewu(int i, int, int, Array4< Real > const &u, Array4< Real const > const &p, Array4< Real const > const &sig, GpuArray< Real, AMREX_SPACEDIM > const &dxinv) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:471
amrex::LoopConcurrent
AMREX_GPU_HOST_DEVICE AMREX_ATTRIBUTE_FLATTEN_FOR void LoopConcurrent(Dim3 lo, Dim3 hi, F const &f) noexcept
Definition: AMReX_Loop.H:149
amrex::mlndlap_restriction_rap
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_restriction_rap(int, int, int, Array4< Real > const &, Array4< Real const > const &, Array4< Real const > const &, Array4< int const > const &) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:573
amrex::mlndlap_gscolor_c
AMREX_GPU_DEVICE AMREX_FORCE_INLINE void mlndlap_gscolor_c(int i, int j, int k, Array4< Real > const &sol, Array4< Real const > const &rhs, Real sig, Array4< int const > const &msk, GpuArray< Real, AMREX_SPACEDIM > const &dxinv, int color) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:617
amrex::aa_interp_face_yz
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real aa_interp_face_yz(Array4< Real const > const &crse, Array4< Real const > const &sig, int i, int j, int k, int ic, int jc, int kc) noexcept
Definition: AMReX_MLNodeLap_3D_K.H:1782
amrex::mlndlap_sum_Df
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real mlndlap_sum_Df(int ii, int jj, Real facx, Real facy, Array4< Real const > const &vel, Box const &velbx, bool is_rz) noexcept
Definition: AMReX_MLNodeLap_2D_K.H:1159
amrex::mlndlap_Ax_fine_contrib_doit
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_Ax_fine_contrib_doit(S const &sig, int i, int j, Box const &ndbx, Box const &ccbx, Array4< Real > const &f, Array4< Real const > const &res, Array4< Real const > const &rhs, Array4< Real const > const &phi, Array4< int const > const &msk, bool is_rz, GpuArray< Real, AMREX_SPACEDIM > const &dxinv) noexcept
Definition: AMReX_MLNodeLap_2D_K.H:1415
amrex::aa_interp_face_xy
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real aa_interp_face_xy(Array4< Real const > const &crse, Array4< Real const > const &sig, int i, int j, int ic, int jc) noexcept
Definition: AMReX_MLNodeLap_2D_K.H:907
amrex::coarsen
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE BoxND< dim > coarsen(const BoxND< dim > &b, int ref_ratio) noexcept
Coarsen BoxND by given (positive) refinement ratio. NOTE: if type(dir) = CELL centered: lo <- lo/rati...
Definition: AMReX_Box.H:1304
amrex::mlndlap_color
AMREX_GPU_DEVICE AMREX_FORCE_INLINE int mlndlap_color(int i, int, int)
Definition: AMReX_MLNodeLap_1D_K.H:579
amrex::Abort
void Abort(const std::string &msg)
Print out message to cerr and exit via abort().
Definition: AMReX.cpp:221
amrex::int
const int[]
Definition: AMReX_BLProfiler.cpp:1664
amrex::mlndlap_crse_resid
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_crse_resid(int, int, int, Array4< Real > const &, Array4< Real const > const &, Array4< int const > const &, Box const &, GpuArray< LinOpBCType, AMREX_SPACEDIM > const &, GpuArray< LinOpBCType, AMREX_SPACEDIM > const &, bool) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:508
amrex::mlndlap_set_dirichlet_mask
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_set_dirichlet_mask(Box const &bx, Array4< int > const &dmsk, Array4< int const > const &omsk, Box const &dom, GpuArray< LinOpBCType, AMREX_SPACEDIM > const &bclo, GpuArray< LinOpBCType, AMREX_SPACEDIM > const &bchi) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:24
amrex::mlndlap_normalize_ha
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_normalize_ha(int i, int, int, Array4< Real > const &x, Array4< Real const > const &sx, Array4< int const > const &msk, GpuArray< Real, AMREX_SPACEDIM > const &dxinv) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:180
amrex::mlndlap_avgdown_coeff_x
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_avgdown_coeff_x(int i, int, int, Array4< Real > const &crse, Array4< Real const > const &fine) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:94
amrex::mlndlap_set_dot_mask
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_set_dot_mask(Box const &bx, Array4< Real > const &dmsk, Array4< int const > const &omsk, Box const &dom, GpuArray< LinOpBCType, AMREX_SPACEDIM > const &bclo, GpuArray< LinOpBCType, AMREX_SPACEDIM > const &bchi) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:51
amrex::mlndlap_gauss_seidel_c
void mlndlap_gauss_seidel_c(Box const &bx, Array4< Real > const &sol, Array4< Real const > const &rhs, Real sig, Array4< int const > const &msk, GpuArray< Real, AMREX_SPACEDIM > const &dxinv) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:309
amrex::mlndlap_interpadd_ha
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_interpadd_ha(int i, int j, int k, Array4< Real > const &fine, Array4< Real const > const &crse, Array4< Real const > const &sig, Array4< int const > const &msk) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:435
amrex::mlndlap_Ax_fine_contrib
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_Ax_fine_contrib(int i, int j, int, Box const &ndbx, Box const &ccbx, Array4< Real > const &f, Array4< Real const > const &res, Array4< Real const > const &rhs, Array4< Real const > const &phi, Array4< Real const > const &sig, Array4< int const > const &msk, bool is_rz, GpuArray< Real, AMREX_SPACEDIM > const &dxinv) noexcept
Definition: AMReX_MLNodeLap_2D_K.H:1459
amrex::aa_interp_line_x
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real aa_interp_line_x(Array4< Real const > const &crse, Array4< Real const > const &sig, int i, int j, int ic, int jc) noexcept
Definition: AMReX_MLNodeLap_2D_K.H:889
amrex::mlndlap_stencil_rap
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_stencil_rap(int, int, int, Array4< Real > const &, Array4< Real const > const &) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:550
amrex::mlndlap_adotx_sten_doit
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real mlndlap_adotx_sten_doit(int i, int j, int k, Array4< Real const > const &x, Array4< Real const > const &sten) noexcept
Definition: AMReX_MLNodeLap_2D_K.H:1823
amrex::tridiagonal_solve
AMREX_FORCE_INLINE void tridiagonal_solve(Array1D< T, 0, 31 > &a_ls, Array1D< T, 0, 31 > &b_ls, Array1D< T, 0, 31 > &c_ls, Array1D< T, 0, 31 > &r_ls, Array1D< T, 0, 31 > &u_ls, Array1D< T, 0, 31 > &gam, int ilen) noexcept
Definition: AMReX_MLABecLap_3D_K.H:432
amrex::mlndlap_rhcc
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real mlndlap_rhcc(int i, int, int, Array4< Real const > const &rhcc, Array4< int const > const &msk) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:458
amrex::mlndlap_mknewu_c
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_mknewu_c(int i, int, int, Array4< Real > const &u, Array4< Real const > const &p, Real sig, GpuArray< Real, AMREX_SPACEDIM > const &dxinv) noexcept
Definition: AMReX_MLNodeLap_1D_K.H:481
amrex::aa_interp_face_xz
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real aa_interp_face_xz(Array4< Real const > const &crse, Array4< Real const > const &sig, int i, int j, int k, int ic, int jc, int kc) noexcept
Definition: AMReX_MLNodeLap_3D_K.H:1768
amrex::neumann_scale
AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real neumann_scale(int i, int j, Box const &nddom, GpuArray< LinOpBCType, AMREX_SPACEDIM > const &bclo, GpuArray< LinOpBCType, AMREX_SPACEDIM > const &bchi) noexcept
Definition: AMReX_MLNodeLap_2D_K.H:1246
amrex::Array4
Definition: AMReX_Array4.H:61