GEOS
HypreMGR.hpp
Go to the documentation of this file.
1 /*
2  * ------------------------------------------------------------------------------------------------------------
3  * SPDX-License-Identifier: LGPL-2.1-only
4  *
5  * Copyright (c) 2016-2024 Lawrence Livermore National Security LLC
6  * Copyright (c) 2018-2024 TotalEnergies
7  * Copyright (c) 2018-2024 The Board of Trustees of the Leland Stanford Junior University
8  * Copyright (c) 2023-2024 Chevron
9  * Copyright (c) 2019- GEOS/GEOSX Contributors
10  * All rights reserved
11  *
12  * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details.
13  * ------------------------------------------------------------------------------------------------------------
14  */
15 
20 #ifndef GEOS_LINEARALGEBRA_INTERFACES_HYPREMGRSTRATEGIES_HPP_
21 #define GEOS_LINEARALGEBRA_INTERFACES_HYPREMGRSTRATEGIES_HPP_
22 
24 
28 
29 #include <_hypre_utilities.h>
30 
31 namespace geos
32 {
33 
38 {
43 };
44 
45 namespace hypre
46 {
47 
48 namespace mgr
49 {
50 
59 template< typename STRATEGY >
61  arrayView1d< int const > const & numComponentsPerField,
62  HyprePrecWrapper & precond,
63  HypreMGRData & mgrData )
64 {
65  STRATEGY strategy( numComponentsPerField );
66  strategy.setup( params, precond, mgrData );
67 }
68 
73 template< int NLEVEL >
75 {
76 public:
77 
78  static constexpr HYPRE_Int numLevels = NLEVEL;
79 
80 protected:
81 
82  HYPRE_Int m_numBlocks{ 0 };
83 
85  HYPRE_Int m_numLabels[numLevels]{ -1 };
86  HYPRE_Int * m_ptrLabels[numLevels]{ nullptr };
87 
89  HYPRE_Int m_levelFRelaxIters[numLevels]{ -1 };
95  HYPRE_Real m_coarseGridThreshold{ 1.0e-20 };
96 
97  // TODO: the following options are currently commented out in MGR's code.
98  // Let's consider their use when re-enable in hypre
99  // HYPRE_Int m_numRestrictSweeps{ -1 }; ///< Number of restrict sweeps
100  // HYPRE_Int m_numInterpSweeps{ -1 }; ///< Number of interpolation sweeps
101 
107  static HYPRE_Int totalNumBlocks( arrayView1d< int const > const & numComponentsPerField )
108  {
109  HYPRE_Int result = 0;
110  for( localIndex i = 0; i < numComponentsPerField.size(); ++i )
111  {
112  result += LvArray::integerConversion< HYPRE_Int >( numComponentsPerField[i] );
113  }
114  return result;
115  }
116 
121  explicit MGRStrategyBase( HYPRE_Int const numBlocks )
122  : m_numBlocks( numBlocks )
123  {
124  for( HYPRE_Int i = 0; i < numLevels; ++i )
125  {
131  }
132  }
133 
137  void setupLabels()
138  {
139  for( HYPRE_Int i = 0; i < numLevels; ++i )
140  {
141  m_numLabels[i] = m_labels[i].size();
142  m_ptrLabels[i] = m_labels[i].data();
143  }
144  }
145 
152  HypreMGRData & mgrData )
153 
154  {
155  // Ensure that if no F-relaxation or global smoothing is chosen the corresponding number
156  // of iteration is set to 0
157  for( HYPRE_Int i = 0; i < numLevels; ++i )
158  {
160  {
161  m_levelFRelaxIters[i] = 0;
162  }
164  {
166  }
167  }
168 
169  GEOS_LAI_CHECK_ERROR( HYPRE_MGRSetCpointsByPointMarkerArray( precond.ptr,
172  mgrData.pointMarkers.data() ) );
173  GEOS_LAI_CHECK_ERROR( HYPRE_MGRSetLevelFRelaxType( precond.ptr, toUnderlyingPtr( m_levelFRelaxType ) ) );
174  GEOS_LAI_CHECK_ERROR( HYPRE_MGRSetLevelNumRelaxSweeps( precond.ptr, m_levelFRelaxIters ) );
175  GEOS_LAI_CHECK_ERROR( HYPRE_MGRSetLevelInterpType( precond.ptr, toUnderlyingPtr( m_levelInterpType ) ) );
176  GEOS_LAI_CHECK_ERROR( HYPRE_MGRSetLevelRestrictType( precond.ptr, toUnderlyingPtr( m_levelRestrictType ) ) );
177  GEOS_LAI_CHECK_ERROR( HYPRE_MGRSetCoarseGridMethod( precond.ptr, toUnderlyingPtr( m_levelCoarseGridMethod ) ) );
178  GEOS_LAI_CHECK_ERROR( HYPRE_MGRSetLevelSmoothType( precond.ptr, toUnderlyingPtr( m_levelGlobalSmootherType ) ) );
179  GEOS_LAI_CHECK_ERROR( HYPRE_MGRSetLevelSmoothIters( precond.ptr, m_levelGlobalSmootherIters ) );
180  GEOS_LAI_CHECK_ERROR( HYPRE_MGRSetTruncateCoarseGridThreshold( precond.ptr, m_coarseGridThreshold ) );
181  GEOS_LAI_CHECK_ERROR( HYPRE_MGRSetNonCpointsToFpoints( precond.ptr, 1 ));
182  GEOS_LAI_CHECK_ERROR( HYPRE_MGRSetNonGalerkinMaxElmts( precond.ptr, 1 ));
183  }
184 
191  integer const & separateComponents )
192  {
193  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGCreate( &solver.ptr ) );
194  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetTol( solver.ptr, 0.0 ) );
195  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxIter( solver.ptr, 1 ) );
196  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxRowSum( solver.ptr, 1.0 ) );
197  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetStrongThreshold( solver.ptr, 0.6 ) );
198  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetPrintLevel( solver.ptr, 0 ) );
199 #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP
202  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumSweeps( solver.ptr, 1 ) );
203 #else
204  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetRelaxOrder( solver.ptr, 1 ) );
205 #endif
206 
207  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumFunctions( solver.ptr, 3 ) );
208  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetFilterFunctions( solver.ptr, separateComponents ) );
209 
210  solver.setup = HYPRE_BoomerAMGSetup;
211  solver.solve = HYPRE_BoomerAMGSolve;
212  solver.destroy = HYPRE_BoomerAMGDestroy;
213  }
214 
226  integer const separateComponents,
227  bool const bubbleCoarse )
228  {
229  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGCreate( &solver.ptr ) );
230  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetTol( solver.ptr, 0.0 ) );
231  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxIter( solver.ptr, 1 ) );
232  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetPrintLevel( solver.ptr, 0 ) );
233  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxRowSum( solver.ptr, 1.0 ) );
234  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetStrongThreshold( solver.ptr, bubbleCoarse ? 0.75 : 0.8 ) );
235  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumFunctions( solver.ptr, 3 ) );
236  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetFilterFunctions( solver.ptr, bubbleCoarse ? 0 : separateComponents ) );
237  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetPMaxElmts( solver.ptr, bubbleCoarse ? 10 : 20 ) );
238 
239  if( !bubbleCoarse )
240  {
241  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggNumLevels( solver.ptr, 1 ) );
242  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr,
244  }
245 
246 #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP
247  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr,
249  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetRelaxType( solver.ptr,
251  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumSweeps( solver.ptr, bubbleCoarse ? 1 : 2 ) );
252 #else
253  HYPRE_Int constexpr l1SymmetricHybridGaussSeidel = 89;
254  HYPRE_Int constexpr gaussianElimination = 9;
255  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCycleRelaxType( solver.ptr, l1SymmetricHybridGaussSeidel, 1 ) );
256  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCycleRelaxType( solver.ptr, l1SymmetricHybridGaussSeidel, 2 ) );
257  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCycleRelaxType( solver.ptr, gaussianElimination, 3 ) );
258  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumSweeps( solver.ptr, bubbleCoarse ? 1 : 2 ) );
259  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetRelaxOrder( solver.ptr, 0 ) );
260 #endif
261 
262  solver.setup = HYPRE_BoomerAMGSetup;
263  solver.solve = HYPRE_BoomerAMGSolve;
264  solver.destroy = HYPRE_BoomerAMGDestroy;
265  }
266 
272  {
273  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGCreate( &solver.ptr ) );
274  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetPrintLevel( solver.ptr, 0 ) );
275  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxIter( solver.ptr, 1 ) );
276  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggNumLevels( solver.ptr, 1 ) );
277  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggPMaxElmts( solver.ptr, 20 ) );
279  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetTol( solver.ptr, 0.0 ) );
280 #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP
284  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumSweeps( solver.ptr, 2 ) );
285  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxRowSum( solver.ptr, 1.0 ) );
286 #else
287  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetRelaxOrder( solver.ptr, 1 ) );
288 #endif
289 
290  solver.setup = HYPRE_BoomerAMGSetup;
291  solver.solve = HYPRE_BoomerAMGSolve;
292  solver.destroy = HYPRE_BoomerAMGDestroy;
293  }
294 
300  {
301  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGCreate( &solver.ptr ) );
302  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetPrintLevel( solver.ptr, 0 ) );
303  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxIter( solver.ptr, 1 ) );
304  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggNumLevels( solver.ptr, 1 ) ); // TODO: keep or not 1 aggressive level?
305  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggPMaxElmts( solver.ptr, 16 ) );
306  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetTol( solver.ptr, 0.0 ) );
307  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumFunctions( solver.ptr, 2 ) ); // pressure and temperature (CPTR)
308 #if GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_CUDA || GEOS_USE_HYPRE_DEVICE == GEOS_USE_HYPRE_HIP
309  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetAggNumLevels( solver.ptr, 0 ) );
310  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetCoarsenType( solver.ptr, toUnderlying( AMGCoarseningType::PMIS ) ) );
312  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetNumSweeps( solver.ptr, 2 ) );
313  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetMaxRowSum( solver.ptr, 1.0 ) );
314 #else
315  GEOS_LAI_CHECK_ERROR( HYPRE_BoomerAMGSetRelaxOrder( solver.ptr, 1 ) );
316 #endif
317 
318  solver.setup = HYPRE_BoomerAMGSetup;
319  solver.solve = HYPRE_BoomerAMGSolve;
320  solver.destroy = HYPRE_BoomerAMGDestroy;
321  }
322 
333  HypreMGRData & mgrData,
334  integer const & separateComponents )
335  {
336  setDisplacementAMG( mgrData.mechSolver, separateComponents );
337  HYPRE_MGRSetFSolver( precond.ptr, mgrData.mechSolver.solve, mgrData.mechSolver.setup, mgrData.mechSolver.ptr );
338  }
345  {
346  /* (Required) Create ILU solver */
347  HYPRE_ILUCreate( &solver.ptr );
348 
349  /* (Recommended) General solver options */
350  int const ilu_type = 0; /* 0, 1, 10, 11, 20, 21, 30, 31, 40, 41, 50 */
351  int const max_iter = 1;
352  double const tol = 0.0;
353  int const reordering = 0; /* 0: none, 1: RCM */
354  int const print_level = 0;
355  HYPRE_ILUSetType( solver.ptr, ilu_type );
356  HYPRE_ILUSetMaxIter( solver.ptr, max_iter );
357  HYPRE_ILUSetTol( solver.ptr, tol );
358  HYPRE_ILUSetLocalReordering( solver.ptr, reordering );
359  HYPRE_ILUSetPrintLevel( solver.ptr, print_level );
360 
361  solver.setup = HYPRE_ILUSetup;
362  solver.solve = HYPRE_ILUSolve;
363  solver.destroy = HYPRE_ILUDestroy;
364  }
365 
366 };
367 
375 void createMGR( LinearSolverParameters const & params,
376  DofManager const * const dofManager,
377  HyprePrecWrapper & precond,
378  HypreMGRData & mgrData );
379 
380 } // namespace mgr
381 
382 } // namespace hypre
383 
384 } // namespace geos
385 
386 #endif /*GEOS_LINEARALGEBRA_INTERFACES_HYPREMGRSTRATEGIES_HPP_*/
void createMGR(LinearSolverParameters const &params, DofManager const *const dofManager, HyprePrecWrapper &precond, HypreMGRData &mgrData)
Create the MGR preconditioner object.
void setStrategy(LinearSolverParameters::MGR const &params, arrayView1d< int const > const &numComponentsPerField, HyprePrecWrapper &precond, HypreMGRData &mgrData)
Helper to simplify MGR setup.
Definition: HypreMGR.hpp:60
The DoFManager is responsible for allocating global dofs, constructing sparsity patterns,...
Definition: DofManager.hpp:45
Helper struct for strategies that provides some basic parameter arrays needed by MGR.
Definition: HypreMGR.hpp:75
MGRCoarseGridMethod m_levelCoarseGridMethod[numLevels]
Coarse grid method for each level.
Definition: HypreMGR.hpp:92
HYPRE_Int m_numBlocks
Number of different matrix blocks treated separately.
Definition: HypreMGR.hpp:82
void setALMDisplacementAMG(HyprePrecWrapper &solver, integer const separateComponents, bool const bubbleCoarse)
Set up one of the two BoomerAMG instances used by the fully coupled single-phase ALM hierarchy.
Definition: HypreMGR.hpp:225
void setDisplacementAMG(HyprePrecWrapper &solver, integer const &separateComponents)
Set up BoomerAMG to perform the solve for the displacement system.
Definition: HypreMGR.hpp:190
MGRInterpolationType m_levelInterpType[numLevels]
Interpolation type for each level.
Definition: HypreMGR.hpp:90
static HYPRE_Int totalNumBlocks(arrayView1d< int const > const &numComponentsPerField)
Total number of dof labels, i.e. the sum of all fields' components.
Definition: HypreMGR.hpp:107
void setILUCoarseSolver(HyprePrecWrapper &solver)
Definition: HypreMGR.hpp:344
MGRStrategyBase(HYPRE_Int const numBlocks)
Constructor.
Definition: HypreMGR.hpp:121
void setPressureTemperatureAMG(HyprePrecWrapper &solver)
Set up BoomerAMG to perform the solve for the pressure/temperature system.
Definition: HypreMGR.hpp:299
static constexpr HYPRE_Int numLevels
Number of levels.
Definition: HypreMGR.hpp:78
stdVector< HYPRE_Int > m_labels[numLevels]
Dof labels kept at each level.
Definition: HypreMGR.hpp:84
void setMechanicsFSolver(HyprePrecWrapper &precond, HypreMGRData &mgrData, integer const &separateComponents)
Set up BoomerAMG to perform the mechanics F-solve for the first F-relaxation.
Definition: HypreMGR.hpp:332
void setupLabels()
Call this after populating lv_cindexes.
Definition: HypreMGR.hpp:137
HYPRE_Int m_levelFRelaxIters[numLevels]
Number of F-relaxation iterations for each level.
Definition: HypreMGR.hpp:89
void setPressureAMG(HyprePrecWrapper &solver)
Set up BoomerAMG to perform the solve for the pressure system.
Definition: HypreMGR.hpp:271
void setReduction(HyprePrecWrapper &precond, HypreMGRData &mgrData)
Helper function that sets the reduction features common to all mgr strategies.
Definition: HypreMGR.hpp:151
MGRFRelaxationType m_levelFRelaxType[numLevels]
F-relaxation type for each level.
Definition: HypreMGR.hpp:88
HYPRE_Int * m_ptrLabels[numLevels]
Pointers to each level's labels, as consumed by MGR.
Definition: HypreMGR.hpp:86
HYPRE_Int m_levelGlobalSmootherIters[numLevels]
Number of global smoother iterations for each level.
Definition: HypreMGR.hpp:94
HYPRE_Real m_coarseGridThreshold
Coarse grid truncation threshold.
Definition: HypreMGR.hpp:95
MGRGlobalSmootherType m_levelGlobalSmootherType[numLevels]
Global smoother type for each level.
Definition: HypreMGR.hpp:93
HYPRE_Int m_numLabels[numLevels]
Number of dof labels kept.
Definition: HypreMGR.hpp:85
MGRRestrictionType m_levelRestrictType[numLevels]
Restriction type for each level.
Definition: HypreMGR.hpp:91
#define GEOS_LAI_CHECK_ERROR(call)
Definition: common.hpp:119
MGRCoarseGridMethod
This enum class specifies the strategy for level coarse grid computation in MGR.
Definition: HypreUtils.hpp:595
@ galerkin
Galerkin coarse grid computation using RAP.
MGRRestrictionType
This enum class specifies the strategy for computing the level restriction operator in MGR.
Definition: HypreUtils.hpp:580
HYPRE_Int getAMGAggressiveInterpolationType(LinearSolverParameters::AMG::AggInterpType const &type)
Returns hypre's identifier of the AMG aggressive interpolation type.
Definition: HypreUtils.hpp:433
MGRFRelaxationType
This enum class specifies the F-relaxation type.
Definition: HypreUtils.hpp:612
@ none
no F-relaxation if performed
HYPRE_Int getAMGRelaxationType(LinearSolverParameters::AMG::SmootherType const &type)
Returns hypre's identifier of the AMG smoother type.
Definition: HypreUtils.hpp:388
@ PMIS
Parallel coarsening algorithm using independent sets.
HYPRE_Int getAMGCoarseningType(LinearSolverParameters::AMG::CoarseningType const &type)
Returns hypre's identifier of the AMG coarsening type.
Definition: HypreUtils.hpp:494
MGRInterpolationType
This enum class specifies the strategy for computing the level intepolation operator in MGR.
Definition: HypreUtils.hpp:565
MGRGlobalSmootherType
This enum class specifies the global smoother type.
Definition: HypreUtils.hpp:635
@ none
no global smoothing is performed (default)
ArrayView< T, 1 > arrayView1d
Alias for 1D array view.
Definition: DataTypes.hpp:179
GEOS_LOCALINDEX_TYPE localIndex
Local index type (for indexing objects within an MPI partition).
Definition: DataTypes.hpp:84
int integer
Signed integer type.
Definition: DataTypes.hpp:81
Array< T, 1 > array1d
Alias for 1D array.
Definition: DataTypes.hpp:175
internal::StdVectorWrapper< T, Allocator, USE_STD_CONTAINER_BOUNDS_CHECKING > stdVector
Container for hypre preconditioner auxiliary data for MGR.
Definition: HypreMGR.hpp:38
HyprePrecWrapper mechSolver
MGR mechanics fine solver pointer and functions.
Definition: HypreMGR.hpp:41
HyprePrecWrapper nestedSolver
Optional nested MGR F-relaxation wrapper.
Definition: HypreMGR.hpp:42
HyprePrecWrapper coarseSolver
MGR coarse solver pointer and functions.
Definition: HypreMGR.hpp:40
array1d< HYPRE_Int > pointMarkers
array1d of unique tags for local degrees of freedom
Definition: HypreMGR.hpp:39
Container for hypre preconditioner function pointers.
Definition: HypreUtils.hpp:51
HYPRE_Solver ptr
pointer to preconditioner
Definition: HypreUtils.hpp:61
SetupFunc setup
pointer to setup function
Definition: HypreUtils.hpp:62
DestroyFunc destroy
pointer to destroy function
Definition: HypreUtils.hpp:64
SolveFunc solve
pointer to apply function
Definition: HypreUtils.hpp:63
@ chebyshev
Chebyshev polynomial smoothing.
@ Falgout
Ruge-Stueben followed by CLJP.
@ PMIS
Parallel coarsening as CLJP but with lower complexities (GPU support)
@ modifiedExtendedE
Modularized Extended+e (GPU support)
Multigrid reduction parameters.
Set of parameters for a linear solver or preconditioner.