ObjectManagerBase.hpp

Go to the documentation of this file.

/*
 * ------------------------------------------------------------------------------------------------------------
 * SPDX-License-Identifier: LGPL-2.1-only
 *
 * Copyright (c) 2016-2024 Lawrence Livermore National Security LLC
 * Copyright (c) 2018-2024 TotalEnergies
 * Copyright (c) 2018-2024 The Board of Trustees of the Leland Stanford Junior University
 * Copyright (c) 2023-2024 Chevron
 * Copyright (c) 2019-     GEOS/GEOSX Contributors
 * All rights reserved
 *
 * See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details.
 * ------------------------------------------------------------------------------------------------------------
 */
 
#ifndef GEOS_MESH_OBJECTMANAGERBASE_HPP_
#define GEOS_MESH_OBJECTMANAGERBASE_HPP_
 
#include "dataRepository/Group.hpp"
#include "common/TimingMacros.hpp"
#include "mpiCommunications/NeighborData.hpp"
 
namespace geos
{
 
class ObjectManagerBase : public dataRepository::Group
{
public:
  ObjectManagerBase() = delete;
 
  explicit ObjectManagerBase( string const & name,
                              dataRepository::Group * const parent );
 
  ~ObjectManagerBase() override;
 
 
  using CatalogInterface = dataRepository::CatalogInterface< ObjectManagerBase, string const &, dataRepository::Group * const >;
 
  static CatalogInterface::CatalogType & getCatalog();
 
  virtual string getCatalogName() const = 0;
 
  using dataRepository::Group::packSize;
  using dataRepository::Group::pack;
 
  virtual localIndex packSize( string_array const & wrapperNames,
                               arrayView1d< localIndex const > const & packList,
                               integer const recursive,
                               bool onDevice,
                               parallelDeviceEvents & events ) const override;
 
  virtual localIndex pack( buffer_unit_type * & buffer,
                           string_array const & wrapperNames,
                           arrayView1d< localIndex const > const & packList,
                           integer const recursive,
                           bool onDevice,
                           parallelDeviceEvents & events ) const override;
 
  virtual localIndex unpack( buffer_unit_type const * & buffer,
                             arrayView1d< localIndex > & packList,
                             integer const recursive,
                             bool onDevice,
                             parallelDeviceEvents & events,
                             MPI_Op op=MPI_REPLACE ) override;
 
  template< bool DO_PACKING >
  localIndex packSets( buffer_unit_type * & buffer,
                       arrayView1d< localIndex const > const & packList ) const;
 
  localIndex unpackSets( buffer_unit_type const * & buffer );
 
  void excludeWrappersFromPacking( std::set< string > const & wrapperNames );
 
  virtual localIndex packGlobalMapsSize( arrayView1d< localIndex const > const & packList,
                                         integer const recursive ) const;
 
  virtual localIndex packGlobalMaps( buffer_unit_type * & buffer,
                                     arrayView1d< localIndex const > const & packList,
                                     integer const recursive ) const;
 
  void setReceiveLists();
 
  virtual localIndex packUpDownMapsSize( arrayView1d< localIndex const > const & packList ) const
  {
    GEOS_UNUSED_VAR( packList );
    return 0;
  }
 
  virtual localIndex packUpDownMaps( buffer_unit_type * & buffer,
                                     arrayView1d< localIndex const > const & packList ) const
  {
    GEOS_UNUSED_VAR( buffer );
    GEOS_UNUSED_VAR( packList );
    return 0;
  }
 
  virtual localIndex unpackUpDownMaps( buffer_unit_type const * & buffer,
                                       array1d< localIndex > & packList,
                                       bool const overwriteUpMaps,
                                       bool const overwriteDownMaps )
  {
    GEOS_UNUSED_VAR( buffer );
    GEOS_UNUSED_VAR( packList );
    GEOS_UNUSED_VAR( overwriteUpMaps );
    GEOS_UNUSED_VAR( overwriteDownMaps );
    return 0;
  }
 
  virtual localIndex unpackGlobalMaps( buffer_unit_type const * & buffer,
                                       localIndex_array & packList,
                                       integer const recursive );
 
  localIndex packParentChildMapsSize( arrayView1d< localIndex const > const & packList ) const
  {
    buffer_unit_type * buffer = nullptr;
    return packParentChildMapsImpl< false >( buffer, packList );
  }
 
  localIndex packParentChildMaps( buffer_unit_type * & buffer,
                                  arrayView1d< localIndex const > const & packList ) const
  {
    return packParentChildMapsImpl< true >( buffer, packList );
  }
 
  localIndex unpackParentChildMaps( buffer_unit_type const * & buffer,
                                    localIndex_array & packList );
 
private:
  template< bool DO_PACKING >
  localIndex packImpl( buffer_unit_type * & buffer,
                       string_array const & wrapperNames,
                       arrayView1d< localIndex const > const & packList,
                       integer const recursive,
                       bool onDevice,
                       parallelDeviceEvents & events ) const;
 
  template< bool DO_PACKING >
  localIndex packGlobalMapsImpl( buffer_unit_type * & buffer,
                                 arrayView1d< localIndex const > const & packList,
                                 integer const recursive ) const;
 
  template< bool DO_PACKING >
  localIndex packParentChildMapsImpl( buffer_unit_type * & buffer,
                                      arrayView1d< localIndex const > const & packList ) const;
 
  //**********************************************************************************************************************
  // functions for compatibility with old data structure
  // TODO Deprecate or modernize all these suckers
 
public:
 
  void moveSets( LvArray::MemorySpace const targetSpace );
 
  localIndex resize( localIndex const newSize,
                     const bool /*assignGlobals*/ )
  {
    dataRepository::Group::resize( newSize );
    return 0;
  }
 
  using dataRepository::Group::resize;
 
  SortedArray< localIndex > & createSet( const string & newSetName );
 
  void constructSetFromSetAndMap( SortedArrayView< localIndex const > const & inputSet,
                                  const array2d< localIndex > & map,
                                  const string & setName );
  void constructSetFromSetAndMap( SortedArrayView< localIndex const > const & inputSet,
                                  const array1d< localIndex_array > & map,
                                  const string & setName );
  void constructSetFromSetAndMap( SortedArrayView< localIndex const > const & inputSet,
                                  ArrayOfArraysView< localIndex const > const & map,
                                  const string & setName );
 
  void constructGlobalToLocalMap();
 
  array1d< globalIndex > constructGlobalListOfBoundaryObjects() const;
 
  virtual ArrayOfSets< globalIndex >
  extractMapFromObjectForAssignGlobalIndexNumbers( ObjectManagerBase const & nodeManager )
  {
    GEOS_UNUSED_VAR( nodeManager );
    return {};
  }
 
  void setGhostRankForSenders( int const neighborRank )
  {
    arrayView1d< localIndex const > const ghostsToSend = getNeighborData( neighborRank ).ghostsToSend();
    array1d< std::pair< globalIndex, int > > & nonLocalGhosts = getNeighborData( neighborRank ).nonLocalGhosts();
    nonLocalGhosts.clear();
 
    for( localIndex const index : ghostsToSend )
    {
      integer & owningRank = m_ghostRank[ index ];
      if( owningRank >= 0 )
      {
        nonLocalGhosts.emplace_back( m_localToGlobalMap[ index ], owningRank );
      }
      else
      {
        owningRank = -1;
      }
    }
  }
 
  localIndex getNumberOfGhosts() const;
 
  localIndex getNumberOfLocalIndices() const;
 
  integer splitObject( localIndex const indexToSplit,
                       int const rank,
                       localIndex & newIndex );
 
  void inheritGhostRankFromParent( std::set< localIndex > const & indices );
 
  void copyObject( localIndex const source, localIndex const destination );
 
  void eraseObject( std::set< localIndex > const & indicesToErase );
 
  virtual void setMaxGlobalIndex();
 
  template< typename TYPE_RELATION >
  static void fixUpDownMaps( TYPE_RELATION & relation,
                             map< localIndex, array1d< globalIndex > > & unmappedIndices,
                             bool const clearIfUnmapped );
 
  template< typename TYPE_RELATION >
  static void fixUpDownMaps( TYPE_RELATION & relation,
                             map< localIndex, SortedArray< globalIndex > > & unmappedIndices,
                             bool const clearIfUnmapped );
 
  static void fixUpDownMaps( ArrayOfSets< localIndex > & relation,
                             unordered_map< globalIndex, localIndex > const & globalToLocal,
                             map< localIndex, SortedArray< globalIndex > > & unmappedIndices,
                             bool const clearIfUnmapped );
 
  static void cleanUpMap( std::set< localIndex > const & targetIndices,
                          array1d< SortedArray< localIndex > > & upmap,
                          arrayView2d< localIndex const > const & downmap );
 
  static void cleanUpMap( std::set< localIndex > const & targetIndices,
                          ArrayOfSetsView< localIndex > const & upmap,
                          arrayView2d< localIndex const > const & downmap );
 
  static void cleanUpMap( std::set< localIndex > const & targetIndices,
                          array1d< SortedArray< localIndex > > & upmap,
                          arrayView1d< arrayView1d< localIndex const > const > const & downmap );
  static void cleanUpMap( std::set< localIndex > const & targetIndices,
                          ArrayOfSetsView< localIndex > const & upmap,
                          arrayView1d< arrayView1d< localIndex const > const > const & downmap );
  static void cleanUpMap( std::set< localIndex > const & targetIndices,
                          ArrayOfSetsView< localIndex > const & upmap,
                          ArrayOfArraysView< localIndex const > const & downmap );
 
  virtual void enforceStateFieldConsistencyPostTopologyChange( std::set< localIndex > const & targetIndices );
 
  static localIndex getParentRecursive( arraySlice1d< localIndex const > const & parentIndices,
                                        localIndex const lookup )
  {
    localIndex rval = lookup;
 
    while( parentIndices[rval] != -1 )
    {
      rval = parentIndices[rval];
    }
 
    return rval;
  }
 
  template< typename FIELD_TRAIT >
  dataRepository::Wrapper< typename FIELD_TRAIT::type > &
  registerField( string const & nameOfRegisteringObject )
  {
    // These are required to work-around the need for instantiation of
    // the static constexpr trait components. This will not be required once
    // we move to c++17.
 
    // This is required for the Tensor classes.
    typename FIELD_TRAIT::dataType defaultValue( FIELD_TRAIT::defaultValue() );
 
    m_registeredField.insert( FIELD_TRAIT::key());
 
    return this->registerWrapper< typename FIELD_TRAIT::type >( FIELD_TRAIT::key() ).
             setApplyDefaultValue( defaultValue ).
             setPlotLevel( FIELD_TRAIT::plotLevel ).
             setRestartFlags( FIELD_TRAIT::restartFlag ).
             setDescription( FIELD_TRAIT::description ).
             setRegisteringObjects( nameOfRegisteringObject );
  }
 
  template< typename FIELD_TRAIT >
  dataRepository::Wrapper< typename FIELD_TRAIT::type > & registerField( FIELD_TRAIT const & fieldTrait,
                                                                         typename FIELD_TRAIT::type * newObject )
  {
 
    m_registeredField.insert( FIELD_TRAIT::key());
 
    return registerWrapper( fieldTrait.key(), newObject ).
             setApplyDefaultValue( fieldTrait.defaultValue() ).
             setPlotLevel( FIELD_TRAIT::plotLevel ).
             setRestartFlags( FIELD_TRAIT::restartFlag ).
             setDescription( FIELD_TRAIT::description );
  }
 
  template< typename FIELD_TRAIT0, typename FIELD_TRAIT1, typename ... FIELD_TRAITS >
  void registerField( string const & nameOfRegisteringObject )
  {
    registerField< FIELD_TRAIT0 >( nameOfRegisteringObject );
    registerField< FIELD_TRAIT1, FIELD_TRAITS... >( nameOfRegisteringObject );
  }
 
  template< typename FIELD_TRAIT >
  GEOS_DECLTYPE_AUTO_RETURN getField() const
  {
    return this->getWrapper< typename FIELD_TRAIT::type >( FIELD_TRAIT::key() ).reference();
  }
 
  template< typename FIELD_TRAIT >
  GEOS_DECLTYPE_AUTO_RETURN getField()
  {
    return this->getWrapper< typename FIELD_TRAIT::type >( FIELD_TRAIT::key() ).reference();
  }
 
  template< typename FIELD_TRAIT >
  bool hasField() const
  {
    return this->hasWrapper( FIELD_TRAIT::key() );
  }
 
  //**********************************************************************************************************************
 
  struct viewKeyStruct
  {
    static constexpr char const * adjacencyListString() { return "adjacencyList"; }
 
    static constexpr char const * domainBoundaryIndicatorString() { return "domainBoundaryIndicator"; }
 
    static constexpr char const * externalSetString() { return "externalSet"; }
 
    static constexpr char const * ghostRankString() { return "ghostRank"; }
 
    static constexpr char const * ghostsToReceiveString() { return "ghostsToReceive"; }
 
    static constexpr char const * globalToLocalMapString() { return "globalToLocalMap"; }
 
    static constexpr char const * isExternalString() { return "isExternal"; }
 
    static constexpr char const * localToGlobalMapString() { return "localToGlobalMap"; }
 
    static constexpr char const * localMaxGlobalIndexString() { return "localMaxGlobalIndex"; }
 
    static constexpr char const * maxGlobalIndexString() { return "maxGlobalIndex"; }
 
    dataRepository::ViewKey externalSet = { externalSetString() };
 
    dataRepository::ViewKey ghostRank = { ghostRankString() };
 
    dataRepository::ViewKey globalToLocalMap = { globalToLocalMapString() };
 
    dataRepository::ViewKey localToGlobalMap = { localToGlobalMapString() };
  }
  m_ObjectManagerBaseViewKeys;
 
  struct groupKeyStruct
  {
    static constexpr char const * setsString() { return "sets"; }
 
    static constexpr char const * neighborDataString() { return "neighborData"; }
 
    dataRepository::GroupKey sets = { setsString() };
 
    dataRepository::GroupKey neighborData{ neighborDataString() };
  }
  m_ObjectManagerBaseGroupKeys;
 
  virtual viewKeyStruct & viewKeys() { return m_ObjectManagerBaseViewKeys; }
 
  virtual viewKeyStruct const & viewKeys() const { return m_ObjectManagerBaseViewKeys; }
  virtual groupKeyStruct & groupKeys() { return m_ObjectManagerBaseGroupKeys; }
  virtual groupKeyStruct const & groupKeys() const { return m_ObjectManagerBaseGroupKeys; }
 
  Group & sets()
  { return m_sets; }
 
  Group const & sets() const
  { return m_sets; }
 
  SortedArray< localIndex > & getSet( string const & setName )
  { return m_sets.getReference< SortedArray< localIndex > >( setName ); }
 
  SortedArrayView< localIndex const > getSet( string const & setName ) const
  { return m_sets.getReference< SortedArray< localIndex > >( setName ).toViewConst(); }
 
  SortedArray< localIndex > & externalSet()
  { return m_sets.getReference< SortedArray< localIndex > >( m_ObjectManagerBaseViewKeys.externalSet ); }
 
  SortedArrayView< localIndex const > externalSet() const
  { return m_sets.getReference< SortedArray< localIndex > >( m_ObjectManagerBaseViewKeys.externalSet ); }
 
  void updateGlobalToLocalMap( localIndex const lid )
  {
    globalIndex const gid = m_localToGlobalMap[ lid ];
    m_localMaxGlobalIndex = std::max( m_localMaxGlobalIndex, gid );
    m_globalToLocalMap[ gid ] = lid;
  }
 
  arrayView1d< globalIndex > localToGlobalMap()
  { return m_localToGlobalMap; }
 
  arrayView1d< globalIndex const > localToGlobalMap() const
  { return m_localToGlobalMap; }
 
  unordered_map< globalIndex, localIndex > const & globalToLocalMap() const
  { return m_globalToLocalMap; }
 
  localIndex globalToLocalMap( globalIndex const gid ) const
  { return m_globalToLocalMap.at( gid ); }
 
  array1d< integer > const & isExternal()
  { return this->m_isExternal; }
 
  arrayView1d< integer const > isExternal() const
  { return this->m_isExternal; }
 
  array1d< integer > const & ghostRank()
  { return this->m_ghostRank; }
 
  arrayView1d< integer const > ghostRank() const
  { return this->m_ghostRank; }
 
  NeighborData & getNeighborData( int const rank )
  { return m_neighborData.at( rank ); }
 
  NeighborData const & getNeighborData( int const rank ) const
  { return m_neighborData.at( rank ); }
 
  void addNeighbor( int const rank )
  {
    string const & rankString = std::to_string( rank );
    m_neighborData.emplace( std::piecewise_construct, std::make_tuple( rank ), std::make_tuple( rankString, &m_neighborGroup ) );
    m_neighborGroup.registerGroup( rankString, &getNeighborData( rank ) );
  }
 
  void removeNeighbor( int const rank )
  {
    m_neighborGroup.deregisterGroup( getNeighborData( rank ).getName() );
    m_neighborData.erase( rank );
  }
 
  globalIndex localMaxGlobalIndex() const
  { return m_localMaxGlobalIndex; }
 
  globalIndex maxGlobalIndex() const
  { return m_maxGlobalIndex; }
 
  std::set< string > const & getRegisteredFields() const { return m_registeredField; }
 
 
  array1d< integer > & getDomainBoundaryIndicator()
  {
    return m_domainBoundaryIndicator;
  }
 
  arrayView1d< integer const > getDomainBoundaryIndicator() const
  {
    return m_domainBoundaryIndicator.toViewConst();
  }
 
  virtual void outputObjectConnectivity() const
  {
    GEOS_ERROR( "Called outputObjectConnectivity in ObjectManagerBase. Function should be implemented." );
  }
 
protected:
  Group m_sets;
 
  std::set< string > m_packingExclusionList;
 
  Group m_neighborGroup;
 
  array1d< globalIndex > m_localToGlobalMap;
 
  unordered_map< globalIndex, localIndex > m_globalToLocalMap;
 
  array1d< integer > m_isExternal;
 
  array1d< integer > m_domainBoundaryIndicator;
 
  array1d< integer > m_ghostRank;
 
  unordered_map< int, NeighborData > m_neighborData;
 
  real64 m_overAllocationFactor = 1.1;
 
  globalIndex m_maxGlobalIndex = -1;
 
  globalIndex m_localMaxGlobalIndex = -1;
 
  std::set< string > m_registeredField = {};
};
 
 
template< typename TYPE_RELATION >
void ObjectManagerBase::fixUpDownMaps( TYPE_RELATION & relation,
                                       map< localIndex, array1d< globalIndex > > & unmappedIndices,
                                       bool const )
{
  GEOS_MARK_FUNCTION;
 
  bool allValuesMapped = true;
  unordered_map< globalIndex, localIndex > const & globalToLocal = relation.relatedObjectGlobalToLocal();
  for( auto & unmappedIndex: unmappedIndices )
  {
    localIndex const li = unmappedIndex.first;
    array1d< globalIndex > const & globalIndices = unmappedIndex.second;
    for( localIndex a = 0; a < globalIndices.size(); ++a )
    {
      if( globalIndices[a] != unmappedLocalIndexValue )
      {
        if( relation[li][a] == unmappedLocalIndexValue )
        {
          relation[li][a] = globalToLocal.at( globalIndices[a] );
        }
        else
        {
          allValuesMapped = false;
        }
      }
      // temporarily disabled this check to allow for the case where the index is not set
      // this entire fixUpDownMaps will be removed in a future PR as the unpacking is modified
      // s.t. there are no invalid unpacked values that are not expected.
      //GEOS_ERROR_IF( relation[li][a] == unmappedLocalIndexValue, "Index not set" );
    }
  }
  GEOS_ERROR_IF( !allValuesMapped, "some values of unmappedIndices were not used" );
  unmappedIndices.clear();
}
 
 
template< typename TYPE_RELATION >
void ObjectManagerBase::fixUpDownMaps( TYPE_RELATION & relation,
                                       map< localIndex, SortedArray< globalIndex > > & unmappedIndices,
                                       bool const clearIfUnmapped )
{
  GEOS_MARK_FUNCTION;
 
  unordered_map< globalIndex, localIndex > const & globalToLocal = relation.RelatedObjectGlobalToLocal();
  for( map< localIndex, SortedArray< globalIndex > >::iterator iter = unmappedIndices.begin();
       iter != unmappedIndices.end();
       ++iter )
  {
    localIndex const li = iter->first;
    if( clearIfUnmapped )
    {
      relation[li].clear();
    }
    else
    {
      SortedArray< globalIndex > const & globalIndices = iter->second;
      for( auto const newGlobalIndex : globalIndices )
      {
        // NOTE: This simply ignores if newGlobalIndex is not found. This is OK if this function is
        // used for an upmap and the object shouldn't exist on this rank. There should be a better
        // way to check this.
        auto iterG2L = globalToLocal.find( newGlobalIndex );
        if( iterG2L != globalToLocal.end() )
        {
          relation[li].insert( iterG2L->second );
        }
      }
    }
  }
  unmappedIndices.clear();
}
 
inline
void ObjectManagerBase::fixUpDownMaps( ArrayOfSets< localIndex > & relation,
                                       unordered_map< globalIndex, localIndex > const & globalToLocal,
                                       map< localIndex, SortedArray< globalIndex > > & unmappedIndices,
                                       bool const clearIfUnmapped )
{
  GEOS_MARK_FUNCTION;
 
  for( map< localIndex, SortedArray< globalIndex > >::iterator iter = unmappedIndices.begin();
       iter != unmappedIndices.end();
       ++iter )
  {
    localIndex const li = iter->first;
    if( clearIfUnmapped )
    {
      relation.clearSet( li );
    }
    else
    {
      SortedArray< globalIndex > const & globalIndices = iter->second;
      for( globalIndex const newGlobalIndex : globalIndices )
      {
        // NOTE: This simply ignores if newGlobalIndex is not found. This is OK if this function is
        // used for an upmap and the object shouldn't exist on this rank. There should be a better
        // way to check this.
        auto iterG2L = globalToLocal.find( newGlobalIndex );
        if( iterG2L != globalToLocal.end() )
        {
          relation.insertIntoSet( li, iterG2L->second );
        }
      }
    }
  }
  unmappedIndices.clear();
}
 
} /* namespace geos */
 
 
typedef geos::ObjectManagerBase ObjectDataStructureBaseT;
 
#endif /* GEOS_MESH_OBJECTMANAGERBASE_HPP_ */