olb::BlockCommunicationNeighborhood< T, D > Class Template Reference

Configurable overlap communication neighborhood of a block. More...

#include <blockCommunicationNeighborhood.h>

Inheritance diagram for olb::BlockCommunicationNeighborhood< T, D >:

Collaboration diagram for olb::BlockCommunicationNeighborhood< T, D >:

Public Member Functions
	BlockCommunicationNeighborhood (CuboidDecomposition< T, D > &cuboidDecomposition, LoadBalancer< T > &loadBalancer, int iC, int padding, MPI_Comm comm)
template<typename FIELD >
void	requestField ()
	Request field and provides local availability.
void	requestCell (LatticeR< D > latticeR)
	Request individual cell for communication.
void	requestOverlap (int width)
	Request all cells in overlap of size width for communication.
void	requestOverlap (int width, BlockIndicatorF< T, D > &indicatorF)
	Request all indicated cells in overlap of width for communication.
void	clearRequestedCells ()
	Remove all requested cells.
void	setFieldAvailability (std::type_index field, bool available)
	Update local availability of previously requested field.
template<typename BLOCK >
void	setFieldsAvailability (int iC, BLOCK &block)
	Update outbound availabilities for locally available neighbor block.
void	maintain ()
void	send (SuperCommunicationTagCoordinator< T > &)
void	receive (SuperCommunicationTagCoordinator< T > &)
void	wait ()
template<typename F >
void	forNeighbors (F f) const
	Calls f(iC) for every neighboring cuboid ID iC.
const std::vector< std::type_index > &	getFieldsCommonWith (int iC) const
const std::vector< CellID > &	getCellsOutboundTo (int iC) const
const std::vector< CellID > &	getCellsInboundFrom (int iC) const
const std::vector< CellID > &	getCellsRequestedFrom (int iC) const

Detailed Description

template<typename T, unsigned D>
class olb::BlockCommunicationNeighborhood< T, D >

Configurable overlap communication neighborhood of a block.

Managed by SuperCommunicator

Definition at line 48 of file blockCommunicationNeighborhood.h.

Constructor & Destructor Documentation

BlockCommunicationNeighborhood()

template<typename T , unsigned D>

olb::BlockCommunicationNeighborhood< T, D >::BlockCommunicationNeighborhood	(	CuboidDecomposition< T, D > &	cuboidDecomposition,
		LoadBalancer< T > &	loadBalancer,
		int	iC,
		int	padding,
		MPI_Comm	comm )

Definition at line 39 of file blockCommunicationNeighborhood.hh.

 :
    _cuboidDecomposition(cuboidDecomposition)
  , _loadBalancer(loadBalancer)
  , _iC(iC)
  , _padding(padding)
#ifdef PARALLEL_MODE_MPI
  , _neighborhoodComm(comm)
#endif
{
  // Ensure that any neighboring cuboids are interacted with during request negotiation
  requestOverlap(1);
  for (auto& [_, cells] : _cellsInboundFrom) {
    cells.clear();
  }
  for (auto& [_, cells] : _cellsRequestedFrom) {
    cells.clear();
  }
}

References olb::BlockCommunicationNeighborhood< T, D >::requestOverlap().

Here is the call graph for this function:

Member Function Documentation

clearRequestedCells()

template<typename T , unsigned D>

void olb::BlockCommunicationNeighborhood< T, D >::clearRequestedCells

(

)

Remove all requested cells.

Definition at line 130 of file blockCommunicationNeighborhood.hh.

{
  _cellsInboundFrom.clear();
  _cellsOutboundTo.clear();
  _cellsRequestedFrom.clear();
}

forNeighbors()

template<typename T , unsigned D>

template<typename F >

void olb::BlockCommunicationNeighborhood< T, D >::forNeighbors ( F f ) const

inline

Calls f(iC) for every neighboring cuboid ID iC.

Definition at line 118 of file blockCommunicationNeighborhood.h.

                               {
    for (const auto& [iC, _] : _cellsRequestedFrom) {
      f(iC);
    }
  }

Here is the caller graph for this function:

getCellsInboundFrom()

template<typename T , unsigned D>

const std::vector< CellID > & olb::BlockCommunicationNeighborhood< T, D >::getCellsInboundFrom

(

int

iC

)

const

Definition at line 268 of file blockCommunicationNeighborhood.hh.

{
  return _cellsInboundFrom.at(iC);
}

Here is the caller graph for this function:

getCellsOutboundTo()

template<typename T , unsigned D>

const std::vector< CellID > & olb::BlockCommunicationNeighborhood< T, D >::getCellsOutboundTo

(

int

iC

)

const

Definition at line 261 of file blockCommunicationNeighborhood.hh.

{
  return _cellsOutboundTo.at(iC);
}

Here is the caller graph for this function:

getCellsRequestedFrom()

template<typename T , unsigned D>

const std::vector< CellID > & olb::BlockCommunicationNeighborhood< T, D >::getCellsRequestedFrom

(

int

iC

)

const

Definition at line 275 of file blockCommunicationNeighborhood.hh.

{
  return _cellsRequestedFrom.at(iC);
}

Here is the caller graph for this function:

getFieldsCommonWith()

template<typename T , unsigned D>

const std::vector< std::type_index > & olb::BlockCommunicationNeighborhood< T, D >::getFieldsCommonWith

(

int

iC

)

const

Definition at line 254 of file blockCommunicationNeighborhood.hh.

{
  return _fieldsCommonWith.at(iC);
}

Here is the caller graph for this function:

maintain()

template<typename T , unsigned D>

void olb::BlockCommunicationNeighborhood< T, D >::maintain

(

)

Definition at line 161 of file blockCommunicationNeighborhood.hh.

{
  for (auto& [iC, _] : _cellsInboundFrom) {
    auto& iCells = _cellsInboundFrom[iC];
    auto& oCells = _cellsRequestedFrom[iC];
    // Compute sorted indices w.r.t. cell IDs in iCells
    std::vector<std::ptrdiff_t> p(iCells.size());
    std::iota(p.begin(), p.end(), 0);
    std::sort(p.begin(), p.end(), [&iCells](auto i, auto j) { return iCells[i] < iCells[j]; });
    // Reduce indices to only include unique values of iCells
    auto pu = std::unique(p.begin(), p.end(), [&iCells](auto i, auto j) { return iCells[i] == iCells[j]; });
    p.erase(pu, p.end());
    // Copy cell IDs at computed indices to new in and out lists
    {
      std::vector<CellID> buffer(p.size());
      std::transform(p.begin(), p.end(), buffer.begin(), [&iCells](auto i) { return iCells[i]; });
      iCells = buffer;
    }
    {
      std::vector<CellID> buffer(p.size());
      std::transform(p.begin(), p.end(), buffer.begin(), [&oCells](auto i) { return oCells[i]; });
      oCells = buffer;
    }
  }
}

receive()

template<typename T , unsigned D>

void olb::BlockCommunicationNeighborhood< T, D >::receive

(

SuperCommunicationTagCoordinator< T > &

coordinator

)

Definition at line 206 of file blockCommunicationNeighborhood.hh.

{
  forNeighbors([&](int iC) {
    std::unique_ptr<bool[]> fieldsAvailable(new bool[_fieldsRequested.size()] { });
    auto& fieldsCommonWith = _fieldsCommonWith[iC];
    fieldsCommonWith.clear();
 
    singleton::mpi().receive(
      fieldsAvailable.get(), _fieldsRequested.size(),
      _loadBalancer.rank(iC),
      coordinator.get(iC, _iC, 0),
      _neighborhoodComm);
 
    for (unsigned iField=0; iField < _fieldsRequested.size(); ++iField) {
      if (fieldsAvailable[iField] && _fieldsAvailable[iField]) {
        fieldsCommonWith.emplace_back(_fieldsRequested[iField]);
      }
    }
 
    static_assert(sizeof(CellID) == 4 || sizeof(CellID) == 8,
                  "CellID must be either 4 or 8 byte unsigned integer");
    std::size_t newOutboundCount = singleton::mpi().probeReceiveSize<CellID>(_loadBalancer.rank(iC),
                                                                             coordinator.get(iC, _iC, 1),
                                                                             _neighborhoodComm);
    _cellsOutboundTo[iC].resize(newOutboundCount);
 
    singleton::mpi().receive(
      _cellsOutboundTo[iC].data(),
      _cellsOutboundTo[iC].size(),
      _loadBalancer.rank(iC),
      coordinator.get(iC, _iC, 1),
      _neighborhoodComm);
  });
}

References olb::SuperCommunicationTagCoordinator< T >::get(), olb::singleton::mpi(), olb::singleton::MpiManager::probeReceiveSize(), and olb::singleton::MpiManager::receive().

Here is the call graph for this function:

requestCell()

template<typename T , unsigned D>

void olb::BlockCommunicationNeighborhood< T, D >::requestCell

(

LatticeR< D >

latticeR

)

Request individual cell for communication.

Definition at line 67 of file blockCommunicationNeighborhood.hh.

{
  const auto& cuboid = _cuboidDecomposition.get(_iC);
  BlockStructureD<D> paddedBlock(cuboid.getExtent(), _padding);
  if (paddedBlock.isPadding(latticeR)) {
    // Read physR using global cuboid geometry instead of local cuboid
    // to resolve periodic boundaries
    auto physR = _cuboidDecomposition.getPhysR(latticeR.withPrefix(_iC));
    if (auto remoteLatticeR = _cuboidDecomposition.getLatticeR(physR)) {
      _cellsInboundFrom[(*remoteLatticeR)[0]].emplace_back(paddedBlock.getCellId(latticeR));
 
      const auto& remoteCuboid = _cuboidDecomposition.get((*remoteLatticeR)[0]);
      BlockStructureD<D> remotePaddedBlock(remoteCuboid.getExtent(), _padding);
 
      _cellsRequestedFrom[(*remoteLatticeR)[0]].emplace_back(
        remotePaddedBlock.getCellId(&(*remoteLatticeR)[1]));
    }
  } else {
    throw std::logic_error("Requested cell is outside of available padding");
  }
}

References olb::BlockStructureD< D >::isPadding().

Here is the call graph for this function:

requestField()

template<typename T , unsigned D>

template<typename FIELD >

void olb::BlockCommunicationNeighborhood< T, D >::requestField ( )

inline

Request field and provides local availability.

Definition at line 83 of file blockCommunicationNeighborhood.h.

                      {
    if (std::find(_fieldsRequested.begin(), _fieldsRequested.end(), typeid(FIELD)) == _fieldsRequested.end()) {
      _fieldsRequested.emplace_back(typeid(FIELD));
      _fieldsAvailable.resize(_fieldsRequested.size());
    }
  }

References olb::cpu::sisd::Column< T >::resize().

Here is the call graph for this function:

requestOverlap() [1/2]

template<typename T , unsigned D>

void olb::BlockCommunicationNeighborhood< T, D >::requestOverlap

(

int

width

)

Request all cells in overlap of size width for communication.

Definition at line 90 of file blockCommunicationNeighborhood.hh.

{
  if (width < 1) {
    throw std::logic_error("Overlap requests must have width >= 1");
  }
  if (width > _padding) {
    throw std::logic_error("Requested overlap exceeds available padding");
  }
 
  const auto& cuboid = _cuboidDecomposition.get(_iC);
  BlockStructureD<D> overlapBlock(cuboid.getExtent(), width);
 
  overlapBlock.forSpatialLocations([&](LatticeR<D> latticeR) {
    if (overlapBlock.isPadding(latticeR)) {
      requestCell(latticeR);
    }
  });
}

Here is the caller graph for this function:

requestOverlap() [2/2]

template<typename T , unsigned D>

void olb::BlockCommunicationNeighborhood< T, D >::requestOverlap	(	int	width,
		BlockIndicatorF< T, D > &	indicatorF )

Request all indicated cells in overlap of width for communication.

Definition at line 110 of file blockCommunicationNeighborhood.hh.

{
  if (width < 1) {
    throw std::logic_error("Overlap requests must have width >= 1");
  }
  if (width > _padding) {
    throw std::logic_error("Requested overlap exceeds available padding");
  }
 
  const auto& cuboid = _cuboidDecomposition.get(_iC);
  BlockStructureD<D> overlapBlock(cuboid.getExtent(), width);
 
  overlapBlock.forSpatialLocations([&](LatticeR<D> latticeR) {
    if (overlapBlock.isPadding(latticeR) && indicatorF(latticeR)) {
      requestCell(latticeR);
    }
  });
}

send()

template<typename T , unsigned D>

void olb::BlockCommunicationNeighborhood< T, D >::send

(

SuperCommunicationTagCoordinator< T > &

coordinator

)

Definition at line 190 of file blockCommunicationNeighborhood.hh.

{
  for (auto& [iC, cells] : _cellsRequestedFrom) {
    _fieldRequests[iC] = std::make_unique<MpiSendRequest>(
      _fieldsAvailable.data(), _fieldsRequested.size(),
      _loadBalancer.rank(iC), coordinator.get(_iC, iC, 0), _neighborhoodComm);
    _fieldRequests[iC]->start();
 
    _cellsRequests[iC] = std::make_unique<MpiSendRequest>(
      cells.data(), cells.size(),
      _loadBalancer.rank(iC), coordinator.get(_iC, iC, 1), _neighborhoodComm);
    _cellsRequests[iC]->start();
  }
}

References olb::SuperCommunicationTagCoordinator< T >::get().

Here is the call graph for this function:

setFieldAvailability()

template<typename T , unsigned D>

void olb::BlockCommunicationNeighborhood< T, D >::setFieldAvailability	(	std::type_index	field,
		bool	available )

Update local availability of previously requested field.

Definition at line 138 of file blockCommunicationNeighborhood.hh.

{
  auto iter = std::find(_fieldsRequested.begin(), _fieldsRequested.end(), field);
  if (iter != _fieldsRequested.end()) {
    _fieldsAvailable[iter - _fieldsRequested.begin()] = available;
  }
}

setFieldsAvailability()

template<typename T , unsigned D>

template<typename BLOCK >

void olb::BlockCommunicationNeighborhood< T, D >::setFieldsAvailability	(	int	iC,
		BLOCK &	block )

Update outbound availabilities for locally available neighbor block.

Used for local non-MPI exchange of field availabilities

Definition at line 148 of file blockCommunicationNeighborhood.hh.

{
  auto& fieldsCommonWith = _fieldsCommonWith[iC];
  fieldsCommonWith.clear();
  for (unsigned iField=0; iField < _fieldsRequested.size(); ++iField) {
    auto field = _fieldsRequested[iField];
    if (block.hasCommunicatable(field) && _fieldsAvailable[iField]) {
      fieldsCommonWith.emplace_back(field);
    }
  }
}

wait()

template<typename T , unsigned D>

void olb::BlockCommunicationNeighborhood< T, D >::wait

(

)

Definition at line 242 of file blockCommunicationNeighborhood.hh.

{
  forNeighbors([&](int iC) {
    _fieldRequests[iC]->wait();
    _cellsRequests[iC]->wait();
  });
}

---

The documentation for this class was generated from the following files:

• src/communication/blockCommunicationNeighborhood.h
• src/communication/blockCommunicationNeighborhood.hh