olb::particles::communication Namespace Reference

Classes
struct	ParticleCommunicator

Functions
template<typename T , typename PARTICLECONTACTTYPE , typename WALLCONTACTTYPE >
std::unordered_set< int >	evalDestRanks (contact::ContactContainer< T, PARTICLECONTACTTYPE, WALLCONTACTTYPE > &contactContainer, const std::size_t globalParticleID)
	evaluates ranks that need the new particle data
template<typename T , typename PARTICLETYPE >
void	communicateEquationsOfMotionResultsIntra (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, const std::size_t &pID, const Vector< T, PARTICLETYPE::d > &position, const Vector< T, PARTICLETYPE::d > &velocity, const Vector< T, utilities::dimensions::convert< PARTICLETYPE::d >::rotation > &angle, const Vector< T, utilities::dimensions::convert< PARTICLETYPE::d >::rotation > &angVelocity)
template<typename T , typename PARTICLETYPE , typename PARTICLECONTACTTYPE , typename WALLCONTACTTYPE >
std::size_t	evalEquationsOfMotionResults (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, contact::ContactContainer< T, PARTICLECONTACTTYPE, WALLCONTACTTYPE > &contactContainer, std::multimap< int, std::unique_ptr< std::uint8_t[]> > &dataMap)
template<typename T , typename PARTICLETYPE >
void	receiveEquationsOfMotionResults (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, MPI_Comm equationsOfMotionComm, singleton::MpiNonBlockingHelper &mpiNbHelper)
template<typename T , typename PARTICLETYPE , typename PARTICLECONTACTTYPE , typename WALLCONTACTTYPE >
void	communicateEquationsOfMotionResults (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, contact::ContactContainer< T, PARTICLECONTACTTYPE, WALLCONTACTTYPE > &contactContainer, MPI_Comm equationsOfMotionComm)
template<typename T , typename PARTICLETYPE >
void	updateSurfacePtr (ParticleSystem< T, PARTICLETYPE > &particleSystem, std::size_t iP)
template<typename T , typename PARTICLETYPE >
std::size_t	appendSerializedParticle (ParticleSystem< T, PARTICLETYPE > &particleSystem, std::uint8_t rawBuffer[])
template<typename T , typename PARTICLETYPE >
std::size_t	insertSerializedParticle (ParticleSystem< T, PARTICLETYPE > &particleSystem, std::size_t idxParticle, std::uint8_t rawBuffer[])
template<typename T , typename PARTICLETYPE >
std::size_t	attachSerializedParticle (ParticleSystem< T, PARTICLETYPE > &particleSystem, std::uint8_t *rawBuffer)
template<typename T , typename PARTICLETYPE >
void	prepareRelocation (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, Particle< T, PARTICLETYPE > &particle, int globiC, int globiCdest, std::multimap< int, std::unique_ptr< std::uint8_t[]> > &rankDataMapRelocationInter, std::vector< std::unique_ptr< std::uint8_t[]> > &dataListRelocationIntra, std::size_t serialSize)
template<typename T , typename PARTICLETYPE >
void	checkSurfaceRelocation (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, const T physDeltaX, Particle< T, PARTICLETYPE > &particle, int globiC, std::multimap< int, std::unique_ptr< std::uint8_t[]> > &rankDataMapRelocationInter, std::vector< std::unique_ptr< std::uint8_t[]> > &dataListRelocationIntra, std::size_t serialSize, const Vector< bool, PARTICLETYPE::d > &periodicity)
template<typename T , typename PARTICLETYPE >
void	checkRelocation (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, const T physDeltaX, std::multimap< int, std::unique_ptr< std::uint8_t[]> > &rankDataMapRelocationInter, std::vector< std::unique_ptr< std::uint8_t[]> > &dataListRelocationIntra, std::size_t serialSize, const Vector< bool, PARTICLETYPE::d > &periodicity)
void	fillSendBuffer (std::multimap< int, std::unique_ptr< std::uint8_t[]> > &rankDataMap, std::map< int, std::vector< std::uint8_t > > &rankDataMapSorted, std::size_t serialSize)
void	sendMappedData (std::map< int, std::vector< std::uint8_t > > &rankDataMapSorted, const std::unordered_set< int > &availableRanks, const std::size_t serialSize, MPI_Comm Comm, singleton::MpiNonBlockingHelper &mpiNbHelper)
template<typename T , typename PARTICLETYPE >
void	checkInvalidationOnReceival (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, std::vector< std::vector< std::size_t > > &receivedLocalIDs, const Vector< bool, PARTICLETYPE::d > &periodicity)
template<typename T , typename PARTICLETYPE >
void	assignParticleToIC (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, std::vector< std::vector< std::size_t > > &receivedLocalIDs, int rankOrig, std::uint8_t *rawBuffer)
template<typename F >
void	receiveAndExecuteForData (const std::unordered_set< int > &availableRanks, std::size_t serialSize, MPI_Comm Comm, singleton::MpiNonBlockingHelper &mpiNbHelper, F f)
template<typename T , typename PARTICLETYPE >
void	receiveParticles (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, std::vector< std::unique_ptr< std::uint8_t[]> > &dataListRelocationIntra, std::size_t serialSize, MPI_Comm particleDistributionComm, singleton::MpiNonBlockingHelper &mpiNbHelper, const Vector< bool, PARTICLETYPE::d > &periodicity)
template<typename T , typename PARTICLETYPE >
void	updateParticleCuboidDistribution (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, const T physDeltaX, MPI_Comm particleDistributionComm, const Vector< bool, PARTICLETYPE::d > &periodicity)
template<typename DATA >
void	collectDataAndAppendVector (std::vector< DATA > &dataVector, std::unordered_set< int > &availableRanks, singleton::MpiNonBlockingHelper &mpiNbHelper, MPI_Comm commGroup=MPI_COMM_WORLD)
template<typename T , typename PARTICLETYPE , unsigned forceDataSize>
void	communicateSurfaceForce (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, std::map< std::size_t, Vector< T, forceDataSize > > &globalIdDataMap, MPI_Comm surfaceForceComm)
template<typename T , typename PARTICLETYPE , unsigned forceDataSize>
void	assignSurfaceForce (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, std::map< std::size_t, Vector< T, forceDataSize > > &globalIdDataMap)
template<typename T , unsigned D, bool verbose = false>
std::vector< int >	getNeighborCuboids (CuboidGeometry< T, D > &cuboidGeometry, unsigned offset, int globiC)
	Get neighbour cuboids.
template<typename T , unsigned D, bool verbose = false>
std::unordered_set< int >	getNeighbourRanksFromCuboidNeighbourhood (SuperStructure< T, D > &superStructure, int rank, std::map< int, std::vector< int > > neighbourhood)
	Get neighbour ranks.
template<typename T , unsigned D, bool verbose = false>
std::unordered_set< int >	getNeighbourRanksFromCuboidNeighbourhood (SuperStructure< T, D > &superStructure, int rank, const std::vector< std::unordered_set< int > > &neighbourhood)
	Get neighbour ranks.
template<typename T , unsigned D, bool verbose = false>
std::unordered_set< int >	getNeighbourRanksFromCuboidNeighbourhood (SuperStructure< T, D > &superStructure, const std::vector< std::unordered_set< int > > &neighbourhood)
	Get neighbour ranks.
template<typename T , unsigned D, bool verbose = false>
std::unordered_set< int >	getNeighbourRanks (SuperStructure< T, D > &superStructure, unsigned offset, int rank)
	Get neighbour ranks.
template<typename T , unsigned D, bool verbose = false>
std::unordered_set< int >	getNeighbourRanks (SuperStructure< T, D > &superStructure, unsigned offset)
	Get neighbour ranks.
template<typename T , typename PARTICLETYPE >
std::size_t	serializeIcDest (int globiCdest, std::uint8_t *bufferiCandParticle)
template<typename T , typename PARTICLETYPE >
void	serializeIcDestAndParticle (int globiCdest, Particle< T, PARTICLETYPE > &particle, std::uint8_t *bufferiCandParticle)
template<typename T , typename PARTICLETYPE >
std::size_t	deserializeIcDest (int &globiCdest, std::uint8_t *bufferRaw)
template<typename T , typename PARTICLETYPE >
void	deserializeIcDestAndParticle (int &globiCdest, Particle< T, PARTICLETYPE > &particle, std::uint8_t *bufferiCandParticle)
template<typename T , typename PARTICLETYPE >
void	checkCuboidSizes (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem)
template<typename T >
T	movePositionToStart (const T position, const T max, const T min)
template<typename T >
T	movePositionToEnd (const T position, const T max, const T min)
template<typename T >
T	applyPeriodicityToPosition (const bool isPeriodic, T position, const T max, const T min)
template<typename T , unsigned D>
Vector< T, D >	getCuboidMin (const CuboidGeometry< T, D > &cuboidGeometry)
	Returns minimal coordinate of domain for periodic particle boundaries.
template<typename T , unsigned D>
Vector< T, D >	getCuboidMax (const CuboidGeometry< T, D > &cuboidGeometry, const PhysR< T, D > &min)
	Returns maximal coordinate of domain for periodic particle boundaries.
template<typename T , unsigned D>
Vector< T, D >	applyPeriodicityToPosition (const CuboidGeometry< T, D > &cuboidGeometry, const Vector< bool, D > &periodicity, PhysR< T, D > position)
	Updates a position if out of bounds and periodic setup is used.
template<typename T , unsigned D>
bool	getCuboid (const CuboidGeometry< T, D > &cuboidGeometry, const Vector< bool, D > &periodicity, const PhysR< T, D > &position, int &iC)
	Function returns true if cuboid was found and gives iC.
template<typename T , typename PARTICLETYPE , typename F >
void	forSystemsInSuperParticleSystem (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, F f)
template<typename T , typename PARTICLETYPE , typename PCONDITION = conditions::valid_particles, typename F >
void	forParticlesInSuperParticleSystem (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, F f)
template<typename T , typename PARTICLETYPE , typename F >
void	doForParticle (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, ParallelParticleLocator &locator, F f)
template<typename T , typename PARTICLETYPE , typename PCONDITION = conditions::valid_particle_centres, bool sync = true>
bool	searchParticleGlobally (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, std::size_t globalIDrequested, std::size_t &localParticleID, int &globiC)
	Search particle in SuperParticleSystem by globalParticleID and return globiC if found.
template<typename T , unsigned D, typename F = std::function<void(int)>, bool domainWarning = false, bool checkDiscretePoints = false>
std::unordered_set< int >	getSurfaceTouchingICs (CuboidGeometry< T, D > &cuboidGeometry, Vector< T, D > position, T circumRadius, const Vector< bool, D > &periodicity, int globiC, F f=[](int){})
	Get a set of surface touching iCs (that are not globiC) Allows to run an optional function per unique globiC.
template<typename T , typename PARTICLETYPE , typename F = std::function<void(int)>>
std::unordered_set< int >	getSurfaceTouchingICs (SuperParticleSystem< T, PARTICLETYPE > &sParticleSystem, Vector< T, PARTICLETYPE::d > position, T circumRadius, const Vector< bool, PARTICLETYPE::d > &periodicity, int globiC, F f=[](int){})
template<typename T , unsigned D, typename F = std::function<void(int)>, bool domainWarning = false>
std::vector< int >	getVectorOfSurfaceTouchingICs (CuboidGeometry< T, D > &cuboidGeometry, Vector< T, D > position, T circumRadius, const Vector< bool, D > &periodicity, int globiC, F f=[](int){})

Function Documentation

appendSerializedParticle()

template<typename T , typename PARTICLETYPE >

std::size_t olb::particles::communication::appendSerializedParticle	(	ParticleSystem< T, PARTICLETYPE > &	particleSystem,
		std::uint8_t	rawBuffer[] )

Definition at line 63 of file relocation.h.

{
  //Extend particle container
  auto& particleContainer = particleSystem.get();
  particleContainer.push_back();
  //Retrieve new particle
  std::size_t idxParticleNew = particleSystem.size()-1;
  auto particleNew = particleSystem.get(idxParticleNew);
  //Deserialise particle
  int globiCdest;
  deserializeIcDestAndParticle(globiCdest,particleNew,rawBuffer);
  //Update surface pointer
  if constexpr( access::providesSurface<PARTICLETYPE>() ){
    updateSurfacePtr( particleSystem, idxParticleNew );
  }
  return particleNew.getId();
}

References deserializeIcDestAndParticle(), olb::particles::ParticleSystem< T, PARTICLETYPE >::get(), olb::particles::ParticleSystem< T, PARTICLETYPE >::size(), and updateSurfacePtr().

Here is the call graph for this function:

Here is the caller graph for this function:

applyPeriodicityToPosition() [1/2]

template<typename T >

T olb::particles::communication::applyPeriodicityToPosition	(	const bool	isPeriodic,
		T	position,
		const T	max,
		const T	min )

Definition at line 266 of file utilities.h.

{
  if(isPeriodic) {
    if(position < min) {
      position = movePositionToEnd(position, max, min);
    }
    else if(position > max) {
      position = movePositionToStart(position, max, min);
    }
  }
  return position;
}

References olb::particles::isPeriodic(), movePositionToEnd(), and movePositionToStart().

Here is the call graph for this function:

Here is the caller graph for this function:

applyPeriodicityToPosition() [2/2]

template<typename T , unsigned D>

Vector< T, D > olb::particles::communication::applyPeriodicityToPosition	(	const CuboidGeometry< T, D > &	cuboidGeometry,
		const Vector< bool, D > &	periodicity,
		PhysR< T, D >	position )

Updates a position if out of bounds and periodic setup is used.

Definition at line 302 of file utilities.h.

{
  const PhysR<T,D> min = getCuboidMin<T,D>(cuboidGeometry);
  const PhysR<T,D> max = getCuboidMax<T,D>(cuboidGeometry, min);
 
  for(unsigned iD=0; iD<D; ++iD) {
    position[iD] = applyPeriodicityToPosition(periodicity[iD],
        position[iD], max[iD], min[iD]);
  }
 
  return position;
}

References applyPeriodicityToPosition().

Here is the call graph for this function:

assignParticleToIC()

template<typename T , typename PARTICLETYPE >

void olb::particles::communication::assignParticleToIC	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		std::vector< std::vector< std::size_t > > &	receivedLocalIDs,
		int	rankOrig,
		std::uint8_t *	rawBuffer )

Definition at line 517 of file relocation.h.

{
  using namespace descriptors;
 
  //Deserialize globiCdest
  int globiCdest;
  deserializeIcDest<T,PARTICLETYPE>( globiCdest,rawBuffer );
 
  //Loop over ParticleSystems in SuperParticleSystem
  forSystemsInSuperParticleSystem( sParticleSystem,
    [&](ParticleSystem<T,PARTICLETYPE>& particleSystem, int iC,
      int globiC){
 
    if (globiC == globiCdest){
 
      //Attach serialized particle to new particleSystem
      [[maybe_unused]] std::size_t particleID =
        attachSerializedParticle(
        particleSystem, rawBuffer);
 
      //Add local particle iD to list (resolved only)
      if constexpr( access::providesSurface<PARTICLETYPE>() ){
        receivedLocalIDs[iC].push_back(particleID);
      }
 
#ifdef VERBOSE_COMMUNICATION
      int rank = singleton::mpi().getRank();
      auto particle = particleSystem.get(particleID);
      std::size_t globalID = particle.template getField<PARALLELIZATION,ID>();
      std::cout << std::endl << "[Particle] RECEIVING";
      if (rankOrig != rank){ std::cout << "(inter)"; } else { std::cout << "(intra)"; }
      std::cout << " Particle"
                << "(globalID=" << globalID << ") "
                << "from iC" << "X" << "(rank=" << rankOrig << ") to iC"
                << globiCdest << "(rank=" << rank << ")" << std::endl;
      particle.template print<true>();
#endif
 
    } //if (globiC == globiCdest)
  }); //forSystemsInSuperParticleSystem
}

References attachSerializedParticle(), forSystemsInSuperParticleSystem(), olb::particles::ParticleSystem< T, PARTICLETYPE >::get(), olb::singleton::MpiManager::getRank(), and olb::singleton::mpi().

Here is the call graph for this function:

Here is the caller graph for this function:

assignSurfaceForce()

template<typename T , typename PARTICLETYPE , unsigned forceDataSize>

void olb::particles::communication::assignSurfaceForce	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		std::map< std::size_t, Vector< T, forceDataSize > > &	globalIdDataMap )

Definition at line 185 of file surfaceForce.h.

{
  using namespace descriptors;
  //Retrieve dimension
  constexpr unsigned D = PARTICLETYPE::d;
  constexpr unsigned Drot = utilities::dimensions::convert<D>::rotation;
  //Loop over particle systems
  communication::forSystemsInSuperParticleSystem( sParticleSystem,
    [&](ParticleSystem<T,PARTICLETYPE>& particleSystem, int iC,
      int globiC){
    //Loop over globalIdDataPairs
    for ( auto globalIdDataPair : globalIdDataMap ){
      //Loop over particles
      // -! Lambda-expression unavailable due to break command
      for (std::size_t iP=0; iP<particleSystem.size(); ++iP){
        auto particle = particleSystem.get(iP);
        //Only consider valid particle centres
        if (conditions::valid_particle_centres::value(particle, globiC)){
          //Retrieve globalIDs
          std::size_t globalID = globalIdDataPair.first;
          std::size_t globalIDiP = particle.template getField<PARALLELIZATION,ID>();
          //Check whether globalID matches
          if (globalIDiP==globalID){
            //Retrieve force data
            auto forceData = globalIdDataPair.second;
            auto force = particle.template getField<FORCING,FORCE>();
            auto torque = particle.template getField<FORCING,TORQUE>();
            //Add force to particle
            for (unsigned iDim=0; iDim<D; ++iDim) {
              force[iDim] += forceData[iDim];
            }
            //Add torque to particle
            if constexpr (utilities::dimensions::convert<PARTICLETYPE::d>::rotation == 1 ) {
              torque += forceData[D];
            }
            else {
              for (unsigned iDim=0; iDim<Drot; ++iDim) {
                torque[iDim] += forceData[iDim+D];
              }
            }
 
#ifdef VERBOSE_COMMUNICATION
    int rank = singleton::mpi().getRank();
    std::cout << "Assigned (on rank=" << rank
              << "): force=" << force
              << std::endl
              << "                  "
              << ", torque=" << torque
              << ", globalID=" << globalID
              << std::endl;
#endif
            //Update particle
            particle.template setField<FORCING,FORCE>( force );
            particle.template setField<FORCING,TORQUE>( torque );
            //Break, since particle already found
            break;
          }
        }
      }
    }
  });
}

References forSystemsInSuperParticleSystem(), olb::particles::ParticleSystem< T, PARTICLETYPE >::get(), olb::singleton::MpiManager::getRank(), olb::singleton::mpi(), olb::particles::ParticleSystem< T, PARTICLETYPE >::size(), and olb::particles::conditions::valid_particle_centres::value().

Here is the call graph for this function:

Here is the caller graph for this function:

attachSerializedParticle()

template<typename T , typename PARTICLETYPE >

std::size_t olb::particles::communication::attachSerializedParticle	(	ParticleSystem< T, PARTICLETYPE > &	particleSystem,
		std::uint8_t *	rawBuffer )

Definition at line 128 of file relocation.h.

{
  using namespace descriptors;
 
  std::size_t particleID;
  //Check whether resolved (or subgrid)
  //- subgrid has to be appended always, as valid particle data will not
  //  be present on new core due to previous direct invalidation
  //- TODO: Check consistency, as for now, invalidated slots are considered
  //  for resolved, but not for sub grid particles
  if constexpr( access::providesSurface<PARTICLETYPE>() ){
    //Attach serialized particle (either at end or at existing index)
    if (particleSystem.size()==0){
      particleID = appendSerializedParticle( particleSystem, rawBuffer );
    } else {
      //Create temporary particle system to retrieve global id
      ParticleSystem<T,PARTICLETYPE> particleSystemTmp(1);
      auto particleTmp = particleSystemTmp.get(0);
      int globiCdummy;
      deserializeIcDestAndParticle(globiCdummy,particleTmp,rawBuffer);
      std::size_t globalID = particleTmp.template getField<PARALLELIZATION,ID>();
      //Check whether particle already existing
      // -! Lambda-expression unavailable due to break command
      bool found=false;
      //Loop over particles
      //- including invalid slots, as they are revalidated
      //  anyways when inserting particle data)
      for (std::size_t iP=0; iP<particleSystem.size(); ++iP){
        auto particle = particleSystem.get(iP);
        auto globalIDiP = particle.template getField<PARALLELIZATION,ID>();
        if (globalIDiP==globalID){
          particleID = insertSerializedParticle( particleSystem, iP, rawBuffer );
          found=true;
          break;
        }
      }
      if (!found){
        particleID = appendSerializedParticle( particleSystem, rawBuffer );
      }
    }
  } else { //if constexpr ( access::providesSurface<PARTICLETYPE>() )
    //Attach serialized particle
    particleID = appendSerializedParticle( particleSystem, rawBuffer );
  }
  return particleID;
}

References appendSerializedParticle(), deserializeIcDestAndParticle(), olb::particles::ParticleSystem< T, PARTICLETYPE >::get(), insertSerializedParticle(), and olb::particles::ParticleSystem< T, PARTICLETYPE >::size().

Here is the call graph for this function:

Here is the caller graph for this function:

checkCuboidSizes()

template<typename T , typename PARTICLETYPE >

void olb::particles::communication::checkCuboidSizes

(

SuperParticleSystem< T, PARTICLETYPE > &

sParticleSystem

)

Definition at line 213 of file utilities.h.

{
  constexpr unsigned D = PARTICLETYPE::d;
  auto& cuboidGeometry = sParticleSystem.getSuperStructure().getCuboidGeometry();
 
  const T physDeltaX = cuboidGeometry.getMotherCuboid().getDeltaR();
 
  T maxCircumRadius{0};
  communication::forParticlesInSuperParticleSystem(
      sParticleSystem,
      [&](Particle<T, PARTICLETYPE>&       particle,
          ParticleSystem<T, PARTICLETYPE>& particleSystem, int globiC) {
          maxCircumRadius = util::max(maxCircumRadius,
              particles::contact::evalCircumRadius(particle, physDeltaX));
      });
#ifdef PARALLEL_MODE_MPI
    singleton::mpi().reduceAndBcast(maxCircumRadius, MPI_MAX,
                                    singleton::mpi().bossId());
#endif
 
  int minCuboidExtent{std::numeric_limits<int>::max()};
  for (int iC=0; iC<cuboidGeometry.getNc(); ++iC){
    for(unsigned iD=0; iD<D; ++iD){
      minCuboidExtent = util::min(minCuboidExtent,
          cuboidGeometry.get(iC).getExtent()[iD]);
    }
  }
#ifdef PARALLEL_MODE_MPI
    singleton::mpi().reduceAndBcast(minCuboidExtent, MPI_MIN,
                                    singleton::mpi().bossId());
#endif
 
  // It's enough that one rank checks for an error
  if (singleton::mpi().getRank() == 0) {
    OLB_ASSERT(minCuboidExtent*physDeltaX > maxCircumRadius,
        "At least one cuboid too small so that the currently implemented "
        "particle parallelization strategy does not work.");
  }
}

References olb::particles::contact::evalCircumRadius(), forParticlesInSuperParticleSystem(), olb::SuperStructure< T, D >::getCuboidGeometry(), olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getSuperStructure(), olb::util::max(), olb::util::min(), olb::singleton::mpi(), OLB_ASSERT, and olb::singleton::MpiManager::reduceAndBcast().

Here is the call graph for this function:

checkInvalidationOnReceival()

template<typename T , typename PARTICLETYPE >

void olb::particles::communication::checkInvalidationOnReceival	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		std::vector< std::vector< std::size_t > > &	receivedLocalIDs,
		const Vector< bool, PARTICLETYPE::d > &	periodicity )

Definition at line 442 of file relocation.h.

{
  using namespace descriptors;
 
  //Retrieve load balancer
  auto& superStructure = sParticleSystem.getSuperStructure();
  auto& loadBalancer = superStructure.getLoadBalancer();
 
  //Loop over valid surfaces and check, whether any data was received
  forParticlesInSuperParticleSystem<T,PARTICLETYPE,conditions::valid_particle_surfaces>(
    sParticleSystem,
    [&](Particle<T,PARTICLETYPE>& particle,
      ParticleSystem<T,PARTICLETYPE>& particleSystem, int globiC){
 
    //Retrieve local iC
    int lociC = loadBalancer.loc(globiC);
 
    //Retrieve local particle iD
    std::size_t locId = particle.getId();
 
#ifdef VERBOSE_COMMUNICATION
    std::cout << "ReceivedLocalIDs:";
    for (int i=0; i<receivedLocalIDs[lociC].size(); ++i){
      auto idPrint = receivedLocalIDs[lociC][i];
      std::cout << idPrint;
      if (i<receivedLocalIDs.size()){ std::cout << ","; }
    }
    std::cout << std::endl;
#endif
 
    //Check wether particle belongs to those having received data (here, if not found)
    if( std::find(receivedLocalIDs[lociC].begin(),receivedLocalIDs[lociC].end(),
      locId) == receivedLocalIDs[lociC].end()){
      //Invalidate particle
      particle.template setField<GENERAL,INVALID>(true);
 
#ifdef VERBOSE_COMMUNICATION
      int rank = singleton::mpi().getRank();
      std::size_t globalID = particle.template getField<PARALLELIZATION,ID>();
      std::cout << std::endl << "[Particle] "
                << "INVALIDATING Particle"
                << "(globalID=" << globalID << ") "
                << "on iC" << globiC << "(rank=" << rank << ")" << std::endl;
    particle.template print<true>();
#endif
 
    } // if( std::find() )
    /* else { */
    /*   // Check if the current block still touches the surface, if not then invalidate the particle */
    /*   auto& cuboidGeometry = superStructure.getCuboidGeometry(); */
    /*   const T physDeltaX = cuboidGeometry.getMotherCuboid().getDeltaR(); */
    /*   const T circumRadius = contact::evalCircumRadius(particle, physDeltaX); */
    /*   const int globiCcenter = particle.template getField<PARALLELIZATION, IC>(); */
    /*   bool touchesSurface = false; */
    /*   communication::getSurfaceTouchingICs( */
    /*       sParticleSystem, access::getPosition(particle), circumRadius, periodicity, globiCcenter, */
    /*       [&](int surfaceTouchingGlobiC){ */
    /*         if(surfaceTouchingGlobiC == globiC) { */
    /*           touchesSurface = true; */
    /*         } */
    /*       }); */
    /*   if(!touchesSurface) { */
    /*     particle.template setField<GENERAL,INVALID>(true); */
    /*   } */
    /* } */
  }); //forParticlesInSuperParticleSystem<T,PARTICLETYPE,valid_particle_surfaces>
}

References olb::particles::Particle< T, PARTICLETYPE >::getId(), olb::SuperStructure< T, D >::getLoadBalancer(), olb::singleton::MpiManager::getRank(), olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getSuperStructure(), and olb::singleton::mpi().

Here is the call graph for this function:

Here is the caller graph for this function:

checkRelocation()

template<typename T , typename PARTICLETYPE >

void olb::particles::communication::checkRelocation	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		const T	physDeltaX,
		std::multimap< int, std::unique_ptr< std::uint8_t[]> > &	rankDataMapRelocationInter,
		std::vector< std::unique_ptr< std::uint8_t[]> > &	dataListRelocationIntra,
		std::size_t	serialSize,
		const Vector< bool, PARTICLETYPE::d > &	periodicity )

Definition at line 252 of file relocation.h.

{
  using namespace descriptors;
  using namespace particles::access;
  constexpr unsigned D = PARTICLETYPE::d;
 
  //Retrieve load balancer and cuboid geometry
  auto& superStructure = sParticleSystem.getSuperStructure();
  auto& cuboidGeometry = superStructure.getCuboidGeometry();
 
  //Loop over valid particles
  forParticlesInSuperParticleSystem( sParticleSystem,
    [&](Particle<T,PARTICLETYPE>& particle,
    ParticleSystem<T,PARTICLETYPE>& particleSystem, int globiC){
 
    //Retrieve centre iC in previous timestep.
    //- used to detect iC change
    //- unly updated for resolved particles
    int globiCcentreLast= -1;
    if constexpr( providesSurface<PARTICLETYPE>() ){
      globiCcentreLast = particle.template getField<PARALLELIZATION,IC>();
    }
    // Retrieve and update particle position
    // if a periodic setup is used and the particle is out of bounds
    const PhysR<T,D> position = applyPeriodicityToPosition(cuboidGeometry,
        periodicity, particle.template getField<GENERAL,POSITION>());
    //Find particle system by position
    int globiCcentre = -1;
    // getC doesn't treat periodicity! We can only use it because we updated it before
    const bool particleCentreInDomain = cuboidGeometry.getC(position, globiCcentre);
    /*
    // Retrieve particle position
    PhysR<T,D> position = particle.template getField<GENERAL,POSITION>();
    //Find particle system by position
    int globiCcentre = -1;
    const bool particleCentreInDomain = getCuboid(
        cuboidGeometry, periodicity, position, globiCcentre);
    */
    // Replace the position with the updated position
    // if no periodicity is used or the particle isn't out of bounds nothing happens
    // (must be here, because it can be that the particle remains on the same block after crossing the boundary)
    // TODO: Add constexpr check if the setup is periodic or not before continuously overwriting the position here
    particle.template setField<GENERAL, POSITION>(position);
    /*if(isPeriodic(periodicity)) {
      const PhysR<T,D> min = getCuboidMin<T,D>(cuboidGeometry);
      const PhysR<T,D> max = getCuboidMax<T,D>(cuboidGeometry, min);
 
      for(unsigned iD=0; iD<D; ++iD) {
        position[iD] = applyPeriodicityToPosition(
            periodicity[iD], position[iD], max[iD], min[iD]);
      }
      particle.template setField<GENERAL, POSITION>(position);
    }*/
 
    //Update iC, if resolved
    if constexpr( providesSurface<PARTICLETYPE>() ){
      particle.template setField<PARALLELIZATION,IC>(globiCcentre);
    }
 
    //Check whether particle has lost the global domain
    if (particleCentreInDomain){
      //Check whether particle centre resides in current iC
      //If yes:
      //- subgrid: no data to be sent
      //- resolved: check surface
      if (globiCcentre == globiC){
        //Check, whether surface parts have to be sent (resolved only)
        if constexpr( providesSurface<PARTICLETYPE>() ){
          checkSurfaceRelocation( sParticleSystem, physDeltaX, particle, globiC,
            rankDataMapRelocationInter, dataListRelocationIntra, serialSize, periodicity );
        }
      //If particle centre does not reside in iC
      } else { // if (globiCcentre == globiC)
        //Differentiate between resolved and subgrid treatment
        if constexpr( providesSurface<PARTICLETYPE>() ){
 
          //Detect center switch
          //- to send data one last time
          //- necessary, since new responsible iC can only send at next timestep
          if(globiCcentre!=globiCcentreLast){
#ifdef VERBOSE_COMMUNICATION
            int rank = singleton::mpi().getRank();
            std::cout << std::endl << "[Particle] "
                      << "CENTRE SWITCH("
                      << globiCcentreLast << "->" << globiCcentre << ")"
                      << " at sending from iC=" << globiC
                      << "(rank=" << rank << ")" << std::endl;
#endif
            //Prepare inter relocation or intra relocation
            //-here: to new responsible ic
            prepareRelocation( sParticleSystem,
              particle, globiC, globiCcentre,
              rankDataMapRelocationInter, dataListRelocationIntra, serialSize);
 
            if constexpr( providesSurface<PARTICLETYPE>() ){
              //Sending to self, to avoid invalidation on receiving
              //- necessary, as responsibility changed
              prepareRelocation( sParticleSystem,
                particle, globiCcentre, globiC,
                rankDataMapRelocationInter, dataListRelocationIntra, serialSize);
 
              //Check, whether surface parts have to be sent (resolved only)
              checkSurfaceRelocation( sParticleSystem, physDeltaX, particle, globiC,
                rankDataMapRelocationInter, dataListRelocationIntra, serialSize, periodicity );
            }
 
          } //if(globiCcentre!=globiCcentreLast)
        } else { //if constexpr( providesSurface<PARTICLETYPE>() )
          //Prepare inter relocation or intra relocation
          prepareRelocation( sParticleSystem,
            particle, globiC, globiCcentre,
            rankDataMapRelocationInter, dataListRelocationIntra, serialSize);
          //Immediate invalidation (subgrid only)
          particle.template setField<GENERAL,INVALID>(true);
        }
      } // else (globiCcentre == globiC)
    } else { //particleCentreInDomain
      //Invalidate particle
      particle.template setField<GENERAL,INVALID>(true);
      //Output warning
      int rank = singleton::mpi().getRank();
      auto globalID = particle.template getField<PARALLELIZATION,ID>();
      std::cout << ">>>> WARNING: Particle " << globalID
           << " lost domain at iC="<< globiC
           << " (rank=" << rank << ") <<<<" << std::endl;
    }
  }); // forParticlesInSuperParticleSystem
}

References applyPeriodicityToPosition(), checkSurfaceRelocation(), forParticlesInSuperParticleSystem(), olb::SuperStructure< T, D >::getCuboidGeometry(), olb::singleton::MpiManager::getRank(), olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getSuperStructure(), olb::singleton::mpi(), and prepareRelocation().

Here is the call graph for this function:

Here is the caller graph for this function:

checkSurfaceRelocation()

template<typename T , typename PARTICLETYPE >

void olb::particles::communication::checkSurfaceRelocation	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		const T	physDeltaX,
		Particle< T, PARTICLETYPE > &	particle,
		int	globiC,
		std::multimap< int, std::unique_ptr< std::uint8_t[]> > &	rankDataMapRelocationInter,
		std::vector< std::unique_ptr< std::uint8_t[]> > &	dataListRelocationIntra,
		std::size_t	serialSize,
		const Vector< bool, PARTICLETYPE::d > &	periodicity )

Definition at line 225 of file relocation.h.

{
  //Retrieve cuboid geometry
  /* auto& superStructure = sParticleSystem.getSuperStructure(); */
  /* auto& cuboidGeometry = superStructure.getCuboidGeometry(); */
  //Retrieve position and circumRadius
  auto position = access::getPosition( particle );
  const T circumRadius = contact::evalCircumRadius(particle, physDeltaX);
  const int globiCcentre = particle.template getField<descriptors::PARALLELIZATION,descriptors::IC>();
  //Get unique iCs on surface and prepare relocation
  getSurfaceTouchingICs(sParticleSystem, position, circumRadius, periodicity, globiCcentre,
      [&](const int globiCdest){
        prepareRelocation( sParticleSystem, particle, globiC, globiCdest,
          rankDataMapRelocationInter, dataListRelocationIntra, serialSize);
      });
}

References olb::particles::contact::evalCircumRadius(), olb::particles::access::getPosition(), getSurfaceTouchingICs(), and prepareRelocation().

Here is the call graph for this function:

Here is the caller graph for this function:

collectDataAndAppendVector()

template<typename DATA >

void olb::particles::communication::collectDataAndAppendVector	(	std::vector< DATA > &	dataVector,
		std::unordered_set< int > &	availableRanks,
		singleton::MpiNonBlockingHelper &	mpiNbHelper,
		MPI_Comm	commGroup = MPI_COMM_WORLD )

Definition at line 734 of file relocation.h.

{
  DATA data;
  auto communicatable = ConcreteCommunicatable(data);
  const int serialSize = communicatable.size();
  //Receive data
  receiveAndExecuteForData( availableRanks,
    serialSize, commGroup, mpiNbHelper,
    [&](int rankOrig, std::uint8_t* rawBuffer){
      //Deserialize buffer
      communicatable.deserialize( rawBuffer );
      //Append data to std::vector
      dataVector.push_back(data);
  });
}

References receiveAndExecuteForData().

Here is the call graph for this function:

communicateEquationsOfMotionResults()

template<typename T , typename PARTICLETYPE , typename PARTICLECONTACTTYPE , typename WALLCONTACTTYPE >

void olb::particles::communication::communicateEquationsOfMotionResults	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		contact::ContactContainer< T, PARTICLECONTACTTYPE, WALLCONTACTTYPE > &	contactContainer,
		MPI_Comm	equationsOfMotionComm )

Definition at line 304 of file equationsOfMotionResults.h.

{
  // TODO: Test if it is faster to only send & receive between processors that must exchange data (that way we don't need to synchronize all processors)
 
#ifdef PARALLEL_MODE_MPI
  // Create map for data
  std::multimap<int, std::unique_ptr<std::uint8_t[]>> dataMap;
 
  std::size_t serialSize =
      evalEquationsOfMotionResults(sParticleSystem, contactContainer, dataMap);
 
  //Create non blocking mpi helper
  singleton::MpiNonBlockingHelper mpiNbHelper;
 
  std::map<int, std::vector<std::uint8_t>> rankDataMapSorted;
  communication::fillSendBuffer(dataMap, rankDataMapSorted, serialSize);
 
  // Send mapped data
  communication::sendMappedData(rankDataMapSorted,
                                sParticleSystem.getNeighbourRanks(), serialSize,
                                equationsOfMotionComm, mpiNbHelper);
 
  receiveEquationsOfMotionResults(sParticleSystem, equationsOfMotionComm,
                                  mpiNbHelper);
#endif
}

References evalEquationsOfMotionResults(), fillSendBuffer(), olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getNeighbourRanks(), receiveEquationsOfMotionResults(), and sendMappedData().

Here is the call graph for this function:

communicateEquationsOfMotionResultsIntra()

template<typename T , typename PARTICLETYPE >

void olb::particles::communication::communicateEquationsOfMotionResultsIntra	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		const std::size_t &	pID,
		const Vector< T, PARTICLETYPE::d > &	position,
		const Vector< T, PARTICLETYPE::d > &	velocity,
		const Vector< T, utilities::dimensions::convert< PARTICLETYPE::d >::rotation > &	angle,
		const Vector< T, utilities::dimensions::convert< PARTICLETYPE::d >::rotation > &	angVelocity )

Definition at line 71 of file equationsOfMotionResults.h.

{
  using namespace descriptors;
 
  particles::communication::forParticlesInSuperParticleSystem(
      sParticleSystem,
      [&](Particle<T, PARTICLETYPE>&       particle,
          ParticleSystem<T, PARTICLETYPE>& particleSystem, int globiC) {
        const std::size_t currGlobalParticleID =
            particle.template getField<PARALLELIZATION, ID>();
 
        if (currGlobalParticleID == pID) {
          particle.template setField<GENERAL, POSITION>(position);
          particle.template setField<MOBILITY, VELOCITY>(velocity);
          particle.template setField<SURFACE, ANGLE>(
              utilities::dimensions::convert<
                  PARTICLETYPE::d>::serialize_rotation(angle));
          particle.template setField<MOBILITY, ANG_VELOCITY>(
              utilities::dimensions::convert<
                  PARTICLETYPE::d>::serialize_rotation(angVelocity));
        }
      });
}

References forParticlesInSuperParticleSystem().

Here is the call graph for this function:

Here is the caller graph for this function:

communicateSurfaceForce()

template<typename T , typename PARTICLETYPE , unsigned forceDataSize>

void olb::particles::communication::communicateSurfaceForce	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		std::map< std::size_t, Vector< T, forceDataSize > > &	globalIdDataMap,
		MPI_Comm	surfaceForceComm )

Definition at line 43 of file surfaceForce.h.

{
  //TODO: add intra and inter core differentiation
  using namespace descriptors;
  //Retrieve dimensions
  constexpr unsigned D = PARTICLETYPE::d;
  constexpr unsigned Drot = utilities::dimensions::convert<D>::rotation;
  //Retrieve loadBalancer
  auto& superStructure = sParticleSystem.getSuperStructure();
  auto& loadBalancer = superStructure.getLoadBalancer();
 
  //Create FieldArrayD for globalID, force and torque
  FieldArrayD<T,PARTICLETYPE,Platform::CPU_SISD,descriptors::FORCE> fieldForce(1);
  using FORCING_EVAL = typename PARTICLETYPE
                     ::template derivedField<descriptors::FORCING>;
  using TORQUE_EVAL = typename FORCING_EVAL
                     ::template derivedField<descriptors::TORQUE>;
  FieldArrayD<T,PARTICLETYPE,Platform::CPU_SISD,TORQUE_EVAL> fieldTorque(1);
  FieldArrayD<T,PARTICLETYPE,Platform::CPU_SISD,descriptors::ID> fieldID(1);
 
  //Create communicatables
  auto communicatableForce = ConcreteCommunicatable(fieldForce);
  auto communicatableTorque = ConcreteCommunicatable(fieldTorque);
  auto communicatableID = ConcreteCommunicatable(fieldID);
  //Retrieve serial size
  const std::vector<unsigned int> indices{0};
  auto serialSize = communicatableForce.size(indices)
                  + communicatableTorque.size(indices)
                  + communicatableID.size(indices);
  //Create rankDataMap
  std::multimap<int,std::unique_ptr<std::uint8_t[]>> rankDataMap;
 
  //Loop over valid particles in particle system
  communication::forParticlesInSuperParticleSystem<T,PARTICLETYPE,
    conditions::valid_particle_surfaces>( sParticleSystem,
    [&](Particle<T,PARTICLETYPE>& particle,
    ParticleSystem<T,PARTICLETYPE>& particleSystem, int globiC){
    //Retrieve globiC of particle center
    int globiCcentre = particle.template getField<PARALLELIZATION,IC>();
    int rankDest = loadBalancer.rank(globiCcentre);
    //Retrieve force and torque and set FieldArray
    auto force = particle.template getField<FORCING,FORCE>();
    auto torque = particle.template getField<FORCING,TORQUE>();
    auto globalID = particle.template getField<PARALLELIZATION,ID>();
#ifdef VERBOSE_COMMUNICATION
    std::cout << "EVALUATING: force=" << force << ", torque=" << torque << ", globalID=" << globalID << std::endl;
#endif
    fieldForce.setField(0, force);
    fieldTorque.setField(0, torque);
    fieldID.setField(0, globalID);
    //Create buffer
    std::unique_ptr<std::uint8_t[]> buffer(new std::uint8_t[serialSize]{ });
    std::uint8_t* bufferRaw = buffer.get();
    //Serialize force
    std::size_t serialIdx = communicatableForce.serialize(indices, bufferRaw);
    //Serialice torque with offset serialIdx
    serialIdx += communicatableTorque.serialize(indices, &bufferRaw[serialIdx]);
    //Serialice globalID with offset serialIdx
    communicatableID.serialize(indices, &bufferRaw[serialIdx]);
    //Add serialized data to rankDataMap
    rankDataMap.insert(std::make_pair(rankDest, std::move(buffer)));
  }); //forParticlesInSuperParticleSystem()
 
  //Retrieve rank of direct neighbours
  auto& listNeighbourRanks = sParticleSystem.getNeighbourRanks();
 
  //Create non blocking mpi helper
  singleton::MpiNonBlockingHelper mpiNbHelper;
 
  //Create ranDataMapSorted (WARNING: has to be existent until all data is received! c.f. #290)
  std::map<int, std::vector<std::uint8_t> > rankDataMapSorted;
 
  //Fill send buffer
  fillSendBuffer( rankDataMap, rankDataMapSorted, serialSize );
 
  //Send mapped data
  communication::sendMappedData( rankDataMapSorted,
      listNeighbourRanks, serialSize, surfaceForceComm, mpiNbHelper );
 
  //Receive data and iterate buffer
  communication::receiveAndExecuteForData( listNeighbourRanks, serialSize,
      surfaceForceComm, mpiNbHelper,
    [&](int rankOrig, std::uint8_t* bufferRaw){
    //Deserialize force
    std::size_t serialIdx = communicatableForce.deserialize(indices, bufferRaw);
    //Deserialize torque with offset serialIdx
    serialIdx += communicatableTorque.deserialize(indices, &bufferRaw[serialIdx]);
    //Deserialize globalID with offset serialIdx
    communicatableID.deserialize(indices, &bufferRaw[serialIdx]);
    //Retrieve vectors
    auto force = fieldForce.getField(0);
    auto torque = fieldTorque.getField(0);
    auto globalID = fieldID.getField(0);
 
#ifdef VERBOSE_COMMUNICATION
    int rank = singleton::mpi().getRank();
    std::cout << "Received (on rank=" << rank
              << " from rank=" << rankOrig
              << "): force=" << force
              << std::endl
              << "                  "
              << ", torque=" << torque
              << ", globalID=" << globalID
              << std::endl;
#endif
 
    //Create forceData Vector
    Vector<T,forceDataSize> forceData;
    for (unsigned iDim=0; iDim<D; ++iDim) {
      forceData[iDim] = force[iDim];
    }
 
    if constexpr (utilities::dimensions::convert<PARTICLETYPE::d>::rotation == 1 ) {
      forceData[D] = torque;
    }
    else {
      for (unsigned iDim=0; iDim<Drot; ++iDim) {
        forceData[iDim+D] = torque[iDim];
      }
    }
 
    //Add id-data-pair to map OR only add data
    auto mapPair = globalIdDataMap.insert( std::pair<std::size_t,
      Vector<T,forceDataSize>>(globalID, forceData) );
    if (mapPair.second==false){ //Checks previous existance
      globalIdDataMap[globalID] += forceData;
    }
  }); //receiveAndExecuteForData()
}

References fillSendBuffer(), forParticlesInSuperParticleSystem(), olb::FieldArrayD< T, DESCRIPTOR, PLATFORM, FIELD >::getField(), olb::SuperStructure< T, D >::getLoadBalancer(), olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getNeighbourRanks(), olb::singleton::MpiManager::getRank(), olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getSuperStructure(), olb::singleton::mpi(), receiveAndExecuteForData(), sendMappedData(), and olb::FieldArrayD< T, DESCRIPTOR, PLATFORM, FIELD >::setField().

Here is the call graph for this function:

Here is the caller graph for this function:

deserializeIcDest()

template<typename T , typename PARTICLETYPE >

std::size_t olb::particles::communication::deserializeIcDest	(	int &	globiCdest,
		std::uint8_t *	bufferRaw )

Definition at line 179 of file utilities.h.

{
  //Define iC field as FieldArrayD with single entry
  FieldArrayD<T,PARTICLETYPE,Platform::CPU_SISD,descriptors::IC> fieldiC(1);
  const std::vector<unsigned int> index{0};
  auto communicatable = ConcreteCommunicatable(fieldiC);
  //Deserialice iC
  communicatable.deserialize(index, bufferRaw);
  globiCdest = fieldiC.getField(0);
  //Return communicatable size
  return communicatable.size(index);
}

References olb::FieldArrayD< T, DESCRIPTOR, PLATFORM, FIELD >::getField().

Here is the call graph for this function:

deserializeIcDestAndParticle()

template<typename T , typename PARTICLETYPE >

void olb::particles::communication::deserializeIcDestAndParticle	(	int &	globiCdest,
		Particle< T, PARTICLETYPE > &	particle,
		std::uint8_t *	bufferiCandParticle )

Definition at line 195 of file utilities.h.

{
  //Deserialize iC and retrieve particle buffer
  std::size_t serialSizeField =
    deserializeIcDest<T,PARTICLETYPE>(globiCdest,bufferiCandParticle);
  std::uint8_t* bufferRawParticle = &bufferiCandParticle[serialSizeField];
  //Deserialize particle
  particle.deserialize(bufferRawParticle);
}

References olb::particles::Particle< T, PARTICLETYPE >::deserialize().

Here is the call graph for this function:

Here is the caller graph for this function:

doForParticle()

template<typename T , typename PARTICLETYPE , typename F >

void olb::particles::communication::doForParticle	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		ParallelParticleLocator &	locator,
		F	f )

Definition at line 198 of file lambdaLoops.h.

{
  //Retrieve rank and only proceed for correct one
  int rank = singleton::mpi().getRank();
  auto& loadBalancer = sParticleSystem.getSuperStructure().getLoadBalancer();
  if (rank==loadBalancer.rank(locator.globiC)){
    //Retrieve ParticleSystem
    auto bParticleSystems = sParticleSystem.getBlockParticleSystems();
    auto& particleSystem = *bParticleSystems[loadBalancer.loc(locator.globiC)];
    //Retrieve particle
    auto particle = particleSystem.get(locator.localID);
    //Call function f for particle
    doForParticle( particle, f );
  }
}

References doForParticle(), olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getBlockParticleSystems(), olb::SuperStructure< T, D >::getLoadBalancer(), olb::singleton::MpiManager::getRank(), olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getSuperStructure(), olb::particles::ParallelParticleLocator::globiC, olb::particles::ParallelParticleLocator::localID, and olb::singleton::mpi().

Here is the call graph for this function:

Here is the caller graph for this function:

evalDestRanks()

template<typename T , typename PARTICLECONTACTTYPE , typename WALLCONTACTTYPE >

std::unordered_set< int > olb::particles::communication::evalDestRanks	(	contact::ContactContainer< T, PARTICLECONTACTTYPE, WALLCONTACTTYPE > &	contactContainer,
		const std::size_t	globalParticleID )

evaluates ranks that need the new particle data

contact.isEmpty() &&

contact.isEmpty() &&

Definition at line 44 of file equationsOfMotionResults.h.

{
  std::unordered_set<int> destRanks;
 
  for (const PARTICLECONTACTTYPE& contact : contactContainer.particleContacts) {
    if (
        (globalParticleID == contact.getIDs()[0] ||
         globalParticleID == contact.getIDs()[1])) {
      destRanks.insert(contact.getResponsibleRank());
    }
  }
  for (const WALLCONTACTTYPE& contact : contactContainer.wallContacts) {
    if (
        globalParticleID == contact.getParticleID()) {
      destRanks.insert(contact.getResponsibleRank());
    }
  }
 
  return destRanks;
}

Here is the caller graph for this function:

evalEquationsOfMotionResults()

template<typename T , typename PARTICLETYPE , typename PARTICLECONTACTTYPE , typename WALLCONTACTTYPE >

std::size_t olb::particles::communication::evalEquationsOfMotionResults	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		contact::ContactContainer< T, PARTICLECONTACTTYPE, WALLCONTACTTYPE > &	contactContainer,
		std::multimap< int, std::unique_ptr< std::uint8_t[]> > &	dataMap )

Definition at line 106 of file equationsOfMotionResults.h.

{
  using namespace descriptors;
 
  //Create FieldArrayD for globalID, force and torque
  using GENERAL_EVAL =
      typename PARTICLETYPE ::template derivedField<descriptors::GENERAL>;
  using MOBILITY_EVAL =
      typename PARTICLETYPE ::template derivedField<descriptors::MOBILITY>;
  using SURFACE_EVAL =
      typename PARTICLETYPE ::template derivedField<descriptors::SURFACE>;
  using POSITION_EVAL =
      typename GENERAL_EVAL ::template derivedField<descriptors::POSITION>;
  using VELOCITY_EVAL =
      typename MOBILITY_EVAL ::template derivedField<descriptors::VELOCITY>;
  using ANGLE_EVAL =
      typename SURFACE_EVAL ::template derivedField<descriptors::ANGLE>;
  using ANGVELOCITY_EVAL =
      typename MOBILITY_EVAL ::template derivedField<descriptors::ANG_VELOCITY>;
  FieldArrayD<T, PARTICLETYPE, Platform::CPU_SISD, descriptors::ID> fieldID(1);
  FieldArrayD<T, PARTICLETYPE, Platform::CPU_SISD, POSITION_EVAL> fieldPosition(
      1);
  FieldArrayD<T, PARTICLETYPE, Platform::CPU_SISD, VELOCITY_EVAL> fieldVelocity(
      1);
  FieldArrayD<T, PARTICLETYPE, Platform::CPU_SISD, ANGLE_EVAL> fieldAngle(1);
  FieldArrayD<T, PARTICLETYPE, Platform::CPU_SISD, ANGVELOCITY_EVAL>
      fieldAngVelocity(1);
 
  //Create communicatables
  auto communicatableID          = ConcreteCommunicatable(fieldID);
  auto communicatablePosition    = ConcreteCommunicatable(fieldPosition);
  auto communicatableVelocity    = ConcreteCommunicatable(fieldVelocity);
  auto communicatableAngle       = ConcreteCommunicatable(fieldAngle);
  auto communicatableAngVelocity = ConcreteCommunicatable(fieldAngVelocity);
 
  //Retrieve serial size
  const std::vector<unsigned int> indices {0};
  const std::size_t               serialSize =
      communicatableID.size(indices) + communicatablePosition.size(indices) +
      communicatableVelocity.size(indices) + communicatableAngle.size(indices) +
      communicatableAngVelocity.size(indices);
 
  // Fill map with data
  communication::forParticlesInSuperParticleSystem(
      sParticleSystem,
      [&](Particle<T, PARTICLETYPE>&       particle,
          ParticleSystem<T, PARTICLETYPE>& particleSystem, int globiC) {
        using PCONDITION = std::conditional_t<
            access::providesSurface<PARTICLETYPE>(),
            conditions::
                valid_particle_centres,   // only consider center for resolved
            conditions::valid_particles>; // only consider valid
 
        if (PCONDITION::value(particle, globiC)) {
          // Get quantities to package
          fieldID.setField(0,
                           particle.template getField<PARALLELIZATION, ID>());
          fieldPosition.setField(0, access::getPosition(particle));
          fieldVelocity.setField(0, access::getVelocity(particle));
          fieldAngle.setField(0, access::getAngle(particle));
          fieldAngVelocity.setField(0, access::getAngularVelocity(particle));
 
          communicateEquationsOfMotionResultsIntra(
              sParticleSystem,
              particle.template getField<PARALLELIZATION, ID>(),
              access::getPosition(particle), access::getVelocity(particle),
              access::getAngle(particle), access::getAngularVelocity(particle));
 
          // Evaluate which ranks need to know the data
          std::unordered_set<int> destRanks {
              evalDestRanks(contactContainer, fieldID.getField(0))};
 
          for (const int destRank : destRanks) {
            std::unique_ptr<std::uint8_t[]> buffer(
                new std::uint8_t[serialSize] {});
            std::uint8_t* bufferRaw = buffer.get();
            std::size_t   serialIdx =
                communicatableID.serialize(indices, bufferRaw);
            serialIdx += communicatablePosition.serialize(
                indices, &bufferRaw[serialIdx]);
            serialIdx += communicatableVelocity.serialize(
                indices, &bufferRaw[serialIdx]);
            serialIdx +=
                communicatableAngle.serialize(indices, &bufferRaw[serialIdx]);
            serialIdx += communicatableAngVelocity.serialize(
                indices, &bufferRaw[serialIdx]);
            dataMap.insert(std::make_pair(destRank, std::move(buffer)));
          }
 
#ifdef VERBOSE_CONTACT_COMMUNICATION
          std::cout << "Rank " << singleton::mpi().getRank()
                    << " sends particle data of ID " << fieldID.getField(0)
                    << " to ";
          for (const int destRank : destRanks) {
            std::cout << destRank << " ";
          }
          std::cout << std::endl;
#endif
        }
      });
 
  return serialSize;
}

References communicateEquationsOfMotionResultsIntra(), evalDestRanks(), forParticlesInSuperParticleSystem(), olb::particles::access::getAngle(), olb::particles::access::getAngularVelocity(), olb::FieldArrayD< T, DESCRIPTOR, PLATFORM, FIELD >::getField(), olb::particles::access::getPosition(), olb::singleton::MpiManager::getRank(), olb::particles::access::getVelocity(), olb::singleton::mpi(), and olb::FieldArrayD< T, DESCRIPTOR, PLATFORM, FIELD >::setField().

Here is the call graph for this function:

Here is the caller graph for this function:

fillSendBuffer()

void olb::particles::communication::fillSendBuffer	(	std::multimap< int, std::unique_ptr< std::uint8_t[]> > &	rankDataMap,
		std::map< int, std::vector< std::uint8_t > > &	rankDataMapSorted,
		std::size_t	serialSize )

Definition at line 387 of file relocation.h.

{
  //Iterate over rank data pairs
  for (auto& rankDataPair : rankDataMap) {
    //Decompose pair
    int rankDest = rankDataPair.first;
    std::uint8_t* buffer = rankDataPair.second.get();
    //Write data to sorted map
    // - creates vector via 3-arg initializer list
    rankDataMapSorted[rankDest].insert(
      rankDataMapSorted[rankDest].end(),
      buffer,
      buffer + serialSize );
  }
}

Here is the caller graph for this function:

forParticlesInSuperParticleSystem()

template<typename T , typename PARTICLETYPE , typename PCONDITION = conditions::valid_particles, typename F >

void olb::particles::communication::forParticlesInSuperParticleSystem	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		F	f )

Definition at line 173 of file lambdaLoops.h.

{
  //Iterate over particle systems
  forSystemsInSuperParticleSystem( sParticleSystem,
    [&](ParticleSystem<T,PARTICLETYPE>& particleSystem, int iC, int globiC){
    //Iterate over particles
    forParticlesInParticleSystem<T,PARTICLETYPE,PCONDITION>( particleSystem,
      [&](Particle<T,PARTICLETYPE>& particle){
      if constexpr (std::is_invocable_v<F,
        Particle<T,PARTICLETYPE>&,ParticleSystem<T,PARTICLETYPE>&,int>)
      {
        f( particle, particleSystem, globiC );
      } else {
        f( particle );
      }
    },globiC);
  });
}

References forSystemsInSuperParticleSystem().

Here is the call graph for this function:

Here is the caller graph for this function:

forSystemsInSuperParticleSystem()

template<typename T , typename PARTICLETYPE , typename F >

void olb::particles::communication::forSystemsInSuperParticleSystem	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		F	f )

Definition at line 154 of file lambdaLoops.h.

{
  //Retrieve load balancer
  auto& loadBalancer = sParticleSystem.getSuperStructure().getLoadBalancer();
  //Loop over iCs
  for (int iC=0; iC<loadBalancer.size(); ++iC){
    int globiC = loadBalancer.glob(iC);
    //Retrieve container
    auto bParticleSystems = sParticleSystem.getBlockParticleSystems();
    auto& particleSystem = *bParticleSystems[iC];
    f( particleSystem, iC, globiC );
  }
}

References olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getBlockParticleSystems(), olb::SuperStructure< T, D >::getLoadBalancer(), and olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getSuperStructure().

Here is the call graph for this function:

Here is the caller graph for this function:

getCuboid()

template<typename T , unsigned D>

bool olb::particles::communication::getCuboid	(	const CuboidGeometry< T, D > &	cuboidGeometry,
		const Vector< bool, D > &	periodicity,
		const PhysR< T, D > &	position,
		int &	iC )

Function returns true if cuboid was found and gives iC.

Definition at line 320 of file utilities.h.

{
  PhysR<T,D> newPosition(
      applyPeriodicityToPosition(cuboidGeometry, periodicity, position));
  return cuboidGeometry.getC(newPosition, iC);
}

References applyPeriodicityToPosition().

Here is the call graph for this function:

Here is the caller graph for this function:

getCuboidMax()

template<typename T , unsigned D>

Vector< T, D > olb::particles::communication::getCuboidMax	(	const CuboidGeometry< T, D > &	cuboidGeometry,
		const PhysR< T, D > &	min )

Returns maximal coordinate of domain for periodic particle boundaries.

Definition at line 293 of file utilities.h.

{
  const T physDeltaX = cuboidGeometry.getMotherCuboid().getDeltaR();
  return min + physDeltaX * cuboidGeometry.getMotherCuboid().getExtent();
}

getCuboidMin()

template<typename T , unsigned D>

Vector< T, D > olb::particles::communication::getCuboidMin

(

const CuboidGeometry< T, D > &

cuboidGeometry

)

Returns minimal coordinate of domain for periodic particle boundaries.

Definition at line 284 of file utilities.h.

{
  const T physDeltaX = cuboidGeometry.getMotherCuboid().getDeltaR();
  return (cuboidGeometry.getMotherCuboid().getOrigin() - 0.5 * physDeltaX);
 
}

getNeighborCuboids()

template<typename T , unsigned D, bool verbose = false>

std::vector< int > olb::particles::communication::getNeighborCuboids	(	CuboidGeometry< T, D > &	cuboidGeometry,
		unsigned	offset,
		int	globiC )

Get neighbour cuboids.

Definition at line 39 of file utilities.h.

{
  std::vector<int> neighbors;
  cuboidGeometry.getNeighbourhood(globiC, neighbors, offset);
 
  //Print, if desired
  if constexpr(verbose){
    std::cout << globiC << ": " << neighbors << std::endl;
  }
 
  return neighbors;
}

getNeighbourRanks() [1/2]

template<typename T , unsigned D, bool verbose = false>

std::unordered_set< int > olb::particles::communication::getNeighbourRanks	(	SuperStructure< T, D > &	superStructure,
		unsigned	offset )

Get neighbour ranks.

Definition at line 141 of file utilities.h.

{
  //Retrive parallization infos
  const int rank = singleton::mpi().getRank();
  return getNeighbourRanks<T,D,false>(superStructure, offset, rank);
}

References olb::singleton::MpiManager::getRank(), and olb::singleton::mpi().

Here is the call graph for this function:

getNeighbourRanks() [2/2]

template<typename T , unsigned D, bool verbose = false>

std::unordered_set< int > olb::particles::communication::getNeighbourRanks	(	SuperStructure< T, D > &	superStructure,
		unsigned	offset,
		int	rank )

Get neighbour ranks.

HOTFIX VERSION (according to #319)

Definition at line 113 of file utilities.h.

{
  std::unordered_set<int> rankNeighbours;
 
#ifdef PARALLEL_MODE_MPI
  auto& loadBalancer = superStructure.getLoadBalancer();
  auto& cuboidGeometry = superStructure.getCuboidGeometry();
 
  // 1. sample complete neighbourhood
  std::map<int,std::vector<int>> neighbourhood;
  evaluateCuboidGeometryNeighbourhood( cuboidGeometry, neighbourhood, offset);
 
  //2. check consistency
  constexpr bool correctFaultyNeighbourhood = true;
  checkCuboidNeighbourhoodConsistency( neighbourhood, correctFaultyNeighbourhood );
 
  //3. evaluate relevant ranks
  rankNeighbours = getNeighbourRanksFromCuboidNeighbourhood<T,D,verbose>(
    superStructure, rank, neighbourhood );
#endif
 
  return rankNeighbours;
}

References olb::checkCuboidNeighbourhoodConsistency(), olb::evaluateCuboidGeometryNeighbourhood(), olb::SuperStructure< T, D >::getCuboidGeometry(), and olb::SuperStructure< T, D >::getLoadBalancer().

Here is the call graph for this function:

getNeighbourRanksFromCuboidNeighbourhood() [1/3]

template<typename T , unsigned D, bool verbose = false>

std::unordered_set< int > olb::particles::communication::getNeighbourRanksFromCuboidNeighbourhood	(	SuperStructure< T, D > &	superStructure,
		const std::vector< std::unordered_set< int > > &	neighbourhood )

Get neighbour ranks.

Definition at line 102 of file utilities.h.

{
  const int rank = singleton::mpi().getRank();
  return getNeighbourRanksFromCuboidNeighbourhood<T,D,verbose>(
    superStructure, rank, neighbourhood);
}

References olb::singleton::MpiManager::getRank(), and olb::singleton::mpi().

Here is the call graph for this function:

getNeighbourRanksFromCuboidNeighbourhood() [2/3]

template<typename T , unsigned D, bool verbose = false>

std::unordered_set< int > olb::particles::communication::getNeighbourRanksFromCuboidNeighbourhood	(	SuperStructure< T, D > &	superStructure,
		int	rank,
		const std::vector< std::unordered_set< int > > &	neighbourhood )

Get neighbour ranks.

Definition at line 84 of file utilities.h.

{
  std::map<int,std::vector<int>> neighbourhoodMap;
 
  for(unsigned i=0; i<neighbourhood.size(); ++i) {
    neighbourhoodMap[i] = std::vector<int>();
    for(int iC : neighbourhood[i]) {
      neighbourhoodMap[i].push_back(iC);
    }
  }
 
  return getNeighbourRanksFromCuboidNeighbourhood<T,D,verbose>(
    superStructure, rank, neighbourhoodMap );
}

getNeighbourRanksFromCuboidNeighbourhood() [3/3]

template<typename T , unsigned D, bool verbose = false>

std::unordered_set< int > olb::particles::communication::getNeighbourRanksFromCuboidNeighbourhood	(	SuperStructure< T, D > &	superStructure,
		int	rank,
		std::map< int, std::vector< int > >	neighbourhood )

Get neighbour ranks.

Definition at line 55 of file utilities.h.

{
  std::unordered_set<int> rankNeighbours;
 
#ifdef PARALLEL_MODE_MPI
  auto& loadBalancer = superStructure.getLoadBalancer();
 
  for ( auto cuboidNeighbourPair : neighbourhood){
    int globiC = cuboidNeighbourPair.first;
    if (loadBalancer.rank(globiC) == rank) {
      std::vector<int>& neighbours = cuboidNeighbourPair.second;
      for (int globiCN : neighbours ){
        rankNeighbours.insert( loadBalancer.rank(globiCN));
      }
    }
  }
 
  if constexpr(verbose){
    std::cout << rank << ": " << rankNeighbours << std::endl;
  }
#endif
 
  return rankNeighbours;
}

References olb::SuperStructure< T, D >::getLoadBalancer().

Here is the call graph for this function:

getSurfaceTouchingICs() [1/2]

template<typename T , unsigned D, typename F = std::function<void(int)>, bool domainWarning = false, bool checkDiscretePoints = false>

std::unordered_set< int > olb::particles::communication::getSurfaceTouchingICs	(	CuboidGeometry< T, D > &	cuboidGeometry,
		Vector< T, D >	position,
		T	circumRadius,
		const Vector< bool, D > &	periodicity,
		int	globiC,
		F	f = [](int){} )

Get a set of surface touching iCs (that are not globiC) Allows to run an optional function per unique globiC.

Definition at line 299 of file particleUtilities.h.

                         {} )
{
  // This option is prone to errors
  if constexpr(checkDiscretePoints) {
    //Set of destination iCs preventing sending to same iC multiple times
    std::unordered_set<int> iCs;
    //Retrieve points on hull
    auto pointsOnHull = discrete_points_on_hull::calculate( position, circumRadius );
    //Iterate over points on hull
    for (const PhysR<T,D>& point : pointsOnHull){
      //Retrieve residence iC
      int globiConHull=0;
      const bool cuboidFound = getCuboid(cuboidGeometry,
          periodicity, point, globiConHull);
      if constexpr(domainWarning){
        if (!cuboidFound){
          std::cerr << "Warning [during getSurfaceTouchingICs]: Cuboid not found for point "
                    << point << std::endl;
        }
      }
      //Check whether point has left responsible ic
      if (cuboidFound && globiConHull!=globiC){
        //Add destination ic to list (to avoid resending)
        if(iCs.insert(globiConHull).second) {
          f(globiConHull);
        }
      } //if (globiConHull!=globiC)
    } //for (auto point : pointsOnHull)
    return iCs;
  }
  else {
    const T physDeltaX = cuboidGeometry.getMotherCuboid().getDeltaR();
    std::vector<int> tmp = communication::getNeighborCuboids<T,D,false>(
          cuboidGeometry, util::ceil(circumRadius/physDeltaX), globiC);
    std::unordered_set<int> iCs;
    for(const int entry : tmp) {
      if( globiC != entry ) {
        iCs.insert(entry);
        f(entry);
      }
    }
    return iCs;
  }
  __builtin_unreachable();
}

References getSurfaceTouchingICs().

Here is the call graph for this function:

Here is the caller graph for this function:

getSurfaceTouchingICs() [2/2]

template<typename T , typename PARTICLETYPE , typename F = std::function<void(int)>>

std::unordered_set< int > olb::particles::communication::getSurfaceTouchingICs	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		Vector< T, PARTICLETYPE::d >	position,
		T	circumRadius,
		const Vector< bool, PARTICLETYPE::d > &	periodicity,
		int	globiC,
		F	f = [](int){} )

Definition at line 350 of file particleUtilities.h.

                         {} )
{
  std::unordered_set<int> neighbors = sParticleSystem.getCuboidNeighborhood()[globiC];
  for(const int neighbor: neighbors) {
    f(neighbor);
  }
  return neighbors;
}

References getSurfaceTouchingICs().

Here is the call graph for this function:

getVectorOfSurfaceTouchingICs()

template<typename T , unsigned D, typename F = std::function<void(int)>, bool domainWarning = false>

std::vector< int > olb::particles::communication::getVectorOfSurfaceTouchingICs	(	CuboidGeometry< T, D > &	cuboidGeometry,
		Vector< T, D >	position,
		T	circumRadius,
		const Vector< bool, D > &	periodicity,
		int	globiC,
		F	f = [](int){} )

Definition at line 367 of file particleUtilities.h.

                         {} )
{
  const std::unordered_set<int> touchedICs{getSurfaceTouchingICs<T,D,F,domainWarning>(
      cuboidGeometry, position, circumRadius, periodicity, globiC, f)};
  std::vector<int> listOfICs;
  listOfICs.insert(listOfICs.end(), touchedICs.begin(), touchedICs.end());
  return listOfICs;
}

References getVectorOfSurfaceTouchingICs().

Here is the call graph for this function:

Here is the caller graph for this function:

insertSerializedParticle()

template<typename T , typename PARTICLETYPE >

std::size_t olb::particles::communication::insertSerializedParticle	(	ParticleSystem< T, PARTICLETYPE > &	particleSystem,
		std::size_t	idxParticle,
		std::uint8_t	rawBuffer[] )

Definition at line 85 of file relocation.h.

{
  using namespace descriptors;
  //Retrieve particle
  auto particle = particleSystem.get(idxParticle);
 
#ifdef VERBOSE_COMMUNICATION
  auto globiCcentreLast = particle.template getField<PARALLELIZATION,IC>();
#endif
 
  //Deserialise particle
  int globiCdest;
  deserializeIcDestAndParticle(globiCdest,particle,rawBuffer);
 
#ifdef VERBOSE_COMMUNICATION
  auto globiCcentre = particle.template getField<PARALLELIZATION,IC>();
 
  if(globiCcentre!=globiCcentreLast){
    int rank = singleton::mpi().getRank();
    std::cout << std::endl << "[Particle] "
              << "CENTRE SWITCH("
              << globiCcentreLast << "->" << globiCcentre << ")"
              << " at receiving at iC=" << "X"
              << "(rank=" << rank << ")" << std::endl;
  }
#endif
 
 
  //Update surface pointer
  if constexpr( access::providesSurface<PARTICLETYPE>() ){
    updateSurfacePtr( particleSystem, idxParticle );
  }
  return particle.getId();
}

References deserializeIcDestAndParticle(), olb::particles::ParticleSystem< T, PARTICLETYPE >::get(), olb::singleton::MpiManager::getRank(), olb::singleton::mpi(), and updateSurfacePtr().

Here is the call graph for this function:

Here is the caller graph for this function:

movePositionToEnd()

template<typename T >

T olb::particles::communication::movePositionToEnd	(	const T	position,
		const T	max,
		const T	min )

Definition at line 260 of file utilities.h.

{
  return max - (min - position);
}

Here is the caller graph for this function:

movePositionToStart()

template<typename T >

T olb::particles::communication::movePositionToStart	(	const T	position,
		const T	max,
		const T	min )

Definition at line 254 of file utilities.h.

{
  return min + (position - max);
}

Here is the caller graph for this function:

prepareRelocation()

template<typename T , typename PARTICLETYPE >

void olb::particles::communication::prepareRelocation	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		Particle< T, PARTICLETYPE > &	particle,
		int	globiC,
		int	globiCdest,
		std::multimap< int, std::unique_ptr< std::uint8_t[]> > &	rankDataMapRelocationInter,
		std::vector< std::unique_ptr< std::uint8_t[]> > &	dataListRelocationIntra,
		std::size_t	serialSize )

Definition at line 182 of file relocation.h.

{
  //Retrieve loadBalancer
  auto& loadBalancer = sParticleSystem.getSuperStructure().getLoadBalancer();
  //Create buffer pointer
  std::unique_ptr<std::uint8_t[]> buffer(new std::uint8_t[serialSize]{ });
 
  //Serialize particle content into buffer
  serializeIcDestAndParticle( globiCdest, particle, buffer.get() );
 
  //Check, whether iCs are on same core
  int rank = singleton::mpi().getRank();
  int rankDest = loadBalancer.rank(globiCdest);
  if (rank != rankDest){
    //Add rank-buffer pair to rankDataMapRelocationInter
    rankDataMapRelocationInter.insert(std::make_pair(rankDest, std::move(buffer)));
  } else {
    //If on same core location can be done without mpi
    dataListRelocationIntra.push_back(std::move(buffer));
  } //else (rank != rankDest)
 
#ifdef VERBOSE_COMMUNICATION
  auto globalID = particle.template getField<
    descriptors::PARALLELIZATION,descriptors::ID>();
  std::cout << std::endl << "[Particle] SENDING";
  if (rank != rankDest){ std::cout << "(inter)"; } else { std::cout << "(intra)"; }
  std::cout << " Particle"
            << "(globalID=" << globalID << ") "
            << "from iC" << globiC << "(rank=" << rank << ") to iC"
            << globiCdest << "(rank=" << rankDest << ")" << std::endl;
  particle.template print<true>();
#endif
}

References olb::SuperStructure< T, D >::getLoadBalancer(), olb::singleton::MpiManager::getRank(), olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getSuperStructure(), olb::singleton::mpi(), and serializeIcDestAndParticle().

Here is the call graph for this function:

Here is the caller graph for this function:

receiveAndExecuteForData()

template<typename F >

void olb::particles::communication::receiveAndExecuteForData	(	const std::unordered_set< int > &	availableRanks,
		std::size_t	serialSize,
		MPI_Comm	Comm,
		singleton::MpiNonBlockingHelper &	mpiNbHelper,
		F	f )

Definition at line 572 of file relocation.h.

{
  //Iterate available ranks
  for (int rankOrig : availableRanks) {
    //Probe whether incomming data on rank
    std::size_t noBytesReceived = singleton::mpi().probeReceiveSize(rankOrig, MPI_BYTE, 1, Comm);
    //Create receive buffer as vector
    std::vector<std::uint8_t> recv_buffer(noBytesReceived);
#ifdef VERBOSE_COMMUNICATION
      int rank = singleton::mpi().getRank();
      std::cout << "Serial(rank=" << rank << "): "
                << "noBytesToBeReceived=" << noBytesReceived << "; " << std::endl;
#endif
    //Receive data via mpi
    singleton::mpi().receive<std::uint8_t>(
      recv_buffer.data(),
      noBytesReceived,
      rankOrig,
      1,
      Comm);
    //Loop over bytes in chunks of serialSize
    for (unsigned iByte=0; iByte < noBytesReceived; iByte += serialSize) {
      //Define raw buffer
      std::uint8_t* rawBuffer = &recv_buffer[iByte];
      //Execute passed function
      f( rankOrig, rawBuffer );
#ifdef VERBOSE_COMMUNICATION
      int rank = singleton::mpi().getRank();
      std::cout << "Serial(rank=" << rank << "): "
                << "noBytesReceived=" << noBytesReceived << "; "
                << "iByte=" << iByte << std::endl;
#endif
      //assignParticleToIC( sParticleSystem, receivedLocalIDs, rankOrig, rawBuffer );
    } // for (int iByte=0; iByte<noBytesReceived; iByte+=serialSize)
  } // for (auto rankOrig : availableRanks)
  singleton::mpi().waitAll(mpiNbHelper);
  //Free mpi helper
  mpiNbHelper.free();
}

References olb::singleton::MpiNonBlockingHelper::free(), olb::singleton::MpiManager::getRank(), olb::singleton::mpi(), olb::singleton::MpiManager::probeReceiveSize(), olb::singleton::MpiManager::receive(), and olb::singleton::MpiManager::waitAll().

Here is the call graph for this function:

Here is the caller graph for this function:

receiveEquationsOfMotionResults()

template<typename T , typename PARTICLETYPE >

void olb::particles::communication::receiveEquationsOfMotionResults	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		MPI_Comm	equationsOfMotionComm,
		singleton::MpiNonBlockingHelper &	mpiNbHelper )

Definition at line 217 of file equationsOfMotionResults.h.

{
  using namespace descriptors;
 
  // Retrieve rank of direct neighbours
  auto& listNeighbourRanks = sParticleSystem.getNeighbourRanks();
 
  //Create FieldArrayD for globalID, force and torque
  using GENERAL_EVAL =
      typename PARTICLETYPE ::template derivedField<descriptors::GENERAL>;
  using MOBILITY_EVAL =
      typename PARTICLETYPE ::template derivedField<descriptors::MOBILITY>;
  using SURFACE_EVAL =
      typename PARTICLETYPE ::template derivedField<descriptors::SURFACE>;
  using POSITION_EVAL =
      typename GENERAL_EVAL ::template derivedField<descriptors::POSITION>;
  using VELOCITY_EVAL =
      typename MOBILITY_EVAL ::template derivedField<descriptors::VELOCITY>;
  using ANGLE_EVAL =
      typename SURFACE_EVAL ::template derivedField<descriptors::ANGLE>;
  using ANGVELOCITY_EVAL =
      typename MOBILITY_EVAL ::template derivedField<descriptors::ANG_VELOCITY>;
  FieldArrayD<T, PARTICLETYPE, Platform::CPU_SISD, descriptors::ID> fieldID(1);
  FieldArrayD<T, PARTICLETYPE, Platform::CPU_SISD, POSITION_EVAL> fieldPosition(
      1);
  FieldArrayD<T, PARTICLETYPE, Platform::CPU_SISD, VELOCITY_EVAL> fieldVelocity(
      1);
  FieldArrayD<T, PARTICLETYPE, Platform::CPU_SISD, ANGLE_EVAL> fieldAngle(1);
  FieldArrayD<T, PARTICLETYPE, Platform::CPU_SISD, ANGVELOCITY_EVAL>
      fieldAngVelocity(1);
 
  //Create communicatables
  auto communicatableID          = ConcreteCommunicatable(fieldID);
  auto communicatablePosition    = ConcreteCommunicatable(fieldPosition);
  auto communicatableVelocity    = ConcreteCommunicatable(fieldVelocity);
  auto communicatableAngle       = ConcreteCommunicatable(fieldAngle);
  auto communicatableAngVelocity = ConcreteCommunicatable(fieldAngVelocity);
 
  // Retrieve serial size
  const std::vector<unsigned int> indices {0};
  const std::size_t               serialSize =
      communicatableID.size(indices) + communicatablePosition.size(indices) +
      communicatableVelocity.size(indices) + communicatableAngle.size(indices) +
      communicatableAngVelocity.size(indices);
 
  // Receive data and iterate buffer
  // TODO: Improve processing of data, do not iterate through the all particles every time (maybe also add main iC to the communicated data to reduce processed particles)
  communication::receiveAndExecuteForData(
      listNeighbourRanks, serialSize, equationsOfMotionComm, mpiNbHelper,
      [&](int rankOrig, std::uint8_t* buffer) {
        std::size_t serialIdx = communicatableID.deserialize(indices, buffer);
        serialIdx +=
            communicatablePosition.deserialize(indices, &buffer[serialIdx]);
        serialIdx +=
            communicatableVelocity.deserialize(indices, &buffer[serialIdx]);
        serialIdx +=
            communicatableAngle.deserialize(indices, &buffer[serialIdx]);
        serialIdx +=
            communicatableAngVelocity.deserialize(indices, &buffer[serialIdx]);
 
        communication::forParticlesInSuperParticleSystem(
            sParticleSystem,
            [&](Particle<T, PARTICLETYPE>&       particle,
                ParticleSystem<T, PARTICLETYPE>& particleSystem, int globiC) {
              const std::size_t currentGlobalParticleID =
                  particle.template getField<PARALLELIZATION, ID>();
              if (fieldID.getField(0) == currentGlobalParticleID) {
                // Override particle data
                particle.template setField<GENERAL, POSITION>(
                    fieldPosition.getField(0));
                particle.template setField<MOBILITY, VELOCITY>(
                    fieldVelocity.getField(0));
                particle.template setField<SURFACE, ANGLE>(
                    fieldAngle.getField(0));
                particle.template setField<MOBILITY, ANG_VELOCITY>(
                    fieldAngVelocity.getField(0));
              }
            });
      });
}

References forParticlesInSuperParticleSystem(), olb::FieldArrayD< T, DESCRIPTOR, PLATFORM, FIELD >::getField(), olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getNeighbourRanks(), and receiveAndExecuteForData().

Here is the call graph for this function:

Here is the caller graph for this function:

receiveParticles()

template<typename T , typename PARTICLETYPE >

void olb::particles::communication::receiveParticles	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		std::vector< std::unique_ptr< std::uint8_t[]> > &	dataListRelocationIntra,
		std::size_t	serialSize,
		MPI_Comm	particleDistributionComm,
		singleton::MpiNonBlockingHelper &	mpiNbHelper,
		const Vector< bool, PARTICLETYPE::d > &	periodicity )

Definition at line 621 of file relocation.h.

{
  using namespace particles::access;
 
  //Retrieve load balancer and cuboid geometry
  auto& superStructure = sParticleSystem.getSuperStructure();
  auto& loadBalancer = superStructure.getLoadBalancer();
 
  //Create list of local particle iDs to check afterwards,
  //  whether data was received (resolved only)
  std::vector<std::vector<std::size_t>> receivedLocalIDs(
    loadBalancer.size());
 
  //Intra core relocations (without MPI)
  for ( auto& buffer : dataListRelocationIntra ){
    //Define raw buffer
    std::uint8_t* rawBuffer = buffer.get();
    int rankOrig = singleton::mpi().getRank();
    assignParticleToIC( sParticleSystem, receivedLocalIDs, rankOrig, rawBuffer );
  }
 
  //Retrieve neighbour ranks
  auto& listNeighbourRanks = sParticleSystem.getExtendedNeighbourRanks();
  //Iterate over inter core received particle data
  receiveAndExecuteForData( listNeighbourRanks, serialSize,
      particleDistributionComm, mpiNbHelper,
    [&](int rankOrig, std::uint8_t* rawBuffer){
    //Assign particle to destination iC
    assignParticleToIC( sParticleSystem, receivedLocalIDs, rankOrig, rawBuffer );
  });
 
  //Check, whether invalidation nccessary
  if constexpr( providesSurface<PARTICLETYPE>() ){
    checkInvalidationOnReceival( sParticleSystem, receivedLocalIDs, periodicity );
  }
}

References assignParticleToIC(), checkInvalidationOnReceival(), olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getExtendedNeighbourRanks(), olb::SuperStructure< T, D >::getLoadBalancer(), olb::singleton::MpiManager::getRank(), olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getSuperStructure(), olb::singleton::mpi(), and receiveAndExecuteForData().

Here is the call graph for this function:

Here is the caller graph for this function:

searchParticleGlobally()

template<typename T , typename PARTICLETYPE , typename PCONDITION = conditions::valid_particle_centres, bool sync = true>

bool olb::particles::communication::searchParticleGlobally	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		std::size_t	globalIDrequested,
		std::size_t &	localParticleID,
		int &	globiC )

Search particle in SuperParticleSystem by globalParticleID and return globiC if found.

• WARNING: Sync is enabled by default, to avoid errors. As it might usually not be necessary though, communication can be disables to increase performance.

Definition at line 240 of file particleUtilities.h.

{
  using namespace descriptors;
  static_assert(PARTICLETYPE::template providesNested<PARALLELIZATION,ID>(), "Field PARALLELIZATION:ID has to be provided");
  //Retrieve load balancer
  auto& loadBalancer = sParticleSystem.getSuperStructure().getLoadBalancer();
  //Initialize return quantities
  localParticleID = 0;
  globiC = 0;
  bool found = false;
  //Loop over particle systems
  for (int iC=0; iC<loadBalancer.size(); ++iC){
    if (!found){
      int globiC = loadBalancer.glob(iC);
      //Retrieve container
      auto bParticleSystems = sParticleSystem.getBlockParticleSystems();
      auto& particleSystem = *bParticleSystems[iC];
      //Search particle on particleSystem
      //NOTE: Can not be substituted with searchParticle( ParticleSystem ..) due to globiC
      for (std::size_t iP=0; iP<particleSystem.size(); ++iP) {
        if (!found){
          auto particle = particleSystem.get(iP);
          //Execute F when particle meets condition
          doWhenMeetingCondition<T,PARTICLETYPE,PCONDITION>( particle,
            [&](Particle<T,PARTICLETYPE> particle){
            auto globalID = particle.template getField<PARALLELIZATION,ID>();
            if (globalID==globalIDrequested){
              localParticleID = particle.getId();
              found=true;
            }
          }, globiC);
        } else {
          break;
        }
      } //for (std::size_t iP=0; iP<particleSystem.size(); ++iP)
    } else {
      break;
    }
  } //for (int iC=0; iC<loadBalancer.size(); ++iC)
#ifdef PARALLEL_MODE_MPI
  if constexpr(sync){
    singleton::mpi().reduceAndBcast(found, MPI_LOR);
    if (found){
      singleton::mpi().reduceAndBcast(globiC, MPI_MAX);
      singleton::mpi().reduceAndBcast(localParticleID, MPI_MAX);
    }
  }
#endif
  return found;
}

References olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getBlockParticleSystems(), olb::particles::Particle< T, PARTICLETYPE >::getId(), olb::SuperStructure< T, D >::getLoadBalancer(), olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getSuperStructure(), and searchParticleGlobally().

Here is the call graph for this function:

Here is the caller graph for this function:

sendMappedData()

void olb::particles::communication::sendMappedData	(	std::map< int, std::vector< std::uint8_t > > &	rankDataMapSorted,
		const std::unordered_set< int > &	availableRanks,
		const std::size_t	serialSize,
		MPI_Comm	Comm,
		singleton::MpiNonBlockingHelper &	mpiNbHelper )

Definition at line 408 of file relocation.h.

{
  //Create map with vector holding individual data (of type std::uint8_t)
  //Iterate over available ranks and retrieve total number of ranks taking part in communication
  int noRelevantRanks = availableRanks.size();
  //Check whether any rank is taking part in communication
  //Allocate mpi requests for relevant ranks
  mpiNbHelper.allocate(noRelevantRanks);
  //Iterate over ranks with index iRank
  int iRank = 0;
  for (int rankEval : availableRanks) {
    //Send data via mpi
    singleton::mpi().iSend<std::uint8_t>(
      rankDataMapSorted[rankEval].data(),          //buffer: pointer to rank section in rankDataMapSorted
      rankDataMapSorted[rankEval].size(),          //count: number of bytes (per rank)
      rankEval,                                    //dest: destination rank
      &mpiNbHelper.get_mpiRequest()[iRank],      //request: request for iRank (iterator for relevant ranks)
      1,                                           //tag
      Comm);                                       //MPI_Comm scope
    iRank += 1;
  }
}

References olb::singleton::MpiNonBlockingHelper::allocate(), olb::singleton::MpiNonBlockingHelper::get_mpiRequest(), olb::singleton::MpiManager::iSend(), and olb::singleton::mpi().

Here is the call graph for this function:

Here is the caller graph for this function:

serializeIcDest()

template<typename T , typename PARTICLETYPE >

std::size_t olb::particles::communication::serializeIcDest	(	int	globiCdest,
		std::uint8_t *	bufferiCandParticle )

Definition at line 151 of file utilities.h.

{
  //Define iC field as FieldArrayD with single entry
  FieldArrayD<T,PARTICLETYPE,Platform::CPU_SISD,descriptors::IC> fieldiC(1);
  fieldiC.setField(0, globiCdest);
  const std::vector<unsigned int> index{0};
  auto communicatable = ConcreteCommunicatable(fieldiC);
  //Serialize iC
  communicatable.serialize(index, bufferiCandParticle);
  //Return communicatable size
  return communicatable.size(index);
}

References olb::FieldArrayD< T, DESCRIPTOR, PLATFORM, FIELD >::setField().

Here is the call graph for this function:

serializeIcDestAndParticle()

template<typename T , typename PARTICLETYPE >

void olb::particles::communication::serializeIcDestAndParticle	(	int	globiCdest,
		Particle< T, PARTICLETYPE > &	particle,
		std::uint8_t *	bufferiCandParticle )

Definition at line 166 of file utilities.h.

{
  //Serialize iC and retrieve particle buffer
  std::size_t serialSizeField =
    serializeIcDest<T,PARTICLETYPE>( globiCdest, bufferiCandParticle );
  std::uint8_t* bufferRawParticle = &bufferiCandParticle[serialSizeField];
  //Serialize particle
  particle.serialize(bufferRawParticle);
}

References olb::particles::Particle< T, PARTICLETYPE >::serialize().

Here is the call graph for this function:

Here is the caller graph for this function:

updateParticleCuboidDistribution()

template<typename T , typename PARTICLETYPE >

void olb::particles::communication::updateParticleCuboidDistribution	(	SuperParticleSystem< T, PARTICLETYPE > &	sParticleSystem,
		const T	physDeltaX,
		MPI_Comm	particleDistributionComm,
		const Vector< bool, PARTICLETYPE::d > &	periodicity )

Definition at line 667 of file relocation.h.

{
  //Retrieve serial size
  std::size_t serialSize = sParticleSystem.getSerialSize();
 
  //Increase serial size by Field holding the globiC of the destination
  // - for that, using FieldArrayD with single entry
  //TODO: Optimization possible here:
  // - Necessary for resolved particles
  // - Not necessary for subgrid: This could be exchanged by on-rank retrieval
  //   of the ic depending on particle position. This would lead to a reduction
  //   in buffer size, which will most likely speed up communication significantly.
  //   A separate treatment will however be very error prone.
  FieldArrayD<T,PARTICLETYPE,Platform::CPU_SISD,descriptors::IC> fieldiC(1);
  const ConstSpan<unsigned> index(0, 1);
  // Workaround for Clang argument deduction bug
  auto communicatable = ConcreteCommunicatable(fieldiC);
  std::size_t serialSizeField = communicatable.size(index);
  serialSize += serialSizeField;
 
  //Check rankDataMapRelocationInter necessity and create relocation map TODO: adapt description
  std::multimap<int,std::unique_ptr<std::uint8_t[]>> rankDataMapRelocationInter;
  std::vector<std::unique_ptr<std::uint8_t[]>> dataListRelocationIntra;
  checkRelocation( sParticleSystem, physDeltaX, rankDataMapRelocationInter,
      dataListRelocationIntra, serialSize, periodicity );
 
#ifdef PARALLEL_MODE_MPI
 
  //Create non blocking mpi helper
  singleton::MpiNonBlockingHelper mpiNbHelper;
 
  //Retrieve rank of direct neighbours
  auto& listNeighbourRanks = sParticleSystem.getExtendedNeighbourRanks();
 
  //Create ranDataMapSorted (WARNING: has to be existent until all data is received! c.f. #290)
  std::map<int, std::vector<std::uint8_t> > rankDataMapSorted;
 
  //Fill send buffer
  fillSendBuffer( rankDataMapRelocationInter, rankDataMapSorted, serialSize );
 
  //Send particles in rankDataMapRelocationInter map via mpi().iSend and return global number of bytes
  sendMappedData( rankDataMapSorted, listNeighbourRanks,
    serialSize, particleDistributionComm, mpiNbHelper );
 
  //Receive particles via mpi().receive
  receiveParticles( sParticleSystem, dataListRelocationIntra,
    serialSize, particleDistributionComm, mpiNbHelper, periodicity );
 
#else
  std::cerr
    << "ERROR: Invalidation treatement for sequential runs not implemented yet!"
    << std::endl;
  throw;
#endif
 
}

References checkRelocation(), fillSendBuffer(), olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getExtendedNeighbourRanks(), olb::particles::SuperParticleSystem< T, PARTICLETYPE >::getSerialSize(), receiveParticles(), and sendMappedData().

Here is the call graph for this function:

Here is the caller graph for this function:

updateSurfacePtr()

template<typename T , typename PARTICLETYPE >

void olb::particles::communication::updateSurfacePtr	(	ParticleSystem< T, PARTICLETYPE > &	particleSystem,
		std::size_t	iP )

Definition at line 42 of file relocation.h.

{
  using namespace descriptors;
  static_assert(PARTICLETYPE::template providesNested<SURFACE,SURFACE_ID>(), "Field SURFACE_ID has to be provided");
  //Retrieve particle
  auto particle = particleSystem.get(iP);
  //Retrieve surface id
  std::size_t idxSurface = particle.template getField<SURFACE,SURFACE_ID>();
  //Retrieve surface ptr
  using SIndicatorBaseType = SmoothIndicatorF<T,T,PARTICLETYPE::d,true>;
  auto& vectorOfIndicators = particleSystem.template getAssociatedData<
    std::vector<std::unique_ptr<SIndicatorBaseType>>>();
  auto sIndicatorPtr = vectorOfIndicators.at( idxSurface ).get();
  //Update particle
  particle.template setField<SURFACE,SINDICATOR>( sIndicatorPtr );
}

References olb::particles::ParticleSystem< T, PARTICLETYPE >::get().

Here is the call graph for this function:

Here is the caller graph for this function: