olb::OrthogonalHeterogeneousLoadBalancer< T > Class Template Reference

Load balancer for heterogeneous CPU-GPU systems. More...

#include <heterogeneousLoadBalancer.h>

Inheritance diagram for olb::OrthogonalHeterogeneousLoadBalancer< T >:

Collaboration diagram for olb::OrthogonalHeterogeneousLoadBalancer< T >:

Public Member Functions
	OrthogonalHeterogeneousLoadBalancer (CuboidDecomposition< T, 3 > &cGeometry, T largeBlockFraction=0.9)
Platform	platform (int loc) const override
void	setPlatform (int loc, Platform platform)
Public Member Functions inherited from olb::LoadBalancer< T >
	LoadBalancer (int size=1)
	Default empty constructor.
	LoadBalancer (int size, std::map< int, int > &loc, std::vector< int > &glob, std::map< int, int > &rank)
	Constructor accepting existing balancing.
	LoadBalancer (int size, std::map< int, int > &loc, std::vector< int > &glob, std::map< int, int > &rank, std::map< int, Platform > &platform)
	Constructor accepting existing heterogeneous balancing.
virtual	~LoadBalancer ()
	Default empty destructor.
void	swap (LoadBalancer< T > &loadBalancer)
	Swap method.
bool	isLocal (const int &glob) const
	returns whether glob is on this process
bool	isLocal (Platform platform) const
	returns whether there is a block on platform in this process
int	loc (const int &glob)
int	loc (int glob) const
int	glob (int loc) const
int	rank (int glob) const
int	size () const
int	getRankSize () const
virtual bool	doOutput (int glob) const
virtual void	setDoOutput (int glob, bool doOutput)
bool	operator== (const LoadBalancer< T > &rhs) const
	equal operator
std::size_t	getNblock () const override
	Number of data blocks for the serializable interface.
std::size_t	getSerializableSize () const override
	Binary size for the serializer.
bool *	getBlock (std::size_t iBlock, std::size_t &sizeBlock, bool loadingMode) override
	Return a pointer to the memory of the current block and its size for the serializable interface.
void	print (bool multiOutput=false) const
Public Member Functions inherited from olb::Serializable
virtual	~Serializable ()=default
template<bool includeLogOutputDir = true>
bool	save (std::string fileName="", const bool enforceUint=false)
	Save Serializable into file fileName
template<bool includeLogOutputDir = true>
bool	load (std::string fileName="", const bool enforceUint=false)
	Load Serializable from file fileName
bool	save (std::uint8_t *buffer)
	Save Serializable into buffer of length getSerializableSize
bool	load (const std::uint8_t *buffer)
	Load Serializable from buffer of length getSerializableSize
virtual void	postLoad ()

Additional Inherited Members
Protected Member Functions inherited from olb::BufferSerializable
template<typename DataType >
void	registerSerializable (const std::size_t iBlock, std::size_t &sizeBlock, std::size_t &currentBlock, size_t &sizeBufferIndex, bool *&dataPtr, DataType &data, const bool loadingMode=false)
	Register Serializable object of dynamic size.
template<typename DataType >
void	registerStdVectorOfVars (const std::size_t iBlock, std::size_t &sizeBlock, std::size_t &currentBlock, size_t &sizeBufferIndex, bool *&dataPtr, std::vector< DataType > &data, const bool loadingMode=false)
	Method for registering a std::vector<DataType> of primitive DataType (int, double, ...)
template<typename DataType >
void	registerStdVectorOfSerializablesOfConstSize (const std::size_t iBlock, std::size_t &sizeBlock, std::size_t &currentBlock, size_t &sizeBufferIndex, bool *&dataPtr, std::vector< DataType > &data, const bool loadingMode=false)
	Method for registering a std::vector<DataType> of constant-sized Serializable
template<typename DataType >
void	registerStdVectorOfSerializables (const std::size_t iBlock, std::size_t &sizeBlock, std::size_t &currentBlock, size_t &sizeBufferIndex, bool *&dataPtr, std::vector< DataType > &data, const bool loadingMode=false)
	Method for registering a std::vector<DataType> of dynamic-sized DataType
template<typename DataTypeKey , typename DataTypeValue >
void	registerMap (const std::size_t iBlock, std::size_t &sizeBlock, std::size_t &currentBlock, size_t &sizeBufferIndex, bool *&dataPtr, std::map< DataTypeKey, DataTypeValue > &data, const bool loadingMode=false)
	Method for registering a std::map<DataTypeKey, DataTypeValue> of fixed-sized types (i.e. int, double)
size_t	addSizeToBuffer (const std::size_t iBlock, std::size_t &sizeBlock, std::size_t &currentBlock, size_t &sizeBufferIndex, bool *&dataPtr, const size_t data) const
	Add a size_t to the sizeBuffer in the n-th util::round and return that size_t in all successive rounds.
Protected Member Functions inherited from olb::Serializable
template<typename DataType >
void	registerVar (const std::size_t iBlock, std::size_t &sizeBlock, std::size_t &currentBlock, bool *&dataPtr, const DataType &data, const size_t arrayLength=1) const
	Register primitive data types (int, double, ...) or arrays of those.
template<typename DataType >
void	registerSerializableOfConstSize (const std::size_t iBlock, std::size_t &sizeBlock, std::size_t &currentBlock, bool *&dataPtr, DataType &data, const bool loadingMode=false)
	Register Serializable object of constant size.
template<typename DataType >
void	registerSerializablesOfConstSize (const std::size_t iBlock, std::size_t &sizeBlock, std::size_t &currentBlock, bool *&dataPtr, DataType *data, const size_t arrayLength, const bool loadingMode=false)
	Register an array of Serializable objects of constant size.
Protected Attributes inherited from olb::LoadBalancer< T >
int	_size
	number of cuboids after shrink -1 in appropriate thread
std::map< int, int >	_loc
	maps global cuboid to (local) thread cuboid
std::vector< int >	_glob
	content is 0,1,2,...,_size
std::map< int, int >	_rank
	maps global cuboid number to the processing thread
std::map< int, Platform >	_platform
	maps global cuboid number to local platform
std::map< int, bool >	_doOutput
	defines if global cuboid number has state doOutput
Protected Attributes inherited from olb::BufferSerializable
std::vector< bool * >	_dataBuffer
	Data buffer for data that has to be buffered between two getBlock() iterations.
std::vector< size_t >	_sizeBuffer
	std::vector of integer buffers (e.g. for std::vector size) to be buffered for the whole iteration process

Detailed Description

template<typename T>
class olb::OrthogonalHeterogeneousLoadBalancer< T >

Load balancer for heterogeneous CPU-GPU systems.

Assigns largest cuboids to GPUs until given largeBlockFraction is reached. Remaining (small) cuboids are assigned to ranks without GPUs.

This balancer should only be used in conjunction with a heterogeneous cuboid decomposition and appropriate, system specific, CPU_SIMD+OpenMP configuration.

Definition at line 209 of file heterogeneousLoadBalancer.h.

Constructor & Destructor Documentation

OrthogonalHeterogeneousLoadBalancer()

template<typename T >

olb::OrthogonalHeterogeneousLoadBalancer< T >::OrthogonalHeterogeneousLoadBalancer ( CuboidDecomposition< T, 3 > & cGeometry, T largeBlockFraction = 0.9 )

inline

Definition at line 214 of file heterogeneousLoadBalancer.h.

                                                                                                    :
    LoadBalancer<T>(0)
  {
    OstreamManager clout(std::cout, "OrthogonalHeterogeneousLoadBalancer");
 
    std::vector<int> rankBuffer(cGeometry.size(), 0);
    std::vector<int> locBuffer(cGeometry.size(), 0);
    std::vector<std::uint8_t> platformBuffer(cGeometry.size(), static_cast<std::uint8_t>(Platform::CPU_SISD));
 
    const auto nRank = singleton::mpi().getSize();
    const auto iRank = singleton::mpi().getRank();
 
    auto& cuboids = cGeometry.cuboids();
    // Prioritize assignment of largest cuboids
    std::sort(cuboids.begin(), cuboids.end(),
              [&](const auto& lhs, const auto& rhs) {
                return lhs.getLatticeVolume() > rhs.getLatticeVolume();
              });
 
    #if defined(PARALLEL_MODE_MPI) || defined(PLATFORM_GPU_CUDA)
    int localPreferredPlatform = static_cast<int>(Platform::CPU_SIMD);
    #endif
    #ifdef PLATFORM_GPU_CUDA
    if (gpu::cuda::device::getCount() > 0) {
      localPreferredPlatform = static_cast<int>(Platform::GPU_CUDA);
    }
    #endif
    std::vector<int> preferredPlatform(cuboids.size(), -1);
    #ifdef PARALLEL_MODE_MPI
    singleton::mpi().gather(&localPreferredPlatform,  1,
                            preferredPlatform.data(), 1);
    #endif
 
    // Distribute cuboids to ranks on rank 0
    if (iRank == 0) {
      std::set<int> cpuRanks;
      std::set<int> gpuRanks;
      for (int jRank=0; jRank < singleton::mpi().getSize(); ++jRank) {
        switch (static_cast<Platform>(preferredPlatform[jRank])) {
          case Platform::CPU_SIMD:
            cpuRanks.insert(jRank);
            break;
          case Platform::GPU_CUDA:
            gpuRanks.insert(jRank);
            break;
          default:
            break;
       }
      }
 
      // Total volume for tracking targeted GPU block fraction
      std::size_t totalVolume = std::accumulate(cuboids.begin(), cuboids.end(),
                                                std::size_t{0},
                                                [](const auto& currVolume, const auto& rhs) -> std::size_t {
                                                  return currVolume + rhs.getLatticeVolume();
                                                });
 
      std::map<int,int> nLoc;
      int jRank = 0;
      int iCuboid = 0;
 
      std::size_t globalAssignedVolume = 0;
      std::vector<std::size_t> localAssignedVolume(nRank, 0);
      // Prevent GPU assignment to CPU ranks
      for (int jRank : cpuRanks) {
        localAssignedVolume[jRank] = std::numeric_limits<std::size_t>::max();
      }
 
      // Assign largest cuboids to GPUs until desired fraction is reached
      do {
        jRank = std::distance(localAssignedVolume.begin(),
                              std::min_element(localAssignedVolume.begin(),
                                               localAssignedVolume.end()));
        rankBuffer[iCuboid] = jRank;
        platformBuffer[iCuboid] = static_cast<std::uint8_t>(Platform::GPU_CUDA);
        locBuffer[iCuboid]  = nLoc[jRank]++;
        localAssignedVolume[jRank] += cuboids[iCuboid].getLatticeVolume();
        globalAssignedVolume += cuboids[iCuboid].getLatticeVolume();
        clout << iCuboid << ", " << jRank << ", assignedVolumeFraction=" << (1.*globalAssignedVolume) / totalVolume << std::endl;
        iCuboid += 1;
      } while (globalAssignedVolume < largeBlockFraction*totalVolume);
 
      for (int iRank : gpuRanks) {
        clout << "assignedVolumeOfRank[" << iRank << "]=" << localAssignedVolume[iRank] / double(totalVolume) << std::endl;
      }
 
      // Prevent CPU assignment to GPU ranks
      for (int jRank : cpuRanks) {
        localAssignedVolume[jRank] = 0;
      }
      for (int jRank : gpuRanks) {
        localAssignedVolume[jRank] = std::numeric_limits<std::size_t>::max();
      }
 
      for (int jCuboid=iCuboid; jCuboid < cGeometry.size(); ++jCuboid) {
        jRank = std::distance(localAssignedVolume.begin(),
                              std::min_element(localAssignedVolume.begin(),
                                               localAssignedVolume.end()));
        rankBuffer[jCuboid] = jRank;
        platformBuffer[jCuboid] = static_cast<std::uint8_t>(Platform::CPU_SIMD);
        locBuffer[jCuboid]  = nLoc[jRank]++;
        localAssignedVolume[jRank] += cuboids[jCuboid].getLatticeVolume();
        clout << jCuboid << ", " << jRank << std::endl;
      }
 
      for (int iRank : cpuRanks) {
        clout << "assignedVolumeOfRank[" << iRank << "]=" << localAssignedVolume[iRank] / double(totalVolume) << std::endl;
      }
    }
 
    #ifdef PARALLEL_MODE_MPI
    // Broadcast assignments to all processes
    singleton::mpi().bCast(rankBuffer.data(), rankBuffer.size());
    singleton::mpi().bCast(locBuffer.data(), locBuffer.size());
    singleton::mpi().bCast(platformBuffer.data(), platformBuffer.size());
    #endif
 
    // Update internal LoadBalancer structure to match given assignment
    for (int iCuboid=0; iCuboid < cGeometry.size(); ++iCuboid) {
      this->_rank[iCuboid] = rankBuffer[iCuboid];
      this->_loc[iCuboid] = locBuffer[iCuboid];
 
      if (rankBuffer[iCuboid] == iRank) {
        this->_glob.resize(std::max(int(this->_glob.size()), this->_loc[iCuboid]+1));
        this->_platform.resize(this->_glob.size());
 
        this->_glob[this->_loc[iCuboid]] = iCuboid;
        this->_platform[this->_loc[iCuboid]] = static_cast<Platform>(platformBuffer[iCuboid]);
 
        this->_size = this->_glob.size();
      }
    }
  }

References olb::LoadBalancer< T >::_glob, olb::LoadBalancer< T >::_loc, olb::LoadBalancer< T >::_rank, olb::LoadBalancer< T >::_size, olb::singleton::MpiManager::bCast(), olb::CPU_SIMD, olb::CPU_SISD, olb::CuboidDecomposition< T, D >::cuboids(), olb::singleton::MpiManager::gather(), olb::gpu::cuda::device::getCount(), olb::singleton::MpiManager::getRank(), olb::singleton::MpiManager::getSize(), olb::GPU_CUDA, olb::singleton::mpi(), and olb::CuboidDecomposition< T, D >::size().

Here is the call graph for this function:

Member Function Documentation

platform()

template<typename T >

Platform olb::OrthogonalHeterogeneousLoadBalancer< T >::platform ( int loc ) const

inlineoverridevirtual

Returns

target platform for processing of local cuboid

Reimplemented from olb::LoadBalancer< T >.

Definition at line 348 of file heterogeneousLoadBalancer.h.

                                            {
    return _platform[loc];
  }

References olb::LoadBalancer< T >::loc().

Here is the call graph for this function:

Here is the caller graph for this function:

setPlatform()

template<typename T >

void olb::OrthogonalHeterogeneousLoadBalancer< T >::setPlatform ( int loc, Platform platform )

inlinevirtual

Reimplemented from olb::LoadBalancer< T >.

Definition at line 352 of file heterogeneousLoadBalancer.h.

                                               {
    _platform[loc] = platform;
  }

References olb::LoadBalancer< T >::loc(), and olb::OrthogonalHeterogeneousLoadBalancer< T >::platform().

Here is the call graph for this function:

---

The documentation for this class was generated from the following file:

• src/communication/heterogeneousLoadBalancer.h