OpenLB 1.7
Loading...
Searching...
No Matches
Classes | Public Member Functions | List of all members
olb::ConcreteBlockCommunicator< ConcreteBlockLattice< T, DESCRIPTOR, Platform::GPU_CUDA > > Class Template Referencefinal

#include <communicator.h>

+ Inheritance diagram for olb::ConcreteBlockCommunicator< ConcreteBlockLattice< T, DESCRIPTOR, Platform::GPU_CUDA > >:
+ Collaboration diagram for olb::ConcreteBlockCommunicator< ConcreteBlockLattice< T, DESCRIPTOR, Platform::GPU_CUDA > >:

Classes

struct  CopyTask
 Wrapper for a local plain-copy block communication request. More...
 
class  HomogeneousCopyTask
 Wrapper for a local plain-copy block communication request. More...
 
class  RecvTask
 Wrapper for a non-blocking block propagation receive request. More...
 
class  SendTask
 Wrapper for a non-blocking block propagation send request. More...
 

Public Member Functions

 ConcreteBlockCommunicator (SuperLattice< T, DESCRIPTOR > &super, LoadBalancer< T > &loadBalancer, SuperCommunicationTagCoordinator< T > &tagCoordinator, MPI_Comm comm, int iC, const BlockCommunicationNeighborhood< T, DESCRIPTOR::d > &neighborhood)
 
 ~ConcreteBlockCommunicator ()
 
void receive () override
 
void send () override
 
void unpack () override
 
void wait () override
 
- Public Member Functions inherited from olb::BlockCommunicator
virtual ~BlockCommunicator ()
 

Detailed Description

template<typename T, typename DESCRIPTOR>
class olb::ConcreteBlockCommunicator< ConcreteBlockLattice< T, DESCRIPTOR, Platform::GPU_CUDA > >

Definition at line 96 of file communicator.h.

Constructor & Destructor Documentation

◆ ConcreteBlockCommunicator()

template<typename T , typename DESCRIPTOR >
olb::ConcreteBlockCommunicator< ConcreteBlockLattice< T, DESCRIPTOR, Platform::GPU_CUDA > >::ConcreteBlockCommunicator ( SuperLattice< T, DESCRIPTOR > & super,
LoadBalancer< T > & loadBalancer,
SuperCommunicationTagCoordinator< T > & tagCoordinator,
MPI_Comm comm,
int iC,
const BlockCommunicationNeighborhood< T, DESCRIPTOR::d > & neighborhood )

Definition at line 743 of file communicator.hh.

751 :
752 _iC(iC)
753#ifdef PARALLEL_MODE_MPI
754, _mpiCommunicator(comm)
755#endif
756{
757#ifdef PARALLEL_MODE_MPI
758 neighborhood.forNeighbors([&](int remoteC) {
759 if (loadBalancer.isLocal(remoteC)) {
760 const Platform remotePlatform = loadBalancer.platform(loadBalancer.loc(remoteC));
761 if (!neighborhood.getCellsInboundFrom(remoteC).empty()) {
762 switch (remotePlatform) {
763 case Platform::GPU_CUDA:
764 // Use manual copy for local GPU-GPU communication due to better performance
765 _copyTasks.emplace_back(new HomogeneousCopyTask(
766 neighborhood.getFieldsCommonWith(remoteC),
767 neighborhood.getCellsInboundFrom(remoteC), super.template getBlock<ConcreteBlockLattice<T,DESCRIPTOR,Platform::GPU_CUDA>>(_iC),
768 neighborhood.getCellsRequestedFrom(remoteC), super.template getBlock<ConcreteBlockLattice<T,DESCRIPTOR,Platform::GPU_CUDA>>(loadBalancer.loc(remoteC))));
769 break;
770 case Platform::CPU_SIMD:
771 // Use manual copy for local GPU-CPU communication due to better performance
772 _copyTasks.emplace_back(new HeterogeneousCopyTask<T,DESCRIPTOR,Platform::CPU_SIMD,Platform::GPU_CUDA>(
773 neighborhood.getFieldsCommonWith(remoteC),
774 neighborhood.getCellsInboundFrom(remoteC), super.template getBlock<ConcreteBlockLattice<T,DESCRIPTOR,Platform::GPU_CUDA>>(_iC),
775 neighborhood.getCellsRequestedFrom(remoteC), super.template getBlock<ConcreteBlockLattice<T,DESCRIPTOR,Platform::CPU_SIMD>>(loadBalancer.loc(remoteC))));
776 break;
777 case Platform::CPU_SISD:
778 // Use manual copy for local GPU-CPU communication due to better performance
779 _copyTasks.emplace_back(new HeterogeneousCopyTask<T,DESCRIPTOR,Platform::CPU_SISD,Platform::GPU_CUDA>(
780 neighborhood.getFieldsCommonWith(remoteC),
781 neighborhood.getCellsInboundFrom(remoteC), super.template getBlock<ConcreteBlockLattice<T,DESCRIPTOR,Platform::GPU_CUDA>>(_iC),
782 neighborhood.getCellsRequestedFrom(remoteC), super.template getBlock<ConcreteBlockLattice<T,DESCRIPTOR,Platform::CPU_SISD>>(loadBalancer.loc(remoteC))));
783 break;
784 default:
785 throw std::runtime_error("Invalid remote PLATFORM");
786 }
787 }
788 } else {
789 // Handling of non-local GPU-GPU and GPU-^GPU communication in general
790 if (!neighborhood.getCellsOutboundTo(remoteC).empty()) {
791 _sendTasks.emplace_back(std::make_unique<SendTask>(
792 _mpiCommunicator, tagCoordinator.get(loadBalancer.glob(_iC), remoteC),
793 loadBalancer.rank(remoteC),
794 neighborhood.getFieldsCommonWith(remoteC),
795 neighborhood.getCellsOutboundTo(remoteC),
796 super.template getBlock<ConcreteBlockLattice<T,DESCRIPTOR,Platform::GPU_CUDA>>(_iC)));
797 }
798 if (!neighborhood.getCellsInboundFrom(remoteC).empty()) {
799 _recvTasks.emplace_back(std::make_unique<RecvTask>(
800 _mpiCommunicator, tagCoordinator.get(remoteC, loadBalancer.glob(_iC)),
801 loadBalancer.rank(remoteC),
802 neighborhood.getFieldsCommonWith(remoteC),
803 neighborhood.getCellsInboundFrom(remoteC),
804 super.template getBlock<ConcreteBlockLattice<T,DESCRIPTOR,Platform::GPU_CUDA>>(_iC)));
805 }
806 }
807 });
808
809#else // not using PARALLEL_MODE_MPI
810 neighborhood.forNeighbors([&](int localC) {
811 if (!neighborhood.getCellsInboundFrom(localC).empty()) {
812 _copyTasks.emplace_back(new HomogeneousCopyTask(
813 neighborhood.getFieldsCommonWith(localC),
814 neighborhood.getCellsInboundFrom(localC), super.template getBlock<ConcreteBlockLattice<T,DESCRIPTOR,Platform::GPU_CUDA>>(_iC),
815 neighborhood.getCellsRequestedFrom(localC), super.template getBlock<ConcreteBlockLattice<T,DESCRIPTOR,Platform::GPU_CUDA>>(loadBalancer.loc(localC))));
816 }
817 });
818#endif
819}

References olb::BlockCommunicationNeighborhood< T, D >::forNeighbors(), olb::BlockCommunicationNeighborhood< T, D >::getCellsInboundFrom(), olb::BlockCommunicationNeighborhood< T, D >::getCellsOutboundTo(), and olb::LoadBalancer< T >::isLocal().

+ Here is the call graph for this function:

◆ ~ConcreteBlockCommunicator()

template<typename T , typename DESCRIPTOR >
olb::ConcreteBlockCommunicator< ConcreteBlockLattice< T, DESCRIPTOR, Platform::GPU_CUDA > >::~ConcreteBlockCommunicator ( )

Definition at line 822 of file communicator.hh.

823{ }

Member Function Documentation

◆ receive()

template<typename T , typename DESCRIPTOR >
void olb::ConcreteBlockCommunicator< ConcreteBlockLattice< T, DESCRIPTOR, Platform::GPU_CUDA > >::receive ( )
overridevirtual

Implements olb::BlockCommunicator.

Definition at line 828 of file communicator.hh.

829{
830 for (auto& task : _recvTasks) {
831 task->receive();
832 }
833}

References olb::ConcreteBlockCommunicator< BLOCK >::receive().

+ Here is the call graph for this function:

◆ send()

template<typename T , typename DESCRIPTOR >
void olb::ConcreteBlockCommunicator< ConcreteBlockLattice< T, DESCRIPTOR, Platform::GPU_CUDA > >::send ( )
overridevirtual

Implements olb::BlockCommunicator.

Definition at line 836 of file communicator.hh.

837{
838 for (auto& task : _sendTasks) {
839 task->prepare();
840 }
841 for (auto& task : _sendTasks) {
842 task->send();
843 }
844 for (auto& task : _copyTasks) {
845 task->copy();
846 }
847}

References olb::ConcreteBlockCommunicator< BLOCK >::send().

+ Here is the call graph for this function:

◆ unpack()

template<typename T , typename DESCRIPTOR >
void olb::ConcreteBlockCommunicator< ConcreteBlockLattice< T, DESCRIPTOR, Platform::GPU_CUDA > >::unpack ( )
overridevirtual

Implements olb::BlockCommunicator.

Definition at line 850 of file communicator.hh.

851{
852 std::set<typename RecvTask::ref> pending(_recvTasks.begin(), _recvTasks.end());
853 while (!pending.empty()) {
854 auto task_iterator = pending.begin();
855 while (task_iterator != pending.end()) {
856 auto& task = *task_iterator;
857 if (task->isDone()) {
858 task->unpack();
859 task_iterator = pending.erase(task_iterator);
860 }
861 else {
862 ++task_iterator;
863 }
864 }
865 }
866}

◆ wait()

template<typename T , typename DESCRIPTOR >
void olb::ConcreteBlockCommunicator< ConcreteBlockLattice< T, DESCRIPTOR, Platform::GPU_CUDA > >::wait ( )
overridevirtual

Implements olb::BlockCommunicator.

Definition at line 869 of file communicator.hh.

870{
871 for (auto& task : _copyTasks) {
872 task->wait();
873 }
874 for (auto& task : _recvTasks) {
875 task->wait();
876 }
877 for (auto& task : _sendTasks) {
878 task->wait();
879 }
880}

References olb::ConcreteBlockCommunicator< BLOCK >::wait().

+ Here is the call graph for this function:

The documentation for this class was generated from the following files: