24#ifndef SUPER_COMMUNICATOR_HH
25#define SUPER_COMMUNICATOR_HH
38template <
typename T,
typename SUPER>
42#ifdef PARALLEL_MODE_MPI
43, _tagCoordinator(super.getLoadBalancer())
47#ifdef PARALLEL_MODE_MPI
48 if (MPI_Comm_dup(MPI_COMM_WORLD, &_neighborhoodComm) != MPI_SUCCESS) {
49 throw std::runtime_error(
"Unable to duplicate MPI communicator");
51 if (MPI_Comm_dup(MPI_COMM_WORLD, &_communicatorComm) != MPI_SUCCESS) {
52 throw std::runtime_error(
"Unable to duplicate MPI communicator");
56 auto& cuboidGeometry = _super.getCuboidGeometry();
57 auto& load = _super.getLoadBalancer();
59 for (
int iC = 0; iC < load.size(); ++iC) {
60 _blockNeighborhoods.emplace_back(
64#ifdef PARALLEL_MODE_MPI
71template <
typename T,
typename SUPER>
74#ifdef PARALLEL_MODE_MPI
75 MPI_Comm_free(&_neighborhoodComm);
76 MPI_Comm_free(&_communicatorComm);
80template <
typename T,
typename SUPER>
83 auto& load = _super.getLoadBalancer();
85 for (
int iC = 0; iC < load.size(); ++iC) {
86 _blockNeighborhoods[iC]->maintain();
89 for (
int iC = 0; iC < load.size(); ++iC) {
90 for (std::type_index field : _fieldsRequested) {
91 _blockNeighborhoods[iC]->setFieldAvailability(
92 field, _super.getBlock(iC).hasCommunicatable(field));
96#ifdef PARALLEL_MODE_MPI
97 _tagCoordinator.template coordinate<SUPER::d>(_blockNeighborhoods);
99 for (
int iC = 0; iC < load.size(); ++iC) {
100 _blockNeighborhoods[iC]->send(_tagCoordinator);
102 for (
int iC = 0; iC < load.size(); ++iC) {
103 _blockNeighborhoods[iC]->receive(_tagCoordinator);
105 for (
int iC = 0; iC < load.size(); ++iC) {
106 _blockNeighborhoods[iC]->wait();
110 for (
int iC = 0; iC < load.size(); ++iC) {
111 _blockNeighborhoods[iC]->forNeighbors([&](
int localC) {
112 _blockNeighborhoods[iC]->setFieldsAvailability(localC, _super.getBlock(load.loc(localC)));
117 _blockCommunicators.clear();
118 _blockCommunicators.resize(load.size());
119 for (
int iC = 0; iC < load.size(); ++iC) {
120 auto* block = &_super.getBlock(iC);
121 _blockCommunicators[iC] = callUsingConcretePlatform<typename SUPER::block_t>(
122 block->getPlatform(),
124 [&](
auto* concreteBlock) -> std::unique_ptr<BlockCommunicator> {
125 return std::make_unique<ConcreteBlockCommunicator<std::remove_reference_t<decltype(*concreteBlock)>>>(
128#ifdef PARALLEL_MODE_MPI
133 *_blockNeighborhoods[iC]);
137#ifdef PARALLEL_MODE_MPI
138 for (
int iC = 0; iC < load.size(); ++iC) {
139 _blockNeighborhoods[iC]->forNeighbors([&](
int remoteC) {
140 _remoteCuboidNeighborhood.emplace(remoteC);
148template <
typename T,
typename SUPER>
151 _blockNeighborhoods[latticeR[0]]->requestCell(latticeR.
data()+1);
156template <
typename T,
typename SUPER>
159 auto& load = _super.getLoadBalancer();
160 for (
int iC = 0; iC < load.size(); ++iC) {
161 _blockNeighborhoods[iC]->requestOverlap(width);
167template <
typename T,
typename SUPER>
170 auto& load = _super.getLoadBalancer();
171 for (
int iC = 0; iC < load.size(); ++iC) {
172 _blockNeighborhoods[iC]->requestOverlap(width, indicatorF->getBlockIndicatorF(iC));
178template <
typename T,
typename SUPER>
181 auto& load = _super.getLoadBalancer();
182 for (
int iC = 0; iC < load.size(); ++iC) {
183 _blockNeighborhoods[iC]->clearRequestedCells();
189template <
typename T,
typename SUPER>
196 throw std::logic_error(
"Requests must be re-exchanged after any changes");
199 auto& load = _super.getLoadBalancer();
200#ifdef PARALLEL_MODE_MPI
201 for (
int iC = 0; iC < load.size(); ++iC) {
202 _blockCommunicators[iC]->receive();
204 for (
int iC = 0; iC < load.size(); ++iC) {
205 _blockCommunicators[iC]->send();
207 for (
int iC = 0; iC < load.size(); ++iC) {
208 _blockCommunicators[iC]->unpack();
210 for (
int iC = 0; iC < load.size(); ++iC) {
211 _blockCommunicators[iC]->wait();
214 for (
int iC = 0; iC < load.size(); ++iC) {
215 _blockCommunicators[iC]->copy();
220template <
typename T,
typename SUPER>
224 return _remoteCuboidNeighborhood;
226 throw std::logic_error(
"Requests must be re-exchanged after any changes");
Configurable overlap communication neighborhood of a block.
Smart pointer for managing the various ways of passing functors around.
void communicate()
Perform communication.
SuperCommunicator(SUPER &super)
void requestOverlap(int width)
Request all cells in overlap of width for communication.
const std::set< int > & getRemoteCuboids() const
Returns set of non-local neighborhood cuboid indices.
void clearRequestedCells()
Remove all requested cells.
void exchangeRequests()
Exchange requests between processes.
void requestCell(LatticeR< SUPER::d+1 > latticeR)
Request single cell in the padding area for communication.
constexpr const T * data() const any_platform
Top level namespace for all of OpenLB.
std::conditional_t< D==2, SuperIndicatorF2D< T >, SuperIndicatorF3D< T > > SuperIndicatorF