OpenLB 1.7
Loading...
Searching...
No Matches
superCommunicator.hh
Go to the documentation of this file.
1/* This file is part of the OpenLB library
2 *
3 * Copyright (C) 2020 Adrian Kummerlaender
4 * E-mail contact: info@openlb.net
5 * The most recent release of OpenLB can be downloaded at
6 * <http://www.openlb.net/>
7 *
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License
10 * as published by the Free Software Foundation; either version 2
11 * of the License, or (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public
19 * License along with this program; if not, write to the Free
20 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
21 * Boston, MA 02110-1301, USA.
22*/
23
24#ifndef SUPER_COMMUNICATOR_HH
25#define SUPER_COMMUNICATOR_HH
26
27#include "superCommunicator.h"
30
31#include <algorithm>
32#include <stdexcept>
33
34namespace olb {
35
36// *INDENT-OFF*
37
38template <typename T, typename SUPER>
40 SUPER& super):
41 _super(super)
42#ifdef PARALLEL_MODE_MPI
43, _tagCoordinator(super.getLoadBalancer())
44, _enabled(false)
45#endif
46{
47#ifdef PARALLEL_MODE_MPI
48 if (MPI_Comm_dup(MPI_COMM_WORLD, &_neighborhoodComm) != MPI_SUCCESS) {
49 throw std::runtime_error("Unable to duplicate MPI communicator");
50 }
51 if (MPI_Comm_dup(MPI_COMM_WORLD, &_communicatorComm) != MPI_SUCCESS) {
52 throw std::runtime_error("Unable to duplicate MPI communicator");
53 }
54#endif
55
56 auto& cuboidGeometry = _super.getCuboidGeometry();
57 auto& load = _super.getLoadBalancer();
58
59 for (int iC = 0; iC < load.size(); ++iC) {
60 _blockNeighborhoods.emplace_back(
61 std::make_unique<BlockCommunicationNeighborhood<T,SUPER::d>>( cuboidGeometry
62 , load, load.glob(iC)
63 , _super.getOverlap()
64#ifdef PARALLEL_MODE_MPI
65 , _neighborhoodComm
66#endif
67 ));
68 }
69}
70
71template <typename T, typename SUPER>
73{
74#ifdef PARALLEL_MODE_MPI
75 MPI_Comm_free(&_neighborhoodComm);
76 MPI_Comm_free(&_communicatorComm);
77#endif
78}
80template <typename T, typename SUPER>
82{
83 auto& load = _super.getLoadBalancer();
84
85 for (int iC = 0; iC < load.size(); ++iC) {
86 _blockNeighborhoods[iC]->maintain();
87 }
88
89 for (int iC = 0; iC < load.size(); ++iC) {
90 for (std::type_index field : _fieldsRequested) {
91 _blockNeighborhoods[iC]->setFieldAvailability(
92 field, _super.getBlock(iC).hasCommunicatable(field));
93 }
94 }
95
96#ifdef PARALLEL_MODE_MPI
97 _tagCoordinator.template coordinate<SUPER::d>(_blockNeighborhoods);
98
99 for (int iC = 0; iC < load.size(); ++iC) {
100 _blockNeighborhoods[iC]->send(_tagCoordinator);
101 }
102 for (int iC = 0; iC < load.size(); ++iC) {
103 _blockNeighborhoods[iC]->receive(_tagCoordinator);
105 for (int iC = 0; iC < load.size(); ++iC) {
106 _blockNeighborhoods[iC]->wait();
107 }
109#else // not using PARALLEL_MODE_MPI
110 for (int iC = 0; iC < load.size(); ++iC) {
111 _blockNeighborhoods[iC]->forNeighbors([&](int localC) {
112 _blockNeighborhoods[iC]->setFieldsAvailability(localC, _super.getBlock(load.loc(localC)));
113 });
115#endif
116
117 _blockCommunicators.clear();
118 _blockCommunicators.resize(load.size());
119 for (int iC = 0; iC < load.size(); ++iC) {
120 auto* block = &_super.getBlock(iC);
121 _blockCommunicators[iC] = callUsingConcretePlatform<typename SUPER::block_t>(
122 block->getPlatform(),
123 block,
124 [&](auto* concreteBlock) -> std::unique_ptr<BlockCommunicator> {
125 return std::make_unique<ConcreteBlockCommunicator<std::remove_reference_t<decltype(*concreteBlock)>>>(
126 _super,
127 load,
128#ifdef PARALLEL_MODE_MPI
129 _tagCoordinator,
130 _communicatorComm,
131#endif
132 iC,
133 *_blockNeighborhoods[iC]);
134 });
135 }
136
137#ifdef PARALLEL_MODE_MPI
138 for (int iC = 0; iC < load.size(); ++iC) {
139 _blockNeighborhoods[iC]->forNeighbors([&](int remoteC) {
140 _remoteCuboidNeighborhood.emplace(remoteC);
141 });
142 }
143#endif
144
145 _ready = true;
146}
147
148template <typename T, typename SUPER>
150{
151 _blockNeighborhoods[latticeR[0]]->requestCell(latticeR.data()+1);
152 _ready = false;
153 _enabled = true;
154}
155
156template <typename T, typename SUPER>
158{
159 auto& load = _super.getLoadBalancer();
160 for (int iC = 0; iC < load.size(); ++iC) {
161 _blockNeighborhoods[iC]->requestOverlap(width);
162 }
163 _ready = false;
164 _enabled = true;
165}
166
167template <typename T, typename SUPER>
169{
170 auto& load = _super.getLoadBalancer();
171 for (int iC = 0; iC < load.size(); ++iC) {
172 _blockNeighborhoods[iC]->requestOverlap(width, indicatorF->getBlockIndicatorF(iC));
173 }
174 _ready = false;
175 _enabled = true;
176}
177
178template <typename T, typename SUPER>
180{
181 auto& load = _super.getLoadBalancer();
182 for (int iC = 0; iC < load.size(); ++iC) {
183 _blockNeighborhoods[iC]->clearRequestedCells();
184 }
185 _ready = false;
186 _enabled = false;
187}
188
189template <typename T, typename SUPER>
191{
192 if (!_enabled) {
193 return;
194 }
195 if (!_ready) {
196 throw std::logic_error("Requests must be re-exchanged after any changes");
197 }
198
199 auto& load = _super.getLoadBalancer();
200#ifdef PARALLEL_MODE_MPI
201 for (int iC = 0; iC < load.size(); ++iC) {
202 _blockCommunicators[iC]->receive();
203 }
204 for (int iC = 0; iC < load.size(); ++iC) {
205 _blockCommunicators[iC]->send();
206 }
207 for (int iC = 0; iC < load.size(); ++iC) {
208 _blockCommunicators[iC]->unpack();
209 }
210 for (int iC = 0; iC < load.size(); ++iC) {
211 _blockCommunicators[iC]->wait();
212 }
213#else // not using PARALLEL_MODE_MPI
214 for (int iC = 0; iC < load.size(); ++iC) {
215 _blockCommunicators[iC]->copy();
216 }
217#endif
218}
219
220template <typename T, typename SUPER>
222{
223 if (_ready) {
224 return _remoteCuboidNeighborhood;
225 } else {
226 throw std::logic_error("Requests must be re-exchanged after any changes");
227 }
228}
229
230// *INDENT-ON*
231
232}
233
234#endif
Configurable overlap communication neighborhood of a block.
Smart pointer for managing the various ways of passing functors around.
Definition functorPtr.h:60
void communicate()
Perform communication.
void requestOverlap(int width)
Request all cells in overlap of width for communication.
const std::set< int > & getRemoteCuboids() const
Returns set of non-local neighborhood cuboid indices.
void clearRequestedCells()
Remove all requested cells.
void exchangeRequests()
Exchange requests between processes.
void requestCell(LatticeR< SUPER::d+1 > latticeR)
Request single cell in the padding area for communication.
Plain old scalar vector.
Definition vector.h:47
constexpr const T * data() const any_platform
Definition vector.h:161
Top level namespace for all of OpenLB.
std::conditional_t< D==2, SuperIndicatorF2D< T >, SuperIndicatorF3D< T > > SuperIndicatorF
Definition aliases.h:208