preCICE v3.1.1
Loading...
Searching...
No Matches
ProvidedPartition.cpp
Go to the documentation of this file.
1#include <algorithm>
2#include <map>
3#include <memory>
4#include <numeric>
5#include <ostream>
6#include <utility>
7#include <vector>
8
10#include "com/Extra.hpp"
11#include "com/SharedPointer.hpp"
12#include "logging/LogMacros.hpp"
13#include "m2n/M2N.hpp"
14#include "m2n/SharedPointer.hpp"
15#include "mesh/BoundingBox.hpp"
16#include "mesh/Mesh.hpp"
17#include "mesh/Vertex.hpp"
21#include "profiling/Event.hpp"
22#include "utils/IntraComm.hpp"
23#include "utils/assertion.hpp"
24
26
27namespace precice::partition {
28
34
36{
38
39 prepare();
40
41 if (_m2ns.empty())
42 return;
43
44 // Temporary globalMesh such that the primary rank also keeps his local mesh
45 mesh::Mesh globalMesh(_mesh->getName(), _mesh->getDimensions(), mesh::Mesh::MESH_ID_UNDEFINED);
46 bool hasMeshBeenGathered = false;
47
48 bool twoLevelInitAlreadyUsed = false;
49
50 for (auto &m2n : _m2ns) {
51 if (m2n->usesTwoLevelInitialization()) {
52
53 PRECICE_CHECK(not twoLevelInitAlreadyUsed, "Two-level initialization does not yet support multiple receivers of a provided mesh. "
54 "Please either switch two-level initialization off in your m2n definition, or "
55 "adapt your mesh setup such that each provided mesh is only received by maximum one "
56 "participant.");
57 twoLevelInitAlreadyUsed = true;
58
59 Event e("partition.broadcastMeshPartitions." + _mesh->getName(), profiling::Synchronize);
60
61 // communicate the total number of vertices to the other participants primary rank
63 _m2ns[0]->getPrimaryRankCommunication()->send(_mesh->getGlobalNumberOfVertices(), 0);
64 }
65
66 // the min and max of global vertex IDs of this rank's partition
67 PRECICE_ASSERT(_mesh->getVertexOffsets().size() == static_cast<decltype(_mesh->getVertexOffsets().size())>(utils::IntraComm::getSize()));
68 const int vertexOffset = _mesh->getVertexOffsets()[utils::IntraComm::getRank()];
69 const int minGlobalVertexID = vertexOffset - _mesh->nVertices();
70 const int maxGlobalVertexID = vertexOffset - 1;
71
72 // each rank sends its min/max global vertex index to connected remote ranks
73 _m2ns[0]->broadcastSend(minGlobalVertexID, *_mesh);
74 _m2ns[0]->broadcastSend(maxGlobalVertexID, *_mesh);
75
76 // each rank sends its mesh partition to connected remote ranks
77 _m2ns[0]->broadcastSendMesh(*_mesh);
78
79 } else {
80
81 if (not hasMeshBeenGathered) {
82 // Gather mesh
83 Event e("partition.gatherMesh." + _mesh->getName(), profiling::Synchronize);
85 globalMesh.addMesh(*_mesh); // Add local primary mesh to global mesh
86 }
87 PRECICE_INFO("Gather mesh {}", _mesh->getName());
91
92 for (Rank secondaryRank : utils::IntraComm::allSecondaryRanks()) {
93 com::receiveMesh(*utils::IntraComm::getCommunication(), secondaryRank, globalMesh);
94 PRECICE_DEBUG("Received sub-mesh, from secondary rank: {}, global vertexCount: {}", secondaryRank, globalMesh.nVertices());
95 }
96 }
99 }
100 hasMeshBeenGathered = true;
101 }
102
103 // Send (global) Mesh
104 PRECICE_INFO("Send global mesh {}", _mesh->getName());
105 Event e("partition.sendGlobalMesh." + _mesh->getName(), profiling::Synchronize);
106
108 PRECICE_CHECK(globalMesh.nVertices() > 0,
109 "The provided mesh \"{}\" is empty. Please set the mesh using setMeshVertex()/setMeshVertices() prior to calling initialize().",
110 globalMesh.getName());
111 com::sendMesh(*m2n->getPrimaryRankCommunication(), 0, globalMesh);
112 }
113 }
114 }
115}
116
118{
120 PRECICE_INFO("Prepare partition for mesh {}", _mesh->getName());
121 Event e("partition.prepareMesh." + _mesh->getName(), profiling::Synchronize);
122
123 int numberOfVertices = _mesh->nVertices();
124
127
128 // set globals IDs on primary rank
129 for (int i = 0; i < numberOfVertices; i++) {
130 _mesh->vertex(i).setGlobalIndex(i);
131 }
132
134 vertexOffsets[0] = numberOfVertices;
135 int globalNumberOfVertices = numberOfVertices;
136
137 // receive number of secondary vertices and fill vertex offsets
138 for (Rank secondaryRank : utils::IntraComm::allSecondaryRanks()) {
139 int numberOfSecondaryRankVertices = -1;
140 utils::IntraComm::getCommunication()->receive(numberOfSecondaryRankVertices, secondaryRank);
141 vertexOffsets[secondaryRank] = numberOfSecondaryRankVertices + vertexOffsets[secondaryRank - 1];
142 utils::IntraComm::getCommunication()->send(globalNumberOfVertices, secondaryRank);
143 globalNumberOfVertices += numberOfSecondaryRankVertices;
144 }
145 PRECICE_ASSERT(std::all_of(vertexOffsets.begin(), vertexOffsets.end(), [](auto i) { return i >= 0; }));
146 PRECICE_ASSERT(_mesh->getVertexOffsets().empty());
147 _mesh->setVertexOffsets(vertexOffsets);
148
149 // set and broadcast global number of vertices
150 _mesh->setGlobalNumberOfVertices(globalNumberOfVertices);
151 PRECICE_DEBUG("Broadcast global number of vertices: {}", globalNumberOfVertices);
152 utils::IntraComm::getCommunication()->broadcast(globalNumberOfVertices);
153
154 // broadcast vertex offsets to secondary ranks
155 PRECICE_DEBUG("My vertex offsets: {}", vertexOffsets);
156 utils::IntraComm::getCommunication()->broadcast(vertexOffsets);
157
158 // fill vertex distribution
159 if (std::any_of(_m2ns.begin(), _m2ns.end(), [](const m2n::PtrM2N &m2n) { return not m2n->usesTwoLevelInitialization(); }) && utils::IntraComm::isPrimary()) {
160 PRECICE_DEBUG("Fill vertex distribution");
161 PRECICE_ASSERT(_mesh->getVertexDistribution().empty());
163 mesh::Mesh::VertexDistribution vertexDistribution;
164 auto & localIds = vertexDistribution[0];
165 localIds.resize(vertexOffsets[0]);
166 std::iota(localIds.begin(), localIds.end(), 0);
167
168 for (Rank secondaryRank : utils::IntraComm::allSecondaryRanks()) {
169 // This always creates an entry for each secondary rank
170 auto &secondaryIds = vertexDistribution[secondaryRank];
171 for (int i = vertexOffsets[secondaryRank - 1]; i < vertexOffsets[secondaryRank]; i++) {
172 secondaryIds.push_back(i);
173 }
174 }
175 PRECICE_ASSERT(vertexDistribution.size() == static_cast<mesh::Mesh::VertexDistribution::size_type>(utils::IntraComm::getSize()));
176 _mesh->setVertexDistribution(std::move(vertexDistribution));
177 }
178 } else if (utils::IntraComm::isSecondary()) {
179
180 // send number of own vertices
181 PRECICE_DEBUG("Send number of vertices: {}", numberOfVertices);
182 utils::IntraComm::getCommunication()->send(numberOfVertices, 0);
183
184 // set global IDs
185 int globalVertexCounter = -1;
186 utils::IntraComm::getCommunication()->receive(globalVertexCounter, 0);
187 PRECICE_DEBUG("Set global vertex indices");
188 for (int i = 0; i < numberOfVertices; i++) {
189 _mesh->vertex(i).setGlobalIndex(globalVertexCounter + i);
190 }
191
192 // set global number of vertices
193 int globalNumberOfVertices = -1;
194 utils::IntraComm::getCommunication()->broadcast(globalNumberOfVertices, 0);
195 PRECICE_ASSERT(globalNumberOfVertices != -1);
196 _mesh->setGlobalNumberOfVertices(globalNumberOfVertices);
197
198 // receive set vertex offsets
199 mesh::Mesh::VertexOffsets vertexOffsets;
200 utils::IntraComm::getCommunication()->broadcast(vertexOffsets, 0);
201 PRECICE_DEBUG("My vertex offsets: {}", vertexOffsets);
202 PRECICE_ASSERT(_mesh->getVertexOffsets().empty());
203 _mesh->setVertexOffsets(std::move(vertexOffsets));
204
205 } else {
206
207 // The only rank of the participant contains all vertices
208 PRECICE_ASSERT(_mesh->getVertexDistribution().empty());
209 _mesh->setVertexDistribution([&] {
210 mesh::Mesh::VertexDistribution vertexDistribution;
211 for (int i = 0; i < numberOfVertices; i++) {
212 vertexDistribution[0].push_back(i);
213 _mesh->vertex(i).setGlobalIndex(i);
214 }
215 return vertexDistribution;
216 }());
217 PRECICE_ASSERT(_mesh->getVertexOffsets().empty());
218 _mesh->setVertexOffsets({numberOfVertices});
219 _mesh->setGlobalNumberOfVertices(numberOfVertices);
220 }
221
222 PRECICE_DEBUG("Set owner information");
223 for (mesh::Vertex &v : _mesh->vertices()) {
224 v.setOwner(true);
225 }
226}
227
229{
231 for (const auto &m2n : _m2ns) {
232 if (m2n->usesTwoLevelInitialization()) {
233 // @todo this will probably not work for more than one m2n
234 PRECICE_ASSERT(_m2ns.size() <= 1);
235 // receive communication map from all remote connected ranks
236 m2n->gatherAllCommunicationMap(_mesh->getCommunicationMap(), *_mesh);
237 }
238 }
239}
240
242{
244 if (_m2ns.empty())
245 return;
246
247 _mesh->clearPartitioning();
248
249 //@todo coupling mode
250
251 //@todo treatment of multiple m2ns
252 if (not _m2ns[0]->usesTwoLevelInitialization())
253 return;
254
255 // each secondary rank sends its bb to the primary rank
256 if (utils::IntraComm::isSecondary()) { // secondary
257 PRECICE_ASSERT(_mesh->getBoundingBox().getDimension() == _mesh->getDimensions(), "The boundingbox of the local mesh is invalid!");
259 } else { // Primary
260
263
264 // to store the collection of bounding boxes
266 mesh::BoundingBox bb(_mesh->getDimensions());
267 bbm.emplace(0, _mesh->getBoundingBox());
268 PRECICE_ASSERT(!bbm.empty(), "The bounding box of the local mesh is invalid!");
269
270 // primary rank receives bbs from secondary ranks and stores them in bbm
271 for (Rank secondaryRank : utils::IntraComm::allSecondaryRanks()) {
272 // initialize bbm
273 bbm.emplace(secondaryRank, bb);
274 com::receiveBoundingBox(*utils::IntraComm::getCommunication(), secondaryRank, bbm.at(secondaryRank));
275 }
276
277 // primary rank sends number of ranks and bbm to the other primary rank
278 _m2ns[0]->getPrimaryRankCommunication()->send(utils::IntraComm::getSize(), 0);
279 com::sendBoundingBoxMap(*_m2ns[0]->getPrimaryRankCommunication(), 0, bbm);
280 }
281
282 // size of the feedbackmap
283 int remoteConnectionMapSize = 0;
284
286
287 // primary rank receives feedback map (map of other participant ranks -> connected ranks at this participant)
288 // from other participants primary rank
289 std::vector<Rank> connectedRanksList = _m2ns[0]->getPrimaryRankCommunication()->receiveRange(0, com::AsVectorTag<Rank>{});
290 remoteConnectionMapSize = connectedRanksList.size();
291
292 mesh::Mesh::CommunicationMap remoteConnectionMap;
293 for (auto &rank : connectedRanksList) {
294 remoteConnectionMap[rank] = {-1};
295 }
296 if (remoteConnectionMapSize != 0) {
297 com::receiveConnectionMap(*_m2ns[0]->getPrimaryRankCommunication(), 0, remoteConnectionMap);
298 }
299
300 // broadcast the received feedbackMap
301 utils::IntraComm::getCommunication()->broadcast(connectedRanksList);
302 if (remoteConnectionMapSize != 0) {
304 }
305
306 // primary rank checks which ranks are connected to it
307 PRECICE_ASSERT(_mesh->getConnectedRanks().empty());
308 _mesh->setConnectedRanks([&] {
309 std::vector<Rank> ranks;
310 for (const auto &remoteRank : remoteConnectionMap) {
311 for (const auto &includedRank : remoteRank.second) {
312 if (utils::IntraComm::getRank() == includedRank) {
313 ranks.push_back(remoteRank.first);
314 }
315 }
316 }
317 return ranks;
318 }());
319
320 } else { // Secondary rank
321 std::vector<Rank> connectedRanksList;
322 utils::IntraComm::getCommunication()->broadcast(connectedRanksList, 0);
323
324 mesh::Mesh::CommunicationMap remoteConnectionMap;
325 if (!connectedRanksList.empty()) {
326 for (Rank rank : connectedRanksList) {
327 remoteConnectionMap[rank] = {-1};
328 }
330 }
331
332 PRECICE_ASSERT(_mesh->getConnectedRanks().empty());
333 _mesh->setConnectedRanks([&] {
334 std::vector<Rank> ranks;
335 for (const auto &remoteRank : remoteConnectionMap) {
336 for (int includedRanks : remoteRank.second) {
337 if (utils::IntraComm::getRank() == includedRanks) {
338 ranks.push_back(remoteRank.first);
339 }
340 }
341 }
342 return ranks;
343 }());
344 }
345}
346
347} // namespace precice::partition
#define PRECICE_DEBUG(...)
Definition LogMacros.hpp:64
#define PRECICE_TRACE(...)
Definition LogMacros.hpp:95
#define PRECICE_INFO(...)
Definition LogMacros.hpp:13
#define PRECICE_CHECK(check,...)
Definition LogMacros.hpp:35
T all_of(T... args)
#define PRECICE_ASSERT(...)
Definition assertion.hpp:87
T at(T... args)
T begin(T... args)
An axis-aligned bounding box around a (partition of a) mesh.
Container and creator for meshes.
Definition Mesh.hpp:39
void addMesh(Mesh &deltaMesh)
Definition Mesh.cpp:309
const std::string & getName() const
Returns the name of the mesh, as set in the config file.
Definition Mesh.cpp:218
static constexpr MeshID MESH_ID_UNDEFINED
Use if the id of the mesh is not necessary.
Definition Mesh.hpp:58
std::size_t nVertices() const
Returns the number of vertices.
Definition Mesh.cpp:63
Vertex of a mesh.
Definition Vertex.hpp:16
Abstract base class for partitions.
Definition Partition.hpp:29
std::vector< m2n::PtrM2N > _m2ns
m2n connection to each connected participant
Definition Partition.hpp:70
void compute() override
All distribution data structures are set up.
void communicate() override
The mesh is gathered and sent to another participant (if required)
void compareBoundingBoxes() override
Intersections between bounding boxes around each rank are computed.
static int getSize()
Number of ranks. This includes ranks from both participants, e.g. minimal size is 2.
Definition IntraComm.cpp:47
static Rank getRank()
Current rank.
Definition IntraComm.cpp:42
static bool isPrimary()
True if this process is running the primary rank.
Definition IntraComm.cpp:52
static auto allSecondaryRanks()
Returns an iterable range over salve ranks [1, _size)
Definition IntraComm.hpp:37
static bool isSecondary()
True if this process is running a secondary rank.
Definition IntraComm.cpp:57
static com::PtrCommunication & getCommunication()
Intra-participant communication.
Definition IntraComm.hpp:31
T emplace(T... args)
T empty(T... args)
T end(T... args)
T iota(T... args)
void sendMesh(Communication &communication, int rankReceiver, const mesh::Mesh &mesh)
Definition Extra.cpp:8
void sendBoundingBox(Communication &communication, int rankReceiver, const mesh::BoundingBox &bb)
Definition Extra.cpp:48
void receiveConnectionMap(Communication &communication, int rankSender, mesh::Mesh::ConnectionMap &cm)
Definition Extra.cpp:33
void receiveMesh(Communication &communication, int rankSender, mesh::Mesh &mesh)
Definition Extra.cpp:13
void broadcastSendConnectionMap(Communication &communication, const mesh::Mesh::ConnectionMap &cm)
Definition Extra.cpp:38
void sendBoundingBoxMap(Communication &communication, int rankReceiver, const mesh::Mesh::BoundingBoxMap &bbm)
Definition Extra.cpp:58
void broadcastReceiveConnectionMap(Communication &communication, mesh::Mesh::ConnectionMap &cm)
Definition Extra.cpp:43
void receiveBoundingBox(Communication &communication, int rankSender, mesh::BoundingBox &bb)
Definition Extra.cpp:53
contains the partitioning of distributed meshes.
Definition Partition.cpp:5
static constexpr SynchronizeTag Synchronize
Convenience instance of the SynchronizeTag.
Definition Event.hpp:21
int Rank
Definition Types.hpp:37
STL namespace.
T push_back(T... args)
T size(T... args)