preCICE v3.1.1
Loading...
Searching...
No Matches
GatherScatterCommunication.cpp
Go to the documentation of this file.
1#include <algorithm>
2#include <cstddef>
3#include <map>
4#include <memory>
5#include <ostream>
6#include <utility>
7
10#include "logging/LogMacros.hpp"
12#include "mesh/Mesh.hpp"
14#include "utils/IntraComm.hpp"
15#include "utils/algorithm.hpp"
16#include "utils/assertion.hpp"
17
18namespace precice::m2n {
21 mesh::PtrMesh mesh)
22 : DistributedCommunication(std::move(mesh)),
23 _com(std::move(com)),
24 _isConnected(false)
25{
26}
27
34
39
41 const std::string &acceptorName,
42 const std::string &requesterName)
43{
44 PRECICE_TRACE(acceptorName, requesterName);
46 _isConnected = true;
47}
48
50 const std::string &acceptorName,
51 const std::string &requesterName)
52{
53 PRECICE_TRACE(acceptorName, requesterName);
55 _isConnected = true;
56}
57
64
65namespace {
66template <class Indices, class Src, class Dst, class Size>
67void add_to_indirect_blocks(
68 const Src & src,
69 const Indices &indices,
70 Size blockSize,
71 Dst & dst)
72{
73 for (size_t i = 0; i < indices.size(); ++i) {
74 auto srcfirst = blockSize * i;
75 auto dstfirst = blockSize * indices[i];
76 PRECICE_ASSERT(srcfirst >= 0 && static_cast<size_t>(srcfirst + blockSize) <= src.size(), srcfirst, blockSize, src.size());
77 PRECICE_ASSERT(dstfirst >= 0 && static_cast<size_t>(dstfirst + blockSize) <= dst.size(), dstfirst, blockSize, dst.size());
78 utils::add_n(&src[srcfirst], blockSize, &dst[dstfirst]);
79 }
80}
81
82template <class Indices, class Src, class Dst, class Size>
83void copy_from_indirect_blocks(
84 const Src & src,
85 const Indices &indices,
86 Size blockSize,
87 Dst & dst)
88{
89 for (size_t i = 0; i < indices.size(); ++i) {
90 auto srcfirst = blockSize * indices[i];
91 auto dstfirst = blockSize * i;
92 PRECICE_ASSERT(srcfirst >= 0 && static_cast<size_t>(srcfirst + blockSize) <= src.size(), srcfirst, blockSize, src.size());
93 PRECICE_ASSERT(dstfirst >= 0 && static_cast<size_t>(dstfirst + blockSize) <= dst.size(), dstfirst, blockSize, dst.size());
94 std::copy_n(&src[srcfirst], blockSize, &dst[dstfirst]);
95 }
96}
97
98} // namespace
99
101{
102 PRECICE_TRACE(itemsToSend.size());
103
104 // Gather data on secondary ranks
105 if (utils::IntraComm::isSecondary()) { // Secondary rank
106 if (!itemsToSend.empty()) {
107 PRECICE_DEBUG("Providing {} elements to gather step", itemsToSend.size());
108 utils::IntraComm::getCommunication()->send(itemsToSend, 0);
109 }
110 return;
111 }
112
113 // Primary rank or coupling mode
115 const auto &vertexDistribution = _mesh->getVertexDistribution();
116 const int globalSize = _mesh->getGlobalNumberOfVertices() * valueDimension;
117 PRECICE_DEBUG("Gathering data on primary ({} elements)", globalSize);
118 std::vector<double> globalItemsToSend(globalSize);
119
120 // Directly copy primary rank data
121 PRECICE_ASSERT(vertexDistribution.count(0) > 0);
122 const auto &primaryDistribution = vertexDistribution.at(0);
123 add_to_indirect_blocks(itemsToSend, primaryDistribution, valueDimension, globalItemsToSend);
124 PRECICE_DEBUG("Directly gathered {} entries from primary", primaryDistribution.size() * valueDimension);
125
126 // Gather data from secondary ranks
127 for (Rank secondaryRank : utils::IntraComm::allSecondaryRanks()) {
130
131 auto iter = vertexDistribution.find(secondaryRank);
132 if (iter == vertexDistribution.end()) {
133 continue;
134 }
135 const auto &secondaryDistribution = iter->second;
136
137 int secondaryRankSize = secondaryDistribution.size() * valueDimension;
138 PRECICE_DEBUG("Gathering {} entries from secondary rank {}", secondaryRankSize, secondaryRank);
139 if (secondaryDistribution.empty()) {
140 continue;
141 }
142 std::vector<double> secondaryRankValues(secondaryRankSize);
143 utils::IntraComm::getCommunication()->receive(span<double>{secondaryRankValues}, secondaryRank);
144 add_to_indirect_blocks(secondaryRankValues, secondaryDistribution, valueDimension, globalItemsToSend);
145 }
146
147 // Send data to other primary
148 PRECICE_DEBUG("Sending gathered data to other participant");
149 _com->sendRange(globalItemsToSend, 0);
150}
151
152void GatherScatterCommunication::receive(precice::span<double> itemsToReceive, int valueDimension)
153{
154 PRECICE_TRACE(itemsToReceive.size());
155
156 // Secondary ranks receive scattered data
157 if (utils::IntraComm::isSecondary()) { // Secondary rank
158 if (!itemsToReceive.empty()) {
159 auto received = utils::IntraComm::getCommunication()->receiveRange(0, com::AsVectorTag<double>{});
160 PRECICE_ASSERT(!received.empty());
161 PRECICE_DEBUG("Received scattered data starting with {}", received[0]);
162 std::copy(received.begin(), received.end(), itemsToReceive.begin());
163 }
164 return;
165 }
166
167 // Primary rank receives and scatters the data
169
170 const int globalSize = _mesh->getGlobalNumberOfVertices() * valueDimension;
171 PRECICE_DEBUG("Receiving {} elements from other participant to scatter", globalSize);
172
173 auto globalItemsToReceive = _com->receiveRange(0, com::AsVectorTag<double>{});
174 PRECICE_ASSERT(globalItemsToReceive.size() == static_cast<std::size_t>(globalSize));
175
176 const auto &vertexDistribution = _mesh->getVertexDistribution();
177
178 // Directly copy primary rank data
179 PRECICE_ASSERT(vertexDistribution.count(0) > 0);
180 const auto &primaryDistribution = vertexDistribution.at(0);
181 copy_from_indirect_blocks(globalItemsToReceive, primaryDistribution, valueDimension, itemsToReceive);
182
183 PRECICE_DEBUG("Directly extracted {} data entries for primary", primaryDistribution.size() * valueDimension);
184
185 // Extract and scatter data to secondary ranks
186 for (Rank secondaryRank : utils::IntraComm::allSecondaryRanks()) {
189
190 auto iter = vertexDistribution.find(secondaryRank);
191 if (iter == vertexDistribution.end()) {
192 continue;
193 }
194 const auto &secondaryDistribution = iter->second;
195
196 int secondarySize = secondaryDistribution.size() * valueDimension;
197 PRECICE_DEBUG("Scattering {} entries to secondary {}", secondarySize, secondarySize);
198 if (secondaryDistribution.empty()) {
199 continue;
200 }
201
202 std::vector<double> secondaryRankValues(secondarySize);
203 copy_from_indirect_blocks(globalItemsToReceive, secondaryDistribution, valueDimension, secondaryRankValues);
204 PRECICE_DEBUG("Scattering data starting with {} to rank {}", secondaryRankValues[0], secondaryRank);
205 utils::IntraComm::getCommunication()->sendRange(secondaryRankValues, secondaryRank);
206 }
207}
208
210 std::string const &acceptorName,
211 std::string const &requesterName)
212{
213 PRECICE_ASSERT(false, "Not available for GatherScatterCommunication.");
214}
215
217 std::string const &acceptorName,
218 std::string const &requesterName)
219{
220 PRECICE_ASSERT(false, "Not available for GatherScatterCommunication.");
221}
222
224{
225 PRECICE_ASSERT(false, "Not available for GatherScatterCommunication.");
226}
227
229{
230 PRECICE_ASSERT(false, "Not available for GatherScatterCommunication.");
231}
232
234{
235 PRECICE_ASSERT(false, "Not available for GatherScatterCommunication.");
236}
237
239{
240 PRECICE_ASSERT(false, "Not available for GatherScatterCommunication.");
241}
242
244{
245 PRECICE_ASSERT(false, "Not available for GatherScatterCommunication.");
246}
247
249{
250 PRECICE_ASSERT(false, "Not available for GatherScatterCommunication.");
251}
252
254{
255 PRECICE_ASSERT(false, "Not available for GatherScatterCommunication.");
256}
257
258} // namespace precice::m2n
#define PRECICE_DEBUG(...)
Definition LogMacros.hpp:64
#define PRECICE_TRACE(...)
Definition LogMacros.hpp:95
#define PRECICE_ASSERT(...)
Definition assertion.hpp:87
Interface for all distributed solver to solver communication classes.
mesh::PtrMesh _mesh
mesh that dictates the distribution of this mapping
com::PtrCommunication _com
primary to primary basic communication
bool isConnected() const override
Returns true, if a connection to a remote participant has been setup.
void broadcastReceiveAllMesh() override
Receive mesh partitions per connected rank on remote participant. Not available for GatherScatterComm...
void acceptConnection(const std::string &acceptorName, const std::string &requesterName) override
Accepts connection from participant, which has to call requestConnection().
GatherScatterCommunication(com::PtrCommunication com, mesh::PtrMesh mesh)
void broadcastSendMesh() override
Broadcasts a mesh to connected ranks on remote participant. Not available for GatherScatterCommunicat...
void broadcastReceiveAll(std::vector< int > &itemToReceive) override
Receives an int per connected rank on remote participant. Not available for GatherScatterCommunicatio...
void completeSecondaryRanksConnection() override
Completes the secondary connections for both acceptor and requester by updating the vertex list in _m...
void closeConnection() override
Disconnects from communication space, i.e. participant.
void receive(precice::span< double > itemsToReceive, int valueDimension) override
All ranks receive an array of doubles (different for each rank).
void broadcastSend(int itemToSend) override
Broadcasts an int to connected ranks on remote participant. Not available for GatherScatterCommunicat...
void scatterAllCommunicationMap(CommunicationMap &localCommunicationMap) override
Scatters a communication map over connected ranks on remote participant. Not available for GatherScat...
void acceptPreConnection(std::string const &acceptorName, std::string const &requesterName) override
void requestConnection(const std::string &acceptorName, const std::string &requesterName) override
Requests connection from participant, which has to call acceptConnection().
void requestPreConnection(std::string const &acceptorName, std::string const &requesterName) override
void send(precice::span< double const > itemsToSend, int valueDimension) override
Sends an array of double values from all ranks (different for each rank).
bool _isConnected
Global communication is set up or not.
void gatherAllCommunicationMap(CommunicationMap &localCommunicationMap) override
Gathers a communication maps from connected ranks on remote participant. Not available for GatherScat...
A C++ 11 implementation of the non-owning C++20 std::span type.
Definition span.hpp:284
PRECICE_SPAN_NODISCARD constexpr bool empty() const noexcept
Definition span.hpp:476
constexpr iterator begin() const noexcept
Definition span.hpp:503
constexpr size_type size() const noexcept
Definition span.hpp:469
static Rank getRank()
Current rank.
Definition IntraComm.cpp:42
static auto allSecondaryRanks()
Returns an iterable range over salve ranks [1, _size)
Definition IntraComm.hpp:37
static bool isSecondary()
True if this process is running a secondary rank.
Definition IntraComm.cpp:57
static com::PtrCommunication & getCommunication()
Intra-participant communication.
Definition IntraComm.hpp:31
T copy(T... args)
T copy_n(T... args)
contains the logic of the parallel communication between participants.
Definition BoundM2N.cpp:12
void add_n(InputIt first, Size count, InOutIt result)
int Rank
Definition Types.hpp:37
STL namespace.