14 #include <dolfinx/graph/AdjacencyList.h>
17 #include <type_traits>
21 #define MPICH_IGNORE_CXX_SEEK 1
56 MPI_Comm
comm()
const;
64 static int rank(MPI_Comm comm);
68 static int size(MPI_Comm comm);
82 const std::vector<int>& send_offsets,
83 const std::vector<T>& send_data);
87 static std::vector<int>
neighbors(MPI_Comm neighbor_comm);
91 static std::size_t
global_offset(MPI_Comm comm, std::size_t range,
96 static std::array<std::int64_t, 2>
local_range(
int process, std::int64_t N,
106 template <
typename T>
112 template <
typename T>
116 throw std::runtime_error(
"MPI data type unknown");
125 inline MPI_Datatype MPI::mpi_type<float>()
130 inline MPI_Datatype MPI::mpi_type<double>()
135 inline MPI_Datatype MPI::mpi_type<std::complex<double>>()
137 return MPI_DOUBLE_COMPLEX;
140 inline MPI_Datatype MPI::mpi_type<short int>()
145 inline MPI_Datatype MPI::mpi_type<int>()
150 inline MPI_Datatype MPI::mpi_type<unsigned int>()
155 inline MPI_Datatype MPI::mpi_type<long int>()
160 inline MPI_Datatype MPI::mpi_type<unsigned long>()
162 return MPI_UNSIGNED_LONG;
165 inline MPI_Datatype MPI::mpi_type<long long>()
167 return MPI_LONG_LONG;
170 inline MPI_Datatype MPI::mpi_type<unsigned long long>()
172 return MPI_UNSIGNED_LONG_LONG;
175 inline MPI_Datatype MPI::mpi_type<bool>()
181 template <
typename T>
182 graph::AdjacencyList<T>
186 const Eigen::Array<std::int32_t, Eigen::Dynamic, 1>& send_offsets
188 const Eigen::Array<T, Eigen::Dynamic, 1>& values_in = send_data.
array();
191 assert(send_data.
num_nodes() == comm_size);
194 std::vector<int> send_size(comm_size);
195 std::adjacent_difference(send_offsets.data() + 1,
196 send_offsets.data() + send_offsets.rows(),
200 std::vector<int> recv_size(comm_size);
201 MPI_Alltoall(send_size.data(), 1, mpi_type<int>(), recv_size.data(), 1,
202 mpi_type<int>(), comm);
205 Eigen::Array<std::int32_t, Eigen::Dynamic, 1> recv_offset(comm_size + 1);
207 std::partial_sum(recv_size.begin(), recv_size.end(), recv_offset.data() + 1);
210 Eigen::Array<T, Eigen::Dynamic, 1> recv_values(recv_offset(comm_size));
211 MPI_Alltoallv(values_in.data(), send_size.data(), send_offsets.data(),
212 mpi_type<T>(), recv_values.data(), recv_size.data(),
213 recv_offset.data(), mpi_type<T>(), comm);
216 std::move(recv_offset));
219 template <
typename T>
222 const std::vector<int>& send_offsets,
223 const std::vector<T>& send_data)
225 assert((
int)send_data.size() == send_offsets.back());
226 assert(send_offsets[0] == 0);
229 std::vector<int> send_sizes(send_offsets.size() - 1, 0);
230 std::vector<int> recv_sizes(send_sizes.size());
231 std::adjacent_difference(send_offsets.begin() + 1, send_offsets.end(),
233 MPI_Neighbor_alltoall(send_sizes.data(), 1, MPI::mpi_type<int>(),
234 recv_sizes.data(), 1, MPI::mpi_type<int>(),
238 Eigen::Array<int, Eigen::Dynamic, 1> recv_offsets(recv_sizes.size() + 1);
240 std::partial_sum(recv_sizes.begin(), recv_sizes.end(),
241 recv_offsets.data() + 1);
243 Eigen::Array<T, Eigen::Dynamic, 1> recv_data(
244 recv_offsets(recv_offsets.rows() - 1));
245 MPI_Neighbor_alltoallv(
246 send_data.data(), send_sizes.data(), send_offsets.data(),
247 MPI::mpi_type<T>(), recv_data.data(), recv_sizes.data(),
248 recv_offsets.data(), MPI::mpi_type<T>(), neighbor_comm);