source: src/comm/mpi/datatypes_local.hpp@ 177495

Last change on this file since 177495 was 0bd47e, checked in by Julian Iseringhausen <isering@…>, 13 years ago

Added MPI classes output.
  • Property mode set to 100644
File size: 17.4 KB
Line 
1/*
2 * vmg - a versatile multigrid solver
3 * Copyright (C) 2012 Institute for Numerical Simulation, University of Bonn
4 *
5 * vmg is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
9 *
10 * vmg is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
19/**
20 * @file datatypes_local.hpp
21 * @author Julian Iseringhausen <isering@ins.uni-bonn.de>
22 * @date Mon Jan 2 18:45:22 2012
23 *
24 * @brief Saves MPI datatypes for ghost cell communication.
25 *
26 */
27
28#ifndef DATATYPES_LOCAL_HPP_
29#define DATATYPES_LOCAL_HPP_
30
31#ifdef HAVE_MPI
32#include <mpi.h>
33
34#include <vector>
35
36#include "base/index.hpp"
37#include "comm/mpi/datatype.hpp"
38
39namespace VMG
40{
41
42namespace MPI
43{
44
45class DatatypesLocal
46{
47public:
48 template <class T>
49 DatatypesLocal(const T& grid, const MPI_Comm& comm, const bool& alloc)
50 {
51 InitDatatypesLocal(grid, comm, alloc);
52 }
53
54 ~DatatypesLocal() {}
55
56 std::vector<Datatype>& Halo() {return _halo;}
57 std::vector<Datatype>& NB() {return _nb;}
58
59 const std::vector<Datatype>& Halo() const {return _halo;}
60 const std::vector<Datatype>& NB() const {return _nb;}
61
62 const std::vector<Index>& Offset() const {return _offset;}
63
64 std::string ToString() const;
65
66private:
67 template <class T>
68 void InitDatatypesLocal(const T& grid, const MPI_Comm& comm, const bool& alloc_buffer);
69
70 std::vector<Datatype> _halo, _nb;
71 std::vector<Index> _offset;
72};
73
74std::ostream& operator<<(std::ostream& out, const DatatypesLocal& dt_local);
75
76namespace
77{
78
79inline int to_1d(const Index& i)
80{
81 return i.Z()+3*(i.Y()+3*i.X());
82}
83
84inline bool _is_valid(const Index& coord, const Index& dims, const Index& periods)
85{
86 return (periods[0] || (coord[0] >= 0 && coord[0] < dims[0])) &&
87 (periods[1] || (coord[1] >= 0 && coord[1] < dims[1])) &&
88 (periods[2] || (coord[2] >= 0 && coord[2] < dims[2]));
89}
90
91}
92
93template <class T>
94void DatatypesLocal::InitDatatypesLocal(const T& grid, const MPI_Comm& comm, const bool& alloc_buffer)
95{
96 if (comm != MPI_COMM_NULL) {
97
98 int index, ranks[27];
99 Index dims, periods, coords;
100 Index sizes, subsizes, starts;
101 Index offset, i;
102 const LocalIndices& l = grid.Local();
103
104 MPI_Cart_get(comm, 3, dims.vec(), periods.vec(), coords.vec());
105
106 for (i.X()=-1; i.X()<=1; ++i.X())
107 for (i.Y()=-1; i.Y()<=1; ++i.Y())
108 for (i.Z()=-1; i.Z()<=1; ++i.Z())
109 if (_is_valid(coords + i, dims, periods))
110 MPI_Cart_rank(comm, (coords + i).vec(), &ranks[to_1d(i+1)]);
111
112 sizes = l.SizeTotal();
113
114 /* -1 0 0 */
115 offset = Index(-1,0,0);
116 if (_is_valid(coords + offset, dims, periods)) {
117 index = to_1d(offset+1);
118 subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.Size().Z());
119 starts = Index(0, l.Begin().Y(), l.Begin().Z());
120 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 0, 1, alloc_buffer));
121 starts = l.Begin();
122 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 0, 1, alloc_buffer));
123 _offset.push_back(offset);
124 }
125
126 /* 1 0 0 */
127 offset = Index(1,0,0);
128 if (_is_valid(coords + offset, dims, periods)) {
129 index = to_1d(offset+1);
130 subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.Size().Z());
131 starts = Index(l.End().X(), l.Begin().Y(), l.Begin().Z());
132 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 1, 0, alloc_buffer));
133 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
134 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 1, 0, alloc_buffer));
135 _offset.push_back(offset);
136 }
137
138 /* 0 -1 0 */
139 offset = Index(0,-1,0);
140 if (_is_valid(coords + offset, dims, periods)) {
141 index = to_1d(offset+1);
142 subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.Size().Z());
143 starts = Index(l.Begin().X(), 0, l.Begin().Z());
144 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 2, 3, alloc_buffer));
145 starts = l.Begin();
146 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 2, 3, alloc_buffer));
147 _offset.push_back(offset);
148 }
149
150 /* 0 1 0 */
151 offset = Index(0,1,0);
152 if (_is_valid(coords + offset, dims, periods)) {
153 index = to_1d(offset+1);
154 subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.Size().Z());
155 starts = Index(l.Begin().X(), l.End().Y(), l.Begin().Z());
156 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 3, 2, alloc_buffer));
157 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
158 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 3, 2, alloc_buffer));
159 _offset.push_back(offset);
160 }
161
162 /* 0 0 -1 */
163 offset = Index(0,0,-1);
164 if (_is_valid(coords + offset, dims, periods)) {
165 index = to_1d(offset+1);
166 subsizes = Index(l.Size().X(), l.Size().Y(), l.HaloSize1().Z());
167 starts = Index(l.Begin().X(), l.Begin().Y(), 0);
168 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 4, 5, alloc_buffer));
169 starts = l.Begin();
170 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 4, 5, alloc_buffer));
171 _offset.push_back(offset);
172 }
173
174 /* 0 0 1 */
175 offset = Index(0,0,1);
176 if (_is_valid(coords + offset, dims, periods)) {
177 index = to_1d(offset+1);
178 subsizes = Index(l.Size().X(), l.Size().Y(), l.HaloSize2().Z());
179 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z());
180 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 5, 4, alloc_buffer));
181 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
182 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 5, 4, alloc_buffer));
183 _offset.push_back(offset);
184 }
185
186 /* -1 -1 0 */
187 offset = Index(-1,-1,0);
188 if (_is_valid(coords + offset, dims, periods)) {
189 index = to_1d(offset+1);
190 subsizes = Index(l.HaloSize1().X(), l.HaloSize1().Y(), l.Size().Z());
191 starts = Index(0, 0, l.Begin().Z());
192 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 6, 7, alloc_buffer));
193 starts = l.Begin();
194 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 6, 7, alloc_buffer));
195 _offset.push_back(offset);
196 }
197
198 /* -1 1 0 */
199 offset = Index(-1,1,0);
200 if (_is_valid(coords + offset, dims, periods)) {
201 index = to_1d(offset+1);
202 subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.Size().Z());
203 starts = Index(0, l.End().Y(), l.Begin().Z());
204 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 8, 9, alloc_buffer));
205 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
206 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 8, 9, alloc_buffer));
207 _offset.push_back(offset);
208 }
209
210 /* 1 -1 0 */
211 offset = Index(1,-1,0);
212 if (_is_valid(coords + offset, dims, periods)) {
213 index = to_1d(offset+1);
214 subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.Size().Z());
215 starts = Index(l.End().X(), 0, l.Begin().Z());
216 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 9, 8, alloc_buffer));
217 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
218 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 9, 8, alloc_buffer));
219 _offset.push_back(offset);
220 }
221
222 /* 1 1 0 */
223 offset = Index(1,1,0);
224 if (_is_valid(coords + offset, dims, periods)) {
225 index = to_1d(offset+1);
226 subsizes = Index(l.HaloSize2().X(), l.HaloSize2().Y(), l.Size().Z());
227 starts = Index(l.End().X(), l.End().Y(), l.Begin().Z());
228 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 7, 6, alloc_buffer));
229 starts = Index(l.End().X()-l.HaloSize2().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
230 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 7, 6, alloc_buffer));
231 _offset.push_back(offset);
232 }
233
234 /* -1 0 -1 */
235 offset = Index(-1,0,-1);
236 if (_is_valid(coords + offset, dims, periods)) {
237 index = to_1d(offset+1);
238 subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.HaloSize1().Z());
239 starts = Index(0, l.Begin().Y(), 0);
240 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 10, 11, alloc_buffer));
241 starts = l.Begin();
242 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 10, 11, alloc_buffer));
243 _offset.push_back(offset);
244 }
245
246 /* -1 0 1 */
247 offset = Index(-1,0,1);
248 if (_is_valid(coords + offset, dims, periods)) {
249 index = to_1d(offset+1);
250 subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.HaloSize2().Z());
251 starts = Index(0, l.Begin().Y(), l.End().Z());
252 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 12, 13, alloc_buffer));
253 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
254 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 12, 13, alloc_buffer));
255 _offset.push_back(offset);
256 }
257
258 /* 1 0 -1 */
259 offset = Index(1,0,-1);
260 if (_is_valid(coords + offset, dims, periods)) {
261 index = to_1d(offset+1);
262 subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.HaloSize1().Z());
263 starts = Index(l.End().X(), l.Begin().Y(), 0);
264 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 13, 12, alloc_buffer));
265 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
266 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 13, 12, alloc_buffer));
267 _offset.push_back(offset);
268 }
269
270 /* 1 0 1 */
271 offset = Index(1,0,1);
272 if (_is_valid(coords + offset, dims, periods)) {
273 index = to_1d(offset+1);
274 subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.HaloSize2().Z());
275 starts = Index(l.End().X(), l.Begin().Y(), l.End().Z());
276 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 11, 10, alloc_buffer));
277 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
278 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 11, 10, alloc_buffer));
279 _offset.push_back(offset);
280 }
281
282 /* 0 -1 -1 */
283 offset = Index(0,-1,-1);
284 if (_is_valid(coords + offset, dims, periods)) {
285 index = to_1d(offset+1);
286 subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.HaloSize1().Z());
287 starts = Index(l.Begin().X(), 0, 0);
288 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15, alloc_buffer));
289 starts = l.Begin();
290 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15, alloc_buffer));
291 _offset.push_back(offset);
292 }
293
294 /* 0 -1 1 */
295 offset = Index(0,-1,1);
296 if (_is_valid(coords + offset, dims, periods)) {
297 index = to_1d(offset+1);
298 subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
299 starts = Index(l.Begin().X(), 0, l.End().Z());
300 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17, alloc_buffer));
301 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
302 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17, alloc_buffer));
303 _offset.push_back(offset);
304 }
305
306 /* 0 1 -1 */
307 offset = Index(0,1,-1);
308 if (_is_valid(coords + offset, dims, periods)) {
309 index = to_1d(offset+1);
310 subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
311 starts = Index(l.Begin().X(), l.End().Y(), 0);
312 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16, alloc_buffer));
313 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
314 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16, alloc_buffer));
315 _offset.push_back(offset);
316 }
317
318 /* 0 1 1 */
319 offset = Index(0,1,1);
320 if (_is_valid(coords + offset, dims, periods)) {
321 index = to_1d(offset+1);
322 subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.HaloSize2().Z());
323 starts = Index(l.Begin().X(), l.End().Y(), l.End().Z());
324 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14, alloc_buffer));
325 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.End().Z()-l.HaloSize2().Z());
326 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14, alloc_buffer));
327 _offset.push_back(offset);
328 }
329
330 /* -1 -1 -1 */
331 offset = Index(-1,-1,-1);
332 if (_is_valid(coords + offset, dims, periods)) {
333 index = to_1d(offset+1);
334 subsizes = l.HaloSize1();
335 starts = 0;
336 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19, alloc_buffer));
337 starts = l.Begin();
338 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19, alloc_buffer));
339 _offset.push_back(offset);
340 }
341
342 /* -1 -1 1 */
343 offset = Index(-1,-1,1);
344 if (_is_valid(coords + offset, dims, periods)) {
345 index = to_1d(offset+1);
346 subsizes = Index(l.HaloSize1().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
347 starts = Index(0, 0, l.End().Z());;
348 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21, alloc_buffer));
349 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
350 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21, alloc_buffer));
351 _offset.push_back(offset);
352 }
353
354 /* -1 1 -1 */
355 offset = Index(-1,1,-1);
356 if (_is_valid(coords + offset, dims, periods)) {
357 index = to_1d(offset+1);
358 subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
359 starts = Index(0, l.End().Y(), 0);
360 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23, alloc_buffer));
361 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
362 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23, alloc_buffer));
363 _offset.push_back(offset);
364 }
365
366 /* 1 -1 -1 */
367 offset = Index(1,-1,-1);
368 if (_is_valid(coords + offset, dims, periods)) {
369 index = to_1d(offset+1);
370 subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.HaloSize1().Z());
371 starts = Index(l.End().X(), 0, 0);
372 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25, alloc_buffer));
373 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
374 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25, alloc_buffer));
375 _offset.push_back(offset);
376 }
377
378 /* -1 1 1 */
379 offset = Index(-1,1,1);
380 if (_is_valid(coords + offset, dims, periods)) {
381 index = to_1d(offset+1);
382 subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.HaloSize2().Z());
383 starts = Index(0, l.End().Y(), l.End().Z());
384 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24, alloc_buffer));
385 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.End().Z()-l.HaloSize2().Z());
386 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24, alloc_buffer));
387 _offset.push_back(offset);
388 }
389
390 /* 1 -1 1 */
391 offset = Index(1,-1,1);
392 if (_is_valid(coords + offset, dims, periods)) {
393 index = to_1d(offset+1);
394 subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
395 starts = Index(l.End().X(), 0, l.End().Z());
396 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22, alloc_buffer));
397 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
398 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22, alloc_buffer));
399 _offset.push_back(offset);
400 }
401
402 /* 1 1 -1 */
403 offset = Index(1,1,-1);
404 if (_is_valid(coords + offset, dims, periods)) {
405 index = to_1d(offset+1);
406 subsizes = Index(l.HaloSize2().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
407 starts = Index(l.End().X(), l.End().Y(), 0);
408 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20, alloc_buffer));
409 starts = Index(l.End().X()-l.HaloSize2().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
410 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20, alloc_buffer));
411 _offset.push_back(offset);
412 }
413
414 /* 1 1 1 */
415 offset = Index(1,1,1);
416 if (_is_valid(coords + offset, dims, periods)) {
417 index = to_1d(offset+1);
418 subsizes = l.HaloSize2();
419 starts = l.End();
420 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18, alloc_buffer));
421 starts = l.End()-l.HaloSize2();
422 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18, alloc_buffer));
423 _offset.push_back(offset);
424 }
425
426 }
427}
428
429}
430
431}
432
433#endif /* HAVE_MPI */
434
435#endif /* DATATYPES_LOCAL_HPP_ */
Note: See TracBrowser for help on using the repository browser.