source: src/comm/mpi/datatypes_local.hpp@ e85cfd

Last change on this file since e85cfd was 8180d8, checked in by Julian Iseringhausen <julian.iseringhausen@…>, 13 years ago

Merge stashed open boundary stuff.
  • Property mode set to 100644
File size: 17.0 KB
Line 
1/*
2 * vmg - a versatile multigrid solver
3 * Copyright (C) 2012 Institute for Numerical Simulation, University of Bonn
4 *
5 * vmg is free software: you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, either version 3 of the License, or
8 * (at your option) any later version.
9 *
10 * vmg is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
19#ifndef DATATYPES_LOCAL_HPP_
20#define DATATYPES_LOCAL_HPP_
21
22#include <vector>
23
24#include "base/index.hpp"
25#include "comm/mpi/datatype.hpp"
26
27namespace VMG
28{
29
30namespace MPI
31{
32
33class DatatypesLocal
34{
35public:
36 template <class T>
37 DatatypesLocal(const T& grid, const MPI_Comm& comm, const bool& alloc)
38 {
39 InitDatatypesLocal(grid, comm, alloc);
40 }
41
42 ~DatatypesLocal() {}
43
44 std::vector<Datatype>& Halo() {return _halo;}
45 std::vector<Datatype>& NB() {return _nb;}
46
47 const std::vector<Datatype>& Halo() const {return _halo;}
48 const std::vector<Datatype>& NB() const {return _nb;}
49
50 const std::vector<Index>& Offset() const {return _offset;}
51
52private:
53 template <class T>
54 void InitDatatypesLocal(const T& grid, const MPI_Comm& comm, const bool& alloc_buffer);
55
56 std::vector<Datatype> _halo, _nb;
57 std::vector<Index> _offset;
58};
59
60namespace
61{
62
63inline int to_1d(const Index& i)
64{
65 return i.Z()+3*(i.Y()+3*i.X());
66}
67
68inline bool _is_valid(const Index& coord, const Index& dims, const Index& periods)
69{
70 return (periods[0] || (coord[0] >= 0 && coord[0] < dims[0])) &&
71 (periods[1] || (coord[1] >= 0 && coord[1] < dims[1])) &&
72 (periods[2] || (coord[2] >= 0 && coord[2] < dims[2]));
73}
74
75}
76
77template <class T>
78void DatatypesLocal::InitDatatypesLocal(const T& grid, const MPI_Comm& comm, const bool& alloc_buffer)
79{
80 if (comm != MPI_COMM_NULL) {
81
82 int index, ranks[27];
83 Index dims, periods, coords;
84 Index sizes, subsizes, starts;
85 Index offset, i;
86 const LocalIndices& l = grid.Local();
87
88 MPI_Cart_get(comm, 3, dims.vec(), periods.vec(), coords.vec());
89
90 for (i.X()=-1; i.X()<=1; ++i.X())
91 for (i.Y()=-1; i.Y()<=1; ++i.Y())
92 for (i.Z()=-1; i.Z()<=1; ++i.Z())
93 if (_is_valid(coords + i, dims, periods))
94 MPI_Cart_rank(comm, (coords + i).vec(), &ranks[to_1d(i+1)]);
95
96 sizes = l.SizeTotal();
97
98 /* -1 0 0 */
99 offset = Index(-1,0,0);
100 if (_is_valid(coords + offset, dims, periods)) {
101 index = to_1d(offset+1);
102 subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.Size().Z());
103 starts = Index(0, l.Begin().Y(), l.Begin().Z());
104 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 0, 1, alloc_buffer));
105 starts = l.Begin();
106 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 0, 1, alloc_buffer));
107 _offset.push_back(offset);
108 }
109
110 /* 1 0 0 */
111 offset = Index(1,0,0);
112 if (_is_valid(coords + offset, dims, periods)) {
113 index = to_1d(offset+1);
114 subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.Size().Z());
115 starts = Index(l.End().X(), l.Begin().Y(), l.Begin().Z());
116 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 1, 0, alloc_buffer));
117 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
118 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 1, 0, alloc_buffer));
119 _offset.push_back(offset);
120 }
121
122 /* 0 -1 0 */
123 offset = Index(0,-1,0);
124 if (_is_valid(coords + offset, dims, periods)) {
125 index = to_1d(offset+1);
126 subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.Size().Z());
127 starts = Index(l.Begin().X(), 0, l.Begin().Z());
128 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 2, 3, alloc_buffer));
129 starts = l.Begin();
130 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 2, 3, alloc_buffer));
131 _offset.push_back(offset);
132 }
133
134 /* 0 1 0 */
135 offset = Index(0,1,0);
136 if (_is_valid(coords + offset, dims, periods)) {
137 index = to_1d(offset+1);
138 subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.Size().Z());
139 starts = Index(l.Begin().X(), l.End().Y(), l.Begin().Z());
140 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 3, 2, alloc_buffer));
141 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
142 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 3, 2, alloc_buffer));
143 _offset.push_back(offset);
144 }
145
146 /* 0 0 -1 */
147 offset = Index(0,0,-1);
148 if (_is_valid(coords + offset, dims, periods)) {
149 index = to_1d(offset+1);
150 subsizes = Index(l.Size().X(), l.Size().Y(), l.HaloSize1().Z());
151 starts = Index(l.Begin().X(), l.Begin().Y(), 0);
152 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 4, 5, alloc_buffer));
153 starts = l.Begin();
154 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 4, 5, alloc_buffer));
155 _offset.push_back(offset);
156 }
157
158 /* 0 0 1 */
159 offset = Index(0,0,1);
160 if (_is_valid(coords + offset, dims, periods)) {
161 index = to_1d(offset+1);
162 subsizes = Index(l.Size().X(), l.Size().Y(), l.HaloSize2().Z());
163 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z());
164 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 5, 4, alloc_buffer));
165 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
166 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 5, 4, alloc_buffer));
167 _offset.push_back(offset);
168 }
169
170 /* -1 -1 0 */
171 offset = Index(-1,-1,0);
172 if (_is_valid(coords + offset, dims, periods)) {
173 index = to_1d(offset+1);
174 subsizes = Index(l.HaloSize1().X(), l.HaloSize1().Y(), l.Size().Z());
175 starts = Index(0, 0, l.Begin().Z());
176 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 6, 7, alloc_buffer));
177 starts = l.Begin();
178 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 6, 7, alloc_buffer));
179 _offset.push_back(offset);
180 }
181
182 /* -1 1 0 */
183 offset = Index(-1,1,0);
184 if (_is_valid(coords + offset, dims, periods)) {
185 index = to_1d(offset+1);
186 subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.Size().Z());
187 starts = Index(0, l.End().Y(), l.Begin().Z());
188 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 8, 9, alloc_buffer));
189 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
190 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 8, 9, alloc_buffer));
191 _offset.push_back(offset);
192 }
193
194 /* 1 -1 0 */
195 offset = Index(1,-1,0);
196 if (_is_valid(coords + offset, dims, periods)) {
197 index = to_1d(offset+1);
198 subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.Size().Z());
199 starts = Index(l.End().X(), 0, l.Begin().Z());
200 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 9, 8, alloc_buffer));
201 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
202 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 9, 8, alloc_buffer));
203 _offset.push_back(offset);
204 }
205
206 /* 1 1 0 */
207 offset = Index(1,1,0);
208 if (_is_valid(coords + offset, dims, periods)) {
209 index = to_1d(offset+1);
210 subsizes = Index(l.HaloSize2().X(), l.HaloSize2().Y(), l.Size().Z());
211 starts = Index(l.End().X(), l.End().Y(), l.Begin().Z());
212 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 7, 6, alloc_buffer));
213 starts = Index(l.End().X()-l.HaloSize2().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
214 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 7, 6, alloc_buffer));
215 _offset.push_back(offset);
216 }
217
218 /* -1 0 -1 */
219 offset = Index(-1,0,-1);
220 if (_is_valid(coords + offset, dims, periods)) {
221 index = to_1d(offset+1);
222 subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.HaloSize1().Z());
223 starts = Index(0, l.Begin().Y(), 0);
224 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 10, 11, alloc_buffer));
225 starts = l.Begin();
226 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 10, 11, alloc_buffer));
227 _offset.push_back(offset);
228 }
229
230 /* -1 0 1 */
231 offset = Index(-1,0,1);
232 if (_is_valid(coords + offset, dims, periods)) {
233 index = to_1d(offset+1);
234 subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.HaloSize2().Z());
235 starts = Index(0, l.Begin().Y(), l.End().Z());
236 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 12, 13, alloc_buffer));
237 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
238 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 12, 13, alloc_buffer));
239 _offset.push_back(offset);
240 }
241
242 /* 1 0 -1 */
243 offset = Index(1,0,-1);
244 if (_is_valid(coords + offset, dims, periods)) {
245 index = to_1d(offset+1);
246 subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.HaloSize1().Z());
247 starts = Index(l.End().X(), l.Begin().Y(), 0);
248 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 13, 12, alloc_buffer));
249 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
250 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 13, 12, alloc_buffer));
251 _offset.push_back(offset);
252 }
253
254 /* 1 0 1 */
255 offset = Index(1,0,1);
256 if (_is_valid(coords + offset, dims, periods)) {
257 index = to_1d(offset+1);
258 subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.HaloSize2().Z());
259 starts = Index(l.End().X(), l.Begin().Y(), l.End().Z());
260 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 11, 10, alloc_buffer));
261 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
262 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 11, 10, alloc_buffer));
263 _offset.push_back(offset);
264 }
265
266 /* 0 -1 -1 */
267 offset = Index(0,-1,-1);
268 if (_is_valid(coords + offset, dims, periods)) {
269 index = to_1d(offset+1);
270 subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.HaloSize1().Z());
271 starts = Index(l.Begin().X(), 0, 0);
272 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15, alloc_buffer));
273 starts = l.Begin();
274 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15, alloc_buffer));
275 _offset.push_back(offset);
276 }
277
278 /* 0 -1 1 */
279 offset = Index(0,-1,1);
280 if (_is_valid(coords + offset, dims, periods)) {
281 index = to_1d(offset+1);
282 subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
283 starts = Index(l.Begin().X(), 0, l.End().Z());
284 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17, alloc_buffer));
285 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
286 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17, alloc_buffer));
287 _offset.push_back(offset);
288 }
289
290 /* 0 1 -1 */
291 offset = Index(0,1,-1);
292 if (_is_valid(coords + offset, dims, periods)) {
293 index = to_1d(offset+1);
294 subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
295 starts = Index(l.Begin().X(), l.End().Y(), 0);
296 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16, alloc_buffer));
297 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
298 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16, alloc_buffer));
299 _offset.push_back(offset);
300 }
301
302 /* 0 1 1 */
303 offset = Index(0,1,1);
304 if (_is_valid(coords + offset, dims, periods)) {
305 index = to_1d(offset+1);
306 subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.HaloSize2().Z());
307 starts = Index(l.Begin().X(), l.End().Y(), l.End().Z());
308 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14, alloc_buffer));
309 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.End().Z()-l.HaloSize2().Z());
310 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14, alloc_buffer));
311 _offset.push_back(offset);
312 }
313
314 /* -1 -1 -1 */
315 offset = Index(-1,-1,-1);
316 if (_is_valid(coords + offset, dims, periods)) {
317 index = to_1d(offset+1);
318 subsizes = l.HaloSize1();
319 starts = 0;
320 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19, alloc_buffer));
321 starts = l.Begin();
322 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19, alloc_buffer));
323 _offset.push_back(offset);
324 }
325
326 /* -1 -1 1 */
327 offset = Index(-1,-1,1);
328 if (_is_valid(coords + offset, dims, periods)) {
329 index = to_1d(offset+1);
330 subsizes = Index(l.HaloSize1().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
331 starts = Index(0, 0, l.End().Z());;
332 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21, alloc_buffer));
333 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
334 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21, alloc_buffer));
335 _offset.push_back(offset);
336 }
337
338 /* -1 1 -1 */
339 offset = Index(-1,1,-1);
340 if (_is_valid(coords + offset, dims, periods)) {
341 index = to_1d(offset+1);
342 subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
343 starts = Index(0, l.End().Y(), 0);
344 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23, alloc_buffer));
345 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
346 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23, alloc_buffer));
347 _offset.push_back(offset);
348 }
349
350 /* 1 -1 -1 */
351 offset = Index(1,-1,-1);
352 if (_is_valid(coords + offset, dims, periods)) {
353 index = to_1d(offset+1);
354 subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.HaloSize1().Z());
355 starts = Index(l.End().X(), 0, 0);
356 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25, alloc_buffer));
357 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
358 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25, alloc_buffer));
359 _offset.push_back(offset);
360 }
361
362 /* -1 1 1 */
363 offset = Index(-1,1,1);
364 if (_is_valid(coords + offset, dims, periods)) {
365 index = to_1d(offset+1);
366 subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.HaloSize2().Z());
367 starts = Index(0, l.End().Y(), l.End().Z());
368 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24, alloc_buffer));
369 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.End().Z()-l.HaloSize2().Z());
370 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24, alloc_buffer));
371 _offset.push_back(offset);
372 }
373
374 /* 1 -1 1 */
375 offset = Index(1,-1,1);
376 if (_is_valid(coords + offset, dims, periods)) {
377 index = to_1d(offset+1);
378 subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
379 starts = Index(l.End().X(), 0, l.End().Z());
380 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22, alloc_buffer));
381 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
382 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22, alloc_buffer));
383 _offset.push_back(offset);
384 }
385
386 /* 1 1 -1 */
387 offset = Index(1,1,-1);
388 if (_is_valid(coords + offset, dims, periods)) {
389 index = to_1d(offset+1);
390 subsizes = Index(l.HaloSize2().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
391 starts = Index(l.End().X(), l.End().Y(), 0);
392 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20, alloc_buffer));
393 starts = Index(l.End().X()-l.HaloSize2().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
394 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20, alloc_buffer));
395 _offset.push_back(offset);
396 }
397
398 /* 1 1 1 */
399 offset = Index(1,1,1);
400 if (_is_valid(coords + offset, dims, periods)) {
401 index = to_1d(offset+1);
402 subsizes = l.HaloSize2();
403 starts = l.End();
404 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18, alloc_buffer));
405 starts = l.End()-l.HaloSize2();
406 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18, alloc_buffer));
407 _offset.push_back(offset);
408 }
409
410 }
411}
412
413}
414
415}
416
417#endif /* DATATYPES_LOCAL_HPP_ */
Note: See TracBrowser for help on using the repository browser.