source: src/comm/mpi/datatypes_local.hpp@ f003a9

Last change on this file since f003a9 was f003a9, checked in by Julian Iseringhausen <isering@…>, 14 years ago

Refactored vmg in order to separate the core library and the particle simulation part properly.

git-svn-id: ​https://svn.version.fz-juelich.de/scafacos/trunk@1798 5161e1c8-67bf-11de-9fd5-51895aff932f
  • Property mode set to 100644
File size: 16.3 KB
Line 
1#ifndef DATATYPES_LOCAL_HPP_
2#define DATATYPES_LOCAL_HPP_
3
4#include <vector>
5
6#include "base/index.hpp"
7#include "comm/mpi/datatype.hpp"
8
9namespace VMG
10{
11
12namespace MPI
13{
14
15class DatatypesLocal
16{
17public:
18 template <class T>
19 DatatypesLocal(const T& grid, const MPI_Comm& comm, const bool& alloc)
20 {
21 InitDatatypesLocal(grid, comm, alloc);
22 }
23
24 ~DatatypesLocal() {}
25
26 std::vector<Datatype>& Halo() {return _halo;}
27 std::vector<Datatype>& NB() {return _nb;}
28
29 const std::vector<Datatype>& Halo() const {return _halo;}
30 const std::vector<Datatype>& NB() const {return _nb;}
31
32 const std::vector<Index>& Offset() const {return _offset;}
33
34private:
35 template <class T>
36 void InitDatatypesLocal(const T& grid, const MPI_Comm& comm, const bool& alloc_buffer);
37
38 std::vector<Datatype> _halo, _nb;
39 std::vector<Index> _offset;
40};
41
42namespace
43{
44
45inline int to_1d(const Index& i)
46{
47 return i.Z()+3*(i.Y()+3*i.X());
48}
49
50inline bool _is_valid(const Index& coord, const Index& dims, const Index& periods)
51{
52 return (periods[0] || (coord[0] >= 0 && coord[0] < dims[0])) &&
53 (periods[1] || (coord[1] >= 0 && coord[1] < dims[1])) &&
54 (periods[2] || (coord[2] >= 0 && coord[2] < dims[2]));
55}
56
57}
58
59template <class T>
60void DatatypesLocal::InitDatatypesLocal(const T& grid, const MPI_Comm& comm, const bool& alloc_buffer)
61{
62 if (comm != MPI_COMM_NULL) {
63
64 int index, ranks[27];
65 Index dims, periods, coords;
66 Index sizes, subsizes, starts;
67 Index offset, i;
68 const LocalIndices& l = grid.Local();
69
70 MPI_Cart_get(comm, 3, dims.vec(), periods.vec(), coords.vec());
71
72 for (i.X()=-1; i.X()<=1; ++i.X())
73 for (i.Y()=-1; i.Y()<=1; ++i.Y())
74 for (i.Z()=-1; i.Z()<=1; ++i.Z())
75 if (_is_valid(coords + i, dims, periods))
76 MPI_Cart_rank(comm, (coords + i).vec(), &ranks[to_1d(i+1)]);
77
78 sizes = l.SizeTotal();
79
80 /* -1 0 0 */
81 offset = Index(-1,0,0);
82 if (_is_valid(coords + offset, dims, periods)) {
83 index = to_1d(offset+1);
84 subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.Size().Z());
85 starts = Index(0, l.Begin().Y(), l.Begin().Z());
86 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 0, 1, alloc_buffer));
87 starts = l.Begin();
88 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 0, 1, alloc_buffer));
89 _offset.push_back(offset);
90 }
91
92 /* 1 0 0 */
93 offset = Index(1,0,0);
94 if (_is_valid(coords + offset, dims, periods)) {
95 index = to_1d(offset+1);
96 subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.Size().Z());
97 starts = Index(l.End().X(), l.Begin().Y(), l.Begin().Z());
98 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 1, 0, alloc_buffer));
99 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
100 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 1, 0, alloc_buffer));
101 _offset.push_back(offset);
102 }
103
104 /* 0 -1 0 */
105 offset = Index(0,-1,0);
106 if (_is_valid(coords + offset, dims, periods)) {
107 index = to_1d(offset+1);
108 subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.Size().Z());
109 starts = Index(l.Begin().X(), 0, l.Begin().Z());
110 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 2, 3, alloc_buffer));
111 starts = l.Begin();
112 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 2, 3, alloc_buffer));
113 _offset.push_back(offset);
114 }
115
116 /* 0 1 0 */
117 offset = Index(0,1,0);
118 if (_is_valid(coords + offset, dims, periods)) {
119 index = to_1d(offset+1);
120 subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.Size().Z());
121 starts = Index(l.Begin().X(), l.End().Y(), l.Begin().Z());
122 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 3, 2, alloc_buffer));
123 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
124 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 3, 2, alloc_buffer));
125 _offset.push_back(offset);
126 }
127
128 /* 0 0 -1 */
129 offset = Index(0,0,-1);
130 if (_is_valid(coords + offset, dims, periods)) {
131 index = to_1d(offset+1);
132 subsizes = Index(l.Size().X(), l.Size().Y(), l.HaloSize1().Z());
133 starts = Index(l.Begin().X(), l.Begin().Y(), 0);
134 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 4, 5, alloc_buffer));
135 starts = l.Begin();
136 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 4, 5, alloc_buffer));
137 _offset.push_back(offset);
138 }
139
140 /* 0 0 1 */
141 offset = Index(0,0,1);
142 if (_is_valid(coords + offset, dims, periods)) {
143 index = to_1d(offset+1);
144 subsizes = Index(l.Size().X(), l.Size().Y(), l.HaloSize2().Z());
145 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z());
146 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 5, 4, alloc_buffer));
147 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
148 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 5, 4, alloc_buffer));
149 _offset.push_back(offset);
150 }
151
152 /* -1 -1 0 */
153 offset = Index(-1,-1,0);
154 if (_is_valid(coords + offset, dims, periods)) {
155 index = to_1d(offset+1);
156 subsizes = Index(l.HaloSize1().X(), l.HaloSize1().Y(), l.Size().Z());
157 starts = Index(0, 0, l.Begin().Z());
158 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 6, 7, alloc_buffer));
159 starts = l.Begin();
160 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 6, 7, alloc_buffer));
161 _offset.push_back(offset);
162 }
163
164 /* -1 1 0 */
165 offset = Index(-1,1,0);
166 if (_is_valid(coords + offset, dims, periods)) {
167 index = to_1d(offset+1);
168 subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.Size().Z());
169 starts = Index(0, l.End().Y(), l.Begin().Z());
170 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 8, 9, alloc_buffer));
171 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
172 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 8, 9, alloc_buffer));
173 _offset.push_back(offset);
174 }
175
176 /* 1 -1 0 */
177 offset = Index(1,-1,0);
178 if (_is_valid(coords + offset, dims, periods)) {
179 index = to_1d(offset+1);
180 subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.Size().Z());
181 starts = Index(l.End().X(), 0, l.Begin().Z());
182 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 9, 8, alloc_buffer));
183 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
184 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 9, 8, alloc_buffer));
185 _offset.push_back(offset);
186 }
187
188 /* 1 1 0 */
189 offset = Index(1,1,0);
190 if (_is_valid(coords + offset, dims, periods)) {
191 index = to_1d(offset+1);
192 subsizes = Index(l.HaloSize2().X(), l.HaloSize2().Y(), l.Size().Z());
193 starts = Index(l.End().X(), l.End().Y(), l.Begin().Z());
194 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 7, 6, alloc_buffer));
195 starts = Index(l.End().X()-l.HaloSize2().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
196 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 7, 6, alloc_buffer));
197 _offset.push_back(offset);
198 }
199
200 /* -1 0 -1 */
201 offset = Index(-1,0,-1);
202 if (_is_valid(coords + offset, dims, periods)) {
203 index = to_1d(offset+1);
204 subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.HaloSize1().Z());
205 starts = Index(0, l.Begin().Y(), 0);
206 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 10, 11, alloc_buffer));
207 starts = l.Begin();
208 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 10, 11, alloc_buffer));
209 _offset.push_back(offset);
210 }
211
212 /* -1 0 1 */
213 offset = Index(-1,0,1);
214 if (_is_valid(coords + offset, dims, periods)) {
215 index = to_1d(offset+1);
216 subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.HaloSize2().Z());
217 starts = Index(0, l.Begin().Y(), l.End().Z());
218 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 12, 13, alloc_buffer));
219 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
220 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 12, 13, alloc_buffer));
221 _offset.push_back(offset);
222 }
223
224 /* 1 0 -1 */
225 offset = Index(1,0,-1);
226 if (_is_valid(coords + offset, dims, periods)) {
227 index = to_1d(offset+1);
228 subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.HaloSize1().Z());
229 starts = Index(l.End().X(), l.Begin().Y(), 0);
230 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 13, 12, alloc_buffer));
231 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
232 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 13, 12, alloc_buffer));
233 _offset.push_back(offset);
234 }
235
236 /* 1 0 1 */
237 offset = Index(1,0,1);
238 if (_is_valid(coords + offset, dims, periods)) {
239 index = to_1d(offset+1);
240 subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.HaloSize2().Z());
241 starts = Index(l.End().X(), l.Begin().Y(), l.End().Z());
242 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 11, 10, alloc_buffer));
243 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
244 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 11, 10, alloc_buffer));
245 _offset.push_back(offset);
246 }
247
248 /* 0 -1 -1 */
249 offset = Index(0,-1,-1);
250 if (_is_valid(coords + offset, dims, periods)) {
251 index = to_1d(offset+1);
252 subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.HaloSize1().Z());
253 starts = Index(l.Begin().X(), 0, 0);
254 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15, alloc_buffer));
255 starts = l.Begin();
256 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15, alloc_buffer));
257 _offset.push_back(offset);
258 }
259
260 /* 0 -1 1 */
261 offset = Index(0,-1,1);
262 if (_is_valid(coords + offset, dims, periods)) {
263 index = to_1d(offset+1);
264 subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
265 starts = Index(l.Begin().X(), 0, l.End().Z());
266 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17, alloc_buffer));
267 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
268 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17, alloc_buffer));
269 _offset.push_back(offset);
270 }
271
272 /* 0 1 -1 */
273 offset = Index(0,1,-1);
274 if (_is_valid(coords + offset, dims, periods)) {
275 index = to_1d(offset+1);
276 subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
277 starts = Index(l.Begin().X(), l.End().Y(), 0);
278 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16, alloc_buffer));
279 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
280 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16, alloc_buffer));
281 _offset.push_back(offset);
282 }
283
284 /* 0 1 1 */
285 offset = Index(0,1,1);
286 if (_is_valid(coords + offset, dims, periods)) {
287 index = to_1d(offset+1);
288 subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.HaloSize2().Z());
289 starts = Index(l.Begin().X(), l.End().Y(), l.End().Z());
290 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14, alloc_buffer));
291 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.End().Z()-l.HaloSize2().Z());
292 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14, alloc_buffer));
293 _offset.push_back(offset);
294 }
295
296 /* -1 -1 -1 */
297 offset = Index(-1,-1,-1);
298 if (_is_valid(coords + offset, dims, periods)) {
299 index = to_1d(offset+1);
300 subsizes = l.HaloSize1();
301 starts = 0;
302 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19, alloc_buffer));
303 starts = l.Begin();
304 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19, alloc_buffer));
305 _offset.push_back(offset);
306 }
307
308 /* -1 -1 1 */
309 offset = Index(-1,-1,1);
310 if (_is_valid(coords + offset, dims, periods)) {
311 index = to_1d(offset+1);
312 subsizes = Index(l.HaloSize1().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
313 starts = Index(0, 0, l.End().Z());;
314 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21, alloc_buffer));
315 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
316 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21, alloc_buffer));
317 _offset.push_back(offset);
318 }
319
320 /* -1 1 -1 */
321 offset = Index(-1,1,-1);
322 if (_is_valid(coords + offset, dims, periods)) {
323 index = to_1d(offset+1);
324 subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
325 starts = Index(0, l.End().Y(), 0);
326 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23, alloc_buffer));
327 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
328 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23, alloc_buffer));
329 _offset.push_back(offset);
330 }
331
332 /* 1 -1 -1 */
333 offset = Index(1,-1,-1);
334 if (_is_valid(coords + offset, dims, periods)) {
335 index = to_1d(offset+1);
336 subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.HaloSize1().Z());
337 starts = Index(l.End().X(), 0, 0);
338 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25, alloc_buffer));
339 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
340 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25, alloc_buffer));
341 _offset.push_back(offset);
342 }
343
344 /* -1 1 1 */
345 offset = Index(-1,1,1);
346 if (_is_valid(coords + offset, dims, periods)) {
347 index = to_1d(offset+1);
348 subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.HaloSize2().Z());
349 starts = Index(0, l.End().Y(), l.End().Z());
350 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24, alloc_buffer));
351 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.End().Z()-l.HaloSize2().Z());
352 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24, alloc_buffer));
353 _offset.push_back(offset);
354 }
355
356 /* 1 -1 1 */
357 offset = Index(1,-1,1);
358 if (_is_valid(coords + offset, dims, periods)) {
359 index = to_1d(offset+1);
360 subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
361 starts = Index(l.End().X(), 0, l.End().Z());
362 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22, alloc_buffer));
363 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
364 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22, alloc_buffer));
365 _offset.push_back(offset);
366 }
367
368 /* 1 1 -1 */
369 offset = Index(1,1,-1);
370 if (_is_valid(coords + offset, dims, periods)) {
371 index = to_1d(offset+1);
372 subsizes = Index(l.HaloSize2().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
373 starts = Index(l.End().X(), l.End().Y(), 0);
374 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20, alloc_buffer));
375 starts = Index(l.End().X()-l.HaloSize2().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
376 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20, alloc_buffer));
377 _offset.push_back(offset);
378 }
379
380 /* 1 1 1 */
381 offset = Index(1,1,1);
382 if (_is_valid(coords + offset, dims, periods)) {
383 index = to_1d(offset+1);
384 subsizes = l.HaloSize2();
385 starts = l.End();
386 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18, alloc_buffer));
387 starts = l.End()-l.HaloSize2();
388 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18, alloc_buffer));
389 _offset.push_back(offset);
390 }
391
392 }
393}
394
395}
396
397}
398
399#endif /* DATATYPES_LOCAL_HPP_ */
Note: See TracBrowser for help on using the repository browser.