source: src/comm/mpi/datatypes_local.cpp@ 716da7

Last change on this file since 716da7 was 716da7, checked in by Julian Iseringhausen <isering@…>, 14 years ago

Fix energy calculation.

git-svn-id: ​https://svn.version.fz-juelich.de/scafacos/trunk@1729 5161e1c8-67bf-11de-9fd5-51895aff932f
  • Property mode set to 100644
File size: 16.3 KB
Line 
1#ifdef HAVE_CONFIG_H
2#include <config.h>
3#endif
4
5#ifdef HAVE_MPI
6#include <mpi.h>
7#ifdef HAVE_MARMOT
8#include <enhancempicalls.h>
9#include <sourceinfompicalls.h>
10#endif
11#else
12#error MPI is needed to compile VMG::MPI::DatatypesLocal
13#endif
14
15#include <map>
16
17#include "base/index.hpp"
18#include "base/linked_cell_list.hpp"
19#include "comm/mpi/datatypes_local.hpp"
20#include "comm/mpi/key.hpp"
21#include "grid/grid.hpp"
22
23using namespace VMG;
24
25static inline int to_1d(const Index& i) __attribute__((always_inline));
26static inline int to_1d(const Index& i)
27{
28 return i.Z()+3*(i.Y()+3*i.X());
29}
30
31static inline bool is_valid(const Index& coord, const Index& dims, const Index& periods)
32{
33 return (periods[0] || (coord[0] >= 0 && coord[0] < dims[0])) &&
34 (periods[1] || (coord[1] >= 0 && coord[1] < dims[1])) &&
35 (periods[2] || (coord[2] >= 0 && coord[2] < dims[2]));
36}
37
38template <class T>
39void VMG::MPI::DatatypesLocal::InitDatatypesLocal(const T& grid, const MPI_Comm& comm, const bool& alloc_buffer)
40{
41 if (comm != MPI_COMM_NULL) {
42
43 int index, ranks[27];
44 Index dims, periods, coords;
45 Index sizes, subsizes, starts;
46 Index offset, i;
47 const LocalIndices& l = grid.Local();
48
49 MPI_Cart_get(comm, 3, dims.vec(), periods.vec(), coords.vec());
50
51 for (i.X()=-1; i.X()<=1; ++i.X())
52 for (i.Y()=-1; i.Y()<=1; ++i.Y())
53 for (i.Z()=-1; i.Z()<=1; ++i.Z())
54 if (is_valid(coords + i, dims, periods))
55 MPI_Cart_rank(comm, (coords + i).vec(), &ranks[to_1d(i+1)]);
56
57 sizes = l.SizeTotal();
58
59 /* -1 0 0 */
60 offset = Index(-1,0,0);
61 if (is_valid(coords + offset, dims, periods)) {
62 index = to_1d(offset+1);
63 subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.Size().Z());
64 starts = Index(0, l.Begin().Y(), l.Begin().Z());
65 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 0, 1, alloc_buffer));
66 starts = l.Begin();
67 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 0, 1, alloc_buffer));
68 _offset.push_back(offset);
69 }
70
71 /* 1 0 0 */
72 offset = Index(1,0,0);
73 if (is_valid(coords + offset, dims, periods)) {
74 index = to_1d(offset+1);
75 subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.Size().Z());
76 starts = Index(l.End().X(), l.Begin().Y(), l.Begin().Z());
77 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 1, 0, alloc_buffer));
78 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
79 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 1, 0, alloc_buffer));
80 _offset.push_back(offset);
81 }
82
83 /* 0 -1 0 */
84 offset = Index(0,-1,0);
85 if (is_valid(coords + offset, dims, periods)) {
86 index = to_1d(offset+1);
87 subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.Size().Z());
88 starts = Index(l.Begin().X(), 0, l.Begin().Z());
89 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 2, 3, alloc_buffer));
90 starts = l.Begin();
91 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 2, 3, alloc_buffer));
92 _offset.push_back(offset);
93 }
94
95 /* 0 1 0 */
96 offset = Index(0,1,0);
97 if (is_valid(coords + offset, dims, periods)) {
98 index = to_1d(offset+1);
99 subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.Size().Z());
100 starts = Index(l.Begin().X(), l.End().Y(), l.Begin().Z());
101 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 3, 2, alloc_buffer));
102 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
103 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 3, 2, alloc_buffer));
104 _offset.push_back(offset);
105 }
106
107 /* 0 0 -1 */
108 offset = Index(0,0,-1);
109 if (is_valid(coords + offset, dims, periods)) {
110 index = to_1d(offset+1);
111 subsizes = Index(l.Size().X(), l.Size().Y(), l.HaloSize1().Z());
112 starts = Index(l.Begin().X(), l.Begin().Y(), 0);
113 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 4, 5, alloc_buffer));
114 starts = l.Begin();
115 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 4, 5, alloc_buffer));
116 _offset.push_back(offset);
117 }
118
119 /* 0 0 1 */
120 offset = Index(0,0,1);
121 if (is_valid(coords + offset, dims, periods)) {
122 index = to_1d(offset+1);
123 subsizes = Index(l.Size().X(), l.Size().Y(), l.HaloSize2().Z());
124 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z());
125 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 5, 4, alloc_buffer));
126 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
127 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 5, 4, alloc_buffer));
128 _offset.push_back(offset);
129 }
130
131 /* -1 -1 0 */
132 offset = Index(-1,-1,0);
133 if (is_valid(coords + offset, dims, periods)) {
134 index = to_1d(offset+1);
135 subsizes = Index(l.HaloSize1().X(), l.HaloSize1().Y(), l.Size().Z());
136 starts = Index(0, 0, l.Begin().Z());
137 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 6, 7, alloc_buffer));
138 starts = l.Begin();
139 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 6, 7, alloc_buffer));
140 _offset.push_back(offset);
141 }
142
143 /* -1 1 0 */
144 offset = Index(-1,1,0);
145 if (is_valid(coords + offset, dims, periods)) {
146 index = to_1d(offset+1);
147 subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.Size().Z());
148 starts = Index(0, l.End().Y(), l.Begin().Z());
149 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 8, 9, alloc_buffer));
150 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
151 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 8, 9, alloc_buffer));
152 _offset.push_back(offset);
153 }
154
155 /* 1 -1 0 */
156 offset = Index(1,-1,0);
157 if (is_valid(coords + offset, dims, periods)) {
158 index = to_1d(offset+1);
159 subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.Size().Z());
160 starts = Index(l.End().X(), 0, l.Begin().Z());
161 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 9, 8, alloc_buffer));
162 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
163 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 9, 8, alloc_buffer));
164 _offset.push_back(offset);
165 }
166
167 /* 1 1 0 */
168 offset = Index(1,1,0);
169 if (is_valid(coords + offset, dims, periods)) {
170 index = to_1d(offset+1);
171 subsizes = Index(l.HaloSize2().X(), l.HaloSize2().Y(), l.Size().Z());
172 starts = Index(l.End().X(), l.End().Y(), l.Begin().Z());
173 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 7, 6, alloc_buffer));
174 starts = Index(l.End().X()-l.HaloSize2().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
175 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 7, 6, alloc_buffer));
176 _offset.push_back(offset);
177 }
178
179 /* -1 0 -1 */
180 offset = Index(-1,0,-1);
181 if (is_valid(coords + offset, dims, periods)) {
182 index = to_1d(offset+1);
183 subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.HaloSize1().Z());
184 starts = Index(0, l.Begin().Y(), 0);
185 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 10, 11, alloc_buffer));
186 starts = l.Begin();
187 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 10, 11, alloc_buffer));
188 _offset.push_back(offset);
189 }
190
191 /* -1 0 1 */
192 offset = Index(-1,0,1);
193 if (is_valid(coords + offset, dims, periods)) {
194 index = to_1d(offset+1);
195 subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.HaloSize2().Z());
196 starts = Index(0, l.Begin().Y(), l.End().Z());
197 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 12, 13, alloc_buffer));
198 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
199 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 12, 13, alloc_buffer));
200 _offset.push_back(offset);
201 }
202
203 /* 1 0 -1 */
204 offset = Index(1,0,-1);
205 if (is_valid(coords + offset, dims, periods)) {
206 index = to_1d(offset+1);
207 subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.HaloSize1().Z());
208 starts = Index(l.End().X(), l.Begin().Y(), 0);
209 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 13, 12, alloc_buffer));
210 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
211 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 13, 12, alloc_buffer));
212 _offset.push_back(offset);
213 }
214
215 /* 1 0 1 */
216 offset = Index(1,0,1);
217 if (is_valid(coords + offset, dims, periods)) {
218 index = to_1d(offset+1);
219 subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.HaloSize2().Z());
220 starts = Index(l.End().X(), l.Begin().Y(), l.End().Z());
221 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 11, 10, alloc_buffer));
222 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
223 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 11, 10, alloc_buffer));
224 _offset.push_back(offset);
225 }
226
227 /* 0 -1 -1 */
228 offset = Index(0,-1,-1);
229 if (is_valid(coords + offset, dims, periods)) {
230 index = to_1d(offset+1);
231 subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.HaloSize1().Z());
232 starts = Index(l.Begin().X(), 0, 0);
233 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15, alloc_buffer));
234 starts = l.Begin();
235 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15, alloc_buffer));
236 _offset.push_back(offset);
237 }
238
239 /* 0 -1 1 */
240 offset = Index(0,-1,1);
241 if (is_valid(coords + offset, dims, periods)) {
242 index = to_1d(offset+1);
243 subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
244 starts = Index(l.Begin().X(), 0, l.End().Z());
245 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17, alloc_buffer));
246 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
247 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17, alloc_buffer));
248 _offset.push_back(offset);
249 }
250
251 /* 0 1 -1 */
252 offset = Index(0,1,-1);
253 if (is_valid(coords + offset, dims, periods)) {
254 index = to_1d(offset+1);
255 subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
256 starts = Index(l.Begin().X(), l.End().Y(), 0);
257 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16, alloc_buffer));
258 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
259 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16, alloc_buffer));
260 _offset.push_back(offset);
261 }
262
263 /* 0 1 1 */
264 offset = Index(0,1,1);
265 if (is_valid(coords + offset, dims, periods)) {
266 index = to_1d(offset+1);
267 subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.HaloSize2().Z());
268 starts = Index(l.Begin().X(), l.End().Y(), l.End().Z());
269 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14, alloc_buffer));
270 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.End().Z()-l.HaloSize2().Z());
271 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14, alloc_buffer));
272 _offset.push_back(offset);
273 }
274
275 /* -1 -1 -1 */
276 offset = Index(-1,-1,-1);
277 if (is_valid(coords + offset, dims, periods)) {
278 index = to_1d(offset+1);
279 subsizes = l.HaloSize1();
280 starts = 0;
281 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19, alloc_buffer));
282 starts = l.Begin();
283 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19, alloc_buffer));
284 _offset.push_back(offset);
285 }
286
287 /* -1 -1 1 */
288 offset = Index(-1,-1,1);
289 if (is_valid(coords + offset, dims, periods)) {
290 index = to_1d(offset+1);
291 subsizes = Index(l.HaloSize1().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
292 starts = Index(0, 0, l.End().Z());;
293 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21, alloc_buffer));
294 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
295 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21, alloc_buffer));
296 _offset.push_back(offset);
297 }
298
299 /* -1 1 -1 */
300 offset = Index(-1,1,-1);
301 if (is_valid(coords + offset, dims, periods)) {
302 index = to_1d(offset+1);
303 subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
304 starts = Index(0, l.End().Y(), 0);
305 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23, alloc_buffer));
306 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
307 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23, alloc_buffer));
308 _offset.push_back(offset);
309 }
310
311 /* 1 -1 -1 */
312 offset = Index(1,-1,-1);
313 if (is_valid(coords + offset, dims, periods)) {
314 index = to_1d(offset+1);
315 subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.HaloSize1().Z());
316 starts = Index(l.End().X(), 0, 0);
317 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25, alloc_buffer));
318 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
319 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25, alloc_buffer));
320 _offset.push_back(offset);
321 }
322
323 /* -1 1 1 */
324 offset = Index(-1,1,1);
325 if (is_valid(coords + offset, dims, periods)) {
326 index = to_1d(offset+1);
327 subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.HaloSize2().Z());
328 starts = Index(0, l.End().Y(), l.End().Z());
329 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24, alloc_buffer));
330 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.End().Z()-l.HaloSize2().Z());
331 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24, alloc_buffer));
332 _offset.push_back(offset);
333 }
334
335 /* 1 -1 1 */
336 offset = Index(1,-1,1);
337 if (is_valid(coords + offset, dims, periods)) {
338 index = to_1d(offset+1);
339 subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
340 starts = Index(l.End().X(), 0, l.End().Z());
341 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22, alloc_buffer));
342 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
343 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22, alloc_buffer));
344 _offset.push_back(offset);
345 }
346
347 /* 1 1 -1 */
348 offset = Index(1,1,-1);
349 if (is_valid(coords + offset, dims, periods)) {
350 index = to_1d(offset+1);
351 subsizes = Index(l.HaloSize2().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
352 starts = Index(l.End().X(), l.End().Y(), 0);
353 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20, alloc_buffer));
354 starts = Index(l.End().X()-l.HaloSize2().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
355 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20, alloc_buffer));
356 _offset.push_back(offset);
357 }
358
359 /* 1 1 1 */
360 offset = Index(1,1,1);
361 if (is_valid(coords + offset, dims, periods)) {
362 index = to_1d(offset+1);
363 subsizes = l.HaloSize2();
364 starts = l.End();
365 _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18, alloc_buffer));
366 starts = l.End()-l.HaloSize2();
367 _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18, alloc_buffer));
368 _offset.push_back(offset);
369 }
370
371 }
372}
373
374template void VMG::MPI::DatatypesLocal::InitDatatypesLocal<VMG::Grid>(const VMG::Grid& grid, const MPI_Comm& comm, const bool& alloc_buffer);
375template void VMG::MPI::DatatypesLocal::InitDatatypesLocal<VMG::Particle::LinkedCellList>(const VMG::Particle::LinkedCellList& grid, const MPI_Comm& comm, const bool& alloc_buffer);
376
Note: See TracBrowser for help on using the repository browser.