source: src/comm/mpi/datatypes_local.cpp@ ac6d04

Last change on this file since ac6d04 was ac6d04, checked in by Julian Iseringhausen <isering@…>, 14 years ago

Merge recent changes of the vmg library into ScaFaCos.

Includes a fix for the communication problems on Jugene.

git-svn-id: ​https://svn.version.fz-juelich.de/scafacos/trunk@1666 5161e1c8-67bf-11de-9fd5-51895aff932f
  • Property mode set to 100644
File size: 13.9 KB
Line 
1#ifdef HAVE_CONFIG_H
2#include <config.h>
3#endif
4
5#ifdef HAVE_MPI
6#include <mpi.h>
7#ifdef HAVE_MARMOT
8#include <enhancempicalls.h>
9#include <sourceinfompicalls.h>
10#endif
11#else
12#error MPI is needed to compile VMG::MPI::DatatypesLocal
13#endif
14
15#include <map>
16
17#include "base/index.hpp"
18#include "base/linked_cell_list.hpp"
19#include "comm/mpi/datatypes_local.hpp"
20#include "comm/mpi/key.hpp"
21#include "grid/grid.hpp"
22
23using namespace VMG;
24
25static inline int to_1d(const int& x, const int& y, const int& z)
26{
27 return z+3*(y+3*x);
28}
29
30static inline bool is_valid(const Index& coord, const Index& dims, const Index& periods)
31{
32 return (periods[0] || (coord[0] >= 0 && coord[0] < dims[0])) &&
33 (periods[1] || (coord[1] >= 0 && coord[1] < dims[1])) &&
34 (periods[2] || (coord[2] >= 0 && coord[2] < dims[2]));
35}
36
37template <class T>
38void VMG::MPI::DatatypesLocal::InitDatatypesLocal(const T& grid, const MPI_Comm& comm)
39{
40 if (comm != MPI_COMM_NULL) {
41
42 int index, ranks[27];
43 Index dims, periods, coords;
44 Index sizes, subsizes, starts;
45 const LocalIndices& l = grid.Local();
46
47 MPI_Cart_get(comm, 3, dims.vec(), periods.vec(), coords.vec());
48
49 for (int i=-1; i<=1; ++i)
50 for (int j=-1; j<=1; ++j)
51 for (int k=-1; k<=1; ++k)
52 if (is_valid(coords + Index(i,j,k), dims, periods))
53 MPI_Cart_rank(comm, (coords + Index(i,j,k)).vec(), &ranks[to_1d(i+1,j+1,k+1)]);
54
55 sizes = l.SizeTotal();
56
57 /* -1 0 0 */
58 if (is_valid(coords + Index(-1,0,0), dims, periods)) {
59 index = to_1d(0,1,1);
60 subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.Size().Z());
61 starts = Index(0, l.Begin().Y(), l.Begin().Z());
62 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 0, 1));
63 starts = l.Begin();
64 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 0, 1));
65 }
66
67 /* 1 0 0 */
68 if (is_valid(coords + Index(1,0,0), dims, periods)) {
69 index = to_1d(2,1,1);
70 subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.Size().Z());
71 starts = Index(l.End().X(), l.Begin().Y(), l.Begin().Z());
72 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 1, 0));
73 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
74 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 1, 0));
75 }
76
77 /* 0 -1 0 */
78 if (is_valid(coords + Index(0,-1,0), dims, periods)) {
79 index = to_1d(1,0,1);
80 subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.Size().Z());
81 starts = Index(l.Begin().X(), 0, l.Begin().Z());
82 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 2, 3));
83 starts = l.Begin();
84 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 2, 3));
85 }
86
87 /* 0 1 0 */
88 if (is_valid(coords + Index(0,1,0), dims, periods)) {
89 index = to_1d(1,2,1);
90 subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.Size().Z());
91 starts = Index(l.Begin().X(), l.End().Y(), l.Begin().Z());
92 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 3, 2));
93 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
94 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 3, 2));
95 }
96
97 /* 0 0 -1 */
98 if (is_valid(coords + Index(0,0,-1), dims, periods)) {
99 index = to_1d(1,1,0);
100 subsizes = Index(l.Size().X(), l.Size().Y(), l.HaloSize1().Z());
101 starts = Index(l.Begin().X(), l.Begin().Y(), 0);
102 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 4, 5));
103 starts = l.Begin();
104 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 4, 5));
105 }
106
107 /* 0 0 1 */
108 if (is_valid(coords + Index(0,0,1), dims, periods)) {
109 index = to_1d(1,1,2);
110 subsizes = Index(l.Size().X(), l.Size().Y(), l.HaloSize2().Z());
111 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z());
112 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 5, 4));
113 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
114 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 5, 4));
115 }
116
117 /* -1 -1 0 */
118 if (is_valid(coords + Index(-1,-1,0), dims, periods)) {
119 index = to_1d(0,0,1);
120 subsizes = Index(l.HaloSize1().X(), l.HaloSize1().Y(), l.Size().Z());
121 starts = Index(0, 0, l.Begin().Z());
122 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 6, 7));
123 starts = l.Begin();
124 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 6, 7));
125 }
126
127 /* -1 1 0 */
128 if (is_valid(coords + Index(-1,1,0), dims, periods)) {
129 index = to_1d(0,2,1);
130 subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.Size().Z());
131 starts = Index(0, l.End().Y(), l.Begin().Z());
132 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 8, 9));
133 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
134 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 8, 9));
135 }
136
137 /* 1 -1 0 */
138 if (is_valid(coords + Index(1,-1,0), dims, periods)) {
139 index = to_1d(2,0,1);
140 subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.Size().Z());
141 starts = Index(l.End().X(), 0, l.Begin().Z());
142 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 9, 8));
143 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
144 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 9, 8));
145 }
146
147 /* 1 1 0 */
148 if (is_valid(coords + Index(1,1,0), dims, periods)) {
149 index = to_1d(2,2,1);
150 subsizes = Index(l.HaloSize2().X(), l.HaloSize2().Y(), l.Size().Z());
151 starts = Index(l.End().X(), l.End().Y(), l.Begin().Z());
152 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 7, 6));
153 starts = Index(l.End().X()-l.HaloSize2().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
154 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 7, 6));
155 }
156
157 /* -1 0 -1 */
158 if (is_valid(coords + Index(-1,0,-1), dims, periods)) {
159 index = to_1d(0,1,0);
160 subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.HaloSize1().Z());
161 starts = Index(0, l.Begin().Y(), 0);
162 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 10, 11));
163 starts = l.Begin();
164 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 10, 11));
165 }
166
167 /* -1 0 1 */
168 if (is_valid(coords + Index(-1,0,1), dims, periods)) {
169 index = to_1d(0,1,2);
170 subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.HaloSize2().Z());
171 starts = Index(0, l.Begin().Y(), l.End().Z());
172 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 12, 13));
173 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
174 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 12, 13));
175 }
176
177 /* 1 0 -1 */
178 if (is_valid(coords + Index(1,0,-1), dims, periods)) {
179 index = to_1d(2,1,0);
180 subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.HaloSize1().Z());
181 starts = Index(l.End().X(), l.Begin().Y(), 0);
182 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 13, 12));
183 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
184 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 13, 12));
185 }
186
187 /* 1 0 1 */
188 if (is_valid(coords + Index(1,0,1), dims, periods)) {
189 index = to_1d(2,1,2);
190 subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.HaloSize2().Z());
191 starts = Index(l.End().X(), l.Begin().Y(), l.End().Z());
192 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 11, 10));
193 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
194 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 11, 10));
195 }
196
197 /* 0 -1 -1 */
198 if (is_valid(coords + Index(0,-1,-1), dims, periods)) {
199 index = to_1d(1,0,0);
200 subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.HaloSize1().Z());
201 starts = Index(l.Begin().X(), 0, 0);
202 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15));
203 starts = l.Begin();
204 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15));
205 }
206
207 /* 0 -1 1 */
208 if (is_valid(coords + Index(0,-1,1), dims, periods)) {
209 index = to_1d(1,0,2);
210 subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
211 starts = Index(l.Begin().X(), 0, l.End().Z());
212 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17));
213 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
214 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17));
215 }
216
217 /* 0 1 -1 */
218 if (is_valid(coords + Index(0,1,-1), dims, periods)) {
219 index = to_1d(1,2,0);
220 subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
221 starts = Index(l.Begin().X(), l.End().Y(), 0);
222 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16));
223 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
224 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16));
225 }
226
227 /* 0 1 1 */
228 if (is_valid(coords + Index(0,1,1), dims, periods)) {
229 index = to_1d(1,2,2);
230 subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.HaloSize2().Z());
231 starts = Index(l.Begin().X(), l.End().Y(), l.End().Z());
232 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14));
233 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.End().Z()-l.HaloSize2().Z());
234 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14));
235 }
236
237 /* -1 -1 -1 */
238 if (is_valid(coords + Index(-1,-1,-1), dims, periods)) {
239 index = to_1d(0,0,0);
240 subsizes = l.HaloSize1();
241 starts = 0;
242 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19));
243 starts = l.Begin();
244 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19));
245 }
246
247 /* -1 -1 1 */
248 if (is_valid(coords + Index(-1,-1,1), dims, periods)) {
249 index = to_1d(0,0,2);
250 subsizes = Index(l.HaloSize1().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
251 starts = Index(0, 0, l.End().Z());;
252 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21));
253 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
254 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21));
255 }
256
257 /* -1 1 -1 */
258 if (is_valid(coords + Index(-1,1,-1), dims, periods)) {
259 index = to_1d(0,2,0);
260 subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
261 starts = Index(0, l.End().Y(), 0);
262 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23));
263 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
264 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23));
265 }
266
267 /* 1 -1 -1 */
268 if (is_valid(coords + Index(1,-1,-1), dims, periods)) {
269 index = to_1d(2,0,0);
270 subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.HaloSize1().Z());
271 starts = Index(l.End().X(), 0, 0);
272 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25));
273 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
274 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25));
275 }
276
277 /* -1 1 1 */
278 if (is_valid(coords + Index(-1,1,1), dims, periods)) {
279 index = to_1d(0,2,2);
280 subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.HaloSize2().Z());
281 starts = Index(0, l.End().Y(), l.End().Z());
282 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24));
283 starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.End().Z()-l.HaloSize2().Z());
284 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24));
285 }
286
287 /* 1 -1 1 */
288 if (is_valid(coords + Index(1,-1,1), dims, periods)) {
289 index = to_1d(2,0,2);
290 subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
291 starts = Index(l.End().X(), 0, l.End().Z());
292 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22));
293 starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
294 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22));
295 }
296
297 /* 1 1 -1 */
298 if (is_valid(coords + Index(1,1,-1), dims, periods)) {
299 index = to_1d(2,2,0);
300 subsizes = Index(l.HaloSize2().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
301 starts = Index(l.End().X(), l.End().Y(), 0);
302 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20));
303 starts = Index(l.End().X()-l.HaloSize2().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
304 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20));
305 }
306
307 /* 1 1 1 */
308 if (is_valid(coords + Index(1,1,1), dims, periods)) {
309 index = to_1d(2,2,2);
310 subsizes = l.HaloSize2();
311 starts = l.End();
312 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18));
313 starts = l.End()-l.HaloSize2();
314 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18));
315 }
316
317 }
318}
319
320template void VMG::MPI::DatatypesLocal::InitDatatypesLocal<VMG::Grid>(const VMG::Grid& grid, const MPI_Comm& comm);
321template void VMG::MPI::DatatypesLocal::InitDatatypesLocal<VMG::Particle::LinkedCellList>(const VMG::Particle::LinkedCellList& grid, const MPI_Comm& comm);
322
Note: See TracBrowser for help on using the repository browser.