source: src/comm/mpi/datatypes_local.cpp@ 1a4dd81

Last change on this file since 1a4dd81 was 894a5f, checked in by Julian Iseringhausen <isering@…>, 14 years ago

Parallel performance update.

git-svn-id: ​https://svn.version.fz-juelich.de/scafacos/trunk@1314 5161e1c8-67bf-11de-9fd5-51895aff932f
  • Property mode set to 100644
File size: 12.6 KB
Line 
1#ifdef HAVE_CONFIG_H
2#include <config.h>
3#endif
4
5#ifdef HAVE_MPI
6#include <mpi.h>
7#else
8#error MPI is needed to compile VMG::MPI::DatatypesLocal
9#endif
10
11#include <map>
12
13#include "base/index.hpp"
14#include "comm/mpi/datatypes_local.hpp"
15#include "comm/mpi/key.hpp"
16#include "grid/grid.hpp"
17
18using namespace VMG;
19
20static inline int to_1d(const int& x, const int& y, const int& z)
21{
22 return z+3*(y+3*x);
23}
24
25VMG::MPI::DatatypesLocal::DatatypesLocal(const Grid& grid, const MPI_Comm& comm)
26{
27 if (comm != MPI_COMM_NULL) {
28
29 int index, my_rank, ranks[27];
30 Index my_coords;
31 Index sizes, subsizes, starts;
32 const LocalIndices& l = grid.Local();
33
34 MPI_Comm_rank(comm, &my_rank);
35 MPI_Cart_coords(comm, my_rank, 3, my_coords.vec());
36
37 for (int i=-1; i<=1; ++i)
38 for (int j=-1; j<=1; ++j)
39 for (int k=-1; k<=1; ++k)
40 MPI_Cart_rank(comm, (my_coords+Index(i,j,k)).vec(), &ranks[to_1d(i+1,j+1,k+1)]);
41
42 sizes = l.SizeTotal();
43
44 /* -1 0 0 */
45 index = to_1d(0,1,1);
46 subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.Size().Y(), l.Size().Z());
47 starts = Index(0, l.Begin().Y(), l.Begin().Z());
48 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 0, 1));
49 starts = l.Begin();
50 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 0, 1));
51
52 /* 1 0 0 */
53 index = to_1d(2,1,1);
54 subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.Size().Y(), l.Size().Z());
55 starts = Index(l.End().X(), l.Begin().Y(), l.Begin().Z());
56 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 1, 0));
57 starts = Index(l.End().X()-l.HaloEnd2().X()+l.HaloBegin2().X(), l.Begin().Y(), l.Begin().Z());
58 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 1, 0));
59
60 /* 0 -1 0 */
61 index = to_1d(1,0,1);
62 subsizes = Index(l.Size().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.Size().Z());
63 starts = Index(l.Begin().X(), 0, l.Begin().Z());
64 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 2, 3));
65 starts = l.Begin();
66 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 2, 3));
67
68 /* 0 1 0 */
69 index = to_1d(1,2,1);
70 subsizes = Index(l.Size().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.Size().Z());
71 starts = Index(l.Begin().X(), l.End().Y(), l.Begin().Z());
72 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 3, 2));
73 starts = Index(l.Begin().X(), l.End().Y()-l.HaloEnd2().Y()+l.HaloBegin2().Y(), l.Begin().Z());
74 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 3, 2));
75
76 /* 0 0 -1 */
77 index = to_1d(1,1,0);
78 subsizes = Index(l.Size().X(), l.Size().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
79 starts = Index(l.Begin().X(), l.Begin().Y(), 0);
80 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 4, 5));
81 starts = l.Begin();
82 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 4, 5));
83
84 /* 0 0 1 */
85 index = to_1d(1,1,2);
86 subsizes = Index(l.Size().X(), l.Size().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
87 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z());
88 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 5, 4));
89 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloEnd2().Z()+l.HaloBegin2().Z());
90 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 5, 4));
91
92 /* -1 -1 0 */
93 index = to_1d(0,0,1);
94 subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.Size().Z());
95 starts = Index(0, 0, l.Begin().Z());
96 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 6, 7));
97 starts = l.Begin();
98 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 6, 7));
99
100 /* -1 1 0 */
101 index = to_1d(0,2,1);
102 subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.Size().Z());
103 starts = Index(0, l.End().Y(), l.Begin().Z());
104 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 8, 9));
105 starts = Index(l.Begin().X(), l.End().Y()-l.HaloEnd2().Y()+l.HaloBegin2().Y(), l.Begin().Z());
106 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 8, 9));
107
108 /* 1 -1 0 */
109 index = to_1d(2,0,1);
110 subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.Size().Z());
111 starts = Index(l.End().X(), 0, l.Begin().Z());
112 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 9, 8));
113 starts = Index(l.End().X()-l.HaloEnd2().X()+l.HaloBegin2().X(), l.Begin().Y(), l.Begin().Z());
114 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 9, 8));
115
116 /* 1 1 0 */
117 index = to_1d(2,2,1);
118 subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.Size().Z());
119 starts = Index(l.End().X(), l.End().Y(), l.Begin().Z());
120 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 7, 6));
121 starts = Index(l.End().X()-l.HaloEnd2().X()+l.HaloBegin2().X(), l.End().Y()-l.HaloEnd2().Y()+l.HaloBegin2().Y(), l.Begin().Z());
122 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 7, 6));
123
124 /* -1 0 -1 */
125 index = to_1d(0,1,0);
126 subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.Size().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
127 starts = Index(0, l.Begin().Y(), 0);
128 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 10, 11));
129 starts = l.Begin();
130 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 10, 11));
131
132 /* -1 0 1 */
133 index = to_1d(0,1,2);
134 subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.Size().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
135 starts = Index(0, l.Begin().Y(), l.End().Z());
136 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 12, 13));
137 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloEnd2().Z()+l.HaloBegin2().Z());
138 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 12, 13));
139
140 /* 1 0 -1 */
141 index = to_1d(2,1,0);
142 subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.Size().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
143 starts = Index(l.End().X(), l.Begin().Y(), 0);
144 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 13, 12));
145 starts = Index(l.End().X()-l.HaloEnd2().X()+l.HaloBegin2().X(), l.Begin().Y(), l.Begin().Z());
146 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 13, 12));
147
148 /* 1 0 1 */
149 index = to_1d(2,1,2);
150 subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.Size().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
151 starts = Index(l.End().X(), l.Begin().Y(), l.End().Z());
152 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 11, 10));
153 starts = Index(l.End().X()-l.HaloEnd2().X()+l.HaloBegin2().X(), l.Begin().Y(), l.End().Z()-l.HaloEnd2().Z()+l.HaloBegin2().Z());
154 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 11, 10));
155
156 /* 0 -1 -1 */
157 index = to_1d(1,0,0);
158 subsizes = Index(l.Size().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
159 starts = Index(l.Begin().X(), 0, 0);
160 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15));
161 starts = l.Begin();
162 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15));
163
164 /* 0 -1 1 */
165 index = to_1d(1,0,2);
166 subsizes = Index(l.Size().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
167 starts = Index(l.Begin().X(), 0, l.End().Z());
168 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17));
169 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloEnd2().Z()+l.HaloBegin2().Z());
170 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17));
171
172 /* 0 1 -1 */
173 index = to_1d(1,2,0);
174 subsizes = Index(l.Size().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
175 starts = Index(l.Begin().X(), l.End().Y(), 0);
176 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16));
177 starts = Index(l.Begin().X(), l.End().Y()-l.HaloEnd2().Y()+l.HaloBegin2().Y(), l.Begin().Z());
178 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16));
179
180 /* 0 1 1 */
181 index = to_1d(1,2,2);
182 subsizes = Index(l.Size().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
183 starts = Index(l.Begin().X(), l.End().Y(), l.End().Z());
184 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14));
185 starts = Index(l.Begin().X(), l.End().Y()-l.HaloEnd2().Y()+l.HaloBegin2().Y(), l.End().Z()-l.HaloEnd2().Z()+l.HaloBegin2().Z());
186 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14));
187
188 /* -1 -1 -1 */
189 index = to_1d(0,0,0);
190 subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
191 starts = 0;
192 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19));
193 starts = l.Begin();
194 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19));
195
196 /* -1 -1 1 */
197 index = to_1d(0,0,2);
198 subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
199 starts = Index(0, 0, l.End().Z());;
200 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21));
201 starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloEnd2().Z()+l.HaloBegin2().Z());
202 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21));
203
204 /* -1 1 -1 */
205 index = to_1d(0,2,0);
206 subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
207 starts = Index(0, l.End().Y(), 0);
208 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23));
209 starts = Index(l.Begin().X(), l.End().Y()-l.HaloEnd2().Y()+l.HaloBegin2().Y(), l.Begin().Z());
210 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23));
211
212 /* 1 -1 -1 */
213 index = to_1d(2,0,0);
214 subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
215 starts = Index(l.End().X(), 0, 0);
216 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25));
217 starts = Index(l.End().X()-l.HaloEnd2().X()+l.HaloBegin2().X(), l.Begin().Y(), l.Begin().Z());
218 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25));
219
220 /* -1 1 1 */
221 index = to_1d(0,2,2);
222 subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
223 starts = Index(0, l.End().Y(), l.End().Z());
224 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24));
225 starts = Index(l.Begin().X(), l.End().Y()-l.HaloEnd2().Y()+l.HaloBegin2().Y(), l.End().Z()-l.HaloEnd2().Z()+l.HaloBegin2().Z());
226 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24));
227
228 /* 1 -1 1 */
229 index = to_1d(2,0,2);
230 subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
231 starts = Index(l.End().X(), 0, l.End().Z());
232 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22));
233 starts = Index(l.End().X()-l.HaloEnd2().X()+l.HaloBegin2().X(), l.Begin().Y(), l.End().Z()-l.HaloEnd2().Z()+l.HaloBegin2().Z());
234 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22));
235
236 /* 1 1 -1 */
237 index = to_1d(2,2,0);
238 subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
239 starts = Index(l.End().X(), l.End().Y(), 0);
240 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20));
241 starts = Index(l.End().X()-l.HaloEnd2().X()+l.HaloBegin2().X(), l.End().Y()-l.HaloEnd2().Y()+l.HaloBegin2().Y(), l.Begin().Z());
242 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20));
243
244 /* 1 1 1 */
245 index = to_1d(2,2,2);
246 subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
247 starts = l.End();
248 halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18));
249 starts = l.End()-l.HaloEnd2()+l.HaloBegin2();
250 nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18));
251
252 }
253}
Note: See TracBrowser for help on using the repository browser.