Ignore:
Timestamp:
Apr 10, 2012, 1:55:49 PM (14 years ago)
Author:
Julian Iseringhausen <isering@…>
Children:
a40eea
Parents:
d24c2f
Message:

Merge recent changes of the vmg library into ScaFaCos.

Includes a fix for the communication problems on Jugene.

git-svn-id: ​https://svn.version.fz-juelich.de/scafacos/trunk@1666 5161e1c8-67bf-11de-9fd5-51895aff932f

File:
1 edited

Legend:

Unmodified
Added
Removed

  • src/comm/mpi/datatypes_local.cpp

    rd24c2f rac6d04  
    55#ifdef HAVE_MPI
    66#include <mpi.h>
     7#ifdef HAVE_MARMOT
     8#include <enhancempicalls.h>
     9#include <sourceinfompicalls.h>
     10#endif
    711#else
    812#error MPI is needed to compile VMG::MPI::DatatypesLocal
    … …  
    1216
    1317#include "base/index.hpp"
     18#include "base/linked_cell_list.hpp"
    1419#include "comm/mpi/datatypes_local.hpp"
    1520#include "comm/mpi/key.hpp"
    … …  
    2328}
    2429
    25 VMG::MPI::DatatypesLocal::DatatypesLocal(const Grid& grid, const MPI_Comm& comm)
     30static inline bool is_valid(const Index& coord, const Index& dims, const Index& periods)
     31{
     32  return (periods[0] || (coord[0] >= 0 && coord[0] < dims[0])) &&
     33         (periods[1] || (coord[1] >= 0 && coord[1] < dims[1])) &&
     34         (periods[2] || (coord[2] >= 0 && coord[2] < dims[2]));
     35}
     36
     37template <class T>
     38void VMG::MPI::DatatypesLocal::InitDatatypesLocal(const T& grid, const MPI_Comm& comm)
    2639{
    2740  if (comm != MPI_COMM_NULL) {
    2841
    29     int index, my_rank, ranks[27];
    30     Index my_coords;
     42    int index, ranks[27];
     43    Index dims, periods, coords;
    3144    Index sizes, subsizes, starts;
    3245    const LocalIndices& l = grid.Local();
    3346
    34     MPI_Comm_rank(comm, &my_rank);
    35     MPI_Cart_coords(comm, my_rank, 3, my_coords.vec());
     47    MPI_Cart_get(comm, 3, dims.vec(), periods.vec(), coords.vec());
    3648
    3749    for (int i=-1; i<=1; ++i)
    3850      for (int j=-1; j<=1; ++j)
    3951        for (int k=-1; k<=1; ++k)
    40           MPI_Cart_rank(comm, (my_coords+Index(i,j,k)).vec(), &ranks[to_1d(i+1,j+1,k+1)]);
     52          if (is_valid(coords + Index(i,j,k), dims, periods))
     53            MPI_Cart_rank(comm, (coords + Index(i,j,k)).vec(), &ranks[to_1d(i+1,j+1,k+1)]);
    4154
    4255    sizes = l.SizeTotal();
    4356
    4457    /* -1 0 0 */
    45     index = to_1d(0,1,1);
    46     subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.Size().Y(), l.Size().Z());
    47     starts = Index(0, l.Begin().Y(), l.Begin().Z());
    48     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 0, 1));
    49     starts = l.Begin();
    50     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 0, 1));
     58    if (is_valid(coords + Index(-1,0,0), dims, periods)) {
     59      index = to_1d(0,1,1);
     60      subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.Size().Z());
     61      starts = Index(0, l.Begin().Y(), l.Begin().Z());
     62      halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 0, 1));
     63      starts = l.Begin();
     64      nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 0, 1));
     65    }
    5166
    5267    /* 1 0 0 */
    53     index = to_1d(2,1,1);
    54     subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.Size().Y(), l.Size().Z());
    55     starts = Index(l.End().X(), l.Begin().Y(), l.Begin().Z());
    56     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 1, 0));
    57     starts = Index(l.End().X()-l.HaloEnd2().X()+l.HaloBegin2().X(), l.Begin().Y(), l.Begin().Z());
    58     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 1, 0));
     68    if (is_valid(coords + Index(1,0,0), dims, periods)) {
     69      index = to_1d(2,1,1);
     70      subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.Size().Z());
     71      starts = Index(l.End().X(), l.Begin().Y(), l.Begin().Z());
     72      halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 1, 0));
     73      starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
     74      nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 1, 0));
     75    }
    5976
    6077    /* 0 -1 0 */
    61     index = to_1d(1,0,1);
    62     subsizes = Index(l.Size().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.Size().Z());
    63     starts = Index(l.Begin().X(), 0, l.Begin().Z());
    64     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 2, 3));
    65     starts = l.Begin();
    66     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 2, 3));
     78    if (is_valid(coords + Index(0,-1,0), dims, periods)) {
     79      index = to_1d(1,0,1);
     80      subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.Size().Z());
     81      starts = Index(l.Begin().X(), 0, l.Begin().Z());
     82      halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 2, 3));
     83      starts = l.Begin();
     84      nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 2, 3));
     85    }
    6786
    6887    /* 0 1 0 */
    69     index = to_1d(1,2,1);
    70     subsizes = Index(l.Size().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.Size().Z());
    71     starts = Index(l.Begin().X(), l.End().Y(), l.Begin().Z());
    72     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 3, 2));
    73     starts = Index(l.Begin().X(), l.End().Y()-l.HaloEnd2().Y()+l.HaloBegin2().Y(), l.Begin().Z());
    74     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 3, 2));
     88    if (is_valid(coords + Index(0,1,0), dims, periods)) {
     89      index = to_1d(1,2,1);
     90      subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.Size().Z());
     91      starts = Index(l.Begin().X(), l.End().Y(), l.Begin().Z());
     92      halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 3, 2));
     93      starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
     94      nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 3, 2));
     95    }
    7596
    7697    /* 0 0 -1 */
    77     index = to_1d(1,1,0);
    78     subsizes = Index(l.Size().X(), l.Size().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
    79     starts = Index(l.Begin().X(), l.Begin().Y(), 0);
    80     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 4, 5));
    81     starts = l.Begin();
    82     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 4, 5));
     98    if (is_valid(coords + Index(0,0,-1), dims, periods)) {
     99      index = to_1d(1,1,0);
     100      subsizes = Index(l.Size().X(), l.Size().Y(), l.HaloSize1().Z());
     101      starts = Index(l.Begin().X(), l.Begin().Y(), 0);
     102      halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 4, 5));
     103      starts = l.Begin();
     104      nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 4, 5));
     105    }
    83106
    84107    /* 0 0 1 */
    85     index = to_1d(1,1,2);
    86     subsizes = Index(l.Size().X(), l.Size().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
    87     starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z());
    88     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 5, 4));
    89     starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloEnd2().Z()+l.HaloBegin2().Z());
    90     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 5, 4));
     108    if (is_valid(coords + Index(0,0,1), dims, periods)) {
     109      index = to_1d(1,1,2);
     110      subsizes = Index(l.Size().X(), l.Size().Y(), l.HaloSize2().Z());
     111      starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z());
     112      halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 5, 4));
     113      starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
     114      nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 5, 4));
     115    }
    91116
    92117    /* -1 -1 0 */
    93     index = to_1d(0,0,1);
    94     subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.Size().Z());
    95     starts = Index(0, 0, l.Begin().Z());
    96     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 6, 7));
    97     starts = l.Begin();
    98     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 6, 7));
     118    if (is_valid(coords + Index(-1,-1,0), dims, periods)) {
     119      index = to_1d(0,0,1);
     120      subsizes = Index(l.HaloSize1().X(), l.HaloSize1().Y(), l.Size().Z());
     121      starts = Index(0, 0, l.Begin().Z());
     122      halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 6, 7));
     123      starts = l.Begin();
     124      nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 6, 7));
     125    }
    99126
    100127    /* -1 1 0 */
    101     index = to_1d(0,2,1);
    102     subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.Size().Z());
    103     starts = Index(0, l.End().Y(), l.Begin().Z());
    104     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 8, 9));
    105     starts = Index(l.Begin().X(), l.End().Y()-l.HaloEnd2().Y()+l.HaloBegin2().Y(), l.Begin().Z());
    106     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 8, 9));
     128    if (is_valid(coords + Index(-1,1,0), dims, periods)) {
     129      index = to_1d(0,2,1);
     130      subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.Size().Z());
     131      starts = Index(0, l.End().Y(), l.Begin().Z());
     132      halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 8, 9));
     133      starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
     134      nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 8, 9));
     135    }
    107136
    108137    /* 1 -1 0 */
    109     index = to_1d(2,0,1);
    110     subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.Size().Z());
    111     starts = Index(l.End().X(), 0, l.Begin().Z());
    112     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 9, 8));
    113     starts = Index(l.End().X()-l.HaloEnd2().X()+l.HaloBegin2().X(), l.Begin().Y(), l.Begin().Z());
    114     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 9, 8));
     138    if (is_valid(coords + Index(1,-1,0), dims, periods)) {
     139      index = to_1d(2,0,1);
     140      subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.Size().Z());
     141      starts = Index(l.End().X(), 0, l.Begin().Z());
     142      halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 9, 8));
     143      starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
     144      nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 9, 8));
     145    }
    115146
    116147    /* 1 1 0 */
    117     index = to_1d(2,2,1);
    118     subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.Size().Z());
    119     starts = Index(l.End().X(), l.End().Y(), l.Begin().Z());
    120     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 7, 6));
    121     starts = Index(l.End().X()-l.HaloEnd2().X()+l.HaloBegin2().X(), l.End().Y()-l.HaloEnd2().Y()+l.HaloBegin2().Y(), l.Begin().Z());
    122     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 7, 6));
     148    if (is_valid(coords + Index(1,1,0), dims, periods)) {
     149      index = to_1d(2,2,1);
     150      subsizes = Index(l.HaloSize2().X(), l.HaloSize2().Y(), l.Size().Z());
     151      starts = Index(l.End().X(), l.End().Y(), l.Begin().Z());
     152      halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 7, 6));
     153      starts = Index(l.End().X()-l.HaloSize2().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
     154      nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 7, 6));
     155    }
    123156
    124157    /* -1 0 -1 */
    125     index = to_1d(0,1,0);
    126     subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.Size().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
    127     starts = Index(0, l.Begin().Y(), 0);
    128     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 10, 11));
    129     starts = l.Begin();
    130     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 10, 11));
     158    if (is_valid(coords + Index(-1,0,-1), dims, periods)) {
     159      index = to_1d(0,1,0);
     160      subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.HaloSize1().Z());
     161      starts = Index(0, l.Begin().Y(), 0);
     162      halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 10, 11));
     163      starts = l.Begin();
     164      nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 10, 11));
     165    }
    131166
    132167    /* -1 0 1 */
    133     index = to_1d(0,1,2);
    134     subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.Size().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
    135     starts = Index(0, l.Begin().Y(), l.End().Z());
    136     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 12, 13));
    137     starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloEnd2().Z()+l.HaloBegin2().Z());
    138     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 12, 13));
     168    if (is_valid(coords + Index(-1,0,1), dims, periods)) {
     169      index = to_1d(0,1,2);
     170      subsizes = Index(l.HaloSize1().X(), l.Size().Y(), l.HaloSize2().Z());
     171      starts = Index(0, l.Begin().Y(), l.End().Z());
     172      halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 12, 13));
     173      starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
     174      nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 12, 13));
     175    }
    139176
    140177    /* 1 0 -1 */
    141     index = to_1d(2,1,0);
    142     subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.Size().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
    143     starts = Index(l.End().X(), l.Begin().Y(), 0);
    144     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 13, 12));
    145     starts = Index(l.End().X()-l.HaloEnd2().X()+l.HaloBegin2().X(), l.Begin().Y(), l.Begin().Z());
    146     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 13, 12));
     178    if (is_valid(coords + Index(1,0,-1), dims, periods)) {
     179      index = to_1d(2,1,0);
     180      subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.HaloSize1().Z());
     181      starts = Index(l.End().X(), l.Begin().Y(), 0);
     182      halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 13, 12));
     183      starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
     184      nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 13, 12));
     185    }
    147186
    148187    /* 1 0 1 */
    149     index = to_1d(2,1,2);
    150     subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.Size().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
    151     starts = Index(l.End().X(), l.Begin().Y(), l.End().Z());
    152     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 11, 10));
    153     starts = Index(l.End().X()-l.HaloEnd2().X()+l.HaloBegin2().X(), l.Begin().Y(), l.End().Z()-l.HaloEnd2().Z()+l.HaloBegin2().Z());
    154     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 11, 10));
     188    if (is_valid(coords + Index(1,0,1), dims, periods)) {
     189      index = to_1d(2,1,2);
     190      subsizes = Index(l.HaloSize2().X(), l.Size().Y(), l.HaloSize2().Z());
     191      starts = Index(l.End().X(), l.Begin().Y(), l.End().Z());
     192      halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 11, 10));
     193      starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
     194      nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 11, 10));
     195    }
    155196
    156197    /* 0 -1 -1 */
    157     index = to_1d(1,0,0);
    158     subsizes = Index(l.Size().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
    159     starts = Index(l.Begin().X(), 0, 0);
    160     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15));
    161     starts = l.Begin();
    162     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15));
     198     if (is_valid(coords + Index(0,-1,-1), dims, periods)) {
     199       index = to_1d(1,0,0);
     200       subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.HaloSize1().Z());
     201       starts = Index(l.Begin().X(), 0, 0);
     202       halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15));
     203       starts = l.Begin();
     204       nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15));
     205     }
    163206
    164207    /* 0 -1 1 */
    165     index = to_1d(1,0,2);
    166     subsizes = Index(l.Size().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
    167     starts = Index(l.Begin().X(), 0, l.End().Z());
    168     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17));
    169     starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloEnd2().Z()+l.HaloBegin2().Z());
    170     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17));
     208     if (is_valid(coords + Index(0,-1,1), dims, periods)) {
     209       index = to_1d(1,0,2);
     210       subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
     211       starts = Index(l.Begin().X(), 0, l.End().Z());
     212       halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17));
     213       starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
     214       nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17));
     215     }
    171216
    172217    /* 0 1 -1 */
    173     index = to_1d(1,2,0);
    174     subsizes = Index(l.Size().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
    175     starts = Index(l.Begin().X(), l.End().Y(), 0);
    176     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16));
    177     starts = Index(l.Begin().X(), l.End().Y()-l.HaloEnd2().Y()+l.HaloBegin2().Y(), l.Begin().Z());
    178     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16));
     218     if (is_valid(coords + Index(0,1,-1), dims, periods)) {
     219       index = to_1d(1,2,0);
     220       subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
     221       starts = Index(l.Begin().X(), l.End().Y(), 0);
     222       halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16));
     223       starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
     224       nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16));
     225     }
    179226
    180227    /* 0 1 1 */
    181     index = to_1d(1,2,2);
    182     subsizes = Index(l.Size().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
    183     starts = Index(l.Begin().X(), l.End().Y(), l.End().Z());
    184     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14));
    185     starts = Index(l.Begin().X(), l.End().Y()-l.HaloEnd2().Y()+l.HaloBegin2().Y(), l.End().Z()-l.HaloEnd2().Z()+l.HaloBegin2().Z());
    186     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14));
     228     if (is_valid(coords + Index(0,1,1), dims, periods)) {
     229       index = to_1d(1,2,2);
     230       subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.HaloSize2().Z());
     231       starts = Index(l.Begin().X(), l.End().Y(), l.End().Z());
     232       halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14));
     233       starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.End().Z()-l.HaloSize2().Z());
     234       nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14));
     235     }
    187236
    188237    /* -1 -1 -1 */
    189     index = to_1d(0,0,0);
    190     subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
    191     starts = 0;
    192     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19));
    193     starts = l.Begin();
    194     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19));
     238     if (is_valid(coords + Index(-1,-1,-1), dims, periods)) {
     239       index = to_1d(0,0,0);
     240       subsizes = l.HaloSize1();
     241       starts = 0;
     242       halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19));
     243       starts = l.Begin();
     244       nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19));
     245     }
    195246
    196247    /* -1 -1 1 */
    197     index = to_1d(0,0,2);
    198     subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
    199     starts = Index(0, 0, l.End().Z());;
    200     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21));
    201     starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloEnd2().Z()+l.HaloBegin2().Z());
    202     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21));
     248     if (is_valid(coords + Index(-1,-1,1), dims, periods)) {
     249       index = to_1d(0,0,2);
     250       subsizes = Index(l.HaloSize1().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
     251       starts = Index(0, 0, l.End().Z());;
     252       halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21));
     253       starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
     254       nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21));
     255     }
    203256
    204257    /* -1 1 -1 */
    205     index = to_1d(0,2,0);
    206     subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
    207     starts = Index(0, l.End().Y(), 0);
    208     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23));
    209     starts = Index(l.Begin().X(), l.End().Y()-l.HaloEnd2().Y()+l.HaloBegin2().Y(), l.Begin().Z());
    210     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23));
     258     if (is_valid(coords + Index(-1,1,-1), dims, periods)) {
     259       index = to_1d(0,2,0);
     260       subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
     261       starts = Index(0, l.End().Y(), 0);
     262       halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23));
     263       starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
     264       nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23));
     265     }
    211266
    212267    /* 1 -1 -1 */
    213     index = to_1d(2,0,0);
    214     subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
    215     starts = Index(l.End().X(), 0, 0);
    216     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25));
    217     starts = Index(l.End().X()-l.HaloEnd2().X()+l.HaloBegin2().X(), l.Begin().Y(), l.Begin().Z());
    218     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25));
     268     if (is_valid(coords + Index(1,-1,-1), dims, periods)) {
     269       index = to_1d(2,0,0);
     270       subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.HaloSize1().Z());
     271       starts = Index(l.End().X(), 0, 0);
     272       halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25));
     273       starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
     274       nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25));
     275     }
    219276
    220277    /* -1 1 1 */
    221     index = to_1d(0,2,2);
    222     subsizes = Index(l.HaloEnd1().X()-l.HaloBegin1().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
    223     starts = Index(0, l.End().Y(), l.End().Z());
    224     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24));
    225     starts = Index(l.Begin().X(), l.End().Y()-l.HaloEnd2().Y()+l.HaloBegin2().Y(), l.End().Z()-l.HaloEnd2().Z()+l.HaloBegin2().Z());
    226     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24));
     278     if (is_valid(coords + Index(-1,1,1), dims, periods)) {
     279       index = to_1d(0,2,2);
     280       subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.HaloSize2().Z());
     281       starts = Index(0, l.End().Y(), l.End().Z());
     282       halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24));
     283       starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.End().Z()-l.HaloSize2().Z());
     284       nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24));
     285     }
    227286
    228287    /* 1 -1 1 */
    229     index = to_1d(2,0,2);
    230     subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.HaloEnd1().Y()-l.HaloBegin1().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
    231     starts = Index(l.End().X(), 0, l.End().Z());
    232     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22));
    233     starts = Index(l.End().X()-l.HaloEnd2().X()+l.HaloBegin2().X(), l.Begin().Y(), l.End().Z()-l.HaloEnd2().Z()+l.HaloBegin2().Z());
    234     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22));
     288     if (is_valid(coords + Index(1,-1,1), dims, periods)) {
     289       index = to_1d(2,0,2);
     290       subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
     291       starts = Index(l.End().X(), 0, l.End().Z());
     292       halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22));
     293       starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
     294       nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22));
     295     }
    235296
    236297    /* 1 1 -1 */
    237     index = to_1d(2,2,0);
    238     subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.HaloEnd1().Z()-l.HaloBegin1().Z());
    239     starts = Index(l.End().X(), l.End().Y(), 0);
    240     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20));
    241     starts = Index(l.End().X()-l.HaloEnd2().X()+l.HaloBegin2().X(), l.End().Y()-l.HaloEnd2().Y()+l.HaloBegin2().Y(), l.Begin().Z());
    242     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20));
     298     if (is_valid(coords + Index(1,1,-1), dims, periods)) {
     299       index = to_1d(2,2,0);
     300       subsizes = Index(l.HaloSize2().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
     301       starts = Index(l.End().X(), l.End().Y(), 0);
     302       halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20));
     303       starts = Index(l.End().X()-l.HaloSize2().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
     304       nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20));
     305     }
    243306
    244307    /* 1 1 1 */
    245     index = to_1d(2,2,2);
    246     subsizes = Index(l.HaloEnd2().X()-l.HaloBegin2().X(), l.HaloEnd2().Y()-l.HaloBegin2().Y(), l.HaloEnd2().Z()-l.HaloBegin2().Z());
    247     starts = l.End();
    248     halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18));
    249     starts = l.End()-l.HaloEnd2()+l.HaloBegin2();
    250     nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18));
     308     if (is_valid(coords + Index(1,1,1), dims, periods)) {
     309       index = to_1d(2,2,2);
     310       subsizes = l.HaloSize2();
     311       starts = l.End();
     312       halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18));
     313       starts = l.End()-l.HaloSize2();
     314       nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18));
     315     }
    251316
    252317  }
    253318}
     319
     320template void VMG::MPI::DatatypesLocal::InitDatatypesLocal<VMG::Grid>(const VMG::Grid& grid, const MPI_Comm& comm);
     321template void VMG::MPI::DatatypesLocal::InitDatatypesLocal<VMG::Particle::LinkedCellList>(const VMG::Particle::LinkedCellList& grid, const MPI_Comm& comm);
     322
Note: See TracChangeset for help on using the changeset viewer.