Changeset 8180d8 for src/comm/mpi

Timestamp:

Apr 5, 2013, 12:39:30 PM (13 years ago)

Author:

Julian Iseringhausen <julian.iseringhausen@…>

Children:

4a709e

Parents:

f57182

Message:

Merge stashed open boundary stuff.

Location:

src/comm/mpi

Files:

• datatype.cpp (modified) (6 diffs)
• datatypes_local.hpp (modified) (4 diffs)
• key.hpp (modified) (1 diff)
• settings.cpp (modified) (11 diffs)

src/comm/mpi/datatype.cpp

@@ -72 +72 @@
     for (i.X()=_starts.X(); i.X()<end.X();  ++i.X())
       for (i.Y()=_starts.Y(); i.Y()<end.Y(); ++i.Y()) {
-        std::memcpy(&_buffer[c], &grid.GetVal(i.X(), i.Y(), _starts.Z()), memcpy_size);
-        c += _subsizes.Z();
+        std::memcpy(&_buffer[c], &grid.GetVal(i.X(), i.Y(), _starts.Z()), memcpy_size);
+        c += _subsizes.Z();
       }
 
@@ -91 +91 @@
     for (i.X()=_starts.X(); i.X()<end.X();  ++i.X())
       for (i.Y()=_starts.Y(); i.Y()<end.Y(); ++i.Y()) {
-        std::memcpy(&_buffer[c], &grid.GetVal(i.X(), i.Y(), _starts.Z()), memcpy_size);
-        c += _subsizes.Z();
+        std::memcpy(&_buffer[c], &grid.GetVal(i.X(), i.Y(), _starts.Z()), memcpy_size);
+        c += _subsizes.Z();
       }
 
@@ -124 +124 @@
     for (i.X()=_starts.X(); i.X()<end.X(); ++i.X())
       for (i.Y()=_starts.Y(); i.Y()<end.Y(); ++i.Y()) {
-        std::memcpy(&grid(i.X(), i.Y(), _starts.Z()), &_buffer[c], memcpy_size);
-        c += _subsizes.Z();
+        std::memcpy(&grid(i.X(), i.Y(), _starts.Z()), &_buffer[c], memcpy_size);
+        c += _subsizes.Z();
       }
 
@@ -142 +142 @@
     for (i.X()=_starts.X(); i.X()<end.X(); ++i.X())
       for (i.Y()=_starts.Y(); i.Y()<end.Y(); ++i.Y())
-        for (i.Z()=_starts.Z(); i.Z()<end.Z(); ++i.Z())
-          grid(i) += *iter++;
+        for (i.Z()=_starts.Z(); i.Z()<end.Z(); ++i.Z())
+          grid(i) += *iter++;
 
     assert(iter == _buffer.end());
@@ -150 +150 @@
 
 void VMG::MPI::Datatype::Set(const GridIteratorSet& bounds, const Grid& grid, const int& rank,
-                             const int& tag_send, const int& tag_receive)
+    const int& tag_send, const int& tag_receive)
 {
   _sizes = grid.Local().SizeTotal();
@@ -166 +166 @@
 
 void VMG::MPI::Datatype::Set(const Index& sizes, const Index& subsizes, const Index& starts, const int& rank,
-                             const int& tag_send, const int& tag_receive)
+    const int& tag_send, const int& tag_receive)
 {
   _sizes = sizes;

src/comm/mpi/datatypes_local.hpp

@@ -37 +37 @@
   DatatypesLocal(const T& grid, const MPI_Comm& comm, const bool& alloc)
   {
-    InitDatatypesLocal(grid, comm, alloc);
+  InitDatatypesLocal(grid, comm, alloc);
   }
 
@@ -69 +69 @@
 {
   return (periods[0] || (coord[0] >= 0 && coord[0] < dims[0])) &&
-         (periods[1] || (coord[1] >= 0 && coord[1] < dims[1])) &&
-         (periods[2] || (coord[2] >= 0 && coord[2] < dims[2]));
+      (periods[1] || (coord[1] >= 0 && coord[1] < dims[1])) &&
+      (periods[2] || (coord[2] >= 0 && coord[2] < dims[2]));
 }
 
@@ -90 +90 @@
     for (i.X()=-1; i.X()<=1; ++i.X())
       for (i.Y()=-1; i.Y()<=1; ++i.Y())
-        for (i.Z()=-1; i.Z()<=1; ++i.Z())
-          if (_is_valid(coords + i, dims, periods))
-            MPI_Cart_rank(comm, (coords + i).vec(), &ranks[to_1d(i+1)]);
+        for (i.Z()=-1; i.Z()<=1; ++i.Z())
+          if (_is_valid(coords + i, dims, periods))
+            MPI_Cart_rank(comm, (coords + i).vec(), &ranks[to_1d(i+1)]);
 
     sizes = l.SizeTotal();
@@ -266 +266 @@
     /* 0 -1 -1 */
     offset = Index(0,-1,-1);
-     if (_is_valid(coords + offset, dims, periods)) {
-       index = to_1d(offset+1);
-       subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.HaloSize1().Z());
-       starts = Index(l.Begin().X(), 0, 0);
-       _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15, alloc_buffer));
-       starts = l.Begin();
-       _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15, alloc_buffer));
-       _offset.push_back(offset);
-     }
+    if (_is_valid(coords + offset, dims, periods)) {
+      index = to_1d(offset+1);
+      subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.HaloSize1().Z());
+      starts = Index(l.Begin().X(), 0, 0);
+      _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15, alloc_buffer));
+      starts = l.Begin();
+      _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 14, 15, alloc_buffer));
+      _offset.push_back(offset);
+    }
 
     /* 0 -1 1 */
-     offset = Index(0,-1,1);
-     if (_is_valid(coords + offset, dims, periods)) {
-       index = to_1d(offset+1);
-       subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
-       starts = Index(l.Begin().X(), 0, l.End().Z());
-       _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17, alloc_buffer));
-       starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
-       _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17, alloc_buffer));
-       _offset.push_back(offset);
-     }
+    offset = Index(0,-1,1);
+    if (_is_valid(coords + offset, dims, periods)) {
+      index = to_1d(offset+1);
+      subsizes = Index(l.Size().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
+      starts = Index(l.Begin().X(), 0, l.End().Z());
+      _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17, alloc_buffer));
+      starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
+      _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 16, 17, alloc_buffer));
+      _offset.push_back(offset);
+    }
 
     /* 0 1 -1 */
-     offset = Index(0,1,-1);
-     if (_is_valid(coords + offset, dims, periods)) {
-       index = to_1d(offset+1);
-       subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
-       starts = Index(l.Begin().X(), l.End().Y(), 0);
-       _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16, alloc_buffer));
-       starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
-       _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16, alloc_buffer));
-       _offset.push_back(offset);
-     }
+    offset = Index(0,1,-1);
+    if (_is_valid(coords + offset, dims, periods)) {
+      index = to_1d(offset+1);
+      subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
+      starts = Index(l.Begin().X(), l.End().Y(), 0);
+      _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16, alloc_buffer));
+      starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
+      _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 17, 16, alloc_buffer));
+      _offset.push_back(offset);
+    }
 
     /* 0 1 1 */
-     offset = Index(0,1,1);
-     if (_is_valid(coords + offset, dims, periods)) {
-       index = to_1d(offset+1);
-       subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.HaloSize2().Z());
-       starts = Index(l.Begin().X(), l.End().Y(), l.End().Z());
-       _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14, alloc_buffer));
-       starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.End().Z()-l.HaloSize2().Z());
-       _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14, alloc_buffer));
-       _offset.push_back(offset);
-     }
+    offset = Index(0,1,1);
+    if (_is_valid(coords + offset, dims, periods)) {
+      index = to_1d(offset+1);
+      subsizes = Index(l.Size().X(), l.HaloSize2().Y(), l.HaloSize2().Z());
+      starts = Index(l.Begin().X(), l.End().Y(), l.End().Z());
+      _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14, alloc_buffer));
+      starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.End().Z()-l.HaloSize2().Z());
+      _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 15, 14, alloc_buffer));
+      _offset.push_back(offset);
+    }
 
     /* -1 -1 -1 */
-     offset = Index(-1,-1,-1);
-     if (_is_valid(coords + offset, dims, periods)) {
-       index = to_1d(offset+1);
-       subsizes = l.HaloSize1();
-       starts = 0;
-       _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19, alloc_buffer));
-       starts = l.Begin();
-       _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19, alloc_buffer));
-       _offset.push_back(offset);
-     }
+    offset = Index(-1,-1,-1);
+    if (_is_valid(coords + offset, dims, periods)) {
+      index = to_1d(offset+1);
+      subsizes = l.HaloSize1();
+      starts = 0;
+      _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19, alloc_buffer));
+      starts = l.Begin();
+      _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 18, 19, alloc_buffer));
+      _offset.push_back(offset);
+    }
 
     /* -1 -1 1 */
-     offset = Index(-1,-1,1);
-     if (_is_valid(coords + offset, dims, periods)) {
-       index = to_1d(offset+1);
-       subsizes = Index(l.HaloSize1().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
-       starts = Index(0, 0, l.End().Z());;
-       _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21, alloc_buffer));
-       starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
-       _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21, alloc_buffer));
-       _offset.push_back(offset);
-     }
+    offset = Index(-1,-1,1);
+    if (_is_valid(coords + offset, dims, periods)) {
+      index = to_1d(offset+1);
+      subsizes = Index(l.HaloSize1().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
+      starts = Index(0, 0, l.End().Z());;
+      _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21, alloc_buffer));
+      starts = Index(l.Begin().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
+      _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 20, 21, alloc_buffer));
+      _offset.push_back(offset);
+    }
 
     /* -1 1 -1 */
-     offset = Index(-1,1,-1);
-     if (_is_valid(coords + offset, dims, periods)) {
-       index = to_1d(offset+1);
-       subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
-       starts = Index(0, l.End().Y(), 0);
-       _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23, alloc_buffer));
-       starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
-       _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23, alloc_buffer));
-       _offset.push_back(offset);
-     }
+    offset = Index(-1,1,-1);
+    if (_is_valid(coords + offset, dims, periods)) {
+      index = to_1d(offset+1);
+      subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
+      starts = Index(0, l.End().Y(), 0);
+      _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23, alloc_buffer));
+      starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
+      _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 22, 23, alloc_buffer));
+      _offset.push_back(offset);
+    }
 
     /* 1 -1 -1 */
-     offset = Index(1,-1,-1);
-     if (_is_valid(coords + offset, dims, periods)) {
-       index = to_1d(offset+1);
-       subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.HaloSize1().Z());
-       starts = Index(l.End().X(), 0, 0);
-       _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25, alloc_buffer));
-       starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
-       _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25, alloc_buffer));
-       _offset.push_back(offset);
-     }
+    offset = Index(1,-1,-1);
+    if (_is_valid(coords + offset, dims, periods)) {
+      index = to_1d(offset+1);
+      subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.HaloSize1().Z());
+      starts = Index(l.End().X(), 0, 0);
+      _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25, alloc_buffer));
+      starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.Begin().Z());
+      _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 24, 25, alloc_buffer));
+      _offset.push_back(offset);
+    }
 
     /* -1 1 1 */
-     offset = Index(-1,1,1);
-     if (_is_valid(coords + offset, dims, periods)) {
-       index = to_1d(offset+1);
-       subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.HaloSize2().Z());
-       starts = Index(0, l.End().Y(), l.End().Z());
-       _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24, alloc_buffer));
-       starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.End().Z()-l.HaloSize2().Z());
-       _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24, alloc_buffer));
-       _offset.push_back(offset);
-     }
+    offset = Index(-1,1,1);
+    if (_is_valid(coords + offset, dims, periods)) {
+      index = to_1d(offset+1);
+      subsizes = Index(l.HaloSize1().X(), l.HaloSize2().Y(), l.HaloSize2().Z());
+      starts = Index(0, l.End().Y(), l.End().Z());
+      _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24, alloc_buffer));
+      starts = Index(l.Begin().X(), l.End().Y()-l.HaloSize2().Y(), l.End().Z()-l.HaloSize2().Z());
+      _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 25, 24, alloc_buffer));
+      _offset.push_back(offset);
+    }
 
     /* 1 -1 1 */
-     offset = Index(1,-1,1);
-     if (_is_valid(coords + offset, dims, periods)) {
-       index = to_1d(offset+1);
-       subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
-       starts = Index(l.End().X(), 0, l.End().Z());
-       _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22, alloc_buffer));
-       starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
-       _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22, alloc_buffer));
-       _offset.push_back(offset);
-     }
+    offset = Index(1,-1,1);
+    if (_is_valid(coords + offset, dims, periods)) {
+      index = to_1d(offset+1);
+      subsizes = Index(l.HaloSize2().X(), l.HaloSize1().Y(), l.HaloSize2().Z());
+      starts = Index(l.End().X(), 0, l.End().Z());
+      _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22, alloc_buffer));
+      starts = Index(l.End().X()-l.HaloSize2().X(), l.Begin().Y(), l.End().Z()-l.HaloSize2().Z());
+      _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 23, 22, alloc_buffer));
+      _offset.push_back(offset);
+    }
 
     /* 1 1 -1 */
-     offset = Index(1,1,-1);
-     if (_is_valid(coords + offset, dims, periods)) {
-       index = to_1d(offset+1);
-       subsizes = Index(l.HaloSize2().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
-       starts = Index(l.End().X(), l.End().Y(), 0);
-       _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20, alloc_buffer));
-       starts = Index(l.End().X()-l.HaloSize2().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
-       _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20, alloc_buffer));
-       _offset.push_back(offset);
-     }
+    offset = Index(1,1,-1);
+    if (_is_valid(coords + offset, dims, periods)) {
+      index = to_1d(offset+1);
+      subsizes = Index(l.HaloSize2().X(), l.HaloSize2().Y(), l.HaloSize1().Z());
+      starts = Index(l.End().X(), l.End().Y(), 0);
+      _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20, alloc_buffer));
+      starts = Index(l.End().X()-l.HaloSize2().X(), l.End().Y()-l.HaloSize2().Y(), l.Begin().Z());
+      _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 21, 20, alloc_buffer));
+      _offset.push_back(offset);
+    }
 
     /* 1 1 1 */
-     offset = Index(1,1,1);
-     if (_is_valid(coords + offset, dims, periods)) {
-       index = to_1d(offset+1);
-       subsizes = l.HaloSize2();
-       starts = l.End();
-       _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18, alloc_buffer));
-       starts = l.End()-l.HaloSize2();
-       _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18, alloc_buffer));
-       _offset.push_back(offset);
-     }
+    offset = Index(1,1,1);
+    if (_is_valid(coords + offset, dims, periods)) {
+      index = to_1d(offset+1);
+      subsizes = l.HaloSize2();
+      starts = l.End();
+      _halo.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18, alloc_buffer));
+      starts = l.End()-l.HaloSize2();
+      _nb.push_back(VMG::MPI::Datatype(sizes, subsizes, starts, ranks[index], 19, 18, alloc_buffer));
+      _offset.push_back(offset);
+    }
 
   }

src/comm/mpi/key.hpp

@@ -54 +54 @@
     end(grid.Global().LocalEnd()),
     size_local(grid.Local().SizeTotal()),
-    size_global(grid.Global().GlobalSize()),
+    size_global(grid.Global().GlobalSizeFinest()),
     level(grid.Level())
   {}

src/comm/mpi/settings.cpp

@@ -96 +96 @@
 
     if (temp_grid->Global().LocalBegin().IsComponentwiseGreaterOrEqual(sol(i-1).Global().LocalBegin()) &&
-        temp_grid->Global().LocalBegin().IsComponentwiseLessOrEqual(sol(i-1).Global().LocalEnd()) &&
-        temp_grid->Global().LocalEnd().IsComponentwiseGreaterOrEqual(sol(i-1).Global().LocalBegin()) &&
-        temp_grid->Global().LocalEnd().IsComponentwiseLessOrEqual(sol(i-1).Global().LocalEnd())) {
+        temp_grid->Global().LocalBegin().IsComponentwiseLessOrEqual(sol(i-1).Global().LocalEnd()) &&
+        temp_grid->Global().LocalEnd().IsComponentwiseGreaterOrEqual(sol(i-1).Global().LocalBegin()) &&
+        temp_grid->Global().LocalEnd().IsComponentwiseLessOrEqual(sol(i-1).Global().LocalEnd())) {
       delete temp_grid;
       coarser_grids.insert(std::make_pair(&sol(i), &sol(i-1)));
@@ -113 +113 @@
 
     if (temp_grid->Global().LocalBegin().IsComponentwiseGreaterOrEqual(sol(i-1).Global().LocalBegin()) &&
-        temp_grid->Global().LocalBegin().IsComponentwiseLessOrEqual(sol(i-1).Global().LocalEnd()) &&
-        temp_grid->Global().LocalEnd().IsComponentwiseGreaterOrEqual(sol(i-1).Global().LocalBegin()) &&
-        temp_grid->Global().LocalEnd().IsComponentwiseLessOrEqual(sol(i-1).Global().LocalEnd())) {
+        temp_grid->Global().LocalBegin().IsComponentwiseLessOrEqual(sol(i-1).Global().LocalEnd()) &&
+        temp_grid->Global().LocalEnd().IsComponentwiseGreaterOrEqual(sol(i-1).Global().LocalBegin()) &&
+        temp_grid->Global().LocalEnd().IsComponentwiseLessOrEqual(sol(i-1).Global().LocalEnd())) {
       delete temp_grid;
       coarser_grids.insert(std::make_pair(&rhs(i), &rhs(i-1)));
@@ -133 +133 @@
 
     if (temp_grid->Global().LocalBegin() == sol(i+1).Global().LocalBegin() &&
-        temp_grid->Global().LocalEnd() == sol(i+1).Global().LocalEnd()) {
+        temp_grid->Global().LocalEnd() == sol(i+1).Global().LocalEnd()) {
       delete temp_grid;
       finer_grids.insert(std::make_pair(&sol(i), &sol(i+1)));
@@ -148 +148 @@
 
     if (temp_grid->Global().LocalBegin() == rhs(i+1).Global().LocalBegin() &&
-        temp_grid->Global().LocalEnd() == rhs(i+1).Global().LocalEnd()) {
+        temp_grid->Global().LocalEnd() == rhs(i+1).Global().LocalEnd()) {
       delete temp_grid;
       finer_grids.insert(std::make_pair(&rhs(i), &rhs(i+1)));
@@ -220 +220 @@
 
   const bool in_communicator = (grid_1->Global().LocalSize().Product() > 0) ||
-                               (grid_2 && grid_2->Global().LocalSize().Product() > 0) ||
-                               (grid_3 && grid_3->Global().LocalSize().Product() > 0);
+      (grid_2 && grid_2->Global().LocalSize().Product() > 0) ||
+      (grid_3 && grid_3->Global().LocalSize().Product() > 0);
 
   MPI_Comm_rank(comm_global, &rank);
@@ -263 +263 @@
     for (iter=communicators_local_unique.begin(); iter!=communicators_local_unique.end(); ++iter) {
       if (*iter != MPI_COMM_NULL) {
-        MPI_Comm_compare(comm_new, *iter, &comm_equal);
-        assert(comm_equal != MPI_SIMILAR);
-        if (comm_equal == MPI_IDENT || comm_equal == MPI_CONGRUENT) {
-          MPI_Comm_free(&comm_new);
-          comm_new = *iter;
-          break;
-        }
+        MPI_Comm_compare(comm_new, *iter, &comm_equal);
+        assert(comm_equal != MPI_SIMILAR);
+        if (comm_equal == MPI_IDENT || comm_equal == MPI_CONGRUENT) {
+          MPI_Comm_free(&comm_new);
+          comm_new = *iter;
+          break;
+        }
       }
     }
@@ -283 +283 @@
   bool dt_is_new = true;
 
-    // Insert into map
+  // Insert into map
   std::pair< std::map<VMG::MPI::KeyUnsorted, VMG::MPI::DatatypesGlobal>::iterator, bool > insert_result =
-    datatypes_global.insert(std::make_pair(VMG::MPI::KeyUnsorted(grid_old, grid_new, direction), VMG::MPI::DatatypesGlobal()));
+      datatypes_global.insert(std::make_pair(VMG::MPI::KeyUnsorted(grid_old, grid_new, direction), VMG::MPI::DatatypesGlobal()));
   VMG::MPI::DatatypesGlobal& dt_global = insert_result.first->second;
   dt_is_new = insert_result.second;
@@ -310 +310 @@
     if (&grid_old == &grid_new) {
       for (int i=0; i<6; ++i)
-        buffer[6*rank+i] = 0;
+        buffer[6*rank+i] = 0;
     }else {
       for (int i=0; i<3; ++i) {
-        buffer[6*rank+i] = grid_old.Global().LocalBegin()[i];
-        buffer[6*rank+i+3] = grid_old.Global().LocalEnd()[i];
+        buffer[6*rank+i] = grid_old.Global().LocalBegin()[i];
+        buffer[6*rank+i+3] = grid_old.Global().LocalEnd()[i];
       }
     }
@@ -324 +324 @@
       // Decide who offers a useful grid part
       for (int i=0; i<size; ++i) {
-        for (int j=0; j<3; ++j) {
-          begin[j] = buffer[6*i+j];
-          end[j] = buffer[6*i+j+3];
-        }
-
-        begin = begin.Clamp(grid_new.Global().LocalBegin(), grid_new.Global().LocalEnd());
-        end = end.Clamp(grid_new.Global().LocalBegin(), grid_new.Global().LocalEnd());
-
-        if ((end-begin).Product() > 0) {
-          // This process has a useful part
-          dt_global.Receive().push_back(VMG::MPI::Datatype(grid_new.Local().SizeTotal(),
-                                                           end - begin,
-                                                           begin - grid_new.Global().LocalBegin() + offset_new,
-                                                           i, 0, 0, true));
-        }
+        for (int j=0; j<3; ++j) {
+          begin[j] = buffer[6*i+j];
+          end[j] = buffer[6*i+j+3];
+        }
+
+        begin = begin.Clamp(grid_new.Global().LocalBegin(), grid_new.Global().LocalEnd());
+        end = end.Clamp(grid_new.Global().LocalBegin(), grid_new.Global().LocalEnd());
+
+        if ((end-begin).Product() > 0) {
+          // This process has a useful part
+          dt_global.Receive().push_back(VMG::MPI::Datatype(grid_new.Local().SizeTotal(),
+              end - begin,
+              begin - grid_new.Global().LocalBegin() + grid_new.Global().GlobalBegin() + offset_new,
+              i, 0, 0, true));
+        }
       }
     }
@@ -355 +355 @@
       for (int i=0; i<size; ++i) {
 
-        if ((i == rank) && (&grid_old == &grid_new))
-          continue;
-
-        for (int j=0; j<3; ++j) {
-          begin[j] = buffer[6*i+j];
-          end[j] = buffer[6*i+j+3];
-        }
-
-        begin = begin.Clamp(grid_old.Global().LocalBegin(), grid_old.Global().LocalEnd());
-        end = end.Clamp(grid_old.Global().LocalBegin(), grid_old.Global().LocalEnd());
-
-        if ((end-begin).Product() > 0) {
-          // This process needs one of my parts
-          dt_global.Send().push_back(VMG::MPI::Datatype(grid_old.Local().SizeTotal(),
-                                                        end - begin,
-                                                        begin - grid_old.Global().LocalBegin() + offset_old,
-                                                        i, 0, 0, true));
-        }
+        if ((i == rank) && (&grid_old == &grid_new))
+          continue;
+
+        for (int j=0; j<3; ++j) {
+          begin[j] = buffer[6*i+j];
+          end[j] = buffer[6*i+j+3];
+        }
+
+        begin = begin.Clamp(grid_old.Global().LocalBegin(), grid_old.Global().LocalEnd());
+        end = end.Clamp(grid_old.Global().LocalBegin(), grid_old.Global().LocalEnd());
+
+        if ((end-begin).Product() > 0) {
+          // This process needs one of my parts
+          dt_global.Send().push_back(VMG::MPI::Datatype(grid_old.Local().SizeTotal(),
+              end - begin,
+              begin - grid_old.Global().LocalBegin() + grid_old.Global().GlobalBegin() + offset_old,
+              i, 0, 0, true));
+        }
       }
     }
@@ -379 +379 @@
 
 MPI_Datatype& VMG::MPI::Settings::Datatype(const Index& begin, const Index& end,
-                                           const Index& size_local, const Index& size_global,
-                                           const int& level)
+    const Index& size_local, const Index& size_global,
+    const int& level)
 {
   KeyUnsorted k(begin, end, size_local, size_global, level, 0);