source: src/comm/comm_mpi.hpp@ 894a5f

/**
 * @file   comm_mpi.hpp
 * @author Julian Iseringhausen <isering@ins.uni-bonn.de>
 * @date   Wed Jun 16 13:21:06 2010
 *
 * @brief  Class for MPI-based communication.
 *
 */

#ifndef COMM_MPI_HPP_
#define COMM_MPI_HPP_

#ifndef HAVE_MPI
#error You need MPI in order to compile CommMPI
#endif

#include <cstdarg>
#include <cstdio>
#include <list>
#include <map>
#include <vector>

#include "base/has_tempgrids.hpp"
#include "comm/comm.hpp"
#include "comm/mpi/settings.hpp"
#include "comm/mpi/has_request_vec.hpp"

namespace VMG
{

class DomainDecomposition;
class GridIteratorSet;
class TempGrid;

class CommMPI : public Comm, HasTempGrids, HasRequestVec
{
public:
  CommMPI(const Boundary& boundary, DomainDecomposition* domain_dec, const MPI_Comm& mpi_comm) :
    Comm(boundary, domain_dec),
    win_created(false)
  {
    InitCommMPI(mpi_comm);
  }

  CommMPI(const Boundary& boundary, DomainDecomposition* domain_dec) :
    Comm(boundary, domain_dec),
    win_created(false)
  {
    InitCommMPI(MPI_COMM_WORLD);
  }

  virtual ~CommMPI();

  Grid& GetCoarserGrid(Multigrid& multigrid);
  Grid& GetFinerGrid(Multigrid& multigrid);
  Grid& GetGlobalCoarseGrid(Multigrid& multigrid);

  void CommFromGhosts(Grid& grid);
  void CommToGhosts(Grid& grid);
  void CommToGhostsRB(Grid& grid, const int& offset);
  void CommSubgrid(Grid& grid_old, Grid& grid_new);
  void CommAddSubgrid(Grid& grid_old, Grid& grid_new);

  void CommToGhostsAsyncStart(Grid& grid);
  void CommToGhostsAsyncFinish();
  void CommFromGhostsAsyncStart(Grid& grid);
  void CommFromGhostsAsyncFinish(Grid& grid);

  vmg_float GlobalSum(const vmg_float& value);
  vmg_float GlobalSumRoot(const vmg_float& value);
  void GlobalSumArray(vmg_float* array, const vmg_int& size);

  void PrintString(const char* format, ...);
  void PrintStringOnce(const char* format, ...);
  void PrintXML(const std::string& filename, const std::string& xml_data);
  void PrintXMLAll(const std::string& filename, const std::string& xml_data);
  void PrintAllSettings();
  void PrintDefect(Grid& sol, Grid& rhs, const char* information);
  void PrintGrid(Grid& grid, const char* information);
  void PrintCorrectedGrid(Grid& grid, const char* information);
  void PrintInnerGrid(Grid& grid, const char* information);
  void PrintInnerCorrectedGrid(Grid& grid, const char* information);

  void CommParticles(const Grid& grid, std::list<Particle::Particle>& particles);
  void CommLCListGhosts(const Grid& grid, Particle::LinkedCellList& lc);
  void CommAddPotential(std::list<Particle::Particle>& particles);
  void CommAddPotential(Particle::LinkedCellList& lc);

  int GlobalRank() const;
  Index GlobalPos() const;
  Index GlobalProcs() const;

  void PostInit(Multigrid& sol, Multigrid& rhs)
  {
    settings.ComputeSettings(sol, rhs, comm_global);
  }

  virtual void DebugPrintError(const Grid& sol, const char* information) {}
  virtual void DebugPrintErrorNorm(Grid& sol) {}
  virtual void DebugPrintGridStructure(Multigrid& multigrid) {}

private:
  void InitCommMPI(const MPI_Comm& comm);

  void CreateOutputFiles(const Grid& grid, const std::stringstream& serial_data, const char* information,
                         const Index& begin_global, const Index& end_global,
                         const Index& begin_local, const Index& end_local);

  void CreateParallelOutputFile(const Grid& grid, MPI_Comm& comm,
                                const int& output_count, const char* information,
                                const Index& begin_global, const Index& end_global,
                                const Index& begin_local, const Index& end_local);

  MPI_File CreateSerialOutputFile(const Grid& grid, MPI_Comm& comm,
                                  const int& output_count, const char* information,
                                  const Index& begin_global, const Index& end_global,
                                  const Index& begin_local, const Index& end_local);

  void FinalizeSerialOutputFile(MPI_File& file);

  std::string CreateOutputDirectory();

  MPI_Comm comm_global;
  bool win_created;
  MPI_Win win;
  MPI_Info info;

  std::vector< std::vector<vmg_float> > receive_buffer;

protected:
  void Isend(Grid& grid, VMG::MPI::Datatype& type, const MPI_Comm& comm, const int& tag);
  void IsendAll(Grid& grid, std::vector<VMG::MPI::Datatype>& types, const MPI_Comm& comm, const int& tag_start);
  void Irecv(Grid& grid, VMG::MPI::Datatype& type, const MPI_Comm& comm, const int& tag);
  void IrecvAll(Grid& grid, std::vector<VMG::MPI::Datatype>& types, const MPI_Comm& comm, const int& tag_start);
  void IrecvAllBuffered(std::vector<VMG::MPI::Datatype>& types, const MPI_Comm& comm, const int& tag_start);
  void AddBufferAll(Grid& grid, std::vector<VMG::MPI::Datatype>& types);

  VMG::MPI::Settings settings;
};

}

#endif /* COMM_MPI_HPP_ */