Initialize at openlb-1-3

1 files changed, 1154 insertions, 0 deletions

diff --git a/src/communication/mpiManager.cpp b/src/communication/mpiManager.cpp
new file mode 100644
index 0000000..07dcdf4
--- /dev/null
+++ b/src/communication/mpiManager.cpp
@@ -0,0 +1,1154 @@
+/*  This file is part of the OpenLB library
+ *
+ *  Copyright (C) 2007 The OpenLB project
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public
+ *  License along with this program; if not, write to the Free
+ *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+*/
+
+/** \file
+ * Wrapper functions that simplify the use of MPI, template instatiations
+ */
+
+#ifdef PARALLEL_MODE_MPI
+
+#include "communication/mpiManager.h"
+#include "core/blockData2D.h"
+#include "core/olbDebug.h"
+#include <iostream>
+#include <algorithm>
+
+#ifndef OLB_PRECOMPILED
+#include "core/blockData2D.hh"
+#endif
+
+
+namespace olb {
+
+namespace singleton {
+
+MpiManager::MpiManager() : ok(false), clout(std::cout,"MpiManager")
+{ }
+
+MpiManager::~MpiManager()
+{
+  if (ok) {
+    MPI_Finalize();
+    ok = false;
+  }
+}
+
+void MpiManager::init(int *argc, char ***argv)
+{
+
+  int ok1 = MPI_Init(argc, argv);
+  int ok2 = MPI_Comm_rank(MPI_COMM_WORLD,&taskId);
+  int ok3 = MPI_Comm_size(MPI_COMM_WORLD,&numTasks);
+  ok = (ok1==0 && ok2==0 && ok3==0);
+  clout << "Sucessfully initialized, numThreads=" << getSize() << std::endl;
+}
+
+int MpiManager::getSize() const
+{
+  return numTasks;
+}
+
+int MpiManager::getRank() const
+{
+  return taskId;
+}
+
+int MpiManager::bossId() const
+{
+  return 0;
+}
+
+bool MpiManager::isMainProcessor() const
+{
+  return bossId() == getRank();
+}
+
+double MpiManager::getTime() const
+{
+  if (!ok) {
+    return 0.;
+  }
+  return MPI_Wtime();
+}
+
+void MpiManager::barrier(MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Barrier(comm);
+}
+
+template <>
+void MpiManager::send<bool>(bool *buf, int count, int dest, int tag, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Send(static_cast<void*>(buf), count, MPI_BYTE, dest, tag, comm);
+}
+
+template <>
+void MpiManager::send<char>(char *buf, int count, int dest, int tag, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Send(static_cast<void*>(buf), count, MPI_CHAR, dest, tag, comm);
+}
+
+template <>
+void MpiManager::send<int>(int *buf, int count, int dest, int tag, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Send(static_cast<void*>(buf), count, MPI_INT, dest, tag, comm);
+}
+
+template <>
+void MpiManager::send<float>(float *buf, int count, int dest, int tag, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Send(static_cast<void*>(buf), count, MPI_FLOAT, dest, tag, comm);
+}
+
+template <>
+void MpiManager::send<double>(double *buf, int count, int dest, int tag, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Send(static_cast<void*>(buf), count, MPI_DOUBLE, dest, tag, comm);
+}
+
+template <>
+void MpiManager::iSend<bool>
+(bool *buf, int count, int dest, MPI_Request* request, int tag, MPI_Comm comm)
+{
+  if (ok) {
+    MPI_Isend(static_cast<void*>(buf), count, MPI_BYTE, dest, tag, comm, request);
+  }
+}
+
+template <>
+void MpiManager::iSend<char>
+(char *buf, int count, int dest, MPI_Request* request, int tag, MPI_Comm comm)
+{
+  if (ok) {
+    MPI_Isend(static_cast<void*>(buf), count, MPI_CHAR, dest, tag, comm, request);
+  }
+}
+
+template <>
+void MpiManager::iSend<int>
+(int *buf, int count, int dest, MPI_Request* request, int tag, MPI_Comm comm)
+{
+  if (ok) {
+    MPI_Isend(static_cast<void*>(buf), count, MPI_INT, dest, tag, comm, request);
+  }
+}
+
+template <>
+void MpiManager::iSend<float>
+(float *buf, int count, int dest, MPI_Request* request, int tag, MPI_Comm comm)
+{
+  if (ok) {
+    MPI_Isend(static_cast<void*>(buf), count, MPI_FLOAT, dest, tag, comm, request);
+  }
+}
+
+template <>
+void MpiManager::iSend<double>
+(double *buf, int count, int dest, MPI_Request* request, int tag, MPI_Comm comm)
+{
+  if (ok) {
+    MPI_Isend(static_cast<void*>(buf), count, MPI_DOUBLE, dest, tag, comm, request);
+  }
+}
+
+
+template <>
+void MpiManager::ibSend<bool>
+(bool *buf, int count, int dest, MPI_Request* request, int tag, MPI_Comm comm)
+{
+  if (ok) {
+    MPI_Ibsend(static_cast<void*>(buf), count, MPI_BYTE, dest, tag, comm, request);
+  }
+}
+
+template <>
+void MpiManager::ibSend<char>
+(char *buf, int count, int dest, MPI_Request* request, int tag, MPI_Comm comm)
+{
+  if (ok) {
+    MPI_Ibsend(static_cast<void*>(buf), count, MPI_CHAR, dest, tag, comm, request);
+  }
+}
+
+template <>
+void MpiManager::ibSend<int>
+(int *buf, int count, int dest, MPI_Request* request, int tag, MPI_Comm comm)
+{
+  if (ok) {
+    MPI_Ibsend(static_cast<void*>(buf), count, MPI_INT, dest, tag, comm, request);
+  }
+}
+
+template <>
+void MpiManager::ibSend<float>
+(float *buf, int count, int dest, MPI_Request* request, int tag, MPI_Comm comm)
+{
+  if (ok) {
+    MPI_Ibsend(static_cast<void*>(buf), count, MPI_FLOAT, dest, tag, comm, request);
+  }
+}
+
+template <>
+void MpiManager::ibSend<double>
+(double *buf, int count, int dest, MPI_Request* request, int tag, MPI_Comm comm)
+{
+  if (ok) {
+    MPI_Ibsend(static_cast<void*>(buf), count, MPI_DOUBLE, dest, tag, comm, request);
+  }
+}
+
+
+template <>
+void MpiManager::iSendRequestFree<bool>
+(bool *buf, int count, int dest, int tag, MPI_Comm comm)
+{
+  MPI_Request request;
+  if (ok) {
+    MPI_Isend(static_cast<void*>(buf), count, MPI_BYTE, dest, tag, comm, &request);
+  }
+  MPI_Request_free(&request);
+}
+
+template <>
+void MpiManager::iSendRequestFree<char>
+(char *buf, int count, int dest, int tag, MPI_Comm comm)
+{
+  MPI_Request request;
+  if (ok) {
+    MPI_Isend(static_cast<void*>(buf), count, MPI_CHAR, dest, tag, comm, &request);
+  }
+  MPI_Request_free(&request);
+}
+
+template <>
+void MpiManager::iSendRequestFree<int>
+(int *buf, int count, int dest, int tag, MPI_Comm comm)
+{
+  MPI_Request request;
+  if (ok) {
+    MPI_Isend(static_cast<void*>(buf), count, MPI_INT, dest, tag, comm, &request);
+  }
+  MPI_Request_free(&request);
+}
+
+template <>
+void MpiManager::iSendRequestFree<float>
+(float *buf, int count, int dest, int tag, MPI_Comm comm)
+{
+  MPI_Request request;
+  if (ok) {
+    MPI_Isend(static_cast<void*>(buf), count, MPI_FLOAT, dest, tag, comm, &request);
+  }
+  MPI_Request_free(&request);
+}
+
+template <>
+void MpiManager::iSendRequestFree<double>
+(double *buf, int count, int dest, int tag, MPI_Comm comm)
+{
+  MPI_Request request;
+  if (ok) {
+    MPI_Isend(static_cast<void*>(buf), count, MPI_DOUBLE, dest, tag, comm, &request);
+  }
+  MPI_Request_free(&request);
+}
+
+template <>
+void MpiManager::receive<bool>(bool *buf, int count, int source, int tag, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Status status;
+  MPI_Recv(static_cast<void*>(buf), count, MPI_BYTE, source, tag, comm, &status);
+}
+
+
+template <>
+void MpiManager::receive<char>(char *buf, int count, int source, int tag, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Status status;
+  MPI_Recv(static_cast<void*>(buf), count, MPI_CHAR, source, tag, comm, &status);
+}
+
+template <>
+void MpiManager::receive<int>(int *buf, int count, int source, int tag, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Status status;
+  MPI_Recv(static_cast<void*>(buf), count, MPI_INT, source, tag, comm, &status);
+}
+
+template <>
+void MpiManager::receive<float>(float *buf, int count, int source, int tag, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Status status;
+  MPI_Recv(static_cast<void*>(buf), count, MPI_FLOAT, source, tag, comm, &status);
+}
+
+template <>
+void MpiManager::receive<double>(double *buf, int count, int source, int tag, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Status status;
+  MPI_Recv(static_cast<void*>(buf), count, MPI_DOUBLE, source, tag, comm, &status);
+}
+
+template <>
+void MpiManager::sendToMaster<bool>(bool* sendBuf, int sendCount, bool iAmRoot, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  if (iAmRoot && !isMainProcessor()) {
+    send(sendBuf, sendCount, 0);
+  }
+  if (isMainProcessor() && !iAmRoot) {
+    receive(sendBuf, sendCount, MPI_ANY_SOURCE);
+  }
+}
+
+template <>
+void MpiManager::sendToMaster<char>(char* sendBuf, int sendCount, bool iAmRoot, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  if (iAmRoot && !isMainProcessor()) {
+    send(sendBuf, sendCount, 0);
+  }
+  if (isMainProcessor() && !iAmRoot) {
+    receive(sendBuf, sendCount, MPI_ANY_SOURCE);
+  }
+}
+
+template <>
+void MpiManager::sendToMaster<int>(int* sendBuf, int sendCount, bool iAmRoot, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  if (iAmRoot && !isMainProcessor()) {
+    send(sendBuf, sendCount, 0);
+  }
+  if (isMainProcessor() && !iAmRoot) {
+    receive(sendBuf, sendCount, MPI_ANY_SOURCE);
+  }
+}
+
+template <>
+void MpiManager::sendToMaster<float>(float* sendBuf, int sendCount, bool iAmRoot, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  if (iAmRoot && !isMainProcessor()) {
+    send(sendBuf, sendCount, 0);
+  }
+  if (isMainProcessor() && !iAmRoot) {
+    receive(sendBuf, sendCount, MPI_ANY_SOURCE);
+  }
+}
+
+template <>
+void MpiManager::sendToMaster<double>(double* sendBuf, int sendCount, bool iAmRoot, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  if (iAmRoot && !isMainProcessor()) {
+    send(sendBuf, sendCount, 0);
+  }
+  if (isMainProcessor() && !iAmRoot) {
+    receive(sendBuf, sendCount, MPI_ANY_SOURCE);
+  }
+}
+
+template <>
+void MpiManager::iRecv<bool>(bool *buf, int count, int source, MPI_Request* request, int tag, MPI_Comm comm)
+{
+  if (ok) {
+    MPI_Irecv(static_cast<void*>(buf), count, MPI_BYTE, source, tag, comm, request);
+  }
+}
+
+template <>
+void MpiManager::iRecv<char>(char *buf, int count, int source, MPI_Request* request, int tag, MPI_Comm comm)
+{
+  if (ok) {
+    MPI_Irecv(static_cast<void*>(buf), count, MPI_CHAR, source, tag, comm, request);
+  }
+}
+
+template <>
+void MpiManager::iRecv<int>(int *buf, int count, int source, MPI_Request* request, int tag, MPI_Comm comm)
+{
+  if (ok) {
+    MPI_Irecv(static_cast<void*>(buf), count, MPI_INT, source, tag, comm, request);
+  }
+}
+
+template <>
+void MpiManager::iRecv<float>(float *buf, int count, int source, MPI_Request* request, int tag, MPI_Comm comm)
+{
+  if (ok) {
+    MPI_Irecv(static_cast<void*>(buf), count, MPI_FLOAT, source, tag, comm, request);
+  }
+}
+
+template <>
+void MpiManager::iRecv<double>(double *buf, int count, int source, MPI_Request* request, int tag, MPI_Comm comm)
+{
+  if (ok) {
+    MPI_Irecv(static_cast<void*>(buf), count, MPI_DOUBLE, source, tag, comm, request);
+  }
+}
+
+template <>
+void MpiManager::sendRecv<bool>
+(bool *sendBuf, bool *recvBuf, int count, int dest, int source, int tag, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Status status;
+  MPI_Sendrecv(static_cast<void*>(sendBuf),
+               count,
+               MPI_BYTE, dest, tag,
+               static_cast<void*>(recvBuf),
+               count,
+               MPI_BYTE, source, tag, comm, &status);
+}
+
+template <>
+void MpiManager::sendRecv<char>
+(char *sendBuf, char *recvBuf, int count, int dest, int source, int tag, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Status status;
+  MPI_Sendrecv(static_cast<void*>(sendBuf),
+               count,
+               MPI_CHAR, dest, tag,
+               static_cast<void*>(recvBuf),
+               count,
+               MPI_CHAR, source, tag, comm, &status);
+}
+
+template <>
+void MpiManager::sendRecv<int>
+(int *sendBuf, int *recvBuf, int count, int dest, int source, int tag, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Status status;
+  MPI_Sendrecv(static_cast<void*>(sendBuf),
+               count,
+               MPI_INT, dest, tag,
+               static_cast<void*>(recvBuf),
+               count,
+               MPI_INT, source, tag, comm, &status);
+}
+
+template <>
+void MpiManager::sendRecv<float>
+(float *sendBuf, float *recvBuf, int count, int dest, int source, int tag, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Status status;
+  MPI_Sendrecv(static_cast<void*>(sendBuf),
+               count,
+               MPI_FLOAT, dest, tag,
+               static_cast<void*>(recvBuf),
+               count,
+               MPI_FLOAT, source, tag, comm, &status);
+}
+
+template <>
+void MpiManager::sendRecv<double>
+(double *sendBuf, double *recvBuf, int count, int dest, int source, int tag, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Status status;
+  MPI_Sendrecv(static_cast<void*>(sendBuf),
+               count,
+               MPI_DOUBLE, dest, tag,
+               static_cast<void*>(recvBuf),
+               count,
+               MPI_DOUBLE, source, tag, comm, &status);
+}
+
+template <>
+void MpiManager::scatterv_impl<bool>(bool* sendBuf, int* sendCounts, int* displs,
+                                     bool* recvBuf, int recvCount, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Scatterv(static_cast<void*>(sendBuf),
+               sendCounts, displs, MPI_BYTE,
+               static_cast<void*>(recvBuf),
+               recvCount, MPI_BYTE, root, comm);
+}
+
+template <>
+void MpiManager::scatterv_impl<char>(char* sendBuf, int* sendCounts, int* displs,
+                                     char* recvBuf, int recvCount, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Scatterv(static_cast<void*>(sendBuf),
+               sendCounts, displs, MPI_CHAR,
+               static_cast<void*>(recvBuf),
+               recvCount, MPI_CHAR, root, comm);
+}
+
+template <>
+void MpiManager::scatterv_impl<int>(int *sendBuf, int* sendCounts, int* displs,
+                                    int* recvBuf, int recvCount, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Scatterv(static_cast<void*>(sendBuf),
+               sendCounts, displs, MPI_INT,
+               static_cast<void*>(recvBuf),
+               recvCount, MPI_INT, root, comm);
+}
+
+template <>
+void MpiManager::scatterv_impl<float>(float *sendBuf, int* sendCounts, int* displs,
+                                      float* recvBuf, int recvCount, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Scatterv(static_cast<void*>(sendBuf),
+               sendCounts, displs, MPI_FLOAT,
+               static_cast<void*>(recvBuf),
+               recvCount, MPI_FLOAT, root, comm);
+}
+
+template <>
+void MpiManager::scatterv_impl<double>(double *sendBuf, int* sendCounts, int* displs,
+                                       double* recvBuf, int recvCount, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Scatterv(static_cast<void*>(sendBuf),
+               sendCounts, displs, MPI_DOUBLE,
+               static_cast<void*>(recvBuf),
+               recvCount, MPI_DOUBLE, root, comm);
+}
+
+template <>
+void MpiManager::gatherv_impl<bool>(bool* sendBuf, int sendCount,
+                                    bool* recvBuf, int* recvCounts, int* displs,
+                                    int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Gatherv(static_cast<void*>(sendBuf), sendCount, MPI_BYTE,
+              static_cast<void*>(recvBuf), recvCounts, displs, MPI_BYTE,
+              root, comm);
+}
+
+template <>
+void MpiManager::gatherv_impl<char>(char* sendBuf, int sendCount,
+                                    char* recvBuf, int* recvCounts, int* displs,
+                                    int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Gatherv(static_cast<void*>(sendBuf), sendCount, MPI_CHAR,
+              static_cast<void*>(recvBuf), recvCounts, displs, MPI_CHAR,
+              root, comm);
+}
+
+template <>
+void MpiManager::gatherv_impl<int>(int* sendBuf, int sendCount,
+                                   int* recvBuf, int* recvCounts, int* displs,
+                                   int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Gatherv(static_cast<void*>(sendBuf), sendCount, MPI_INT,
+              static_cast<void*>(recvBuf), recvCounts, displs, MPI_INT,
+              root, comm);
+}
+
+template <>
+void MpiManager::gatherv_impl<float>(float* sendBuf, int sendCount,
+                                     float* recvBuf, int* recvCounts, int* displs,
+                                     int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Gatherv(static_cast<void*>(sendBuf), sendCount, MPI_FLOAT,
+              static_cast<void*>(recvBuf), recvCounts, displs, MPI_FLOAT,
+              root, comm);
+}
+
+template <>
+void MpiManager::gatherv_impl<double>(double* sendBuf, int sendCount,
+                                      double* recvBuf, int* recvCounts, int* displs,
+                                      int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Gatherv(static_cast<void*>(sendBuf), sendCount, MPI_DOUBLE,
+              static_cast<void*>(recvBuf), recvCounts, displs, MPI_DOUBLE,
+              root, comm);
+}
+
+template <>
+void MpiManager::bCast<bool>(bool* sendBuf, int sendCount, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Bcast(static_cast<void*>(sendBuf),
+            sendCount, MPI_BYTE, root, comm);
+}
+
+template <>
+void MpiManager::bCast<char>(char* sendBuf, int sendCount, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Bcast(static_cast<void*>(sendBuf),
+            sendCount, MPI_CHAR, root, comm);
+}
+
+template <>
+void MpiManager::bCast<int>(int* sendBuf, int sendCount, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Bcast(static_cast<void*>(sendBuf),
+            sendCount, MPI_INT, root, comm);
+}
+
+template <>
+void MpiManager::bCast<float>(float* sendBuf, int sendCount, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Bcast(static_cast<void*>(sendBuf),
+            sendCount, MPI_FLOAT, root, comm);
+}
+
+template <>
+void MpiManager::bCast<double>(double* sendBuf, int sendCount, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Bcast(static_cast<void*>(sendBuf),
+            sendCount, MPI_DOUBLE, root, comm);
+}
+
+template <>
+void MpiManager::bCast<std::string>(std::string* sendBuf, int sendCount, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  int length = (int) sendBuf->size();
+  MPI_Bcast(static_cast<void*>(&length), 1, MPI_INT, root, comm);
+  char* buffer = new char[length+1];
+  if (getRank()==root) {
+    std::copy(sendBuf->c_str(), sendBuf->c_str()+length+1, buffer);
+  }
+  MPI_Bcast(static_cast<void*>(buffer), length+1, MPI_CHAR, root, comm);
+  if (getRank()!=root) {
+    *sendBuf = buffer;
+  }
+  delete [] buffer;
+}
+
+void MpiManager::bCast(BlockData2D<double,double>& sendData, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Bcast(static_cast<void*>(sendData.getRawData()),
+            sendData.getDataSize(), MPI_DOUBLE, root, comm);
+}
+
+template <>
+void MpiManager::bCastThroughMaster<bool>(bool* sendBuf, int sendCount, bool iAmRoot, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  if (iAmRoot && !isMainProcessor()) {
+    send(sendBuf, sendCount, 0);
+  }
+  if (isMainProcessor() && !iAmRoot) {
+    receive(sendBuf, sendCount, MPI_ANY_SOURCE);
+  }
+  bCast(sendBuf, sendCount, 0);
+}
+
+template <>
+void MpiManager::bCastThroughMaster<char>(char* sendBuf, int sendCount, bool iAmRoot, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  if (iAmRoot && !isMainProcessor()) {
+    send(sendBuf, sendCount, 0);
+  }
+  if (isMainProcessor() && !iAmRoot) {
+    receive(sendBuf, sendCount, MPI_ANY_SOURCE);
+  }
+  bCast(sendBuf, sendCount, 0);
+}
+
+template <>
+void MpiManager::bCastThroughMaster<int>(int* sendBuf, int sendCount, bool iAmRoot, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  if (iAmRoot && !isMainProcessor()) {
+    send(sendBuf, sendCount, 0);
+  }
+  if (isMainProcessor() && !iAmRoot) {
+    receive(sendBuf, sendCount, MPI_ANY_SOURCE);
+  }
+  bCast(sendBuf, sendCount, 0);
+}
+
+template <>
+void MpiManager::bCastThroughMaster<float>(float* sendBuf, int sendCount, bool iAmRoot, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  if (iAmRoot && !isMainProcessor()) {
+    send(sendBuf, sendCount, 0);
+  }
+  if (isMainProcessor() && !iAmRoot) {
+    receive(sendBuf, sendCount, MPI_ANY_SOURCE);
+  }
+  bCast(sendBuf, sendCount, 0);
+}
+
+template <>
+void MpiManager::bCastThroughMaster<double>(double* sendBuf, int sendCount, bool iAmRoot, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  if (iAmRoot && !isMainProcessor()) {
+    send(sendBuf, sendCount, 0);
+  }
+  if (isMainProcessor() && !iAmRoot) {
+    receive(sendBuf, sendCount, MPI_ANY_SOURCE);
+  }
+  bCast(sendBuf, sendCount, 0);
+}
+
+template <>
+void MpiManager::reduce<bool>(bool& sendVal, bool& recvVal,  MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Reduce(static_cast<void*>(&sendVal),
+             static_cast<void*>(&recvVal), 1, MPI_BYTE, op, root, comm);
+}
+
+template <>
+void MpiManager::reduce<char>(char& sendVal, char& recvVal,  MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Reduce(static_cast<void*>(&sendVal),
+             static_cast<void*>(&recvVal), 1, MPI_CHAR, op, root, comm);
+}
+
+template <>
+void MpiManager::reduce<int>(int& sendVal, int& recvVal,  MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Reduce(static_cast<void*>(&sendVal),
+             static_cast<void*>(&recvVal), 1, MPI_INT, op, root, comm);
+}
+
+template <>
+void MpiManager::reduce<float>(float& sendVal, float& recvVal,  MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Reduce(static_cast<void*>(&sendVal),
+             static_cast<void*>(&recvVal), 1, MPI_FLOAT, op, root, comm);
+}
+
+template <>
+void MpiManager::reduce<double>(double& sendVal, double& recvVal,  MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Reduce(static_cast<void*>(&sendVal),
+             static_cast<void*>(&recvVal), 1, MPI_DOUBLE, op, root, comm);
+}
+
+
+template <>
+void MpiManager::reduce<BlockData2D<double,int> >(BlockData2D<double,int>& sendVal, BlockData2D<double,int>& recvVal,  MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Reduce(static_cast<void*>(sendVal.getRawData()),
+             static_cast<void*>(recvVal.getRawData()),
+             sendVal.getDataSize(), MPI_DOUBLE, op, root, comm);
+}
+
+template <>
+void MpiManager::reduce<BlockData2D<double,double> >(BlockData2D<double,double>& sendVal, BlockData2D<double,double>& recvVal,  MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Reduce(static_cast<void*>(sendVal.getRawData()),
+             static_cast<void*>(recvVal.getRawData()),
+             sendVal.getDataSize(), MPI_DOUBLE, op, root, comm);
+}
+
+/*template <>
+void MpiManager::reduceVect<bool>(std::vector<bool>& sendVal, std::vector<bool>& recvVal,
+                                  MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) return;
+  MPI_Reduce(static_cast<void*>(&(sendVal[0])),
+             static_cast<void*>(&(recvVal[0])),
+             sendVal.size(), MPI_BYTE, op, root, comm);
+}
+*/
+template <>
+void MpiManager::reduceVect<char>(std::vector<char>& sendVal, std::vector<char>& recvVal,
+                                  MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Reduce(static_cast<void*>(&(sendVal[0])),
+             static_cast<void*>(&(recvVal[0])),
+             sendVal.size(), MPI_CHAR, op, root, comm);
+}
+
+template <>
+void MpiManager::reduceVect<int>(std::vector<int>& sendVal, std::vector<int>& recvVal,
+                                 MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Reduce(static_cast<void*>(&(sendVal[0])),
+             static_cast<void*>(&(recvVal[0])),
+             sendVal.size(), MPI_INT, op, root, comm);
+}
+
+template <>
+void MpiManager::reduceVect<float>(std::vector<float>& sendVal, std::vector<float>& recvVal,
+                                   MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Reduce(static_cast<void*>(&(sendVal[0])),
+             static_cast<void*>(&(recvVal[0])),
+             sendVal.size(), MPI_FLOAT, op, root, comm);
+}
+
+template <>
+void MpiManager::reduceVect<double>(std::vector<double>& sendVal, std::vector<double>& recvVal,
+                                    MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Reduce(static_cast<void*>(&(sendVal[0])),
+             static_cast<void*>(&(recvVal[0])),
+             sendVal.size(), MPI_DOUBLE, op, root, comm);
+}
+
+template <>
+void MpiManager::reduceAndBcast<bool>(bool& reductVal, MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  char recvVal;
+  MPI_Reduce(static_cast<void*>(&reductVal), static_cast<void*>(&recvVal), 1, MPI_BYTE, op, root, comm);
+  reductVal = recvVal;
+  MPI_Bcast(&reductVal, 1, MPI_BYTE, root, comm);
+
+}
+
+template <>
+void MpiManager::reduceAndBcast<char>(char& reductVal, MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  char recvVal;
+  MPI_Reduce(&reductVal, &recvVal, 1, MPI_CHAR, op, root, comm);
+  reductVal = recvVal;
+  MPI_Bcast(&reductVal, 1, MPI_CHAR, root, comm);
+
+}
+
+template <>
+void MpiManager::reduceAndBcast<int>(int& reductVal, MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  int recvVal;
+  MPI_Reduce(&reductVal, &recvVal, 1, MPI_INT, op, root, comm);
+  reductVal = recvVal;
+  MPI_Bcast(&reductVal, 1, MPI_INT, root, comm);
+
+}
+
+template <>
+void MpiManager::reduceAndBcast<float>(float& reductVal, MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  float recvVal;
+  MPI_Reduce(&reductVal, &recvVal, 1, MPI_FLOAT, op, root, comm);
+  reductVal = recvVal;
+  MPI_Bcast(&reductVal, 1, MPI_FLOAT, root, comm);
+
+}
+
+template <>
+void MpiManager::reduceAndBcast<double>(double& reductVal, MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  double recvVal;
+  MPI_Reduce(&reductVal, &recvVal, 1, MPI_DOUBLE, op, root, comm);
+  reductVal = recvVal;
+  MPI_Bcast(&reductVal, 1, MPI_DOUBLE, root, comm);
+
+}
+
+template <>
+void MpiManager::reduceAndBcast<long>(long& reductVal, MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  long recvVal;
+  MPI_Reduce(&reductVal, &recvVal, 1, MPI_LONG, op, root, comm);
+  reductVal = recvVal;
+  MPI_Bcast(&reductVal, 1, MPI_LONG, root, comm);
+
+}
+
+template <>
+void MpiManager::reduceAndBcast<unsigned long>(unsigned long& reductVal, MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+  unsigned long recvVal;
+  MPI_Reduce(&reductVal, &recvVal, 1, MPI_UNSIGNED_LONG, op, root, comm);
+  reductVal = recvVal;
+  MPI_Bcast(&reductVal, 1, MPI_UNSIGNED_LONG, root, comm);
+
+}
+
+template <>
+void MpiManager::reduceAndBcast<BlockData2D<double,double>>(BlockData2D<double,double>& reductVal, MPI_Op op, int root, MPI_Comm comm)
+{
+  if (!ok) {
+    return;
+  }
+
+  BlockData2D<double,double> recvVal(reductVal.getNx(), reductVal.getNy());
+
+  MPI_Reduce(static_cast<void*>(reductVal.getRawData()),
+             static_cast<void*>(recvVal.getRawData()),
+             reductVal.getDataSize(), MPI_DOUBLE, op, root, comm);
+
+  reductVal.swap(recvVal);
+
+  MPI_Bcast(static_cast<void*>(reductVal.getRawData()),
+            reductVal.getDataSize(), MPI_DOUBLE, root, comm);
+}
+
+void MpiManager::wait(MPI_Request* request, MPI_Status* status)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Wait(request, status);
+}
+
+void MpiManager::waitAll(MpiNonBlockingHelper& mpiNbHelper)
+{
+  if (!ok) {
+    return;
+  }
+  MPI_Waitall(mpiNbHelper.get_size(), mpiNbHelper.get_mpiRequest(), mpiNbHelper.get_mpiStatus());
+}
+
+
+MpiNonBlockingHelper::MpiNonBlockingHelper()
+{
+  _size = 0;
+}
+
+MpiNonBlockingHelper::MpiNonBlockingHelper(
+  MpiNonBlockingHelper const& rhs )
+{
+  _size          = rhs._size;
+  if (_size!=0) {
+    allocate(_size);
+    for (unsigned i=0; i<_size; i++) {
+      _mpiRequest[i] = rhs._mpiRequest[i];
+      _mpiStatus[i]  = rhs._mpiStatus[i];
+    }
+  }
+}
+
+MpiNonBlockingHelper MpiNonBlockingHelper::operator= (
+  MpiNonBlockingHelper rhs )
+{
+  MpiNonBlockingHelper tmp(rhs);
+  return tmp;
+}
+
+void MpiNonBlockingHelper::swap ( MpiNonBlockingHelper& rhs )
+{
+  std::swap(_size, rhs._size);
+  std::swap(_mpiRequest, rhs._mpiRequest);
+  std::swap(_mpiStatus, rhs._mpiStatus);
+}
+
+MpiNonBlockingHelper::~MpiNonBlockingHelper()
+{
+  free();
+}
+
+void MpiNonBlockingHelper::allocate(unsigned i)
+{
+  free();
+  _size = i;
+  _mpiRequest = new MPI_Request [i];
+  _mpiStatus  = new MPI_Status [i];
+}
+
+void MpiNonBlockingHelper::free()
+{
+  if (_size!=0) {
+    delete [] _mpiRequest;
+    delete [] _mpiStatus;
+    _size = 0;
+  }
+}
+
+unsigned const& MpiNonBlockingHelper::get_size() const
+{
+  return _size;
+}
+
+MPI_Request* MpiNonBlockingHelper::get_mpiRequest() const
+{
+  return _mpiRequest;
+}
+
+MPI_Request* MpiNonBlockingHelper::get_mpiRequest(int i) const
+{
+  assert(size_t(i) < _size);
+  return &_mpiRequest[i];
+}
+
+MPI_Status* MpiNonBlockingHelper::get_mpiStatus() const
+{
+  return _mpiStatus;
+}
+
+
+}  // namespace singleton
+
+
+}
+