Initialize at openlb-1-3

37 files changed, 5364 insertions, 0 deletions

diff --git a/src/communication/MakeHeader b/src/communication/MakeHeader
new file mode 100644
index 0000000..169f7c1
--- /dev/null
+++ b/src/communication/MakeHeader
@@ -0,0 +1,38 @@
+#  This file is part of the OpenLB library
+#
+#  Copyright (C) 2007 Mathias J. Krause
+#  E-mail contact: info@openlb.net
+#  The most recent release of OpenLB can be downloaded at
+#  <http://www.openlb.net/>
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU General Public License
+#  as published by the Free Software Foundation; either version 2
+#  of the License, or (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public 
+#  License along with this program; if not, write to the Free 
+#  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+#  Boston, MA  02110-1301, USA.
+
+
+generic        := loadBalancer \
+                  mpiManager \
+                  ompManager
+
+precompiled    := blockLoadBalancer \
+                  communicator2D \
+                  communicator3D \
+                  cuboidNeighbourhood2D \
+                  cuboidNeighbourhood3D \
+                  heuristicLoadBalancer \
+                  loadBalancer \
+                  mpiManager \
+                  ompManager \
+                  superStructure2D \
+                  superStructure3D
diff --git a/src/communication/blockLoadBalancer.cpp b/src/communication/blockLoadBalancer.cpp
new file mode 100644
index 0000000..35558e9
--- /dev/null
+++ b/src/communication/blockLoadBalancer.cpp
@@ -0,0 +1,29 @@
+/*  This file is part of the OpenLB library
+ *
+ *  Copyright (C) 2007 Mathias Krause
+ *  E-mail contact: info@openlb.net
+ *  The most recent release of OpenLB can be downloaded at
+ *  <http://www.openlb.net/>
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public
+ *  License along with this program; if not, write to the Free
+ *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+*/
+
+#include "communication/blockLoadBalancer.h"
+#include "communication/blockLoadBalancer.hh"
+
+namespace olb {
+template class BlockLoadBalancer<double>;
+}
diff --git a/src/communication/blockLoadBalancer.h b/src/communication/blockLoadBalancer.h
new file mode 100644
index 0000000..0aa0f2e
--- /dev/null
+++ b/src/communication/blockLoadBalancer.h
@@ -0,0 +1,57 @@
+/*  This file is part of the OpenLB library
+ *
+ *  Copyright (C) 2007 Mathias Krause
+ *  E-mail contact: info@openlb.net
+ *  The most recent release of OpenLB can be downloaded at
+ *  <http://www.openlb.net/>
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public
+ *  License along with this program; if not, write to the Free
+ *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+*/
+
+
+#ifndef BLOCK_LOAD_BALANCER_H
+#define BLOCK_LOAD_BALANCER_H
+
+
+#include "communication/loadBalancer.h"
+
+
+
+namespace olb {
+
+template<typename T> class CuboidGeometry3D;
+template<typename T> class CuboidGeometry2D;
+
+
+template<typename T>
+class BlockLoadBalancer : public LoadBalancer<T> {
+private:
+  int _locChunkSize;
+  int _firstGlobNum;
+  int _lastGlobNum;
+public:
+  BlockLoadBalancer() {}
+  BlockLoadBalancer(int rank, int size, int globChunkSize, int offset);
+  BlockLoadBalancer(CuboidGeometry2D<T>& cGeometry);
+  BlockLoadBalancer(CuboidGeometry3D<T>& cGeometry);
+  void init_chunkD(int rank, int size, int globChunkSize, int offset);
+  int locChunkSize() const;
+  int firstGlobNum() const;
+  int lastGlobNum() const;
+};
+}  // namespace olb
+
+#endif
diff --git a/src/communication/blockLoadBalancer.hh b/src/communication/blockLoadBalancer.hh
new file mode 100644
index 0000000..0c61af1
--- /dev/null
+++ b/src/communication/blockLoadBalancer.hh
@@ -0,0 +1,105 @@
+/*  This file is part of the OpenLB library
+ *
+ *  Copyright (C) 2007 Mathias Krause
+ *  E-mail contact: info@openlb.net
+ *  The most recent release of OpenLB can be downloaded at
+ *  <http://www.openlb.net/>
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public
+ *  License along with this program; if not, write to the Free
+ *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+*/
+
+#ifndef BLOCK_LOAD_BALANCER_HH
+#define BLOCK_LOAD_BALANCER_HH
+
+#include <vector>
+#include <map>
+#include "communication/mpiManager.h"
+#include "communication/blockLoadBalancer.h"
+#include "geometry/cuboidGeometry2D.h"
+#include "geometry/cuboidGeometry3D.h"
+#include "core/olbDebug.h"
+
+namespace olb {
+
+template<typename T>
+BlockLoadBalancer<T>::BlockLoadBalancer(int rank, int size, int globChunkSize, int offset)
+{
+  init_chunkD(rank, size, globChunkSize, offset);
+}
+
+template<typename T>
+BlockLoadBalancer<T>::BlockLoadBalancer(CuboidGeometry3D<T>& cGeometry)
+{
+  init_chunkD(singleton::mpi().getRank(), singleton::mpi().getSize(), cGeometry.getNc(), 0);
+}
+
+template<typename T>
+BlockLoadBalancer<T>::BlockLoadBalancer(CuboidGeometry2D<T>& cGeometry)
+{
+  init_chunkD(singleton::mpi().getRank(), singleton::mpi().getSize(), cGeometry.getNc(), 0);
+}
+
+template<typename T>
+void BlockLoadBalancer<T>::init_chunkD(int rank, int size, int globChunkSize, int offset)
+{
+
+  OLB_PRECONDITION(rank>=0 && size>=1 && offset>=0)
+  OLB_PRECONDITION(size<=globChunkSize &&  rank<size);
+
+  // nice way to calculate # of chunks per processor
+  this->_locChunkSize = (globChunkSize+size-rank-1)/size;
+  this->_size = _locChunkSize;
+  if (rank+1 <= globChunkSize-(globChunkSize/size)*size) {
+    this->_firstGlobNum = globChunkSize/size * rank + rank + offset;
+    this->_lastGlobNum  = this->_firstGlobNum + this->_locChunkSize - 1;
+  } else {
+    this->_firstGlobNum = globChunkSize/size * rank + globChunkSize - (globChunkSize/size)*size + offset;
+    this->_lastGlobNum  = this->_firstGlobNum + this->_locChunkSize - 1;
+  }
+  for (int i=0; i<this->_locChunkSize; i++) {
+    this->_loc[this->_firstGlobNum + i] = i;
+    this->_glob.push_back(this->_firstGlobNum + i);
+  }
+  int temp = offset;
+  for (int iRank=0; iRank<size; iRank++) {
+    int iLocChunkSize = (globChunkSize+size-iRank-1)/size;
+    for (int i=0; i<iLocChunkSize; i++) {
+      this->_rank[temp] = iRank;
+      temp++;
+    }
+  }
+}
+
+template<typename T>
+int BlockLoadBalancer<T>::locChunkSize() const
+{
+  return _locChunkSize;
+}
+
+template<typename T>
+int BlockLoadBalancer<T>::firstGlobNum() const
+{
+  return _firstGlobNum;
+}
+
+template<typename T>
+int BlockLoadBalancer<T>::lastGlobNum() const
+{
+  return _lastGlobNum;
+}
+
+}  // namespace olb
+#endif
diff --git a/src/communication/communication2D.h b/src/communication/communication2D.h
new file mode 100644
index 0000000..904c55d
--- /dev/null
+++ b/src/communication/communication2D.h
@@ -0,0 +1,41 @@
+/*  This file is part of the OpenLB library
+ *
+ *  Copyright (C) 2007 The OpenLB project
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public
+ *  License along with this program; if not, write to the Free
+ *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+*/
+
+/** \file
+ * Groups all the include files for 2D dynamics in
+ * the dataStructure directory.
+ */
+
+#ifndef COMMUNICATION_2D_H
+#define COMMUNICATION_2D_H
+
+#include "blockLoadBalancer.h"
+#include "communicator2D.h"
+#include "cuboidNeighbourhood2D.h"
+#include "heuristicLoadBalancer.h"
+#include "loadBalancer.h"
+#include "mpiManager.h"
+#include "ompManager.h"
+#include "superStructure2D.h"
+
+
+
+
+#endif
diff --git a/src/communication/communication2D.hh b/src/communication/communication2D.hh
new file mode 100644
index 0000000..797ca0f
--- /dev/null
+++ b/src/communication/communication2D.hh
@@ -0,0 +1,38 @@
+/*  This file is part of the OpenLB library
+ *
+ *  Copyright (C) 2007 The OpenLB project
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public
+ *  License along with this program; if not, write to the Free
+ *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+*/
+
+/** \file
+ * Groups all the include files for 2D dynamics in
+ * the dataStructure directory.
+ */
+
+#ifndef COMMUNICATION_2D_HH
+#define COMMUNICATION_2D_HH
+
+#include "blockLoadBalancer.hh"
+#include "communicator2D.hh"
+#include "cuboidNeighbourhood2D.hh"
+#include "heuristicLoadBalancer.hh"
+#include "loadBalancer.hh"
+#include "superStructure2D.hh"
+
+
+
+#endif
diff --git a/src/communication/communication3D.h b/src/communication/communication3D.h
new file mode 100644
index 0000000..7ca6eca
--- /dev/null
+++ b/src/communication/communication3D.h
@@ -0,0 +1,42 @@
+/*  This file is part of the OpenLB library
+ *
+ *  Copyright (C) 2007 The OpenLB project
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public
+ *  License along with this program; if not, write to the Free
+ *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+*/
+
+/** \file
+ * Groups all the include files for 2D dynamics in
+ * the dataStructure directory.
+ */
+
+#ifndef COMMUNICATION_3D_H
+#define COMMUNICATION_3D_H
+
+#include "blockLoadBalancer.h"
+#include "communicator3D.h"
+#include "cuboidNeighbourhood3D.h"
+#include "heuristicLoadBalancer.h"
+#include "loadBalancer.h"
+#include "mpiManager.h"
+#include "ompManager.h"
+#include "superStructure3D.h"
+
+
+
+
+
+#endif
diff --git a/src/communication/communication3D.hh b/src/communication/communication3D.hh
new file mode 100644
index 0000000..d4d4705
--- /dev/null
+++ b/src/communication/communication3D.hh
@@ -0,0 +1,38 @@
+/*  This file is part of the OpenLB library
+ *
+ *  Copyright (C) 2007 The OpenLB project
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public
+ *  License along with this program; if not, write to the Free
+ *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+*/
+
+/** \file
+ * Groups all the include files for 2D dynamics in
+ * the dataStructure directory.
+ */
+
+#ifndef COMMUNICATION_3D_HH
+#define COMMUNICATION_3D_HH
+
+#include "blockLoadBalancer.hh"
+#include "communicator3D.hh"
+#include "cuboidNeighbourhood3D.hh"
+#include "heuristicLoadBalancer.hh"
+#include "loadBalancer.hh"
+#include "superStructure3D.hh"
+
+
+
+#endif
diff --git a/src/communication/communicator2D.cpp b/src/communication/communicator2D.cpp
new file mode 100644
index 0000000..e509a06
--- /dev/null
+++ b/src/communication/communicator2D.cpp
@@ -0,0 +1,37 @@
+/*  This file is part of the OpenLB library
+*
+*  Copyright (C) 2007, 2014 Mathias J. Krause
+*  E-mail contact: info@openlb.net
+*  The most recent release of OpenLB can be downloaded at
+*  <http://www.openlb.net/>
+*
+*  This program is free software; you can redistribute it and/or
+*  modify it under the terms of the GNU General Public License
+*  as published by the Free Software Foundation; either version 2
+*  of the License, or (at your option) any later version.
+*
+*  This program is distributed in the hope that it will be useful,
+*  but WITHOUT ANY WARRANTY; without even the implied warranty of
+*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*  GNU General Public License for more details.
+*
+*  You should have received a copy of the GNU General Public
+*  License along with this program; if not, write to the Free
+*  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+*  Boston, MA  02110-1301, USA.
+*/
+
+/** \file
+ * A communincator provides a cuboids with cells of other
+ * cuboids -- template instantiation.
+ */
+
+#include "communication/mpiManager.h"
+#include "communication/communicator2D.h"
+#include "communication/communicator2D.hh"
+
+namespace olb {
+
+template class Communicator2D<double>;
+
+}
diff --git a/src/communication/communicator2D.h b/src/communication/communicator2D.h
new file mode 100644
index 0000000..8be8c2a
--- /dev/null
+++ b/src/communication/communicator2D.h
@@ -0,0 +1,99 @@
+/*  This file is part of the OpenLB library
+ *
+ *  Copyright (C) 2007, 2014 Mathias J. Krause
+ *  E-mail contact: info@openlb.net
+ *  The most recent release of OpenLB can be downloaded at
+ *  <http://www.openlb.net/>
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public
+ *  License along with this program; if not, write to the Free
+ *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+*/
+
+/** \file
+ * A communincator provides a cuboids with cells of other
+ * cuboids -- header file.
+ */
+
+
+#ifndef COMMUNICATOR_2D_H
+#define COMMUNICATOR_2D_H
+
+#include <vector>
+#include "communication/cuboidNeighbourhood2D.h"
+#include "communication/superStructure2D.h"
+
+
+/// All OpenLB code is contained in this namespace.
+namespace olb {
+
+/// A communicator provides a cuboid with data of other
+/// cuboids.
+/** For each cuboid a cuboid neighbourhood is defined. A communicator
+ * coordinates the communication of the single cuboids, in which the data
+ * is stored locally.
+ *
+ * There are methods to add needed cells, send, receive and write
+ * data of the cell.
+ *
+ * This class is not intended to be derived from.
+ */
+
+
+template<typename T> class CuboidNeighbourhood2D;
+template<typename T> class SuperStructure2D;
+
+template<typename T>
+class Communicator2D {
+
+private:
+  /// Stores neighbourhoods information for a single cuboid
+  std::vector<CuboidNeighbourhood2D<T> > _nh;
+  /// Reference to the super structure
+  SuperStructure2D<T>& _superStructure;
+  /// Number of cuboids in the structure
+  int _nC;
+  /// Indicates if there was an initialization done
+  bool _initDone;
+public:
+  /// Constructor
+  Communicator2D(SuperStructure2D<T>& superStructure);
+  /// Initializes the cuboid neighbourhoods
+  void init_nh();
+
+  /// Adds a cell to the cuboid neigbourhood iCloc
+  /// that is then able to get data of the cell
+  void add_cell(int iCloc, int iX, int iY);
+  /// Adds all cells of the overlap to its associated cuboid
+  /// neigbourhood
+  void add_cells(int overlap);
+  /// Initializes the communicator
+  void init();
+  /// Resets all cuboids neighbourhoods
+  void reset();
+
+  /// Sends the needed data
+  void send();
+  /// Receives the needed data
+  void receive();
+  /// Writes all data to the block lattices of the super structure
+  void write();
+
+  /// Access to internal data for the tests
+  std::vector<CuboidNeighbourhood2D<T>>& get_nh();
+};
+
+}  // namespace olb
+
+#endif
diff --git a/src/communication/communicator2D.hh b/src/communication/communicator2D.hh
new file mode 100644
index 0000000..86a5dcb
--- /dev/null
+++ b/src/communication/communicator2D.hh
@@ -0,0 +1,185 @@
+/*  This file is part of the OpenLB library
+ *
+ *  Copyright (C) 2007, 2014 Mathias J. Krause
+ *  E-mail contact: info@openlb.net
+ *  The most recent release of OpenLB can be downloaded at
+ *  <http://www.openlb.net/>
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public
+ *  License along with this program; if not, write to the Free
+ *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+*/
+
+/** \file
+ * A communincator provides a cuboids with cells of other
+ * cuboids -- generic implementation.
+ */
+
+
+#ifndef COMMUNICATOR_2D_HH
+#define COMMUNICATOR_2D_HH
+
+#include "communication/mpiManager.h"
+#include <vector>
+#include "communication/communicator2D.h"
+#include "communication/cuboidNeighbourhood2D.h"
+#include "geometry/cuboidGeometry2D.h"
+#include "communication/superStructure2D.h"
+
+
+namespace olb {
+
+/////////////////// Class Communicator2D //////////////////////
+
+
+template<typename T>
+Communicator2D<T>::Communicator2D(SuperStructure2D<T>& superStructure):_superStructure(superStructure)
+{
+  _initDone = false;
+}
+
+template<typename T>
+void Communicator2D<T>::init_nh()
+{
+
+  _nC = _superStructure.getCuboidGeometry().getNc();
+
+  for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) {
+    CuboidNeighbourhood2D<T> nh(_superStructure,_superStructure.getLoadBalancer().glob(iC));
+    _nh.push_back(nh);
+  }
+}
+
+template<typename T>
+void Communicator2D<T>::add_cell(int iCloc, int iX, int iY)
+{
+
+  _nh[iCloc].add_inCell(iX,iY);
+}
+
+template<typename T>
+void Communicator2D<T>::add_cells(int overlap)
+{
+
+  for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) {
+    _nh[iC].add_inCells(overlap);
+  }
+}
+
+template<typename T>
+void Communicator2D<T>::init()
+{
+
+  reset();
+  for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) {
+    _nh[iC].init_inCN();
+    for (int i=0; i<_nh[iC].get_inCellsSize(); i++) {
+      int ID = _nh[iC].get_inCell(i).latticeR[0];
+#ifdef PARALLEL_MODE_MPI
+      if ( singleton::mpi().getRank() ==_superStructure.getLoadBalancer().rank(ID) )
+#endif
+      {
+        Cell2D<T> temp;
+        temp.physR[0] = _nh[iC].get_inCell(i).physR[0];
+        temp.physR[1] = _nh[iC].get_inCell(i).physR[1];
+        _superStructure.getCuboidGeometry().getLatticeR(temp.physR, temp.latticeR);
+        temp.latticeR[0]    = _superStructure.getLoadBalancer().glob(iC);
+        _nh[_superStructure.getLoadBalancer().loc(ID)].add_outCell(temp);
+      }
+    }
+  }
+
+  for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) {
+    _nh[iC].init_outCN();
+  }
+
+#ifdef PARALLEL_MODE_MPI
+  for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) {
+    _nh[iC].finish_comm();
+  }
+  for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) {
+    _nh[iC].bufSend_inCells();
+  }
+  for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) {
+    _nh[iC].recWrite_outCells();
+  }
+  for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) {
+    _nh[iC].finish_comm();
+  }
+#endif
+}
+
+template<typename T>
+void Communicator2D<T>::reset()
+{
+
+  if (_initDone) {
+    for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) {
+      _nh[iC].reset();
+    }
+    _initDone = false;
+  }
+}
+
+template<typename T>
+void Communicator2D<T>::send()
+{
+
+  for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) {
+    _nh[iC].buffer_outData();
+#ifdef PARALLEL_MODE_MPI
+    _nh[iC].send_outData();
+#endif
+  }
+}
+
+template<typename T>
+void Communicator2D<T>::receive()
+{
+
+  for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) {
+#ifdef PARALLEL_MODE_MPI
+    _nh[iC].receive_inData();
+#else
+    for (int i=0; i<_nh[iC].get_inCsize(); i++) {
+      _nh[iC].get_inData()[_nh[iC].get_inC(i)] =
+        _nh[_nh[iC].get_inC(i)].get_outData()[iC];
+    }
+#endif
+  }
+#ifdef PARALLEL_MODE_MPI
+  for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) {
+    _nh[iC].finish_comm();
+  }
+#endif
+}
+
+template<typename T>
+void Communicator2D<T>::write()
+{
+
+  for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) {
+    _nh[iC].write_inData();
+  }