From 94d3e79a8617f88dc0219cfdeedfa3147833719d Mon Sep 17 00:00:00 2001 From: Adrian Kummerlaender Date: Mon, 24 Jun 2019 14:43:36 +0200 Subject: Initialize at openlb-1-3 --- src/communication/MakeHeader | 38 + src/communication/blockLoadBalancer.cpp | 29 + src/communication/blockLoadBalancer.h | 57 ++ src/communication/blockLoadBalancer.hh | 105 +++ src/communication/communication2D.h | 41 + src/communication/communication2D.hh | 38 + src/communication/communication3D.h | 42 + src/communication/communication3D.hh | 38 + src/communication/communicator2D.cpp | 37 + src/communication/communicator2D.h | 99 +++ src/communication/communicator2D.hh | 185 +++++ src/communication/communicator3D.cpp | 37 + src/communication/communicator3D.h | 97 +++ src/communication/communicator3D.hh | 181 +++++ src/communication/cuboidNeighbourhood2D.cpp | 38 + src/communication/cuboidNeighbourhood2D.h | 195 +++++ src/communication/cuboidNeighbourhood2D.hh | 452 +++++++++++ src/communication/cuboidNeighbourhood3D.cpp | 38 + src/communication/cuboidNeighbourhood3D.h | 190 +++++ src/communication/cuboidNeighbourhood3D.hh | 449 +++++++++++ src/communication/heuristicLoadBalancer.cpp | 29 + src/communication/heuristicLoadBalancer.h | 85 ++ src/communication/heuristicLoadBalancer.hh | 402 ++++++++++ src/communication/loadBalancer.cpp | 31 + src/communication/loadBalancer.h | 208 +++++ src/communication/loadBalancer.hh | 167 ++++ src/communication/module.mk | 27 + src/communication/mpiManager.cpp | 1154 +++++++++++++++++++++++++++ src/communication/mpiManager.h | 279 +++++++ src/communication/ompManager.cpp | 59 ++ src/communication/ompManager.h | 49 ++ src/communication/superStructure2D.cpp | 34 + src/communication/superStructure2D.h | 91 +++ src/communication/superStructure2D.hh | 114 +++ src/communication/superStructure3D.cpp | 34 + src/communication/superStructure3D.h | 95 +++ src/communication/superStructure3D.hh | 120 +++ 37 files changed, 5364 insertions(+) create mode 100644 src/communication/MakeHeader create mode 100644 src/communication/blockLoadBalancer.cpp create mode 100644 src/communication/blockLoadBalancer.h create mode 100644 src/communication/blockLoadBalancer.hh create mode 100644 src/communication/communication2D.h create mode 100644 src/communication/communication2D.hh create mode 100644 src/communication/communication3D.h create mode 100644 src/communication/communication3D.hh create mode 100644 src/communication/communicator2D.cpp create mode 100644 src/communication/communicator2D.h create mode 100644 src/communication/communicator2D.hh create mode 100644 src/communication/communicator3D.cpp create mode 100644 src/communication/communicator3D.h create mode 100644 src/communication/communicator3D.hh create mode 100644 src/communication/cuboidNeighbourhood2D.cpp create mode 100644 src/communication/cuboidNeighbourhood2D.h create mode 100644 src/communication/cuboidNeighbourhood2D.hh create mode 100644 src/communication/cuboidNeighbourhood3D.cpp create mode 100644 src/communication/cuboidNeighbourhood3D.h create mode 100644 src/communication/cuboidNeighbourhood3D.hh create mode 100644 src/communication/heuristicLoadBalancer.cpp create mode 100644 src/communication/heuristicLoadBalancer.h create mode 100644 src/communication/heuristicLoadBalancer.hh create mode 100644 src/communication/loadBalancer.cpp create mode 100644 src/communication/loadBalancer.h create mode 100644 src/communication/loadBalancer.hh create mode 100644 src/communication/module.mk create mode 100644 src/communication/mpiManager.cpp create mode 100644 src/communication/mpiManager.h create mode 100644 src/communication/ompManager.cpp create mode 100644 src/communication/ompManager.h create mode 100644 src/communication/superStructure2D.cpp create mode 100644 src/communication/superStructure2D.h create mode 100644 src/communication/superStructure2D.hh create mode 100644 src/communication/superStructure3D.cpp create mode 100644 src/communication/superStructure3D.h create mode 100644 src/communication/superStructure3D.hh (limited to 'src/communication') diff --git a/src/communication/MakeHeader b/src/communication/MakeHeader new file mode 100644 index 0000000..169f7c1 --- /dev/null +++ b/src/communication/MakeHeader @@ -0,0 +1,38 @@ +# This file is part of the OpenLB library +# +# Copyright (C) 2007 Mathias J. Krause +# E-mail contact: info@openlb.net +# The most recent release of OpenLB can be downloaded at +# +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with this program; if not, write to the Free +# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. + + +generic := loadBalancer \ + mpiManager \ + ompManager + +precompiled := blockLoadBalancer \ + communicator2D \ + communicator3D \ + cuboidNeighbourhood2D \ + cuboidNeighbourhood3D \ + heuristicLoadBalancer \ + loadBalancer \ + mpiManager \ + ompManager \ + superStructure2D \ + superStructure3D diff --git a/src/communication/blockLoadBalancer.cpp b/src/communication/blockLoadBalancer.cpp new file mode 100644 index 0000000..35558e9 --- /dev/null +++ b/src/communication/blockLoadBalancer.cpp @@ -0,0 +1,29 @@ +/* This file is part of the OpenLB library + * + * Copyright (C) 2007 Mathias Krause + * E-mail contact: info@openlb.net + * The most recent release of OpenLB can be downloaded at + * + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. +*/ + +#include "communication/blockLoadBalancer.h" +#include "communication/blockLoadBalancer.hh" + +namespace olb { +template class BlockLoadBalancer; +} diff --git a/src/communication/blockLoadBalancer.h b/src/communication/blockLoadBalancer.h new file mode 100644 index 0000000..0aa0f2e --- /dev/null +++ b/src/communication/blockLoadBalancer.h @@ -0,0 +1,57 @@ +/* This file is part of the OpenLB library + * + * Copyright (C) 2007 Mathias Krause + * E-mail contact: info@openlb.net + * The most recent release of OpenLB can be downloaded at + * + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. +*/ + + +#ifndef BLOCK_LOAD_BALANCER_H +#define BLOCK_LOAD_BALANCER_H + + +#include "communication/loadBalancer.h" + + + +namespace olb { + +template class CuboidGeometry3D; +template class CuboidGeometry2D; + + +template +class BlockLoadBalancer : public LoadBalancer { +private: + int _locChunkSize; + int _firstGlobNum; + int _lastGlobNum; +public: + BlockLoadBalancer() {} + BlockLoadBalancer(int rank, int size, int globChunkSize, int offset); + BlockLoadBalancer(CuboidGeometry2D& cGeometry); + BlockLoadBalancer(CuboidGeometry3D& cGeometry); + void init_chunkD(int rank, int size, int globChunkSize, int offset); + int locChunkSize() const; + int firstGlobNum() const; + int lastGlobNum() const; +}; +} // namespace olb + +#endif diff --git a/src/communication/blockLoadBalancer.hh b/src/communication/blockLoadBalancer.hh new file mode 100644 index 0000000..0c61af1 --- /dev/null +++ b/src/communication/blockLoadBalancer.hh @@ -0,0 +1,105 @@ +/* This file is part of the OpenLB library + * + * Copyright (C) 2007 Mathias Krause + * E-mail contact: info@openlb.net + * The most recent release of OpenLB can be downloaded at + * + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. +*/ + +#ifndef BLOCK_LOAD_BALANCER_HH +#define BLOCK_LOAD_BALANCER_HH + +#include +#include +#include "communication/mpiManager.h" +#include "communication/blockLoadBalancer.h" +#include "geometry/cuboidGeometry2D.h" +#include "geometry/cuboidGeometry3D.h" +#include "core/olbDebug.h" + +namespace olb { + +template +BlockLoadBalancer::BlockLoadBalancer(int rank, int size, int globChunkSize, int offset) +{ + init_chunkD(rank, size, globChunkSize, offset); +} + +template +BlockLoadBalancer::BlockLoadBalancer(CuboidGeometry3D& cGeometry) +{ + init_chunkD(singleton::mpi().getRank(), singleton::mpi().getSize(), cGeometry.getNc(), 0); +} + +template +BlockLoadBalancer::BlockLoadBalancer(CuboidGeometry2D& cGeometry) +{ + init_chunkD(singleton::mpi().getRank(), singleton::mpi().getSize(), cGeometry.getNc(), 0); +} + +template +void BlockLoadBalancer::init_chunkD(int rank, int size, int globChunkSize, int offset) +{ + + OLB_PRECONDITION(rank>=0 && size>=1 && offset>=0) + OLB_PRECONDITION(size<=globChunkSize && rank_locChunkSize = (globChunkSize+size-rank-1)/size; + this->_size = _locChunkSize; + if (rank+1 <= globChunkSize-(globChunkSize/size)*size) { + this->_firstGlobNum = globChunkSize/size * rank + rank + offset; + this->_lastGlobNum = this->_firstGlobNum + this->_locChunkSize - 1; + } else { + this->_firstGlobNum = globChunkSize/size * rank + globChunkSize - (globChunkSize/size)*size + offset; + this->_lastGlobNum = this->_firstGlobNum + this->_locChunkSize - 1; + } + for (int i=0; i_locChunkSize; i++) { + this->_loc[this->_firstGlobNum + i] = i; + this->_glob.push_back(this->_firstGlobNum + i); + } + int temp = offset; + for (int iRank=0; iRank_rank[temp] = iRank; + temp++; + } + } +} + +template +int BlockLoadBalancer::locChunkSize() const +{ + return _locChunkSize; +} + +template +int BlockLoadBalancer::firstGlobNum() const +{ + return _firstGlobNum; +} + +template +int BlockLoadBalancer::lastGlobNum() const +{ + return _lastGlobNum; +} + +} // namespace olb +#endif diff --git a/src/communication/communication2D.h b/src/communication/communication2D.h new file mode 100644 index 0000000..904c55d --- /dev/null +++ b/src/communication/communication2D.h @@ -0,0 +1,41 @@ +/* This file is part of the OpenLB library + * + * Copyright (C) 2007 The OpenLB project + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. +*/ + +/** \file + * Groups all the include files for 2D dynamics in + * the dataStructure directory. + */ + +#ifndef COMMUNICATION_2D_H +#define COMMUNICATION_2D_H + +#include "blockLoadBalancer.h" +#include "communicator2D.h" +#include "cuboidNeighbourhood2D.h" +#include "heuristicLoadBalancer.h" +#include "loadBalancer.h" +#include "mpiManager.h" +#include "ompManager.h" +#include "superStructure2D.h" + + + + +#endif diff --git a/src/communication/communication2D.hh b/src/communication/communication2D.hh new file mode 100644 index 0000000..797ca0f --- /dev/null +++ b/src/communication/communication2D.hh @@ -0,0 +1,38 @@ +/* This file is part of the OpenLB library + * + * Copyright (C) 2007 The OpenLB project + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. +*/ + +/** \file + * Groups all the include files for 2D dynamics in + * the dataStructure directory. + */ + +#ifndef COMMUNICATION_2D_HH +#define COMMUNICATION_2D_HH + +#include "blockLoadBalancer.hh" +#include "communicator2D.hh" +#include "cuboidNeighbourhood2D.hh" +#include "heuristicLoadBalancer.hh" +#include "loadBalancer.hh" +#include "superStructure2D.hh" + + + +#endif diff --git a/src/communication/communication3D.h b/src/communication/communication3D.h new file mode 100644 index 0000000..7ca6eca --- /dev/null +++ b/src/communication/communication3D.h @@ -0,0 +1,42 @@ +/* This file is part of the OpenLB library + * + * Copyright (C) 2007 The OpenLB project + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. +*/ + +/** \file + * Groups all the include files for 2D dynamics in + * the dataStructure directory. + */ + +#ifndef COMMUNICATION_3D_H +#define COMMUNICATION_3D_H + +#include "blockLoadBalancer.h" +#include "communicator3D.h" +#include "cuboidNeighbourhood3D.h" +#include "heuristicLoadBalancer.h" +#include "loadBalancer.h" +#include "mpiManager.h" +#include "ompManager.h" +#include "superStructure3D.h" + + + + + +#endif diff --git a/src/communication/communication3D.hh b/src/communication/communication3D.hh new file mode 100644 index 0000000..d4d4705 --- /dev/null +++ b/src/communication/communication3D.hh @@ -0,0 +1,38 @@ +/* This file is part of the OpenLB library + * + * Copyright (C) 2007 The OpenLB project + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. +*/ + +/** \file + * Groups all the include files for 2D dynamics in + * the dataStructure directory. + */ + +#ifndef COMMUNICATION_3D_HH +#define COMMUNICATION_3D_HH + +#include "blockLoadBalancer.hh" +#include "communicator3D.hh" +#include "cuboidNeighbourhood3D.hh" +#include "heuristicLoadBalancer.hh" +#include "loadBalancer.hh" +#include "superStructure3D.hh" + + + +#endif diff --git a/src/communication/communicator2D.cpp b/src/communication/communicator2D.cpp new file mode 100644 index 0000000..e509a06 --- /dev/null +++ b/src/communication/communicator2D.cpp @@ -0,0 +1,37 @@ +/* This file is part of the OpenLB library +* +* Copyright (C) 2007, 2014 Mathias J. Krause +* E-mail contact: info@openlb.net +* The most recent release of OpenLB can be downloaded at +* +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public +* License along with this program; if not, write to the Free +* Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +* Boston, MA 02110-1301, USA. +*/ + +/** \file + * A communincator provides a cuboids with cells of other + * cuboids -- template instantiation. + */ + +#include "communication/mpiManager.h" +#include "communication/communicator2D.h" +#include "communication/communicator2D.hh" + +namespace olb { + +template class Communicator2D; + +} diff --git a/src/communication/communicator2D.h b/src/communication/communicator2D.h new file mode 100644 index 0000000..8be8c2a --- /dev/null +++ b/src/communication/communicator2D.h @@ -0,0 +1,99 @@ +/* This file is part of the OpenLB library + * + * Copyright (C) 2007, 2014 Mathias J. Krause + * E-mail contact: info@openlb.net + * The most recent release of OpenLB can be downloaded at + * + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. +*/ + +/** \file + * A communincator provides a cuboids with cells of other + * cuboids -- header file. + */ + + +#ifndef COMMUNICATOR_2D_H +#define COMMUNICATOR_2D_H + +#include +#include "communication/cuboidNeighbourhood2D.h" +#include "communication/superStructure2D.h" + + +/// All OpenLB code is contained in this namespace. +namespace olb { + +/// A communicator provides a cuboid with data of other +/// cuboids. +/** For each cuboid a cuboid neighbourhood is defined. A communicator + * coordinates the communication of the single cuboids, in which the data + * is stored locally. + * + * There are methods to add needed cells, send, receive and write + * data of the cell. + * + * This class is not intended to be derived from. + */ + + +template class CuboidNeighbourhood2D; +template class SuperStructure2D; + +template +class Communicator2D { + +private: + /// Stores neighbourhoods information for a single cuboid + std::vector > _nh; + /// Reference to the super structure + SuperStructure2D& _superStructure; + /// Number of cuboids in the structure + int _nC; + /// Indicates if there was an initialization done + bool _initDone; +public: + /// Constructor + Communicator2D(SuperStructure2D& superStructure); + /// Initializes the cuboid neighbourhoods + void init_nh(); + + /// Adds a cell to the cuboid neigbourhood iCloc + /// that is then able to get data of the cell + void add_cell(int iCloc, int iX, int iY); + /// Adds all cells of the overlap to its associated cuboid + /// neigbourhood + void add_cells(int overlap); + /// Initializes the communicator + void init(); + /// Resets all cuboids neighbourhoods + void reset(); + + /// Sends the needed data + void send(); + /// Receives the needed data + void receive(); + /// Writes all data to the block lattices of the super structure + void write(); + + /// Access to internal data for the tests + std::vector>& get_nh(); +}; + +} // namespace olb + +#endif diff --git a/src/communication/communicator2D.hh b/src/communication/communicator2D.hh new file mode 100644 index 0000000..86a5dcb --- /dev/null +++ b/src/communication/communicator2D.hh @@ -0,0 +1,185 @@ +/* This file is part of the OpenLB library + * + * Copyright (C) 2007, 2014 Mathias J. Krause + * E-mail contact: info@openlb.net + * The most recent release of OpenLB can be downloaded at + * + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. +*/ + +/** \file + * A communincator provides a cuboids with cells of other + * cuboids -- generic implementation. + */ + + +#ifndef COMMUNICATOR_2D_HH +#define COMMUNICATOR_2D_HH + +#include "communication/mpiManager.h" +#include +#include "communication/communicator2D.h" +#include "communication/cuboidNeighbourhood2D.h" +#include "geometry/cuboidGeometry2D.h" +#include "communication/superStructure2D.h" + + +namespace olb { + +/////////////////// Class Communicator2D ////////////////////// + + +template +Communicator2D::Communicator2D(SuperStructure2D& superStructure):_superStructure(superStructure) +{ + _initDone = false; +} + +template +void Communicator2D::init_nh() +{ + + _nC = _superStructure.getCuboidGeometry().getNc(); + + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + CuboidNeighbourhood2D nh(_superStructure,_superStructure.getLoadBalancer().glob(iC)); + _nh.push_back(nh); + } +} + +template +void Communicator2D::add_cell(int iCloc, int iX, int iY) +{ + + _nh[iCloc].add_inCell(iX,iY); +} + +template +void Communicator2D::add_cells(int overlap) +{ + + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].add_inCells(overlap); + } +} + +template +void Communicator2D::init() +{ + + reset(); + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].init_inCN(); + for (int i=0; i<_nh[iC].get_inCellsSize(); i++) { + int ID = _nh[iC].get_inCell(i).latticeR[0]; +#ifdef PARALLEL_MODE_MPI + if ( singleton::mpi().getRank() ==_superStructure.getLoadBalancer().rank(ID) ) +#endif + { + Cell2D temp; + temp.physR[0] = _nh[iC].get_inCell(i).physR[0]; + temp.physR[1] = _nh[iC].get_inCell(i).physR[1]; + _superStructure.getCuboidGeometry().getLatticeR(temp.physR, temp.latticeR); + temp.latticeR[0] = _superStructure.getLoadBalancer().glob(iC); + _nh[_superStructure.getLoadBalancer().loc(ID)].add_outCell(temp); + } + } + } + + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].init_outCN(); + } + +#ifdef PARALLEL_MODE_MPI + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].finish_comm(); + } + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].bufSend_inCells(); + } + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].recWrite_outCells(); + } + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].finish_comm(); + } +#endif +} + +template +void Communicator2D::reset() +{ + + if (_initDone) { + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].reset(); + } + _initDone = false; + } +} + +template +void Communicator2D::send() +{ + + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].buffer_outData(); +#ifdef PARALLEL_MODE_MPI + _nh[iC].send_outData(); +#endif + } +} + +template +void Communicator2D::receive() +{ + + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { +#ifdef PARALLEL_MODE_MPI + _nh[iC].receive_inData(); +#else + for (int i=0; i<_nh[iC].get_inCsize(); i++) { + _nh[iC].get_inData()[_nh[iC].get_inC(i)] = + _nh[_nh[iC].get_inC(i)].get_outData()[iC]; + } +#endif + } +#ifdef PARALLEL_MODE_MPI + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].finish_comm(); + } +#endif +} + +template +void Communicator2D::write() +{ + + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].write_inData(); + } +} + +template +std::vector >& Communicator2D::get_nh() +{ + return _nh; +} + +} + +#endif diff --git a/src/communication/communicator3D.cpp b/src/communication/communicator3D.cpp new file mode 100644 index 0000000..1e8c905 --- /dev/null +++ b/src/communication/communicator3D.cpp @@ -0,0 +1,37 @@ +/* This file is part of the OpenLB library +* +* Copyright (C) 2007, 2014 Mathias J. Krause +* E-mail contact: info@openlb.net +* The most recent release of OpenLB can be downloaded at +* +* +* This program is free software; you can redistribute it and/or +* modify it under the terms of the GNU General Public License +* as published by the Free Software Foundation; either version 2 +* of the License, or (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public +* License along with this program; if not, write to the Free +* Software Foundation, Inc., 51 Franklin Street, Fifth Floor, +* Boston, MA 02110-1301, USA. +*/ + +/** \file + * A communincator provides a cuboids with cells of other + * cuboids -- template instantiation. + */ + +#include "communication/mpiManager.h" +#include "communication/communicator3D.h" +#include "communication/communicator3D.hh" + +namespace olb { + +template class Communicator3D; + +} diff --git a/src/communication/communicator3D.h b/src/communication/communicator3D.h new file mode 100644 index 0000000..b7f3780 --- /dev/null +++ b/src/communication/communicator3D.h @@ -0,0 +1,97 @@ +/* This file is part of the OpenLB library + * + * Copyright (C) 2007, 2014 Mathias J. Krause + * E-mail contact: info@openlb.net + * The most recent release of OpenLB can be downloaded at + * + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. +*/ + +/** \file + * A communincator provides a cuboids with cells of other + * cuboids -- header file. + */ + + +#ifndef COMMUNICATOR_3D_H +#define COMMUNICATOR_3D_H + +#include +#include "communication/cuboidNeighbourhood3D.h" +#include "communication/superStructure3D.h" + + +/// All OpenLB code is contained in this namespace. +namespace olb { + +/// A communicator provides a cuboid with data of other +/// cuboids. +/** For each cuboid a cuboid neighbourhood is defined. A communicator + * coordinates the communication of the single cuboids, in which the data + * is stored locally. + * + * There are methods to add needed cells, send, receive and write + * data of the cell. + * + * This class is not intended to be derived from. + */ + + +template class CuboidNeighbourhood3D; +template class SuperStructure3D; + +template +class Communicator3D { + +private: + /// Stores neighbourhoods information for a single cuboid + std::vector > _nh; + /// Reference to the super structure + SuperStructure3D& _superStructure; + /// Number of cuboids in the structure + int _nC; + /// Indicates if there was an initialization done + bool _initDone; +public: + ~Communicator3D() {}; + /// Constructor + Communicator3D(SuperStructure3D& superStructure); + /// Initializes the cuboid neighbourhoods + void init_nh(); + + /// Adds a cell to the cuboid neigbourhood iCloc + /// that is then able to get data of the cell + void add_cell(int iCloc, int iX, int iY, int iZ); + /// Adds all cells of the overlap to its associated cuboid + /// neigbourhood + void add_cells(int overlap); + /// Initializes the communicator + void init(); + /// Resets all cuboids neighbourhoods + void reset(); + + /// Sends the needed data + void send(); + /// Receives the needed data + void receive(); + /// Writes all data to the block lattices of the super structure + void write(); +}; + +} // namespace olb + +#endif diff --git a/src/communication/communicator3D.hh b/src/communication/communicator3D.hh new file mode 100644 index 0000000..000df93 --- /dev/null +++ b/src/communication/communicator3D.hh @@ -0,0 +1,181 @@ +/* This file is part of the OpenLB library + * + * Copyright (C) 2007, 2014 Mathias J. Krause + * E-mail contact: info@openlb.net + * The most recent release of OpenLB can be downloaded at + * + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. +*/ + +/** \file + * A communincator provides a cuboids with cells of other + * cuboids -- generic implementation. + */ + + +#ifndef COMMUNICATOR_3D_HH +#define COMMUNICATOR_3D_HH + +#include "communication/mpiManager.h" +#include +#include "communication/communicator3D.h" +#include "communication/cuboidNeighbourhood3D.h" +#include "geometry/cuboidGeometry3D.h" +#include "communication/superStructure3D.h" + + +namespace olb { + +/////////////////// Class Communicator3D ////////////////////// + + +template +Communicator3D::Communicator3D(SuperStructure3D& superStructure):_superStructure(superStructure) +{ + _initDone = false; +} + +template +void Communicator3D::init_nh() +{ + + _nC = _superStructure.getCuboidGeometry().getNc(); + + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + CuboidNeighbourhood3D nh(_superStructure,_superStructure.getLoadBalancer().glob(iC)); + _nh.push_back(nh); + } +} + +template +void Communicator3D::add_cell(int iCloc, int iX, int iY, int iZ) +{ + + _nh[iCloc].add_inCell(iX,iY,iZ); +} + +template +void Communicator3D::add_cells(int overlap) +{ + + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].add_inCells(overlap); + } +} + +template +void Communicator3D::init() +{ + + reset(); + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].init_inCN(); + for (int i=0; i<_nh[iC].get_inCellsSize(); i++) { + int ID = _nh[iC].get_inCell(i).latticeR[0]; +#ifdef PARALLEL_MODE_MPI + if ( singleton::mpi().getRank() ==_superStructure.getLoadBalancer().rank(ID) ) +#endif + { + Cell3D temp; + temp.physR[0] = _nh[iC].get_inCell(i).physR[0]; + temp.physR[1] = _nh[iC].get_inCell(i).physR[1]; + temp.physR[2] = _nh[iC].get_inCell(i).physR[2]; + _superStructure.getCuboidGeometry().getLatticeR(temp.latticeR, temp.physR); + temp.latticeR[0] = _superStructure.getLoadBalancer().glob(iC); + _nh[_superStructure.getLoadBalancer().loc(ID)].add_outCell(temp); + } + } + } + + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].init_outCN(); + } + +#ifdef PARALLEL_MODE_MPI + singleton::MpiNonBlockingHelper helper; + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].finish_comm(); + } + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].bufSend_inCells(helper); + } + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].recWrite_outCells(); + } + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + singleton::mpi().waitAll(helper); + } +#endif +} + +template +void Communicator3D::reset() +{ + + if (_initDone) { + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].reset(); + } + _initDone = false; + } +} + +template +void Communicator3D::send() +{ + + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].buffer_outData(); +#ifdef PARALLEL_MODE_MPI + _nh[iC].send_outData(); +#endif + } +} + +template +void Communicator3D::receive() +{ + + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { +#ifdef PARALLEL_MODE_MPI + _nh[iC].receive_inData(); +#else + for (int i=0; i<_nh[iC].get_inCsize(); i++) { + _nh[iC].get_inData()[_nh[iC].get_inC(i)] = + _nh[_nh[iC].get_inC(i)].get_outData()[iC]; + } +#endif + } +#ifdef PARALLEL_MODE_MPI + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].finish_comm(); + } +#endif +} + +template +void Communicator3D::write() +{ + + for (int iC=0; iC<_superStructure.getLoadBalancer().size(); iC++) { + _nh[iC].write_inData(); + } +} + +} + +#endif diff --git a/src/communication/cuboidNeighbourhood2D.cpp b/src/communication/cuboidNeighbourhood2D.cpp new file mode 100644 index 0000000..68d9a0b --- /dev/null +++ b/src/communication/cuboidNeighbourhood2D.cpp @@ -0,0 +1,38 @@ +/* This file is part of the OpenLB library + * + * Copyright (C) 2007 Mathias J. Krause + * E-mail contact: info@openlb.net + * The most recent release of OpenLB can be downloaded at + * + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. +*/ + +/** \file + * The description of a 2D cuboid neighbourhood -- template instantiation. + */ + + +#include "communication/mpiManager.h" +#include "communication/cuboidNeighbourhood2D.h" +#include "communication/cuboidNeighbourhood2D.hh" + +namespace olb { + +template struct Cell2D; +template class CuboidNeighbourhood2D; + +} diff --git a/src/communication/cuboidNeighbourhood2D.h b/src/communication/cuboidNeighbourhood2D.h new file mode 100644 index 0000000..91b37d3 --- /dev/null +++ b/src/communication/cuboidNeighbourhood2D.h @@ -0,0 +1,195 @@ +/* This file is part of the OpenLB library + * + * Copyright (C) 2007, 2014 Mathias J. Krause + * E-mail contact: info@openlb.net + * The most recent release of OpenLB can be downloaded at + * + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. +*/ + +/** \file + * The description of a 2D cuboid neighbourhood -- header file. + */ + + +#ifndef CUBOID_NEIGHBOURHOOD_2D_H +#define CUBOID_NEIGHBOURHOOD_2D_H + +#include "communication/mpiManager.h" +#include +#include "communication/superStructure2D.h" + + +/// All OpenLB code is contained in this namespace. +namespace olb { + +/// Single 2D cuboid neighbourhoods are the basic component of a +/// 2D communicator +/** For each cuboid a cuboid neighbourhood is defined. It stores the + * needed cell coordinates (Cell2D) of other cuboids. Futhermore this + * class provides basic initialization and communication methods + * for the class Communicator2D. + * + * WARNING: For unstructured grids there is an interpolation needed + * for the method buffer_outData which is not yet implemented! + * + * This class is not intended to be derived from. + */ + + +template class SuperStructure2D; + +template +struct Cell2D { + + Cell2D() + { + latticeR.resize(3); + physR.resize(2); + }; + + // local position latticeR + std::vector latticeR; + //int iC, iX, iY; + // global position physR + //T x, y; + std::vector physR; + + bool operator==(Cell2D const& rhs) const + { + return latticeR == rhs.latticeR; + }; + + Cell2D(Cell2D const& rhs) = default; + + /// Copy assignment + Cell2D& operator=(Cell2D const& rhs) = default; + +}; + + +template +class CuboidNeighbourhood2D { +private: + /// Cuboid ID + int _iCglob; + /// Number of cubboids in the structure + int _nC; + /// Delta of the cuboid + T _deltaC; + /// Reference to the super structure + SuperStructure2D& _superStructure; + + /// Number of data to be transfered + int _nData; + /// Size of underlying data type + int _nDataType; + + /// Internal needed cells + std::vector > _inCells; + /// External needed cells + std::vector > _outCells; + /// Internal needed neighbour cuboid and number of cells + std::vector _inC; + std::vector _inN; + /// External needed neighbour cuboid and number of cells + std::vector _outC; + std::vector _outN; + /// Buffer for the internal needed data + bool **_inData; + /// Buffer for the external needed data + bool **_outData; + /// Buffer for the internal needed data + T **_inDataCoordinates; + /// Buffer for the external needed data + T **_outDataCoordinates; + /// Indecates if there was an initialization done + bool _initInCNdone; + int* _tempInCN; + /// Indecates if there was an initialization done + bool _initOutCNdone; + /// Handels the MPI communication +#ifdef PARALLEL_MODE_MPI + singleton::MpiNonBlockingHelper _mpiNbHelper; +#endif +public: + /// Constructor + //CuboidNeighbourhood2D() {}; + /// Constructor + CuboidNeighbourhood2D(SuperStructure2D& superStructure, int iC); + /// Copy construction + CuboidNeighbourhood2D(CuboidNeighbourhood2D const& rhs); + /// Copy assignment + CuboidNeighbourhood2D operator=(CuboidNeighbourhood2D rhs); + /// Destructor + ~CuboidNeighbourhood2D(); + + /// Read only access to _inCells + Cell2D const& get_inCell(int i) const; + /// Returns the number of cells in _inCells + int get_inCellsSize() const; + /// Read only access to _inC + int const& get_inC(int i) const; + /// Returns the number of cells in _inC + int get_inCsize() const; + int get_outCellsSize() const; + /// Read and write access to **_inData + bool** get_inData(); + /// Read and write access to **_outData + bool** get_outData(); + + /// Adds a cell to the vector _inCells + void add_inCell(Cell2D cell); + /// Adds a cell to the vector _outCells + void add_outCell(Cell2D cell); + /// Adds a cell to the vector _inCells + /// if the cell is not already there and + /// if there is another cuboid which can deliver the information + void add_inCell(int iX, int iY); + /// Adds all cells with the distance overlap*_delta to + /// the vector _inCells + void add_inCells(int overlap); + /// Initializes _inC and _inN + void init_inCN(); + /// Initializes _outC and _outN + void init_outCN(); + /// Initialization Helper + void bufSend_inCells(); + /// Initialization Helper + void recWrite_outCells(); + /// Finishes a communication step + void finish_comm(); + + /// Buffers data to be send + // WARNING: Here is interpolation needed if globX, globY + // are not integers. This needs to be fixed if one will + // use unstructured grids. + void buffer_outData(); + /// Sends data (pure mpi method) + void send_outData(); + /// Receives data (pure mpi method) + void receive_inData(); + /// Writes all data to the corresponding lattice cells + void write_inData(); + /// Resets neighbourhood after initialization (init_inCN + /// and init_outCN) + void reset(); +}; + +} // namespace olb + +#endif diff --git a/src/communication/cuboidNeighbourhood2D.hh b/src/communication/cuboidNeighbourhood2D.hh new file mode 100644 index 0000000..ea7f8d6 --- /dev/null +++ b/src/communication/cuboidNeighbourhood2D.hh @@ -0,0 +1,452 @@ +/* This file is part of the OpenLB library + * + * Copyright (C) 2007, 2014 Mathias J. Krause + * E-mail contact: info@openlb.net + * The most recent release of OpenLB can be downloaded at + * + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. +*/ + +/** \file + * The description of a 2D cuboid neighbourhood -- generic implementation. + */ + + +#ifndef CUBOID_NEIGHBOURHOOD_2D_HH +#define CUBOID_NEIGHBOURHOOD_2D_HH + +#include +#include + +#include "communication/mpiManager.h" +#include "communication/cuboidNeighbourhood2D.h" +#include "geometry/cuboidGeometry2D.h" +#include "dynamics/dynamics.h" +#include "core/cell.h" +#include "communication/superStructure2D.h" + + +namespace olb { + + +//////////////// Class CuboidNeighbourhood2D ////////////////// + +template +CuboidNeighbourhood2D::CuboidNeighbourhood2D( + SuperStructure2D& superStructure, int iC):_superStructure(superStructure) +{ + _iCglob = iC; + _nC = _superStructure.getCuboidGeometry().getNc(); + _deltaC = _superStructure.getCuboidGeometry().get(iC).getDeltaR(); + _nData = _superStructure.getDataSize(); + _nDataType = _superStructure.getDataTypeSize(); + _initInCNdone = false; + _initOutCNdone = false; +} + +template +CuboidNeighbourhood2D::CuboidNeighbourhood2D ( + CuboidNeighbourhood2D const& rhs ):_superStructure(rhs._superStructure) +{ + _iCglob = rhs._iCglob; + _nC = rhs._nC; + _deltaC = rhs._deltaC; + _inCells = rhs._inCells; + _outCells = rhs._outCells; + _inC = rhs._inC; + _inN = rhs._inN; + _outC = rhs._outC; + _outN = rhs._outN; + _nData = rhs._nData; + _nDataType = rhs._nDataType; + _initInCNdone = false; + _initOutCNdone = false; +} + +template +CuboidNeighbourhood2D CuboidNeighbourhood2D::operator= ( + CuboidNeighbourhood2D rhs ) +{ + CuboidNeighbourhood2D tmp(rhs); + return tmp; +} + +template +CuboidNeighbourhood2D::~CuboidNeighbourhood2D() +{ + reset(); +} + + +template +Cell2D const& CuboidNeighbourhood2D::get_inCell(int i) const +{ + return _inCells[i]; +} + +template +int CuboidNeighbourhood2D::get_inCellsSize() const +{ + return _inCells.size(); +} + +template +int CuboidNeighbourhood2D::get_outCellsSize() const +{ + return _outCells.size(); +} + +template +int const& CuboidNeighbourhood2D::get_inC(int i) const +{ + return _inC[i]; +} + +template +int CuboidNeighbourhood2D::get_inCsize() const +{ + return _inC.size(); +} + +template +bool** CuboidNeighbourhood2D::get_inData() +{ + return _inData; +} + +template +bool** CuboidNeighbourhood2D::get_outData() +{ + return _outData; +} + + +template +void CuboidNeighbourhood2D::add_inCell(Cell2D cell) +{ + _inCells.push_back(cell); +} + +template +void CuboidNeighbourhood2D::add_outCell(Cell2D cell) +{ + _outCells.push_back(cell); +} + +template +void CuboidNeighbourhood2D::add_inCell(int iX, int iY) +{ + + Cell2D found; + found.latticeR[0] = _iCglob; + found.latticeR[1] = iX; + found.latticeR[2] = iY; + found.physR = _superStructure.getCuboidGeometry().getPhysR(found.latticeR); + if (_superStructure.getCuboidGeometry().getC(found.physR, found.latticeR[0]) ) { + for (unsigned i=0; i<_inCells.size(); i++) { + if (_inCells[i]==found) { + return; + } + } + _inCells.push_back(found); + } +} + +template +void CuboidNeighbourhood2D::add_inCells(int overlap) +{ + + int nX = _superStructure.getCuboidGeometry().get(_iCglob).getNx(); + int nY = _superStructure.getCuboidGeometry().get(_iCglob).getNy(); + + for (int iX=0; iX nX + overlap - 1 || + iY < overlap || iY > nY + overlap - 1 ) { + Cell2D found; + found.latticeR[0] = _iCglob; + found.latticeR[1] = iX - overlap; + found.latticeR[2] = iY - overlap; + + found.physR = _superStructure.getCuboidGeometry().getPhysR(found.latticeR); + + if (_superStructure.getCuboidGeometry().getC(found.physR, found.latticeR[0]) ) { + _inCells.push_back(found); + } + } + } + } +} + +template +void CuboidNeighbourhood2D::init_inCN() +{ + + _inC.clear(); + _inN.clear(); + + _inData = new bool* [_nC]; + _inDataCoordinates = new T* [_nC]; + _tempInCN = new int [_nC]; + for (int i=0; i<_nC; i++) { + _tempInCN[i]=0; + } + + for (unsigned i=0; i<_inCells.size(); i++) { + _tempInCN[_inCells[i].latticeR[0]]++; + } + for (int i=0; i<_nC; i++) { + if (_tempInCN[i]!=0) { + _inC.push_back(i); + _inN.push_back(_tempInCN[i]); +#ifdef PARALLEL_MODE_MPI + _inData[i] = new bool [_tempInCN[i]*_nData*_nDataType]; + _inDataCoordinates[i] = new T [_tempInCN[i]*2]; +#endif + } +#ifdef PARALLEL_MODE_MPI + else { + _inData[i] = NULL; + _inDataCoordinates[i] = NULL; + } +#endif + } + +#ifdef PARALLEL_MODE_MPI + int counter=0; + for (int i=0; i<_nC; i++) { + int dRank = _superStructure.getLoadBalancer().rank(i); + if ( singleton::mpi().getRank() != dRank ) { + counter++; + } + } + _mpiNbHelper.allocate(counter); + counter=0; + for (int i=0; i<_nC; i++) { + int dRank = _superStructure.getLoadBalancer().rank(i); + if ( singleton::mpi().getRank() != dRank ) { + singleton::mpi().iSend(&_tempInCN[i] , 1, dRank, &_mpiNbHelper.get_mpiRequest()[counter], _iCglob); + counter++; + } + } +#endif + + _initInCNdone = true; +} + +template +void CuboidNeighbourhood2D::init_outCN() +{ + + _outC.clear(); + _outN.clear(); + _outData = new bool* [_nC]; + _outDataCoordinates = new T* [_nC]; + + std::vector temp(_nC,0); + + for (unsigned i=0; i<_outCells.size(); i++) { + temp[_outCells[i].latticeR[0]]++; + } + + for (int i=0; i<_nC; i++) { +#ifdef PARALLEL_MODE_MPI + int sRank = _superStructure.getLoadBalancer().rank(i); + if ( singleton::mpi().getRank() != sRank ) { + singleton::mpi().receive(&temp[i], 1, sRank, i); + } +#endif + if (temp[i]!=0) { + _outC.push_back(i); + _outN.push_back(temp[i]); + } + _outData[i] = new bool [temp[i]*_nData*_nDataType]; + _outDataCoordinates[i] = new T [temp[i]*2]; + } + + _initOutCNdone = true; +} + +template +void CuboidNeighbourhood2D::bufSend_inCells() +{ + +#ifdef PARALLEL_MODE_MPI + _mpiNbHelper.free(); + + std::vector temp(_nC,0); + for (unsigned i=0; i<_inCells.size(); i++) { + int iC = _inCells[i].latticeR[0]; + if (singleton::mpi().getRank() != _superStructure.getLoadBalancer().rank(iC)) { + _inDataCoordinates[iC][2*temp[iC]] = _inCells[i].physR[0]; + _inDataCoordinates[iC][2*temp[iC]+1] = _inCells[i].physR[1]; + temp[iC]++; + } + } + + int counter=0; + for (unsigned iC=0; iC<_inC.size(); iC++) { + //int dRank = _superStructure.get_load().rank(_inC[iC]); + //if ( singleton::mpi().getRank() != dRank ) + counter++; + } + + _mpiNbHelper.allocate(counter); + counter=0; + for (unsigned iC=0; iC<_inC.size(); iC++) { + int dRank = _superStructure.getLoadBalancer().rank(_inC[iC]); + //if ( singleton::mpi().getRank() != dRank ) { + singleton::mpi().iSend( _inDataCoordinates[_inC[iC]], + _inN[iC]*2, dRank, &_mpiNbHelper.get_mpiRequest()[counter], _iCglob); + counter++; + //} + } +#endif +} + +template +void CuboidNeighbourhood2D::recWrite_outCells() +{ + +#ifdef PARALLEL_MODE_MPI + for (unsigned iC=0; iC<_outC.size(); iC++) { + int sRank = _superStructure.getLoadBalancer().rank(_outC[iC]); + singleton::mpi().receive(_outDataCoordinates[_outC[iC]], _outN[iC]*2, sRank, _outC[iC]); + if ( singleton::mpi().getRank() != sRank ) { + //singleton::mpi().receive(_outDataCoordinates[_outC[iC]], _outN[iC]*2, sRank, _outC[iC]); + Cell2D found; + for (int i=0; i<_outN[iC]; i++) { + found.physR[0] = _outDataCoordinates[_outC[iC]][2*i]; + found.physR[1] = _outDataCoordinates[_outC[iC]][2*i+1]; + _superStructure.getCuboidGeometry().getLatticeR(found.physR, found.latticeR); + found.latticeR[0] = _outC[iC]; + _outCells.push_back(found); + } + } + } +#endif +} + +template +void CuboidNeighbourhood2D::finish_comm() +{ + +#ifdef PARALLEL_MODE_MPI + singleton::mpi().waitAll(_mpiNbHelper); +#endif + +} + +template +void CuboidNeighbourhood2D::buffer_outData() +{ + + std::vector temp(_nC,0); + int iCloc = _superStructure.getLoadBalancer().loc(_iCglob); + for (unsigned i=0; i<_outCells.size(); i++) { + int iC = _outCells[i].latticeR[0]; + // WARNING: Here is interpolation needed if globX, globY + // are not integers. This needs to be fixed if one will + // use unstructured grids. + //for (int iData=0; iData<_nData; iData++) { + // memcpy(_outData[iC] + (temp[iC]*_nData + iData)*_nDataType, _superStructure(iCloc,iX+overlap,iY+overlap,iZ+overlap,iData), _nDataType); + //} + std::memcpy(_outData[iC] + temp[iC]*_nData*_nDataType, _superStructure(iCloc,_outCells[i].latticeR[1],_outCells[i].latticeR[2],0), _nDataType*_nData); + temp[iC]++; + } +} + +template +void CuboidNeighbourhood2D::send_outData() +{ +#ifdef PARALLEL_MODE_MPI + for (unsigned iC=0; iC<_outC.size(); iC++) { + int dRank = _superStructure.getLoadBalancer().rank(_outC[iC]); + singleton::mpi().iSend( _outData[_outC[iC]], + _outN[iC]*_nData*_nDataType, dRank, &_mpiNbHelper.get_mpiRequest()[iC], _iCglob); + } +#endif +} + +template +void CuboidNeighbourhood2D::receive_inData() +{ +#ifdef PARALLEL_MODE_MPI + for (unsigned iC=0; iC<_inC.size(); iC++) { + int sRank = _superStructure.getLoadBalancer().rank(_inC[iC]); + singleton::mpi().receive(_inData[_inC[iC]], _inN[iC]*_nData*_nDataType, sRank, _inC[iC]); + } +#endif +} + +template +void CuboidNeighbourhood2D::write_inData() +{ + + int iCloc = _superStructure.getLoadBalancer().loc(_iCglob); + std::vector temp(_nC,0); + for (unsigned i=0; i<_inCells.size(); i++) { + int iC = _inCells[i].latticeR[0]; + //for (int iData=0; iData<_nData; iData++) { + // memcpy(_superStructure(iCloc,iX+overlap,iY+overlap,iZ+overlap,iData), _inData[iC] + (temp[iC]*_nData + iData)*_nDataType, _nDataType); + //} + //memcpy(_superStructure(iCloc,iX+overlap,iY+overlap,iZ+overlap,0), _inData[iC] + temp[iC]*_nData*_nDataType, _nData*_nDataType); + //temp[iC]++; + std::memcpy(_superStructure(iCloc,_inCells[i].latticeR[1],_inCells[i].latticeR[2],0), _inData[iC] + temp[iC]*_nData*_nDataType, _nData*_nDataType); + temp[iC]++; + } +} + +template +void CuboidNeighbourhood2D::reset() +{ + + if (_initInCNdone) { +#ifdef PARALLEL_MODE_MPI + for (int iC=0; iC<_nC; iC++) { + delete[] _inData[iC]; + delete[] _inDataCoordinates[iC]; + } +#endif + delete[] _inData; + delete[] _inDataCoordinates; + delete[] _tempInCN; + _initInCNdone = false; + } + if (_initOutCNdone) { + for (int iC=0; iC<_nC; iC++) { + delete[] _outData[iC]; + delete[] _outDataCoordinates[iC]; + } + delete[] _outData; + delete[] _outDataCoordinates; +#ifdef PARALLEL_MODE_MPI + _mpiNbHelper.free(); +#endif + _initOutCNdone = false; + } + _inCells.clear(); + _outCells.clear(); + _inC.clear(); + _outC.clear(); + _inN.clear(); + _outN.clear(); +} + +} // namespace olb + +#endif diff --git a/src/communication/cuboidNeighbourhood3D.cpp b/src/communication/cuboidNeighbourhood3D.cpp new file mode 100644 index 0000000..c063f89 --- /dev/null +++ b/src/communication/cuboidNeighbourhood3D.cpp @@ -0,0 +1,38 @@ +/* This file is part of the OpenLB library + * + * Copyright (C) 2007 Mathias J. Krause + * E-mail contact: info@openlb.net + * The most recent release of OpenLB can be downloaded at + * + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. +*/ + +/** \file + * The description of a 3D cuboid neighbourhood -- template instantiation. + */ + + +#include "communication/mpiManager.h" +#include "communication/cuboidNeighbourhood3D.h" +#include "communication/cuboidNeighbourhood3D.hh" + +namespace olb { + +template struct Cell3D; +template class CuboidNeighbourhood3D; + +} diff --git a/src/communication/cuboidNeighbourhood3D.h b/src/communication/cuboidNeighbourhood3D.h new file mode 100644 index 0000000..639102c --- /dev/null +++ b/src/communication/cuboidNeighbourhood3D.h @@ -0,0 +1,190 @@ +/* This file is part of the OpenLB library + * + * Copyright (C) 2007, 2014 Mathias J. Krause + * E-mail contact: info@openlb.net + * The most recent release of OpenLB can be downloaded at + * + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. +*/ + +/** \file + * The description of a 3D cuboid neighbourhood -- header file. + */ + + +#ifndef CUBOID_NEIGHBOURHOOD_3D_H +#define CUBOID_NEIGHBOURHOOD_3D_H + +#include "communication/mpiManager.h" +#include +#include "communication/superStructure3D.h" + + +/// All OpenLB code is contained in this namespace. +namespace olb { + +template class SuperStructure3D; + + +/// Single 3D cuboid neighbourhoods are the basic component of a +/// 3D communicator +/** For each cuboid a cuboid neighbourhood is defined. It stores the + * needed cell coordinates (Cell3D) of other cuboids. Futhermore this + * class provides basic initialization and communication methods + * for the class Communicator3D. + * + * WARNING: For unstructured grids there is an interpolation needed + * for the method buffer_outData which is not yet implemented! + * + * This class is not intended to be derived from. + */ +template +struct Cell3D { + Cell3D() : latticeR {0,0,0,0} {}; + // local position latticeR: iC, iX, iY, iZ; + int latticeR[4]; + // global position physR: x, y, z; + T physR[3]; + + bool operator==(Cell3D const& rhs) const + { + return latticeR == rhs.latticeR; + }; + + /// Copy constructor + Cell3D(Cell3D const& rhs) = default; + + /// Copy assignment + Cell3D& operator=(Cell3D const& rhs) = default; + +}; + + +template +class CuboidNeighbourhood3D { +private: + /// Cuboid ID + int _iCglob; + /// Number of cubboids in the structure + int _nC; + /// Delta of the cuboid + T _deltaC; + /// Reference to the super structure + SuperStructure3D& _superStructure; + + /// Number of data to be transfered + int _nData; + /// Size of underlying data type + int _nDataType; + + /// Internal needed cells + std::vector > _inCells; + /// External needed cells + std::vector > _outCells; + /// Internal needed neighbour cuboid and number of cells + std::vector _inC; + std::vector _inN; + /// External needed neighbour cuboid and number of cells + std::vector _outC; + std::vector _outN; + /// Buffer for the internal needed data + bool **_inData; + /// Buffer for the external needed data + bool **_outData; + /// Buffer for the internal needed data + T **_inDataCoordinates; + /// Buffer for the external needed data + T **_outDataCoordinates; + /// Indecates if there was an initialization done + bool _initInCNdone; + int* _tempInCN; + /// Indecates if there was an initialization done + bool _initOutCNdone; + /// Handels the MPI communication +#ifdef PARALLEL_MODE_MPI + singleton::MpiNonBlockingHelper _mpiNbHelper; +#endif +public: + /// Constructor + //CuboidNeighbourhood2D() {}; + /// Constructor + CuboidNeighbourhood3D(SuperStructure3D& superStructure, int iC); + /// Copy construction + CuboidNeighbourhood3D(CuboidNeighbourhood3D const& rhs); + /// Copy assignment + CuboidNeighbourhood3D operator=(CuboidNeighbourhood3D rhs); + /// Destructor + ~CuboidNeighbourhood3D(); + + /// Read only access to _inCells + Cell3D const& get_inCell(int i) const; + /// Returns the number of cells in _inCells + int get_inCellsSize() const; + /// Read only access to _inC + int const& get_inC(int i) const; + /// Returns the number of cells in _inC + int get_inCsize() const; + /// Read and write access to **_inData + bool** get_inData(); + /// Read and write access to **_outData + bool** get_outData(); + + /// Adds a cell to the vector _inCells + void add_inCell(Cell3D cell); + /// Adds a cell to the vector _outCells + void add_outCell(Cell3D cell); + /// Adds a cell to the vector _inCells +