mpiManager.h

Go to the documentation of this file.

/*  This file is part of the OpenLB library
 *
 *  Copyright (C) 2007 The OpenLB project
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU General Public License
 *  as published by the Free Software Foundation; either version 2
 *  of the License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public
 *  License along with this program; if not, write to the Free
 *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 *  Boston, MA  02110-1301, USA.
*/
 
#ifndef MPI_MANAGER_H
#define MPI_MANAGER_H
 
#include "core/platform/platform.h"
 
#ifdef PARALLEL_MODE_MPI
#include "mpi.h"
 
#include <vector>
#include <memory>
#endif
 
#include <string>
#include "io/ostreamManager.h"
#include "utilities/aDiff.h"
 
namespace olb {
 
template <unsigned D, typename T, typename U>
class BlockData;
 
namespace singleton {
 
#ifdef PARALLEL_MODE_MPI
 
class MpiNonBlockingHelper {
private:
  unsigned _size;
  std::unique_ptr<MPI_Request[]> _mpiRequest;
  std::unique_ptr<MPI_Status[]> _mpiStatus;
public:
  MpiNonBlockingHelper();
  ~MpiNonBlockingHelper() = default;
 
  MpiNonBlockingHelper(MpiNonBlockingHelper&& rhs) = default;
  MpiNonBlockingHelper(const MpiNonBlockingHelper&) = delete;
  MpiNonBlockingHelper& operator=(const MpiNonBlockingHelper&) = delete;
 
  void allocate(unsigned i);
  void free();
 
  unsigned get_size() const;
 
  MPI_Request* get_mpiRequest(int i=0) const;
  MPI_Status* get_mpiStatus(int i=0) const;
 
  void start(int i);
  void wait(int i);
  bool isDone(int i);
 
  void swap(MpiNonBlockingHelper& rhs);
};
 
 
class MpiManager {
public:
  void init(int *argc, char ***argv, bool verbose=true);
  int getSize() const;
  int getRank() const;
  int bossId() const;
  bool isMainProcessor() const;
  double getTime() const;
 
  void barrier(MPI_Comm comm = MPI_COMM_WORLD);
 
  void synchronizeIO(unsigned tDelay = 100, MPI_Comm comm = MPI_COMM_WORLD);
 
  template <typename T>
  void send(T *buf, int count, int dest, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD);
  template <typename T,unsigned DIM>
  void send(util::ADf<T,DIM> *buf, int count, int dest, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD);
  template<typename... args>
  void send(std::vector<args...>& vec, int dest, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD){
    send( vec.data(), vec.size(), dest, tag, comm );
  }
  template<class T, std::size_t N>
  void send(std::array<T,N>& array, int dest, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD){
    send( array.data(), array.size(), dest, tag, comm );
  }
 
  template <typename T>
  void sendInit(T *buf, int count, int dest, MPI_Request* request, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD);
  template <typename T,unsigned DIM>
  void sendInit(util::ADf<T,DIM> *buf, int count, int dest, MPI_Request* request, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD);
 
  template <typename T>
  void iSend(T *buf, int count, int dest, MPI_Request* request, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD);
  template <typename T,unsigned DIM>
  void iSend(util::ADf<T,DIM> *buf, int count, int dest, MPI_Request* request, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD);
 
  template <typename T>
  void ibSend(T *buf, int count, int dest, MPI_Request* request, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD);
  template <typename T,unsigned DIM>
  void ibSend(util::ADf<T,DIM> *buf, int count, int dest, MPI_Request* request, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD);
 
  std::size_t probeReceiveSize(int source, MPI_Datatype type, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD);
  template <typename TYPE>
  std::size_t probeReceiveSize(int source, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD);
 
  template <typename T>
  void receive(T *buf, int count, int source, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD);
  template <typename T,unsigned DIM>
  void receive(util::ADf<T,DIM> *buf, int count, int source, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD);
  template<typename... args>
  void receive(std::vector<args...>& vec, int source, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD){
    receive( vec.data(), vec.size(), source, tag, comm );
  }
  template<class T, std::size_t N>
  void receive(std::array<T,N>& array, int source, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD){
    receive( array.data(), array.size(), source, tag, comm );
  }
 
  template <typename T>
  void recvInit(T *buf, int count, int dest, MPI_Request* request, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD);
  template <typename T,unsigned DIM>
  void recvInit(util::ADf<T,DIM> *buf, int count, int dest, MPI_Request* request, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD);
 
  template <typename T>
  void iRecv(T *buf, int count, int source, MPI_Request* request, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD);
  template <typename T,unsigned DIM>
  void iRecv(util::ADf<T,DIM> *buf, int count, int source, MPI_Request* request, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD);
 
  template <typename T>
  void sendRecv(T *sendBuf, T *recvBuf, int count, int dest, int source, int tag = 0,
                MPI_Comm comm = MPI_COMM_WORLD);
  template <typename T,unsigned DIM>
  void sendRecv(util::ADf<T,DIM> *sendBuf, util::ADf<T,DIM> *recvBuf, int count,
                int dest, int source, int tag = 0, MPI_Comm comm = MPI_COMM_WORLD);
 
  template <typename T>
  void sendToMaster(T* sendBuf, int sendCount, bool iAmRoot, MPI_Comm comm = MPI_COMM_WORLD);
 
  template <typename T>
  void scatterv(T *sendBuf, int* sendCounts, int* displs,
                T* recvBuf, int recvCount, int root = 0, MPI_Comm comm = MPI_COMM_WORLD);
 
  template <typename T>
  void gather(T* sendBuf, int sendCount, T* recvBuf, int recvCount,
              int root = 0, MPI_Comm comm = MPI_COMM_WORLD);
 
  template <typename T>
  void gatherv(T* sendBuf, int sendCount, T* recvBuf, int* recvCounts, int* displs,
               int root = 0, MPI_Comm comm = MPI_COMM_WORLD);
 
  template <typename T>
  void bCast(T* sendBuf, int sendCount, int root = 0, MPI_Comm comm = MPI_COMM_WORLD);
  template <typename T,unsigned DIM>
  void bCast(util::ADf<T,DIM>* sendBuf, int sendCount, int root = 0, MPI_Comm comm = MPI_COMM_WORLD);
  template <typename T,unsigned DIM>
  void bCast(BlockData<2,util::ADf<T,DIM>,util::ADf<T,DIM>>& sendData, int root = 0, MPI_Comm comm = MPI_COMM_WORLD);
  template <typename T>
  void bCast(T& sendVal, int root = 0, MPI_Comm comm = MPI_COMM_WORLD);
 
  template <typename T>
  void bCastThroughMaster(T* sendBuf, int sendCount, bool iAmRoot, MPI_Comm comm = MPI_COMM_WORLD);
  template <typename T,unsigned DIM>
  void bCastThroughMaster(util::ADf<T,DIM>* sendBuf, int sendCount, bool iAmRoot, MPI_Comm comm = MPI_COMM_WORLD);
 
  void bCast(std::string& message, int root = 0);
  void bCast(BlockData<2,double,double>& sendData, int root = 0, MPI_Comm comm = MPI_COMM_WORLD);
  void bCast(BlockData<2,float,float>& sendData, int root = 0, MPI_Comm comm = MPI_COMM_WORLD);
 
  template <typename T>
  void reduce(T& sendVal, T& recvVal, MPI_Op op, int root = 0, MPI_Comm = MPI_COMM_WORLD);
  template <typename T,unsigned DIM>
  void reduce(util::ADf<T,DIM>& sendVal, util::ADf<T,DIM>& recvVal,
              MPI_Op op, int root = 0, MPI_Comm = MPI_COMM_WORLD);
  template <typename T,unsigned DIM>
  void reduce(BlockData<2,util::ADf<T,DIM>,util::ADf<T,DIM>>& sendVal,
              BlockData<2,util::ADf<T,DIM>,util::ADf<T,DIM>>& recvVal,
              MPI_Op op, int root = 0, MPI_Comm comm = MPI_COMM_WORLD);
 
  template <typename T>
  void reduceVect(std::vector<T>& sendVal, std::vector<T>& recvVal,
                  MPI_Op op, int root = 0, MPI_Comm comm = MPI_COMM_WORLD);
 
  template <typename T>
  void reduceAndBcast(T& reductVal, MPI_Op op, int root = 0, MPI_Comm comm = MPI_COMM_WORLD);
  template <typename T,unsigned DIM>
  void reduceAndBcast(util::ADf<T,DIM>& reductVal, MPI_Op op, int root = 0, MPI_Comm comm = MPI_COMM_WORLD);
 
  void wait(MPI_Request* request, MPI_Status* status);
 
  void waitAll(MpiNonBlockingHelper& mpiNbHelper);
 
private:
  MpiManager();
  ~MpiManager();
private:
  int numTasks, taskId;
  bool ok;
  mutable OstreamManager clout;
 
  friend MpiManager& mpi();
};
 
#else
 
class MpiManager {
public:
  void init(int *argc, char ***argv, bool verbose=false) { }
  int getSize() const
  {
    return 1;
  }
  int getRank() const
  {
    return 0;
  }
  int bossId() const
  {
    return 0;
  }
  bool isMainProcessor() const
  {
    return true;
  }
 
  void barrier() const {};
 
  friend MpiManager& mpi();
};
 
#endif
 
MpiManager& mpi();
 
}  // namespace singleton
 
}  // namespace olb
 
 
#endif  // MPI_MANAGER_H