escript Revision_
EsysMPI.h
Go to the documentation of this file.
1
2/*****************************************************************************
3*
4* Copyright (c) 2003-2020 by The University of Queensland
5* http://www.uq.edu.au
6*
7* Primary Business: Queensland, Australia
8* Licensed under the Apache License, version 2.0
9* http://www.apache.org/licenses/LICENSE-2.0
10*
11* Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12* Development 2012-2013 by School of Earth Sciences
13* Development from 2014-2017 by Centre for Geoscience Computing (GeoComp)
14* Development from 2019 by School of Earth and Environmental Sciences
15**
16*****************************************************************************/
17
18#ifndef __ESCRIPT_ESYSMPI_H__
19#define __ESCRIPT_ESYSMPI_H__
20
21#include "system_dep.h"
22
23#include <escript/DataTypes.h>
24
25#include <ctime>
26#include <sstream>
27
28#include <boost/shared_ptr.hpp>
29
30#ifdef _OPENMP
31#include <omp.h>
32#endif
33
34#ifdef ESYS_MPI
35#include <mpi.h>
36
37#ifdef ESYS_INDEXTYPE_LONG
38#define MPI_DIM_T MPI_LONG
39#else
40#define MPI_DIM_T MPI_INT
41#endif
42
43#else
44 typedef int MPI_Comm;
45 typedef int MPI_Request;
46 typedef int MPI_Op;
47 typedef int MPI_Status;
48 #define MPI_INT 6
49 #define MPI_DOUBLE 11
50 #define MPI_COMM_WORLD 91
51 #define MPI_COMM_NULL 0
52
53 // MPI_Op replacements for non-MPI - these values are arbitrary
54 #define MPI_SUM 100
55 #define MPI_MIN 101
56 #define MPI_MAX 102
57
58 #define MPI_OP_NULL 17
59 // end MPI_op
60
61#endif // ESYS_MPI
62
63namespace escript {
64
69inline int getSubWorldTag()
70{
71 return (('S'<< 24) + ('u' << 16) + ('b' << 8) + 'W')%1010201;
72}
73
74class JMPI_;
75
76typedef boost::shared_ptr<JMPI_> JMPI;
77
81JMPI makeInfo(MPI_Comm comm, bool owncom=false);
82
84{
85public:
86 ~JMPI_();
87
89 DataTypes::dim_t setDistribution(DataTypes::index_t min_id,
90 DataTypes::index_t max_id,
91 DataTypes::index_t* distribution);
92
94 void split(DataTypes::dim_t N, DataTypes::dim_t* local_N,
95 DataTypes::index_t* offset);
96
99 inline int mod_rank(int k) const
100 {
101 int out=0;
102#ifdef ESYS_MPI
103 if (size > 1) {
104 const int q = k/size;
105 if (k > 0) {
106 out=k-size*q;
107 } else if (k < 0) {
108 out=k-size*(q-1);
109 }
110 }
111#endif
112 return out;
113 }
114
116 inline std::string appendRankToFileName(const std::string& fileName) const
117 {
118#ifdef ESYS_MPI
119 if (size > 1) {
120 std::stringstream ss;
121 ss << fileName << '.';
122 ss.fill('0');
123 ss.width(4);
124 ss << rank;
125 return ss.str();
126 }
127#endif
128 return fileName;
129 }
130
132 inline int counter() const
133 {
134 return msg_tag_counter;
135 }
136
138 inline void incCounter(int i=1)
139 {
140 msg_tag_counter+=i;
141 // there is no particular significance here other than being 7 digits
142 // and prime (because why not). It just needs to be big.
143 msg_tag_counter %= 1010201;
144 }
145
147 inline void setCounter(int value)
148 {
149 msg_tag_counter = value%1010201;
150 }
151
153 inline bool isValid() const
154 {
155 return comm!=MPI_COMM_NULL;
156 }
157
158 int size;
159 int rank;
161
162private:
163 JMPI_(MPI_Comm comm, bool owncomm);
164 friend ESCRIPT_DLL_API JMPI makeInfo(MPI_Comm comm, bool owncom);
165#pragma clang diagnostic push
166#pragma clang diagnostic ignored "-Wunused-private-field"
168#pragma clang diagnostic pop
170};
171
172// Does not cope with nested calls
174{
175public:
176 NoCOMM_WORLD();
178 static bool active();
179};
180
183bool checkResult(int input, int& output, const JMPI& comm);
184
189bool shipString(const char* src, char** dest, MPI_Comm& comm);
190
192inline double gettime()
193{
194 double out;
195#ifdef ESYS_MPI
196 out = MPI_Wtime();
197#else
198#ifdef _OPENMP
199 out=omp_get_wtime();
200#else
201 out=((double) clock())/CLOCKS_PER_SEC;
202#endif
203#endif
204 return out;
205}
206
207} // namespace escript
208
209#endif // __ESCRIPT_ESYSMPI_H__
210
#define MPI_COMM_NULL
Definition EsysMPI.h:51
int MPI_Op
Definition EsysMPI.h:46
int MPI_Request
Definition EsysMPI.h:45
int MPI_Comm
Definition EsysMPI.h:44
int MPI_Status
Definition EsysMPI.h:47
Definition EsysMPI.h:84
int mod_rank(int k) const
Definition EsysMPI.h:99
void incCounter(int i=1)
increments the message tag counter by i
Definition EsysMPI.h:138
bool isValid() const
returns true if this has a valid MPI communicator
Definition EsysMPI.h:153
void setCounter(int value)
sets the message tag counter to value
Definition EsysMPI.h:147
int msg_tag_counter
Definition EsysMPI.h:169
bool ownscomm
Definition EsysMPI.h:167
std::string appendRankToFileName(const std::string &fileName) const
appends MPI rank to a file name if MPI size > 1
Definition EsysMPI.h:116
int counter() const
returns the current value of the message tag counter
Definition EsysMPI.h:132
int size
Definition EsysMPI.h:158
MPI_Comm comm
Definition EsysMPI.h:160
int rank
Definition EsysMPI.h:159
Definition EsysMPI.h:174
NoCOMM_WORLD()
Definition EsysMPI.cpp:224
~NoCOMM_WORLD()
Definition EsysMPI.cpp:232
static bool active()
Definition EsysMPI.cpp:237
#define ESCRIPT_DLL_API
Definition escriptcore/src/system_dep.h:30
index_t dim_t
Definition DataTypes.h:66
int index_t
type for array/matrix indices used both globally and on each rank
Definition DataTypes.h:61
Definition AbstractContinuousDomain.cpp:23
bool checkResult(int res, int &mres, const JMPI &info)
Everyone puts in their error code and everyone gets the largest one.
Definition EsysMPI.cpp:110
JMPI makeInfo(MPI_Comm comm, bool owncom)
Definition EsysMPI.cpp:29
int getSubWorldTag()
tag reserved for use by SubWorld code This value should be higher than the modulus used in JMPI_::set...
Definition EsysMPI.h:69
double gettime()
returns the current ticks for timing
Definition EsysMPI.h:192
bool shipString(const char *src, char **dest, MPI_Comm &comm)
Definition EsysMPI.cpp:164
boost::shared_ptr< JMPI_ > JMPI
Definition EsysMPI.h:76