escript Revision_
blocktools.h
Go to the documentation of this file.
1
2/*****************************************************************************
3*
4* Copyright (c) 2014-2018 by The University of Queensland
5* http://www.uq.edu.au
6*
7* Primary Business: Queensland, Australia
8* Licensed under the Apache License, version 2.0
9* http://www.apache.org/licenses/LICENSE-2.0
10*
11* Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12* Development 2012-2013 by School of Earth Sciences
13* Development from 2014-2017 by Centre for Geoscience Computing (GeoComp)
14* Development from 2019 by School of Earth and Environmental Sciences
15**
16*****************************************************************************/
17
18#ifndef __RIPLEY_BLOCKTOOLS_H__
19#define __RIPLEY_BLOCKTOOLS_H__
20
21/* This file contains two main classes for dealing with a large 3D region which
22 * has been divided into a 3D Grid of Blocks (usually to be distributed).
23 * Each block is divided into 27 subblocks. The first and last subblocks in
24 * each dimension are cubes.
25 *
26 * class Block manages a single block. It has methods for copying between a
27 * flat array (storing all the values in the 3D block) and buffers storing
28 * individual subblocks. These buffers can be used to transfer individual
29 * subblocks to other blocks (using some external means).
30 *
31 * class BlockGrid deals with the position of a given block in relation to the
32 * rest of the blocks in the grid. (In an MPI setting, there would be one block
33 * per rank.)
34 * It also requires transfers of subblocks into and out of the block in order
35 * to make the whole (global) array consistent.
36 *
37 * Each block has a region "inset" wide in from each edge which is shared with
38 * neighbouring blocks. Where these regions overlap with another block, the
39 * block closest to block 0,0,0 is used.
40 * Or more precisely, values move left->right then lowy->highy and finally
41 * lowz -> highz.
42 *
43 * Please don't mix external calls into this file, it may be useful to separate
44 * it for debugging purposes.
45 *
46 * Types required:
47 * neighbourID_t - Stores the label of a neighbouring block.
48 * In an MPI setting, this will be the type used refer to
49 * ranks
50 * coord_t - Stores a position of a block in the grid of blocks
51 * (it could be within one dimension or overall in a flat
52 * structure. It is not (necessarily) the same as
53 * neighbourID_t because coord_t should be _unsigned_ and
54 * there is no guarantee that neighbourID_t will even be
55 * an integral type.
56 */
57
58#include <escript/EsysMPI.h>
59
60#include <vector>
61
62typedef int neighbourID_t; // This should be the MPI_rank type
63typedef unsigned coord_t; // if we ever get more than 2^32 ranks, we have other problems
64
65typedef std::pair<neighbourID_t, int> neighpair;
66typedef std::vector<neighpair> neighbourvector;
67
68
69typedef struct
70{
71public:
72 neighbourID_t sourceID; // ranks involved in communication
74 int tag;
75 unsigned char srcbuffid; // number of buffer to use for coms
76 unsigned char destbuffid;
77} message;
78
79typedef std::vector<message> messvec;
80
81
83{
84public:
85 BlockGrid(coord_t maxx, coord_t maxy, coord_t maxz);
86
88
89
90 // generate all incoming com messages for this block.
91 // for each subblock (27 of them), there may be an x, y, z direction to search in
92 void generateInNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec& v);
93
94
95 // generate all outgoing com messages for this block
96 void generateOutNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec& v);
97
98private:
102};
103
104
105
106/* Do not ask about buffers for sub-block 1,1,1 (also known as #13)
107 They do not exist, such buffers would be:
108 1) big
109 2) unnecessary since the centre sub-block is not sent anywhere
110
111Note that this class does not deal with data transfer between blocks
112Sub-blocks are copied to and from buffers. Other code is required to
113actually move the data.
114
115"dpp" == doubles per point and gives the number of doubles that make up each "point"
116This is required to calculate offsets and buffer sizes.
117*/
118
119class Block
120{
121public:
122 // s? specifiy the [local] size (in points) of each dimension
123 Block(size_t sx, size_t sy, size_t sz, size_t inset, size_t xmidlen,
124 size_t ymidlen, size_t zmidlen, unsigned int dpp=1);
125
126 ~Block();
127
128 // Out buffers are loaded with the contents of the flat array and are
129 // to be sent to other blocks
130 double* getOutBuffer(unsigned char subx, unsigned char suby, unsigned char subz);
131 double* getOutBuffer(unsigned char bid);
132
133 // In buffers are populated from external communications
134 // and copied back to the flat array
135 double* getInBuffer(unsigned char subx, unsigned char suby, unsigned char subz);
136 double* getInBuffer(unsigned char bid);
137
138 // return number of doubles in the given block
139 size_t getBuffSize(unsigned char subx, unsigned char suby, unsigned char subz);
140 size_t getBuffSize(unsigned char bid);
141
142 // where does the subblock specified start in a source array
143 size_t startOffset(unsigned char subx, unsigned char suby, unsigned char subz);
144
145 // debug only
146 void displayBlock(unsigned char subx, unsigned char suby, unsigned char subz, bool out);
147
148 // Copy a 3d region from a flat array into a buffer
149 void copyToBuffer(unsigned char buffid, double* src);
150
151 // Copy a 3d region from a buffer into a flat array
152 void copyFromBuffer(unsigned char buffid, double* dest);
153
154
155 void copyAllToBuffer(double* src);
156
157 void copyUsedFromBuffer(double* dest);
158
159 void setUsed(unsigned char buffid);
160
161private:
162
163 // determines the dimensions of each subblock
164 void populateDimsTable();
165 void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen);
166 void createBuffArrays(double* startaddress, double* buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen);
167
168
169 double* inbuff;
170 double* outbuff;
171 size_t buffoffsets[27]; // offsets of the various blocks within the buffer arrays
172 size_t flatoffsets[27]; // starting point of each block within a flat array
173 bool used[27];
174 size_t dims[27][3]; // dimension of each subblock
175 size_t sx;
176 size_t sy;
177 size_t sz;
178 size_t inset;
179 size_t xmidlen;
180 size_t ymidlen;
181 size_t zmidlen;
182 double* inbuffptr[27];
183 double* outbuffptr[27];
184 const unsigned int dpsize; // number of doubles which make up a point
185};
186
187// Returns the MPI message tag to use for a transfer between the two subblocks
188int getTag(unsigned char sourcex, unsigned char sourcey, unsigned char sourcez,
189 unsigned char targetx, unsigned char targety, unsigned char targetz);
190
191// computes the tag based on the destination and the direction it comes from
192// the booleans indicate whether a negative shift in that direction is required
193int getTag(unsigned char destx, unsigned char desty, unsigned char destz,
194 bool deltax, bool deltay, bool deltaz);
195
196
197// the booleans indicate whether a negative shift in that direction is required
198unsigned char getSrcBuffID(unsigned char destx, unsigned char desty,
199 unsigned char destz, bool deltax, bool deltay,
200 bool deltaz);
201
202
203/* Now the 2D versions */
204
205// 2D version
207{
208public:
209 BlockGrid2(coord_t maxx, coord_t maxy);
210
212
213 // generate all incoming com messages for this block.
214 // for each subblock (9 of them), there may be an x, y direction to search in
215 void generateInNeighbours(coord_t blockx, coord_t blocky, messvec& v);
216
217 // generate all outgoing com messages for this block
218 void generateOutNeighbours(coord_t blockx, coord_t blocky, messvec& v);
219
220private:
223};
224
225// The 2D version - there is no block 4
227{
228public:
229 // s? specifiy the [local] size (in points) of each dimension
230 Block2(size_t sx, size_t sy, size_t inset, size_t xmidlen,
231 size_t ymidlen, unsigned int dpp=1);
232
233 ~Block2();
234
235 // Out buffers are loaded with the contents of the flat array and are
236 // to be sent to other blocks
237 double* getOutBuffer(unsigned char subx, unsigned char suby);
238 double* getOutBuffer(unsigned char bid);
239
240 // In buffers are populated from external communications
241 // and copied back to the flat array
242 double* getInBuffer(unsigned char subx, unsigned char suby);
243 double* getInBuffer(unsigned char bid);
244
245 // return number of doubles in the given block
246 size_t getBuffSize(unsigned char subx, unsigned char suby);
247 size_t getBuffSize(unsigned char bid);
248
249 // where does the subblock specified start in a source array
250 size_t startOffset(unsigned char subx, unsigned char suby);
251
252 // debug only
253 void displayBlock(unsigned char subx, unsigned char suby, bool out);
254
255 // Copy a 3d region from a flat array into a buffer
256 void copyToBuffer(unsigned char buffid, double* src);
257
258 // Copy a 3d region from a buffer into a flat array
259 void copyFromBuffer(unsigned char buffid, double* dest);
260
261
262 void copyAllToBuffer(double* src);
263
264 void copyUsedFromBuffer(double* dest);
265
266 void setUsed(unsigned char buffid);
267
268private:
269
270 // determines the dimensions of each subblock
271 void populateDimsTable();
272 void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen);
273 void createBuffArrays(double* startaddress, double* buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen);
274
275
276 double* inbuff;
277 double* outbuff;
278 size_t buffoffsets[9]; // offsets of the various blocks within the buffer arrays
279 size_t flatoffsets[9]; // starting point of each block within a flat array
280 bool used[9];
281 size_t dims[9][2]; // dimension of each subblock
282 size_t sx;
283 size_t sy;
284 size_t inset;
285 size_t xmidlen;
286 size_t ymidlen;
287 double* inbuffptr[9];
288 double* outbuffptr[9];
289 const unsigned int dpsize; // number of doubles which make up a point
290};
291
292
293// Returns the MPI message tag to use for a transfer between the two subblocks
294int getTag2(unsigned char sourcex, unsigned char sourcey, unsigned char targetx, unsigned char targety);
295
296// computes the tag based on the destination and the direction it comes from
297// the booleans indicate whether a negative shift in that direction is required
298int getTag2(unsigned char destx, unsigned char desty, bool deltax, bool deltay);
299
300
301// the booleans indicate whether a negative shift in that direction is required
302unsigned char getSrcBuffID2(unsigned char destx, unsigned char desty, bool deltax, bool deltay);
303
304#endif // __RIPLEY_BLOCKTOOLS_H__
305
unsigned char getSrcBuffID2(unsigned char destx, unsigned char desty, bool deltax, bool deltay)
Definition blocktools2.cpp:381
std::pair< neighbourID_t, int > neighpair
Definition blocktools.h:65
int getTag2(unsigned char sourcex, unsigned char sourcey, unsigned char targetx, unsigned char targety)
Definition blocktools2.cpp:365
unsigned coord_t
Definition blocktools.h:63
int neighbourID_t
Definition blocktools.h:62
int getTag(unsigned char sourcex, unsigned char sourcey, unsigned char sourcez, unsigned char targetx, unsigned char targety, unsigned char targetz)
Definition blocktools.cpp:413
unsigned char getSrcBuffID(unsigned char destx, unsigned char desty, unsigned char destz, bool deltax, bool deltay, bool deltaz)
Definition blocktools.cpp:431
std::vector< message > messvec
Definition blocktools.h:79
std::vector< neighpair > neighbourvector
Definition blocktools.h:66
Definition blocktools.h:227
size_t inset
Definition blocktools.h:284
double * getOutBuffer(unsigned char subx, unsigned char suby)
Definition blocktools2.cpp:101
Block2(size_t sx, size_t sy, size_t inset, size_t xmidlen, size_t ymidlen, unsigned int dpp=1)
Definition blocktools2.cpp:247
size_t xmidlen
Definition blocktools.h:285
void populateDimsTable()
Definition blocktools2.cpp:166
void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen)
Definition blocktools2.cpp:184
void displayBlock(unsigned char subx, unsigned char suby, bool out)
Definition blocktools2.cpp:289
size_t dims[9][2]
Definition blocktools.h:281
void createBuffArrays(double *startaddress, double *buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen)
Definition blocktools2.cpp:206
const unsigned int dpsize
Definition blocktools.h:289
size_t flatoffsets[9]
Definition blocktools.h:279
size_t getBuffSize(unsigned char subx, unsigned char suby)
Definition blocktools2.cpp:121
~Block2()
Definition blocktools2.cpp:160
size_t buffoffsets[9]
Definition blocktools.h:278
void copyAllToBuffer(double *src)
Definition blocktools2.cpp:221
size_t sx
Definition blocktools.h:282
size_t startOffset(unsigned char subx, unsigned char suby)
Definition blocktools2.cpp:277
void copyToBuffer(unsigned char buffid, double *src)
Definition blocktools2.cpp:318
void copyUsedFromBuffer(double *dest)
Definition blocktools2.cpp:234
void setUsed(unsigned char buffid)
Definition blocktools2.cpp:216
double * outbuffptr[9]
Definition blocktools.h:288
double * inbuffptr[9]
Definition blocktools.h:287
size_t sy
Definition blocktools.h:283
double * inbuff
Definition blocktools.h:276
void copyFromBuffer(unsigned char buffid, double *dest)
Definition blocktools2.cpp:346
double * outbuff
Definition blocktools.h:277
double * getInBuffer(unsigned char subx, unsigned char suby)
Definition blocktools2.cpp:111
size_t ymidlen
Definition blocktools.h:286
bool used[9]
Definition blocktools.h:280
Definition blocktools.h:207
BlockGrid2(coord_t maxx, coord_t maxy)
Definition blocktools2.cpp:24
coord_t xmax
Definition blocktools.h:221
void generateOutNeighbours(coord_t blockx, coord_t blocky, messvec &v)
Definition blocktools2.cpp:68
void generateInNeighbours(coord_t blockx, coord_t blocky, messvec &v)
Definition blocktools2.cpp:35
neighbourID_t getNID(coord_t x, coord_t y) const
Definition blocktools2.cpp:28
coord_t ymax
Definition blocktools.h:222
Definition blocktools.h:83
coord_t zmax
Definition blocktools.h:101
neighbourID_t getNID(coord_t x, coord_t y, coord_t z) const
Definition blocktools.cpp:29
coord_t ymax
Definition blocktools.h:100
BlockGrid(coord_t maxx, coord_t maxy, coord_t maxz)
Definition blocktools.cpp:25
void generateInNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec &v)
Definition blocktools.cpp:44
coord_t xmax
Definition blocktools.h:99
void generateOutNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec &v)
Definition blocktools.cpp:83
Definition blocktools.h:120
size_t getBuffSize(unsigned char subx, unsigned char suby, unsigned char subz)
Definition blocktools.cpp:143
bool used[27]
Definition blocktools.h:173
void copyToBuffer(unsigned char buffid, double *src)
Definition blocktools.cpp:358
~Block()
Definition blocktools.cpp:182
size_t ymidlen
Definition blocktools.h:180
size_t dims[27][3]
Definition blocktools.h:174
double * getOutBuffer(unsigned char subx, unsigned char suby, unsigned char subz)
Definition blocktools.cpp:123
void copyFromBuffer(unsigned char buffid, double *dest)
Definition blocktools.cpp:387
void displayBlock(unsigned char subx, unsigned char suby, unsigned char subz, bool out)
Definition blocktools.cpp:324
void createBuffArrays(double *startaddress, double *buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen)
Definition blocktools.cpp:237
void setUsed(unsigned char buffid)
Definition blocktools.cpp:247
size_t zmidlen
Definition blocktools.h:181
double * getInBuffer(unsigned char subx, unsigned char suby, unsigned char subz)
Definition blocktools.cpp:133
size_t inset
Definition blocktools.h:178
const unsigned int dpsize
Definition blocktools.h:184
size_t xmidlen
Definition blocktools.h:179
size_t sz
Definition blocktools.h:177
void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen)
Definition blocktools.cpp:215
size_t buffoffsets[27]
Definition blocktools.h:171
size_t sy
Definition blocktools.h:176
size_t sx
Definition blocktools.h:175
size_t flatoffsets[27]
Definition blocktools.h:172
double * inbuff
Definition blocktools.h:169
void copyUsedFromBuffer(double *dest)
Definition blocktools.cpp:265
double * inbuffptr[27]
Definition blocktools.h:182
size_t startOffset(unsigned char subx, unsigned char suby, unsigned char subz)
Definition blocktools.cpp:310
void copyAllToBuffer(double *src)
Definition blocktools.cpp:252
Block(size_t sx, size_t sy, size_t sz, size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen, unsigned int dpp=1)
Definition blocktools.cpp:278
void populateDimsTable()
Definition blocktools.cpp:188
double * outbuffptr[27]
Definition blocktools.h:183
double * outbuff
Definition blocktools.h:170
Definition blocktools.h:70
unsigned char destbuffid
Definition blocktools.h:76
unsigned char srcbuffid
Definition blocktools.h:75
neighbourID_t destID
Definition blocktools.h:73
int tag
Definition blocktools.h:74
neighbourID_t sourceID
Definition blocktools.h:72