27#ifndef EWOMS_DOMESTIC_OVERLAP_FROM_BCRS_MATRIX_HH
28#define EWOMS_DOMESTIC_OVERLAP_FROM_BCRS_MATRIX_HH
65 template <
class BCRSMatrix>
72 , globalIndices_(foreignOverlap_)
79 MPI_Comm_rank(MPI_COMM_WORLD, &tmp);
81 MPI_Comm_size(MPI_COMM_WORLD, &tmp);
82 worldSize_ =
static_cast<unsigned>(tmp);
85 buildDomesticOverlap_();
87 blackList_.updateNativeToDomesticMap(*
this);
96 for (
unsigned domIdx = 0; domIdx <
numDomestic(); ++domIdx) {
103 std::map<int, MpiBuffer<unsigned> *> sizeBufferMap;
105 auto peerIt = peerSet_.begin();
106 const auto& peerEndIt = peerSet_.end();
107 for (; peerIt != peerEndIt; ++peerIt) {
109 sizeBufferMap[*peerIt] = &buffer;
111 buffer.send(*peerIt);
114 peerIt = peerSet_.begin();
115 for (; peerIt != peerEndIt; ++peerIt) {
116 MpiBuffer<unsigned> rcvBuffer(1);
117 rcvBuffer.receive(*peerIt);
119 assert(rcvBuffer[0] == domesticOverlapWithPeer_.find(*peerIt)->second.size());
122 peerIt = peerSet_.begin();
123 for (; peerIt != peerEndIt; ++peerIt) {
124 sizeBufferMap[*peerIt]->wait();
125 delete sizeBufferMap[*peerIt];
139 {
return worldSize_; }
154 && foreignOverlap_.
isBorder(mapExternalToInternal_(domesticIdx));
164 && foreignOverlap_.
isBorderWith(mapExternalToInternal_(domesticIdx),
173 {
return foreignOverlap_.
numFront(peerRank); }
182 Index internalDomesticIdx = mapExternalToInternal_(domesticIdx);
186 const auto& domOverlap = domesticOverlapByIndex_[internalDomesticIdx];
187 return domOverlap.size() > 0
188 && domOverlap.begin()->second == foreignOverlap_.
overlapSize();
195 {
return blackList_; }
202 {
return domesticOverlapByIndex_[mapExternalToInternal_(domesticIdx)].size(); }
227 {
return foreignOverlap_.
numLocal(); }
246 {
return mapExternalToInternal_(domesticIdx) <
static_cast<Index>(
numLocal()); }
256 return foreignOverlap_.
iAmMasterOf(mapExternalToInternal_(domesticIdx));
263 {
return masterRank_[
static_cast<unsigned>(mapExternalToInternal_(domesticIdx))]; }
269 { globalIndices_.
print(); }
279 return mapInternalToExternal_(internalIdx);
293 Index internalIdx = mapExternalToInternal_(domIdx);
307 return mapInternalToExternal_(localIdx);
316 return !this->
isLocal(domesticIdx)
317 || this->foreignOverlap_.
isInOverlap(mapExternalToInternal_(domesticIdx));
326 Index internalIdx = mapExternalToInternal_(domesticIdx);
327 return this->foreignOverlap_.
isFrontFor(peerRank, internalIdx);
336 mapExternalToInternal_(domesticIdx));
355 return mapInternalToExternal_(internalIdx);
363 {
return domesticOverlapWithPeer_.at(peerRank).size(); }
372 Index internalIdx = domesticOverlapWithPeer_.at(peerRank)[overlapOffset];
373 return mapInternalToExternal_(internalIdx);
377 void buildDomesticOverlap_()
380 peerSet_ = foreignOverlap_.
peerSet();
384 domesticOverlapByIndex_.resize(
numLocal());
385 borderDistance_.resize(
numLocal(), 0);
387 PeerSet::const_iterator peerIt;
388 PeerSet::const_iterator peerEndIt = peerSet_.end();
391 peerIt = peerSet_.begin();
392 for (; peerIt != peerEndIt; ++peerIt) {
394 sendIndicesToPeer_(peerRank);
398 peerIt = peerSet_.begin();
399 for (; peerIt != peerEndIt; ++peerIt) {
400 ProcessRank peerRank = *peerIt;
401 receiveIndicesFromPeer_(peerRank);
405 peerIt = peerSet_.begin();
406 for (; peerIt != peerEndIt; ++peerIt) {
407 ProcessRank peerRank = *peerIt;
408 waitSendIndices_(peerRank);
412 void updateMasterRanks_()
416 masterRank_.resize(nDomestic);
420 for (
unsigned i = 0; i < nLocal; ++i) {
426 for (
size_t i = nLocal; i < nDomestic; ++i)
427 masterRank_[i] = std::numeric_limits<ProcessRank>::max();
431 auto peerIt = peerSet_.begin();
432 const auto& peerEndIt = peerSet_.end();
433 for (; peerIt != peerEndIt; ++peerIt) {
434 const auto& overlapWithPeer = domesticOverlapWithPeer_.find(*peerIt)->second;
436 auto idxIt = overlapWithPeer.begin();
437 const auto& idxEndIt = overlapWithPeer.end();
438 for (; idxIt != idxEndIt; ++idxIt) {
439 if (*idxIt >= 0 && foreignOverlap_.
isLocal(*idxIt))
442 masterRank_[
static_cast<unsigned>(*idxIt)] = std::min(masterRank_[
static_cast<unsigned>(*idxIt)], *peerIt);
447 void sendIndicesToPeer_([[maybe_unused]] ProcessRank peerRank)
455 size_t numIndices = foreignOverlap.size();
456 numIndicesSendBuffer_[peerRank] =
new MpiBuffer<size_t>(1);
457 (*numIndicesSendBuffer_[peerRank])[0] = numIndices;
458 numIndicesSendBuffer_[peerRank]->send(peerRank);
461 indicesSendBuffer_[peerRank] =
new MpiBuffer<IndexDistanceNpeers>(numIndices);
464 auto overlapIt = foreignOverlap.begin();
465 const auto& overlapEndIt = foreignOverlap.end();
466 for (
unsigned i = 0; overlapIt != overlapEndIt; ++overlapIt, ++i) {
467 Index localIdx = overlapIt->index;
471 IndexDistanceNpeers tmp;
473 tmp.borderDistance = borderDistance;
474 tmp.numPeers =
static_cast<unsigned>(
numPeers);
476 (*indicesSendBuffer_[peerRank])[i] = tmp;
479 indicesSendBuffer_[peerRank]->send(peerRank);
483 void waitSendIndices_(ProcessRank peerRank)
485 numIndicesSendBuffer_[peerRank]->wait();
486 delete numIndicesSendBuffer_[peerRank];
488 indicesSendBuffer_[peerRank]->wait();
489 delete indicesSendBuffer_[peerRank];
492 void receiveIndicesFromPeer_([[maybe_unused]] ProcessRank peerRank)
497 MpiBuffer<size_t> numIndicesRecvBuff(1);
498 numIndicesRecvBuff.receive(peerRank);
499 numIndices =
static_cast<int>(numIndicesRecvBuff[0]);
502 MpiBuffer<IndexDistanceNpeers> recvBuff(
static_cast<size_t>(numIndices));
503 recvBuff.receive(peerRank);
504 for (
unsigned i = 0; i < static_cast<unsigned>(numIndices); ++i) {
505 Index globalIdx = recvBuff[i].index;
512 globalIndices_.
addIndex(newDomesticIdx, globalIdx);
515 borderDistance_.resize(newSize, std::numeric_limits<int>::max());
516 domesticOverlapByIndex_.resize(newSize);
523 domesticOverlapByIndex_[
static_cast<unsigned>(domesticIdx)][
static_cast<unsigned>(peerRank)] = borderDistance;
524 domesticOverlapWithPeer_[
static_cast<unsigned>(peerRank)].push_back(domesticIdx);
527 assert(globalIdx >= 0);
528 assert(domesticIdx >= 0);
529 assert(!(borderDistance == 0 && !foreignOverlap_.
isLocal(domesticIdx)));
530 assert(!(borderDistance > 0 && foreignOverlap_.
isLocal(domesticIdx)));
532 borderDistance_[
static_cast<unsigned>(domesticIdx)] = std::min(borderDistance, borderDistance_[
static_cast<unsigned>(domesticIdx)]);
546 void setupDebugMapping_()
553 Index mapInternalToExternal_(Index internalIdx)
const
554 {
return internalIdx; }
560 Index mapExternalToInternal_(Index externalIdx)
const
561 {
return externalIdx; }
565 ForeignOverlap foreignOverlap_;
567 BlackList blackList_;
571 std::vector<BorderDistance> borderDistance_;
572 std::vector<ProcessRank> masterRank_;
574 std::map<ProcessRank, MpiBuffer<size_t> *> numIndicesSendBuffer_;
575 std::map<ProcessRank, MpiBuffer<IndexDistanceNpeers> *> indicesSendBuffer_;
576 GlobalIndices globalIndices_;
Expresses which degrees of freedom are blacklisted for the parallel linear solvers and which domestic...
Expresses which degrees of freedom are blacklisted for the parallel linear solvers and which domestic...
Definition blacklist.hh:49
This class creates and manages the foreign overlap given an initial list of border indices and a BCRS...
Definition domesticoverlapfrombcrsmatrix.hh:53
void print() const
Print the foreign overlap for debugging purposes.
Definition domesticoverlapfrombcrsmatrix.hh:268
ProcessRank myRank() const
Returns the rank of the current process.
Definition domesticoverlapfrombcrsmatrix.hh:132
size_t numFront(ProcessRank peerRank) const
Returns the number of indices on the front within a given peer rank's grid partition.
Definition domesticoverlapfrombcrsmatrix.hh:172
Index nativeToDomestic(Index nativeIdx) const
Returns a domestic index given a native one.
Definition domesticoverlapfrombcrsmatrix.hh:302
Index domesticToGlobal(Index domIdx) const
Returns a global index given a domestic one.
Definition domesticoverlapfrombcrsmatrix.hh:285
bool peerHasIndex(int peerRank, Index domesticIdx) const
Returns true iff a domestic index is seen by a peer rank.
Definition domesticoverlapfrombcrsmatrix.hh:333
bool isLocal(Index domesticIdx) const
Return true if a domestic index is local for the process.
Definition domesticoverlapfrombcrsmatrix.hh:245
const BlackList & blackList() const
Returns the object which represents the black-listed native indices.
Definition domesticoverlapfrombcrsmatrix.hh:194
size_t numDomestic() const
Returns the number domestic indices.
Definition domesticoverlapfrombcrsmatrix.hh:236
unsigned overlapSize() const
Returns the size of the overlap region.
Definition domesticoverlapfrombcrsmatrix.hh:207
Index foreignOverlapOffsetToDomesticIdx(ProcessRank peerRank, unsigned overlapOffset) const
Returns the domestic index given an offset in the foreign overlap of a peer process with the local pr...
Definition domesticoverlapfrombcrsmatrix.hh:351
unsigned worldSize() const
Returns the number of processes in the global MPI communicator.
Definition domesticoverlapfrombcrsmatrix.hh:138
bool isFrontFor(ProcessRank peerRank, Index domesticIdx) const
Returns true if a given domestic index is a front index for a peer rank.
Definition domesticoverlapfrombcrsmatrix.hh:324
bool isBorderWith(Index domesticIdx, ProcessRank peerRank) const
Returns true iff a domestic index is on the border with a given peer process.
Definition domesticoverlapfrombcrsmatrix.hh:161
bool iAmMasterOf(Index domesticIdx) const
Return true iff the current process is the master of a given domestic index.
Definition domesticoverlapfrombcrsmatrix.hh:252
bool isFront(Index domesticIdx) const
Returns true iff a domestic index is on the front.
Definition domesticoverlapfrombcrsmatrix.hh:178
Index globalToDomestic(Index globalIdx) const
Returns a domestic index given a global one.
Definition domesticoverlapfrombcrsmatrix.hh:274
size_t domesticOverlapSize(ProcessRank peerRank) const
Returns number of indices which are contained in the domestic overlap with a peer.
Definition domesticoverlapfrombcrsmatrix.hh:362
DomesticOverlapFromBCRSMatrix(const BCRSMatrix &A, const BorderList &borderList, const BlackList &blackList, unsigned overlapSize)
Constructs the foreign overlap given a BCRS matrix and an initial list of border indices.
Definition domesticoverlapfrombcrsmatrix.hh:66
ProcessRank masterRank(Index domesticIdx) const
Return the rank of a master process for a domestic index.
Definition domesticoverlapfrombcrsmatrix.hh:262
size_t numLocal() const
Returns the number local indices.
Definition domesticoverlapfrombcrsmatrix.hh:226
const PeerSet & peerSet() const
Return the set of process ranks which share an overlap with the current process.
Definition domesticoverlapfrombcrsmatrix.hh:145
size_t numNative() const
Returns the number native indices.
Definition domesticoverlapfrombcrsmatrix.hh:217
size_t foreignOverlapSize(ProcessRank peerRank) const
Returns number of indices which are contained in the foreign overlap with a peer.
Definition domesticoverlapfrombcrsmatrix.hh:343
bool isInOverlap(Index domesticIdx) const
Returns true if a given domestic index is either in the foreign or in the domestic overlap.
Definition domesticoverlapfrombcrsmatrix.hh:314
bool isBorder(Index domesticIdx) const
Returns true iff a domestic index is a border index.
Definition domesticoverlapfrombcrsmatrix.hh:151
size_t numPeers(Index domesticIdx) const
Returns the number of processes which "see" a given index.
Definition domesticoverlapfrombcrsmatrix.hh:201
Index domesticOverlapOffsetToDomesticIdx(ProcessRank peerRank, Index overlapOffset) const
Returns the domestic index given an offset in the domestic overlap of a peer process with the local p...
Definition domesticoverlapfrombcrsmatrix.hh:370
Index domesticToNative(Index domIdx) const
Returns a native index given a domestic one.
Definition domesticoverlapfrombcrsmatrix.hh:291
This class creates and manages the foreign overlap given an initial list of border indices and a BCRS...
Definition foreignoverlapfrombcrsmatrix.hh:60
const PeerSet & peerSet() const
Return the set of process ranks which share an overlap with the current process.
Definition foreignoverlapfrombcrsmatrix.hh:260
size_t numLocal() const
Returns the number of local indices.
Definition foreignoverlapfrombcrsmatrix.hh:279
bool isInOverlap(Index localIdx) const
Returns true if a given local index is in the foreign overlap of any rank.
Definition foreignoverlapfrombcrsmatrix.hh:323
const OverlapWithPeer & foreignOverlapWithPeer(ProcessRank peerRank) const
Return the list of (local indices, border distance, number of processes) triples which are in the ove...
Definition foreignoverlapfrombcrsmatrix.hh:197
unsigned overlapSize() const
Returns the size of the overlap region.
Definition foreignoverlapfrombcrsmatrix.hh:143
bool isBorder(Index localIdx) const
Returns true iff a local index is a border index.
Definition foreignoverlapfrombcrsmatrix.hh:149
Index nativeToLocal(Index nativeIdx) const
Convert a native index to a local one.
Definition foreignoverlapfrombcrsmatrix.hh:294
bool peerHasIndex(ProcessRank peerRank, Index localIdx) const
Returns true iff a local index is seen by a peer rank.
Definition foreignoverlapfrombcrsmatrix.hh:217
size_t numNative() const
Returns the number of native indices.
Definition foreignoverlapfrombcrsmatrix.hh:273
const std::map< ProcessRank, BorderDistance > & foreignOverlapByLocalIndex(Index localIdx) const
Return the map of (peer rank, border distance) for a given local index.
Definition foreignoverlapfrombcrsmatrix.hh:208
bool isFrontFor(ProcessRank peerRank, Index localIdx) const
Returns whether a given local index is on the front of a given peer rank.
Definition foreignoverlapfrombcrsmatrix.hh:245
size_t numFront(ProcessRank peerRank) const
Returns the number of front indices of a peer process in the local partition.
Definition foreignoverlapfrombcrsmatrix.hh:227
Index localToNative(Index localIdx) const
Convert a local index to a native one.
Definition foreignoverlapfrombcrsmatrix.hh:300
ProcessRank masterRank(Index localIdx) const
Return the rank of the master process of an index.
Definition foreignoverlapfrombcrsmatrix.hh:171
bool isBorderWith(Index localIdx, ProcessRank peerRank) const
Returns true iff a local index is a border index shared with a given peer process.
Definition foreignoverlapfrombcrsmatrix.hh:156
bool iAmMasterOf(Index localIdx) const
Return true if the current rank is the "master" of an index.
Definition foreignoverlapfrombcrsmatrix.hh:182
bool isLocal(Index domesticIdx) const
Returns true iff a domestic index is local.
Definition foreignoverlapfrombcrsmatrix.hh:285
This class maps domestic row indices to and from "global" indices which is used to construct an algeb...
Definition globalindices.hh:56
void print() const
Prints the global indices of all domestic indices for debugging purposes.
Definition globalindices.hh:190
void addIndex(Index domesticIdx, Index globalIdx)
Add an index to the domestic<->global mapping.
Definition globalindices.hh:127
size_t numDomestic() const
Returns the number domestic indices.
Definition globalindices.hh:121
Index domesticToGlobal(Index domesticIdx) const
Converts a domestic index to a global one.
Definition globalindices.hh:88
Index globalToDomestic(Index globalIdx) const
Converts a global index to a domestic one.
Definition globalindices.hh:98
bool hasGlobalIndex(Index globalIdx) const
Return true iff a given global index already exists.
Definition globalindices.hh:183
A set of process ranks.
Definition overlaptypes.hh:149
Simplifies handling of buffers to be used in conjunction with MPI.
Definition mpibuffer.hh:46
This class creates and manages the foreign overlap given an initial list of border indices and a BCRS...
This class maps domestic row indices to and from "global" indices which is used to construct an algeb...
Simplifies handling of buffers to be used in conjunction with MPI.
This file contains a set of helper functions used by VFPProd / VFPInj.
Definition blackoilboundaryratevector.hh:37
std::vector< std::map< ProcessRank, BorderDistance > > OverlapByIndex
Maps each index to a list of processes .
Definition overlaptypes.hh:176
unsigned ProcessRank
The type of the rank of a process.
Definition overlaptypes.hh:49
unsigned BorderDistance
The type representing the distance of an index to the border.
Definition overlaptypes.hh:54
int Index
The type of an index of a degree of freedom.
Definition overlaptypes.hh:44
std::list< BorderIndex > BorderList
This class managages a list of indices which are on the border of a process' partition of the grid.
Definition overlaptypes.hh:120
std::map< ProcessRank, DomesticOverlapWithPeer > DomesticOverlapByRank
A type mapping the process rank to the list of domestic indices which are owned by the peer.
Definition overlaptypes.hh:187