69 template <
class BCRSMatrix>
82 MPI_Comm_rank(MPI_COMM_WORLD, &tmp);
89 createLocalIndices_();
95 for (; it != endIt; ++it) {
100 localBorderIndices_.insert(localIdx);
111 peerSet_ = neighborPeerSet_;
123 for (; borderIt != borderEndIt; ++borderIt) {
124 minBorderDist = std::min(minBorderDist, borderIt->borderDistance);
129 foreignOverlapByLocalIndex_.resize(
numLocal());
130 extendForeignOverlap_(A, initialSeedList, minBorderDist,
overlapSize);
134 computeMasterRanks_();
137 groupForeignOverlapByRank_();
144 {
return overlapSize_; }
150 {
return localBorderIndices_.count(localIdx) > 0; }
158 const auto& indexOverlap = foreignOverlapByLocalIndex_[
static_cast<unsigned>(localIdx)];
159 const auto& borderDistIt = indexOverlap.find(peerRank);
160 if (borderDistIt == indexOverlap.end())
164 return borderDistIt->second == 0;
172 {
return masterRank_[
static_cast<unsigned>(localIdx)]; }
183 {
return masterRank_[
static_cast<unsigned>(localIdx)] == myRank_; }
190 {
return borderList_; }
199 assert(foreignOverlapByRank_.find(peerRank) != foreignOverlapByRank_.end());
200 return foreignOverlapByRank_.find(peerRank)->second;
207 const std::map<ProcessRank, BorderDistance> &
211 return foreignOverlapByLocalIndex_[
static_cast<unsigned>(localIdx)];
219 const auto& idxOverlap = foreignOverlapByLocalIndex_[localIdx];
220 return idxOverlap.find(peerRank) != idxOverlap.end();
229 const auto& peerOverlap = foreignOverlapByRank_.find(peerRank)->second;
232 auto it = peerOverlap.begin();
233 const auto& endIt = peerOverlap.end();
234 for (; it != endIt; ++it) {
235 if (it->borderDistance == overlapSize_)
247 const auto& idxOverlap = foreignOverlapByLocalIndex_[localIdx];
249 auto it = idxOverlap.find(peerRank);
250 if (it == idxOverlap.end())
253 return it->second == overlapSize_;
268 {
return neighborPeerSet_; }
274 {
return numNative_; }
280 {
return numLocal_; }
286 {
return static_cast<unsigned>(domesticIdx) <
numLocal(); }
295 {
return nativeToLocalIndices_[
static_cast<unsigned>(nativeIdx)]; }
302 assert(localIdx <
static_cast<Index>(localToNativeIndices_.size()));
303 return localToNativeIndices_[
static_cast<unsigned>(localIdx)];
310 {
return blackList_; }
317 {
return foreignOverlapByLocalIndex_[
static_cast<unsigned>(localIdx)].size(); }
324 {
return foreignOverlapByLocalIndex_[
static_cast<unsigned>(localIdx)].size() > 0; }
331 auto it = foreignOverlapByRank_.begin();
332 const auto& endIt = foreignOverlapByRank_.end();
333 for (; it != endIt; ++it) {
334 std::cout <<
"Overlap rows(distance) for rank " << it->first <<
": ";
336 auto rowIt = it->second.begin();
337 const auto& rowEndIt = it->second.end();
338 for (; rowIt != rowEndIt; ++rowIt)
339 std::cout << rowIt->index <<
"(" << rowIt->borderDistance <<
") ";
340 std::cout <<
"\n" << std::flush;
348 template <
class BCRSMatrix>
349 void extendForeignOverlap_(
const BCRSMatrix& A,
355 addNonNeighborOverlapIndices_(A, seedList, borderDistance);
358 auto seedIt = seedList.begin();
359 const auto& seedEndIt = seedList.end();
360 for (; seedIt != seedEndIt; ++seedIt) {
363 unsigned distance = borderDistance;
366 if (foreignOverlapByLocalIndex_[
static_cast<unsigned>(localIdx)].count(peerRank) == 0)
367 foreignOverlapByLocalIndex_[
static_cast<unsigned>(localIdx)][peerRank] = distance;
377 SeedList nextSeedList;
378 seedIt = seedList.begin();
379 for (; seedIt != seedEndIt; ++seedIt) {
380 Index nativeRowIdx = seedIt->index;
383 ProcessRank peerRank = seedIt->peerRank;
388 using ColIterator =
typename BCRSMatrix::ConstColIterator;
389 ColIterator colIt = A[
static_cast<unsigned>(nativeRowIdx)].begin();
390 ColIterator colEndIt = A[
static_cast<unsigned>(nativeRowIdx)].end();
391 for (; colIt != colEndIt; ++colIt) {
392 Index nativeColIdx =
static_cast<Index
>(colIt.index());
400 else if (foreignOverlapByLocalIndex_[
static_cast<unsigned>(localColIdx)].count(peerRank) > 0)
404 bool hasIndex =
false;
405 typename SeedList::iterator sIt = nextSeedList.begin();
406 typename SeedList::iterator sEndIt = nextSeedList.end();
407 for (; sIt != sEndIt; ++sIt) {
408 if (sIt->index == nativeColIdx && sIt->peerRank == peerRank) {
418 IndexRankDist newTuple;
419 newTuple.index = nativeColIdx;
420 newTuple.peerRank = peerRank;
421 newTuple.borderDistance = seedIt->borderDistance + 1;
422 nextSeedList.push_back(newTuple);
430 extendForeignOverlap_(A, nextSeedList, borderDistance + 1,
overlapSize);
434 void createLocalIndices_()
438 for (
unsigned nativeIdx = 0; nativeIdx < numNative_;) {
439 if (!blackList_.hasIndex(
static_cast<Index>(nativeIdx))) {
440 localToNativeIndices_.push_back(
static_cast<Index>(nativeIdx));
441 nativeToLocalIndices_.push_back(
static_cast<Index>(localIdx));
446 nativeToLocalIndices_.push_back(-1);
451 numLocal_ = localToNativeIndices_.size();
454 Index localToPeerIdx_(Index localIdx, ProcessRank peerRank)
const
456 auto it = borderList_.begin();
457 const auto& endIt = borderList_.end();
458 for (; it != endIt; ++it) {
459 if (it->localIdx == localIdx && it->peerRank == peerRank)
466 template <
class BCRSMatrix>
467 void addNonNeighborOverlapIndices_(
const BCRSMatrix&,
468 [[maybe_unused]] SeedList& seedList,
469 [[maybe_unused]] BorderDistance borderDist)
476 std::map<ProcessRank, std::vector<BorderIndex> > borderIndices;
480 auto it = seedList.begin();
481 const auto& endIt = seedList.end();
482 for (; it != endIt; ++it) {
486 BorderIndex borderHandle;
487 borderHandle.localIdx = localIdx;
488 borderHandle.peerRank = it->peerRank;
489 borderHandle.borderDistance = it->borderDistance;
492 auto neighborIt = foreignOverlapByLocalIndex_[
static_cast<unsigned>(localIdx)].begin();
493 const auto& neighborEndIt = foreignOverlapByLocalIndex_[
static_cast<unsigned>(localIdx)].end();
494 for (; neighborIt != neighborEndIt; ++neighborIt) {
495 if (neighborIt->second != 0)
498 else if (neighborIt->first == borderHandle.peerRank)
503 Index peerIdx = localToPeerIdx_(localIdx, neighborIt->first);
508 borderHandle.peerIdx = peerIdx;
509 borderIndices[neighborIt->first].push_back(borderHandle);
516 std::map<ProcessRank, Opm::MpiBuffer<unsigned> > numIndicesSendBufs;
517 std::map<ProcessRank, Opm::MpiBuffer<BorderIndex> > indicesSendBufs;
520 for (; peerIt != peerEndIt; ++peerIt) {
522 size_t numIndices = borderIndices[peerRank].size();
523 numIndicesSendBufs[peerRank].resize(1);
524 numIndicesSendBufs[peerRank][0] =
static_cast<unsigned>(numIndices);
526 const auto& peerBorderIndices = borderIndices[peerRank];
527 indicesSendBufs[peerRank].resize(numIndices);
529 auto tmpIt = peerBorderIndices.begin();
530 const auto& tmpEndIt = peerBorderIndices.end();
532 for (; tmpIt != tmpEndIt; ++tmpIt, ++i) {
533 indicesSendBufs[peerRank][i] = *tmpIt;
539 for (; peerIt != peerEndIt; ++peerIt) {
541 numIndicesSendBufs[neighborPeer].send(neighborPeer);
542 indicesSendBufs[neighborPeer].send(neighborPeer);
546 std::map<ProcessRank, MpiBuffer<unsigned> > numIndicesRcvBufs;
547 std::map<ProcessRank, MpiBuffer<BorderIndex> > indicesRcvBufs;
549 for (; peerIt != peerEndIt; ++peerIt) {
551 auto& numIndicesRcvBuf = numIndicesRcvBufs[neighborPeer];
552 auto& indicesRcvBuf = indicesRcvBufs[neighborPeer];
554 numIndicesRcvBuf.resize(1);
555 numIndicesRcvBuf.receive(neighborPeer);
556 unsigned numIndices = numIndicesRcvBufs[neighborPeer][0];
557 indicesRcvBuf.resize(numIndices);
558 indicesRcvBuf.receive(neighborPeer);
563 for (
unsigned i = 0; i < numIndices; ++i) {
566 std::swap(indicesRcvBuf[i].localIdx, indicesRcvBuf[i].peerIdx);
570 Index localIdx = indicesRcvBuf[i].localIdx;
574 const auto& distIt = foreignOverlapByLocalIndex_[
static_cast<unsigned>(localIdx)].find(peerRank);
575 if (distIt != foreignOverlapByLocalIndex_[
static_cast<unsigned>(localIdx)].end())
579 bool inSeedList =
false;
580 auto seedIt = seedList.begin();
581 const auto& seedEndIt = seedList.end();
582 for (; seedIt != seedEndIt; ++seedIt) {
583 if (seedIt->index == localIdx && seedIt->peerRank == peerRank) {
591 IndexRankDist seedEntry;
592 seedEntry.index = localIdx;
593 seedEntry.peerRank = peerRank;
594 seedEntry.borderDistance = borderDist;
595 seedList.push_back(seedEntry);
598 peerSet_.insert(peerRank);
604 for (; peerIt != peerEndIt; ++peerIt) {
606 numIndicesSendBufs[neighborPeer].wait();
607 indicesSendBufs[neighborPeer].wait();
615 void computeMasterRanks_()
618 masterRank_.resize(numLocal_);
619 for (
unsigned localIdx = 0; localIdx < numLocal_; ++localIdx) {
625 auto it = foreignOverlapByLocalIndex_[
static_cast<unsigned>(localIdx)].begin();
626 const auto& endIt = foreignOverlapByLocalIndex_[
static_cast<unsigned>(localIdx)].end();
627 for (; it != endIt; ++it) {
628 if (it->second == 0) {
635 masterRank_[
static_cast<unsigned>(localIdx)] =
masterRank;
642 void groupForeignOverlapByRank_()
646 size_t numLocal = foreignOverlapByLocalIndex_.size();
647 for (
unsigned localIdx = 0; localIdx <
numLocal; ++localIdx) {
649 auto it = foreignOverlapByLocalIndex_[localIdx].begin();
650 const auto& endIt = foreignOverlapByLocalIndex_[localIdx].end();
651 size_t nRanks = foreignOverlapByLocalIndex_[localIdx].size();
652 for (; it != endIt; ++it) {
653 IndexDistanceNpeers tmp;
654 tmp.index =
static_cast<Index>(localIdx);
655 tmp.borderDistance = it->second;
656 tmp.numPeers =
static_cast<unsigned>(nRanks);
657 foreignOverlapByRank_[it->first].push_back(tmp);
666 PeerSet neighborPeerSet_;
672 const BlackList& blackList_;
675 std::vector<Index> nativeToLocalIndices_;
676 std::vector<Index> localToNativeIndices_;
680 std::vector<ProcessRank> masterRank_;
684 std::set<Index> localBorderIndices_;
695 unsigned overlapSize_;