86 GetPropType<TypeTag, Properties::EquilGrid>,
87 GetPropType<TypeTag, Properties::GridView>,
88 GetPropType<TypeTag, Properties::ElementMapper>,
89 GetPropType<TypeTag, Properties::Scalar>>
97 using Element =
typename GridView::template Codim<0>::Entity;
106 static void registerParameters()
115 :
BaseType(simulator.vanguard().schedule(),
116 simulator.vanguard().eclState(),
117 simulator.vanguard().summaryConfig(),
118 simulator.vanguard().grid(),
119 ((simulator.vanguard().grid().comm().rank() == 0)
120 ? &simulator.vanguard().equilGrid()
122 simulator.vanguard().gridView(),
123 simulator.vanguard().cartesianIndexMapper(),
124 ((simulator.vanguard().grid().comm().rank() == 0)
125 ? &simulator.vanguard().equilCartesianIndexMapper()
128 , simulator_(simulator)
130 this->damarisUpdate_ = true ;
132 this->rank_ = this->simulator_.vanguard().grid().comm().rank() ;
133 this->nranks_ = this->simulator_.vanguard().grid().comm().size();
135 this->elements_rank_offsets_.resize(this->nranks_);
145 const auto& gridView = this->simulator_.gridView();
146 const auto& interior_elements = elements(gridView, Dune::Partitions::interior);
148 this->numElements_ = std::distance(interior_elements.begin(), interior_elements.end());
151 if (this->nranks_ > 1) {
152 auto smryCfg = (this->rank_ == 0)
153 ? this->eclIO_->finalSummaryConfig()
156 eclBroadcast(this->simulator_.vanguard().grid().comm(), smryCfg);
158 this->damarisOutputModule_ = std::make_unique<OutputBlackOilModule<TypeTag>>
159 (simulator, smryCfg, this->collectOnIORank_);
162 this->damarisOutputModule_ = std::make_unique<OutputBlackOilModule<TypeTag>>
163 (simulator, this->eclIO_->finalSummaryConfig(), this->collectOnIORank_);
166 wanted_vars_set_ = Opm::DamarisOutput::getSetOfIncludedVariables<TypeTag>();
175 const int reportStepNum = simulator_.episodeIndex() + 1;
176 const auto& cc = simulator_.vanguard().grid().comm();
180 this->damarisOutputModule_->invalidateLocalData() ;
181 this->prepareLocalCellData(isSubStep, reportStepNum);
182 this->damarisOutputModule_->outputErrorLog(cc);
185 auto localWellData = simulator_.problem().wellModel().wellData();
189 if (localCellData.size() == 0) {
190 this->damarisOutputModule_->assignToSolution(localCellData);
194 this->damarisOutputModule_->addRftDataToWells(localWellData, reportStepNum);
197 if (damarisUpdate_ ==
true) {
202 dam_err_ = DamarisOutput::setupWritingPars(cc, numElements_, elements_rank_offsets_);
205 this->setGlobalIndexForDamaris() ;
209 this->writeDamarisGridOutput() ;
213 this->damarisUpdate_ =
false;
217 int cell_data_written = 0 ;
219 for (
const auto& damVar : localCellData )
222 const std::string& name = damVar.first ;
225 if (wanted_vars_set_.count(name) || wanted_vars_set_.empty())
227 const data::CellData& dataCol = damVar.second ;
228 OpmLog::debug(fmt::format(
"Name of Damaris Variable : ( rank:{}) name: {} ", rank_, name));
233 dam_err_ = DamarisOutput::setPosition(name.c_str(), this->elements_rank_offsets_[rank_]);
238 if (dataCol.data<
double>().size() >=
static_cast<std::vector<double>::size_type
>(this->numElements_)) {
239 dam_err_ = DamarisOutput::write(name.c_str(), dataCol.data<
double>().data()) ;
241 OpmLog::info(fmt::format(
"( rank:{}) The variable \"{}\" was found to be of a different size {} (not {}).", rank_, name, dataCol.data<
double>().size(), this->numElements_ ));
244 catch (std::bad_variant_access
const& ex) {
246 if (dataCol.data<
int>().size() >=
static_cast<std::vector<int>::size_type
>(this->numElements_)) {
247 dam_err_ = DamarisOutput::write(name.c_str(), dataCol.data<
int>().data()) ;
249 OpmLog::info(fmt::format(
"( rank:{}) The variable \"{}\" was found to be of a different size {} (not {}).", rank_, name, dataCol.data<
int>().size(), this->numElements_ ));
252 ++cell_data_written ;
255 DamarisOutput::handleError(dam_err_, cc,
"setPosition() and write() for available variables");
257 if (!cell_data_written) {
258 OpmLog::info(fmt::format(
"( rank:{}) No simulation data written to the Damaris server - check --damaris-limit-variables command line option (if used) has valid variable name(s) and that the Damaris XML file contains variable names that are available in your simulation.", rank_));
260 OpmLog::debug(fmt::format(
"( rank:{}) {} Damaris Variables written to the Damaris servers", rank_, cell_data_written));
276 if (this->damarisOutputModule_->getPRESSURE_ptr() !=
nullptr)
278 dam_err_ = DamarisOutput::endIteration();
280 DamarisOutput::handleError(dam_err_, cc,
"endIteration()");
290 std::unordered_set<std::string> wanted_vars_set_ ;
293 std::unique_ptr<OutputBlackOilModule<TypeTag>> damarisOutputModule_;
294 std::vector<unsigned long long> elements_rank_offsets_ ;
295 bool damarisUpdate_ =
false;
297 static bool enableDamarisOutput_()
299 static bool enable = Parameters::Get<Parameters::EnableDamarisOutput>();
303 void setGlobalIndexForDamaris ()
305 const auto& cc = simulator_.
vanguard().grid().comm();
309 dam_err_ = DamarisOutput::setPosition(
"GLOBAL_CELL_INDEX", elements_rank_offsets_[rank_]);
310 DamarisOutput::handleError(dam_err_, cc,
"setPosition() for GLOBAL_CELL_INDEX");
316 DamarisVarInt mpi_rank_var(1, {
"n_elements_local"},
"MPI_RANK", rank_);
317 mpi_rank_var.setDamarisPosition({
static_cast<int64_t
>(elements_rank_offsets_[rank_])});
321 if (this->collectOnIORank_.isParallel()) {
322 const std::vector<int>& local_to_global =
323 this->collectOnIORank_.localIdxToGlobalIdxMapping();
324 dam_err_ = DamarisOutput::write(
"GLOBAL_CELL_INDEX", local_to_global.data());
326 std::vector<int> local_to_global_filled ;
327 local_to_global_filled.resize(this->numElements_) ;
328 std::iota(local_to_global_filled.begin(), local_to_global_filled.end(), 0);
329 dam_err_ = DamarisOutput::write(
"GLOBAL_CELL_INDEX", local_to_global_filled.data());
331 DamarisOutput::handleError(dam_err_, cc,
"write() for GLOBAL_CELL_INDEX");
333 mpi_rank_var.setDamarisParameterAndShmem( {this->numElements_ } ) ;
335 std::fill(mpi_rank_var.data(), mpi_rank_var.data() + numElements_, rank_);
339 const auto& outputDir = simulator_.
vanguard().eclState().cfg().io().getOutputDir();
340 if (outputDir.size() > 0) {
341 dam_err_ = DamarisOutput::setParameter(
"path_string_length", outputDir.size()) ;
342 DamarisOutput::handleError(dam_err_, cc,
"setParameter() for path_string_length");
343 dam_err_ = DamarisOutput::write(
"OUTPUTDIR", outputDir.c_str());
344 DamarisOutput::handleError(dam_err_, cc,
"write() for OUTPUTDIR");
348 void writeDamarisGridOutput()
350 const auto& gridView = simulator_.
gridView();
351 GridDataOutput::SimMeshDataAccessor geomData(gridView, Dune::Partitions::interior) ;
354 const bool hasPolyCells = geomData.polyhedralCellPresent() ;
355 if ( hasPolyCells ) {
356 OpmLog::error(fmt::format(
"ERORR: rank {} The DUNE geometry grid has polyhedral elements - These elements are currently not supported.", rank_ ));
370 DamarisVarDbl var_x(1, {
"n_coords_local"},
"coordset/coords/values/x", rank_) ;
374 var_x.setDamarisParameterAndShmem( { geomData.getNVertices() } ) ;
376 DamarisVarDbl var_y(1, {
"n_coords_local"},
"coordset/coords/values/y", rank_) ;
377 var_y.setDamarisParameterAndShmem( { geomData.getNVertices() } ) ;
379 DamarisVarDbl var_z(1, {
"n_coords_local"},
"coordset/coords/values/z", rank_) ;
380 var_z.setDamarisParameterAndShmem( { geomData.getNVertices() } ) ;
383 if ( geomData.writeGridPoints(var_x, var_y, var_z) < 0)
384 DUNE_THROW(Dune::IOError, geomData.getError() );
399 DamarisVarInt var_connectivity(1, {
"n_connectivity_ph"},
400 "topologies/topo/elements/connectivity", rank_) ;
401 var_connectivity.setDamarisParameterAndShmem({ geomData.getNCorners()}) ;
402 DamarisVarInt var_offsets(1, {
"n_offsets_types_ph"},
403 "topologies/topo/elements/offsets", rank_) ;
404 var_offsets.setDamarisParameterAndShmem({ geomData.getNCells()+1}) ;
405 DamarisVarChar var_types(1, {
"n_offsets_types_ph"},
406 "topologies/topo/elements/types", rank_) ;
407 var_types.setDamarisParameterAndShmem({ geomData.getNCells()}) ;
411 GridDataOutput::ConnectivityVertexOrder vtkorder = GridDataOutput::VTK ;
413 i = geomData.writeConnectivity(var_connectivity, vtkorder) ;
414 if ( i != geomData.getNCorners())
415 DUNE_THROW(Dune::IOError, geomData.getError());
417 i = geomData.writeOffsetsCells(var_offsets);
418 if ( i != geomData.getNCells()+1)
419 DUNE_THROW(Dune::IOError,geomData.getError());
421 i = geomData.writeCellTypes(var_types) ;
422 if ( i != geomData.getNCells())
423 DUNE_THROW(Dune::IOError,geomData.getError());
425 catch (std::exception& e)
427 OpmLog::error(e.what());
431 void prepareLocalCellData(
const bool isSubStep,
432 const int reportStepNum)
434 OPM_TIMEBLOCK(prepareLocalCellData);
435 if (damarisOutputModule_->localDataValid()) {
439 const auto& gridView = simulator_.
vanguard().gridView();
440 const int num_interior = detail::
441 countLocalInteriorCellsGridView(gridView);
442 const bool log = this->collectOnIORank_.isIORank();
444 damarisOutputModule_->allocBuffers(num_interior, reportStepNum,
445 isSubStep, log,
false);
447 ElementContext elemCtx(simulator_);
448 OPM_BEGIN_PARALLEL_TRY_CATCH();
450 OPM_TIMEBLOCK(prepareCellBasedData);
451 for (
const auto& elem : elements(gridView, Dune::Partitions::interior)) {
452 elemCtx.updatePrimaryStencil(elem);
453 elemCtx.updatePrimaryIntensiveQuantities(0);
455 damarisOutputModule_->processElement(elemCtx);
458 if(!simulator_.
model().linearizer().getFlowsInfo().empty()){
459 OPM_TIMEBLOCK(prepareFlowsData);
460 for (
const auto& elem : elements(gridView, Dune::Partitions::interior)) {
461 elemCtx.updatePrimaryStencil(elem);
462 elemCtx.updatePrimaryIntensiveQuantities(0);
463 damarisOutputModule_->processElementFlows(elemCtx);
467 OPM_TIMEBLOCK(prepareBlockData);
468 for (
const auto& elem : elements(gridView, Dune::Partitions::interior)) {
469 elemCtx.updatePrimaryStencil(elem);
470 elemCtx.updatePrimaryIntensiveQuantities(0);
471 damarisOutputModule_->processElementBlockData(elemCtx);
475 OPM_TIMEBLOCK(prepareFluidInPlace);
477#pragma omp parallel for
479 for (
int dofIdx=0; dofIdx < num_interior; ++dofIdx){
480 const auto& intQuants = *(simulator_.
model().cachedIntensiveQuantities(dofIdx, 0));
481 const auto totVolume = simulator_.
model().dofTotalVolume(dofIdx);
482 damarisOutputModule_->updateFluidInPlace(dofIdx, intQuants, totVolume);
485 damarisOutputModule_->validateLocalData();
486 OPM_END_PARALLEL_TRY_CATCH(
"DamarisWriter::prepareLocalCellData() failed: ", simulator_.
vanguard().grid().comm());