From a9a3cee8392a2520951d1ca109b9a992911d2318 Mon Sep 17 00:00:00 2001 From: Hans Kallekleiv <16436291+HansKallekleiv@users.noreply.github.com> Date: Tue, 26 Oct 2021 22:20:48 +0200 Subject: [PATCH] Utility to export data to qc well log upscaling in Webviz --- setup.py | 1 + src/fmu/tools/rms/upscaling_qc/__init__.py | 0 src/fmu/tools/rms/upscaling_qc/_types.py | 73 ++++++++ .../tools/rms/upscaling_qc/upscaling_qc.py | 136 ++++++++++++++ tests/rms/test_upscaling_qc.py | 177 ++++++++++++++++++ 5 files changed, 387 insertions(+) create mode 100644 src/fmu/tools/rms/upscaling_qc/__init__.py create mode 100644 src/fmu/tools/rms/upscaling_qc/_types.py create mode 100644 src/fmu/tools/rms/upscaling_qc/upscaling_qc.py create mode 100644 tests/rms/test_upscaling_qc.py diff --git a/setup.py b/setup.py index cc3ff5d4..82988b13 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ def parse_requirements(filename): "pre-commit", "pytest", "pytest-cov", + "types-dataclasses", "types-PyYAML", ] diff --git a/src/fmu/tools/rms/upscaling_qc/__init__.py b/src/fmu/tools/rms/upscaling_qc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/fmu/tools/rms/upscaling_qc/_types.py b/src/fmu/tools/rms/upscaling_qc/_types.py new file mode 100644 index 00000000..fb2299a3 --- /dev/null +++ b/src/fmu/tools/rms/upscaling_qc/_types.py @@ -0,0 +1,73 @@ +from typing import List, Union, Dict +from enum import Enum +from dataclasses import dataclass, field + + +class UpscalingQCFiles(Enum): + WELLS = "well.csv" + BLOCKEDWELLS = "bw.csv" + GRID = "grid.csv" + METADATA = "metadata.json" + + def __fspath__(self): + return self.value + + +@dataclass +class WellSource: + names: List[str] = field(default_factory=list) + trajectory: str = "Drilled trajectory" + logrun: str = "log" + + +@dataclass +class BlockedWellSource: + grid: str + bwname: str + names: List[str] = field(default_factory=list) + + +@dataclass +class Context: + properties: Union[List[str], Dict[str, str]] + selectors: Union[List[str], Dict[str, str]] + + +@dataclass +class WellContext(Context): + wells: WellSource = WellSource() + + @classmethod + def from_dict(cls, data) -> "WellContext": + wells = data.pop("wells", {}) + return WellContext(wells=WellSource(**wells), **data) + + +@dataclass +class GridContext(Context): + grid: str + + @classmethod + def from_dict(cls, data) -> "GridContext": + return GridContext(**data) + + +@dataclass +class BlockedWellContext(Context): + wells: BlockedWellSource + + @classmethod + def from_dict(cls, data) -> "BlockedWellContext": + wells = data.pop("wells", {}) + return BlockedWellContext(wells=BlockedWellSource(**wells), **data) + + +@dataclass +class MetaData: + selectors: List[str] + properties: List[str] + well_names: List[str] + trajectory: str + logrun: str + grid_name: str + bw_name: str diff --git a/src/fmu/tools/rms/upscaling_qc/upscaling_qc.py b/src/fmu/tools/rms/upscaling_qc/upscaling_qc.py new file mode 100644 index 00000000..65878b56 --- /dev/null +++ b/src/fmu/tools/rms/upscaling_qc/upscaling_qc.py @@ -0,0 +1,136 @@ +from pathlib import Path +from typing import List, Union, Dict, Set +from dataclasses import asdict +import json + +import pandas as pd +from fmu.tools.qcproperties._grid2df import GridProps2df +from fmu.tools.qcproperties._well2df import WellLogs2df +from fmu.tools.qcdata import QCData + +from fmu.tools.rms.upscaling_qc._types import ( + WellContext, + GridContext, + BlockedWellContext, + UpscalingQCFiles, + MetaData, +) + + +class RMSUpscalingQC: + def __init__( + self, project, well_data: dict, grid_data: dict, bw_data: dict + ) -> None: + self._project = project + self._well_data = WellContext.from_dict(well_data) + self._grid_data = GridContext.from_dict(grid_data) + self._bw_data = BlockedWellContext.from_dict(bw_data) + self._validate_grid_name() + self._validate_properties() + self._validate_selectors() + self._set_well_names() + + def _set_well_names(self) -> None: + self._well_data.wells.names = self.well_names + self._bw_data.wells.names = self.well_names + + def _validate_grid_name(self) -> None: + if self._grid_data.grid != self._bw_data.wells.grid: + raise ValueError("Different grids given for blocked well and grid.") + + def _validate_properties(self) -> None: + if ( + not self._to_set(self._well_data.properties) + == self._to_set(self._bw_data.properties) + == self._to_set(self._grid_data.properties) + ): + raise ValueError("Data sources do not have the same properties!") + + def _validate_selectors(self) -> None: + if ( + not self._to_set(self._well_data.selectors) + == self._to_set(self._bw_data.selectors) + == self._to_set(self._grid_data.selectors) + ): + raise ValueError("Data sources do not have the same selectors!") + + @staticmethod + def _to_set(values: Union[List, Dict]) -> Set[str]: + if isinstance(values, list): + return set(values) + return set(list(values.keys())) + + @property + def _selectors(self) -> List[str]: + if isinstance(self._well_data.selectors, list): + return self._well_data.selectors + return list(self._well_data.selectors.keys()) + + @property + def _properties(self) -> List[str]: + if isinstance(self._well_data.properties, list): + return self._well_data.properties + return list(self._well_data.properties.keys()) + + @property + def _grid_name(self) -> str: + return self._grid_data.grid + + @property + def _metadata(self) -> MetaData: + return MetaData( + selectors=self._selectors, + properties=self._properties, + well_names=self.well_names, + trajectory=self._well_data.wells.trajectory, + logrun=self._well_data.wells.logrun, + grid_name=self._grid_data.grid, + bw_name=self._bw_data.wells.bwname, + ) + + @property + def well_names(self) -> List[str]: + try: + grid = self._project.grid_models[self._grid_data.grid] + return grid.blocked_wells_set[self._bw_data.wells.bwname].get_well_names() + except ValueError: + return [] + + def _get_well_data(self) -> pd.DataFrame: + _ = WellLogs2df( + project=self._project, data=asdict(self._well_data), xtgdata=QCData() + ) + return _.dataframe + + def _get_bw_data(self) -> pd.DataFrame: + _ = WellLogs2df( + project=self._project, + data=asdict(self._bw_data), + xtgdata=QCData(), + blockedwells=True, + ) + return _.dataframe + + def _get_grid_data(self) -> pd.DataFrame: + _ = GridProps2df( + project=self._project, data=asdict(self._grid_data), xtgdata=QCData() + ) + return _.dataframe + + def get_statistics(self) -> pd.DataFrame: + for _, df in self._get_well_data().groupby("ZONE"): + print(df) + + def to_disk(self, path: str = "../../share/results/tables/upscaling_qc") -> None: + folder = Path(path) + + if not folder.parent.is_dir(): + print(f"Cannot create folder. Ensure that {folder.parent} exists.") + folder.mkdir(exist_ok=True) + print("Extracting data...") + self._get_well_data().to_csv(folder / UpscalingQCFiles.WELLS, index=False) + self._get_bw_data().to_csv(folder / UpscalingQCFiles.BLOCKEDWELLS, index=False) + self._get_grid_data().to_csv(folder / UpscalingQCFiles.GRID, index=False) + with open(folder / UpscalingQCFiles.METADATA, "w") as fp: + json.dump(asdict(self._metadata), fp, indent=4) + print(f"Done. Output written to {folder}.") diff --git a/tests/rms/test_upscaling_qc.py b/tests/rms/test_upscaling_qc.py new file mode 100644 index 00000000..dafedac6 --- /dev/null +++ b/tests/rms/test_upscaling_qc.py @@ -0,0 +1,177 @@ +"""Run tests in RMS. + +Creates a tmp RMS project in given version which is used as fixture for all other Roxar +API dependent tests. + +This requires a ROXAPI license, and to be ran in a "roxenvbash" environment; hence +the decorator "roxapilicense" + +""" +from pathlib import Path +from os.path import isdir +import shutil +import json + +import numpy as np +import pytest + +import xtgeo + +try: + import roxar + import roxar.jobs + +except ImportError: + pass + +from fmu.tools.rms.upscaling_qc.upscaling_qc import RMSUpscalingQC +from fmu.tools.rms.upscaling_qc._types import UpscalingQCFiles, MetaData + +# ====================================================================================== +# settings to create RMS project! + +TMPD = Path("TMP") +TMPD.mkdir(parents=True, exist_ok=True) + +TPATH = Path("../xtgeo-testdata") +PROJNAME = "tmp_project_qcupscaling.rmsxxx" + +PRJ = str(TMPD / PROJNAME) + +GRIDDATA1 = TPATH / "3dgrids/reek/reek_sim_grid.roff" +PORODATA1 = TPATH / "3dgrids/reek/reek_sim_poro.roff" +ZONEDATA1 = TPATH / "3dgrids/reek/reek_sim_zone.roff" +GRIDNAME1 = "Simgrid" +PORONAME1 = "PORO" +ZONENAME1 = "Zone" + +WELLSFOLDER1 = TPATH / "wells/reek/1" +WELLS1 = ["OP_1.w", "OP_2.w", "OP_6.w"] + +BW_JOB_SPEC = { + "BlockedWellsName": "BW", + "Continuous Blocked Log": [ + { + "Name": "Poro", + } + ], + "Wells": [["Wells", "OP_1"], ["Wells", "OP_2"], ["Wells", "OP_6"]], + "Zone Blocked Log": [ + { + "Name": "Zonelog", + "ScaleUpType": "SUBGRID_BIAS", + "ThicknessWeighting": "MD_WEIGHT", + "ZoneLogArray": [1, 2], + } + ], +} + + +@pytest.mark.skipunlessroxar +@pytest.fixture(name="roxar_project") +def fixture_create_project(): + """Create a tmp RMS project for testing, populate with basic data. + + After the yield command, the teardown phase will remove the tmp RMS project. + """ + prj1 = str(PRJ) + + print("\n******** Setup RMS project!\n") + if isdir(prj1): + print("Remove existing project! (1)") + shutil.rmtree(prj1) + + project = roxar.Project.create() + + rox = xtgeo.RoxUtils(project) + print("Roxar version is", rox.roxversion) + print("RMS version is", rox.rmsversion(rox.roxversion)) + assert "1." in rox.roxversion + + for wfile in WELLS1: + wobj = xtgeo.well_from_file(WELLSFOLDER1 / wfile) + wobj.dataframe["Zonelog"] = 1 + + wobj.to_roxar(project, wobj.name, logrun="log", trajectory="Drilled trajectory") + + # populate with grid and props + grd = xtgeo.grid_from_file(GRIDDATA1) + grd.to_roxar(project, GRIDNAME1) + por = xtgeo.gridproperty_from_file(PORODATA1, name=PORONAME1) + por.to_roxar(project, GRIDNAME1, PORONAME1) + zon = xtgeo.gridproperty_from_file(ZONEDATA1, name=ZONENAME1) + zon.values = zon.values.astype(np.uint8) + zon.values = 1 + zon.to_roxar(project, GRIDNAME1, ZONENAME1) + + # Create blocked wells in grid + bw_job = roxar.jobs.Job.create( + owner=["Grid models", "Simgrid", "Grid"], type="Block Wells", name="BW" + ) + bw_job.set_arguments(BW_JOB_SPEC) + bw_job.save() + + bw_job.execute(0) + + # save project (both an initla version and a work version) and exit + project.save_as(prj1) + project.close() + + yield prj1 + + print("\n******* Teardown RMS project!\n") + + if isdir(prj1): + print("Remove existing project! (1)") + shutil.rmtree(prj1) + + +@pytest.mark.skipunlessroxar +def test_upscaling_qc(tmpdir, roxar_project): + """Test qcreset metod in roxapi.""" + # ================================================================================== + + rox = xtgeo.RoxUtils(roxar_project, readonly=True) + project = rox.project + + well_data = { + "selectors": { + "ZONE": { + "name": "Zonelog", + } + }, + "properties": {"PORO": {"name": "Poro"}}, + } + bw_data = { + "selectors": { + "ZONE": { + "name": "Zonelog", + } + }, + "properties": {"PORO": {"name": "Poro"}}, + "wells": {"grid": "Simgrid", "bwname": "BW"}, + } + grid_data = { + "selectors": {"ZONE": {"name": "Zone"}}, + "properties": ["PORO"], + "grid": "Simgrid", + } + ups = RMSUpscalingQC( + project=project, well_data=well_data, bw_data=bw_data, grid_data=grid_data + ) + well_df = ups._get_well_data() + bw_df = ups._get_bw_data() + grid_df = ups._get_grid_data() + for data in [well_df, bw_df, grid_df]: + assert set(data.columns) == set(["ZONE", "PORO"]) + assert well_df["PORO"].mean() == pytest.approx(0.1725, abs=0.0001) + assert bw_df["PORO"].mean() == pytest.approx(0.1765, abs=0.0001) + assert grid_df["PORO"].mean() == pytest.approx(0.1677, abs=0.0001) + ups.to_disk(path=str(tmpdir / "upscalingqc")) + files = Path(tmpdir / "upscalingqc").glob("**/*") + assert set([fn.name for fn in files]) == set( + [item.value for item in UpscalingQCFiles] + ) + ups.get_statistics() + with open(Path(tmpdir / "upscalingqc" / UpscalingQCFiles.METADATA), "r") as fp: + assert MetaData(**json.load(fp)) == ups._metadata