added test for combined type plus property model. made adjustments to…

… pass nsrdb tests.

mlclouds/__init__.py

@@ -7,19 +7,18 @@
 MLCLOUDSDIR = os.path.dirname(os.path.realpath(__file__))
 TESTDATADIR = os.path.join(os.path.dirname(MLCLOUDSDIR), 'tests', 'data')
 
+LEG_DIR = os.path.join(MLCLOUDSDIR, 'model/legacy')
 CTYPE_DIR = os.path.join(MLCLOUDSDIR, 'model/cloud_type')
-CTYPE_CONFIG_FPATH = os.path.join(CTYPE_DIR, 'config.json')
-CTYPE_MODEL_FPATH = os.path.join(CTYPE_DIR, 'outputs/mlclouds_model.pkl')
-
 CPROP_DIR = os.path.join(MLCLOUDSDIR, 'model/cloud_properties')
-CPROP_CONFIG_FPATH = os.path.join(CPROP_DIR, 'config.json')
+COMBINED_DIR = os.path.join(MLCLOUDSDIR, 'model/type_plus_properties')
+CONFIG_FPATH = os.path.join(LEG_DIR, 'config.json')
+
 CPROP_MODEL_FPATH = os.path.join(CPROP_DIR, 'outputs/mlclouds_model.pkl')
+CTYPE_MODEL_FPATH = os.path.join(CTYPE_DIR, 'outputs/mlclouds_model.pkl')
+COMBINED_MODEL_FPATH = os.path.join(COMBINED_DIR, 'outputs/mlclouds_model.pkl')
+LEG_MODEL_FPATH = os.path.join(LEG_DIR, 'outputs/mlclouds_model.pkl')
 
 MODEL_FPATH = {
     'cloud_type_model_path': CTYPE_MODEL_FPATH,
     'cloud_prop_model_path': CPROP_MODEL_FPATH,
 }
-
-LEG_DIR = os.path.join(MLCLOUDSDIR, 'model/legacy')
-LEG_CONFIG_FPATH = os.path.join(LEG_DIR, 'config.json')
-LEG_MODEL_FPATH = os.path.join(LEG_DIR, 'outputs/mlclouds_model.pkl')

mlclouds/model/multi_step.py

@@ -1,5 +1,7 @@
 """Wrapped phygnn model with methods to produce needed outputs for NSRDB."""
 
+import numpy as np
+
 from mlclouds.p_fun import decode_sky_type, encode_sky_type
 
 from .base import MLCloudsModel
@@ -58,11 +60,15 @@ def predict(self, features, **kwargs):
         """
         if self.cloud_type_model is not None:
             out = self.cloud_type_model.predict(features, **kwargs)
-            features['flag'] = decode_sky_type(out['cloud_type'])
+            features['flag'] = decode_sky_type(out)
         out = self.model.predict(features, **kwargs)
 
         if 'flag' in features:
-            out['cloud_type'] = encode_sky_type(features['flag'])
+            ctype = encode_sky_type(features)
+            if hasattr(out, 'columns'):
+                out['cloud_type'] = ctype
+            else:
+                out = np.concatenate([out, ctype[:, None]], axis=-1)
         return out
 
     @property

mlclouds/p_fun.py

@@ -7,6 +7,7 @@
 import logging
 
 import numpy as np
+import pandas as pd
 import tensorflow as tf
 
 from mlclouds.tdhi import t_calc_dhi
@@ -33,18 +34,26 @@ def _get_sky_type(clear_fraction, ice_fraction, water_fraction):
 
 def decode_sky_type(cloud_type):
     """Decode integer cloud types as strings."""
-    cloud_type[cloud_type == 0] = 'clear'
-    cloud_type[cloud_type == 3] = 'water_cloud'
-    cloud_type[cloud_type == 6] = 'ice_cloud'
-    return cloud_type
+    if 'cloud_type' in cloud_type:
+        df = cloud_type.copy()
+    else:
+        df = pd.DataFrame(columns=['flag'], data=cloud_type)
+    df.loc[df['flag'] == 0, 'flag'] = 'clear'
+    df.loc[df['flag'] == 3, 'flag'] = 'water_cloud'
+    df.loc[df['flag'] == 6, 'flag'] = 'ice_cloud'
+    return df['flag'].values
 
 
 def encode_sky_type(cloud_type):
     """Encode string cloud types as integers."""
-    cloud_type[cloud_type == 'clear'] = 0
-    cloud_type[cloud_type == 'water_cloud'] = 3
-    cloud_type[cloud_type == 'ice_cloud'] = 6
-    return cloud_type
+    if 'flag' in cloud_type:
+        df = cloud_type.copy()
+    else:
+        df = pd.DataFrame(columns=['flag'], data=cloud_type)
+    df.loc[df['flag'] == 'clear', 'flag'] = 0
+    df.loc[df['flag'] == 'water_cloud', 'flag'] = 3
+    df.loc[df['flag'] == 'ice_cloud', 'flag'] = 6
+    return df['flag'].values.astype(int)
 
 
 def get_sky_type(model, y_true, y_predicted, p, labels=None):

mlclouds/utilities.py

@@ -8,7 +8,7 @@
 
 import pandas as pd
 
-from mlclouds import LEG_CONFIG_FPATH
+from mlclouds import CONFIG_FPATH
 from mlclouds.p_fun import p_fun_all_sky, p_fun_dummy
 
 logger = logging.getLogger(__name__)
@@ -60,7 +60,7 @@
 
 
 # Phygnn model configuration
-with open(LEG_CONFIG_FPATH) as f:
+with open(CONFIG_FPATH) as f:
     CONFIG = json.load(f)
 
 

tests/test_combined_model.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+"""
+Test loading and running of the saved production mlclouds model
+"""
+
+import numpy as np
+import pandas as pd
+
+from mlclouds import COMBINED_MODEL_FPATH
+from mlclouds.model.base import MLCloudsModel
+
+
+def test_load_and_run():
+    """Test that the combined cloud type plus cloud properties mlclouds model
+    loads and runs a dummy prediction"""
+
+    defaults = {
+        'solar_zenith_angle': 10,
+        'refl_0_65um_nom': 90,
+        'refl_3_75um_nom': 90,
+        'temp_3_75um_nom': 300,
+        'temp_11_0um_nom': 300,
+        'air_temperature': 10,
+        'dew_point': 10,
+        'cld_press_acha': 800,
+        'relative_humidity': 80,
+        'total_precipitable_water': 5,
+        'surface_albedo': 0.1,
+    }
+
+    model = MLCloudsModel.load(COMBINED_MODEL_FPATH)
+
+    assert any(fn in model.feature_names for fn in defaults)
+    label_names = (
+        'cld_opd_dcomp',
+        'cld_reff_dcomp',
+        'clear_fraction',
+        'ice_fraction',
+        'water_fraction',
+    )
+    output_names = ('cld_opd_dcomp', 'cld_reff_dcomp', 'cloud_type')
+    assert all(ln in model.label_names for ln in label_names)
+    assert all(on in model.output_names for on in output_names)
+
+    assert len(model.history) >= 190
+    assert model.history['training_loss'].values[-1] < 1
+
+    missing = [fn for fn in model.feature_names if fn not in defaults]
+    if any(missing):
+        msg = 'Need to update test with default feature inputs for {}.'.format(
+            missing
+        )
+        raise KeyError(msg)
+
+    features = {fn: defaults[fn] * np.ones(10) for fn in model.feature_names}
+    features = pd.DataFrame(features)
+
+    out = model.predict(features, table=True)
+
+    assert (out['cld_opd_dcomp'] > 0).all()
+    assert (out['cld_opd_dcomp'] < 100).all()
+
+    assert (out['cld_reff_dcomp'] > 0).all()
+    assert (out['cld_reff_dcomp'] < 20).all()
+
+    assert out['cloud_type'].isin([0, 3, 6]).all()

tests/test_prod_model.py → tests/test_multistep_model.py

@@ -11,8 +11,8 @@
 
 
 def test_multistep_load_and_run():
-    """Test that the multistep model with cloud type model and cloud property
-    model loads and runs a dummy prediction"""
+    """Test that the multistep cloud type prediction followed by cloud property
+    prediction model loads and runs a dummy prediction"""
 
     defaults = {
         'solar_zenith_angle': 10,
@@ -28,17 +28,20 @@ def test_multistep_load_and_run():
     }
 
     model = MultiCloudsModel.load(CPROP_MODEL_FPATH, CTYPE_MODEL_FPATH)
-
     assert any(fn in model.feature_names for fn in defaults)
-    assert 'cld_opd_dcomp' in model.label_names
-    assert 'cld_reff_dcomp' in model.label_names
-    assert 'clear_fraction' in model.cloud_type_model.label_names
-    assert 'ice_fraction' in model.cloud_type_model.label_names
-    assert 'water_fraction' in model.cloud_type_model.label_names
-
-    assert 'cld_opd_dcomp' in model.output_names
-    assert 'cld_reff_dcomp' in model.output_names
-    assert 'cloud_type' in model.output_names
+    cprops = (
+        'cld_opd_dcomp',
+        'cld_reff_dcomp',
+    )
+    ctypes = (
+        'clear_fraction',
+        'ice_fraction',
+        'water_fraction',
+    )
+    output_names = ('cld_opd_dcomp', 'cld_reff_dcomp', 'cloud_type')
+    assert all(cp in model.label_names for cp in cprops)
+    assert all(ct in model.cloud_type_model.label_names for ct in ctypes)
+    assert all(on in model.output_names for on in output_names)
 
     assert len(model.history) >= 190
     assert model.history['training_loss'].values[-1] < 1