Clean CATS & SNN (#320)

* new CATS Model (#317)

* New CBPF Classifier

* Making PEP8 compatible

* More changes to be PEP8 compatible

* Fixing load model to make compatible with PEP8 rules

* updated doc for predict_nn in CATS

* fix typo o predict_nn docs

* Chaging number of 0 class preds from 52 to 56

* New CATS model + fix on processor.py

* Issue/302/snn (#315)

* updated models snn

* models qith correct training log

* Fix test

* Test against last SNN version

* Fix typo

* 5 classes

---------

Co-authored-by: JulienPeloton <[email protected]>

* Update test with new SNN broad model

* Bump to 4.3.0

---------

Co-authored-by: André Santos <[email protected]>
Co-authored-by: Anais Möller <[email protected]>

fink_science/__init__.py

@@ -12,4 +12,4 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-__version__ = "4.2.0"
+__version__ = "4.3.0"

fink_science/cats/processor.py

@@ -115,7 +115,7 @@ def predict_nn(
     >>> df = df.withColumn('preds', predict_nn(*args))
     >>> df = df.withColumn('argmax', F.expr('array_position(preds, array_max(preds)) - 1'))
     >>> df.filter(df['argmax'] == 0).count()
-    56
+    55
     """
 
     filter_dict = {'u': 1, 'g': 2, 'r': 3, 'i': 4, 'z': 5, 'Y': 6}
@@ -142,7 +142,6 @@ def predict_nn(
                              z_final_err.values[i]])
 
     flux = psFlux.apply(lambda x: norm_column(x))
-    mjd = midpointTai.apply(lambda x: norm_column(x))
     error = psFluxErr.apply(lambda x: norm_column(x))
 
     flux = keras.utils.pad_sequences(flux,
@@ -181,7 +180,7 @@ def predict_nn(
     if model is None:
         # Load pre-trained model
         curdir = os.path.dirname(os.path.abspath(__file__))
-        model_path = curdir + '/data/models/cats_models/model_cut_meta'
+        model_path = curdir + '/data/models/cats_models/model_meta_0'
     else:
         model_path = model.values[0]
 

fink_science/data/models/snn_models/elasticc_broad/data_norm.json

@@ -1,67 +1,67 @@
 {
     "FLUXCALERR_Y": {
-        "mean": -6.856283187866211,
+        "mean": -6.074979305267334,
         "min": 0.0,
-        "std": 7.766845703125
+        "std": 8.142868995666504
     },
     "FLUXCALERR_g": {
-        "mean": -6.856283187866211,
+        "mean": -6.074979305267334,
         "min": 0.0,
-        "std": 7.766845703125
+        "std": 8.142868995666504
     },
     "FLUXCALERR_i": {
-        "mean": -6.856283187866211,
+        "mean": -6.074979305267334,
         "min": 0.0,
-        "std": 7.766845703125
+        "std": 8.142868995666504
     },
     "FLUXCALERR_r": {
-        "mean": -6.856283187866211,
+        "mean": -6.074979305267334,
         "min": 0.0,
-        "std": 7.766845703125
+        "std": 8.142868995666504
     },
     "FLUXCALERR_u": {
-        "mean": -6.856283187866211,
+        "mean": -6.074979305267334,
         "min": 0.0,
-        "std": 7.766845703125
+        "std": 8.142868995666504
     },
     "FLUXCALERR_z": {
-        "mean": -6.856283187866211,
+        "mean": -6.074979305267334,
         "min": 0.0,
-        "std": 7.766845703125
+        "std": 8.142868995666504
     },
     "FLUXCAL_Y": {
-        "mean": 15.653460502624512,
+        "mean": 15.682472229003906,
         "min": -2000.0,
-        "std": 0.006687534041702747
+        "std": 0.005008767358958721
     },
     "FLUXCAL_g": {
-        "mean": 15.653460502624512,
+        "mean": 15.682472229003906,
         "min": -2000.0,
-        "std": 0.006687534041702747
+        "std": 0.005008767358958721
     },
     "FLUXCAL_i": {
-        "mean": 15.653460502624512,
+        "mean": 15.682472229003906,
         "min": -2000.0,
-        "std": 0.006687534041702747
+        "std": 0.005008767358958721
     },
     "FLUXCAL_r": {
-        "mean": 15.653460502624512,
+        "mean": 15.682472229003906,
         "min": -2000.0,
-        "std": 0.006687534041702747
+        "std": 0.005008767358958721
     },
     "FLUXCAL_u": {
-        "mean": 15.653460502624512,
+        "mean": 15.682472229003906,
         "min": -2000.0,
-        "std": 0.006687534041702747
+        "std": 0.005008767358958721
     },
     "FLUXCAL_z": {
-        "mean": 15.653460502624512,
+        "mean": 15.682472229003906,
         "min": -2000.0,
-        "std": 0.006687534041702747
+        "std": 0.005008767358958721
     },
     "delta_time": {
-        "mean": -0.0453844778239727,
+        "mean": -0.0457201711833477,
         "min": 0.0,
-        "std": 4.170915126800537
+        "std": 3.821138620376587
     }
 }

fink_science/data/models/snn_models/elasticc_broad/training_log.json

@@ -1 +1 @@
-{"vanilla_S_0_CLF_5_R_zspe_photometry_DF_1.0_N_cosmo_quantile_lstm_32x2_0.05_128_True_mean_C": {"training_time": 74.79755640029907, "AUC": -1, "Acc": 0.8714918759231906, "loss": 0.2891275076622856, "data_types_training": "['0 Ia [10, 11, 12, 20, 21, 25, 26, 27, 30, 31, 32, 35, 36, 37]'\n '1 Long [40, 42, 45, 46, 59]' '2 Fast [51, 82, 84, 87, 88, 89]'\n '3 NonPeriodic [60]' '4 Periodic [80, 83, 90, 91]']"}}
+{"vanilla_S_0_CLF_5_R_zspe_photometry_DF_1.0_N_cosmo_quantile_lstm_32x2_0.05_128_True_mean_C": {"training_time": 166.83654022216797, "AUC": -1, "Acc": 0.8907657657657657, "loss": 0.29699913359411517, "data_types_training": "['0 Ia [10, 11, 12, 20, 21, 25, 26, 27, 30, 31, 32, 35, 36, 37]'\n '1 Long [40, 42, 45, 46, 59]' '2 Fast [50, 51, 82, 84, 87, 88, 89]'\n '3 NonPeriodic [60]' '4 Periodic [80, 83, 90, 91]']"}}

fink_science/snn/processor.py

@@ -390,7 +390,7 @@ def snn_broad_elasticc(
     >>> df = df.withColumn('snn_max_prob', F.col('preds').getItem(1))
 
     >>> df.filter(df['snn_class'] == 0).count()
-    22
+    11
     """
     # No a priori cuts
     mask = np.ones(len(diaSourceId), dtype=bool)