Add MASE metric

examples/simple/time_series_forecasting/api_forecasting.py

@@ -41,6 +41,7 @@ def run_ts_forecasting_example(dataset='australia', horizon: int = 30, timeout:
                                    TsForecastingParams(forecast_length=horizon)).task_params,
                   timeout=timeout,
                   n_jobs=-1,
+                  metric='mase',
                   with_tuning=with_tuning,
                   cv_folds=2, preset='fast_train')
 
@@ -50,7 +51,7 @@ def run_ts_forecasting_example(dataset='australia', horizon: int = 30, timeout:
     # use model to obtain two-step in-sample forecast
     in_sample_forecast = model.predict(test_data, validation_blocks=validation_blocks)
     print('Metrics for two-step in-sample forecast: ',
-          model.get_metrics(metric_names=['rmse', 'mae', 'mape'],
+          model.get_metrics(metric_names=['mase', 'mae', 'mape'],
                             validation_blocks=validation_blocks))
 
     # plot forecasting result
@@ -68,7 +69,7 @@ def run_ts_forecasting_example(dataset='australia', horizon: int = 30, timeout:
         model.plot_prediction()
 
     # use model to obtain two-step out-of-sample forecast
-    out_of_sample_forecast = model.forecast(test_data, horizon=20)
+    out_of_sample_forecast = model.forecast(test_data, horizon=horizon * 2)
     # we can not calculate metrics because we do not have enough future values
     if visualization:
         model.plot_prediction()
@@ -77,4 +78,4 @@ def run_ts_forecasting_example(dataset='australia', horizon: int = 30, timeout:
 
 
 if __name__ == '__main__':
-    run_ts_forecasting_example(dataset='salaries', horizon=10, timeout=10., visualization=True)
+    run_ts_forecasting_example(dataset='beer', horizon=14, timeout=2., visualization=True)

fedot/api/api_utils/metrics.py

@@ -5,7 +5,7 @@
 from fedot.core.composer.metrics import Metric
 from fedot.core.repository.quality_metrics_repository import (ClassificationMetricsEnum, ClusteringMetricsEnum,
                                                               ComplexityMetricsEnum, RegressionMetricsEnum, MetricType,
-                                                              MetricsEnum)
+                                                              MetricsEnum, TimeSeriesForecastingMetricsEnum)
 from fedot.core.repository.tasks import Task
 from fedot.utilities.define_metric_by_task import MetricByTask
 
@@ -25,6 +25,7 @@ class ApiMetrics:
         'mse': RegressionMetricsEnum.MSE,
         'msle': RegressionMetricsEnum.MSLE,
         'mape': RegressionMetricsEnum.MAPE,
+        'mase': TimeSeriesForecastingMetricsEnum.MASE,
         'smape': RegressionMetricsEnum.SMAPE,
         'r2': RegressionMetricsEnum.R2,
         'rmse': RegressionMetricsEnum.RMSE,

fedot/core/composer/metrics.py

@@ -8,6 +8,7 @@
 from sklearn.metrics import (accuracy_score, auc, f1_score, log_loss, mean_absolute_error,
                              mean_absolute_percentage_error, mean_squared_error, mean_squared_log_error,
                              precision_score, r2_score, roc_auc_score, roc_curve, silhouette_score)
+from sktime.performance_metrics.forecasting import mean_absolute_scaled_error
 
 from fedot.core.data.data import InputData, OutputData
 from fedot.core.pipelines.pipeline import Pipeline
@@ -232,6 +233,15 @@ def metric(reference: InputData, predicted: OutputData) -> float:
         return mean_absolute_error(y_true=reference.target, y_pred=predicted.predict)
 
 
+class MASE(QualityMetric):
+    default_value = sys.maxsize
+
+    @staticmethod
+    def metric(reference: InputData, predicted: OutputData) -> float:
+        history_series = reference.features
+        return mean_absolute_scaled_error(y_true=reference.target, y_pred=predicted.predict, y_train=history_series)
+
+
 class R2(QualityMetric):
     default_value = 0
 

fedot/core/repository/quality_metrics_repository.py

@@ -6,7 +6,7 @@
 
 from fedot.core.composer.metrics import (ComputationTime, Accuracy, F1, Logloss, MAE,
                                          MAPE, SMAPE, MSE, MSLE, Metric, NodeNum, Precision, R2,
-                                         RMSE, ROCAUC, Silhouette, StructuralComplexity)
+                                         RMSE, ROCAUC, Silhouette, StructuralComplexity, MASE)
 
 
 class MetricsEnum(Enum):
@@ -53,6 +53,18 @@ class RegressionMetricsEnum(QualityMetricsEnum):
     RMSE_penalty = 'rmse_pen'
 
 
+class TimeSeriesForecastingMetricsEnum(QualityMetricsEnum):
+    MASE = 'mase'
+    RMSE = 'rmse'
+    MSE = 'mse'
+    MSLE = 'neg_mean_squared_log_error'
+    MAPE = 'mape'
+    SMAPE = 'smape'
+    MAE = 'mae'
+    R2 = 'r2'
+    RMSE_penalty = 'rmse_pen'
+
+
 class MetricsRepository:
     _metrics_implementations = {
         # classification
@@ -73,6 +85,9 @@ class MetricsRepository:
         RegressionMetricsEnum.RMSE_penalty: RMSE.get_value_with_penalty,
         RegressionMetricsEnum.R2: R2.get_value,
 
+        # ts forecasting
+        TimeSeriesForecastingMetricsEnum.MASE: MASE.get_value,
+
         # clustering
         ClusteringMetricsEnum.silhouette: Silhouette.get_value,
 

fedot/utilities/define_metric_by_task.py

@@ -1,21 +1,21 @@
 from typing import List
 
 from fedot.core.data.data import InputData, OutputData
-from fedot.core.repository.tasks import TaskTypesEnum
 from fedot.core.repository.quality_metrics_repository import (
     MetricsEnum,
     RegressionMetricsEnum,
     ClassificationMetricsEnum,
     ClusteringMetricsEnum,
-    MetricsRepository
+    MetricsRepository, TimeSeriesForecastingMetricsEnum
 )
+from fedot.core.repository.tasks import TaskTypesEnum
 
 
 class MetricByTask:
     __metric_by_task = {TaskTypesEnum.regression: RegressionMetricsEnum.RMSE,
                         TaskTypesEnum.classification: ClassificationMetricsEnum.ROCAUC_penalty,
                         TaskTypesEnum.clustering: ClusteringMetricsEnum.silhouette,
-                        TaskTypesEnum.ts_forecasting: RegressionMetricsEnum.RMSE,
+                        TaskTypesEnum.ts_forecasting: TimeSeriesForecastingMetricsEnum.RMSE,
                         }
 
     @staticmethod

requirements.txt

@@ -14,6 +14,7 @@ statsmodels>=0.12.0
 ete3>=3.1.0
 networkx>=2.4, !=2.7.*, !=2.8.1, !=2.8.2, !=2.8.3
 scikit_learn>=1.0.0; python_version >= '3.8'
+sktime
 
 # Analysis and optimizations
 hyperopt==0.2.7