diff --git a/examples/simple/time_series_forecasting/api_forecasting.py b/examples/simple/time_series_forecasting/api_forecasting.py index aeeb09edb2..dbb2c52b5d 100644 --- a/examples/simple/time_series_forecasting/api_forecasting.py +++ b/examples/simple/time_series_forecasting/api_forecasting.py @@ -41,6 +41,7 @@ def run_ts_forecasting_example(dataset='australia', horizon: int = 30, timeout: TsForecastingParams(forecast_length=horizon)).task_params, timeout=timeout, n_jobs=-1, + metric='mase', with_tuning=with_tuning, cv_folds=2, preset='fast_train') @@ -50,7 +51,7 @@ def run_ts_forecasting_example(dataset='australia', horizon: int = 30, timeout: # use model to obtain two-step in-sample forecast in_sample_forecast = model.predict(test_data, validation_blocks=validation_blocks) print('Metrics for two-step in-sample forecast: ', - model.get_metrics(metric_names=['rmse', 'mae', 'mape'], + model.get_metrics(metric_names=['mase', 'mae', 'mape'], validation_blocks=validation_blocks)) # plot forecasting result @@ -68,7 +69,7 @@ def run_ts_forecasting_example(dataset='australia', horizon: int = 30, timeout: model.plot_prediction() # use model to obtain two-step out-of-sample forecast - out_of_sample_forecast = model.forecast(test_data, horizon=20) + out_of_sample_forecast = model.forecast(test_data, horizon=horizon * 2) # we can not calculate metrics because we do not have enough future values if visualization: model.plot_prediction() @@ -77,4 +78,4 @@ def run_ts_forecasting_example(dataset='australia', horizon: int = 30, timeout: if __name__ == '__main__': - run_ts_forecasting_example(dataset='salaries', horizon=10, timeout=10., visualization=True) + run_ts_forecasting_example(dataset='beer', horizon=14, timeout=2., visualization=True) diff --git a/fedot/api/api_utils/metrics.py b/fedot/api/api_utils/metrics.py index a0e83283b6..47b1d3af01 100644 --- a/fedot/api/api_utils/metrics.py +++ b/fedot/api/api_utils/metrics.py @@ -5,7 +5,7 @@ from fedot.core.composer.metrics import Metric from fedot.core.repository.quality_metrics_repository import (ClassificationMetricsEnum, ClusteringMetricsEnum, ComplexityMetricsEnum, RegressionMetricsEnum, MetricType, - MetricsEnum) + MetricsEnum, TimeSeriesForecastingMetricsEnum) from fedot.core.repository.tasks import Task from fedot.utilities.define_metric_by_task import MetricByTask @@ -25,6 +25,7 @@ class ApiMetrics: 'mse': RegressionMetricsEnum.MSE, 'msle': RegressionMetricsEnum.MSLE, 'mape': RegressionMetricsEnum.MAPE, + 'mase': TimeSeriesForecastingMetricsEnum.MASE, 'smape': RegressionMetricsEnum.SMAPE, 'r2': RegressionMetricsEnum.R2, 'rmse': RegressionMetricsEnum.RMSE, diff --git a/fedot/api/main.py b/fedot/api/main.py index 52aadb7726..05ef793e65 100644 --- a/fedot/api/main.py +++ b/fedot/api/main.py @@ -95,9 +95,11 @@ class Fedot: - classification -> \ :class:`~fedot.core.repository.quality_metrics_repository.ClassificationMetricsEnum` - - regression & time series forcasting -> \ + - regression -> \ :class:`~fedot.core.repository.quality_metrics_repository.RegressionMetricsEnum` - - pipeline complexity (task-independent)-> \ + - time series forcasting -> \ + :class:`~fedot.core.repository.quality_metrics_repository.TimeSeriesForecastingMetricsEnum` + - pipeline complexity (task-independent) -> \ :class:`~fedot.core.repository.quality_metrics_repository.ComplexityMetricsEnum` collect_intermediate_metric (bool): save metrics for intermediate (non-root) nodes in composed diff --git a/fedot/core/composer/metrics.py b/fedot/core/composer/metrics.py index d3d9e42c3e..22f5196daa 100644 --- a/fedot/core/composer/metrics.py +++ b/fedot/core/composer/metrics.py @@ -8,6 +8,7 @@ from sklearn.metrics import (accuracy_score, auc, f1_score, log_loss, mean_absolute_error, mean_absolute_percentage_error, mean_squared_error, mean_squared_log_error, precision_score, r2_score, roc_auc_score, roc_curve, silhouette_score) +from sktime.performance_metrics.forecasting import mean_absolute_scaled_error from fedot.core.data.data import InputData, OutputData from fedot.core.pipelines.pipeline import Pipeline @@ -232,6 +233,15 @@ def metric(reference: InputData, predicted: OutputData) -> float: return mean_absolute_error(y_true=reference.target, y_pred=predicted.predict) +class MASE(QualityMetric): + default_value = sys.maxsize + + @staticmethod + def metric(reference: InputData, predicted: OutputData) -> float: + history_series = reference.features + return mean_absolute_scaled_error(y_true=reference.target, y_pred=predicted.predict, y_train=history_series) + + class R2(QualityMetric): default_value = 0 diff --git a/fedot/core/repository/quality_metrics_repository.py b/fedot/core/repository/quality_metrics_repository.py index 66125fdd10..b6ce6cf857 100644 --- a/fedot/core/repository/quality_metrics_repository.py +++ b/fedot/core/repository/quality_metrics_repository.py @@ -6,7 +6,7 @@ from fedot.core.composer.metrics import (ComputationTime, Accuracy, F1, Logloss, MAE, MAPE, SMAPE, MSE, MSLE, Metric, NodeNum, Precision, R2, - RMSE, ROCAUC, Silhouette, StructuralComplexity) + RMSE, ROCAUC, Silhouette, StructuralComplexity, MASE) class MetricsEnum(Enum): @@ -53,6 +53,18 @@ class RegressionMetricsEnum(QualityMetricsEnum): RMSE_penalty = 'rmse_pen' +class TimeSeriesForecastingMetricsEnum(QualityMetricsEnum): + MASE = 'mase' + RMSE = 'rmse' + MSE = 'mse' + MSLE = 'neg_mean_squared_log_error' + MAPE = 'mape' + SMAPE = 'smape' + MAE = 'mae' + R2 = 'r2' + RMSE_penalty = 'rmse_pen' + + class MetricsRepository: _metrics_implementations = { # classification @@ -73,6 +85,9 @@ class MetricsRepository: RegressionMetricsEnum.RMSE_penalty: RMSE.get_value_with_penalty, RegressionMetricsEnum.R2: R2.get_value, + # ts forecasting + TimeSeriesForecastingMetricsEnum.MASE: MASE.get_value, + # clustering ClusteringMetricsEnum.silhouette: Silhouette.get_value, diff --git a/fedot/utilities/define_metric_by_task.py b/fedot/utilities/define_metric_by_task.py index 3c5b3779e2..c168ad1605 100644 --- a/fedot/utilities/define_metric_by_task.py +++ b/fedot/utilities/define_metric_by_task.py @@ -1,21 +1,21 @@ from typing import List from fedot.core.data.data import InputData, OutputData -from fedot.core.repository.tasks import TaskTypesEnum from fedot.core.repository.quality_metrics_repository import ( MetricsEnum, RegressionMetricsEnum, ClassificationMetricsEnum, ClusteringMetricsEnum, - MetricsRepository + MetricsRepository, TimeSeriesForecastingMetricsEnum ) +from fedot.core.repository.tasks import TaskTypesEnum class MetricByTask: __metric_by_task = {TaskTypesEnum.regression: RegressionMetricsEnum.RMSE, TaskTypesEnum.classification: ClassificationMetricsEnum.ROCAUC_penalty, TaskTypesEnum.clustering: ClusteringMetricsEnum.silhouette, - TaskTypesEnum.ts_forecasting: RegressionMetricsEnum.RMSE, + TaskTypesEnum.ts_forecasting: TimeSeriesForecastingMetricsEnum.RMSE, } @staticmethod @@ -23,7 +23,8 @@ def get_default_quality_metrics(task_type: TaskTypesEnum) -> List[MetricsEnum]: return [MetricByTask.__metric_by_task.get(task_type)] @staticmethod - def compute_default_metric(task_type: TaskTypesEnum, true: InputData, predicted: OutputData, round_up_to: int = 6) -> float: + def compute_default_metric(task_type: TaskTypesEnum, true: InputData, predicted: OutputData, + round_up_to: int = 6) -> float: """Returns the value of metric defined by task""" metric_id = MetricByTask.get_default_quality_metrics(task_type)[0] metric = MetricsRepository.metric_class_by_id(metric_id) diff --git a/requirements.txt b/requirements.txt index ada1d0e987..7dc30e2982 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ statsmodels>=0.12.0 ete3>=3.1.0 networkx>=2.4, !=2.7.*, !=2.8.1, !=2.8.2, !=2.8.3 scikit_learn>=1.0.0; python_version >= '3.8' +sktime==0.16.1 # Analysis and optimizations hyperopt==0.2.7