diff --git a/cases/credit_scoring/credit_scoring_problem.py b/cases/credit_scoring/credit_scoring_problem.py index 028029bd72..21111e3059 100644 --- a/cases/credit_scoring/credit_scoring_problem.py +++ b/cases/credit_scoring/credit_scoring_problem.py @@ -1,6 +1,7 @@ import logging import os import random +from pathlib import Path import numpy as np from sklearn.metrics import roc_auc_score as roc_auc @@ -39,7 +40,8 @@ def run_credit_scoring_problem(train_file_path, test_file_path, metrics = automl.get_metrics() if automl.history: - print(automl.history.get_leaderboard()) + lb = automl.history.get_leaderboard() + Path(os.path.join('D:/', "leaderboard.csv")).write_text(lb) if visualization: automl.current_pipeline.show() @@ -68,6 +70,6 @@ def get_scoring_data(): full_path_train, full_path_test = get_scoring_data() run_credit_scoring_problem(full_path_train, full_path_test, - timeout=5, + timeout=2, visualization=True, with_tuning=True) diff --git a/cases/metocean_forecasting_problem.py b/cases/metocean_forecasting_problem.py index b1832d489c..f2e768d8db 100644 --- a/cases/metocean_forecasting_problem.py +++ b/cases/metocean_forecasting_problem.py @@ -45,6 +45,7 @@ def run_metocean_forecasting_problem(train_file_path, test_file_path, fedot = Fedot(problem='ts_forecasting', task_params=TsForecastingParams(forecast_length=forecast_length), timeout=timeout, logging_level=logging.DEBUG) + fedot.current_pipeline pipeline = fedot.fit(features=historical_data, target=ssh_history) fedot.forecast(historical_data) diff --git a/docs/source/benchmarks/amlb_res.csv b/docs/source/benchmarks/amlb_res.csv index 07fb820c62..569f98431d 100644 --- a/docs/source/benchmarks/amlb_res.csv +++ b/docs/source/benchmarks/amlb_res.csv @@ -1,32 +1,32 @@ Dataset name,Metric name,AutoGluon,FEDOT,H2O,LAMA -APSFailure,auc,0.99,0.991,,0.992 -Amazon_employee_access,auc,0.857,0.865,,0.879 +APSFailure,auc,0.99,0.991,0.992,0.992 +Amazon_employee_access,auc,0.857,0.865,0.873,0.879 Australian,auc,0.94,0.939,0.939,0.945 -Covertype,neg_logloss,-0.071,-0.117,, -Fashion-MNIST,neg_logloss,-0.329,-0.373,,-0.248 -Jannis,neg_logloss,-0.728,-0.737,,-0.664 -KDDCup09_appetency,auc,0.804,0.822,,0.85 +Covertype,neg_logloss,-0.071,-0.117,-0.265, +Fashion-MNIST,neg_logloss,-0.329,-0.373,-0.38,-0.248 +Jannis,neg_logloss,-0.728,-0.737,-0.691,-0.664 +KDDCup09_appetency,auc,0.804,0.822,0.829,0.85 MiniBooNE,auc,0.982,0.981,,0.988 -Shuttle,neg_logloss,-0.001,-0.001,,-0.001 -Volkert,neg_logloss,-0.917,-1.097,,-0.806 -adult,auc,0.91,0.925,,0.932 -bank-marketing,auc,0.931,0.935,,0.94 +Shuttle,neg_logloss,-0.001,-0.001,-0.0,-0.001 +Volkert,neg_logloss,-0.917,-1.097,-0.976,-0.806 +adult,auc,0.91,0.925,0.931,0.932 +bank-marketing,auc,0.931,0.935,0.939,0.94 blood-transfusion,auc,0.69,0.759,0.765,0.75 car,neg_logloss,-0.117,-0.011,-0.004,-0.002 christine,auc,0.804,0.812,0.823,0.83 cnae-9,neg_logloss,-0.332,-0.211,-0.175,-0.156 -connect-4,neg_logloss,-0.502,-0.456,,-0.337 +connect-4,neg_logloss,-0.502,-0.456,-0.338,-0.337 credit-g,auc,0.795,0.778,0.789,0.796 dilbert,neg_logloss,-0.148,-0.159,-0.05,-0.033 fabert,neg_logloss,-0.788,-0.895,-0.752,-0.766 guillermo,auc,0.9,0.891,,0.926 jasmine,auc,0.883,0.888,0.887,0.88 -jungle chess,neg_logloss,-0.431,-0.193,,-0.149 +jungle chess,neg_logloss,-0.431,-0.193,-0.24,-0.149 kc1,auc,0.822,0.843,,0.831 kr-vs-kp,auc,0.999,1.0,,1.0 mfeat-factors,neg_logloss,-0.161,-0.094,,-0.082 -nomao,auc,0.995,0.994,,0.997 -numerai28_6,auc,0.517,0.529,,0.531 +nomao,auc,0.995,0.994,0.996,0.997 +numerai28_6,auc,0.517,0.529,0.531,0.531 phoneme,auc,0.965,0.965,,0.965 segment,neg_logloss,-0.094,-0.062,,-0.061 sylvine,auc,0.985,0.988,,0.988 diff --git a/docs/source/benchmarks/amlb_res.html b/docs/source/benchmarks/amlb_res.html new file mode 100644 index 0000000000..d0c976963d --- /dev/null +++ b/docs/source/benchmarks/amlb_res.html @@ -0,0 +1,719 @@ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Classification statistics +
+ + framework + + AutoGluon + + FEDOT + + H2O + + LAMA +
+ Dataset name + + Metric name + + + + +
+ APSFailure + + auc + + 0.990 + + 0.991 + + 0.992 + + 0.992 +
+ Amazon_employee_access + + auc + + 0.857 + + 0.865 + + 0.873 + + 0.879 +
+ Australian + + auc + + 0.940 + + 0.939 + + 0.938 + + 0.945 +
+ Covertype + + neg_logloss + + -0.071 + + -0.117 + + -0.265 + + nan +
+ Fashion-MNIST + + neg_logloss + + -0.329 + + -0.373 + + -0.380 + + -0.248 +
+ Jannis + + neg_logloss + + -0.728 + + -0.737 + + -0.691 + + -0.664 +
+ KDDCup09_appetency + + auc + + 0.804 + + 0.822 + + 0.829 + + 0.850 +
+ MiniBooNE + + auc + + 0.982 + + 0.981 + + nan + + 0.988 +
+ Shuttle + + neg_logloss + + -0.001 + + -0.001 + + -0.000 + + -0.001 +
+ Volkert + + neg_logloss + + -0.917 + + -1.097 + + -0.976 + + -0.806 +
+ adult + + auc + + 0.910 + + 0.925 + + 0.931 + + 0.932 +
+ bank-marketing + + auc + + 0.931 + + 0.935 + + 0.939 + + 0.940 +
+ blood-transfusion + + auc + + 0.690 + + 0.759 + + 0.754 + + 0.750 +
+ car + + neg_logloss + + -0.117 + + -0.011 + + -0.003 + + -0.002 +
+ christine + + auc + + 0.804 + + 0.812 + + 0.815 + + 0.830 +
+ cnae-9 + + neg_logloss + + -0.332 + + -0.211 + + -0.262 + + -0.156 +
+ connect-4 + + neg_logloss + + -0.502 + + -0.456 + + -0.338 + + -0.337 +
+ credit-g + + auc + + 0.795 + + 0.778 + + 0.798 + + 0.796 +
+ dilbert + + neg_logloss + + -0.148 + + -0.159 + + -0.103 + + -0.033 +
+ fabert + + neg_logloss + + -0.788 + + -0.895 + + -0.792 + + -0.766 +
+ guillermo + + auc + + 0.900 + + 0.891 + + nan + + 0.926 +
+ jasmine + + auc + + 0.883 + + 0.888 + + 0.888 + + 0.880 +
+ jungle chess + + neg_logloss + + -0.431 + + -0.193 + + -0.240 + + -0.149 +
+ kc1 + + auc + + 0.822 + + 0.843 + + nan + + 0.831 +
+ kr-vs-kp + + auc + + 0.999 + + 1.000 + + 1.000 + + 1.000 +
+ mfeat-factors + + neg_logloss + + -0.161 + + -0.094 + + -0.093 + + -0.082 +
+ nomao + + auc + + 0.995 + + 0.994 + + 0.996 + + 0.997 +
+ numerai28_6 + + auc + + 0.517 + + 0.529 + + 0.531 + + 0.531 +
+ phoneme + + auc + + 0.965 + + 0.965 + + 0.968 + + 0.965 +
+ segment + + neg_logloss + + -0.094 + + -0.062 + + -0.060 + + -0.061 +
+ sylvine + + auc + + 0.985 + + 0.988 + + 0.989 + + 0.988 +
+ vehicle + + neg_logloss + + -0.515 + + -0.354 + + -0.331 + + -0.404 +
+
diff --git a/docs/source/benchmarks/img_benchmarks/stats.png b/docs/source/benchmarks/img_benchmarks/stats.png new file mode 100644 index 0000000000..63f68efd5e Binary files /dev/null and b/docs/source/benchmarks/img_benchmarks/stats.png differ diff --git a/docs/source/benchmarks/tabular.rst b/docs/source/benchmarks/tabular.rst index f8a4099a10..8e7801c7ba 100644 --- a/docs/source/benchmarks/tabular.rst +++ b/docs/source/benchmarks/tabular.rst @@ -1,10 +1,21 @@ Tabular data ------------ -Here are overall classification problem results across popular AutoML frameworks: +Here are overall classification problem results across state-of-the-art AutoML frameworks +using `AutoMlBenchmark `__ test suite: -.. csv-table:: Classification statistics - :file: amlb_res.csv - :align: center - :widths: auto - :header-rows: 1 +.. raw:: html + :file: amlb_res.html + + +The results are obtained using sever based on Xeon Cascadelake (2900MHz) +with 12 cores and 24GB memory for experiments with the local infrastructure. 1h8c configuration was used for AMLB. + +Despite the obtained metrics being a bit different from AMLB's `paper `__ +the results confirm that FEDOT is competitive with SOTA solutions. + +The statistical analysis was conducted using the Friedman t-test. +The results of experiments and analysis confirm that FEDOT results are statistically indistinguishable +from SOTA competitors H2O, AutoGluon and LAMA (see below). + +.. image:: img_benchmarks/stats.png diff --git a/fedot/core/utils.py b/fedot/core/utils.py index 87edba84ad..9ca374b602 100644 --- a/fedot/core/utils.py +++ b/fedot/core/utils.py @@ -3,7 +3,7 @@ import random import tempfile from pathlib import Path -from typing import Optional +from typing import Optional, Union import numpy as np import pandas as pd @@ -89,3 +89,44 @@ def set_random_seed(seed: Optional[int]): np.random.seed(seed) random.seed(seed) RandomStateHandler.MODEL_FITTING_SEED = seed + + +def df_to_html(df: pd.DataFrame, save_path: Union[str, os.PathLike], name: str = 'table', caption: str = ''): + ''' + Makes html table out of DataFrame look like csv-table. + Requires BeatifulSoup to be installed. + + Args: + df: the pd.DataFrame to convert to html + save_path: where to save the output + name: output table identificator + caption: caption above the output table + ''' + from bs4 import BeautifulSoup + + df_styler = df.round(3).style.highlight_max(props='color: blue; font-weight: bold;', axis=1) + df_styler.format(precision=3) + if caption: + df_styler.set_caption(caption) + df_styler.set_table_attributes(( + 'style="width: 100%; border-collapse: collapse;' + 'font-family: Lato,proxima-nova,Helvetica Neue,Arial,sans-serif;"' + )) + df_styler.set_table_styles([ + {'selector': 'table, th, td', + 'props': 'border: 1px solid #e1e4e5; text-align: center; font-size: .9rem;'}, + {'selector': 'th, td', + 'props': 'padding: 8px 16px;'}, + {'selector': 'tr', + 'props': 'background-color: #fff;'}, + {'selector': 'tbody tr:nth-child(odd)', + 'props': 'background-color: #f3f6f6;'} + ]) + file = Path(save_path) + df_styler.to_html(file, table_uuid=name) + + doc = BeautifulSoup(file.read_text(), 'html.parser') + table = doc.find('table') + if table.parent.name != 'div': + table = table.wrap(doc.new_tag('div', style='overflow: auto;')) + file.write_text(doc.prettify())