From 4b3d116c7eb7e60295d4ec7889e3a20f10c7ed9d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 2 Jul 2019 02:49:20 +0430 Subject: [PATCH 001/107] Update pytest from 4.6.3 to 5.0.0 --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index c6e04f30..00ab5621 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ art==3.7 numpy==1.16.4 codecov==2.0.15 -pytest==4.6.3 +pytest==5.0.0 pytest-cov==2.7.1 setuptools==41.0.1 vulture==1.0 From 910a272ed51f6e073c262716da1ee74a226ae13b Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 2 Jul 2019 02:49:21 +0430 Subject: [PATCH 002/107] Update bandit from 1.6.1 to 1.6.2 --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 00ab5621..7a6ae73b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -5,6 +5,6 @@ pytest==5.0.0 pytest-cov==2.7.1 setuptools==41.0.1 vulture==1.0 -bandit==1.6.1 +bandit==1.6.2 pydocstyle==3.0.0 From b01327eac168b8ac2849aa15241d5a837c2fd58d Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 2 Jul 2019 12:10:13 +0430 Subject: [PATCH 003/107] fix : dev-requirements updated --- dev-requirements.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 7a6ae73b..faf654ff 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,10 +1,10 @@ art==3.7 numpy==1.16.4 -codecov==2.0.15 -pytest==5.0.0 -pytest-cov==2.7.1 -setuptools==41.0.1 -vulture==1.0 -bandit==1.6.2 -pydocstyle==3.0.0 +codecov>=2.0.15 +pytest>=4.3.1 +pytest-cov>=2.6.1 +setuptools>=40.8.0 +vulture>=1.0 +bandit>=1.5.1 +pydocstyle>=3.0.0 From 331d11f8d029c99ce04b469afb0bd031cf022846 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 8 Jul 2019 13:48:09 +0430 Subject: [PATCH 004/107] feat : TI_calc function added #214 --- pycm/pycm_class_func.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pycm/pycm_class_func.py b/pycm/pycm_class_func.py index 588d6e41..abcace31 100644 --- a/pycm/pycm_class_func.py +++ b/pycm/pycm_class_func.py @@ -5,6 +5,28 @@ from .pycm_interpret import * +def TI_calc(TP, FP, FN, alpha, beta): + """ + Calculate TI (Tversky index). + + :param TP: true positive + :type TP : int + :param FP: false positive + :type FP : int + :param FN: false negative + :type FN : int + :param alpha: alpha coefficient + :type alpha : float + :param beta: beta coefficient + :type beta: float + :return: TI as float + """ + try: + TI = TP/(TP+alpha*FN + beta*FP) + return TI + except Exception: + return "None" + def OOC_calc(TP, TOP, P): """ Calculate OOC (Otsuka-Ochiai coefficient). From 148ad3fbf9a0a91b7cc3c98e8593f450e44251a9 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 8 Jul 2019 13:53:15 +0430 Subject: [PATCH 005/107] fix : TI method added to object #214 --- pycm/pycm_obj.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pycm/pycm_obj.py b/pycm/pycm_obj.py index da29f39d..615ed6e5 100644 --- a/pycm/pycm_obj.py +++ b/pycm/pycm_obj.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ConfusionMatrix module.""" from __future__ import division -from .pycm_class_func import class_statistics, F_calc, IBA_calc +from .pycm_class_func import class_statistics, F_calc, IBA_calc, TI_calc from .pycm_overall_func import overall_statistics from .pycm_output import * from .pycm_util import * @@ -401,6 +401,24 @@ def IBA_alpha(self, alpha): except Exception: return {} + def TI(self, alpha, beta): + """ + Calculate TI (Tversky index). + + :param alpha: alpha coefficient + :type alpha : float + :param beta: beta coefficient + :type beta: float + :return: TI as float + """ + try: + TI_dict={} + for i in self.classes: + TI_dict[i] = TI_calc(self.TP[i],self.FP[i],self.FN[i],alpha,beta) + return TI_dict + except Exception: + return {} + def __repr__(self): """ Confusion matrix object representation method. From 9fc6b49168b3bcdf37fa1af076632d43fc43e6ee Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 8 Jul 2019 14:47:09 +0430 Subject: [PATCH 006/107] fix : testcases updated #214 --- Test/function_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Test/function_test.py b/Test/function_test.py index 34475e1d..76e80e70 100644 --- a/Test/function_test.py +++ b/Test/function_test.py @@ -167,6 +167,8 @@ >>> del cm.classes >>> cm.IBA_alpha(2) {} +>>> cm.TI(2,3) +{} >>> kappa_analysis_koch(-0.1) 'Poor' >>> kappa_analysis_koch(0) @@ -609,4 +611,11 @@ 0.9636363636363636 >>> cm.OOC[1] 0.9383838571303771 +>>> cm = ConfusionMatrix(matrix={1:{1:63,0:1},0:{0:50,1:2}}) # Verified Case +>>> cm.TI(alpha=1,beta=1)[1] +0.9545454545454546 +>>> cm.TI(alpha=2,beta=8)[1] +0.7777777777777778 +>>> cm.TI(alpha=2,beta=8)[0] +0.8064516129032258 """ From 8ecd49f98e150b6c18f3f38b92eaebd676b26088 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 8 Jul 2019 14:48:46 +0430 Subject: [PATCH 007/107] fix : autopep8 fix #214 --- pycm/pycm_class_func.py | 3 ++- pycm/pycm_interpret.py | 2 +- pycm/pycm_obj.py | 17 +++++++++++------ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/pycm/pycm_class_func.py b/pycm/pycm_class_func.py index abcace31..dc9315c5 100644 --- a/pycm/pycm_class_func.py +++ b/pycm/pycm_class_func.py @@ -22,11 +22,12 @@ def TI_calc(TP, FP, FN, alpha, beta): :return: TI as float """ try: - TI = TP/(TP+alpha*FN + beta*FP) + TI = TP / (TP + alpha * FN + beta * FP) return TI except Exception: return "None" + def OOC_calc(TP, TOP, P): """ Calculate OOC (Otsuka-Ochiai coefficient). diff --git a/pycm/pycm_interpret.py b/pycm/pycm_interpret.py index f74f6e21..d3787949 100644 --- a/pycm/pycm_interpret.py +++ b/pycm/pycm_interpret.py @@ -72,7 +72,7 @@ def V_analysis(V): return "Strong" if V >= 0.8: return "Very Strong" - except Exception: # pragma: no cover + except Exception: # pragma: no cover return "None" diff --git a/pycm/pycm_obj.py b/pycm/pycm_obj.py index 615ed6e5..a43f9f48 100644 --- a/pycm/pycm_obj.py +++ b/pycm/pycm_obj.py @@ -204,7 +204,8 @@ def save_stat( file.write(matrix + normalized_matrix + stat + one_vs_all) file.close() if address: - message = os.path.join(os.getcwd(), name + ".pycm") # pragma: no cover + message = os.path.join( + os.getcwd(), name + ".pycm") # pragma: no cover return {"Status": True, "Message": message} except Exception as e: return {"Status": False, "Message": str(e)} @@ -261,7 +262,8 @@ def save_html( html_file.write(html_end(VERSION)) html_file.close() if address: - message = os.path.join(os.getcwd(), name + ".html") # pragma: no cover + message = os.path.join( + os.getcwd(), name + ".html") # pragma: no cover return {"Status": True, "Message": message} except Exception as e: return {"Status": False, "Message": str(e)} @@ -309,7 +311,8 @@ def save_csv( csv_matrix_data = csv_matrix_print(self.classes, matrix) csv_matrix_file.write(csv_matrix_data) if address: - message = os.path.join(os.getcwd(), name + ".csv") # pragma: no cover + message = os.path.join( + os.getcwd(), name + ".csv") # pragma: no cover return {"Status": True, "Message": message} except Exception as e: return {"Status": False, "Message": str(e)} @@ -360,7 +363,8 @@ def save_obj( dump_dict["Predict-Vector"] = None json.dump(dump_dict, obj_file) if address: - message = os.path.join(os.getcwd(), name + ".obj") # pragma: no cover + message = os.path.join( + os.getcwd(), name + ".obj") # pragma: no cover return {"Status": True, "Message": message} except Exception as e: return {"Status": False, "Message": str(e)} @@ -412,9 +416,10 @@ def TI(self, alpha, beta): :return: TI as float """ try: - TI_dict={} + TI_dict = {} for i in self.classes: - TI_dict[i] = TI_calc(self.TP[i],self.FP[i],self.FN[i],alpha,beta) + TI_dict[i] = TI_calc( + self.TP[i], self.FP[i], self.FN[i], alpha, beta) return TI_dict except Exception: return {} From a65ea5472852aaab8e387b0f8c2524c8e9d7f1c2 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 8 Jul 2019 15:26:21 +0430 Subject: [PATCH 008/107] doc : TI added to Document #214 --- Document/Document.ipynb | 63 ++++++++++++++++++++++++++++++++++++++--- README.md | 2 ++ 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/Document/Document.ipynb b/Document/Document.ipynb index 3f30899a..607bd061 100644 --- a/Document/Document.ipynb +++ b/Document/Document.ipynb @@ -111,7 +111,8 @@ "
  • Adjusted G-Mean
    • \n", "
  • Adjusted F-Score
    • \n", "
  • Overlap Coefficient
    • \n", - "
  • Otsuka Ochiai Coefficient
    • \n", + "
  • Otsuka Ochiai Coefficient
    • \n", + "
  • Tversky Index
    • \n", " \n", "  \n", "
  • Overall Statistics
    • \n", @@ -1073,7 +1074,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -1131,7 +1132,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -4661,6 +4662,58 @@ "" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### TI (Tversky index)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The Tversky index, named after Amos Tversky, is an asymmetric similarity measure on sets that compares a variant to a prototype. The Tversky index can be seen as a generalization of Dice's coefficient and Tanimoto coefficient [[54]](#ref54).\n", + "\n", + "Wikipedia page" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$TI(\\alpha,\\beta)=\\frac{TP}{TP+\\alpha FN+\\beta FP}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'L1': 0.42857142857142855, 'L2': 0.1111111111111111, 'L3': 0.1875}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.TI(2,3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
      \n", + "
    • Notice : new in version 2.4
      • \n", + "
    " + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -9750,7 +9803,9 @@ "\n", "
    52- M. Vijaymeena and K. Kavitha, \"A survey on similarity measures in text mining,\" Machine Learning and Applications: An International Journal, vol. 3, no. 2, pp. 19-28, 2016.
    \n", "\n", - "
    53- Y. Otsuka, \"The faunal character of the Japanese Pleistocene marine Mollusca, as evidence of climate having become colder during the Pleistocene in Japan,\" Biogeograph. Soc. Japan, vol. 6, pp. 165-170, 1936.
    " + "
    53- Y. Otsuka, \"The faunal character of the Japanese Pleistocene marine Mollusca, as evidence of climate having become colder during the Pleistocene in Japan,\" Biogeograph. Soc. Japan, vol. 6, pp. 165-170, 1936.
    \n", + "\n", + "
    54- Tversky, Amos (1977). \"Features of Similarity\" . Psychological Review. 84 (4): 327–352. doi:10.1037/0033-295x.84.4.327
    " ] } ], diff --git a/README.md b/README.md index 3f6a417b..0b8fb30a 100644 --- a/README.md +++ b/README.md @@ -694,6 +694,8 @@ or send an email to [info@pycm.ir](mailto:info@pycm.ir "info@pycm.ir").
    53- Y. Otsuka, "The faunal character of the Japanese Pleistocene marine Mollusca, as evidence of climate having become colder during the Pleistocene in Japan," Biogeograph. Soc. Japan, vol. 6, pp. 165-170, 1936.
    +
    54- Tversky, Amos (1977). "Features of Similarity" . Psychological Review. 84 (4): 327–352. doi:10.1037/0033-295x.84.4.327
    + ## Cite From b834b52c8cd47f30dde411259738ece21ab358a6 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 8 Jul 2019 15:41:17 +0430 Subject: [PATCH 009/107] doc : minor edit in TI method docstring #214 --- pycm/pycm_obj.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycm/pycm_obj.py b/pycm/pycm_obj.py index a43f9f48..b4f6d97d 100644 --- a/pycm/pycm_obj.py +++ b/pycm/pycm_obj.py @@ -407,7 +407,7 @@ def IBA_alpha(self, alpha): def TI(self, alpha, beta): """ - Calculate TI (Tversky index). + Calculate Tversky index. :param alpha: alpha coefficient :type alpha : float From d5e0e1466657521944c74486b00a7da15e993625 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 8 Jul 2019 15:43:35 +0430 Subject: [PATCH 010/107] fix : minor edit in testcases #214 --- Test/function_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Test/function_test.py b/Test/function_test.py index 76e80e70..141df33c 100644 --- a/Test/function_test.py +++ b/Test/function_test.py @@ -614,6 +614,10 @@ >>> cm = ConfusionMatrix(matrix={1:{1:63,0:1},0:{0:50,1:2}}) # Verified Case >>> cm.TI(alpha=1,beta=1)[1] 0.9545454545454546 +>>> cm.TI(alpha=0.5,beta=0.5)[1] == cm.F1[1] +True +>>> cm.TI(alpha=0.5,beta=0.5)[0] == cm.F1[0] +True >>> cm.TI(alpha=2,beta=8)[1] 0.7777777777777778 >>> cm.TI(alpha=2,beta=8)[0] From 5c840daa02f934577a4beecf364c29a09057c5d8 Mon Sep 17 00:00:00 2001 From: alirezazolanvari Date: Mon, 8 Jul 2019 18:18:25 +0430 Subject: [PATCH 011/107] reference 54 reformated #214 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0b8fb30a..b93b6e36 100644 --- a/README.md +++ b/README.md @@ -694,7 +694,7 @@ or send an email to [info@pycm.ir](mailto:info@pycm.ir "info@pycm.ir").
    53- Y. Otsuka, "The faunal character of the Japanese Pleistocene marine Mollusca, as evidence of climate having become colder during the Pleistocene in Japan," Biogeograph. Soc. Japan, vol. 6, pp. 165-170, 1936.
    -
    54- Tversky, Amos (1977). "Features of Similarity" . Psychological Review. 84 (4): 327–352. doi:10.1037/0033-295x.84.4.327
    +
    54- A. Tversky, "Features of similarity," Psychological review, vol. 84, no. 4, p. 327, 1977.
    From b0567974e11e3b576cbb78f97c6e87de4ffc5b87 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 9 Jul 2019 12:32:06 +0430 Subject: [PATCH 012/107] doc : References updated #214 --- Document/Document.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Document/Document.ipynb b/Document/Document.ipynb index 607bd061..6ee53310 100644 --- a/Document/Document.ipynb +++ b/Document/Document.ipynb @@ -9805,7 +9805,7 @@ "\n", "
    53- Y. Otsuka, \"The faunal character of the Japanese Pleistocene marine Mollusca, as evidence of climate having become colder during the Pleistocene in Japan,\" Biogeograph. Soc. Japan, vol. 6, pp. 165-170, 1936.
    \n", "\n", - "
    54- Tversky, Amos (1977). \"Features of Similarity\" . Psychological Review. 84 (4): 327–352. doi:10.1037/0033-295x.84.4.327
    " + "
    54- A. Tversky, \"Features of similarity,\" Psychological review, vol. 84, no. 4, p. 327, 1977.
    " ] } ], From 39e5def59741684b9c1dd831bfca0886c2008a12 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 9 Jul 2019 12:43:01 +0430 Subject: [PATCH 013/107] feat : AUPR added #216 --- pycm/pycm_class_func.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pycm/pycm_class_func.py b/pycm/pycm_class_func.py index dc9315c5..07bdc0a0 100644 --- a/pycm/pycm_class_func.py +++ b/pycm/pycm_class_func.py @@ -470,18 +470,18 @@ def CEN_calc(classes, table, TOP, P, class_name, modified=False): return "None" -def AUC_calc(TNR, TPR): +def AUC_calc(item, TPR): """ - Calculate AUC (Area under the ROC curve for each class). + Calculate AUC/AUPR (Area under the ROC/PR curve for each class). - :param TNR: specificity or true negative rate - :type TNR : float + :param item: TNR or PPV + :type item : float :param TPR: sensitivity, recall, hit rate, or true positive rate :type TPR : float - :return: AUC as float + :return: AUC/AUPR as float """ try: - return (TNR + TPR) / 2 + return (item + TPR) / 2 except Exception: return "None" @@ -711,6 +711,7 @@ def class_statistics(TP, TN, FP, FN, classes, table): AGF = {} OC = {} OOC = {} + AUPR = {} for i in TP.keys(): POP[i] = TP[i] + TN[i] + FP[i] + FN[i] P[i] = TP[i] + FN[i] @@ -765,6 +766,7 @@ def class_statistics(TP, TN, FP, FN, classes, table): AGF[i] = AGF_calc(TP[i], FP[i], FN[i], TN[i]) OC[i] = OC_calc(TP[i], TOP[i], P[i]) OOC[i] = OOC_calc(TP[i], TOP[i], P[i]) + AUPR[i] = AUC_calc(PPV[i], TPR[i]) for i in TP.keys(): BCD[i] = BCD_calc(TOP, P, AM[i]) result = { @@ -825,5 +827,6 @@ def class_statistics(TP, TN, FP, FN, classes, table): "MCCI": MCCI, "AGF": AGF, "OC": OC, - "OOC": OOC} + "OOC": OOC, + "AUPR": AUPR} return result From 47e2830e6c004cef4d446d38ed08d8bd6aec4576 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 9 Jul 2019 12:44:15 +0430 Subject: [PATCH 014/107] fix : AUPR added to object #216 --- pycm/pycm_obj.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pycm/pycm_obj.py b/pycm/pycm_obj.py index b4f6d97d..15e5a7e3 100644 --- a/pycm/pycm_obj.py +++ b/pycm/pycm_obj.py @@ -563,6 +563,7 @@ def __class_stat_init__(cm): cm.AGF = cm.class_stat["AGF"] cm.OC = cm.class_stat["OC"] cm.OOC = cm.class_stat["OOC"] + cm.AUPR = cm.class_stat["AUPR"] def __overall_stat_init__(cm): From 5a6b4f0c7fa8b2e2668db5052345baaac227b9d7 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 9 Jul 2019 12:48:15 +0430 Subject: [PATCH 015/107] fix : AUPR links added #216 --- pycm/pycm_param.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pycm/pycm_param.py b/pycm/pycm_param.py index bd6ff3ac..ba70cd71 100644 --- a/pycm/pycm_param.py +++ b/pycm/pycm_param.py @@ -230,7 +230,8 @@ "AGM": "Adjusted geometric mean", "AGF": "Adjusted F-score", "OC": "Overlap coefficient", - "OOC": "Otsuka-Ochiai coefficient"} + "OOC": "Otsuka-Ochiai coefficient", + "AUPR": "Area under the PR curve"} PARAMS_LINK = { "TPR": "TPR-(True-positive-rate)", @@ -343,9 +344,10 @@ "SOA6(Matthews)": "SOA6-(Matthews's-benchmark)", "AGF": "AGF-(Adjusted-F-score)", "OC": "OC-(Overlap-coefficient)", - "OOC": "OOC-(Otsuka-Ochiai-coefficient)"} + "OOC": "OOC-(Otsuka-Ochiai-coefficient)", + "AUPR": ""} -CAPITALIZE_FILTER = ["BCD", "AUCI", "Q", "AGF", "OOC"] +CAPITALIZE_FILTER = ["BCD", "AUCI", "Q", "AGF", "OOC", "AUPR"] BENCHMARK_COLOR = { "PLRI": { From 62f9040a9a8d9974022e7f7567ef49d8a887a904 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 9 Jul 2019 13:04:55 +0430 Subject: [PATCH 016/107] fix : testcases updated #216 --- Test/function_test.py | 2 ++ Test/output_test.py | 2 ++ Test/overall_test.py | 7 +++++++ 3 files changed, 11 insertions(+) diff --git a/Test/function_test.py b/Test/function_test.py index 141df33c..1084460f 100644 --- a/Test/function_test.py +++ b/Test/function_test.py @@ -566,6 +566,8 @@ 0.55 >>> cm.AUC[1] 0.75 +>>> cm.AUPR[1] +0.5355263157894736 >>> cm.OP[1] 0.6473333333333334 >>> cm.IBA[1] diff --git a/Test/output_test.py b/Test/output_test.py index c6e45c47..83875bef 100644 --- a/Test/output_test.py +++ b/Test/output_test.py @@ -163,6 +163,7 @@ AM(Difference between automatic and manual classification) 11 -9 -1 -1 AUC(Area under the roc curve) None 0.5625 0.63725 0.5 AUCI(AUC value interpretation) None Poor Fair Poor +AUPR(Area under the PR curve) None 0.61607 0.41667 None BCD(Bray-Curtis dissimilarity) 0.275 0.225 0.025 0.025 BM(Informedness or bookmaker informedness) None 0.125 0.27451 0.0 CEN(Confusion entropy) 0.33496 0.35708 0.53895 0.0 @@ -327,6 +328,7 @@ AM(Difference between automatic and manual classification) 4 0 -4 AUC(Area under the roc curve) 0.86667 0.61111 0.63889 AUCI(AUC value interpretation) Very Good Fair Fair +AUPR(Area under the PR curve) 0.8 0.33333 0.625 BCD(Bray-Curtis dissimilarity) 0.09524 0.0 0.09524 BM(Informedness or bookmaker informedness) 0.73333 0.22222 0.27778 CEN(Confusion entropy) 0.25 0.52832 0.56439 diff --git a/Test/overall_test.py b/Test/overall_test.py index 0d665253..cbe69876 100644 --- a/Test/overall_test.py +++ b/Test/overall_test.py @@ -88,6 +88,7 @@ AM(Difference between automatic and manual classification) 2 -1 -1 AUC(Area under the roc curve) 0.88889 0.61111 0.58333 AUCI(AUC value interpretation) Very Good Fair Poor +AUPR(Area under the PR curve) 0.8 0.41667 0.55 BCD(Bray-Curtis dissimilarity) 0.08333 0.04167 0.04167 BM(Informedness or bookmaker informedness) 0.77778 0.22222 0.16667 CEN(Confusion entropy) 0.25 0.49658 0.60442 @@ -220,6 +221,7 @@ AM(Difference between automatic and manual classification) 2 -1 -1 AUC(Area under the roc curve) 0.88889 0.61111 0.58333 AUCI(AUC value interpretation) Very Good Fair Poor +AUPR(Area under the PR curve) 0.8 0.41667 0.55 BCD(Bray-Curtis dissimilarity) 0.08333 0.04167 0.04167 BM(Informedness or bookmaker informedness) 0.77778 0.22222 0.16667 CEN(Confusion entropy) 0.25 0.49658 0.60442 @@ -367,6 +369,7 @@ AM(Difference between automatic and manual classification) 11 -9 -1 -1 AUC(Area under the roc curve) None 0.5625 0.63725 0.5 AUCI(AUC value interpretation) None Poor Fair Poor +AUPR(Area under the PR curve) None 0.61607 0.41667 None BCD(Bray-Curtis dissimilarity) 0.275 0.225 0.025 0.025 BM(Informedness or bookmaker informedness) None 0.125 0.27451 0.0 CEN(Confusion entropy) 0.33496 0.35708 0.53895 0.0 @@ -486,6 +489,7 @@ AM(Difference between automatic and manual classification) 11 -9 -1 -1 AUC(Area under the roc curve) None 0.5625 0.63725 0.5 AUCI(AUC value interpretation) None Poor Fair Poor +AUPR(Area under the PR curve) None 0.61607 0.41667 None BCD(Bray-Curtis dissimilarity) 0.275 0.225 0.025 0.025 BM(Informedness or bookmaker informedness) None 0.125 0.27451 0.0 CEN(Confusion entropy) 0.33496 0.35708 0.53895 0.0 @@ -718,6 +722,7 @@ AM(Difference between automatic and manual classification) 1 0 -1 AUC(Area under the roc curve) 0.74167 0.66667 0.80952 AUCI(AUC value interpretation) Good Fair Very Good +AUPR(Area under the PR curve) 0.72115 0.55556 0.73333 BCD(Bray-Curtis dissimilarity) 0.01852 0.0 0.01852 BM(Informedness or bookmaker informedness) 0.48333 0.33333 0.61905 CEN(Confusion entropy) 0.45994 0.66249 0.47174 @@ -850,6 +855,7 @@ AM(Difference between automatic and manual classification) 1 0 -1 AUC(Area under the roc curve) 0.74167 0.66667 0.80952 AUCI(AUC value interpretation) Good Fair Very Good +AUPR(Area under the PR curve) 0.72115 0.55556 0.73333 BCD(Bray-Curtis dissimilarity) 0.01852 0.0 0.01852 BM(Informedness or bookmaker informedness) 0.48333 0.33333 0.61905 CEN(Confusion entropy) 0.45994 0.66249 0.47174 @@ -984,6 +990,7 @@ AM(Difference between automatic and manual classification) 4 0 -4 AUC(Area under the roc curve) 0.86667 0.61111 0.63889 AUCI(AUC value interpretation) Very Good Fair Fair +AUPR(Area under the PR curve) 0.8 0.33333 0.625 BCD(Bray-Curtis dissimilarity) 0.09524 0.0 0.09524 BM(Informedness or bookmaker informedness) 0.73333 0.22222 0.27778 CEN(Confusion entropy) 0.25 0.52832 0.56439 From e8e6cde38a9f0daae797ec06dc7483c86d53aade Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 9 Jul 2019 13:09:12 +0430 Subject: [PATCH 017/107] fix : testcases updated #216 --- Test/function_test.py | 203 +++++++++++++++++++++--------------------- 1 file changed, 102 insertions(+), 101 deletions(-) diff --git a/Test/function_test.py b/Test/function_test.py index 1084460f..17101362 100644 --- a/Test/function_test.py +++ b/Test/function_test.py @@ -349,107 +349,108 @@ 8-AUCI 9-AUNP 10-AUNU -11-BCD -12-BM -13-Bennett S -14-CBA -15-CEN -16-Chi-Squared -17-Chi-Squared DF -18-Conditional Entropy -19-Cramer V -20-Cross Entropy -21-DOR -22-DP -23-DPI -24-ERR -25-F0.5 -26-F1 -27-F1 Macro -28-F1 Micro -29-F2 -30-FDR -31-FN -32-FNR -33-FOR -34-FP -35-FPR -36-G -37-GI -38-GM -39-Gwet AC1 -40-Hamming Loss -41-IBA -42-IS -43-J -44-Joint Entropy -45-KL Divergence -46-Kappa -47-Kappa 95% CI -48-Kappa No Prevalence -49-Kappa Standard Error -50-Kappa Unbiased -51-LS -52-Lambda A -53-Lambda B -54-MCC -55-MCCI -56-MCEN -57-MK -58-Mutual Information -59-N -60-NIR -61-NLR -62-NLRI -63-NPV -64-OC -65-OOC -66-OP -67-Overall ACC -68-Overall CEN -69-Overall J -70-Overall MCC -71-Overall MCEN -72-Overall RACC -73-Overall RACCU -74-P -75-P-Value -76-PLR -77-PLRI -78-POP -79-PPV -80-PPV Macro -81-PPV Micro -82-PRE -83-Pearson C -84-Phi-Squared -85-Q -86-RACC -87-RACCU -88-RCI -89-RR -90-Reference Entropy -91-Response Entropy -92-SOA1(Landis & Koch) -93-SOA2(Fleiss) -94-SOA3(Altman) -95-SOA4(Cicchetti) -96-SOA5(Cramer) -97-SOA6(Matthews) -98-Scott PI -99-Standard Error -100-TN -101-TNR -102-TON -103-TOP -104-TP -105-TPR -106-TPR Macro -107-TPR Micro -108-Y -109-Zero-one Loss -110-dInd -111-sInd +11-AUPR +12-BCD +13-BM +14-Bennett S +15-CBA +16-CEN +17-Chi-Squared +18-Chi-Squared DF +19-Conditional Entropy +20-Cramer V +21-Cross Entropy +22-DOR +23-DP +24-DPI +25-ERR +26-F0.5 +27-F1 +28-F1 Macro +29-F1 Micro +30-F2 +31-FDR +32-FN +33-FNR +34-FOR +35-FP +36-FPR +37-G +38-GI +39-GM +40-Gwet AC1 +41-Hamming Loss +42-IBA +43-IS +44-J +45-Joint Entropy +46-KL Divergence +47-Kappa +48-Kappa 95% CI +49-Kappa No Prevalence +50-Kappa Standard Error +51-Kappa Unbiased +52-LS +53-Lambda A +54-Lambda B +55-MCC +56-MCCI +57-MCEN +58-MK +59-Mutual Information +60-N +61-NIR +62-NLR +63-NLRI +64-NPV +65-OC +66-OOC +67-OP +68-Overall ACC +69-Overall CEN +70-Overall J +71-Overall MCC +72-Overall MCEN +73-Overall RACC +74-Overall RACCU +75-P +76-P-Value +77-PLR +78-PLRI +79-POP +80-PPV +81-PPV Macro +82-PPV Micro +83-PRE +84-Pearson C +85-Phi-Squared +86-Q +87-RACC +88-RACCU +89-RCI +90-RR +91-Reference Entropy +92-Response Entropy +93-SOA1(Landis & Koch) +94-SOA2(Fleiss) +95-SOA3(Altman) +96-SOA4(Cicchetti) +97-SOA5(Cramer) +98-SOA6(Matthews) +99-Scott PI +100-Standard Error +101-TN +102-TNR +103-TON +104-TOP +105-TP +106-TPR +107-TPR Macro +108-TPR Micro +109-Y +110-Zero-one Loss +111-dInd +112-sInd >>> online_help("J") ... >>> online_help(4) From de996019385ebce6ee28c520897e3ef0b724c09a Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 9 Jul 2019 13:13:37 +0430 Subject: [PATCH 018/107] fix : minor edit in AUC description --- Test/output_test.py | 4 ++-- Test/overall_test.py | 18 +++++++++--------- pycm/pycm_param.py | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Test/output_test.py b/Test/output_test.py index 83875bef..e01876ff 100644 --- a/Test/output_test.py +++ b/Test/output_test.py @@ -161,7 +161,7 @@ AGF(Adjusted F-score) 0.0 0.33642 0.56659 0.0 AGM(Adjusted geometric mean) None 0.56694 0.7352 0 AM(Difference between automatic and manual classification) 11 -9 -1 -1 -AUC(Area under the roc curve) None 0.5625 0.63725 0.5 +AUC(Area under the ROC curve) None 0.5625 0.63725 0.5 AUCI(AUC value interpretation) None Poor Fair Poor AUPR(Area under the PR curve) None 0.61607 0.41667 None BCD(Bray-Curtis dissimilarity) 0.275 0.225 0.025 0.025 @@ -326,7 +326,7 @@ AGF(Adjusted F-score) 0.90694 0.54433 0.55442 AGM(Adjusted geometric mean) 0.80509 0.70336 0.66986 AM(Difference between automatic and manual classification) 4 0 -4 -AUC(Area under the roc curve) 0.86667 0.61111 0.63889 +AUC(Area under the ROC curve) 0.86667 0.61111 0.63889 AUCI(AUC value interpretation) Very Good Fair Fair AUPR(Area under the PR curve) 0.8 0.33333 0.625 BCD(Bray-Curtis dissimilarity) 0.09524 0.0 0.09524 diff --git a/Test/overall_test.py b/Test/overall_test.py index cbe69876..a58e0f2b 100644 --- a/Test/overall_test.py +++ b/Test/overall_test.py @@ -86,7 +86,7 @@ AGF(Adjusted F-score) 0.9136 0.53995 0.5516 AGM(Adjusted geometric mean) 0.83729 0.692 0.60712 AM(Difference between automatic and manual classification) 2 -1 -1 -AUC(Area under the roc curve) 0.88889 0.61111 0.58333 +AUC(Area under the ROC curve) 0.88889 0.61111 0.58333 AUCI(AUC value interpretation) Very Good Fair Poor AUPR(Area under the PR curve) 0.8 0.41667 0.55 BCD(Bray-Curtis dissimilarity) 0.08333 0.04167 0.04167 @@ -219,7 +219,7 @@ AGF(Adjusted F-score) 0.9136 0.53995 0.5516 AGM(Adjusted geometric mean) 0.83729 0.692 0.60712 AM(Difference between automatic and manual classification) 2 -1 -1 -AUC(Area under the roc curve) 0.88889 0.61111 0.58333 +AUC(Area under the ROC curve) 0.88889 0.61111 0.58333 AUCI(AUC value interpretation) Very Good Fair Poor AUPR(Area under the PR curve) 0.8 0.41667 0.55 BCD(Bray-Curtis dissimilarity) 0.08333 0.04167 0.04167 @@ -367,7 +367,7 @@ AGF(Adjusted F-score) 0.0 0.33642 0.56659 0.0 AGM(Adjusted geometric mean) None 0.56694 0.7352 0 AM(Difference between automatic and manual classification) 11 -9 -1 -1 -AUC(Area under the roc curve) None 0.5625 0.63725 0.5 +AUC(Area under the ROC curve) None 0.5625 0.63725 0.5 AUCI(AUC value interpretation) None Poor Fair Poor AUPR(Area under the PR curve) None 0.61607 0.41667 None BCD(Bray-Curtis dissimilarity) 0.275 0.225 0.025 0.025 @@ -487,7 +487,7 @@ AGF(Adjusted F-score) 0.0 0.33642 0.56659 0.0 AGM(Adjusted geometric mean) None 0.56694 0.7352 0 AM(Difference between automatic and manual classification) 11 -9 -1 -1 -AUC(Area under the roc curve) None 0.5625 0.63725 0.5 +AUC(Area under the ROC curve) None 0.5625 0.63725 0.5 AUCI(AUC value interpretation) None Poor Fair Poor AUPR(Area under the PR curve) None 0.61607 0.41667 None BCD(Bray-Curtis dissimilarity) 0.275 0.225 0.025 0.025 @@ -553,7 +553,7 @@ Classes 100 200 500 600 ACC(Accuracy) 0.45 0.45 0.85 0.95 -AUC(Area under the roc curve) None 0.5625 0.63725 0.5 +AUC(Area under the ROC curve) None 0.5625 0.63725 0.5 TNR(Specificity or true negative rate) 0.45 0.75 0.94118 1.0 TPR(Sensitivity, recall, hit rate, or true positive rate) None 0.375 0.33333 0.0 @@ -567,7 +567,7 @@ Classes 100 ACC(Accuracy) 0.45 -AUC(Area under the roc curve) None +AUC(Area under the ROC curve) None TNR(Specificity or true negative rate) 0.45 TPR(Sensitivity, recall, hit rate, or true positive rate) None @@ -720,7 +720,7 @@ AGF(Adjusted F-score) 0.75595 0.65734 0.79543 AGM(Adjusted geometric mean) 0.73866 0.70552 0.86488 AM(Difference between automatic and manual classification) 1 0 -1 -AUC(Area under the roc curve) 0.74167 0.66667 0.80952 +AUC(Area under the ROC curve) 0.74167 0.66667 0.80952 AUCI(AUC value interpretation) Good Fair Very Good AUPR(Area under the PR curve) 0.72115 0.55556 0.73333 BCD(Bray-Curtis dissimilarity) 0.01852 0.0 0.01852 @@ -853,7 +853,7 @@ AGF(Adjusted F-score) 0.75595 0.65734 0.79543 AGM(Adjusted geometric mean) 0.73866 0.70552 0.86488 AM(Difference between automatic and manual classification) 1 0 -1 -AUC(Area under the roc curve) 0.74167 0.66667 0.80952 +AUC(Area under the ROC curve) 0.74167 0.66667 0.80952 AUCI(AUC value interpretation) Good Fair Very Good AUPR(Area under the PR curve) 0.72115 0.55556 0.73333 BCD(Bray-Curtis dissimilarity) 0.01852 0.0 0.01852 @@ -988,7 +988,7 @@ AGF(Adjusted F-score) 0.90694 0.54433 0.55442 AGM(Adjusted geometric mean) 0.80509 0.70336 0.66986 AM(Difference between automatic and manual classification) 4 0 -4 -AUC(Area under the roc curve) 0.86667 0.61111 0.63889 +AUC(Area under the ROC curve) 0.86667 0.61111 0.63889 AUCI(AUC value interpretation) Very Good Fair Fair AUPR(Area under the PR curve) 0.8 0.33333 0.625 BCD(Bray-Curtis dissimilarity) 0.09524 0.0 0.09524 diff --git a/pycm/pycm_param.py b/pycm/pycm_param.py index ba70cd71..5216950e 100644 --- a/pycm/pycm_param.py +++ b/pycm/pycm_param.py @@ -347,7 +347,7 @@ "OOC": "OOC-(Otsuka-Ochiai-coefficient)", "AUPR": ""} -CAPITALIZE_FILTER = ["BCD", "AUCI", "Q", "AGF", "OOC", "AUPR"] +CAPITALIZE_FILTER = ["BCD", "AUCI", "Q", "AGF", "OOC", "AUPR", "AUC"] BENCHMARK_COLOR = { "PLRI": { From 9743d59ff2c6bef92d256d5019ac362a2d825502 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 9 Jul 2019 13:38:11 +0430 Subject: [PATCH 019/107] doc : AUPR added to Document #216 --- Document/Document.ipynb | 60 +++++++++++++++++++++++++++++++++++++++-- README.md | 4 +++ pycm/pycm_param.py | 2 +- 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/Document/Document.ipynb b/Document/Document.ipynb index 6ee53310..7c50f9b1 100644 --- a/Document/Document.ipynb +++ b/Document/Document.ipynb @@ -112,7 +112,8 @@ "
  • Adjusted F-Score
    • \n", "
  • Overlap Coefficient
    • \n", "
  • Otsuka Ochiai Coefficient
    • \n", - "
  • Tversky Index
    • \n", + "
  • Tversky Index
    • \n", + "
  • Area Under The PR Curve
    • \n", " \n", "  \n", "
  • Overall Statistics
    • \n", @@ -4714,6 +4715,57 @@ "" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### AUPR (Area under the PR curve)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Are under the precision-recall curve [[55]](#ref55) [[56]](#ref56)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "$$AUPR=\\frac{TPR+PPV}{2}$$" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'L1': 0.8, 'L2': 0.41666666666666663, 'L3': 0.55}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.AUPR" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
      \n", + "
    • Notice : new in version 2.4
      • \n", + "
    • Notice : this is an approximate calculation of AUPR
      • \n", + "
    " + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -9805,7 +9857,11 @@ "\n", "
    53- Y. Otsuka, \"The faunal character of the Japanese Pleistocene marine Mollusca, as evidence of climate having become colder during the Pleistocene in Japan,\" Biogeograph. Soc. Japan, vol. 6, pp. 165-170, 1936.
    \n", "\n", - "
    54- A. Tversky, \"Features of similarity,\" Psychological review, vol. 84, no. 4, p. 327, 1977.
    " + "
    54- A. Tversky, \"Features of similarity,\" Psychological review, vol. 84, no. 4, p. 327, 1977.
    \n", + "\n", + "
    55- Boyd, K., Eng, K.H. and Page, C.D., 2013, September. Area under the precision-recall curve: point estimates and confidence intervals. In Joint European conference on machine learning and knowledge discovery in databases (pp. 451-466). Springer, Berlin, Heidelberg.
    \n", + "\n", + "
    56- Davis, J. and Goadrich, M., 2006, June. The relationship between Precision-Recall and ROC curves. In Proceedings of the 23rd international conference on Machine learning (pp. 233-240). ACM.
    " ] } ], diff --git a/README.md b/README.md index b93b6e36..ce8cacb0 100644 --- a/README.md +++ b/README.md @@ -696,6 +696,10 @@ or send an email to [info@pycm.ir](mailto:info@pycm.ir "info@pycm.ir").
    54- A. Tversky, "Features of similarity," Psychological review, vol. 84, no. 4, p. 327, 1977.
    +
    55- Boyd, K., Eng, K.H. and Page, C.D., 2013, September. Area under the precision-recall curve: point estimates and confidence intervals. In Joint European conference on machine learning and knowledge discovery in databases (pp. 451-466). Springer, Berlin, Heidelberg.
    + +
    56- Davis, J. and Goadrich, M., 2006, June. The relationship between Precision-Recall and ROC curves. In Proceedings of the 23rd international conference on Machine learning (pp. 233-240). ACM.
    + ## Cite diff --git a/pycm/pycm_param.py b/pycm/pycm_param.py index 5216950e..a2ed25d0 100644 --- a/pycm/pycm_param.py +++ b/pycm/pycm_param.py @@ -345,7 +345,7 @@ "AGF": "AGF-(Adjusted-F-score)", "OC": "OC-(Overlap-coefficient)", "OOC": "OOC-(Otsuka-Ochiai-coefficient)", - "AUPR": ""} + "AUPR": "AUPR-(Area-under-the-PR-curve)"} CAPITALIZE_FILTER = ["BCD", "AUCI", "Q", "AGF", "OOC", "AUPR", "AUC"] From 43a88eb92e115489a060ed372ac1f7ffc38aa616 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 9 Jul 2019 13:40:02 +0430 Subject: [PATCH 020/107] doc : CHANGELOG updated #216 #214 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79484f7f..f1d20727 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Tversky index (TI) +- Are under the PR curve (AUPR) ## [2.3] - 2019-06-27 ### Added - Adjusted F-score (AGF) From 9471d0296598e481e1844a4d67e63986d72b3c82 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 9 Jul 2019 14:30:53 +0430 Subject: [PATCH 021/107] doc : AUPR description updated #216 --- Document/Document.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Document/Document.ipynb b/Document/Document.ipynb index 7c50f9b1..3cd154d6 100644 --- a/Document/Document.ipynb +++ b/Document/Document.ipynb @@ -4726,7 +4726,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Are under the precision-recall curve [[55]](#ref55) [[56]](#ref56)." + "A PR curve is plotting precision against recall. The precision recall area under curve (AUPR) is just the area under the PR curve. The higher it is, the better the model is [[55]](#ref55) [[56]](#ref56).\n", + "\n" ] }, { From 71463e3aeed8da2c3cf686826fb98507bb17537a Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 9 Jul 2019 14:31:41 +0430 Subject: [PATCH 022/107] doc : CHANGELOG updated --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1d20727..917201f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Tversky index (TI) - Are under the PR curve (AUPR) +### Changed +- `AUC_calc` function modified ## [2.3] - 2019-06-27 ### Added - Adjusted F-score (AGF) From 0002b422d4f36d3b0b9d1e1cf92d505e698db202 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Fri, 12 Jul 2019 01:04:34 +0430 Subject: [PATCH 023/107] doc : p-value description updated #225 --- Document/Document.ipynb | 9 ++++++--- README.md | 2 ++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Document/Document.ipynb b/Document/Document.ipynb index 3cd154d6..0a807e70 100644 --- a/Document/Document.ipynb +++ b/Document/Document.ipynb @@ -7251,7 +7251,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The no information error rate is the error rate when the input and output are independent." + "Largest class percentage in the data [[57]](#ref57)." ] }, { @@ -7301,7 +7301,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "In statistical hypothesis testing, the p-value or probability value or significance is, for a given statistical model, the probability that, when the null hypothesis is true, the statistical summary (such as the absolute value of the sample mean difference between two compared groups) would be greater than or equal to the actual observed results [[31]](#ref31).\n", + "In statistical hypothesis testing, the p-value or probability value is, for a given statistical model, the probability that, when the null hypothesis is true, the statistical summary (such as the absolute value of the sample mean difference between two compared groups) would be greater than or equal to the actual observed results [[31]](#ref31) . \n", + "Here an one-sided binomial test to see if the accuracy is better than the no information rate [[57]](#ref57).\n", "\n", "\n", "\n", @@ -9862,7 +9863,9 @@ "\n", "
    55- Boyd, K., Eng, K.H. and Page, C.D., 2013, September. Area under the precision-recall curve: point estimates and confidence intervals. In Joint European conference on machine learning and knowledge discovery in databases (pp. 451-466). Springer, Berlin, Heidelberg.
    \n", "\n", - "
    56- Davis, J. and Goadrich, M., 2006, June. The relationship between Precision-Recall and ROC curves. In Proceedings of the 23rd international conference on Machine learning (pp. 233-240). ACM.
    " + "
    56- Davis, J. and Goadrich, M., 2006, June. The relationship between Precision-Recall and ROC curves. In Proceedings of the 23rd international conference on Machine learning (pp. 233-240). ACM.
    \n", + "\n", + "
    57- Kuhn, M., 2008. Building predictive models in R using the caret package. Journal of statistical software, 28(5), pp.1-26.
    " ] } ], diff --git a/README.md b/README.md index ce8cacb0..db147558 100644 --- a/README.md +++ b/README.md @@ -700,6 +700,8 @@ or send an email to [info@pycm.ir](mailto:info@pycm.ir "info@pycm.ir").
    56- Davis, J. and Goadrich, M., 2006, June. The relationship between Precision-Recall and ROC curves. In Proceedings of the 23rd international conference on Machine learning (pp. 233-240). ACM.
    +
    57- Kuhn, M., 2008. Building predictive models in R using the caret package. Journal of statistical software, 28(5), pp.1-26.
    + ## Cite From 4c9b6eeac031f4280a1c77ce6473df2a0db717b2 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Fri, 12 Jul 2019 01:05:22 +0430 Subject: [PATCH 024/107] doc : CHANGELOG updated #225 --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 917201f3..47ad7857 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added - Tversky index (TI) -- Are under the PR curve (AUPR) +- Area under the PR curve (AUPR) ### Changed - `AUC_calc` function modified +- Document modified ## [2.3] - 2019-06-27 ### Added - Adjusted F-score (AGF) From 159dfef1925e6f6733bb91c3f85223029444c2fc Mon Sep 17 00:00:00 2001 From: alirezazolanvari Date: Sat, 13 Jul 2019 10:08:21 +0430 Subject: [PATCH 025/107] reference 55, 56, and 57 reformated #216 #225 --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index db147558..5d530632 100644 --- a/README.md +++ b/README.md @@ -696,11 +696,11 @@ or send an email to [info@pycm.ir](mailto:info@pycm.ir "info@pycm.ir").
    54- A. Tversky, "Features of similarity," Psychological review, vol. 84, no. 4, p. 327, 1977.
    -
    55- Boyd, K., Eng, K.H. and Page, C.D., 2013, September. Area under the precision-recall curve: point estimates and confidence intervals. In Joint European conference on machine learning and knowledge discovery in databases (pp. 451-466). Springer, Berlin, Heidelberg.
    +
    55- K. Boyd, K. H. Eng, and C. D. Page, "Area under the precision-recall curve: point estimates and confidence intervals," in Joint European conference on machine learning and knowledge discovery in databases, 2013, pp. 451-466: Springer.
    -
    56- Davis, J. and Goadrich, M., 2006, June. The relationship between Precision-Recall and ROC curves. In Proceedings of the 23rd international conference on Machine learning (pp. 233-240). ACM.
    +
    56- J. Davis and M. Goadrich, "The relationship between Precision-Recall and ROC curves," in Proceedings of the 23rd international conference on Machine learning, 2006, pp. 233-240: ACM.
    -
    57- Kuhn, M., 2008. Building predictive models in R using the caret package. Journal of statistical software, 28(5), pp.1-26.
    +
    57- M. Kuhn, "Building predictive models in R using the caret package," Journal of statistical software, vol. 28, no. 5, pp. 1-26, 2008.
    From 8a1539b84d73e9647ff41d7d3d529125d4de770a Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 13 Jul 2019 11:06:32 +0430 Subject: [PATCH 026/107] feat : SUMMARY_CLASS and SUMMARY_OVERALL added #217 --- pycm/pycm_param.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pycm/pycm_param.py b/pycm/pycm_param.py index a2ed25d0..7c914074 100644 --- a/pycm/pycm_param.py +++ b/pycm/pycm_param.py @@ -46,6 +46,10 @@ BALANCE_RATIO_THRESHOLD = 3 +SUMMARY_OVERALL = ["ACC Macro","Kappa","Overall ACC","SOA1(Landis & Koch)","Zero-one Loss","F1 Macro","TPR Macro","PPV Macro"] + +SUMMARY_CLASS = ["ACC","AUC","AUCI","F1","TPR","PPV","TP","FP","FN","TN","N","P","POP","TOP","TON"] + BINARY_RECOMMEND = ["ACC", "TPR", "PPV", "AUC", "AUCI", "TNR", "F1"] MULTICLASS_RECOMMEND = [ From 3ec26f81d50ea02660da1ba6a1e2d7f84fc06d01 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 13 Jul 2019 11:09:46 +0430 Subject: [PATCH 027/107] fix : summary mode added to stat method #217 --- pycm/pycm_obj.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pycm/pycm_obj.py b/pycm/pycm_obj.py index 15e5a7e3..839f9ec6 100644 --- a/pycm/pycm_obj.py +++ b/pycm/pycm_obj.py @@ -126,7 +126,7 @@ def print_normalized_matrix(self, one_vs_all=False, class_name=None): table = normalized_table_calc(classes, table) print(table_print(classes, table)) - def stat(self, overall_param=None, class_param=None, class_name=None): + def stat(self, overall_param=None, class_param=None, class_name=None, summary=False): """ Print statistical measures table. @@ -136,15 +136,22 @@ def stat(self, overall_param=None, class_param=None, class_name=None): :type class_param : list :param class_name : class name (sub set of classes), Example :[1,2,3] :type class_name : list + :param summary : summary mode flag + :type summary : bool :return: None """ classes = class_filter(self.classes, class_name) + class_list = class_param + overall_list = overall_param + if summary : + class_list = SUMMARY_CLASS + overall_list = SUMMARY_OVERALL print( stat_print( classes, self.class_stat, self.overall_stat, - self.digit, overall_param, class_param)) + self.digit, overall_list, class_list)) def __str__(self): """ From c3fbd78ecc8bdef43b63155e145ad61d39ac7805 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 13 Jul 2019 11:11:11 +0430 Subject: [PATCH 028/107] fix : summary mode added to save_stat method #217 --- pycm/pycm_obj.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pycm/pycm_obj.py b/pycm/pycm_obj.py index 839f9ec6..5b1f1c7a 100644 --- a/pycm/pycm_obj.py +++ b/pycm/pycm_obj.py @@ -143,7 +143,7 @@ def stat(self, overall_param=None, class_param=None, class_name=None, summary=Fa classes = class_filter(self.classes, class_name) class_list = class_param overall_list = overall_param - if summary : + if summary: class_list = SUMMARY_CLASS overall_list = SUMMARY_OVERALL print( @@ -171,7 +171,8 @@ def save_stat( address=True, overall_param=None, class_param=None, - class_name=None): + class_name=None, + summary=False): """ Save ConfusionMatrix in .pycm (flat file format). @@ -185,10 +186,17 @@ def save_stat( :type class_param : list :param class_name : class name (sub set of classes), Example :[1,2,3] :type class_name : list + :param summary : summary mode flag + :type summary : bool :return: saving Status as dict {"Status":bool , "Message":str} """ try: message = None + class_list = class_param + overall_list = overall_param + if summary: + class_list = SUMMARY_CLASS + overall_list = SUMMARY_OVERALL file = open(name + ".pycm", "w") matrix = "Matrix : \n\n" + table_print(self.classes, self.table) + "\n\n" @@ -207,7 +215,7 @@ def save_stat( classes, self.class_stat, self.overall_stat, - self.digit, overall_param, class_param) + self.digit, overall_list, class_list) file.write(matrix + normalized_matrix + stat + one_vs_all) file.close() if address: From af3925b1f504b939ba388b18d9e19f5a2763e3c0 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 13 Jul 2019 11:12:27 +0430 Subject: [PATCH 029/107] fix : summary mode added to save_html method #217 --- pycm/pycm_obj.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pycm/pycm_obj.py b/pycm/pycm_obj.py index 5b1f1c7a..09e565cb 100644 --- a/pycm/pycm_obj.py +++ b/pycm/pycm_obj.py @@ -231,7 +231,7 @@ def save_html( address=True, overall_param=None, class_param=None, - class_name=None, color=(0, 0, 0), normalize=False): + class_name=None, color=(0, 0, 0), normalize=False, summary=False): """ Save ConfusionMatrix in HTML file. @@ -249,9 +249,16 @@ def save_html( :type color : tuple :param normalize : save normalize matrix flag :type normalize : bool + :param summary : summary mode flag + :type summary : bool :return: saving Status as dict {"Status":bool , "Message":str} """ try: + class_list = class_param + overall_list = overall_param + if summary: + class_list = SUMMARY_CLASS + overall_list = SUMMARY_OVERALL message = None table = self.table if normalize: @@ -264,7 +271,7 @@ def save_html( html_overall_stat( self.overall_stat, self.digit, - overall_param, + overall_list, self.recommended_list)) class_stat_classes = class_filter(self.classes, class_name) html_file.write( @@ -272,7 +279,7 @@ def save_html( class_stat_classes, self.class_stat, self.digit, - class_param, + class_list, self.recommended_list)) html_file.write(html_end(VERSION)) html_file.close() From 36967d8c3f65bd22925e1b1cb100448c459706b2 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 13 Jul 2019 11:13:41 +0430 Subject: [PATCH 030/107] fix : summary mode added to save_csv method #217 --- pycm/pycm_obj.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pycm/pycm_obj.py b/pycm/pycm_obj.py index 09e565cb..22d1a360 100644 --- a/pycm/pycm_obj.py +++ b/pycm/pycm_obj.py @@ -297,7 +297,8 @@ def save_csv( class_param=None, class_name=None, matrix_save=True, - normalize=False): + normalize=False, + summary=False): """ Save ConfusionMatrix in CSV file. @@ -313,9 +314,14 @@ def save_csv( :type matrix_save : bool :param normalize : save normalize matrix flag :type normalize : bool + :param summary : summary mode flag + :type summary : bool :return: saving Status as dict {"Status":bool , "Message":str} """ try: + class_list = class_param + if summary: + class_list = SUMMARY_CLASS message = None classes = class_filter(self.classes, class_name) csv_file = open(name + ".csv", "w") @@ -323,7 +329,7 @@ def save_csv( classes, self.class_stat, self.digit, - class_param) + class_list) csv_file.write(csv_data) if matrix_save: matrix = self.table From 659b92563d618800bc10531a27b117bac046f8a4 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 13 Jul 2019 14:17:00 +0430 Subject: [PATCH 031/107] fix : testcases updated #226 --- Test/overall_test.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/Test/overall_test.py b/Test/overall_test.py index a58e0f2b..ba531843 100644 --- a/Test/overall_test.py +++ b/Test/overall_test.py @@ -5,6 +5,8 @@ >>> import json >>> y_actu = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2] >>> y_pred = [0, 0, 2, 1, 0, 2, 1, 0, 2, 0, 2, 2] +>>> y_actu_copy = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2] +>>> y_pred_copy = [0, 0, 2, 1, 0, 2, 1, 0, 2, 0, 2, 2] >>> cm = ConfusionMatrix(y_actu, y_pred) >>> cm pycm.ConfusionMatrix(classes: [0, 1, 2]) @@ -143,6 +145,10 @@ sInd(Similarity index) 0.84287 0.52209 0.57508 >>> cm.relabel({0:"L1",1:"L2",2:"L3"}) +>>> y_actu == y_actu_copy +True +>>> y_pred == y_pred_copy +True >>> print(cm) Predict L1 L2 L3 Actual @@ -287,8 +293,14 @@ 0.2222222222222221 >>> import numpy as np >>> y_test = np.array([600, 200, 200, 200, 200, 200, 200, 200, 500, 500, 500, 200, 200, 200, 200, 200, 200, 200, 200, 200]) +>>> y_test_copy = np.array([600, 200, 200, 200, 200, 200, 200, 200, 500, 500, 500, 200, 200, 200, 200, 200, 200, 200, 200, 200]) >>> y_pred = np.array([100, 200, 200, 100, 100, 200, 200, 200, 100, 200, 500, 100, 100, 100, 100, 100, 100, 100, 500, 200]) +>>> y_pred_copy = np.array([100, 200, 200, 100, 100, 200, 200, 200, 100, 200, 500, 100, 100, 100, 100, 100, 100, 100, 500, 200]) >>> cm=ConfusionMatrix(y_test, y_pred) +>>> type(y_pred) == type(y_pred_copy) +True +>>> type(y_test) == type(y_test_copy) +True >>> print(cm) Predict 100 200 500 600 Actual @@ -634,7 +646,11 @@ ... return 1 ... else: ... return 0 ->>> cm_6 = ConfusionMatrix([0,0,1,0],[0.87,0.34,0.9,0.12],threshold=activation, transpose=2) +>>> y_pred_act = [0.87,0.34,0.9,0.12] +>>> y_pred_act_copy = [0.87,0.34,0.9,0.12] +>>> cm_6 = ConfusionMatrix([0,0,1,0],y_pred_act,threshold=activation, transpose=2) +>>> y_pred_act_copy == y_pred_act +True >>> cm_6.print_matrix() Predict 0 1 Actual @@ -643,7 +659,11 @@ >>> cm = ConfusionMatrix(matrix={1:{1:0,2:0},2:{1:0,2:0}}) >>> cm pycm.ConfusionMatrix(classes: [1, 2]) ->>> cm = ConfusionMatrix(matrix={"Class1":{"Class1":9,"Class2":3,"Class3":0},"Class2":{"Class1":3,"Class2":5,"Class3":1},"Class3":{"Class1":1,"Class2":1,"Class3":4}}) +>>> matrix1 = {"Class1":{"Class1":9,"Class2":3,"Class3":0},"Class2":{"Class1":3,"Class2":5,"Class3":1},"Class3":{"Class1":1,"Class2":1,"Class3":4}} +>>> matrix1_copy = {"Class1":{"Class1":9,"Class2":3,"Class3":0},"Class2":{"Class1":3,"Class2":5,"Class3":1},"Class3":{"Class1":1,"Class2":1,"Class3":4}} +>>> cm = ConfusionMatrix(matrix=matrix1) +>>> matrix1 == matrix1_copy +True >>> print(cm) Predict Class1 Class2 Class3 Actual @@ -776,7 +796,9 @@ dInd(Distance index) 0.36553 0.4969 0.33672 sInd(Similarity index) 0.74153 0.64864 0.7619 ->>> cm = ConfusionMatrix(matrix={"Class1":{"Class1":9,"Class2":3,"Class3":1},"Class2":{"Class1":3,"Class2":5,"Class3":1},"Class3":{"Class1":0,"Class2":1,"Class3":4}},transpose=True) +>>> cm = ConfusionMatrix(matrix=matrix1,transpose=True) +>>> matrix1 == matrix1_copy +True >>> print(cm) Predict Class1 Class2 Class3 Actual @@ -911,7 +933,11 @@ >>> y_actu = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2] >>> y_pred = [0, 0, 2, 1, 0, 2, 1, 0, 2, 0, 2, 2] ->>> cm = ConfusionMatrix(y_actu, y_pred, sample_weight=[2, 2, 2, 2, 3, 1, 1, 2, 2, 1, 1, 2]) +>>> weight = [2, 2, 2, 2, 3, 1, 1, 2, 2, 1, 1, 2] +>>> weight_copy = [2, 2, 2, 2, 3, 1, 1, 2, 2, 1, 1, 2] +>>> cm = ConfusionMatrix(y_actu, y_pred, sample_weight=weight) +>>> weight_copy == weight +True >>> print(cm) Predict 0 1 2 Actual From 88c1dd7127cf3c7cab412d8cb2f82aab2c78f45b Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 13 Jul 2019 14:36:49 +0430 Subject: [PATCH 032/107] fix : transpose bug fixed #226 --- Test/overall_test.py | 2 ++ pycm/pycm_util.py | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Test/overall_test.py b/Test/overall_test.py index ba531843..692455cf 100644 --- a/Test/overall_test.py +++ b/Test/overall_test.py @@ -796,6 +796,8 @@ dInd(Distance index) 0.36553 0.4969 0.33672 sInd(Similarity index) 0.74153 0.64864 0.7619 +>>> matrix1 = {"Class1":{"Class1":9,"Class2":3,"Class3":1},"Class2":{"Class1":3,"Class2":5,"Class3":1},"Class3":{"Class1":0,"Class2":1,"Class3":4}} +>>> matrix1_copy = {"Class1":{"Class1":9,"Class2":3,"Class3":1},"Class2":{"Class1":3,"Class2":5,"Class3":1},"Class3":{"Class1":0,"Class2":1,"Class3":4}} >>> cm = ConfusionMatrix(matrix=matrix1,transpose=True) >>> matrix1 == matrix1_copy True diff --git a/pycm/pycm_util.py b/pycm/pycm_util.py index 9890521a..7f41764f 100644 --- a/pycm/pycm_util.py +++ b/pycm/pycm_util.py @@ -217,7 +217,7 @@ def transpose_func(classes, table): :type table : dict :return: transposed table as dict """ - transposed_table = table + transposed_table = {k:table[k].copy() for k in classes} for i, item1 in enumerate(classes): for j, item2 in enumerate(classes): if i > j: @@ -238,6 +238,7 @@ def matrix_params_from_table(table, transpose=False): :return: [classes_list,table,TP,TN,FP,FN] """ classes = sorted(table.keys()) + table_temp = table map_dict = {k: 0 for k in classes} TP_dict = map_dict.copy() TN_dict = map_dict.copy() @@ -255,8 +256,8 @@ def matrix_params_from_table(table, transpose=False): temp = FN_dict FN_dict = FP_dict FP_dict = temp - table = transpose_func(classes, table) - return [classes, table, TP_dict, TN_dict, FP_dict, FN_dict] + table_temp = transpose_func(classes, table) + return [classes, table_temp, TP_dict, TN_dict, FP_dict, FN_dict] def matrix_params_calc(actual_vector, predict_vector, sample_weight): From 678c589cec5ee0bb91f895e0088328587e594851 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 13 Jul 2019 14:59:35 +0430 Subject: [PATCH 033/107] fix : testcases updated #226 --- Test/overall_test.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Test/overall_test.py b/Test/overall_test.py index 692455cf..6f043761 100644 --- a/Test/overall_test.py +++ b/Test/overall_test.py @@ -1104,7 +1104,11 @@ >>> cm_comp1_temp = ConfusionMatrix(matrix={0:{0:2,1:50,2:6},1:{0:5,1:50,2:3},2:{0:1,1:7,2:50}}) >>> cm_comp1 == cm_comp1_temp True ->>> cp = Compare({"model1":cm_comp1,"model2":cm_comp2}) +>>> compare_input = {"model1":cm_comp1,"model2":cm_comp2} +>>> compare_input_copy = {"model1":cm_comp1,"model2":cm_comp2} +>>> cp = Compare(compare_input) +>>> compare_input == compare_input_copy +True >>> cp pycm.Compare(classes: [0, 1, 2]) >>> cp.scores == {'model1': {'overall': 2.55, 'class': 7.05}, 'model2': {'overall': 1.98333, 'class': 4.55}} @@ -1127,7 +1131,11 @@ 1 model1 7.05 2.55 2 model2 4.55 1.98333 ->>> cp = Compare({"model1":cm_comp1,"model2":cm_comp2},by_class=True,weight={0:5,1:1,2:1}) +>>> weight = {0:5,1:1,2:1} +>>> weight_copy = {0:5,1:1,2:1} +>>> cp = Compare({"model1":cm_comp1,"model2":cm_comp2},by_class=True,weight=weight) +>>> weight == weight_copy +True >>> print(cp) Best : model2 From 1dcb672ddd58654631ce270fdbe71e8901072404 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 13 Jul 2019 15:36:50 +0430 Subject: [PATCH 034/107] fix : testcases updated for summary mode #217 --- Test/output_test.py | 16 ++++++++++++++++ Test/overall_test.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/Test/output_test.py b/Test/output_test.py index e01876ff..68119230 100644 --- a/Test/output_test.py +++ b/Test/output_test.py @@ -13,6 +13,9 @@ >>> save_stat=cm.save_stat("test_filtered",address=False,overall_param=["Kappa","Scott PI"],class_param=["TPR","TNR","ACC","AUC"]) >>> save_stat=={'Status': True, 'Message': None} True +>>> save_stat=cm.save_stat("test_summary",address=False,summary=True) +>>> save_stat=={'Status': True, 'Message': None} +True >>> save_stat=cm.save_stat("test_filtered2",address=False,overall_param=["Kappa","Scott PI"],class_param=["TPR","TNR","ACC","AUC"],class_name=["L1","L2"]) >>> save_stat=={'Status': True, 'Message': None} True @@ -28,6 +31,9 @@ >>> save_stat=cm.save_html("test_normalized",address=False,normalize=True) >>> save_stat=={'Status': True, 'Message': None} True +>>> save_stat=cm.save_html("test_summary",address=False,summary=True) +>>> save_stat=={'Status': True, 'Message': None} +True >>> save_stat=cm.save_html("test_filtered",address=False,overall_param=["Kappa","Scott PI"],class_param=["TPR","TNR","ACC","AUC"]) >>> save_stat=={'Status': True, 'Message': None} True @@ -55,6 +61,9 @@ >>> save_stat=cm.save_csv("test_normalized",address=False,normalize=True) >>> save_stat=={'Status': True, 'Message': None} True +>>> save_stat=cm.save_csv("test_summary",address=False,summary=True,matrix_save=False) +>>> save_stat=={'Status': True, 'Message': None} +True >>> save_stat=cm.save_csv("test_filtered",address=False,class_param=["TPR","TNR","ACC","AUC"]) >>> save_stat=={'Status': True, 'Message': None} True @@ -397,6 +406,10 @@ 0.627145631592811 >>> cm_file.transpose True +>>> cm.matrix == cm_file.matrix +True +>>> cm.normalized_matrix == cm_file.normalized_matrix +True >>> json.dump({"Actual-Vector": None, "Digit": 5, "Predict-Vector": None, "Matrix": {"0": {"0": 3, "1": 0, "2": 2}, "1": {"0": 0, "1": 1, "2": 1}, "2": {"0": 0, "1": 2, "2": 3}}, "Transpose": True,"Sample-Weight": None},open("test5.obj","w")) >>> cm_file=ConfusionMatrix(file=open("test5.obj","r")) >>> cm_file.transpose @@ -432,15 +445,18 @@ >>> os.remove("test_filtered.csv") >>> os.remove("test_filtered_matrix.csv") >>> os.remove("test_filtered.pycm") +>>> os.remove("test_summary.pycm") >>> os.remove("test_filtered2.html") >>> os.remove("test_filtered3.html") >>> os.remove("test_filtered4.html") >>> os.remove("test_filtered5.html") +>>> os.remove("test_summary.html") >>> os.remove("test_colored.html") >>> os.remove("test_colored2.html") >>> os.remove("test_filtered2.csv") >>> os.remove("test_filtered3.csv") >>> os.remove("test_filtered4.csv") +>>> os.remove("test_summary.csv") >>> os.remove("test_filtered2.pycm") >>> os.remove("test_filtered3.pycm") >>> os.remove("test2.obj") diff --git a/Test/overall_test.py b/Test/overall_test.py index 6f043761..ee814d98 100644 --- a/Test/overall_test.py +++ b/Test/overall_test.py @@ -555,6 +555,37 @@ dInd(Distance index) None 0.67315 0.66926 1.0 sInd(Similarity index) None 0.52401 0.52676 0.29289 +>>> cm.stat(summary=True) +Overall Statistics : + +ACC Macro 0.675 +F1 Macro 0.23043 +Kappa 0.07801 +Overall ACC 0.35 +PPV Macro None +SOA1(Landis & Koch) Slight +TPR Macro None +Zero-one Loss 13 + +Class Statistics : + +Classes 100 200 500 600 +ACC(Accuracy) 0.45 0.45 0.85 0.95 +AUC(Area under the ROC curve) None 0.5625 0.63725 0.5 +AUCI(AUC value interpretation) None Poor Fair Poor +F1(F1 score - harmonic mean of precision and sensitivity) 0.0 0.52174 0.4 0.0 +FN(False negative/miss/type 2 error) 0 10 2 1 +FP(False positive/type 1 error/false alarm) 11 1 1 0 +N(Condition negative) 20 4 17 19 +P(Condition positive or support) 0 16 3 1 +POP(Population) 20 20 20 20 +PPV(Precision or positive predictive value) 0.0 0.85714 0.5 None +TN(True negative/correct rejection) 9 3 16 19 +TON(Test outcome negative) 9 13 18 20 +TOP(Test outcome positive) 11 7 2 0 +TP(True positive/hit) 0 6 1 0 +TPR(Sensitivity, recall, hit rate, or true positive rate) None 0.375 0.33333 0.0 + >>> cm.stat(overall_param=["Kappa","Scott PI"],class_param=["TPR","TNR","ACC","AUC"]) Overall Statistics : From 82b4f012337bb288cd1c5e8da9540591574b0a3f Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 13 Jul 2019 16:39:35 +0430 Subject: [PATCH 035/107] doc : Document updated for summary mode #217 --- Document/Document.ipynb | 178 +++++++++++++++++++- Document/Document_Files/cm1_summary.csv | 16 ++ Document/Document_Files/cm1_summary.html | 202 +++++++++++++++++++++++ Document/Document_Files/cm1_summary.pycm | 86 ++++++++++ 4 files changed, 478 insertions(+), 4 deletions(-) create mode 100644 Document/Document_Files/cm1_summary.csv create mode 100644 Document/Document_Files/cm1_summary.html create mode 100644 Document/Document_Files/cm1_summary.pycm diff --git a/Document/Document.ipynb b/Document/Document.ipynb index 0a807e70..5978bebd 100644 --- a/Document/Document.ipynb +++ b/Document/Document.ipynb @@ -8458,6 +8458,52 @@ "cm.stat(overall_param=[\"Kappa\"],class_param=[\"ACC\",\"AUC\",\"TPR\"],class_name=[\"L1\",\"L3\"])" ] }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overall Statistics : \n", + "\n", + "ACC Macro 0.72222\n", + "F1 Macro 0.56515\n", + "Kappa 0.35484\n", + "Overall ACC 0.58333\n", + "PPV Macro 0.61111\n", + "SOA1(Landis & Koch) Fair\n", + "TPR Macro 0.56667\n", + "Zero-one Loss 5\n", + "\n", + "Class Statistics :\n", + "\n", + "Classes L1 L2 L3 \n", + "ACC(Accuracy) 0.83333 0.75 0.58333 \n", + "AUC(Area under the ROC curve) 0.8 0.65 0.58571 \n", + "AUCI(AUC value interpretation) Very Good Fair Poor \n", + "F1(F1 score - harmonic mean of precision and sensitivity) 0.75 0.4 0.54545 \n", + "FN(False negative/miss/type 2 error) 2 1 2 \n", + "FP(False positive/type 1 error/false alarm) 0 2 3 \n", + "N(Condition negative) 7 10 7 \n", + "P(Condition positive or support) 5 2 5 \n", + "POP(Population) 12 12 12 \n", + "PPV(Precision or positive predictive value) 1.0 0.33333 0.5 \n", + "TN(True negative/correct rejection) 7 8 4 \n", + "TON(Test outcome negative) 9 9 6 \n", + "TOP(Test outcome positive) 3 3 6 \n", + "TP(True positive/hit) 3 1 3 \n", + "TPR(Sensitivity, recall, hit rate, or true positive rate) 0.6 0.5 0.6 \n", + "\n" + ] + } + ], + "source": [ + "cm.stat(summary=True)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -8471,7 +8517,8 @@ "source": [ "1. `overall_param` : overall statistics names for print (type : `list`)\n", "2. `class_param` : class statistics names for print (type : `list`)\n", - "3. `class_name` : class names for print (sub set of classes) (type : `list`)\n" + "3. `class_name` : class names for print (sub set of classes) (type : `list`)\n", + "4. `summary` : summary mode flag (type : `bool`)\n" ] }, { @@ -8501,6 +8548,15 @@ "" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
      \n", + "
    • Notice : `summary` , new in version 2.4
      • \n", + "
    " + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -8661,6 +8717,34 @@ "Open File" ] }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cm1_summary.pycm',\n", + " 'Status': True}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.save_stat(os.path.join(\"Document_Files\",\"cm1_summary\"),summary=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Open File" + ] + }, { "cell_type": "code", "execution_count": 172, @@ -8697,7 +8781,8 @@ "2. `address` : flag for address return (type : `bool`)\n", "3. `overall_param` : overall statistics names for save (type : `list`)\n", "4. `class_param` : class statistics names for save (type : `list`)\n", - "5. `class_name` : class names for print (sub set of classes) (type : `list`)" + "5. `class_name` : class names for print (sub set of classes) (type : `list`)\n", + "6. `summary` : summary mode flag (type : `bool`)" ] }, { @@ -8727,6 +8812,15 @@ "" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
      \n", + "
    • Notice : `summary` , new in version 2.4
      • \n", + "
    " + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -8902,6 +8996,34 @@ "Open File" ] }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cm1_summary.html',\n", + " 'Status': True}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.save_html(os.path.join(\"Document_Files\",\"cm1_summary\"),summary=True,normalize=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Open File" + ] + }, { "cell_type": "code", "execution_count": 179, @@ -8939,7 +9061,8 @@ "3. `overall_param` : overall statistics names for save (type : `list`)\n", "4. `class_param` : class statistics names for save (type : `list`)\n", "5. `class_name` : class names for print (sub set of classes) (type : `list`)\n", - "6. `color` : matrix color (R,G,B) (type : `tuple`/`str`)" + "6. `color` : matrix color (R,G,B) (type : `tuple`/`str`)\n", + "7. `summary` : summary mode flag (type : `bool`)" ] }, { @@ -8996,6 +9119,15 @@ "" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
      \n", + "
    • Notice : `summary` , new in version 2.4
      • \n", + "
    " + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -9123,6 +9255,34 @@ "Open Matrix File" ] }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Message': 'D:\\\\For Asus Laptop\\\\projects\\\\pycm\\\\Document\\\\Document_Files\\\\cm1_summary.csv',\n", + " 'Status': True}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cm.save_csv(os.path.join(\"Document_Files\",\"cm1_summary\"),summary=True,matrix_save=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Open Stat File" + ] + }, { "cell_type": "code", "execution_count": 184, @@ -9160,7 +9320,8 @@ "4. `class_param` : class statistics names for save (type : `list`)\n", "5. `class_name` : class names for print (sub set of classes) (type : `list`)\n", "6. `matrix_save` : flag for saving matrix in seperate CSV file (type : `bool`)\n", - "7. `normalize` : flag for saving normalized matrix instead of matrix (type : `bool`)" + "7. `normalize` : flag for saving normalized matrix instead of matrix (type : `bool`)\n", + "8. `summary` : summary mode flag (type : `bool`)" ] }, { @@ -9199,6 +9360,15 @@ "" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
      \n", + "
    • Notice : `summary` , new in version 2.4
      • \n", + "
    " + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/Document/Document_Files/cm1_summary.csv b/Document/Document_Files/cm1_summary.csv new file mode 100644 index 00000000..c3ad2329 --- /dev/null +++ b/Document/Document_Files/cm1_summary.csv @@ -0,0 +1,16 @@ +Class,"L1","L2","L3" +ACC,0.83333,0.75,0.58333 +AUC,0.8,0.65,0.58571 +AUCI,Very Good,Fair,Poor +F1,0.75,0.4,0.54545 +TPR,0.6,0.5,0.6 +PPV,1.0,0.33333,0.5 +TP,3,1,3 +FP,0,2,3 +FN,2,1,2 +TN,7,8,4 +N,7,10,7 +P,5,2,5 +POP,12,12,12 +TOP,3,3,6 +TON,9,9,6 diff --git a/Document/Document_Files/cm1_summary.html b/Document/Document_Files/cm1_summary.html new file mode 100644 index 00000000..b660938e --- /dev/null +++ b/Document/Document_Files/cm1_summary.html @@ -0,0 +1,202 @@ + + +Document_Files\cm1_summary + + +

    PyCM Report

    Dataset Type :

    +
      + +
    • Multi-Class Classification
      • + +
    • Balanced
      • +
    +

    Note 1 : Recommended statistics for this type of classification highlighted in aqua

    +

    Note 2 : The recommender system assumes that the input is the result of classification over the whole data rather than just a part of it. +If the confusion matrix is the result of test data classification, the recommendation is not valid.

    +

    Confusion Matrix (Normalized):

    + + + + + +
    ActualPredict + + + + + + + + + + + + + + + + + + + + + + + + + +
    L1L2L3
    L10.60.00.4
    L20.00.50.5
    L30.00.40.6
    +
    +

    Overall Statistics :

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    ACC Macro0.72222
    F1 Macro0.56515
    Kappa0.35484
    Overall ACC0.58333
    PPV Macro0.61111
    SOA1(Landis & Koch)Fair
    TPR Macro0.56667
    Zero-one Loss5
    +

    Class Statistics :

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    ClassL1L2L3Description
    ACC0.833330.750.58333Accuracy
    AUC0.80.650.58571Area under the ROC curve
    AUCIVery GoodFairPoorAUC value interpretation
    F10.750.40.54545F1 score - harmonic mean of precision and sensitivity
    TPR0.60.50.6Sensitivity, recall, hit rate, or true positive rate
    PPV1.00.333330.5Precision or positive predictive value
    TP313True positive/hit
    FP023False positive/type 1 error/false alarm
    FN212False negative/miss/type 2 error
    TN784True negative/correct rejection
    N7107Condition negative
    P525Condition positive or support
    POP121212Population
    TOP336Test outcome positive
    TON996Test outcome negative
    + +

    Generated By PyCM Version 2.3

    + \ No newline at end of file diff --git a/Document/Document_Files/cm1_summary.pycm b/Document/Document_Files/cm1_summary.pycm new file mode 100644 index 00000000..4532d89d --- /dev/null +++ b/Document/Document_Files/cm1_summary.pycm @@ -0,0 +1,86 @@ +Matrix : + +Predict L1 L2 L3 +Actual +L1 3 0 2 + +L2 0 1 1 + +L3 0 2 3 + + + +Normalized Matrix : + +Predict L1 L2 L3 +Actual +L1 0.6 0.0 0.4 + +L2 0.0 0.5 0.5 + +L3 0.0 0.4 0.6 + + + +Overall Statistics : + +ACC Macro 0.72222 +F1 Macro 0.56515 +Kappa 0.35484 +Overall ACC 0.58333 +PPV Macro 0.61111 +SOA1(Landis & Koch) Fair +TPR Macro 0.56667 +Zero-one Loss 5 + +Class Statistics : + +Classes L1 L2 L3 +ACC(Accuracy) 0.83333 0.75 0.58333 +AUC(Area under the ROC curve) 0.8 0.65 0.58571 +AUCI(AUC value interpretation) Very Good Fair Poor +F1(F1 score - harmonic mean of precision and sensitivity) 0.75 0.4 0.54545 +FN(False negative/miss/type 2 error) 2 1 2 +FP(False positive/type 1 error/false alarm) 0 2 3 +N(Condition negative) 7 10 7 +P(Condition positive or support) 5 2 5 +POP(Population) 12 12 12 +PPV(Precision or positive predictive value) 1.0 0.33333 0.5 +TN(True negative/correct rejection) 7 8 4 +TON(Test outcome negative) 9 9 6 +TOP(Test outcome positive) 3 3 6 +TP(True positive/hit) 3 1 3 +TPR(Sensitivity, recall, hit rate, or true positive rate) 0.6 0.5 0.6 + +One-Vs-All : + +L1-Vs-All : + +Predict L1 ~ +Actual +L1 3 2 + +~ 0 7 + + + +L2-Vs-All : + +Predict L2 ~ +Actual +L2 1 1 + +~ 2 8 + + + +L3-Vs-All : + +Predict L3 ~ +Actual +L3 3 2 + +~ 3 4 + + + From 8c1dbf0ed9c486f07967bd7c904a990af553f28c Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 13 Jul 2019 16:43:53 +0430 Subject: [PATCH 036/107] doc : references reformated #216 #225 --- Document/Document.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Document/Document.ipynb b/Document/Document.ipynb index 5978bebd..198d0307 100644 --- a/Document/Document.ipynb +++ b/Document/Document.ipynb @@ -10031,11 +10031,11 @@ "\n", "
    54- A. Tversky, \"Features of similarity,\" Psychological review, vol. 84, no. 4, p. 327, 1977.
    \n", "\n", - "
    55- Boyd, K., Eng, K.H. and Page, C.D., 2013, September. Area under the precision-recall curve: point estimates and confidence intervals. In Joint European conference on machine learning and knowledge discovery in databases (pp. 451-466). Springer, Berlin, Heidelberg.
    \n", + "
    55- K. Boyd, K. H. Eng, and C. D. Page, \"Area under the precision-recall curve: point estimates and confidence intervals,\" in Joint European conference on machine learning and knowledge discovery in databases, 2013, pp. 451-466: Springer.
    \n", "\n", - "
    56- Davis, J. and Goadrich, M., 2006, June. The relationship between Precision-Recall and ROC curves. In Proceedings of the 23rd international conference on Machine learning (pp. 233-240). ACM.
    \n", + "
    56- J. Davis and M. Goadrich, \"The relationship between Precision-Recall and ROC curves,\" in Proceedings of the 23rd international conference on Machine learning, 2006, pp. 233-240: ACM.
    \n", "\n", - "
    57- Kuhn, M., 2008. Building predictive models in R using the caret package. Journal of statistical software, 28(5), pp.1-26.
    " + "
    57- M. Kuhn, \"Building predictive models in R using the caret package,\" Journal of statistical software, vol. 28, no. 5, pp. 1-26, 2008.
    " ] } ], From 0d249c328903f3c1adade2af464de4090090b81f Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 13 Jul 2019 16:52:34 +0430 Subject: [PATCH 037/107] doc : CHANGELOG updated #217 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47ad7857..292489c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - `AUC_calc` function modified - Document modified +- `summary` parameter added to `save_html`,`save_stat`,`save_csv` and `stat` methods ## [2.3] - 2019-06-27 ### Added - Adjusted F-score (AGF) From 691d2c34b17febe3d62bc5287cec6b45e15631aa Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sun, 14 Jul 2019 18:14:41 +0430 Subject: [PATCH 038/107] fix : update exceptions class in pycm_class_func #227 --- pycm/pycm_class_func.py | 58 ++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/pycm/pycm_class_func.py b/pycm/pycm_class_func.py index 07bdc0a0..4bc7d16e 100644 --- a/pycm/pycm_class_func.py +++ b/pycm/pycm_class_func.py @@ -24,7 +24,7 @@ def TI_calc(TP, FP, FN, alpha, beta): try: TI = TP / (TP + alpha * FN + beta * FP) return TI - except Exception: + except (ZeroDivisionError, TypeError): return "None" @@ -43,7 +43,7 @@ def OOC_calc(TP, TOP, P): try: OOC = TP / (math.sqrt(TOP * P)) return OOC - except Exception: + except (ZeroDivisionError, TypeError, ValueError): return "None" @@ -62,7 +62,7 @@ def OC_calc(TP, TOP, P): try: overlap_coef = TP / min(TOP, P) return overlap_coef - except Exception: + except (ZeroDivisionError, TypeError): return "None" @@ -85,7 +85,7 @@ def AGF_calc(TP, FP, FN, TN): F05_inv = F_calc(TP=TN, FP=FN, FN=FP, beta=0.5) AGF = math.sqrt(F2 * F05_inv) return AGF - except Exception: + except (TypeError, ValueError): return "None" @@ -112,7 +112,7 @@ def AGM_calc(TPR, TNR, GM, N, POP): else: result = (GM + TNR * n) / (1 + n) return result - except Exception: + except (ZeroDivisionError, TypeError): return "None" @@ -134,7 +134,7 @@ def Q_calc(TP, TN, FP, FN): OR = (TP * TN) / (FP * FN) result = (OR - 1) / (OR + 1) return result - except Exception: + except (ZeroDivisionError, TypeError): return "None" @@ -151,7 +151,7 @@ def TTPN_calc(item1, item2): try: result = item1 / (item1 + item2) return result - except ZeroDivisionError: + except (ZeroDivisionError, TypeError): return "None" @@ -166,7 +166,7 @@ def FXR_calc(item): try: result = 1 - item return result - except Exception: + except TypeError: return "None" @@ -248,7 +248,7 @@ def MK_BM_calc(item1, item2): try: result = item1 + item2 - 1 return result - except Exception: + except TypeError: return "None" @@ -265,7 +265,7 @@ def LR_calc(item1, item2): try: result = item1 / item2 return result - except Exception: + except (ZeroDivisionError, TypeError): return "None" @@ -282,7 +282,7 @@ def PRE_calc(P, POP): try: result = P / POP return result - except Exception: + except (ZeroDivisionError, TypeError): return "None" @@ -299,7 +299,7 @@ def G_calc(item1, item2): try: result = math.sqrt(item1 * item2) return result - except Exception: + except (TypeError, ValueError): return "None" @@ -318,7 +318,7 @@ def RACC_calc(TOP, P, POP): try: result = (TOP * P) / ((POP) ** 2) return result - except Exception: + except (ZeroDivisionError, TypeError): return "None" @@ -337,7 +337,7 @@ def RACCU_calc(TOP, P, POP): try: result = ((TOP + P) / (2 * POP))**2 return result - except Exception: + except (ZeroDivisionError, TypeError): return "None" @@ -351,7 +351,7 @@ def ERR_calc(ACC): """ try: return 1 - ACC - except Exception: + except TypeError: return "None" @@ -369,7 +369,7 @@ def jaccard_index_calc(TP, TOP, P): """ try: return TP / (TOP + P - TP) - except Exception: + except (ZeroDivisionError, TypeError): return "None" @@ -391,7 +391,7 @@ def IS_calc(TP, FP, FN, POP): result = -math.log(((TP + FN) / POP), 2) + \ math.log((TP / (TP + FP)), 2) return result - except Exception: + except (ZeroDivisionError, TypeError, ValueError): return "None" @@ -428,7 +428,7 @@ def CEN_misclassification_calc( result -= table[subject_class][subject_class] result = table[i][j] / result return result - except Exception: + except (ZeroDivisionError, TypeError): return "None" @@ -466,7 +466,7 @@ def CEN_calc(classes, table, TOP, P, class_name, modified=False): if result != 0: result = result * (-1) return result - except Exception: + except (ZeroDivisionError, TypeError, ValueError): return "None" @@ -482,7 +482,7 @@ def AUC_calc(item, TPR): """ try: return (item + TPR) / 2 - except Exception: + except TypeError: return "None" @@ -499,7 +499,7 @@ def dInd_calc(TNR, TPR): try: result = math.sqrt(((1 - TNR)**2) + ((1 - TPR)**2)) return result - except Exception: + except TypeError: return "None" @@ -513,7 +513,7 @@ def sInd_calc(dInd): """ try: return 1 - (dInd / (math.sqrt(2))) - except Exception: + except (ZeroDivisionError, TypeError): return "None" @@ -531,7 +531,7 @@ def DP_calc(TPR, TNR): X = TPR / (1 - TPR) Y = TNR / (1 - TNR) return (math.sqrt(3) / math.pi) * (math.log(X, 10) + math.log(Y, 10)) - except Exception: + except (ZeroDivisionError, TypeError, ValueError): return "None" @@ -545,7 +545,7 @@ def GI_calc(AUC): """ try: return 2 * AUC - 1 - except Exception: + except TypeError: return "None" @@ -561,7 +561,7 @@ def lift_calc(PPV, PRE): """ try: return PPV / PRE - except Exception: + except (ZeroDivisionError, TypeError): return "None" @@ -577,7 +577,7 @@ def AM_calc(TOP, P): """ try: return TOP - P - except Exception: + except TypeError: return "None" @@ -596,7 +596,7 @@ def OP_calc(ACC, TPR, TNR): try: RI = abs(TNR - TPR) / (TPR + TNR) return ACC - RI - except Exception: + except (ZeroDivisionError, TypeError): return "None" @@ -615,7 +615,7 @@ def IBA_calc(TPR, TNR, alpha=1): try: IBA = (1 + alpha * (TPR - TNR)) * TPR * TNR return IBA - except Exception: + except TypeError: return "None" @@ -635,7 +635,7 @@ def BCD_calc(TOP, P, AM): TOP_sum = sum(TOP.values()) P_sum = sum(P.values()) return abs(AM) / (P_sum + TOP_sum) - except Exception: + except (ZeroDivisionError, TypeError, AttributeError): return "None" From e793af9a7fde6f1fe6d7b25a5123ca80db0eaa2a Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 15 Jul 2019 16:04:04 +0430 Subject: [PATCH 039/107] fix : update exceptions class in pycm_class_func #227 --- pycm/pycm_class_func.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pycm/pycm_class_func.py b/pycm/pycm_class_func.py index 4bc7d16e..e74d590d 100644 --- a/pycm/pycm_class_func.py +++ b/pycm/pycm_class_func.py @@ -187,7 +187,7 @@ def ACC_calc(TP, TN, FP, FN): try: result = (TP + TN) / (TP + TN + FN + FP) return result - except ZeroDivisionError: + except (ZeroDivisionError, TypeError): return "None" @@ -209,7 +209,7 @@ def F_calc(TP, FP, FN, beta): result = ((1 + (beta)**2) * TP) / \ ((1 + (beta)**2) * TP + FP + (beta**2) * FN) return result - except ZeroDivisionError: + except (ZeroDivisionError, TypeError): return "None" @@ -231,7 +231,7 @@ def MCC_calc(TP, TN, FP, FN): result = (TP * TN - FP * FN) / \ (math.sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN))) return result - except ZeroDivisionError: + except (ZeroDivisionError, TypeError, ValueError): return "None" @@ -499,7 +499,7 @@ def dInd_calc(TNR, TPR): try: result = math.sqrt(((1 - TNR)**2) + ((1 - TPR)**2)) return result - except TypeError: + except (TypeError, ValueError): return "None" From a6d0b1c93f76fed43f89f88f1987c8e35bd6708d Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 15 Jul 2019 16:09:48 +0430 Subject: [PATCH 040/107] fix : testcases updated #227 --- Test/function_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Test/function_test.py b/Test/function_test.py index 17101362..2d800731 100644 --- a/Test/function_test.py +++ b/Test/function_test.py @@ -150,6 +150,8 @@ 'None' >>> F_calc(TP=3,FP=2,FN=1,beta=5) 0.7428571428571429 +>>> TI_calc("None",0,0,0,0) +'None' >>> ERR_calc(None) 'None' >>> ERR_calc(0.1) From eba980a27f2a85ef7ce8e892481289d49efba2ef Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 15 Jul 2019 16:18:16 +0430 Subject: [PATCH 041/107] fix : sample_weight bug for numpy.array fixed #227 --- pycm/pycm_obj.py | 9 +++++---- pycm/pycm_util.py | 2 ++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pycm/pycm_obj.py b/pycm/pycm_obj.py index 22d1a360..cc13d83a 100644 --- a/pycm/pycm_obj.py +++ b/pycm/pycm_obj.py @@ -369,6 +369,7 @@ def save_obj( obj_file = open(name + ".obj", "w") actual_vector_temp = self.actual_vector predict_vector_temp = self.predict_vector + weights_vector_temp = self.weights matrix_temp = {k: self.table[k].copy() for k in self.classes} matrix_items = [] for i in self.classes: @@ -377,11 +378,13 @@ def save_obj( actual_vector_temp = actual_vector_temp.tolist() if isinstance(predict_vector_temp, numpy.ndarray): predict_vector_temp = predict_vector_temp.tolist() + if isinstance(weights_vector_temp, numpy.ndarray): + weights_vector_temp = weights_vector_temp.tolist() dump_dict = {"Actual-Vector": actual_vector_temp, "Predict-Vector": predict_vector_temp, "Matrix": matrix_items, "Digit": self.digit, - "Sample-Weight": self.weights, + "Sample-Weight": weights_vector_temp, "Transpose": self.transpose} if save_stat: dump_dict["Class-Stat"] = self.class_stat @@ -798,9 +801,7 @@ def __obj_vector_handler__( actual_vector, predict_vector) matrix_param = matrix_params_calc( actual_vector, predict_vector, sample_weight) - if isinstance(sample_weight, list): + if isinstance(sample_weight, (list, numpy.ndarray)): cm.weights = sample_weight - if isinstance(sample_weight, numpy.ndarray): - cm.weights = sample_weight.tolist() return matrix_param diff --git a/pycm/pycm_util.py b/pycm/pycm_util.py index 7f41764f..64cd2147 100644 --- a/pycm/pycm_util.py +++ b/pycm/pycm_util.py @@ -276,6 +276,8 @@ def matrix_params_calc(actual_vector, predict_vector, sample_weight): actual_vector = actual_vector.tolist() if isinstance(predict_vector, numpy.ndarray): predict_vector = predict_vector.tolist() + if isinstance(sample_weight, numpy.ndarray): + sample_weight = sample_weight.tolist() classes = set(actual_vector).union(set(predict_vector)) classes = sorted(classes) map_dict = {k: 0 for k in classes} From db73ecae3e3b05b4ffba6ea7641572712e258464 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 15 Jul 2019 16:36:33 +0430 Subject: [PATCH 042/107] fix : testcases updated #227 --- Test/function_test.py | 7 +++++-- Test/output_test.py | 8 ++++++++ Test/overall_test.py | 5 +++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Test/function_test.py b/Test/function_test.py index 2d800731..79b2aec2 100644 --- a/Test/function_test.py +++ b/Test/function_test.py @@ -162,15 +162,18 @@ 0.34 >>> cm.F_beta(4)["L3"] 0.504950495049505 ->>> cm.F_beta(None) -{} +>>> cm.F_beta(None) == {'L3': 'None', 'L1': 'None', 'L2': 'None'} +True >>> cm.IBA_alpha(None) == {'L3': 'None', 'L1': 'None', 'L2': 'None'} True >>> del cm.classes +>>> del cm.TP >>> cm.IBA_alpha(2) {} >>> cm.TI(2,3) {} +>>> cm.F_beta(2) +{} >>> kappa_analysis_koch(-0.1) 'Poor' >>> kappa_analysis_koch(0) diff --git a/Test/output_test.py b/Test/output_test.py index 68119230..1fadd1e4 100644 --- a/Test/output_test.py +++ b/Test/output_test.py @@ -264,6 +264,13 @@ >>> save_obj=={'Status': True, 'Message': None} True >>> cm_file_3=ConfusionMatrix(file=open("test3.obj","r")) +>>> cm = ConfusionMatrix(y_actu, y_pred, sample_weight=np.array([2, 2, 2, 2, 3, 1, 1, 2, 2, 1, 1, 2])) +>>> save_obj=cm.save_obj("test3_np",address=False) +>>> save_obj=={'Status': True, 'Message': None} +True +>>> cm_file_3_np=ConfusionMatrix(file=open("test3_np.obj","r")) +>>> cm_file_3_np == cm_file_3 +True >>> cm_file_3.print_matrix() Predict 0 1 2 Actual @@ -461,6 +468,7 @@ >>> os.remove("test_filtered3.pycm") >>> os.remove("test2.obj") >>> os.remove("test3.obj") +>>> os.remove("test3_np.obj") >>> os.remove("test4.obj") >>> os.remove("test5.obj") >>> os.remove("test6.obj") diff --git a/Test/overall_test.py b/Test/overall_test.py index ee814d98..9104e737 100644 --- a/Test/overall_test.py +++ b/Test/overall_test.py @@ -1103,6 +1103,11 @@ dInd(Distance index) 0.26667 0.67586 0.54716 sInd(Similarity index) 0.81144 0.52209 0.6131 +>>> cm2 = ConfusionMatrix(y_actu, y_pred, sample_weight=np.array(weight)) +>>> isinstance(cm2.weights,np.ndarray) +True +>>> cm2 == cm +True >>> cm = ConfusionMatrix([1,2,3,4],[1,2,3,"4"]) >>> cm pycm.ConfusionMatrix(classes: ['1', '2', '3', '4']) From 0b379813eed7f4dd305ac9e1416b49686b5f389b Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 15 Jul 2019 18:38:20 +0430 Subject: [PATCH 043/107] doc : CHANGELOG updated #226,#227 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 292489c0..454cee3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `AUC_calc` function modified - Document modified - `summary` parameter added to `save_html`,`save_stat`,`save_csv` and `stat` methods +- `sample_weight` bug in `numpy` array format fixed +- Inputs manipulation bug fixed ## [2.3] - 2019-06-27 ### Added - Adjusted F-score (AGF) From 227b92b17194564503e1d8f702d30056cf79ef26 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 15 Jul 2019 18:38:41 +0430 Subject: [PATCH 044/107] fix : autopep8 fix --- pycm/pycm_obj.py | 7 ++++++- pycm/pycm_param.py | 27 +++++++++++++++++++++++++-- pycm/pycm_util.py | 2 +- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/pycm/pycm_obj.py b/pycm/pycm_obj.py index cc13d83a..ad66159c 100644 --- a/pycm/pycm_obj.py +++ b/pycm/pycm_obj.py @@ -126,7 +126,12 @@ def print_normalized_matrix(self, one_vs_all=False, class_name=None): table = normalized_table_calc(classes, table) print(table_print(classes, table)) - def stat(self, overall_param=None, class_param=None, class_name=None, summary=False): + def stat( + self, + overall_param=None, + class_param=None, + class_name=None, + summary=False): """ Print statistical measures table. diff --git a/pycm/pycm_param.py b/pycm/pycm_param.py index 7c914074..92909b37 100644 --- a/pycm/pycm_param.py +++ b/pycm/pycm_param.py @@ -46,9 +46,32 @@ BALANCE_RATIO_THRESHOLD = 3 -SUMMARY_OVERALL = ["ACC Macro","Kappa","Overall ACC","SOA1(Landis & Koch)","Zero-one Loss","F1 Macro","TPR Macro","PPV Macro"] +SUMMARY_OVERALL = [ + "ACC Macro", + "Kappa", + "Overall ACC", + "SOA1(Landis & Koch)", + "Zero-one Loss", + "F1 Macro", + "TPR Macro", + "PPV Macro"] -SUMMARY_CLASS = ["ACC","AUC","AUCI","F1","TPR","PPV","TP","FP","FN","TN","N","P","POP","TOP","TON"] +SUMMARY_CLASS = [ + "ACC", + "AUC", + "AUCI", + "F1", + "TPR", + "PPV", + "TP", + "FP", + "FN", + "TN", + "N", + "P", + "POP", + "TOP", + "TON"] BINARY_RECOMMEND = ["ACC", "TPR", "PPV", "AUC", "AUCI", "TNR", "F1"] diff --git a/pycm/pycm_util.py b/pycm/pycm_util.py index 64cd2147..39d93ab2 100644 --- a/pycm/pycm_util.py +++ b/pycm/pycm_util.py @@ -217,7 +217,7 @@ def transpose_func(classes, table): :type table : dict :return: transposed table as dict """ - transposed_table = {k:table[k].copy() for k in classes} + transposed_table = {k: table[k].copy() for k in classes} for i, item1 in enumerate(classes): for j, item2 in enumerate(classes): if i > j: From c01e79cfef2dd4505b188cacdc6e6e95bd8dbf80 Mon Sep 17 00:00:00 2001 From: alirezazolanvari Date: Sat, 20 Jul 2019 10:25:21 +0430 Subject: [PATCH 045/107] pycm_compare fixed #228 --- pycm/pycm_compare.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pycm/pycm_compare.py b/pycm/pycm_compare.py index 594bd35c..6b312877 100644 --- a/pycm/pycm_compare.py +++ b/pycm/pycm_compare.py @@ -7,7 +7,7 @@ from .pycm_obj import ConfusionMatrix import os import numpy - +from warnings import warn class pycmCompareError(Exception): """Compare error class.""" @@ -91,9 +91,11 @@ def __init__(self, cm_dict, by_class=False, weight=None, digit=5): self.best = cm_dict[max_class_name] self.best_name = max_overall_name else: - print('Warning: ' + COMPARE_RESULT_WARNING) + warn(COMPARE_RESULT_WARNING, RuntimeWarning) + # print('Warning: ' + COMPARE_RESULT_WARNING) else: - print('Warning: ' + COMPARE_RESULT_WARNING) + warn(COMPARE_RESULT_WARNING, RuntimeWarning) + # print('Warning: ' + COMPARE_RESULT_WARNING) def print_report(self): """ From b4b7802bfad842573b68809b553c2c101d58645a Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 20 Jul 2019 16:02:59 +0430 Subject: [PATCH 046/107] fix : table_print function warnings fixed #228 --- pycm/pycm_obj.py | 11 ++++++++++- pycm/pycm_output.py | 2 -- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pycm/pycm_obj.py b/pycm/pycm_obj.py index ad66159c..d89d5592 100644 --- a/pycm/pycm_obj.py +++ b/pycm/pycm_obj.py @@ -107,6 +107,8 @@ def print_matrix(self, one_vs_all=False, class_name=None): [classes, table] = one_vs_all_func( classes, table, self.TP, self.TN, self.FP, self.FN, class_name) print(table_print(classes, table)) + if len(classes) >= CLASS_NUMBER_THRESHOLD: + warn(CLASS_NUMBER_WARNING, RuntimeWarning) def print_normalized_matrix(self, one_vs_all=False, class_name=None): """ @@ -125,6 +127,8 @@ def print_normalized_matrix(self, one_vs_all=False, class_name=None): classes, table, self.TP, self.TN, self.FP, self.FN, class_name) table = normalized_table_calc(classes, table) print(table_print(classes, table)) + if len(classes) >= CLASS_NUMBER_THRESHOLD: + warn(CLASS_NUMBER_WARNING, RuntimeWarning) def stat( self, @@ -168,6 +172,8 @@ def __str__(self): result += "\n" * 4 result += stat_print(self.classes, self.class_stat, self.overall_stat, self.digit) + if len(self.classes) >= CLASS_NUMBER_THRESHOLD: + warn(CLASS_NUMBER_WARNING, RuntimeWarning) return result def save_stat( @@ -199,6 +205,7 @@ def save_stat( message = None class_list = class_param overall_list = overall_param + warning_message = "" if summary: class_list = SUMMARY_CLASS overall_list = SUMMARY_OVERALL @@ -221,7 +228,9 @@ def save_stat( self.class_stat, self.overall_stat, self.digit, overall_list, class_list) - file.write(matrix + normalized_matrix + stat + one_vs_all) + if len(self.classes) >= CLASS_NUMBER_THRESHOLD: + warning_message = "\n" + "Warning : " + CLASS_NUMBER_WARNING + "\n" + file.write(matrix + normalized_matrix + stat + one_vs_all + warning_message) file.close() if address: message = os.path.join( diff --git a/pycm/pycm_output.py b/pycm/pycm_output.py index 2de245a1..40ab4a95 100644 --- a/pycm/pycm_output.py +++ b/pycm/pycm_output.py @@ -308,8 +308,6 @@ def table_print(classes, table): row = [table[key][i] for i in classes] result += shift % str(key) + \ shift * classes_len % tuple(map(str, row)) + "\n\n" - if classes_len >= CLASS_NUMBER_THRESHOLD: - result += "\n" + "Warning : " + CLASS_NUMBER_WARNING + "\n" return result From ccad71025887b4e10233792cf0dffc58acb0d238 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 20 Jul 2019 16:12:07 +0430 Subject: [PATCH 047/107] fix : stat_print function warnings fixed #228 --- pycm/pycm_obj.py | 2 ++ pycm/pycm_output.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pycm/pycm_obj.py b/pycm/pycm_obj.py index d89d5592..32f6285f 100644 --- a/pycm/pycm_obj.py +++ b/pycm/pycm_obj.py @@ -161,6 +161,8 @@ def stat( self.class_stat, self.overall_stat, self.digit, overall_list, class_list)) + if len(classes) >= CLASS_NUMBER_THRESHOLD: + warn(CLASS_NUMBER_WARNING, RuntimeWarning) def __str__(self): """ diff --git a/pycm/pycm_output.py b/pycm/pycm_output.py index 40ab4a95..f4c2c2e9 100644 --- a/pycm/pycm_output.py +++ b/pycm/pycm_output.py @@ -419,8 +419,6 @@ def stat_print( result += key + "(" + params_text + ")" + " " * ( shift - len(key) - len(PARAMS_DESCRIPTION[key]) + 5) + class_shift_format * classes_len % tuple( map(rounder_map, row)) + "\n" - if classes_len >= CLASS_NUMBER_THRESHOLD: - result += "\n" + "Warning : " + CLASS_NUMBER_WARNING + "\n" return result From b5e63512d0f26905e0a882602cfe02596743eec9 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 20 Jul 2019 16:15:38 +0430 Subject: [PATCH 048/107] fix : comapre testcases modified #228 --- Test/overall_test.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Test/overall_test.py b/Test/overall_test.py index 9104e737..1f175576 100644 --- a/Test/overall_test.py +++ b/Test/overall_test.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """ >>> from pycm import * +>>> from pytest import warns >>> import os >>> import json >>> y_actu = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2] @@ -1183,8 +1184,8 @@ pycm.ConfusionMatrix(classes: [0, 1, 2]) >>> cp.best_name 'model2' ->>> cp2 = Compare({"model1":cm_comp1,"model2":cm_comp1}) -Warning: Confusion matrices are too close and the best one can not be recognized. +>>> with warns(RuntimeWarning, match='Confusion matrices are too close'): +... cp2 = Compare({"model1":cm_comp1,"model2":cm_comp1}) >>> cp2.scores == {'model1': {'overall': 2.55, 'class': 7.05}, 'model2': {'overall': 2.55, 'class': 7.05}} True >>> cp2.best @@ -1199,8 +1200,8 @@ 1 cm2 10.7 5.8 2 cm1 7.9 4.48333 ->>> cp3 = Compare({"cm1":cm1,"cm2":cm2},weight={0:200,1:1,2:1}) -Warning: Confusion matrices are too close and the best one can not be recognized. +>>> with warns(RuntimeWarning, match='Confusion matrices are too close'): +... cp3 = Compare({"cm1":cm1,"cm2":cm2},weight={0:200,1:1,2:1}) >>> print(cp3) Best : None From bb7d0cf6bce0536c8ad1f5c29b672ce94783737c Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 20 Jul 2019 16:22:30 +0430 Subject: [PATCH 049/107] fix : warnings imported in pycm_obj #228 --- pycm/pycm_obj.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pycm/pycm_obj.py b/pycm/pycm_obj.py index 32f6285f..2b011ec0 100644 --- a/pycm/pycm_obj.py +++ b/pycm/pycm_obj.py @@ -10,6 +10,7 @@ import json import types import numpy +from warnings import warn class pycmVectorError(Exception): From 9c3006b25def61b0d1cfbeba92f148770aea9791 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 20 Jul 2019 16:44:42 +0430 Subject: [PATCH 050/107] fix : overall_test updated for large cm #228 --- Test/overall_test.py | 315 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 315 insertions(+) diff --git a/Test/overall_test.py b/Test/overall_test.py index 1f175576..bd56c4f4 100644 --- a/Test/overall_test.py +++ b/Test/overall_test.py @@ -1132,6 +1132,321 @@ >>> cm = ConfusionMatrix(matrix={1:{1:60,2:9,3:1,4:0,5:0,6:0},2:{1:23,2:48,3:0,4:2,5:2,6:1},3:{1:11,2:5,3:60,4:0,5:0,6:0},4:{1:0,2:2,3:0,4:60,5:1,6:3},5:{1:2,2:1,3:0,4:0,5:60,6:2},6:{1:1,2:2,3:0,4:2,5:1,6:60}}) >>> set(cm.recommended_list) == set(MULTICLASS_RECOMMEND) True +>>> large_cm = ConfusionMatrix(list(range(10)),list(range(10))) +>>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'): +... large_cm.print_matrix() +Predict 0 1 2 3 4 5 6 7 8 9 +Actual +0 1 0 0 0 0 0 0 0 0 0 + +1 0 1 0 0 0 0 0 0 0 0 + +2 0 0 1 0 0 0 0 0 0 0 + +3 0 0 0 1 0 0 0 0 0 0 + +4 0 0 0 0 1 0 0 0 0 0 + +5 0 0 0 0 0 1 0 0 0 0 + +6 0 0 0 0 0 0 1 0 0 0 + +7 0 0 0 0 0 0 0 1 0 0 + +8 0 0 0 0 0 0 0 0 1 0 + +9 0 0 0 0 0 0 0 0 0 1 + +>>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'): +... large_cm.print_normalized_matrix() +Predict 0 1 2 3 4 5 6 7 8 9 +Actual +0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 + +1 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 + +2 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 + +3 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 + +4 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 + +5 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 + +6 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 + +7 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 + +8 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 + +9 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 + +>>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'): +... large_cm.stat() +Overall Statistics : + +95% CI (1.0,1.0) +ACC Macro 1.0 +AUNP 1.0 +AUNU 1.0 +Bennett S 1.0 +CBA 1.0 +Chi-Squared 90.0 +Chi-Squared DF 81 +Conditional Entropy -0.0 +Cramer V 1.0 +Cross Entropy 3.32193 +F1 Macro 1.0 +F1 Micro 1.0 +Gwet AC1 1.0 +Hamming Loss 0.0 +Joint Entropy 3.32193 +KL Divergence 0.0 +Kappa 1.0 +Kappa 95% CI (1.0,1.0) +Kappa No Prevalence 1.0 +Kappa Standard Error 0.0 +Kappa Unbiased 1.0 +Lambda A 1.0 +Lambda B 1.0 +Mutual Information 3.32193 +NIR 0.1 +Overall ACC 1.0 +Overall CEN 0.0 +Overall J (10.0,1.0) +Overall MCC 1.0 +Overall MCEN 0.0 +Overall RACC 0.1 +Overall RACCU 0.1 +P-Value 0.0 +PPV Macro 1.0 +PPV Micro 1.0 +Pearson C 0.94868 +Phi-Squared 9.0 +RCI 1.0 +RR 1.0 +Reference Entropy 3.32193 +Response Entropy 3.32193 +SOA1(Landis & Koch) Almost Perfect +SOA2(Fleiss) Excellent +SOA3(Altman) Very Good +SOA4(Cicchetti) Excellent +SOA5(Cramer) Very Strong +SOA6(Matthews) Very Strong +Scott PI 1.0 +Standard Error 0.0 +TPR Macro 1.0 +TPR Micro 1.0 +Zero-one Loss 0 + +Class Statistics : + +Classes 0 1 2 3 4 5 6 7 8 9 +ACC(Accuracy) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +AGF(Adjusted F-score) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +AGM(Adjusted geometric mean) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +AM(Difference between automatic and manual classification) 0 0 0 0 0 0 0 0 0 0 +AUC(Area under the ROC curve) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +AUCI(AUC value interpretation) Excellent Excellent Excellent Excellent Excellent Excellent Excellent Excellent Excellent Excellent +AUPR(Area under the PR curve) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +BCD(Bray-Curtis dissimilarity) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +BM(Informedness or bookmaker informedness) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +CEN(Confusion entropy) 0 0 0 0 0 0 0 0 0 0 +DOR(Diagnostic odds ratio) None None None None None None None None None None +DP(Discriminant power) None None None None None None None None None None +DPI(Discriminant power interpretation) None None None None None None None None None None +ERR(Error rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +F0.5(F0.5 score) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +F1(F1 score - harmonic mean of precision and sensitivity) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +F2(F2 score) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +FDR(False discovery rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +FN(False negative/miss/type 2 error) 0 0 0 0 0 0 0 0 0 0 +FNR(Miss rate or false negative rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +FOR(False omission rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +FP(False positive/type 1 error/false alarm) 0 0 0 0 0 0 0 0 0 0 +FPR(Fall-out or false positive rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +G(G-measure geometric mean of precision and sensitivity) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +GI(Gini index) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +GM(G-mean geometric mean of specificity and sensitivity) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +IBA(Index of balanced accuracy) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +IS(Information score) 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 +J(Jaccard index) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +LS(Lift score) 10.0 10.0 10.0 10.0 10.0 10.0 10.0 10.0 10.0 10.0 +MCC(Matthews correlation coefficient) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +MCCI(Matthews correlation coefficient interpretation) Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong +MCEN(Modified confusion entropy) 0 0 0 0 0 0 0 0 0 0 +MK(Markedness) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +N(Condition negative) 9 9 9 9 9 9 9 9 9 9 +NLR(Negative likelihood ratio) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +NLRI(Negative likelihood ratio interpretation) Good Good Good Good Good Good Good Good Good Good +NPV(Negative predictive value) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +OC(Overlap coefficient) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +OOC(Otsuka-Ochiai coefficient) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +OP(Optimized precision) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +P(Condition positive or support) 1 1 1 1 1 1 1 1 1 1 +PLR(Positive likelihood ratio) None None None None None None None None None None +PLRI(Positive likelihood ratio interpretation) None None None None None None None None None None +POP(Population) 10 10 10 10 10 10 10 10 10 10 +PPV(Precision or positive predictive value) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +PRE(Prevalence) 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 +Q(Yule Q - coefficient of colligation) None None None None None None None None None None +RACC(Random accuracy) 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 +RACCU(Random accuracy unbiased) 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 +TN(True negative/correct rejection) 9 9 9 9 9 9 9 9 9 9 +TNR(Specificity or true negative rate) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +TON(Test outcome negative) 9 9 9 9 9 9 9 9 9 9 +TOP(Test outcome positive) 1 1 1 1 1 1 1 1 1 1 +TP(True positive/hit) 1 1 1 1 1 1 1 1 1 1 +TPR(Sensitivity, recall, hit rate, or true positive rate) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +Y(Youden index) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +dInd(Distance index) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +sInd(Similarity index) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 + +>>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'): +... print(large_cm) +Predict 0 1 2 3 4 5 6 7 8 9 +Actual +0 1 0 0 0 0 0 0 0 0 0 + +1 0 1 0 0 0 0 0 0 0 0 + +2 0 0 1 0 0 0 0 0 0 0 + +3 0 0 0 1 0 0 0 0 0 0 + +4 0 0 0 0 1 0 0 0 0 0 + +5 0 0 0 0 0 1 0 0 0 0 + +6 0 0 0 0 0 0 1 0 0 0 + +7 0 0 0 0 0 0 0 1 0 0 + +8 0 0 0 0 0 0 0 0 1 0 + +9 0 0 0 0 0 0 0 0 0 1 + + + +Overall Statistics : + +95% CI (1.0,1.0) +ACC Macro 1.0 +AUNP 1.0 +AUNU 1.0 +Bennett S 1.0 +CBA 1.0 +Chi-Squared 90.0 +Chi-Squared DF 81 +Conditional Entropy -0.0 +Cramer V 1.0 +Cross Entropy 3.32193 +F1 Macro 1.0 +F1 Micro 1.0 +Gwet AC1 1.0 +Hamming Loss 0.0 +Joint Entropy 3.32193 +KL Divergence 0.0 +Kappa 1.0 +Kappa 95% CI (1.0,1.0) +Kappa No Prevalence 1.0 +Kappa Standard Error 0.0 +Kappa Unbiased 1.0 +Lambda A 1.0 +Lambda B 1.0 +Mutual Information 3.32193 +NIR 0.1 +Overall ACC 1.0 +Overall CEN 0.0 +Overall J (10.0,1.0) +Overall MCC 1.0 +Overall MCEN 0.0 +Overall RACC 0.1 +Overall RACCU 0.1 +P-Value 0.0 +PPV Macro 1.0 +PPV Micro 1.0 +Pearson C 0.94868 +Phi-Squared 9.0 +RCI 1.0 +RR 1.0 +Reference Entropy 3.32193 +Response Entropy 3.32193 +SOA1(Landis & Koch) Almost Perfect +SOA2(Fleiss) Excellent +SOA3(Altman) Very Good +SOA4(Cicchetti) Excellent +SOA5(Cramer) Very Strong +SOA6(Matthews) Very Strong +Scott PI 1.0 +Standard Error 0.0 +TPR Macro 1.0 +TPR Micro 1.0 +Zero-one Loss 0 + +Class Statistics : + +Classes 0 1 2 3 4 5 6 7 8 9 +ACC(Accuracy) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +AGF(Adjusted F-score) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +AGM(Adjusted geometric mean) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +AM(Difference between automatic and manual classification) 0 0 0 0 0 0 0 0 0 0 +AUC(Area under the ROC curve) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +AUCI(AUC value interpretation) Excellent Excellent Excellent Excellent Excellent Excellent Excellent Excellent Excellent Excellent +AUPR(Area under the PR curve) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +BCD(Bray-Curtis dissimilarity) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +BM(Informedness or bookmaker informedness) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +CEN(Confusion entropy) 0 0 0 0 0 0 0 0 0 0 +DOR(Diagnostic odds ratio) None None None None None None None None None None +DP(Discriminant power) None None None None None None None None None None +DPI(Discriminant power interpretation) None None None None None None None None None None +ERR(Error rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +F0.5(F0.5 score) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +F1(F1 score - harmonic mean of precision and sensitivity) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +F2(F2 score) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +FDR(False discovery rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +FN(False negative/miss/type 2 error) 0 0 0 0 0 0 0 0 0 0 +FNR(Miss rate or false negative rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +FOR(False omission rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +FP(False positive/type 1 error/false alarm) 0 0 0 0 0 0 0 0 0 0 +FPR(Fall-out or false positive rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +G(G-measure geometric mean of precision and sensitivity) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +GI(Gini index) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +GM(G-mean geometric mean of specificity and sensitivity) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +IBA(Index of balanced accuracy) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +IS(Information score) 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 +J(Jaccard index) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +LS(Lift score) 10.0 10.0 10.0 10.0 10.0 10.0 10.0 10.0 10.0 10.0 +MCC(Matthews correlation coefficient) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +MCCI(Matthews correlation coefficient interpretation) Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong +MCEN(Modified confusion entropy) 0 0 0 0 0 0 0 0 0 0 +MK(Markedness) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +N(Condition negative) 9 9 9 9 9 9 9 9 9 9 +NLR(Negative likelihood ratio) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +NLRI(Negative likelihood ratio interpretation) Good Good Good Good Good Good Good Good Good Good +NPV(Negative predictive value) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +OC(Overlap coefficient) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +OOC(Otsuka-Ochiai coefficient) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +OP(Optimized precision) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +P(Condition positive or support) 1 1 1 1 1 1 1 1 1 1 +PLR(Positive likelihood ratio) None None None None None None None None None None +PLRI(Positive likelihood ratio interpretation) None None None None None None None None None None +POP(Population) 10 10 10 10 10 10 10 10 10 10 +PPV(Precision or positive predictive value) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +PRE(Prevalence) 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 +Q(Yule Q - coefficient of colligation) None None None None None None None None None None +RACC(Random accuracy) 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 +RACCU(Random accuracy unbiased) 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 +TN(True negative/correct rejection) 9 9 9 9 9 9 9 9 9 9 +TNR(Specificity or true negative rate) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +TON(Test outcome negative) 9 9 9 9 9 9 9 9 9 9 +TOP(Test outcome positive) 1 1 1 1 1 1 1 1 1 1 +TP(True positive/hit) 1 1 1 1 1 1 1 1 1 1 +TPR(Sensitivity, recall, hit rate, or true positive rate) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +Y(Youden index) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +dInd(Distance index) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +sInd(Similarity index) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 + >>> cm_comp1 = ConfusionMatrix(matrix={0:{0:2,1:50,2:6},1:{0:5,1:50,2:3},2:{0:1,1:7,2:50}}) >>> cm_comp2 = ConfusionMatrix(matrix={0:{0:50,1:2,2:6},1:{0:50,1:5,2:3},2:{0:1,1:55,2:2}}) >>> cm_comp1 == cm_comp2 From 96da1c3f784b6fbabf03c3b83307ac33920bda37 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 20 Jul 2019 17:02:41 +0430 Subject: [PATCH 051/107] fix : error_test updated for save_stat #228 --- Test/output_test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Test/output_test.py b/Test/output_test.py index 1fadd1e4..1cb4b095 100644 --- a/Test/output_test.py +++ b/Test/output_test.py @@ -22,6 +22,10 @@ >>> save_stat=cm.save_stat("test_filtered3",address=False,overall_param=["Kappa","Scott PI"],class_param=["TPR","TNR","ACC","AUC"],class_name=[]) >>> save_stat=={'Status': True, 'Message': None} True +>>> large_cm = ConfusionMatrix(list(range(20)),list(range(20))) +>>> save_stat = large_cm.save_stat("test_large",address=False) +>>> save_stat == {'Status': True, 'Message': None} +True >>> save_stat=cm.save_stat("/asdasd,qweqwe.eo/",address=True) >>> save_stat=={'Status': False, 'Message': "[Errno 2] No such file or directory: '/asdasd,qweqwe.eo/.pycm'"} True @@ -452,6 +456,7 @@ >>> os.remove("test_filtered.csv") >>> os.remove("test_filtered_matrix.csv") >>> os.remove("test_filtered.pycm") +>>> os.remove("test_large.pycm") >>> os.remove("test_summary.pycm") >>> os.remove("test_filtered2.html") >>> os.remove("test_filtered3.html") From 5709d00d35853cc73f6a29dcc8f42ef1835997c9 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 20 Jul 2019 17:10:06 +0430 Subject: [PATCH 052/107] fix : output_test updated for save_html method --- Test/output_test.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Test/output_test.py b/Test/output_test.py index 1cb4b095..2720c22f 100644 --- a/Test/output_test.py +++ b/Test/output_test.py @@ -59,6 +59,13 @@ >>> save_stat=cm.save_html("test_colored2",address=False,color="Beige") >>> save_stat=={'Status': True, 'Message': None} True +>>> long_name_cm = ConfusionMatrix(matrix={'SVM-Classifier':{'SVM-Classifier':25,'NN-Classifier':2},'NN-Classifier':{'SVM-Classifier':3,'NN-Classifier':50}}) +>>> save_stat=long_name_cm.save_html("test_long_name",address=False,color="Pink") +>>> save_stat=={'Status': True, 'Message': None} +True +>>> save_stat=cm.save_html("/asdasd,qweqwe.eo/",address=True) +>>> save_stat=={'Status': False, 'Message': "[Errno 2] No such file or directory: '/asdasd,qweqwe.eo/.html'"} +True >>> save_stat=cm.save_csv("test",address=False) >>> save_stat=={'Status': True, 'Message': None} True @@ -80,9 +87,6 @@ >>> save_stat=cm.save_csv("test_filtered4",address=False,class_param=[],class_name=[100],matrix_save=False) >>> save_stat=={'Status': True, 'Message': None} True ->>> save_stat=cm.save_html("/asdasd,qweqwe.eo/",address=True) ->>> save_stat=={'Status': False, 'Message': "[Errno 2] No such file or directory: '/asdasd,qweqwe.eo/.html'"} -True >>> save_stat=cm.save_csv("/asdasd,qweqwe.eo/",address=True) >>> save_stat=={'Status': False, 'Message': "[Errno 2] No such file or directory: '/asdasd,qweqwe.eo/.csv'"} True @@ -462,6 +466,7 @@ >>> os.remove("test_filtered3.html") >>> os.remove("test_filtered4.html") >>> os.remove("test_filtered5.html") +>>> os.remove("test_long_name.html") >>> os.remove("test_summary.html") >>> os.remove("test_colored.html") >>> os.remove("test_colored2.html") From 6d3affc383257ed0a0c35d261ff285aac9ac7de1 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 20 Jul 2019 17:35:17 +0430 Subject: [PATCH 053/107] fix : minor edit in appveyor config --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 12a822b9..bc834885 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -22,10 +22,10 @@ environment: PYTHON_VERSION: "3.6.0" PYTHON_ARCH: "32" - PYTHON: "C:\\Python34" - PYTHON_VERSION: "3.4.0" + PYTHON_VERSION: "3.4.8" PYTHON_ARCH: "64" - PYTHON: "C:\\Python34" - PYTHON_VERSION: "3.4.0" + PYTHON_VERSION: "3.4.8" PYTHON_ARCH: "32" - PYTHON: "C:\\Python27" PYTHON_VERSION: "2.7.2" From 7d0009fef7bd745b574eb86e465bc17bddb2cf73 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sat, 20 Jul 2019 17:56:36 +0430 Subject: [PATCH 054/107] fix : minor edit in overall_test #228 --- Test/overall_test.py | 450 ++++++++++++++++++++++--------------------- 1 file changed, 226 insertions(+), 224 deletions(-) diff --git a/Test/overall_test.py b/Test/overall_test.py index bd56c4f4..f758f712 100644 --- a/Test/overall_test.py +++ b/Test/overall_test.py @@ -1132,7 +1132,7 @@ >>> cm = ConfusionMatrix(matrix={1:{1:60,2:9,3:1,4:0,5:0,6:0},2:{1:23,2:48,3:0,4:2,5:2,6:1},3:{1:11,2:5,3:60,4:0,5:0,6:0},4:{1:0,2:2,3:0,4:60,5:1,6:3},5:{1:2,2:1,3:0,4:0,5:60,6:2},6:{1:1,2:2,3:0,4:2,5:1,6:60}}) >>> set(cm.recommended_list) == set(MULTICLASS_RECOMMEND) True ->>> large_cm = ConfusionMatrix(list(range(10)),list(range(10))) +>>> large_cm = ConfusionMatrix(list(range(10))+[2,3,5],list(range(10))+[1,7,2]) >>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'): ... large_cm.print_matrix() Predict 0 1 2 3 4 5 6 7 8 9 @@ -1141,13 +1141,13 @@ 1 0 1 0 0 0 0 0 0 0 0 -2 0 0 1 0 0 0 0 0 0 0 +2 0 1 1 0 0 0 0 0 0 0 -3 0 0 0 1 0 0 0 0 0 0 +3 0 0 0 1 0 0 0 1 0 0 4 0 0 0 0 1 0 0 0 0 0 -5 0 0 0 0 0 1 0 0 0 0 +5 0 0 1 0 0 1 0 0 0 0 6 0 0 0 0 0 0 1 0 0 0 @@ -1165,13 +1165,13 @@ 1 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -2 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +2 0.0 0.5 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -3 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 +3 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.5 0.0 0.0 4 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 -5 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 +5 0.0 0.0 0.5 0.0 0.0 0.5 0.0 0.0 0.0 0.0 6 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 @@ -1185,122 +1185,122 @@ ... large_cm.stat() Overall Statistics : -95% CI (1.0,1.0) -ACC Macro 1.0 -AUNP 1.0 -AUNU 1.0 -Bennett S 1.0 -CBA 1.0 -Chi-Squared 90.0 +95% CI (0.5402,0.99827) +ACC Macro 0.95385 +AUNP 0.87121 +AUNU 0.91212 +Bennett S 0.74359 +CBA 0.75 +Chi-Squared 91.0 Chi-Squared DF 81 -Conditional Entropy -0.0 -Cramer V 1.0 -Cross Entropy 3.32193 -F1 Macro 1.0 -F1 Micro 1.0 -Gwet AC1 1.0 -Hamming Loss 0.0 -Joint Entropy 3.32193 -KL Divergence 0.0 -Kappa 1.0 -Kappa 95% CI (1.0,1.0) -Kappa No Prevalence 1.0 -Kappa Standard Error 0.0 -Kappa Unbiased 1.0 -Lambda A 1.0 -Lambda B 1.0 -Mutual Information 3.32193 -NIR 0.1 -Overall ACC 1.0 -Overall CEN 0.0 -Overall J (10.0,1.0) -Overall MCC 1.0 -Overall MCEN 0.0 -Overall RACC 0.1 -Overall RACCU 0.1 +Conditional Entropy 0.46154 +Cramer V 0.88192 +Cross Entropy 3.39275 +F1 Macro 0.81667 +F1 Micro 0.76923 +Gwet AC1 0.7438 +Hamming Loss 0.23077 +Joint Entropy 3.70044 +KL Divergence 0.15385 +Kappa 0.74342 +Kappa 95% CI (0.48877,0.99807) +Kappa No Prevalence 0.53846 +Kappa Standard Error 0.12992 +Kappa Unbiased 0.74172 +Lambda A 0.72727 +Lambda B 0.72727 +Mutual Information 2.77736 +NIR 0.15385 +Overall ACC 0.76923 +Overall CEN 0.09537 +Overall J (7.33333,0.73333) +Overall MCC 0.75333 +Overall MCEN 0.10746 +Overall RACC 0.10059 +Overall RACCU 0.10651 P-Value 0.0 -PPV Macro 1.0 -PPV Micro 1.0 -Pearson C 0.94868 -Phi-Squared 9.0 -RCI 1.0 -RR 1.0 -Reference Entropy 3.32193 -Response Entropy 3.32193 -SOA1(Landis & Koch) Almost Perfect -SOA2(Fleiss) Excellent -SOA3(Altman) Very Good +PPV Macro 0.85 +PPV Micro 0.76923 +Pearson C 0.93541 +Phi-Squared 7.0 +RCI 0.8575 +RR 1.3 +Reference Entropy 3.2389 +Response Entropy 3.2389 +SOA1(Landis & Koch) Substantial +SOA2(Fleiss) Intermediate to Good +SOA3(Altman) Good SOA4(Cicchetti) Excellent SOA5(Cramer) Very Strong -SOA6(Matthews) Very Strong -Scott PI 1.0 -Standard Error 0.0 -TPR Macro 1.0 -TPR Micro 1.0 -Zero-one Loss 0 +SOA6(Matthews) Strong +Scott PI 0.74172 +Standard Error 0.11685 +TPR Macro 0.85 +TPR Micro 0.76923 +Zero-one Loss 3 Class Statistics : Classes 0 1 2 3 4 5 6 7 8 9 -ACC(Accuracy) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -AGF(Adjusted F-score) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -AGM(Adjusted geometric mean) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -AM(Difference between automatic and manual classification) 0 0 0 0 0 0 0 0 0 0 -AUC(Area under the ROC curve) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -AUCI(AUC value interpretation) Excellent Excellent Excellent Excellent Excellent Excellent Excellent Excellent Excellent Excellent -AUPR(Area under the PR curve) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -BCD(Bray-Curtis dissimilarity) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -BM(Informedness or bookmaker informedness) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -CEN(Confusion entropy) 0 0 0 0 0 0 0 0 0 0 -DOR(Diagnostic odds ratio) None None None None None None None None None None -DP(Discriminant power) None None None None None None None None None None -DPI(Discriminant power interpretation) None None None None None None None None None None -ERR(Error rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -F0.5(F0.5 score) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -F1(F1 score - harmonic mean of precision and sensitivity) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -F2(F2 score) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -FDR(False discovery rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -FN(False negative/miss/type 2 error) 0 0 0 0 0 0 0 0 0 0 -FNR(Miss rate or false negative rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -FOR(False omission rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -FP(False positive/type 1 error/false alarm) 0 0 0 0 0 0 0 0 0 0 -FPR(Fall-out or false positive rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -G(G-measure geometric mean of precision and sensitivity) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -GI(Gini index) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -GM(G-mean geometric mean of specificity and sensitivity) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -IBA(Index of balanced accuracy) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -IS(Information score) 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 -J(Jaccard index) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -LS(Lift score) 10.0 10.0 10.0 10.0 10.0 10.0 10.0 10.0 10.0 10.0 -MCC(Matthews correlation coefficient) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -MCCI(Matthews correlation coefficient interpretation) Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong -MCEN(Modified confusion entropy) 0 0 0 0 0 0 0 0 0 0 -MK(Markedness) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -N(Condition negative) 9 9 9 9 9 9 9 9 9 9 -NLR(Negative likelihood ratio) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -NLRI(Negative likelihood ratio interpretation) Good Good Good Good Good Good Good Good Good Good -NPV(Negative predictive value) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -OC(Overlap coefficient) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -OOC(Otsuka-Ochiai coefficient) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -OP(Optimized precision) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -P(Condition positive or support) 1 1 1 1 1 1 1 1 1 1 -PLR(Positive likelihood ratio) None None None None None None None None None None -PLRI(Positive likelihood ratio interpretation) None None None None None None None None None None -POP(Population) 10 10 10 10 10 10 10 10 10 10 -PPV(Precision or positive predictive value) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -PRE(Prevalence) 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 -Q(Yule Q - coefficient of colligation) None None None None None None None None None None -RACC(Random accuracy) 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 -RACCU(Random accuracy unbiased) 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 -TN(True negative/correct rejection) 9 9 9 9 9 9 9 9 9 9 -TNR(Specificity or true negative rate) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -TON(Test outcome negative) 9 9 9 9 9 9 9 9 9 9 -TOP(Test outcome positive) 1 1 1 1 1 1 1 1 1 1 +ACC(Accuracy) 1.0 0.92308 0.84615 0.92308 1.0 0.92308 1.0 0.92308 1.0 1.0 +AGF(Adjusted F-score) 1.0 0.90468 0.6742 0.71965 1.0 0.71965 1.0 0.90468 1.0 1.0 +AGM(Adjusted geometric mean) 1.0 0.93786 0.78186 0.84135 1.0 0.84135 1.0 0.93786 1.0 1.0 +AM(Difference between automatic and manual classification) 0 1 0 -1 0 -1 0 1 0 0 +AUC(Area under the ROC curve) 1.0 0.95833 0.70455 0.75 1.0 0.75 1.0 0.95833 1.0 1.0 +AUCI(AUC value interpretation) Excellent Excellent Good Good Excellent Good Excellent Excellent Excellent Excellent +AUPR(Area under the PR curve) 1.0 0.75 0.5 0.75 1.0 0.75 1.0 0.75 1.0 1.0 +BCD(Bray-Curtis dissimilarity) 0.0 0.03846 0.0 0.03846 0.0 0.03846 0.0 0.03846 0.0 0.0 +BM(Informedness or bookmaker informedness) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 +CEN(Confusion entropy) 0 0.1267 0.23981 0.1267 0 0.1267 0 0.1267 0 0 +DOR(Diagnostic odds ratio) None None 10.0 None None None None None None None +DP(Discriminant power) None None 0.55133 None None None None None None None +DPI(Discriminant power interpretation) None None Poor None None None None None None None +ERR(Error rate) 0.0 0.07692 0.15385 0.07692 0.0 0.07692 0.0 0.07692 0.0 0.0 +F0.5(F0.5 score) 1.0 0.55556 0.5 0.83333 1.0 0.83333 1.0 0.55556 1.0 1.0 +F1(F1 score - harmonic mean of precision and sensitivity) 1.0 0.66667 0.5 0.66667 1.0 0.66667 1.0 0.66667 1.0 1.0 +F2(F2 score) 1.0 0.83333 0.5 0.55556 1.0 0.55556 1.0 0.83333 1.0 1.0 +FDR(False discovery rate) 0.0 0.5 0.5 0.0 0.0 0.0 0.0 0.5 0.0 0.0 +FN(False negative/miss/type 2 error) 0 0 1 1 0 1 0 0 0 0 +FNR(Miss rate or false negative rate) 0.0 0.0 0.5 0.5 0.0 0.5 0.0 0.0 0.0 0.0 +FOR(False omission rate) 0.0 0.0 0.09091 0.08333 0.0 0.08333 0.0 0.0 0.0 0.0 +FP(False positive/type 1 error/false alarm) 0 1 1 0 0 0 0 1 0 0 +FPR(Fall-out or false positive rate) 0.0 0.08333 0.09091 0.0 0.0 0.0 0.0 0.08333 0.0 0.0 +G(G-measure geometric mean of precision and sensitivity) 1.0 0.70711 0.5 0.70711 1.0 0.70711 1.0 0.70711 1.0 1.0 +GI(Gini index) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 +GM(G-mean geometric mean of specificity and sensitivity) 1.0 0.95743 0.6742 0.70711 1.0 0.70711 1.0 0.95743 1.0 1.0 +IBA(Index of balanced accuracy) 1.0 0.99306 0.2686 0.25 1.0 0.25 1.0 0.99306 1.0 1.0 +IS(Information score) 3.70044 2.70044 1.70044 2.70044 3.70044 2.70044 3.70044 2.70044 3.70044 3.70044 +J(Jaccard index) 1.0 0.5 0.33333 0.5 1.0 0.5 1.0 0.5 1.0 1.0 +LS(Lift score) 13.0 6.5 3.25 6.5 13.0 6.5 13.0 6.5 13.0 13.0 +MCC(Matthews correlation coefficient) 1.0 0.677 0.40909 0.677 1.0 0.677 1.0 0.677 1.0 1.0 +MCCI(Matthews correlation coefficient interpretation) Very Strong Moderate Weak Moderate Very Strong Moderate Very Strong Moderate Very Strong Very Strong +MCEN(Modified confusion entropy) 0 0.11991 0.2534 0.11991 0 0.11991 0 0.11991 0 0 +MK(Markedness) 1.0 0.5 0.40909 0.91667 1.0 0.91667 1.0 0.5 1.0 1.0 +N(Condition negative) 12 12 11 11 12 11 12 12 12 12 +NLR(Negative likelihood ratio) 0.0 0.0 0.55 0.5 0.0 0.5 0.0 0.0 0.0 0.0 +NLRI(Negative likelihood ratio interpretation) Good Good Negligible Negligible Good Negligible Good Good Good Good +NPV(Negative predictive value) 1.0 1.0 0.90909 0.91667 1.0 0.91667 1.0 1.0 1.0 1.0 +OC(Overlap coefficient) 1.0 1.0 0.5 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +OOC(Otsuka-Ochiai coefficient) 1.0 0.70711 0.5 0.70711 1.0 0.70711 1.0 0.70711 1.0 1.0 +OP(Optimized precision) 1.0 0.8796 0.55583 0.58974 1.0 0.58974 1.0 0.8796 1.0 1.0 +P(Condition positive or support) 1 1 2 2 1 2 1 1 1 1 +PLR(Positive likelihood ratio) None 12.0 5.5 None None None None 12.0 None None +PLRI(Positive likelihood ratio interpretation) None Good Fair None None None None Good None None +POP(Population) 13 13 13 13 13 13 13 13 13 13 +PPV(Precision or positive predictive value) 1.0 0.5 0.5 1.0 1.0 1.0 1.0 0.5 1.0 1.0 +PRE(Prevalence) 0.07692 0.07692 0.15385 0.15385 0.07692 0.15385 0.07692 0.07692 0.07692 0.07692 +Q(Yule Q - coefficient of colligation) None None 0.81818 None None None None None None None +RACC(Random accuracy) 0.00592 0.01183 0.02367 0.01183 0.00592 0.01183 0.00592 0.01183 0.00592 0.00592 +RACCU(Random accuracy unbiased) 0.00592 0.01331 0.02367 0.01331 0.00592 0.01331 0.00592 0.01331 0.00592 0.00592 +TN(True negative/correct rejection) 12 11 10 11 12 11 12 11 12 12 +TNR(Specificity or true negative rate) 1.0 0.91667 0.90909 1.0 1.0 1.0 1.0 0.91667 1.0 1.0 +TON(Test outcome negative) 12 11 11 12 12 12 12 11 12 12 +TOP(Test outcome positive) 1 2 2 1 1 1 1 2 1 1 TP(True positive/hit) 1 1 1 1 1 1 1 1 1 1 -TPR(Sensitivity, recall, hit rate, or true positive rate) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -Y(Youden index) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -dInd(Distance index) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -sInd(Similarity index) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +TPR(Sensitivity, recall, hit rate, or true positive rate) 1.0 1.0 0.5 0.5 1.0 0.5 1.0 1.0 1.0 1.0 +Y(Youden index) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 +dInd(Distance index) 0.0 0.08333 0.5082 0.5 0.0 0.5 0.0 0.08333 0.0 0.0 +sInd(Similarity index) 1.0 0.94107 0.64065 0.64645 1.0 0.64645 1.0 0.94107 1.0 1.0 >>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'): ... print(large_cm) @@ -1310,13 +1310,13 @@ 1 0 1 0 0 0 0 0 0 0 0 -2 0 0 1 0 0 0 0 0 0 0 +2 0 1 1 0 0 0 0 0 0 0 -3 0 0 0 1 0 0 0 0 0 0 +3 0 0 0 1 0 0 0 1 0 0 4 0 0 0 0 1 0 0 0 0 0 -5 0 0 0 0 0 1 0 0 0 0 +5 0 0 1 0 0 1 0 0 0 0 6 0 0 0 0 0 0 1 0 0 0 @@ -1328,124 +1328,126 @@ + + Overall Statistics : -95% CI (1.0,1.0) -ACC Macro 1.0 -AUNP 1.0 -AUNU 1.0 -Bennett S 1.0 -CBA 1.0 -Chi-Squared 90.0 +95% CI (0.5402,0.99827) +ACC Macro 0.95385 +AUNP 0.87121 +AUNU 0.91212 +Bennett S 0.74359 +CBA 0.75 +Chi-Squared 91.0 Chi-Squared DF 81 -Conditional Entropy -0.0 -Cramer V 1.0 -Cross Entropy 3.32193 -F1 Macro 1.0 -F1 Micro 1.0 -Gwet AC1 1.0 -Hamming Loss 0.0 -Joint Entropy 3.32193 -KL Divergence 0.0 -Kappa 1.0 -Kappa 95% CI (1.0,1.0) -Kappa No Prevalence 1.0 -Kappa Standard Error 0.0 -Kappa Unbiased 1.0 -Lambda A 1.0 -Lambda B 1.0 -Mutual Information 3.32193 -NIR 0.1 -Overall ACC 1.0 -Overall CEN 0.0 -Overall J (10.0,1.0) -Overall MCC 1.0 -Overall MCEN 0.0 -Overall RACC 0.1 -Overall RACCU 0.1 +Conditional Entropy 0.46154 +Cramer V 0.88192 +Cross Entropy 3.39275 +F1 Macro 0.81667 +F1 Micro 0.76923 +Gwet AC1 0.7438 +Hamming Loss 0.23077 +Joint Entropy 3.70044 +KL Divergence 0.15385 +Kappa 0.74342 +Kappa 95% CI (0.48877,0.99807) +Kappa No Prevalence 0.53846 +Kappa Standard Error 0.12992 +Kappa Unbiased 0.74172 +Lambda A 0.72727 +Lambda B 0.72727 +Mutual Information 2.77736 +NIR 0.15385 +Overall ACC 0.76923 +Overall CEN 0.09537 +Overall J (7.33333,0.73333) +Overall MCC 0.75333 +Overall MCEN 0.10746 +Overall RACC 0.10059 +Overall RACCU 0.10651 P-Value 0.0 -PPV Macro 1.0 -PPV Micro 1.0 -Pearson C 0.94868 -Phi-Squared 9.0 -RCI 1.0 -RR 1.0 -Reference Entropy 3.32193 -Response Entropy 3.32193 -SOA1(Landis & Koch) Almost Perfect -SOA2(Fleiss) Excellent -SOA3(Altman) Very Good +PPV Macro 0.85 +PPV Micro 0.76923 +Pearson C 0.93541 +Phi-Squared 7.0 +RCI 0.8575 +RR 1.3 +Reference Entropy 3.2389 +Response Entropy 3.2389 +SOA1(Landis & Koch) Substantial +SOA2(Fleiss) Intermediate to Good +SOA3(Altman) Good SOA4(Cicchetti) Excellent SOA5(Cramer) Very Strong -SOA6(Matthews) Very Strong -Scott PI 1.0 -Standard Error 0.0 -TPR Macro 1.0 -TPR Micro 1.0 -Zero-one Loss 0 +SOA6(Matthews) Strong +Scott PI 0.74172 +Standard Error 0.11685 +TPR Macro 0.85 +TPR Micro 0.76923 +Zero-one Loss 3 Class Statistics : Classes 0 1 2 3 4 5 6 7 8 9 -ACC(Accuracy) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -AGF(Adjusted F-score) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -AGM(Adjusted geometric mean) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -AM(Difference between automatic and manual classification) 0 0 0 0 0 0 0 0 0 0 -AUC(Area under the ROC curve) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -AUCI(AUC value interpretation) Excellent Excellent Excellent Excellent Excellent Excellent Excellent Excellent Excellent Excellent -AUPR(Area under the PR curve) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -BCD(Bray-Curtis dissimilarity) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -BM(Informedness or bookmaker informedness) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -CEN(Confusion entropy) 0 0 0 0 0 0 0 0 0 0 -DOR(Diagnostic odds ratio) None None None None None None None None None None -DP(Discriminant power) None None None None None None None None None None -DPI(Discriminant power interpretation) None None None None None None None None None None -ERR(Error rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -F0.5(F0.5 score) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -F1(F1 score - harmonic mean of precision and sensitivity) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -F2(F2 score) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -FDR(False discovery rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -FN(False negative/miss/type 2 error) 0 0 0 0 0 0 0 0 0 0 -FNR(Miss rate or false negative rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -FOR(False omission rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -FP(False positive/type 1 error/false alarm) 0 0 0 0 0 0 0 0 0 0 -FPR(Fall-out or false positive rate) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -G(G-measure geometric mean of precision and sensitivity) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -GI(Gini index) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -GM(G-mean geometric mean of specificity and sensitivity) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -IBA(Index of balanced accuracy) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -IS(Information score) 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 3.32193 -J(Jaccard index) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -LS(Lift score) 10.0 10.0 10.0 10.0 10.0 10.0 10.0 10.0 10.0 10.0 -MCC(Matthews correlation coefficient) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -MCCI(Matthews correlation coefficient interpretation) Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong Very Strong -MCEN(Modified confusion entropy) 0 0 0 0 0 0 0 0 0 0 -MK(Markedness) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -N(Condition negative) 9 9 9 9 9 9 9 9 9 9 -NLR(Negative likelihood ratio) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -NLRI(Negative likelihood ratio interpretation) Good Good Good Good Good Good Good Good Good Good -NPV(Negative predictive value) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -OC(Overlap coefficient) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -OOC(Otsuka-Ochiai coefficient) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -OP(Optimized precision) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -P(Condition positive or support) 1 1 1 1 1 1 1 1 1 1 -PLR(Positive likelihood ratio) None None None None None None None None None None -PLRI(Positive likelihood ratio interpretation) None None None None None None None None None None -POP(Population) 10 10 10 10 10 10 10 10 10 10 -PPV(Precision or positive predictive value) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -PRE(Prevalence) 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 0.1 -Q(Yule Q - coefficient of colligation) None None None None None None None None None None -RACC(Random accuracy) 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 -RACCU(Random accuracy unbiased) 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 -TN(True negative/correct rejection) 9 9 9 9 9 9 9 9 9 9 -TNR(Specificity or true negative rate) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -TON(Test outcome negative) 9 9 9 9 9 9 9 9 9 9 -TOP(Test outcome positive) 1 1 1 1 1 1 1 1 1 1 +ACC(Accuracy) 1.0 0.92308 0.84615 0.92308 1.0 0.92308 1.0 0.92308 1.0 1.0 +AGF(Adjusted F-score) 1.0 0.90468 0.6742 0.71965 1.0 0.71965 1.0 0.90468 1.0 1.0 +AGM(Adjusted geometric mean) 1.0 0.93786 0.78186 0.84135 1.0 0.84135 1.0 0.93786 1.0 1.0 +AM(Difference between automatic and manual classification) 0 1 0 -1 0 -1 0 1 0 0 +AUC(Area under the ROC curve) 1.0 0.95833 0.70455 0.75 1.0 0.75 1.0 0.95833 1.0 1.0 +AUCI(AUC value interpretation) Excellent Excellent Good Good Excellent Good Excellent Excellent Excellent Excellent +AUPR(Area under the PR curve) 1.0 0.75 0.5 0.75 1.0 0.75 1.0 0.75 1.0 1.0 +BCD(Bray-Curtis dissimilarity) 0.0 0.03846 0.0 0.03846 0.0 0.03846 0.0 0.03846 0.0 0.0 +BM(Informedness or bookmaker informedness) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 +CEN(Confusion entropy) 0 0.1267 0.23981 0.1267 0 0.1267 0 0.1267 0 0 +DOR(Diagnostic odds ratio) None None 10.0 None None None None None None None +DP(Discriminant power) None None 0.55133 None None None None None None None +DPI(Discriminant power interpretation) None None Poor None None None None None None None +ERR(Error rate) 0.0 0.07692 0.15385 0.07692 0.0 0.07692 0.0 0.07692 0.0 0.0 +F0.5(F0.5 score) 1.0 0.55556 0.5 0.83333 1.0 0.83333 1.0 0.55556 1.0 1.0 +F1(F1 score - harmonic mean of precision and sensitivity) 1.0 0.66667 0.5 0.66667 1.0 0.66667 1.0 0.66667 1.0 1.0 +F2(F2 score) 1.0 0.83333 0.5 0.55556 1.0 0.55556 1.0 0.83333 1.0 1.0 +FDR(False discovery rate) 0.0 0.5 0.5 0.0 0.0 0.0 0.0 0.5 0.0 0.0 +FN(False negative/miss/type 2 error) 0 0 1 1 0 1 0 0 0 0 +FNR(Miss rate or false negative rate) 0.0 0.0 0.5 0.5 0.0 0.5 0.0 0.0 0.0 0.0 +FOR(False omission rate) 0.0 0.0 0.09091 0.08333 0.0 0.08333 0.0 0.0 0.0 0.0 +FP(False positive/type 1 error/false alarm) 0 1 1 0 0 0 0 1 0 0 +FPR(Fall-out or false positive rate) 0.0 0.08333 0.09091 0.0 0.0 0.0 0.0 0.08333 0.0 0.0 +G(G-measure geometric mean of precision and sensitivity) 1.0 0.70711 0.5 0.70711 1.0 0.70711 1.0 0.70711 1.0 1.0 +GI(Gini index) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 +GM(G-mean geometric mean of specificity and sensitivity) 1.0 0.95743 0.6742 0.70711 1.0 0.70711 1.0 0.95743 1.0 1.0 +IBA(Index of balanced accuracy) 1.0 0.99306 0.2686 0.25 1.0 0.25 1.0 0.99306 1.0 1.0 +IS(Information score) 3.70044 2.70044 1.70044 2.70044 3.70044 2.70044 3.70044 2.70044 3.70044 3.70044 +J(Jaccard index) 1.0 0.5 0.33333 0.5 1.0 0.5 1.0 0.5 1.0 1.0 +LS(Lift score) 13.0 6.5 3.25 6.5 13.0 6.5 13.0 6.5 13.0 13.0 +MCC(Matthews correlation coefficient) 1.0 0.677 0.40909 0.677 1.0 0.677 1.0 0.677 1.0 1.0 +MCCI(Matthews correlation coefficient interpretation) Very Strong Moderate Weak Moderate Very Strong Moderate Very Strong Moderate Very Strong Very Strong +MCEN(Modified confusion entropy) 0 0.11991 0.2534 0.11991 0 0.11991 0 0.11991 0 0 +MK(Markedness) 1.0 0.5 0.40909 0.91667 1.0 0.91667 1.0 0.5 1.0 1.0 +N(Condition negative) 12 12 11 11 12 11 12 12 12 12 +NLR(Negative likelihood ratio) 0.0 0.0 0.55 0.5 0.0 0.5 0.0 0.0 0.0 0.0 +NLRI(Negative likelihood ratio interpretation) Good Good Negligible Negligible Good Negligible Good Good Good Good +NPV(Negative predictive value) 1.0 1.0 0.90909 0.91667 1.0 0.91667 1.0 1.0 1.0 1.0 +OC(Overlap coefficient) 1.0 1.0 0.5 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +OOC(Otsuka-Ochiai coefficient) 1.0 0.70711 0.5 0.70711 1.0 0.70711 1.0 0.70711 1.0 1.0 +OP(Optimized precision) 1.0 0.8796 0.55583 0.58974 1.0 0.58974 1.0 0.8796 1.0 1.0 +P(Condition positive or support) 1 1 2 2 1 2 1 1 1 1 +PLR(Positive likelihood ratio) None 12.0 5.5 None None None None 12.0 None None +PLRI(Positive likelihood ratio interpretation) None Good Fair None None None None Good None None +POP(Population) 13 13 13 13 13 13 13 13 13 13 +PPV(Precision or positive predictive value) 1.0 0.5 0.5 1.0 1.0 1.0 1.0 0.5 1.0 1.0 +PRE(Prevalence) 0.07692 0.07692 0.15385 0.15385 0.07692 0.15385 0.07692 0.07692 0.07692 0.07692 +Q(Yule Q - coefficient of colligation) None None 0.81818 None None None None None None None +RACC(Random accuracy) 0.00592 0.01183 0.02367 0.01183 0.00592 0.01183 0.00592 0.01183 0.00592 0.00592 +RACCU(Random accuracy unbiased) 0.00592 0.01331 0.02367 0.01331 0.00592 0.01331 0.00592 0.01331 0.00592 0.00592 +TN(True negative/correct rejection) 12 11 10 11 12 11 12 11 12 12 +TNR(Specificity or true negative rate) 1.0 0.91667 0.90909 1.0 1.0 1.0 1.0 0.91667 1.0 1.0 +TON(Test outcome negative) 12 11 11 12 12 12 12 11 12 12 +TOP(Test outcome positive) 1 2 2 1 1 1 1 2 1 1 TP(True positive/hit) 1 1 1 1 1 1 1 1 1 1 -TPR(Sensitivity, recall, hit rate, or true positive rate) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -Y(Youden index) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -dInd(Distance index) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -sInd(Similarity index) 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +TPR(Sensitivity, recall, hit rate, or true positive rate) 1.0 1.0 0.5 0.5 1.0 0.5 1.0 1.0 1.0 1.0 +Y(Youden index) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 +dInd(Distance index) 0.0 0.08333 0.5082 0.5 0.0 0.5 0.0 0.08333 0.0 0.0 +sInd(Similarity index) 1.0 0.94107 0.64065 0.64645 1.0 0.64645 1.0 0.94107 1.0 1.0 >>> cm_comp1 = ConfusionMatrix(matrix={0:{0:2,1:50,2:6},1:{0:5,1:50,2:3},2:{0:1,1:7,2:50}}) >>> cm_comp2 = ConfusionMatrix(matrix={0:{0:50,1:2,2:6},1:{0:50,1:5,2:3},2:{0:1,1:55,2:2}}) From 9ba5a7121e4fd3ce3531ed73a8b4a3acaeda2602 Mon Sep 17 00:00:00 2001 From: sadrasabouri Date: Wed, 24 Jul 2019 23:53:18 +0430 Subject: [PATCH 055/107] Typo fixed. --- Otherfiles/test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Otherfiles/test.sh b/Otherfiles/test.sh index c9cd0ef7..0e5e61ff 100644 --- a/Otherfiles/test.sh +++ b/Otherfiles/test.sh @@ -4,8 +4,8 @@ python Otherfiles/version_check.py if [ "$TRAVIS_PYTHON_VERSION" = '3.6' ] then - python -m vulture --min-confidence 80 --exclude=pycm,build,.eggs --sort-by-size . - python -m bandit -r pycm -s B311 + python -m vulture --min-confidence 80 --exclude=pycm,build,.eggs --sort-by-size + python -m bandit -r pycm -s B311 python -m pydocstyle --match-dir=pycm fi - python -m cProfile -s cumtime pycm/pycm_profile.py \ No newline at end of file + python -m cProfile -s cumtime pycm/pycm_profile.py From 59cd6a85918465b019766d6d82d8dcc6063bea44 Mon Sep 17 00:00:00 2001 From: sadrasabouri Date: Wed, 24 Jul 2019 23:54:30 +0430 Subject: [PATCH 056/107] Minimum confidence set to 65% --- Otherfiles/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Otherfiles/test.sh b/Otherfiles/test.sh index 0e5e61ff..dbbf745a 100644 --- a/Otherfiles/test.sh +++ b/Otherfiles/test.sh @@ -4,7 +4,7 @@ python Otherfiles/version_check.py if [ "$TRAVIS_PYTHON_VERSION" = '3.6' ] then - python -m vulture --min-confidence 80 --exclude=pycm,build,.eggs --sort-by-size + python -m vulture --min-confidence 65 --exclude=pycm,build,.eggs --sort-by-size python -m bandit -r pycm -s B311 python -m pydocstyle --match-dir=pycm fi From 0a7458ce66f664a27875bd5bf1d5cde0a27c3b8a Mon Sep 17 00:00:00 2001 From: sadrasabouri Date: Wed, 24 Jul 2019 23:56:34 +0430 Subject: [PATCH 057/107] pycm and Otherfiles folder and setup.py vulture check added. --- Otherfiles/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Otherfiles/test.sh b/Otherfiles/test.sh index dbbf745a..9be25c4a 100644 --- a/Otherfiles/test.sh +++ b/Otherfiles/test.sh @@ -4,7 +4,7 @@ python Otherfiles/version_check.py if [ "$TRAVIS_PYTHON_VERSION" = '3.6' ] then - python -m vulture --min-confidence 65 --exclude=pycm,build,.eggs --sort-by-size + python -m vulture pycm/ Otherfiles/ setup.py --min-confidence 65 --exclude=build,.eggs --sort-by-size python -m bandit -r pycm -s B311 python -m pydocstyle --match-dir=pycm fi From 5b58572e043fcf9c491bfeaae7472aac9a028b4b Mon Sep 17 00:00:00 2001 From: sadrasabouri Date: Wed, 24 Jul 2019 23:59:33 +0430 Subject: [PATCH 058/107] chmod +x --- Otherfiles/test.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 Otherfiles/test.sh diff --git a/Otherfiles/test.sh b/Otherfiles/test.sh old mode 100644 new mode 100755 From 1ad5414c27b1b0a3b65f9d6c92e42e50a95e854d Mon Sep 17 00:00:00 2001 From: sadrasabouri Date: Thu, 25 Jul 2019 00:03:12 +0430 Subject: [PATCH 059/107] fix : CHANGELOG.md updated#229 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 454cee3a..7daead7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] +- vulture section edited ### Added - Tversky index (TI) - Area under the PR curve (AUPR) From 673ba6af2d7f6206324b432aea6e4f49084dc543 Mon Sep 17 00:00:00 2001 From: sadrasabouri Date: Thu, 25 Jul 2019 00:03:12 +0430 Subject: [PATCH 060/107] fix : CHANGELOG.md updated#229 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 454cee3a..7daead7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] +- vulture section edited ### Added - Tversky index (TI) - Area under the PR curve (AUPR) From 1c742bf0e4c5d8815c64543873a02073c2eef44a Mon Sep 17 00:00:00 2001 From: alirezazolanvari Date: Fri, 26 Jul 2019 14:49:22 +0430 Subject: [PATCH 061/107] printed warnings eliminated from compare_print report function #228 --- pycm/pycm_output.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pycm/pycm_output.py b/pycm/pycm_output.py index f4c2c2e9..859ba406 100644 --- a/pycm/pycm_output.py +++ b/pycm/pycm_output.py @@ -5,6 +5,7 @@ from .pycm_param import * from .pycm_util import class_filter, rounder import webbrowser +from warnings import warn def html_init(name): @@ -457,8 +458,6 @@ def compare_report_print(sorted_list, scores, best_name): result += ("".join(shifts)) % (str(rank + 1), str(cm), str(scores[cm]["class"])) + str(scores[cm]["overall"]) + "\n" prev_rank = rank - if best_name is None: - result += "\nWarning: " + COMPARE_RESULT_WARNING return result From b77aa47acfd02e734eb113ae029e1f64dc752d29 Mon Sep 17 00:00:00 2001 From: alirezazolanvari Date: Fri, 26 Jul 2019 14:50:28 +0430 Subject: [PATCH 062/107] comapre testcases modified #228 --- Test/overall_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Test/overall_test.py b/Test/overall_test.py index f758f712..687c6ca3 100644 --- a/Test/overall_test.py +++ b/Test/overall_test.py @@ -1526,7 +1526,6 @@ 1 cm1 604.9 4.48333 2 cm2 567.9 5.8 -Warning: Confusion matrices are too close and the best one can not be recognized. >>> cp3.best >>> cp3.best_name """ From 3a1700667f4ef2552f10a29f0dc1be99db7ca976 Mon Sep 17 00:00:00 2001 From: sadrasabouri Date: Sat, 27 Jul 2019 13:55:56 +0430 Subject: [PATCH 063/107] add : TRAVIS check added. --- Otherfiles/test.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Otherfiles/test.sh b/Otherfiles/test.sh index 9be25c4a..5878a9a1 100755 --- a/Otherfiles/test.sh +++ b/Otherfiles/test.sh @@ -2,7 +2,15 @@ set -x python -m pytest Test --cov=pycm --cov-report=term python Otherfiles/version_check.py - if [ "$TRAVIS_PYTHON_VERSION" = '3.6' ] + + #Check if we are in TRAVIS + isInTRAVIS=false + if [ "$CI" = 'true' ] && [ "$TRAVIS" = 'true' ] + then + $isInTRAVIS=true + fi + + if [ "$isInTRAVIS" = 'false' ] || { [ "$isInTRAVIS" = 'true' ] && [ "$TRAVIS_PYTHON_VERSION" = '3.6' ]; } then python -m vulture pycm/ Otherfiles/ setup.py --min-confidence 65 --exclude=build,.eggs --sort-by-size python -m bandit -r pycm -s B311 From 271f0fd163436a057b6d92d2c030b2a8742e3942 Mon Sep 17 00:00:00 2001 From: sadrasabouri Date: Sat, 27 Jul 2019 14:04:13 +0430 Subject: [PATCH 064/107] fix : some typo --- Otherfiles/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Otherfiles/test.sh b/Otherfiles/test.sh index 5878a9a1..18a6fb65 100755 --- a/Otherfiles/test.sh +++ b/Otherfiles/test.sh @@ -7,7 +7,7 @@ isInTRAVIS=false if [ "$CI" = 'true' ] && [ "$TRAVIS" = 'true' ] then - $isInTRAVIS=true + isInTRAVIS=true fi if [ "$isInTRAVIS" = 'false' ] || { [ "$isInTRAVIS" = 'true' ] && [ "$TRAVIS_PYTHON_VERSION" = '3.6' ]; } From 6ebea062a144bdabb9bc144b3dfb39b3a02b51c5 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sun, 28 Jul 2019 16:22:01 +0430 Subject: [PATCH 065/107] fix : alt_link parameter added #232 --- pycm/pycm_output.py | 14 +++++++++++--- pycm/pycm_param.py | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/pycm/pycm_output.py b/pycm/pycm_output.py index 859ba406..a11e8a19 100644 --- a/pycm/pycm_output.py +++ b/pycm/pycm_output.py @@ -153,7 +153,8 @@ def html_overall_stat( overall_stat, digit=5, overall_param=None, - recommended_list=()): + recommended_list=(), + alt_link = False): """ Return HTML report file overall stat. @@ -165,6 +166,8 @@ def html_overall_stat( :type overall_param : list :param recommended_list: recommended statistics list :type recommended_list : list or tuple + :param alt_link: alternative link for document flag + :type alt_link: bool :return: html_overall_stat as str """ result = "" @@ -200,7 +203,8 @@ def html_class_stat( class_stat, digit=5, class_param=None, - recommended_list=()): + recommended_list=(), + alt_link=False): """ Return HTML report file class_stat. @@ -214,6 +218,8 @@ def html_class_stat( :type class_param : list :param recommended_list: recommended statistics list :type recommended_list : list or tuple + :param alt_link: alternative link for document flag + :type alt_link: bool :return: html_class_stat as str """ result = "" @@ -461,12 +467,14 @@ def compare_report_print(sorted_list, scores, best_name): return result -def online_help(param=None): +def online_help(param=None,alt_link=False): """ Open online document in web browser. :param param: input parameter :type param : int or str + :param alt_link: alternative link for document flag + :type alt_link: bool :return: None """ try: diff --git a/pycm/pycm_param.py b/pycm/pycm_param.py index 92909b37..2d9969ee 100644 --- a/pycm/pycm_param.py +++ b/pycm/pycm_param.py @@ -197,6 +197,7 @@ RECOMMEND_WARNING) DOCUMENT_ADR = "http://www.pycm.ir/doc/index.html#" +DOCUMENT_ADR_ALT = "https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Document.ipynb" PARAMS_DESCRIPTION = { "TPR": "sensitivity, recall, hit rate, or true positive rate", From ee872c36271835b8dfae6636ff74dfb5ad031460 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sun, 28 Jul 2019 16:26:23 +0430 Subject: [PATCH 066/107] fix : alt_link added to object #232 --- pycm/pycm_obj.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pycm/pycm_obj.py b/pycm/pycm_obj.py index 2b011ec0..c12a2a99 100644 --- a/pycm/pycm_obj.py +++ b/pycm/pycm_obj.py @@ -248,7 +248,7 @@ def save_html( address=True, overall_param=None, class_param=None, - class_name=None, color=(0, 0, 0), normalize=False, summary=False): + class_name=None, color=(0, 0, 0), normalize=False, summary=False, alt_link=False): """ Save ConfusionMatrix in HTML file. @@ -268,6 +268,8 @@ def save_html( :type normalize : bool :param summary : summary mode flag :type summary : bool + :param alt_link: alternative link for document flag + :type alt_link: bool :return: saving Status as dict {"Status":bool , "Message":str} """ try: @@ -289,7 +291,8 @@ def save_html( self.overall_stat, self.digit, overall_list, - self.recommended_list)) + self.recommended_list, + alt_link)) class_stat_classes = class_filter(self.classes, class_name) html_file.write( html_class_stat( @@ -297,7 +300,8 @@ def save_html( self.class_stat, self.digit, class_list, - self.recommended_list)) + self.recommended_list, + alt_link)) html_file.write(html_end(VERSION)) html_file.close() if address: From 7630d8168fe919396fe2817e6b16569fec378b03 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sun, 28 Jul 2019 16:43:59 +0430 Subject: [PATCH 067/107] doc : Document updated #232 --- Document/Document.ipynb | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/Document/Document.ipynb b/Document/Document.ipynb index 198d0307..0d694d5a 100644 --- a/Document/Document.ipynb +++ b/Document/Document.ipynb @@ -1191,6 +1191,7 @@ "\n", ">>> from pycm import online_help\n", ">>> online_help(\"J\")\n", + ">>> online_help(\"J\", alt_link=True)\n", ">>> online_help(\"SOA1(Landis & Koch)\")\n", ">>> online_help(2)\n", "\n", @@ -1204,6 +1205,13 @@ "* List of items are available by calling `online_help()` (without argument)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* If PyCM website is not available, set `alt_link = True`" + ] + }, { "cell_type": "code", "execution_count": 29, @@ -1335,6 +1343,15 @@ "online_help()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
      \n", + "
    • Notice : `alt_link` , new in version 2.4
      • \n", + "
    " + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -9061,8 +9078,9 @@ "3. `overall_param` : overall statistics names for save (type : `list`)\n", "4. `class_param` : class statistics names for save (type : `list`)\n", "5. `class_name` : class names for print (sub set of classes) (type : `list`)\n", - "6. `color` : matrix color (R,G,B) (type : `tuple`/`str`)\n", - "7. `summary` : summary mode flag (type : `bool`)" + "6. `color` : matrix color (R,G,B) (type : `tuple`/`str`), support X11 color names\n", + "7. `summary` : summary mode flag (type : `bool`)\n", + "8. `alt_link` : alternative link for document flag (type : `bool`)" ] }, { @@ -9106,7 +9124,7 @@ "metadata": {}, "source": [ "
      \n", - "
    • Notice : `color` support X11 color names
      • \n", + "
    • Notice : `normalize`, new in version 2.0
      • \n", "
    " ] }, @@ -9115,7 +9133,7 @@ "metadata": {}, "source": [ "
      \n", - "
    • Notice : `normalize`, new in version 2.0
      • \n", + "
    • Notice : `summary` and `alt_link` , new in version 2.4
      • \n", "
    " ] }, @@ -9124,7 +9142,7 @@ "metadata": {}, "source": [ "
      \n", - "
    • Notice : `summary` , new in version 2.4
      • \n", + "
    • Notice : If PyCM website is not available, set `alt_link = True`
      • \n", "
    " ] }, From 8df9d2b8a1befb25214756aa3f3ddaf6db8a4149 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sun, 28 Jul 2019 16:49:17 +0430 Subject: [PATCH 068/107] fix : conditions added #232 --- pycm/pycm_output.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pycm/pycm_output.py b/pycm/pycm_output.py index a11e8a19..7be6df94 100644 --- a/pycm/pycm_output.py +++ b/pycm/pycm_output.py @@ -170,6 +170,9 @@ def html_overall_stat( :type alt_link: bool :return: html_overall_stat as str """ + document_link = DOCUMENT_ADR + if alt_link: + document_link = DOCUMENT_ADR_ALT result = "" result += "

    Overall Statistics :

    \n" result += '\n' @@ -185,7 +188,7 @@ def html_overall_stat( background_color = RECOMMEND_BACKGROUND_COLOR result += '\n' result += '\n' + background_color) + document_link + PARAMS_LINK[i] + '" style="text-decoration:None;">' + str(i) + '\n' if i in BENCHMARK_LIST: background_color = BENCHMARK_COLOR[i][overall_stat[i]] result += ' - + @@ -314,6 +314,13 @@

    Class Statistics :

    + + + + + + + @@ -679,5 +686,5 @@

    Class Statistics :

    ' + str(i) + ''.format( @@ -222,6 +225,9 @@ def html_class_stat( :type alt_link: bool :return: html_class_stat as str """ + document_link = DOCUMENT_ADR + if alt_link: + document_link = DOCUMENT_ADR_ALT result = "" result += "

    Class Statistics :

    \n" result += '\n' @@ -244,7 +250,7 @@ def html_class_stat( background_color = RECOMMEND_BACKGROUND_COLOR result += '\n' result += '\n' + background_color) + document_link + PARAMS_LINK[i] + '" style="text-decoration:None;">' + str(i) + '\n' for j in classes: if i in BENCHMARK_LIST: background_color = BENCHMARK_COLOR[i][class_stat[i][j]] @@ -478,12 +484,15 @@ def online_help(param=None,alt_link=False): :return: None """ try: + document_link = DOCUMENT_ADR + if alt_link: + document_link = DOCUMENT_ADR_ALT PARAMS_LINK_KEYS = sorted(PARAMS_LINK.keys()) if param in PARAMS_LINK_KEYS: - webbrowser.open_new_tab(DOCUMENT_ADR + PARAMS_LINK[param]) + webbrowser.open_new_tab(document_link + PARAMS_LINK[param]) elif param in range(1, len(PARAMS_LINK_KEYS) + 1): webbrowser.open_new_tab( - DOCUMENT_ADR + PARAMS_LINK[PARAMS_LINK_KEYS[param - 1]]) + document_link + PARAMS_LINK[PARAMS_LINK_KEYS[param - 1]]) else: print("Please choose one parameter : \n") print('Example : online_help("J") or online_help(2)\n') From 6d42bd530dd13ff94144455d60860146555eb98a Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sun, 28 Jul 2019 16:54:47 +0430 Subject: [PATCH 069/107] fix : output_test updated #232 --- Test/output_test.py | 4 ++++ pycm/pycm_param.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Test/output_test.py b/Test/output_test.py index 2720c22f..50702916 100644 --- a/Test/output_test.py +++ b/Test/output_test.py @@ -35,6 +35,9 @@ >>> save_stat=cm.save_html("test_normalized",address=False,normalize=True) >>> save_stat=={'Status': True, 'Message': None} True +>>> save_stat=cm.save_html("test_alt",address=False,normalize=True,alt_link=True) +>>> save_stat=={'Status': True, 'Message': None} +True >>> save_stat=cm.save_html("test_summary",address=False,summary=True) >>> save_stat=={'Status': True, 'Message': None} True @@ -467,6 +470,7 @@ >>> os.remove("test_filtered4.html") >>> os.remove("test_filtered5.html") >>> os.remove("test_long_name.html") +>>> os.remove("test_alt.html") >>> os.remove("test_summary.html") >>> os.remove("test_colored.html") >>> os.remove("test_colored2.html") diff --git a/pycm/pycm_param.py b/pycm/pycm_param.py index 2d9969ee..9db10c39 100644 --- a/pycm/pycm_param.py +++ b/pycm/pycm_param.py @@ -197,7 +197,7 @@ RECOMMEND_WARNING) DOCUMENT_ADR = "http://www.pycm.ir/doc/index.html#" -DOCUMENT_ADR_ALT = "https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Document.ipynb" +DOCUMENT_ADR_ALT = "https://nbviewer.jupyter.org/github/sepandhaghighi/pycm/blob/master/Document/Document.ipynb#" PARAMS_DESCRIPTION = { "TPR": "sensitivity, recall, hit rate, or true positive rate", From a9398b9f8e0db97df549b1782ef972c93175bab9 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sun, 28 Jul 2019 16:57:06 +0430 Subject: [PATCH 070/107] fix : function_test updated #232 --- Test/function_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Test/function_test.py b/Test/function_test.py index 79b2aec2..97e36ea5 100644 --- a/Test/function_test.py +++ b/Test/function_test.py @@ -458,6 +458,8 @@ 112-sInd >>> online_help("J") ... +>>> online_help("J",alt_link=True) +... >>> online_help(4) ... >>> NIR_calc({'Class2': 804, 'Class1': 196},1000) # Verified Case From c7ca20ede7ead057236a661f489c60df5e336563 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sun, 28 Jul 2019 16:57:29 +0430 Subject: [PATCH 071/107] fix : autopep8 fix #232 --- pycm/pycm_compare.py | 1 + pycm/pycm_obj.py | 16 ++++++++++++++-- pycm/pycm_output.py | 4 ++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/pycm/pycm_compare.py b/pycm/pycm_compare.py index 6b312877..0e918045 100644 --- a/pycm/pycm_compare.py +++ b/pycm/pycm_compare.py @@ -9,6 +9,7 @@ import numpy from warnings import warn + class pycmCompareError(Exception): """Compare error class.""" diff --git a/pycm/pycm_obj.py b/pycm/pycm_obj.py index c12a2a99..35a61603 100644 --- a/pycm/pycm_obj.py +++ b/pycm/pycm_obj.py @@ -233,7 +233,12 @@ def save_stat( self.digit, overall_list, class_list) if len(self.classes) >= CLASS_NUMBER_THRESHOLD: warning_message = "\n" + "Warning : " + CLASS_NUMBER_WARNING + "\n" - file.write(matrix + normalized_matrix + stat + one_vs_all + warning_message) + file.write( + matrix + + normalized_matrix + + stat + + one_vs_all + + warning_message) file.close() if address: message = os.path.join( @@ -248,7 +253,14 @@ def save_html( address=True, overall_param=None, class_param=None, - class_name=None, color=(0, 0, 0), normalize=False, summary=False, alt_link=False): + class_name=None, + color=( + 0, + 0, + 0), + normalize=False, + summary=False, + alt_link=False): """ Save ConfusionMatrix in HTML file. diff --git a/pycm/pycm_output.py b/pycm/pycm_output.py index 7be6df94..45e61645 100644 --- a/pycm/pycm_output.py +++ b/pycm/pycm_output.py @@ -154,7 +154,7 @@ def html_overall_stat( digit=5, overall_param=None, recommended_list=(), - alt_link = False): + alt_link=False): """ Return HTML report file overall stat. @@ -473,7 +473,7 @@ def compare_report_print(sorted_list, scores, best_name): return result -def online_help(param=None,alt_link=False): +def online_help(param=None, alt_link=False): """ Open online document in web browser. From 5dc8af21b096afedee3fa353c8e81d93db78eca9 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Sun, 28 Jul 2019 16:59:32 +0430 Subject: [PATCH 072/107] doc : CHANGELOG updated #232 --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7daead7f..c3272036 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,6 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] -- vulture section edited ### Added - Tversky index (TI) - Area under the PR curve (AUPR) @@ -15,6 +14,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `summary` parameter added to `save_html`,`save_stat`,`save_csv` and `stat` methods - `sample_weight` bug in `numpy` array format fixed - Inputs manipulation bug fixed +- Test system modified +- `alt_link` parameter added to `save_html` method and `online_help` function ## [2.3] - 2019-06-27 ### Added - Adjusted F-score (AGF) From 37d9b4ed79107df2b78d9dafe52d52d1c11508b1 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 11:46:09 +0430 Subject: [PATCH 073/107] fix : compare assign handler added --- pycm/pycm_compare.py | 70 ++++++++++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/pycm/pycm_compare.py b/pycm/pycm_compare.py index 0e918045..7e2a1511 100644 --- a/pycm/pycm_compare.py +++ b/pycm/pycm_compare.py @@ -51,32 +51,10 @@ def __init__(self, cm_dict, by_class=False, weight=None, digit=5): :param digit: precision digit (default value : 5) :type digit : int """ - if not isinstance(cm_dict, dict): - raise pycmCompareError(COMPARE_FORMAT_ERROR) - if not all(isinstance(item, ConfusionMatrix) - for item in cm_dict.values()): - raise pycmCompareError(COMPARE_TYPE_ERROR) - if not list_check_equal([getattr(item, "POP") - for item in cm_dict.values()]): - raise pycmCompareError(COMPARE_DOMAIN_ERROR) - if len(cm_dict) < 2: - raise pycmCompareError(COMPARE_NUMBER_ERROR) - self.classes = list(cm_dict.values())[0].classes - self.weight = {k: 1 for k in self.classes} - self.digit = digit - self.best = None - self.best_name = None + self.scores = None self.sorted = None - self.scores = {k: {"overall": 0, "class": 0}.copy() - for k in cm_dict.keys()} - if weight is not None: - if not isinstance(weight, dict): - raise pycmCompareError(COMPARE_WEIGHT_ERROR) - if list(weight.keys()) == self.classes and all( - [isfloat(x) for x in weight.values()]): - self.weight = weight - else: - raise pycmCompareError(COMPARE_WEIGHT_ERROR) + self.classes = None + __compare_assign_handler__(self,cm_dict,weight,digit) __compare_class_handler__(self, cm_dict) __compare_overall_handler__(self, cm_dict) __compare_rounder__(self, cm_dict) @@ -240,3 +218,45 @@ def __compare_sort_handler__(compare): max_overall_score, max_class_name, max_class_score) + +def __compare_assign_handler__(compare,cm_dict,weight,digit): + """ + Assign basic parameters to Comapre. + + :param compare: Compare + :type compare : pycm.Compare object + :param cm_dict: cm's dictionary + :type cm_dict : dict + :param digit: precision digit (default value : 5) + :type digit : int + :param weight: class weights + :type weight: dict + :return: None + """ + if not isinstance(cm_dict, dict): + raise pycmCompareError(COMPARE_FORMAT_ERROR) + if not all(isinstance(item, ConfusionMatrix) + for item in cm_dict.values()): + raise pycmCompareError(COMPARE_TYPE_ERROR) + if not list_check_equal([getattr(item, "POP") + for item in cm_dict.values()]): + raise pycmCompareError(COMPARE_DOMAIN_ERROR) + if len(cm_dict) < 2: + raise pycmCompareError(COMPARE_NUMBER_ERROR) + compare.classes = list(cm_dict.values())[0].classes + compare.weight = {k: 1 for k in compare.classes} + compare.digit = digit + compare.best = None + compare.best_name = None + compare.sorted = None + compare.scores = {k: {"overall": 0, "class": 0}.copy() + for k in cm_dict.keys()} + if weight is not None: + if not isinstance(weight, dict): + raise pycmCompareError(COMPARE_WEIGHT_ERROR) + if list(weight.keys()) == compare.classes and all( + [isfloat(x) for x in weight.values()]): + compare.weight = weight + else: + raise pycmCompareError(COMPARE_WEIGHT_ERROR) + From 43963beafeae0c529069a94e47d443d0248a17cd Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 11:50:42 +0430 Subject: [PATCH 074/107] fix : minor edit in online_help function --- pycm/pycm_output.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pycm/pycm_output.py b/pycm/pycm_output.py index 45e61645..381f9c18 100644 --- a/pycm/pycm_output.py +++ b/pycm/pycm_output.py @@ -487,16 +487,16 @@ def online_help(param=None, alt_link=False): document_link = DOCUMENT_ADR if alt_link: document_link = DOCUMENT_ADR_ALT - PARAMS_LINK_KEYS = sorted(PARAMS_LINK.keys()) - if param in PARAMS_LINK_KEYS: + params_link_keys = sorted(PARAMS_LINK.keys()) + if param in params_link_keys: webbrowser.open_new_tab(document_link + PARAMS_LINK[param]) - elif param in range(1, len(PARAMS_LINK_KEYS) + 1): + elif param in range(1, len(params_link_keys) + 1): webbrowser.open_new_tab( - document_link + PARAMS_LINK[PARAMS_LINK_KEYS[param - 1]]) + document_link + PARAMS_LINK[params_link_keys[param - 1]]) else: print("Please choose one parameter : \n") print('Example : online_help("J") or online_help(2)\n') - for index, item in enumerate(PARAMS_LINK_KEYS): + for index, item in enumerate(params_link_keys): print(str(index + 1) + "-" + item) except Exception: print("Error in online help") From 3b621dddcf6c0008914c5e475cb80b0e759e27a6 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 11:55:46 +0430 Subject: [PATCH 075/107] fix : compare test file splitted --- Test/compare_test.py | 83 ++++++++++++++++++++++++++++++++++++++++++++ Test/overall_test.py | 79 ----------------------------------------- 2 files changed, 83 insertions(+), 79 deletions(-) create mode 100644 Test/compare_test.py diff --git a/Test/compare_test.py b/Test/compare_test.py new file mode 100644 index 00000000..174ec907 --- /dev/null +++ b/Test/compare_test.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +""" +>>> from pycm import * +>>> cm_comp1 = ConfusionMatrix(matrix={0:{0:2,1:50,2:6},1:{0:5,1:50,2:3},2:{0:1,1:7,2:50}}) +>>> cm_comp2 = ConfusionMatrix(matrix={0:{0:50,1:2,2:6},1:{0:50,1:5,2:3},2:{0:1,1:55,2:2}}) +>>> cm_comp1 == cm_comp2 +False +>>> cm_comp1 == 2 +False +>>> cm_comp1_temp = ConfusionMatrix(matrix={0:{0:2,1:50,2:6},1:{0:5,1:50,2:3},2:{0:1,1:7,2:50}}) +>>> cm_comp1 == cm_comp1_temp +True +>>> compare_input = {"model1":cm_comp1,"model2":cm_comp2} +>>> compare_input_copy = {"model1":cm_comp1,"model2":cm_comp2} +>>> cp = Compare(compare_input) +>>> compare_input == compare_input_copy +True +>>> cp +pycm.Compare(classes: [0, 1, 2]) +>>> cp.scores == {'model1': {'overall': 2.55, 'class': 7.05}, 'model2': {'overall': 1.98333, 'class': 4.55}} +True +>>> cp.best +pycm.ConfusionMatrix(classes: [0, 1, 2]) +>>> cp.best_name +'model1' +>>> print(cp) +Best : model1 + +Rank Name Class-Score Overall-Score +1 model1 7.05 2.55 +2 model2 4.55 1.98333 + +>>> cp.print_report() +Best : model1 + +Rank Name Class-Score Overall-Score +1 model1 7.05 2.55 +2 model2 4.55 1.98333 + +>>> weight = {0:5,1:1,2:1} +>>> weight_copy = {0:5,1:1,2:1} +>>> cp = Compare({"model1":cm_comp1,"model2":cm_comp2},by_class=True,weight=weight) +>>> weight == weight_copy +True +>>> print(cp) +Best : model2 + +Rank Name Class-Score Overall-Score +1 model2 13.55 1.98333 +2 model1 11.65 2.55 + +>>> cp.best +pycm.ConfusionMatrix(classes: [0, 1, 2]) +>>> cp.best_name +'model2' +>>> with warns(RuntimeWarning, match='Confusion matrices are too close'): +... cp2 = Compare({"model1":cm_comp1,"model2":cm_comp1}) +>>> cp2.scores == {'model1': {'overall': 2.55, 'class': 7.05}, 'model2': {'overall': 2.55, 'class': 7.05}} +True +>>> cp2.best +>>> cp2.best_name +>>> cm1 = ConfusionMatrix(matrix={0:{0:50,1:0,2:0},1:{0:0,1:35,2:15},2:{0:0,1:16,2:34}}) +>>> cm2 = ConfusionMatrix(matrix={0:{0:48,1:2,2:0},1:{0:3,1:46,2:1},2:{0:8,1:2,2:40}}) +>>> cp3 = Compare({"cm1":cm1,"cm2":cm2}) +>>> print(cp3) +Best : cm2 + +Rank Name Class-Score Overall-Score +1 cm2 10.7 5.8 +2 cm1 7.9 4.48333 + +>>> with warns(RuntimeWarning, match='Confusion matrices are too close'): +... cp3 = Compare({"cm1":cm1,"cm2":cm2},weight={0:200,1:1,2:1}) +>>> print(cp3) +Best : None + +Rank Name Class-Score Overall-Score +1 cm1 604.9 4.48333 +2 cm2 567.9 5.8 + +>>> cp3.best +>>> cp3.best_name +""" diff --git a/Test/overall_test.py b/Test/overall_test.py index 687c6ca3..acbe3908 100644 --- a/Test/overall_test.py +++ b/Test/overall_test.py @@ -1449,83 +1449,4 @@ dInd(Distance index) 0.0 0.08333 0.5082 0.5 0.0 0.5 0.0 0.08333 0.0 0.0 sInd(Similarity index) 1.0 0.94107 0.64065 0.64645 1.0 0.64645 1.0 0.94107 1.0 1.0 ->>> cm_comp1 = ConfusionMatrix(matrix={0:{0:2,1:50,2:6},1:{0:5,1:50,2:3},2:{0:1,1:7,2:50}}) ->>> cm_comp2 = ConfusionMatrix(matrix={0:{0:50,1:2,2:6},1:{0:50,1:5,2:3},2:{0:1,1:55,2:2}}) ->>> cm_comp1 == cm_comp2 -False ->>> cm_comp1 == 2 -False ->>> cm_comp1_temp = ConfusionMatrix(matrix={0:{0:2,1:50,2:6},1:{0:5,1:50,2:3},2:{0:1,1:7,2:50}}) ->>> cm_comp1 == cm_comp1_temp -True ->>> compare_input = {"model1":cm_comp1,"model2":cm_comp2} ->>> compare_input_copy = {"model1":cm_comp1,"model2":cm_comp2} ->>> cp = Compare(compare_input) ->>> compare_input == compare_input_copy -True ->>> cp -pycm.Compare(classes: [0, 1, 2]) ->>> cp.scores == {'model1': {'overall': 2.55, 'class': 7.05}, 'model2': {'overall': 1.98333, 'class': 4.55}} -True ->>> cp.best -pycm.ConfusionMatrix(classes: [0, 1, 2]) ->>> cp.best_name -'model1' ->>> print(cp) -Best : model1 - -Rank Name Class-Score Overall-Score -1 model1 7.05 2.55 -2 model2 4.55 1.98333 - ->>> cp.print_report() -Best : model1 - -Rank Name Class-Score Overall-Score -1 model1 7.05 2.55 -2 model2 4.55 1.98333 - ->>> weight = {0:5,1:1,2:1} ->>> weight_copy = {0:5,1:1,2:1} ->>> cp = Compare({"model1":cm_comp1,"model2":cm_comp2},by_class=True,weight=weight) ->>> weight == weight_copy -True ->>> print(cp) -Best : model2 - -Rank Name Class-Score Overall-Score -1 model2 13.55 1.98333 -2 model1 11.65 2.55 - ->>> cp.best -pycm.ConfusionMatrix(classes: [0, 1, 2]) ->>> cp.best_name -'model2' ->>> with warns(RuntimeWarning, match='Confusion matrices are too close'): -... cp2 = Compare({"model1":cm_comp1,"model2":cm_comp1}) ->>> cp2.scores == {'model1': {'overall': 2.55, 'class': 7.05}, 'model2': {'overall': 2.55, 'class': 7.05}} -True ->>> cp2.best ->>> cp2.best_name ->>> cm1 = ConfusionMatrix(matrix={0:{0:50,1:0,2:0},1:{0:0,1:35,2:15},2:{0:0,1:16,2:34}}) ->>> cm2 = ConfusionMatrix(matrix={0:{0:48,1:2,2:0},1:{0:3,1:46,2:1},2:{0:8,1:2,2:40}}) ->>> cp3 = Compare({"cm1":cm1,"cm2":cm2}) ->>> print(cp3) -Best : cm2 - -Rank Name Class-Score Overall-Score -1 cm2 10.7 5.8 -2 cm1 7.9 4.48333 - ->>> with warns(RuntimeWarning, match='Confusion matrices are too close'): -... cp3 = Compare({"cm1":cm1,"cm2":cm2},weight={0:200,1:1,2:1}) ->>> print(cp3) -Best : None - -Rank Name Class-Score Overall-Score -1 cm1 604.9 4.48333 -2 cm2 567.9 5.8 - ->>> cp3.best ->>> cp3.best_name """ From d316c17efaf0bcb6e6b949b50c2e3cf26afe7b8a Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 12:02:48 +0430 Subject: [PATCH 076/107] fix : minor edit in compare_test --- Test/compare_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Test/compare_test.py b/Test/compare_test.py index 174ec907..4b24cfde 100644 --- a/Test/compare_test.py +++ b/Test/compare_test.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """ >>> from pycm import * +>>> from pytest import warns >>> cm_comp1 = ConfusionMatrix(matrix={0:{0:2,1:50,2:6},1:{0:5,1:50,2:3},2:{0:1,1:7,2:50}}) >>> cm_comp2 = ConfusionMatrix(matrix={0:{0:50,1:2,2:6},1:{0:50,1:5,2:3},2:{0:1,1:55,2:2}}) >>> cm_comp1 == cm_comp2 From 7f397e93ab45df24159f58b6076ad547478bec48 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 12:08:36 +0430 Subject: [PATCH 077/107] doc : CHANGELOG updated --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3272036..76200f54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Inputs manipulation bug fixed - Test system modified - `alt_link` parameter added to `save_html` method and `online_help` function +- `Compare` class tests moved to `compare_test.py` ## [2.3] - 2019-06-27 ### Added - Adjusted F-score (AGF) From 10e8b04c763af5f54d4c1e47ed6f930e8900e1b2 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 12:08:56 +0430 Subject: [PATCH 078/107] fix : autopep8 fix --- pycm/pycm_compare.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pycm/pycm_compare.py b/pycm/pycm_compare.py index 7e2a1511..031cbda7 100644 --- a/pycm/pycm_compare.py +++ b/pycm/pycm_compare.py @@ -54,7 +54,7 @@ def __init__(self, cm_dict, by_class=False, weight=None, digit=5): self.scores = None self.sorted = None self.classes = None - __compare_assign_handler__(self,cm_dict,weight,digit) + __compare_assign_handler__(self, cm_dict, weight, digit) __compare_class_handler__(self, cm_dict) __compare_overall_handler__(self, cm_dict) __compare_rounder__(self, cm_dict) @@ -219,7 +219,8 @@ def __compare_sort_handler__(compare): max_class_name, max_class_score) -def __compare_assign_handler__(compare,cm_dict,weight,digit): + +def __compare_assign_handler__(compare, cm_dict, weight, digit): """ Assign basic parameters to Comapre. @@ -250,7 +251,7 @@ def __compare_assign_handler__(compare,cm_dict,weight,digit): compare.best_name = None compare.sorted = None compare.scores = {k: {"overall": 0, "class": 0}.copy() - for k in cm_dict.keys()} + for k in cm_dict.keys()} if weight is not None: if not isinstance(weight, dict): raise pycmCompareError(COMPARE_WEIGHT_ERROR) @@ -259,4 +260,3 @@ def __compare_assign_handler__(compare,cm_dict,weight,digit): compare.weight = weight else: raise pycmCompareError(COMPARE_WEIGHT_ERROR) - From 5a059650ba2b0702f8d9c8765c865739b889b55e Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 12:15:36 +0430 Subject: [PATCH 079/107] fix : minor edit in travis and appveyor config for __main__ test --- .travis.yml | 1 + appveyor.yml | 1 + pycm/__main__.py | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ec0c9c0e..8b2070a1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,7 @@ install: - pip install -r requirements.txt - python setup.py install - python -m pycm test + - python -m pycm before_script: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip install --upgrade --upgrade-strategy=only-if-needed -r dev-requirements.txt --user ; fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then pip install --upgrade --upgrade-strategy=only-if-needed -r dev-requirements.txt ; fi diff --git a/appveyor.yml b/appveyor.yml index bc834885..c2db212e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -43,6 +43,7 @@ install: test_script: - "%PYTHON%/python.exe -m pycm test" + - "%PYTHON%/python.exe -m pycm" - "%PYTHON%/Scripts/pip.exe install -r dev-requirements.txt" - "%PYTHON%/python.exe -m pytest Test --cov=pycm --cov-report=term" - "%PYTHON%/python.exe Otherfiles/version_check.py" diff --git a/pycm/__main__.py b/pycm/__main__.py index 35c94ead..b0471d10 100644 --- a/pycm/__main__.py +++ b/pycm/__main__.py @@ -5,7 +5,7 @@ import sys from .pycm_obj import * from .pycm_output import * -from art import tprint +#from art import tprint if __name__ == "__main__": From c99cf83b7ea72eb8a249742c6aaf9a3bcd01b9fc Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 12:18:17 +0430 Subject: [PATCH 080/107] fix : __main__ import fixed --- pycm/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycm/__main__.py b/pycm/__main__.py index b0471d10..35c94ead 100644 --- a/pycm/__main__.py +++ b/pycm/__main__.py @@ -5,7 +5,7 @@ import sys from .pycm_obj import * from .pycm_output import * -#from art import tprint +from art import tprint if __name__ == "__main__": From 268d619a4c241afde0c980fb07db1f9ba9df6a09 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 17:16:39 +0430 Subject: [PATCH 081/107] fix : unused import removed from pycm_output --- pycm/pycm_output.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pycm/pycm_output.py b/pycm/pycm_output.py index 381f9c18..5825a3fe 100644 --- a/pycm/pycm_output.py +++ b/pycm/pycm_output.py @@ -3,9 +3,8 @@ from __future__ import division from functools import partial from .pycm_param import * -from .pycm_util import class_filter, rounder +from .pycm_util import rounder import webbrowser -from warnings import warn def html_init(name): From 8d3d54debe6a9a1e648f81cd2ab896c10e70c209 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 17:31:21 +0430 Subject: [PATCH 082/107] fix : minor edit in __init__ method of ConfusionMatrix --- pycm/pycm_obj.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pycm/pycm_obj.py b/pycm/pycm_obj.py index 35a61603..ce2da868 100644 --- a/pycm/pycm_obj.py +++ b/pycm/pycm_obj.py @@ -72,6 +72,7 @@ def __init__( self.predict_vector = predict_vector self.digit = digit self.weights = None + self.classes = None if isinstance(transpose, bool): self.transpose = transpose else: From 77e64cb980b1546ce0de37cdbcb3b1245939b62f Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 17:39:30 +0430 Subject: [PATCH 083/107] fix : unused variables in __compare_sort_handler removed --- pycm/pycm_compare.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/pycm/pycm_compare.py b/pycm/pycm_compare.py index 031cbda7..a25910c5 100644 --- a/pycm/pycm_compare.py +++ b/pycm/pycm_compare.py @@ -59,8 +59,7 @@ def __init__(self, cm_dict, by_class=False, weight=None, digit=5): __compare_overall_handler__(self, cm_dict) __compare_rounder__(self, cm_dict) scores_list = list(self.scores.values()) - (max_overall_name, max_overall_score, max_class_name, - max_class_score) = __compare_sort_handler__(self) + (max_overall_name, max_class_name) = __compare_sort_handler__(self) if scores_list.count(self.scores[max_class_name]) == 1: if by_class: self.best = cm_dict[max_class_name] @@ -194,7 +193,7 @@ def __compare_sort_handler__(compare): :param compare: Compare :type compare : pycm.Compare object - :return: (max_overall_name,max_overall_score,max_class_name,max_class_score) as tuple + :return: (max_overall_name,max_class_name) as tuple """ sorted_by_class = sorted( compare.scores, @@ -211,13 +210,9 @@ def __compare_sort_handler__(compare): compare.sorted = sorted_by_class max_overall_name = sorted_by_overall[0] max_class_name = sorted_by_class[0] - max_class_score = compare.scores[max_class_name]["class"] - max_overall_score = compare.scores[max_overall_name]["overall"] - return ( - max_overall_name, - max_overall_score, - max_class_name, - max_class_score) + #max_class_score = compare.scores[max_class_name]["class"] + #max_overall_score = compare.scores[max_overall_name]["overall"] + return (max_overall_name,max_class_name) def __compare_assign_handler__(compare, cm_dict, weight, digit): From 0adbd7158e6907d02445e04f1054df7918bac986 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 17:55:35 +0430 Subject: [PATCH 084/107] fix : minor edit in test.sh --- Otherfiles/test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/Otherfiles/test.sh b/Otherfiles/test.sh index 18a6fb65..3002ff3c 100755 --- a/Otherfiles/test.sh +++ b/Otherfiles/test.sh @@ -1,3 +1,4 @@ + #!/bin/bash set -e set -x python -m pytest Test --cov=pycm --cov-report=term From 546349523758fddd1651040383dba2f94331cd9f Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 17:58:41 +0430 Subject: [PATCH 085/107] fix : minor edit in chi_square_calc function --- pycm/pycm_overall_func.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycm/pycm_overall_func.py b/pycm/pycm_overall_func.py index 95ce464a..0d663c4f 100644 --- a/pycm/pycm_overall_func.py +++ b/pycm/pycm_overall_func.py @@ -517,7 +517,7 @@ def chi_square_calc(classes, table, TOP, P, POP): try: result = 0 for i in classes: - for index, j in enumerate(classes): + for j in classes: expected = (TOP[j] * P[i]) / (POP[i]) result += ((table[i][j] - expected)**2) / expected return result From 2130fccc620d1ee4f6232b9bfe349f0fe95fa8f3 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 17:59:29 +0430 Subject: [PATCH 086/107] fix : autopep8 fix --- pycm/pycm_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycm/pycm_compare.py b/pycm/pycm_compare.py index a25910c5..b6dacf8e 100644 --- a/pycm/pycm_compare.py +++ b/pycm/pycm_compare.py @@ -212,7 +212,7 @@ def __compare_sort_handler__(compare): max_class_name = sorted_by_class[0] #max_class_score = compare.scores[max_class_name]["class"] #max_overall_score = compare.scores[max_overall_name]["overall"] - return (max_overall_name,max_class_name) + return (max_overall_name, max_class_name) def __compare_assign_handler__(compare, cm_dict, weight, digit): From 733505500fcf9543f46be4e17066dd8febb4f7bd Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 18:09:55 +0430 Subject: [PATCH 087/107] fix : isINTRAVIS renamed to IS_IN_TRAVIS --- Otherfiles/test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Otherfiles/test.sh b/Otherfiles/test.sh index 3002ff3c..cb20576d 100755 --- a/Otherfiles/test.sh +++ b/Otherfiles/test.sh @@ -5,13 +5,13 @@ python Otherfiles/version_check.py #Check if we are in TRAVIS - isInTRAVIS=false + IS_IN_TRAVIS=false if [ "$CI" = 'true' ] && [ "$TRAVIS" = 'true' ] then - isInTRAVIS=true + IS_IN_TRAVIS=true fi - if [ "$isInTRAVIS" = 'false' ] || { [ "$isInTRAVIS" = 'true' ] && [ "$TRAVIS_PYTHON_VERSION" = '3.6' ]; } + if [ "$IS_IN_TRAVIS" = 'false' ] || { [ "$IS_IN_TRAVIS" = 'true' ] && [ "$TRAVIS_PYTHON_VERSION" = '3.6' ]; } then python -m vulture pycm/ Otherfiles/ setup.py --min-confidence 65 --exclude=build,.eggs --sort-by-size python -m bandit -r pycm -s B311 From 933d9ddfffe4bfd4e6e584cf377e30ed6804c006 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 18:30:50 +0430 Subject: [PATCH 088/107] fix : output_test updated --- Test/output_test.py | 12 ++++++++++++ pycm/pycm_compare.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Test/output_test.py b/Test/output_test.py index 50702916..0a1bfa92 100644 --- a/Test/output_test.py +++ b/Test/output_test.py @@ -441,6 +441,17 @@ False >>> cm_file.matrix == {'1': {'1': 1, '2': 1, '0': 0}, '2': {'1': 2, '2': 3, '0': 0}, '0': {'1': 0, '2': 2, '0': 3}} True +>>> json.dump({"Actual-Vector": ['1', '1', '2', '2', '2', '2', '2', '0', '0', '0', '0', '0'], "Digit": 5, "Predict-Vector": ['1', '2', '1', '1', '2', '2', '2', '2', '2', '0', '0', '0'], "Matrix": {"0": {"0": 3, "1": 0, "2": 2}, "1": {"0": 0, "1": 1, "2": 1}, "2": {"0": 0, "1": 2, "2": 3}}},open("test7.obj","w")) +>>> cm_file=ConfusionMatrix(file=open("test7.obj","r")) +>>> cm_file.weights +>>> cm_file.transpose +False +>>> cm_file.matrix == {'1': {'1': 1, '2': 1, '0': 0}, '2': {'1': 2, '2': 3, '0': 0}, '0': {'1': 0, '2': 2, '0': 3}} +True +>>> cm_file.actual_vector == ['1', '1', '2', '2', '2', '2', '2', '0', '0', '0', '0', '0'] +True +>>> cm_file.predict_vector == ['1', '2', '1', '1', '2', '2', '2', '2', '2', '0', '0', '0'] +True >>> cm_comp1 = ConfusionMatrix(matrix={0:{0:2,1:50,2:6},1:{0:5,1:50,2:3},2:{0:1,1:7,2:50}}) >>> cm_comp2 = ConfusionMatrix(matrix={0:{0:50,1:2,2:6},1:{0:50,1:5,2:3},2:{0:1,1:55,2:2}}) >>> cp = Compare({"model1":cm_comp1,"model2":cm_comp2}) @@ -486,6 +497,7 @@ >>> os.remove("test4.obj") >>> os.remove("test5.obj") >>> os.remove("test6.obj") +>>> os.remove("test7.obj") >>> os.remove("test.pycm") >>> os.remove("test.comp") """ diff --git a/pycm/pycm_compare.py b/pycm/pycm_compare.py index b6dacf8e..780ffc54 100644 --- a/pycm/pycm_compare.py +++ b/pycm/pycm_compare.py @@ -106,7 +106,7 @@ def save_report( file.write(report) file.close() if address: - message = os.path.join(os.getcwd(), name + ".comp") + message = os.path.join(os.getcwd(), name + ".comp") # pragma: no cover return {"Status": True, "Message": message} except Exception as e: return {"Status": False, "Message": str(e)} From d71e4debab871b675dec1d01cc7e76f9e3cb4207 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 18:38:05 +0430 Subject: [PATCH 089/107] fix : warning test splitted --- Test/overall_test.py | 318 ------------------------------------------ Test/warning_test.py | 322 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 322 insertions(+), 318 deletions(-) create mode 100644 Test/warning_test.py diff --git a/Test/overall_test.py b/Test/overall_test.py index acbe3908..8dcf1f60 100644 --- a/Test/overall_test.py +++ b/Test/overall_test.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """ >>> from pycm import * ->>> from pytest import warns >>> import os >>> import json >>> y_actu = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2] @@ -1132,321 +1131,4 @@ >>> cm = ConfusionMatrix(matrix={1:{1:60,2:9,3:1,4:0,5:0,6:0},2:{1:23,2:48,3:0,4:2,5:2,6:1},3:{1:11,2:5,3:60,4:0,5:0,6:0},4:{1:0,2:2,3:0,4:60,5:1,6:3},5:{1:2,2:1,3:0,4:0,5:60,6:2},6:{1:1,2:2,3:0,4:2,5:1,6:60}}) >>> set(cm.recommended_list) == set(MULTICLASS_RECOMMEND) True ->>> large_cm = ConfusionMatrix(list(range(10))+[2,3,5],list(range(10))+[1,7,2]) ->>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'): -... large_cm.print_matrix() -Predict 0 1 2 3 4 5 6 7 8 9 -Actual -0 1 0 0 0 0 0 0 0 0 0 - -1 0 1 0 0 0 0 0 0 0 0 - -2 0 1 1 0 0 0 0 0 0 0 - -3 0 0 0 1 0 0 0 1 0 0 - -4 0 0 0 0 1 0 0 0 0 0 - -5 0 0 1 0 0 1 0 0 0 0 - -6 0 0 0 0 0 0 1 0 0 0 - -7 0 0 0 0 0 0 0 1 0 0 - -8 0 0 0 0 0 0 0 0 1 0 - -9 0 0 0 0 0 0 0 0 0 1 - ->>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'): -... large_cm.print_normalized_matrix() -Predict 0 1 2 3 4 5 6 7 8 9 -Actual -0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 - -1 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 - -2 0.0 0.5 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 - -3 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.5 0.0 0.0 - -4 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 - -5 0.0 0.0 0.5 0.0 0.0 0.5 0.0 0.0 0.0 0.0 - -6 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 - -7 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 - -8 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 - -9 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 - ->>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'): -... large_cm.stat() -Overall Statistics : - -95% CI (0.5402,0.99827) -ACC Macro 0.95385 -AUNP 0.87121 -AUNU 0.91212 -Bennett S 0.74359 -CBA 0.75 -Chi-Squared 91.0 -Chi-Squared DF 81 -Conditional Entropy 0.46154 -Cramer V 0.88192 -Cross Entropy 3.39275 -F1 Macro 0.81667 -F1 Micro 0.76923 -Gwet AC1 0.7438 -Hamming Loss 0.23077 -Joint Entropy 3.70044 -KL Divergence 0.15385 -Kappa 0.74342 -Kappa 95% CI (0.48877,0.99807) -Kappa No Prevalence 0.53846 -Kappa Standard Error 0.12992 -Kappa Unbiased 0.74172 -Lambda A 0.72727 -Lambda B 0.72727 -Mutual Information 2.77736 -NIR 0.15385 -Overall ACC 0.76923 -Overall CEN 0.09537 -Overall J (7.33333,0.73333) -Overall MCC 0.75333 -Overall MCEN 0.10746 -Overall RACC 0.10059 -Overall RACCU 0.10651 -P-Value 0.0 -PPV Macro 0.85 -PPV Micro 0.76923 -Pearson C 0.93541 -Phi-Squared 7.0 -RCI 0.8575 -RR 1.3 -Reference Entropy 3.2389 -Response Entropy 3.2389 -SOA1(Landis & Koch) Substantial -SOA2(Fleiss) Intermediate to Good -SOA3(Altman) Good -SOA4(Cicchetti) Excellent -SOA5(Cramer) Very Strong -SOA6(Matthews) Strong -Scott PI 0.74172 -Standard Error 0.11685 -TPR Macro 0.85 -TPR Micro 0.76923 -Zero-one Loss 3 - -Class Statistics : - -Classes 0 1 2 3 4 5 6 7 8 9 -ACC(Accuracy) 1.0 0.92308 0.84615 0.92308 1.0 0.92308 1.0 0.92308 1.0 1.0 -AGF(Adjusted F-score) 1.0 0.90468 0.6742 0.71965 1.0 0.71965 1.0 0.90468 1.0 1.0 -AGM(Adjusted geometric mean) 1.0 0.93786 0.78186 0.84135 1.0 0.84135 1.0 0.93786 1.0 1.0 -AM(Difference between automatic and manual classification) 0 1 0 -1 0 -1 0 1 0 0 -AUC(Area under the ROC curve) 1.0 0.95833 0.70455 0.75 1.0 0.75 1.0 0.95833 1.0 1.0 -AUCI(AUC value interpretation) Excellent Excellent Good Good Excellent Good Excellent Excellent Excellent Excellent -AUPR(Area under the PR curve) 1.0 0.75 0.5 0.75 1.0 0.75 1.0 0.75 1.0 1.0 -BCD(Bray-Curtis dissimilarity) 0.0 0.03846 0.0 0.03846 0.0 0.03846 0.0 0.03846 0.0 0.0 -BM(Informedness or bookmaker informedness) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 -CEN(Confusion entropy) 0 0.1267 0.23981 0.1267 0 0.1267 0 0.1267 0 0 -DOR(Diagnostic odds ratio) None None 10.0 None None None None None None None -DP(Discriminant power) None None 0.55133 None None None None None None None -DPI(Discriminant power interpretation) None None Poor None None None None None None None -ERR(Error rate) 0.0 0.07692 0.15385 0.07692 0.0 0.07692 0.0 0.07692 0.0 0.0 -F0.5(F0.5 score) 1.0 0.55556 0.5 0.83333 1.0 0.83333 1.0 0.55556 1.0 1.0 -F1(F1 score - harmonic mean of precision and sensitivity) 1.0 0.66667 0.5 0.66667 1.0 0.66667 1.0 0.66667 1.0 1.0 -F2(F2 score) 1.0 0.83333 0.5 0.55556 1.0 0.55556 1.0 0.83333 1.0 1.0 -FDR(False discovery rate) 0.0 0.5 0.5 0.0 0.0 0.0 0.0 0.5 0.0 0.0 -FN(False negative/miss/type 2 error) 0 0 1 1 0 1 0 0 0 0 -FNR(Miss rate or false negative rate) 0.0 0.0 0.5 0.5 0.0 0.5 0.0 0.0 0.0 0.0 -FOR(False omission rate) 0.0 0.0 0.09091 0.08333 0.0 0.08333 0.0 0.0 0.0 0.0 -FP(False positive/type 1 error/false alarm) 0 1 1 0 0 0 0 1 0 0 -FPR(Fall-out or false positive rate) 0.0 0.08333 0.09091 0.0 0.0 0.0 0.0 0.08333 0.0 0.0 -G(G-measure geometric mean of precision and sensitivity) 1.0 0.70711 0.5 0.70711 1.0 0.70711 1.0 0.70711 1.0 1.0 -GI(Gini index) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 -GM(G-mean geometric mean of specificity and sensitivity) 1.0 0.95743 0.6742 0.70711 1.0 0.70711 1.0 0.95743 1.0 1.0 -IBA(Index of balanced accuracy) 1.0 0.99306 0.2686 0.25 1.0 0.25 1.0 0.99306 1.0 1.0 -IS(Information score) 3.70044 2.70044 1.70044 2.70044 3.70044 2.70044 3.70044 2.70044 3.70044 3.70044 -J(Jaccard index) 1.0 0.5 0.33333 0.5 1.0 0.5 1.0 0.5 1.0 1.0 -LS(Lift score) 13.0 6.5 3.25 6.5 13.0 6.5 13.0 6.5 13.0 13.0 -MCC(Matthews correlation coefficient) 1.0 0.677 0.40909 0.677 1.0 0.677 1.0 0.677 1.0 1.0 -MCCI(Matthews correlation coefficient interpretation) Very Strong Moderate Weak Moderate Very Strong Moderate Very Strong Moderate Very Strong Very Strong -MCEN(Modified confusion entropy) 0 0.11991 0.2534 0.11991 0 0.11991 0 0.11991 0 0 -MK(Markedness) 1.0 0.5 0.40909 0.91667 1.0 0.91667 1.0 0.5 1.0 1.0 -N(Condition negative) 12 12 11 11 12 11 12 12 12 12 -NLR(Negative likelihood ratio) 0.0 0.0 0.55 0.5 0.0 0.5 0.0 0.0 0.0 0.0 -NLRI(Negative likelihood ratio interpretation) Good Good Negligible Negligible Good Negligible Good Good Good Good -NPV(Negative predictive value) 1.0 1.0 0.90909 0.91667 1.0 0.91667 1.0 1.0 1.0 1.0 -OC(Overlap coefficient) 1.0 1.0 0.5 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -OOC(Otsuka-Ochiai coefficient) 1.0 0.70711 0.5 0.70711 1.0 0.70711 1.0 0.70711 1.0 1.0 -OP(Optimized precision) 1.0 0.8796 0.55583 0.58974 1.0 0.58974 1.0 0.8796 1.0 1.0 -P(Condition positive or support) 1 1 2 2 1 2 1 1 1 1 -PLR(Positive likelihood ratio) None 12.0 5.5 None None None None 12.0 None None -PLRI(Positive likelihood ratio interpretation) None Good Fair None None None None Good None None -POP(Population) 13 13 13 13 13 13 13 13 13 13 -PPV(Precision or positive predictive value) 1.0 0.5 0.5 1.0 1.0 1.0 1.0 0.5 1.0 1.0 -PRE(Prevalence) 0.07692 0.07692 0.15385 0.15385 0.07692 0.15385 0.07692 0.07692 0.07692 0.07692 -Q(Yule Q - coefficient of colligation) None None 0.81818 None None None None None None None -RACC(Random accuracy) 0.00592 0.01183 0.02367 0.01183 0.00592 0.01183 0.00592 0.01183 0.00592 0.00592 -RACCU(Random accuracy unbiased) 0.00592 0.01331 0.02367 0.01331 0.00592 0.01331 0.00592 0.01331 0.00592 0.00592 -TN(True negative/correct rejection) 12 11 10 11 12 11 12 11 12 12 -TNR(Specificity or true negative rate) 1.0 0.91667 0.90909 1.0 1.0 1.0 1.0 0.91667 1.0 1.0 -TON(Test outcome negative) 12 11 11 12 12 12 12 11 12 12 -TOP(Test outcome positive) 1 2 2 1 1 1 1 2 1 1 -TP(True positive/hit) 1 1 1 1 1 1 1 1 1 1 -TPR(Sensitivity, recall, hit rate, or true positive rate) 1.0 1.0 0.5 0.5 1.0 0.5 1.0 1.0 1.0 1.0 -Y(Youden index) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 -dInd(Distance index) 0.0 0.08333 0.5082 0.5 0.0 0.5 0.0 0.08333 0.0 0.0 -sInd(Similarity index) 1.0 0.94107 0.64065 0.64645 1.0 0.64645 1.0 0.94107 1.0 1.0 - ->>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'): -... print(large_cm) -Predict 0 1 2 3 4 5 6 7 8 9 -Actual -0 1 0 0 0 0 0 0 0 0 0 - -1 0 1 0 0 0 0 0 0 0 0 - -2 0 1 1 0 0 0 0 0 0 0 - -3 0 0 0 1 0 0 0 1 0 0 - -4 0 0 0 0 1 0 0 0 0 0 - -5 0 0 1 0 0 1 0 0 0 0 - -6 0 0 0 0 0 0 1 0 0 0 - -7 0 0 0 0 0 0 0 1 0 0 - -8 0 0 0 0 0 0 0 0 1 0 - -9 0 0 0 0 0 0 0 0 0 1 - - - - - -Overall Statistics : - -95% CI (0.5402,0.99827) -ACC Macro 0.95385 -AUNP 0.87121 -AUNU 0.91212 -Bennett S 0.74359 -CBA 0.75 -Chi-Squared 91.0 -Chi-Squared DF 81 -Conditional Entropy 0.46154 -Cramer V 0.88192 -Cross Entropy 3.39275 -F1 Macro 0.81667 -F1 Micro 0.76923 -Gwet AC1 0.7438 -Hamming Loss 0.23077 -Joint Entropy 3.70044 -KL Divergence 0.15385 -Kappa 0.74342 -Kappa 95% CI (0.48877,0.99807) -Kappa No Prevalence 0.53846 -Kappa Standard Error 0.12992 -Kappa Unbiased 0.74172 -Lambda A 0.72727 -Lambda B 0.72727 -Mutual Information 2.77736 -NIR 0.15385 -Overall ACC 0.76923 -Overall CEN 0.09537 -Overall J (7.33333,0.73333) -Overall MCC 0.75333 -Overall MCEN 0.10746 -Overall RACC 0.10059 -Overall RACCU 0.10651 -P-Value 0.0 -PPV Macro 0.85 -PPV Micro 0.76923 -Pearson C 0.93541 -Phi-Squared 7.0 -RCI 0.8575 -RR 1.3 -Reference Entropy 3.2389 -Response Entropy 3.2389 -SOA1(Landis & Koch) Substantial -SOA2(Fleiss) Intermediate to Good -SOA3(Altman) Good -SOA4(Cicchetti) Excellent -SOA5(Cramer) Very Strong -SOA6(Matthews) Strong -Scott PI 0.74172 -Standard Error 0.11685 -TPR Macro 0.85 -TPR Micro 0.76923 -Zero-one Loss 3 - -Class Statistics : - -Classes 0 1 2 3 4 5 6 7 8 9 -ACC(Accuracy) 1.0 0.92308 0.84615 0.92308 1.0 0.92308 1.0 0.92308 1.0 1.0 -AGF(Adjusted F-score) 1.0 0.90468 0.6742 0.71965 1.0 0.71965 1.0 0.90468 1.0 1.0 -AGM(Adjusted geometric mean) 1.0 0.93786 0.78186 0.84135 1.0 0.84135 1.0 0.93786 1.0 1.0 -AM(Difference between automatic and manual classification) 0 1 0 -1 0 -1 0 1 0 0 -AUC(Area under the ROC curve) 1.0 0.95833 0.70455 0.75 1.0 0.75 1.0 0.95833 1.0 1.0 -AUCI(AUC value interpretation) Excellent Excellent Good Good Excellent Good Excellent Excellent Excellent Excellent -AUPR(Area under the PR curve) 1.0 0.75 0.5 0.75 1.0 0.75 1.0 0.75 1.0 1.0 -BCD(Bray-Curtis dissimilarity) 0.0 0.03846 0.0 0.03846 0.0 0.03846 0.0 0.03846 0.0 0.0 -BM(Informedness or bookmaker informedness) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 -CEN(Confusion entropy) 0 0.1267 0.23981 0.1267 0 0.1267 0 0.1267 0 0 -DOR(Diagnostic odds ratio) None None 10.0 None None None None None None None -DP(Discriminant power) None None 0.55133 None None None None None None None -DPI(Discriminant power interpretation) None None Poor None None None None None None None -ERR(Error rate) 0.0 0.07692 0.15385 0.07692 0.0 0.07692 0.0 0.07692 0.0 0.0 -F0.5(F0.5 score) 1.0 0.55556 0.5 0.83333 1.0 0.83333 1.0 0.55556 1.0 1.0 -F1(F1 score - harmonic mean of precision and sensitivity) 1.0 0.66667 0.5 0.66667 1.0 0.66667 1.0 0.66667 1.0 1.0 -F2(F2 score) 1.0 0.83333 0.5 0.55556 1.0 0.55556 1.0 0.83333 1.0 1.0 -FDR(False discovery rate) 0.0 0.5 0.5 0.0 0.0 0.0 0.0 0.5 0.0 0.0 -FN(False negative/miss/type 2 error) 0 0 1 1 0 1 0 0 0 0 -FNR(Miss rate or false negative rate) 0.0 0.0 0.5 0.5 0.0 0.5 0.0 0.0 0.0 0.0 -FOR(False omission rate) 0.0 0.0 0.09091 0.08333 0.0 0.08333 0.0 0.0 0.0 0.0 -FP(False positive/type 1 error/false alarm) 0 1 1 0 0 0 0 1 0 0 -FPR(Fall-out or false positive rate) 0.0 0.08333 0.09091 0.0 0.0 0.0 0.0 0.08333 0.0 0.0 -G(G-measure geometric mean of precision and sensitivity) 1.0 0.70711 0.5 0.70711 1.0 0.70711 1.0 0.70711 1.0 1.0 -GI(Gini index) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 -GM(G-mean geometric mean of specificity and sensitivity) 1.0 0.95743 0.6742 0.70711 1.0 0.70711 1.0 0.95743 1.0 1.0 -IBA(Index of balanced accuracy) 1.0 0.99306 0.2686 0.25 1.0 0.25 1.0 0.99306 1.0 1.0 -IS(Information score) 3.70044 2.70044 1.70044 2.70044 3.70044 2.70044 3.70044 2.70044 3.70044 3.70044 -J(Jaccard index) 1.0 0.5 0.33333 0.5 1.0 0.5 1.0 0.5 1.0 1.0 -LS(Lift score) 13.0 6.5 3.25 6.5 13.0 6.5 13.0 6.5 13.0 13.0 -MCC(Matthews correlation coefficient) 1.0 0.677 0.40909 0.677 1.0 0.677 1.0 0.677 1.0 1.0 -MCCI(Matthews correlation coefficient interpretation) Very Strong Moderate Weak Moderate Very Strong Moderate Very Strong Moderate Very Strong Very Strong -MCEN(Modified confusion entropy) 0 0.11991 0.2534 0.11991 0 0.11991 0 0.11991 0 0 -MK(Markedness) 1.0 0.5 0.40909 0.91667 1.0 0.91667 1.0 0.5 1.0 1.0 -N(Condition negative) 12 12 11 11 12 11 12 12 12 12 -NLR(Negative likelihood ratio) 0.0 0.0 0.55 0.5 0.0 0.5 0.0 0.0 0.0 0.0 -NLRI(Negative likelihood ratio interpretation) Good Good Negligible Negligible Good Negligible Good Good Good Good -NPV(Negative predictive value) 1.0 1.0 0.90909 0.91667 1.0 0.91667 1.0 1.0 1.0 1.0 -OC(Overlap coefficient) 1.0 1.0 0.5 1.0 1.0 1.0 1.0 1.0 1.0 1.0 -OOC(Otsuka-Ochiai coefficient) 1.0 0.70711 0.5 0.70711 1.0 0.70711 1.0 0.70711 1.0 1.0 -OP(Optimized precision) 1.0 0.8796 0.55583 0.58974 1.0 0.58974 1.0 0.8796 1.0 1.0 -P(Condition positive or support) 1 1 2 2 1 2 1 1 1 1 -PLR(Positive likelihood ratio) None 12.0 5.5 None None None None 12.0 None None -PLRI(Positive likelihood ratio interpretation) None Good Fair None None None None Good None None -POP(Population) 13 13 13 13 13 13 13 13 13 13 -PPV(Precision or positive predictive value) 1.0 0.5 0.5 1.0 1.0 1.0 1.0 0.5 1.0 1.0 -PRE(Prevalence) 0.07692 0.07692 0.15385 0.15385 0.07692 0.15385 0.07692 0.07692 0.07692 0.07692 -Q(Yule Q - coefficient of colligation) None None 0.81818 None None None None None None None -RACC(Random accuracy) 0.00592 0.01183 0.02367 0.01183 0.00592 0.01183 0.00592 0.01183 0.00592 0.00592 -RACCU(Random accuracy unbiased) 0.00592 0.01331 0.02367 0.01331 0.00592 0.01331 0.00592 0.01331 0.00592 0.00592 -TN(True negative/correct rejection) 12 11 10 11 12 11 12 11 12 12 -TNR(Specificity or true negative rate) 1.0 0.91667 0.90909 1.0 1.0 1.0 1.0 0.91667 1.0 1.0 -TON(Test outcome negative) 12 11 11 12 12 12 12 11 12 12 -TOP(Test outcome positive) 1 2 2 1 1 1 1 2 1 1 -TP(True positive/hit) 1 1 1 1 1 1 1 1 1 1 -TPR(Sensitivity, recall, hit rate, or true positive rate) 1.0 1.0 0.5 0.5 1.0 0.5 1.0 1.0 1.0 1.0 -Y(Youden index) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 -dInd(Distance index) 0.0 0.08333 0.5082 0.5 0.0 0.5 0.0 0.08333 0.0 0.0 -sInd(Similarity index) 1.0 0.94107 0.64065 0.64645 1.0 0.64645 1.0 0.94107 1.0 1.0 - """ diff --git a/Test/warning_test.py b/Test/warning_test.py new file mode 100644 index 00000000..37cb06d3 --- /dev/null +++ b/Test/warning_test.py @@ -0,0 +1,322 @@ +# -*- coding: utf-8 -*- +""" +>>> from pycm import * +>>> from pytest import warns +>>> large_cm = ConfusionMatrix(list(range(10))+[2,3,5],list(range(10))+[1,7,2]) +>>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'): +... large_cm.print_matrix() +Predict 0 1 2 3 4 5 6 7 8 9 +Actual +0 1 0 0 0 0 0 0 0 0 0 + +1 0 1 0 0 0 0 0 0 0 0 + +2 0 1 1 0 0 0 0 0 0 0 + +3 0 0 0 1 0 0 0 1 0 0 + +4 0 0 0 0 1 0 0 0 0 0 + +5 0 0 1 0 0 1 0 0 0 0 + +6 0 0 0 0 0 0 1 0 0 0 + +7 0 0 0 0 0 0 0 1 0 0 + +8 0 0 0 0 0 0 0 0 1 0 + +9 0 0 0 0 0 0 0 0 0 1 + +>>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'): +... large_cm.print_normalized_matrix() +Predict 0 1 2 3 4 5 6 7 8 9 +Actual +0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 + +1 0.0 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 + +2 0.0 0.5 0.5 0.0 0.0 0.0 0.0 0.0 0.0 0.0 + +3 0.0 0.0 0.0 0.5 0.0 0.0 0.0 0.5 0.0 0.0 + +4 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 0.0 0.0 + +5 0.0 0.0 0.5 0.0 0.0 0.5 0.0 0.0 0.0 0.0 + +6 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 0.0 + +7 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.0 + +8 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 + +9 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 + +>>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'): +... large_cm.stat() +Overall Statistics : + +95% CI (0.5402,0.99827) +ACC Macro 0.95385 +AUNP 0.87121 +AUNU 0.91212 +Bennett S 0.74359 +CBA 0.75 +Chi-Squared 91.0 +Chi-Squared DF 81 +Conditional Entropy 0.46154 +Cramer V 0.88192 +Cross Entropy 3.39275 +F1 Macro 0.81667 +F1 Micro 0.76923 +Gwet AC1 0.7438 +Hamming Loss 0.23077 +Joint Entropy 3.70044 +KL Divergence 0.15385 +Kappa 0.74342 +Kappa 95% CI (0.48877,0.99807) +Kappa No Prevalence 0.53846 +Kappa Standard Error 0.12992 +Kappa Unbiased 0.74172 +Lambda A 0.72727 +Lambda B 0.72727 +Mutual Information 2.77736 +NIR 0.15385 +Overall ACC 0.76923 +Overall CEN 0.09537 +Overall J (7.33333,0.73333) +Overall MCC 0.75333 +Overall MCEN 0.10746 +Overall RACC 0.10059 +Overall RACCU 0.10651 +P-Value 0.0 +PPV Macro 0.85 +PPV Micro 0.76923 +Pearson C 0.93541 +Phi-Squared 7.0 +RCI 0.8575 +RR 1.3 +Reference Entropy 3.2389 +Response Entropy 3.2389 +SOA1(Landis & Koch) Substantial +SOA2(Fleiss) Intermediate to Good +SOA3(Altman) Good +SOA4(Cicchetti) Excellent +SOA5(Cramer) Very Strong +SOA6(Matthews) Strong +Scott PI 0.74172 +Standard Error 0.11685 +TPR Macro 0.85 +TPR Micro 0.76923 +Zero-one Loss 3 + +Class Statistics : + +Classes 0 1 2 3 4 5 6 7 8 9 +ACC(Accuracy) 1.0 0.92308 0.84615 0.92308 1.0 0.92308 1.0 0.92308 1.0 1.0 +AGF(Adjusted F-score) 1.0 0.90468 0.6742 0.71965 1.0 0.71965 1.0 0.90468 1.0 1.0 +AGM(Adjusted geometric mean) 1.0 0.93786 0.78186 0.84135 1.0 0.84135 1.0 0.93786 1.0 1.0 +AM(Difference between automatic and manual classification) 0 1 0 -1 0 -1 0 1 0 0 +AUC(Area under the ROC curve) 1.0 0.95833 0.70455 0.75 1.0 0.75 1.0 0.95833 1.0 1.0 +AUCI(AUC value interpretation) Excellent Excellent Good Good Excellent Good Excellent Excellent Excellent Excellent +AUPR(Area under the PR curve) 1.0 0.75 0.5 0.75 1.0 0.75 1.0 0.75 1.0 1.0 +BCD(Bray-Curtis dissimilarity) 0.0 0.03846 0.0 0.03846 0.0 0.03846 0.0 0.03846 0.0 0.0 +BM(Informedness or bookmaker informedness) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 +CEN(Confusion entropy) 0 0.1267 0.23981 0.1267 0 0.1267 0 0.1267 0 0 +DOR(Diagnostic odds ratio) None None 10.0 None None None None None None None +DP(Discriminant power) None None 0.55133 None None None None None None None +DPI(Discriminant power interpretation) None None Poor None None None None None None None +ERR(Error rate) 0.0 0.07692 0.15385 0.07692 0.0 0.07692 0.0 0.07692 0.0 0.0 +F0.5(F0.5 score) 1.0 0.55556 0.5 0.83333 1.0 0.83333 1.0 0.55556 1.0 1.0 +F1(F1 score - harmonic mean of precision and sensitivity) 1.0 0.66667 0.5 0.66667 1.0 0.66667 1.0 0.66667 1.0 1.0 +F2(F2 score) 1.0 0.83333 0.5 0.55556 1.0 0.55556 1.0 0.83333 1.0 1.0 +FDR(False discovery rate) 0.0 0.5 0.5 0.0 0.0 0.0 0.0 0.5 0.0 0.0 +FN(False negative/miss/type 2 error) 0 0 1 1 0 1 0 0 0 0 +FNR(Miss rate or false negative rate) 0.0 0.0 0.5 0.5 0.0 0.5 0.0 0.0 0.0 0.0 +FOR(False omission rate) 0.0 0.0 0.09091 0.08333 0.0 0.08333 0.0 0.0 0.0 0.0 +FP(False positive/type 1 error/false alarm) 0 1 1 0 0 0 0 1 0 0 +FPR(Fall-out or false positive rate) 0.0 0.08333 0.09091 0.0 0.0 0.0 0.0 0.08333 0.0 0.0 +G(G-measure geometric mean of precision and sensitivity) 1.0 0.70711 0.5 0.70711 1.0 0.70711 1.0 0.70711 1.0 1.0 +GI(Gini index) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 +GM(G-mean geometric mean of specificity and sensitivity) 1.0 0.95743 0.6742 0.70711 1.0 0.70711 1.0 0.95743 1.0 1.0 +IBA(Index of balanced accuracy) 1.0 0.99306 0.2686 0.25 1.0 0.25 1.0 0.99306 1.0 1.0 +IS(Information score) 3.70044 2.70044 1.70044 2.70044 3.70044 2.70044 3.70044 2.70044 3.70044 3.70044 +J(Jaccard index) 1.0 0.5 0.33333 0.5 1.0 0.5 1.0 0.5 1.0 1.0 +LS(Lift score) 13.0 6.5 3.25 6.5 13.0 6.5 13.0 6.5 13.0 13.0 +MCC(Matthews correlation coefficient) 1.0 0.677 0.40909 0.677 1.0 0.677 1.0 0.677 1.0 1.0 +MCCI(Matthews correlation coefficient interpretation) Very Strong Moderate Weak Moderate Very Strong Moderate Very Strong Moderate Very Strong Very Strong +MCEN(Modified confusion entropy) 0 0.11991 0.2534 0.11991 0 0.11991 0 0.11991 0 0 +MK(Markedness) 1.0 0.5 0.40909 0.91667 1.0 0.91667 1.0 0.5 1.0 1.0 +N(Condition negative) 12 12 11 11 12 11 12 12 12 12 +NLR(Negative likelihood ratio) 0.0 0.0 0.55 0.5 0.0 0.5 0.0 0.0 0.0 0.0 +NLRI(Negative likelihood ratio interpretation) Good Good Negligible Negligible Good Negligible Good Good Good Good +NPV(Negative predictive value) 1.0 1.0 0.90909 0.91667 1.0 0.91667 1.0 1.0 1.0 1.0 +OC(Overlap coefficient) 1.0 1.0 0.5 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +OOC(Otsuka-Ochiai coefficient) 1.0 0.70711 0.5 0.70711 1.0 0.70711 1.0 0.70711 1.0 1.0 +OP(Optimized precision) 1.0 0.8796 0.55583 0.58974 1.0 0.58974 1.0 0.8796 1.0 1.0 +P(Condition positive or support) 1 1 2 2 1 2 1 1 1 1 +PLR(Positive likelihood ratio) None 12.0 5.5 None None None None 12.0 None None +PLRI(Positive likelihood ratio interpretation) None Good Fair None None None None Good None None +POP(Population) 13 13 13 13 13 13 13 13 13 13 +PPV(Precision or positive predictive value) 1.0 0.5 0.5 1.0 1.0 1.0 1.0 0.5 1.0 1.0 +PRE(Prevalence) 0.07692 0.07692 0.15385 0.15385 0.07692 0.15385 0.07692 0.07692 0.07692 0.07692 +Q(Yule Q - coefficient of colligation) None None 0.81818 None None None None None None None +RACC(Random accuracy) 0.00592 0.01183 0.02367 0.01183 0.00592 0.01183 0.00592 0.01183 0.00592 0.00592 +RACCU(Random accuracy unbiased) 0.00592 0.01331 0.02367 0.01331 0.00592 0.01331 0.00592 0.01331 0.00592 0.00592 +TN(True negative/correct rejection) 12 11 10 11 12 11 12 11 12 12 +TNR(Specificity or true negative rate) 1.0 0.91667 0.90909 1.0 1.0 1.0 1.0 0.91667 1.0 1.0 +TON(Test outcome negative) 12 11 11 12 12 12 12 11 12 12 +TOP(Test outcome positive) 1 2 2 1 1 1 1 2 1 1 +TP(True positive/hit) 1 1 1 1 1 1 1 1 1 1 +TPR(Sensitivity, recall, hit rate, or true positive rate) 1.0 1.0 0.5 0.5 1.0 0.5 1.0 1.0 1.0 1.0 +Y(Youden index) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 +dInd(Distance index) 0.0 0.08333 0.5082 0.5 0.0 0.5 0.0 0.08333 0.0 0.0 +sInd(Similarity index) 1.0 0.94107 0.64065 0.64645 1.0 0.64645 1.0 0.94107 1.0 1.0 + +>>> with warns(RuntimeWarning, match='The confusion matrix is a high dimension matrix'): +... print(large_cm) +Predict 0 1 2 3 4 5 6 7 8 9 +Actual +0 1 0 0 0 0 0 0 0 0 0 + +1 0 1 0 0 0 0 0 0 0 0 + +2 0 1 1 0 0 0 0 0 0 0 + +3 0 0 0 1 0 0 0 1 0 0 + +4 0 0 0 0 1 0 0 0 0 0 + +5 0 0 1 0 0 1 0 0 0 0 + +6 0 0 0 0 0 0 1 0 0 0 + +7 0 0 0 0 0 0 0 1 0 0 + +8 0 0 0 0 0 0 0 0 1 0 + +9 0 0 0 0 0 0 0 0 0 1 + + + + + +Overall Statistics : + +95% CI (0.5402,0.99827) +ACC Macro 0.95385 +AUNP 0.87121 +AUNU 0.91212 +Bennett S 0.74359 +CBA 0.75 +Chi-Squared 91.0 +Chi-Squared DF 81 +Conditional Entropy 0.46154 +Cramer V 0.88192 +Cross Entropy 3.39275 +F1 Macro 0.81667 +F1 Micro 0.76923 +Gwet AC1 0.7438 +Hamming Loss 0.23077 +Joint Entropy 3.70044 +KL Divergence 0.15385 +Kappa 0.74342 +Kappa 95% CI (0.48877,0.99807) +Kappa No Prevalence 0.53846 +Kappa Standard Error 0.12992 +Kappa Unbiased 0.74172 +Lambda A 0.72727 +Lambda B 0.72727 +Mutual Information 2.77736 +NIR 0.15385 +Overall ACC 0.76923 +Overall CEN 0.09537 +Overall J (7.33333,0.73333) +Overall MCC 0.75333 +Overall MCEN 0.10746 +Overall RACC 0.10059 +Overall RACCU 0.10651 +P-Value 0.0 +PPV Macro 0.85 +PPV Micro 0.76923 +Pearson C 0.93541 +Phi-Squared 7.0 +RCI 0.8575 +RR 1.3 +Reference Entropy 3.2389 +Response Entropy 3.2389 +SOA1(Landis & Koch) Substantial +SOA2(Fleiss) Intermediate to Good +SOA3(Altman) Good +SOA4(Cicchetti) Excellent +SOA5(Cramer) Very Strong +SOA6(Matthews) Strong +Scott PI 0.74172 +Standard Error 0.11685 +TPR Macro 0.85 +TPR Micro 0.76923 +Zero-one Loss 3 + +Class Statistics : + +Classes 0 1 2 3 4 5 6 7 8 9 +ACC(Accuracy) 1.0 0.92308 0.84615 0.92308 1.0 0.92308 1.0 0.92308 1.0 1.0 +AGF(Adjusted F-score) 1.0 0.90468 0.6742 0.71965 1.0 0.71965 1.0 0.90468 1.0 1.0 +AGM(Adjusted geometric mean) 1.0 0.93786 0.78186 0.84135 1.0 0.84135 1.0 0.93786 1.0 1.0 +AM(Difference between automatic and manual classification) 0 1 0 -1 0 -1 0 1 0 0 +AUC(Area under the ROC curve) 1.0 0.95833 0.70455 0.75 1.0 0.75 1.0 0.95833 1.0 1.0 +AUCI(AUC value interpretation) Excellent Excellent Good Good Excellent Good Excellent Excellent Excellent Excellent +AUPR(Area under the PR curve) 1.0 0.75 0.5 0.75 1.0 0.75 1.0 0.75 1.0 1.0 +BCD(Bray-Curtis dissimilarity) 0.0 0.03846 0.0 0.03846 0.0 0.03846 0.0 0.03846 0.0 0.0 +BM(Informedness or bookmaker informedness) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 +CEN(Confusion entropy) 0 0.1267 0.23981 0.1267 0 0.1267 0 0.1267 0 0 +DOR(Diagnostic odds ratio) None None 10.0 None None None None None None None +DP(Discriminant power) None None 0.55133 None None None None None None None +DPI(Discriminant power interpretation) None None Poor None None None None None None None +ERR(Error rate) 0.0 0.07692 0.15385 0.07692 0.0 0.07692 0.0 0.07692 0.0 0.0 +F0.5(F0.5 score) 1.0 0.55556 0.5 0.83333 1.0 0.83333 1.0 0.55556 1.0 1.0 +F1(F1 score - harmonic mean of precision and sensitivity) 1.0 0.66667 0.5 0.66667 1.0 0.66667 1.0 0.66667 1.0 1.0 +F2(F2 score) 1.0 0.83333 0.5 0.55556 1.0 0.55556 1.0 0.83333 1.0 1.0 +FDR(False discovery rate) 0.0 0.5 0.5 0.0 0.0 0.0 0.0 0.5 0.0 0.0 +FN(False negative/miss/type 2 error) 0 0 1 1 0 1 0 0 0 0 +FNR(Miss rate or false negative rate) 0.0 0.0 0.5 0.5 0.0 0.5 0.0 0.0 0.0 0.0 +FOR(False omission rate) 0.0 0.0 0.09091 0.08333 0.0 0.08333 0.0 0.0 0.0 0.0 +FP(False positive/type 1 error/false alarm) 0 1 1 0 0 0 0 1 0 0 +FPR(Fall-out or false positive rate) 0.0 0.08333 0.09091 0.0 0.0 0.0 0.0 0.08333 0.0 0.0 +G(G-measure geometric mean of precision and sensitivity) 1.0 0.70711 0.5 0.70711 1.0 0.70711 1.0 0.70711 1.0 1.0 +GI(Gini index) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 +GM(G-mean geometric mean of specificity and sensitivity) 1.0 0.95743 0.6742 0.70711 1.0 0.70711 1.0 0.95743 1.0 1.0 +IBA(Index of balanced accuracy) 1.0 0.99306 0.2686 0.25 1.0 0.25 1.0 0.99306 1.0 1.0 +IS(Information score) 3.70044 2.70044 1.70044 2.70044 3.70044 2.70044 3.70044 2.70044 3.70044 3.70044 +J(Jaccard index) 1.0 0.5 0.33333 0.5 1.0 0.5 1.0 0.5 1.0 1.0 +LS(Lift score) 13.0 6.5 3.25 6.5 13.0 6.5 13.0 6.5 13.0 13.0 +MCC(Matthews correlation coefficient) 1.0 0.677 0.40909 0.677 1.0 0.677 1.0 0.677 1.0 1.0 +MCCI(Matthews correlation coefficient interpretation) Very Strong Moderate Weak Moderate Very Strong Moderate Very Strong Moderate Very Strong Very Strong +MCEN(Modified confusion entropy) 0 0.11991 0.2534 0.11991 0 0.11991 0 0.11991 0 0 +MK(Markedness) 1.0 0.5 0.40909 0.91667 1.0 0.91667 1.0 0.5 1.0 1.0 +N(Condition negative) 12 12 11 11 12 11 12 12 12 12 +NLR(Negative likelihood ratio) 0.0 0.0 0.55 0.5 0.0 0.5 0.0 0.0 0.0 0.0 +NLRI(Negative likelihood ratio interpretation) Good Good Negligible Negligible Good Negligible Good Good Good Good +NPV(Negative predictive value) 1.0 1.0 0.90909 0.91667 1.0 0.91667 1.0 1.0 1.0 1.0 +OC(Overlap coefficient) 1.0 1.0 0.5 1.0 1.0 1.0 1.0 1.0 1.0 1.0 +OOC(Otsuka-Ochiai coefficient) 1.0 0.70711 0.5 0.70711 1.0 0.70711 1.0 0.70711 1.0 1.0 +OP(Optimized precision) 1.0 0.8796 0.55583 0.58974 1.0 0.58974 1.0 0.8796 1.0 1.0 +P(Condition positive or support) 1 1 2 2 1 2 1 1 1 1 +PLR(Positive likelihood ratio) None 12.0 5.5 None None None None 12.0 None None +PLRI(Positive likelihood ratio interpretation) None Good Fair None None None None Good None None +POP(Population) 13 13 13 13 13 13 13 13 13 13 +PPV(Precision or positive predictive value) 1.0 0.5 0.5 1.0 1.0 1.0 1.0 0.5 1.0 1.0 +PRE(Prevalence) 0.07692 0.07692 0.15385 0.15385 0.07692 0.15385 0.07692 0.07692 0.07692 0.07692 +Q(Yule Q - coefficient of colligation) None None 0.81818 None None None None None None None +RACC(Random accuracy) 0.00592 0.01183 0.02367 0.01183 0.00592 0.01183 0.00592 0.01183 0.00592 0.00592 +RACCU(Random accuracy unbiased) 0.00592 0.01331 0.02367 0.01331 0.00592 0.01331 0.00592 0.01331 0.00592 0.00592 +TN(True negative/correct rejection) 12 11 10 11 12 11 12 11 12 12 +TNR(Specificity or true negative rate) 1.0 0.91667 0.90909 1.0 1.0 1.0 1.0 0.91667 1.0 1.0 +TON(Test outcome negative) 12 11 11 12 12 12 12 11 12 12 +TOP(Test outcome positive) 1 2 2 1 1 1 1 2 1 1 +TP(True positive/hit) 1 1 1 1 1 1 1 1 1 1 +TPR(Sensitivity, recall, hit rate, or true positive rate) 1.0 1.0 0.5 0.5 1.0 0.5 1.0 1.0 1.0 1.0 +Y(Youden index) 1.0 0.91667 0.40909 0.5 1.0 0.5 1.0 0.91667 1.0 1.0 +dInd(Distance index) 0.0 0.08333 0.5082 0.5 0.0 0.5 0.0 0.08333 0.0 0.0 +sInd(Similarity index) 1.0 0.94107 0.64065 0.64645 1.0 0.64645 1.0 0.94107 1.0 1.0 + +""" From a8badae81d64c5ae1fd87fb7c18cdcaeb33ca425 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Mon, 29 Jul 2019 18:38:52 +0430 Subject: [PATCH 090/107] doc : CHANGELOG updated --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76200f54..422908d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Test system modified - `alt_link` parameter added to `save_html` method and `online_help` function - `Compare` class tests moved to `compare_test.py` +- Warning tests moved to `warning_test.py` ## [2.3] - 2019-06-27 ### Added - Adjusted F-score (AGF) From d3ca1fdb4d995e6f5c604dfc446ff5ed3679620a Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 30 Jul 2019 07:01:00 +0430 Subject: [PATCH 091/107] doc : FUNDING.yml added --- .github/FUNDING.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..8fba89bb --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +custom: https://www.pycm.ir/donate.html \ No newline at end of file From e97e79491660082d8b4098ac1d7e064f0c0ccadf Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 30 Jul 2019 07:01:50 +0430 Subject: [PATCH 092/107] doc : CHANGELOG updated --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 422908d4..6dab8352 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Tversky index (TI) - Area under the PR curve (AUPR) +- `FUNDING.yml` ### Changed - `AUC_calc` function modified - Document modified From fe87c65cfe2bccddc39a0ee661f2b82b96bcf2e0 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 30 Jul 2019 07:45:03 +0430 Subject: [PATCH 093/107] doc : supported versions warning added to Document --- Document/Document.ipynb | 7 +++++++ README.md | 2 ++ 2 files changed, 9 insertions(+) diff --git a/Document/Document.ipynb b/Document/Document.ipynb index 0d694d5a..e949311a 100644 --- a/Document/Document.ipynb +++ b/Document/Document.ipynb @@ -230,6 +230,13 @@ "## Installation\t" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "⚠️ PyCM 2.4 is the last version to support **Python 2.7** & **Python 3.4**" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/README.md b/README.md index 5d530632..b370bbc9 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,8 @@ PyCM is the swiss-army knife of confusion matrices, targeted mainly at data scie ## Installation +⚠️ PyCM 2.4 is the last version to support **Python 2.7** & **Python 3.4** + ### Source code - Download [Version 2.3](https://github.com/sepandhaghighi/pycm/archive/v2.3.zip) or [Latest Source ](https://github.com/sepandhaghighi/pycm/archive/dev.zip) - Run `pip install -r requirements.txt` or `pip3 install -r requirements.txt` (Need root access) From 8a767dce97c83d65dda42e27849af9fbe2bba9e8 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 30 Jul 2019 15:31:25 +0430 Subject: [PATCH 094/107] rel : migrate to version 2.4 --- CHANGELOG.md | 4 +++- README.md | 4 ++-- pycm/pycm_param.py | 2 +- setup.py | 4 ++-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dab8352..f51df1a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [2.4] - 2019-07-30 ### Added - Tversky index (TI) - Area under the PR curve (AUPR) @@ -408,7 +409,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - TPR - documents and `README.md` -[Unreleased]: https://github.com/sepandhaghighi/pycm/compare/v2.3...dev +[Unreleased]: https://github.com/sepandhaghighi/pycm/compare/v2.4...dev +[2.4]: https://github.com/sepandhaghighi/pycm/compare/v2.3...v2.4 [2.3]: https://github.com/sepandhaghighi/pycm/compare/v2.2...v2.3 [2.2]: https://github.com/sepandhaghighi/pycm/compare/v2.1...v2.2 [2.1]: https://github.com/sepandhaghighi/pycm/compare/v2.0...v2.1 diff --git a/README.md b/README.md index b370bbc9..32351162 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ PyCM is the swiss-army knife of confusion matrices, targeted mainly at data scie ⚠️ PyCM 2.4 is the last version to support **Python 2.7** & **Python 3.4** ### Source code -- Download [Version 2.3](https://github.com/sepandhaghighi/pycm/archive/v2.3.zip) or [Latest Source ](https://github.com/sepandhaghighi/pycm/archive/dev.zip) +- Download [Version 2.4](https://github.com/sepandhaghighi/pycm/archive/v2.4.zip) or [Latest Source ](https://github.com/sepandhaghighi/pycm/archive/dev.zip) - Run `pip install -r requirements.txt` or `pip3 install -r requirements.txt` (Need root access) - Run `python3 setup.py install` or `python setup.py install` (Need root access) @@ -105,7 +105,7 @@ PyCM is the swiss-army knife of confusion matrices, targeted mainly at data scie - Check [Python Packaging User Guide](https://packaging.python.org/installing/) -- Run `pip install pycm==2.3` or `pip3 install pycm==2.3` (Need root access) +- Run `pip install pycm==2.4` or `pip3 install pycm==2.4` (Need root access) ### Conda diff --git a/pycm/pycm_param.py b/pycm/pycm_param.py index 9db10c39..571ee3e9 100644 --- a/pycm/pycm_param.py +++ b/pycm/pycm_param.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """Parameters and constants.""" -VERSION = "2.3" +VERSION = "2.4" OVERVIEW = ''' diff --git a/setup.py b/setup.py index 7423244d..e742092a 100644 --- a/setup.py +++ b/setup.py @@ -36,14 +36,14 @@ def read_description(): setup( name='pycm', packages=['pycm'], - version='2.3', + version='2.4', description='Multi-class confusion matrix library in Python', long_description=read_description(), long_description_content_type='text/markdown', author='Sepand Haghighi', author_email='sepand@qpage.ir', url='https://github.com/sepandhaghighi/pycm', - download_url='https://github.com/sepandhaghighi/pycm/tarball/v2.3', + download_url='https://github.com/sepandhaghighi/pycm/tarball/v2.4', keywords="confusion-matrix python3 python machine_learning ML", project_urls={ 'Webpage': 'http://pycm.shaghighi.ir', From b2c406d56b0ef43d09c946d77af010e9b9e7962d Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 30 Jul 2019 15:37:34 +0430 Subject: [PATCH 095/107] doc : Document and examples updated for version 2.4 --- Document/Document.ipynb | 601 ++++++++++---------- Document/Document_Files/cm1.csv | 1 + Document/Document_Files/cm1.html | 11 +- Document/Document_Files/cm1.obj | 2 +- Document/Document_Files/cm1.pycm | 3 +- Document/Document_Files/cm1_colored.html | 11 +- Document/Document_Files/cm1_colored2.html | 11 +- Document/Document_Files/cm1_filtered.html | 4 +- Document/Document_Files/cm1_filtered.pycm | 2 +- Document/Document_Files/cm1_filtered2.html | 4 +- Document/Document_Files/cm1_filtered2.pycm | 2 +- Document/Document_Files/cm1_no_vectors.obj | 2 +- Document/Document_Files/cm1_normalized.html | 11 +- Document/Document_Files/cm1_stat.obj | 2 +- Document/Document_Files/cm1_summary.html | 2 +- Document/Example1.ipynb | 2 +- Document/Example1_Files/cm1.html | 11 +- Document/Example1_Files/cm2.html | 11 +- Document/Example1_Files/cm3.html | 11 +- Document/Example2.ipynb | 8 +- Document/Example3.ipynb | 3 +- Document/Example5.ipynb | 6 +- Document/Example6.ipynb | 66 +-- Document/Example7.ipynb | 16 +- 24 files changed, 431 insertions(+), 372 deletions(-) diff --git a/Document/Document.ipynb b/Document/Document.ipynb index e949311a..54a37174 100644 --- a/Document/Document.ipynb +++ b/Document/Document.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Version : 2.3\n", + "### Version : 2.4\n", "-----" ] }, @@ -242,7 +242,7 @@ "metadata": {}, "source": [ "### Source code\n", - "- Download [Version 2.3](https://github.com/sepandhaghighi/pycm/archive/v2.3.zip) or [Latest Source ](https://github.com/sepandhaghighi/pycm/archive/dev.zip)\n", + "- Download [Version 2.4](https://github.com/sepandhaghighi/pycm/archive/v2.4.zip) or [Latest Source ](https://github.com/sepandhaghighi/pycm/archive/dev.zip)\n", "- Run `pip install -r requirements.txt` or `pip3 install -r requirements.txt` (Need root access)\n", "- Run `python3 setup.py install` or `python setup.py install` (Need root access)" ] @@ -255,7 +255,7 @@ "\n", "\n", "- Check [Python Packaging User Guide](https://packaging.python.org/installing/) \n", - "- Run `pip install pycm==2.3` or `pip3 install pycm==2.3` (Need root access)" + "- Run `pip install pycm==2.4` or `pip3 install pycm==2.4` (Need root access)" ] }, { @@ -423,6 +423,7 @@ " 'AM': {0: 2, 1: -1, 2: -1},\n", " 'AUC': {0: 0.8888888888888888, 1: 0.611111111111111, 2: 0.5833333333333333},\n", " 'AUCI': {0: 'Very Good', 1: 'Fair', 2: 'Poor'},\n", + " 'AUPR': {0: 0.8, 1: 0.41666666666666663, 2: 0.55},\n", " 'BCD': {0: 0.08333333333333333,\n", " 1: 0.041666666666666664,\n", " 2: 0.041666666666666664},\n", @@ -833,6 +834,7 @@ " 'AM': {0: 2, 1: -1, 2: -1},\n", " 'AUC': {0: 0.8888888888888888, 1: 0.611111111111111, 2: 0.5833333333333333},\n", " 'AUCI': {0: 'Very Good', 1: 'Fair', 2: 'Poor'},\n", + " 'AUPR': {0: 0.8, 1: 0.41666666666666663, 2: 0.55},\n", " 'BCD': {0: 0.08333333333333333,\n", " 1: 0.041666666666666664,\n", " 2: 0.041666666666666664},\n", @@ -1082,7 +1084,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -1140,7 +1142,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -1242,107 +1244,108 @@ "8-AUCI\n", "9-AUNP\n", "10-AUNU\n", - "11-BCD\n", - "12-BM\n", - "13-Bennett S\n", - "14-CBA\n", - "15-CEN\n", - "16-Chi-Squared\n", - "17-Chi-Squared DF\n", - "18-Conditional Entropy\n", - "19-Cramer V\n", - "20-Cross Entropy\n", - "21-DOR\n", - "22-DP\n", - "23-DPI\n", - "24-ERR\n", - "25-F0.5\n", - "26-F1\n", - "27-F1 Macro\n", - "28-F1 Micro\n", - "29-F2\n", - "30-FDR\n", - "31-FN\n", - "32-FNR\n", - "33-FOR\n", - "34-FP\n", - "35-FPR\n", - "36-G\n", - "37-GI\n", - "38-GM\n", - "39-Gwet AC1\n", - "40-Hamming Loss\n", - "41-IBA\n", - "42-IS\n", - "43-J\n", - "44-Joint Entropy\n", - "45-KL Divergence\n", - "46-Kappa\n", - "47-Kappa 95% CI\n", - "48-Kappa No Prevalence\n", - "49-Kappa Standard Error\n", - "50-Kappa Unbiased\n", - "51-LS\n", - "52-Lambda A\n", - "53-Lambda B\n", - "54-MCC\n", - "55-MCCI\n", - "56-MCEN\n", - "57-MK\n", - "58-Mutual Information\n", - "59-N\n", - "60-NIR\n", - "61-NLR\n", - "62-NLRI\n", - "63-NPV\n", - "64-OC\n", - "65-OOC\n", - "66-OP\n", - "67-Overall ACC\n", - "68-Overall CEN\n", - "69-Overall J\n", - "70-Overall MCC\n", - "71-Overall MCEN\n", - "72-Overall RACC\n", - "73-Overall RACCU\n", - "74-P\n", - "75-P-Value\n", - "76-PLR\n", - "77-PLRI\n", - "78-POP\n", - "79-PPV\n", - "80-PPV Macro\n", - "81-PPV Micro\n", - "82-PRE\n", - "83-Pearson C\n", - "84-Phi-Squared\n", - "85-Q\n", - "86-RACC\n", - "87-RACCU\n", - "88-RCI\n", - "89-RR\n", - "90-Reference Entropy\n", - "91-Response Entropy\n", - "92-SOA1(Landis & Koch)\n", - "93-SOA2(Fleiss)\n", - "94-SOA3(Altman)\n", - "95-SOA4(Cicchetti)\n", - "96-SOA5(Cramer)\n", - "97-SOA6(Matthews)\n", - "98-Scott PI\n", - "99-Standard Error\n", - "100-TN\n", - "101-TNR\n", - "102-TON\n", - "103-TOP\n", - "104-TP\n", - "105-TPR\n", - "106-TPR Macro\n", - "107-TPR Micro\n", - "108-Y\n", - "109-Zero-one Loss\n", - "110-dInd\n", - "111-sInd\n" + "11-AUPR\n", + "12-BCD\n", + "13-BM\n", + "14-Bennett S\n", + "15-CBA\n", + "16-CEN\n", + "17-Chi-Squared\n", + "18-Chi-Squared DF\n", + "19-Conditional Entropy\n", + "20-Cramer V\n", + "21-Cross Entropy\n", + "22-DOR\n", + "23-DP\n", + "24-DPI\n", + "25-ERR\n", + "26-F0.5\n", + "27-F1\n", + "28-F1 Macro\n", + "29-F1 Micro\n", + "30-F2\n", + "31-FDR\n", + "32-FN\n", + "33-FNR\n", + "34-FOR\n", + "35-FP\n", + "36-FPR\n", + "37-G\n", + "38-GI\n", + "39-GM\n", + "40-Gwet AC1\n", + "41-Hamming Loss\n", + "42-IBA\n", + "43-IS\n", + "44-J\n", + "45-Joint Entropy\n", + "46-KL Divergence\n", + "47-Kappa\n", + "48-Kappa 95% CI\n", + "49-Kappa No Prevalence\n", + "50-Kappa Standard Error\n", + "51-Kappa Unbiased\n", + "52-LS\n", + "53-Lambda A\n", + "54-Lambda B\n", + "55-MCC\n", + "56-MCCI\n", + "57-MCEN\n", + "58-MK\n", + "59-Mutual Information\n", + "60-N\n", + "61-NIR\n", + "62-NLR\n", + "63-NLRI\n", + "64-NPV\n", + "65-OC\n", + "66-OOC\n", + "67-OP\n", + "68-Overall ACC\n", + "69-Overall CEN\n", + "70-Overall J\n", + "71-Overall MCC\n", + "72-Overall MCEN\n", + "73-Overall RACC\n", + "74-Overall RACCU\n", + "75-P\n", + "76-P-Value\n", + "77-PLR\n", + "78-PLRI\n", + "79-POP\n", + "80-PPV\n", + "81-PPV Macro\n", + "82-PPV Micro\n", + "83-PRE\n", + "84-Pearson C\n", + "85-Phi-Squared\n", + "86-Q\n", + "87-RACC\n", + "88-RACCU\n", + "89-RCI\n", + "90-RR\n", + "91-Reference Entropy\n", + "92-Response Entropy\n", + "93-SOA1(Landis & Koch)\n", + "94-SOA2(Fleiss)\n", + "95-SOA3(Altman)\n", + "96-SOA4(Cicchetti)\n", + "97-SOA5(Cramer)\n", + "98-SOA6(Matthews)\n", + "99-Scott PI\n", + "100-Standard Error\n", + "101-TN\n", + "102-TNR\n", + "103-TON\n", + "104-TOP\n", + "105-TP\n", + "106-TPR\n", + "107-TPR Macro\n", + "108-TPR Micro\n", + "109-Y\n", + "110-Zero-one Loss\n", + "111-dInd\n", + "112-sInd\n" ] } ], @@ -4712,7 +4715,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 103, "metadata": {}, "outputs": [ { @@ -4721,7 +4724,7 @@ "{'L1': 0.42857142857142855, 'L2': 0.1111111111111111, 'L3': 0.1875}" ] }, - "execution_count": 4, + "execution_count": 103, "metadata": {}, "output_type": "execute_result" } @@ -4763,7 +4766,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 104, "metadata": {}, "outputs": [ { @@ -4772,7 +4775,7 @@ "{'L1': 0.8, 'L2': 0.41666666666666663, 'L3': 0.55}" ] }, - "execution_count": 4, + "execution_count": 104, "metadata": {}, "output_type": "execute_result" } @@ -4823,7 +4826,7 @@ }, { "cell_type": "code", - "execution_count": 103, + "execution_count": 105, "metadata": {}, "outputs": [ { @@ -4832,7 +4835,7 @@ "0.35483870967741943" ] }, - "execution_count": 103, + "execution_count": 105, "metadata": {}, "output_type": "execute_result" } @@ -4873,7 +4876,7 @@ }, { "cell_type": "code", - "execution_count": 104, + "execution_count": 106, "metadata": {}, "outputs": [ { @@ -4882,7 +4885,7 @@ "0.34426229508196726" ] }, - "execution_count": 104, + "execution_count": 106, "metadata": {}, "output_type": "execute_result" } @@ -4923,7 +4926,7 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": 107, "metadata": {}, "outputs": [ { @@ -4932,7 +4935,7 @@ "0.16666666666666674" ] }, - "execution_count": 105, + "execution_count": 107, "metadata": {}, "output_type": "execute_result" } @@ -4973,7 +4976,7 @@ }, { "cell_type": "code", - "execution_count": 106, + "execution_count": 108, "metadata": {}, "outputs": [ { @@ -4982,7 +4985,7 @@ "0.2203645326012817" ] }, - "execution_count": 106, + "execution_count": 108, "metadata": {}, "output_type": "execute_result" } @@ -5023,7 +5026,7 @@ }, { "cell_type": "code", - "execution_count": 107, + "execution_count": 109, "metadata": {}, "outputs": [ { @@ -5032,7 +5035,7 @@ "(-0.07707577422109269, 0.7867531935759315)" ] }, - "execution_count": 107, + "execution_count": 109, "metadata": {}, "output_type": "execute_result" } @@ -5082,7 +5085,7 @@ }, { "cell_type": "code", - "execution_count": 108, + "execution_count": 110, "metadata": {}, "outputs": [ { @@ -5091,7 +5094,7 @@ "6.6000000000000005" ] }, - "execution_count": 108, + "execution_count": 110, "metadata": {}, "output_type": "execute_result" } @@ -5132,7 +5135,7 @@ }, { "cell_type": "code", - "execution_count": 109, + "execution_count": 111, "metadata": {}, "outputs": [ { @@ -5141,7 +5144,7 @@ "4" ] }, - "execution_count": 109, + "execution_count": 111, "metadata": {}, "output_type": "execute_result" } @@ -5184,7 +5187,7 @@ }, { "cell_type": "code", - "execution_count": 110, + "execution_count": 112, "metadata": {}, "outputs": [ { @@ -5193,7 +5196,7 @@ "0.55" ] }, - "execution_count": 110, + "execution_count": 112, "metadata": {}, "output_type": "execute_result" } @@ -5236,7 +5239,7 @@ }, { "cell_type": "code", - "execution_count": 111, + "execution_count": 113, "metadata": {}, "outputs": [ { @@ -5245,7 +5248,7 @@ "0.5244044240850758" ] }, - "execution_count": 111, + "execution_count": 113, "metadata": {}, "output_type": "execute_result" } @@ -5288,7 +5291,7 @@ }, { "cell_type": "code", - "execution_count": 112, + "execution_count": 114, "metadata": {}, "outputs": [ { @@ -5297,7 +5300,7 @@ "0.14231876063832777" ] }, - "execution_count": 112, + "execution_count": 114, "metadata": {}, "output_type": "execute_result" } @@ -5340,7 +5343,7 @@ }, { "cell_type": "code", - "execution_count": 113, + "execution_count": 115, "metadata": {}, "outputs": [ { @@ -5349,7 +5352,7 @@ "(0.30438856248221097, 0.8622781041844558)" ] }, - "execution_count": 113, + "execution_count": 115, "metadata": {}, "output_type": "execute_result" } @@ -5400,7 +5403,7 @@ }, { "cell_type": "code", - "execution_count": 114, + "execution_count": 116, "metadata": {}, "outputs": [ { @@ -5409,7 +5412,7 @@ "0.37500000000000006" ] }, - "execution_count": 114, + "execution_count": 116, "metadata": {}, "output_type": "execute_result" } @@ -5459,7 +5462,7 @@ }, { "cell_type": "code", - "execution_count": 115, + "execution_count": 117, "metadata": {}, "outputs": [ { @@ -5468,7 +5471,7 @@ "0.34426229508196726" ] }, - "execution_count": 115, + "execution_count": 117, "metadata": {}, "output_type": "execute_result" } @@ -5523,7 +5526,7 @@ }, { "cell_type": "code", - "execution_count": 116, + "execution_count": 118, "metadata": {}, "outputs": [ { @@ -5532,7 +5535,7 @@ "0.3893129770992367" ] }, - "execution_count": 116, + "execution_count": 118, "metadata": {}, "output_type": "execute_result" } @@ -5587,7 +5590,7 @@ }, { "cell_type": "code", - "execution_count": 117, + "execution_count": 119, "metadata": {}, "outputs": [ { @@ -5596,7 +5599,7 @@ "1.4833557549816874" ] }, - "execution_count": 117, + "execution_count": 119, "metadata": {}, "output_type": "execute_result" } @@ -5651,7 +5654,7 @@ }, { "cell_type": "code", - "execution_count": 118, + "execution_count": 120, "metadata": {}, "outputs": [ { @@ -5660,7 +5663,7 @@ "1.5" ] }, - "execution_count": 118, + "execution_count": 120, "metadata": {}, "output_type": "execute_result" } @@ -5724,7 +5727,7 @@ }, { "cell_type": "code", - "execution_count": 119, + "execution_count": 121, "metadata": {}, "outputs": [ { @@ -5733,7 +5736,7 @@ "1.5833333333333335" ] }, - "execution_count": 119, + "execution_count": 121, "metadata": {}, "output_type": "execute_result" } @@ -5788,7 +5791,7 @@ }, { "cell_type": "code", - "execution_count": 120, + "execution_count": 122, "metadata": {}, "outputs": [ { @@ -5797,7 +5800,7 @@ "2.4591479170272446" ] }, - "execution_count": 120, + "execution_count": 122, "metadata": {}, "output_type": "execute_result" } @@ -5854,7 +5857,7 @@ }, { "cell_type": "code", - "execution_count": 121, + "execution_count": 123, "metadata": {}, "outputs": [ { @@ -5863,7 +5866,7 @@ "0.9757921620455572" ] }, - "execution_count": 121, + "execution_count": 123, "metadata": {}, "output_type": "execute_result" } @@ -5920,7 +5923,7 @@ }, { "cell_type": "code", - "execution_count": 122, + "execution_count": 124, "metadata": {}, "outputs": [ { @@ -5929,7 +5932,7 @@ "0.09997757835164581" ] }, - "execution_count": 122, + "execution_count": 124, "metadata": {}, "output_type": "execute_result" } @@ -6001,7 +6004,7 @@ }, { "cell_type": "code", - "execution_count": 123, + "execution_count": 125, "metadata": {}, "outputs": [ { @@ -6010,7 +6013,7 @@ "0.5242078379544428" ] }, - "execution_count": 123, + "execution_count": 125, "metadata": {}, "output_type": "execute_result" } @@ -6053,7 +6056,7 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 126, "metadata": {}, "outputs": [ { @@ -6062,7 +6065,7 @@ "0.42857142857142855" ] }, - "execution_count": 124, + "execution_count": 126, "metadata": {}, "output_type": "execute_result" } @@ -6105,7 +6108,7 @@ }, { "cell_type": "code", - "execution_count": 125, + "execution_count": 127, "metadata": {}, "outputs": [ { @@ -6114,7 +6117,7 @@ "0.16666666666666666" ] }, - "execution_count": 125, + "execution_count": 127, "metadata": {}, "output_type": "execute_result" } @@ -6186,7 +6189,7 @@ }, { "cell_type": "code", - "execution_count": 126, + "execution_count": 128, "metadata": {}, "outputs": [ { @@ -6195,7 +6198,7 @@ "'Fair'" ] }, - "execution_count": 126, + "execution_count": 128, "metadata": {}, "output_type": "execute_result" } @@ -6255,7 +6258,7 @@ }, { "cell_type": "code", - "execution_count": 127, + "execution_count": 129, "metadata": {}, "outputs": [ { @@ -6264,7 +6267,7 @@ "'Poor'" ] }, - "execution_count": 127, + "execution_count": 129, "metadata": {}, "output_type": "execute_result" } @@ -6332,7 +6335,7 @@ }, { "cell_type": "code", - "execution_count": 128, + "execution_count": 130, "metadata": {}, "outputs": [ { @@ -6341,7 +6344,7 @@ "'Fair'" ] }, - "execution_count": 128, + "execution_count": 130, "metadata": {}, "output_type": "execute_result" } @@ -6405,7 +6408,7 @@ }, { "cell_type": "code", - "execution_count": 129, + "execution_count": 131, "metadata": {}, "outputs": [ { @@ -6414,7 +6417,7 @@ "'Poor'" ] }, - "execution_count": 129, + "execution_count": 131, "metadata": {}, "output_type": "execute_result" } @@ -6486,7 +6489,7 @@ }, { "cell_type": "code", - "execution_count": 130, + "execution_count": 132, "metadata": {}, "outputs": [ { @@ -6495,7 +6498,7 @@ "'Relatively Strong'" ] }, - "execution_count": 130, + "execution_count": 132, "metadata": {}, "output_type": "execute_result" } @@ -6564,7 +6567,7 @@ }, { "cell_type": "code", - "execution_count": 131, + "execution_count": 133, "metadata": {}, "outputs": [ { @@ -6573,7 +6576,7 @@ "'Weak'" ] }, - "execution_count": 131, + "execution_count": 133, "metadata": {}, "output_type": "execute_result" } @@ -6623,7 +6626,7 @@ }, { "cell_type": "code", - "execution_count": 132, + "execution_count": 134, "metadata": {}, "outputs": [ { @@ -6632,7 +6635,7 @@ "0.5833333333333334" ] }, - "execution_count": 132, + "execution_count": 134, "metadata": {}, "output_type": "execute_result" } @@ -6673,7 +6676,7 @@ }, { "cell_type": "code", - "execution_count": 133, + "execution_count": 135, "metadata": {}, "outputs": [ { @@ -6682,7 +6685,7 @@ "0.3541666666666667" ] }, - "execution_count": 133, + "execution_count": 135, "metadata": {}, "output_type": "execute_result" } @@ -6723,7 +6726,7 @@ }, { "cell_type": "code", - "execution_count": 134, + "execution_count": 136, "metadata": {}, "outputs": [ { @@ -6732,7 +6735,7 @@ "0.3645833333333333" ] }, - "execution_count": 134, + "execution_count": 136, "metadata": {}, "output_type": "execute_result" } @@ -6773,7 +6776,7 @@ }, { "cell_type": "code", - "execution_count": 135, + "execution_count": 137, "metadata": {}, "outputs": [ { @@ -6782,7 +6785,7 @@ "0.5833333333333334" ] }, - "execution_count": 135, + "execution_count": 137, "metadata": {}, "output_type": "execute_result" } @@ -6823,7 +6826,7 @@ }, { "cell_type": "code", - "execution_count": 136, + "execution_count": 138, "metadata": {}, "outputs": [ { @@ -6832,7 +6835,7 @@ "0.5833333333333334" ] }, - "execution_count": 136, + "execution_count": 138, "metadata": {}, "output_type": "execute_result" } @@ -6873,7 +6876,7 @@ }, { "cell_type": "code", - "execution_count": 137, + "execution_count": 139, "metadata": {}, "outputs": [ { @@ -6882,7 +6885,7 @@ "0.5833333333333334" ] }, - "execution_count": 137, + "execution_count": 139, "metadata": {}, "output_type": "execute_result" } @@ -6923,7 +6926,7 @@ }, { "cell_type": "code", - "execution_count": 138, + "execution_count": 140, "metadata": {}, "outputs": [ { @@ -6932,7 +6935,7 @@ "0.611111111111111" ] }, - "execution_count": 138, + "execution_count": 140, "metadata": {}, "output_type": "execute_result" } @@ -6973,7 +6976,7 @@ }, { "cell_type": "code", - "execution_count": 139, + "execution_count": 141, "metadata": {}, "outputs": [ { @@ -6982,7 +6985,7 @@ "0.5666666666666668" ] }, - "execution_count": 139, + "execution_count": 141, "metadata": {}, "output_type": "execute_result" } @@ -7023,7 +7026,7 @@ }, { "cell_type": "code", - "execution_count": 140, + "execution_count": 142, "metadata": {}, "outputs": [ { @@ -7032,7 +7035,7 @@ "0.5651515151515151" ] }, - "execution_count": 140, + "execution_count": 142, "metadata": {}, "output_type": "execute_result" } @@ -7073,7 +7076,7 @@ }, { "cell_type": "code", - "execution_count": 141, + "execution_count": 143, "metadata": {}, "outputs": [ { @@ -7082,7 +7085,7 @@ "0.7222222222222223" ] }, - "execution_count": 141, + "execution_count": 143, "metadata": {}, "output_type": "execute_result" } @@ -7137,7 +7140,7 @@ }, { "cell_type": "code", - "execution_count": 142, + "execution_count": 144, "metadata": {}, "outputs": [ { @@ -7146,7 +7149,7 @@ "(1.225, 0.4083333333333334)" ] }, - "execution_count": 142, + "execution_count": 144, "metadata": {}, "output_type": "execute_result" } @@ -7187,7 +7190,7 @@ }, { "cell_type": "code", - "execution_count": 143, + "execution_count": 145, "metadata": {}, "outputs": [ { @@ -7196,7 +7199,7 @@ "0.41666666666666663" ] }, - "execution_count": 143, + "execution_count": 145, "metadata": {}, "output_type": "execute_result" } @@ -7237,7 +7240,7 @@ }, { "cell_type": "code", - "execution_count": 144, + "execution_count": 146, "metadata": {}, "outputs": [ { @@ -7246,7 +7249,7 @@ "5" ] }, - "execution_count": 144, + "execution_count": 146, "metadata": {}, "output_type": "execute_result" } @@ -7287,7 +7290,7 @@ }, { "cell_type": "code", - "execution_count": 145, + "execution_count": 147, "metadata": {}, "outputs": [ { @@ -7296,7 +7299,7 @@ "0.4166666666666667" ] }, - "execution_count": 145, + "execution_count": 147, "metadata": {}, "output_type": "execute_result" } @@ -7364,7 +7367,7 @@ }, { "cell_type": "code", - "execution_count": 146, + "execution_count": 148, "metadata": {}, "outputs": [ { @@ -7373,7 +7376,7 @@ "0.18926430237560654" ] }, - "execution_count": 146, + "execution_count": 148, "metadata": {}, "output_type": "execute_result" } @@ -7421,7 +7424,7 @@ }, { "cell_type": "code", - "execution_count": 147, + "execution_count": 149, "metadata": {}, "outputs": [ { @@ -7430,7 +7433,7 @@ "0.4638112995385119" ] }, - "execution_count": 147, + "execution_count": 149, "metadata": {}, "output_type": "execute_result" } @@ -7485,7 +7488,7 @@ }, { "cell_type": "code", - "execution_count": 148, + "execution_count": 150, "metadata": {}, "outputs": [ { @@ -7494,7 +7497,7 @@ "0.5189369467580801" ] }, - "execution_count": 148, + "execution_count": 150, "metadata": {}, "output_type": "execute_result" } @@ -7556,7 +7559,7 @@ }, { "cell_type": "code", - "execution_count": 149, + "execution_count": 151, "metadata": {}, "outputs": [ { @@ -7565,7 +7568,7 @@ "0.36666666666666664" ] }, - "execution_count": 149, + "execution_count": 151, "metadata": {}, "output_type": "execute_result" } @@ -7606,7 +7609,7 @@ }, { "cell_type": "code", - "execution_count": 150, + "execution_count": 152, "metadata": {}, "outputs": [ { @@ -7615,7 +7618,7 @@ "4.0" ] }, - "execution_count": 150, + "execution_count": 152, "metadata": {}, "output_type": "execute_result" } @@ -7658,7 +7661,7 @@ }, { "cell_type": "code", - "execution_count": 151, + "execution_count": 153, "metadata": {}, "outputs": [ { @@ -7667,7 +7670,7 @@ "0.4777777777777778" ] }, - "execution_count": 151, + "execution_count": 153, "metadata": {}, "output_type": "execute_result" } @@ -7708,7 +7711,7 @@ }, { "cell_type": "code", - "execution_count": 152, + "execution_count": 154, "metadata": {}, "outputs": [ { @@ -7717,7 +7720,7 @@ "0.6785714285714285" ] }, - "execution_count": 152, + "execution_count": 154, "metadata": {}, "output_type": "execute_result" } @@ -7758,7 +7761,7 @@ }, { "cell_type": "code", - "execution_count": 153, + "execution_count": 155, "metadata": {}, "outputs": [ { @@ -7767,7 +7770,7 @@ "0.6857142857142857" ] }, - "execution_count": 153, + "execution_count": 155, "metadata": {}, "output_type": "execute_result" } @@ -7830,7 +7833,7 @@ }, { "cell_type": "code", - "execution_count": 154, + "execution_count": 156, "metadata": {}, "outputs": [ { @@ -7839,7 +7842,7 @@ "0.3533932006492363" ] }, - "execution_count": 154, + "execution_count": 156, "metadata": {}, "output_type": "execute_result" } @@ -7880,7 +7883,7 @@ }, { "cell_type": "code", - "execution_count": 155, + "execution_count": 157, "metadata": {}, "outputs": [ { @@ -7889,7 +7892,7 @@ "0.5956833971812706" ] }, - "execution_count": 155, + "execution_count": 157, "metadata": {}, "output_type": "execute_result" } @@ -7923,7 +7926,7 @@ }, { "cell_type": "code", - "execution_count": 156, + "execution_count": 158, "metadata": {}, "outputs": [ { @@ -8005,8 +8008,9 @@ "AGF(Adjusted F-score) 0.72859 0.62869 0.61009 \n", "AGM(Adjusted geometric mean) 0.85764 0.70861 0.58034 \n", "AM(Difference between automatic and manual classification) -2 1 1 \n", - "AUC(Area under the roc curve) 0.8 0.65 0.58571 \n", + "AUC(Area under the ROC curve) 0.8 0.65 0.58571 \n", "AUCI(AUC value interpretation) Very Good Fair Poor \n", + "AUPR(Area under the PR curve) 0.8 0.41667 0.55 \n", "BCD(Bray-Curtis dissimilarity) 0.08333 0.04167 0.04167 \n", "BM(Informedness or bookmaker informedness) 0.6 0.3 0.17143 \n", "CEN(Confusion entropy) 0.25 0.49658 0.60442 \n", @@ -8076,7 +8080,7 @@ }, { "cell_type": "code", - "execution_count": 157, + "execution_count": 159, "metadata": {}, "outputs": [ { @@ -8101,7 +8105,7 @@ }, { "cell_type": "code", - "execution_count": 158, + "execution_count": 160, "metadata": {}, "outputs": [ { @@ -8112,7 +8116,7 @@ " 'L3': {'L1': 0, 'L2': 2, 'L3': 3}}" ] }, - "execution_count": 158, + "execution_count": 160, "metadata": {}, "output_type": "execute_result" } @@ -8123,7 +8127,7 @@ }, { "cell_type": "code", - "execution_count": 159, + "execution_count": 161, "metadata": {}, "outputs": [ { @@ -8186,7 +8190,7 @@ }, { "cell_type": "code", - "execution_count": 160, + "execution_count": 162, "metadata": {}, "outputs": [ { @@ -8211,7 +8215,7 @@ }, { "cell_type": "code", - "execution_count": 161, + "execution_count": 163, "metadata": {}, "outputs": [ { @@ -8222,7 +8226,7 @@ " 'L3': {'L1': 0.0, 'L2': 0.4, 'L3': 0.6}}" ] }, - "execution_count": 161, + "execution_count": 163, "metadata": {}, "output_type": "execute_result" } @@ -8233,7 +8237,7 @@ }, { "cell_type": "code", - "execution_count": 162, + "execution_count": 164, "metadata": {}, "outputs": [ { @@ -8296,7 +8300,7 @@ }, { "cell_type": "code", - "execution_count": 163, + "execution_count": 165, "metadata": {}, "outputs": [ { @@ -8366,8 +8370,9 @@ "AGF(Adjusted F-score) 0.72859 0.62869 0.61009 \n", "AGM(Adjusted geometric mean) 0.85764 0.70861 0.58034 \n", "AM(Difference between automatic and manual classification) -2 1 1 \n", - "AUC(Area under the roc curve) 0.8 0.65 0.58571 \n", + "AUC(Area under the ROC curve) 0.8 0.65 0.58571 \n", "AUCI(AUC value interpretation) Very Good Fair Poor \n", + "AUPR(Area under the PR curve) 0.8 0.41667 0.55 \n", "BCD(Bray-Curtis dissimilarity) 0.08333 0.04167 0.04167 \n", "BM(Informedness or bookmaker informedness) 0.6 0.3 0.17143 \n", "CEN(Confusion entropy) 0.25 0.49658 0.60442 \n", @@ -8430,7 +8435,7 @@ }, { "cell_type": "code", - "execution_count": 164, + "execution_count": 166, "metadata": {}, "outputs": [ { @@ -8445,7 +8450,7 @@ "\n", "Classes L1 L2 L3 \n", "ACC(Accuracy) 0.83333 0.75 0.58333 \n", - "AUC(Area under the roc curve) 0.8 0.65 0.58571 \n", + "AUC(Area under the ROC curve) 0.8 0.65 0.58571 \n", "TPR(Sensitivity, recall, hit rate, or true positive rate) 0.6 0.5 0.6 \n", "\n" ] @@ -8457,7 +8462,7 @@ }, { "cell_type": "code", - "execution_count": 165, + "execution_count": 167, "metadata": {}, "outputs": [ { @@ -8472,7 +8477,7 @@ "\n", "Classes L1 L3 \n", "ACC(Accuracy) 0.83333 0.58333 \n", - "AUC(Area under the roc curve) 0.8 0.58571 \n", + "AUC(Area under the ROC curve) 0.8 0.58571 \n", "TPR(Sensitivity, recall, hit rate, or true positive rate) 0.6 0.6 \n", "\n" ] @@ -8484,7 +8489,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 168, "metadata": {}, "outputs": [ { @@ -8590,7 +8595,7 @@ }, { "cell_type": "code", - "execution_count": 166, + "execution_count": 169, "metadata": {}, "outputs": [ { @@ -8612,7 +8617,7 @@ }, { "cell_type": "code", - "execution_count": 167, + "execution_count": 170, "metadata": {}, "outputs": [ { @@ -8641,7 +8646,7 @@ }, { "cell_type": "code", - "execution_count": 168, + "execution_count": 171, "metadata": {}, "outputs": [], "source": [ @@ -8659,7 +8664,7 @@ }, { "cell_type": "code", - "execution_count": 169, + "execution_count": 172, "metadata": {}, "outputs": [ { @@ -8669,7 +8674,7 @@ " 'Status': True}" ] }, - "execution_count": 169, + "execution_count": 172, "metadata": {}, "output_type": "execute_result" } @@ -8687,7 +8692,7 @@ }, { "cell_type": "code", - "execution_count": 170, + "execution_count": 173, "metadata": {}, "outputs": [ { @@ -8697,7 +8702,7 @@ " 'Status': True}" ] }, - "execution_count": 170, + "execution_count": 173, "metadata": {}, "output_type": "execute_result" } @@ -8715,7 +8720,7 @@ }, { "cell_type": "code", - "execution_count": 171, + "execution_count": 174, "metadata": {}, "outputs": [ { @@ -8725,7 +8730,7 @@ " 'Status': True}" ] }, - "execution_count": 171, + "execution_count": 174, "metadata": {}, "output_type": "execute_result" } @@ -8743,7 +8748,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 175, "metadata": {}, "outputs": [ { @@ -8753,7 +8758,7 @@ " 'Status': True}" ] }, - "execution_count": 5, + "execution_count": 175, "metadata": {}, "output_type": "execute_result" } @@ -8771,7 +8776,7 @@ }, { "cell_type": "code", - "execution_count": 172, + "execution_count": 176, "metadata": {}, "outputs": [ { @@ -8781,7 +8786,7 @@ " 'Status': False}" ] }, - "execution_count": 172, + "execution_count": 176, "metadata": {}, "output_type": "execute_result" } @@ -8854,7 +8859,7 @@ }, { "cell_type": "code", - "execution_count": 173, + "execution_count": 177, "metadata": {}, "outputs": [ { @@ -8864,7 +8869,7 @@ " 'Status': True}" ] }, - "execution_count": 173, + "execution_count": 177, "metadata": {}, "output_type": "execute_result" } @@ -8882,7 +8887,7 @@ }, { "cell_type": "code", - "execution_count": 174, + "execution_count": 178, "metadata": {}, "outputs": [ { @@ -8892,7 +8897,7 @@ " 'Status': True}" ] }, - "execution_count": 174, + "execution_count": 178, "metadata": {}, "output_type": "execute_result" } @@ -8910,7 +8915,7 @@ }, { "cell_type": "code", - "execution_count": 175, + "execution_count": 179, "metadata": {}, "outputs": [ { @@ -8920,7 +8925,7 @@ " 'Status': True}" ] }, - "execution_count": 175, + "execution_count": 179, "metadata": {}, "output_type": "execute_result" } @@ -8938,7 +8943,7 @@ }, { "cell_type": "code", - "execution_count": 176, + "execution_count": 180, "metadata": {}, "outputs": [ { @@ -8948,7 +8953,7 @@ " 'Status': True}" ] }, - "execution_count": 176, + "execution_count": 180, "metadata": {}, "output_type": "execute_result" } @@ -8966,7 +8971,7 @@ }, { "cell_type": "code", - "execution_count": 177, + "execution_count": 181, "metadata": {}, "outputs": [ { @@ -8976,7 +8981,7 @@ " 'Status': True}" ] }, - "execution_count": 177, + "execution_count": 181, "metadata": {}, "output_type": "execute_result" } @@ -8994,7 +8999,7 @@ }, { "cell_type": "code", - "execution_count": 178, + "execution_count": 182, "metadata": {}, "outputs": [ { @@ -9004,7 +9009,7 @@ " 'Status': True}" ] }, - "execution_count": 178, + "execution_count": 182, "metadata": {}, "output_type": "execute_result" } @@ -9022,7 +9027,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 183, "metadata": {}, "outputs": [ { @@ -9032,7 +9037,7 @@ " 'Status': True}" ] }, - "execution_count": 6, + "execution_count": 183, "metadata": {}, "output_type": "execute_result" } @@ -9050,7 +9055,7 @@ }, { "cell_type": "code", - "execution_count": 179, + "execution_count": 184, "metadata": {}, "outputs": [ { @@ -9060,7 +9065,7 @@ " 'Status': False}" ] }, - "execution_count": 179, + "execution_count": 184, "metadata": {}, "output_type": "execute_result" } @@ -9162,7 +9167,7 @@ }, { "cell_type": "code", - "execution_count": 180, + "execution_count": 185, "metadata": {}, "outputs": [ { @@ -9172,7 +9177,7 @@ " 'Status': True}" ] }, - "execution_count": 180, + "execution_count": 185, "metadata": {}, "output_type": "execute_result" } @@ -9192,7 +9197,7 @@ }, { "cell_type": "code", - "execution_count": 181, + "execution_count": 186, "metadata": {}, "outputs": [ { @@ -9202,7 +9207,7 @@ " 'Status': True}" ] }, - "execution_count": 181, + "execution_count": 186, "metadata": {}, "output_type": "execute_result" } @@ -9222,7 +9227,7 @@ }, { "cell_type": "code", - "execution_count": 182, + "execution_count": 187, "metadata": {}, "outputs": [ { @@ -9232,7 +9237,7 @@ " 'Status': True}" ] }, - "execution_count": 182, + "execution_count": 187, "metadata": {}, "output_type": "execute_result" } @@ -9252,7 +9257,7 @@ }, { "cell_type": "code", - "execution_count": 183, + "execution_count": 188, "metadata": {}, "outputs": [ { @@ -9262,7 +9267,7 @@ " 'Status': True}" ] }, - "execution_count": 183, + "execution_count": 188, "metadata": {}, "output_type": "execute_result" } @@ -9282,7 +9287,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 189, "metadata": {}, "outputs": [ { @@ -9292,7 +9297,7 @@ " 'Status': True}" ] }, - "execution_count": 7, + "execution_count": 189, "metadata": {}, "output_type": "execute_result" } @@ -9310,7 +9315,7 @@ }, { "cell_type": "code", - "execution_count": 184, + "execution_count": 190, "metadata": {}, "outputs": [ { @@ -9320,7 +9325,7 @@ " 'Status': False}" ] }, - "execution_count": 184, + "execution_count": 190, "metadata": {}, "output_type": "execute_result" } @@ -9403,7 +9408,7 @@ }, { "cell_type": "code", - "execution_count": 185, + "execution_count": 191, "metadata": {}, "outputs": [ { @@ -9413,7 +9418,7 @@ " 'Status': True}" ] }, - "execution_count": 185, + "execution_count": 191, "metadata": {}, "output_type": "execute_result" } @@ -9431,7 +9436,7 @@ }, { "cell_type": "code", - "execution_count": 186, + "execution_count": 192, "metadata": {}, "outputs": [ { @@ -9441,7 +9446,7 @@ " 'Status': True}" ] }, - "execution_count": 186, + "execution_count": 192, "metadata": {}, "output_type": "execute_result" } @@ -9459,7 +9464,7 @@ }, { "cell_type": "code", - "execution_count": 187, + "execution_count": 193, "metadata": {}, "outputs": [ { @@ -9469,7 +9474,7 @@ " 'Status': True}" ] }, - "execution_count": 187, + "execution_count": 193, "metadata": {}, "output_type": "execute_result" } @@ -9487,7 +9492,7 @@ }, { "cell_type": "code", - "execution_count": 188, + "execution_count": 194, "metadata": {}, "outputs": [ { @@ -9497,7 +9502,7 @@ " 'Status': False}" ] }, - "execution_count": 188, + "execution_count": 194, "metadata": {}, "output_type": "execute_result" } @@ -9559,7 +9564,7 @@ }, { "cell_type": "code", - "execution_count": 189, + "execution_count": 195, "metadata": {}, "outputs": [ { @@ -9569,7 +9574,7 @@ " 'Status': True}" ] }, - "execution_count": 189, + "execution_count": 195, "metadata": {}, "output_type": "execute_result" } @@ -9587,7 +9592,7 @@ }, { "cell_type": "code", - "execution_count": 190, + "execution_count": 196, "metadata": {}, "outputs": [ { @@ -9597,7 +9602,7 @@ " 'Status': False}" ] }, - "execution_count": 190, + "execution_count": 196, "metadata": {}, "output_type": "execute_result" } @@ -9639,7 +9644,7 @@ }, { "cell_type": "code", - "execution_count": 191, + "execution_count": 197, "metadata": {}, "outputs": [ { @@ -9659,7 +9664,7 @@ }, { "cell_type": "code", - "execution_count": 192, + "execution_count": 198, "metadata": { "scrolled": true }, @@ -9681,7 +9686,7 @@ }, { "cell_type": "code", - "execution_count": 193, + "execution_count": 199, "metadata": {}, "outputs": [ { @@ -9701,7 +9706,7 @@ }, { "cell_type": "code", - "execution_count": 194, + "execution_count": 200, "metadata": {}, "outputs": [ { @@ -9721,7 +9726,7 @@ }, { "cell_type": "code", - "execution_count": 195, + "execution_count": 201, "metadata": {}, "outputs": [ { @@ -9741,7 +9746,7 @@ }, { "cell_type": "code", - "execution_count": 196, + "execution_count": 202, "metadata": {}, "outputs": [ { @@ -9761,7 +9766,7 @@ }, { "cell_type": "code", - "execution_count": 197, + "execution_count": 203, "metadata": {}, "outputs": [ { @@ -9781,7 +9786,7 @@ }, { "cell_type": "code", - "execution_count": 198, + "execution_count": 204, "metadata": {}, "outputs": [ { @@ -9801,7 +9806,7 @@ }, { "cell_type": "code", - "execution_count": 199, + "execution_count": 205, "metadata": {}, "outputs": [ { @@ -9821,7 +9826,7 @@ }, { "cell_type": "code", - "execution_count": 200, + "execution_count": 206, "metadata": {}, "outputs": [ { @@ -9841,7 +9846,7 @@ }, { "cell_type": "code", - "execution_count": 201, + "execution_count": 207, "metadata": {}, "outputs": [ { @@ -9861,7 +9866,7 @@ }, { "cell_type": "code", - "execution_count": 202, + "execution_count": 208, "metadata": {}, "outputs": [ { diff --git a/Document/Document_Files/cm1.csv b/Document/Document_Files/cm1.csv index 97d81cbc..168bf6d0 100644 --- a/Document/Document_Files/cm1.csv +++ b/Document/Document_Files/cm1.csv @@ -5,6 +5,7 @@ AGM,0.85764,0.70861,0.58034 AM,-2,1,1 AUC,0.8,0.65,0.58571 AUCI,Very Good,Fair,Poor +AUPR,0.8,0.41667,0.55 BCD,0.08333,0.04167,0.04167 BM,0.6,0.3,0.17143 CEN,0.25,0.49658,0.60442 diff --git a/Document/Document_Files/cm1.html b/Document/Document_Files/cm1.html index e551a7b1..bec5ec92 100644 --- a/Document/Document_Files/cm1.html +++ b/Document/Document_Files/cm1.html @@ -304,7 +304,7 @@

    Class Statistics :

    - + @@ -314,6 +314,13 @@

    Class Statistics :

    + + + + + + + @@ -679,5 +686,5 @@

    Class Statistics :

    ' + str(i) + '0.8 0.65 0.58571Area under the roc curveArea under the ROC curve
    AUCI AUC value interpretation
    AUPR0.80.416670.55Area under the PR curve
    BCD 0.08333 0.04167
    -

    Generated By PyCM Version 2.3

    +

    Generated By PyCM Version 2.4

    \ No newline at end of file diff --git a/Document/Document_Files/cm1.obj b/Document/Document_Files/cm1.obj index fce0e389..dc291064 100644 --- a/Document/Document_Files/cm1.obj +++ b/Document/Document_Files/cm1.obj @@ -1 +1 @@ -{"Transpose": true, "Predict-Vector": null, "Sample-Weight": null, "Actual-Vector": null, "Matrix": [["L1", [["L2", 0], ["L1", 3], ["L3", 2]]], ["L2", [["L2", 1], ["L1", 0], ["L3", 1]]], ["L3", [["L2", 2], ["L1", 0], ["L3", 3]]]], "Digit": 5} \ No newline at end of file +{"Sample-Weight": null, "Predict-Vector": null, "Matrix": [["L1", [["L2", 0], ["L1", 3], ["L3", 2]]], ["L2", [["L2", 1], ["L1", 0], ["L3", 1]]], ["L3", [["L2", 2], ["L1", 0], ["L3", 3]]]], "Digit": 5, "Transpose": true, "Actual-Vector": null} \ No newline at end of file diff --git a/Document/Document_Files/cm1.pycm b/Document/Document_Files/cm1.pycm index a982afd7..2bbb3d24 100644 --- a/Document/Document_Files/cm1.pycm +++ b/Document/Document_Files/cm1.pycm @@ -85,8 +85,9 @@ ACC(Accuracy) 0.83333 AGF(Adjusted F-score) 0.72859 0.62869 0.61009 AGM(Adjusted geometric mean) 0.85764 0.70861 0.58034 AM(Difference between automatic and manual classification) -2 1 1 -AUC(Area under the roc curve) 0.8 0.65 0.58571 +AUC(Area under the ROC curve) 0.8 0.65 0.58571 AUCI(AUC value interpretation) Very Good Fair Poor +AUPR(Area under the PR curve) 0.8 0.41667 0.55 BCD(Bray-Curtis dissimilarity) 0.08333 0.04167 0.04167 BM(Informedness or bookmaker informedness) 0.6 0.3 0.17143 CEN(Confusion entropy) 0.25 0.49658 0.60442 diff --git a/Document/Document_Files/cm1_colored.html b/Document/Document_Files/cm1_colored.html index 7ddbb012..0b646ac5 100644 --- a/Document/Document_Files/cm1_colored.html +++ b/Document/Document_Files/cm1_colored.html @@ -304,7 +304,7 @@

    Class Statistics :

    		0.8 0.65 0.58571Area under the roc curveArea under the ROC curve
    AUCI AUC value interpretation
    AUPR0.80.416670.55Area under the PR curve
    BCD 0.08333 0.04167
    -

    Generated By PyCM Version 2.3

    +

    Generated By PyCM Version 2.4

    \ No newline at end of file diff --git a/Document/Document_Files/cm1_colored2.html b/Document/Document_Files/cm1_colored2.html index a3283cd2..3398f027 100644 --- a/Document/Document_Files/cm1_colored2.html +++ b/Document/Document_Files/cm1_colored2.html @@ -304,7 +304,7 @@

    Class Statistics :

    0.8 0.65 0.58571 -Area under the roc curve +Area under the ROC curve AUCI @@ -314,6 +314,13 @@

    Class Statistics :

    AUC value interpretation +AUPR +0.8 +0.41667 +0.55 +Area under the PR curve + + BCD 0.08333 0.04167 @@ -679,5 +686,5 @@

    Class Statistics :

    -

    Generated By PyCM Version 2.3

    +

    Generated By PyCM Version 2.4

    \ No newline at end of file diff --git a/Document/Document_Files/cm1_filtered.html b/Document/Document_Files/cm1_filtered.html index 5e9aa4cd..89493c39 100644 --- a/Document/Document_Files/cm1_filtered.html +++ b/Document/Document_Files/cm1_filtered.html @@ -75,7 +75,7 @@

    Class Statistics :

    0.8 0.65 0.58571 -Area under the roc curve +Area under the ROC curve TPR @@ -86,5 +86,5 @@

    Class Statistics :

    -

    Generated By PyCM Version 2.3

    +

    Generated By PyCM Version 2.4

    \ No newline at end of file diff --git a/Document/Document_Files/cm1_filtered.pycm b/Document/Document_Files/cm1_filtered.pycm index 3e96b7e0..cd47a692 100644 --- a/Document/Document_Files/cm1_filtered.pycm +++ b/Document/Document_Files/cm1_filtered.pycm @@ -30,7 +30,7 @@ Class Statistics : Classes L1 L2 L3 ACC(Accuracy) 0.83333 0.75 0.58333 -AUC(Area under the roc curve) 0.8 0.65 0.58571 +AUC(Area under the ROC curve) 0.8 0.65 0.58571 TPR(Sensitivity, recall, hit rate, or true positive rate) 0.6 0.5 0.6 One-Vs-All : diff --git a/Document/Document_Files/cm1_filtered2.html b/Document/Document_Files/cm1_filtered2.html index caa871ed..c223d0ae 100644 --- a/Document/Document_Files/cm1_filtered2.html +++ b/Document/Document_Files/cm1_filtered2.html @@ -69,7 +69,7 @@

    Class Statistics :

    AUC 0.8 -Area under the roc curve +Area under the ROC curve TPR @@ -78,5 +78,5 @@

    Class Statistics :

    -

    Generated By PyCM Version 2.3

    +

    Generated By PyCM Version 2.4

    \ No newline at end of file diff --git a/Document/Document_Files/cm1_filtered2.pycm b/Document/Document_Files/cm1_filtered2.pycm index f7b6c082..1a84dcad 100644 --- a/Document/Document_Files/cm1_filtered2.pycm +++ b/Document/Document_Files/cm1_filtered2.pycm @@ -30,7 +30,7 @@ Class Statistics : Classes L1 ACC(Accuracy) 0.83333 -AUC(Area under the roc curve) 0.8 +AUC(Area under the ROC curve) 0.8 TPR(Sensitivity, recall, hit rate, or true positive rate) 0.6 One-Vs-All : diff --git a/Document/Document_Files/cm1_no_vectors.obj b/Document/Document_Files/cm1_no_vectors.obj index fce0e389..dc291064 100644 --- a/Document/Document_Files/cm1_no_vectors.obj +++ b/Document/Document_Files/cm1_no_vectors.obj @@ -1 +1 @@ -{"Transpose": true, "Predict-Vector": null, "Sample-Weight": null, "Actual-Vector": null, "Matrix": [["L1", [["L2", 0], ["L1", 3], ["L3", 2]]], ["L2", [["L2", 1], ["L1", 0], ["L3", 1]]], ["L3", [["L2", 2], ["L1", 0], ["L3", 3]]]], "Digit": 5} \ No newline at end of file +{"Sample-Weight": null, "Predict-Vector": null, "Matrix": [["L1", [["L2", 0], ["L1", 3], ["L3", 2]]], ["L2", [["L2", 1], ["L1", 0], ["L3", 1]]], ["L3", [["L2", 2], ["L1", 0], ["L3", 3]]]], "Digit": 5, "Transpose": true, "Actual-Vector": null} \ No newline at end of file diff --git a/Document/Document_Files/cm1_normalized.html b/Document/Document_Files/cm1_normalized.html index a03ddb31..0a6d3c71 100644 --- a/Document/Document_Files/cm1_normalized.html +++ b/Document/Document_Files/cm1_normalized.html @@ -304,7 +304,7 @@

    Class Statistics :

    0.8 0.65 0.58571 -Area under the roc curve +Area under the ROC curve AUCI @@ -314,6 +314,13 @@

    Class Statistics :

    AUC value interpretation +AUPR +0.8 +0.41667 +0.55 +Area under the PR curve + + BCD 0.08333 0.04167 @@ -679,5 +686,5 @@

    Class Statistics :

    -

    Generated By PyCM Version 2.3

    +

    Generated By PyCM Version 2.4

    \ No newline at end of file diff --git a/Document/Document_Files/cm1_stat.obj b/Document/Document_Files/cm1_stat.obj index a448c1c6..df3ebf57 100644 --- a/Document/Document_Files/cm1_stat.obj +++ b/Document/Document_Files/cm1_stat.obj @@ -1 +1 @@ -{"Class-Stat": {"Y": {"L2": 0.30000000000000004, "L1": 0.6000000000000001, "L3": 0.17142857142857126}, "FNR": {"L2": 0.5, "L1": 0.4, "L3": 0.4}, "FOR": {"L2": 0.11111111111111116, "L1": 0.2222222222222222, "L3": 0.33333333333333337}, "BM": {"L2": 0.30000000000000004, "L1": 0.6000000000000001, "L3": 0.17142857142857126}, "F0.5": {"L2": 0.35714285714285715, "L1": 0.8823529411764706, "L3": 0.5172413793103449}, "OC": {"L2": 0.5, "L1": 1.0, "L3": 0.6}, "TPR": {"L2": 0.5, "L1": 0.6, "L3": 0.6}, "FDR": {"L2": 0.6666666666666667, "L1": 0.0, "L3": 0.5}, "N": {"L2": 10, "L1": 7, "L3": 7}, "FPR": {"L2": 0.19999999999999996, "L1": 0.0, "L3": 0.4285714285714286}, "MK": {"L2": 0.2222222222222221, "L1": 0.7777777777777777, "L3": 0.16666666666666652}, "J": {"L2": 0.25, "L1": 0.6, "L3": 0.375}, "PLRI": {"L2": "Poor", "L1": "None", "L3": "Poor"}, "AGF": {"L2": 0.6286946134619315, "L1": 0.7285871475307653, "L3": 0.610088876086563}, "OOC": {"L2": 0.4082482904638631, "L1": 0.7745966692414834, "L3": 0.5477225575051661}, "PLR": {"L2": 2.5000000000000004, "L1": "None", "L3": 1.4}, "DPI": {"L2": "Poor", "L1": "None", "L3": "Poor"}, "TNR": {"L2": 0.8, "L1": 1.0, "L3": 0.5714285714285714}, "AGM": {"L2": 0.708612108382005, "L1": 0.8576400016262, "L3": 0.5803410802752335}, "F2": {"L2": 0.45454545454545453, "L1": 0.6521739130434783, "L3": 0.5769230769230769}, "MCC": {"L2": 0.25819888974716115, "L1": 0.6831300510639732, "L3": 0.1690308509457033}, "POP": {"L2": 12, "L1": 12, "L3": 12}, "IBA": {"L2": 0.27999999999999997, "L1": 0.36, "L3": 0.35265306122448975}, "AUCI": {"L2": "Fair", "L1": "Very Good", "L3": "Poor"}, "MCCI": {"L2": "Negligible", "L1": "Moderate", "L3": "Negligible"}, "dInd": {"L2": 0.5385164807134504, "L1": 0.4, "L3": 0.5862367008195198}, "AUC": {"L2": 0.65, "L1": 0.8, "L3": 0.5857142857142856}, "P": {"L2": 2, "L1": 5, "L3": 5}, "MCEN": {"L2": 0.5, "L1": 0.2643856189774724, "L3": 0.6875}, "ACC": {"L2": 0.75, "L1": 0.8333333333333334, "L3": 0.5833333333333334}, "NLRI": {"L2": "Negligible", "L1": "Poor", "L3": "Negligible"}, "F1": {"L2": 0.4, "L1": 0.75, "L3": 0.5454545454545454}, "FP": {"L2": 2, "L1": 0, "L3": 3}, "FN": {"L2": 1, "L1": 2, "L3": 2}, "Q": {"L2": 0.6, "L1": "None", "L3": 0.3333333333333333}, "AM": {"L2": 1, "L1": -2, "L3": 1}, "sInd": {"L2": 0.6192113447068046, "L1": 0.717157287525381, "L3": 0.5854680534700882}, "TOP": {"L2": 3, "L1": 3, "L3": 6}, "G": {"L2": 0.408248290463863, "L1": 0.7745966692414834, "L3": 0.5477225575051661}, "GM": {"L2": 0.6324555320336759, "L1": 0.7745966692414834, "L3": 0.5855400437691198}, "IS": {"L2": 0.9999999999999998, "L1": 1.2630344058337937, "L3": 0.26303440583379367}, "PPV": {"L2": 0.3333333333333333, "L1": 1.0, "L3": 0.5}, "ERR": {"L2": 0.25, "L1": 0.16666666666666663, "L3": 0.41666666666666663}, "TN": {"L2": 8, "L1": 7, "L3": 4}, "RACC": {"L2": 0.041666666666666664, "L1": 0.10416666666666667, "L3": 0.20833333333333334}, "TON": {"L2": 9, "L1": 9, "L3": 6}, "DOR": {"L2": 4.000000000000001, "L1": "None", "L3": 1.9999999999999998}, "NLR": {"L2": 0.625, "L1": 0.4, "L3": 0.7000000000000001}, "BCD": {"L2": 0.041666666666666664, "L1": 0.08333333333333333, "L3": 0.041666666666666664}, "TP": {"L2": 1, "L1": 3, "L3": 3}, "CEN": {"L2": 0.49657842846620864, "L1": 0.25, "L3": 0.6044162769630221}, "PRE": {"L2": 0.16666666666666666, "L1": 0.4166666666666667, "L3": 0.4166666666666667}, "OP": {"L2": 0.5192307692307692, "L1": 0.5833333333333334, "L3": 0.5589430894308943}, "DP": {"L2": 0.33193306999649924, "L1": "None", "L3": 0.1659665349982495}, "LS": {"L2": 2.0, "L1": 2.4, "L3": 1.2}, "GI": {"L2": 0.30000000000000004, "L1": 0.6000000000000001, "L3": 0.17142857142857126}, "RACCU": {"L2": 0.04340277777777778, "L1": 0.1111111111111111, "L3": 0.21006944444444442}, "NPV": {"L2": 0.8888888888888888, "L1": 0.7777777777777778, "L3": 0.6666666666666666}}, "Transpose": true, "Predict-Vector": null, "Sample-Weight": null, "Actual-Vector": null, "Matrix": [["L1", [["L2", 0], ["L1", 3], ["L3", 2]]], ["L2", [["L2", 1], ["L1", 0], ["L3", 1]]], ["L3", [["L2", 2], ["L1", 0], ["L3", 3]]]], "Overall-Stat": {"Overall J": [1.225, 0.4083333333333334], "KL Divergence": 0.09997757835164581, "Pearson C": 0.5956833971812706, "Cramer V": 0.5244044240850758, "F1 Macro": 0.5651515151515151, "RR": 4.0, "Cross Entropy": 1.5833333333333335, "95% CI": [0.30438856248221097, 0.8622781041844558], "SOA1(Landis & Koch)": "Fair", "PPV Macro": 0.611111111111111, "AUNU": 0.6785714285714285, "Overall ACC": 0.5833333333333334, "Overall RACC": 0.3541666666666667, "Kappa": 0.35483870967741943, "Hamming Loss": 0.41666666666666663, "Overall CEN": 0.4638112995385119, "Lambda B": 0.16666666666666666, "SOA5(Cramer)": "Relatively Strong", "Bennett S": 0.37500000000000006, "P-Value": 0.18926430237560654, "PPV Micro": 0.5833333333333334, "Chi-Squared DF": 4, "Lambda A": 0.42857142857142855, "Overall RACCU": 0.3645833333333333, "Kappa No Prevalence": 0.16666666666666674, "Kappa Unbiased": 0.34426229508196726, "Standard Error": 0.14231876063832777, "Overall MCC": 0.36666666666666664, "ACC Macro": 0.7222222222222223, "Reference Entropy": 1.4833557549816874, "Kappa 95% CI": [-0.07707577422109269, 0.7867531935759315], "Scott PI": 0.34426229508196726, "SOA3(Altman)": "Fair", "AUNP": 0.6857142857142857, "Joint Entropy": 2.4591479170272446, "Phi-Squared": 0.55, "Kappa Standard Error": 0.2203645326012817, "NIR": 0.4166666666666667, "Response Entropy": 1.5, "F1 Micro": 0.5833333333333334, "Zero-one Loss": 5, "SOA2(Fleiss)": "Poor", "TPR Macro": 0.5666666666666668, "RCI": 0.3533932006492363, "Conditional Entropy": 0.9757921620455572, "Chi-Squared": 6.6000000000000005, "SOA6(Matthews)": "Weak", "CBA": 0.4777777777777778, "Mutual Information": 0.5242078379544428, "TPR Micro": 0.5833333333333334, "Gwet AC1": 0.3893129770992367, "SOA4(Cicchetti)": "Poor", "Overall MCEN": 0.5189369467580801}, "Digit": 5} \ No newline at end of file +{"Sample-Weight": null, "Class-Stat": {"PRE": {"L2": 0.16666666666666666, "L1": 0.4166666666666667, "L3": 0.4166666666666667}, "BM": {"L2": 0.30000000000000004, "L1": 0.6000000000000001, "L3": 0.17142857142857126}, "ERR": {"L2": 0.25, "L1": 0.16666666666666663, "L3": 0.41666666666666663}, "RACCU": {"L2": 0.04340277777777778, "L1": 0.1111111111111111, "L3": 0.21006944444444442}, "AUC": {"L2": 0.65, "L1": 0.8, "L3": 0.5857142857142856}, "MCEN": {"L2": 0.5, "L1": 0.2643856189774724, "L3": 0.6875}, "NLR": {"L2": 0.625, "L1": 0.4, "L3": 0.7000000000000001}, "G": {"L2": 0.408248290463863, "L1": 0.7745966692414834, "L3": 0.5477225575051661}, "DP": {"L2": 0.33193306999649924, "L1": "None", "L3": 0.1659665349982495}, "dInd": {"L2": 0.5385164807134504, "L1": 0.4, "L3": 0.5862367008195198}, "Q": {"L2": 0.6, "L1": "None", "L3": 0.3333333333333333}, "DPI": {"L2": "Poor", "L1": "None", "L3": "Poor"}, "IBA": {"L2": 0.27999999999999997, "L1": 0.36, "L3": 0.35265306122448975}, "PLRI": {"L2": "Poor", "L1": "None", "L3": "Poor"}, "IS": {"L2": 0.9999999999999998, "L1": 1.2630344058337937, "L3": 0.26303440583379367}, "CEN": {"L2": 0.49657842846620864, "L1": 0.25, "L3": 0.6044162769630221}, "ACC": {"L2": 0.75, "L1": 0.8333333333333334, "L3": 0.5833333333333334}, "FOR": {"L2": 0.11111111111111116, "L1": 0.2222222222222222, "L3": 0.33333333333333337}, "TP": {"L2": 1, "L1": 3, "L3": 3}, "GI": {"L2": 0.30000000000000004, "L1": 0.6000000000000001, "L3": 0.17142857142857126}, "TN": {"L2": 8, "L1": 7, "L3": 4}, "TON": {"L2": 9, "L1": 9, "L3": 6}, "BCD": {"L2": 0.041666666666666664, "L1": 0.08333333333333333, "L3": 0.041666666666666664}, "NPV": {"L2": 0.8888888888888888, "L1": 0.7777777777777778, "L3": 0.6666666666666666}, "MK": {"L2": 0.2222222222222221, "L1": 0.7777777777777777, "L3": 0.16666666666666652}, "MCC": {"L2": 0.25819888974716115, "L1": 0.6831300510639732, "L3": 0.1690308509457033}, "MCCI": {"L2": "Negligible", "L1": "Moderate", "L3": "Negligible"}, "FPR": {"L2": 0.19999999999999996, "L1": 0.0, "L3": 0.4285714285714286}, "DOR": {"L2": 4.000000000000001, "L1": "None", "L3": 1.9999999999999998}, "NLRI": {"L2": "Negligible", "L1": "Poor", "L3": "Negligible"}, "GM": {"L2": 0.6324555320336759, "L1": 0.7745966692414834, "L3": 0.5855400437691198}, "OP": {"L2": 0.5192307692307692, "L1": 0.5833333333333334, "L3": 0.5589430894308943}, "RACC": {"L2": 0.041666666666666664, "L1": 0.10416666666666667, "L3": 0.20833333333333334}, "OOC": {"L2": 0.4082482904638631, "L1": 0.7745966692414834, "L3": 0.5477225575051661}, "F0.5": {"L2": 0.35714285714285715, "L1": 0.8823529411764706, "L3": 0.5172413793103449}, "F2": {"L2": 0.45454545454545453, "L1": 0.6521739130434783, "L3": 0.5769230769230769}, "OC": {"L2": 0.5, "L1": 1.0, "L3": 0.6}, "FNR": {"L2": 0.5, "L1": 0.4, "L3": 0.4}, "PLR": {"L2": 2.5000000000000004, "L1": "None", "L3": 1.4}, "F1": {"L2": 0.4, "L1": 0.75, "L3": 0.5454545454545454}, "TOP": {"L2": 3, "L1": 3, "L3": 6}, "J": {"L2": 0.25, "L1": 0.6, "L3": 0.375}, "N": {"L2": 10, "L1": 7, "L3": 7}, "AGF": {"L2": 0.6286946134619315, "L1": 0.7285871475307653, "L3": 0.610088876086563}, "TNR": {"L2": 0.8, "L1": 1.0, "L3": 0.5714285714285714}, "P": {"L2": 2, "L1": 5, "L3": 5}, "TPR": {"L2": 0.5, "L1": 0.6, "L3": 0.6}, "sInd": {"L2": 0.6192113447068046, "L1": 0.717157287525381, "L3": 0.5854680534700882}, "AUCI": {"L2": "Fair", "L1": "Very Good", "L3": "Poor"}, "AGM": {"L2": 0.708612108382005, "L1": 0.8576400016262, "L3": 0.5803410802752335}, "LS": {"L2": 2.0, "L1": 2.4, "L3": 1.2}, "FDR": {"L2": 0.6666666666666667, "L1": 0.0, "L3": 0.5}, "AUPR": {"L2": 0.41666666666666663, "L1": 0.8, "L3": 0.55}, "AM": {"L2": 1, "L1": -2, "L3": 1}, "FP": {"L2": 2, "L1": 0, "L3": 3}, "FN": {"L2": 1, "L1": 2, "L3": 2}, "PPV": {"L2": 0.3333333333333333, "L1": 1.0, "L3": 0.5}, "Y": {"L2": 0.30000000000000004, "L1": 0.6000000000000001, "L3": 0.17142857142857126}, "POP": {"L2": 12, "L1": 12, "L3": 12}}, "Predict-Vector": null, "Matrix": [["L1", [["L2", 0], ["L1", 3], ["L3", 2]]], ["L2", [["L2", 1], ["L1", 0], ["L3", 1]]], ["L3", [["L2", 2], ["L1", 0], ["L3", 3]]]], "Digit": 5, "Transpose": true, "Overall-Stat": {"Overall CEN": 0.4638112995385119, "Conditional Entropy": 0.9757921620455572, "Standard Error": 0.14231876063832777, "Chi-Squared DF": 4, "Kappa No Prevalence": 0.16666666666666674, "SOA4(Cicchetti)": "Poor", "PPV Micro": 0.5833333333333334, "Hamming Loss": 0.41666666666666663, "SOA3(Altman)": "Fair", "ACC Macro": 0.7222222222222223, "Kappa Standard Error": 0.2203645326012817, "Reference Entropy": 1.4833557549816874, "Pearson C": 0.5956833971812706, "AUNU": 0.6785714285714285, "Cross Entropy": 1.5833333333333335, "Joint Entropy": 2.4591479170272446, "Bennett S": 0.37500000000000006, "Overall J": [1.225, 0.4083333333333334], "SOA5(Cramer)": "Relatively Strong", "TPR Micro": 0.5833333333333334, "Phi-Squared": 0.55, "TPR Macro": 0.5666666666666668, "Zero-one Loss": 5, "Lambda A": 0.42857142857142855, "Overall MCEN": 0.5189369467580801, "Kappa Unbiased": 0.34426229508196726, "Scott PI": 0.34426229508196726, "PPV Macro": 0.611111111111111, "Kappa 95% CI": [-0.07707577422109269, 0.7867531935759315], "SOA1(Landis & Koch)": "Fair", "F1 Micro": 0.5833333333333334, "Chi-Squared": 6.6000000000000005, "Overall RACC": 0.3541666666666667, "SOA6(Matthews)": "Weak", "SOA2(Fleiss)": "Poor", "Gwet AC1": 0.3893129770992367, "Kappa": 0.35483870967741943, "Response Entropy": 1.5, "AUNP": 0.6857142857142857, "95% CI": [0.30438856248221097, 0.8622781041844558], "CBA": 0.4777777777777778, "RR": 4.0, "Mutual Information": 0.5242078379544428, "F1 Macro": 0.5651515151515151, "Lambda B": 0.16666666666666666, "Overall RACCU": 0.3645833333333333, "KL Divergence": 0.09997757835164581, "Overall MCC": 0.36666666666666664, "NIR": 0.4166666666666667, "P-Value": 0.18926430237560654, "RCI": 0.3533932006492363, "Cramer V": 0.5244044240850758, "Overall ACC": 0.5833333333333334}, "Actual-Vector": null} \ No newline at end of file diff --git a/Document/Document_Files/cm1_summary.html b/Document/Document_Files/cm1_summary.html index b660938e..c16f7671 100644 --- a/Document/Document_Files/cm1_summary.html +++ b/Document/Document_Files/cm1_summary.html @@ -198,5 +198,5 @@

    Class Statistics :

    -

    Generated By PyCM Version 2.3

    +

    Generated By PyCM Version 2.4

    \ No newline at end of file diff --git a/Document/Example1.ipynb b/Document/Example1.ipynb index d92f2257..a77dd680 100644 --- a/Document/Example1.ipynb +++ b/Document/Example1.ipynb @@ -38,7 +38,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "You are using pip version 19.0.2, however version 19.1.1 is available.\n", + "You are using pip version 19.0.2, however version 19.2.1 is available.\n", "You should consider upgrading via the 'python -m pip install --upgrade pip' command.\n" ] } diff --git a/Document/Example1_Files/cm1.html b/Document/Example1_Files/cm1.html index 70dba8a5..8fa14c4b 100644 --- a/Document/Example1_Files/cm1.html +++ b/Document/Example1_Files/cm1.html @@ -304,7 +304,7 @@

    Class Statistics :

    1.0 0.8125 0.89655 -Area under the roc curve +Area under the ROC curve AUCI @@ -314,6 +314,13 @@

    Class Statistics :

    AUC value interpretation +AUPR +1.0 +0.8125 +0.8 +Area under the PR curve + + BCD 0.0 0.07895 @@ -679,5 +686,5 @@

    Class Statistics :

    -

    Generated By PyCM Version 2.3

    +

    Generated By PyCM Version 2.4

    \ No newline at end of file diff --git a/Document/Example1_Files/cm2.html b/Document/Example1_Files/cm2.html index f9029862..c745bfc1 100644 --- a/Document/Example1_Files/cm2.html +++ b/Document/Example1_Files/cm2.html @@ -304,7 +304,7 @@

    Class Statistics :

    1.0 0.96875 0.98276 -Area under the roc curve +Area under the ROC curve AUCI @@ -314,6 +314,13 @@

    Class Statistics :

    AUC value interpretation +AUPR +1.0 +0.96875 +0.95 +Area under the PR curve + + BCD 0.0 0.01316 @@ -679,5 +686,5 @@

    Class Statistics :

    -

    Generated By PyCM Version 2.3

    +

    Generated By PyCM Version 2.4

    \ No newline at end of file diff --git a/Document/Example1_Files/cm3.html b/Document/Example1_Files/cm3.html index 01ee1e5e..fcc1fbce 100644 --- a/Document/Example1_Files/cm3.html +++ b/Document/Example1_Files/cm3.html @@ -304,7 +304,7 @@

    Class Statistics :

    1.0 0.90057 0.81609 -Area under the roc curve +Area under the ROC curve AUCI @@ -314,6 +314,13 @@

    Class Statistics :

    AUC value interpretation +AUPR +1.0 +0.88542 +0.7619 +Area under the PR curve + + BCD 0.0 0.02632 @@ -679,5 +686,5 @@

    Class Statistics :

    -

    Generated By PyCM Version 2.3

    +

    Generated By PyCM Version 2.4

    \ No newline at end of file diff --git a/Document/Example2.ipynb b/Document/Example2.ipynb index 81aeee40..5903bdae 100644 --- a/Document/Example2.ipynb +++ b/Document/Example2.ipynb @@ -26,13 +26,13 @@ "output_type": "stream", "text": [ "Requirement already satisfied: matplotlib in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (2.2.2)\n", - "Requirement already satisfied: pytz in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib) (2018.3)\n", - "Requirement already satisfied: six>=1.10 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib) (1.11.0)\n", "Requirement already satisfied: cycler>=0.10 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib) (0.10.0)\n", + "Requirement already satisfied: six>=1.10 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib) (1.11.0)\n", "Requirement already satisfied: numpy>=1.7.1 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib) (1.15.2)\n", - "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib) (2.2.0)\n", "Requirement already satisfied: python-dateutil>=2.1 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib) (2.6.1)\n", + "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib) (2.2.0)\n", "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib) (1.0.1)\n", + "Requirement already satisfied: pytz in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib) (2018.3)\n", "Requirement already satisfied: setuptools in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from kiwisolver>=1.0.1->matplotlib) (39.2.0)\n" ] }, @@ -40,7 +40,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "You are using pip version 19.0.2, however version 19.1.1 is available.\n", + "You are using pip version 19.0.2, however version 19.2.1 is available.\n", "You should consider upgrading via the 'python -m pip install --upgrade pip' command.\n" ] } diff --git a/Document/Example3.ipynb b/Document/Example3.ipynb index 60140a49..2234dea2 100644 --- a/Document/Example3.ipynb +++ b/Document/Example3.ipynb @@ -135,8 +135,9 @@ "AGF(Adjusted F-score) 0.68041 0.89087 \n", "AGM(Adjusted geometric mean) 0.82426 0.65533 \n", "AM(Difference between automatic and manual classification) -1 1 \n", - "AUC(Area under the roc curve) 0.75 0.75 \n", + "AUC(Area under the ROC curve) 0.75 0.75 \n", "AUCI(AUC value interpretation) Good Good \n", + "AUPR(Area under the PR curve) 0.75 0.9 \n", "BCD(Bray-Curtis dissimilarity) 0.08333 0.08333 \n", "BM(Informedness or bookmaker informedness) 0.5 0.5 \n", "CEN(Confusion entropy) 0.52832 0.35221 \n", diff --git a/Document/Example5.ipynb b/Document/Example5.ipynb index 39da3543..35ce0416 100644 --- a/Document/Example5.ipynb +++ b/Document/Example5.ipynb @@ -130,8 +130,9 @@ "AGF(Adjusted F-score) 0.9136 0.53995 0.5516 \n", "AGM(Adjusted geometric mean) 0.83729 0.692 0.60712 \n", "AM(Difference between automatic and manual classification) 2 -1 -1 \n", - "AUC(Area under the roc curve) 0.88889 0.61111 0.58333 \n", + "AUC(Area under the ROC curve) 0.88889 0.61111 0.58333 \n", "AUCI(AUC value interpretation) Very Good Fair Poor \n", + "AUPR(Area under the PR curve) 0.8 0.41667 0.55 \n", "BCD(Bray-Curtis dissimilarity) 0.08333 0.04167 0.04167 \n", "BM(Informedness or bookmaker informedness) 0.77778 0.22222 0.16667 \n", "CEN(Confusion entropy) 0.25 0.49658 0.60442 \n", @@ -316,8 +317,9 @@ "AGF(Adjusted F-score) 0.94152 0.68599 0.76783 \n", "AGM(Adjusted geometric mean) 0.91704 0.81486 0.81018 \n", "AM(Difference between automatic and manual classification) 12 -5 -7 \n", - "AUC(Area under the roc curve) 0.9434 0.72029 0.81502 \n", + "AUC(Area under the ROC curve) 0.9434 0.72029 0.81502 \n", "AUCI(AUC value interpretation) Excellent Good Very Good \n", + "AUPR(Area under the PR curve) 0.83333 0.58333 0.86996 \n", "BCD(Bray-Curtis dissimilarity) 0.04615 0.01923 0.02692 \n", "BM(Informedness or bookmaker informedness) 0.88679 0.44058 0.63004 \n", "CEN(Confusion entropy) 0.23219 0.44655 0.28458 \n", diff --git a/Document/Example6.ipynb b/Document/Example6.ipynb index 0c360401..68ea939f 100644 --- a/Document/Example6.ipynb +++ b/Document/Example6.ipynb @@ -226,11 +226,11 @@ "Class4 0.0 0.0 2e-05 0.99998 \n", "\n", "\n", - "ACC: {'Class4': 0.9999500199920032, 'Class1': 0.9999750099960016, 'Class2': 0.9999500199920032, 'Class3': 0.9999250299880048}\n", - "MCC: {'Class4': 0.9333083339583177, 'Class1': 0.8944160139432883, 'Class2': 0.7999750068731099, 'Class3': 0.7302602381427055}\n", - "CEN: {'Class4': 0.0001575200922489127, 'Class1': 0.13625493172565745, 'Class2': 0.25701944178769376, 'Class3': 0.3649884090288471}\n", - "MCEN: {'Class4': 0.00029569133318617423, 'Class1': 0.17964888034078544, 'Class2': 0.3333333333333333, 'Class3': 0.4654427710721536}\n", - "DP: {'Class4': 3.1691421556058055, 'Class1': 'None', 'Class2': 2.869241573973406, 'Class3': 2.7032690544190636}\n", + "ACC: {'Class3': 0.9999250299880048, 'Class1': 0.9999750099960016, 'Class2': 0.9999500199920032, 'Class4': 0.9999500199920032}\n", + "MCC: {'Class3': 0.7302602381427055, 'Class1': 0.8944160139432883, 'Class2': 0.7999750068731099, 'Class4': 0.9333083339583177}\n", + "CEN: {'Class3': 0.3649884090288471, 'Class1': 0.13625493172565745, 'Class2': 0.25701944178769376, 'Class4': 0.0001575200922489127}\n", + "MCEN: {'Class3': 0.4654427710721536, 'Class1': 0.17964888034078544, 'Class2': 0.3333333333333333, 'Class4': 0.00029569133318617423}\n", + "DP: {'Class3': 2.7032690544190636, 'Class1': 'None', 'Class2': 2.869241573973406, 'Class4': 3.1691421556058055}\n", "Kappa: 0.8666333383326446\n", "RCI: 0.8711441699127427\n", "SOA1: Almost Perfect\n" @@ -285,11 +285,11 @@ "Class4 0.25 0.25 0.25 0.25 \n", "\n", "\n", - "ACC: {'Class4': 0.625, 'Class1': 0.625, 'Class2': 0.625, 'Class3': 0.625}\n", - "MCC: {'Class4': 0.0, 'Class1': 0.0, 'Class2': 0.0, 'Class3': 0.0}\n", - "CEN: {'Class4': 0.8704188162777186, 'Class1': 0.8704188162777186, 'Class2': 0.8704188162777186, 'Class3': 0.8704188162777186}\n", - "MCEN: {'Class4': 0.9308855421443073, 'Class1': 0.9308855421443073, 'Class2': 0.9308855421443073, 'Class3': 0.9308855421443073}\n", - "DP: {'Class4': 0.0, 'Class1': 0.0, 'Class2': 0.0, 'Class3': 0.0}\n", + "ACC: {'Class3': 0.625, 'Class1': 0.625, 'Class2': 0.625, 'Class4': 0.625}\n", + "MCC: {'Class3': 0.0, 'Class1': 0.0, 'Class2': 0.0, 'Class4': 0.0}\n", + "CEN: {'Class3': 0.8704188162777186, 'Class1': 0.8704188162777186, 'Class2': 0.8704188162777186, 'Class4': 0.8704188162777186}\n", + "MCEN: {'Class3': 0.9308855421443073, 'Class1': 0.9308855421443073, 'Class2': 0.9308855421443073, 'Class4': 0.9308855421443073}\n", + "DP: {'Class3': 0.0, 'Class1': 0.0, 'Class2': 0.0, 'Class4': 0.0}\n", "Kappa: 0.0\n", "RCI: 0.0\n", "SOA1: Slight\n" @@ -344,13 +344,13 @@ "Class4 0.76923 0.07692 0.07692 0.07692 \n", "\n", "\n", - "ACC: {'Class4': 0.4, 'Class1': 0.4, 'Class2': 0.76, 'Class3': 0.76}\n", - "MCC: {'Class4': -0.2358640882624316, 'Class1': -0.2358640882624316, 'Class2': 0.10714285714285714, 'Class3': 0.10714285714285714}\n", - "CEN: {'Class4': 0.6392779429225796, 'Class1': 0.6392779429225794, 'Class2': 0.8704188162777186, 'Class3': 0.8704188162777186}\n", - "MCEN: {'Class4': 0.647512271542988, 'Class1': 0.647512271542988, 'Class2': 0.9308855421443073, 'Class3': 0.9308855421443073}\n", - "DP: {'Class4': -0.3319330699964992, 'Class1': -0.33193306999649924, 'Class2': 0.16596653499824943, 'Class3': 0.16596653499824943}\n", - "Kappa: -0.07361963190184051\n", - "RCI: 0.1160303056449364\n", + "ACC: {'Class3': 0.76, 'Class1': 0.4, 'Class2': 0.76, 'Class4': 0.4}\n", + "MCC: {'Class3': 0.10714285714285714, 'Class1': -0.2358640882624316, 'Class2': 0.10714285714285714, 'Class4': -0.2358640882624316}\n", + "CEN: {'Class3': 0.8704188162777186, 'Class1': 0.6392779429225794, 'Class2': 0.8704188162777186, 'Class4': 0.6392779429225796}\n", + "MCEN: {'Class3': 0.9308855421443073, 'Class1': 0.647512271542988, 'Class2': 0.9308855421443073, 'Class4': 0.647512271542988}\n", + "DP: {'Class3': 0.16596653499824943, 'Class1': -0.33193306999649924, 'Class2': 0.16596653499824943, 'Class4': -0.3319330699964992}\n", + "Kappa: -0.07361963190184047\n", + "RCI: 0.11603030564493641\n", "SOA1: Poor\n" ] } @@ -403,12 +403,12 @@ "Class4 0.76923 0.07692 0.07692 0.07692 \n", "\n", "\n", - "ACC: {'Class4': 0.000998502246630055, 'Class1': 0.000998502246630055, 'Class2': 0.999400898652022, 'Class3': 0.999400898652022}\n", - "MCC: {'Class4': -0.43266656861311537, 'Class1': -0.43266656861311537, 'Class2': 0.24970032963739885, 'Class3': 0.24970032963739885}\n", - "CEN: {'Class4': 0.0029588592520426657, 'Class1': 0.0029588592520426657, 'Class2': 0.8704188162777186, 'Class3': 0.8704188162777186}\n", - "MCEN: {'Class4': 0.002903385725603509, 'Class1': 0.002903385725603509, 'Class2': 0.9308855421443073, 'Class3': 0.9308855421443073}\n", - "DP: {'Class4': -1.9423127303715728, 'Class1': -1.9423127303715728, 'Class2': 1.6794055876913858, 'Class3': 1.6794055876913858}\n", - "Kappa: -0.0003990813465900263\n", + "ACC: {'Class3': 0.999400898652022, 'Class1': 0.000998502246630055, 'Class2': 0.999400898652022, 'Class4': 0.000998502246630055}\n", + "MCC: {'Class3': 0.24970032963739885, 'Class1': -0.43266656861311537, 'Class2': 0.24970032963739885, 'Class4': -0.43266656861311537}\n", + "CEN: {'Class3': 0.8704188162777186, 'Class1': 0.0029588592520426657, 'Class2': 0.8704188162777186, 'Class4': 0.0029588592520426657}\n", + "MCEN: {'Class3': 0.9308855421443073, 'Class1': 0.002903385725603509, 'Class2': 0.9308855421443073, 'Class4': 0.002903385725603509}\n", + "DP: {'Class3': 1.6794055876913858, 'Class1': -1.9423127303715728, 'Class2': 1.6794055876913858, 'Class4': -1.9423127303715728}\n", + "Kappa: -0.0003990813465900262\n", "RCI: 0.5536610475678805\n", "SOA1: Poor\n" ] @@ -462,11 +462,11 @@ "Class4 0.25 0.25 0.25 0.25 \n", "\n", "\n", - "ACC: {'Class4': 0.36538461538461536, 'Class1': 0.7115384615384616, 'Class2': 0.7115384615384616, 'Class3': 0.7115384615384616}\n", - "MCC: {'Class4': 0.0, 'Class1': 0.0, 'Class2': 0.0, 'Class3': 0.0}\n", - "CEN: {'Class4': 0.6522742127953861, 'Class1': 0.6392779429225794, 'Class2': 0.6392779429225794, 'Class3': 0.6392779429225794}\n", - "MCEN: {'Class4': 0.7144082229288313, 'Class1': 0.647512271542988, 'Class2': 0.647512271542988, 'Class3': 0.647512271542988}\n", - "DP: {'Class4': 0.0, 'Class1': 0.0, 'Class2': 0.0, 'Class3': 0.0}\n", + "ACC: {'Class3': 0.7115384615384616, 'Class1': 0.7115384615384616, 'Class2': 0.7115384615384616, 'Class4': 0.36538461538461536}\n", + "MCC: {'Class3': 0.0, 'Class1': 0.0, 'Class2': 0.0, 'Class4': 0.0}\n", + "CEN: {'Class3': 0.6392779429225794, 'Class1': 0.6392779429225794, 'Class2': 0.6392779429225794, 'Class4': 0.6522742127953861}\n", + "MCEN: {'Class3': 0.647512271542988, 'Class1': 0.647512271542988, 'Class2': 0.647512271542988, 'Class4': 0.7144082229288313}\n", + "DP: {'Class3': 0.0, 'Class1': 0.0, 'Class2': 0.0, 'Class4': 0.0}\n", "Kappa: 0.0\n", "RCI: 0.0\n", "SOA1: Slight\n" @@ -521,11 +521,11 @@ "Class4 0.25 0.25 0.25 0.25 \n", "\n", "\n", - "ACC: {'Class4': 0.25014995501349596, 'Class1': 0.7499500149955014, 'Class2': 0.7499500149955014, 'Class3': 0.7499500149955014}\n", - "MCC: {'Class4': 0.0, 'Class1': 0.0, 'Class2': 0.0, 'Class3': 0.0}\n", - "CEN: {'Class4': 0.539296694603886, 'Class1': 0.0029588592520426657, 'Class2': 0.0029588592520426657, 'Class3': 0.0029588592520426657}\n", - "MCEN: {'Class4': 0.580710610324597, 'Class1': 0.002903385725603509, 'Class2': 0.002903385725603509, 'Class3': 0.002903385725603509}\n", - "DP: {'Class4': 0.0, 'Class1': 0.0, 'Class2': 0.0, 'Class3': 0.0}\n", + "ACC: {'Class3': 0.7499500149955014, 'Class1': 0.7499500149955014, 'Class2': 0.7499500149955014, 'Class4': 0.25014995501349596}\n", + "MCC: {'Class3': 0.0, 'Class1': 0.0, 'Class2': 0.0, 'Class4': 0.0}\n", + "CEN: {'Class3': 0.0029588592520426657, 'Class1': 0.0029588592520426657, 'Class2': 0.0029588592520426657, 'Class4': 0.539296694603886}\n", + "MCEN: {'Class3': 0.002903385725603509, 'Class1': 0.002903385725603509, 'Class2': 0.002903385725603509, 'Class4': 0.580710610324597}\n", + "DP: {'Class3': 0.0, 'Class1': 0.0, 'Class2': 0.0, 'Class4': 0.0}\n", "Kappa: 0.0\n", "RCI: 0.0\n", "SOA1: Slight\n" diff --git a/Document/Example7.ipynb b/Document/Example7.ipynb index 15a3f497..d60ecf31 100644 --- a/Document/Example7.ipynb +++ b/Document/Example7.ipynb @@ -24,16 +24,16 @@ "output_type": "stream", "text": [ "Requirement already satisfied: seaborn in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (0.9.0)\n", - "Requirement already satisfied: matplotlib>=1.4.3 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from seaborn) (2.2.2)\n", "Requirement already satisfied: scipy>=0.14.0 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from seaborn) (1.1.0)\n", - "Requirement already satisfied: pandas>=0.15.2 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from seaborn) (0.22.0)\n", + "Requirement already satisfied: matplotlib>=1.4.3 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from seaborn) (2.2.2)\n", "Requirement already satisfied: numpy>=1.9.3 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from seaborn) (1.15.2)\n", - "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib>=1.4.3->seaborn) (2.2.0)\n", + "Requirement already satisfied: pandas>=0.15.2 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from seaborn) (0.22.0)\n", "Requirement already satisfied: pytz in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib>=1.4.3->seaborn) (2018.3)\n", - "Requirement already satisfied: six>=1.10 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib>=1.4.3->seaborn) (1.11.0)\n", + "Requirement already satisfied: python-dateutil>=2.1 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib>=1.4.3->seaborn) (2.6.1)\n", + "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib>=1.4.3->seaborn) (2.2.0)\n", "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib>=1.4.3->seaborn) (1.0.1)\n", + "Requirement already satisfied: six>=1.10 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib>=1.4.3->seaborn) (1.11.0)\n", "Requirement already satisfied: cycler>=0.10 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib>=1.4.3->seaborn) (0.10.0)\n", - "Requirement already satisfied: python-dateutil>=2.1 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from matplotlib>=1.4.3->seaborn) (2.6.1)\n", "Requirement already satisfied: setuptools in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from kiwisolver>=1.0.1->matplotlib>=1.4.3->seaborn) (39.2.0)\n" ] }, @@ -41,7 +41,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "You are using pip version 19.0.2, however version 19.1.1 is available.\n", + "You are using pip version 19.0.2, however version 19.2.1 is available.\n", "You should consider upgrading via the 'python -m pip install --upgrade pip' command.\n" ] }, @@ -50,9 +50,9 @@ "output_type": "stream", "text": [ "Requirement already satisfied: pandas in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (0.22.0)\n", - "Requirement already satisfied: pytz>=2011k in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from pandas) (2018.3)\n", "Requirement already satisfied: numpy>=1.9.0 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from pandas) (1.15.2)\n", "Requirement already satisfied: python-dateutil>=2 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from pandas) (2.6.1)\n", + "Requirement already satisfied: pytz>=2011k in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from pandas) (2018.3)\n", "Requirement already satisfied: six>=1.5 in c:\\users\\sepkjaer\\appdata\\local\\programs\\python\\python35-32\\lib\\site-packages (from python-dateutil>=2->pandas) (1.11.0)\n" ] }, @@ -60,7 +60,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "You are using pip version 19.0.2, however version 19.1.1 is available.\n", + "You are using pip version 19.0.2, however version 19.2.1 is available.\n", "You should consider upgrading via the 'python -m pip install --upgrade pip' command.\n" ] } From 4a21d0f280c554b41138c01803d1284228d73317 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 30 Jul 2019 15:39:05 +0430 Subject: [PATCH 096/107] doc : outputs updated for version 2.4 --- Otherfiles/test.csv | 1 + Otherfiles/test.html | 11 +++++++++-- Otherfiles/test.obj | 2 +- Otherfiles/test.pycm | 3 ++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Otherfiles/test.csv b/Otherfiles/test.csv index 97d81cbc..168bf6d0 100644 --- a/Otherfiles/test.csv +++ b/Otherfiles/test.csv @@ -5,6 +5,7 @@ AGM,0.85764,0.70861,0.58034 AM,-2,1,1 AUC,0.8,0.65,0.58571 AUCI,Very Good,Fair,Poor +AUPR,0.8,0.41667,0.55 BCD,0.08333,0.04167,0.04167 BM,0.6,0.3,0.17143 CEN,0.25,0.49658,0.60442 diff --git a/Otherfiles/test.html b/Otherfiles/test.html index e551a7b1..bec5ec92 100644 --- a/Otherfiles/test.html +++ b/Otherfiles/test.html @@ -304,7 +304,7 @@

    Class Statistics :

    0.8 0.65 0.58571 -Area under the roc curve +Area under the ROC curve AUCI @@ -314,6 +314,13 @@

    Class Statistics :

    AUC value interpretation +AUPR +0.8 +0.41667 +0.55 +Area under the PR curve + + BCD 0.08333 0.04167 @@ -679,5 +686,5 @@

    Class Statistics :

    -

    Generated By PyCM Version 2.3

    +

    Generated By PyCM Version 2.4

    \ No newline at end of file diff --git a/Otherfiles/test.obj b/Otherfiles/test.obj index fce0e389..dc291064 100644 --- a/Otherfiles/test.obj +++ b/Otherfiles/test.obj @@ -1 +1 @@ -{"Transpose": true, "Predict-Vector": null, "Sample-Weight": null, "Actual-Vector": null, "Matrix": [["L1", [["L2", 0], ["L1", 3], ["L3", 2]]], ["L2", [["L2", 1], ["L1", 0], ["L3", 1]]], ["L3", [["L2", 2], ["L1", 0], ["L3", 3]]]], "Digit": 5} \ No newline at end of file +{"Sample-Weight": null, "Predict-Vector": null, "Matrix": [["L1", [["L2", 0], ["L1", 3], ["L3", 2]]], ["L2", [["L2", 1], ["L1", 0], ["L3", 1]]], ["L3", [["L2", 2], ["L1", 0], ["L3", 3]]]], "Digit": 5, "Transpose": true, "Actual-Vector": null} \ No newline at end of file diff --git a/Otherfiles/test.pycm b/Otherfiles/test.pycm index a982afd7..2bbb3d24 100644 --- a/Otherfiles/test.pycm +++ b/Otherfiles/test.pycm @@ -85,8 +85,9 @@ ACC(Accuracy) 0.83333 AGF(Adjusted F-score) 0.72859 0.62869 0.61009 AGM(Adjusted geometric mean) 0.85764 0.70861 0.58034 AM(Difference between automatic and manual classification) -2 1 1 -AUC(Area under the roc curve) 0.8 0.65 0.58571 +AUC(Area under the ROC curve) 0.8 0.65 0.58571 AUCI(AUC value interpretation) Very Good Fair Poor +AUPR(Area under the PR curve) 0.8 0.41667 0.55 BCD(Bray-Curtis dissimilarity) 0.08333 0.04167 0.04167 BM(Informedness or bookmaker informedness) 0.6 0.3 0.17143 CEN(Confusion entropy) 0.25 0.49658 0.60442 From 73a94df2f56da416bd76d498ac851cd0175e0359 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 30 Jul 2019 15:45:39 +0430 Subject: [PATCH 097/107] rel : version_check script updated for version 2.4 --- Otherfiles/version_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Otherfiles/version_check.py b/Otherfiles/version_check.py index c00c1355..6398cb14 100644 --- a/Otherfiles/version_check.py +++ b/Otherfiles/version_check.py @@ -4,7 +4,7 @@ import sys import codecs Failed = 0 -VERSION = "2.3" +VERSION = "2.4" SETUP_ITEMS = [ From 32bea91870ef147e1cb0b3f426ed1265efd317da Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 30 Jul 2019 15:52:04 +0430 Subject: [PATCH 098/107] fix : minor edit in setup.py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index e742092a..6d3fc038 100644 --- a/setup.py +++ b/setup.py @@ -58,7 +58,6 @@ def read_description(): 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', From 2b2ab5aede284f6ceb4e5ed608833d1f2fa24209 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 30 Jul 2019 16:45:18 +0430 Subject: [PATCH 099/107] doc : CHANGELOG updated --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f51df1a0..a36e05b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `sample_weight` bug in `numpy` array format fixed - Inputs manipulation bug fixed - Test system modified +- Warning system modified - `alt_link` parameter added to `save_html` method and `online_help` function - `Compare` class tests moved to `compare_test.py` - Warning tests moved to `warning_test.py` From 09c78d4f7206099b686aa1ac48e09185bd1e9d26 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 30 Jul 2019 16:49:22 +0430 Subject: [PATCH 100/107] fix : autopep8 fix --- pycm/pycm_compare.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pycm/pycm_compare.py b/pycm/pycm_compare.py index 780ffc54..15d1888d 100644 --- a/pycm/pycm_compare.py +++ b/pycm/pycm_compare.py @@ -106,7 +106,8 @@ def save_report( file.write(report) file.close() if address: - message = os.path.join(os.getcwd(), name + ".comp") # pragma: no cover + message = os.path.join( + os.getcwd(), name + ".comp") # pragma: no cover return {"Status": True, "Message": message} except Exception as e: return {"Status": False, "Message": str(e)} From 455ec7f430989b41e6bcc313b2754b7b634fb54b Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 30 Jul 2019 16:49:55 +0430 Subject: [PATCH 101/107] doc : CHANGELOG updated --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a36e05b4..ea12146e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] -## [2.4] - 2019-07-30 +## [2.4] - 2019-07-31 ### Added - Tversky index (TI) - Area under the PR curve (AUPR) From e176c2586174745d8ece71af843f11ebb384007d Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 30 Jul 2019 17:53:34 +0430 Subject: [PATCH 102/107] doc : README examples updated --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 32351162..826f381f 100644 --- a/README.md +++ b/README.md @@ -205,8 +205,9 @@ ACC(Accuracy) 0.83333 AGF(Adjusted F-score) 0.9136 0.53995 0.5516 AGM(Adjusted geometric mean) 0.83729 0.692 0.60712 AM(Difference between automatic and manual classification) 2 -1 -1 -AUC(Area under the roc curve) 0.88889 0.61111 0.58333 +AUC(Area under the ROC curve) 0.88889 0.61111 0.58333 AUCI(AUC value interpretation) Very Good Fair Poor +AUPR(Area under the PR curve) 0.8 0.41667 0.55 BCD(Bray-Curtis dissimilarity) 0.08333 0.04167 0.04167 BM(Informedness or bookmaker informedness) 0.77778 0.22222 0.16667 CEN(Confusion entropy) 0.25 0.49658 0.60442 @@ -367,8 +368,9 @@ ACC(Accuracy) 0.75 AGF(Adjusted F-score) 0.53979 0.81325 AGM(Adjusted geometric mean) 0.73991 0.5108 AM(Difference between automatic and manual classification) -2 2 -AUC(Area under the roc curve) 0.66667 0.66667 +AUC(Area under the ROC curve) 0.66667 0.66667 AUCI(AUC value interpretation) Fair Fair +AUPR(Area under the PR curve) 0.66667 0.85714 BCD(Bray-Curtis dissimilarity) 0.125 0.125 BM(Informedness or bookmaker informedness) 0.33333 0.33333 CEN(Confusion entropy) 0.5 0.43083 @@ -420,7 +422,7 @@ TP(True positive/hit) 1 TPR(Sensitivity, recall, hit rate, or true positive rate) 0.33333 1.0 Y(Youden index) 0.33333 0.33333 dInd(Distance index) 0.66667 0.66667 -sInd(Similarity index) 0.5286 0.5286 +sInd(Similarity index) 0.5286 0.5286 >>> cm3 = ConfusionMatrix(matrix={"Class1": {"Class1": 1, "Class2":0}, "Class2": {"Class1": 2, "Class2": 5}},transpose=True) # Transpose Matrix >>> cm3.print_matrix() From d56d10eb8b76986fce289a69ab31150246c7c141 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 30 Jul 2019 18:06:09 +0430 Subject: [PATCH 103/107] doc : summary mode example added to README --- README.md | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 826f381f..7dabb953 100644 --- a/README.md +++ b/README.md @@ -423,7 +423,38 @@ TPR(Sensitivity, recall, hit rate, or true positive rate) 0.33333 Y(Youden index) 0.33333 0.33333 dInd(Distance index) 0.66667 0.66667 sInd(Similarity index) 0.5286 0.5286 - + +>>> cm2.stat(summary=True) +Overall Statistics : + +ACC Macro 0.75 +F1 Macro 0.66667 +Kappa 0.38462 +Overall ACC 0.75 +PPV Macro 0.85714 +SOA1(Landis & Koch) Fair +TPR Macro 0.66667 +Zero-one Loss 2 + +Class Statistics : + +Classes Class1 Class2 +ACC(Accuracy) 0.75 0.75 +AUC(Area under the ROC curve) 0.66667 0.66667 +AUCI(AUC value interpretation) Fair Fair +F1(F1 score - harmonic mean of precision and sensitivity) 0.5 0.83333 +FN(False negative/miss/type 2 error) 2 0 +FP(False positive/type 1 error/false alarm) 0 2 +N(Condition negative) 5 3 +P(Condition positive or support) 3 5 +POP(Population) 8 8 +PPV(Precision or positive predictive value) 1.0 0.71429 +TN(True negative/correct rejection) 5 1 +TON(Test outcome negative) 7 1 +TOP(Test outcome positive) 1 7 +TP(True positive/hit) 1 5 +TPR(Sensitivity, recall, hit rate, or true positive rate) 0.33333 1.0 + >>> cm3 = ConfusionMatrix(matrix={"Class1": {"Class1": 1, "Class2":0}, "Class2": {"Class1": 2, "Class2": 5}},transpose=True) # Transpose Matrix >>> cm3.print_matrix() Predict Class1 Class2 From 3327d48a33e78015c2ea9d1b1dd7fe5f224ec6cb Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 30 Jul 2019 23:35:57 +0430 Subject: [PATCH 104/107] fix : minor edit in conditional_entropy_calc function --- pycm/pycm_overall_func.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycm/pycm_overall_func.py b/pycm/pycm_overall_func.py index 0d663c4f..8e921455 100644 --- a/pycm/pycm_overall_func.py +++ b/pycm/pycm_overall_func.py @@ -391,7 +391,7 @@ def conditional_entropy_calc(classes, table, P, POP): result = 0 for i in classes: temp = 0 - for index, j in enumerate(classes): + for j in classes: p_prime = 0 if P[i] != 0: p_prime = table[i][j] / P[i] From b2a5d16738034f2c5bdcceeeaeafcd9ce1a6d6c8 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 30 Jul 2019 23:36:45 +0430 Subject: [PATCH 105/107] fix : minor edit in joint_entropy_calc function --- pycm/pycm_overall_func.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pycm/pycm_overall_func.py b/pycm/pycm_overall_func.py index 8e921455..91f25174 100644 --- a/pycm/pycm_overall_func.py +++ b/pycm/pycm_overall_func.py @@ -364,7 +364,7 @@ def joint_entropy_calc(classes, table, POP): try: result = 0 for i in classes: - for index, j in enumerate(classes): + for j in classes: p_prime = table[i][j] / POP[i] if p_prime != 0: result += p_prime * math.log(p_prime, 2) From efb006cb93965d7848222d69a2f96c9cfcc7e0a9 Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Tue, 30 Jul 2019 23:40:18 +0430 Subject: [PATCH 106/107] doc : online help section modified --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7dabb953..a3e02cb4 100644 --- a/README.md +++ b/README.md @@ -507,6 +507,7 @@ pycm.ConfusionMatrix(classes: ['L1', 'L2', 'L3']) ``` * List of items are available by calling `online_help()` (without argument) +* If PyCM website is not available, set `alt_link = True` (new in `version 2.4`) ### Parameter recommender From 80b7b327677dff1f65821bbf94b8fb7c06a4deea Mon Sep 17 00:00:00 2001 From: sepandhaghighi Date: Wed, 31 Jul 2019 00:01:08 +0430 Subject: [PATCH 107/107] fix : remove leading spaces before the shebang --- Otherfiles/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Otherfiles/test.sh b/Otherfiles/test.sh index cb20576d..cb9ad54d 100755 --- a/Otherfiles/test.sh +++ b/Otherfiles/test.sh @@ -1,4 +1,4 @@ - #!/bin/bash +#!/bin/bash set -e set -x python -m pytest Test --cov=pycm --cov-report=term