master_script.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports and Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "import skelm\n",
    "import sklearn.linear_model\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import random\n",
    "from sklearn.feature_selection import SelectKBest, f_classif, RFE\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "from sklearn.decomposition import PCA\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "from saving_outputs import *\n",
    "from load_data import *\n",
    "from masks import *\n",
    "from decoding import *\n",
    "from plots import *\n",
    "from utility import *\n",
    "from metrics import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from_who = \"our\"   # 2 possibilities : \"mohameds\" or \"our\"\n",
    "voxel_size = \"3\" # 2 possibilities : \"2\" or \"3\"\n",
    "radius_mask = \"10\" # [mm] from 5 to 11\n",
    "use_t_maps = False # use t-maps or beta-maps\n",
    "correction = \"_Tcorrected\" # a t-value has been used to create ROIs or no,\n",
    "use_pca = False # use PCA or not\n",
    "components_pca = 20 # number of components to keep in PCA\n",
    "use_k_selector = False # use k-selector or not\n",
    "use_rfe = False # use RFE or not\n",
    "features_rfe = 100 # number of features to keep in RFE\n",
    "rfe_type = \"LR\" # type of RFE estimator to use\n",
    "\n",
    "maps_folder=\"brain_maps/\"+from_who+\"_maps\"+\"_\"+voxel_size\n",
    "masks_folder=\"masks/\"+from_who+\"_masks\"+\"_\"+voxel_size+\"_radius=\"+radius_mask+correction\n",
    "\n",
    "SEED = 0\n",
    "random.seed(SEED)\n",
    "classes = ['Up', 'Down', 'Right', 'Left']\n",
    "nb_runs = 12\n",
    "labels = {'vis' : np.array(classes*nb_runs), 'aud' : np.array(classes*nb_runs)}\n",
    "labels_same = np.array(classes*nb_runs)\n",
    "subjects_ids = range(1, 24)\n",
    "n_subjects = len(subjects_ids)\n",
    "n_individual_perms = 1000\n",
    "\n",
    "within_modal_tasks_regions = [([\"vis\"], [\"V5_L\", \"V5_R\"]),\n",
    "                           ([\"vis\"], [\"PT_L\", \"PT_R\"]),\n",
    "                           ([\"aud\"], [\"V5_L\", \"V5_R\"]),\n",
    "                           ([\"aud\"], [\"PT_L\", \"PT_R\"])]\n",
    "\n",
    "cross_modal_tasks_regions = [([\"vis\", \"aud\"], [\"V5_L\", \"V5_R\"]),\n",
    "                            ([\"vis\", \"aud\"], [\"PT_L\", \"PT_R\"])]\n",
    "\n",
    "def k_selector(n_voxel):\n",
    "    return int(0.7*n_voxel)\n",
    "\n",
    "std_scaler = sklearn.preprocessing.StandardScaler()\n",
    "\n",
    "selector = SelectKBest(score_func=f_classif)\n",
    "\n",
    "classifiers = {\n",
    "    #'svm':sklearn.svm.SVC(random_state=SEED),\n",
    "    #'LR':sklearn.linear_model.LogisticRegression(random_state=SEED, max_iter=1000),\n",
    "    #'KNN':sklearn.neighbors.KNeighborsClassifier(),\n",
    "    'perceptron':sklearn.linear_model.Perceptron(random_state=SEED),\n",
    "    'ELM':skelm.ELMClassifier(random_state=SEED),\n",
    "    }\n",
    "\n",
    "param_grids = {\n",
    "    'svm':{\n",
    "        #'svm__C': [10**x for x in np.linspace(-3,3,num=20)],\n",
    "        'svm__C': [1],\n",
    "        #'svm__gamma': [1, 0.1],\n",
    "        'svm__kernel': ['linear'],\n",
    "        #'svm__decision_function_shape': ['ovo', 'ovr'],\n",
    "        #'svm__gamma': [10**x for x in np.linspace(-3,3,num=13)],\n",
    "        #'svm__gamma': [\"scale\", \"auto\"],\n",
    "    },\n",
    "    'LR':{\n",
    "        #'LR__C': [10**x for x in np.linspace(-3,3,num=20)],\n",
    "        'LR__C': [1],\n",
    "        #'LR__max_iter': [1000]\n",
    "        #'LR__multi_class': ['ovr', 'multinomial']\n",
    "    },\n",
    "    'KNN':{\n",
    "        #'KNN__n_neighbors': [1,3,5,10,20],\n",
    "        'KNN__n_neighbors': [5],\n",
    "        'KNN__weights':[\"uniform\", \"distance\"],\n",
    "    },\n",
    "    'perceptron':{\n",
    "        'perceptron__penalty':[\"l1\", \"l2\", \"elasticnet\"],\n",
    "        'perceptron__max_iter':[1000],\n",
    "        'perceptron__alpha':[0.0001,0.001,0.01,0.1,1],\n",
    "        'perceptron__tol':[0.0001,0.001,0.01,0.1,1],\n",
    "    },\n",
    "    'ELM':{\n",
    "        'ELM__n_neurons': [100,200,300,400,500],\n",
    "        'ELM__alpha': [0.1, 0.5, 1, 2, 5, 10, 20],\n",
    "        'ELM__activation': [\"identity\", \"logistic\", \"tanh\", \"relu\"],\n",
    "        'ELM__include_original_features': [True, False],\n",
    "    }\n",
    "}\n",
    "cv_scheme = list()\n",
    "for i in range(11):\n",
    "    full_idx = range(44)\n",
    "    idx_te = [i*4,i*4+1,i*4+2,i*4+3]\n",
    "    idx_tr = [x for x in full_idx if x not in idx_te]\n",
    "    tr_te_splits = [idx_tr, idx_te]\n",
    "    cv_scheme.append(tr_te_splits)\n",
    "\n",
    "models = dict()\n",
    "identity_transformer = FunctionTransformer(func=lambda x: x)\n",
    "for name in classifiers:\n",
    "    estimator_class = classifiers[name].__class__\n",
    "    params = classifiers[name].get_params()\n",
    "    #rfe_estimator = estimator_class().set_params(params) # not used at the moment\n",
    "    pipeline = Pipeline([('scaler', std_scaler),\n",
    "                         ('pca', (PCA(n_components=components_pca, random_state=SEED) if use_pca else identity_transformer)),\n",
    "                         ('kbest', (selector if use_k_selector else identity_transformer)),\n",
    "                         ('rfe', (RFE(sklearn.linear_model.LogisticRegression(random_state=SEED,C=0.5, max_iter=1000), n_features_to_select=features_rfe, step=5) if use_rfe else identity_transformer)),\n",
    "                         (name, classifiers[name])\n",
    "                         ])\n",
    "    GS = GridSearchCV(pipeline, param_grids[name], cv = cv_scheme, n_jobs = 4)\n",
    "    models[name] = GS\n",
    "    \n",
    "decoder = Decoder(n_perm=n_individual_perms, models=models, n_classes=len(classes), n_splits=nb_runs, seed=SEED, verbose=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "maps_masked, masks_exist = load_full_data(subjects_ids, len(classes), nb_runs, maps_folder, masks_folder, is_from_mohamed=(from_who==\"mohameds\"), use_t_maps=use_t_maps)\n",
    "decoder.set_masks_exist(masks_exist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_voxels = maps_masked[0][\"vis\"][0][\"V5_L\"].shape[1]\n",
    "for name in classifiers:\n",
    "    if isinstance(decoder.models[name].estimator.steps[1][1], SelectKBest): \n",
    "        decoder.models[name].estimator.steps[1][1].set_params(k = k_selector(n_voxels))\n",
    "print(\"initially \"+str(n_voxels)+\" voxels, selected = \"+str(k_selector(n_voxels))+\" voxels.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# Within-modality decoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Invalid parameter activation for estimator ELMClassifier(random_state=0). Check the list of available parameters with `estimator.get_params().keys()`.",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mValueError\u001B[0m                                Traceback (most recent call last)",
      "\u001B[1;32m~\\AppData\\Local\\Temp/ipykernel_12324/4154023467.py\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[1;32m----> 1\u001B[1;33m \u001B[0mconfusion_matrixes_within\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mval_scores_within\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mdecoder\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mwithin_modality_decoding\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mmaps_masked\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mlabels\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0msubjects_ids\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mwithin_modal_tasks_regions\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m      2\u001B[0m \u001B[0mconfusion_matrixes_within\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mchange_confusion_matrixes_org\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mconfusion_matrixes_within\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0msubjects_ids\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mmodels\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mkeys\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m      3\u001B[0m \u001B[0mval_scores_within\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mchange_confusion_matrixes_org\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mval_scores_within\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0msubjects_ids\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mmodels\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mkeys\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\OneDrive\\Bureau\\Mémoire\\MasterThesisGit\\decoding.py\u001B[0m in \u001B[0;36mwithin_modality_decoding\u001B[1;34m(self, maps, labels, subjects_ids, tasks_regions)\u001B[0m\n\u001B[0;32m    180\u001B[0m             \u001B[1;31m# within-modality decoding : training on a task and decoding on other samples from same task\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    181\u001B[0m             \u001B[1;32mfor\u001B[0m \u001B[0mtasks\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mregions\u001B[0m \u001B[1;32min\u001B[0m \u001B[0mtasks_regions\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 182\u001B[1;33m                 \u001B[0m_\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mcf\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0m_\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mvs\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mclassify_tasks_regions\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mmaps\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mi\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mlabels\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mtasks\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mregions\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mi\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdo_pval\u001B[0m\u001B[1;33m=\u001B[0m\u001B[1;32mFalse\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    183\u001B[0m                 \u001B[0mconf_matrixes\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mi\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mupdate\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mcf\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    184\u001B[0m                 \u001B[0mval_scores\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mi\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mupdate\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mvs\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\OneDrive\\Bureau\\Mémoire\\MasterThesisGit\\decoding.py\u001B[0m in \u001B[0;36mclassify_tasks_regions\u001B[1;34m(self, brain_maps, labels, tasks_names, regions_names, id_subj, do_pval)\u001B[0m\n\u001B[0;32m    169\u001B[0m                     \u001B[0m_maps\u001B[0m \u001B[1;33m=\u001B[0m \u001B[1;33m[\u001B[0m\u001B[0mmaps\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mregion_name\u001B[0m\u001B[1;33m]\u001B[0m \u001B[1;32mfor\u001B[0m \u001B[0mmaps\u001B[0m \u001B[1;32min\u001B[0m \u001B[0mbrain_maps\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mtask_name\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    170\u001B[0m                     \u001B[0mkey\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mtask_name\u001B[0m \u001B[1;33m+\u001B[0m \u001B[1;34m\"_\"\u001B[0m \u001B[1;33m+\u001B[0m \u001B[0mregion_name\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 171\u001B[1;33m                     \u001B[0mp_values\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mkey\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mconf_matrixes\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mkey\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mconf_matrixes_perms\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mkey\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mval_scores\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mkey\u001B[0m\u001B[1;33m]\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mclassify\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0m_maps\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mlabels\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mtask_name\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdo_pval\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    172\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    173\u001B[0m         \u001B[1;32mreturn\u001B[0m \u001B[0mp_values\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mconf_matrixes\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mconf_matrixes_perms\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mval_scores\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\OneDrive\\Bureau\\Mémoire\\MasterThesisGit\\decoding.py\u001B[0m in \u001B[0;36mclassify\u001B[1;34m(self, brain_maps, labels, do_pval)\u001B[0m\n\u001B[0;32m    143\u001B[0m         :return: cross-validation confusion matrix, p-value \"\"\"\n\u001B[0;32m    144\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 145\u001B[1;33m         \u001B[0mconf_matrix\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mval_scores\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mcross_validate\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mbrain_maps\u001B[0m\u001B[1;33m[\u001B[0m\u001B[1;36m0\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mlabels\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    146\u001B[0m         \u001B[0mbase_score\u001B[0m \u001B[1;33m=\u001B[0m \u001B[1;33m[\u001B[0m\u001B[0maccuracy\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mconf_matrix\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mname\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mn_classes\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;32mfor\u001B[0m \u001B[0mname\u001B[0m \u001B[1;32min\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mmodels\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    147\u001B[0m         \u001B[0mp_val\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mconf_matrix_perm\u001B[0m \u001B[1;33m=\u001B[0m \u001B[1;32mNone\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;32mNone\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\OneDrive\\Bureau\\Mémoire\\MasterThesisGit\\decoding.py\u001B[0m in \u001B[0;36mcross_validate\u001B[1;34m(self, brain_map, labels, return_model, brain_map_2)\u001B[0m\n\u001B[0;32m     83\u001B[0m             \u001B[1;32mfor\u001B[0m \u001B[0mname\u001B[0m \u001B[1;32min\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mmodels\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     84\u001B[0m                 \u001B[0mmodel\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mmodels\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mname\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 85\u001B[1;33m                 \u001B[0mmodel\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mfit\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mX_train\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0my_train\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     86\u001B[0m                 \u001B[0mpredictions\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mmodel\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mpredict\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mX_test\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     87\u001B[0m                 \u001B[0mtest_score_associated\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mmodel\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mscore\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mX_test\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0my_test\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\miniconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py\u001B[0m in \u001B[0;36mfit\u001B[1;34m(self, X, y, groups, **fit_params)\u001B[0m\n\u001B[0;32m    889\u001B[0m                 \u001B[1;32mreturn\u001B[0m \u001B[0mresults\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    890\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 891\u001B[1;33m             \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_run_search\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mevaluate_candidates\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    892\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    893\u001B[0m             \u001B[1;31m# multimetric is determined here because in the case of a callable\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\miniconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py\u001B[0m in \u001B[0;36m_run_search\u001B[1;34m(self, evaluate_candidates)\u001B[0m\n\u001B[0;32m   1390\u001B[0m     \u001B[1;32mdef\u001B[0m \u001B[0m_run_search\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mevaluate_candidates\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1391\u001B[0m         \u001B[1;34m\"\"\"Search all candidates in param_grid\"\"\"\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m-> 1392\u001B[1;33m         \u001B[0mevaluate_candidates\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mParameterGrid\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mparam_grid\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m   1393\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1394\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\miniconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py\u001B[0m in \u001B[0;36mevaluate_candidates\u001B[1;34m(candidate_params, cv, more_results)\u001B[0m\n\u001B[0;32m    836\u001B[0m                     )\n\u001B[0;32m    837\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 838\u001B[1;33m                 out = parallel(\n\u001B[0m\u001B[0;32m    839\u001B[0m                     delayed(_fit_and_score)(\n\u001B[0;32m    840\u001B[0m                         \u001B[0mclone\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mbase_estimator\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m,\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\miniconda3\\lib\\site-packages\\joblib\\parallel.py\u001B[0m in \u001B[0;36m__call__\u001B[1;34m(self, iterable)\u001B[0m\n\u001B[0;32m   1054\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1055\u001B[0m             \u001B[1;32mwith\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_backend\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mretrieval_context\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m-> 1056\u001B[1;33m                 \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mretrieve\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m   1057\u001B[0m             \u001B[1;31m# Make sure that we get a last message telling us we are done\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1058\u001B[0m             \u001B[0melapsed_time\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mtime\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mtime\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;33m-\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_start_time\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\miniconda3\\lib\\site-packages\\joblib\\parallel.py\u001B[0m in \u001B[0;36mretrieve\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m    933\u001B[0m             \u001B[1;32mtry\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    934\u001B[0m                 \u001B[1;32mif\u001B[0m \u001B[0mgetattr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_backend\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;34m'supports_timeout'\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;32mFalse\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 935\u001B[1;33m                     \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_output\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mextend\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mjob\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mget\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mtimeout\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mtimeout\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    936\u001B[0m                 \u001B[1;32melse\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    937\u001B[0m                     \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_output\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mextend\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mjob\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mget\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\miniconda3\\lib\\site-packages\\joblib\\_parallel_backends.py\u001B[0m in \u001B[0;36mwrap_future_result\u001B[1;34m(future, timeout)\u001B[0m\n\u001B[0;32m    540\u001B[0m         AsyncResults.get from multiprocessing.\"\"\"\n\u001B[0;32m    541\u001B[0m         \u001B[1;32mtry\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 542\u001B[1;33m             \u001B[1;32mreturn\u001B[0m \u001B[0mfuture\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mresult\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mtimeout\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mtimeout\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    543\u001B[0m         \u001B[1;32mexcept\u001B[0m \u001B[0mCfTimeoutError\u001B[0m \u001B[1;32mas\u001B[0m \u001B[0me\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    544\u001B[0m             \u001B[1;32mraise\u001B[0m \u001B[0mTimeoutError\u001B[0m \u001B[1;32mfrom\u001B[0m \u001B[0me\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\miniconda3\\lib\\concurrent\\futures\\_base.py\u001B[0m in \u001B[0;36mresult\u001B[1;34m(self, timeout)\u001B[0m\n\u001B[0;32m    437\u001B[0m                 \u001B[1;32mraise\u001B[0m \u001B[0mCancelledError\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    438\u001B[0m             \u001B[1;32melif\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_state\u001B[0m \u001B[1;33m==\u001B[0m \u001B[0mFINISHED\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 439\u001B[1;33m                 \u001B[1;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m__get_result\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    440\u001B[0m             \u001B[1;32melse\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    441\u001B[0m                 \u001B[1;32mraise\u001B[0m \u001B[0mTimeoutError\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32m~\\miniconda3\\lib\\concurrent\\futures\\_base.py\u001B[0m in \u001B[0;36m__get_result\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m    386\u001B[0m     \u001B[1;32mdef\u001B[0m \u001B[0m__get_result\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    387\u001B[0m         \u001B[1;32mif\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_exception\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 388\u001B[1;33m             \u001B[1;32mraise\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_exception\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    389\u001B[0m         \u001B[1;32melse\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    390\u001B[0m             \u001B[1;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_result\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;31mValueError\u001B[0m: Invalid parameter activation for estimator ELMClassifier(random_state=0). Check the list of available parameters with `estimator.get_params().keys()`."
     ]
    }
   ],
   "source": [
    "confusion_matrixes_within, val_scores_within = decoder.within_modality_decoding(maps_masked, labels, subjects_ids, within_modal_tasks_regions)\n",
    "confusion_matrixes_within = change_confusion_matrixes_org(confusion_matrixes_within, subjects_ids, models.keys())\n",
    "val_scores_within = change_confusion_matrixes_org(val_scores_within, subjects_ids, models.keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# Cross-modal decoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "confusion_matrixes_cross, val_scores_cross = decoder.cross_modality_decoding(maps_masked, labels, subjects_ids, cross_modal_tasks_regions)\n",
    "confusion_matrixes_cross = change_confusion_matrixes_org(confusion_matrixes_cross, subjects_ids, models.keys())\n",
    "val_scores_cross = change_confusion_matrixes_org(val_scores_cross, subjects_ids, models.keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# Bootstrapping to assess group-level significance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "n_single_perm = 50\n",
    "n_bootstrap = 100_000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "within_cf_100_perm = decoder.score_bootstrapped_permutations(n_single_perm, labels_same, within_modal_tasks_regions,maps_masked,n_subjects, within_modality=True)\n",
    "within_cf_100_perm = change_cfm_bootstrap_org(within_cf_100_perm, subjects_ids, models.keys(), n_single_perm)\n",
    "\n",
    "cross_cf_100_perm = decoder.score_bootstrapped_permutations(n_single_perm, labels_same, cross_modal_tasks_regions,maps_masked,n_subjects, within_modality=False)\n",
    "cross_cf_100_perm = change_cfm_bootstrap_org(cross_cf_100_perm, subjects_ids, models.keys(), n_single_perm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "bootstrapped_distribution_cross = dict()\n",
    "bootstrapped_distribution_within = dict()\n",
    "for name in models.keys():\n",
    "    within_scores_100_perm = compute_accuracy_bootstrap(n_subjects, n_single_perm, within_cf_100_perm[name], len(classes))\n",
    "    bootstrapped_distribution_within[name] = compute_bootstrap_distribution(n_bootstrap, n_subjects, within_scores_100_perm, n_single_perm)\n",
    "\n",
    "    cross_scores_100_perm = compute_accuracy_bootstrap(n_subjects, n_single_perm, cross_cf_100_perm[name], len(classes))\n",
    "    bootstrapped_distribution_cross[name] = compute_bootstrap_distribution(n_bootstrap, n_subjects, cross_scores_100_perm, n_single_perm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saving results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type_maps = \"_t_maps_\" if use_t_maps else \"_beta_maps_\" \n",
    "out_directory = \"out/\"+from_who+type_maps\n",
    "\n",
    "for name in classifiers:\n",
    "    out_dir = out_directory+str(classifiers[name])+\"_\"+voxel_size+\"_radius=\"+radius_mask+correction + (\"_pca\"+str(components_pca) if use_pca else \"\") + (\"_\" + \"k_selector\" if use_k_selector else \"\")+ (\"_\" + \"rfe\"+str(features_rfe) + rfe_type if use_rfe else \"\")+\"/\"\n",
    "    create_directory(out_dir)\n",
    "    save_dicts(out_dir+\"masks_exist.csv\", masks_exist, list(masks_exist[0].keys()), subjects_ids)\n",
    "    \n",
    "    save_dicts(out_dir+\"confusion_matrixes_within.csv\", confusion_matrixes_within[name], list(confusion_matrixes_within[name][0].keys()), subjects_ids)\n",
    "    save_dicts(out_dir+\"validation_scores_within.csv\", val_scores_within[name], list(val_scores_within[name][0].keys()), subjects_ids)\n",
    "    acc_within = compute_metric(out_dir, subjects_ids, {'name' : 'accuracy', 'function':accuracy}, \"within\", masks_exist, len(classes), ret = True)\n",
    "\n",
    "    compute_metric(out_dir, subjects_ids, {'name' : 'recall', 'function':recall}, \"within\", masks_exist, len(classes))\n",
    "    compute_metric(out_dir, subjects_ids, {'name' : 'precision', 'function':precision}, \"within\", masks_exist, len(classes))\n",
    "\n",
    "    within_modality_group_results = average_dicos(acc_within)\n",
    "    save_dicts(out_dir+\"group_scores_within.csv\", [within_modality_group_results], list(within_modality_group_results.keys()), [0])\n",
    "\n",
    "\n",
    "    save_dicts(out_dir+\"confusion_matrixes_cross.csv\", confusion_matrixes_cross[name], list(confusion_matrixes_cross[name][0].keys()), subjects_ids)\n",
    "    save_dicts(out_dir+\"validation_scores_cross.csv\", val_scores_cross[name], list(val_scores_cross[name][0].keys()), subjects_ids)\n",
    "    acc_cross = compute_metric(out_dir, subjects_ids, {'name' : 'accuracy', 'function':accuracy}, \"cross\", masks_exist, len(classes), ret = True)\n",
    "    \n",
    "    compute_metric(out_dir, subjects_ids, {'name' : 'recall', 'function':recall}, \"cross\", masks_exist, len(classes))\n",
    "    compute_metric(out_dir, subjects_ids, {'name' : 'precision', 'function':precision}, \"cross\", masks_exist, len(classes))\n",
    "\n",
    "    cross_modality_group_results = average_dicos(acc_cross)\n",
    "    save_dicts(out_dir+\"group_scores_cross.csv\", [cross_modality_group_results], list(cross_modality_group_results.keys()), [0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "for name in classifiers:\n",
    "    out_dir = out_directory+str(classifiers[name])+\"_\"+voxel_size+\"_radius=\"+radius_mask + (\"_\" + \"pca\" if use_pca else \"\") + (\"_\" + \"k_selector\" if use_k_selector else \"\")+(\"_\" + \"rfe\"+str(features_rfe) + rfe_type if use_rfe else \"\")+\"/\"\n",
    "    save_dicts(out_dir+\"accuracy_bootstraps_within.csv\", [bootstrapped_distribution_within[name]], list(bootstrapped_distribution_within[name][0].keys()), range(n_bootstrap))\n",
    "    save_dicts(out_dir+\"accuracy_bootstraps_cross.csv\", [bootstrapped_distribution_cross[name]], list(bootstrapped_distribution_cross[name][0].keys()), range(n_bootstrap))\n",
    "\n",
    "    cv_group_df = retrieve_cv_metric(out_dir, \"group_scores\")\n",
    "    bootstrap_df  = retrieve_bootstrap_metric(out_dir, \"accuracy\")\n",
    "    pvals = compute_p_val_bootstrap(bootstrap_df, cv_group_df)\n",
    "    save_dicts(out_dir+\"estimated_pval_bootstrap.csv\", [pvals], list(pvals.keys()), [0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Plotting results (from files of saved results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "type_maps = \"_t_maps_\" if use_t_maps else \"_beta_maps_\"\n",
    "out_directory = \"out/\"+from_who+type_maps#+\"_2tests_\"\n",
    "for name in classifiers:\n",
    "    out_dir = out_directory+str(classifiers[name])+\"_\"+voxel_size+\"_radius=\"+radius_mask+correction + (\"_\" + \"pca\"+str(components_pca) if use_pca else \"\") + (\"_\" + \"k_selector\" if use_k_selector else \"\")+(\"_\" + \"rfe\"+str(features_rfe) + rfe_type if use_rfe else \"\")+\"/\"\n",
    "    masks_exist = retrieve_masks_exist(out_dir)\n",
    "    cv_group_df = retrieve_cv_metric(out_dir, \"group_scores\")\n",
    "    cv_df = retrieve_cv_metric(out_dir, \"accuracy\")\n",
    "    cfm_df = retrieve_cv_matrixes(out_dir)\n",
    "    val_scores_df = retrieve_val_scores(out_dir)\n",
    "    pvals = retrieve_pvals(out_dir, default_keys=cv_df.columns)\n",
    "\n",
    "    plt_dir = \"plots/\"+from_who+type_maps+str(classifiers[name])+\"_\"+voxel_size+\"_radius=\"+radius_mask+correction + (\"_\" + \"pca\"+str(components_pca) if use_pca else \"\") + (\"_\" + \"k_selector\" if use_k_selector else \"\")+(\"_\" + \"rfe\"+str(features_rfe) + rfe_type if use_rfe else \"\")\n",
    "    create_directory(plt_dir)\n",
    "    plotter = Plotter(plt_dir, subjects_ids)\n",
    "    plotter.plot_cv_score_with_points(cv_df, pvals, chance_level = True)\n",
    "    compute_accuracy_variance(out_dir, \"within\")\n",
    "    compute_accuracy_variance(out_dir, \"cross\")\n",
    "    #plotter.plot_validation_scores_hyper_param(val_scores_df, \"C\", param_grids[\"LR\"][\"LR__C\"], masks_exist, chance_level=True, log10_scale=True)\n",
    "\n",
    "    group_cfm = compute_group_confusion_matrix(cfm_df, subjects_ids)\n",
    "    plotter.plot_group_confusion_matrix(group_cfm, classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "for name in classifiers:\n",
    "    out_dir = out_directory+str(classifiers[name])+\"_\"+voxel_size+\"_radius=\"+radius_mask+\"/\"\n",
    "    bootstrap_df  = retrieve_bootstrap_metric(out_dir, \"accuracy\")\n",
    "    cv_group_df = retrieve_cv_metric(out_dir, \"group_scores\")\n",
    "    pvals = retrieve_pvals(out_dir)\n",
    "    \n",
    "    plotter.plot_bootstrap(bootstrap_df, cv_group_df, pvals, 30)"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "# ANOVA tests"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "folders = [\"out/our_t_maps_SVC(random_state=0)_3_radius=10_Tcorrected/\",\n",
    "           \"out/our_t_maps_KNeighborsClassifier()_3_radius=10_Tcorrected/\",\n",
    "           ]\n",
    "\n",
    "compute_anova(folders)\n",
    "print(\"-------------------------------\")\n",
    "compute_repeated_anova(folders)\n",
    "\n",
    "#print('-----------------------------------------------------')\n",
    "#compute_anova_demeaned(folders[0], folders[1])\n",
    "#print('-----------------------------------------------------')\n",
    "#compute_two_sided_t_test_demeaned(folders[0], folders[1])\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparing classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "subjects_ids = range(1,24)\n",
    "out_folders = [\"out/our_t_maps_SVC(random_state=0)_3_radius=10_Tcorrected/\",\n",
    "               \"out/our_t_maps_SVC(random_state=0)_3_radius=10_Tcorrected_rfe100SVM/\",\n",
    "               \"out/our_t_maps_SVC(random_state=0)_3_radius=10_Tcorrected_rfe50SVM/\",\n",
    "               \"out/our_t_maps_SVC(random_state=0)_3_radius=10_Tcorrected_rfe25SVM/\",\n",
    "               \"out/our_t_maps_SVC(random_state=0)_3_radius=10_Tcorrected_rfe10SVM/\",\n",
    "               ]\n",
    "\n",
    "classifiers_names = [\"SVM\", \"SVM with RFE(100)\", \"SVM with RFE(50)\", \"SVM with RFE(25)\", \"SVM with RFE(10)\"]\n",
    "\n",
    "for folder in out_folders :\n",
    "    compute_accuracy_variance(folder, \"within\")\n",
    "    compute_accuracy_variance(folder, \"cross\")\n",
    "\n",
    "plotter = Plotter(\"plots/comparing_classifiers_rfe_svm_\"+\"_\".join(classifiers_names), subjects_ids)\n",
    "plotter.plot_tests_scores_from_different_folders(out_folders, classifiers_names, \"Influence of RFE\", \"Classifier\", p_vals=False)\n",
    "plotter.plot_accuracy_std_from_different_folders(out_folders, classifiers_names, \"CInfluence of RFE\", \"Classifier\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "subjects_ids = range(1,24)\n",
    "out_folders = [\"out/our_t_maps_SVC(random_state=0)_3_radius=10_Tcorrected/\",\n",
    "               \"out/our_t_maps_LogisticRegression(max_iter=1000, random_state=0)_3_radius=10_Tcorrected/\",\n",
    "               \"out/our_t_maps_KNeighborsClassifier()_3_radius=10_Tcorrected/\",\n",
    "               #Perceptron\n",
    "               #\"out/our_t_maps_ELMClassifier(random_state=0)_3_radius=10_Tcorrected/\"\n",
    "               ]\n",
    "\n",
    "classifiers_names = [\"SVM\", \"LR\",\"KNN\"]#, \"Perceptron\", \"ELM\"]\n",
    "\n",
    "for folder in out_folders :\n",
    "    compute_accuracy_variance(folder, \"within\")\n",
    "    compute_accuracy_variance(folder, \"cross\")\n",
    "\n",
    "plotter = Plotter(\"plots/comparing_classifiers_\"+\"_\".join(classifiers_names), subjects_ids)\n",
    "plotter.plot_tests_scores_from_different_folders(out_folders, classifiers_names, \"Comparing classifiers\", \"Classifier\", p_vals=False)\n",
    "plotter.plot_accuracy_std_from_different_folders(out_folders, classifiers_names, \"Comparing classifiers\", \"Classifier\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "subjects_ids = range(1,24)\n",
    "out_folders = [\"out/our_t_maps_LogisticRegression(max_iter=1000, random_state=0)_3_radius=10_Tcorrected/\",\n",
    "               \"out/our_t_maps_LogisticRegression(max_iter=1000, random_state=0)_3_radius=10_Tcorrected_rfe100SVM/\",\n",
    "               \"out/our_t_maps_LogisticRegression(max_iter=1000, random_state=0)_3_radius=10_Tcorrected_rfe100LR/\"\n",
    "               ]\n",
    "\n",
    "classifiers_names = [\"LR\", \"LR with SVM-RFE(100)\", \"LR with LR-RFE(100)\"]\n",
    "\n",
    "for folder in out_folders :\n",
    "    compute_accuracy_variance(folder, \"within\")\n",
    "    compute_accuracy_variance(folder, \"cross\")\n",
    "\n",
    "plotter = Plotter(\"plots/comparing_classifiers_rfe_lr_\"+\"_\".join(classifiers_names), subjects_ids)\n",
    "plotter.plot_tests_scores_from_different_folders(out_folders, classifiers_names, \"Influence of RFE\", \"Classifier\", p_vals=False)\n",
    "plotter.plot_accuracy_std_from_different_folders(out_folders, classifiers_names, \"CInfluence of RFE\", \"Classifier\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subjects_ids = range(1,24)\n",
    "x = dict()\n",
    "type_maps = \"_t_maps_\" if use_t_maps else \"_beta_maps_\"\n",
    "x[\"2\"] = [81, 123, 179, 257, 389, 515]\n",
    "x[\"3\"] = [19, 33, 57, 81, 123, 171]\n",
    "acc_group_combination = dict()\n",
    "var_group_combination = dict()\n",
    "radiuses = range(5,11)\n",
    "voxel_sizes = [\"2\", \"3\"]\n",
    "\n",
    "for voxel_size in voxel_sizes:\n",
    "    acc_group_combination[voxel_size]= dict((radius,None) for radius in radiuses)\n",
    "    var_group_combination[voxel_size] = dict((radius,None) for radius in radiuses)\n",
    "    for radius in radiuses:\n",
    "        folder = \"out/our_\"+type_maps+str(voxel_size)+\"_radius=\"+str(radius)+\"_\"+str(classifiers[\"svm\"])+\"selecting_20.0%/\"\n",
    "        acc = retrieve_cv_metric(folder, \"group_scores\")\n",
    "        dfw = pd.read_csv(folder+\"var_within.csv\", index_col=0)\n",
    "        dfc = pd.read_csv(folder+\"var_cross.csv\", index_col=0)\n",
    "        df = pd.concat([dfw, dfc])\n",
    "        new_cols = df.index[1:]\n",
    "        temp_df = pd.DataFrame(columns=new_cols, index=[1], dtype=float)\n",
    "        temp_df[new_cols] = df[df.columns[0]].values[1:]\n",
    "\n",
    "        acc_group_combination[voxel_size][radius] = acc\n",
    "        var_group_combination[voxel_size][radius] = temp_df\n",
    "        #if not os.path.exists(folder+\"accuracy_cross.csv\"): os.rename(folder+\"cross_cross.csv\", folder+\"accuracy_cross.csv\")\n",
    "        #compute_accuracy_variance(folder, \"within\")\n",
    "        #compute_accuracy_variance(folder, \"cross\")\n",
    "\n",
    "colors = {\"2\":\"tab:green\",\"3\":\"tab:orange\"}\n",
    "plt_directory = \"plots/\"+from_who+\"_\"+type_maps+\"vsizes_radiuses_OK\"+str(classifiers[\"svm\"])+\"/\"\n",
    "create_directory(plt_directory)\n",
    "plotter = Plotter(plt_directory, subjects_ids)\n",
    "scores_var = dict()\n",
    "\n",
    "for i, analysis in enumerate(acc_group_combination[\"2\"][5]):\n",
    "    # # faire le plot\n",
    "    # for voxel_size in  voxel_sizes:\n",
    "    #      scores = [acc_group_combination[voxel_size][radius][analysis] for radius in radiuses]\n",
    "    # #     scores_var[analysis] = np.var(scores*100)\n",
    "    # #     print(analysis)\n",
    "    # #     print((max(scores)-min(scores))*100)\n",
    "    #      plt.plot(radiuses, scores, '-o', color = colors[voxel_size], label = voxel_size+\" mm\")\n",
    "    #      plt.axhline(0.25, color=\"gray\", alpha=0.5)\n",
    "    # title = plotter.generate_title(\"Accuracy\",analysis,0)\n",
    "    # plt.ylim(0.2,0.5)\n",
    "    # plotter.save(title+ \", depending on ROI radius\",\"\",\"mean accuracy\",\"radius [mm]\",legend=True)\n",
    "\n",
    "    # for voxel_size in  voxel_sizes:\n",
    "    #     scores = [acc_group_combination[voxel_size][radius][analysis] for radius in radiuses]\n",
    "    #     #n_voxels = list(map(k_selector, np.array(voxel_amounts[voxel_size])))\n",
    "    #     plt.plot(np.array(x[voxel_size]), scores, '-o', color = colors[voxel_size], label = voxel_size+\" mm\")\n",
    "    #     #print(\"number of voxels : \"+str(voxel_amounts[voxel_size]))\n",
    "    #     #print(\"number of voxels selected : \"+str(n_voxels))\n",
    "    #     plt.axhline(0.25, color=\"gray\", alpha=0.5)\n",
    "    # title = plotter.generate_title(\"Accuracy\",analysis,0)\n",
    "    # plt.ylim(0.2,0.5)\n",
    "    # plotter.save(title+\", depending on amount of voxels\",\"\",\"mean accuracy\",\"voxel amount\",legend=True)\n",
    "\n",
    "    for voxel_size in  voxel_sizes:\n",
    "        scores = [var_group_combination[voxel_size][radius][analysis] for radius in radiuses]\n",
    "        #n_voxels = list(map(k_selector, np.array(voxel_amounts[voxel_size])))\n",
    "        plt.plot(np.array(x[voxel_size]), scores, '-o', color = colors[voxel_size], label = voxel_size+\" mm\")\n",
    "        #print(\"number of voxels : \"+str(voxel_amounts[voxel_size]))\n",
    "        #print(\"number of voxels selected : \"+str(n_voxels))\n",
    "        #plt.axhline(0.25, color=\"gray\", alpha=0.5)\n",
    "    title = plotter.generate_title(\"Variance\",analysis,0)\n",
    "    plt.ylim(0, 0.014)\n",
    "    plotter.save(title+\", depending on amount of voxels\",\"\",\"accuracy variance\",\"voxel amount\",legend=True)\n",
    "\n",
    "# out_folders = list()\n",
    "# labels = list()\n",
    "# for kn in [\"2\", \"3\", \"5\"]:\n",
    "#     for sm in [\"15\", \"30\", \"50\"]:\n",
    "#         out_folders.append(\"out/our__t_maps_3_radius=10_SVC(C=1, random_state=0)smote\"+sm+\"kn\"+kn+\"/\")\n",
    "#         labels.append(\"s=\"+sm+\"k=\"+kn)\n",
    "\n",
    "# out_folders.append(\"out/our_3_radius=10_SVC(C=1, random_state=0)/\")\n",
    "# labels.append(\"no smote\")\n",
    "\n",
    "# print(out_folders)\n",
    "\n",
    "\n",
    "# plotter = Plotter(\"plots/comparing_smote_\"+\"_\".join(labels), subjects_ids)\n",
    "# plotter.plot_tests_scores_from_different_folders(out_folders, labels, \"SMOTE\", \"parameters\")\n",
    "# plotter.plot_accuracy_var_from_different_folders(out_folders, labels)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "9650cb4e16cdd4a8e8e2d128bf38d875813998db22a3c986335f89e0cb4d7bb2"
  },
  "kernelspec": {
   "display_name": "Python 3.8.6 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}