Modeling_Ensemble.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9440dcf5-a4e9-439a-a74f-10c93797607a",
   "metadata": {},
   "source": [
    "# Estimating VaR in EURUSD from IV using ML and QR"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01243841-1668-4fcb-a6bf-8c8f1bbad4a7",
   "metadata": {},
   "source": [
    "## Modeling-Ensemble"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f93b66d8-517b-4b34-b72d-69181ac00ec4",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e87e1d4b-e1aa-4c24-9e2a-5bc68003d27a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "import joblib\n",
    "import statsmodels.api as sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c9763ee0-5184-4a58-8e7f-b44b542976be",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data_scale.pickle', 'rb') as f:\n",
    "    data_scale = pickle.load(f)\n",
    "\n",
    "with open('model_qr.pickle', 'rb') as f:\n",
    "    model_qr = pickle.load(f)\n",
    "\n",
    "with open('model_qr2.pickle', 'rb') as f:\n",
    "    model_qr2 = pickle.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "892e1146-2d6f-47b2-b114-64263774488b",
   "metadata": {
    "tags": []
   },
   "source": [
    "### QR-IM generated dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bebb625c-7363-44df-a671-b3f6600ca8b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "quantiles = [0.01, 0.025, 0.05, 0.95, 0.975, 0.99]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9066be47-a1da-4cc5-acf5-0658a9bff951",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_spot_is_x = data_scale['df_spot_is'].iloc[:,:-1]\n",
    "df_spot2_is_x = data_scale['df_spot2_is'].iloc[:,:-1]\n",
    "df_spread_is_x = data_scale['df_spread_is'].iloc[:,:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3aecc262-fea0-439d-ac79-460a57ea01a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "predict_qr = dict()\n",
    "for quantile, model in model_qr.items():\n",
    "    predict_qr[quantile] = model.predict(df_spot_is_x[['IV_ATM', 'RR_25D']])\n",
    "\n",
    "predict_qr2 = dict()\n",
    "for quantile, model in model_qr2.items():\n",
    "    predict_qr2[quantile] = model.predict(df_spot2_is_x)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6742c68f-6741-4193-819d-8060d6452dc8",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Random Forest: QR-IM dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e1904ce6-afcc-45bd-83a8-80a7389b331a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import TimeSeriesSplit, GridSearchCV\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "10296ba9-781e-4290-b8e7-d6a4b0354a60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### Random Forest-Quantile 0.01 ######\n",
      "Split 1 MSE: 1.5372573898666862e-06\n",
      "Split 2 MSE: 1.3769145815939367e-06\n",
      "Split 3 MSE: 3.3664631710369686e-07\n",
      "Split 4 MSE: 1.0985739536903793e-06\n",
      "Split 5 MSE: 1.5561282695046664e-06\n",
      "Quantile 0.01 Final MSE: 2.222583260116153e-08\n",
      "###### Random Forest-Quantile 0.025 ######\n",
      "Split 1 MSE: 8.187781523877505e-07\n",
      "Split 2 MSE: 6.659763101596655e-07\n",
      "Split 3 MSE: 1.3845404744418917e-07\n",
      "Split 4 MSE: 5.419814389889644e-07\n",
      "Split 5 MSE: 7.430899463823402e-07\n",
      "Quantile 0.025 Final MSE: 1.4767511204939047e-08\n",
      "###### Random Forest-Quantile 0.05 ######\n",
      "Split 1 MSE: 5.07264761112262e-07\n",
      "Split 2 MSE: 2.5215753113659697e-07\n",
      "Split 3 MSE: 8.474787638059223e-08\n",
      "Split 4 MSE: 2.443515531098794e-07\n",
      "Split 5 MSE: 3.097383569878273e-07\n",
      "Quantile 0.05 Final MSE: 1.0386889337692175e-08\n",
      "###### Random Forest-Quantile 0.95 ######\n",
      "Split 1 MSE: 7.237117932229468e-07\n",
      "Split 2 MSE: 3.508318796746841e-08\n",
      "Split 3 MSE: 3.855484463910626e-08\n",
      "Split 4 MSE: 3.81808961624331e-08\n",
      "Split 5 MSE: 2.9759481988601283e-08\n",
      "Quantile 0.95 Final MSE: 1.8513548668444453e-09\n",
      "###### Random Forest-Quantile 0.975 ######\n",
      "Split 1 MSE: 1.4472597825916562e-06\n",
      "Split 2 MSE: 9.552324233516423e-08\n",
      "Split 3 MSE: 7.741404362746854e-08\n",
      "Split 4 MSE: 1.822298238121602e-07\n",
      "Split 5 MSE: 1.8573648063768397e-07\n",
      "Quantile 0.975 Final MSE: 4.416633306635169e-09\n",
      "###### Random Forest-Quantile 0.99 ######\n",
      "Split 1 MSE: 1.7069762520195059e-06\n",
      "Split 2 MSE: 9.094373045718341e-08\n",
      "Split 3 MSE: 8.776931179644952e-08\n",
      "Split 4 MSE: 1.2207255695397246e-07\n",
      "Split 5 MSE: 1.0723562292073309e-07\n",
      "Quantile 0.99 Final MSE: 2.943249980934056e-09\n"
     ]
    }
   ],
   "source": [
    "rf_models = dict()\n",
    "for quantile in quantiles:\n",
    "    print(f'###### Random Forest-Quantile {quantile} ######')\n",
    "    X = df_spread_is_x\n",
    "    y = predict_qr[quantile]\n",
    "\n",
    "    # more sample weight toward recent data\n",
    "    sample_weights = np.arange(1,len(y)+1)\n",
    "\n",
    "    param_grid = {\n",
    "       'n_estimators': [50, 100, 200],\n",
    "       'max_depth': [None, 10, 20],\n",
    "       'min_samples_split': [2, 5, 10],\n",
    "       'min_samples_leaf': [1, 2, 4]\n",
    "    }\n",
    "\n",
    "    tscv = TimeSeriesSplit(n_splits=5)\n",
    "\n",
    "    best_params = []\n",
    "    best_scores = []\n",
    "\n",
    "    # walk forward cv\n",
    "    split = 0\n",
    "    for train_index, test_index in tscv.split(X):\n",
    "        # train test split\n",
    "        split += 1\n",
    "        X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n",
    "        y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n",
    "        weights_train = sample_weights[train_index]\n",
    "        weights_test = sample_weights[test_index]\n",
    "\n",
    "        # random forest regression\n",
    "        rf = RandomForestRegressor()\n",
    "        grid_search = GridSearchCV(rf,\n",
    "                                   param_grid,\n",
    "                                   cv=TimeSeriesSplit(n_splits=3),\n",
    "                                   n_jobs=-1)\n",
    "        grid_search.fit(X_train, y_train,\n",
    "                        sample_weight=weights_train)\n",
    "\n",
    "        # hyper-parameter tuning\n",
    "        best_params.append(grid_search.best_params_)\n",
    "        best_rf = grid_search.best_estimator_\n",
    "        y_pred = best_rf.predict(X_test)\n",
    "        mse = mean_squared_error(y_test, y_pred)\n",
    "        best_scores.append(mse)\n",
    "        print(f'Split {split} MSE: {mse}')\n",
    "\n",
    "    # final rf model\n",
    "    mean_score = np.mean(best_scores)\n",
    "    final_rf = RandomForestRegressor(**best_params[-1])\n",
    "    final_rf.fit(X, y, sample_weight=sample_weights)\n",
    "    y_pred = final_rf.predict(X)\n",
    "    mse = mean_squared_error(y, y_pred)\n",
    "    print(f'Quantile {quantile} Final MSE: {mse}')\n",
    "    \n",
    "    # append final model\n",
    "    rf_models[quantile] = final_rf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "614d61e2-0d9f-4899-ac79-ce121b49b815",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['rf_models.pkl']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(rf_models, 'rf_models.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0959c5b5-2775-4c0a-b69b-dd0bcae1891f",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### XGBoost(1): QR-IM Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4fe1e013-bb38-4c23-922f-8573a72c80bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "57bbc3b2-8cfc-4e40-9c63-a3d6a319c01f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### XGBoost 1-Quantile 0.01 ######\n",
      "Split 1 MSE: 1.4538978117731475e-06\n",
      "Split 2 MSE: 4.75001941221467e-07\n",
      "Split 3 MSE: 1.8595648193373418e-07\n",
      "Split 4 MSE: 3.1902499504692565e-07\n",
      "Split 5 MSE: 1.270345096580873e-06\n",
      "Quantile 0.01 Final MSE: 3.73324165510311e-09\n",
      "###### XGBoost 1-Quantile 0.025 ######\n",
      "Split 1 MSE: 1.1013882985999929e-06\n",
      "Split 2 MSE: 2.966276045765665e-07\n",
      "Split 3 MSE: 1.1210354686473428e-07\n",
      "Split 4 MSE: 1.8390904552084965e-07\n",
      "Split 5 MSE: 6.648778214190647e-07\n",
      "Quantile 0.025 Final MSE: 1.1314575891788625e-09\n",
      "###### XGBoost 1-Quantile 0.05 ######\n",
      "Split 1 MSE: 6.776148489626505e-07\n",
      "Split 2 MSE: 1.1561446876514746e-07\n",
      "Split 3 MSE: 5.5242373590808064e-08\n",
      "Split 4 MSE: 5.5743852018208567e-08\n",
      "Split 5 MSE: 2.3833745400932406e-07\n",
      "Quantile 0.05 Final MSE: 1.15565290844337e-09\n",
      "###### XGBoost 1-Quantile 0.95 ######\n",
      "Split 1 MSE: 6.015262453932209e-07\n",
      "Split 2 MSE: 2.353188681254225e-08\n",
      "Split 3 MSE: 4.0422646843336135e-08\n",
      "Split 4 MSE: 1.6837918919426172e-08\n",
      "Split 5 MSE: 2.42577002107031e-08\n",
      "Quantile 0.95 Final MSE: 6.990359227297153e-10\n",
      "###### XGBoost 1-Quantile 0.975 ######\n",
      "Split 1 MSE: 1.0463952251145614e-06\n",
      "Split 2 MSE: 1.2650153339891388e-07\n",
      "Split 3 MSE: 4.973117612631e-08\n",
      "Split 4 MSE: 6.580569514538863e-08\n",
      "Split 5 MSE: 1.4048092719630825e-07\n",
      "Quantile 0.975 Final MSE: 1.2010601471303763e-09\n",
      "###### XGBoost 1-Quantile 0.99 ######\n",
      "Split 1 MSE: 1.3161475514888447e-06\n",
      "Split 2 MSE: 9.34593734587435e-08\n",
      "Split 3 MSE: 6.153911704812807e-08\n",
      "Split 4 MSE: 5.955780005938785e-08\n",
      "Split 5 MSE: 3.715877724160863e-08\n",
      "Quantile 0.99 Final MSE: 4.70803561360839e-09\n"
     ]
    }
   ],
   "source": [
    "xgb1_models = dict()\n",
    "for quantile in quantiles:\n",
    "    print(f'###### XGBoost 1-Quantile {quantile} ######')\n",
    "    X = df_spread_is_x\n",
    "    y = predict_qr[quantile]\n",
    "\n",
    "    # more sample weight toward recent data\n",
    "    sample_weights = np.arange(1,len(y)+1)\n",
    "\n",
    "    param_grid = {\n",
    "        'n_estimators': [50, 100, 200],\n",
    "        'max_depth': [3, 5, 7],\n",
    "        'learning_rate': [0.01, 0.1, 0.2],\n",
    "        'subsample': [0.8, 0.9, 1.0]\n",
    "    }\n",
    "\n",
    "    tscv = TimeSeriesSplit(n_splits=5)\n",
    "\n",
    "    best_params = []\n",
    "    best_scores = []\n",
    "\n",
    "    # walk forward cv\n",
    "    split = 0\n",
    "    for train_index, test_index in tscv.split(X):\n",
    "        # train test split\n",
    "        split += 1\n",
    "        X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n",
    "        y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n",
    "        weights_train = sample_weights[train_index]\n",
    "        weights_test = sample_weights[test_index]\n",
    "\n",
    "        # xgboost regression\n",
    "        xgb_reg = xgb.XGBRegressor()\n",
    "        grid_search = GridSearchCV(xgb_reg,\n",
    "                                   param_grid,\n",
    "                                   cv=TimeSeriesSplit(n_splits=3),\n",
    "                                   n_jobs=-1)\n",
    "        grid_search.fit(X_train, y_train,\n",
    "                        sample_weight=weights_train)\n",
    "\n",
    "        # hyper-parameter tuning\n",
    "        best_params.append(grid_search.best_params_)\n",
    "        best_xgb_reg = grid_search.best_estimator_\n",
    "        y_pred = best_xgb_reg.predict(X_test)\n",
    "        mse = mean_squared_error(y_test, y_pred)\n",
    "        best_scores.append(mse)\n",
    "        print(f'Split {split} MSE: {mse}')\n",
    "\n",
    "    # final xgboost model\n",
    "    mean_score = np.mean(best_scores)\n",
    "    final_xgb_reg = xgb.XGBRegressor(**best_params[-1])\n",
    "    final_xgb_reg.fit(X, y, sample_weight=sample_weights)\n",
    "    y_pred = final_xgb_reg.predict(X)\n",
    "    mse = mean_squared_error(y, y_pred)\n",
    "    print(f'Quantile {quantile} Final MSE: {mse}')\n",
    "    \n",
    "    # append final model\n",
    "    xgb1_models[quantile] = final_xgb_reg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "af7b4b16-8439-4e9f-9b4b-376d32849444",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['xgb1_models.pkl']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(xgb1_models, 'xgb1_models.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35bb8054-a842-4754-879f-c5b7e98f5fbf",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### XGBoost(2): Gradient Boosting generated Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0e598282-9dae-41bd-8550-26e5cbfca278",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.metrics import make_scorer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f111428c-8fd1-4a39-8ce4-726f0ae16910",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "#### Gradient Boosting Dataset Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9aaf93f7-c8b3-48bb-a1e2-5a3b4ed7c18c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(action='ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0e28058e-927f-467b-bcbe-fa7428302928",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.01 Quantile Split 1 Pinball: 0.0002378448149821997\n",
      "0.025 Quantile Split 1 Pinball: 0.00042888888875717006\n",
      "0.05 Quantile Split 1 Pinball: 0.0006942403629171711\n",
      "0.95 Quantile Split 1 Pinball: 0.000650690898486907\n",
      "0.975 Quantile Split 1 Pinball: 0.00035494438381243837\n",
      "0.99 Quantile Split 1 Pinball: 0.00016030157935206592\n",
      "0.01 Quantile Split 2 Pinball: 0.00015327019836672786\n",
      "0.025 Quantile Split 2 Pinball: 0.00031679740669540555\n",
      "0.05 Quantile Split 2 Pinball: 0.0005492848940740808\n",
      "0.95 Quantile Split 2 Pinball: 0.0005536378095091283\n",
      "0.975 Quantile Split 2 Pinball: 0.00031995668921588183\n",
      "0.99 Quantile Split 2 Pinball: 0.0001545547052830905\n",
      "0.01 Quantile Split 3 Pinball: 0.0001947464338308918\n",
      "0.025 Quantile Split 3 Pinball: 0.0003836716748994674\n",
      "0.05 Quantile Split 3 Pinball: 0.0006162678308358285\n",
      "0.95 Quantile Split 3 Pinball: 0.0006941101193075007\n",
      "0.975 Quantile Split 3 Pinball: 0.0004049889469382913\n",
      "0.99 Quantile Split 3 Pinball: 0.0001762919434705711\n",
      "0.01 Quantile Split 4 Pinball: 0.0002455341203134282\n",
      "0.025 Quantile Split 4 Pinball: 0.00045518320774414985\n",
      "0.05 Quantile Split 4 Pinball: 0.0007412457717014685\n",
      "0.95 Quantile Split 4 Pinball: 0.0006712115757676397\n",
      "0.975 Quantile Split 4 Pinball: 0.00039379747217469755\n",
      "0.99 Quantile Split 4 Pinball: 0.00018607701416717155\n",
      "0.01 Quantile Split 5 Pinball: 0.00014837908180310527\n",
      "0.025 Quantile Split 5 Pinball: 0.00029351185523032025\n",
      "0.05 Quantile Split 5 Pinball: 0.0004918422201422972\n",
      "0.95 Quantile Split 5 Pinball: 0.0005548507572747678\n",
      "0.975 Quantile Split 5 Pinball: 0.00034377838626117993\n",
      "0.99 Quantile Split 5 Pinball: 0.00018219239323196603\n",
      "0.01 Quantile Final Pinball: 0.00018219239323196603\n",
      "0.025 Quantile Final Pinball: 0.00018219239323196603\n",
      "0.05 Quantile Final Pinball: 0.00018219239323196603\n",
      "0.95 Quantile Final Pinball: 0.00018219239323196603\n",
      "0.975 Quantile Final Pinball: 0.00018219239323196603\n",
      "0.99 Quantile Final Pinball: 0.00018219239323196603\n"
     ]
    }
   ],
   "source": [
    "sample_weights = np.arange(1,len(y)+1)\n",
    "\n",
    "param_grid = {\n",
    "   'n_estimators': [50, 100, 200],\n",
    "   'max_depth': [3, 5, 7],\n",
    "   'learning_rate': [0.01, 0.1, 0.2],\n",
    "   'subsample': [0.8, 0.9, 1.0]\n",
    "}\n",
    "\n",
    "def pinball_loss(y_true, y_pred, quantile):\n",
    "    errors = y_true - y_pred\n",
    "    return np.maximum((quantile - 1) * errors, quantile * errors).mean()\n",
    "\n",
    "pinball_scorer = make_scorer(pinball_loss, greater_is_better=False)\n",
    "\n",
    "tscv = TimeSeriesSplit(n_splits=5)\n",
    "\n",
    "best_params = {q: [] for q in quantiles}\n",
    "best_scores = {q: [] for q in quantiles}\n",
    "\n",
    "X = df_spread_is_x\n",
    "y = data_scale['df_spread_is'].iloc[:,-1]\n",
    "\n",
    "# walk forward cv\n",
    "split = 0\n",
    "for train_index, test_index in tscv.split(X):\n",
    "    split += 1\n",
    "    X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n",
    "    y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n",
    "    weights_train = sample_weights[train_index]\n",
    "    weights_test = sample_weights[test_index]\n",
    "\n",
    "    for quantile in quantiles:\n",
    "        # gradient boosting quantile regressor\n",
    "        gb_quantile = GradientBoostingRegressor(loss='quantile',\n",
    "                                                alpha=quantile)\n",
    "        grid_search = GridSearchCV(gb_quantile, param_grid,\n",
    "                                   cv=TimeSeriesSplit(n_splits=3),\n",
    "                                   n_jobs=-1, scoring=pinball_scorer)\n",
    "        grid_search.fit(X_train, y_train, sample_weight=weights_train)\n",
    "\n",
    "        # hyper-parameter tuning\n",
    "        best_params[quantile].append(grid_search.best_params_)\n",
    "        best_gb_quantile = GradientBoostingRegressor(\n",
    "            loss='quantile', alpha=quantile, **grid_search.best_params_)\n",
    "        best_gb_quantile.fit(X_train, y_train)\n",
    "        y_pred = best_gb_quantile.predict(X_test)\n",
    "        pinball = pinball_loss(y_test, y_pred, quantile)\n",
    "        best_scores[quantile].append(pinball)\n",
    "        print(f'{quantile} Quantile Split {split} Pinball: {pinball}')\n",
    "\n",
    "# append final gb model\n",
    "mean_scores = {q: np.mean(scores) for q, scores in best_scores.items()}\n",
    "\n",
    "final_models = dict()\n",
    "for quantile in quantiles:\n",
    "    final_gb_quantile = GradientBoostingRegressor(\n",
    "        loss='quantile', alpha=quantile, **best_params[quantile][-1])\n",
    "    final_gb_quantile.fit(X, y, sample_weight=sample_weights)\n",
    "    print(f'{quantile} Quantile Final Pinball: {pinball}')\n",
    "    final_models[quantile] = final_gb_quantile\n",
    "\n",
    "# generate target dataset\n",
    "predict_gb = dict()\n",
    "for quantile, model in final_models.items():\n",
    "    predict_gb[quantile] = model.predict(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dea6b624-f2eb-4a39-944f-212ede44ebbc",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "#### XGBoost Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7ecc3732-f6c9-4d25-87ed-8b31b1f01713",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### XGBoost 2-Quantile 0.01 ######\n",
      "Split 1 MSE: 1.2311076090509598e-06\n",
      "Split 2 MSE: 1.4220845725933297e-09\n",
      "Split 3 MSE: 2.133823595126439e-09\n",
      "Split 4 MSE: 8.189131011101105e-08\n",
      "Split 5 MSE: 3.6296829162357874e-07\n",
      "Quantile 0.01 Final MSE: 4.1795049736772594e-10\n",
      "###### XGBoost 2-Quantile 0.025 ######\n",
      "Split 1 MSE: 1.8113438297072164e-06\n",
      "Split 2 MSE: 4.253891751419835e-08\n",
      "Split 3 MSE: 7.4087652114714755e-09\n",
      "Split 4 MSE: 1.1007960470231408e-07\n",
      "Split 5 MSE: 2.838393889900539e-08\n",
      "Quantile 0.025 Final MSE: 1.8450699658662916e-08\n",
      "###### XGBoost 2-Quantile 0.05 ######\n",
      "Split 1 MSE: 1.0684456037104718e-07\n",
      "Split 2 MSE: 4.707845043784861e-09\n",
      "Split 3 MSE: 1.3322735505801273e-09\n",
      "Split 4 MSE: 4.495082448522923e-08\n",
      "Split 5 MSE: 1.8001521006539026e-08\n",
      "Quantile 0.05 Final MSE: 7.252422511738843e-11\n",
      "###### XGBoost 2-Quantile 0.95 ######\n",
      "Split 1 MSE: 2.204245096612315e-07\n",
      "Split 2 MSE: 4.066226356678968e-09\n",
      "Split 3 MSE: 6.327930314627804e-08\n",
      "Split 4 MSE: 3.423943381369469e-08\n",
      "Split 5 MSE: 2.2832181128347086e-09\n",
      "Quantile 0.95 Final MSE: 8.726890489478189e-10\n",
      "###### XGBoost 2-Quantile 0.975 ######\n",
      "Split 1 MSE: 3.581327374483766e-08\n",
      "Split 2 MSE: 7.857329505084958e-08\n",
      "Split 3 MSE: 4.3892263381579913e-08\n",
      "Split 4 MSE: 1.708708837839894e-08\n",
      "Split 5 MSE: 4.1461774825315956e-09\n",
      "Quantile 0.975 Final MSE: 1.2552322901717314e-10\n",
      "###### XGBoost 2-Quantile 0.99 ######\n",
      "Split 1 MSE: 2.0059071769516698e-08\n",
      "Split 2 MSE: 3.955158164890513e-08\n",
      "Split 3 MSE: 7.957725591148133e-09\n",
      "Split 4 MSE: 1.9028550477738983e-08\n",
      "Split 5 MSE: 4.248255690431038e-09\n",
      "Quantile 0.99 Final MSE: 2.0331812091356983e-10\n"
     ]
    }
   ],
   "source": [
    "xgb2_models = dict()\n",
    "for quantile in quantiles:\n",
    "    print(f'###### XGBoost 2-Quantile {quantile} ######')\n",
    "    X = df_spread_is_x\n",
    "    y = pd.Series(predict_gb[quantile],\n",
    "                  index=df_spread_is_x.index)\n",
    "\n",
    "    # more sample weight toward recent data\n",
    "    sample_weights = np.arange(1,len(y)+1)\n",
    "\n",
    "    param_grid = {\n",
    "        'n_estimators': [50, 100, 200],\n",
    "        'max_depth': [3, 5, 7],\n",
    "        'learning_rate': [0.01, 0.1, 0.2],\n",
    "        'subsample': [0.8, 0.9, 1.0]\n",
    "    }\n",
    "\n",
    "    tscv = TimeSeriesSplit(n_splits=5)\n",
    "\n",
    "    best_params = []\n",
    "    best_scores = []\n",
    "\n",
    "    # walk forward cv\n",
    "    split = 0\n",
    "    for train_index, test_index in tscv.split(X):\n",
    "        # train test split\n",
    "        split += 1\n",
    "        X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n",
    "        y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n",
    "        weights_train = sample_weights[train_index]\n",
    "        weights_test = sample_weights[test_index]\n",
    "\n",
    "        # xgboost regression\n",
    "        xgb_reg = xgb.XGBRegressor()\n",
    "        grid_search = GridSearchCV(xgb_reg,\n",
    "                                   param_grid,\n",
    "                                   cv=TimeSeriesSplit(n_splits=3),\n",
    "                                   n_jobs=-1)\n",
    "        grid_search.fit(X_train, y_train,\n",
    "                        sample_weight=weights_train)\n",
    "\n",
    "        # hyper-parameter tuning\n",
    "        best_params.append(grid_search.best_params_)\n",
    "        best_xgb_reg = grid_search.best_estimator_\n",
    "        y_pred = best_xgb_reg.predict(X_test)\n",
    "        mse = mean_squared_error(y_test, y_pred)\n",
    "        best_scores.append(mse)\n",
    "        print(f'Split {split} MSE: {mse}')\n",
    "\n",
    "    # final xgboost model\n",
    "    mean_score = np.mean(best_scores)\n",
    "    final_xgb_reg = xgb.XGBRegressor(**best_params[-1])\n",
    "    final_xgb_reg.fit(X, y, sample_weight=sample_weights)\n",
    "    y_pred = final_xgb_reg.predict(X)\n",
    "    mse = mean_squared_error(y, y_pred)\n",
    "    print(f'Quantile {quantile} Final MSE: {mse}')\n",
    "    \n",
    "    # append final model\n",
    "    xgb2_models[quantile] = final_xgb_reg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8c83321e-89f0-4fcf-abb1-bc2aae74ead6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['xgb2_models.pkl']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(xgb2_models, 'xgb2_models.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f953d54-dd4c-46b8-a896-64778538df0a",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### LightGBM(1): QR-IM Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2208eb9e-7c84-4c4c-abf5-8bbf8cfe28d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightgbm as lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "436b57c2-6b11-47ad-81d9-df7f69f25b36",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### LightGBM 1-Quantile 0.01 ######\n",
      "Split 1 MSE: 1.790465551848179e-06\n",
      "Split 2 MSE: 9.627052156662542e-07\n",
      "Split 3 MSE: 3.597757162072677e-07\n",
      "Split 4 MSE: 8.675631925287862e-07\n",
      "Split 5 MSE: 1.296956982940743e-06\n",
      "Quantile 0.01 Final MSE: 2.6685382721792005e-08\n",
      "###### LightGBM 1-Quantile 0.025 ######\n",
      "Split 1 MSE: 1.264882471472374e-06\n",
      "Split 2 MSE: 5.098047351216236e-07\n",
      "Split 3 MSE: 2.2073421165387137e-07\n",
      "Split 4 MSE: 5.021876463981952e-07\n",
      "Split 5 MSE: 5.843591095688133e-07\n",
      "Quantile 0.025 Final MSE: 9.975297604435664e-09\n",
      "###### LightGBM 1-Quantile 0.05 ######\n",
      "Split 1 MSE: 8.716430045283189e-07\n",
      "Split 2 MSE: 2.5328299198276267e-07\n",
      "Split 3 MSE: 1.153239065355161e-07\n",
      "Split 4 MSE: 1.8748413253295058e-07\n",
      "Split 5 MSE: 2.2849973482750662e-07\n",
      "Quantile 0.05 Final MSE: 5.287941503344655e-09\n",
      "###### LightGBM 1-Quantile 0.95 ######\n",
      "Split 1 MSE: 8.911824249584444e-07\n",
      "Split 2 MSE: 1.17292202350402e-07\n",
      "Split 3 MSE: 9.270400134322852e-08\n",
      "Split 4 MSE: 3.003311297913809e-08\n",
      "Split 5 MSE: 1.8205264568242003e-08\n",
      "Quantile 0.95 Final MSE: 2.672225701071331e-09\n",
      "###### LightGBM 1-Quantile 0.975 ######\n",
      "Split 1 MSE: 1.7355223073334503e-06\n",
      "Split 2 MSE: 1.521519624203991e-07\n",
      "Split 3 MSE: 1.1025592402505453e-07\n",
      "Split 4 MSE: 1.418771571315025e-07\n",
      "Split 5 MSE: 9.886803514176807e-08\n",
      "Quantile 0.975 Final MSE: 4.129287038786904e-09\n",
      "###### LightGBM 1-Quantile 0.99 ######\n",
      "Split 1 MSE: 2.0153922641895386e-06\n",
      "Split 2 MSE: 2.5750954947334386e-07\n",
      "Split 3 MSE: 2.2262842331618967e-07\n",
      "Split 4 MSE: 8.627533273644633e-08\n",
      "Split 5 MSE: 5.841023624369944e-08\n",
      "Quantile 0.99 Final MSE: 8.833962011093853e-09\n"
     ]
    }
   ],
   "source": [
    "lgb1_models = dict()\n",
    "for quantile in quantiles:\n",
    "    print(f'###### LightGBM 1-Quantile {quantile} ######')\n",
    "    X = df_spread_is_x\n",
    "    y = predict_qr[quantile]\n",
    "\n",
    "    # more sample weight toward recent data\n",
    "    sample_weights = np.arange(1,len(y)+1)\n",
    "\n",
    "    param_grid = {\n",
    "        'n_estimators': [50, 100, 200],\n",
    "        'max_depth': [3, 5, 7],\n",
    "        'learning_rate': [0.01, 0.1, 0.2],\n",
    "        'subsample': [0.8, 0.9, 1.0],\n",
    "        'verbose': [-1]\n",
    "    }\n",
    "\n",
    "    tscv = TimeSeriesSplit(n_splits=5)\n",
    "\n",
    "    best_params = []\n",
    "    best_scores = []\n",
    "\n",
    "    # walk forward cv\n",
    "    split = 0\n",
    "    for train_index, test_index in tscv.split(X):\n",
    "        # train test split\n",
    "        split += 1\n",
    "        X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n",
    "        y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n",
    "        weights_train = sample_weights[train_index]\n",
    "        weights_test = sample_weights[test_index]\n",
    "\n",
    "        # light gbm regression\n",
    "        lgb_reg = lgb.LGBMRegressor()\n",
    "        grid_search = GridSearchCV(lgb_reg,\n",
    "                                   param_grid,\n",
    "                                   cv=TimeSeriesSplit(n_splits=3),\n",
    "                                   n_jobs=-1)\n",
    "        grid_search.fit(X_train, y_train,\n",
    "                        sample_weight=weights_train)\n",
    "\n",
    "        # hyper-parameter tuning\n",
    "        best_params.append(grid_search.best_params_)\n",
    "        best_lgb_reg = grid_search.best_estimator_\n",
    "        y_pred = best_lgb_reg.predict(X_test)\n",
    "        mse = mean_squared_error(y_test, y_pred)\n",
    "        best_scores.append(mse)\n",
    "        print(f'Split {split} MSE: {mse}')\n",
    "\n",
    "    # final light gbm model\n",
    "    mean_score = np.mean(best_scores)\n",
    "    final_lgb_reg = lgb.LGBMRegressor(**best_params[-1])\n",
    "    final_lgb_reg.fit(X, y, sample_weight=sample_weights)\n",
    "    y_pred = final_lgb_reg.predict(X)\n",
    "    mse = mean_squared_error(y, y_pred)\n",
    "    print(f'Quantile {quantile} Final MSE: {mse}')\n",
    "    \n",
    "    # append final model\n",
    "    lgb1_models[quantile] = final_lgb_reg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "06b178a6-e5d7-4cd9-99c3-a7589ece14ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['lgb1_models.pkl']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(lgb1_models, 'lgb1_models.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "922f0ebd-f465-496e-bcf6-8a3e6f780547",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### LightGBM(2): QR-IM Dataset with Interaction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "9515ed53-7ea4-4c3b-b7af-4cc0930bbb7e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "###### LightGBM 2-Quantile 0.01 ######\n",
      "Split 1 MSE: 1.1085141491131679e-05\n",
      "Split 2 MSE: 7.348411305157841e-06\n",
      "Split 3 MSE: 1.0134850772903324e-05\n",
      "Split 4 MSE: 1.653311672308595e-05\n",
      "Split 5 MSE: 8.175006581755998e-06\n",
      "Quantile 0.01 Final MSE: 5.029330781105214e-07\n",
      "###### LightGBM 2-Quantile 0.025 ######\n",
      "Split 1 MSE: 7.020032761227292e-06\n",
      "Split 2 MSE: 7.726545378044094e-06\n",
      "Split 3 MSE: 6.197562497218393e-06\n",
      "Split 4 MSE: 2.286527481739401e-05\n",
      "Split 5 MSE: 9.24679587853899e-06\n",
      "Quantile 0.025 Final MSE: 1.6247147942826845e-06\n",
      "###### LightGBM 2-Quantile 0.05 ######\n",
      "Split 1 MSE: 3.278468857751274e-06\n",
      "Split 2 MSE: 3.0390651476461033e-06\n",
      "Split 3 MSE: 3.3969692112810778e-06\n",
      "Split 4 MSE: 5.3624084201086635e-06\n",
      "Split 5 MSE: 2.7638088528592243e-06\n",
      "Quantile 0.05 Final MSE: 2.8768163274020064e-07\n",
      "###### LightGBM 2-Quantile 0.95 ######\n",
      "Split 1 MSE: 4.346665866513345e-06\n",
      "Split 2 MSE: 2.3928797244780634e-06\n",
      "Split 3 MSE: 1.0850504979028253e-05\n",
      "Split 4 MSE: 8.283108506942155e-06\n",
      "Split 5 MSE: 6.843265742394865e-06\n",
      "Quantile 0.95 Final MSE: 3.66297752666823e-06\n",
      "###### LightGBM 2-Quantile 0.975 ######\n",
      "Split 1 MSE: 4.261607414968728e-06\n",
      "Split 2 MSE: 3.046241257575433e-06\n",
      "Split 3 MSE: 1.7004202809348148e-05\n",
      "Split 4 MSE: 7.368769291236649e-06\n",
      "Split 5 MSE: 9.005972246796683e-06\n",
      "Quantile 0.975 Final MSE: 4.894007298622431e-06\n",
      "###### LightGBM 2-Quantile 0.99 ######\n",
      "Split 1 MSE: 4.69877424164264e-06\n",
      "Split 2 MSE: 3.351577143729858e-06\n",
      "Split 3 MSE: 1.9133450675481774e-05\n",
      "Split 4 MSE: 2.0401395760250767e-05\n",
      "Split 5 MSE: 1.3147960321798619e-05\n",
      "Quantile 0.99 Final MSE: 7.693101689294037e-06\n"
     ]
    }
   ],
   "source": [
    "lgb2_models = dict()\n",
    "for quantile in quantiles:\n",
    "    print(f'###### LightGBM 2-Quantile {quantile} ######')\n",
    "    y = predict_qr2[quantile]\n",
    "    X = df_spread_is_x[df_spread_is_x.index>=y.index[0]]\n",
    "\n",
    "    # more sample weight toward recent data\n",
    "    sample_weights = np.arange(1,len(y)+1)\n",
    "\n",
    "    param_grid = {\n",
    "        'n_estimators': [50, 100, 200],\n",
    "        'max_depth': [3, 5, 7],\n",
    "        'learning_rate': [0.01, 0.1, 0.2],\n",
    "        'subsample': [0.8, 0.9, 1.0],\n",
    "        'verbose': [-1]\n",
    "    }\n",
    "\n",
    "    tscv = TimeSeriesSplit(n_splits=5)\n",
    "\n",
    "    best_params = []\n",
    "    best_scores = []\n",
    "\n",
    "    # walk forward cv\n",
    "    split = 0\n",
    "    for train_index, test_index in tscv.split(X):\n",
    "        # train test split\n",
    "        split += 1\n",
    "        X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n",
    "        y_train, y_test = y.iloc[train_index], y.iloc[test_index]\n",
    "        weights_train = sample_weights[train_index]\n",
    "        weights_test = sample_weights[test_index]\n",
    "\n",
    "        # light gbm regression\n",
    "        lgb_reg = lgb.LGBMRegressor()\n",
    "        grid_search = GridSearchCV(lgb_reg,\n",
    "                                   param_grid,\n",
    "                                   cv=TimeSeriesSplit(n_splits=3),\n",
    "                                   n_jobs=-1)\n",
    "        grid_search.fit(X_train, y_train,\n",
    "                        sample_weight=weights_train)\n",
    "\n",
    "        # hyper-parameter tuning\n",
    "        best_params.append(grid_search.best_params_)\n",
    "        best_lgb_reg = grid_search.best_estimator_\n",
    "        y_pred = best_lgb_reg.predict(X_test)\n",
    "        mse = mean_squared_error(y_test, y_pred)\n",
    "        best_scores.append(mse)\n",
    "        print(f'Split {split} MSE: {mse}')\n",
    "\n",
    "    # final light gbm model\n",
    "    mean_score = np.mean(best_scores)\n",
    "    final_lgb_reg = lgb.LGBMRegressor(**best_params[-1])\n",
    "    final_lgb_reg.fit(X, y, sample_weight=sample_weights)\n",
    "    y_pred = final_lgb_reg.predict(X)\n",
    "    mse = mean_squared_error(y, y_pred)\n",
    "    print(f'Quantile {quantile} Final MSE: {mse}')\n",
    "    \n",
    "    # append final model\n",
    "    lgb2_models[quantile] = final_lgb_reg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "72e033f4-4b7d-43e8-8206-bfb896fe659d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['lgb2_models.pkl']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(lgb2_models, 'lgb2_models.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46f23659-1ade-40a3-bee0-10b9ba860d1f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}