GNN_refactor_schnet_cat3Head.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import torch\n",
    "from torch_geometric.data import Data,DataLoader\n",
    "from functions_refactor import *\n",
    "from pytorch_util import *\n",
    "from torch.optim import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# model parameters\n",
    "reuse = False\n",
    "block = schnet_block\n",
    "head = cat3Head\n",
    "data = '../Data/{}_data_ACSF_expand.pickle'\n",
    "batch_size = 32\n",
    "dim = 128\n",
    "epochs = 20\n",
    "clip = 2\n",
    "layer1 = 2\n",
    "layer2 = 2\n",
    "factor = 2\n",
    "lr = 1e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dl,val_dl = get_data(data,batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model = GNN(reuse,block,head,dim,layer1,layer2,factor,**data_dict[data]).to('cuda:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "paras = trainable_parameter(model)\n",
    "opt = Adam(paras,lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:0, train_loss: +0.815, val_loss: +0.211, \n",
      "train_vector: +2.90|+1.43|+0.52|-0.01|+0.14|+0.83|+0.65|+0.06, \n",
      "val_vector  : +1.35|+0.72|+0.20|-0.67|-0.48|+0.69|+0.38|-0.50\n",
      "\n",
      "epoch:1, train_loss: +0.061, val_loss: +0.032, \n",
      "train_vector: +0.98|+0.45|+0.07|-0.73|-0.60|+0.62|+0.33|-0.61, \n",
      "val_vector  : +1.03|+0.31|+0.05|-0.59|-0.79|+0.66|+0.28|-0.70\n",
      "\n",
      "epoch:2, train_loss: -0.124, val_loss: -0.141, \n",
      "train_vector: +0.71|+0.18|-0.09|-0.96|-0.82|+0.55|+0.24|-0.79, \n",
      "val_vector  : +0.65|+0.28|-0.17|-0.58|-1.08|+0.54|+0.16|-0.92\n",
      "\n",
      "epoch:3, train_loss: -0.230, val_loss: -0.261, \n",
      "train_vector: +0.59|+0.04|-0.19|-1.10|-0.95|+0.51|+0.17|-0.91, \n",
      "val_vector  : +0.50|+0.35|-0.27|-1.23|-1.05|+0.49|+0.11|-0.99\n",
      "\n",
      "epoch:4, train_loss: -0.305, val_loss: -0.385, \n",
      "train_vector: +0.49|-0.07|-0.25|-1.18|-1.05|+0.48|+0.13|-0.99, \n",
      "val_vector  : +0.38|-0.27|-0.31|-1.16|-1.22|+0.48|+0.07|-1.05\n",
      "\n",
      "epoch:5, train_loss: -0.358, val_loss: -0.424, \n",
      "train_vector: +0.44|-0.12|-0.30|-1.26|-1.12|+0.46|+0.09|-1.05, \n",
      "val_vector  : +0.48|-0.24|-0.34|-1.45|-1.27|+0.45|+0.03|-1.07\n",
      "\n",
      "epoch:6, train_loss: -0.407, val_loss: -0.356, \n",
      "train_vector: +0.39|-0.17|-0.35|-1.31|-1.19|+0.43|+0.06|-1.11, \n",
      "val_vector  : +0.77|+0.14|-0.39|-1.36|-1.28|+0.43|+0.01|-1.16\n",
      "\n",
      "epoch:7, train_loss: -0.446, val_loss: -0.522, \n",
      "train_vector: +0.35|-0.23|-0.38|-1.36|-1.23|+0.41|+0.03|-1.16, \n",
      "val_vector  : +0.53|-0.62|-0.45|-1.55|-1.23|+0.39|-0.04|-1.22\n",
      "\n",
      "epoch:8, train_loss: -0.484, val_loss: -0.501, \n",
      "train_vector: +0.32|-0.30|-0.41|-1.39|-1.28|+0.39|+0.00|-1.20, \n",
      "val_vector  : +0.70|-0.58|-0.42|-1.55|-1.23|+0.38|-0.05|-1.25\n",
      "\n",
      "epoch:9, train_loss: -0.513, val_loss: -0.579, \n",
      "train_vector: +0.29|-0.31|-0.43|-1.43|-1.31|+0.37|-0.04|-1.24, \n",
      "val_vector  : +0.30|-0.56|-0.49|-1.45|-1.38|+0.35|-0.12|-1.29\n",
      "\n",
      "epoch:10, train_loss: -0.543, val_loss: -0.607, \n",
      "train_vector: +0.27|-0.36|-0.46|-1.46|-1.34|+0.36|-0.09|-1.27, \n",
      "val_vector  : +0.48|-0.66|-0.49|-1.59|-1.45|+0.33|-0.17|-1.30\n",
      "\n",
      "epoch:11, train_loss: -0.575, val_loss: -0.456, \n",
      "train_vector: +0.23|-0.40|-0.48|-1.49|-1.38|+0.34|-0.13|-1.30, \n",
      "val_vector  : +0.27|+0.58|-0.50|-1.44|-1.39|+0.34|-0.20|-1.32\n",
      "\n",
      "epoch:12, train_loss: -0.597, val_loss: -0.553, \n",
      "train_vector: +0.21|-0.41|-0.50|-1.52|-1.41|+0.32|-0.15|-1.33, \n",
      "val_vector  : +0.24|-0.07|-0.54|-1.48|-1.42|+0.31|-0.22|-1.25\n",
      "\n",
      "epoch:13, train_loss: -0.617, val_loss: -0.652, \n",
      "train_vector: +0.20|-0.42|-0.52|-1.54|-1.43|+0.30|-0.18|-1.35, \n",
      "val_vector  : +0.52|-0.75|-0.57|-1.66|-1.52|+0.28|-0.24|-1.29\n",
      "\n",
      "epoch:14, train_loss: -0.640, val_loss: -0.626, \n",
      "train_vector: +0.17|-0.46|-0.53|-1.55|-1.46|+0.29|-0.20|-1.37, \n",
      "val_vector  : +0.50|-0.42|-0.58|-1.54|-1.54|+0.28|-0.27|-1.44\n",
      "\n",
      "epoch:15, train_loss: -0.661, val_loss: -0.721, \n",
      "train_vector: +0.16|-0.51|-0.55|-1.58|-1.47|+0.28|-0.22|-1.39, \n",
      "val_vector  : +0.15|-0.57|-0.59|-1.72|-1.55|+0.26|-0.29|-1.45\n",
      "\n",
      "epoch:16, train_loss: -0.678, val_loss: -0.729, \n",
      "train_vector: +0.14|-0.51|-0.56|-1.59|-1.50|+0.26|-0.25|-1.42, \n",
      "val_vector  : +0.12|-0.52|-0.61|-1.74|-1.60|+0.24|-0.33|-1.38\n",
      "\n",
      "epoch:17, train_loss: -0.697, val_loss: -0.755, \n",
      "train_vector: +0.13|-0.52|-0.58|-1.63|-1.52|+0.25|-0.28|-1.44, \n",
      "val_vector  : +0.38|-0.81|-0.63|-1.73|-1.61|+0.23|-0.37|-1.50\n",
      "\n",
      "epoch:18, train_loss: -0.712, val_loss: -0.720, \n",
      "train_vector: +0.12|-0.51|-0.59|-1.63|-1.54|+0.24|-0.32|-1.46, \n",
      "val_vector  : +0.12|-0.51|-0.61|-1.63|-1.52|+0.23|-0.38|-1.47\n",
      "\n",
      "epoch:19, train_loss: -0.732, val_loss: -0.728, \n",
      "train_vector: +0.10|-0.55|-0.60|-1.65|-1.56|+0.23|-0.37|-1.47, \n",
      "val_vector  : +0.26|-0.59|-0.63|-1.47|-1.67|+0.21|-0.44|-1.51\n",
      "\n",
      "Training completed in 748.5707676410675s\n"
     ]
    }
   ],
   "source": [
    "model,train_loss_perType,val_loss_perType = train(opt,model,epochs,train_dl,val_dl,paras,clip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(train_loss_perType,val_loss_perType,reuse,block,\\\n",
    "             head,data,batch_size,dim,clip,layer1,layer2,factor,epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_model(model,opt,reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### train on each type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = GNN(reuse,block,head,dim,layer1,layer2,factor,**data_dict[data]).to('cuda:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_type =1e-4/4\n",
    "epochs_type = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "paras = trainable_parameter(model)\n",
    "opt = Adam(paras,lr=lr_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name = !ls ../Model/\n",
    "file_name = file_name[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training type: 0\n",
      "epoch:0, train_loss: +0.069, val_loss: -0.064, \n",
      "train_vector: +0.07|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.06|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.026, val_loss: +0.310, \n",
      "train_vector: -0.03|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.31|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.072, val_loss: -0.131, \n",
      "train_vector: -0.07|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.13|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.105, val_loss: +0.163, \n",
      "train_vector: -0.11|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.16|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.132, val_loss: +0.020, \n",
      "train_vector: -0.13|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.02|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.155, val_loss: +0.028, \n",
      "train_vector: -0.16|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.03|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.175, val_loss: +0.066, \n",
      "train_vector: -0.18|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.07|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.192, val_loss: -0.094, \n",
      "train_vector: -0.19|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.09|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.218, val_loss: -0.088, \n",
      "train_vector: -0.22|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.09|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.231, val_loss: -0.252, \n",
      "train_vector: -0.23|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.25|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 380.4856460094452s\n",
      "\n",
      "Training type: 1\n",
      "epoch:0, train_loss: -0.521, val_loss: -0.726, \n",
      "train_vector: +0.00|-0.52|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.73|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.787, val_loss: -0.853, \n",
      "train_vector: +0.00|-0.79|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.85|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.847, val_loss: -0.755, \n",
      "train_vector: +0.00|-0.85|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.76|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.911, val_loss: -0.973, \n",
      "train_vector: +0.00|-0.91|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.97|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.932, val_loss: -0.683, \n",
      "train_vector: +0.00|-0.93|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.68|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.902, val_loss: -0.538, \n",
      "train_vector: +0.00|-0.90|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.54|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.982, val_loss: -0.792, \n",
      "train_vector: +0.00|-0.98|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.79|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.023, val_loss: -0.733, \n",
      "train_vector: +0.00|-1.02|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.73|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.036, val_loss: -1.089, \n",
      "train_vector: +0.00|-1.04|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.09|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.057, val_loss: -0.941, \n",
      "train_vector: +0.00|-1.06|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.94|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 128.70867013931274s\n",
      "\n",
      "Training type: 2\n",
      "epoch:0, train_loss: -0.659, val_loss: -0.741, \n",
      "train_vector: +0.00|+0.00|-0.66|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.74|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.749, val_loss: -0.806, \n",
      "train_vector: +0.00|+0.00|-0.75|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.81|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.811, val_loss: -0.887, \n",
      "train_vector: +0.00|+0.00|-0.81|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.89|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.857, val_loss: -0.928, \n",
      "train_vector: +0.00|+0.00|-0.86|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.93|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.898, val_loss: -0.911, \n",
      "train_vector: +0.00|+0.00|-0.90|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.91|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.933, val_loss: -0.959, \n",
      "train_vector: +0.00|+0.00|-0.93|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.96|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.964, val_loss: -1.018, \n",
      "train_vector: +0.00|+0.00|-0.96|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.02|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.990, val_loss: -1.027, \n",
      "train_vector: +0.00|+0.00|-0.99|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.03|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.015, val_loss: -0.996, \n",
      "train_vector: +0.00|+0.00|-1.02|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.036, val_loss: -1.097, \n",
      "train_vector: +0.00|+0.00|-1.04|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.10|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 390.7286388874054s\n",
      "\n",
      "Training type: 3\n",
      "epoch:0, train_loss: -1.660, val_loss: -1.816, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.66|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.82|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.762, val_loss: -1.855, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.76|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.86|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.808, val_loss: -1.805, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.81|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.81|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.848, val_loss: -1.540, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.85|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.54|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.872, val_loss: -2.007, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.87|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.01|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.900, val_loss: -1.911, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.90|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.91|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.924, val_loss: -1.902, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.92|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.90|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.949, val_loss: -1.720, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.95|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.72|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.964, val_loss: -1.897, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.96|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.90|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.979, val_loss: -1.659, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.98|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.66|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 382.73200273513794s\n",
      "\n",
      "Training type: 4\n",
      "epoch:0, train_loss: -1.537, val_loss: -1.694, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.54|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.69|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.684, val_loss: -1.733, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.68|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.73|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.731, val_loss: -1.710, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.73|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.71|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.773, val_loss: -1.846, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.77|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.85|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.809, val_loss: -1.832, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.81|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.83|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.828, val_loss: -1.912, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.83|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.91|+0.00|+0.00|+0.00\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:6, train_loss: -1.854, val_loss: -1.846, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.85|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.85|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.874, val_loss: -1.881, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.87|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.88|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.894, val_loss: -1.953, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.89|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.95|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.909, val_loss: -1.859, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.91|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.86|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 189.78208541870117s\n",
      "\n",
      "Training type: 5\n",
      "epoch:0, train_loss: +0.124, val_loss: +0.023, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.12|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.02|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.016, val_loss: -0.092, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.02|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.09|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.113, val_loss: -0.180, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.11|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.18|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.184, val_loss: -0.195, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.18|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.20|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.240, val_loss: -0.280, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.24|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.28|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.286, val_loss: -0.283, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.29|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.28|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.327, val_loss: -0.361, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.33|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.36|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.361, val_loss: -0.406, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.36|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.41|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.395, val_loss: -0.416, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.40|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.42|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.425, val_loss: -0.460, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.43|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.46|+0.00|+0.00\n",
      "\n",
      "Training completed in 395.52741169929504s\n",
      "\n",
      "Training type: 6\n",
      "epoch:0, train_loss: -0.495, val_loss: -0.631, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.49|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.63|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.639, val_loss: -0.688, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.64|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.69|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.715, val_loss: -0.753, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.72|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.75|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.772, val_loss: -0.804, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.77|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.80|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.819, val_loss: -0.809, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.82|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.81|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.857, val_loss: -0.869, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.86|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.87|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.888, val_loss: -0.944, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.89|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.94|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.916, val_loss: -0.930, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.92|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.93|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.951, val_loss: -0.936, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.95|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.94|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.977, val_loss: -0.953, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.98|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.95|+0.00\n",
      "\n",
      "Training completed in 356.16362047195435s\n",
      "\n",
      "Training type: 7\n",
      "epoch:0, train_loss: -1.462, val_loss: -1.531, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.46, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.53\n",
      "\n",
      "epoch:1, train_loss: -1.567, val_loss: -1.506, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.57, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.51\n",
      "\n",
      "epoch:2, train_loss: -1.622, val_loss: -1.601, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.62, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.60\n",
      "\n",
      "epoch:3, train_loss: -1.666, val_loss: -1.638, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.67, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.64\n",
      "\n",
      "epoch:4, train_loss: -1.699, val_loss: -1.654, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.70, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.65\n",
      "\n",
      "epoch:5, train_loss: -1.729, val_loss: -1.693, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.73, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.69\n",
      "\n",
      "epoch:6, train_loss: -1.759, val_loss: -1.663, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.76, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.66\n",
      "\n",
      "epoch:7, train_loss: -1.785, val_loss: -1.724, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.79, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.72\n",
      "\n",
      "epoch:8, train_loss: -1.810, val_loss: -1.729, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.81, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.73\n",
      "\n",
      "epoch:9, train_loss: -1.825, val_loss: -1.764, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.82, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.76\n",
      "\n",
      "Training completed in 213.36850094795227s\n"
     ]
    }
   ],
   "source": [
    "for i in range(8):\n",
    "    data_type = data.split('.pickle')[0]+ '_type_'+str(i)+'.pickle'\n",
    "    train_dl,val_dl = get_data(data_type,batch_size)\n",
    "\n",
    "    checkpoint = torch.load('../Model/'+file_name)\n",
    "    model.load_state_dict(checkpoint['model_state_dict'])\n",
    "    opt.load_state_dict(checkpoint['optimizer_state_dict'])\n",
    "\n",
    "    for para in opt.param_groups:\n",
    "        para['lr'] = lr_type\n",
    "    \n",
    "    print('\\nTraining type: {}'.format(i))\n",
    "    model,train_loss_perType,val_loss_perType = train(opt,model,epochs_type,train_dl,val_dl,paras,clip,True)\n",
    "\n",
    "    save_results(train_loss_perType,val_loss_perType,reuse,block,\\\n",
    "                 head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'type_'+str(i))\n",
    "\n",
    "    save_model(model,opt,reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'type_'+str(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train on each type from start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "head = cat3Head_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_type =1e-4\n",
    "epochs_type = 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training type: 0\n",
      "epoch:0, train_loss: +0.955, val_loss: +0.476, \n",
      "train_vector: +0.95|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.48|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: +0.467, val_loss: +0.587, \n",
      "train_vector: +0.47|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.59|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: +0.325, val_loss: +0.178, \n",
      "train_vector: +0.32|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.18|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: +0.212, val_loss: +0.232, \n",
      "train_vector: +0.21|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.23|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: +0.143, val_loss: +0.036, \n",
      "train_vector: +0.14|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.04|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: +0.095, val_loss: -0.003, \n",
      "train_vector: +0.10|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: +0.048, val_loss: +0.032, \n",
      "train_vector: +0.05|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.03|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.003, val_loss: -0.089, \n",
      "train_vector: -0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.09|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.029, val_loss: +0.280, \n",
      "train_vector: -0.03|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.28|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.048, val_loss: +0.131, \n",
      "train_vector: -0.05|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.13|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -0.081, val_loss: -0.228, \n",
      "train_vector: -0.08|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.23|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -0.108, val_loss: -0.045, \n",
      "train_vector: -0.11|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.05|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -0.125, val_loss: +0.054, \n",
      "train_vector: -0.13|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.05|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -0.165, val_loss: +0.033, \n",
      "train_vector: -0.17|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.03|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -0.172, val_loss: -0.138, \n",
      "train_vector: -0.17|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.14|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -0.203, val_loss: +0.063, \n",
      "train_vector: -0.20|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.06|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -0.210, val_loss: -0.332, \n",
      "train_vector: -0.21|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.33|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -0.229, val_loss: -0.239, \n",
      "train_vector: -0.23|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.24|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -0.235, val_loss: -0.321, \n",
      "train_vector: -0.24|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.32|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -0.278, val_loss: -0.359, \n",
      "train_vector: -0.28|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.36|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -0.276, val_loss: -0.172, \n",
      "train_vector: -0.28|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.17|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -0.284, val_loss: -0.120, \n",
      "train_vector: -0.28|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.12|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -0.302, val_loss: -0.353, \n",
      "train_vector: -0.30|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.35|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -0.319, val_loss: -0.401, \n",
      "train_vector: -0.32|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.40|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -0.320, val_loss: -0.387, \n",
      "train_vector: -0.32|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.39|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 963.8991708755493s\n",
      "\n",
      "Training type: 1\n",
      "epoch:0, train_loss: +1.415, val_loss: +1.562, \n",
      "train_vector: +0.00|+1.42|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+1.56|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: +0.561, val_loss: +0.642, \n",
      "train_vector: +0.00|+0.56|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.64|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: +0.191, val_loss: -0.016, \n",
      "train_vector: +0.00|+0.19|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.02|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: +0.068, val_loss: +0.247, \n",
      "train_vector: +0.00|+0.07|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.25|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.033, val_loss: +1.369, \n",
      "train_vector: +0.00|-0.03|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+1.37|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.196, val_loss: -0.340, \n",
      "train_vector: +0.00|-0.20|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.34|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.233, val_loss: -0.387, \n",
      "train_vector: +0.00|-0.23|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.39|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.192, val_loss: -0.133, \n",
      "train_vector: +0.00|-0.19|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.13|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.364, val_loss: -0.212, \n",
      "train_vector: +0.00|-0.36|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.21|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.372, val_loss: -0.543, \n",
      "train_vector: +0.00|-0.37|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.54|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -0.374, val_loss: +0.064, \n",
      "train_vector: +0.00|-0.37|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.06|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -0.399, val_loss: -0.703, \n",
      "train_vector: +0.00|-0.40|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.70|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -0.554, val_loss: +0.342, \n",
      "train_vector: +0.00|-0.55|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.34|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -0.512, val_loss: -0.643, \n",
      "train_vector: +0.00|-0.51|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.64|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -0.556, val_loss: -0.272, \n",
      "train_vector: +0.00|-0.56|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.27|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -0.534, val_loss: -0.585, \n",
      "train_vector: +0.00|-0.53|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.58|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -0.578, val_loss: -0.931, \n",
      "train_vector: +0.00|-0.58|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.93|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -0.530, val_loss: -0.943, \n",
      "train_vector: +0.00|-0.53|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.94|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -0.605, val_loss: -0.765, \n",
      "train_vector: +0.00|-0.61|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.77|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -0.671, val_loss: -0.921, \n",
      "train_vector: +0.00|-0.67|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.92|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -0.595, val_loss: -0.797, \n",
      "train_vector: +0.00|-0.60|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.80|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -0.647, val_loss: -0.210, \n",
      "train_vector: +0.00|-0.65|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.21|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:22, train_loss: -0.674, val_loss: -0.502, \n",
      "train_vector: +0.00|-0.67|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.50|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -0.650, val_loss: -0.788, \n",
      "train_vector: +0.00|-0.65|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.79|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -0.708, val_loss: -0.561, \n",
      "train_vector: +0.00|-0.71|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.56|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 333.60658383369446s\n",
      "\n",
      "Training type: 2\n",
      "epoch:0, train_loss: -0.235, val_loss: -0.207, \n",
      "train_vector: +0.00|+0.00|-0.24|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.21|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.680, val_loss: -0.749, \n",
      "train_vector: +0.00|+0.00|-0.68|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.75|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.843, val_loss: -0.908, \n",
      "train_vector: +0.00|+0.00|-0.84|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.91|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.943, val_loss: -0.990, \n",
      "train_vector: +0.00|+0.00|-0.94|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.99|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.020, val_loss: -1.060, \n",
      "train_vector: +0.00|+0.00|-1.02|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.06|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.081, val_loss: -1.131, \n",
      "train_vector: +0.00|+0.00|-1.08|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.13|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.130, val_loss: -1.167, \n",
      "train_vector: +0.00|+0.00|-1.13|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.17|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.175, val_loss: -1.080, \n",
      "train_vector: +0.00|+0.00|-1.18|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.08|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.213, val_loss: -1.250, \n",
      "train_vector: +0.00|+0.00|-1.21|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.25|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.247, val_loss: -1.171, \n",
      "train_vector: +0.00|+0.00|-1.25|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.17|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.279, val_loss: -1.222, \n",
      "train_vector: +0.00|+0.00|-1.28|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.22|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.308, val_loss: -1.236, \n",
      "train_vector: +0.00|+0.00|-1.31|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.24|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.336, val_loss: -1.276, \n",
      "train_vector: +0.00|+0.00|-1.34|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.28|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.359, val_loss: -1.382, \n",
      "train_vector: +0.00|+0.00|-1.36|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.38|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.382, val_loss: -1.341, \n",
      "train_vector: +0.00|+0.00|-1.38|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.34|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -1.403, val_loss: -1.324, \n",
      "train_vector: +0.00|+0.00|-1.40|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.32|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -1.422, val_loss: -1.410, \n",
      "train_vector: +0.00|+0.00|-1.42|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.41|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -1.440, val_loss: -1.285, \n",
      "train_vector: +0.00|+0.00|-1.44|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.28|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -1.457, val_loss: -1.445, \n",
      "train_vector: +0.00|+0.00|-1.46|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.44|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -1.474, val_loss: -1.457, \n",
      "train_vector: +0.00|+0.00|-1.47|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.46|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -1.489, val_loss: -1.468, \n",
      "train_vector: +0.00|+0.00|-1.49|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.47|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -1.504, val_loss: -1.414, \n",
      "train_vector: +0.00|+0.00|-1.50|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.41|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -1.518, val_loss: -1.479, \n",
      "train_vector: +0.00|+0.00|-1.52|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.48|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -1.530, val_loss: -1.436, \n",
      "train_vector: +0.00|+0.00|-1.53|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.44|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -1.541, val_loss: -1.501, \n",
      "train_vector: +0.00|+0.00|-1.54|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.50|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 993.2755978107452s\n",
      "\n",
      "Training type: 3\n",
      "epoch:0, train_loss: -0.593, val_loss: -1.083, \n",
      "train_vector: +0.00|+0.00|+0.00|-0.59|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.08|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.137, val_loss: -1.075, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.14|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.07|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.338, val_loss: -1.501, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.34|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.50|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.414, val_loss: -1.528, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.41|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.53|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.511, val_loss: -1.275, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.51|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.27|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.600, val_loss: -1.489, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.60|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.49|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.658, val_loss: -1.175, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.66|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.17|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.705, val_loss: -1.922, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.70|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.92|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.740, val_loss: -1.707, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.74|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.71|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.773, val_loss: -1.336, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.77|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.34|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.783, val_loss: -1.781, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.78|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.78|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.836, val_loss: -1.544, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.84|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.54|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.849, val_loss: -1.917, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.85|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.92|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.879, val_loss: -1.664, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.88|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.66|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.920, val_loss: -1.831, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.92|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.83|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -1.925, val_loss: -1.748, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.92|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.75|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -1.962, val_loss: -1.576, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.96|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.58|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -1.970, val_loss: -1.466, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.97|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.47|+0.00|+0.00|+0.00|+0.00\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:18, train_loss: -1.995, val_loss: -2.031, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.99|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.03|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -2.012, val_loss: -1.796, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.01|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.80|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -2.022, val_loss: -1.650, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.02|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.65|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -2.038, val_loss: -1.849, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.04|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.85|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -2.077, val_loss: -1.475, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.08|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.47|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -2.065, val_loss: -1.962, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.07|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.96|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -2.087, val_loss: -2.187, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.09|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.19|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 956.6354224681854s\n",
      "\n",
      "Training type: 4\n",
      "epoch:0, train_loss: -0.436, val_loss: -0.760, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-0.44|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-0.76|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.034, val_loss: -1.235, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.03|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.24|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.246, val_loss: -1.311, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.25|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.31|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.390, val_loss: -1.282, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.39|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.28|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.485, val_loss: -1.451, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.48|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.45|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.570, val_loss: -1.570, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.57|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.57|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.639, val_loss: -1.717, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.64|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.72|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.684, val_loss: -1.573, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.68|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.57|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.735, val_loss: -1.697, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.73|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.70|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.772, val_loss: -1.857, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.77|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.86|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.812, val_loss: -1.842, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.81|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.84|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.842, val_loss: -1.793, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.84|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.79|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.875, val_loss: -1.906, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.88|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.91|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.897, val_loss: -1.964, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.90|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.96|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.921, val_loss: -1.964, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.92|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.96|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -1.940, val_loss: -1.988, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.94|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.99|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -1.971, val_loss: -1.751, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.97|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.75|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -1.997, val_loss: -1.926, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.93|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -2.010, val_loss: -2.012, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.01|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.01|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -2.018, val_loss: -2.010, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.02|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.01|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -2.054, val_loss: -1.932, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.05|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.93|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -2.060, val_loss: -1.987, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.06|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.99|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -2.084, val_loss: -2.075, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.08|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.07|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -2.092, val_loss: -2.083, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.09|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.08|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -2.123, val_loss: -2.097, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.12|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.10|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 475.0164542198181s\n",
      "\n",
      "Training type: 5\n",
      "epoch:0, train_loss: +0.119, val_loss: -0.200, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.12|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.20|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.314, val_loss: -0.342, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.31|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.34|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.476, val_loss: -0.554, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.48|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.55|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.578, val_loss: -0.630, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.58|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.63|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.657, val_loss: -0.700, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.66|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.70|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.718, val_loss: -0.688, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.72|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.69|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.770, val_loss: -0.785, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.77|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.78|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.815, val_loss: -0.735, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.82|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.74|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.852, val_loss: -0.853, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.85|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.85|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.887, val_loss: -0.895, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.89|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.89|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -0.918, val_loss: -0.930, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.92|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.93|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -0.946, val_loss: -0.950, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.95|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.95|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -0.972, val_loss: -0.952, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.97|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.95|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -0.994, val_loss: -0.974, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.99|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.97|+0.00|+0.00\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:14, train_loss: -1.017, val_loss: -0.991, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.02|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.99|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -1.034, val_loss: -1.007, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.03|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.01|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -1.052, val_loss: -1.041, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.05|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.04|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -1.070, val_loss: -1.052, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.07|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.05|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -1.086, val_loss: -1.039, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.09|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.04|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -1.100, val_loss: -1.085, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.10|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.09|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -1.118, val_loss: -1.075, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.12|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.08|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -1.131, val_loss: -1.073, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.13|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.07|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -1.143, val_loss: -1.065, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.14|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.06|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -1.153, val_loss: -1.102, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.15|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.10|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -1.166, val_loss: -1.135, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.17|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.14|+0.00|+0.00\n",
      "\n",
      "Training completed in 1016.17418217659s\n",
      "\n",
      "Training type: 6\n",
      "epoch:0, train_loss: -0.224, val_loss: -0.586, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.22|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.59|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.775, val_loss: -0.890, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.78|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.89|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.953, val_loss: -0.956, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.95|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.96|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.065, val_loss: -1.031, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.06|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.03|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.145, val_loss: -1.049, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.15|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.05|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.205, val_loss: -1.198, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.20|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.20|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.261, val_loss: -1.224, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.26|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.22|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.300, val_loss: -1.325, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.30|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.32|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.335, val_loss: -1.216, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.34|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.22|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.360, val_loss: -1.377, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.36|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.38|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.390, val_loss: -1.433, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.39|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.43|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.423, val_loss: -1.370, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.42|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.37|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.450, val_loss: -1.389, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.45|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.39|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.472, val_loss: -1.424, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.47|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.42|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.493, val_loss: -1.432, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.49|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.43|+0.00\n",
      "\n",
      "epoch:15, train_loss: -1.510, val_loss: -1.441, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.51|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.44|+0.00\n",
      "\n",
      "epoch:16, train_loss: -1.517, val_loss: -1.463, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.52|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.46|+0.00\n",
      "\n",
      "epoch:17, train_loss: -1.533, val_loss: -1.566, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.53|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.57|+0.00\n",
      "\n",
      "epoch:18, train_loss: -1.557, val_loss: -1.431, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.56|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.43|+0.00\n",
      "\n",
      "epoch:19, train_loss: -1.567, val_loss: -1.591, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.57|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.59|+0.00\n",
      "\n",
      "epoch:20, train_loss: -1.565, val_loss: -1.448, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.57|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.45|+0.00\n",
      "\n",
      "epoch:21, train_loss: -1.593, val_loss: -1.173, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.59|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.17|+0.00\n",
      "\n",
      "epoch:22, train_loss: -1.602, val_loss: -1.468, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.60|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.47|+0.00\n",
      "\n",
      "epoch:23, train_loss: -1.607, val_loss: -1.580, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.61|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.58|+0.00\n",
      "\n",
      "epoch:24, train_loss: -1.633, val_loss: -1.633, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.63|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.63|+0.00\n",
      "\n",
      "Training completed in 912.7614831924438s\n",
      "\n",
      "Training type: 7\n",
      "epoch:0, train_loss: -0.736, val_loss: -1.012, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.74, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.01\n",
      "\n",
      "epoch:1, train_loss: -1.214, val_loss: -1.180, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.21, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.18\n",
      "\n",
      "epoch:2, train_loss: -1.429, val_loss: -1.402, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.43, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.40\n",
      "\n",
      "epoch:3, train_loss: -1.560, val_loss: -1.536, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.56, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.54\n",
      "\n",
      "epoch:4, train_loss: -1.665, val_loss: -1.486, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.66, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.49\n",
      "\n",
      "epoch:5, train_loss: -1.736, val_loss: -1.734, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.74, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.73\n",
      "\n",
      "epoch:6, train_loss: -1.810, val_loss: -1.801, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.81, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.80\n",
      "\n",
      "epoch:7, train_loss: -1.866, val_loss: -1.717, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.87, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.72\n",
      "\n",
      "epoch:8, train_loss: -1.913, val_loss: -1.861, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.91, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.86\n",
      "\n",
      "epoch:9, train_loss: -1.955, val_loss: -1.874, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.96, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.87\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:10, train_loss: -1.999, val_loss: -1.705, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.70\n",
      "\n",
      "epoch:11, train_loss: -2.031, val_loss: -1.955, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.03, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.96\n",
      "\n",
      "epoch:12, train_loss: -2.072, val_loss: -1.972, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.07, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.97\n",
      "\n",
      "epoch:13, train_loss: -2.096, val_loss: -1.947, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.10, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.95\n",
      "\n",
      "epoch:14, train_loss: -2.128, val_loss: -1.994, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.13, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.99\n",
      "\n",
      "epoch:15, train_loss: -2.156, val_loss: -2.026, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.16, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.03\n",
      "\n",
      "epoch:16, train_loss: -2.176, val_loss: -2.019, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.18, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.02\n",
      "\n",
      "epoch:17, train_loss: -2.203, val_loss: -2.056, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.20, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.06\n",
      "\n",
      "epoch:18, train_loss: -2.221, val_loss: -2.074, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.22, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.07\n",
      "\n",
      "epoch:19, train_loss: -2.248, val_loss: -2.093, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.25, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.09\n",
      "\n",
      "epoch:20, train_loss: -2.268, val_loss: -2.049, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.27, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.05\n",
      "\n",
      "epoch:21, train_loss: -2.289, val_loss: -2.063, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.29, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.06\n",
      "\n",
      "epoch:22, train_loss: -2.303, val_loss: -2.114, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.30, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.11\n",
      "\n",
      "epoch:23, train_loss: -2.330, val_loss: -2.167, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.33, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.17\n",
      "\n",
      "epoch:24, train_loss: -2.342, val_loss: -2.186, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.34, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.19\n",
      "\n",
      "Training completed in 545.3477826118469s\n"
     ]
    }
   ],
   "source": [
    "for i in range(8):\n",
    "    # load type data\n",
    "    data_type = '../Data/{}_data_ACSF_expand_type_'+str(i)+'.pickle'\n",
    "    train_dl,val_dl = get_data(data_type,batch_size)\n",
    "    \n",
    "    # init model\n",
    "    model = GNN(reuse,block,head,dim,layer1,layer2,factor,**data_dict[data]).to('cuda:0')\n",
    "    paras = trainable_parameter(model)\n",
    "    opt = Adam(paras,lr=lr_type)\n",
    "    \n",
    "    print('\\nTraining type: {}'.format(i))\n",
    "    model,train_loss_perType,val_loss_perType = train(opt,model,epochs_type,train_dl,val_dl,paras,clip,True)\n",
    "\n",
    "    save_results(train_loss_perType,val_loss_perType,reuse,block,\\\n",
    "                 head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'type_'+str(i)+'_start')\n",
    "\n",
    "    save_model(model,opt,reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'type_'+str(i)+'_start')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### two stage bigger head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_type =1e-4/2\n",
    "epochs_type = 10\n",
    "file_name = \"False_<class 'functions_refactor.schnet_block'>_<class 'functions_refactor.cat3Head'>__data_ACSF_expand.pickle_32_128_2_2_2_2_20_base.tar\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training type: 0\n",
      "\n",
      "Training head\n",
      "epoch:0, train_loss: +0.589, val_loss: +0.061, \n",
      "train_vector: +0.59|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.06|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: +0.008, val_loss: +0.214, \n",
      "train_vector: +0.01|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.21|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.005, val_loss: +0.116, \n",
      "train_vector: -0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.12|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.008, val_loss: +0.121, \n",
      "train_vector: -0.01|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.12|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.010, val_loss: +0.239, \n",
      "train_vector: -0.01|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.24|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 85.70982599258423s\n",
      "\n",
      "Training all\n",
      "epoch:5, train_loss: +0.050, val_loss: +0.155, \n",
      "train_vector: +0.05|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.15|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.012, val_loss: -0.143, \n",
      "train_vector: -0.01|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.14|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.050, val_loss: +0.034, \n",
      "train_vector: -0.05|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.03|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.095, val_loss: -0.247, \n",
      "train_vector: -0.09|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.25|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.113, val_loss: -0.093, \n",
      "train_vector: -0.11|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.09|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -0.148, val_loss: -0.185, \n",
      "train_vector: -0.15|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.19|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -0.163, val_loss: -0.129, \n",
      "train_vector: -0.16|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.13|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -0.185, val_loss: +0.175, \n",
      "train_vector: -0.19|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.18|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -0.210, val_loss: -0.260, \n",
      "train_vector: -0.21|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.26|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -0.226, val_loss: -0.156, \n",
      "train_vector: -0.23|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.16|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 377.3226683139801s\n",
      "\n",
      "Training type: 1\n",
      "\n",
      "Training head\n",
      "epoch:0, train_loss: +0.832, val_loss: -0.475, \n",
      "train_vector: +0.00|+0.83|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.47|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.307, val_loss: -0.487, \n",
      "train_vector: +0.00|-0.31|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.49|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.381, val_loss: -0.612, \n",
      "train_vector: +0.00|-0.38|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.61|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.439, val_loss: -0.491, \n",
      "train_vector: +0.00|-0.44|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.49|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.482, val_loss: -0.450, \n",
      "train_vector: +0.00|-0.48|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.45|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 29.753875255584717s\n",
      "\n",
      "Training all\n",
      "epoch:5, train_loss: -0.593, val_loss: -0.850, \n",
      "train_vector: +0.00|-0.59|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.85|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.745, val_loss: -0.686, \n",
      "train_vector: +0.00|-0.74|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.69|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.769, val_loss: -0.298, \n",
      "train_vector: +0.00|-0.77|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.30|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.836, val_loss: -0.582, \n",
      "train_vector: +0.00|-0.84|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.58|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.817, val_loss: -1.131, \n",
      "train_vector: +0.00|-0.82|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.13|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -0.890, val_loss: -1.165, \n",
      "train_vector: +0.00|-0.89|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.16|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -0.864, val_loss: -0.995, \n",
      "train_vector: +0.00|-0.86|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.99|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -0.887, val_loss: -0.762, \n",
      "train_vector: +0.00|-0.89|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.76|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -0.921, val_loss: -1.242, \n",
      "train_vector: +0.00|-0.92|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.24|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -0.957, val_loss: -1.120, \n",
      "train_vector: +0.00|-0.96|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.12|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 130.74536728858948s\n",
      "\n",
      "Training type: 2\n",
      "\n",
      "Training head\n",
      "epoch:0, train_loss: -0.454, val_loss: -0.591, \n",
      "train_vector: +0.00|+0.00|-0.45|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.59|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.625, val_loss: -0.609, \n",
      "train_vector: +0.00|+0.00|-0.62|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.61|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.640, val_loss: -0.642, \n",
      "train_vector: +0.00|+0.00|-0.64|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.64|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.651, val_loss: -0.643, \n",
      "train_vector: +0.00|+0.00|-0.65|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.64|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.659, val_loss: -0.676, \n",
      "train_vector: +0.00|+0.00|-0.66|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.68|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 90.99793410301208s\n",
      "\n",
      "Training all\n",
      "epoch:5, train_loss: -0.702, val_loss: -0.810, \n",
      "train_vector: +0.00|+0.00|-0.70|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.81|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.799, val_loss: -0.884, \n",
      "train_vector: +0.00|+0.00|-0.80|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.88|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.867, val_loss: -0.869, \n",
      "train_vector: +0.00|+0.00|-0.87|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.87|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.921, val_loss: -0.980, \n",
      "train_vector: +0.00|+0.00|-0.92|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.98|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.970, val_loss: -1.036, \n",
      "train_vector: +0.00|+0.00|-0.97|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.04|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.008, val_loss: -1.064, \n",
      "train_vector: +0.00|+0.00|-1.01|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.06|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.045, val_loss: -1.077, \n",
      "train_vector: +0.00|+0.00|-1.05|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.08|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.078, val_loss: -1.132, \n",
      "train_vector: +0.00|+0.00|-1.08|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.13|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.106, val_loss: -1.132, \n",
      "train_vector: +0.00|+0.00|-1.11|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.13|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.138, val_loss: -1.168, \n",
      "train_vector: +0.00|+0.00|-1.14|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.17|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 394.71592378616333s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training type: 3\n",
      "\n",
      "Training head\n",
      "epoch:0, train_loss: -1.242, val_loss: -1.765, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.24|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.76|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.703, val_loss: -1.774, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.70|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.77|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.733, val_loss: -1.693, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.73|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.69|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.748, val_loss: -1.865, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.75|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.86|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.766, val_loss: -1.651, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.77|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.65|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 87.28697752952576s\n",
      "\n",
      "Training all\n",
      "epoch:5, train_loss: -1.664, val_loss: -1.873, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.66|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.87|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.735, val_loss: -1.177, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.73|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.18|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.769, val_loss: -1.918, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.77|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.92|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.785, val_loss: -1.711, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.78|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.71|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.835, val_loss: -1.970, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.84|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.97|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.869, val_loss: -1.968, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.87|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.97|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.887, val_loss: -1.926, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.89|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.93|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.898, val_loss: -1.785, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.90|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.79|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.928, val_loss: -1.965, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.93|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.97|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.947, val_loss: -1.816, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.95|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.82|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 379.6869592666626s\n",
      "\n",
      "Training type: 4\n",
      "\n",
      "Training head\n",
      "epoch:0, train_loss: -1.046, val_loss: -1.436, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.05|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.44|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.472, val_loss: -1.587, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.47|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.59|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.518, val_loss: -1.575, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.52|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.57|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.551, val_loss: -1.541, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.55|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.54|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.564, val_loss: -1.633, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.56|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.63|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 44.453752756118774s\n",
      "\n",
      "Training all\n",
      "epoch:5, train_loss: -1.581, val_loss: -1.561, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.58|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.56|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.683, val_loss: -1.759, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.68|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.76|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.740, val_loss: -1.836, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.74|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.84|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.770, val_loss: -1.857, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.77|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.86|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.817, val_loss: -1.901, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.82|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.90|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.848, val_loss: -1.742, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.85|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.74|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.874, val_loss: -1.880, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.87|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.88|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.893, val_loss: -1.884, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.89|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.88|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.915, val_loss: -1.967, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.91|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.97|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.940, val_loss: -2.003, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.94|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 192.33296394348145s\n",
      "\n",
      "Training type: 5\n",
      "\n",
      "Training head\n",
      "epoch:0, train_loss: +0.316, val_loss: +0.194, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.32|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.19|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: +0.194, val_loss: +0.184, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.19|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.18|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: +0.171, val_loss: +0.176, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.17|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.18|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: +0.149, val_loss: +0.131, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.15|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.13|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: +0.128, val_loss: +0.128, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.13|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.13|+0.00|+0.00\n",
      "\n",
      "Training completed in 94.3971426486969s\n",
      "\n",
      "Training all\n",
      "epoch:5, train_loss: -0.027, val_loss: -0.147, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.03|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.15|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.203, val_loss: -0.285, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.20|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.28|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.308, val_loss: -0.318, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.31|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.32|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.386, val_loss: -0.389, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.39|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.39|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.450, val_loss: -0.489, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.45|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.49|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -0.506, val_loss: -0.507, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.51|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.51|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -0.556, val_loss: -0.584, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.56|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.58|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -0.600, val_loss: -0.598, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.60|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.60|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -0.642, val_loss: -0.605, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.64|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.60|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -0.678, val_loss: -0.705, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.68|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.70|+0.00|+0.00\n",
      "\n",
      "Training completed in 405.3645977973938s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training type: 6\n",
      "\n",
      "Training head\n",
      "epoch:0, train_loss: -0.193, val_loss: -0.417, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.19|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.42|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.423, val_loss: -0.460, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.42|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.46|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.449, val_loss: -0.413, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.45|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.41|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.474, val_loss: -0.513, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.47|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.51|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.493, val_loss: -0.516, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.49|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.52|+0.00\n",
      "\n",
      "Training completed in 84.60186243057251s\n",
      "\n",
      "Training all\n",
      "epoch:5, train_loss: -0.620, val_loss: -0.683, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.62|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.68|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.775, val_loss: -0.741, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.78|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.74|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.870, val_loss: -0.925, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.87|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.93|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.929, val_loss: -1.004, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.93|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.986, val_loss: -1.001, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.99|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.027, val_loss: -1.021, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.03|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.02|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.072, val_loss: -1.078, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.07|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.08|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.110, val_loss: -1.127, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.11|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.13|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.143, val_loss: -1.151, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.14|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.15|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.169, val_loss: -1.180, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.17|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.18|+0.00\n",
      "\n",
      "Training completed in 364.97120213508606s\n",
      "\n",
      "Training type: 7\n",
      "\n",
      "Training head\n",
      "epoch:0, train_loss: -1.089, val_loss: -1.322, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.09, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.32\n",
      "\n",
      "epoch:1, train_loss: -1.395, val_loss: -1.397, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.39, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.40\n",
      "\n",
      "epoch:2, train_loss: -1.437, val_loss: -1.468, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.44, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.47\n",
      "\n",
      "epoch:3, train_loss: -1.456, val_loss: -1.454, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.46, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.45\n",
      "\n",
      "epoch:4, train_loss: -1.479, val_loss: -1.448, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.48, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.45\n",
      "\n",
      "Training completed in 51.48229789733887s\n",
      "\n",
      "Training all\n",
      "epoch:5, train_loss: -1.500, val_loss: -1.539, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.50, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.54\n",
      "\n",
      "epoch:6, train_loss: -1.598, val_loss: -1.564, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.60, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.56\n",
      "\n",
      "epoch:7, train_loss: -1.665, val_loss: -1.536, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.67, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.54\n",
      "\n",
      "epoch:8, train_loss: -1.713, val_loss: -1.675, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.71, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.68\n",
      "\n",
      "epoch:9, train_loss: -1.758, val_loss: -1.668, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.76, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.67\n",
      "\n",
      "epoch:10, train_loss: -1.805, val_loss: -1.779, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.80, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.78\n",
      "\n",
      "epoch:11, train_loss: -1.847, val_loss: -1.769, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.85, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.77\n",
      "\n",
      "epoch:12, train_loss: -1.872, val_loss: -1.829, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.87, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.83\n",
      "\n",
      "epoch:13, train_loss: -1.905, val_loss: -1.827, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.90, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.83\n",
      "\n",
      "epoch:14, train_loss: -1.932, val_loss: -1.881, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.93, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.88\n",
      "\n",
      "Training completed in 220.315260887146s\n"
     ]
    }
   ],
   "source": [
    "for i in range(8):\n",
    "    # load type data\n",
    "    data_type = '../Data/{}_data_ACSF_expand_type_'+str(i)+'.pickle'\n",
    "    train_dl,val_dl = get_data(data_type,batch_size)\n",
    "    \n",
    "    # load base model\n",
    "    model = GNN(reuse,block,head,dim,layer1,layer2,factor,**data_dict[data]).to('cuda:0')\n",
    "    checkpoint = torch.load('../Model/'+file_name)\n",
    "    model.load_state_dict(checkpoint['model_state_dict'])\n",
    "\n",
    "    set_requires_grad(model, False)\n",
    "    model.head = cat3Head_type(dim,8,data_dict[data]['edge_in4']).to('cuda:0')\n",
    "    \n",
    "    paras = trainable_parameter(model)\n",
    "    opt = Adam(paras,lr=lr_type) \n",
    "    # train head\n",
    "    print('\\nTraining type: {}'.format(i))\n",
    "    print('\\nTraining head')\n",
    "    model,train_loss_perType,val_loss_perType = train(opt,model,5,train_dl,val_dl,paras,clip,True)\n",
    "    \n",
    "    # train all\n",
    "    set_requires_grad(model, True)\n",
    "    paras = trainable_parameter(model)\n",
    "    opt = Adam(paras,lr=lr_type)    \n",
    "    print('\\nTraining all')\n",
    "    model,train_loss_perType,val_loss_perType = train(opt,model,epochs_type,train_dl,val_dl,paras,clip,True,train_loss_perType,val_loss_perType)\n",
    "    \n",
    "    save_results(train_loss_perType,val_loss_perType,reuse,block,\\\n",
    "                 cat3Head_type,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'type_'+str(i)+'_2stage')\n",
    "\n",
    "    save_model(model,opt,reuse,block,cat3Head_type,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'type_'+str(i)+'_2stage')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### make submissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_submission(reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "head = cat3Head_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_submission(reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'_start')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2Stage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "head = cat3Head_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_submission(reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'_2stage')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}