GNN_refactor_NNConv_cat3Head.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import torch\n",
    "from torch_geometric.data import Data,DataLoader\n",
    "from functions_refactor import *\n",
    "from pytorch_util import *\n",
    "from torch.optim import Adam\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# model parameters\n",
    "reuse = False\n",
    "block = NNConv_block\n",
    "head = cat3Head_type\n",
    "data = '../Data/{}_data_ACSF_expand.pickle'\n",
    "batch_size = 32\n",
    "dim = 128\n",
    "epochs = 20\n",
    "clip = 2\n",
    "layer1 = 2\n",
    "layer2 = 2\n",
    "factor = 2\n",
    "lr = 1e-4\n",
    "epochs_type = 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train on each type from start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training type: 0\n",
      "epoch:0, train_loss: +0.896, val_loss: +0.131, \n",
      "train_vector: +0.90|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.13|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: +0.113, val_loss: -0.043, \n",
      "train_vector: +0.11|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.04|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.023, val_loss: -0.134, \n",
      "train_vector: -0.02|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.13|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.102, val_loss: -0.200, \n",
      "train_vector: -0.10|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.20|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.166, val_loss: -0.253, \n",
      "train_vector: -0.17|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.25|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.216, val_loss: -0.243, \n",
      "train_vector: -0.22|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.24|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.261, val_loss: -0.180, \n",
      "train_vector: -0.26|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.18|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.299, val_loss: -0.341, \n",
      "train_vector: -0.30|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.34|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.332, val_loss: -0.372, \n",
      "train_vector: -0.33|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.37|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.365, val_loss: -0.387, \n",
      "train_vector: -0.36|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.39|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -0.398, val_loss: -0.318, \n",
      "train_vector: -0.40|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.32|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -0.418, val_loss: -0.446, \n",
      "train_vector: -0.42|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.45|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -0.438, val_loss: -0.528, \n",
      "train_vector: -0.44|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.53|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -0.468, val_loss: -0.452, \n",
      "train_vector: -0.47|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.45|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -0.488, val_loss: -0.510, \n",
      "train_vector: -0.49|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.51|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -0.509, val_loss: -0.480, \n",
      "train_vector: -0.51|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.48|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -0.518, val_loss: -0.524, \n",
      "train_vector: -0.52|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.52|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -0.538, val_loss: -0.585, \n",
      "train_vector: -0.54|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.58|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -0.566, val_loss: -0.607, \n",
      "train_vector: -0.57|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.61|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -0.583, val_loss: -0.473, \n",
      "train_vector: -0.58|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.47|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -0.592, val_loss: -0.570, \n",
      "train_vector: -0.59|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.57|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -0.612, val_loss: -0.496, \n",
      "train_vector: -0.61|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.50|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -0.618, val_loss: -0.586, \n",
      "train_vector: -0.62|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.59|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -0.635, val_loss: -0.630, \n",
      "train_vector: -0.63|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.63|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -0.655, val_loss: -0.667, \n",
      "train_vector: -0.66|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.67|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 1608.482215642929s\n",
      "\n",
      "Training type: 1\n",
      "epoch:0, train_loss: +0.718, val_loss: +0.097, \n",
      "train_vector: +0.00|+0.72|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.10|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.149, val_loss: -0.319, \n",
      "train_vector: +0.00|-0.15|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.32|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.306, val_loss: -0.434, \n",
      "train_vector: +0.00|-0.31|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.43|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.451, val_loss: -0.540, \n",
      "train_vector: +0.00|-0.45|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.54|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.498, val_loss: -0.652, \n",
      "train_vector: +0.00|-0.50|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.65|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.561, val_loss: -0.625, \n",
      "train_vector: +0.00|-0.56|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.62|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.643, val_loss: -0.156, \n",
      "train_vector: +0.00|-0.64|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.16|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.684, val_loss: -0.832, \n",
      "train_vector: +0.00|-0.68|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.83|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.709, val_loss: -0.617, \n",
      "train_vector: +0.00|-0.71|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.62|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.764, val_loss: -0.713, \n",
      "train_vector: +0.00|-0.76|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.71|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -0.778, val_loss: -0.861, \n",
      "train_vector: +0.00|-0.78|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.86|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -0.830, val_loss: -0.893, \n",
      "train_vector: +0.00|-0.83|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.89|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -0.848, val_loss: -0.887, \n",
      "train_vector: +0.00|-0.85|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.89|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -0.865, val_loss: -0.653, \n",
      "train_vector: +0.00|-0.86|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.65|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -0.924, val_loss: -0.985, \n",
      "train_vector: +0.00|-0.92|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.99|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -0.925, val_loss: -0.907, \n",
      "train_vector: +0.00|-0.93|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.91|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -0.958, val_loss: -1.041, \n",
      "train_vector: +0.00|-0.96|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.04|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -0.984, val_loss: -0.942, \n",
      "train_vector: +0.00|-0.98|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.94|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -0.986, val_loss: -1.050, \n",
      "train_vector: +0.00|-0.99|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.05|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -1.016, val_loss: -1.055, \n",
      "train_vector: +0.00|-1.02|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.05|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -1.029, val_loss: -1.051, \n",
      "train_vector: +0.00|-1.03|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.05|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -1.053, val_loss: -1.091, \n",
      "train_vector: +0.00|-1.05|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.09|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:22, train_loss: -1.055, val_loss: -1.011, \n",
      "train_vector: +0.00|-1.06|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.01|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -1.069, val_loss: -1.071, \n",
      "train_vector: +0.00|-1.07|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.07|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -1.084, val_loss: -0.851, \n",
      "train_vector: +0.00|-1.08|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.85|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 497.0621814727783s\n",
      "\n",
      "Training type: 2\n",
      "epoch:0, train_loss: -0.254, val_loss: -0.609, \n",
      "train_vector: +0.00|+0.00|-0.25|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.61|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.709, val_loss: -0.816, \n",
      "train_vector: +0.00|+0.00|-0.71|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.82|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.877, val_loss: -0.972, \n",
      "train_vector: +0.00|+0.00|-0.88|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.97|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.990, val_loss: -1.060, \n",
      "train_vector: +0.00|+0.00|-0.99|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.06|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.077, val_loss: -1.116, \n",
      "train_vector: +0.00|+0.00|-1.08|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.12|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.144, val_loss: -1.171, \n",
      "train_vector: +0.00|+0.00|-1.14|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.17|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.207, val_loss: -1.214, \n",
      "train_vector: +0.00|+0.00|-1.21|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.21|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.258, val_loss: -1.244, \n",
      "train_vector: +0.00|+0.00|-1.26|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.24|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.304, val_loss: -1.329, \n",
      "train_vector: +0.00|+0.00|-1.30|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.33|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.348, val_loss: -1.347, \n",
      "train_vector: +0.00|+0.00|-1.35|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.35|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.386, val_loss: -1.385, \n",
      "train_vector: +0.00|+0.00|-1.39|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.39|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.420, val_loss: -1.412, \n",
      "train_vector: +0.00|+0.00|-1.42|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.41|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.453, val_loss: -1.445, \n",
      "train_vector: +0.00|+0.00|-1.45|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.44|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.482, val_loss: -1.456, \n",
      "train_vector: +0.00|+0.00|-1.48|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.46|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.511, val_loss: -1.465, \n",
      "train_vector: +0.00|+0.00|-1.51|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.47|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -1.538, val_loss: -1.461, \n",
      "train_vector: +0.00|+0.00|-1.54|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.46|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -1.563, val_loss: -1.509, \n",
      "train_vector: +0.00|+0.00|-1.56|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.51|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -1.587, val_loss: -1.552, \n",
      "train_vector: +0.00|+0.00|-1.59|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.55|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -1.609, val_loss: -1.536, \n",
      "train_vector: +0.00|+0.00|-1.61|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.54|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -1.629, val_loss: -1.559, \n",
      "train_vector: +0.00|+0.00|-1.63|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.56|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -1.650, val_loss: -1.576, \n",
      "train_vector: +0.00|+0.00|-1.65|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.58|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -1.670, val_loss: -1.599, \n",
      "train_vector: +0.00|+0.00|-1.67|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.60|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -1.688, val_loss: -1.602, \n",
      "train_vector: +0.00|+0.00|-1.69|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.60|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -1.705, val_loss: -1.593, \n",
      "train_vector: +0.00|+0.00|-1.71|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.59|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -1.724, val_loss: -1.637, \n",
      "train_vector: +0.00|+0.00|-1.72|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.64|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 1593.133712053299s\n",
      "\n",
      "Training type: 3\n",
      "epoch:0, train_loss: -0.649, val_loss: -1.182, \n",
      "train_vector: +0.00|+0.00|+0.00|-0.65|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.18|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.262, val_loss: -1.476, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.26|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.48|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.467, val_loss: -1.585, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.47|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.58|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.557, val_loss: -1.740, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.56|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.74|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.651, val_loss: -1.759, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.65|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.76|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.728, val_loss: -1.845, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.73|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.84|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.792, val_loss: -1.718, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.79|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.72|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.831, val_loss: -1.902, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.83|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.90|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.879, val_loss: -1.902, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.88|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.90|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.918, val_loss: -1.942, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.92|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.94|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.947, val_loss: -2.028, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.95|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.03|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.987, val_loss: -1.955, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.99|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.96|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -2.006, val_loss: -1.987, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.01|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.99|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -2.021, val_loss: -1.806, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.02|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.81|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -2.063, val_loss: -2.021, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.06|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.02|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -2.085, val_loss: -2.095, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.09|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.09|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -2.094, val_loss: -2.155, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.09|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.15|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -2.129, val_loss: -2.223, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.13|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.22|+0.00|+0.00|+0.00|+0.00\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:18, train_loss: -2.149, val_loss: -1.951, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.15|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.95|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -2.153, val_loss: -2.233, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.15|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.23|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -2.179, val_loss: -2.084, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.18|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.08|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -2.203, val_loss: -2.141, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.20|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.14|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -2.210, val_loss: -2.266, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.21|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.27|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -2.217, val_loss: -2.266, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.22|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.27|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -2.250, val_loss: -2.096, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.25|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.10|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 1552.5156981945038s\n",
      "\n",
      "Training type: 4\n",
      "epoch:0, train_loss: -0.665, val_loss: -1.139, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-0.67|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.14|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.274, val_loss: -1.370, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.27|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.37|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.461, val_loss: -1.478, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.46|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.48|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.602, val_loss: -1.686, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.60|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.69|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.688, val_loss: -1.708, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.69|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.71|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.763, val_loss: -1.723, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.76|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.72|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.828, val_loss: -1.862, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.83|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.86|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.874, val_loss: -1.885, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.87|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.89|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.922, val_loss: -1.881, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.92|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.88|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.960, val_loss: -1.884, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.96|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.88|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.998, val_loss: -1.918, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.92|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -2.037, val_loss: -1.995, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.04|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.99|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -2.069, val_loss: -1.844, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.07|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.84|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -2.098, val_loss: -2.038, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.10|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.04|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -2.135, val_loss: -2.089, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.14|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.09|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -2.149, val_loss: -2.012, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.15|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.01|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -2.180, val_loss: -2.061, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.18|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.06|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -2.196, val_loss: -2.091, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.20|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.09|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -2.237, val_loss: -2.159, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.24|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.16|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -2.248, val_loss: -2.142, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.25|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.14|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -2.271, val_loss: -2.193, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.27|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.19|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -2.293, val_loss: -2.202, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.29|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.20|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -2.313, val_loss: -2.198, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.31|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.20|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -2.328, val_loss: -2.199, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.33|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.20|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -2.353, val_loss: -2.155, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.35|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.15|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 725.1556434631348s\n",
      "\n",
      "Training type: 5\n",
      "epoch:0, train_loss: +0.089, val_loss: -0.232, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.09|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.23|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.334, val_loss: -0.429, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.33|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.43|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.497, val_loss: -0.558, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.50|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.56|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.611, val_loss: -0.649, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.61|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.65|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.699, val_loss: -0.730, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.70|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.73|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.771, val_loss: -0.772, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.77|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.77|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.833, val_loss: -0.837, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.83|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.84|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.888, val_loss: -0.889, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.89|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.89|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.935, val_loss: -0.919, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.94|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.92|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.977, val_loss: -0.957, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.98|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.96|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.015, val_loss: -0.975, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.02|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.98|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.051, val_loss: -1.022, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.05|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.02|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.081, val_loss: -1.047, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.08|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.05|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.112, val_loss: -1.064, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.11|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.06|+0.00|+0.00\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:14, train_loss: -1.138, val_loss: -1.105, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.14|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.10|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -1.165, val_loss: -1.115, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.16|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.11|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -1.188, val_loss: -1.135, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.19|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.14|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -1.211, val_loss: -1.131, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.21|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.13|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -1.230, val_loss: -1.156, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.23|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.16|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -1.251, val_loss: -1.158, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.25|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.16|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -1.270, val_loss: -1.184, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.27|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.18|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -1.286, val_loss: -1.183, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.29|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.18|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -1.304, val_loss: -1.211, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.30|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.21|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -1.318, val_loss: -1.234, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.32|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.23|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -1.335, val_loss: -1.235, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.33|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.24|+0.00|+0.00\n",
      "\n",
      "Training completed in 1619.0221407413483s\n",
      "\n",
      "Training type: 6\n",
      "epoch:0, train_loss: -0.189, val_loss: -0.597, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.19|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.60|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.723, val_loss: -0.852, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.72|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.85|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.905, val_loss: -0.865, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.90|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.86|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.001, val_loss: -1.089, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.09|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.089, val_loss: -1.151, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.09|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.15|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.144, val_loss: -1.129, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.14|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.13|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.183, val_loss: -1.181, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.18|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.18|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.235, val_loss: -1.252, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.24|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.25|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.269, val_loss: -1.246, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.27|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.25|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.303, val_loss: -1.278, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.30|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.28|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.325, val_loss: -1.307, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.32|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.31|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.351, val_loss: -1.331, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.35|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.33|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.371, val_loss: -1.373, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.37|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.37|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.401, val_loss: -1.416, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.40|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.42|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.425, val_loss: -1.424, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.43|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.42|+0.00\n",
      "\n",
      "epoch:15, train_loss: -1.449, val_loss: -1.405, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.45|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.40|+0.00\n",
      "\n",
      "epoch:16, train_loss: -1.469, val_loss: -1.457, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.47|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.46|+0.00\n",
      "\n",
      "epoch:17, train_loss: -1.481, val_loss: -1.430, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.48|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.43|+0.00\n",
      "\n",
      "epoch:18, train_loss: -1.495, val_loss: -1.453, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.50|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.45|+0.00\n",
      "\n",
      "epoch:19, train_loss: -1.517, val_loss: -1.446, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.52|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.45|+0.00\n",
      "\n",
      "epoch:20, train_loss: -1.528, val_loss: -1.399, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.53|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.40|+0.00\n",
      "\n",
      "epoch:21, train_loss: -1.529, val_loss: -1.536, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.53|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.54|+0.00\n",
      "\n",
      "epoch:22, train_loss: -1.537, val_loss: -1.415, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.54|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.42|+0.00\n",
      "\n",
      "epoch:23, train_loss: -1.556, val_loss: -1.502, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.56|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.50|+0.00\n",
      "\n",
      "epoch:24, train_loss: -1.574, val_loss: -1.531, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.57|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.53|+0.00\n",
      "\n",
      "Training completed in 1491.4294884204865s\n",
      "\n",
      "Training type: 7\n",
      "epoch:0, train_loss: -0.924, val_loss: -1.292, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.92, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.29\n",
      "\n",
      "epoch:1, train_loss: -1.420, val_loss: -1.531, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.42, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.53\n",
      "\n",
      "epoch:2, train_loss: -1.608, val_loss: -1.708, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.61, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.71\n",
      "\n",
      "epoch:3, train_loss: -1.738, val_loss: -1.768, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.74, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.77\n",
      "\n",
      "epoch:4, train_loss: -1.839, val_loss: -1.818, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.84, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.82\n",
      "\n",
      "epoch:5, train_loss: -1.916, val_loss: -1.894, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.92, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.89\n",
      "\n",
      "epoch:6, train_loss: -1.994, val_loss: -1.917, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.99, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.92\n",
      "\n",
      "epoch:7, train_loss: -2.055, val_loss: -1.955, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.05, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.96\n",
      "\n",
      "epoch:8, train_loss: -2.104, val_loss: -1.962, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.10, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.96\n",
      "\n",
      "epoch:9, train_loss: -2.154, val_loss: -2.069, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.15, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.07\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:10, train_loss: -2.197, val_loss: -2.032, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.20, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.03\n",
      "\n",
      "epoch:11, train_loss: -2.235, val_loss: -2.106, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.24, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.11\n",
      "\n",
      "epoch:12, train_loss: -2.271, val_loss: -2.117, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.27, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.12\n",
      "\n",
      "epoch:13, train_loss: -2.306, val_loss: -2.156, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.31, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.16\n",
      "\n",
      "epoch:14, train_loss: -2.339, val_loss: -2.173, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.34, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.17\n",
      "\n",
      "epoch:15, train_loss: -2.369, val_loss: -2.200, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.37, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.20\n",
      "\n",
      "epoch:16, train_loss: -2.393, val_loss: -2.245, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.39, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.24\n",
      "\n",
      "epoch:17, train_loss: -2.424, val_loss: -2.239, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.42, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.24\n",
      "\n",
      "epoch:18, train_loss: -2.449, val_loss: -2.264, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.45, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.26\n",
      "\n",
      "epoch:19, train_loss: -2.477, val_loss: -2.253, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.48, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.25\n",
      "\n",
      "epoch:20, train_loss: -2.496, val_loss: -2.259, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.50, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.26\n",
      "\n",
      "epoch:21, train_loss: -2.524, val_loss: -2.295, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.52, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.30\n",
      "\n",
      "epoch:22, train_loss: -2.545, val_loss: -2.298, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.54, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.30\n",
      "\n",
      "epoch:23, train_loss: -2.567, val_loss: -2.355, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.57, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.36\n",
      "\n",
      "epoch:24, train_loss: -2.583, val_loss: -2.367, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.58, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.37\n",
      "\n",
      "Training completed in 821.7700595855713s\n"
     ]
    }
   ],
   "source": [
    "for i in range(8):\n",
    "    # load type data\n",
    "    data_type = '../Data/{}_data_ACSF_expand_type_'+str(i)+'.pickle'\n",
    "    train_dl,val_dl = get_data(data_type,batch_size)\n",
    "    \n",
    "    # init model\n",
    "    model = GNN(reuse,block,head,dim,layer1,layer2,factor,**data_dict[data]).to('cuda:0')\n",
    "    paras = trainable_parameter(model)\n",
    "    opt = Adam(paras,lr=lr)\n",
    "    \n",
    "    print('\\nTraining type: {}'.format(i))\n",
    "    model,train_loss_perType,val_loss_perType = train(opt,model,epochs_type,train_dl,val_dl,paras,clip,True)\n",
    "\n",
    "    save_results(train_loss_perType,val_loss_perType,reuse,block,\\\n",
    "                 head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'type_'+str(i)+'_start')\n",
    "\n",
    "    save_model(model,opt,reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'type_'+str(i)+'_start')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Continue training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs_type = 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = GNN(reuse,block,head,dim,layer1,layer2,factor,**data_dict[data]).to('cuda:0')\n",
    "paras = trainable_parameter(model)\n",
    "opt = Adam(paras,lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training type: 0\n",
      "epoch:0, train_loss: -0.663, val_loss: -0.669, \n",
      "train_vector: -0.66|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.67|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.674, val_loss: -0.593, \n",
      "train_vector: -0.67|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.59|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.687, val_loss: -0.632, \n",
      "train_vector: -0.69|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.63|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.695, val_loss: -0.486, \n",
      "train_vector: -0.70|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.49|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.710, val_loss: -0.711, \n",
      "train_vector: -0.71|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.71|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.725, val_loss: -0.664, \n",
      "train_vector: -0.73|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.66|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.736, val_loss: -0.548, \n",
      "train_vector: -0.74|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.55|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.742, val_loss: -0.753, \n",
      "train_vector: -0.74|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.75|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.759, val_loss: -0.755, \n",
      "train_vector: -0.76|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.75|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.770, val_loss: -0.671, \n",
      "train_vector: -0.77|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.67|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -0.778, val_loss: -0.617, \n",
      "train_vector: -0.78|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.62|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -0.786, val_loss: -0.750, \n",
      "train_vector: -0.79|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.75|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -0.799, val_loss: -0.783, \n",
      "train_vector: -0.80|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.78|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -0.801, val_loss: -0.566, \n",
      "train_vector: -0.80|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.57|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -0.817, val_loss: -0.792, \n",
      "train_vector: -0.82|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.79|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -0.820, val_loss: -0.795, \n",
      "train_vector: -0.82|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.79|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -0.829, val_loss: -0.806, \n",
      "train_vector: -0.83|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.81|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -0.833, val_loss: -0.711, \n",
      "train_vector: -0.83|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.71|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -0.851, val_loss: -0.768, \n",
      "train_vector: -0.85|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.77|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -0.854, val_loss: -0.792, \n",
      "train_vector: -0.85|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.79|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -0.864, val_loss: -0.769, \n",
      "train_vector: -0.86|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.77|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -0.868, val_loss: -0.772, \n",
      "train_vector: -0.87|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.77|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -0.879, val_loss: -0.808, \n",
      "train_vector: -0.88|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.81|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -0.883, val_loss: -0.797, \n",
      "train_vector: -0.88|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.80|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -0.897, val_loss: -0.699, \n",
      "train_vector: -0.90|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.70|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 1575.3348979949951s\n",
      "\n",
      "Training type: 1\n",
      "epoch:0, train_loss: -0.982, val_loss: -0.945, \n",
      "train_vector: +0.00|-0.98|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.95|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.968, val_loss: -1.011, \n",
      "train_vector: +0.00|-0.97|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.01|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.010, val_loss: -1.098, \n",
      "train_vector: +0.00|-1.01|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.10|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.017, val_loss: -1.074, \n",
      "train_vector: +0.00|-1.02|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.07|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.036, val_loss: -1.009, \n",
      "train_vector: +0.00|-1.04|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.01|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.053, val_loss: -1.069, \n",
      "train_vector: +0.00|-1.05|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.07|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.060, val_loss: -0.930, \n",
      "train_vector: +0.00|-1.06|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.93|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.070, val_loss: -1.017, \n",
      "train_vector: +0.00|-1.07|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.02|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.101, val_loss: -1.022, \n",
      "train_vector: +0.00|-1.10|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.02|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.129, val_loss: -1.155, \n",
      "train_vector: +0.00|-1.13|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.15|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.136, val_loss: -1.121, \n",
      "train_vector: +0.00|-1.14|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.12|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.161, val_loss: -1.118, \n",
      "train_vector: +0.00|-1.16|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.12|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.145, val_loss: -1.075, \n",
      "train_vector: +0.00|-1.14|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.07|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.151, val_loss: -1.133, \n",
      "train_vector: +0.00|-1.15|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.13|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.192, val_loss: -1.199, \n",
      "train_vector: +0.00|-1.19|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.20|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -1.202, val_loss: -0.928, \n",
      "train_vector: +0.00|-1.20|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.93|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -1.174, val_loss: -1.220, \n",
      "train_vector: +0.00|-1.17|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.22|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -1.180, val_loss: -1.178, \n",
      "train_vector: +0.00|-1.18|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.18|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -1.230, val_loss: -0.899, \n",
      "train_vector: +0.00|-1.23|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.90|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -1.222, val_loss: -1.167, \n",
      "train_vector: +0.00|-1.22|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.17|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -1.255, val_loss: -1.188, \n",
      "train_vector: +0.00|-1.26|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.19|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -1.258, val_loss: -1.247, \n",
      "train_vector: +0.00|-1.26|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.25|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:22, train_loss: -1.252, val_loss: -1.170, \n",
      "train_vector: +0.00|-1.25|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.17|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -1.218, val_loss: -1.252, \n",
      "train_vector: +0.00|-1.22|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.25|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -1.287, val_loss: -1.235, \n",
      "train_vector: +0.00|-1.29|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.23|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 481.2448470592499s\n",
      "\n",
      "Training type: 2\n",
      "epoch:0, train_loss: -1.740, val_loss: -1.659, \n",
      "train_vector: +0.00|+0.00|-1.74|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.66|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.753, val_loss: -1.658, \n",
      "train_vector: +0.00|+0.00|-1.75|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.66|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.772, val_loss: -1.687, \n",
      "train_vector: +0.00|+0.00|-1.77|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.69|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.785, val_loss: -1.701, \n",
      "train_vector: +0.00|+0.00|-1.79|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.70|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.797, val_loss: -1.693, \n",
      "train_vector: +0.00|+0.00|-1.80|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.69|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.815, val_loss: -1.710, \n",
      "train_vector: +0.00|+0.00|-1.81|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.71|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.829, val_loss: -1.696, \n",
      "train_vector: +0.00|+0.00|-1.83|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.70|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.840, val_loss: -1.728, \n",
      "train_vector: +0.00|+0.00|-1.84|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.73|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.850, val_loss: -1.726, \n",
      "train_vector: +0.00|+0.00|-1.85|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.73|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.863, val_loss: -1.742, \n",
      "train_vector: +0.00|+0.00|-1.86|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.74|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.878, val_loss: -1.722, \n",
      "train_vector: +0.00|+0.00|-1.88|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.72|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.887, val_loss: -1.764, \n",
      "train_vector: +0.00|+0.00|-1.89|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.76|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.898, val_loss: -1.757, \n",
      "train_vector: +0.00|+0.00|-1.90|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.76|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.910, val_loss: -1.760, \n",
      "train_vector: +0.00|+0.00|-1.91|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.76|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.921, val_loss: -1.768, \n",
      "train_vector: +0.00|+0.00|-1.92|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.77|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -1.931, val_loss: -1.788, \n",
      "train_vector: +0.00|+0.00|-1.93|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.79|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -1.942, val_loss: -1.790, \n",
      "train_vector: +0.00|+0.00|-1.94|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.79|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -1.947, val_loss: -1.784, \n",
      "train_vector: +0.00|+0.00|-1.95|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.78|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -1.962, val_loss: -1.800, \n",
      "train_vector: +0.00|+0.00|-1.96|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.80|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -1.969, val_loss: -1.811, \n",
      "train_vector: +0.00|+0.00|-1.97|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.81|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -1.977, val_loss: -1.809, \n",
      "train_vector: +0.00|+0.00|-1.98|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.81|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -1.988, val_loss: -1.815, \n",
      "train_vector: +0.00|+0.00|-1.99|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.81|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -1.995, val_loss: -1.814, \n",
      "train_vector: +0.00|+0.00|-2.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.81|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -2.005, val_loss: -1.823, \n",
      "train_vector: +0.00|+0.00|-2.01|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.82|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -2.015, val_loss: -1.826, \n",
      "train_vector: +0.00|+0.00|-2.02|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.83|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 1590.8499782085419s\n",
      "\n",
      "Training type: 3\n",
      "epoch:0, train_loss: -2.186, val_loss: -2.126, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.19|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.13|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -2.196, val_loss: -2.191, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.20|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.19|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -2.211, val_loss: -2.217, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.21|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.22|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -2.236, val_loss: -2.305, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.24|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.31|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -2.241, val_loss: -1.910, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.24|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.91|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -2.252, val_loss: -2.227, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.25|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.23|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -2.274, val_loss: -1.984, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.27|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.98|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -2.292, val_loss: -2.271, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.29|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.27|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -2.306, val_loss: -2.298, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.31|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.30|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -2.317, val_loss: -2.386, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.32|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.39|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -2.310, val_loss: -2.276, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.31|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.28|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -2.336, val_loss: -2.014, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.34|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.01|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -2.335, val_loss: -2.335, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.33|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.33|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -2.364, val_loss: -2.261, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.36|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.26|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -2.371, val_loss: -2.206, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.37|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.21|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -2.378, val_loss: -2.392, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.38|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.39|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -2.391, val_loss: -2.245, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.39|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.25|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -2.396, val_loss: -2.315, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.40|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.32|+0.00|+0.00|+0.00|+0.00\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:18, train_loss: -2.403, val_loss: -2.401, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.40|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.40|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -2.416, val_loss: -2.412, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.42|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.41|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -2.438, val_loss: -2.270, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.44|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.27|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -2.438, val_loss: -2.455, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.44|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.46|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -2.437, val_loss: -2.287, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.44|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.29|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -2.442, val_loss: -2.392, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.44|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.39|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -2.444, val_loss: -2.251, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.44|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.25|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 1547.6689801216125s\n",
      "\n",
      "Training type: 4\n",
      "epoch:0, train_loss: -2.300, val_loss: -2.218, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.30|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.22|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -2.321, val_loss: -2.220, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.32|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.22|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -2.340, val_loss: -2.169, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.34|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.17|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -2.361, val_loss: -2.212, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.36|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.21|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -2.368, val_loss: -2.173, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.37|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.17|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -2.389, val_loss: -2.225, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.39|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.22|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -2.402, val_loss: -2.249, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.40|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.25|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -2.430, val_loss: -2.238, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.43|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.24|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -2.433, val_loss: -2.249, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.43|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.25|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -2.442, val_loss: -2.244, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.44|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.24|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -2.464, val_loss: -2.316, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.46|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.32|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -2.474, val_loss: -2.299, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.47|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.30|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -2.491, val_loss: -2.320, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.49|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.32|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -2.504, val_loss: -2.294, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.50|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.29|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -2.517, val_loss: -2.340, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.52|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.34|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -2.537, val_loss: -2.308, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.54|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.31|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -2.536, val_loss: -2.345, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.54|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.34|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -2.562, val_loss: -2.362, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.56|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.36|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -2.570, val_loss: -2.354, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.57|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.35|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -2.582, val_loss: -2.345, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.58|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.35|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -2.593, val_loss: -2.318, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.59|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.32|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -2.610, val_loss: -2.325, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.61|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.32|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -2.614, val_loss: -2.336, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.61|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.34|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -2.623, val_loss: -2.328, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.62|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.33|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -2.641, val_loss: -2.347, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.64|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.35|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 716.0526428222656s\n",
      "\n",
      "Training type: 5\n",
      "epoch:0, train_loss: -1.337, val_loss: -1.242, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.34|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.24|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.351, val_loss: -1.245, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.35|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.24|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.365, val_loss: -1.250, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.37|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.25|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.378, val_loss: -1.244, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.38|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.24|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.391, val_loss: -1.272, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.39|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.27|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.404, val_loss: -1.276, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.40|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.28|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.415, val_loss: -1.286, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.41|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.29|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.427, val_loss: -1.303, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.43|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.30|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.440, val_loss: -1.307, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.44|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.31|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.450, val_loss: -1.312, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.45|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.31|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.462, val_loss: -1.323, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.46|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.32|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.472, val_loss: -1.335, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.47|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.34|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.480, val_loss: -1.329, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.48|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.33|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.490, val_loss: -1.346, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.49|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.35|+0.00|+0.00\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:14, train_loss: -1.502, val_loss: -1.348, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.50|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.35|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -1.512, val_loss: -1.349, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.51|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.35|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -1.521, val_loss: -1.369, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.52|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.37|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -1.527, val_loss: -1.359, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.53|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.36|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -1.539, val_loss: -1.376, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.54|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.38|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -1.544, val_loss: -1.373, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.54|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.37|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -1.554, val_loss: -1.371, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.55|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.37|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -1.562, val_loss: -1.377, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.56|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.38|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -1.570, val_loss: -1.378, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.57|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.38|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -1.575, val_loss: -1.392, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.57|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.39|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -1.586, val_loss: -1.393, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.59|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.39|+0.00|+0.00\n",
      "\n",
      "Training completed in 1613.044191122055s\n",
      "\n",
      "Training type: 6\n",
      "epoch:0, train_loss: -1.582, val_loss: -1.553, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.58|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.55|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.597, val_loss: -1.533, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.60|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.53|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.620, val_loss: -1.549, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.62|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.55|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.623, val_loss: -1.563, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.62|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.56|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.639, val_loss: -1.551, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.64|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.55|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.651, val_loss: -1.547, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.65|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.55|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.665, val_loss: -1.544, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.66|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.54|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.669, val_loss: -1.629, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.67|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.63|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.671, val_loss: -1.584, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.67|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.58|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.677, val_loss: -1.608, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.68|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.61|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.687, val_loss: -1.467, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.69|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.47|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.708, val_loss: -1.644, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.71|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.64|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.710, val_loss: -1.636, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.71|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.64|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.718, val_loss: -1.614, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.72|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.61|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.725, val_loss: -1.647, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.72|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.65|+0.00\n",
      "\n",
      "epoch:15, train_loss: -1.741, val_loss: -1.636, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.74|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.64|+0.00\n",
      "\n",
      "epoch:16, train_loss: -1.746, val_loss: -1.679, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.75|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.68|+0.00\n",
      "\n",
      "epoch:17, train_loss: -1.734, val_loss: -1.729, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.73|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.73|+0.00\n",
      "\n",
      "epoch:18, train_loss: -1.748, val_loss: -1.584, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.75|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.58|+0.00\n",
      "\n",
      "epoch:19, train_loss: -1.755, val_loss: -1.644, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.75|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.64|+0.00\n",
      "\n",
      "epoch:20, train_loss: -1.763, val_loss: -1.627, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.76|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.63|+0.00\n",
      "\n",
      "epoch:21, train_loss: -1.774, val_loss: -1.705, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.77|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.71|+0.00\n",
      "\n",
      "epoch:22, train_loss: -1.783, val_loss: -1.719, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.78|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.72|+0.00\n",
      "\n",
      "epoch:23, train_loss: -1.788, val_loss: -1.668, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.79|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.67|+0.00\n",
      "\n",
      "epoch:24, train_loss: -1.793, val_loss: -1.712, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.79|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.71|+0.00\n",
      "\n",
      "Training completed in 1487.5258553028107s\n",
      "\n",
      "Training type: 7\n",
      "epoch:0, train_loss: -2.602, val_loss: -2.352, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.60, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.35\n",
      "\n",
      "epoch:1, train_loss: -2.622, val_loss: -2.333, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.62, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.33\n",
      "\n",
      "epoch:2, train_loss: -2.644, val_loss: -2.391, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.64, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.39\n",
      "\n",
      "epoch:3, train_loss: -2.665, val_loss: -2.374, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.67, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.37\n",
      "\n",
      "epoch:4, train_loss: -2.674, val_loss: -2.374, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.67, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.37\n",
      "\n",
      "epoch:5, train_loss: -2.692, val_loss: -2.401, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.69, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.40\n",
      "\n",
      "epoch:6, train_loss: -2.689, val_loss: -2.368, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.69, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.37\n",
      "\n",
      "epoch:7, train_loss: -2.726, val_loss: -2.414, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.73, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.41\n",
      "\n",
      "epoch:8, train_loss: -2.738, val_loss: -2.394, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.74, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.39\n",
      "\n",
      "epoch:9, train_loss: -2.750, val_loss: -2.421, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.75, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.42\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:10, train_loss: -2.765, val_loss: -2.436, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.77, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.44\n",
      "\n",
      "epoch:11, train_loss: -2.778, val_loss: -2.439, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.78, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.44\n",
      "\n",
      "epoch:12, train_loss: -2.794, val_loss: -2.449, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.79, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.45\n",
      "\n",
      "epoch:13, train_loss: -2.814, val_loss: -2.437, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.81, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.44\n",
      "\n",
      "epoch:14, train_loss: -2.805, val_loss: -2.409, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.80, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.41\n",
      "\n",
      "epoch:15, train_loss: -2.839, val_loss: -2.453, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.84, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.45\n",
      "\n",
      "epoch:16, train_loss: -2.848, val_loss: -2.465, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.85, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.47\n",
      "\n",
      "epoch:17, train_loss: -2.855, val_loss: -2.488, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.86, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.49\n",
      "\n",
      "epoch:18, train_loss: -2.871, val_loss: -2.476, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.87, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.48\n",
      "\n",
      "epoch:19, train_loss: -2.888, val_loss: -2.477, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.89, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.48\n",
      "\n",
      "epoch:20, train_loss: -2.889, val_loss: -2.487, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.89, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.49\n",
      "\n",
      "epoch:21, train_loss: -2.895, val_loss: -2.463, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.90, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.46\n",
      "\n",
      "epoch:22, train_loss: -2.916, val_loss: -2.486, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.92, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.49\n",
      "\n",
      "epoch:23, train_loss: -2.921, val_loss: -2.489, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.92, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.49\n",
      "\n",
      "epoch:24, train_loss: -2.932, val_loss: -2.450, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.93, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.45\n",
      "\n",
      "Training completed in 818.4898104667664s\n"
     ]
    }
   ],
   "source": [
    "for i in range(8):\n",
    "    # load type data\n",
    "    data_type = '../Data/{}_data_ACSF_expand_type_'+str(i)+'.pickle'\n",
    "    train_dl,val_dl = get_data(data_type,batch_size)\n",
    "    \n",
    "    # init model\n",
    "    file_name = '../Model/{}.tar'.format('_'.join([str(i).split('}')[1] if '}' in str(i) else str(i) \\\n",
    "                                    for i in [reuse,block,head,data,batch_size,dim,clip,\\\n",
    "                                          layer1,layer2,factor,epochs,'type_'+str(i)+'_start']]))\n",
    "    checkpoint = torch.load(file_name)\n",
    "    model.load_state_dict(checkpoint['model_state_dict'])\n",
    "    opt.load_state_dict(checkpoint['optimizer_state_dict'])\n",
    "    scheduler = ReduceLROnPlateau(opt, 'min',factor=0.4,patience=6)\n",
    "    \n",
    "    print('\\nTraining type: {}'.format(i))\n",
    "    model,train_loss_perType,val_loss_perType = train(opt,model,epochs_type,train_dl,val_dl,paras,clip,True,scheduler=scheduler)\n",
    "\n",
    "    save_results(train_loss_perType,val_loss_perType,reuse,block,\\\n",
    "                 head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'type_'+str(i)+'_start')\n",
    "\n",
    "    save_model(model,opt,reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'type_'+str(i)+'_start')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### make submissions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_submission(reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'_start')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}