GNN_refactor_NNConv_swapHead.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import torch\n",
    "from torch_geometric.data import Data,DataLoader\n",
    "from functions_refactor import *\n",
    "from pytorch_util import *\n",
    "from torch.optim import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# model parameters\n",
    "reuse = False\n",
    "block = NNConv_block\n",
    "head = swapHead_type\n",
    "data = '../Data/{}_data_ACSF_expand.pickle'\n",
    "batch_size = 32\n",
    "dim = 128\n",
    "epochs = 20\n",
    "clip = 2\n",
    "layer1 = 2\n",
    "layer2 = 2\n",
    "factor = 2\n",
    "lr = 1e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dl,val_dl = get_data(data,batch_size)\n",
    "\n",
    "model = GNN(reuse,block,head,dim,layer1,layer2,factor,**data_dict[data]).to('cuda:0')\n",
    "\n",
    "paras = trainable_parameter(model)\n",
    "opt = Adam(paras,lr=lr)\n",
    "\n",
    "model,train_loss_perType,val_loss_perType = train(opt,model,epochs,train_dl,val_dl,paras,clip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(train_loss_perType,val_loss_perType,reuse,block,\\\n",
    "             head,data,batch_size,dim,clip,layer1,layer2,factor,epochs)\n",
    "\n",
    "save_model(model,opt,reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### train on each type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = GNN(reuse,block,head,dim,layer1,layer2,factor,**data_dict[data]).to('cuda:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_type =1e-4/2\n",
    "epochs_type = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "paras = trainable_parameter(model)\n",
    "opt = Adam(paras,lr=lr_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name = !ls ../Model/\n",
    "file_name = file_name[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training type: 0\n",
      "epoch:0, train_loss: +0.069, val_loss: -0.064, \n",
      "train_vector: +0.07|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.06|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.026, val_loss: +0.310, \n",
      "train_vector: -0.03|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.31|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.072, val_loss: -0.131, \n",
      "train_vector: -0.07|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.13|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.105, val_loss: +0.163, \n",
      "train_vector: -0.11|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.16|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.132, val_loss: +0.020, \n",
      "train_vector: -0.13|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.02|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.155, val_loss: +0.028, \n",
      "train_vector: -0.16|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.03|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.175, val_loss: +0.066, \n",
      "train_vector: -0.18|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.07|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.192, val_loss: -0.094, \n",
      "train_vector: -0.19|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.09|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.218, val_loss: -0.088, \n",
      "train_vector: -0.22|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.09|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.231, val_loss: -0.252, \n",
      "train_vector: -0.23|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.25|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 380.4856460094452s\n",
      "\n",
      "Training type: 1\n",
      "epoch:0, train_loss: -0.521, val_loss: -0.726, \n",
      "train_vector: +0.00|-0.52|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.73|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.787, val_loss: -0.853, \n",
      "train_vector: +0.00|-0.79|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.85|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.847, val_loss: -0.755, \n",
      "train_vector: +0.00|-0.85|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.76|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.911, val_loss: -0.973, \n",
      "train_vector: +0.00|-0.91|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.97|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.932, val_loss: -0.683, \n",
      "train_vector: +0.00|-0.93|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.68|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.902, val_loss: -0.538, \n",
      "train_vector: +0.00|-0.90|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.54|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.982, val_loss: -0.792, \n",
      "train_vector: +0.00|-0.98|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.79|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.023, val_loss: -0.733, \n",
      "train_vector: +0.00|-1.02|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.73|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.036, val_loss: -1.089, \n",
      "train_vector: +0.00|-1.04|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.09|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.057, val_loss: -0.941, \n",
      "train_vector: +0.00|-1.06|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.94|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 128.70867013931274s\n",
      "\n",
      "Training type: 2\n",
      "epoch:0, train_loss: -0.659, val_loss: -0.741, \n",
      "train_vector: +0.00|+0.00|-0.66|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.74|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.749, val_loss: -0.806, \n",
      "train_vector: +0.00|+0.00|-0.75|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.81|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.811, val_loss: -0.887, \n",
      "train_vector: +0.00|+0.00|-0.81|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.89|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.857, val_loss: -0.928, \n",
      "train_vector: +0.00|+0.00|-0.86|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.93|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.898, val_loss: -0.911, \n",
      "train_vector: +0.00|+0.00|-0.90|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.91|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.933, val_loss: -0.959, \n",
      "train_vector: +0.00|+0.00|-0.93|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.96|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.964, val_loss: -1.018, \n",
      "train_vector: +0.00|+0.00|-0.96|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.02|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.990, val_loss: -1.027, \n",
      "train_vector: +0.00|+0.00|-0.99|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.03|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.015, val_loss: -0.996, \n",
      "train_vector: +0.00|+0.00|-1.02|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.036, val_loss: -1.097, \n",
      "train_vector: +0.00|+0.00|-1.04|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.10|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 390.7286388874054s\n",
      "\n",
      "Training type: 3\n",
      "epoch:0, train_loss: -1.660, val_loss: -1.816, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.66|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.82|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.762, val_loss: -1.855, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.76|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.86|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.808, val_loss: -1.805, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.81|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.81|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.848, val_loss: -1.540, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.85|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.54|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.872, val_loss: -2.007, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.87|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.01|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.900, val_loss: -1.911, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.90|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.91|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.924, val_loss: -1.902, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.92|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.90|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.949, val_loss: -1.720, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.95|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.72|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.964, val_loss: -1.897, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.96|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.90|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.979, val_loss: -1.659, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.98|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.66|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 382.73200273513794s\n",
      "\n",
      "Training type: 4\n",
      "epoch:0, train_loss: -1.537, val_loss: -1.694, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.54|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.69|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.684, val_loss: -1.733, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.68|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.73|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.731, val_loss: -1.710, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.73|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.71|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.773, val_loss: -1.846, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.77|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.85|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.809, val_loss: -1.832, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.81|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.83|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.828, val_loss: -1.912, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.83|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.91|+0.00|+0.00|+0.00\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:6, train_loss: -1.854, val_loss: -1.846, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.85|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.85|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.874, val_loss: -1.881, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.87|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.88|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.894, val_loss: -1.953, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.89|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.95|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.909, val_loss: -1.859, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.91|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.86|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 189.78208541870117s\n",
      "\n",
      "Training type: 5\n",
      "epoch:0, train_loss: +0.124, val_loss: +0.023, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.12|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.02|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.016, val_loss: -0.092, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.02|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.09|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.113, val_loss: -0.180, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.11|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.18|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.184, val_loss: -0.195, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.18|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.20|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.240, val_loss: -0.280, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.24|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.28|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.286, val_loss: -0.283, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.29|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.28|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.327, val_loss: -0.361, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.33|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.36|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.361, val_loss: -0.406, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.36|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.41|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.395, val_loss: -0.416, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.40|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.42|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.425, val_loss: -0.460, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.43|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.46|+0.00|+0.00\n",
      "\n",
      "Training completed in 395.52741169929504s\n",
      "\n",
      "Training type: 6\n",
      "epoch:0, train_loss: -0.495, val_loss: -0.631, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.49|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.63|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.639, val_loss: -0.688, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.64|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.69|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.715, val_loss: -0.753, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.72|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.75|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.772, val_loss: -0.804, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.77|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.80|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.819, val_loss: -0.809, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.82|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.81|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.857, val_loss: -0.869, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.86|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.87|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.888, val_loss: -0.944, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.89|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.94|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.916, val_loss: -0.930, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.92|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.93|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.951, val_loss: -0.936, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.95|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.94|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.977, val_loss: -0.953, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.98|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.95|+0.00\n",
      "\n",
      "Training completed in 356.16362047195435s\n",
      "\n",
      "Training type: 7\n",
      "epoch:0, train_loss: -1.462, val_loss: -1.531, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.46, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.53\n",
      "\n",
      "epoch:1, train_loss: -1.567, val_loss: -1.506, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.57, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.51\n",
      "\n",
      "epoch:2, train_loss: -1.622, val_loss: -1.601, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.62, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.60\n",
      "\n",
      "epoch:3, train_loss: -1.666, val_loss: -1.638, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.67, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.64\n",
      "\n",
      "epoch:4, train_loss: -1.699, val_loss: -1.654, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.70, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.65\n",
      "\n",
      "epoch:5, train_loss: -1.729, val_loss: -1.693, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.73, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.69\n",
      "\n",
      "epoch:6, train_loss: -1.759, val_loss: -1.663, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.76, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.66\n",
      "\n",
      "epoch:7, train_loss: -1.785, val_loss: -1.724, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.79, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.72\n",
      "\n",
      "epoch:8, train_loss: -1.810, val_loss: -1.729, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.81, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.73\n",
      "\n",
      "epoch:9, train_loss: -1.825, val_loss: -1.764, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.82, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.76\n",
      "\n",
      "Training completed in 213.36850094795227s\n"
     ]
    }
   ],
   "source": [
    "for i in range(8):\n",
    "    data_type = data.split('.pickle')[0]+ '_type_'+str(i)+'.pickle'\n",
    "    train_dl,val_dl = get_data(data_type,batch_size)\n",
    "\n",
    "    checkpoint = torch.load('../Model/'+file_name)\n",
    "    model.load_state_dict(checkpoint['model_state_dict'])\n",
    "    opt.load_state_dict(checkpoint['optimizer_state_dict'])\n",
    "\n",
    "    for para in opt.param_groups:\n",
    "        para['lr'] = lr_type\n",
    "    \n",
    "    print('\\nTraining type: {}'.format(i))\n",
    "    model,train_loss_perType,val_loss_perType = train(opt,model,epochs_type,train_dl,val_dl,paras,clip,True)\n",
    "\n",
    "    save_results(train_loss_perType,val_loss_perType,reuse,block,\\\n",
    "                 head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'type_'+str(i))\n",
    "\n",
    "    save_model(model,opt,reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'type_'+str(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train on each type from start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "head = swapHead_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_type =1e-4\n",
    "epochs_type = 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training type: 0\n",
      "epoch:0, train_loss: +0.892, val_loss: +0.112, \n",
      "train_vector: +0.89|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.11|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: +0.104, val_loss: -0.042, \n",
      "train_vector: +0.10|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.04|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.015, val_loss: -0.147, \n",
      "train_vector: -0.02|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.15|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.108, val_loss: -0.171, \n",
      "train_vector: -0.11|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.17|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.168, val_loss: -0.244, \n",
      "train_vector: -0.17|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.24|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.222, val_loss: -0.277, \n",
      "train_vector: -0.22|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.28|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.264, val_loss: -0.309, \n",
      "train_vector: -0.26|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.31|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.297, val_loss: -0.333, \n",
      "train_vector: -0.30|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.33|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.339, val_loss: -0.250, \n",
      "train_vector: -0.34|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.25|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.365, val_loss: -0.401, \n",
      "train_vector: -0.37|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.40|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -0.397, val_loss: -0.463, \n",
      "train_vector: -0.40|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.46|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -0.424, val_loss: -0.448, \n",
      "train_vector: -0.42|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.45|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -0.444, val_loss: -0.445, \n",
      "train_vector: -0.44|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.45|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -0.470, val_loss: -0.506, \n",
      "train_vector: -0.47|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.51|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -0.493, val_loss: -0.546, \n",
      "train_vector: -0.49|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.55|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -0.506, val_loss: -0.516, \n",
      "train_vector: -0.51|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.52|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -0.527, val_loss: -0.502, \n",
      "train_vector: -0.53|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.50|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -0.551, val_loss: -0.541, \n",
      "train_vector: -0.55|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.54|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -0.565, val_loss: -0.561, \n",
      "train_vector: -0.56|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.56|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -0.584, val_loss: -0.508, \n",
      "train_vector: -0.58|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.51|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -0.599, val_loss: -0.598, \n",
      "train_vector: -0.60|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.60|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -0.616, val_loss: -0.590, \n",
      "train_vector: -0.62|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.59|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -0.631, val_loss: -0.617, \n",
      "train_vector: -0.63|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.62|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -0.647, val_loss: -0.378, \n",
      "train_vector: -0.65|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.38|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -0.657, val_loss: -0.595, \n",
      "train_vector: -0.66|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.60|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 1608.5755877494812s\n",
      "\n",
      "Training type: 1\n",
      "epoch:0, train_loss: +0.944, val_loss: -0.002, \n",
      "train_vector: +0.00|+0.94|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.179, val_loss: -0.418, \n",
      "train_vector: +0.00|-0.18|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.42|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.342, val_loss: -0.581, \n",
      "train_vector: +0.00|-0.34|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.58|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.475, val_loss: -0.566, \n",
      "train_vector: +0.00|-0.47|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.57|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.566, val_loss: -0.581, \n",
      "train_vector: +0.00|-0.57|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.58|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.627, val_loss: -0.602, \n",
      "train_vector: +0.00|-0.63|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.60|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.659, val_loss: -0.740, \n",
      "train_vector: +0.00|-0.66|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.74|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.706, val_loss: -0.872, \n",
      "train_vector: +0.00|-0.71|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.87|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.761, val_loss: -0.901, \n",
      "train_vector: +0.00|-0.76|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.90|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.808, val_loss: -0.648, \n",
      "train_vector: +0.00|-0.81|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.65|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -0.837, val_loss: -0.967, \n",
      "train_vector: +0.00|-0.84|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.97|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -0.877, val_loss: -0.867, \n",
      "train_vector: +0.00|-0.88|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.87|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -0.873, val_loss: -0.823, \n",
      "train_vector: +0.00|-0.87|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.82|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -0.924, val_loss: -0.637, \n",
      "train_vector: +0.00|-0.92|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.64|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -0.944, val_loss: -1.031, \n",
      "train_vector: +0.00|-0.94|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.03|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -0.956, val_loss: -1.053, \n",
      "train_vector: +0.00|-0.96|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.05|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -1.000, val_loss: -1.084, \n",
      "train_vector: +0.00|-1.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.08|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -1.034, val_loss: -1.005, \n",
      "train_vector: +0.00|-1.03|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.01|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -1.028, val_loss: -1.012, \n",
      "train_vector: +0.00|-1.03|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.01|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -1.080, val_loss: -0.983, \n",
      "train_vector: +0.00|-1.08|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.98|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -1.078, val_loss: -1.139, \n",
      "train_vector: +0.00|-1.08|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.14|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -1.096, val_loss: -1.188, \n",
      "train_vector: +0.00|-1.10|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.19|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:22, train_loss: -1.102, val_loss: -0.929, \n",
      "train_vector: +0.00|-1.10|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.93|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -1.143, val_loss: -1.092, \n",
      "train_vector: +0.00|-1.14|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.09|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -1.114, val_loss: -1.020, \n",
      "train_vector: +0.00|-1.11|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.02|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 494.7766773700714s\n",
      "\n",
      "Training type: 2\n",
      "epoch:0, train_loss: -0.123, val_loss: -0.429, \n",
      "train_vector: +0.00|+0.00|-0.12|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.43|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.550, val_loss: -0.697, \n",
      "train_vector: +0.00|+0.00|-0.55|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.70|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.741, val_loss: -0.783, \n",
      "train_vector: +0.00|+0.00|-0.74|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.78|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.867, val_loss: -0.933, \n",
      "train_vector: +0.00|+0.00|-0.87|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.93|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.955, val_loss: -1.004, \n",
      "train_vector: +0.00|+0.00|-0.96|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.031, val_loss: -1.069, \n",
      "train_vector: +0.00|+0.00|-1.03|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.07|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.093, val_loss: -1.136, \n",
      "train_vector: +0.00|+0.00|-1.09|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.14|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.150, val_loss: -1.166, \n",
      "train_vector: +0.00|+0.00|-1.15|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.17|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.195, val_loss: -1.209, \n",
      "train_vector: +0.00|+0.00|-1.19|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.21|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.241, val_loss: -1.236, \n",
      "train_vector: +0.00|+0.00|-1.24|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.24|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.280, val_loss: -1.278, \n",
      "train_vector: +0.00|+0.00|-1.28|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.28|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.314, val_loss: -1.320, \n",
      "train_vector: +0.00|+0.00|-1.31|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.32|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.349, val_loss: -1.354, \n",
      "train_vector: +0.00|+0.00|-1.35|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.35|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.382, val_loss: -1.377, \n",
      "train_vector: +0.00|+0.00|-1.38|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.38|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.410, val_loss: -1.379, \n",
      "train_vector: +0.00|+0.00|-1.41|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.38|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -1.438, val_loss: -1.414, \n",
      "train_vector: +0.00|+0.00|-1.44|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.41|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -1.460, val_loss: -1.418, \n",
      "train_vector: +0.00|+0.00|-1.46|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.42|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -1.485, val_loss: -1.462, \n",
      "train_vector: +0.00|+0.00|-1.49|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.46|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -1.510, val_loss: -1.442, \n",
      "train_vector: +0.00|+0.00|-1.51|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.44|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -1.532, val_loss: -1.446, \n",
      "train_vector: +0.00|+0.00|-1.53|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.45|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -1.551, val_loss: -1.498, \n",
      "train_vector: +0.00|+0.00|-1.55|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.50|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -1.572, val_loss: -1.508, \n",
      "train_vector: +0.00|+0.00|-1.57|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.51|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -1.589, val_loss: -1.532, \n",
      "train_vector: +0.00|+0.00|-1.59|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.53|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -1.611, val_loss: -1.511, \n",
      "train_vector: +0.00|+0.00|-1.61|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.51|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -1.627, val_loss: -1.551, \n",
      "train_vector: +0.00|+0.00|-1.63|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.55|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 1592.6144738197327s\n",
      "\n",
      "Training type: 3\n",
      "epoch:0, train_loss: -0.613, val_loss: -1.188, \n",
      "train_vector: +0.00|+0.00|+0.00|-0.61|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.19|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.223, val_loss: -1.230, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.22|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.23|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.407, val_loss: -1.635, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.41|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.64|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.509, val_loss: -1.691, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.51|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.69|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.604, val_loss: -1.800, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.60|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.80|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.666, val_loss: -1.628, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.67|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.63|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.729, val_loss: -1.618, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.73|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.62|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.781, val_loss: -1.802, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.78|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.80|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.808, val_loss: -1.826, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.81|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.83|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.848, val_loss: -1.803, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.85|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.80|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.889, val_loss: -1.943, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.89|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.94|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.921, val_loss: -1.781, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.92|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.78|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.946, val_loss: -1.893, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.95|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.89|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.969, val_loss: -1.827, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.97|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.83|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -2.002, val_loss: -2.154, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.15|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -2.012, val_loss: -2.092, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.01|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.09|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -2.030, val_loss: -2.084, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.03|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.08|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -2.063, val_loss: -2.032, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.06|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.03|+0.00|+0.00|+0.00|+0.00\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:18, train_loss: -2.080, val_loss: -2.162, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.08|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.16|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -2.110, val_loss: -2.231, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.11|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.23|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -2.114, val_loss: -2.195, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.11|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.19|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -2.138, val_loss: -2.162, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.14|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.16|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -2.158, val_loss: -1.886, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.16|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.89|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -2.179, val_loss: -2.233, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.18|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.23|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -2.189, val_loss: -2.238, \n",
      "train_vector: +0.00|+0.00|+0.00|-2.19|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-2.24|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 1548.5870463848114s\n",
      "\n",
      "Training type: 4\n",
      "epoch:0, train_loss: -0.623, val_loss: -1.108, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-0.62|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.11|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.243, val_loss: -1.310, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.24|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.31|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.423, val_loss: -1.525, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.42|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.53|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.540, val_loss: -1.552, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.54|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.55|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.622, val_loss: -1.679, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.62|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.68|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.683, val_loss: -1.751, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.68|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.75|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.762, val_loss: -1.712, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.76|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.71|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.823, val_loss: -1.858, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.82|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.86|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.867, val_loss: -1.867, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.87|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.87|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.920, val_loss: -1.936, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.92|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.94|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.967, val_loss: -1.933, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.97|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.93|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -2.001, val_loss: -1.910, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.91|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -2.036, val_loss: -2.049, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.04|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.05|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -2.073, val_loss: -2.033, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.07|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.03|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -2.099, val_loss: -2.044, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.10|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.04|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -2.125, val_loss: -2.104, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.12|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.10|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -2.151, val_loss: -2.122, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.15|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.12|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -2.171, val_loss: -2.114, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.17|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.11|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -2.201, val_loss: -2.112, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.20|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.11|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -2.235, val_loss: -2.116, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.23|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.12|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -2.236, val_loss: -2.163, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.24|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.16|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -2.268, val_loss: -2.054, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.27|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.05|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -2.288, val_loss: -2.187, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.29|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.19|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -2.306, val_loss: -2.196, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.31|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.20|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -2.325, val_loss: -1.998, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-2.33|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 725.5519800186157s\n",
      "\n",
      "Training type: 5\n",
      "epoch:0, train_loss: +0.365, val_loss: +0.049, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.37|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.05|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.075, val_loss: -0.195, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.07|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.19|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.258, val_loss: -0.320, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.26|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.32|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.382, val_loss: -0.438, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.38|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.44|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.475, val_loss: -0.510, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.47|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.51|+0.00|+0.00\n",
      "\n",
      "epoch:5, train_loss: -0.550, val_loss: -0.576, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.55|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.58|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.612, val_loss: -0.628, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.61|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.63|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.667, val_loss: -0.647, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.67|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.65|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.713, val_loss: -0.710, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.71|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.71|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.757, val_loss: -0.758, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.76|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.76|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -0.793, val_loss: -0.779, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.79|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.78|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -0.829, val_loss: -0.808, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.83|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.81|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -0.860, val_loss: -0.818, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.86|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.82|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -0.891, val_loss: -0.854, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.89|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.85|+0.00|+0.00\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:14, train_loss: -0.917, val_loss: -0.883, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.92|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.88|+0.00|+0.00\n",
      "\n",
      "epoch:15, train_loss: -0.942, val_loss: -0.896, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.94|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.90|+0.00|+0.00\n",
      "\n",
      "epoch:16, train_loss: -0.964, val_loss: -0.922, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.96|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.92|+0.00|+0.00\n",
      "\n",
      "epoch:17, train_loss: -0.988, val_loss: -0.934, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.99|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.93|+0.00|+0.00\n",
      "\n",
      "epoch:18, train_loss: -1.008, val_loss: -0.958, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.01|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.96|+0.00|+0.00\n",
      "\n",
      "epoch:19, train_loss: -1.032, val_loss: -0.947, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.03|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.95|+0.00|+0.00\n",
      "\n",
      "epoch:20, train_loss: -1.046, val_loss: -0.981, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.05|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.98|+0.00|+0.00\n",
      "\n",
      "epoch:21, train_loss: -1.067, val_loss: -0.992, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.07|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.99|+0.00|+0.00\n",
      "\n",
      "epoch:22, train_loss: -1.085, val_loss: -1.005, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.08|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.00|+0.00|+0.00\n",
      "\n",
      "epoch:23, train_loss: -1.101, val_loss: -1.026, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.10|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.03|+0.00|+0.00\n",
      "\n",
      "epoch:24, train_loss: -1.116, val_loss: -1.037, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-1.12|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-1.04|+0.00|+0.00\n",
      "\n",
      "Training completed in 1618.184473991394s\n",
      "\n",
      "Training type: 6\n",
      "epoch:0, train_loss: +0.086, val_loss: -0.445, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.09|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.45|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.586, val_loss: -0.696, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.59|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.70|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.771, val_loss: -0.789, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.77|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.79|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.887, val_loss: -0.958, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.89|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.96|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.971, val_loss: -0.925, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.97|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.92|+0.00\n",
      "\n",
      "epoch:5, train_loss: -1.033, val_loss: -1.075, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.03|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.07|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.084, val_loss: -1.104, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.08|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.10|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.139, val_loss: -1.136, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.14|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.14|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.184, val_loss: -1.069, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.18|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.07|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.220, val_loss: -1.142, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.22|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.14|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.245, val_loss: -1.200, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.24|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.20|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.286, val_loss: -1.233, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.29|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.23|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.307, val_loss: -1.160, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.31|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.16|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.337, val_loss: -1.333, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.34|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.33|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.353, val_loss: -1.347, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.35|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.35|+0.00\n",
      "\n",
      "epoch:15, train_loss: -1.377, val_loss: -1.349, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.38|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.35|+0.00\n",
      "\n",
      "epoch:16, train_loss: -1.389, val_loss: -1.354, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.39|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.35|+0.00\n",
      "\n",
      "epoch:17, train_loss: -1.413, val_loss: -1.234, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.41|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.23|+0.00\n",
      "\n",
      "epoch:18, train_loss: -1.405, val_loss: -1.298, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.40|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.30|+0.00\n",
      "\n",
      "epoch:19, train_loss: -1.430, val_loss: -1.403, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.43|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.40|+0.00\n",
      "\n",
      "epoch:20, train_loss: -1.459, val_loss: -1.450, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.46|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.45|+0.00\n",
      "\n",
      "epoch:21, train_loss: -1.482, val_loss: -1.450, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.48|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.45|+0.00\n",
      "\n",
      "epoch:22, train_loss: -1.487, val_loss: -1.502, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.49|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.50|+0.00\n",
      "\n",
      "epoch:23, train_loss: -1.506, val_loss: -1.371, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.51|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.37|+0.00\n",
      "\n",
      "epoch:24, train_loss: -1.508, val_loss: -1.435, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.51|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.44|+0.00\n",
      "\n",
      "Training completed in 1492.8860969543457s\n",
      "\n",
      "Training type: 7\n",
      "epoch:0, train_loss: -0.845, val_loss: -0.993, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.84, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.99\n",
      "\n",
      "epoch:1, train_loss: -1.331, val_loss: -1.460, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.33, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.46\n",
      "\n",
      "epoch:2, train_loss: -1.529, val_loss: -1.582, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.53, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.58\n",
      "\n",
      "epoch:3, train_loss: -1.649, val_loss: -1.666, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.65, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.67\n",
      "\n",
      "epoch:4, train_loss: -1.745, val_loss: -1.731, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.75, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.73\n",
      "\n",
      "epoch:5, train_loss: -1.828, val_loss: -1.824, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.83, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.82\n",
      "\n",
      "epoch:6, train_loss: -1.899, val_loss: -1.886, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.90, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.89\n",
      "\n",
      "epoch:7, train_loss: -1.966, val_loss: -1.906, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.97, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.91\n",
      "\n",
      "epoch:8, train_loss: -2.026, val_loss: -1.953, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.03, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.95\n",
      "\n",
      "epoch:9, train_loss: -2.069, val_loss: -1.968, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.07, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.97\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:10, train_loss: -2.114, val_loss: -2.034, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.11, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.03\n",
      "\n",
      "epoch:11, train_loss: -2.153, val_loss: -2.050, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.15, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.05\n",
      "\n",
      "epoch:12, train_loss: -2.186, val_loss: -2.059, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.19, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.06\n",
      "\n",
      "epoch:13, train_loss: -2.225, val_loss: -2.118, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.23, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.12\n",
      "\n",
      "epoch:14, train_loss: -2.255, val_loss: -2.129, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.26, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.13\n",
      "\n",
      "epoch:15, train_loss: -2.281, val_loss: -2.177, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.28, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.18\n",
      "\n",
      "epoch:16, train_loss: -2.313, val_loss: -2.135, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.31, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.13\n",
      "\n",
      "epoch:17, train_loss: -2.344, val_loss: -2.201, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.34, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.20\n",
      "\n",
      "epoch:18, train_loss: -2.369, val_loss: -2.193, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.37, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.19\n",
      "\n",
      "epoch:19, train_loss: -2.395, val_loss: -2.245, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.40, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.24\n",
      "\n",
      "epoch:20, train_loss: -2.418, val_loss: -2.248, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.42, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.25\n",
      "\n",
      "epoch:21, train_loss: -2.441, val_loss: -2.274, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.44, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.27\n",
      "\n",
      "epoch:22, train_loss: -2.462, val_loss: -2.262, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.46, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.26\n",
      "\n",
      "epoch:23, train_loss: -2.486, val_loss: -2.290, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.49, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.29\n",
      "\n",
      "epoch:24, train_loss: -2.503, val_loss: -2.256, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.50, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-2.26\n",
      "\n",
      "Training completed in 823.6473083496094s\n"
     ]
    }
   ],
   "source": [
    "for i in range(8):\n",
    "    # load type data\n",
    "    data_type = '../Data/{}_data_ACSF_expand_type_'+str(i)+'.pickle'\n",
    "    train_dl,val_dl = get_data(data_type,batch_size)\n",
    "    \n",
    "    # init model\n",
    "    model = GNN(reuse,block,head,dim,layer1,layer2,factor,**data_dict[data]).to('cuda:0')\n",
    "    paras = trainable_parameter(model)\n",
    "    opt = Adam(paras,lr=lr_type)\n",
    "    \n",
    "    print('\\nTraining type: {}'.format(i))\n",
    "    model,train_loss_perType,val_loss_perType = train(opt,model,epochs_type,train_dl,val_dl,paras,clip,True)\n",
    "\n",
    "    save_results(train_loss_perType,val_loss_perType,reuse,block,\\\n",
    "                 head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'type_'+str(i)+'_start')\n",
    "\n",
    "    save_model(model,opt,reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'type_'+str(i)+'_start')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### two stage bigger head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_type =1e-4/2\n",
    "epochs_type = 10\n",
    "file_name = \"False_<class 'functions_refactor.schnet_block'>_<class 'functions_refactor.cat3Head'>__data_ACSF_expand.pickle_32_128_2_2_2_2_20_base.tar\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training type: 0\n",
      "\n",
      "Training head\n",
      "epoch:0, train_loss: +0.589, val_loss: +0.061, \n",
      "train_vector: +0.59|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.06|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: +0.008, val_loss: +0.214, \n",
      "train_vector: +0.01|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.21|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.005, val_loss: +0.116, \n",
      "train_vector: -0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.12|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.008, val_loss: +0.121, \n",
      "train_vector: -0.01|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.12|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.010, val_loss: +0.239, \n",
      "train_vector: -0.01|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.24|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 85.70982599258423s\n",
      "\n",
      "Training all\n",
      "epoch:5, train_loss: +0.050, val_loss: +0.155, \n",
      "train_vector: +0.05|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.15|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.012, val_loss: -0.143, \n",
      "train_vector: -0.01|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.14|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.050, val_loss: +0.034, \n",
      "train_vector: -0.05|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.03|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.095, val_loss: -0.247, \n",
      "train_vector: -0.09|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.25|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.113, val_loss: -0.093, \n",
      "train_vector: -0.11|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.09|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -0.148, val_loss: -0.185, \n",
      "train_vector: -0.15|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.19|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -0.163, val_loss: -0.129, \n",
      "train_vector: -0.16|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.13|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -0.185, val_loss: +0.175, \n",
      "train_vector: -0.19|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.18|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -0.210, val_loss: -0.260, \n",
      "train_vector: -0.21|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.26|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -0.226, val_loss: -0.156, \n",
      "train_vector: -0.23|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : -0.16|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 377.3226683139801s\n",
      "\n",
      "Training type: 1\n",
      "\n",
      "Training head\n",
      "epoch:0, train_loss: +0.832, val_loss: -0.475, \n",
      "train_vector: +0.00|+0.83|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.47|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.307, val_loss: -0.487, \n",
      "train_vector: +0.00|-0.31|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.49|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.381, val_loss: -0.612, \n",
      "train_vector: +0.00|-0.38|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.61|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.439, val_loss: -0.491, \n",
      "train_vector: +0.00|-0.44|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.49|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.482, val_loss: -0.450, \n",
      "train_vector: +0.00|-0.48|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.45|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 29.753875255584717s\n",
      "\n",
      "Training all\n",
      "epoch:5, train_loss: -0.593, val_loss: -0.850, \n",
      "train_vector: +0.00|-0.59|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.85|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.745, val_loss: -0.686, \n",
      "train_vector: +0.00|-0.74|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.69|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.769, val_loss: -0.298, \n",
      "train_vector: +0.00|-0.77|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.30|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.836, val_loss: -0.582, \n",
      "train_vector: +0.00|-0.84|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.58|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.817, val_loss: -1.131, \n",
      "train_vector: +0.00|-0.82|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.13|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -0.890, val_loss: -1.165, \n",
      "train_vector: +0.00|-0.89|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.16|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -0.864, val_loss: -0.995, \n",
      "train_vector: +0.00|-0.86|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.99|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -0.887, val_loss: -0.762, \n",
      "train_vector: +0.00|-0.89|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-0.76|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -0.921, val_loss: -1.242, \n",
      "train_vector: +0.00|-0.92|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.24|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -0.957, val_loss: -1.120, \n",
      "train_vector: +0.00|-0.96|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|-1.12|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 130.74536728858948s\n",
      "\n",
      "Training type: 2\n",
      "\n",
      "Training head\n",
      "epoch:0, train_loss: -0.454, val_loss: -0.591, \n",
      "train_vector: +0.00|+0.00|-0.45|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.59|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.625, val_loss: -0.609, \n",
      "train_vector: +0.00|+0.00|-0.62|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.61|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.640, val_loss: -0.642, \n",
      "train_vector: +0.00|+0.00|-0.64|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.64|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.651, val_loss: -0.643, \n",
      "train_vector: +0.00|+0.00|-0.65|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.64|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.659, val_loss: -0.676, \n",
      "train_vector: +0.00|+0.00|-0.66|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.68|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 90.99793410301208s\n",
      "\n",
      "Training all\n",
      "epoch:5, train_loss: -0.702, val_loss: -0.810, \n",
      "train_vector: +0.00|+0.00|-0.70|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.81|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.799, val_loss: -0.884, \n",
      "train_vector: +0.00|+0.00|-0.80|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.88|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.867, val_loss: -0.869, \n",
      "train_vector: +0.00|+0.00|-0.87|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.87|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.921, val_loss: -0.980, \n",
      "train_vector: +0.00|+0.00|-0.92|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-0.98|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.970, val_loss: -1.036, \n",
      "train_vector: +0.00|+0.00|-0.97|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.04|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.008, val_loss: -1.064, \n",
      "train_vector: +0.00|+0.00|-1.01|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.06|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.045, val_loss: -1.077, \n",
      "train_vector: +0.00|+0.00|-1.05|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.08|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.078, val_loss: -1.132, \n",
      "train_vector: +0.00|+0.00|-1.08|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.13|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.106, val_loss: -1.132, \n",
      "train_vector: +0.00|+0.00|-1.11|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.13|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.138, val_loss: -1.168, \n",
      "train_vector: +0.00|+0.00|-1.14|+0.00|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|-1.17|+0.00|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 394.71592378616333s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training type: 3\n",
      "\n",
      "Training head\n",
      "epoch:0, train_loss: -1.242, val_loss: -1.765, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.24|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.76|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.703, val_loss: -1.774, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.70|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.77|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.733, val_loss: -1.693, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.73|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.69|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.748, val_loss: -1.865, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.75|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.86|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.766, val_loss: -1.651, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.77|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.65|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 87.28697752952576s\n",
      "\n",
      "Training all\n",
      "epoch:5, train_loss: -1.664, val_loss: -1.873, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.66|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.87|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.735, val_loss: -1.177, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.73|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.18|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.769, val_loss: -1.918, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.77|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.92|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.785, val_loss: -1.711, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.78|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.71|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.835, val_loss: -1.970, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.84|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.97|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.869, val_loss: -1.968, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.87|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.97|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.887, val_loss: -1.926, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.89|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.93|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.898, val_loss: -1.785, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.90|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.79|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.928, val_loss: -1.965, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.93|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.97|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.947, val_loss: -1.816, \n",
      "train_vector: +0.00|+0.00|+0.00|-1.95|+0.00|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|-1.82|+0.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 379.6869592666626s\n",
      "\n",
      "Training type: 4\n",
      "\n",
      "Training head\n",
      "epoch:0, train_loss: -1.046, val_loss: -1.436, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.05|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.44|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: -1.472, val_loss: -1.587, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.47|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.59|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: -1.518, val_loss: -1.575, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.52|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.57|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: -1.551, val_loss: -1.541, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.55|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.54|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: -1.564, val_loss: -1.633, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.56|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.63|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 44.453752756118774s\n",
      "\n",
      "Training all\n",
      "epoch:5, train_loss: -1.581, val_loss: -1.561, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.58|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.56|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -1.683, val_loss: -1.759, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.68|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.76|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -1.740, val_loss: -1.836, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.74|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.84|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -1.770, val_loss: -1.857, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.77|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.86|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -1.817, val_loss: -1.901, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.82|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.90|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.848, val_loss: -1.742, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.85|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.74|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.874, val_loss: -1.880, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.87|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.88|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.893, val_loss: -1.884, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.89|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.88|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.915, val_loss: -1.967, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.91|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-1.97|+0.00|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.940, val_loss: -2.003, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|-1.94|+0.00|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|-2.00|+0.00|+0.00|+0.00\n",
      "\n",
      "Training completed in 192.33296394348145s\n",
      "\n",
      "Training type: 5\n",
      "\n",
      "Training head\n",
      "epoch:0, train_loss: +0.316, val_loss: +0.194, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.32|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.19|+0.00|+0.00\n",
      "\n",
      "epoch:1, train_loss: +0.194, val_loss: +0.184, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.19|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.18|+0.00|+0.00\n",
      "\n",
      "epoch:2, train_loss: +0.171, val_loss: +0.176, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.17|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.18|+0.00|+0.00\n",
      "\n",
      "epoch:3, train_loss: +0.149, val_loss: +0.131, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.15|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.13|+0.00|+0.00\n",
      "\n",
      "epoch:4, train_loss: +0.128, val_loss: +0.128, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.13|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.13|+0.00|+0.00\n",
      "\n",
      "Training completed in 94.3971426486969s\n",
      "\n",
      "Training all\n",
      "epoch:5, train_loss: -0.027, val_loss: -0.147, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.03|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.15|+0.00|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.203, val_loss: -0.285, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.20|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.28|+0.00|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.308, val_loss: -0.318, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.31|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.32|+0.00|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.386, val_loss: -0.389, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.39|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.39|+0.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.450, val_loss: -0.489, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.45|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.49|+0.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -0.506, val_loss: -0.507, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.51|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.51|+0.00|+0.00\n",
      "\n",
      "epoch:11, train_loss: -0.556, val_loss: -0.584, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.56|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.58|+0.00|+0.00\n",
      "\n",
      "epoch:12, train_loss: -0.600, val_loss: -0.598, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.60|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.60|+0.00|+0.00\n",
      "\n",
      "epoch:13, train_loss: -0.642, val_loss: -0.605, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.64|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.60|+0.00|+0.00\n",
      "\n",
      "epoch:14, train_loss: -0.678, val_loss: -0.705, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|-0.68|+0.00|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|-0.70|+0.00|+0.00\n",
      "\n",
      "Training completed in 405.3645977973938s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training type: 6\n",
      "\n",
      "Training head\n",
      "epoch:0, train_loss: -0.193, val_loss: -0.417, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.19|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.42|+0.00\n",
      "\n",
      "epoch:1, train_loss: -0.423, val_loss: -0.460, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.42|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.46|+0.00\n",
      "\n",
      "epoch:2, train_loss: -0.449, val_loss: -0.413, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.45|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.41|+0.00\n",
      "\n",
      "epoch:3, train_loss: -0.474, val_loss: -0.513, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.47|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.51|+0.00\n",
      "\n",
      "epoch:4, train_loss: -0.493, val_loss: -0.516, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.49|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.52|+0.00\n",
      "\n",
      "Training completed in 84.60186243057251s\n",
      "\n",
      "Training all\n",
      "epoch:5, train_loss: -0.620, val_loss: -0.683, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.62|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.68|+0.00\n",
      "\n",
      "epoch:6, train_loss: -0.775, val_loss: -0.741, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.78|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.74|+0.00\n",
      "\n",
      "epoch:7, train_loss: -0.870, val_loss: -0.925, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.87|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.93|+0.00\n",
      "\n",
      "epoch:8, train_loss: -0.929, val_loss: -1.004, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.93|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.00|+0.00\n",
      "\n",
      "epoch:9, train_loss: -0.986, val_loss: -1.001, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-0.99|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.00|+0.00\n",
      "\n",
      "epoch:10, train_loss: -1.027, val_loss: -1.021, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.03|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.02|+0.00\n",
      "\n",
      "epoch:11, train_loss: -1.072, val_loss: -1.078, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.07|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.08|+0.00\n",
      "\n",
      "epoch:12, train_loss: -1.110, val_loss: -1.127, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.11|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.13|+0.00\n",
      "\n",
      "epoch:13, train_loss: -1.143, val_loss: -1.151, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.14|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.15|+0.00\n",
      "\n",
      "epoch:14, train_loss: -1.169, val_loss: -1.180, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.17|+0.00, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.18|+0.00\n",
      "\n",
      "Training completed in 364.97120213508606s\n",
      "\n",
      "Training type: 7\n",
      "\n",
      "Training head\n",
      "epoch:0, train_loss: -1.089, val_loss: -1.322, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.09, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.32\n",
      "\n",
      "epoch:1, train_loss: -1.395, val_loss: -1.397, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.39, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.40\n",
      "\n",
      "epoch:2, train_loss: -1.437, val_loss: -1.468, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.44, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.47\n",
      "\n",
      "epoch:3, train_loss: -1.456, val_loss: -1.454, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.46, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.45\n",
      "\n",
      "epoch:4, train_loss: -1.479, val_loss: -1.448, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.48, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.45\n",
      "\n",
      "Training completed in 51.48229789733887s\n",
      "\n",
      "Training all\n",
      "epoch:5, train_loss: -1.500, val_loss: -1.539, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.50, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.54\n",
      "\n",
      "epoch:6, train_loss: -1.598, val_loss: -1.564, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.60, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.56\n",
      "\n",
      "epoch:7, train_loss: -1.665, val_loss: -1.536, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.67, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.54\n",
      "\n",
      "epoch:8, train_loss: -1.713, val_loss: -1.675, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.71, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.68\n",
      "\n",
      "epoch:9, train_loss: -1.758, val_loss: -1.668, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.76, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.67\n",
      "\n",
      "epoch:10, train_loss: -1.805, val_loss: -1.779, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.80, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.78\n",
      "\n",
      "epoch:11, train_loss: -1.847, val_loss: -1.769, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.85, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.77\n",
      "\n",
      "epoch:12, train_loss: -1.872, val_loss: -1.829, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.87, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.83\n",
      "\n",
      "epoch:13, train_loss: -1.905, val_loss: -1.827, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.90, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.83\n",
      "\n",
      "epoch:14, train_loss: -1.932, val_loss: -1.881, \n",
      "train_vector: +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.93, \n",
      "val_vector  : +0.00|+0.00|+0.00|+0.00|+0.00|+0.00|+0.00|-1.88\n",
      "\n",
      "Training completed in 220.315260887146s\n"
     ]
    }
   ],
   "source": [
    "for i in range(8):\n",
    "    # load type data\n",
    "    data_type = '../Data/{}_data_ACSF_expand_type_'+str(i)+'.pickle'\n",
    "    train_dl,val_dl = get_data(data_type,batch_size)\n",
    "    \n",
    "    # load base model\n",
    "    model = GNN(reuse,block,head,dim,layer1,layer2,factor,**data_dict[data]).to('cuda:0')\n",
    "    checkpoint = torch.load('../Model/'+file_name)\n",
    "    model.load_state_dict(checkpoint['model_state_dict'])\n",
    "\n",
    "    set_requires_grad(model, False)\n",
    "    model.head = cat3Head_type(dim,8,data_dict[data]['edge_in4']).to('cuda:0')\n",
    "    \n",
    "    paras = trainable_parameter(model)\n",
    "    opt = Adam(paras,lr=lr_type) \n",
    "    # train head\n",
    "    print('\\nTraining type: {}'.format(i))\n",
    "    print('\\nTraining head')\n",
    "    model,train_loss_perType,val_loss_perType = train(opt,model,5,train_dl,val_dl,paras,clip,True)\n",
    "    \n",
    "    # train all\n",
    "    set_requires_grad(model, True)\n",
    "    paras = trainable_parameter(model)\n",
    "    opt = Adam(paras,lr=lr_type)    \n",
    "    print('\\nTraining all')\n",
    "    model,train_loss_perType,val_loss_perType = train(opt,model,epochs_type,train_dl,val_dl,paras,clip,True,train_loss_perType,val_loss_perType)\n",
    "    \n",
    "    save_results(train_loss_perType,val_loss_perType,reuse,block,\\\n",
    "                 cat3Head_type,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'type_'+str(i)+'_2stage')\n",
    "\n",
    "    save_model(model,opt,reuse,block,cat3Head_type,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'type_'+str(i)+'_2stage')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### make submissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_submission(reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "head = cat3Head_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_submission(reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'_start')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2Stage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "head = cat3Head_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_submission(reuse,block,head,data,batch_size,dim,clip,layer1,layer2,factor,epochs,'_2stage')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}