GNN_edge_5fold_0727_pseudoLabel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import torch\n",
    "from torch_geometric.data import Data,DataLoader\n",
    "from functions_refactor import *\n",
    "from pytorch_util import *\n",
    "from torch.optim import Adam\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fixed parameters\n",
    "block = MEGNet_block\n",
    "head_mol,head_atom,head_edge = head_mol,head_atom,head_edge\n",
    "clip = 2\n",
    "batch_size = 32\n",
    "reuse = False\n",
    "lr = 1e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# changing parameters\n",
    "head = feedforwardHead_Update\n",
    "data = '../Data/{}_data_SOAP_expand_PCA_otherInfo.pickle'\n",
    "dim = 256\n",
    "logLoss = False\n",
    "weight = 0.04\n",
    "layer1 = 4\n",
    "layer2 = 5\n",
    "factor = 4\n",
    "epochs = 110"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "prefix = '_'.join([str(i).split('}')[1] if '}' in str(i) else str(i) \\\n",
    "                                        for i in [head,data,dim,logLoss,weight,layer1,layer2,factor,epochs,'pseudoLabel']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv('../Data/train.csv')\n",
    "test_df = pd.read_csv('../Data/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "folds = []\n",
    "for f in range(5):\n",
    "    with open(data.format('train').split('pickle')[0][:-1]+'_f'+str(f)+'.pickle', 'rb') as handle:\n",
    "        folds.append(pickle.load(handle))\n",
    "folds = [[Data(**d) for d in fold] for fold in folds]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pseudoLabel for test\n",
    "data_pseudoLabel = '../Data/{}_data_ACSF_SOAP_atomInfo_otherInfo_pseudoLabel_0807.pickle'\n",
    "with open(data_pseudoLabel.format('test'), 'rb') as handle:\n",
    "    test_list_pseudoLabel = pickle.load(handle)\n",
    "    \n",
    "test_list_pseudoLabel = [Data(**d) for d in test_list_pseudoLabel]\n",
    "test_dl_pseudoLabel = DataLoader(test_list_pseudoLabel,batch_size,shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for i in range(5):\n",
    "    print('\\nstart fold '+str(i))\n",
    "    # parpare data\n",
    "    train_list = []\n",
    "    val_list = []\n",
    "    for j in range(5):\n",
    "        if i == j:\n",
    "            val_list.extend(folds[j])\n",
    "        else:\n",
    "            train_list.extend(folds[j])\n",
    "    \n",
    "    train_dl = DataLoader(train_list,batch_size,shuffle=True)\n",
    "    val_dl = DataLoader(val_list,batch_size,shuffle=False)\n",
    "    \n",
    "    # train model\n",
    "    model = GNN_multiHead(reuse,block,head,head_mol,head_atom,head_edge,\\\n",
    "                          dim,layer1,layer2,factor,**data_dict[data]).to('cuda:0')\n",
    "    paras = trainable_parameter(model)\n",
    "    opt = Adam(paras,lr=lr)\n",
    "    scheduler = ReduceLROnPlateau(opt, 'min',factor=0.5,patience=5)\n",
    "        \n",
    "    model,train_loss_perType,val_loss_perType,bestWeight = train_type_earlyStop_5fold_pseudoLabel(opt,model,epochs_type,train_dl,val_dl,paras,clip,\\\n",
    "                                                            scheduler=scheduler,logLoss=logLoss,weight=weight,test_dl=test_dl_pseudoLabel)\n",
    "    # predict oof for each type\n",
    "    for type_i in range(8):\n",
    "        # load val data and type_id\n",
    "        with open(data.format('train').split('pickle')[0][:-1]+'_f'+str(i)+'_type_'+str(type_i)+'.pickle', 'rb') as handle:\n",
    "            test_data = pickle.load(handle)\n",
    "        test_list = [Data(**d) for d in test_data]\n",
    "        test_dl = DataLoader(test_list,batch_size,shuffle=False)\n",
    "        \n",
    "        with open(data.format('train').split('pickle')[0][:-1]+'_f'+str(i)+'_type_'+str(type_i)+'_id.pickle', 'rb') as handle:\n",
    "            test_id = pickle.load(handle)\n",
    "    \n",
    "        # load model\n",
    "        model.load_state_dict(bestWeight[type_i])\n",
    "    \n",
    "        # predict\n",
    "        model.eval()\n",
    "        yhat_list = []\n",
    "        with torch.no_grad():\n",
    "            for data_torch in test_dl:\n",
    "                data_torch = data_torch.to('cuda:0')\n",
    "                yhat_list.append(model(data_torch,False,True))\n",
    "        yhat = torch.cat(yhat_list).cpu().detach().numpy()        \n",
    "    \n",
    "        # join\n",
    "        assert yhat.shape[0]==test_id.shape[0],'yhat and test_id should have same shape'\n",
    "        submit_ = dict(zip(test_id,yhat))\n",
    "        train_df['fold'+str(i)+'_type'+str(type_i)] = train_df.id.map(submit_)\n",
    "    \n",
    "    # predict test\n",
    "    for type_i in range(8):\n",
    "        # load val data and type_id\n",
    "        with open(data.format('test').split('pickle')[0][:-1]+'_type_'+str(type_i)+'.pickle', 'rb') as handle:\n",
    "            test_data = pickle.load(handle)\n",
    "        test_list = [Data(**d) for d in test_data]\n",
    "        test_dl = DataLoader(test_list,batch_size,shuffle=False)\n",
    "        \n",
    "        with open(data.format('test').split('pickle')[0][:-1]+'_id_type_'+str(type_i)+'.pickle', 'rb') as handle:\n",
    "            test_id = pickle.load(handle)\n",
    "    \n",
    "        # load model\n",
    "        model.load_state_dict(bestWeight[type_i])\n",
    "    \n",
    "        # predict\n",
    "        model.eval()\n",
    "        yhat_list = []\n",
    "        with torch.no_grad():\n",
    "            for data_torch in test_dl:\n",
    "                data_torch = data_torch.to('cuda:0')\n",
    "                yhat_list.append(model(data_torch,False,True))\n",
    "        yhat = torch.cat(yhat_list).cpu().detach().numpy()        \n",
    "    \n",
    "        # join\n",
    "        assert yhat.shape[0]==test_id.shape[0],'yhat and test_id should have same shape'\n",
    "        submit_ = dict(zip(test_id,yhat))\n",
    "        test_df['fold'+str(i)+'_type'+str(type_i)] = test_df.id.map(submit_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#assert set(test.iloc[:,5:].isnull().sum(1)) == set([7*5])\n",
    "test_df['yhat'] = np.nanmean(test_df.iloc[:,5:],1)\n",
    "#test = test[['id','yhat']]\n",
    "test_df.to_csv('../Data/test_oof_'+prefix,index=False)\n",
    "\n",
    "#assert set(train.iloc[:,6:].isnull().sum(1)) == set([train.iloc[:,6:].shape[1]-1])\n",
    "train_df['yhat'] = np.nanmean(train_df.iloc[:,6:],1)\n",
    "#train = train[['id','yhat']]\n",
    "train_df.to_csv('../Data/train_oof_'+prefix,index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}