Pre_process_and_train1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Pre-process_and_train.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMRUVMdf7yBO8nHGvSV4lWp",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/BatoolMM/DeepSulfation/blob/master/Pre_process_and_train1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jUaFnVj1KG-P"
      },
      "source": [
        "!pip install -Uqq fastbook\n",
        "!pip install fastcore==1.0.9\n",
        "!pip install utils"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x-Hv1efgJZ1f"
      },
      "source": [
        "import pandas as pd\n",
        "import csv\n",
        "import numpy as np\n",
        "import keras.utils.np_utils as kutils\n",
        "import matplotlib.pyplot as plt\n",
        "from utils import *\n",
        "import fastbook\n",
        "from fastbook import *\n",
        "from fastai.vision.all import *\n",
        "from fastai.vision.widgets import ImageClassifierCleaner"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vwqk_-l1Juj-"
      },
      "source": [
        "file = '/content/CG-20-306_SD1.xlsx'\n",
        "training = pd.read_excel(file, sheet_name=0)\n",
        "test =pd.read_excel(file, sheet_name=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CFr8ZLiKSvTV",
        "outputId": "79a69dfa-3ad6-4445-8786-99018feeb476",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 111
        }
      },
      "source": [
        "training.head(2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>POSITIVE</th>\n",
              "      <th>Unnamed: 1</th>\n",
              "      <th>Unnamed: 2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>P14543</td>\n",
              "      <td>ATTNGVVPADVILGTEDGAEYDDEDEDYDLATTRLGLEDVG</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>P14543</td>\n",
              "      <td>PADVILGTEDGAEYDDEDEDYDLATTRLGLEDVGTTPFSYK</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  POSITIVE                                 Unnamed: 1  Unnamed: 2\n",
              "0   P14543  ATTNGVVPADVILGTEDGAEYDDEDEDYDLATTRLGLEDVG         2.0\n",
              "1   P14543  PADVILGTEDGAEYDDEDEDYDLATTRLGLEDVGTTPFSYK         2.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "juFU_v-qc9BW",
        "outputId": "78d0c4f5-7af7-492a-95af-14e7abee891d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 111
        }
      },
      "source": [
        "removed_row = training['POSITIVE'].eq('NEGATIVE')\n",
        "mask = removed_row.cumsum().eq(0)\n",
        "training['Target'] = np.where(mask, '1', '0')\n",
        "training = training[~removed_row].copy()\n",
        "training.head(2)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>POSITIVE</th>\n",
              "      <th>Unnamed: 1</th>\n",
              "      <th>Unnamed: 2</th>\n",
              "      <th>Target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>P14543</td>\n",
              "      <td>ATTNGVVPADVILGTEDGAEYDDEDEDYDLATTRLGLEDVG</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>P14543</td>\n",
              "      <td>PADVILGTEDGAEYDDEDEDYDLATTRLGLEDVGTTPFSYK</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  POSITIVE                                 Unnamed: 1  Unnamed: 2 Target\n",
              "0   P14543  ATTNGVVPADVILGTEDGAEYDDEDEDYDLATTRLGLEDVG         2.0      1\n",
              "1   P14543  PADVILGTEDGAEYDDEDEDYDLATTRLGLEDVGTTPFSYK         2.0      1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xnyobyNnc9Xe",
        "outputId": "d34654cf-c071-46d9-906f-181c33eaf196",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "training.columns"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['POSITIVE', 'Unnamed: 1', 'Unnamed: 2', 'Target'], dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RedTYxnOJuuS",
        "outputId": "1379a4f7-8d0b-4cef-90d4-7bd2126e88e1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "training.rename(\n",
        "    columns = {'POSITIVE':'ID',\n",
        "               'Unnamed: 1':'seq',\n",
        "               'Unnamed: 2': 'site',\n",
        "               'Target': 'Label'}, \n",
        "               inplace = True)\n",
        "training.drop('site', axis=1, inplace=True)\n",
        "\n",
        "def indexe(string, char):\n",
        "  for i, j in enumerate(char):\n",
        "    if j == char :\n",
        "      yield i\n",
        "\n",
        "def ind(my_list, desired_element):\n",
        "    return [index for index, element in enumerate(my_list) if element == desired_element]\n",
        "\n",
        "\n",
        "training['pos'] = training['seq'].apply(lambda x: ind(x,'Y'))\n",
        "training.head(5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ID</th>\n",
              "      <th>seq</th>\n",
              "      <th>Label</th>\n",
              "      <th>pos</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>P14543</td>\n",
              "      <td>ATTNGVVPADVILGTEDGAEYDDEDEDYDLATTRLGLEDVG</td>\n",
              "      <td>1</td>\n",
              "      <td>[20, 27]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>P14543</td>\n",
              "      <td>PADVILGTEDGAEYDDEDEDYDLATTRLGLEDVGTTPFSYK</td>\n",
              "      <td>1</td>\n",
              "      <td>[13, 20, 39]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>P10493</td>\n",
              "      <td>TAKGVVSADVNLDLDDDGADYEDEDYDLVTSHLGLEDVATP</td>\n",
              "      <td>1</td>\n",
              "      <td>[20, 25]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>P10493</td>\n",
              "      <td>VSADVNLDLDDDGADYEDEDYDLVTSHLGLEDVATPSPSHS</td>\n",
              "      <td>1</td>\n",
              "      <td>[15, 20]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>P13521</td>\n",
              "      <td>ENKPYALNSEKNFPMDMSDDYETQQWPERKLKHMQFPPMYE</td>\n",
              "      <td>1</td>\n",
              "      <td>[4, 20, 39]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       ID                                        seq Label           pos\n",
              "0  P14543  ATTNGVVPADVILGTEDGAEYDDEDEDYDLATTRLGLEDVG     1      [20, 27]\n",
              "1  P14543  PADVILGTEDGAEYDDEDEDYDLATTRLGLEDVGTTPFSYK     1  [13, 20, 39]\n",
              "2  P10493  TAKGVVSADVNLDLDDDGADYEDEDYDLVTSHLGLEDVATP     1      [20, 25]\n",
              "3  P10493  VSADVNLDLDDDGADYEDEDYDLVTSHLGLEDVATPSPSHS     1      [15, 20]\n",
              "4  P13521  ENKPYALNSEKNFPMDMSDDYETQQWPERKLKHMQFPPMYE     1   [4, 20, 39]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZcYhCwDqFl2m",
        "outputId": "6f88c9de-c25d-42e3-93c2-1710929a15e8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 111
        }
      },
      "source": [
        "test.tail(2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>POSITIVE</th>\n",
              "      <th>Unnamed: 1</th>\n",
              "      <th>Unnamed: 2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>159</th>\n",
              "      <td>P08238</td>\n",
              "      <td>KNIVKKCLELFSELAEDKENYKKFYEAFSKNLKLGIHEDST</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>160</th>\n",
              "      <td>Q03834</td>\n",
              "      <td>MLHSDLATFCLAIREEPGNFYNETQLDSSTIVQKLNTKIFG</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    POSITIVE                                 Unnamed: 1  Unnamed: 2\n",
              "159   P08238  KNIVKKCLELFSELAEDKENYKKFYEAFSKNLKLGIHEDST         NaN\n",
              "160   Q03834  MLHSDLATFCLAIREEPGNFYNETQLDSSTIVQKLNTKIFG         NaN"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DAyN9Dq-ERuR",
        "outputId": "15d19aba-2795-4c2c-efc5-3ca1474dedd2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "removed_row_test = test['POSITIVE'].eq('NEGATIVE')\n",
        "mask = removed_row_test.cumsum().eq(0)\n",
        "test['Target'] = np.where(mask, '1', '0')\n",
        "test = test[~removed_row_test].copy()\n",
        "test.head(5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>POSITIVE</th>\n",
              "      <th>Unnamed: 1</th>\n",
              "      <th>Unnamed: 2</th>\n",
              "      <th>Target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>P49682</td>\n",
              "      <td>DHQVLNDAEVAALLENFSSSYDYGENESDSCCTSPPCPQDF</td>\n",
              "      <td>6.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>P49682</td>\n",
              "      <td>QVLNDAEVAALLENFSSSYDYGENESDSCCTSPPCPQDFSL</td>\n",
              "      <td>6.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Q7M3V5</td>\n",
              "      <td>DLMDFLLEYEDEDRSKRFDDYGHMRFGKRGGEEQFDDYGHM</td>\n",
              "      <td>2.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Q8NBP7</td>\n",
              "      <td>LLLLLLGPAGARAQEDEDGDYEELVLALRSEEDGLAEAPEH</td>\n",
              "      <td>23.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Q9NZ53</td>\n",
              "      <td>PGSGFPSEENEESRILQPPQYFWEEEEELNDSSLDLGPTAD</td>\n",
              "      <td>7.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  POSITIVE                                 Unnamed: 1  Unnamed: 2 Target\n",
              "0   P49682  DHQVLNDAEVAALLENFSSSYDYGENESDSCCTSPPCPQDF         6.0      1\n",
              "1   P49682  QVLNDAEVAALLENFSSSYDYGENESDSCCTSPPCPQDFSL         6.0      1\n",
              "2   Q7M3V5  DLMDFLLEYEDEDRSKRFDDYGHMRFGKRGGEEQFDDYGHM         2.0      1\n",
              "3   Q8NBP7  LLLLLLGPAGARAQEDEDGDYEELVLALRSEEDGLAEAPEH        23.0      1\n",
              "4   Q9NZ53  PGSGFPSEENEESRILQPPQYFWEEEEELNDSSLDLGPTAD         7.0      1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dWoLYKwCESSy"
      },
      "source": [
        "test.rename(\n",
        "    columns = {'POSITIVE':'ID',\n",
        "               'Unnamed: 1':'seq',\n",
        "               'Unnamed: 2': 'site',\n",
        "               'Target': 'Label'},\n",
        "            inplace = True)\n",
        "test.drop('site', axis=1, inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z3_GACisyWWn",
        "outputId": "609bc3d8-f09a-4d54-9bd2-dca4d9c96306",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "#test['pos'] = test['seq'].apply(lambda x: for i, j in enumerate(x) if j = 'Y' )\n",
        "#test['pos'] = test['seq'].apply(lambda x: ind(x,'Y'))\n",
        "\n",
        "test['pos'] = test['seq'].apply(lambda x: ind(x,'Y'))\n",
        "\n",
        "test.head(5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ID</th>\n",
              "      <th>seq</th>\n",
              "      <th>Label</th>\n",
              "      <th>pos</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>P49682</td>\n",
              "      <td>DHQVLNDAEVAALLENFSSSYDYGENESDSCCTSPPCPQDF</td>\n",
              "      <td>1</td>\n",
              "      <td>[20, 22]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>P49682</td>\n",
              "      <td>QVLNDAEVAALLENFSSSYDYGENESDSCCTSPPCPQDFSL</td>\n",
              "      <td>1</td>\n",
              "      <td>[18, 20]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Q7M3V5</td>\n",
              "      <td>DLMDFLLEYEDEDRSKRFDDYGHMRFGKRGGEEQFDDYGHM</td>\n",
              "      <td>1</td>\n",
              "      <td>[8, 20, 37]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Q8NBP7</td>\n",
              "      <td>LLLLLLGPAGARAQEDEDGDYEELVLALRSEEDGLAEAPEH</td>\n",
              "      <td>1</td>\n",
              "      <td>[20]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Q9NZ53</td>\n",
              "      <td>PGSGFPSEENEESRILQPPQYFWEEEEELNDSSLDLGPTAD</td>\n",
              "      <td>1</td>\n",
              "      <td>[20]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       ID                                        seq Label          pos\n",
              "0  P49682  DHQVLNDAEVAALLENFSSSYDYGENESDSCCTSPPCPQDF     1     [20, 22]\n",
              "1  P49682  QVLNDAEVAALLENFSSSYDYGENESDSCCTSPPCPQDFSL     1     [18, 20]\n",
              "2  Q7M3V5  DLMDFLLEYEDEDRSKRFDDYGHMRFGKRGGEEQFDDYGHM     1  [8, 20, 37]\n",
              "3  Q8NBP7  LLLLLLGPAGARAQEDEDGDYEELVLALRSEEDGLAEAPEH     1         [20]\n",
              "4  Q9NZ53  PGSGFPSEENEESRILQPPQYFWEEEEELNDSSLDLGPTAD     1         [20]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hOkBFOF3NuVn"
      },
      "source": [
        "training.to_csv('cleaned_training.csv')\n",
        "test.to_csv('cleaned_test.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G7BPOtC6Bah8",
        "outputId": "96d2cf64-ae25-4467-96c3-ab5342536a45",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "test['pos'] = 21\n",
        "training['pos'] = 21\n",
        "test.head()\n",
        "training.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ID</th>\n",
              "      <th>seq</th>\n",
              "      <th>Label</th>\n",
              "      <th>pos</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>P14543</td>\n",
              "      <td>ATTNGVVPADVILGTEDGAEYDDEDEDYDLATTRLGLEDVG</td>\n",
              "      <td>1</td>\n",
              "      <td>21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>P14543</td>\n",
              "      <td>PADVILGTEDGAEYDDEDEDYDLATTRLGLEDVGTTPFSYK</td>\n",
              "      <td>1</td>\n",
              "      <td>21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>P10493</td>\n",
              "      <td>TAKGVVSADVNLDLDDDGADYEDEDYDLVTSHLGLEDVATP</td>\n",
              "      <td>1</td>\n",
              "      <td>21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>P10493</td>\n",
              "      <td>VSADVNLDLDDDGADYEDEDYDLVTSHLGLEDVATPSPSHS</td>\n",
              "      <td>1</td>\n",
              "      <td>21</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>P13521</td>\n",
              "      <td>ENKPYALNSEKNFPMDMSDDYETQQWPERKLKHMQFPPMYE</td>\n",
              "      <td>1</td>\n",
              "      <td>21</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       ID                                        seq Label  pos\n",
              "0  P14543  ATTNGVVPADVILGTEDGAEYDDEDEDYDLATTRLGLEDVG     1   21\n",
              "1  P14543  PADVILGTEDGAEYDDEDEDYDLATTRLGLEDVGTTPFSYK     1   21\n",
              "2  P10493  TAKGVVSADVNLDLDDDGADYEDEDYDLVTSHLGLEDVATP     1   21\n",
              "3  P10493  VSADVNLDLDDDGADYEDEDYDLVTSHLGLEDVATPSPSHS     1   21\n",
              "4  P13521  ENKPYALNSEKNFPMDMSDDYETQQWPERKLKHMQFPPMYE     1   21"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3SjPoijDMVei",
        "outputId": "10d094d7-5fd1-4be0-83b7-40deff21d0d5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        }
      },
      "source": [
        "cols_training = training.columns.tolist()\n",
        "#cols_training = cols_training[0] + cols_training[3] + cols_training[1] + cols_training[2]\n",
        "cols_training = cols_training[-1:] + cols_training[:-1]\n",
        "training = training[cols_training]\n",
        "training.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Label</th>\n",
              "      <th>pos</th>\n",
              "      <th>seq</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ID</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>P14543</th>\n",
              "      <td>1</td>\n",
              "      <td>21</td>\n",
              "      <td>ATTNGVVPADVILGTEDGAEYDDEDEDYDLATTRLGLEDVG</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>P14543</th>\n",
              "      <td>1</td>\n",
              "      <td>21</td>\n",
              "      <td>PADVILGTEDGAEYDDEDEDYDLATTRLGLEDVGTTPFSYK</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>P10493</th>\n",
              "      <td>1</td>\n",
              "      <td>21</td>\n",
              "      <td>TAKGVVSADVNLDLDDDGADYEDEDYDLVTSHLGLEDVATP</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>P10493</th>\n",
              "      <td>1</td>\n",
              "      <td>21</td>\n",
              "      <td>VSADVNLDLDDDGADYEDEDYDLVTSHLGLEDVATPSPSHS</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>P13521</th>\n",
              "      <td>1</td>\n",
              "      <td>21</td>\n",
              "      <td>ENKPYALNSEKNFPMDMSDDYETQQWPERKLKHMQFPPMYE</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       Label  pos                                        seq\n",
              "ID                                                          \n",
              "P14543     1   21  ATTNGVVPADVILGTEDGAEYDDEDEDYDLATTRLGLEDVG\n",
              "P14543     1   21  PADVILGTEDGAEYDDEDEDYDLATTRLGLEDVGTTPFSYK\n",
              "P10493     1   21  TAKGVVSADVNLDLDDDGADYEDEDYDLVTSHLGLEDVATP\n",
              "P10493     1   21  VSADVNLDLDDDGADYEDEDYDLVTSHLGLEDVATPSPSHS\n",
              "P13521     1   21  ENKPYALNSEKNFPMDMSDDYETQQWPERKLKHMQFPPMYE"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ph5db6cJBa9T"
      },
      "source": [
        "training.to_csv('cleaned_training_pos.csv')\n",
        "test.to_csv('cleaned_test_pos.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zJ2hbYAGBbIS"
      },
      "source": [
        "training['targetY'] = training['seq'].apply(lambda x: kutils.to_categorical(x))\n",
        "#targetY = kutils.to_categorical(training['seq'])\n",
        "\n",
        "ONE_HOT_SIZE = 21\n",
        "        # _aminos = 'ACDEFGHIKLMNPQRSTVWY*'\n",
        "letterDict = {}\n",
        "letterDict[\"A\"] = 0\n",
        "letterDict[\"C\"] = 1\n",
        "letterDict[\"D\"] = 2\n",
        "letterDict[\"E\"] = 3\n",
        "letterDict[\"F\"] = 4\n",
        "letterDict[\"G\"] = 5\n",
        "letterDict[\"H\"] = 6\n",
        "letterDict[\"I\"] = 7\n",
        "letterDict[\"K\"] = 8\n",
        "letterDict[\"L\"] = 9\n",
        "letterDict[\"M\"] = 10\n",
        "letterDict[\"N\"] = 11\n",
        "letterDict[\"P\"] = 12\n",
        "letterDict[\"Q\"] = 13\n",
        "letterDict[\"R\"] = 14\n",
        "letterDict[\"S\"] = 15\n",
        "letterDict[\"T\"] = 16\n",
        "letterDict[\"V\"] = 17\n",
        "letterDict[\"W\"] = 18\n",
        "letterDict[\"Y\"] = 19\n",
        "letterDict[\"*\"] = 20\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Gv2B3cFJu2R"
      },
      "source": [
        "def getMatrixLabel(positive_position_file_name,sites, window_size=51, empty_aa = '*'):\n",
        "    # input format   label, proteinName, postion,shortsequence,\n",
        "    prot = []  # list of protein name\n",
        "    pos = []  # list of position with protein name\n",
        "    rawseq = []\n",
        "    all_label = []\n",
        "\n",
        "    short_seqs = []\n",
        "    half_len = (window_size - 1) / 2\n",
        "\n",
        "    with open(positive_position_file_name, 'r') as rf:\n",
        "        reader = csv.reader(rf)\n",
        "        for row in reader:\n",
        "\n",
        "            position = int(row[2])\n",
        "            sseq = row[3]\n",
        "            rawseq.append(row[3])\n",
        "            center = sseq[position - 1]\n",
        "            if center in sites:\n",
        "                all_label.append(int(row[0]))\n",
        "                prot.append(row[1])\n",
        "                pos.append(row[2])\n",
        "\n",
        "                #short seq\n",
        "                if position - half_len > 0:\n",
        "                    start = position - half_len\n",
        "                    left_seq = sseq[start - 1:position - 1]\n",
        "                else:\n",
        "                    left_seq = sseq[0:position - 1]\n",
        "\n",
        "                end = len(sseq)\n",
        "                if position + half_len < end:\n",
        "                    end = position + half_len\n",
        "                right_seq = sseq[position:end]\n",
        "\n",
        "                if len(left_seq) < half_len:\n",
        "                    nb_lack = half_len - len(left_seq)\n",
        "                    left_seq = ''.join([empty_aa for count in range(nb_lack)]) + left_seq\n",
        "\n",
        "                if len(right_seq) < half_len:\n",
        "                    nb_lack = half_len - len(right_seq)\n",
        "                    right_seq = right_seq + ''.join([empty_aa for count in range(nb_lack)])\n",
        "                shortseq = left_seq + center + right_seq\n",
        "                short_seqs.append(shortseq)\n",
        "                # coding = one_hot_concat(shortseq)\n",
        "                # all_codings.append(coding)\n",
        "\n",
        "        targetY = kutils.to_categorical(all_label)\n",
        "\n",
        "        ONE_HOT_SIZE = 21\n",
        "        # _aminos = 'ACDEFGHIKLMNPQRSTVWY*'\n",
        "        letterDict = {}\n",
        "        letterDict[\"A\"] = 0\n",
        "        letterDict[\"C\"] = 1\n",
        "        letterDict[\"D\"] = 2\n",
        "        letterDict[\"E\"] = 3\n",
        "        letterDict[\"F\"] = 4\n",
        "        letterDict[\"G\"] = 5\n",
        "        letterDict[\"H\"] = 6\n",
        "        letterDict[\"I\"] = 7\n",
        "        letterDict[\"K\"] = 8\n",
        "        letterDict[\"L\"] = 9\n",
        "        letterDict[\"M\"] = 10\n",
        "        letterDict[\"N\"] = 11\n",
        "        letterDict[\"P\"] = 12\n",
        "        letterDict[\"Q\"] = 13\n",
        "        letterDict[\"R\"] = 14\n",
        "        letterDict[\"S\"] = 15\n",
        "        letterDict[\"T\"] = 16\n",
        "        letterDict[\"V\"] = 17\n",
        "        letterDict[\"W\"] = 18\n",
        "        letterDict[\"Y\"] = 19\n",
        "        letterDict[\"*\"] = 20\n",
        "\n",
        "        Matr = np.zeros((len(short_seqs), window_size, ONE_HOT_SIZE))\n",
        "        samplenumber = 0\n",
        "        for seq in short_seqs:\n",
        "            AANo = 0\n",
        "            for AA in seq:\n",
        "                index = letterDict[AA]\n",
        "                Matr[samplenumber][AANo][index] = 1\n",
        "                AANo = AANo+1\n",
        "            samplenumber = samplenumber + 1\n",
        "\n",
        "    return Matr, targetY\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}