etc/code_practices/ResNet18_CIFAR10_Train.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ResNet18_CIFAR10_Train",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOPIzvdWtH9nlTSWQL5mAew",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c33e7c07bfc847efa64429ee4eb640db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_987ce5cdc16f4c568f979204fa7506f2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0dea8eb11bbc4c779008942559cff4db",
              "IPY_MODEL_0ec3fad637ae4f5d8f9aa3a84e2cd7bd"
            ]
          }
        },
        "987ce5cdc16f4c568f979204fa7506f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0dea8eb11bbc4c779008942559cff4db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ff98ccdfba314fe5a6d9e477c6d4bd4b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a1d1034a83034942b9174d8a46447ef0"
          }
        },
        "0ec3fad637ae4f5d8f9aa3a84e2cd7bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_516e46a6995e4c10b928f2dbbbdd005a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 170500096/? [00:20&lt;00:00, 100483168.85it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1c9448bb0e7b4d4bb219f9313bff333f"
          }
        },
        "ff98ccdfba314fe5a6d9e477c6d4bd4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a1d1034a83034942b9174d8a46447ef0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "516e46a6995e4c10b928f2dbbbdd005a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1c9448bb0e7b4d4bb219f9313bff333f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ndb796/Deep-Learning-Paper-Review-and-Practice/blob/master/code_practices/ResNet18_CIFAR10_Train.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x2SSsQMSknm2"
      },
      "source": [
        "#### ResNet18 모델 정의 및 인스턴스 초기화"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zpUcgk5xkgGZ"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.backends.cudnn as cudnn\n",
        "import torch.optim as optim\n",
        "import os\n",
        "\n",
        "\n",
        "# ResNet18을 위해 최대한 간단히 수정한 BasicBlock 클래스 정의\n",
        "class BasicBlock(nn.Module):\n",
        "    def __init__(self, in_planes, planes, stride=1):\n",
        "        super(BasicBlock, self).__init__()\n",
        "\n",
        "        # 3x3 필터를 사용 (너비와 높이를 줄일 때는 stride 값 조절)\n",
        "        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(planes) # 배치 정규화(batch normalization)\n",
        "\n",
        "        # 3x3 필터를 사용 (패딩을 1만큼 주기 때문에 너비와 높이가 동일)\n",
        "        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)\n",
        "        self.bn2 = nn.BatchNorm2d(planes) # 배치 정규화(batch normalization)\n",
        "\n",
        "        self.shortcut = nn.Sequential() # identity인 경우\n",
        "        if stride != 1: # stride가 1이 아니라면, Identity mapping이 아닌 경우\n",
        "            self.shortcut = nn.Sequential(\n",
        "                nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride, bias=False),\n",
        "                nn.BatchNorm2d(planes)\n",
        "            )\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = F.relu(self.bn1(self.conv1(x)))\n",
        "        out = self.bn2(self.conv2(out))\n",
        "        out += self.shortcut(x) # (핵심) skip connection\n",
        "        out = F.relu(out)\n",
        "        return out\n",
        "\n",
        "\n",
        "# ResNet 클래스 정의\n",
        "class ResNet(nn.Module):\n",
        "    def __init__(self, block, num_blocks, num_classes=10):\n",
        "        super(ResNet, self).__init__()\n",
        "        self.in_planes = 64\n",
        "\n",
        "        # 64개의 3x3 필터(filter)를 사용\n",
        "        self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(64)\n",
        "        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)\n",
        "        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)\n",
        "        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)\n",
        "        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)\n",
        "        self.linear = nn.Linear(512, num_classes)\n",
        "\n",
        "    def _make_layer(self, block, planes, num_blocks, stride):\n",
        "        strides = [stride] + [1] * (num_blocks - 1)\n",
        "        layers = []\n",
        "        for stride in strides:\n",
        "            layers.append(block(self.in_planes, planes, stride))\n",
        "            self.in_planes = planes # 다음 레이어를 위해 채널 수 변경\n",
        "        return nn.Sequential(*layers)\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = F.relu(self.bn1(self.conv1(x)))\n",
        "        out = self.layer1(out)\n",
        "        out = self.layer2(out)\n",
        "        out = self.layer3(out)\n",
        "        out = self.layer4(out)\n",
        "        out = F.avg_pool2d(out, 4)\n",
        "        out = out.view(out.size(0), -1)\n",
        "        out = self.linear(out)\n",
        "        return out\n",
        "\n",
        "\n",
        "# ResNet18 함수 정의\n",
        "def ResNet18():\n",
        "    return ResNet(BasicBlock, [2, 2, 2, 2])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nCNacrgtktlr"
      },
      "source": [
        "#### 데이터셋(Dataset) 다운로드 및 불러오기"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EmmQZ8p5kq_C",
        "outputId": "ae6624d1-332f-412c-b2fc-51141afb1520",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125,
          "referenced_widgets": [
            "c33e7c07bfc847efa64429ee4eb640db",
            "987ce5cdc16f4c568f979204fa7506f2",
            "0dea8eb11bbc4c779008942559cff4db",
            "0ec3fad637ae4f5d8f9aa3a84e2cd7bd",
            "ff98ccdfba314fe5a6d9e477c6d4bd4b",
            "a1d1034a83034942b9174d8a46447ef0",
            "516e46a6995e4c10b928f2dbbbdd005a",
            "1c9448bb0e7b4d4bb219f9313bff333f"
          ]
        }
      },
      "source": [
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "\n",
        "transform_train = transforms.Compose([\n",
        "    transforms.RandomCrop(32, padding=4),\n",
        "    transforms.RandomHorizontalFlip(),\n",
        "    transforms.ToTensor(),\n",
        "])\n",
        "\n",
        "transform_test = transforms.Compose([\n",
        "    transforms.ToTensor(),\n",
        "])\n",
        "\n",
        "train_dataset = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train)\n",
        "test_dataset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)\n",
        "\n",
        "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=128, shuffle=True, num_workers=4)\n",
        "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=100, shuffle=False, num_workers=4)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar-10-python.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c33e7c07bfc847efa64429ee4eb640db",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ./data/cifar-10-python.tar.gz to ./data\n",
            "Files already downloaded and verified\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dl1-47E7pHD_"
      },
      "source": [
        "#### 환경 설정 및 학습(Training) 함수 정의"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bhm_eVykk-Z8"
      },
      "source": [
        "device = 'cuda'\n",
        "\n",
        "net = ResNet18()\n",
        "net = net.to(device)\n",
        "net = torch.nn.DataParallel(net)\n",
        "cudnn.benchmark = True\n",
        "\n",
        "learning_rate = 0.1\n",
        "file_name = 'resnet18_cifar10.pt'\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = optim.SGD(net.parameters(), lr=learning_rate, momentum=0.9, weight_decay=0.0002)\n",
        "\n",
        "\n",
        "def train(epoch):\n",
        "    print('\\n[ Train epoch: %d ]' % epoch)\n",
        "    net.train()\n",
        "    train_loss = 0\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    for batch_idx, (inputs, targets) in enumerate(train_loader):\n",
        "        inputs, targets = inputs.to(device), targets.to(device)\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        benign_outputs = net(inputs)\n",
        "        loss = criterion(benign_outputs, targets)\n",
        "        loss.backward()\n",
        "\n",
        "        optimizer.step()\n",
        "        train_loss += loss.item()\n",
        "        _, predicted = benign_outputs.max(1)\n",
        "\n",
        "        total += targets.size(0)\n",
        "        correct += predicted.eq(targets).sum().item()\n",
        "        \n",
        "        if batch_idx % 100 == 0:\n",
        "            print('\\nCurrent batch:', str(batch_idx))\n",
        "            print('Current benign train accuracy:', str(predicted.eq(targets).sum().item() / targets.size(0)))\n",
        "            print('Current benign train loss:', loss.item())\n",
        "\n",
        "    print('\\nTotal benign train accuarcy:', 100. * correct / total)\n",
        "    print('Total benign train loss:', train_loss)\n",
        "\n",
        "\n",
        "def test(epoch):\n",
        "    print('\\n[ Test epoch: %d ]' % epoch)\n",
        "    net.eval()\n",
        "    loss = 0\n",
        "    correct = 0\n",
        "    total = 0\n",
        "\n",
        "    for batch_idx, (inputs, targets) in enumerate(test_loader):\n",
        "        inputs, targets = inputs.to(device), targets.to(device)\n",
        "        total += targets.size(0)\n",
        "\n",
        "        outputs = net(inputs)\n",
        "        loss += criterion(outputs, targets).item()\n",
        "\n",
        "        _, predicted = outputs.max(1)\n",
        "        correct += predicted.eq(targets).sum().item()\n",
        "\n",
        "    print('\\nTest accuarcy:', 100. * correct / total)\n",
        "    print('Test average loss:', loss / total)\n",
        "\n",
        "    state = {\n",
        "        'net': net.state_dict()\n",
        "    }\n",
        "    if not os.path.isdir('checkpoint'):\n",
        "        os.mkdir('checkpoint')\n",
        "    torch.save(state, './checkpoint/' + file_name)\n",
        "    print('Model Saved!')\n",
        "\n",
        "\n",
        "def adjust_learning_rate(optimizer, epoch):\n",
        "    lr = learning_rate\n",
        "    if epoch >= 100:\n",
        "        lr /= 10\n",
        "    if epoch >= 150:\n",
        "        lr /= 10\n",
        "    for param_group in optimizer.param_groups:\n",
        "        param_group['lr'] = lr"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0mv5z7CEMRrn"
      },
      "source": [
        "#### 학습(Training) 진행"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IFdh-H1MPf0c"
      },
      "source": [
        "* 대략 20번의 epoch 이후에도 85%가량의 test accuracy를 얻을 수 있습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4voLj7TKlaB1",
        "outputId": "76463b85-2f4f-4bc0-a945-c73cc669c67e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# for epoch in range(0, 200):\n",
        "for epoch in range(0, 20):\n",
        "    adjust_learning_rate(optimizer, epoch)\n",
        "    train(epoch)\n",
        "    test(epoch)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "[ Train epoch: 0 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.078125\n",
            "Current benign train loss: 2.342548370361328\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.234375\n",
            "Current benign train loss: 1.9885613918304443\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.3984375\n",
            "Current benign train loss: 1.592935562133789\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.375\n",
            "Current benign train loss: 1.6573998928070068\n",
            "\n",
            "Total benign train accuarcy: 32.606\n",
            "Total benign train loss: 728.0853297710419\n",
            "\n",
            "[ Test epoch: 0 ]\n",
            "\n",
            "Test accuarcy: 32.37\n",
            "Test average loss: 0.019251463556289674\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 1 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.4765625\n",
            "Current benign train loss: 1.467000961303711\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.4609375\n",
            "Current benign train loss: 1.41520094871521\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.4765625\n",
            "Current benign train loss: 1.416500210762024\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.5390625\n",
            "Current benign train loss: 1.2315757274627686\n",
            "\n",
            "Total benign train accuarcy: 49.85\n",
            "Total benign train loss: 533.2066541910172\n",
            "\n",
            "[ Test epoch: 1 ]\n",
            "\n",
            "Test accuarcy: 45.72\n",
            "Test average loss: 0.015757604146003724\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 2 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.6484375\n",
            "Current benign train loss: 1.0243165493011475\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.5703125\n",
            "Current benign train loss: 1.1561983823776245\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.6328125\n",
            "Current benign train loss: 1.0709519386291504\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.6328125\n",
            "Current benign train loss: 1.077557921409607\n",
            "\n",
            "Total benign train accuarcy: 60.638\n",
            "Total benign train loss: 429.05279356241226\n",
            "\n",
            "[ Test epoch: 2 ]\n",
            "\n",
            "Test accuarcy: 55.7\n",
            "Test average loss: 0.012377450782060623\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 3 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.5859375\n",
            "Current benign train loss: 1.0553513765335083\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.7109375\n",
            "Current benign train loss: 0.8117691874504089\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.734375\n",
            "Current benign train loss: 0.782296895980835\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.7265625\n",
            "Current benign train loss: 0.8579391837120056\n",
            "\n",
            "Total benign train accuarcy: 66.628\n",
            "Total benign train loss: 368.8282547593117\n",
            "\n",
            "[ Test epoch: 3 ]\n",
            "\n",
            "Test accuarcy: 62.94\n",
            "Test average loss: 0.011110778278112412\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 4 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.6953125\n",
            "Current benign train loss: 0.8858131170272827\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.671875\n",
            "Current benign train loss: 0.869682788848877\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.734375\n",
            "Current benign train loss: 0.8007809519767761\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.6953125\n",
            "Current benign train loss: 0.8192347288131714\n",
            "\n",
            "Total benign train accuarcy: 70.534\n",
            "Total benign train loss: 325.6062932610512\n",
            "\n",
            "[ Test epoch: 4 ]\n",
            "\n",
            "Test accuarcy: 68.61\n",
            "Test average loss: 0.009179449623823166\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 5 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.7109375\n",
            "Current benign train loss: 0.7147080898284912\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.7265625\n",
            "Current benign train loss: 0.7322123050689697\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.7421875\n",
            "Current benign train loss: 0.7949008345603943\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.8046875\n",
            "Current benign train loss: 0.622573971748352\n",
            "\n",
            "Total benign train accuarcy: 74.776\n",
            "Total benign train loss: 283.41117030382156\n",
            "\n",
            "[ Test epoch: 5 ]\n",
            "\n",
            "Test accuarcy: 66.08\n",
            "Test average loss: 0.010336347603797912\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 6 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.7890625\n",
            "Current benign train loss: 0.6463558673858643\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.7421875\n",
            "Current benign train loss: 0.7392880916595459\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.71875\n",
            "Current benign train loss: 0.747829794883728\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.8046875\n",
            "Current benign train loss: 0.541568398475647\n",
            "\n",
            "Total benign train accuarcy: 78.13\n",
            "Total benign train loss: 246.6013989150524\n",
            "\n",
            "[ Test epoch: 6 ]\n",
            "\n",
            "Test accuarcy: 79.19\n",
            "Test average loss: 0.006162648651003837\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 7 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.8359375\n",
            "Current benign train loss: 0.4995025396347046\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.8203125\n",
            "Current benign train loss: 0.5459635853767395\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.7421875\n",
            "Current benign train loss: 0.7259557843208313\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.8203125\n",
            "Current benign train loss: 0.5176252126693726\n",
            "\n",
            "Total benign train accuarcy: 80.286\n",
            "Total benign train loss: 221.22545537352562\n",
            "\n",
            "[ Test epoch: 7 ]\n",
            "\n",
            "Test accuarcy: 79.12\n",
            "Test average loss: 0.005947562873363495\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 8 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.8203125\n",
            "Current benign train loss: 0.4396432638168335\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.8125\n",
            "Current benign train loss: 0.5141220092773438\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.8203125\n",
            "Current benign train loss: 0.510535478591919\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.828125\n",
            "Current benign train loss: 0.5050277709960938\n",
            "\n",
            "Total benign train accuarcy: 82.262\n",
            "Total benign train loss: 199.77128410339355\n",
            "\n",
            "[ Test epoch: 8 ]\n",
            "\n",
            "Test accuarcy: 75.92\n",
            "Test average loss: 0.007395538124442101\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 9 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.859375\n",
            "Current benign train loss: 0.36461132764816284\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.859375\n",
            "Current benign train loss: 0.4459194242954254\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.8203125\n",
            "Current benign train loss: 0.4709767997264862\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.90625\n",
            "Current benign train loss: 0.32980111241340637\n",
            "\n",
            "Total benign train accuarcy: 83.904\n",
            "Total benign train loss: 184.7817738056183\n",
            "\n",
            "[ Test epoch: 9 ]\n",
            "\n",
            "Test accuarcy: 80.95\n",
            "Test average loss: 0.0058574866235256194\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 10 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.828125\n",
            "Current benign train loss: 0.4373864531517029\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.8671875\n",
            "Current benign train loss: 0.3559974431991577\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.8203125\n",
            "Current benign train loss: 0.48067837953567505\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.875\n",
            "Current benign train loss: 0.3984924554824829\n",
            "\n",
            "Total benign train accuarcy: 84.892\n",
            "Total benign train loss: 171.66229909658432\n",
            "\n",
            "[ Test epoch: 10 ]\n",
            "\n",
            "Test accuarcy: 82.99\n",
            "Test average loss: 0.005103023992478847\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 11 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.875\n",
            "Current benign train loss: 0.3510317802429199\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.875\n",
            "Current benign train loss: 0.32915860414505005\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.921875\n",
            "Current benign train loss: 0.3297663927078247\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.8828125\n",
            "Current benign train loss: 0.34490451216697693\n",
            "\n",
            "Total benign train accuarcy: 86.196\n",
            "Total benign train loss: 156.00640574097633\n",
            "\n",
            "[ Test epoch: 11 ]\n",
            "\n",
            "Test accuarcy: 83.43\n",
            "Test average loss: 0.005025931619107723\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 12 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.8984375\n",
            "Current benign train loss: 0.316066712141037\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.828125\n",
            "Current benign train loss: 0.4465980529785156\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.875\n",
            "Current benign train loss: 0.3925016522407532\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.8125\n",
            "Current benign train loss: 0.5082284808158875\n",
            "\n",
            "Total benign train accuarcy: 86.922\n",
            "Total benign train loss: 147.18924406170845\n",
            "\n",
            "[ Test epoch: 12 ]\n",
            "\n",
            "Test accuarcy: 81.88\n",
            "Test average loss: 0.0055881109565496445\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 13 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.8828125\n",
            "Current benign train loss: 0.344944030046463\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.8828125\n",
            "Current benign train loss: 0.29982665181159973\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.9140625\n",
            "Current benign train loss: 0.26996269822120667\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.8828125\n",
            "Current benign train loss: 0.32363879680633545\n",
            "\n",
            "Total benign train accuarcy: 87.74\n",
            "Total benign train loss: 138.93322916328907\n",
            "\n",
            "[ Test epoch: 13 ]\n",
            "\n",
            "Test accuarcy: 84.9\n",
            "Test average loss: 0.004571232542395592\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 14 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.921875\n",
            "Current benign train loss: 0.22633010149002075\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.90625\n",
            "Current benign train loss: 0.2942282557487488\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.90625\n",
            "Current benign train loss: 0.2622967064380646\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.859375\n",
            "Current benign train loss: 0.35620009899139404\n",
            "\n",
            "Total benign train accuarcy: 88.148\n",
            "Total benign train loss: 133.99209225177765\n",
            "\n",
            "[ Test epoch: 14 ]\n",
            "\n",
            "Test accuarcy: 83.2\n",
            "Test average loss: 0.004986019828915596\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 15 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.8671875\n",
            "Current benign train loss: 0.3578110337257385\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.859375\n",
            "Current benign train loss: 0.44672730565071106\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.90625\n",
            "Current benign train loss: 0.3109784722328186\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.890625\n",
            "Current benign train loss: 0.321890652179718\n",
            "\n",
            "Total benign train accuarcy: 88.54\n",
            "Total benign train loss: 129.44860856235027\n",
            "\n",
            "[ Test epoch: 15 ]\n",
            "\n",
            "Test accuarcy: 83.95\n",
            "Test average loss: 0.00497876156270504\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 16 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.9296875\n",
            "Current benign train loss: 0.22490273416042328\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.8984375\n",
            "Current benign train loss: 0.23266059160232544\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.8671875\n",
            "Current benign train loss: 0.46236950159072876\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.8671875\n",
            "Current benign train loss: 0.36238518357276917\n",
            "\n",
            "Total benign train accuarcy: 89.232\n",
            "Total benign train loss: 121.14673912525177\n",
            "\n",
            "[ Test epoch: 16 ]\n",
            "\n",
            "Test accuarcy: 84.81\n",
            "Test average loss: 0.00471608787626028\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 17 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.8984375\n",
            "Current benign train loss: 0.3090561628341675\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.90625\n",
            "Current benign train loss: 0.28426840901374817\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.9140625\n",
            "Current benign train loss: 0.261736661195755\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.890625\n",
            "Current benign train loss: 0.2979798913002014\n",
            "\n",
            "Total benign train accuarcy: 89.666\n",
            "Total benign train loss: 117.36104936897755\n",
            "\n",
            "[ Test epoch: 17 ]\n",
            "\n",
            "Test accuarcy: 79.67\n",
            "Test average loss: 0.006324853277206421\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 18 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.9296875\n",
            "Current benign train loss: 0.20375630259513855\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.921875\n",
            "Current benign train loss: 0.3001909852027893\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.875\n",
            "Current benign train loss: 0.253219336271286\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.9140625\n",
            "Current benign train loss: 0.2335602045059204\n",
            "\n",
            "Total benign train accuarcy: 89.984\n",
            "Total benign train loss: 113.86838936805725\n",
            "\n",
            "[ Test epoch: 18 ]\n",
            "\n",
            "Test accuarcy: 83.34\n",
            "Test average loss: 0.00514583497941494\n",
            "Model Saved!\n",
            "\n",
            "[ Train epoch: 19 ]\n",
            "\n",
            "Current batch: 0\n",
            "Current benign train accuracy: 0.8984375\n",
            "Current benign train loss: 0.36447006464004517\n",
            "\n",
            "Current batch: 100\n",
            "Current benign train accuracy: 0.8828125\n",
            "Current benign train loss: 0.3062523901462555\n",
            "\n",
            "Current batch: 200\n",
            "Current benign train accuracy: 0.8828125\n",
            "Current benign train loss: 0.3175150454044342\n",
            "\n",
            "Current batch: 300\n",
            "Current benign train accuracy: 0.9296875\n",
            "Current benign train loss: 0.19365999102592468\n",
            "\n",
            "Total benign train accuarcy: 90.082\n",
            "Total benign train loss: 111.900165989995\n",
            "\n",
            "[ Test epoch: 19 ]\n",
            "\n",
            "Test accuarcy: 86.42\n",
            "Test average loss: 0.004216953121125698\n",
            "Model Saved!\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}