Copy_of_DCEC.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of DCEC.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOmbY1YKzo9gcjm9aRNGltF",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/RushaliRajesh/DCEC/blob/main/Copy_of_DCEC.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "aK1MqIZXvjob",
        "outputId": "16859ea0-60af-4e91-85c1-ae9068573d54"
      },
      "source": [
        "from keras.layers import Conv2D, Conv2DTranspose, Dense, Flatten, Reshape\n",
        "from keras.models import Sequential, Model\n",
        "from keras.utils.vis_utils import plot_model\n",
        "import numpy as np\n",
        "from sklearn.metrics import normalized_mutual_info_score, adjusted_rand_score\n",
        "\n",
        "nmi = normalized_mutual_info_score\n",
        "ari = adjusted_rand_score\n",
        "\n",
        "\n",
        "def acc(y_true, y_pred):\n",
        "    \"\"\"\n",
        "    Calculate clustering accuracy. Require scikit-learn installed\n",
        "    # Arguments\n",
        "        y: true labels, numpy.array with shape `(n_samples,)`\n",
        "        y_pred: predicted labels, numpy.array with shape `(n_samples,)`\n",
        "    # Return\n",
        "        accuracy, in [0,1]\n",
        "    \"\"\"\n",
        "    y_true = y_true.astype(np.int64)\n",
        "    assert y_pred.size == y_true.size\n",
        "    D = max(y_pred.max(), y_true.max()) + 1\n",
        "    w = np.zeros((D, D), dtype=np.int64)\n",
        "    for i in range(y_pred.size):\n",
        "        w[y_pred[i], y_true[i]] += 1\n",
        "    !pip install scikit-learn==0.22.2\n",
        "    from sklearn.utils.linear_assignment_ import linear_assignment\n",
        "    ind = linear_assignment(w.max() - w)\n",
        "    return float(sum([w[i, j] for i, j in ind])) * 1.0 / y_pred.size\n",
        "\n",
        "def CAE(input_shape=(28, 28, 1), filters=[32, 64, 128, 10]):\n",
        "    model = Sequential()\n",
        "    if input_shape[0] % 8 == 0:\n",
        "        pad3 = 'same'\n",
        "    else:\n",
        "        pad3 = 'valid'\n",
        "    model.add(Conv2D(filters[0], 5, strides=2, padding='same', activation='relu', name='conv1', input_shape=input_shape))\n",
        "\n",
        "    model.add(Conv2D(filters[1], 5, strides=2, padding='same', activation='relu', name='conv2'))\n",
        "\n",
        "    model.add(Conv2D(filters[2], 3, strides=2, padding=pad3, activation='relu', name='conv3'))\n",
        "\n",
        "    model.add(Flatten())\n",
        "    model.add(Dense(units=filters[3], name='embedding'))\n",
        "    model.add(Dense(units=filters[2]*int(input_shape[0]/8)*int(input_shape[0]/8), activation='relu'))\n",
        "\n",
        "    model.add(Reshape((int(input_shape[0]/8), int(input_shape[0]/8), filters[2])))\n",
        "    model.add(Conv2DTranspose(filters[1], 3, strides=2, padding=pad3, activation='relu', name='deconv3'))\n",
        "\n",
        "    model.add(Conv2DTranspose(filters[0], 5, strides=2, padding='same', activation='relu', name='deconv2'))\n",
        "\n",
        "    model.add(Conv2DTranspose(input_shape[2], 5, strides=2, padding='same', name='deconv1'))\n",
        "    model.summary()\n",
        "    return model\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    from time import time\n",
        "\n",
        "    # setting the hyper parameters\n",
        "    import argparse\n",
        "\n",
        "    class Args:\n",
        "      dataset='mnist'\n",
        "      n_clusters=10\n",
        "      batch_size=256\n",
        "      epochs=20\n",
        "      save_dir='results/temp'\n",
        "    args=Args()\n",
        "    print(args)\n",
        "\n",
        "    import os\n",
        "    if not os.path.exists(args.save_dir):\n",
        "        os.makedirs(args.save_dir)\n",
        "\n",
        "\n",
        "    import numpy as np\n",
        "\n",
        "\n",
        "    def load_mnist():\n",
        "        # the data, shuffled and split between train and test sets\n",
        "        from keras.datasets import mnist\n",
        "        (x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
        "\n",
        "        x = np.concatenate((x_train, x_test))\n",
        "        y = np.concatenate((y_train, y_test))\n",
        "        x = x.reshape(-1, 28, 28, 1).astype('float32')\n",
        "        x = x/255.\n",
        "        print('MNIST:', x.shape)\n",
        "        return x, y\n",
        "\n",
        "\n",
        "    # # load dataset\n",
        "    # from datasets import load_mnist, load_usps\n",
        "    # if args.dataset == 'mnist':\n",
        "    x, y = load_mnist()\n",
        "    #print(x)\n",
        "    # elif args.dataset == 'usps':\n",
        "    #     x, y = load_usps('data/usps')\n",
        "\n",
        "    # define the model\n",
        "    model = CAE(input_shape=x.shape[1:], filters=[32, 64, 128, 10])\n",
        "    plot_model(model, to_file=args.save_dir + '/%s-pretrain-model.png' % args.dataset, show_shapes=True)\n",
        "    model.summary()\n",
        "\n",
        "    # compile the model and callbacks\n",
        "    optimizer = 'adam'\n",
        "    model.compile(optimizer=optimizer, loss='mse')\n",
        "    \n",
        "    # begin training\n",
        "    t0 = time()\n",
        "    model.fit(x, x, batch_size=args.batch_size, epochs=args.epochs)\n",
        "    print('Training time: ', time() - t0)\n",
        "    #model.save(args.save_dir + '/%s-pretrain-model-%d.h5' % (args.dataset, args.epochs))\n",
        "\n",
        "    # extract features\n",
        "    feature_model = Model(inputs=model.input, outputs=model.get_layer(name='embedding').output)\n",
        "    features = feature_model.predict(x)\n",
        "    print('feature shape=', features.shape)\n",
        "\n",
        "    # use features for clustering\n",
        "    from sklearn.cluster import KMeans\n",
        "    km = KMeans(n_clusters=args.n_clusters)\n",
        "\n",
        "    features = np.reshape(features, newshape=(features.shape[0], -1))\n",
        "    pred = km.fit_predict(features)\n",
        "\n",
        "    from sklearn.metrics.cluster import normalized_mutual_info_score   \n",
        "    from sklearn.metrics import accuracy_score \n",
        "    from sklearn.metrics.cluster import adjusted_rand_score\n",
        "    #from . import metrics\n",
        "    print('acc=', acc(y, pred), 'nmi=', nmi(y, pred), 'ari=', ari(y, pred))"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<__main__.Args object at 0x7fb721f08550>\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 0s 0us/step\n",
            "11501568/11490434 [==============================] - 0s 0us/step\n",
            "MNIST: (70000, 28, 28, 1)\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv1 (Conv2D)              (None, 14, 14, 32)        832       \n",
            "                                                                 \n",
            " conv2 (Conv2D)              (None, 7, 7, 64)          51264     \n",
            "                                                                 \n",
            " conv3 (Conv2D)              (None, 3, 3, 128)         73856     \n",
            "                                                                 \n",
            " flatten (Flatten)           (None, 1152)              0         \n",
            "                                                                 \n",
            " embedding (Dense)           (None, 10)                11530     \n",
            "                                                                 \n",
            " dense (Dense)               (None, 1152)              12672     \n",
            "                                                                 \n",
            " reshape (Reshape)           (None, 3, 3, 128)         0         \n",
            "                                                                 \n",
            " deconv3 (Conv2DTranspose)   (None, 7, 7, 64)          73792     \n",
            "                                                                 \n",
            " deconv2 (Conv2DTranspose)   (None, 14, 14, 32)        51232     \n",
            "                                                                 \n",
            " deconv1 (Conv2DTranspose)   (None, 28, 28, 1)         801       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 275,979\n",
            "Trainable params: 275,979\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv1 (Conv2D)              (None, 14, 14, 32)        832       \n",
            "                                                                 \n",
            " conv2 (Conv2D)              (None, 7, 7, 64)          51264     \n",
            "                                                                 \n",
            " conv3 (Conv2D)              (None, 3, 3, 128)         73856     \n",
            "                                                                 \n",
            " flatten (Flatten)           (None, 1152)              0         \n",
            "                                                                 \n",
            " embedding (Dense)           (None, 10)                11530     \n",
            "                                                                 \n",
            " dense (Dense)               (None, 1152)              12672     \n",
            "                                                                 \n",
            " reshape (Reshape)           (None, 3, 3, 128)         0         \n",
            "                                                                 \n",
            " deconv3 (Conv2DTranspose)   (None, 7, 7, 64)          73792     \n",
            "                                                                 \n",
            " deconv2 (Conv2DTranspose)   (None, 14, 14, 32)        51232     \n",
            "                                                                 \n",
            " deconv1 (Conv2DTranspose)   (None, 28, 28, 1)         801       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 275,979\n",
            "Trainable params: 275,979\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/20\n",
            "274/274 [==============================] - 83s 299ms/step - loss: 0.0438\n",
            "Epoch 2/20\n",
            "274/274 [==============================] - 82s 299ms/step - loss: 0.0193\n",
            "Epoch 3/20\n",
            "274/274 [==============================] - 82s 298ms/step - loss: 0.0167\n",
            "Epoch 4/20\n",
            "274/274 [==============================] - 82s 298ms/step - loss: 0.0154\n",
            "Epoch 5/20\n",
            "274/274 [==============================] - 82s 298ms/step - loss: 0.0146\n",
            "Epoch 6/20\n",
            "274/274 [==============================] - 84s 305ms/step - loss: 0.0141\n",
            "Epoch 7/20\n",
            "274/274 [==============================] - 82s 299ms/step - loss: 0.0136\n",
            "Epoch 8/20\n",
            "274/274 [==============================] - 82s 300ms/step - loss: 0.0133\n",
            "Epoch 9/20\n",
            "274/274 [==============================] - 82s 300ms/step - loss: 0.0130\n",
            "Epoch 10/20\n",
            "274/274 [==============================] - 82s 301ms/step - loss: 0.0127\n",
            "Epoch 11/20\n",
            "274/274 [==============================] - 83s 301ms/step - loss: 0.0125\n",
            "Epoch 12/20\n",
            "274/274 [==============================] - 83s 302ms/step - loss: 0.0123\n",
            "Epoch 13/20\n",
            "274/274 [==============================] - 83s 304ms/step - loss: 0.0121\n",
            "Epoch 14/20\n",
            "274/274 [==============================] - 84s 306ms/step - loss: 0.0119\n",
            "Epoch 15/20\n",
            "274/274 [==============================] - 83s 302ms/step - loss: 0.0117\n",
            "Epoch 16/20\n",
            "274/274 [==============================] - 83s 302ms/step - loss: 0.0116\n",
            "Epoch 17/20\n",
            "274/274 [==============================] - 83s 302ms/step - loss: 0.0115\n",
            "Epoch 18/20\n",
            "274/274 [==============================] - 83s 302ms/step - loss: 0.0113\n",
            "Epoch 19/20\n",
            "274/274 [==============================] - 83s 302ms/step - loss: 0.0112\n",
            "Epoch 20/20\n",
            "274/274 [==============================] - 83s 302ms/step - loss: 0.0111\n",
            "Training time:  1651.209172487259\n",
            "feature shape= (70000, 10)\n",
            "Collecting scikit-learn==0.22.2\n",
            "  Downloading scikit_learn-0.22.2-cp37-cp37m-manylinux1_x86_64.whl (7.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 7.1 MB 4.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Installing collected packages: scikit-learn\n",
            "  Attempting uninstall: scikit-learn\n",
            "    Found existing installation: scikit-learn 1.0.1\n",
            "    Uninstalling scikit-learn-1.0.1:\n",
            "      Successfully uninstalled scikit-learn-1.0.1\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "imbalanced-learn 0.8.1 requires scikit-learn>=0.24, but you have scikit-learn 0.22.2 which is incompatible.\u001b[0m\n",
            "Successfully installed scikit-learn-0.22.2\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "sklearn"
                ]
              }
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "acc= 0.8072285714285714 nmi= 0.7411959865499761 ari= 0.689977924408917\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:22: FutureWarning: The linear_assignment_ module is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "---qJVmz5RCr",
        "outputId": "2a857fb8-5369-4bc7-b0be-6228b40a5504"
      },
      "source": [
        "def acc(y_true, y_pred):\n",
        "    \"\"\"\n",
        "    Calculate clustering accuracy. Require scikit-learn installed\n",
        "    # Arguments\n",
        "        y: true labels, numpy.array with shape `(n_samples,)`\n",
        "        y_pred: predicted labels, numpy.array with shape `(n_samples,)`\n",
        "    # Return\n",
        "        accuracy, in [0,1]\n",
        "    \"\"\"\n",
        "    y_true = y_true.astype(np.int64)\n",
        "    assert y_pred.size == y_true.size\n",
        "    D = max(y_pred.max(), y_true.max()) + 1\n",
        "    w = np.zeros((D, D), dtype=np.int64)\n",
        "    print(w)\n",
        "    for i in range(y_pred.size):\n",
        "        w[y_pred[i], y_true[i]] += 1\n",
        "    print(w)\n",
        "    !pip install scikit-learn==0.22.2\n",
        "    from sklearn.utils.linear_assignment_ import linear_assignment\n",
        "    ind = linear_assignment(w.max() - w)\n",
        "    print(ind)\n",
        "    return float(sum(([w[i, j] for i, j in ind])) * 1.0 / y_pred.size)\n",
        "acc(y,pred)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[[0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]\n",
            " [0 0 0 0 0 0 0 0 0 0]]\n",
            "[[  35 7791   56   36   39   31  189   93   93   70]\n",
            " [6756    0  101   20    6   33   51   19   38   41]\n",
            " [   2    2   56   74    3    2    0 6670   12   57]\n",
            " [   0   56 6527   64    2    0    0   61    2    1]\n",
            " [   3    0    0   86    0 5628  220    0   52    6]\n",
            " [   0    8   16   25 3711   17    6  187   43 2527]\n",
            " [  41    1   26   10   24  154 6354    0   34    1]\n",
            " [  30   12   18    7 3037   20    5  235   85 4096]\n",
            " [  34    7  102  184    2  112   50   26 6375  100]\n",
            " [   2    0   88 6635    0  316    1    2   91   59]]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "[[0 1]\n",
            " [1 0]\n",
            " [2 7]\n",
            " [3 2]\n",
            " [4 5]\n",
            " [5 4]\n",
            " [6 6]\n",
            " [7 9]\n",
            " [8 8]\n",
            " [9 3]]\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "0.8649"
            ]
          },
          "execution_count": 29,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zXM_7ivdappe"
      },
      "source": [
        "from scipy.optimize import linear_sum_assignment as linear_assignment"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gmwBDqTWl04E",
        "outputId": "81551b2e-b66e-41bb-9e90-d8de6274dd05"
      },
      "source": [
        "pip install ."
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[31mERROR: Directory '.' is not installable. Neither 'setup.py' nor 'pyproject.toml' found.\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J-kUku7M8v3a",
        "outputId": "7b232731-4385-4bf1-e32c-0d64b7c6d7db"
      },
      "source": [
        "from time import time\n",
        "import numpy as np\n",
        "import keras.backend as K\n",
        "from tensorflow.keras.layers import Layer, InputSpec\n",
        "from keras.models import Model\n",
        "from keras.utils.vis_utils import plot_model\n",
        "from sklearn.cluster import KMeans\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Conv2D, Conv2DTranspose, Dense, Flatten, Reshape\n",
        "\n",
        "#from ConvAE import CAE\n",
        "\n",
        "\n",
        "class ClusteringLayer(Layer):\n",
        "    \"\"\"\n",
        "    Clustering layer converts input sample (feature) to soft label, i.e. a vector that represents the probability of the\n",
        "    sample belonging to each cluster. The probability is calculated with student's t-distribution.\n",
        "    # Example\n",
        "    ```\n",
        "        model.add(ClusteringLayer(n_clusters=10))\n",
        "    ```\n",
        "    # Arguments\n",
        "        n_clusters: number of clusters.\n",
        "        weights: list of Numpy array with shape `(n_clusters, n_features)` witch represents the initial cluster centers.\n",
        "        alpha: parameter in Student's t-distribution. Default to 1.0.\n",
        "    # Input shape\n",
        "        2D tensor with shape: `(n_samples, n_features)`.\n",
        "    # Output shape\n",
        "        2D tensor with shape: `(n_samples, n_clusters)`.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, n_clusters, weights=None, alpha=1.0, **kwargs):\n",
        "        if 'input_shape' not in kwargs and 'input_dim' in kwargs:\n",
        "            kwargs['input_shape'] = (kwargs.pop('input_dim'),)\n",
        "        super(ClusteringLayer, self).__init__(**kwargs)\n",
        "        self.n_clusters = n_clusters\n",
        "        self.alpha = alpha\n",
        "        self.initial_weights = weights\n",
        "        self.input_spec = InputSpec(ndim=2)\n",
        "\n",
        "    def build(self, input_shape):\n",
        "        assert len(input_shape) == 2\n",
        "        input_dim = input_shape[1]\n",
        "        self.input_spec = InputSpec(dtype=K.floatx(), shape=(None, input_dim))\n",
        "        self.clusters = self.add_weight(shape=(self.n_clusters, input_dim), initializer='glorot_uniform', name='clusters')\n",
        "        if self.initial_weights is not None:\n",
        "            self.set_weights(self.initial_weights)\n",
        "            del self.initial_weights\n",
        "        self.built = True\n",
        "\n",
        "    def call(self, inputs, **kwargs):\n",
        "        \"\"\" student t-distribution, as same as used in t-SNE algorithm.\n",
        "                 q_ij = 1/(1+dist(x_i, u_j)^2), then normalize it.\n",
        "        Arguments:\n",
        "            inputs: the variable containing data, shape=(n_samples, n_features)\n",
        "        Return:\n",
        "            q: student's t-distribution, or soft labels for each sample. shape=(n_samples, n_clusters)\n",
        "        \"\"\"\n",
        "        q = 1.0 / (1.0 + (K.sum(K.square(K.expand_dims(inputs, axis=1) - self.clusters), axis=2) / self.alpha))\n",
        "        q **= (self.alpha + 1.0) / 2.0\n",
        "        q = K.transpose(K.transpose(q) / K.sum(q, axis=1))\n",
        "        return q\n",
        "\n",
        "    def compute_output_shape(self, input_shape):\n",
        "        assert input_shape and len(input_shape) == 2\n",
        "        return input_shape[0], self.n_clusters\n",
        "\n",
        "    def get_config(self):\n",
        "        config = {'n_clusters': self.n_clusters}\n",
        "        base_config = super(ClusteringLayer, self).get_config()\n",
        "        return dict(list(base_config.items()) + list(config.items()))\n",
        "\n",
        "\n",
        "class DCEC(object):\n",
        "    def __init__(self,\n",
        "                 input_shape,\n",
        "                 filters=[32, 64, 128, 10],\n",
        "                 n_clusters=10,\n",
        "                 alpha=1.0):\n",
        "\n",
        "        super(DCEC, self).__init__()\n",
        "\n",
        "        self.n_clusters = n_clusters\n",
        "        self.input_shape = input_shape\n",
        "        self.alpha = alpha\n",
        "        self.pretrained = False\n",
        "        self.y_pred = []\n",
        "\n",
        "        self.cae = CAE(input_shape, filters)\n",
        "        hidden = self.cae.get_layer(name='embedding').output\n",
        "        self.encoder = Model(inputs=self.cae.input, outputs=hidden)\n",
        "\n",
        "        # Define DCEC model\n",
        "        clustering_layer = ClusteringLayer(self.n_clusters, name='clustering')(hidden)\n",
        "        self.model = Model(inputs=self.cae.input,\n",
        "                           outputs=[clustering_layer, self.cae.output])\n",
        "\n",
        "    def pretrain(self, x, batch_size=256, epochs=20, optimizer='adam', save_dir='results/temp'):\n",
        "        print('...Pretraining...')\n",
        "        self.cae.compile(optimizer=optimizer, loss='mse')\n",
        "        #from keras.callbacks import CSVLogger\n",
        "        #csv_logger = CSVLogger(args.save_dir + '/pretrain_log.csv')\n",
        "\n",
        "        # begin training\n",
        "        t0 = time()\n",
        "        self.cae.fit(x, x, batch_size=batch_size, epochs=epochs)\n",
        "        print('Pretraining time: ', time() - t0)\n",
        "        self.cae.save(save_dir + '/pretrain_cae_model.h5')\n",
        "        print('Pretrained weights are saved to %s/pretrain_cae_model.h5' % save_dir)\n",
        "        self.pretrained = True\n",
        "\n",
        "    def load_weights(self, weights_path):\n",
        "        self.model.load_weights(weights_path)\n",
        "\n",
        "    def extract_feature(self, x):  # extract features from before clustering layer\n",
        "        return self.encoder.predict(x)\n",
        "\n",
        "    def predict(self, x):\n",
        "        q, _ = self.model.predict(x, verbose=0)\n",
        "        return q.argmax(1)\n",
        "\n",
        "    @staticmethod\n",
        "    def target_distribution(q):\n",
        "        weight = q ** 2 / q.sum(0)\n",
        "        return (weight.T / weight.sum(1)).T\n",
        "\n",
        "    def compile(self, loss=['kld', 'mse'], loss_weights=[1, 1], optimizer='adam'):\n",
        "        self.model.compile(loss=loss, loss_weights=loss_weights, optimizer=optimizer)\n",
        "\n",
        "    def fit(self, x, y=None, batch_size=256, maxiter=2e4, tol=1e-3,\n",
        "            update_interval=140, cae_weights=None, save_dir='./results/temp'):\n",
        "\n",
        "        print('Update interval', update_interval)\n",
        "        save_interval = x.shape[0] / batch_size * 5\n",
        "        print('Save interval', save_interval)\n",
        "\n",
        "        # Step 1: pretrain if necessary\n",
        "        t0 = time()\n",
        "        if not self.pretrained and cae_weights is None:\n",
        "            print('...pretraining CAE using default hyper-parameters:')\n",
        "            print('   optimizer=\\'adam\\';   epochs=200')\n",
        "            self.pretrain(x, batch_size, save_dir=save_dir)\n",
        "            self.pretrained = True\n",
        "        elif cae_weights is not None:\n",
        "            self.cae.load_weights(cae_weights)\n",
        "            print('cae_weights is loaded successfully.')\n",
        "\n",
        "        # Step 2: initialize cluster centers using k-means\n",
        "        t1 = time()\n",
        "        print('Initializing cluster centers with k-means.')\n",
        "        kmeans = KMeans(n_clusters=self.n_clusters, n_init=20)\n",
        "        self.y_pred = kmeans.fit_predict(self.encoder.predict(x))\n",
        "        y_pred_last = np.copy(self.y_pred)\n",
        "        self.model.get_layer(name='clustering').set_weights([kmeans.cluster_centers_])\n",
        "\n",
        "        # Step 3: deep clustering\n",
        "        # logging file\n",
        "        import csv, os\n",
        "        if not os.path.exists(save_dir):\n",
        "            os.makedirs(save_dir)\n",
        "        logfile = open(save_dir + '/dcec_log.csv', 'w')\n",
        "        logwriter = csv.DictWriter(logfile, fieldnames=['iter', 'acc', 'nmi', 'ari', 'L', 'Lc', 'Lr'])\n",
        "        logwriter.writeheader()\n",
        "\n",
        "        t2 = time()\n",
        "        loss = [0, 0, 0]\n",
        "        index = 0\n",
        "        for ite in range(int(maxiter)):\n",
        "            if ite % update_interval == 0:\n",
        "                q, _ = self.model.predict(x, verbose=0)\n",
        "                p = self.target_distribution(q)  # update the auxiliary target distribution p\n",
        "\n",
        "                # evaluate the clustering performance\n",
        "                self.y_pred = q.argmax(1)\n",
        "                if y is not None:\n",
        "                    acc1 = np.round(acc(y, self.y_pred), 5)\n",
        "                    nmi1 = np.round(nmi(y, self.y_pred), 5)\n",
        "                    ari1 = np.round(ari(y, self.y_pred), 5)\n",
        "                    loss = np.round(loss, 5)\n",
        "                    logdict = dict(iter=ite, acc=acc1, nmi=nmi1, ari=ari1, L=loss[0], Lc=loss[1], Lr=loss[2])\n",
        "                    logwriter.writerow(logdict)\n",
        "                    print('Iter', ite, ': Acc', acc, ', nmi', nmi, ', ari', ari, '; loss=', loss)\n",
        "\n",
        "                # check stop criterion\n",
        "                delta_label = np.sum(self.y_pred != y_pred_last).astype(np.float32) / self.y_pred.shape[0]\n",
        "                y_pred_last = np.copy(self.y_pred)\n",
        "                if ite > 0 and delta_label < tol:\n",
        "                    print('delta_label ', delta_label, '< tol ', tol)\n",
        "                    print('Reached tolerance threshold. Stopping training.')\n",
        "                    logfile.close()\n",
        "                    break\n",
        "\n",
        "            # train on batch\n",
        "            if (index + 1) * batch_size > x.shape[0]:\n",
        "                loss = self.model.train_on_batch(x=x[index * batch_size::],\n",
        "                                                 y=[p[index * batch_size::], x[index * batch_size::]])\n",
        "                index = 0\n",
        "            else:\n",
        "                loss = self.model.train_on_batch(x=x[index * batch_size:(index + 1) * batch_size],\n",
        "                                                 y=[p[index * batch_size:(index + 1) * batch_size],\n",
        "                                                    x[index * batch_size:(index + 1) * batch_size]])\n",
        "                index += 1\n",
        "\n",
        "            # save intermediate model\n",
        "            if ite % save_interval == 0:\n",
        "                # save DCEC model checkpoints\n",
        "                print('saving model to:', save_dir + '/dcec_model_' + str(ite) + '.h5')\n",
        "                self.model.save_weights(save_dir + '/dcec_model_' + str(ite) + '.h5')\n",
        "\n",
        "            ite += 1\n",
        "        # save the trained model\n",
        "        logfile.close()\n",
        "        print('saving model to:', save_dir + '/dcec_model_final.h5')\n",
        "        self.model.save_weights(save_dir + '/dcec_model_final.h5')\n",
        "        t3 = time()\n",
        "        print('Pretrain time:  ', t1 - t0)\n",
        "        print('Clustering time:', t3 - t1)\n",
        "        print('Total time:     ', t3 - t0)\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    # setting the hyper parameters\n",
        "    import argparse\n",
        "    class Args:\n",
        "      dataset='mnist'\n",
        "      n_clusters=10\n",
        "      batch_size=256\n",
        "      maxiter=2e4\n",
        "      gamma=0.1\n",
        "      update_interval=140\n",
        "      tol=0.001\n",
        "      cae_weights=None\n",
        "      save_dir='results/temp'\n",
        "    \n",
        "    args = Args()\n",
        "    print(args)\n",
        "\n",
        "    import os\n",
        "    if not os.path.exists(args.save_dir):\n",
        "        os.makedirs(args.save_dir)\n",
        "\n",
        "    # load dataset\n",
        "    #from datasets import load_mnist, load_usps\n",
        "    if args.dataset == 'mnist':\n",
        "        x, y = load_mnist()\n",
        "    # elif args.dataset == 'usps':\n",
        "    #     x, y = load_usps('data/usps')\n",
        "    # elif args.dataset == 'mnist-test':\n",
        "    #     x, y = load_mnist()\n",
        "        x, y = x[60000:], y[60000:]\n",
        "\n",
        "    # prepare the DCEC model\n",
        "    dcec = DCEC(input_shape=x.shape[1:], filters=[32, 64, 128, 10], n_clusters=args.n_clusters)\n",
        "    plot_model(dcec.model, to_file=args.save_dir + '/dcec_model.png', show_shapes=True)\n",
        "    dcec.model.summary()\n",
        "\n",
        "    # begin clustering.\n",
        "    optimizer = 'adam'\n",
        "    dcec.compile(loss=['kld', 'mse'], loss_weights=[args.gamma, 1], optimizer=optimizer)\n",
        "    dcec.fit(x, y=y, tol=args.tol, maxiter=args.maxiter,\n",
        "             update_interval=args.update_interval,\n",
        "             save_dir=args.save_dir,\n",
        "             cae_weights=args.cae_weights)\n",
        "    y_pred = dcec.y_pred\n",
        "    print('acc = %.4f, nmi = %.4f, ari = %.4f' % (acc(y, y_pred), nmi(y, y_pred), ari(y, y_pred)))"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<__main__.Args object at 0x7fb70c708dd0>\n",
            "MNIST: (70000, 28, 28, 1)\n",
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv1 (Conv2D)              (None, 14, 14, 32)        832       \n",
            "                                                                 \n",
            " conv2 (Conv2D)              (None, 7, 7, 64)          51264     \n",
            "                                                                 \n",
            " conv3 (Conv2D)              (None, 3, 3, 128)         73856     \n",
            "                                                                 \n",
            " flatten_1 (Flatten)         (None, 1152)              0         \n",
            "                                                                 \n",
            " embedding (Dense)           (None, 10)                11530     \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 1152)              12672     \n",
            "                                                                 \n",
            " reshape_1 (Reshape)         (None, 3, 3, 128)         0         \n",
            "                                                                 \n",
            " deconv3 (Conv2DTranspose)   (None, 7, 7, 64)          73792     \n",
            "                                                                 \n",
            " deconv2 (Conv2DTranspose)   (None, 14, 14, 32)        51232     \n",
            "                                                                 \n",
            " deconv1 (Conv2DTranspose)   (None, 28, 28, 1)         801       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 275,979\n",
            "Trainable params: 275,979\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Model: \"model_2\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " conv1_input (InputLayer)       [(None, 28, 28, 1)]  0           []                               \n",
            "                                                                                                  \n",
            " conv1 (Conv2D)                 (None, 14, 14, 32)   832         ['conv1_input[0][0]']            \n",
            "                                                                                                  \n",
            " conv2 (Conv2D)                 (None, 7, 7, 64)     51264       ['conv1[0][0]']                  \n",
            "                                                                                                  \n",
            " conv3 (Conv2D)                 (None, 3, 3, 128)    73856       ['conv2[0][0]']                  \n",
            "                                                                                                  \n",
            " flatten_1 (Flatten)            (None, 1152)         0           ['conv3[0][0]']                  \n",
            "                                                                                                  \n",
            " embedding (Dense)              (None, 10)           11530       ['flatten_1[0][0]']              \n",
            "                                                                                                  \n",
            " dense_1 (Dense)                (None, 1152)         12672       ['embedding[0][0]']              \n",
            "                                                                                                  \n",
            " reshape_1 (Reshape)            (None, 3, 3, 128)    0           ['dense_1[0][0]']                \n",
            "                                                                                                  \n",
            " deconv3 (Conv2DTranspose)      (None, 7, 7, 64)     73792       ['reshape_1[0][0]']              \n",
            "                                                                                                  \n",
            " deconv2 (Conv2DTranspose)      (None, 14, 14, 32)   51232       ['deconv3[0][0]']                \n",
            "                                                                                                  \n",
            " clustering (ClusteringLayer)   (None, 10)           100         ['embedding[0][0]']              \n",
            "                                                                                                  \n",
            " deconv1 (Conv2DTranspose)      (None, 28, 28, 1)    801         ['deconv2[0][0]']                \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 276,079\n",
            "Trainable params: 276,079\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n",
            "Update interval 140\n",
            "Save interval 195.3125\n",
            "...pretraining CAE using default hyper-parameters:\n",
            "   optimizer='adam';   epochs=200\n",
            "...Pretraining...\n",
            "Epoch 1/20\n",
            "40/40 [==============================] - 13s 297ms/step - loss: 0.0781\n",
            "Epoch 2/20\n",
            "40/40 [==============================] - 12s 298ms/step - loss: 0.0612\n",
            "Epoch 3/20\n",
            "40/40 [==============================] - 12s 298ms/step - loss: 0.0452\n",
            "Epoch 4/20\n",
            "40/40 [==============================] - 12s 298ms/step - loss: 0.0328\n",
            "Epoch 5/20\n",
            "40/40 [==============================] - 12s 297ms/step - loss: 0.0268\n",
            "Epoch 6/20\n",
            "40/40 [==============================] - 12s 297ms/step - loss: 0.0236\n",
            "Epoch 7/20\n",
            "40/40 [==============================] - 12s 299ms/step - loss: 0.0217\n",
            "Epoch 8/20\n",
            "40/40 [==============================] - 12s 297ms/step - loss: 0.0203\n",
            "Epoch 9/20\n",
            "40/40 [==============================] - 12s 299ms/step - loss: 0.0195\n",
            "Epoch 10/20\n",
            "40/40 [==============================] - 12s 299ms/step - loss: 0.0187\n",
            "Epoch 11/20\n",
            "40/40 [==============================] - 12s 298ms/step - loss: 0.0180\n",
            "Epoch 12/20\n",
            "40/40 [==============================] - 12s 298ms/step - loss: 0.0176\n",
            "Epoch 13/20\n",
            "40/40 [==============================] - 12s 297ms/step - loss: 0.0172\n",
            "Epoch 14/20\n",
            "40/40 [==============================] - 12s 297ms/step - loss: 0.0168\n",
            "Epoch 15/20\n",
            "40/40 [==============================] - 12s 297ms/step - loss: 0.0165\n",
            "Epoch 16/20\n",
            "40/40 [==============================] - 12s 297ms/step - loss: 0.0163\n",
            "Epoch 17/20\n",
            "40/40 [==============================] - 12s 297ms/step - loss: 0.0159\n",
            "Epoch 18/20\n",
            "40/40 [==============================] - 12s 297ms/step - loss: 0.0157\n",
            "Epoch 19/20\n",
            "40/40 [==============================] - 12s 299ms/step - loss: 0.0155\n",
            "Epoch 20/20\n",
            "40/40 [==============================] - 12s 299ms/step - loss: 0.0152\n",
            "Pretraining time:  239.08419394493103\n",
            "Pretrained weights are saved to results/temp/pretrain_cae_model.h5\n",
            "Initializing cluster centers with k-means.\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 0 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0 0 0]\n",
            "saving model to: results/temp/dcec_model_0.h5\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 140 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.02057 0.05511 0.01506]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 280 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.02142 0.07464 0.01396]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 420 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.0346  0.15147 0.01945]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 560 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.02519 0.11616 0.01357]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 700 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.03663 0.16076 0.02055]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 840 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.02228 0.09297 0.01298]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 980 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.03368 0.14637 0.01904]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 1120 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.02096 0.08233 0.01272]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 1260 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.03331 0.1352  0.01979]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 1400 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.02385 0.07953 0.0159 ]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 1540 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.03181 0.12502 0.01931]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 1680 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.01903 0.07251 0.01178]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 1820 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.02841 0.11041 0.01737]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 1960 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.01861 0.07385 0.01122]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 2100 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.02974 0.11098 0.01865]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 2240 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.02037 0.07    0.01337]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 2380 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.02761 0.10836 0.01678]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 2520 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.0164  0.05786 0.01061]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 2660 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.02835 0.10347 0.018  ]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 2800 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.01576 0.0572  0.01004]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 2940 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.02701 0.09625 0.01738]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 3080 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.01644 0.05871 0.01057]\n",
            "saving model to: results/temp/dcec_model_3125.h5\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 3220 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.02673 0.09956 0.01677]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 3360 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.0153  0.06884 0.00841]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 3500 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.02452 0.08902 0.01561]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 3640 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.01784 0.06513 0.01132]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 3780 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.02585 0.08748 0.0171 ]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 3920 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.01498 0.05523 0.00946]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 4060 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.02395 0.07997 0.01595]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 4200 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.01628 0.06885 0.00939]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 4340 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.02287 0.07712 0.01515]\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iter 4480 : Acc <function acc at 0x7fb7970730e0> , nmi <function normalized_mutual_info_score at 0x7fb7214abd40> , ari <function adjusted_rand_score at 0x7fb7214ab950> ; loss= [0.01302 0.04382 0.00864]\n",
            "delta_label  0.0008 < tol  0.001\n",
            "Reached tolerance threshold. Stopping training.\n",
            "saving model to: results/temp/dcec_model_final.h5\n",
            "Pretrain time:   239.14146614074707\n",
            "Clustering time: 1582.9631507396698\n",
            "Total time:      1822.1046168804169\n",
            "Requirement already satisfied: scikit-learn==0.22.2 in /usr/local/lib/python3.7/dist-packages (0.22.2)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.19.5)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n",
            "acc = 0.7893, nmi = 0.7596, ari = 0.6894\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/linear_assignment_.py:128: FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
            "  FutureWarning)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aT_nigoCUJ4-"
      },
      "source": [
        "from sklearn.metrics.cluster import normalized_mutual_info_score   \n",
        "from sklearn.metrics import accuracy_score \n",
        "from sklearn.metrics.cluster import adjusted_rand_score\n",
        "def load_mnist():\n",
        "        # the data, shuffled and split between train and test sets\n",
        "        from keras.datasets import mnist\n",
        "        (x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
        "\n",
        "        x = np.concatenate((x_train, x_test))\n",
        "        y = np.concatenate((y_train, y_test))\n",
        "        x = x.reshape(-1, 28, 28, 1).astype('float32')\n",
        "        x = x/255.\n",
        "        print('MNIST:', x.shape)\n",
        "        return x, y"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "scQmCPJwb9Z4",
        "outputId": "b9721917-1f8b-406a-907b-fc757ed09094"
      },
      "source": [
        "y_pred.shape,y.shape"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((10000,), (10000,))"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J-fKf6-6XIcT"
      },
      "source": [
        "def CAE(input_shape=(28, 28, 1), filters=[32, 64, 128, 10]):\n",
        "    model = Sequential()\n",
        "    if input_shape[0] % 8 == 0:\n",
        "        pad3 = 'same'\n",
        "    else:\n",
        "        pad3 = 'valid'\n",
        "    model.add(Conv2D(filters[0], 5, strides=2, padding='same', activation='relu', name='conv1', input_shape=input_shape))\n",
        "\n",
        "    model.add(Conv2D(filters[1], 5, strides=2, padding='same', activation='relu', name='conv2'))\n",
        "\n",
        "    model.add(Conv2D(filters[2], 3, strides=2, padding=pad3, activation='relu', name='conv3'))\n",
        "\n",
        "    model.add(Flatten())\n",
        "    model.add(Dense(units=filters[3], name='embedding'))\n",
        "    model.add(Dense(units=filters[2]*int(input_shape[0]/8)*int(input_shape[0]/8), activation='relu'))\n",
        "\n",
        "    model.add(Reshape((int(input_shape[0]/8), int(input_shape[0]/8), filters[2])))\n",
        "    model.add(Conv2DTranspose(filters[1], 3, strides=2, padding=pad3, activation='relu', name='deconv3'))\n",
        "\n",
        "    model.add(Conv2DTranspose(filters[0], 5, strides=2, padding='same', activation='relu', name='deconv2'))\n",
        "\n",
        "    model.add(Conv2DTranspose(input_shape[2], 5, strides=2, padding='same', name='deconv1'))\n",
        "    model.summary()\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xun5SPj-XSqq"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}