aa_testing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "27bad576-35ce-4253-92db-a434dd4e0d9b",
   "metadata": {},
   "source": [
    "## Download and read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "d927b607-64ce-4079-91bd-1da9d862fe81",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from scipy import stats\n",
    "import pandahouse\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "02fd20a2-dd4b-467c-b027-b50c32d5d420",
   "metadata": {},
   "outputs": [],
   "source": [
    "def select(query):\n",
    "    return pandahouse.read_clickhouse(query, connection=connection)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "7c3b52da-5c90-4dc5-bdc3-6b107f2fc0b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "connection = {\n",
    "    'database': 'simulator_20221120',\n",
    "    'host': 'https://clickhouse.lab.karpov.courses',\n",
    "    'user': 'student',\n",
    "    'password': 'dpo_python_2020'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "f491f1e4-1dc5-4b0a-8605-3f7d7ceba5dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "query = '''\n",
    "select exp_group, user_id,\n",
    "sum(action = 'like') as likes,\n",
    "sum(action = 'view') as views,\n",
    "likes / views as ctr\n",
    "from {db}.feed_actions \n",
    "where toDate(time) between '2022-10-26' and '2022-11-01' and exp_group in (2, 3)\n",
    "group by exp_group, user_id\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "9acf37c9-e81d-4310-b266-21794394b5a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = select(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "745c8697-e680-451c-ab95-fba4aba88544",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>exp_group</th>\n",
       "      <th>user_id</th>\n",
       "      <th>likes</th>\n",
       "      <th>views</th>\n",
       "      <th>ctr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>115383</td>\n",
       "      <td>11</td>\n",
       "      <td>36</td>\n",
       "      <td>0.305556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>123580</td>\n",
       "      <td>15</td>\n",
       "      <td>92</td>\n",
       "      <td>0.163043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>131473</td>\n",
       "      <td>26</td>\n",
       "      <td>148</td>\n",
       "      <td>0.175676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>23985</td>\n",
       "      <td>12</td>\n",
       "      <td>66</td>\n",
       "      <td>0.181818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>24523</td>\n",
       "      <td>17</td>\n",
       "      <td>120</td>\n",
       "      <td>0.141667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17044</th>\n",
       "      <td>3</td>\n",
       "      <td>122008</td>\n",
       "      <td>4</td>\n",
       "      <td>31</td>\n",
       "      <td>0.129032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17045</th>\n",
       "      <td>2</td>\n",
       "      <td>25383</td>\n",
       "      <td>16</td>\n",
       "      <td>57</td>\n",
       "      <td>0.280702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17046</th>\n",
       "      <td>3</td>\n",
       "      <td>122190</td>\n",
       "      <td>25</td>\n",
       "      <td>74</td>\n",
       "      <td>0.337838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17047</th>\n",
       "      <td>2</td>\n",
       "      <td>108927</td>\n",
       "      <td>8</td>\n",
       "      <td>33</td>\n",
       "      <td>0.242424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17048</th>\n",
       "      <td>2</td>\n",
       "      <td>17310</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>0.142857</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17049 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       exp_group  user_id  likes  views       ctr\n",
       "0              3   115383     11     36  0.305556\n",
       "1              3   123580     15     92  0.163043\n",
       "2              2   131473     26    148  0.175676\n",
       "3              2    23985     12     66  0.181818\n",
       "4              2    24523     17    120  0.141667\n",
       "...          ...      ...    ...    ...       ...\n",
       "17044          3   122008      4     31  0.129032\n",
       "17045          2    25383     16     57  0.280702\n",
       "17046          3   122190     25     74  0.337838\n",
       "17047          2   108927      8     33  0.242424\n",
       "17048          2    17310      2     14  0.142857\n",
       "\n",
       "[17049 rows x 5 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2622ab38-9b7f-4557-9745-f18678c77aad",
   "metadata": {},
   "source": [
    "## Histogram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "a7621c18-5248-441a-811c-fb7323cdb6c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>likes</th>\n",
       "      <th>views</th>\n",
       "      <th>ctr</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>exp_group</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8480</td>\n",
       "      <td>8480</td>\n",
       "      <td>8480</td>\n",
       "      <td>8480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8569</td>\n",
       "      <td>8569</td>\n",
       "      <td>8569</td>\n",
       "      <td>8569</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           user_id  likes  views   ctr\n",
       "exp_group                             \n",
       "2             8480   8480   8480  8480\n",
       "3             8569   8569   8569  8569"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('exp_group').count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5742199-0977-418e-a812-bb9a24156fbb",
   "metadata": {},
   "source": [
    "Размеры групп примерно одинаковые."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "8ecd8985-1b77-4277-966c-5e9487777e6d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='ctr', ylabel='Count'>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsoAAAH0CAYAAAAkDgsAAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAz60lEQVR4nO3df3xU1Z3/8fe9d2ZC+BkSAiSIINRqVttaS2urhdVEDD8SsLTWbKqtUta20F3b3aLUH+CPqgVctVZZ9dvWrmsXV1fFEgX8wdcv4rZWrQoYFBsRUQKBhKj8SObX/f5hTcE5EyZx5t6Zyev5ePh4kHNm5n5ynMA7Z849x3Jd1xUAAACAw9h+FwAAAABkI4IyAAAAYEBQBgAAAAwIygAAAIABQRkAAAAwICgDAAAABgRlAAAAwCDgdwGZtnfvfsXj3m0VXVIyUK2t+zy7Xj5jLNOL8UwfxjJ9GMv0YSzTh7FMn1wYS9u2NHToAGNf3gfleNz1NCh/dE2kB2OZXoxn+jCW6cNYpg9jmT6MZfrk8liy9AIAAAAwICgDAAAABnm/9AIAAKAvicWi2rt3t6LRsN+lqKXFVjwe97sMSVIgENLQoaVynNTjL0EZAAAgj+zdu1v9+vXXgAEjZVmWr7UEAraiUf+Dsuu62r//fe3du1vDhpWl/DyWXgAAAOSRaDSsAQMG+x6Ss4llWRowYHCPZ9kJygAAAHmGkJyoN2NCUAYAAAAMCMoAAACAAUEZAAAAMCAoAwAAwDfRaNTvEpJiezgAAIA+YM+e3br55iV65ZWXVFjYX9/8Zr2qq6fqO9/5B/3rvy7QV786SQcOHNCFF9brggvmaOrUGl133VUKhUJ699139Oqrm3Tcccfriiuu1siR3W+x9qc//VE337xEbW1tmjx5it56601VV09Tbe3ZeuyxlVq58mFVVJyg1asf09lnf1319d/WzTcv0XPP/a8KCvqptvZsffvbs2Xbtn796zv17rvvaOHCayVJzc07dM45M/T0039UIBDQD394kU488bN68cU/adu2bTr55C/osssWafDgIZ94zJhRBgAAyHPxeFyXXPJjfepTn9bDD6/SLbcs0/33L9fmzY366U8XasmS67R3b5t++cubdOyxn9bUqTVdz3388dW64II5evTRJ/WpT31aV199RbfXam9v1xVXXKrvf/+HWrNmrY4+eow2bnzlsMc0Nr6q8vJR+v3v1+g735mtm29eov379+n++x/RbbfdpdWrH9Njj/0+5e9v9epH9dOfLtQjj6yW4zi65ZYbezZASRCUAQAA8tzmzY1qb9+rCy/8RwWDQY0adZRmzDhbTz31uL70pS/rjDOqdPHFP9Af/vCs5s+/7LDnnnrqaTrppJMVCoV00UVz9eqrG7Vr186k1/rDH9brmGPG6e//vlKBQEDnnFOnkpJhhz2mpGSYvvGNOgUCAQUCQT311OP63vd+qP79B6isrFx1dd/S6tWPpfz9VVdP07hxn1JhYaHmzPmB1q59QrFYrGeDZMDSCwAAgDy3c2ezWlv3aMqU07vaYrG4Pve5kyRJM2bM0oMP3q/zz79QQ4YUHfbc4cNHdv25f//+Gjx4sPbs2aMRI0bKZM+ePRo+fETX15ZlqbR0+GGPGTHib/3vvdeuaDR62HKOkSPLtGfP7pS/v0OvN3JkmaLRqN57r13FxSUpv4YJQRkAACDPjRgxQmVl5brvvocT+mKxmJYsuU5TpkzXww//j6ZPn6Gjjhrd1d/S8rfZ4wMHDuj999/XsGHDEl7nI8OGDdOzz7Z0fe26rnbvbvnYo/52+MeQIUUKBALaubNZxxwzTpK0a9dODRtWKkkqLCxUR0dH1+NbW1sTrtnSsqvrz7t27VQgEEgI/L3B0gsAAIA8V1Fxgvr376977/2tOjs7FIvF9Oabf9Hmza/qP//zblmWpZ/+dKHq68/Xz3626LBlC3/4w//qlVdeViQS0a9+9e864YQTk84mS9JXvvJVNTX9RevWPa1oNKqHHrpfra17kj7ecRxVVk7WXXct04ED+7VzZ7P++79/p+rqaZKkY4/9tF555c/auXOn9u3bp3vvvTvhNR5/fJW2bn1THR0d+tWv7tDpp1fJcZxPMGIfIigDAADkOcdxtGTJLXrjjS0655yZmj79TP385z/Tiy8+r/vu+52uuOJqOY6jb33rO7Is6d57f9v13MmTq3X33Xdp2rRKvf76a7ryymu7vVZRUZGuvfbnWrbsVlVXV2rr1q06/vi/UzAYTPqcH/1ovgoLC/XNb87U3LlzNHnyFE2fPkOS9MUvflmVlWfpggvq9N3vnqdTT52Y8Pzq6mm6/vqrNGNGtcLhsH70o/m9G6iPsVzXddPySlmqtXWf4nHvvsXS0kHavfsDz66XzxjL9GI804exTB/GMn0Yy/TJ9bHcuXObRo4ck5bXuu66q1RaOlwXXTS3V88PBGyFw1F97WvTtGjRz3TyyRPSUtehfvjDi7q2njsS09jYtqWSkoHGxzOjDOSZQieuwlhHwn/RAwf8Lg0A0Ec899wf9MEHHygcDuuee34jydUJJ5zod1k9xs18QL4Jh7W9YVVC8/hZteJHHgCQDq+88pJ+8pN/NvY98cQz2rRpg66++gpFoxGNGXOMrr/+31RQ0M/jKj85ll6kWa5/XJNNGMveKYx1JA3K78UIyunAezN9GMv0YSzTJ9fHMp1LLz6pQMBWNBr3u4wuLL0AAAAA0oCgDAAAABgQlAEAAAADgjIAAABgwJ09AAAA6DJ0SD8FQskPB+mtaDiive91HPmBWYSgDAAAgC6BUFCbf3VPWl7Ltq2u3ccq5nxb0pGD8nvvtevaaxfq3XffUTAY1FFHHa358y/T0KFD01JTT7D0AgAAAFnDsizV139by5c/pHvu+W+NGnWU7rjjl77UQlAGAABA1hg8eMhhR12fcMKJ2rlzpy+1EJQBAACQleLxuB5++EF99auTfLk+QRkAAABZ6eabl6p//0J9/evf9OX63MwHAACArHPbbbfonXfe1uLFN8u2/ZnbJSgDAAAgq9x55+16/fXNWrr0FwqFQr7VQVAG8o1tybWshOZYPO5DMQCAXBMNR/66lVv6XzcVb77ZpP/8z7s1evTR+v73Z0uSysrKdcMNN6a9piMhKAN5xnVdbdnWltB+dNyVEvMzAACH+fBQkPQcDBII2IpGezZRM27ceK1f/0Jarv9JcTMfAAAAYEBQBgAAAAwIygAAAIABQRkAAAAwICgDAAAABgRlAAAAwIDt4QAAANBlSFF/hYJO2l83HInpvfYDKT32pz/9V+3YsUO2bamwsL9+/OP5OvbY49Je05EQlAEAANAlFHT0fx56JS2vZduW4nFXkvSPsz6X8vMuv/xqDRw4UJL0zDNP64YbrtFvfvO7tNTUEyy9AAAAQFb5KCRL0r59+2RZ/kRWZpQBAACQdX7+82v1pz/9UZJ04423+lIDM8oAAADIOgsWXKmHHnpUF100V8uW/cKXGgjKAAAAyFpTpkzXn//8ot57r93zaxOUAQAAkDUOHDigXbt2dn29fv06DR48WIMHD/G8FtYoAwAAIGt0dBzUlVcuUEfHQdm2o8GDB2vx4ptlWZbntRCUAQAA0CUcifVoK7eevG4qiotLdNddv0379XuDoAwAAIAuqR4KkopAwFY0Gk/b63mNNcoAAACAAUEZAAAAMCAoAwAA5BnXdf0uIev0ZkwIygAAAHkkEAhp//73CcuHcF1X+/e/r0Ag1KPncTMfAABAHhk6tFR79+7Wvn3tfpci27YVj2fHzXyBQEhDh5b27DkZqgUAAAA+cJyAhg0r87sMSVJp6SDt3v2B32X0GksvAAAAAANmlIEs4QQdhQ17TYYCtmIpbtIOAADSh6AMZIlwNK6V65oS2msnjZfjQz0AAPR1ni+9uO2223Tcccdpy5YtkqSXX35ZM2bMUHV1tWbPnq3W1taux3bXBwAAAGSSp0H51Vdf1csvv6xRo0ZJkuLxuObPn6+FCxdqzZo1mjBhgm688cYj9gEAAACZ5llQDofDuuaaa3TVVVd1tW3atEkFBQWaMGGCJKmurk6rV68+Yh8AAACQaZ6tUf7FL36hGTNm6Kijjupqa25uVnl5edfXxcXFisfjam9v77avqKgo5euWlAxMS/09UVo6yPNr5qu+NJZt73dowICChPZQKKDiwf1Sfp19LZ0KBs0/2n1pPDONsUwfxjJ9GMv0YSzTJ5fH0pOg/NJLL2nTpk36yU9+4sXlDtPauk/xuHcn0+T6foHZpK+NZcyytH9/Z0J7OBzt0Tj0c11FIlFjX18az0zqa+/NTGIs04exTB/GMn1yYSxt20o6sepJUH7++efV1NSkqqoqSdLOnTv13e9+V+eff7527NjR9bi2tjbZtq2ioiKVlZUl7QMAAAAyzZM1yhdddJHWr1+vtWvXau3atRo5cqR+/etfa86cOero6NALL7wgSbrvvvs0ZcoUSdKJJ56YtA8AAADINF/3UbZtW0uWLNGiRYvU2dmpUaNGaenSpUfsAwAAADLNl6C8du3arj+ffPLJWrlypfFx3fUBAAAAmeT5gSMAAABALuAIayBLDLBjqqwoMrZ3xPidFgAArxGUgWwRCWvj8hUJzVXz6iU79X2UAQBAejBNBQAAABgQlAEAAAADgjIAAABgQFAGAAAADLiZD8hRTtBROBo39Fie1wIAQD4iKANZzrIsxazE8BuPuVq5rimhve60ci/KAgAg7xGUgSznuuZAXDNpvA/VAADQd7BGGQAAADAgKAMAAAAGBGUAAADAgKAMAAAAGHAzH5ABhU5cCocTO0IhHYzx+ykAALmAoAxkQjis7Q2rEppH10yVnH49eqlAwFFlRVFC+0A71tvqAABACgjKQJZzIxFtXL4iob1qXr33xQAA0IfwGTAAAABgwIwy4KFA0FFhpMPYZ1uux9UAAIDuEJQBD8XDEW1/dI2xb0xNtcfVAACA7rD0AgAAADBgRhnwkiW5luV3FQAAIAUEZcBjW7a1GdvHelsGAAA4ApZeAAAAAAYEZQAAAMCAoAwAAAAYEJQBAAAAA4IyAAAAYEBQBgAAAAwIygAAAIAB+ygDfZwTdBSOxo19oYCtWCTmcUUAAGQHgjKQowIBR5UVRQntjtwevU44GtfKdU3GvtpJ4+X0pjgAAPIAQRnIUW4koo3LVyS0l82t874YAADyEGuUAQAAAAOCMgAAAGBAUAYAAAAMCMoAAACAAUEZAAAAMCAoAwAAAAZsDwf0EZakmGUlthvaAAAAQRnoM+KujAeL1Ewan7ZrFDpxKRxO7AiFdDDGB1gAgNxCUAaQPuGwtjesSmgeXTNVcvr5UBAAAL3HFA8AAABgQFAGAAAADAjKAAAAgAFBGQAAADAgKAMAAAAGBGUAAADAgO3hgEywLbkc5AEAQE4jKAMZ4LqutmxrS2gf630pAACgl1h6AQAAABgQlAEAAAADll4AfUQgYKuyoiihfaAd874YAAByAEEZ6CPcSEQbl69IaK+aV+99MQAA5ACWXgAAAAAGBGUAAADAgKAMAAAAGBCUAQAAAAOCMgAAAGBAUAYAAAAMCMoAAACAAUEZAAAAMODAEQDpY1tyLcvYDgBAriEoA0gb13W1ZVtbQvto15XIygCAHENQBpDUADsmKxxO7AiFdDDGyi0AQH4jKANILhLW9oZVCc2ja6ZKTj8fCgIAwDtMCQEAAAAGBGUAAADAgKAMAAAAGLBGGejjAgFHlRVFxj5HrrfFAACQRQjKQB/nRiLauHyFsa9sbh37IgMA+iyCMoBusS8yAKCvYo0yAAAAYEBQBgAAAAwIygAAAIABa5SBT8AJOgpH44YeFvACAJDrCMrAJxCOxrVyXVNCe91p5T5UAwAA0omlFwAAAIABQRkAAAAwICgDAAAABgRlAAAAwICgDAAAABgQlAEAAAADgjIAAABgQFAGAAAADAjKAAAAgIFnJ/PNnTtX77zzjmzbVv/+/XXllVeqoqJCW7du1YIFC9Te3q6ioiItXrxYY8eOlaRu+wAAAIBM8mxGefHixfr973+vFStWaPbs2brsssskSYsWLVJ9fb3WrFmj+vp6LVy4sOs53fUBAAAAmeRZUB40aFDXn/ft2yfLstTa2qrGxkbV1NRIkmpqatTY2Ki2trZu+wAAAIBM82zphSRdfvnlevbZZ+W6rn71q1+publZI0aMkOM4kiTHcTR8+HA1NzfLdd2kfcXFxV6WDQAAgD7I06B83XXXSZJWrFihJUuW6OKLL874NUtKBmb8Gh9XWjroyA9CSrJ9LNve79CAAQWGHkvBoPnHK1l7d31+tSfrsyzL+P9mX0tnjx6fy/Lt+/ETY5k+jGX6MJbpk8tj6WlQ/sjZZ5+thQsXauTIkdq1a5disZgcx1EsFlNLS4vKysrkum7Svp5obd2neNzN0HeSqLR0kHbv/sCz6+WzXBjLmGVp//5OQ4+rSCRqfE6y9u76/GpP1ue6rvH/TT/X/H0ne3yuyoX3Zq5gLNOHsUwfxjJ9cmEsbdtKOrHqyRrl/fv3q7m5uevrtWvXasiQISopKVFFRYUaGhokSQ0NDaqoqFBxcXG3fQAAAECmeTKjfPDgQV188cU6ePCgbNvWkCFDdMcdd8iyLF111VVasGCBli1bpsGDB2vx4sVdz+uuDwAAAMgkT4LysGHDdP/99xv7xo8frwceeKDHfUA2GGDHVFlRlNDuyLvlPgAAIDN8WaMM5I1IWBuXr0hoLptb530tAAAgrTjCGgAAADAgKAMAAAAGBGUAAADAgKAMAAAAGHAzH5ACJ+goHI0beizPawEAAN4gKAMpCEfjWrmuKaG97rRyH6oBAABeYOkFAAAAYEBQBgAAAAwIygAAAIABQRkAAAAwICgDAAAABgRlAAAAwICgDAAAABgQlAEAAAADgjIAAABgQFAGAAAADDjCGkjBADumyoqihHZHrvfFAAAATxCUgVREwtq4fEVCc9ncOu9rAQAAnmDpBQAAAGBAUAYAAAAMCMoAAACAAUEZAAAAMOBmPgA9Fgg4Kox0JLTbFruAAADyB0EZQI+5kYi2N6xJaB9TU+1DNQAAZAZLLwAAAAADZpQB+MYJOgpH48a+UMBWLBLzuCIAAP6GoAzAN+FoXCvXNRn7aieNl+NxPQAAHIqlFwAAAIABQRkAAAAwICgDAAAABgRlAAAAwICgDAAAABgQlAEAAAADgjIAAABgQFAGAAAADAjKAAAAgAFBGQAAADDgCGsAGWdZlmKWZWwHACBbEZQBZJzrulq5rimhve70saqsKDI+Z4AdU0eMD70AAP4hKAPwTySsjctXGLuq5tVLdj9v6wEA4BBM1wAAAAAGKQflVatWGdtXr16dtmIAAACAbJFyUL788suN7QsXLkxbMQAAAEC2OOIa5e3bt0v68Gacj/58aF8oFMpMZQAAAICPjhiUJ0+eLMuy5LquJk+efFjfsGHD9E//9E8ZKw4AAADwyxGD8muvvSZJOu+883TvvfdmvCAAucFlD2QAQJ5LeXs4QjKAQ23Z1pbQNtb7MgAAyJiUg/L27dt1yy23aPPmzTpw4MBhfU8//XS66wIAIyfoKByNJ7SHArZikZgPFQEA8lXKQfknP/mJRo8erUsvvVSFhYWZrAkAkgpH48ZT/monjZfjQz0AgPyVclB+4403tHz5ctk2Z5QAAAAg/6Wcer/4xS+qsbExk7UAAAAAWSPlGeVRo0Zpzpw5mjx5soYNG3ZY38UXX5z2wgAAAAA/pRyUDx48qDPOOEPRaFQ7d+7MZE0AAACA71IOyjfccEMm6wAAAACySo+2h0tm9OjRaSkGAAAAyBYpB+VDj7L+iPXXk7k2b96c/soAAAAAH6UclD86yvoju3fv1m233aYJEyakvSgAAADAb73eFLm0tFSXX365brrppnTWAwAAAGSFlGeUTd58800dPHgwXbUAyFOBgKPKiqKEdkdu4oMBAMgSKQfl+vr6rjXJ0ofbxf3lL3/RvHnzMlIYgPzhRiLauHxFQnvZ3DrviwEAIEUpB+VzzjnnsK8LCwt1/PHHa+zYsemuCQAAAPBdykH5a1/7WibrAAAAALJKyjfzRSIR3XrrraqqqtJnPvMZVVVV6dZbb1U4HM5kfQAAAIAvUp5RXrp0qTZs2KCrr75a5eXl2rFjh5YtW6Z9+/bpsssuy2SNAAAAgOdSDsqrV6/WI488oqFDh0qSxo0bp7/7u7/TzJkzCcoAAADIOykvvTj0RL5U2gEAAIBclnJQnjJlin7wgx/omWeeUVNTk9atW6d58+ZpypQpmawPAAAA8EXKSy/mz5+vf//3f9c111yjlpYWjRgxQtOnT9cPfvCDTNYHAAAA+OKIM8ovvviili5dqlAopIsvvlhPPPGEXnnlFT3++OMKh8NqbGz0ok4AAADAU0cMynfeeae++MUvGvtOOeUU3XHHHWkvCgAAAPDbEYPy5s2bNXHiRGPfqaeeqk2bNqW9KAAAAMBvRwzK+/btUyQSMfZFo1Ht378/7UUBAAAAfjtiUB43bpzWr19v7Fu/fr3GjRuX9qIAvzhBRzHLSvhPsvwuDQAAeOyIu15ccMEFWrRokeLxuM4880zZtq14PK4nn3xS11xzjRYsWOBFnYAnwtG4Vq5rSmivO63ch2oAAICfjhiUa2trtWfPHl166aWKRCIqKipSe3u7gsGg/vmf/1k1NTVe1AkAkqQBdkyVFUXG9o5YylvDAwBwRCnto3zhhRfqnHPO0UsvvaT29nYVFRXp85//vAYOHJjp+gDgcJGwNi5fkdBcNa9esvt5Xw8AIG+lfODIwIEDk+5+AQAAAOQbPqcEAAAADAjKAAAAgEHKSy8AwEtW19Z8CT2e1wIA6JsIysAhku2o4Mj1vpg+znVdtuoDAPiKoAwcKsmOCmVz67yvBQAA+Io1ygAAAIABQRkAAAAwICgDAAAABqxRBpCVAgGHGysBAL7yJCjv3btXl1xyid5++22FQiGNGTNG11xzjYqLi/Xyyy9r4cKF6uzs1KhRo7R06VKVlJRIUrd9APKbG4lwYyUAwFeeLL2wLEtz5szRmjVrtHLlSo0ePVo33nij4vG45s+fr4ULF2rNmjWaMGGCbrzxRknqtg8AAADINE+CclFRkU455ZSur0866STt2LFDmzZtUkFBgSZMmCBJqqur0+rVqyWp2z4AAAAg0zy/mS8ej2v58uWqrKxUc3Ozysv/dnhAcXGx4vG42tvbu+0DAAAAMs3zm/muvfZa9e/fX+edd56eeOKJjF+vpGRgxq/xcaWlgzy/Zr7yeiz3tXQqGDT/WGS63Ytr5FJNPX2OZVmevl/4OU8fxjJ9GMv0YSzTJ5fH0tOgvHjxYm3btk133HGHbNtWWVmZduzY0dXf1tYm27ZVVFTUbV9PtLbuUzzu3V3ypaWDtHv3B55dL59laiwLnbgUDhv7bMtVJBI19mW63Ytr5FJNPX2O67qe/ezxc54+jGX6MJbpw1imTy6MpW1bSSdWPQvKN910kzZt2qS77rpLoVBIknTiiSeqo6NDL7zwgiZMmKD77rtPU6ZMOWIf8ImEw9resMrYNaam2uNiAABAtvIkKL/xxhu68847NXbsWNXVfbi101FHHaXbb79dS5Ys0aJFiw7bAk6SbNtO2gcAAABkmidB+dhjj9Xrr79u7Dv55JO1cuXKHvcBAAAAmcTJfADyQiDgqDDSkdgRCulgzPMNfgAAeYCgDCAvuJGItjesSWgfXTNVcvr5UBEAINcxzQIAAAAYEJQBAAAAA4IyAAAAYEBQBgAAAAwIygAAAIABQRkAAAAwYHs45K1CJy6FwwnttuX6UA0AAMg1BGXkr3BY2xtWJTSPmV7tQzEAACDXsPQCAAAAMCAoAwAAAAYsvUDfY0muZfldBQAAyHIEZfRJW7a1GdvHelsGAADIYiy9AAAAAAwIygAAAIABQRkAAAAwICgDAAAABgRlAAAAwICgDAAAABiwPRyA/GZbihn2zQ4FbMUiMR8KAgDkCoIygLzmuq5WrmtKaK+dNF6OD/UAAHIHSy8AAAAAA2aUAeS1QMBRZUVRQvsAO6aOGHMFAIDkCMoA8pobiWjj8hUJ7VXz6iW7n/cFAQByBkEZQJ8UCDgqjHQkdoRCOshMMwBABGUAfZQbiWh7w5qE9tE1UyWHmWYAAEEZQB/mGraNk21oAwD0SQRlAH3Wlm1tCW2jXVciKwMARFAGkEeMM8QAAPQSQRlA3jDNEI/1vgwAQJ7g1m4AAADAgKAMAAAAGBCUAQAAAAOCMgAAAGBAUAYAAAAMCMoAAACAAUEZAAAAMCAoAwAAAAYEZQAAAMCAk/kAIEsUOnEpHDZ3hkI6GGNuAwC8RFAGgGwRDmt7wypj1+iaqZLTz+OCAKBvIygjf9mWXMvyuwoAAJCjCMrIW67rasu2toT2sd6XAgAAchAL3gAAAAADZpQB4BCBgKPCSIe5M9M31HW3XMhmGREAeI2gDACHcCMRbW9YY+zL9A11yZYLSdJo15XIygDgKZZeAAAAAAYEZQAAAMCAoAwAAAAYEJQBAAAAA4IyAAAAYEBQBgAAAAwIygAAAIAB+ygDwMdw6AcAQCIoA0CCdB364QQdhaPxhPZQwFYsEutteQAAjxCUASBDwtG4Vq5rSmivnTRejg/1AAB6hjXKAAAAgAFBGQAAADAgKAMAAAAGrFEGgBQFA44U7dC+lk71c92udrugQAci7IgBAPmGoAwAKYpHInpq2X0KBgOKRKJd7VXz6iW7n4+VAQAygaUXAAAAgAFBGQAAADAgKAMAAAAGBGUAAADAgKAMAAAAGBCUAQAAAAOCMgAAAGBAUAYAAAAMCMoAAACAAUEZAAAAMCAoAwAAAAYBvwsAPikn6CgcjRt6LM9rAQAA+YOgjJwXjsa1cl1TQnvdaeU+VAMAAPIFSy8AAAAAA4IyAAAAYEBQBgAAAAwIygAAAIABN/MhZyTb3cKy2N0C2WmAHVNlRZGxvSPGPAUAZDuCMnJGst0taiaN96Ea4G8sy1LM9AtbJKKNy1ckNFfNq5fsfpkvDADwiRCUAeATcl2XLQoBIA8RlJEzkn2MPdCOeV8MAADIewRl5I5IOPnH2AAAAGnG3SQAAACAAUEZAAAAMCAoAwAAAAaeBOXFixersrJSxx13nLZs2dLVvnXrVp177rmqrq7Wueeeq7feeiulPgAAACDTPAnKVVVV+t3vfqdRo0Yd1r5o0SLV19drzZo1qq+v18KFC1PqAwAAADLNk6A8YcIElZWVHdbW2tqqxsZG1dTUSJJqamrU2Niotra2bvsAAAAAL/i2PVxzc7NGjBghx3EkSY7jaPjw4Wpubpbrukn7iouL/SoZAAAAfUje76NcUjLQ82uWlg7y/Jr56tCx3NfSqWDQ9Ja1NGBAgbHd/Hglbe+uL9Ptfl47G2vy89qp1HT4Y3r2HrQsy/j3RPL3ePLn5IN8/b78wFimD2OZPrk8lr4F5bKyMu3atUuxWEyO4ygWi6mlpUVlZWVyXTdpX0+1tu5TPO5m4DswKy0dpN27P/Dsevns42PZz3UViUQNj3S1f3+nsd38eCVt764v0+1+Xjsba/Lz2keqKRgMfOwxPXsPuq5r/Hsi+Xs8+XNyHX9npg9jmT6MZfrkwljatpV0YtW37eFKSkpUUVGhhoYGSVJDQ4MqKipUXFzcbR8AAADgBU9mlH/2s5/p8ccf1549e3ThhReqqKhIjz76qK666iotWLBAy5Yt0+DBg7V48eKu53TXBwAAAGSaJ0H5iiuu0BVXXJHQPn78eD3wwAPG53TXBwAAAGQaJ/MBAAAABgRlAAAAwCDvt4cDgEwLBBxVVhQltDvybscdAED6EZQB4BNyIxFtXL4iob1sbp33xQAA0oalFwAAAIABM8oA4DHLshSzLFOP57UAAJIjKAOAx1zX1cp1TQntdaeV+1ANACAZll4AAAAABswoA0AeKnTiUjic2BEK6WCMORIASAVBGQDyUTis7Q2rEppH10yVnH4+FAQAuYdpBQAAAMCAGWXkPA57QK7hPQsAuYGgjJzHYQ/INbxnASA3sPQCAAAAMCAoAwAAAAYsvUDW+Whbq3BbVIWxaFe7bbF+EwAAeIegjOzz122tBgwo0P79nV3NY2qqfSwKAAD0NSy9AAAAAAwIygAAAIABSy8AIIc5QUfhaDyh3bItuZaV+ATb0AYAMCIoA0AOC0fjWrmuKaG97rRybdnWltA+2nUlsjIApISlFwAAAIABQRkAAAAwICgDAAAABgRlAAAAwICgDAAAABgQlAEAAAADgjIAAABgwD7KyD5/PSjhYGfUfGACAACABwjKyDqu62rLtjYFgwFFItGu9rH+lQQAAPogll4AAAAABgRlAAAAwICgDAAAABgQlAEAAAADgjIAAABgQFAGAAAADNgeDr4pdOJSOJzQbluuD9UAAAAcjqAM/4TD2t6wKqF5TE21D8UAuWmAHVNlRVFCuyN+4QSAT4qgDAC5LBLWxuUrEprL5tZ5XwsA5BnWKAMAAAAGBGUAAADAgKUXyKhkN+xJkm1LrmV5XBEAAEBqCMrIrCQ37Ekf3rS3ZVtbQvvYDJcE5KJAwFFhpCOhnV1iACBzCMoAkAPcSETbG9YktLNLDABkDmuUAQAAAANmlAEASXV3n4FCIR2MMd8CIH8RlAEAyXVzn8HomqmS08/jggDAO0wFAAAAAAYEZQAAAMCAoAwAAAAYsEYZANArgaCjflHD3s4FBToQ4TAhALmPoAwAkBN0FI7GEzvs5IE3HonoqWX3JbRXzauXbG7yA5D7CMoA0IckO+HPdUJ6cN1bCe11p5Wn7RpsJwcg1xCUkVm2JdfiI1ggW7iRiN5+9PGE9mPOnq7KiqKEdkc9PyI72SmCbCcHINcQlJFRrutqy7Y2Y99Yb0sB8Femn8mxkYg2Ll+R0F42t65X1zD9ghwIJs40h9uiKnRsZpoBZCWCMgAg7ZKF8Y/PZjuOrVHTqiWrwKvSACBlBOU0ix44oMKYYW2elFPr85Ld2BMK2IpFYik/XmLZBZAu+bCM6eMBOhgMqNx1+asCQFYiKKdZvKMzL457DUfjWrmuKaG9dtJ4OT14fG9uBAJgZpyl9b4MAOgzcmN6EwAAAPAYM8oAgJxT6MSlcDixI4eWuAHIfgRlAEC3snJtdDhsXOaWS0vcAGQ/gjIAoFts8Zj9nKCjtvc7FPvYLzXJbsAGkBqCMgDAV5ZlJQQ8iZDXE+FoXGtffFv793ce1p7sBmwAqSEop1ksHk/+MaWdhR9fAoDPXNc175pz+liFYoZ1yJJsq+cnBgJATxGU0ywWT34S3eg83it0gB1L2/G3ACBJipjXIUvSmOnVHhcDoC8iKKNHBgXiind2JrQ7caX1+FsAQOoG2DGdeswAxWKFCe0d7AIC9BpBGT0S7+zUU7f/V0L7WQRiAPBPJKxX//v3ikSihzVXzauXbHYBAXqLoNzHJduL1CkIspQCAAD0aQTlvi7JXqRjaqpZSgHAE4GA0/NfzK0k+zv38KZpJ+goHI0ntLPjBgCJoAwA8JkbiST9xby7w05MN0739KbpcDRu3HGDbdUASARlAEAW47ATAH4iKAMAkAYs4wDyD0EZAIA0YBkHkH8Iyn2dbXW7BhAAcgnHYQNIJ4JyH+e65pMEx3pfCgB8YsmOw2ZWF0BvEJQBAH3WADtm3JqOE+0ASARlAEBfFgkbt6bjRDsAEkEZAJBHkh1eMigQ1wdR0wyx+R6NZGudJdY7A30JQbkP6B90Fe/sNPaxZg9APkl2eMnwefVa+eyOhPa608rNr5NkrbMk1Z0+VqFYOPE5oVDPigWQ9QjKfUC8s1NP3f5fxr6zOJIaAHomEtb2hlUJzUfVTvWhGACZxJ0KAAAAgAEzynkk2alQydbgAQAyLxiwFeHEPiAnEZTzSLJToZKtwQMA9JxlJTuoyTwpEY7G1WD6uznJWmeFQjrI1nRAViAoAwDyXrLdMBy5PX6tZAc1HROwjdcYaCeZNU6y1nl0zVTJYWs6IBsQlAEAeS/ZbhhlabyhOdk1qubVp+0a6ZJsqR7LQYDDEZRzUKETl8JsTQQAmZJsBlrq3Sy0SbIlHIGgo8JIR+IT0rgkI9lSPY76Bg5HUM5F0YjefnR1QvPoGrYmAoB0SDY7LPV8FjpZ6LbduHEJx9hIRNsb1iS0e7EkI9mNh4MCcbkd5v34A/1CinQmTt7YBQU6EOFmcuS2rA/KW7du1YIFC9Te3q6ioiItXrxYY8eO9bssX/V0fVy6Zj8AAD3Xm2UfxpsF7cyHzlA8ovbd7Qntg4sL9XZD4gSNJI2pqdZTy+5LaD/zh99SzHAMeD4s70j2ya4kbsbMM1kflBctWqT6+nrNnDlTjzzyiBYuXKh77rnH77Kykhdr8AAAmWeaDBntuj3e7TMYcKRo4jKOQChknliJdCb9d8S800dyyU43nFV5rPF48GQBujfrqZOdSJtslvvjwTfcFlVhLJo89IbNN2JKyWf+k4brPhqsc2U8sjoot7a2qrGxUXfffbckqaamRtdee63a2tpUXFyc0mvYHvwGfijLtdW/aJC5z7Z7VI8TdIwfgVlyjNewHPO1k7X35jnpak/lOYFAQMFoNKtqysdrZ2NNfl47lZr60nsz0zUFAoGcGg8/r207jmSbA4QlR4VDBh32vpQkNx7TH363MuHxE8+vVdOqpxLay8+vTVrTO+9FjNc+Jkm9wVBQZ322JKF9gBXVrvb3E9oHDx+i/U7i9xdzpf/7wtsJ7TNPHa24zLO6dkR61vB9T7pwlvorMVzbcemd//fM3752bMVjcR19VqVkuEYgYCswaKDx2lbAkW34RcCKx7TjkGt85OizKmUZrmEVhLQ/mjgeAwJxuYalLgoG1ZEkYPZz4lLE8P+vm+ek83VM2SfZeJRXnS7b9nalfHfZzHJdN2s/l9+0aZMuvfRSPfroo11t06ZN09KlS3XCCSf4WBkAAADyXfbMbQMAAABZJKuDcllZmXbt2qVY7MOPSWKxmFpaWlRWVuZzZQAAAMh3WR2US0pKVFFRoYaGBklSQ0ODKioqUl6fDAAAAPRWVq9RlqSmpiYtWLBA77//vgYPHqzFixdr3LhxfpcFAACAPJf1QRkAAADwQ1YvvQAAAAD8QlAGAAAADAjKAAAAgAFBGQAAADAgKPfC1q1bde6556q6ulrnnnuu3nrrrYTHxGIxXX311TrzzDM1efJkPfDAA94XmiNSGc/169dr1qxZOvHEE7V48WLvi8wRqYzl7bffrunTp6u2tlazZs3SM88kHiGK1MbywQcfVG1trWbOnKna2lrdc8893heaA1IZy4+8+eab+tznPsfPeRKpjOUvf/lLfeUrX9HMmTM1c+ZMXX311d4XmgNSfV8+9thjqq2tVU1NjWpra7Vnzx5vC80BqYzlJZdc0vWenDlzpo4//ng99VTiMepZx0WPnX/++e6KFStc13XdFStWuOeff37CYx5++GF39uzZbiwWc1tbW92JEye627dv97rUnJDKeL711ltuY2Oje9NNN7k///nPvS4xZ6QyluvWrXMPHDjguq7rbt682f3CF77gHjx40NM6c0EqY/nBBx+48Xi868+nn366u3nzZk/rzAWpjKXrum40GnXPO+8891/+5V/4OU8ilbG89dZbGb8UpDKWGzZscKdOneq2tLS4ruu677//vtvR0eFpnbkg1Z/xj2zevNn90pe+5HZ2dnpR3ifCjHIPtba2qrGxUTU1NZKkmpoaNTY2qq2t7bDHPfbYYzrnnHNk27aKi4t15plnavXq1X6UnNVSHc8xY8aooqJCgUDAjzJzQqpjOXHiRBUWFkqSjjvuOLmuq/b2dq/LzWqpjuXAgQNlWZYkqaOjQ5FIpOtrfCjVsZSku+66S6effrrGjh3rcZW5oSdjie6lOpa//e1vNXv2bJWWlkqSBg0apIKCAs/rzWa9eV/+z//8j2praxUKhbwqs9cIyj3U3NysESNGyHEcSZLjOBo+fLiam5sTHldeXt71dVlZmXbu3Olprbkg1fHEkfVmLFesWKGjjz5aI0eO9KrMnNCTsXzqqac0ffp0nXHGGZozZ46OO+44r8vNaqmO5Wuvvab169frggsu8KHK3NCT9+Wjjz6q2tpazZ49Wy+99JLXpWa9VMeyqalJ27dv17e+9S197Wtf07Jly+Ry/MRhevpvTzgc1sqVK/X1r3/dyzJ7jek5oI/605/+pF/84hf6zW9+43cpOa2qqkpVVVXasWOH5s2bp0mTJnF6aA9FIhFdeeWVuuGGG7r+sUXv1dXV6fvf/76CwaCeffZZzZ07V4899piGDh3qd2k5JxaL6fXXX9fdd9+tcDisOXPmqLy8XGeffbbfpeWsJ598UuXl5aqoqPC7lJQwo9xDZWVl2rVrl2KxmKQPf4haWlpUVlaW8LgdO3Z0fd3c3MysnUGq44kj68lYvvTSS5o/f75uv/12Qp1Bb96X5eXl+sxnPqOnn37aoypzQypjuXv3br399tu66KKLVFlZqf/4j//Q/fffryuvvNKvsrNSqu/L0tJSBYNBSdJpp52msrIyvfHGG57Xm81SHcvy8nJNmTJFoVBIAwcOVFVVlTZs2OBHyVmrp39fPvjggzkzmywRlHuspKREFRUVamhokCQ1NDSooqJCxcXFhz1uypQpeuCBBxSPx9XW1qYnn3xS1dXVfpSc1VIdTxxZqmO5YcMG/fjHP9att96qE044wY9Ss16qY9nU1NT157a2Nj333HP69Kc/7Wmt2S6VsSwvL9dzzz2ntWvXau3atfrOd76jb37zm7r22mv9Kjsrpfq+3LVrV9efN2/erHfffVfHHHOMp7Vmu1THsqamRuvXr5fruopEIvrjH/+o448/3o+Ss1ZP/h3fuXOnXnzxRdXW1npdZu/5fDNhTvrLX/7ifuMb33DPOuss9xvf+Ibb1NTkuq7rzpkzx92wYYPruh/evb1w4UK3qqrKraqqcu+77z4/S85qqYzn888/706cONH9/Oc/75500knuxIkT3XXr1vlZdlZKZSxnzZrlnnLKKe6MGTO6/nvttdf8LDsrpTKW1113nTtt2jR3xowZbm1trXvPPff4WXLWSmUsD8WuDcmlMpaXXHKJO336dLe2ttadNWuW+/TTT/tZctZKZSxjsZh7/fXXu1OmTHGnTZvmXn/99W4sFvOz7KyU6s/4smXL3B/96Ed+ldkrluuyKh0AAAD4OJZeAAAAAAYEZQAAAMCAoAwAAAAYEJQBAAAAA4IyAAAAYEBQBgAAAAwIygDQRzz00EP6h3/4B7/LAICcQVAGAHT56BhaAIDEgSMAkIeam5t13XXX6YUXXpDrujr++OP15z//WdFoVP369ZPjOHrhhRe0YMECFRQUaMeOHXr++ee1bNkynXrqqX6XDwBZIeB3AQCA9IrFYvre976nL3/5y1q7dq0cx9HGjRv19ttv64EHHtDy5csPe3xDQ4Puuusu3XnnnYpEIj5VDQDZh6UXAJBnNmzYoJaWFl1yySXq37+/CgoKNGHChKSPr6qq0he+8AXZtq2CggIPKwWA7EZQBoA809zcrPLycgUCqX1oWFZWluGKACA3EZQBIM+UlZWpublZ0Wj0sHbLsnyqCAByE0EZAPLMZz/7WZWWlurf/u3fdODAAXV2durFF19USUmJdu3apXA47HeJAJATCMoAkGccx9Edd9yhbdu26YwzztCkSZO0atUqffnLX9anPvUpffWrX9Upp5zid5kAkPXYHg4AAAAwYEYZAAAAMCAoAwAAAAYEZQAAAMCAoAwAAAAYEJQBAAAAA4IyAAAAYEBQBgAAAAwIygAAAIABQRkAAAAw+P+6vq9mRpqEKwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 842.4x595.44 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.set(rc={'figure.figsize': (11.7, 8.27)})\n",
    "sns.histplot(data=df, x='ctr', hue='exp_group', kde=False, palette=['r', 'b'], alpha=0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36b3b063-930d-4e57-88de-013d293c8dc7",
   "metadata": {},
   "source": [
    "Метрики CTR в группах примерно одинаковые."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54a153fb-0ffd-4684-b5f9-977e77418279",
   "metadata": {},
   "source": [
    "## Bootstrapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "90ab6bef-469e-4c21-8d6d-6a3f953cbf1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "group_A = df[df['exp_group'] == 2].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "fd7b3480-f413-4205-b43d-d326f1b638ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "group_B = df[df['exp_group'] == 3].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "fa610e01-a697-402d-98a8-76089dfb07d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "pvalues = []\n",
    "for i in range(10000):\n",
    "    statistic, pvalue = stats.ttest_ind(group_A['ctr'].sample(500, replace=False), group_B['ctr'].sample(500, replace=False), equal_var=False)\n",
    "    pvalues.append(pvalue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "882d8f31-e319-407c-b292-de35c8a5a18f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<seaborn.axisgrid.FacetGrid at 0x7fa5b9160d00>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVwAAAFcCAYAAACEFgYsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAYBElEQVR4nO3df2yVZ93H8U9PaRsWyrqWAqcFQyCTVTHq6EKySB7XDltYi24BIRUUEEnYnMQFAhmjLbANS3FxZCAzmmncMjL+kEnBlpnGh9XoHAYktfzYGJuYHn6d2mfjZ0/PuZ8/Fipoe3rf7Tnf84P3K1lCz/XtdX+v0/LZfa5zn5sMx3EcAQDizpfoBgDgTkHgAoARAhcAjBC4AGCEwAUAIwQuABgZYXWgsrIyZWdnKycnR5K0evVqzZw5U0ePHlVtba1u3Lih4uJiNTY2qqCgQJKijgFAqsmwug63rKxMu3bt0mc/+9m+xyKRiCoqKrRlyxaVlpZq586dOnv2rLZs2RJ1zItg8LIiEfdLvOeeu/Svf131dIxkxnqSG+tJbkNZT2Fh7oBjCd1SaG9vV05OjkpLSyVJCxcuVHNz86Bj8TRiRGbcj2GJ9SQ31pPcYr0esy0F6dNtBMdxNH36dD311FMKBAIqKirqG8/Pz1ckElF3d3fUsby8PMu2ASAmzAL3tddek9/vV09Pj5577jlt2rRJs2bNivtxCwpGef6eaC8JUhHrSW6sJ7nFcj1mgev3+yVJ2dnZqqmp0cqVK/Xtb39bnZ2dfTVdXV3y+XzKy8uT3+8fcMwLr3u4hYW5unjxE0/HSGasJ7mxnuQ2lPUkfA/36tWr+uSTT5t2HEcHDhxQSUmJpk2bpuvXr+vw4cOSpN27d6uyslKSoo4BQCoyOcMNBoN68sknFQ6HFYlENGXKFNXV1cnn82nr1q2qq6u77dIvSVHHACAVmV0WlihsKbCeZMZ6kltKbikAAAhcADBD4AKAEQIXAIwQuABghMAFACMELgAYMb15DZAqRt89UjnZ7v963Ojp1cf/dy2OHSEdELhAP3KyR2jNi//rur5x1f/EsRukC7YUAMAIZ7hAmmE7JHkRuECaYTskebGlAABGOMNFUvDyMriwMNfzy2CvL7OBeOA3EEnB7cvgrKwRCoV6Pb8M5mU2kgGBi5QU6o2k3b+dlShen0veZBs6AhcpKWuEjzPWGOG5tMObZgBghDNcIAXwpl964CcIpAAvb/rF+yX/YHu+/znGnu+/EbgAPIm253vzKpJbsef7b+zhAoARznD/A5fIAIgXAvc/eL1E5vknZroO6FBvWFkjMl3PTZgD6YXAHSYvAd246n+43hFIIK9Xe4R6IzE9PoF7B+M2frjTJPoj3gRuGhkoQKNtecRr+0QioGEjla5RTo0u4Up///fu7zKdm7z+35uPgCIZJdM1yoMhcJMYN2jBnSiVzli9Ss9VpQnOKJEOhnLikK6/9wQugLjixOHf+KQZABghcAHACFsKQALc+sYQb4zeOQhcIAFuXsoU7bK9W6XzvuadhC0FADBC4AKAEQIXAIwQuABghDfNEDd8NBm4HYGLuPF6r2Ag3bGlAABGCFwAMMKWAhAD7FfDDQIXiAHuiAU32FIAACMELgAYIXABwAiBCwBGCFwAMELgAoARAhcAjBC4AGDEPHBfeuklTZ06VadOnZIkHT16VHPnzlVFRYWWLVumYDDYVxttDABSjWng/v3vf9fRo0dVXFwsSYpEIlqzZo1qa2vV0tKi0tJSbdu2bdAxAEhFZoHb09OjTZs2qb6+vu+x9vZ25eTkqLS0VJK0cOFCNTc3DzoGAKnILHBffPFFzZ07VxMmTOh7LBAIqKioqO/r/Px8RSIRdXd3Rx0DgFRkcvOaI0eOqL29XatXr7Y43G0KCkZ5/p6sLG9Pi5f6eM49UH20ORLRz3Brb9YlU+/Dqfe67nj2Eot6r7+D8e5nuHPH8i5wJoH77rvv6vTp0yovL5cknTt3Tt/97ne1ePFidXZ29tV1dXXJ5/MpLy9Pfr9/wDEvgsHLikQc1/WFhbkKhXo9HcNLfTzn7q8+K2tE1Dms+xlu7a3rSabeh1o/2M9nqPMnaq0DrSeZnnuvc1+8+Imn+mgBbbKlsGLFCrW1tam1tVWtra0aP368fvGLX2j58uW6fv26Dh8+LEnavXu3KisrJUnTpk0bcAwAUlFC74fr8/m0detW1dXV6caNGyouLlZjY+OgYwCQihISuK2trX1/vv/++7Vv375+66KNAUCq4ZNmAGCEwAUAIwQuABghcAHACIELAEYIXAAwQuACgBECFwCMELgAYITABQAjBC4AGCFwAcAIgQsARghcADBC4AKAEQIXAIwQuABghMAFACMELgAYIXABwAiBCwBGCFwAMELgAoARAhcAjBC4AGCEwAUAIwQuABghcAHACIELAEYIXAAwQuACgBECFwCMELgAYITABQAjBC4AGCFwAcAIgQsARghcADBC4AKAEQIXAIwQuABghMAFACMELgAYIXABwAiBCwBGCFwAMELgAoARAhcAjBC4AGCEwAUAIwQuABghcAHACIELAEZGWB3o8ccf1z//+U/5fD7ddddd2rBhg0pKSnTmzBmtW7dO3d3dysvLU0NDgyZNmiRJUccAINWYneE2NDTot7/9rfbu3atly5bp6aefliTV1dWppqZGLS0tqqmpUW1tbd/3RBsDgFRjFri5ubl9f758+bIyMjIUDAbV0dGhqqoqSVJVVZU6OjrU1dUVdQwAUpHZloIkrV+/Xn/84x/lOI5+/vOfKxAIaNy4ccrMzJQkZWZmauzYsQoEAnIcZ8Cx/Px8y7YBICZMA/e5556TJO3du1dbt27VqlWr4n7MgoJRnr8nK8vb0+KlPp5zD1QfbY5E9DPc2pt1ydT7cOq9rjuevcSi3uvvYLz7Ge7chYW5gxe5ZBq4N33jG99QbW2txo8fr/PnzyscDiszM1PhcFgXLlyQ3++X4zgDjnkRDF5WJOK4ri8szFUo1OvpGF7q4zl3f/VZWSOizmHdz3Brb11PMvU+1PrBfj5DnT9Rax1oPcn03Hud++LFTzzVRwtokz3cK1euKBAI9H3d2tqqu+++WwUFBSopKVFTU5MkqampSSUlJcrPz486BgCpyOQM99q1a1q1apWuXbsmn8+nu+++W7t27VJGRobq6+u1bt067dy5U6NHj1ZDQ0Pf90UbA4BUYxK4Y8aM0RtvvNHv2JQpU7Rnzx7PYwCQavikGQAYIXABwAiBCwBGCFwAMELgAoARAhcAjBC4AGCEwAUAIwQuABhxHbi/+93v+n28ubk5Zs0AQDpzHbjr16/v93H+FQYAcGfQeymcPXtWkuQ4Tt+fbx3Lzs6OT2cAkGYGDdxZs2YpIyNDjuNo1qxZt42NGTNGTz75ZNyaA4B0MmjgnjhxQpK0aNEivfrqq3FvCADSles9XMIWAIbH9f1wz549q5/85Cc6fvy4rl69etvYH/7wh1j3BQBpx3Xgrl69WhMnTtTatWs1cuTIePYEAGnJdeC+9957ev311+Xz8VkJABgK1+n5wAMPqKOjI569AEBac32GW1xcrOXLl2vWrFkaM2bMbWOrVq2KeWMAkG5cB+61a9f00EMPqbe3V+fOnYtnTwCQllwH7pYtW+LZBwCkPU+XhQ1k4sSJMWkGANKZ68C99SO+N2VkZEiSjh8/HvvOACDNuA7cmx/xvenixYt66aWXVFpaGvOmACAdDfmi2sLCQq1fv14vvPBCLPsBgLQ1rE8xfPDBB7p27VqsegGAtOZ6S6GmpqZvz1b69DKx999/X0888URcGgOAdOM6cOfPn3/b1yNHjtR9992nSZMmxbonAEhLrgP30UcfjWcfAJD2XO/hhkIhbd++XeXl5frCF76g8vJybd++XT09PfHsDwDShusz3MbGRh07dkwbN25UUVGROjs7tXPnTl2+fFlPP/10PHsEgLTgOnCbm5v15ptv6p577pEkTZ48WZ/73Of09a9/ncAFABdcbync+gkzN48DAG7nOnArKyu1cuVKvf322zp9+rQOHTqkJ554QpWVlfHsDwDShusthTVr1uinP/2pNm3apAsXLmjcuHF65JFHtHLlynj2BwBpY9Az3L/+9a9qbGxUdna2Vq1apbfeekt/+9vfdPDgQfX09PCvQACAS4MG7ssvv6wHHnig37EZM2Zo165dMW8KANLRoIF7/PhxzZw5s9+xBx98UO3t7TFvCgDS0aCBe/nyZYVCoX7Hent7deXKlZg3BQDpaNDAnTx5stra2voda2tr0+TJk2PeFACko0EDd8mSJaqrq9PBgwcViUQkSZFIRAcPHlR9fb2WLl0a9yYBIB0MellYdXW1Ll26pLVr1yoUCikvL0/d3d3KysrSD37wA1VVVVn0CQApz9V1uEuXLtX8+fN15MgRdXd3Ky8vT1/+8pc1atSoePcHAGnD9QcfRo0aNeDVCgCAwQ3rn9gBALhH4AKAEQIXAIwQuABghMAFACMELgAYIXABwAiBCwBGCFwAMGISuP/617/0ve99TxUVFaqurtb3v/99dXV1SZKOHj2quXPnqqKiQsuWLVMwGOz7vmhjAJBqTAI3IyNDy5cvV0tLi/bt26eJEydq27ZtikQiWrNmjWpra9XS0qLS0lJt27ZNkqKOAUAqMgncvLw8zZgxo+/rL33pS+rs7FR7e7tycnJUWloqSVq4cKGam5slKeoYAKQi1zeviZVIJKLXX39dZWVlCgQCKioq6hvLz89XJBJRd3d31LG8vDzXxyso8H5Hs6wsb0+Ll/p4zj1QfbQ5EtHPcGtv1iVT78Op97ruePYSi3qvv4Px7me4cxcW5nqqj8Y8cDdv3qy77rpLixYt0ltvvRX34wWDlxWJOK7rCwtzFQr1ejqGl/p4zt1ffVbWiKhzWPcz3Npb15NMvQ+1frCfz1DnT9RaB1pPMj33Xue+ePETT/XRAto0cBsaGvTRRx9p165d8vl88vv96uzs7Bvv6uqSz+dTXl5e1DEASEVml4W98MILam9v144dO5SdnS1JmjZtmq5fv67Dhw9Lknbv3q3KyspBxwAgFZmc4b733nt6+eWXNWnSJC1cuFCSNGHCBO3YsUNbt25VXV2dbty4oeLiYjU2NkqSfD7fgGMAkIpMAvfee+/VyZMn+x27//77tW/fPs9jAJBq+KQZABghcAHACIELAEYIXAAwQuACgBECFwCMELgAYITABQAjBC4AGCFwAcAIgQsARghcADBC4AKAEQIXAIwQuABghMAFACMELgAYIXABwAiBCwBGCFwAMELgAoARAhcAjBC4AGCEwAUAIwQuABghcAHACIELAEYIXAAwQuACgBECFwCMELgAYITABQAjBC4AGCFwAcAIgQsARghcADBC4AKAEQIXAIwQuABghMAFACMELgAYIXABwAiBCwBGCFwAMELgAoARAhcAjBC4AGCEwAUAIwQuABghcAHACIELAEYIXAAwQuACgBGTwG1oaFBZWZmmTp2qU6dO9T1+5swZLViwQBUVFVqwYIE+/PBDV2MAkIpMAre8vFyvvfaaiouLb3u8rq5ONTU1amlpUU1NjWpra12NAUAqMgnc0tJS+f3+2x4LBoPq6OhQVVWVJKmqqkodHR3q6uqKOgYAqWpEog4cCAQ0btw4ZWZmSpIyMzM1duxYBQIBOY4z4Fh+fr6n4xQUjPLcW1aWt6fFS3085x6oPtociehnuLU365Kp9+HUe113PHuJRb3X38F49zPcuQsLcz3VR5OwwLUSDF5WJOK4ri8szFUo1OvpGF7q4zl3f/VZWSOizmHdz3Brb11PMvU+1PrBfj5DnT9Rax1oPcn03Hud++LFTzzVRwvohAWu3+/X+fPnFQ6HlZmZqXA4rAsXLsjv98txnAHHACBVJeyysIKCApWUlKipqUmS1NTUpJKSEuXn50cdA4BUZXKG++yzz+rgwYO6dOmSli5dqry8PO3fv1/19fVat26ddu7cqdGjR6uhoaHve6KNAUAqMgncZ555Rs8888x/PT5lyhTt2bOn3++JNgYAqYhPmgGAEQIXAIwQuABghMAFACMELgAYIXABwAiBCwBGCFwAMELgAoARAhcAjBC4AGCEwAUAIwQuABghcAHACIELAEYIXAAwQuACgBECFwCMELgAYITABQAjBC4AGCFwAcAIgQsARghcADBC4AKAEQIXAIwQuABghMAFACMELgAYIXABwAiBCwBGCFwAMELgAoARAhcAjBC4AGCEwAUAIwQuABghcAHACIELAEYIXAAwQuACgBECFwCMELgAYITABQAjBC4AGCFwAcAIgQsARghcADBC4AKAEQIXAIwQuABghMAFACMELgAYSfrAPXPmjBYsWKCKigotWLBAH374YaJbAoAhSfrAraurU01NjVpaWlRTU6Pa2tpEtwQAQzIi0Q1EEwwG1dHRoVdeeUWSVFVVpc2bN6urq0v5+fmu5vD5Mjwf957cnLjVx3Pu/upHZI1QbygzafoZbu2t60mm3odaP9jPZ6jzJ2qtA60nmZ57r3MPJUMGkuE4jhOz2WKsvb1da9eu1f79+/semzNnjhobG/X5z38+gZ0BgHdJv6UAAOkiqQPX7/fr/PnzCofDkqRwOKwLFy7I7/cnuDMA8C6pA7egoEAlJSVqamqSJDU1NamkpMT1/i0AJJOk3sOVpNOnT2vdunX6+OOPNXr0aDU0NGjy5MmJbgsAPEv6wAWAdJHUWwoAkE4IXAAwQuACgBECFwCM3JGB6+aGOOFwWBs3btTDDz+sWbNmac+ePfaNuuRmPTt27NAjjzyi6upqPfbYY3r77bftG3XJyw2LPvjgA33xi19UQ0ODXYMeuV3PgQMHVF1draqqKlVXV+vSpUu2jbrkZj3BYFArVqxQdXW1Zs+erfr6evX29to360JDQ4PKyso0depUnTp1qt+amOWBcwdavHixs3fvXsdxHGfv3r3O4sWL/6vmN7/5jbNs2TInHA47wWDQmTlzpnP27FnrVl1xs55Dhw45V69edRzHcY4fP+5Mnz7duXbtmmmfbrlZj+M4Tm9vr7No0SLnqaeecn70ox9ZtuiJm/UcO3bMmT17tnPhwgXHcRzn448/dq5fv27ap1tu1vPss8/2/Ux6enqcefPmOfv37zft0613333X6ezsdB566CHn5MmT/dbEKg/uuDPcmzfEqaqqkvTpDXE6OjrU1dV1W92BAwc0f/58+Xw+5efn6+GHH1Zzc3MiWo7K7XpmzpypkSNHSpKmTp0qx3HU3d1t3e6g3K5Hkn72s5/pq1/9qiZNmmTcpXtu1/PLX/5Sy5YtU2FhoSQpNzdXOTnebrJiwe16MjIydOXKFUUiEfX09CgUCmncuHGJaHlQpaWlg356NVZ5cMcFbiAQ0Lhx45SZ+ekdjTIzMzV27FgFAoH/qisqKur72u/369y5c6a9uuF2Pbfau3evPvOZz2j8+PFWbbrmdj0nTpxQW1ublixZkoAu3XO7ntOnT+vs2bP61re+pUcffVQ7d+6Uk4SXyLtdz+OPP64zZ87oK1/5St9/06dPT0TLMRGrPLjjAvdO95e//EUvvviifvzjHye6lSELhULasGGDNm7c2PcXP9WFw2GdPHlSr7zyin7961/r0KFDevPNNxPd1pA1Nzdr6tSpamtr06FDh3T48OGkfIVo7Y4LXLc3xPH7/ers7Oz7OhAIJOUZoZcb/Bw5ckRr1qzRjh07kvbj0W7Wc/HiRf3jH//QihUrVFZWpl/96ld64403tGHDhkS1PSC3P5+ioiJVVlYqOztbo0aNUnl5uY4dO5aIlqNyu55XX31Vc+fOlc/nU25ursrKyvTOO+8kouWYiFUe3HGB6/aGOJWVldqzZ48ikYi6urr0+9//XhUVFYloOSq36zl27Jh++MMfavv27Ul9L2E36ykqKtI777yj1tZWtba26jvf+Y6++c1vavPmzYlqe0Bufz5VVVVqa2uT4zgKhUL685//rPvuuy8RLUfldj0TJkzQoUOHJEk9PT3605/+pHvvvde831iJWR4M6+29FPX+++878+bNc772ta858+bNc06fPu04juMsX77cOXbsmOM4n74DXltb65SXlzvl5eXO7t27E9lyVG7W89hjjzkzZsxw5s6d2/ffiRMnEtn2gNys51bbt29P6qsU3KwnHA47zz//vFNZWenMmTPHef75551wOJzItgfkZj0fffSRs2TJEqeqqsqZPXu2U19f74RCoUS2PaDNmzc7M2fOdEpKSpwHH3zQmTNnjuM48ckDbl4DAEbuuC0FAEgUAhcAjBC4AGCEwAUAIwQuABghcAHACIELAEYIXAAw8v+QZg064zyFkQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.displot(pvalues)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fcca50d8-72b4-41af-97de-6891da35a757",
   "metadata": {},
   "source": [
    "Распределение p-values похоже на равномерное. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "a7b7edf2-9d0a-45ca-984a-12d71d334c74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.05"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(np.array(pvalues) <= 0.05) / 10000"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f500526d-b2dc-4d75-a08d-ac97a6b80d58",
   "metadata": {},
   "source": [
    "## Summary"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed702db2-b448-4435-9574-13781fd88eb6",
   "metadata": {},
   "source": [
    "Система сплитования в целом работает корректно. Ключевая метрика между группами не отличается в тесте и, основываясь на данных bootstrapping, в целом.\n",
    "Группы не отличаются между собой и можно утверждать, что возникшее после воздействия отличие будет вызвано воздействием.  "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}