diff --git a/.gitignore b/.gitignore index 19e35c6a..8cad9e17 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,11 @@ lightning_logs/ # pytest coverage files .coverage* +# Sphinx +docs/build/ +.vscode/ +examples/ode/lorenz63/.ipynb_checkpoints/ + # build artifacts build/ dist/ diff --git a/examples/ode/DuffingEquation/DuffingTest_DNN.ipynb b/examples/ode/DuffingEquation/DuffingTest_DNN.ipynb new file mode 100644 index 00000000..3a821964 --- /dev/null +++ b/examples/ode/DuffingEquation/DuffingTest_DNN.ipynb @@ -0,0 +1,732 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "DuffingTest_DNN.ipynb", + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5Z8Z-OzS6Z-G", + "outputId": "f83e0d3c-fb76-4938-8aac-3e01134a1969" + }, + "source": [ + "!pip install pytorch_lightning" + ], + "execution_count": 1, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting pytorch_lightning\n", + " Downloading pytorch_lightning-1.4.9-py3-none-any.whl (925 kB)\n", + "\u001b[?25l\r\u001b[K |▍ | 10 kB 21.7 MB/s eta 0:00:01\r\u001b[K |▊ | 20 kB 16.9 MB/s eta 0:00:01\r\u001b[K |█ | 30 kB 20.8 MB/s eta 0:00:01\r\u001b[K |█▍ | 40 kB 13.1 MB/s eta 0:00:01\r\u001b[K |█▊ | 51 kB 10.1 MB/s eta 0:00:01\r\u001b[K |██▏ | 61 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██▌ | 71 kB 7.8 MB/s eta 0:00:01\r\u001b[K |██▉ | 81 kB 8.5 MB/s eta 0:00:01\r\u001b[K |███▏ | 92 kB 7.8 MB/s eta 0:00:01\r\u001b[K |███▌ | 102 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████ | 112 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████▎ | 122 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████▋ | 133 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████ | 143 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████▎ | 153 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████▋ | 163 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████ | 174 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████▍ | 184 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████▊ | 194 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████ | 204 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████▍ | 215 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████▉ | 225 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████▏ | 235 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████▌ | 245 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████▉ | 256 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████████▏ | 266 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████████▋ | 276 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████████ | 286 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████████▎ | 296 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████████▋ | 307 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████████ | 317 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████████▎ | 327 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████████▊ | 337 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████████ | 348 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████████▍ | 358 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████████▊ | 368 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████████████ | 378 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████████████▌ | 389 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████████████▉ | 399 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████████████▏ | 409 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████████████▌ | 419 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████████████▉ | 430 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████████████▏ | 440 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████████████▋ | 450 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████████████ | 460 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████████████▎ | 471 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████████████▋ | 481 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████████████████ | 491 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████████████████▍ | 501 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████████████████▊ | 512 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████████████████ | 522 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████████████████▍ | 532 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████████████████▊ | 542 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████████████████▏ | 552 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████████████████▌ | 563 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████████████████▉ | 573 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████████████████▏ | 583 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████████████████▌ | 593 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████████████████▉ | 604 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████████████████████▎ | 614 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████████████████████▋ | 624 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████████████████████ | 634 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████████████████████▎ | 645 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████████████████████▋ | 655 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████████████████████ | 665 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████████████████████▍ | 675 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████████████████████▊ | 686 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████████████████████ | 696 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████████████████████▍ | 706 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████████████████████▊ | 716 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████████████████████████▏ | 727 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████████████████████████▌ | 737 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████████████████████████▉ | 747 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████████████████████████▏ | 757 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████████████████████████▌ | 768 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████████████████████████ | 778 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████████████████████████▎ | 788 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████████████████████████▋ | 798 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████████████████████████ | 808 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████████████████████████▎ | 819 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████████████████████████▊ | 829 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████████████████████████████ | 839 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████████████████████████████▍ | 849 kB 8.4 MB/s eta 0:00:01\r\u001b[K |█████████████████████████████▊ | 860 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████████████████████████████ | 870 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████████████████████████████▍ | 880 kB 8.4 MB/s eta 0:00:01\r\u001b[K |██████████████████████████████▉ | 890 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████████████████████████████▏| 901 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████████████████████████████▌| 911 kB 8.4 MB/s eta 0:00:01\r\u001b[K |███████████████████████████████▉| 921 kB 8.4 MB/s eta 0:00:01\r\u001b[K |████████████████████████████████| 925 kB 8.4 MB/s \n", + "\u001b[?25hCollecting pyDeprecate==0.3.1\n", + " Downloading pyDeprecate-0.3.1-py3-none-any.whl (10 kB)\n", + "Requirement already satisfied: tqdm>=4.41.0 in /usr/local/lib/python3.7/dist-packages (from pytorch_lightning) (4.62.3)\n", + "Collecting PyYAML>=5.1\n", + " Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n", + "\u001b[K |████████████████████████████████| 596 kB 39.5 MB/s \n", + "\u001b[?25hRequirement already satisfied: torch>=1.6 in /usr/local/lib/python3.7/dist-packages (from pytorch_lightning) (1.9.0+cu111)\n", + "Requirement already satisfied: numpy>=1.17.2 in /usr/local/lib/python3.7/dist-packages (from pytorch_lightning) (1.19.5)\n", + "Collecting torchmetrics>=0.4.0\n", + " Downloading torchmetrics-0.5.1-py3-none-any.whl (282 kB)\n", + "\u001b[K |████████████████████████████████| 282 kB 51.8 MB/s \n", + "\u001b[?25hCollecting fsspec[http]!=2021.06.0,>=2021.05.0\n", + " Downloading fsspec-2021.10.1-py3-none-any.whl (125 kB)\n", + "\u001b[K |████████████████████████████████| 125 kB 36.6 MB/s \n", + "\u001b[?25hRequirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.7/dist-packages (from pytorch_lightning) (21.0)\n", + "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from pytorch_lightning) (3.7.4.3)\n", + "Requirement already satisfied: tensorboard>=2.2.0 in /usr/local/lib/python3.7/dist-packages (from pytorch_lightning) (2.6.0)\n", + "Collecting future>=0.17.1\n", + " Downloading future-0.18.2.tar.gz (829 kB)\n", + "\u001b[K |████████████████████████████████| 829 kB 20.7 MB/s \n", + "\u001b[?25hCollecting aiohttp\n", + " Downloading aiohttp-3.7.4.post0-cp37-cp37m-manylinux2014_x86_64.whl (1.3 MB)\n", + "\u001b[K |████████████████████████████████| 1.3 MB 40.5 MB/s \n", + "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from fsspec[http]!=2021.06.0,>=2021.05.0->pytorch_lightning) (2.23.0)\n", + "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=17.0->pytorch_lightning) (2.4.7)\n", + "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch_lightning) (3.3.4)\n", + "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch_lightning) (0.12.0)\n", + "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch_lightning) (1.41.0)\n", + "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch_lightning) (1.8.0)\n", + "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch_lightning) (1.35.0)\n", + "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch_lightning) (57.4.0)\n", + "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch_lightning) (1.0.1)\n", + "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch_lightning) (0.37.0)\n", + "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch_lightning) (0.4.6)\n", + "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch_lightning) (0.6.1)\n", + "Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.2.0->pytorch_lightning) (3.17.3)\n", + "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from absl-py>=0.4->tensorboard>=2.2.0->pytorch_lightning) (1.15.0)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard>=2.2.0->pytorch_lightning) (4.7.2)\n", + "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard>=2.2.0->pytorch_lightning) (4.2.4)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard>=2.2.0->pytorch_lightning) (0.2.8)\n", + "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->pytorch_lightning) (1.3.0)\n", + "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard>=2.2.0->pytorch_lightning) (4.8.1)\n", + "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard>=2.2.0->pytorch_lightning) (0.4.8)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch_lightning) (2021.5.30)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch_lightning) (3.0.4)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch_lightning) (1.24.3)\n", + "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch_lightning) (2.10)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->pytorch_lightning) (3.1.1)\n", + "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch_lightning) (21.2.0)\n", + "Collecting yarl<2.0,>=1.0\n", + " Downloading yarl-1.7.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n", + "\u001b[K |████████████████████████████████| 271 kB 68.3 MB/s \n", + "\u001b[?25hCollecting multidict<7.0,>=4.5\n", + " Downloading multidict-5.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (160 kB)\n", + "\u001b[K |████████████████████████████████| 160 kB 67.3 MB/s \n", + "\u001b[?25hCollecting async-timeout<4.0,>=3.0\n", + " Downloading async_timeout-3.0.1-py3-none-any.whl (8.2 kB)\n", + "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard>=2.2.0->pytorch_lightning) (3.6.0)\n", + "Building wheels for collected packages: future\n", + " Building wheel for future (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for future: filename=future-0.18.2-py3-none-any.whl size=491070 sha256=8c063018aa301dc492d85fbe4044ef8bf855372d849f3b6f40c44a7b52ea1f17\n", + " Stored in directory: /root/.cache/pip/wheels/56/b0/fe/4410d17b32f1f0c3cf54cdfb2bc04d7b4b8f4ae377e2229ba0\n", + "Successfully built future\n", + "Installing collected packages: multidict, yarl, async-timeout, fsspec, aiohttp, torchmetrics, PyYAML, pyDeprecate, future, pytorch-lightning\n", + " Attempting uninstall: PyYAML\n", + " Found existing installation: PyYAML 3.13\n", + " Uninstalling PyYAML-3.13:\n", + " Successfully uninstalled PyYAML-3.13\n", + " Attempting uninstall: future\n", + " Found existing installation: future 0.16.0\n", + " Uninstalling future-0.16.0:\n", + " Successfully uninstalled future-0.16.0\n", + "Successfully installed PyYAML-6.0 aiohttp-3.7.4.post0 async-timeout-3.0.1 fsspec-2021.10.1 future-0.18.2 multidict-5.2.0 pyDeprecate-0.3.1 pytorch-lightning-1.4.9 torchmetrics-0.5.1 yarl-1.7.0\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "zu5Sh81e6N9a" + }, + "source": [ + "import torch\n", + "from pytorch_lightning import LightningModule\n", + "import torch.nn.functional as F\n", + "from torch import nn\n", + "from torch.utils.data import DataLoader, TensorDataset\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import mpl_toolkits.mplot3d.axes3d as p3" + ], + "execution_count": 2, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "Pf5K_82g5blO" + }, + "source": [ + "class ODEDNNSolver(LightningModule):\n", + " def __init__(\n", + " self, ode, init_vars, init_coeffs, dt, solver=\"euler\", outvar=None, **kwargs\n", + " ):\n", + " super().__init__(**kwargs)\n", + "\n", + " if ode.keys() != init_vars.keys():\n", + " raise ValueError(\"Inconsistent keys in ode and init_vars\")\n", + "\n", + " if solver == \"euler\":\n", + " self.step_solver = self.euler_step\n", + " elif solver == \"rk4\":\n", + " self.step_solver = self.runge_kutta_4_step\n", + " else:\n", + " raise ValueError(f\"Unrecognized solver {solver}\")\n", + "\n", + " for name, value in init_coeffs.items():\n", + " self.register_parameter(name, nn.Parameter(torch.tensor(value)))\n", + " \n", + " self.ode = ode\n", + " self.var_names = ode.keys()\n", + " self.init_vars = {\n", + " name: torch.tensor(value, device=self.device)\n", + " for name, value in init_vars.items()\n", + " }\n", + " self.coeffs = {name: param for name, param in self.named_parameters()}\n", + "\n", + " self.dnn = nn.Sequential(\n", + " nn.Linear(1,10),\n", + " nn.ReLU(),\n", + " nn.Linear(10,1),\n", + " nn.Tanh()\n", + " )\n", + "\n", + " self.outvar = self.var_names if outvar is None else outvar\n", + " self.dt = dt\n", + " self.criterion = F.mse_loss\n", + "\n", + " def euler_step(self, prev_val):\n", + " pred = {name: value.unsqueeze(0) for name, value in self.init_vars.items()}\n", + " for var in self.var_names:\n", + " pred[var] = prev_val[var] + self.ode[var](prev_val, self.coeffs, self.dnn) * self.dt\n", + " return pred\n", + "\n", + " def runge_kutta_4_step(self, prev_val):\n", + " pred = {name: value.unsqueeze(0) for name, value in self.init_vars.items()}\n", + "\n", + " k_1 = prev_val\n", + " k_2 = {}\n", + " k_3 = {}\n", + " k_4 = {}\n", + "\n", + " for var in self.var_names:\n", + " k_2[var] = (\n", + " prev_val[var] + self.ode[var](prev_val, self.coeffs, self.dnn) * 0.5 * self.dt\n", + " )\n", + "\n", + " for var in self.var_names:\n", + " k_3[var] = prev_val[var] + self.ode[var](k_2, self.coeffs, self.dnn) * 0.5 * self.dt\n", + "\n", + " for var in self.var_names:\n", + " k_4[var] = prev_val[var] + self.ode[var](k_3, self.coeffs, self.dnn) * self.dt\n", + "\n", + " for var in self.var_names:\n", + " result = self.ode[var](k_1, self.coeffs, self.dnn) / 6\n", + " result += self.ode[var](k_2, self.coeffs, self.dnn) / 3\n", + " result += self.ode[var](k_3, self.coeffs, self.dnn) / 3\n", + " result += self.ode[var](k_4, self.coeffs, self.dnn) / 6\n", + " pred[var] = prev_val[var] + result * self.dt\n", + "\n", + " return pred\n", + "\n", + " def solver(self, nt):\n", + " pred = {name: value.unsqueeze(0) for name, value in self.init_vars.items()}\n", + "\n", + " for n in range(nt - 1):\n", + " # create dictionary containing values from previous time step\n", + " prev_val = {var: pred[var][[n]] for var in self.var_names}\n", + " new_val = self.step_solver(prev_val)\n", + " for var in self.var_names:\n", + " pred[var] = torch.cat([pred[var], new_val[var]])\n", + "\n", + " # reformat output to contain desired (observed) variables\n", + " return torch.stack([pred[var] for var in self.outvar], dim=1)\n", + "\n", + " def forward(self, nt):\n", + " return self.solver(nt)\n", + "\n", + " def get_coeffs(self):\n", + " return {name: param.item() for name, param in self.named_parameters()}\n", + "\n", + " def fit(\n", + " self,\n", + " x,\n", + " optim,\n", + " optim_params=None,\n", + " max_epochs=10,\n", + " scheduler=None,\n", + " scheduler_params=None,\n", + " ):\n", + " \"\"\"Fits model to the given data by comparing the whole dataset\n", + " Args:\n", + " x (torch.Tensor): Original time series data\n", + " optim (torch.optim): Optimizer\n", + " optim_params: Optimizer parameters\n", + " max_epochs (int): Number of training epochs\n", + " scheduler (torch.optim.lr_scheduler): Learning rate scheduler\n", + " scheduler_params: Learning rate scheduler parameters\n", + " \"\"\"\n", + " dataset = TensorDataset(x)\n", + " n = dataset.__len__()\n", + " loader = DataLoader(dataset, batch_size=n)\n", + "\n", + " if optim_params is not None:\n", + " optimizer = optim(self.parameters(), **optim_params)\n", + " else:\n", + " optimizer = optim(self.parameters())\n", + "\n", + " if scheduler is not None:\n", + " if scheduler_params is not None:\n", + " lr_scheduler = scheduler(optimizer, **scheduler_params)\n", + " else:\n", + " lr_scheduler = scheduler(optimizer)\n", + "\n", + " for epoch in range(max_epochs):\n", + " for i, data in enumerate(loader, 0):\n", + " self.zero_grad()\n", + " loss = self._step(data, i, n)\n", + " loss.backward(retain_graph=True)\n", + " optimizer.step()\n", + " if scheduler is not None:\n", + " lr_scheduler.step()\n", + "\n", + " print(\"Epoch: \" + str(epoch) + \"\\t Loss: \" + str(loss))\n", + " print(self.coeffs)\n", + " # Testing to see that self.dnns changes\n", + " print(\"DNN(1)\", self.dnn(torch.Tensor([1.])))\n", + "\n", + " def fit_random_sample(\n", + " self,\n", + " x,\n", + " optim,\n", + " optim_params=None,\n", + " max_epochs=10,\n", + " batch_size=64,\n", + " scheduler=None,\n", + " scheduler_params=None,\n", + " ):\n", + " \"\"\"Fits model to the given data by using random samples for each batch\n", + " Args:\n", + " x (torch.Tensor): Original time series data\n", + " optim (torch.optim): Optimizer\n", + " optim_params: Optimizer parameters\n", + " max_epochs (int): Number of training epochs\n", + " batch_size (int): Batch size for torch.utils.data.DataLoader\n", + " scheduler (torch.optim.lr_scheduler): Learning rate scheduler\n", + " scheduler_params: Learning rate scheduler parameters\n", + " \"\"\"\n", + " dataset = TensorDataset(x)\n", + " loader = DataLoader(dataset, batch_size=batch_size)\n", + "\n", + " if optim_params is not None:\n", + " optimizer = optim(self.parameters(), **optim_params)\n", + " else:\n", + " optimizer = optim(self.parameters())\n", + "\n", + " if scheduler is not None:\n", + " if scheduler_params is not None:\n", + " lr_scheduler = scheduler(optimizer, **scheduler_params)\n", + " else:\n", + " lr_scheduler = scheduler(optimizer)\n", + "\n", + " for epoch in range(max_epochs):\n", + " for i, data in enumerate(loader, 0):\n", + " self.zero_grad()\n", + "\n", + " n = data[0].shape[0]\n", + "\n", + " if n < 3:\n", + " continue\n", + "\n", + " # Takes a random data point from \"data\"\n", + " ri = torch.randint(low=0, high=n - 2, size=()).item()\n", + " single_point = data[0][ri : ri + 1, :]\n", + " init_point = {\n", + " var: single_point[0, i] for i, var in enumerate(self.var_names)\n", + " }\n", + "\n", + " pred = {\n", + " name: value.unsqueeze(0) for name, value in self.init_vars.items()\n", + " }\n", + "\n", + " pred = self.step_solver(init_point)\n", + "\n", + " predictions = torch.stack([pred[var] for var in self.outvar], dim=0)\n", + "\n", + " # Compare numerical integration data with next data point\n", + " loss = self.criterion(predictions, data[0][ri + 1, :])\n", + "\n", + " loss.backward(retain_graph=True)\n", + " optimizer.step()\n", + "\n", + " if scheduler is not None:\n", + " lr_scheduler.step()\n", + "\n", + " print(\"Epoch: \" + str(epoch) + \"\\t Loss: \" + str(loss))\n", + " print(self.coeffs)\n", + " # Testing to see that self.dnns changes\n", + " print(\"DNN(1)\", self.dnn(torch.Tensor([1.])))\n", + "\n", + " def _step(self, batch, batch_idx, num_batches):\n", + " (x,) = batch\n", + " nt = x.shape[0]\n", + " pred = self(nt)\n", + " return self.criterion(pred, x)" + ], + "execution_count": 3, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "vhzCXiGS7vcj" + }, + "source": [ + "# Duffing equation: Second order ODE system\n", + "dt = 0.01\n", + "\n", + "def x_prime(prev_val, coeffs, dnns):\n", + " return prev_val[\"x_\"]\n", + "\n", + "def x_prime_prime(prev_val, coeffs, dnns):\n", + " return 0.8*torch.cos(0.5*prev_val[\"t\"]) - coeffs[\"d\"]*prev_val[\"x_\"] - coeffs[\"a\"]*prev_val[\"x\"] - coeffs[\"b\"]*prev_val[\"x\"]*prev_val[\"x\"]*prev_val[\"x\"]\n", + "\n", + "def t_prime(prev_val, coeffs, dnns):\n", + " return 1\n", + "\n", + "ode = {\"x\": x_prime, \"x_\": x_prime_prime, \"t\": t_prime}\n", + "\n", + "# Initial conditions [0,0,0]\n", + "ode_init = {\"x\": 0, \"x_\": 0, \"t\": 0}\n", + "\n", + "# Constants (Parameters)\n", + "ode_coeffs = {\"a\": 0.1, \"b\": 0.5, \"d\": 0.2}" + ], + "execution_count": 4, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0-Om8KZe7zHS" + }, + "source": [ + "# 4th Order Runge-Kutta - Data Generation for nt = 1000" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "NwF7ddNl71cq" + }, + "source": [ + "ode_solver = ODEDNNSolver(\n", + " ode=ode,\n", + " init_vars=ode_init,\n", + " init_coeffs=ode_coeffs,\n", + " dt=dt,\n", + " solver=\"rk4\"\n", + ")\n", + "\n", + "result = ode_solver(1000)" + ], + "execution_count": 5, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 265 + }, + "id": "UnDf8azV8W0p", + "outputId": "6eca4784-0e7d-4537-95d8-94ec220d1156" + }, + "source": [ + "result_np = result.detach().numpy() # Convert to numpy array\n", + "\n", + "# 2D plot of X and Z\n", + "plt.plot(result_np[:,2], result_np[:,0])\n", + "\n", + "plt.show()" + ], + "execution_count": 6, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXiU5dn+8e+VnSQQCAkBsi/sIFvCroKgAlYBFUTrWq1apa6t2vparbbWtrbVt1KVIq644IagKG4ooGwBCQRCICQhK0kgJISQfe7fH4n98VLCNsszmbk+x5GDmcnD3OcInnl4lvsWYwxKKaU8n4/VAZRSSrmGFr5SSnkJLXyllPISWvhKKeUltPCVUspL+Fkd4GQiIiJMQkKC1TGUUqrD2Lx58wFjTOSJvufWhZ+QkEB6errVMZRSqsMQkX3tfU8P6SillJdwSOGLyCIRKReRzHa+P1FEqkVka9vX7xwxrlJKqdPnqEM6rwDPAa+dZJs1xpifOGg8pZRSZ8ghe/jGmNVApSPeSymllHO48hj+WBHJEJFPRWSQC8dVSimF667S2QLEG2OOiMh0YCnQ50QbisitwK0AcXFxLoqnlFKezyV7+MaYw8aYI22PVwD+IhLRzrYLjDGpxpjUyMgTXkqqlFLqLLhkD19EegJlxhgjIqNo/UFz0BVjq9NXfbSJbcVV5JQfoaa+mSB/H6K6BDE8thux4Z0QEasjKqXs4JDCF5G3gIlAhIgUAY8C/gDGmBeAK4FfiEgzUAfMNToRv1swxvD1rnJeW7ePtTkHaLGd+I8lKTKEK0fGcO2YeLoE+bs4pVLKEcSdezc1NdXonbbOk55fyWPLd5BZfJjeYUFcOqw35/WJpF/PzoR18qeh2UZh5VE25VfycUYpG/MrCevkz12T+3DjuAR8fXSPXyl3IyKbjTGpJ/yeFr73qW9q4c+f7eKV7/PpHdaJu6f0YdbwaPx9T35KJ7O4mr+uzObb3RWMiOvKP64aRnz3EBelVkqdDi189R9lh+u57fXNbC2s4vqx8TwwtT+hgad/ZM8Yw0dbS3h02Q4A/vXTEYxPOeH5d6WUBU5W+DqXjhfZU1bDZc+tZXdZDS9cO4LHZww+o7IHEBFmDo9m+bwJRHUJ5PpFG3l/c5GTEiulHEkL30vsKKnmqgXrsRn44I5xTB3cy673i+sezAd3jGdMUji/ei+DtzcWOCipUspZtPC9wK79h7l6wXqC/HxYcttY+vfs4pD3DQ3046Ub0ji/byQPfbCdpT8UO+R9lVLOoYXv4Uqq6rhx0SY6Bfjyzm1jSYxw7EnWIH9fXrxuJGOSwvn1exl8l3PAoe+vlHIcLXwPVl3XxI0vb6S2oZlXbhpFbHiwU8YJ9PPlxetSSYoI5bbXN7O7rMYp4yil7KOF76FabIZ5b24h70AtL14/kgG9HHMYpz1hnfx55WdpdArw5bbXN3O4vsmp4ymlzpwWvof6+xfZrNlzgCdmDGZcsmsum+wV1on514ygsPIo9y/JwNbOXbtKKWto4XuglTv2M3/VXq4eFcvcUa6dcXRUYji/nT6AL3aW8dLaPJeOrZQ6OS18D1NYeZRfLclgaEwYj11mzbIDN41P4KKBUfx1ZTZZpYctyaCU+m9a+B6kxWa4b8lWAJ67ZgSBfr6W5BARnrriHMKC/bnn7a3UN7VYkkMp9X9p4XuQ57/JYVP+IR6fOchpV+ScrvCQAP5y5Tlkl9Xw15XZlmZRSrXSwvcQGYVVPPPlHi4d2puZw6KtjgPApH49uHZMHIu+y2NLwSGr4yjl9bTwPUBdYwv3vLOVHp0D+cPMwW61UMlD0wbQs0sQv3l/O43NNqvjKOXVtPA9wDNf7ibvQC1Pzx5KWCf3WpwkNNCPx2cMJrushn+vybU6jlJeTQu/g9teVM2/1+Ry9ahYxrnpNMUXDoxi+pCePPvVHnIrjlgdRymvpYXfgTW12Hjg/W1EhAby0LQBVsc5qccuHUSgnw+/+2gH7rwGg1KeTAu/A1uwOpes0sM8MXOw2x3KOV6PLkHcd2Ff1uYc4POdZVbHUcoraeF3UHsrjvDsV3uYPqQnFw/qaXWc03LtmHj6RoXyh0926rX5SllAC78DMsbw8Ifb6eTva9ndtGfD39eHRy8dRGFlnU67oJQFtPA7oI+3lbI+t5IHpvajR+cgq+OckfEpEUwd1JPnvs6htLrO6jhKeRWHFL6ILBKRchHJbOf7IiL/KyI5IrJNREY4YlxvVNvQzB8/yWJwdBfmprl2YjRHefiSAbQYw18+0ztwlXIlR+3hvwJMPcn3pwF92r5uBZ530LheZ/6qHPYfruf3lw3C18d9brA6E7Hhwdw8IZGlW4vZUVJtdRylvIZDCt8YsxqoPMkmM4DXTKv1QFcRsW8VbS+Ud6CWf6/J5YoRMYyMD7c6jl1uPz+ZsE7+/Fn38pVyGVcdw48GCo95XtT22n8RkVtFJF1E0isqKlwSriMwxvD75TsI9PPlwWn9rI5jt7BO/syblMLq3RW6Dq5SLuJ2J22NMQuMManGmNTIyEir47iNr7LK+Sa7gnum9OlwJ2rbc93YeKK7duJPn2bp6lhKuYCrCr8YiD3meUzba+o01De18PjHO+nTI5QbxiVYHcdhAv18uf+ivmQWH2b5thKr4yjl8VxV+MuA69uu1hkDVBtjSl00dof379W5FFQe5bHLBuHv63b/KLPLzGHRDOjVhac/z9bZNJVyMkddlvkWsA7oJyJFInKziNwuIre3bbICyAVygH8DdzhiXG9QdOgo87/JYfqQnox308nR7OHjIzw4tR+FlXW8vanA6jhKeTQ/R7yJMebqU3zfAHc6Yixv8+SKLAAevmSgxUmc5/y+kaQldGP+qhzmpMYS5G/N0oxKeTrPOj7gYdbuOcCK7fuZNymF6K6drI7jNCLCvRf2pexwA29t1L18pZxFC99NNbXYeGz5DuLCg7nl3CSr4zjduOQIxiSF869v9lLXqBOrKeUMWvhu6tXv88kpP8Kjlw70mkMc907pS0VNA4s37LM6ilIeSQvfDZUfrueZL/cwqV8kkwdEWR3HZUYndWdCSgTPf7OXo43NVsdRyuNo4buhpz7bRWOzjd9d2nGmPnaUey/sy8HaRl5bp3v5SjmaFr6bSc+v5IMtxfz8vEQSI0KsjuNyI+O7MbFfJC9+u5cjDbqXr5QjaeG7kRab4Xcf7aBXWBB3TkqxOo5l7p3Sl0NHm3j1+3yroyjlUbTw3chbGwvYWXqYhy8ZQHCAQ26R6JCGxnZlUr9IFq7J1WP5SjmQFr6bqKxt5K8rsxmTFM4lQ3Tm6HkX9OHQ0Sbe3KDX5SvlKFr4buKvK7M50tDM4zMGI9IxFzZxpJHx3Rib1J0Fq3N1wXOlHEQL3w1sK6ri7U0F3Dgugb5Rna2O4zZ+eUEK5TUNvLu5yOooSnkELXyL2WyGRz7aQURoIPdM6WN1HLcyNrk7w+O68sI3e2lq0Zk0lbKXFr7FlqQXklFYxW+n96dzkL/VcdyKiPDLC1Iorqrjo606X75S9tLCt1DV0Ub+/Nku0hK6MXPYCVd89HqT+vVgYK8u/GtVDi26KpZSdtHCt9DfPt9NdV0Tv79MT9S2R0S4c1IKuQdq+TRT18xRyh5a+BbJLK5m8YZ9XDcmnoG9u1gdx61NHdyT5MgQnvs6h9alFZRSZ0ML3wI2m+F3H2XSLTiA+y7qZ3Uct+frI9wxMYVd+2v4Kqvc6jhKdVha+BZYkl7IloIqHpzWn7BOeqL2dMwY1pvorp144du9VkdRqsPSwnexipoGnlyRxajEcGaPjLE6Tofh5+vDz89NJH3fIdLzK62Oo1SHpIXvYk98vJP6JhtPzhqiJ2rP0Jy0WLoF+/Pi6lyroyjVIWnhu9A32eUsyyjhFxOTSekRanWcDic4wI/rxybwxc4ycsprrI6jVIfjkMIXkakiki0iOSLy0Am+f6OIVIjI1ravWxwxbkdS19jCIx9lkhQZwh2Tkq2O02FdPzaeIH8fFuhevlJnzO7CFxFfYD4wDRgIXC0iA0+w6TvGmGFtXwvtHbejefarPRRW1vHkrCEE+nnHGrXO0D00kDmpsXz4QzFlh+utjqNUh+KIPfxRQI4xJtcY0wi8DcxwwPt6jKzSw/x7TS5zUmMYk9Td6jgd3s/PTaLFZli0Ns/qKEp1KI4o/Gig8JjnRW2vHe8KEdkmIu+JSKwDxu0Qmlps/OrdDLoF+/Pb6QOsjuMRYsODueSc3izeUMDh+iar4yjVYbjqpO1yIMEYcw7wBfBqexuKyK0iki4i6RUVFS6K5zz/WrWXHSWH+cPMIXQNDrA6jse47bwkjjQ06wIpSp0BRxR+MXDsHntM22v/YYw5aIxpaHu6EBjZ3psZYxYYY1KNMamRkZEOiGedHSXV/PPrPcwY1pupg3taHcejDI4O49w+ESxam0dDsy6QotTpcEThbwL6iEiiiAQAc4Flx24gIseu2XcZkOWAcd1aY7ONX727jW4hATx26SCr43ik285LprymgaU/FJ96Y6WU/YVvjGkG5gEraS3yJcaYHSLyuIhc1rbZXSKyQ0QygLuAG+0d1909tyqHrNLDPDlrCN1C9FCOM4xP6c6g3l14cXUuNp06WalTcsgxfGPMCmNMX2NMsjHmj22v/c4Ys6zt8W+MMYOMMUONMZOMMbscMa67+qHgEPNX5TBreDQXDoyyOo7HEhFuPz+Z3Ipavtqlk6opdSp6p62D1dQ3cdfbP9CzSxCPXaaHcpxt2uCeRHftxMI1eiOWUqeihe9gjyzNpPhQHc/OHaYzYbqAn68PN41PYENeJZnF1VbHUcqtaeE70Adbili6tYS7J/clNSHc6jheY05aLKGBfrykN2IpdVJa+A6Sf6CWR5ZmMiohnHkXpFgdx6t0CfLnqrRYlmeUsL9ap1tQqj1a+A5Q19jC7W9sxs/Xh3/MHYavj0577Go3jkvAZgyvrsu3OopSbksL307GGH774Xayy2p4du4wort2sjqSV4oND2bq4J4sXr+P2oZmq+Mo5Za08O302rp9fPhDMfdO6cvEfj2sjuPVbp6QxOH6Zt7fUmR1FKXckha+HTblV/LExzuZ3L8H8ybpcXurjYzvxvC4rixam0eL3oil1H/Rwj9L+QdqufW1dGLDg/n7VcPw0eP2buGWCUnkHzzKV1llVkdRyu1o4Z+FQ7WN3PTKJgBevjFNr7d3IxcPimq9EUsv0VTqv2jhn6GG5hZue30zxYfqWHB9KgkRIVZHUsf48UasjXmVbC/SG7GUOpYW/hlobrFx3zsZbMyv5K+zzyFNb65yS///RiydbkGpY2nhnyabzfDg+9v5ZHsp/3PJAGYMO9GiXsod/Hgj1sfbSimtrrM6jlJuQwv/NBhjeHTZDt7fUsS9U/pyy7lJVkdSp/DjjVivr9tndRSl3IYW/im02AwPL83k9fX7uO28JO6arJdfdgSx4cFMGRDFWxsLqG/SFbGUAi38k2pstnH32z/w5oYC7piYzEPT+iOil192FDeOT+DQ0SaWZZRYHUWp02aMobK20Snv7eeUd/UANfVNzHvzB77dXcFvpvXntvOTrY6kztDYpO70i+rMq9/nM3tkjP6wVm6rqcXGprxKvswq56tdZbTYDGsemOTwv7Na+CeQf6CWW15LJ+9ALU9dPoS5o+KsjqTOgohw/bh4Hv4wk/R9h/SqKuVWSqrqWLf3IKuyy/l2dwU19c0E+PkwLrk7kwdE0WIz+Plq4TvV17vKuPedDETg9Z+NYlxKhNWRlB1mDY/mz5/u4pXv87XwlaXKa+pZt/cg63MPsm7vQfIPHgUgIjSAaYN7MnlAFBNSIggJdF4ta+G3qW9q4ckVWby2bh/9e3ZmwXWpxHUPtjqWslNwgB9XpcWy6Lt8Sqvr6BWms5kq16ioaWBTfiXr9h5kXe5BcsqPANA5yI/RieFcNzaBsUnd6d+zs8umZtHCB9bsqeDRZTvIrajl5gmJ/PrifgT5+1odSznI9WMTWLg2j8XrC/jVxf2sjqM81P7qejbkHWR9biUb8w6yt6IWgOAAX9ISwpk9Moaxyd0Z1DvMsjUzvLrw91Yc4emV2XyauZ/47sG8fvMozu0TaXUs5WCx4cFM7h/FmxsLmHdBiv4wVw5RdrieNXsOsCH3IBvyKimobD1E0znQj7TEcGanxjI6MZzB0WH4+7rHBZEOKXwRmQo8C/gCC40xTx33/UDgNWAkcBC4yhiT74ixz5Qxhi0Fh3j5u3w+2V5KkJ8vv7qo9WYqLQLPddP4BL7MKuPjbaVcOTLG6jiqA6pvaiE9/xDf7i5n9e4DZJfVANA12J9RCeHcMC6B0YnhDOjVxW1XvbO78EXEF5gPXAgUAZtEZJkxZucxm90MHDLGpIjIXODPwFX2jn26GpttZBRVsXp3BZ9sLyW3opaQAF9uPz+ZmyckEhEa6KooyiLjkrvTp0cor3yfxxUjovUSTXVaDtc38XVWOZ9mlvLt7grqm2wE+PqQmtCN34zoz7l9Il16DN5ejtjDHwXkGGNyAUTkbWAGcGzhzwAea3v8HvCciIgxximrVDyyNJPquiZq6psoOlRH3oFamm0GH4HU+HBuPy+Z6ef0ItSJZ8OVe2m9RDOBR5ZmsqXgECPj9YoddWL1TS18lrmfZRklrN1zgMYWG1FdApmTGsvEfpGMSepOcEDH7A5HpI4GCo95XgSMbm8bY0yziFQD3YEDx7+ZiNwK3AoQF3d217+v3lOBAKFBfsR3D+HCgVEMiQ5jXHIEYcE6d723unx4NH/5bBcvf5evha/+D2MM24qqWZJeyLKMEmrqm4nu2onrx8YzbUhPhsd26zB78Sfjdj+mjDELgAUAqampZ/UvgG9/PcmhmZRnCAn0Y05qLK9+n8/+6np6hgVZHUlZrKnFxortpSxck8f24mqC/H2YPrjXf064ekLJH8sRhV8MxB7zPKbttRNtUyQifkAYrSdvlXKp68fGs+i7PN7csI/7LtJLNL3V0cZm3txQwKK1eZRU15MUGcITMwczY1hvugR57lEARxT+JqCPiCTSWuxzgWuO22YZcAOwDrgS+NpZx++VOpn47iGc3zeStzcV8svJfdzmcjnlGg3NLby1oYDnVuVw4EgjoxPDeWLmYCb16+Fxe/MnYnfhtx2TnwespPWyzEXGmB0i8jiQboxZBrwEvC4iOUAlrT8UlLLEtaPjueW1dL7YWcb0Ib2sjqNcwBjD0q3FPL1yN8VVdYxODOfF6/p53bkchxzDN8asAFYc99rvjnlcD8x2xFhK2WtS/x5Ed+3EG+v3aeF7gez9NTzyUSYb8yoZEh3GU1cMYUJKhFdemut2J22VcjZfH+HqUbE8/flu9lYcITky1OpIygnqm1r4xxe7Wbg2j85Bfjw5awhz02K94tBNe/QApvJKc9Ji8fMRFq8vsDqKcoLtRdX85J9reXF1LleOiOHr+ydyzeg4ry570MJXXqpH5yCmDu7Je5sLqWvUJRA9RYvN8M+v9jDrX99RU9/Eaz8bxZ+vPIfwkACro7kFLXzlta4dE8/h+maWb9MlED3BodpGbnplE3/7YjfThvRi5T3ncV5fnQzxWHoMX3mt0Ynh9OkRyuL1+5iTGnvq36Dc1raiKn7xxhYqahp4ctYQrh4V65UnZU9F9/CV1xIRfjo6joyiarYVVVkdR52lFdtLufKFdRhjePf2sVwzOk7Lvh1a+MqrXT4yhk7+vryxfp/VUdQZMsawcE0ud765hSHRYSz/5QSGxna1OpZb08JXXq1LkD8zhvVmWUYJ1UebrI6jTpPNZnj845384ZMspg7qyeJbRtNdpzk/JS185fWuHRNPfZON97cUWR1FnYYWm+HB97fx8nf53DQ+geeuGaGLF50mLXzl9QZHhzEstiuLN+xDp3hyby02w6/fzeDdzUXcPbkPv/vJQLddXcodaeErRete/t6KWtbnVlodRbWjxWa4b8lWPvihmPsu7Mu9F/bVk7NnSAtfKeCSIb3oHOTH25v0zlt3ZIzhf5Zu56OtJfz64n7cNbmP1ZE6JC18pYBOAb7MGh7Np5n7qTraaHUcdZy/fb6btzYWMm9SCndOSrE6Toelha9Um7lpcTQ22/hgy/Hr9ygrLVqbx3Orcrh6VCz3X9TX6jgdmha+Um0G9u7C0Jgw3t5UoCdv3cRnmaU8/vFOpg7qyR9mDtFj9nbSwlfqGHNHxbG77AhbCvTOW6vtKKnm3ncyGB7XlWfmDtOrcRxAC1+pY1w6tDfBAb68vVFP3lqpoqaBn7+aTtdgf168bqReZ+8gWvhKHSM00I/Lhvbm422l1NTrnbdWaGhu4fY3NlN5tJF/X59Kj85BVkfyGFr4Sh1n7qg46ppa+GirTptshT+t2MXmfYf42+xhDI4OszqOR9HCV+o4Q2PC6N+zs16Tb4FPt5fyyvf53DIhkUvO0fWGHU0LX6njiAhXj4ojs/gwmcXVVsfxGgUHj/LAe9sYFtuVB6b2tzqOR9LCV+oEZg6LJtDPh7f05K1LNDS3cOebWxCB564ZToCfVpMz2PVfVUTCReQLEdnT9mu3drZrEZGtbV/L7BlTKVcIC/bnkiG9+GhrCUcbm62O4/H+/vluthdX8/TsocR0C7Y6jsey98foQ8BXxpg+wFdtz0+kzhgzrO3rMjvHVMol5o6K40hDMx9vK7U6ikfblF/JgjW5XDM6josG9bQ6jkezt/BnAK+2PX4VmGnn+ynlNtISupEcGaLX5DtRbUMz9y/JILZbMA9PH2B1HI9nb+FHGWN+3P3ZD0S1s12QiKSLyHoROekPBRG5tW3b9IqKCjvjKXX2RIS5aXFsKahid1mN1XE80h9XZFF46ChPzx5KSKCf1XE83ikLX0S+FJHME3zNOHY70zr5SHsTkMQbY1KBa4BnRCS5vfGMMQuMManGmNTIyMgz+SxKOdzlI6Lx9xWWbCq0OorHWb27gjc3FHDruUmMSgy3Oo5XOOWPVGPMlPa+JyJlItLLGFMqIr2A8nbeo7jt11wR+QYYDuw9u8hKuU730EAm94/iwx+KeXBaf/x99eoRR6hrbOHhpdtJigzh3gt1BkxXsfdv7zLghrbHNwAfHb+BiHQTkcC2xxHAeGCnneMq5TKzU2M4WNvI17tOuD+jzsIzX+2msLKOP80aovPkuJC9hf8UcKGI7AGmtD1HRFJFZGHbNgOAdBHJAFYBTxljtPBVh3F+30giOwfybroucu4IO0sOs3BNHlelxjI6qbvVcbyKXWdJjDEHgckneD0duKXt8ffAEHvGUcpKfr4+XD4imoVr8iivqdfJvOzQYjP85sPtdAv25zfT9W5aV9MDkkqdhtkjY2mxGZb+oKth2eON9fvIKKzikZ8MpGtwgNVxvI4WvlKnIaVHKCPiurIkvUhXwzpLB4808PTn2ZzbJ4LLhva2Oo5X0sJX6jTNSY0lp/wIWwt1Nayz8bcvdlPX2MKjlw7UpQotooWv1Gm65JxeBPn7sERP3p6xHSXVvLWxgOvHJpDSo7PVcbyWFr5Sp6lzkD/Th/Ti44wS6hpbrI7TYRhj+P3ynXQLDuDuyX2sjuPVtPCVOgOzR8ZS09DMZzt0QrXTtWL7fjbmVXL/RX0JC/a3Oo5X08JX6gyMTgwnLjxYr8k/TfVNLTy5IosBvbowNy3O6jheTwtfqTPg4yNcOTKG7/cepLDyqNVx3N5r6/IprqrjkZ8MwNdHT9RaTQtfqTN0xcgYROC9zbqXfzLVR5uYv2ovE/tFMi45wuo4Ci18pc5YdNdOTEiJ4L3NRdhsek1+e57/di+H65t44GK9o9ZdaOErdRZmp8ZSXFXHutyDVkdxS6XVdbz8XR4zh0UzsHcXq+OoNlr4Sp2FiwZG0SXIjyXpOk/+iTz75R6Mgft06mO3ooWv1FkI8vdlxrBoPsvcT3Vdk9Vx3EpOeQ1L0gu5dkw8seG6ILk70cJX6izNTo2hodnG8owSq6O4ladX7iY4wI95F6RYHUUdRwtfqbM0JDqM/j07864e1vmP3WU1fLZjPz8bn0B4iM6G6W608JU6SyKt1+RnFFWTvV8XOQeYvyqH4ABfbhqfaHUUdQJa+ErZYdbwaPx8RPfygfwDtSzPKOHaMfF00717t6SFr5QduocGMmVA6yLnjc02q+NY6oVv9+Ln68MtE3Tv3l1p4Stlpzlpush5SVUd728pYm5aLD266BKQ7koLXyk7ndcnkh6dA736sM6C1bkYA7edn2x1FHUSWvhK2cnP14crRsawKruc8sP1VsdxuYqaBt7aWMCs4dFEd+1kdRx1EnYVvojMFpEdImITkdSTbDdVRLJFJEdEHrJnTKXc0eyRMdgMfOCFi5y/tDaPphYbv5ioe/fuzt49/EzgcmB1exuIiC8wH5gGDASuFpGBdo6rlFtJigwlLaEbS9ILvWqR86qjjby+Lp9LzulNUmSo1XHUKdhV+MaYLGNM9ik2GwXkGGNyjTGNwNvADHvGVcodzU6NJbeili0Fh6yO4jKvfJ9PbWMLd07SvfuOwBXH8KOBY89mFbW9dkIicquIpItIekVFhdPDKeUolwzpRXCAL0s2ecc8+Ucamnn5u3ymDIiif0+dEbMjOGXhi8iXIpJ5gi+n7KUbYxYYY1KNMamRkZHOGEIppwgJ9OOSIb34eFsJtQ3NVsdxujfW76O6rknnzOlA/E61gTFmip1jFAOxxzyPaXtNKY8zJy2WdzcXsWJ7KbNTY0/9Gzqo+qYWFq7J49w+EQyL7Wp1HHWaXHFIZxPQR0QSRSQAmAssc8G4Srlcanw3EiNCPH6R83c2FXLgSAN3TtK9+47E3ssyZ4lIETAW+EREVra93ltEVgAYY5qBecBKIAtYYozZYV9spdyTiDA7NYaN+ZXkVhyxOo5TNDbbePHbvaQldGN0YrjVcdQZsPcqnQ+NMTHGmEBjTJQx5uK210uMMdOP2W6FMaavMSbZGPNHe0Mr5c6uGBGDjwcvcv7hD0WUVNdz56QURMTqOOoM6J22SjlYVJcgJvbrwftbimhu8awJ1ZpbbDz/zV6GRIdxfl+9qKKj0cJXygnmpMZQdriBNXsOWB3FoT7ZXhSABCkAAArFSURBVEr+waPcOSlZ9+47IC18pZzggv5RhIcEeNQi5zabYf6qHPr0COWigT2tjqPOgha+Uk4Q4OfDrOHRfJlVRmVto9VxHOKLrDJ2lx3hzkkp+Pjo3n1HpIWvlJPMTo2hqcWw1AMmVDOmde8+LjyYn5zTy+o46ixp4SvlJP17duGcmDCPmFBtzZ4DbCuq5o6Jyfj5am10VPonp5QTzU6NZdf+GjKKqq2OYpfnvs6hV1gQl4+IsTqKsoMWvlJONHNYb4IDfFm8fp/VUc7axrxKNuZXcut5SQT4aWV0ZPqnp5QTdQ7yZ8aw3izfVkL10Sar45yV51blEBEawNy0OKujKDtp4SvlZD8dHU99k40Pfuh4d95mFFaxencFN09IolOAr9VxlJ208JVyssHRYQyN7criDQUd7uTt/FU5dAny49oxunfvCbTwlXKBn46OI6f8CBvyKq2Octqy99fw+c4ybhqfSOcgf6vjKAfQwlfKBS49pzddgvxYvKHA6iin7V/f5BAS4MtN4xOsjqIcRAtfKRfoFODLFSNj+CyzlIqaBqvjnFL+gVqWZ5Rw7Zh4ugYHWB1HOYgWvlIu8tPR8TS1GN7d7P7z6zz/zV78fH24+dxEq6MoB9LCV8pFUnqEMiYpnMXrC9x62uTCyqO8v6WIq9Ni6dE5yOo4yoG08JVyoRvHJVJcVcfnO8usjtKu57/di48It09MtjqKcjAtfKVc6MKBUcSFB/PS2jyro5xQSVUd76YXMicthl5hnayOoxxMC18pF/L1EW4cl8DmfYfYWlhldZz/8sK3ewH4xURdnNwTaeEr5WJz0mLpHOjndnv5+6vreXtjIVeOjCW6q+7deyItfKVcLDTQj6vSYlmxvZSSqjqr4/zHi6v3YjOGO/TYvcfSwlfKAjeMS8AYw6vr8q2OAkB5TT1vbijg8hHRxIYHWx1HOYldhS8is0Vkh4jYRCT1JNvli8h2EdkqIun2jKmUJ4gND2ba4F68uaGA6jrrZ9Gc/3UOzTbDHXrs3qPZu4efCVwOrD6NbScZY4YZY9r9waCUN/nFxGRq6pt5fV2+pTkKK4/y5sYCrkqLJSEixNIsyrnsKnxjTJYxJttRYZTyJoOjw7igfw9eWptHbUOzZTn+8cVufES4e3IfyzIo13DVMXwDfC4im0Xk1pNtKCK3iki6iKRXVFS4KJ5S1ph3QQqHjjbxpkWTqu3af5gPtxZz0/hEorroXbWe7pSFLyJfikjmCb5mnME4E4wxI4BpwJ0icl57GxpjFhhjUo0xqZGRkWcwhFIdz4i4bkxIieDF1bnUN7W4fPynV2bTOdCPX5yvV+Z4g1MWvjFmijFm8Am+PjrdQYwxxW2/lgMfAqPOPrJSnmXeBSkcONLA6+tcu+7t9zkH+DKrnNsnJhMWrPPdewOnH9IRkRAR6fzjY+AiWk/2KqWAMUndOa9vJPO/yXHZFTvNLTZ+v3wnseGd+Nl4nRHTW9h7WeYsESkCxgKfiMjKttd7i8iKts2igLUikgFsBD4xxnxmz7hKeZoHp/ajuq7pP1MbONviDQVkl9Xw8PSBBPnrWrXews+e32yM+ZDWQzTHv14CTG97nAsMtWccpTzdoN5hzBwWzaK1eVw/Nt6pE5dV1jbyt8+zGZ/SnYsHRTltHOV+9E5bpdzEfRf2xQBPrtjl1HGeXJFFbWMLj146CBFx6ljKvWjhK+UmYsODuXNiCsszSli92zmXJH+TXc57m4u47bwk+kZ1dsoYyn1p4SvlRm6fmERSRAiPfJTp8Ms0a+qb+O0H20npEcpdepOVV9LCV8qNBPr58sTMwew7eJRnv9rj0Pd+4uOd7D9cz1+uPEdP1HopLXyl3Mz4lAiuSo3lhW/38n3OAYe85/ubi1iSXsQdE1MYEdfNIe+pOh4tfKXc0KOXDSQxIoR73tnKwSMNdr1X9v4a/mdpJmOSwrlnih7K8WZa+Eq5oeAAP/559XCq6pq4/Y3NNDSf3fH8ssP13PTyRkKD/PjfucPx89X/5b2Z/ukr5aYG9Q7j6dlD2ZR/iPuXZNDcYjuj33/wSAM3LNpIdV0TL9+YRg+dHM3r2XXjlVLKuS4b2pvSqjr+9OkubMbwzFXDCfA79X5aSVUd1y/aSGHlURbekMrg6DAXpFXuTgtfKTd32/nJ+PoIf/gki+Kqdfxz7nDiure/DOHXu8q4f0kGTS2GV382ijFJ3V2YVrkzLXylOoBbzk0iumsnHnx/Gxc98y03jkvkqrRYEttWqGpusbExv5JFa/P4MqucAb26MP+a4SRFhlqcXLkTMcZYnaFdqampJj1dl8BV6kfFVXX85bNdLMsowRjoFuxPaJAfFTUN1DfZ6BLkx+0Tk/nZ+ES91t5Licjm9paS1cJXqgMqqarj8x37yS6rob7JRnhIACPiujF5QA8tei93ssLXQzpKdUC9u3biRp3HXp0hvSxTKaW8hBa+Ukp5CS18pZTyElr4SinlJbTwlVLKS2jhK6WUl9DCV0opL6GFr5RSXsKt77QVkQpg31n+9gjAMcsFdRz6mb2DfmbPZ8/njTfGRJ7oG25d+PYQkfT2bi/2VPqZvYN+Zs/nrM+rh3SUUspLaOErpZSX8OTCX2B1AAvoZ/YO+pk9n1M+r8cew1dKKfV/efIevlJKqWNo4SullJfwuMIXkakiki0iOSLykNV5nE1EYkVklYjsFJEdInK31ZlcRUR8ReQHEfnY6iyuICJdReQ9EdklIlkiMtbqTM4mIve2/b3OFJG3RCTI6kyOJiKLRKRcRDKPeS1cRL4QkT1tv3ZzxFgeVfgi4gvMB6YBA4GrRWSgtamcrhm43xgzEBgD3OkFn/lHdwNZVodwoWeBz4wx/YGhePhnF5Fo4C4g1RgzGPAF5lqbyileAaYe99pDwFfGmD7AV23P7eZRhQ+MAnKMMbnGmEbgbWCGxZmcyhhTaozZ0va4htYSiLY2lfOJSAxwCbDQ6iyuICJhwHnASwDGmEZjTJW1qVzCD+gkIn5AMFBicR6HM8asBiqPe3kG8Grb41eBmY4Yy9MKPxooPOZ5EV5Qfj8SkQRgOLDB2iQu8QzwAGCzOoiLJAIVwMtth7EWikiI1aGcyRhTDDwNFAClQLUx5nNrU7lMlDGmtO3xfiDKEW/qaYXvtUQkFHgfuMcYc9jqPM4kIj8Byo0xm63O4kJ+wAjgeWPMcKAWB/0z3121HbeeQesPu95AiIhca20q1zOt18475Pp5Tyv8YiD2mOcxba95NBHxp7XsFxtjPrA6jwuMBy4TkXxaD9tdICJvWBvJ6YqAImPMj/96e4/WHwCebAqQZ4ypMMY0AR8A4yzO5CplItILoO3Xcke8qacV/iagj4gkikgArSd4llmcyalERGg9rptljPm71XlcwRjzG2NMjDEmgdY/46+NMR6952eM2Q8Uiki/tpcmAzstjOQKBcAYEQlu+3s+GQ8/UX2MZcANbY9vAD5yxJv6OeJN3IUxpllE5gEraT2jv8gYs8PiWM42HrgO2C4iW9te+60xZoWFmZRz/BJY3LYzkwvcZHEepzLGbBCR94AttF6N9gMeOMWCiLwFTAQiRKQIeBR4ClgiIjfTOkX8HIeMpVMrKKWUd/C0QzpKKaXaoYWvlFJeQgtfKaW8hBa+Ukp5CS18pZTyElr4SinlJbTwlVLKS/w/TlcYSqWkWBcAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ObTghBli7-no" + }, + "source": [ + "# Euler's method for training" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "87VAZZtR7-n1" + }, + "source": [ + "def x_prime_prime_train(prev_val, coeffs, dnns):\n", + " return dnns(torch.Tensor([prev_val[\"t\"]]))[0] - coeffs[\"d\"]*prev_val[\"x_\"] - coeffs[\"a\"]*prev_val[\"x\"] - coeffs[\"b\"]*prev_val[\"x\"]*prev_val[\"x\"]*prev_val[\"x\"]\n" + ], + "execution_count": 7, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "dTx1Q0PV7-n2" + }, + "source": [ + "ode_train_coeffs = {\"a\": 0., \"b\": 0., \"d\": 0.}\n", + "ode_train = {\"x\": x_prime, \"x_\": x_prime_prime_train, \"t\": t_prime}\n", + "\n", + "ode_solver_train = ODEDNNSolver(\n", + " ode=ode_train,\n", + " init_vars=ode_init,\n", + " init_coeffs=ode_train_coeffs,\n", + " dt=dt,\n", + " solver=\"rk4\"\n", + ")" + ], + "execution_count": 8, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "67eN5ZyB7-n2", + "outputId": "1588284b-d8f3-42ed-ff4e-0428b39394e8" + }, + "source": [ + "ode_solver_train.fit_random_sample(\n", + " result,torch.optim.Adam,\n", + " {\"lr\": 0.02},\n", + " max_epochs=20,\n", + " scheduler=torch.optim.lr_scheduler.MultiStepLR,\n", + " scheduler_params={\"milestones\": [10],\"gamma\": 0.2}\n", + ")" + ], + "execution_count": 9, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Epoch: 0\t Loss: tensor(6.2093e-06, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.1419, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.1420, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.0721, requires_grad=True)}\n", + "DNN(1) tensor([0.2365], grad_fn=)\n", + "Epoch: 1\t Loss: tensor(2.0705e-05, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.2504, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.2516, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.1099, requires_grad=True)}\n", + "DNN(1) tensor([0.1373], grad_fn=)\n", + "Epoch: 2\t Loss: tensor(2.1068e-05, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.2731, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.2960, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.1083, requires_grad=True)}\n", + "DNN(1) tensor([0.2051], grad_fn=)\n", + "Epoch: 3\t Loss: tensor(2.6809e-05, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.2646, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3165, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.0769, requires_grad=True)}\n", + "DNN(1) tensor([0.3410], grad_fn=)\n", + "Epoch: 4\t Loss: tensor(2.4307e-05, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.2557, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3321, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.0659, requires_grad=True)}\n", + "DNN(1) tensor([0.2867], grad_fn=)\n", + "Epoch: 5\t Loss: tensor(2.1748e-05, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.2179, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3312, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.0237, requires_grad=True)}\n", + "DNN(1) tensor([0.4414], grad_fn=)\n", + "Epoch: 6\t Loss: tensor(2.6169e-05, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.1903, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3348, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.0237, requires_grad=True)}\n", + "DNN(1) tensor([0.3845], grad_fn=)\n", + "Epoch: 7\t Loss: tensor(1.6772e-05, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.1749, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3486, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.0295, requires_grad=True)}\n", + "DNN(1) tensor([0.4565], grad_fn=)\n", + "Epoch: 8\t Loss: tensor(3.1222e-05, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.1566, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3485, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.0641, requires_grad=True)}\n", + "DNN(1) tensor([0.2051], grad_fn=)\n", + "Epoch: 9\t Loss: tensor(5.5046e-06, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.1387, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3774, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.0270, requires_grad=True)}\n", + "DNN(1) tensor([0.4718], grad_fn=)\n", + "Epoch: 10\t Loss: tensor(8.6054e-06, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.1340, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3802, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.0232, requires_grad=True)}\n", + "DNN(1) tensor([0.4493], grad_fn=)\n", + "Epoch: 11\t Loss: tensor(1.7816e-05, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.1227, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3761, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.0155, requires_grad=True)}\n", + "DNN(1) tensor([0.4175], grad_fn=)\n", + "Epoch: 12\t Loss: tensor(1.5981e-05, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.1157, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3760, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.0098, requires_grad=True)}\n", + "DNN(1) tensor([0.4440], grad_fn=)\n", + "Epoch: 13\t Loss: tensor(1.0512e-05, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.1122, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3779, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.0064, requires_grad=True)}\n", + "DNN(1) tensor([0.4677], grad_fn=)\n", + "Epoch: 14\t Loss: tensor(1.4457e-05, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.1096, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3788, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.0031, requires_grad=True)}\n", + "DNN(1) tensor([0.4644], grad_fn=)\n", + "Epoch: 15\t Loss: tensor(1.2071e-05, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.1031, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3770, requires_grad=True), 'd': Parameter containing:\n", + "tensor(0.0009, requires_grad=True)}\n", + "DNN(1) tensor([0.4661], grad_fn=)\n", + "Epoch: 16\t Loss: tensor(1.5328e-05, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.0963, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3764, requires_grad=True), 'd': Parameter containing:\n", + "tensor(0.0040, requires_grad=True)}\n", + "DNN(1) tensor([0.4691], grad_fn=)\n", + "Epoch: 17\t Loss: tensor(1.4537e-05, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.0909, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3771, requires_grad=True), 'd': Parameter containing:\n", + "tensor(0.0068, requires_grad=True)}\n", + "DNN(1) tensor([0.4822], grad_fn=)\n", + "Epoch: 18\t Loss: tensor(1.4585e-05, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.0887, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3810, requires_grad=True), 'd': Parameter containing:\n", + "tensor(0.0104, requires_grad=True)}\n", + "DNN(1) tensor([0.4979], grad_fn=)\n", + "Epoch: 19\t Loss: tensor(1.1779e-05, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.0851, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3836, requires_grad=True), 'd': Parameter containing:\n", + "tensor(0.0162, requires_grad=True)}\n", + "DNN(1) tensor([0.5046], grad_fn=)\n" + ] + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 266 + }, + "id": "IOMjwKY27-n2", + "outputId": "1e8218d2-8200-4b08-9ae5-ec7152cfe18d" + }, + "source": [ + "result = ode_solver_train(1000)\n", + "\n", + "result_np = result.detach().numpy() # Convert to numpy array\n", + "\n", + "# 2D plot of X and Z\n", + "plt.plot(result_np[:,2], result_np[:,0])\n", + "\n", + "plt.show()" + ], + "execution_count": 10, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD5CAYAAAAk7Y4VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhV1dn+8e+TmYSQQBIgJGQAwiSjRIYEcAAUnHC2KAqKonWstbZW/dW+1qq1rbWtCqWi4IiKoqgIIqOIAonMECAJgSRAEgiZ52T9/kh8X0oTpjPsMzyf68rFGXbOug9w3dnZZ+21xRiDUkopz+djdQCllFLOoYWvlFJeQgtfKaW8hBa+Ukp5CS18pZTyElr4SinlJfzs8SIi8gZwJVBojBnQyvMXAZ8B+1se+sQY88zpXjcyMtIkJCTYI6JSSnmF9PT0o8aYqNaes0vhA/OAV4C3TrHNt8aYK8/mRRMSEkhLS7Mll1JKeRUROdDWc3Y5pGOMWQsU2+O1lFJKOYYzj+GPEpGtIvKViJzX1kYiMlNE0kQkraioyInxlFLKszmr8H8E4o0xg4F/Ap+2taExZo4xJtkYkxwV1ephKKWUUufAKYVvjCkzxlS03F4C+ItIpDPGVkop1cwphS8iXUVEWm4Pbxn3mDPGVkop1cxe0zLfBy4CIkUkD3ga8AcwxswGbgB+LiINQDXwM6PLdCqllFPZpfCNMVNO8/wrNE/bVEopZRF7zcNXHsYYQ97xarbmlXC4pIaa+kYC/HyI6diO87qFkRARTMtROqWUm9DCV/8ht7iKDzbl8umWfPKOV7e5XWzHdlw5qBu3joije6dgJyZUSp0rLXwFwJHSGv6+Yi8fpuVhjGFs7yhmju3BsPiOdO8UTLC/LzUNTRw8VkX6weOszihkztos5qzN4oZhsfzq0j507hBk9dtQSp2CuPJnp8nJyUaXVnAsYwzvb8zlj1/uoq6xiVtHxDNzbA+6hbc77fceKqlm7rr9vPV9Dv6+Pjx1RX+mDO+uh3qUspCIpBtjklt9Tgvfe5XV1PPLD7bwze5CUnpG8MJ1g4iLOPvDMweOVfLEou18l3mMCf278NJNgwkN8ndAYqXU6Zyq8HV5ZC+VXVTBNa9+x+o9RTx9VX/emTHinMoeID4ihLfvHMFTV/RjVUYh189aT25xlZ0TK6VspYXvhbbllXDdrPWUVNXzzl0juCM1ER8f2w7D+PgId43pwfw7h3O4tIZrX/uO3YfL7JRYKWUPWvheJi2nmFv/vYGQAD8W3ZfCyB4Rdn391F6RLLovFT8fH2759w/sOqSlr5Sr0ML3ItvySpj2xkaiQgP56N5RxEeEOGScXp3bs2DmSIL8fbnl9R/YW1DukHGUUmdHC99LZBdVMP3NTXQMCeD9mSPPaBaOLRIiQ1gwcyT+vj5Mf2MjBWU1Dh1PKXV6WvheoLCshtvmbkSAt2eMoIuT5svHR4Tw5vQLKK2uZ9obGymvqXfKuEqp1mnhe7ia+kbufjud41V1zLtjOImRjjmM05YBMWHMmjqMzMIKfvnhVpqaXHcasFKeTgvfgxljeOrTHWzNLeGlm4YwMDbMkhxje0fxxOX9WL6rgFlrsizJoJTSwvdo89bnsDA9j4fGJTFxQFdLs9yRmsDkId34y9d7WLtXL12plBW08D1UWk4xz365m/H9uvCLcUlWx0FEeP66gfTpEsrDCzZTqB/iKuV0WvgeqLSqnocXbCEmvB0v3TzY5pOq7CU4wI9XbhlKVV0jjy3chisv66GUJ9LC9zDGGH7z8TYKymr4x5ShdHCxNW16dQ7lqSv6sWZvEfPX51gdRymvooXvYd7dcJClO4/w2GV9GNI93Oo4rZo6Mp5L+nbmua8y9KQspZxIC9+DZBaW84cvdjG2dxR3j+lhdZw2iQh/un4QoYF+/OqjrTQ0NlkdSSmvoIXvIRoam3j0o20EB/jylxsHucxx+7ZEhQby+6vPY1teKfP00I5STqGF7yHmfJvN1twSnpk8gM6h7nHlqSsHRTO+X2f+8vUeDh7T5ZSVcjQtfA+wt6Ccl5fv4/KBXblyULTVcc6YiPCHawbg5+PDE4u266wdpRxMC9/NNTQ28auPttI+yI9nJg9wu8sLRoe14/FJfVmXeZSF6XlWx1HKo9ml8EXkDREpFJEdbTwvIvIPEckUkW0icr49xlXwr7XZbMsr5Q+TBxDZPtDqOOfkluFxJMd35IWvMiit0gXWlHIUe+3hzwMmnuL5SUBSy9dMYJadxvVqmYXlvPzNXq4YFM0VbnQo52Q+PsL/TD6P41V1/O2bvVbHUcpj2aXwjTFrgeJTbDIZeMs0+wEIFxH3bSgXYIzhyUU7CA7w45mrz7M6js3O6xbGrSPieev7HL00olIO4qxj+DFA7gn381oe+y8iMlNE0kQkrahIF9lqyyc/5rNhfzGPT+pLhJseyjnZo5f2JqydP08v3qkf4CrlAC73oa0xZo4xJtkYkxwVFWV1HJdUUlXHH5fs5vy4cG5O7m51HLsJDw7g1xP7snF/MYu3HrI6jlIex1mFnw+c2EyxLY+pc/CnpXsora7nj9cOdPkTrM7WTcndGRgTxnNLdlNV12B1HKU8irMKfzFwe8tsnZFAqTHmsJPG9ijpB47z/saD3JmaQL/oDlbHsTtfH+Hpq/pTUFbL69/utzqOUh7FXtMy3we+B/qISJ6IzBCRe0Xk3pZNlgDZQCbwb+A+e4zrbRoam3hy0Xaiw4L4xfjeVsdxmOSETkwa0JXZa7IoLNd185WyFz97vIgxZsppnjfA/fYYy5vNW59DxpFyZk8dRkigXf7pXNZvJvZl+a4CXv5mH89dO9DqOEp5BJf70Fa17lBJNS8t38slfTtz2XldrI7jcAmRIUwdGc+CjQfZp0soK2UXWvhu4pnPd9FkDP9z9Xlut3zCuXpoXBIhgX48/1WG1VGU8gha+G5gZUYBS3ce4cFLkujeKdjqOE7TKSSABy7uxcqMQtZnHrU6jlJuTwvfxVXXNfK7z3bSq3N7l76oiaNMS0kgJrwdLyzN0JOxlLKRFr6Le2XVPvKOV/PsNQMI8PO+f64gf19+MT6JbXmlLNtZYHUcpdya9zWIG8ksLGfO2myuPz+WkT0irI5jmWuHxtAjKoSXlu+hsUn38pU6V1r4LurExdGeuLyv1XEs5efrw6MT+rC3oILFW/UEbaXOlRa+i/LExdFsMWlAV/pHd+Bvy/dRrxc9V+qcaOG7oJKqOp7zwMXRbOHjIzx2WR8OFlfxYVru6b9BKfVftPBd0J+W7qGkup5nr/G8xdFscVGfKIbFd+QfK/ZRU99odRyl3I4Wvov5aXG0O1IS6N/N8xZHs4VI815+QVkt7/xwwOo4SrkdLXwX0tDYxFOf7mheHG2C5y6OZouRPSIYkxTJa6uzqKzV5ZOVOhta+C5k3vrmy/s9fVV/2nv44mi2+MX43hRX1ulevlJnSQvfRfzn4mhdrY7j0obFd2RMUiRz1mbrRVKUOgta+C7i94t3et3iaLZ4eFwSxyrrePeHg1ZHUcptaOG7gOW7Cvh6VwEPj+vtVYuj2SI5oROpvSL419osqut0xo5SZ0IL32KVtQ08/dkO+nQJ5a4xiVbHcSsPj+vN0Yo63t2gx/KVOhNa+BZ7+Zu9HCqt4bnrBuDvq/8cZ2N4YidG9YjgX2uzdV6+UmdAG8ZCuw6V8cZ3OUwZ3p1h8Z2sjuOWHh6fRFF5Le9t0GP5Sp2OFr5FGpsMTyzaTng7f34z0bsXR7PFyB4RjEjsxOw1WbqXr9RpaOFb5L2NB9mSW8JTV/YjPDjA6jhu7eHxSRSW17Jgo+7lK3UqWvgWKCyr4cWlGaT2iuCaITFWx3F7o3pEMDyhE7PWZFHboHv5SrVFC9/JjDE89ekO6hqa+MPkATrn3g5EhIfGJVFQVsvH6bpevlJtsUvhi8hEEdkjIpki8ngrz08XkSIR2dLydZc9xnVHX24/zNe7CnhkQm96RLW3Oo7HSO0VweDYMGavyaJB18tXbqa8pp61e4t4+Zu93DU/jelvbnTIODYv2CIivsCrwAQgD9gkIouNMbtO2vQDY8wDto7nzoor63j6s50MjAnjrtE6596eRIT7Lu7FPW+n8+X2w0zWQ2XKxZXV1LNk22G+2nGE9VlHqW80iECvqPb0je6AMcbuRwDssULXcCDTGJMNICILgMnAyYXv9Z75fCel1fW8c9cI/HTOvd1N6NeFpM7teW1VFlcN6qbXElAuKbOwnDe/y2HR5nyq6hqJ6xTMnamJjE6KZEj3cEKD/B02tj0KPwY48RJEecCIVra7XkTGAnuBR4wxXnXZopUZBXy65RAPjUuiX7Suc+8IPj7CfRf35JEPtrIyo5Dx/btYHUmp/5VztJK/fbOXxVsP4e/rw+TB3Zg6Mp5BsWFO+yzPWWvwfg68b4ypFZF7gPnAJa1tKCIzgZkAcXFxTornWGU19TzxyQ56d2nPAxf3sjqOR7tqUDf++vVeXlmVybh+nfVDcWW5ytoG/r5iH3PX7cffV7hnbE9mju1BpxDnT8e2x3GFfODEC6/Gtjz2v4wxx4wxtS13XweGtfVixpg5xphkY0xyVFSUHeJZ7w+f76KwvIYXbxhMgJ8eynEkP18f7r2wJ1tyS/g++5jVcZSXW76rgAkvrWHO2mxuHBbL2l9fzOOT+lpS9mCfwt8EJIlIoogEAD8DFp+4gYhEn3D3amC3HcZ1C8t2HuGj9Dzuu6gXQ7qHWx3HK9wwLJao0EBmrc6yOoryUlV1Dfxm4TbufiuNDu38+fjno3jh+kF0Dg2yNJfNh3SMMQ0i8gCwDPAF3jDG7BSRZ4A0Y8xi4CERuRpoAIqB6baO6w6Kymv57SfbGRDTgYfGJVkdx2sE+fty1+hEnv8qg625JQzWH7TKiXbkl/LQ+5vZf6yS+y7qySMTervMwohijLE6Q5uSk5NNWlqa1THOiTGGu+ansS7zKF88OJqkLqFWR/IqFbUNpDy/glE9I/jXbclWx1FeYvHWQzz20VbCg/35281DSOkZ6fQMIpJujGn1P71r/NjxQB9symVFRiG/mdhXy94C7QP9mJ6ayLKdBewrKLc6jvJwTU2GPy/L4KH3NzMoNowvHxpjSdmfjha+A2QVVfDMF7tI7RXB9JQEq+N4rTtSEggO8NVj+cqh6hqaeHDBZl5dlcXPLujOu3eNJLJ9oNWxWqWFb2c19Y3c/+6PBPn78tcbh+jJPxbqGBLALcPj+GzrIXKLq6yOozxQVV0DM+Zv4stth/ntpL48f91Al56J57rJ3NQzX+wi40g5f71pMF3DrP1EXsGMMYkIMHfdfqujKA9TUlXHra9v4LvMo7x4/SDuubCny5/3oYVvR59vPcR7Gw5yz4U9uLhPZ6vjKCA6rB2Th8TwwaZcjlfWWR1HeYjS6nqmzt3AzvwyXrv1fG66oPvpv8kFaOHbyf6jlfz2k+0Mi+/Iry7tY3UcdYKZY3tQXd/IOz/oxc6V7cpr6pn2xkb2HCnnX7cNY+KA6NN/k4vQwreDitoG7nk7DT9f4R9ThrrMnFvVrE/XUC7uE8X873P0MojKJlV1Ddw5bxPb80t55Zbzubive/0mr81ko6Ymw6MfbiGrqJJXbzmfmPB2VkdSrZg5tidHK+r4+Mc8q6MoN1Xf2MS97/xI+oHjvHzzEC47r6vVkc6aFr6N/rkyk2U7C3ji8n6k9nK9ebeq2cgenRgcG8br3+6nscl1TzZUrskYw5OLtrN2bxF/vHYgVw3uZnWkc6KFb4OlO47wt2/2ct35MdyZmmB1HHUKIsLMsT3Zf7SS5buOWB1HuZl/rszkw7Q8HrykF1OGu+8qvlr45yj9wHEeXrCZId3Dee7agS4/HUvBxAFdiesUzOw12bjykiLKtSxMz+Ol5c07dr+c0NvqODbRwj8H2UUV3DV/E9FhQcydlkyQv6/VkdQZ8PUR7h6TyJbcEjblHLc6jnIDaTnF/PaTbaT2iuCF6wa5/Y6dFv5ZKiqvZdqbG/ERYf6dw4lw0VOoVetuGNadTiEBzFmryy2oUysoq+Hn7/5It/B2vHbLMJc+g/ZMuf87cKLiyjpum7uBo+V1zJ1+AfERIVZHUmepXYAvt4+K55vdhbqommpTbUMj976TTmVtA3NuSyYs2HHXmXUmLfwzVFJVx9TXN7D/aCWvT0vWi5m4sdtHJRDk78OctdlWR1EuyBjD05/tZPPBEv5642D6dPWc1W618M9AaVU9t83dSGZhBXNuT9bpl26uU0gANyV359Mt+RSU1VgdR7mYj9LyWLApl/sv7smkge5zFu2Z0MI/jcOl1dz4r/XsOVLOrKnnc2Fvz7jOrre7a3QPGpsMb3yni6qp/5NZWM7vFu8gpWcEv5zgeUukaOGfQlZRBTfM+p5DJTXMu+MCxvXrYnUkZSdxEcFMGhjNez8cpLym3uo4ygXU1DfywHubCQnw4+Wbh+DrgUuba+G34bvMo9wwaz21DY0smDmSFD2M43HuGduD8toGPtiUa3UU5QKe/fL/ljbv3MEzlzbXwj+JMYbXv83mtrkbiGwfyMJ7UxgQE2Z1LOUAg2LDGZ7QiXnrc2hobLI6jrLQV9sP884PB7lnbA8u8uClzbXwT3CsopZ73k7n2S93c2n/riy6P5WESJ166clmjEkk73g1X+8qsDqKssiR0hp+8/E2BncP51EPX9rcz+oAruLrnUd4YtF2yqobePLyfswYnaiXJ/QC4/t1IT4imNe/zeZyD5uRoU7PGMOvP95GfaPh7zcP8YiTq07Fs9/dGcgqqmD6mxuZ+XY6ke0DWfxgKneP7aFl7yV8fYQ7UhL48WAJ6Qd0uQVvs2BTLmv3FvHby/t6xW/zdil8EZkoIntEJFNEHm/l+UAR+aDl+Q0ikmCPcW2RXVTBbxZu47K/rSU95zhPXdGPxQ+Mpm/XDlZHU052Y3J3QoP8eEOve+tVcourePaLXaT2imDqiHir4ziFzYd0RMQXeBWYAOQBm0RksTFm1wmbzQCOG2N6icjPgD8BN9s69tmqqmvgm92FfJyex9p9RQT4+nDLiDgevCSJqFBdE8dbhQT6ccuIOP69Npvc4iq6dwq2OpJysKYmw68+2oqI8OINg73mN3p7HMMfDmQaY7IBRGQBMBk4sfAnA79vub0QeEVExDhojdrc4ioqahuoqG0g73gV+woqSD9wnB8PHqe+0dAtLIgHL+7FbaMStOgVANNTEpj77X7mr8/hqSv7Wx1HOdj873PYsL+YF28Y5FVXqbNH4ccAJ05kzgNGtLWNMaZBREqBCOCoHcb/L+NfWkNtw/9Ns/PzEXp3CeXO1EQu7BPFyMQIr/mJrs5MdFg7rhgUzYJNuTw8PonQIM9YLEv9t/ySav68bA8X9YnixmGxVsdxKpebpSMiM4GZAHFx53ZlmRdvGESArw8hgX50Cw8iPiJELyyuTmvG6EQ+23KIDzblcteYHlbHUQ5gjOF3n+7AGPjD5AFuv7792bJH4ecD3U+4H9vyWGvb5ImIHxAGHGvtxYwxc4A5AMnJyed0yGfykJhz+Tbl5X46EevN73KYnpKAn+4keJyvdhxhRUYhT13Rzys/q7HH/+hNQJKIJIpIAPAzYPFJ2ywGprXcvgFY6ajj90rZYsaYRPJLqlm2U0/E8jSl1fU8vXgnA2I6MD0lweo4lrC58I0xDcADwDJgN/ChMWaniDwjIle3bDYXiBCRTOCXwH9N3VTKFfx0ItbcdbpWvqd5cWkGxypqef7aQV7725tdjuEbY5YAS0567Hcn3K4BbrTHWEo5kq+PcGdqIk8v3kn6geMMi+9odSRlB+kHinl3w0FmjE5kYKz3ro3lnT/mlDqFG4bF0kFPxPIYjU2Gpz7dSbewIH45obfVcSylha/USUIC/ZgyIo6vdhwmt7jK6jjKRu9tOMDuw2U8eUV/QgJdbmKiU2nhK9WK6SkJ+Igwf32O1VGUDY5X1vGXr/cyqkcElw/sanUcy2nhK9WK6LB2TBzQlQ/ScqmsbbA6jjpHf/l6DxW1Dfz+6vO8bs59a7TwlWrDHamJlNc08Mnmk08rUe5gR34p7208yG0j4+nTNdTqOC5BC1+pNpwfF86g2DDmfbefpiY9bcSdGGP4/eKddAwO4BEv/6D2RFr4SrVBRJiekkBWUSXrMh2y7JNykMVbD5F24Di/vqwPYe10XaSfaOErdQpXDIomsn0g8/TDW7dRU9/Ii0v3MCCmAzcldz/9N3gRLXylTiHQz5dbR8SxMqOQ/UcrrY6jzsC89Tnkl1TzxOX9dFXck2jhK3Uat46Mw99Xp2i6g+LKOl5dmcklfTuT0jPS6jguRwtfqdPoHBrElYO6sTA9j/KaeqvjqFP4x4p9VNY18NtJfa2O4pK08JU6A9NTEqiobWBhep7VUVQbco5W8s4PB7j5gjiSuug0zNZo4St1BgZ3D2doXDjz1+foFE0X9eKyDAL8fHhkQpLVUVyWFr5SZ+iO1ERyjlWxZm+R1VHUSdIPFLNk+xFmju1B59Agq+O4LC18pc7QpAFd6dIhkDe+01U0XYkxhueXZNA5NJCZY/XSlKeiha/UGfL39WHqiHi+3XeUzMJyq+OoFqv3FJF24DgPjUsiOMC7V8M8HS18pc7CLSPiCPDz0ROxXERTk+EvX++he6d2epLVGdDCV+osRLQP5OrB3fg4PZ/Sap2iabWlO4+w81AZvxjXmwA/rbPT0b8hpc7S9JQEqusb+Sgt1+ooXq2xyfDS8r306tyea4bGWB3HLWjhK3WWBsSEMTyhE/PW59CoUzQt8+nmfDILK3h0Qm98dQmFM6KFr9Q5mJ6aQN7xalbsLrA6ileqa2ji5RV7GRDTgYkD9EpWZ0oLX6lzcGn/LnQLC2L+9zlWR/FKH6blkltczaOX9tErWZ0FLXylzoGfrw+3joznu8xj7C3QKZrOVFPfyD9X7iM5viMX9Y6yOo5b0cJX6hxNGd48RVNX0XSu9zYcpKCsll9dpnv3Z8umwheRTiKyXET2tfzZsY3tGkVkS8vXYlvGVMpVdAoJYPLgbnzyo07RdJaa+kZmr8liVI8IRvaIsDqO27F1D/9xYIUxJglY0XK/NdXGmCEtX1fbOKZSLmOaTtF0qg/Tciksr+XBcb2sjuKWbC38ycD8ltvzgWtsfD2l3MqAmDCS4zvy1vcHdIqmg9U2NDJrdRYXJHRklO7dnxNbC7+LMeZwy+0jQJc2tgsSkTQR+UFETvlDQURmtmybVlSkqxIq1zc9NYGDxVWs3lNodRSP9nF6PodLa3hoXJIeuz9Hp11pSES+AVqb6PrkiXeMMUZE2trFiTfG5ItID2CliGw3xmS1tqExZg4wByA5OVl3mZTLu+y8rnTtEMS89TmM69fWPo+yRX1jE6+uymRoXDije+mlC8/VaQvfGDO+redEpEBEoo0xh0UkGmh1F8cYk9/yZ7aIrAaGAq0WvlLuxt/Xh1tHxPHX5XvJLKygV+f2VkfyOIt+zCe/pJpnrx2ge/c2sPWQzmJgWsvtacBnJ28gIh1FJLDldiSQCuyycVylXMqUEXEE+Prw1vc5VkfxOA2NTbyyKpNBsWE6795Gthb+C8AEEdkHjG+5j4gki8jrLdv0A9JEZCuwCnjBGKOFrzxKZPtArhwczcd6oXO7+2zLIQ4WV/HQJXrs3lY2XS3AGHMMGNfK42nAXS231wMDbRlHKXcwPSWBT37MZ2F6HnekJlodxyM0NhleWZVJ/+gOjOvX2eo4bk/PtFXKTgbF6oXO7e2LbYfYf7SSh8b10r17O9DCV8qOpqckNF/ofJ9OKbaVMYbXVmXRu0t7Lu2vK2Lagxa+UnY0aUA0UaGBur6OHazaU8iegnLuvbAnPrrevV1o4StlRwF+zVM0V+8pIruowuo4bm3W6ixiwttx1eBuVkfxGFr4StnZLSPi8PcV3vr+gNVR3NamnGI25Rzn7jGJ+PtqTdmL/k0qZWedQ4O4YmA0C9PzqKhtsDqOW5q9OotOIQHcfEGc1VE8iha+Ug4wLSWBitoGPvkxz+oobifjSBkrMgqZnpJAuwBfq+N4FC18pRxgaFxHBseGMU+naJ61f63JJjjAl9tHxVsdxeNo4SvlINNTE8guqmRd5lGro7iN3OIqFm89xC3D4wgPDrA6jsfRwlfKQS4fGE1k+wCdonkWXv82Gx+BGWP0TGVH0MJXykEC/Xy5ZXgcK/cUcuBYpdVxXN7RiloWbMrl2qExRIe1szqOR9LCV8qBbh0Zj6/oFM0zMX99DnWNTcwc29PqKB5LC18pB+rSIYhJA6P5MC2XSp2i2aaK2gbmr8/hsv5d9XoCDqSFr5SDTU+Jp7ymgUWb862O4rLe33CQspoG7r1I9+4dSQtfKQc7P64jA2I6MH99DsboFM2T1TY08vq6bFJ6RjCke7jVcTyaFr5SDiYiTE9JZF9hBeuzjlkdx+V8tvkQBWW13Huh7t07mha+Uk5w5aBoOoUEME+naP6HxibD7LVZnNetA2OS9OLkjqaFr5QTBPn7MmV4d1bsLiC3uMrqOC5j+a4jZBdVcu+FPfUCJ06gha+Uk0wdGY+I8PYPOkUTmi9wMmtNNnGdgrl8YLTVcbyCFr5SThId1o6J53Xlg025VNc1Wh3Hchv2F7M1t4S7x/bAVy9w4hRa+Eo50bSUBEqr6/l0i07RnL0mi8j2Adw4LNbqKF5DC18pJ7ogoSP9ozsw7zvvnqK5+3AZq/cUMT0lgSB/XQLZWbTwlXKi5imaCewpKOeH7GKr41hmztrmJZCnjtQlkJ3JpsIXkRtFZKeINIlI8im2mygie0QkU0Qet2VMpdzd1UO60THY32tX0cw73rwE8hRdAtnpbN3D3wFcB6xtawMR8QVeBSYB/YEpItLfxnGVcltB/r7cfEEcX+86Qn5JtdVxnG7uuv0IMGO0LoHsbDYVvjFmtzFmz2k2Gw5kGmOyjTF1wAJgsi3jKuXubmu5mtPbXraK5vHKOhZszOXqId3oFq5LIDubM47hxwC5J9zPa3msVSIyU0TSRCStqKjI4eGUskJMeDsu7d+VBZsOUlPvPVM03/r+ANX1jbqMgkVOWwgksVQAAAuRSURBVPgi8o2I7GjlyyF76caYOcaYZGNMclRUlCOGUMolTEtJoKSqnsVbDlkdxSmq6xqZ/30O4/p2pneXUKvjeCW/021gjBlv4xj5QPcT7se2PKaUVxvZoxN9u4Yyb30ONybHevzSAh+l51JcWcc9undvGWcc0tkEJIlIoogEAD8DFjthXKVcmogwLSWBXYfL2JRz3Oo4DtXQ2MSctdmcHxfOBQkdrY7jtWydlnmtiOQBo4AvRWRZy+PdRGQJgDGmAXgAWAbsBj40xuy0LbZSnuGaITGEtfP8KZpLdhwh73i1LpJmsdMe0jkVY8wiYFErjx8CLj/h/hJgiS1jKeWJ2gX4cvMF3Zm7bj+HS6s98uLdxhhmr86iZ1QI4/t1sTqOV9MzbZWy2G0j4zHG8I6HrqL57b6j7Dpcxj1je+Kji6RZSgtfKYt17xTMuH5deH9jrkdO0Xx1VSZdOgQyeWg3q6N4PS18pVzA9JQEiivr+GLbYauj2NXG/cVs2F/MPWN7Euini6RZTQtfKReQ0jOCpM7tPe5C56+syiQiJIApw+OsjqLQwlfKJYgI01MT2J5fyob9nrGK5ra8EtbuLWLGmETaBejevSvQwlfKRVx/fiyR7QOYtTrL6ih28crKTDoE+XGbLoHsMrTwlXIRQf6+3JGayJq9RezIL7U6jk0yjpTx9a4C7khNJDTI3+o4qoUWvlIu5LZR8YQG+jFrjXvv5b+6KouQAF/uSE2wOoo6gRa+Ui6kQ5A/t46M56vth9l/tNLqOOcku6iCL7YdYuqoeL3AiYvRwlfKxdw5OgE/Xx/mrHXPvfzXVmcR4OvDXaN7WB1FnUQLXykX0zk0iJuSY/k4PZ+Cshqr45yV7KIKPvkxj1tHxBMVGmh1HHUSLXylXNDMMT1paGri9W+zrY5yVl7+Zh+Bfr78/CJdAtkVaeEr5YLiIoK5enA33vnhIEXltVbHOSN7jpTz+bZDTE9N0L17F6WFr5SLemhcErUNjcx2kxk7f1u+l/YBftwzVo/duyotfKVcVI+o9lx3fizv/HDA5Y/lb88rZenOI8wYk6gzc1yYFr5SLuyhS5JobDK8tirT6iin9NLyPYQH+3Pn6ESro6hT0MJXyoXFRQRzY3Is72/MJb+k2uo4rdqQfYxVe4q4Z2xPOuhZtS5NC18pF/fAJUlA89o0rqapyfDsl7vpFhakZ9W6AS18pVxcTHg7bhkRx4dpuWQWllsd5z98tjWf7fmlPDaxD0H+uiKmq9PCV8oNPHhJL4IDfHluSYbVUf5XTX0jf166h4ExYUweHGN1HHUGtPCVcgMR7QN58JJerMwoZN2+o1bHAWDuuv0cKq3hySv66bVq3YQWvlJuYlpKAt07tePZL3fR2GTtVbEOlVTz6qpMLu3fhZE9IizNos6cFr5SbiLQz5fHJ/Yj40g5H6blWprlfz7fSZMx/L8r+1uaQ50dmwpfRG4UkZ0i0iQiyafYLkdEtovIFhFJs2VMpbzZ5QO7MjyhE39amsGxCmuWXFiZUcCynQU8eEkS3TsFW5JBnRtb9/B3ANcBa89g24uNMUOMMW3+YFBKnZqI8Nx1A6isbeDZL3c7ffzqukZ+99lOenVuz91jdAkFd2NT4Rtjdhtj9tgrjFLq9Hp1DuXnF/Zk0eZ8vt1X5NSxX1yWQd7xav4weQABfnpE2N0461/MAF+LSLqIzDzVhiIyU0TSRCStqMi5/5mVchf3XdyLxMgQnly0g8raBqeMuT7rKG9+l8Pto+IZ1VM/qHVHpy18EflGRHa08jX5LMYZbYw5H5gE3C8iY9va0BgzxxiTbIxJjoqKOoshlPIeQf6+vHDdQHKPV/HM57scPl55TT2PfbSNhIhgHp/U1+HjKcfwO90Gxpjxtg5ijMlv+bNQRBYBwzmz4/5KqTaM6BHBzy/syWurs7ioTxSTBkY7ZBxjDI9/vJ3DpdV8dG8KwQGnrQ3lohx+SEdEQkQk9KfbwKU0f9irlLLRIxN6Mzg2jMc/2c7BY1UOGWPuuv18uf0wj13Wl2HxHR0yhnIOW6dlXisiecAo4EsRWdbyeDcRWdKyWRdgnYhsBTYCXxpjltoyrlKqmb+vD/+YMhSAu99Ko8LOx/N/yD7G819lcGn/Ltx7oc7KcXdijLVn7J1KcnKySUvTaftKnc66fUeZ9uZGLu4Txeypw/Dztf2X94wjZdw0+3siQwP59P5UXfrYTYhIelvT33VelVIeYHRSJE9f1Z9vdhfy6EdbbV564eCxKqa9sZF2Ab68dedwLXsPoZ++KOUhbh+VQHlNA39etgdfEV64ftA5zZXffbiM29/YSH1jEwtmjiS2o55N6ym08JXyIPdf3IumJsNfl+8lv6Sa2VOH0THkzK8xu2J3AY98sIXgAD/eu2cUSV1CHZhWOZse0lHKwzw4LomXbx7C5twSLn15LUt3HOF0n9WVVNXx/z7dwYz5acR2DGbhz7XsPZHu4Svlga4ZGkOvzu15bOE27n0nncHdw5meEs/FfToTHty8x9/YZNhzpJzPtubzwaZcyqrrmZ6SwOOT+urVqzyUztJRyoPVNzbxYVous9dkkVvcfBH0yPaBBPr5UFJVR2VdI74+wvh+nXl4XG/6d+tgcWJlq1PN0tE9fKU8mL+vD7eOiGfKBXFszSthfdYxDh6ror6xiQ7t/BkYE8bopEi6dAiyOqpyAi18pbyAj48wNK4jQ+P0TFlvph/aKqWUl9DCV0opL6GFr5RSXkILXymlvIQWvlJKeQktfKWU8hJa+Eop5SW08JVSyku49NIKIlIEHDjHb48EjtoxjjvQ9+wd9D17Plveb7wxJqq1J1y68G0hImltrSfhqfQ9ewd9z57PUe9XD+kopZSX0MJXSikv4cmFP8fqABbQ9+wd9D17Poe8X489hq+UUuo/efIevlJKqRNo4SullJfwuMIXkYkiskdEMkXkcavzOJqIdBeRVSKyS0R2isjDVmdyFhHxFZHNIvKF1VmcQUTCRWShiGSIyG4RGWV1JkcTkUda/l/vEJH3RcTjLs0lIm+ISKGI7DjhsU4islxE9rX8aZcr13hU4YuIL/AqMAnoD0wRkf7WpnK4BuBRY0x/YCRwvxe85588DOy2OoQT/R1YaozpCwzGw9+7iMQADwHJxpgBgC/wM2tTOcQ8YOJJjz0OrDDGJAErWu7bzKMKHxgOZBpjso0xdcACYLLFmRzKGHPYGPNjy+1ymksgxtpUjiciscAVwOtWZ3EGEQkDxgJzAYwxdcaYEmtTOYUf0E5E/IBg4JDFeezOGLMWKD7p4cnA/Jbb84Fr7DGWpxV+DJB7wv08vKD8fiIiCcBQYIO1SZziZeDXQJPVQZwkESgC3mw5jPW6iIRYHcqRjDH5wF+Ag8BhoNQY87W1qZymizHmcMvtI0AXe7yopxW+1xKR9sDHwC+MMWVW53EkEbkSKDTGpFudxYn8gPOBWcaYoUAldvo131W1HLeeTPMPu25AiIhMtTaV85nmufN2mT/vaYWfD3Q/4X5sy2MeTUT8aS77d40xn1idxwlSgatFJIfmw3aXiMg71kZyuDwgzxjz029vC2n+AeDJxgP7jTFFxph64BMgxeJMzlIgItEALX8W2uNFPa3wNwFJIpIoIgE0f8Cz2OJMDiUiQvNx3d3GmJeszuMMxpjfGmNijTEJNP8brzTGePSenzHmCJArIn1aHhoH7LIwkjMcBEaKSHDL//NxePgH1SdYDExruT0N+MweL+pnjxdxFcaYBhF5AFhG8yf6bxhjdlocy9FSgduA7SKypeWxJ4wxSyzMpBzjQeDdlp2ZbOAOi/M4lDFmg4gsBH6keTbaZjxwiQUReR+4CIgUkTzgaeAF4EMRmUHzEvE32WUsXVpBKaW8g6cd0lFKKdUGLXyllPISWvhKKeUltPCVUspLaOErpZSX0MJXSikvoYWvlFJe4v8DKIwkJMHByFkAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9adPvysOrfMr" + }, + "source": [ + "# Mean Squared Error Loss" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "kwktnR93ripb" + }, + "source": [ + "ode_solver = ODEDNNSolver(\n", + " ode=ode,\n", + " init_vars=ode_init,\n", + " init_coeffs=ode_coeffs,\n", + " dt=dt,\n", + " solver=\"rk4\"\n", + ")\n", + "\n", + "result_train = ode_solver(1000)" + ], + "execution_count": 11, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "aVYankyRrwoW", + "outputId": "bc16f6f1-c234-44cb-c9e6-ce0b4f5e8f45" + }, + "source": [ + "loss = torch.nn.MSELoss()\n", + "loss(result_train,result)" + ], + "execution_count": 12, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "tensor(0.0529, grad_fn=)" + ] + }, + "metadata": {}, + "execution_count": 12 + } + ] + } + ] +} diff --git a/examples/ode/DuffingEquation/DuffingTest_HybridODE.ipynb b/examples/ode/DuffingEquation/DuffingTest_HybridODE.ipynb new file mode 100644 index 00000000..ac7191c4 --- /dev/null +++ b/examples/ode/DuffingEquation/DuffingTest_HybridODE.ipynb @@ -0,0 +1,398 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "zu5Sh81e6N9a" + }, + "outputs": [], + "source": [ + "import torch\n", + "from pytorch_lightning import LightningModule\n", + "import torch.nn.functional as F\n", + "from torch import nn\n", + "from torch.utils.data import DataLoader, TensorDataset\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import mpl_toolkits.mplot3d.axes3d as p3" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " from torchts.nn.models.ode import ODESolver\n", + " from torchts.nn.models.hybridode import HybridODENet\n", + "except ModuleNotFoundError:\n", + " import sys\n", + " sys.path.append(\"../../..\")\n", + " from torchts.nn.models.ode import ODESolver\n", + " from torchts.nn.models.hybridode import HybridODENet" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "vhzCXiGS7vcj" + }, + "outputs": [], + "source": [ + "# Duffing equation: Second order ODE system\n", + "dt = 0.01\n", + "\n", + "def x_prime(prev_val, coeffs, dnns):\n", + " return prev_val[\"x_\"]\n", + "\n", + "def x_prime_prime(prev_val, coeffs, dnns):\n", + " return 0.8*torch.cos(0.5*prev_val[\"t\"]) - coeffs[\"d\"]*prev_val[\"x_\"] - coeffs[\"a\"]*prev_val[\"x\"] - coeffs[\"b\"]*prev_val[\"x\"]*prev_val[\"x\"]*prev_val[\"x\"]\n", + "\n", + "def t_prime(prev_val, coeffs, dnns):\n", + " return 1\n", + "\n", + "ode = {\"x\": x_prime, \"x_\": x_prime_prime, \"t\": t_prime}\n", + "\n", + "# Initial conditions [0,0,0]\n", + "ode_init = {\"x\": 0, \"x_\": 0, \"t\": 0}\n", + "\n", + "# Constants (Parameters)\n", + "ode_coeffs = {\"a\": 0.1, \"b\": 0.5, \"d\": 0.2}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0-Om8KZe7zHS" + }, + "source": [ + "# 4th Order Runge-Kutta - Data Generation for nt = 1000" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "NwF7ddNl71cq" + }, + "outputs": [], + "source": [ + "ode_solver = HybridODENet(\n", + " ode=ode,\n", + " dnns={},\n", + " init_vars=ode_init,\n", + " init_coeffs=ode_coeffs,\n", + " dt=dt,\n", + " optimizer=None\n", + ")\n", + "\n", + "result = ode_solver(1000)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 265 + }, + "id": "UnDf8azV8W0p", + "outputId": "6eca4784-0e7d-4537-95d8-94ec220d1156" + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAGdCAYAAADaPpOnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAA9hAAAPYQGoP6dpAABOSUlEQVR4nO3dd3yU9eEH8M9zd8llX8jeIYQsEkYIGxmCBAERNw5wVVscVaTYSvXnapXWqrVqHVi3omgZgoASZQphBAgECCOQkL3JXeblxvP740I0BSGB3H1vfN6v1/3B5bnchzT1PjzfJcmyLIOIiIjIQShEByAiIiLqCZYXIiIicigsL0RERORQWF6IiIjIobC8EBERkUNheSEiIiKHwvJCREREDoXlhYiIiByKSnSA3mY2m1FeXg5fX19IkiQ6DhEREXWDLMtobGxEREQEFIoL31txuvJSXl6O6Oho0TGIiIjoEpSUlCAqKuqC1zhdefH19QVg+cv7+fkJTkNERETdodPpEB0d3fk5fiFOV17ODhX5+fmxvBARETmY7kz54IRdIiIicigsL0RERORQWF6IiIjIobC8EBERkUOxannZunUrZs6ciYiICEiShFWrVl3w+s2bN0OSpHMeR48etWZMIiIiciBWXW3U3NyMwYMH45577sGNN97Y7dcdO3asy0qh4OBga8QjIiIiB2TV8jJt2jRMmzatx68LCQmBv79/7wciIiIih2eXc17S09MRHh6OyZMnY9OmTRe8Vq/XQ6fTdXkQERGR87Kr8hIeHo4lS5Zg+fLlWLFiBZKSkjB58mRs3br1V1+zePFiaDSazgePBiAiInJukizLsk3eSJKwcuVKXHfddT163cyZMyFJElavXn3er+v1euj1+s4/n91eWKvVcoddIiIiB6HT6aDRaLr1+W1Xd17OZ9SoUThx4sSvfl2tVnceBcAjAYiIiJyf3ZeX/fv3Izw8XHQMIiIishNWXW3U1NSEgoKCzj8XFhYiNzcXAQEBiImJwaJFi1BWVoZPPvkEAPDaa6+hb9++SE1NRXt7Oz777DMsX74cy5cvt2ZMcmHVujYcKNXiVE0TGloNMJllaDzdENXHE2mRGvQL8u7WIWFERGQ7Vi0vOTk5uPLKKzv/vGDBAgDAXXfdhY8++ggVFRUoLi7u/Hp7ezsWLlyIsrIyeHp6IjU1FWvXrsX06dOtGZNcTH1zO5btKcG3B8txuPzCq9Mi/T0xfWAY7hgZi75B3jZKSEREF2KzCbu20pMJP+RaGlra8fqPBfhidzFaDSYAgEICEkJ8kRTmi0AfdyglCdpWAwpqmnCkXAe90dx53bSB4fjj1CTEBrLEEBH1tp58flv1zguRPZBlGV/llOBv64/iTIsBAJAW6YfbR8Ti6rQwBHi7n/d1bQYTNh+rxrI9Jdh0rAZrD1Yg60gV5k2Ix+8n9Yeb0u6njBEROSXeeSGnpm0x4IkVB7H+UCUAICnUF3+ekYLxCUE9mstytFKHF9bmY9uJWgDA4CgNXr8tnXdhiIh6SU8+v1leyGnlV+hw38c5KGtohZtSwsLMJPzmijioLvGOiSzLWJtXgSdXHoK21QA/DxXenTsMo+MDezk5EZHrcap9XoguxbYTNbj5nWyUNbQiNtALyx8Yg99NiL/k4gJYNlq8ZlAE1j86DkNj/KFrM+LOD3Zhxb7SXkxOREQXw/JCTufbg+W458M9aNIbMTIuAKsfugKDovx77ftH+Hti6f2jMGNgOAwmGQu+OoDPdp7ute9PREQXxvJCTmV9XgUe/TIXRrOMmYMj8MlvRkDj5dbr7+PhpsQbt6XjN1fEAQCeWnUIX+4uvsiriIioN7C8kNPIOlKF33+xHyazjBuGRuJfs4dArVJa7f0UCglPzUjBvWMtBWbRyjx8k1tmtfcjIiILlhdyCvuLz+DhpftgNMuYNSQC/7hpMBQK6++MK0kS/u+aFNw1OhayDDz+9UHsLqy3+vsSEbkylhdyeCX1Lbj/kxzojWZMSg7BKzcPhtIGxeUsSZLwzMxUXJ0ahnaTGb/9NAenapps9v5ERK6G5YUcmrbVgHs+2oPapnYMCPfDG7elX9aKokulUEj45+whGBylQUOLAb/9dC+a9Uab5yAicgUsL+SwzGYZf/gqFwXVTQjz88AHdw+Ht1rcptGe7kq8d9cwhPqpUVDdhCdW5MHJtlEiIrILLC/ksJZsO4Uf8qvhrlLgvTuHIUzjIToSQnw98O/bh0KlkLDmQDk+yeYSaiKi3sbyQg5p56k6/OP7YwCAZ2emYmCURnCinw3rG4AnpiUDAF5Ym4/8igufXE1ERD3D8kIOp65J//OS6PRI3DYiWnSkc/zmijhclRKCdpMZjy3Lhd5oEh2JiMhpsLyQQ5FlGYtW5KGmUY+EEB/89fq0Hh2waCuSJGHxDYMQ6O2Oo5WNeDXruOhIREROg+WFHMryfWXYcKQKbkoJr906BF7u4iboXkywrxqLbxgIAFiy9RR2nqoTnIiIyDmwvJDDKD3TgmdXHwYAzL8qEakR9jPP5ddkpobhlmFRkGXgT8sPos3A4SMiosvF8kIOwWyWsfDrA2jSG5ER2wfzJsSLjtRt/3fNAIT6qXG6rgVvbDwhOg4RkcNjeSGH8OnO09h5qh5e7kq8eottd9C9XL4ebnju2jQAwLtbTuFYZaPgREREjo3lhexepbatc1n0E9OSERvoLThRz12dFoYpA0JhNMtYtOIgzGZuXkdEdKlYXsjuPbP6EJr0RqTH+GPOyFjRcS7Zc9emwttdiX3FDfhyT4noOEREDovlheza94cr8f3hKqgUEhbfMNAmJ0VbS4S/JxZkJgEAXt5wDNpWg+BERESOieWF7FZjmwHPfGNZXXT/+H5IDvMTnOjy3Tk6FvHB3qhvbsfrP3LyLhHRpWB5Ibv1yobjqNS1ITbQC49OThAdp1e4KRV4emYqAODjHUUoqG4SnIiIyPGwvJBdOlKuwyfZRQCAv16XBg83pdhAvWhCYjAmJ4fAaJbx17VHRMchInI4LC9kd2RZxnNrDsMsAzMGhmNcQrDoSL3uqWsGwE0pYfOxGmw8WiU6DhGRQ2F5Ibuz/lAldhXWQ61SYNH0ZNFxrCIuyBv3jo0DAPxt/VGYuHSaiKjbWF7IrrQZTHhhbT4A4HcT4hHVx0twIut58Mr+0Hi64XhVE1bsKxUdh4jIYbC8kF15d8splDW0IkLjgQcc6AiAS6HxdMNDV1r+jv/MOs5zj4iIuonlhexGWUMr3t5SAABYND0Fnu7OM0n319w5ui/CNR4o17bh0+zTouMQETkElheyG39bfxRtBjNG9A3ANYPCRcexCQ83JR6bkggAeHNTATeuIyLqBpYXsgv7i89gzYFySBLw9MwBkCTH3Um3p24cGoWEEB9oWw14d8tJ0XGIiOweywsJJ8syFq87CsDyQZ4WqRGcyLaUCgmPT7UcG/DRjiLUNekFJyIism8sLyTcj/nV2F1kWRq9oGMIxdVMGRCKgZEatLSb8N62QtFxiIjsGssLCWU0mfH37yx3Xe4ZG4cIf0/BicSQJAnzr7IcgfBJNu++EBFdCMsLCfXfvaU4Ud0Efy83PDDRuZdGX8yk5BAMirLcfVmy7ZToOEREdovlhYRpbTfhnz8cBwA83LFhmyvrcvdlx2nU8u4LEdF5sbyQMB9sL0SVTo+oPp6YOzpWdBy7cGVSCAZHadBqMOG9rbz7QkR0PiwvJERdkx5vb7YsC358ahLUKuffkK47LHdfLJOWP8nm3RciovNheSEh3thYgCa9EWmRfpg5KEJ0HLsyMSkYg6P90Wow4YOfuPKIiOh/sbyQzZXUt+DzXZat8J+4OgUKhetsSNcdkiThoY7Jy59mn4aujbvuEhH9klXLy9atWzFz5kxERERAkiSsWrXqoq/ZsmULMjIy4OHhgX79+uGdd96xZkQS4I2NJ2AwyRjbPxBXJASJjmOXrkoJRUKIDxr1Rp55RET0P6xaXpqbmzF48GC8+eab3bq+sLAQ06dPx7hx47B//378+c9/xiOPPILly5dbMybZ0KmaJizfVwYA+ENmkuA09kuhkDqXjn+4vZAnThMR/YLKmt982rRpmDZtWrevf+eddxATE4PXXnsNAJCSkoKcnBy8/PLLuPHGG62UkmzptR9OwGSWMTk5BENj+oiOY9dmDo7AKxuOo6yhFV/llODO0X1FRyIisgt2NeclOzsbmZmZXZ6bOnUqcnJyYDCcf9xfr9dDp9N1eZB9Olqpw5qD5QCABZmueQxAT7gpFfjdhH4AgHe3nILBZBaciIjIPthVeamsrERoaGiX50JDQ2E0GlFbW3ve1yxevBgajabzER0dbYuodAle3XAcsgzMGBiO1AjXOnzxUt0yLBpBPu4oa2jFmgPlouMQEdkFuyovgGWlxS/Jsnze589atGgRtFpt56OkpMTqGannDpY2YMORKigk4LEpCaLjOAwPNyXuGRsHAHhr80mYzbLgRERE4tlVeQkLC0NlZWWX56qrq6FSqRAYGHje16jVavj5+XV5kP15ZYPlGIDr0iPRP8RXcBrHMnd0LHzVKhRUN+GH/CrRcYiIhLOr8jJ69GhkZWV1eW7Dhg0YNmwY3Nxc+9wbR7anqB5bjtdApZAwfzLnuvSUn4cb7hhlOT7hP9u4aR0RkVXLS1NTE3Jzc5GbmwvAshQ6NzcXxcXFACxDPnfeeWfn9fPmzcPp06exYMEC5Ofn44MPPsD777+PhQsXWjMmWZEsy/jH98cAALcMj0ZMoJfgRI7p7jF9oVJI2F1UjwMlDaLjEBEJZdXykpOTg/T0dKSnpwMAFixYgPT0dDz99NMAgIqKis4iAwBxcXFYt24dNm/ejCFDhuAvf/kLXn/9dS6TdmA/FdRid2E93FUK/H5Sf9FxHFaYxgPXDrYco/DeNh7YSESuTZLPzoh1EjqdDhqNBlqtlvNfBJNlGde9tQMHShpwz9i+eGZmquhIDu1IuQ7TX98GpULClscnIqoP72IRkfPoyee3Xc15IefyY341DpQ0wNNNiQcn8q7L5RoQ4Yex/QNhMsv4cHuR6DhERMKwvJBVmM0yXt5gmety99i+CPZVC07kHO4bZ9m0btmeEh7YSEQui+WFrGLdoQocrWyEr1qF343vJzqO05iYGIyEEB806Y1Ytpt7GhGRa2J5oV5nNJnxapZlX5f7xvWDv5e74ETOQ5Ik3DfOsmndh9sLeWQAEbkklhfqdd/kluNUTTP8vdxw7xV9RcdxOrOGRCLIxx3l2jasy6sQHYeIyOZYXqhXGUxm/OvHEwCA342Ph68HNxfsbR5uys4Tpv+zrRBOtmCQiOiiWF6oVy3fW4ri+hYE+bjjrjGxouM4rTmjYqFWKZBXpsW+4gbRcYiIbIrlhXqN3mjCGxsLAADzJsTDy10lOJHzCvB2x6whlk3rPtpRJDYMEZGNsbxQr/kqpxRlDa0I9VNjzijedbG2u8b0BQCsz6tApbZNbBgiIhtieaFe0WYw4d8dd10eurI/PNyUghM5v9QIDUbEBcBolvH5rtOi4xAR2QzLC/WKpbuKUalrQ4TGA7OHR4uO4zLu7rj7snRXMfRGk9gwREQ2wvJCl6213YS3Np8EADw8KQFqFe+62ErmgFCEazxQ19yObw9w2TQRuQaWF7psn2QXobZJj+gAT9w8LEp0HJeiUiowd7RlftFHO4q4bJqIXALLC12WJr0R72yx3HV5ZFIC3JT8lbK1W4fHcNk0EbkUftLQZfl4RxHOtBjQL8gb16dHio7jkrhsmohcDcsLXTJdmwFLtp4CADx6VQJUvOsiDJdNE5Er4acNXbL3txVC22pAQogPrhkUITqOS+OyaSJyJSwvdEkaWtrxwU+FAID5VyVCqZAEJ6J7uGyaiFwEywtdkiVbT6FRb0RymC+mpYWJjkMApgwIRUTHsum1B7lsmoicF8sL9Vhdk75zYuiCKYlQ8K6LXVApFbij41iGz3Zy6IiInBfLC/XYu1tPoaXdhEFRGkwZECo6Dv3CLcOi4aaUsK+4AYfLtaLjEBFZBcsL9Uh1Yxs+yS4CADw2JRGSxLsu9iTYV42r08IBAJ/tLBachojIOlheqEfe2nQSbQYz0mP8MTExWHQcOo85I2MAAN/klkHXZhCchoio97G8ULeV1Ld0LsNdmJnEuy52akRcABJDfdDSbsLKfWWi4xAR9TqWF+q21344AYNJxtj+gRjbP0h0HPoVkiRhzi8m7vK8IyJyNiwv1C0nqhqxcn8pAOCPU5MFp6GLuT49El7uSpyobsKuwnrRcYiIehXLC3XLyxuOwSwDV6eGYXC0v+g4dBG+Hm64ruOsKS6bJiJnw/JCF5Vb0oDvD1dBIQELpyaKjkPdNGekZejou0OVqG7keUdE5DxYXuiiXvruKADghqFR6B/iKzgNddeACD8MjfGH0Szjqz0louMQEfUalhe6oJ9O1GLHyTq4KxWYf1WC6DjUQ3NHW+6+LN1VDJOZE3eJyDmwvNCvkmUZ//jectfl9pExiOrjJTgR9dS0tHD08XJDubYNG49Wi45DRNQrWF7oV31/uBIHSrXwclfi4Un9RcehS+DhpsQtw6MBcOIuETkPlhc6L5NZxssbjgMA7rsiDkE+asGJ6FLdMSIWkgRsOV6D03XNouMQEV02lhc6r+X7SlFQ3QR/LzfcN76f6Dh0GWICvTA+wXKUw9JdPO+IiBwfywudo7XdhFc2HAMAPDSxP/w83AQnost1dsfdr/eWQm80CU5DRHR5WF7oHO//dApVOj2i+njizjGxouNQL7gyKRihfmrUN7cj60iV6DhERJeF5YW6qGnU4+3NJwEAj09NglqlFJyIeoNKqcDsYZaJu1/s5tARETk2lhfq4l8/HkdzuwmDojSYOShCdBzqRbcMj4YkAdsL6jhxl4gcGssLdSqobsIXuy07sf55egoUCklwIupNUX28MCHRMnH3S+64S0QOjOWFOv1t/VGYzDKuSgnFqH6BouOQFdw2IgYA8HVOCdqNZsFpiIgujU3Ky1tvvYW4uDh4eHggIyMD27Zt+9VrN2/eDEmSznkcPXrUFlFd1q5TdfghvwpKhYQnpiWLjkNWMik5BMG+atQ2tePHfE7cJSLHZPXysmzZMsyfPx9PPvkk9u/fj3HjxmHatGkoLr7wpMFjx46hoqKi85GQwHN1rMVslvHiunwAwK3Do9E/xEdwIrIWN6UCtwyLAgAs5cRdInJQVi8vr776Kn7zm9/gvvvuQ0pKCl577TVER0fj7bffvuDrQkJCEBYW1vlQKrnqxVrWHCzHgVItvN2VmH9Voug4ZGW3DrcMHW07UYuS+hbBaYiIes6q5aW9vR179+5FZmZml+czMzOxY8eOC742PT0d4eHhmDx5MjZt2vSr1+n1euh0ui4P6r6WdiMWr7MMyT0wMR7BvjwGwNlFB3hhXEIQAODLPbz7QkSOx6rlpba2FiaTCaGhoV2eDw0NRWVl5XlfEx4ejiVLlmD58uVYsWIFkpKSMHnyZGzduvW81y9evBgajabzER0d3et/D2f21qaTqNS1IaqPJ+4bx2MAXMXtHRN3v8ophcHEibtE5FhUtngTSeq65FaW5XOeOyspKQlJSUmdfx49ejRKSkrw8ssvY/z48edcv2jRIixYsKDzzzqdjgWmm4rrWrBk2ykAwFMzUuDhxqE5V3HVgFAE+ahR06jHxqPVmJoaJjoSEVG3WfXOS1BQEJRK5Tl3Waqrq8+5G3Mho0aNwokTJ877NbVaDT8/vy4P6p4X1h1Bu9GMMfGB/PByMW5KBW7umLjLHXeJyNFYtby4u7sjIyMDWVlZXZ7PysrCmDFjuv199u/fj/Dw8N6O59K2F9Ti+8OWpdHPzEz91Tth5LxuHW65Q7nleA1Kz3DiLhE5DqsPGy1YsABz587FsGHDMHr0aCxZsgTFxcWYN28eAMuwT1lZGT755BMAwGuvvYa+ffsiNTUV7e3t+Oyzz7B8+XIsX77c2lFdhtFkxnNrDgMA5oyMQVKYr+BEJEJsoDfG9g/E9oI6fLWnBAsyky7+IiIiO2D18jJ79mzU1dXh+eefR0VFBdLS0rBu3TrExlpOK66oqOiy50t7ezsWLlyIsrIyeHp6IjU1FWvXrsX06dOtHdVlfLbzNI5XNaGPlxsem8Kl0a7sthEx2F5Qh2U5JXhkcgJUSm66TUT2T5JlWRYdojfpdDpoNBpotVrOfzmPmkY9Jr+yGbo2I/5yXRrmjooVHYkEajeaMXrxj6hrbsd7dw7DlAHdn4tGRNSbevL5zX9muZgX1+VD12ZEaoQfbhvOVVmuzl2lwE0ZnLhLRI6F5cWF7Cioxcr9ZZAk4MXrB3KIgAAAsztK7OZj1ajUtglOQ0R0cfz0chF6owlPrToEAJg7KhaDo/3FBiK70S/YByP6BsAsA8v3lYqOQ0R0USwvLuLdLadwqrYZwb5qLJzKVSXU1S0dd1++yimB2exU0+CIyAmxvLiAotpmvLmpAADwf9cMgJ+Hm+BEZG+mDwyDj1qF03Ut2F1ULzoOEdEFsbw4OVmW8X/fHEK70YxxCUGYOYib/dG5vNxVmDnY8rvx1Z4SwWmIiC6M5cXJLd9Xhm0nauGuUuD5WWncSZd+1S3DLENH6w5VQNdmEJyGiOjXsbw4sSpdG57v2En3sasSERfkLTgR2bMh0f5ICPFBm8GM1bnlouMQEf0qlhcnJcsynlx5CLo2IwZFaXD/uDjRkcjOSZLUuWz66xwOHRGR/WJ5cVKrD5Tjh/wquCkl/OOmwdzThbrl+vRIuCklHCjV4milTnQcIqLz4ieaE6pp1OPZ1Zbhot9PSuDBi9RtgT5qXJViOSJgGSfuEpGdYnlxMrIs45nVh3CmxYCUcD88MDFedCRyMGcn7q7cXwa90SQ4DRHRuVhenMyq3DKsy6uEUiHhHzcNghuHi6iHxicGI8zPAw0tBvxwpFp0HCKic/CTzYmU1Lfg6VWW4aJHJycgLVIjOBE5IqVC6jyscRkn7hKRHWJ5cRJGkxmPLctFo96IjNg+eJDDRXQZbh5mKS/bTtSgrKFVcBoioq5YXpzE25tPIuf0GfioVXht9hCuLqLLEhvojVH9AiDLwH9zeFgjEdkXfsI5gdySBrz24wkAwPOzUhEd4CU4ETmDzj1f9vKwRiKyLywvDk7bYsDDS/fBZJZxzaBwXJ8eKToSOYlpaeHw9VCh9Ewrsk/ViY5DRNSJ5cWBmc0yFnyVi9IzrYgO8MQL1w/k2UXUazzclLh2cAQA7vlCRPaF5cWBvbv1FH48Wg13lQJv35EBjaeb6EjkZM4OHX13uBLaFh7WSET2geXFQWWfrMM/vj8KAHju2lQuiyarGBipQXKYL9qNZqzKLRMdh4gIAMuLQ6rSteGRL/fDLAM3DI3ErR3/OibqbZIkde64u3wfVx0RkX1geXEwbQYTfvtJDmoa9UgK9cVfr0vjPBeyqllDIqBSSDhYqsWxykbRcYiIWF4ciSzLePy/B3GgVIs+Xm54785h8HJXiY5FTi7QR41JySEAePeFiOwDy4sDeWNjAdYcKIdKIeGtOzIQE8j9XMg2buw4LmDl/jIYTWbBaYjI1bG8OIg1B8rxatZxAMBfrkvD6PhAwYnIlVyZFIIAb3fUNOqx7USt6DhEJIDZLGPv6Xo8t+Yw/vDVAaFZOObgAH46UYsFX+UCAO4Z2xe3jYgRG4hcjrtKgVlDIvDh9iL8d28pruwYRiIi52YwmbHrVD2yjlQi60gVyrVtACwHuD45IwUB3u5CcrG82Lm8Ui1+92kODCYZMwaG46kZA0RHIhd149AofLi9CFlHqqBtMUDjxX2FiJxRY5sBW47XIOtIFTYerUZjm7Hzaz5qFaYMCMU1g8LhoxZXIVhe7FhhbTPu/nA3mttNGBMfiFdnD4ZSwZVFJEZqhB+Sw3xxtLIRqw+WY+6oWNGRiKgXmMwy8sq02Ha8BtsKarHv9BkYf3GeWZCPOyYnh2LKgFBckRAEDzelwLQWLC92qqi2Gbct2Ym65nakRfrh3bkZUKvE/8KQ65IkCTdlROGva/Px372lLC9EDqykvgU/FdRi24kabC+og7a16w7a/YK8MWVAKDJTQzEkuo/d/cOZ5cUOFXYUl0pdGxJCfPDh3SPg68Fb9CTerCGRWLz+KA6UNKCguhH9Q3xFRyKibjjT3I6dp+qw42QdfiqoRWFtc5ev+3qoMDY+CFckBGFcQhBiA70FJe0elhc787/FZen9oxDsqxYdiwgAEOyrxpVJwfghvxr/3VuGJ6Yli45EROfRpDdid2EddhRYCkt+pQ7yzyNBUCokpEf7d5SVYAyO0kCldJwFyCwvdiSvVIt7PtqN2qZ2JIT44IvfjkKQD4sL2ZebMqLwQ341Vu4vxeNTk+zudjKRK2ozmLDv9BnsOFmHHSdrcaBUC9Mv5q0AQGKoD8bEB2F0fCBGxwfCz4Hv6LO82IltJ2ow79O9aG43YUC4Hz75zQgWF7JLVyaHwN/LDVU6PX4qqMWExGDRkYhcjizLOFyuw5bjNdheUIuc02fQbuy6gWRMgBfGdBSV0fGBCPH1EJS297G82IEvdxfjqVWHYDTLGNs/EO/MyeAcF7JbapUSswZH4OPs0/jv3lKWFyIbaWhpx7YTtdh8rAZbT9SgplHf5eshvmqMiQ/EmP5BGN0vENEBzrsLO8uLQO1GM55bcxif7yoGAFw7OAIv3zwY7irHGXck13RTRjQ+zj6N7w9XQttqgMaTZZvIGk7VNOG7w5X4Mb8a+4vP4JcjQV7uSoyJD8T4xGCMiQ9CfLC3yxzUy/IiSOmZFsz/Mhc5p89AkoCFmUl4cGK8y/zikWNLi/RDYqgPjlc1Ye3BCtw+krs+E/UGWZaRX9GI7w5X4rtDFThe1dTl64mhPpiYFIIJicEY1rePy26hwfIiwDe5ZXhq1SE0thnh66HCv24dgknJoaJjEXXb2T1fXlx3FP/dW8LyQnSZiutasHJ/GVbuL0VRXUvn8yqFhDH9gzA1NRRXJoUgwt9TYEr7wfJiQ5XaNvzl2yNYm1cBABga449/zh5i9+vpic7nuiGR+Pt3x7CvuAEna5oQH+wjOhKRQ9G1GbDuYAVW7CvD7qL6zufVKgUmJAbj6rQwTE4O5VEc58HyYgMGkxkf7yjCP7OOo7ndBKVCwiOTEvDQlfEOta6e6JdC/DwwPiEIm47VYPneUvzxau75QtQd+RU6fJJ9Gqv2l6HVYAIASBIwNj4INwyNxNTUMHgLPDfIEfCnY0UGkxkr95XhjU0nUFLfCgBIj/HHX2alIS1SIzgd0eW7KSMam47VYOX+Mvwhk3u+EP0ag8mM7w9X4pMdp7vcZYkP9saNGVG4Pj0S4RoOCXWXTf7Z/9ZbbyEuLg4eHh7IyMjAtm3bLnj9li1bkJGRAQ8PD/Tr1w/vvPOOLWL2mromPd7dchKTXtmMPy4/iJL6VgT5uOPvNw7E8nljWFzIaUxOCYHG0w0V2jZkn6wTHYfI7rQZTPh052lc+fJmPLx0P3YX1UOpkDBjYDiW/XYUflgwAQ9O7M/i0kNWv/OybNkyzJ8/H2+99RbGjh2Ld999F9OmTcORI0cQE3PuJL/CwkJMnz4d999/Pz777DNs374dDz74IIKDg3HjjTdaO+4lq29ux6aj1cg6UoUfj1bBYLKsZwv0dse8CfGYMyoWnu6uOSucnJeHmxIzBoVj6a5irNhfiisSgkRHIrILLe1GfL6zGO9tO4Xqjv1YAr3dccfIGNw+MhZhGufZME4ESZZl+eKXXbqRI0di6NChePvttzufS0lJwXXXXYfFixefc/2f/vQnrF69Gvn5+Z3PzZs3DwcOHEB2dvZF30+n00Gj0UCr1cLPz693/hIAjCYzDpQ2oM1gRmu7Cbo2A0rqW1FU14wDJQ049T+HXA2O0uC2ETG4dkgEvNw5OkfOK6eoHje9kw1vdyX2PHUVf9/JpRlMZny5pwT/+uEEapsspSVC44Hfju+H2cNj+I/YC+jJ57dV/yvT3t6OvXv34oknnujyfGZmJnbs2HHe12RnZyMzM7PLc1OnTsX7778Pg8EAN7eus671ej30+p93GdTpdL2UviuDScaNb1+4PKWE+2HKgFBcnRqGARG9V5yI7FlGbB/EBHihuL4FWUeqMGtIpOhIRDYnyzLWH6rEP74/1nlic0yAFx6+sj+uS4/k5qO9zKrlpba2FiaTCaGhXfcwCQ0NRWVl5XlfU1lZed7rjUYjamtrER4e3uVrixcvxnPPPde7wc9DrVIgNtALHiolPNyV8FErEeXvhegAT6RGajAkyh99vN2tnoPI3kiShOvSI/H6jyewYl8Zywu5nPwKHf5v1SHknD4DwDI89MjkBNw2IoalxUpscn/3f3eNlWX5gjvJnu/68z0PAIsWLcKCBQs6/6zT6RAdHX05cc9LoZCw5fEre/37EjmD6zvKy7YTNahubHOqA+CIfk1jmwGv/XACH+0ogsksw9NNifvHxeH+8f14Pp2VWbW8BAUFQalUnnOXpbq6+py7K2eFhYWd93qVSoXAwMBzrler1VCrefoykUhxQd4YEu2P3JIGrDlQgd9cESc6EpFVZR2pwlOr8lCls0xbmJYWhv+7ZgB3wLURq97Pcnd3R0ZGBrKysro8n5WVhTFjxpz3NaNHjz7n+g0bNmDYsGHnzHchIvtxw1DLcNHK/aWCkxBZj7bVgAVf5eL+T3JQpdOjb6AXPr53BN6ek8HiYkNWH4xbsGAB/vOf/+CDDz5Afn4+HnvsMRQXF2PevHkALMM+d955Z+f18+bNw+nTp7FgwQLk5+fjgw8+wPvvv4+FCxdaOyoRXYZrBkVApZBwqEyHE1WNouMQ9botx2sw9Z9bsWJfGSQJ+N34fvhu/nhMSAwWHc3lWH3Oy+zZs1FXV4fnn38eFRUVSEtLw7p16xAbGwsAqKioQHFxcef1cXFxWLduHR577DH8+9//RkREBF5//XW73uOFiIAAb3dMTArGD/nVWLm/jMcFkNMwmMx4ecMxvLvlFADLMOnLNw9CRmyA4GSuy+r7vNiatfZ5IaKLW3uwAg8t3YdIf09s++OVUPC4AHJw5Q2t+P0X+7G3YyXR3FGx+PP0FO7XYgV2s88LEbmWySkh8FWrUNbQil2F9Rgdf+4keyJHseV4DR79cj8aWgzwVavw95sGYfrA8Iu/kKyOC9CJqNd4uCk7/+O+an+Z4DREl0aWZXzwUyHu+XA3GloMGBipwbePXMHiYkdYXoioV13fsepoXV4F2gwmwWmIeqbdaMaiFXl4/tsjMMvAzRlR+O8DoxEb6C06Gv0CywsR9aoRfQMQ6e+JRr0RP+RXiY5D1G3aFgPmvL8LX+4pgUICnpqRgpduGgS1ivNb7A3LCxH1KoVCwqwhEQCAlfs4dESOoVLbhlvezcbuwnr4qlV4/67huG9cvwvuBk/isLwQUa+7Pt0ydLTleA3qmvQXuZpIrFM1Tbjx7R04VtWIEF81vn5gNK5MDhEdiy6A5YWIel1CqC/SIv1gNMv49mCF6DhEvyqvVIub3slGWUMr4oK8sfyBMUgO4zYb9o7lhYis4vr0KADACq46Ijt1oKQBt/9nJ+qb25EW6Yev541GdICX6FjUDSwvRGQV1w6OgFIh4UBJA07VNImOQ9TFwdIGzHl/FxrbjBjetw++uH8Ugnx4yK+jYHkhIqsI9lVjXEIQAGBVbrngNEQ/yyvVYs5/LMVlWGwffHjPCPh68OBfR8LyQkRWc3bV0ZoD5XCyk0jIQR0p12HO+7ugazMiI7YPPrp3BHzU3Gze0bC8EJHVTBkQBg83BQprm5FXphUdh1xccV0L7vpwN7StBgyN8cdH9wxncXFQLC9EZDU+ahWuSgkFAHzDoSMSqLZJjzs/2IWaRj2Sw3w5VOTgWF6IyKpmDbHs+fLtwXKYzBw6Ittr0htxz4d7UFTXgqg+nvj43hHQeLK4ODKWFyKyqvGJQfDzUKFKp8euwjrRccjFGExmPPDZXuSVaRHg7Y5P7h2BUD8P0bHoMrG8EJFVqVU/nzS9mkNHZGPPrzmCbSdq4eWuxAd3D0e/YB/RkagXsLwQkdVd27HqaP2hSuiNPGmabOPT7CJ8uvM0JAn4163pGBLtLzoS9RKWFyKyupFxgQjxVUPbasDW47Wi45AL2F5Qi2fXHAEAPD41CVMGhApORL2J5YWIrE6pkDBzsOXuyze5PC6ArKuwthkPfr4PJrOM69Mj8cCEeNGRqJexvBCRTZzdsO6H/Co0642C05Czam03Yd6ne6FtNSA9xh+LbxgISZJEx6JexvJCRDYxMFKDuCBvtBnMyDpSJToOOSFZlvHUqkM4VtWIYF813p2TAQ83pehYZAUsL0RkE5LEoSOyrq9ySrB8XykUEvD6rekI4ZJop8XyQkQ2c21Hedl2ohb1ze2C05AzOVKuw9PfHAYA/CEzCaPjAwUnImtieSEim+kf4oO0SD8YzTLW5VWIjkNOorHNgAc/3wu90Ywrk4I5QdcFsLwQkU2dvfvCDeuotzy7+giK6loQofHAq7cMgULBCbrOjuWFiGxq5uAISBKwu6geZQ2touOQg1uXV9E5z+W1W9PRx9tddCSyAZYXIrKpcI0nRvQNAAB8e4B3X+jSVWrb8OeVeQCABybGY0RcgOBEZCssL0Rkc2dPmv6GQ0d0icxmGY//9wAaWgwYGKnBo5MTRUciG2J5ISKbm5YWBpVCwpEKHQqqG0XHIQf00Y4ibDtRCw83Bf45ewjcVfw4cyX8X5uIbK6PtzsmJAYD4N0X6rnC2mb8/bujAIAnp6egfwhPinY1LC9EJMTZk6a/PVgBWZYFpyFHYTbLeGL5QeiNZoztH4g5o2JFRyIBWF6ISIjJKaFQqxQorG3G4XKd6DjkIL7YU4xdhfXwdFPibzcM4rlFLorlhYiE8FGrMCk5BIDl7gvRxZQ3tGLxOstw0eNTkxAd4CU4EYnC8kJEwlwz6OzQUTmHjuiCzh662KQ3Ij3GH3eN6Ss6EgnE8kJEwkxKDoGXuxKlZ1pxoFQrOg7ZsdUHyrHxaDXclQq8dOMgKLmLrktjeSEiYTzdlZicEgoAWHuQq47o/HRtBvx1bT4A4KEr+yMh1FdwIhKN5YWIhLpmUDgAYO3BCpjNHDqic72WdQI1jXrEBXlj3sR+ouOQHWB5ISKhJiQGw0etQrm2DftLzoiOQ3Ymv0KHj7OLAADPXZsKtUopNhDZBZYXIhLKw02JKQMsQ0drDnDVEf1MlmX836pDMJllTB8YhvEdGxsSsbwQkXBnh47W5VXAxKEj6rBiXxlyTp+Bl7sST80YIDoO2RGrlpczZ85g7ty50Gg00Gg0mDt3LhoaGi74mrvvvhuSJHV5jBo1ypoxiUiwcQnB8PNQobpRjz1F9aLjkB3QtRmweL1lku4jkxMQ4e8pOBHZE6uWl9tvvx25ubn47rvv8N133yE3Nxdz58696OuuvvpqVFRUdD7WrVtnzZhEJJi7SoGpqWEALHu+EP17UwFqm9oRH+yNe8fGiY5DdkZlrW+cn5+P7777Djt37sTIkSMBAO+99x5Gjx6NY8eOISkp6Vdfq1arERYWZq1oRGSHrhkcga/3lmJ9XiWenZkKlZKj2q6qpL4FH/5UBAB4asYAnhhN57Dab0R2djY0Gk1ncQGAUaNGQaPRYMeOHRd87ebNmxESEoLExETcf//9qK6u/tVr9Xo9dDpdlwcROZ4x8YHo4+WGuuZ27Crk0JEr+9v6o2g3mTEuIQgTkzhJl85ltfJSWVmJkJCQc54PCQlBZWXlr75u2rRp+Pzzz7Fx40a88sor2LNnDyZNmgS9Xn/e6xcvXtw5p0aj0SA6OrrX/g5EZDtuSgWuTrNM3OXQkevKKarH2rwKKCTgyRkpPHiRzqvH5eXZZ589Z0Lt/z5ycnIA4Ly/dLIsX/CXcfbs2ZgxYwbS0tIwc+ZMrF+/HsePH8fatWvPe/2iRYug1Wo7HyUlJT39KxGRnZjZsepo/aFKGExmwWnI1sxmGX/p2El39vBoJIf5CU5E9qrHc14efvhh3HrrrRe8pm/fvjh48CCqqqrO+VpNTQ1CQ0O7/X7h4eGIjY3FiRMnzvt1tVoNtVrd7e9HRPZrZL9ABPm4o7apHdsLajEx6dy7t+S8Vh8ox4GSBni7K/HYlETRcciO9bi8BAUFISgo6KLXjR49GlqtFrt378aIESMAALt27YJWq8WYMWO6/X51dXUoKSlBeHh4T6MSkYNRKiRMSwvHpztP49uDFSwvLkRvNOEf3x8DADwwMR4hvh6CE5E9s9qcl5SUFFx99dW4//77sXPnTuzcuRP3338/rrnmmi4rjZKTk7Fy5UoAQFNTExYuXIjs7GwUFRVh8+bNmDlzJoKCgnD99ddbKyoR2ZGzG9Z9f7gSeqNJcBqylS93l6CsoRUhvmr85gqeX0QXZtX1Z59//jkGDhyIzMxMZGZmYtCgQfj000+7XHPs2DFotVoAgFKpRF5eHmbNmoXExETcddddSExMRHZ2Nnx9eYookSsY3jcAoX5qNLYZse14reg4ZAMt7Ua8sbEAAPD7yQnwdOf5RXRhVtvnBQACAgLw2WefXfAaWf55K3BPT098//331oxERHZOoZAwfWA4PtxehLV5FbhqQPfnyJFj+nB7EWqb9IgJ8MLsYVwxShfHnX+IyO5cMygCAJB1pAptBg4dOTNtiwHvbjkJAFgwJZEb0lG38LeEiOzO0Bh/RPp7oklvxOZjNaLjkBW9u/UkdG1GJIX6YubgCNFxyEGwvBCR3ZEkCdMHWo4IWX+oQnAaspbqxjZ8uL0IALBwahKUCm5IR93D8kJEdmnaQMuqox/zqzl05KT+vbEArQYT0mP8cVUKl8VT97G8EJFdSo/2R4TGA016I7ad4KojZ1OpbcMXuy07oj+emcRjAKhHWF6IyC5JktR51tG6PA4dOZt3tpxEu8mMEX0DMKb/xTc+JfollhcislszBlnmvfxwpIob1jmRal0bvthdDAB4ZHKC4DTkiFheiMhupUf3QZifBxr1RvzEoSOnsWTrKeiNZgyN8cfY/oGi45ADYnkhIrulUEi4Os1y92Uth46cQm2THp/v+vmuC+e60KVgeSEiuzaj46yjLA4dOYX/bCtEq8GEQVEaTEgMFh2HHBTLCxHZtYyYPgjxtZx1tKOgTnQcugz1ze34JLsIAPDIJN51oUvH8kJEdk2hkDCNQ0dO4YOfCtHSbsKAcD9M5r4udBlYXojI7p3dsG7D4Uq0G82C09ClaGwz4OMdRQCARyb3510XuiwsL0Rk94b3DUCQjxq6NiN2nOSqI0f0xe5iNOqNiA/2RuaAMNFxyMGxvBCR3VMqJFydFgqAG9Y5onajGe//VAgA+N34eCh4hhFdJpYXInII088OHR2pgsHEoSNHsiq3DFU6PUL91JiVzpOj6fKxvBCRQxjRNwCB3u5oaDEg+yRXHTkKs1nGkq2nAAD3jo2DWqUUnIicAcsLETkElVKBqR2rjtYf4tCRo9h4tBoF1U3wVatw28gY0XHISbC8EJHDmNExdPT94SoYOXTkEN7dehIAcPuoGPh5uAlOQ86C5YWIHMbIuAAEeLujvrkdO0/Vi45DF7H3dD32FJ2Bm1LCvWPjRMchJ8LyQkQOQ6VUYGpqx6ojDh3ZvXe3WOa6XJ8eiVA/D8FpyJmwvBCRQ5mW1jF0dKiSQ0d2rKC6CVn5VQCA346PF5yGnA3LCxE5lNHxgfD3ckNdczt2F3HoyF59uL0QsgxclRKK/iE+ouOQk2F5ISKH4qZUIHMAN6yzZw0t7Vi+rxQA8JsrONeFeh/LCxE5nLMb1n13qAomsyw4Df2vL/eUoM1gRkq4H0b1CxAdh5wQywsROZwx8UHw81ChtkmPPRw6sisGk7nzAMZ7x/blAYxkFSwvRORw3FUKZKZaNqzj0JF9+f5wJSq0bQj0dsfMwTwKgKyD5YWIHNL0gWd3262EmUNHduPD7UUAgDtGxcLDjUcBkHWwvBCRQ7qifzB8PVSoadQj5/QZ0XEIQG5JA/aetmxKN2cUjwIg62F5ISKH5K5SYApXHdmVD7cXAgBmDopAiC83pSPrYXkhIoc1vWPDuvWHKjh0JFiVrg1rD1pK5D08CoCsjOWFiBzWuMQg+KhVqNLpsa+YQ0cifZp9GkazjOF9+2BglEZ0HHJyLC9E5LDUKiWuSgkBAKzl0JEweqMJS3cXA+BdF7INlhcicmhnN6xbn8dVR6J8d6gS9c3tCPPz6Nz9mMiaWF6IyKGNTwyGj1qFSl0b9pdw6EiEpbssd11mD4+GSsmPFbI+/pYRkUPzcFNi8tmho4OVgtO4noLqJuwqrIdCAm4dES06DrkIlhcicnidQ0dcdWRzZ++6TEoORbjGU3AachUsL0Tk8CYkBsPbXYkKbRtySxtEx3EZbQZT5+nRd4zkpnRkOywvROTwLENHHRvWHeSqI1tZe7AC2lYDIv09MT4xWHQcciEsL0TkFH4eOqqELHPoyBbOLo++bUQ0lAqeHk22w/JCRE5hYpJl6KisoRW5JQ2i4zi9o5U67D19BiqFhFuGcaIu2ZZVy8sLL7yAMWPGwMvLC/7+/t16jSzLePbZZxEREQFPT09MnDgRhw8ftmZMInICHm5KTErhWUe2cnai7pQBoQjx4zlGZFtWLS/t7e24+eab8cADD3T7NS+99BJeffVVvPnmm9izZw/CwsIwZcoUNDY2WjEpETmDGQPDAADr8jh0ZE0t7Uas3FcGALhjZKzgNOSKrFpennvuOTz22GMYOHBgt66XZRmvvfYannzySdxwww1IS0vDxx9/jJaWFixdutSaUYnICUxMCoFXx9DRgVKt6DhOa82BcjTqjYgN9MKY+EDRccgF2dWcl8LCQlRWViIzM7PzObVajQkTJmDHjh3nfY1er4dOp+vyICLX5OGmxKRky4Z1HDqyns87hoxuHxEDBSfqkgB2VV4qKy27Y4aGdj0bIzQ0tPNr/2vx4sXQaDSdj+hoThwjcmUzOlYdrT1YwaEjK8gr1eJgqRbuSgVuyogSHYdcVI/Ly7PPPgtJki74yMnJuaxQktS1ycuyfM5zZy1atAharbbzUVJSclnvTUSObWJSCDzdLENHBzl01OuW7j4NAJiaFoZAH7XgNOSqVD19wcMPP4xbb731gtf07dv3ksKEhVkm21VWViI8PLzz+erq6nPuxpylVquhVvP/QERk4emuxKSUEKw9WIF1eRUYHO0vOpLTaGwz4JvccgDcUZfE6nF5CQoKQlBQkDWyIC4uDmFhYcjKykJ6ejoAy4qlLVu24O9//7tV3pOInM/0tHCsPViBtXkVeGJa8q/euaWe+Sa3HC3tJsQHe2NkXIDoOOTCrDrnpbi4GLm5uSguLobJZEJubi5yc3PR1NTUeU1ycjJWrlwJwDJcNH/+fLz44otYuXIlDh06hLvvvhteXl64/fbbrRmViJzIlcnB8HBToPRMK/LKOHTUG2RZ/nmi7shYFkISqsd3Xnri6aefxscff9z557N3UzZt2oSJEycCAI4dOwat9uf/uPzxj39Ea2srHnzwQZw5cwYjR47Ehg0b4Ovra82oROREvNxVmJQcgnV5lViXV4lBUf6iIzm83JIG5Ffo4K5S4MahkaLjkIuTZCebjq/T6aDRaKDVauHn5yc6DhEJ8u3Bcjy8dD9iAryw5fGJvFNwmRZ+fQD/3VuKG4ZG4tVbhoiOQ06oJ5/fdrVUmoiot0xKDoGHmwLF9S04XM79ny6HtsWAbw9yoi7ZD5YXInJKXu4qXJlk2bBuLTesuywr9peizWBGcpgvhsb0ER2HiOWFiJzX9I4N69blccO6SyXLcuchjLePjOHwG9kFlhciclqTkkOgVilwuo5DR5dqT9EZnKhugqebEtelc6Iu2QeWFyJyWt7qn4eOeNbRpVm6y7Kj7rWDI+Dn4SY4DZEFywsRObXpgzh0dKnqm9uxLs9yrtwdozhRl+wHywsRObXJHUNHRXUtOFLBoaOeWL63FO0mM9Ii/bhXDtkVlhcicmreahUmJgUD4NBRT8iyjKW7OybqjogVnIaoK5YXInJ6P686quTQUTdln6xDYW0zfNQqXDskQnQcoi5YXojI6U1OCYVapUBhbTOHjrrp8467LrOGRMBHbdWTZIh6jOWFiJyezy9WHa05wKGji6lp1OP7Qx0TdUdyyIjsD8sLEbmEs0Mfaw6Uc+joIr7eWwKjWcaQaH8MiOAZcWR/WF6IyCVMSg6Bt7sSZQ2t2Fd8RnQcu2U2y/iiY8iI5xiRvWJ5ISKX4OGmRGZqGABgdW654DT2a1tBLUrqW+HrocI1gzhRl+wTywsRuYxrB1s+jNfmVcBoMgtOY58+32nZUffGoVHwdFcKTkN0fiwvROQyrkgIQh8vN9Q2tWPnqXrRcexOpbYNPx6tBsAhI7JvLC9E5DLclApM69jzZfWBMsFp7M+yPSUwmWWM6BuAhFBf0XGIfhXLCxG5lLNDR+sPVUJvNAlOYz9MZhnL9nTsqMu7LmTnWF6IyKWM6BuAMD8PNLYZseVYjeg4dmPzsWqUa9vQx8sNV6eFiY5DdEEsL0TkUhQKCdcMOjt0xFVHZ32+y3LX5aaMKHi4caIu2TeWFyJyOWc3rPshvwrNeqPgNOKVnmnBpmOWibq3jeCQEdk/lhcicjkDIzWIDfRCm8GMH/KrRMcRbtmeEsgyMCY+EP2CfUTHIboolhcicjmSJHVO3HX1DesMJjOW7SkBwIm65DhYXojIJZ0tL1tP1OBMc7vgNOL8mF+F6kY9gnzckTmAE3XJMbC8EJFLSgj1RUq4HwwmGd/mue5J02cn6t48LBruKn4kkGPgbyoRuawbh0YCAFbsKxWcRIzC2mZsO1ELSQJu50RdciAsL0Tksq4dEgGFBOwvbsCpmibRcWxu6S7LOUYTE4MRHeAlOA1R97G8EJHLCvH1wPjEYADAyv2udVxAm8GEr/da7jjNGRUrOA1Rz7C8EJFLu2FoFABgxb4ymM2y4DS2s/ZgBRpaDIj098TEpBDRcYh6hOWFiFxa5oBQ+KpVKGtoxe4i1zlp+rOOIaPbR8ZAqZAEpyHqGZYXInJpHm5KTO84adpVJu4eLtdif3ED3JQSbhkWLToOUY+xvBCRy7uhY9XRurxKtLY7/0nTn+20LI+emhqGYF+14DREPcfyQkQub3jfAET18UST3ogNRypFx7GqxjYDvsm1TE7mRF1yVCwvROTyFAoJN6Sf3fPFuVcdrdxfhpZ2ExJCfDAyLkB0HKJLwvJCRATg+o5VR9tO1KBK1yY4jXXIsozPdlom6t4xMgaSxIm65JhYXoiIAMQFeSMjtg/MMvDfvc45cXdP0Rkcr2qCp5sSN2REiY5DdMlYXoiIOtw63LLyZtmeEqfc8+Xj7CIAlkMp/TzcxIYhugwsL0REHWYMCoevWoXi+hbsPFUnOk6vKm9oxXeHLJOR7x7bV2wYosvE8kJE1MHLXYVrh0QAAL7YUyI4Te/6dOdpmMwyRvULQEq4n+g4RJeF5YWI6Bdu6zhd+ftDlTjT3C44Te9obTfhi92WvV3uGRsnOA3R5bNqeXnhhRcwZswYeHl5wd/fv1uvufvuuyFJUpfHqFGjrBmTiKhTWqQGaZF+aDeZscJJDmtclVuGhhYDogM8cVVKqOg4RJfNquWlvb0dN998Mx544IEeve7qq69GRUVF52PdunVWSkhEdK7Zwy13X77cXQxZduyJu7Is48PthQCAu0b35TlG5BRU1vzmzz33HADgo48+6tHr1Go1wsLCrJCIiOjiZg2JwItr83Giugn7is8gI9ZxN3PLPlmH41VN8HJX4maeY0ROwi7nvGzevBkhISFITEzE/fffj+rq6l+9Vq/XQ6fTdXkQEV0OPw83XDPIcljjJ9mnBae5PB9sLwIA3JQRBY0nl0eTc7C78jJt2jR8/vnn2LhxI1555RXs2bMHkyZNgl6vP+/1ixcvhkaj6XxER/NfFkR0+e4a0xcAsC6vAtWNjrnj7qmaJvx4tArAz38fImfQ4/Ly7LPPnjOh9n8fOTk5lxxo9uzZmDFjBtLS0jBz5kysX78ex48fx9q1a897/aJFi6DVajsfJSXOtbyRiMRIi9QgI7YPDCYZS3cVi45zSd7bVghZBiYnhyA+2Ed0HKJe0+M5Lw8//DBuvfXWC17Tt2/fS81zjvDwcMTGxuLEiRPn/bparYZazSPdiaj33TWmL/aePoPPdxXjwYn94a6yu5vVv6q6sQ3L91mOOZg3MV5wGqLe1ePyEhQUhKCgIGtkOa+6ujqUlJQgPDzcZu9JRAQA09LCEOKrRnWjHusPVWDWkEjRkbrtw+1FaDeaMTTGH8Ni+4iOQ9SrrPrPiOLiYuTm5qK4uBgmkwm5ubnIzc1FU1NT5zXJyclYuXIlAKCpqQkLFy5EdnY2ioqKsHnzZsycORNBQUG4/vrrrRmViOgcbkoF7hgZCwD4aEeR2DA90Nhm6Dw9et6EeJ4eTU7HquXl6aefRnp6Op555hk0NTUhPT0d6enpXebEHDt2DFqtFgCgVCqRl5eHWbNmITExEXfddRcSExORnZ0NX19fa0YlIjqv20ZGw00pYX9xAw6WNoiO0y1f7C5GY5sR8cHe3JSOnJIkO/oOTP9Dp9NBo9FAq9XCz4/ndxDR5Zv/5X6syi3HtYMj8Ppt6aLjXFC70YxxL21ElU6Pl24chFuGcwUmOYaefH47zuwzIiJB7hvXDwDw7cFyFNe1CE5zYV/vLUGVTo9QPzVmpUeIjkNkFSwvREQXkRapwfjEYJhl4L1tp0TH+VXtRjP+vbEAgGWui1qlFJyIyDpYXoiIuuGBCZblxl/llKCm8fybZor2VU4JyrVtCPFVd56OTeSMWF6IiLphVL8ADIn2h95oxkc7CkXHOYfeaMJbmyx3XR6cGA8PN951IefF8kJE1A2SJGFex92XT7JPo7HNIDhRV1/llKJc24ZQPzVu5V0XcnIsL0RE3ZQ5IBTxwd5obDPig5+KRMfp1GYwdc51eXBif951IafH8kJE1E0KhYT5VyUCAP6z7RQaWtoFJ7J4/6dCVOraEOnvidlcGk0ugOWFiKgHZgwMR3KYLxr1Rry7VfzKo7omPd7efBIAsHBqIu+6kEtgeSEi6gGFQsIfMpMAAB9tLxK+8uiNjQVo0huRGuGHWYMd5+wlosvB8kJE1ENXpYRgcLQ/Wg0mvPbDcWE5TtY0dZ5h9OfpKVAoeIYRuQaWFyKiHpIkCX+elgzAco7QkXKdzTPIsoynvzkEo1nGpOQQjO0fZPMMRKKwvBARXYKR/QIxY1A4zDLw/LeHYetj4tYcrMD2gjqoVQo8OzPVpu9NJBrLCxHRJVo0LRlqlQI7T9Vj/aFKm71vY5sBf/32CADgoSv7IybQy2bvTWQPWF6IiC5RVB8v/K5j47rn1hyGttU2G9e9uO4oqhv1iAvyxm/H97PJexLZE5YXIqLL8ODEePQL8kaVTt95N8SaNh2txhe7iwEAL1yfxqXR5JJYXoiILoOHmxIv3TQIkgR8vbcUm45VW+296pvb8cflBwEA946Nw5h4TtIl18TyQkR0mYb1DcC9Y+MAAE8sP4i6pt7f+8VslvGn5QdR06hH/xAf/PHqpF5/DyJHwfJCRNQLFmYmoV+wZfjokS/3w2Tu3dVHb20uQNaRKrgrFfjnLUM4XEQujeWFiKgXeLor8c6cDHi6KbG9oA4vbzjWa9/7hyNVeCXLshneX65LxcAoTa99byJHxPJCRNRLEkN98bcbBwIA3t58Eh/vKLrs77nrVB0eWroPsgzcPjIGs4fHXPb3JHJ0LC9ERL1o1pBIPNZx8vSzaw5j+d7SS/5eOUX1uO/jHOiNZlyVEoLnruVmdEQAywsRUa97ZHJ/3Dk6FrIM/OHrA3hny8ke78D73aFK3PGfXWjUGzEiLgBv3j4Ubkr+J5sIYHkhIup1kiTh2ZmpuH+cZQXS39YfxSNf5kLbcvFN7NoMJry4Lh/zPtvbccclFB/fM4ITdIl+QSU6ABGRM1IoJDw5YwDCNZ54YV0+1hwox/aCWjw4MR43D4uGxtOty/Wt7SasOVCOf/14AmUNrQCAu8f0xVMzUqDiHReiLiTZ1qeJWZlOp4NGo4FWq4Wfn5/oOEREyC1pwB++ysXJmmYAgLtKgSFR/ugb5AWFJKH0TCv2nj6DVoMJABCu8cBz16YiMzVMZGwim+rJ5zfLCxGRDRhNZnyVU4qPdxThWFXjea+JDvDEnJGxuHN0X3i6c5iIXAvLC8sLEdkpWZZxqrYZ+06fQXWjHkaTjBA/NdJj/JEY4guFQhIdkUiInnx+c84LEZENSZKE+GAfxAf7iI5C5LA4C4yIiIgcCssLERERORSWFyIiInIoLC9ERETkUFheiIiIyKGwvBAREZFDYXkhIiIih8LyQkRERA6F5YWIiIgcCssLERERORSWFyIiInIoLC9ERETkUFheiIiIyKE43anSsiwDsBytTURERI7h7Of22c/xC3G68tLY2AgAiI6OFpyEiIiIeqqxsREajeaC10hydyqOAzGbzSgvL4evry8kSerV763T6RAdHY2SkhL4+fn16vemn/HnbDv8WdsGf862wZ+zbVjr5yzLMhobGxEREQGF4sKzWpzuzotCoUBUVJRV38PPz4//x7AB/pxthz9r2+DP2Tb4c7YNa/ycL3bH5SxO2CUiIiKHwvJCREREDoXlpQfUajWeeeYZqNVq0VGcGn/OtsOftW3w52wb/Dnbhj38nJ1uwi4RERE5N955ISIiIofC8kJEREQOheWFiIiIHArLCxERETkUlpdueuuttxAXFwcPDw9kZGRg27ZtoiM5ncWLF2P48OHw9fVFSEgIrrvuOhw7dkx0LKe3ePFiSJKE+fPni47idMrKyjBnzhwEBgbCy8sLQ4YMwd69e0XHcjpGoxFPPfUU4uLi4OnpiX79+uH555+H2WwWHc2hbd26FTNnzkRERAQkScKqVau6fF2WZTz77LOIiIiAp6cnJk6ciMOHD9skG8tLNyxbtgzz58/Hk08+if3792PcuHGYNm0aiouLRUdzKlu2bMFDDz2EnTt3IisrC0ajEZmZmWhubhYdzWnt2bMHS5YswaBBg0RHcTpnzpzB2LFj4ebmhvXr1+PIkSN45ZVX4O/vLzqa0/n73/+Od955B2+++Sby8/Px0ksv4R//+AfeeOMN0dEcWnNzMwYPHow333zzvF9/6aWX8Oqrr+LNN9/Enj17EBYWhilTpnSeMWhVMl3UiBEj5Hnz5nV5Ljk5WX7iiScEJXIN1dXVMgB5y5YtoqM4pcbGRjkhIUHOysqSJ0yYID/66KOiIzmVP/3pT/IVV1whOoZLmDFjhnzvvfd2ee6GG26Q58yZIyiR8wEgr1y5svPPZrNZDgsLk//2t791PtfW1iZrNBr5nXfesXoe3nm5iPb2duzduxeZmZldns/MzMSOHTsEpXINWq0WABAQECA4iXN66KGHMGPGDFx11VWiozil1atXY9iwYbj55psREhKC9PR0vPfee6JjOaUrrrgCP/74I44fPw4AOHDgAH766SdMnz5dcDLnVVhYiMrKyi6fjWq1GhMmTLDJZ6PTHczY22pra2EymRAaGtrl+dDQUFRWVgpK5fxkWcaCBQtwxRVXIC0tTXQcp/Pll19i37592LNnj+goTuvUqVN4++23sWDBAvz5z3/G7t278cgjj0CtVuPOO+8UHc+p/OlPf4JWq0VycjKUSiVMJhNeeOEF3HbbbaKjOa2zn3/n+2w8ffq01d+f5aWbJEnq8mdZls95jnrPww8/jIMHD+Knn34SHcXplJSU4NFHH8WGDRvg4eEhOo7TMpvNGDZsGF588UUAQHp6Og4fPoy3336b5aWXLVu2DJ999hmWLl2K1NRU5ObmYv78+YiIiMBdd90lOp5TE/XZyPJyEUFBQVAqlefcZamurj6ncVLv+P3vf4/Vq1dj69atiIqKEh3H6ezduxfV1dXIyMjofM5kMmHr1q148803odfroVQqBSZ0DuHh4RgwYECX51JSUrB8+XJBiZzX448/jieeeAK33norAGDgwIE4ffo0Fi9ezPJiJWFhYQAsd2DCw8M7n7fVZyPnvFyEu7s7MjIykJWV1eX5rKwsjBkzRlAq5yTLMh5++GGsWLECGzduRFxcnOhITmny5MnIy8tDbm5u52PYsGG44447kJuby+LSS8aOHXvOUv/jx48jNjZWUCLn1dLSAoWi68eZUqnkUmkriouLQ1hYWJfPxvb2dmzZssUmn42889INCxYswNy5czFs2DCMHj0aS5YsQXFxMebNmyc6mlN56KGHsHTpUnzzzTfw9fXtvNul0Wjg6ekpOJ3z8PX1PWcekbe3NwIDAzm/qBc99thjGDNmDF588UXccsst2L17N5YsWYIlS5aIjuZ0Zs6ciRdeeAExMTFITU3F/v378eqrr+Lee+8VHc2hNTU1oaCgoPPPhYWFyM3NRUBAAGJiYjB//ny8+OKLSEhIQEJCAl588UV4eXnh9ttvt344q69nchL//ve/5djYWNnd3V0eOnQol+9aAYDzPj788EPR0Zwel0pbx5o1a+S0tDRZrVbLycnJ8pIlS0RHcko6nU5+9NFH5ZiYGNnDw0Pu16+f/OSTT8p6vV50NIe2adOm8/43+a677pJl2bJc+plnnpHDwsJktVotjx8/Xs7Ly7NJNkmWZdn6FYmIiIiod3DOCxERETkUlhciIiJyKCwvRERE5FBYXoiIiMihsLwQERGRQ2F5ISIiIofC8kJEREQOheWFiIiIHArLCxERETkUlhciIiJyKCwvRERE5FBYXoiIiMih/D/oNOCYHZSxFAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "result_np = result.detach().numpy() # Convert to numpy array\n", + "\n", + "# 2D plot of X and Z\n", + "plt.plot(result_np[:,2], result_np[:,0])\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ObTghBli7-no" + }, + "source": [ + "# Euler's method for training" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "87VAZZtR7-n1" + }, + "outputs": [], + "source": [ + "def x_prime_prime_train(prev_val, coeffs, dnns):\n", + " return dnns[\"f\"](torch.Tensor([prev_val[\"t\"]]))[0] - coeffs[\"d\"]*prev_val[\"x_\"] - coeffs[\"a\"]*prev_val[\"x\"] - coeffs[\"b\"]*prev_val[\"x\"]*prev_val[\"x\"]*prev_val[\"x\"]\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "dnns = {\"f\": nn.Sequential(\n", + " nn.Linear(1,10),\n", + " nn.ReLU(),\n", + " nn.Linear(10,1),\n", + " nn.Tanh()\n", + " )}" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "id": "dTx1Q0PV7-n2" + }, + "outputs": [], + "source": [ + "ode_train_coeffs = {\"a\": 0., \"b\": 0., \"d\": 0.}\n", + "ode_train = {\"x\": x_prime, \"x_\": x_prime_prime_train, \"t\": t_prime}\n", + "\n", + "ode_solver_train = HybridODENet(\n", + " ode=ode_train,\n", + " dnns=dnns,\n", + " init_vars=ode_init,\n", + " init_coeffs=ode_train_coeffs,\n", + " dt=dt,\n", + " solver=\"euler\",\n", + " optimizer=torch.optim.Adam,\n", + " optimizer_args={\"lr\": 0.5},\n", + " scheduler=torch.optim.lr_scheduler.MultiStepLR,\n", + " scheduler_args={\"milestones\": [5],\"gamma\": 0.2}\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "ode_solver_train.register_parameter(\"f0\", dnns[\"f\"][0].weight)\n", + "ode_solver_train.register_parameter(\"f2\", dnns[\"f\"][2].weight)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Parameter containing:\n", + "tensor(0., requires_grad=True)\n", + "Parameter containing:\n", + "tensor(0., requires_grad=True)\n", + "Parameter containing:\n", + "tensor(0., requires_grad=True)\n", + "Parameter containing:\n", + "tensor([[ 0.2142],\n", + " [-0.8496],\n", + " [ 0.5126],\n", + " [-0.8355],\n", + " [-0.4257],\n", + " [-0.9374],\n", + " [ 0.4995],\n", + " [-0.3751],\n", + " [-0.7042],\n", + " [-0.2644]], requires_grad=True)\n", + "Parameter containing:\n", + "tensor([[ 0.3312, 0.3463, 0.0904, 0.1946, -0.0459, -0.0237, 0.1171, 0.2378,\n", + " -0.0938, -0.1383]], requires_grad=True)\n" + ] + } + ], + "source": [ + "for param in ode_solver_train.parameters():\n", + " print(param)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "67eN5ZyB7-n2", + "outputId": "1588284b-d8f3-42ed-ff4e-0428b39394e8" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GPU available: False, used: False\n", + "TPU available: False, using: 0 TPU cores\n", + "IPU available: False, using: 0 IPUs\n", + "\n", + " | Name | Type | Params\n", + "------------------------------\n", + "------------------------------\n", + "23 Trainable params\n", + "0 Non-trainable params\n", + "23 Total params\n", + "0.000 Total estimated model params size (MB)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 0: 0%| | 0/1 [00:00\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m ode_solver_train.fit(\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mmax_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m )\n", + "\u001b[0;32m~/Documents/GitHub/torchTS/examples/ode/DuffingEquation/../../../torchts/nn/model.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, max_epochs, batch_size)\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[0mloader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDataLoader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[0mtrainer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrainer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_epochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 59\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mprepare_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model, train_dataloaders, val_dataloaders, datamodule, train_dataloader, ckpt_path)\u001b[0m\n\u001b[1;32m 735\u001b[0m )\n\u001b[1;32m 736\u001b[0m \u001b[0mtrain_dataloaders\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_dataloader\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 737\u001b[0;31m self._call_and_handle_interrupt(\n\u001b[0m\u001b[1;32m 738\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit_impl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_dataloaders\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_dataloaders\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdatamodule\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mckpt_path\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 739\u001b[0m )\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36m_call_and_handle_interrupt\u001b[0;34m(self, trainer_fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m 680\u001b[0m \"\"\"\n\u001b[1;32m 681\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 682\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mtrainer_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 683\u001b[0m \u001b[0;31m# TODO: treat KeyboardInterrupt as BaseException (delete the code below) in v1.7\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 684\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36m_fit_impl\u001b[0;34m(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)\u001b[0m\n\u001b[1;32m 770\u001b[0m \u001b[0;31m# TODO: ckpt_path only in v1.7\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 771\u001b[0m \u001b[0mckpt_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mckpt_path\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresume_from_checkpoint\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 772\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mckpt_path\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mckpt_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 773\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 774\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstopped\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, model, ckpt_path)\u001b[0m\n\u001b[1;32m 1192\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1193\u001b[0m \u001b[0;31m# dispatch `start_training` or `start_evaluating` or `start_predicting`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1194\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1196\u001b[0m \u001b[0;31m# plugin will finalized fitting (e.g. ddp_spawn will load trained model)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36m_dispatch\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1272\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_type_plugin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_predicting\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1273\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1274\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_type_plugin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_training\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1275\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1276\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mrun_stage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py\u001b[0m in \u001b[0;36mstart_training\u001b[0;34m(self, trainer)\u001b[0m\n\u001b[1;32m 200\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mstart_training\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"pl.Trainer\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 201\u001b[0m \u001b[0;31m# double dispatch to initiate the training loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 202\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_results\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_stage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 203\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 204\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mstart_evaluating\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"pl.Trainer\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mrun_stage\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1282\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredicting\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1283\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1284\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1285\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1286\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_pre_training_routine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36m_run_train\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1312\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1313\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_detect_anomaly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_detect_anomaly\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1314\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1315\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1316\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_run_evaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0m_EVALUATE_OUTPUT\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/loops/base.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 144\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_advance_start\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madvance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 146\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_advance_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestarting\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/loops/fit_loop.py\u001b[0m in \u001b[0;36madvance\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 232\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 233\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"run_training_epoch\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_fetcher\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 236\u001b[0m \u001b[0;31m# the global step is manually decreased here due to backwards compatibility with existing loggers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/loops/base.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 144\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_advance_start\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madvance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 146\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_advance_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestarting\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py\u001b[0m in \u001b[0;36madvance\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 191\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 192\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"run_training_batch\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 193\u001b[0;31m \u001b[0mbatch_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 194\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_progress\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mincrement_processed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/loops/base.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 144\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_advance_start\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madvance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 146\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_advance_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestarting\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/loops/batch/training_batch_loop.py\u001b[0m in \u001b[0;36madvance\u001b[0;34m(self, batch, batch_idx)\u001b[0m\n\u001b[1;32m 86\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlightning_module\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautomatic_optimization\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 87\u001b[0m \u001b[0moptimizers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_get_active_optimizers\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer_frequencies\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msplit_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 89\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmanual_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msplit_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/loops/base.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 144\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_advance_start\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madvance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 146\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_advance_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrestarting\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py\u001b[0m in \u001b[0;36madvance\u001b[0;34m(self, batch, *args, **kwargs)\u001b[0m\n\u001b[1;32m 213\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 214\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0madvance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# type: ignore[override]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 215\u001b[0;31m result = self._run_optimization(\n\u001b[0m\u001b[1;32m 216\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 217\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_idx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py\u001b[0m in \u001b[0;36m_run_optimization\u001b[0;34m(self, split_batch, batch_idx, optimizer, opt_idx)\u001b[0m\n\u001b[1;32m 264\u001b[0m \u001b[0;31m# gradient update with accumulated gradients\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 265\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 266\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimizer_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 267\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 268\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconsume_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py\u001b[0m in \u001b[0;36m_optimizer_step\u001b[0;34m(self, optimizer, opt_idx, batch_idx, train_step_and_backward_closure)\u001b[0m\n\u001b[1;32m 376\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 377\u001b[0m \u001b[0;31m# model hook\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 378\u001b[0;31m lightning_module.optimizer_step(\n\u001b[0m\u001b[1;32m 379\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcurrent_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 380\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/core/lightning.py\u001b[0m in \u001b[0;36moptimizer_step\u001b[0;34m(self, epoch, batch_idx, optimizer, optimizer_idx, optimizer_closure, on_tpu, using_native_amp, using_lbfgs)\u001b[0m\n\u001b[1;32m 1650\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1651\u001b[0m \"\"\"\n\u001b[0;32m-> 1652\u001b[0;31m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclosure\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptimizer_closure\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1653\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1654\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0moptimizer_zero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer_idx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/core/optimizer.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure, **kwargs)\u001b[0m\n\u001b[1;32m 162\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mtrainer\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprofiler_action\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 164\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccelerator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimizer_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/accelerators/accelerator.py\u001b[0m in \u001b[0;36moptimizer_step\u001b[0;34m(self, optimizer, opt_idx, closure, model, **kwargs)\u001b[0m\n\u001b[1;32m 334\u001b[0m \"\"\"\n\u001b[1;32m 335\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlightning_module\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 336\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprecision_plugin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 337\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 338\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0moptimizer_zero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcurrent_epoch\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_idx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py\u001b[0m in \u001b[0;36moptimizer_step\u001b[0;34m(self, model, optimizer, optimizer_idx, closure, **kwargs)\u001b[0m\n\u001b[1;32m 161\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLightningModule\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 162\u001b[0m \u001b[0mclosure\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpartial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wrap_closure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 163\u001b[0;31m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclosure\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclosure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 164\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 165\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_track_grad_norm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"pl.Trainer\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/torch/optim/lr_scheduler.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 65\u001b[0m \u001b[0minstance\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_step_count\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 66\u001b[0m \u001b[0mwrapped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minstance\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 67\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mwrapped\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 69\u001b[0m \u001b[0;31m# Note that the returned function here is no longer a bound method,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 27\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m 64\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mclosure\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 65\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 66\u001b[0;31m \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 67\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 68\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mgroup\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparam_groups\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py\u001b[0m in \u001b[0;36m_wrap_closure\u001b[0;34m(self, model, optimizer, optimizer_idx, closure)\u001b[0m\n\u001b[1;32m 146\u001b[0m \u001b[0mconsistent\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mthe\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mPrecisionPlugin\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0msubclasses\u001b[0m \u001b[0mthat\u001b[0m \u001b[0mcannot\u001b[0m \u001b[0;32mpass\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclosure\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mdirectly\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 147\u001b[0m \"\"\"\n\u001b[0;32m--> 148\u001b[0;31m \u001b[0mclosure_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 149\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_after_closure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 150\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mclosure_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 158\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 159\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 160\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclosure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 161\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_result\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 162\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py\u001b[0m in \u001b[0;36mclosure\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 153\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backward_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mstep_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclosure_loss\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 154\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_profiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"backward\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 155\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backward_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclosure_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 156\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 157\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mstep_output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/loops/optimization/optimizer_loop.py\u001b[0m in \u001b[0;36mbackward_fn\u001b[0;34m(loss)\u001b[0m\n\u001b[1;32m 325\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 326\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mbackward_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 327\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccelerator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 328\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 329\u001b[0m \u001b[0;31m# check if model weights are nan\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/accelerators/accelerator.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, closure_loss, *args, **kwargs)\u001b[0m\n\u001b[1;32m 309\u001b[0m \u001b[0mclosure_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprecision_plugin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpre_backward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlightning_module\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 310\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 311\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprecision_plugin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlightning_module\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 312\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 313\u001b[0m \u001b[0mclosure_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprecision_plugin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpost_backward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlightning_module\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/plugins/precision/precision_plugin.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, model, closure_loss, optimizer, *args, **kwargs)\u001b[0m\n\u001b[1;32m 89\u001b[0m \u001b[0;31m# do backward pass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLightningModule\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclosure_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 92\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_backward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclosure_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/pytorch_lightning/core/lightning.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, loss, optimizer, optimizer_idx, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1432\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1433\u001b[0m \"\"\"\n\u001b[0;32m-> 1434\u001b[0;31m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1435\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1436\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mtoggle_optimizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mOptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLightningOptimizer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer_idx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 220\u001b[0m create_graph=create_graph)\n\u001b[0;32m--> 221\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 222\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 223\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/opt/anaconda3/envs/torchTS/lib/python3.9/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m 128\u001b[0m \u001b[0mretain_graph\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 130\u001b[0;31m Variable._execution_engine.run_backward(\n\u001b[0m\u001b[1;32m 131\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 132\u001b[0m allow_unreachable=True) # allow_unreachable flag\n", + "\u001b[0;31mRuntimeError\u001b[0m: Trying to backward through the graph a second time, but the saved intermediate results have already been freed. Specify retain_graph=True when calling backward the first time." + ] + } + ], + "source": [ + "ode_solver_train.fit(\n", + " result,result,\n", + " max_epochs=1,\n", + " batch_size=result.shape[0]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 266 + }, + "id": "IOMjwKY27-n2", + "outputId": "1e8218d2-8200-4b08-9ae5-ec7152cfe18d" + }, + "outputs": [], + "source": [ + "result_train = ode_solver_train(1000)\n", + "\n", + "result_np = result.detach().numpy() # Convert to numpy array\n", + "\n", + "# 2D plot of X and Z\n", + "plt.plot(result_np[:,2], result_np[:,0])\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9adPvysOrfMr" + }, + "source": [ + "# Mean Squared Error Loss" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "aVYankyRrwoW", + "outputId": "bc16f6f1-c234-44cb-c9e6-ce0b4f5e8f45" + }, + "outputs": [], + "source": [ + "loss = torch.nn.MSELoss()\n", + "loss(result_train,result)" + ] + } + ], + "metadata": { + "colab": { + "name": "DuffingTest_DNN.ipynb", + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/examples/ode/DuffingEquation/DuffingTest_fitRandomSample_with_w_RK4.ipynb b/examples/ode/DuffingEquation/DuffingTest_fitRandomSample_with_w_RK4.ipynb new file mode 100644 index 00000000..54e4a9e1 --- /dev/null +++ b/examples/ode/DuffingEquation/DuffingTest_fitRandomSample_with_w_RK4.ipynb @@ -0,0 +1,467 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.chdir(\"../../../\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "from torchts.nn.models.ode import ODESolver\n", + "import numpy as np\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import mpl_toolkits.mplot3d.axes3d as p3" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Duffing equation: Second order ODE system\n", + "dt = 0.01\n", + "\n", + "def x_prime(prev_val, coeffs):\n", + " return prev_val[\"x_\"]\n", + "\n", + "def x_prime_prime(prev_val, coeffs):\n", + " return coeffs[\"g\"]*torch.cos(coeffs[\"w\"]*prev_val[\"t\"]) - coeffs[\"d\"]*prev_val[\"x_\"] - coeffs[\"a\"]*prev_val[\"x\"] - coeffs[\"b\"]*prev_val[\"x\"]*prev_val[\"x\"]*prev_val[\"x\"]\n", + "\n", + "def t_prime(prev_val, coeffs):\n", + " return 1\n", + "\n", + "ode = {\"x\": x_prime, \"x_\": x_prime_prime, \"t\": t_prime}\n", + "\n", + "# Initial conditions [0,0,0]\n", + "ode_init = {\"x\": 0, \"x_\": 0, \"t\": 0}\n", + "\n", + "# Constants (Parameters)\n", + "ode_coeffs = {\"a\": 0.1, \"b\": 0.5, \"d\": 0.2, \"g\": 0.8, \"w\": 0.5}\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 4th Order Runge-Kutta - Data Generation for nt = 1000" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "ode_solver = ODESolver(\n", + " ode=ode,\n", + " init_vars=ode_init,\n", + " init_coeffs=ode_coeffs,\n", + " dt=dt,\n", + " solver=\"rk4\",\n", + " optimizer=None\n", + ")\n", + "\n", + "result = ode_solver(1000)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[0.0000e+00, 0.0000e+00, 0.0000e+00],\n", + " [3.9973e-05, 7.9920e-03, 1.0000e-02],\n", + " [1.5979e-04, 1.5968e-02, 2.0000e-02],\n", + " ...,\n", + " [1.3451e-01, 1.5991e-01, 9.9701e+00],\n", + " [1.3612e-01, 1.6162e-01, 9.9801e+00],\n", + " [1.3774e-01, 1.6335e-01, 9.9901e+00]], grad_fn=)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAGdCAYAAADaPpOnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAA9hAAAPYQGoP6dpAABNI0lEQVR4nO3deVzUZeIH8M93BhgOYZD7vhREwQPxvtPEKyozsyyttty1sjKqLWt/Xbub27lumbW23WqnmpZaUp4lHqh4oAIqct/qDPfAzPf3xyBFooIy88zxeb9e84fDDPNxIubj8zzf55FkWZZBREREZCUUogMQERERdQbLCxEREVkVlhciIiKyKiwvREREZFVYXoiIiMiqsLwQERGRVWF5ISIiIqvC8kJERERWxUF0gK5mMBhQXFwMd3d3SJIkOg4RERF1gCzLqK6uRlBQEBSKy4+t2Fx5KS4uRmhoqOgYREREdBUKCgoQEhJy2cfYXHlxd3cHYPzLe3h4CE5DREREHaHVahEaGtr6OX45NldeLkwVeXh4sLwQERFZmY4s+eCCXSIiIrIqLC9ERERkVVheiIiIyKqYtLzs2LEDycnJCAoKgiRJ+Pbbby/7+G3btkGSpItuJ06cMGVMIiIisiImXbBbW1uL/v37495778WMGTM6/LysrKw2i219fX1NEY+IiIiskEnLy5QpUzBlypROP8/Pzw+enp5dH4iIiIisnkWueUlISEBgYCAmTJiArVu3XvaxjY2N0Gq1bW5ERERkuyyqvAQGBmL58uVYvXo11qxZg169emHChAnYsWPHJZ+zePFiqNXq1ht31yUiIrJtkizLslleSJKwdu1a3HzzzZ16XnJyMiRJwvr169v9emNjIxobG1v/fGGHPo1Gw03qiIiIrIRWq4Vare7Q57dFjby0Z9iwYcjJybnk11UqVetuutxVl4iIyPZZfHk5ePAgAgMDRccgIiIiC2HSq41qampw8uTJ1j/n5uYiIyMDXl5eCAsLw6JFi1BUVIRPP/0UALBkyRJEREQgLi4OOp0OK1aswOrVq7F69WpTxiQiIiIrYtLykp6ejuuuu671zykpKQCAu+++Gx9//DFKSkqQn5/f+nWdTocnnngCRUVFcHFxQVxcHDZs2ICpU6eaMibZKYNBxunKWhwuPI8STQPO1+ngqFTAw8URPX27IT5YjQC1s+iYRET0B2ZbsGsunVnwQ/bpRKkWX+4rwIbDJSivbrzsY+OCPDCtXyDuGByG7m5OZkpIRGR/OvP5zfJCdiOzWIM3Nmdjy4ny1vucHRXoG6xGhLcburs5oUlvwNlaHbJKq5FdVg2D/NvjZg8JxyMTesLTlSWGiKirdebz26TTRkSWoKaxGa/+cAIrdufBIAMKCZgUF4AZA0MwJsYXTg7tr1uvqmlE6rEyfLY7D5nFWnz4ay7WHCzEM1N7Y2ZiCCRJMvPfhIiIAI68kI07UqjBw58fwJmqOgDADf0C8XhSL0T6uHX4e8iyjJ05lfjHhmPILqsBACT18cert/bjKAwRURfhtBHLCwFYuScPL6zPRJNeRrCnC169tR9G9vS56u/XrDfg/Z25eDM1C016GRHervjf3YPR069bF6YmIrJPNrVJHVFnGQwyXvnhBJ5dexRNehlJffyx4ZFR11RcAMBBqcAD43pg7YMjEezpgjNVdZi+7FfsOlnZRcmJiKgjWF7IpugNMp745hDe3XYKAPDY9TH475zELp3eiQ9WY92CkUgM747qhmbc8/E+bMsqv/ITiYioS7C8kM0wGGQ8+c0hrDlQBKVCwmu39sOj10ebZGGtTzcVVs0biol9/KFrNuDPn+7H1hMsMERE5sDyQjZBlmU8s/ZIa3FZekcCZg4y7QnjKgcl3pk9EJPi/KHTG/CXFfux78xZk74mERGxvJCNeDM1G1/sK4BCApbMGoApfc1zHpaTgwJLZw/E9b2NIzD3f5KOk+U1ZnltIiJ7xfJCVu/r9AK8vcV4hta/bumH5P5BZn19R6UCb9+RgAGhntDUN+HuD/eisubyO/cSEdHVY3khq7brZCUWrTkCAFhwXU/cNti0U0WX4uKkxAd3D0KEtyuKztfjoZUH0Kw3CMlCRGTrWF7IahWeq8ODqw6g2SAjuX8QUibGCM3j3U2F/909CG5OSuzJPYvFm04IzUNEZKtYXsgqNTbr8dCqgzhf14T+IWq8dms/KBTit+vv6eeON24bAAD44JdcfH+4WGwgIiIbxPJCVunlDcdxqOA81C6OeOfOgXB2VIqO1GpyfAAeGNcDALBozREUna8XnIiIyLawvJDV2XC4BJ+k5QEA/j2rP0K6uwpOdLHHJ8YgIcwT1Q3NeOzLDOgNNnUKBxGRUCwvZFVKNPVYtOYwAOCBcT0wPtZfcKL2OSgVWDJrANyclNibexbvbT8lOhIRkc1geSGrIcsy/vrNYWgbmtEvRC18ge6VhHu74cWb4gEA/07NxtEijeBERES2geWFrMZnu/OwM6cSKgcF3rxtAByVlv/jO2NgMKbEB6DZIOOp1Yd5+TQRURew/N/+RABOVdTg5Y3HAQBPT4lFT79ughN1jCRJeOmmeKhdHJFZrMX7O3NFRyIisnosL2TxDAYZT359CA1NBozs6Y27h0eIjtQpvu4q/N8NfQAAS37KRm5lreBERETWjeWFLN7Kvfk4kH8ebk5KvHprf4vYz6WzZgwMxuhoHzQ2G/D06sOQZV59RER0tVheyKKVaRvwastOtU9O6oVgTxfBia6OJEl4eXpfODsqsCf3LNYf4uZ1RERXi+WFLNqL32WiurEZ/UM9McfKpov+KNTLFQuu6wkAeHnjcdQ0NgtORERknVheyGL9fLwMG4+UQqmQsHh6XyitcLroj+4fHYVwb1eUaRvx9pYc0XGIiKwSywtZpNrGZjy3LhMAcP+oSPQJ8hCcqGs4OyrxXMvi3Q9/ycWpihrBiYiIrA/LC1mkt7ecRNH5eoR0d8Gj10eLjtOlJvT2x/hYPzTpZbywPpOLd4mIOonlhSzOmcpafPiLcT+UF5Lj4OrkIDhR13vuhj5wUiqwM6cS27IrRMchIrIqLC9kcf6x4Rh0egPGxPhiQm8/0XFMIsLHDfeMjAAA/GvjCR7cSETUCSwvZFG2Z1fgp+PlcFBIeO6G3pAk61+keykPjesJtYsjssqqsXp/oeg4RERWg+WFLEaT3oC/f38MADB3eAR6+rkLTmRaalfH1kun30jNQr1OLzgREZF1YHkhi/FZWh5OltfA283J5hbpXsrcEeEI6e6CMm0jPvyV5x4REXUEywtZhKqaRvz7p2wAwBOTekHt4ig4kXmoHJR4clIvAMC7206hqqZRcCIiIsvH8kIW4T8/56C6oRlxQR64bVCo6DhmldwvCH2D1ahpbMZ720+JjkNEZPFYXki40xU1WLUnHwDw7LTeNrGTbmcoFBJSkmIAAJ+m5aFc2yA4ERGRZWN5IeFe+zELzQYZ42P9MKKHj+g4QoyL8cXAME80NhuwbBtHX4iILoflhYQ6kH8Om46WQiEBT02OFR1HGEmS8HiSce3Lqj35KDpfLzgREZHlYnkhYWRZxuKNxwEAtyaGoFeAbV8afSUjenhjWJQXdHoDlm45KToOEZHFYnkhYX46Xo59Z85B5aDAYxNjRMcR7vejL1+nFyC/qk5wIiIiy8TyQkI06w341ybjqMt9oyIRqHYRnMgyDI7wwuhoHzQbZLy1JUd0HCIii8TyQkJ8vb8Qpypq0d3VEfPH9RAdx6KktIxCrT1YxNEXIqJ2sLyQ2dXr9Ph3qnFDuofHR8PD2T42pOuohLDuGB3tA71Bxns7eOUREdEfmbS87NixA8nJyQgKCoIkSfj222+v+Jzt27cjMTERzs7OiIqKwnvvvWfKiCTAp2lnUF7diJDuLrhzWJjoOBbp4fHG4xG+SS9EqYb7vhAR/Z5Jy0ttbS369++PpUuXdujxubm5mDp1KkaPHo2DBw/imWeewSOPPILVq1ebMiaZUXVDE95t2UX20QnRUDkoBSeyTEMivTAkwnjl0fIdp0XHISKyKA6m/OZTpkzBlClTOvz49957D2FhYViyZAkAoHfv3khPT8frr7+OGTNmmCglmdMHv+TifF0TonzdMD0hWHQci7ZgfE/M/XAvVu3Nw0PX9YB3N5XoSEREFsGi1rykpaUhKSmpzX2TJk1Ceno6mpqa2n1OY2MjtFptmxtZpnO1Onyw03hy8mPXx8BBaVE/fhZndLQP+oWo0dBk4InTRES/Y1GfHqWlpfD3929zn7+/P5qbm1FZWdnucxYvXgy1Wt16Cw21r0P9rMl/d5xGdWMzegd6YFrfQNFxLJ4kSVhwXU8AwKe78qCpb7/AExHZG4sqL4DxF/bvybLc7v0XLFq0CBqNpvVWUFBg8ozUeeXVDfh4l3H04PGJMVDY2eGLV+v63v7o5e+O6sZmfLrrjOg4REQWwaLKS0BAAEpLS9vcV15eDgcHB3h7e7f7HJVKBQ8PjzY3sjzLtp5CQ5MBA0I9MaG3n+g4VkOhkPDQeOPoy4e/5qJepxeciIhIPIsqL8OHD0dqamqb+zZv3oxBgwbB0ZF7gVirovP1WLUnHwDwRFKvS46iUfum9Q1EmJcrztU14ev9HFkkIjJpeampqUFGRgYyMjIAGC+FzsjIQH6+8YNs0aJFmDt3buvj58+fj7y8PKSkpOD48eP48MMP8cEHH+CJJ54wZUwysbd/zoFOb8CwKC+M7Nn+CBpdmlIh4f7RkQCA/+3Mhd4gC05ERCSWSctLeno6EhISkJCQAABISUlBQkICnnvuOQBASUlJa5EBgMjISGzcuBHbtm3DgAED8Pe//x1vvfUWL5O2YrmVtfh6fyEA4MlJHHW5WjMTQ9Hd1RH5Z+vwY2bplZ9ARGTDJPnCilgbodVqoVarodFouP7FAjz6xUGsyyjGdb188dG9Q0THsWpvpmbjrZ9z0D/UE98+OIJFkIhsSmc+vy1qzQvZlqzSaqw/VAwAeDypl+A01u/u4eFQOShwqOA89uaeFR2HiEgYlhcymTc2Z0GWgSnxAYgPVouOY/W8u6lwa2IIAPDIACKyaywvZBKHC89j87EySBKQMjFGdBybcf/oKEgS8POJcuSUVYuOQ0QkBMsLmcS/U7MBADcPCEa0v7vgNLYj0scNk/oEAADe38nRFyKyTywv1OX2553D1qwKKBUSHp0QLTqOzZk3JgoA8O3BYpRrGwSnISIyP5YX6nJLfjKOuswYGIwIHzfBaWxPYnh3DArvDp3egI95ZAAR2SGWF+pSe3PPYmdOJRwUEh4ez1EXU7l/tHH05fO9+Who4pEBRGRfWF6oS72ZmgUAuG1wKEK9XAWnsV0T+/gjpLsLztU14duDRaLjEBGZFcsLdZldJyux+/RZOCkVWHBdT9FxbJpSIeGeEREAjAc22thek0REl8XyQl1ClmW80XKF0R1DQhHk6SI4ke2bOSgUrk5KZJfVYNepKtFxiIjMhuWFusSOnErszzsHlYMCD3LUxSzULo6Y2bJp3Ye/5ApOQ0RkPiwvdM1kWcabm41rXe4aFg5/D2fBiezH3S1TR1uyypFbWSs2DBGRmbC80DXbcqIchwo1cHFUYv7YHqLj2JUo324YH+sHWQY+4WXTRGQnWF7omsiyjDdb1rrMHREOX3eV4ET2596REQCAr9MLoG1oEhuGiMgMWF7omvyYWYbMYi3cnJT4yxiOuogwqqcPov26oVanx1f7CkTHISIyOZYXumoGg9x6htG9IyPh5eYkOJF9kiQJ946MBAB8knYGegMvmyYi28byQldt49ESZJVVw13lgHktO76SGNMTguHp6oiCs/X46XiZ6DhERCbF8kJXRW+QseSnHADAfaMjoXZ1FJzIvrk4KXHHkDAAwEe/8rJpIrJtLC90Vb47VIyT5TVQuzjiT6MiRcchAHOHh0OpkLD79FlklVaLjkNEZDIsL9RpzXoD/vOzcdTlz2Oi4OHMURdLEKh2wcTe/gCAz3afERuGiMiEWF6o09YeLEJuZS283JxaN0kjyzB3eDgAYO2BIlTzsmkislEsL9QpTXoD3tpiHHX5y5godFM5CE5Evze8hzd6tlw2veYAT5smItvE8kKd8nV6IQrO1sOnmwpzh0eIjkN/IEkS5gwzjr58tjuPp00TkU1ieaEOa2jS462WtS4PjOsBFyel4ETUnlsGBsPNSYmT5TVI42nTRGSDWF6ow1bszkOptgFBamfcOTRMdBy6BHdnR0wfGAwA+DQtT3AaIqKux/JCHVLd0IR3tp4EADx6fTScHTnqYskuTOmlHi9DiaZebBgioi7G8kId8sEvuThX14QoHzfMGBgiOg5dQYy/O4ZGekFvkLFqT77oOEREXYrlha7obK0O/9tp3LU1JSkGDkr+2FiDC6Mvn+8tgK7ZIDYMEVEX4qcQXdG7206iprEZcUEemBofKDoOdVBSnD/8PVSorGnEpqMlouMQEXUZlhe6rBJNPT5pWfT5xKReUCgkwYmooxyVitbzjj7jwl0isiEsL3RZb/18ErpmA4ZEeGFcjK/oONRJs4eEwUEhIT3vHI4Va0XHISLqEiwvdEm5lbX4Kr0AAPDk5F6QJI66WBs/D2dMig8AwPOOiMh2sLzQJf07NRt6g4zrevlicISX6Dh0lea27Lj77cFiaOp53hERWT+WF2rXsWIt1h8qBgA8ntRLcBq6FkMivdDL3x31TXqsPVAoOg4R0TVjeaF2/euHEwCAG/oFIj5YLTgNXQtJkjC7ZUfkVXvzed4REVk9lhe6yM6cCuzIroCjUsKTkzjqYgumDwyGi6MS2WU12J93TnQcIqJrwvJCbRgMMhZvNI663Dk0HOHeboITUVfwcHZEcn/jHj0rueMuEVk5lhdqY+3BIhwr0cJd5YBHJkSLjkNdaPZQ48LdDUdKcK5WJzgNEdHVY3mhVg1NeryxOQsA8OB1PeHl5iQ4EXWl/iFqxAV5QNdswGou3CUiK8byQq0++vUMijUNCFI7496REaLjUBfjwl0ishUsLwTAePjisq0nARgvjXZ2VApORKZw04BguDkpcbqiFrtPnxUdh4joqpilvCxbtgyRkZFwdnZGYmIidu7cecnHbtu2DZIkXXQ7ceKEOaLarbe35KC6sRm9Az1wc0Kw6DhkIt1UDrip5b/vyj0874iIrJPJy8uXX36JhQsX4tlnn8XBgwcxevRoTJkyBfn5l7/iISsrCyUlJa236GguHjWVvKparNht/CB7ZmoslDx80abNbjms8cfMUlTWNApOQ0TUeSYvL2+++Sbuu+8+3H///ejduzeWLFmC0NBQvPvuu5d9np+fHwICAlpvSiWnMUzl5Y3H0aSXMTraB6OjefiirYsPVqN/iBpNehnf7OfCXSKyPiYtLzqdDvv370dSUlKb+5OSkrBr167LPjchIQGBgYGYMGECtm7desnHNTY2QqvVtrlRx/16shI/ZpZBqZDwt2l9RMchM7mz5bLpVXvyYTBw4S4RWReTlpfKykro9Xr4+/u3ud/f3x+lpaXtPicwMBDLly/H6tWrsWbNGvTq1QsTJkzAjh072n384sWLoVarW2+hoaFd/vewVc16A1767hgA4K6hYegV4C44EZnLDf0D4a5yQP7ZOvx6qlJ0HCKiTnEwx4tIUts1FLIsX3TfBb169UKvXr9tST98+HAUFBTg9ddfx5gxYy56/KJFi5CSktL6Z61WywLTQZ/vzUdWWTU8XR3x2MQY0XHIjFydHDB9YDA+TcvDqj35nC4kIqti0pEXHx8fKJXKi0ZZysvLLxqNuZxhw4YhJyen3a+pVCp4eHi0udGVna/T4c3UbABAysQYeLpyQzp7c2HPl83HylCubRCchoio40xaXpycnJCYmIjU1NQ296empmLEiBEd/j4HDx5EYGBgV8eza0t+ysG5uibE+HdrvfqE7EtsgAcSw7tDb5DxVXqB6DhERB1m8mmjlJQUzJkzB4MGDcLw4cOxfPly5OfnY/78+QCM0z5FRUX49NNPAQBLlixBREQE4uLioNPpsGLFCqxevRqrV682dVS7kVNWjc9aLo1+PjkODkruVWivZg8Jw/68c/h8bwEeGNeTl8kTkVUweXmZNWsWqqqq8NJLL6GkpATx8fHYuHEjwsONVzuUlJS02fNFp9PhiSeeQFFREVxcXBAXF4cNGzZg6tSppo5qF2RZxkvfH4PeICOpjz9G9vQRHYkEmtYvEC9+l4mi8/X49WQlxsRw7QsRWT5JtrEDTrRaLdRqNTQaDde/tGPD4RI8tOoAnBwUSH1sDMK93URHIsGeX3cUn6TlYVrfQLxz50DRcYjITnXm85vzBXakuqEJL32fCQB4YGwPFhcCAMwafGHhbimquOMuEVkBlhc78u/UHJRpGxHh7YoHxvUQHYcsRJ8gD/Rr2XF37cEi0XGIiK6I5cVOZBZr8PGuXADASzfF89RoamPWYOPeSF/sK4CNzSQTkQ1iebEDBoOMv317FAYZmNY3kIsy6SI39g+Ci6MSJ8trcCD/nOg4RESXxfJiB75ML8DB/PNwc1Li/27g+UV0MXdnR0zrZ9xL6Yu93POFiCwby4uNK69uwL82nQAApCT1QoDaWXAislS3t0wdfX+4BNUNTYLTEBFdGsuLjXt+XSY09U2IC/LA3cPDRcchC5YY3h09fN1Q36THd4dKRMchIroklhcbtulICTYdLYWDQsKrt/bjTrp0WZIk4faWy6a/3Jd/hUcTEYnDTzMbda5Wh/9b17Kny7geiAtSC05E1mD6wGA4KiUcKtTgWLFWdBwionaxvNiov39/DJU1jYj264YF43uKjkNWwqebChP7GE9852GNRGSpWF5s0NYT5VhzsAiSBLxyaz+oHLinC3XchR131xwoREOTXnAaIqKLsbzYGE19E55ZewQAcN/ISAwM6y44EVmbUT19EOzpAm1DM37MLBUdh4joIiwvNua5dUdRomlAhLcrHk/qJToOWSGlQsLMQSEAuOcLEVkmlhcbsi6jCOsyiqFUSHhz1gC4OHG6iK7OzEGhkCQg7XQVzlTWio5DRNQGy4uNKDpfj799exQAsOC6npwuomsS7OmCMdHGYyS4cJeILA3Liw0wGGQ8/lUGqhuaMSDUk1cXUZe4sOPu1/sL0aw3CE5DRPQblhcbsHznaew+fRYujkr8e9YAOHIzOuoCE3r7w9vNCRXVjdiaVSE6DhFRK37KWbn0M2fx2o9ZAIDnkvsg0sdNcCKyFU4OCsxINC7c5Y67RGRJWF6sWFVNIxasOgi9QUZy/6DWYX6irnJby1VHW7MqUF7dIDgNEZERy4uVMhhkPPbVIZRqGxDl44bFt/SFJEmiY5GN6ennjgGhntAbZHx7sEh0HCIiACwvVmvZtpPYkV0BlYMCy+4aiG4qB9GRyEZd2PPl6/RCyLIsOA0REcuLVdqZU4E3U7MBAH+/OR6xAR6CE5EtS+4fBJWDAjnlNThUqBEdh4iI5cXanK6owUMrD8AgAzMTQ3DbIK5zIdPycHbE5PgAAMDX3POFiCwAy4sV0TY04f5P06FtaMbAME/8Y3q86EhkJ2YmGkvy+kPFPKyRiIRjebESeoOMRz4/iNMVtQhUO+O9OYk8LZrMZkQPbwR7uqCahzUSkQVgebECsizjpe8ysS2rAs6OCrw/dxD83J1FxyI7olBImDEwGADwzf5CwWmIyN6xvFiBZdtO4ZO0PEgS8MbMAYgPVouORHbo1papo19OVqLofL3gNERkz1heLNzX6QW/7aB7Qx9M6xcoOBHZqzBvVwyN9IIsA2s4+kJEArG8WLAtJ8rw9JojAIC/jI3CvSMjBSciezez5eq2bw5wzxciEoflxUJtyyrH/M8OQG+QMT0hGE9NihUdiQhT+wbAzUmJvKo67DtzTnQcIjKzmsZmrN5fiPe2nxKag9uyWqAd2RX482f7odMbMDkuAK/e2g8KBbf+J/FcnRwwrV8gvkovxNfpBRgS6SU6EhGZWJPegF9yKrH2YBE2HytFQ5MBLo5KzBkWDjdBu7uzvFiYnTkVmPdpOnTNBiT18cfbsxPgqOQAGVmOmYNC8VV6ITYcKcELN8YJ++VFRKbT2KzHLzmV2HS0FKnHyqCpb2r9WpSPG25OCEazQdzUMX/rWJDvDxcj5ctD0OkNuL63P5bOHsjiQhZnUHh3RPq4IbeyFhuPlLSugyEi61av02N7djk2HS3Fz8fLUdPY3Po1n25OuKFfEKYnBKNfiFr4QcAsLxbi07QzeH59JmTZuK7g37MGwMmBxYUsjyRJuDUxBK/9mIWv9xeyvBBZsTOVtdiWVY6tWRXYfboKjc2G1q/5e6gwOS4Ak+MDMSTSC0oLWr7A8iKY3iDjlR9OYPmO0wCAOcPC8cKNcRb1Q0L0R7cMDMYbm7OwN/cs8qpqEe7tJjoSEXVAQ5Mee3LPYltWObZlVSC3srbN10O6u2BKvLGwJIR6Wux6S5YXgTR1TXj4i4PYkV0BAHjs+hg8MqGn8OE4oisJVLtgVLQvdmRX4Jv9hXg8qZfoSER0CflVddieU4FtJ8qx61QV6n93PpmDQsLgCC9cF+uLcb38EO3XzSo+g1heBDlapMHDnx9EbmUtnB0VeH1mf9zQL0h0LKIOm5kYgh3ZFVi9vxCPXR9jsf9CI7I3dbpm7D5dhe1ZFdiRU3nR6Iq/hwrX9fLDuF5+GNnTG+7OjoKSXj2WFzMzGGS8v/M0Xt+chSa9jGBPFyyfm4i4IG75T9ZlYh9/uDs7oFjTgN25VRjRw0d0JCK7JMsyTpRWY0d2BXbkVGBf7jno9L+tXXFQSBgY1h3jYn1xXS8/xAa4W8XoyuWwvJjRyfIaPLv2CPbkngUATI4LwOJb+qK7m5PgZESd5+yoxA39AvH53gKsOVDE8kJkRpr6JuzIrsD27ArszKlAmbaxzddDurtgbIwvxsT4YkQP6xxduRyWFzOo1+mxbNtJvLf9FJr0MlwclXg+uQ9mDQ61+vZL9u2WgSH4fG8BNh0pwUs3xcHVib9SiEyl4Gwdfjpehp+Ol2HP6bNt9llxdlRgeJQ3xsT4YmyMLyJ93Gz684W/aUyosVmPz/fk451tp1BRbWzF42P98OKNcQj1chWcjujaDQrvjlAvFxScrcfmzDLcnBAsOhKRzZBlGZnFWmw6WoKfjpUjq6y6zdd7+nXD+Fg/jIn2xaCI7nB2VApKan5mKS/Lli3Da6+9hpKSEsTFxWHJkiUYPXr0JR+/fft2pKSkIDMzE0FBQfjrX/+K+fPnmyNqlyivbsBX+wqwck8+SjQNAIBQLxc8O7UPJsX523QbJvsiSRJuSQjBf37OweoDhSwvRF0gu6wa3x0qxveHS9ostlUqJAyO6I7re/vj+t7+iPCx3y0KTF5evvzySyxcuBDLli3DyJEj8d///hdTpkzBsWPHEBYWdtHjc3NzMXXqVMybNw8rVqzAr7/+igcffBC+vr6YMWOGqeNeNU1dE7ZkleGHlp0JLwznBXg44+EJPTEzMZSbzpFNumVgMP7zcw5+PVmJMm0D/D2cRUcisjqlmgasPlCI9RnFbUZYVA4KjI/1w6S4AIzr5QtPV66RBABJNvG59kOHDsXAgQPx7rvvtt7Xu3dv3HzzzVi8ePFFj3/qqaewfv16HD9+vPW++fPn49ChQ0hLS7vi62m1WqjVamg0Gnh4eHTNXwLGq4QKz9WjVteMOl0zztU2Ie9sHU5V1OBA3jlklVXj9+9kQpgn7hoajmn9Au1qKI/s063v7kJ63jk8MzUWfx7TQ3QcIqvQpDdgy4lyfLWvAFuzynFhCYujUsLYGF8k9w/ChN7+6GYn54d15vPbpO+ITqfD/v378fTTT7e5PykpCbt27Wr3OWlpaUhKSmpz36RJk/DBBx+gqakJjo5tV0w3NjaisfG3VdZarbaL0rdV36THmNe2XvYx0X7dMCkuAFP7BqJPUNcVJyJLd8vAEKTnncPq/UWYNzqKU6NEl1GqacCnaWfwVXohKmt++/waEuGFWxNDMCkuAGpX27o6qKuZtLxUVlZCr9fD39+/zf3+/v4oLS1t9zmlpaXtPr65uRmVlZUIDAxs87XFixfjxRdf7Nrg7XBxVMLNSQkXJwe4qZRwd3ZAmJcrwrzc0D9EjcTw7vDjcDnZqWl9A/HCd5nIKqvGsRIt9y0iaseRQg0++OU0vj9c0rq0wKebE2YkhuC2QaHo4dtNcELrYZaxqD/+K0yW5cv+y6y9x7d3PwAsWrQIKSkprX/WarUIDe36g+IUCgmZL03u8u9LZAvUro6Y2NsfG46UYM2BIpYXohayLGN7dgWWbTuFvS17fAHAkEgv3DsiAtf38YejkushO8uk5cXHxwdKpfKiUZby8vKLRlcuCAgIaPfxDg4O8Pb2vujxKpUKKpWq60IT0VW5ZWAwNhwpwbqMIiyaEgsH/kImO3ahtCz5KQcZBecBGHe6Te4fhPtGRSI+mAX/Wpi0vDg5OSExMRGpqamYPn166/2pqam46aab2n3O8OHD8d1337W5b/PmzRg0aNBF612IyHKMifGFt5sTKmt02JlTieti/URHIhJi16lKvPpDVmtpcXZU4K6h4bh/dBQC1Fxe0BVMPm2UkpKCOXPmYNCgQRg+fDiWL1+O/Pz81n1bFi1ahKKiInz66acAjFcWLV26FCkpKZg3bx7S0tLwwQcf4PPPPzd1VCK6Bo5KBZL7B+HjXWew+kAhywvZnVMVNVi88QR+Ol4GwFha5gwLx5/H9ICvO2cIupLJy8usWbNQVVWFl156CSUlJYiPj8fGjRsRHh4OACgpKUF+fn7r4yMjI7Fx40Y89thjeOeddxAUFIS33nrLovd4ISKjGQND8PGuM9h8rAya+iaoXThaSrbvfJ0OS37KwYrdeWg2yFAqJNw1NAwLxkeztJiIyfd5MTdT7fNCRFcmyzKS/r0DOeU1+NctfXH7kIs3oiSyFbIsY11GMf7+/TFU1eoAABNi/bBoam/09OOVQ53Vmc9vrqgjoi4jSRJuGRgCAFhzoEhwGiLTyauqxdwP92LhlxmoqtUh2q8bVtw3FB/cM5jFxQxYXoioS92cEARJAvaeOYuCs3Wi4xB1KVmW8cmuM5i0ZAd25lTCyUGBJ5JisOGR0RgV7SM6nt1geSGiLhWodsHIHsZf4msPcvSFbEeZtgFzP9yL59dnoqHJgOFR3vhx4RgsGB/Ns+vMjO82EXW5WwYaT5dec6AQNrasjuzUD0dLWkdbVA4KvJDcByvvH4pIOz7ZWST7OO2JiMxqUlwAXJ2O4kxVHQ4WnMfAsO6iIxFdlSa9Af/adAIf/JILAIgP9sCSWQPQ089dcDL7xpEXIupybioHJPUx7qK9PqNYcBqiq1OqacAdy3e3Fpc/j4nCmgdGsrhYAJYXIjKJmwYYp46+P1yMZr1BcBqiztmbexY3vL0T6Xnn4K5ywHt3JeKZqb25tsVC8L8CEZnEqGgfeLUcF/DLyUrRcYg6bM2BQtz5v92orNEhNsAd6x8ehcnxAaJj0e+wvBCRSTgqFbihXyAAYB2njsgKyLKMNzdnIeWrQ2jSy5gcF4C1D47kolwLxPJCRCZzYerox8xS1Ov0gtMQXZqu2YCFX2bgrS0nAQDzx/bAsjsHwsVJKTgZtYflhYhMZmCYJ0K9XFCn0yO15bA6IktTr9Nj3qfpWJdRDAeFhFdm9MXTU2KhUEiio9ElsLwQkclIkoSb+htHX9ZxwzqyQNqGJsz9cA+2Z1fAxVGJD+4ZjFmDeSaXpWN5ISKTujkhCACwPbsCZ1sOryOyBFU1jZj9/m7sO3MO7s4O+Oy+IRgb4ys6FnUAywsRmVRPP3fEBXmg2SBj45ES0XGIAABna3WY/f4eHC3SwtvNCV/8eRgGRXiJjkUdxPJCRCZ3c8vC3XUZnDoi8TR1Tbjrf3uQVVYNP3cVvpo/HHFBatGxqBNYXojI5JL7G0+a3nfmHArP8aRpEqe6ZY3LsRItfLo5YdW8Yejh2010LOoklhciMrkAtTOGRXoD4J4vJE6drhn3frQPhwo16O7qiJX3D0NPPxYXa8TyQkRmcWHhLs86IhGa9QY8tPIA0vPOwcPZAZ/dNxS9AnhGkbVieSEis5gcHwgnpQJZZdU4XqIVHYfsiCzLeHbtUWzNqoCzowIf3TsE8cFc42LNWF6IyCzULo4YH+sHAPiWC3fJjP7zcw6+TC+AQgLevmMgEsO7i45E14jlhYjM5sLU0XcZxTAYZMFpyB58ta8AS37KAQD8/eZ4TOzjLzgRdQWWFyIym3G9/ODu7IBiTQP2nTkrOg7ZuN2nq/DM2iMAgAXX9cSdQ8MFJ6KuwvJCRGbj7KjElPgAAMC3XLhLJlRwtg4PrjyAZoOMG/sH4fGkGNGRqAuxvBCRWV3YsG7jkRLomg2C05AtqtM148+f7cfZWh3igz3w6q39IEk8ZNGWsLwQkVkNjfKGv4cKmvombM+uEB2HbIwsy3jy68M43rIJ3fI5g+DsqBQdi7oYywsRmZVSIeGGfi0Ldw9x6oi61rvbT2HDkRI4KiW8e1cigjxdREciE2B5ISKzS+5vLC8/HS9DvU4vOA3Zir25Z/HG5mwAwAs3xmEwD1q0WSwvRGR2/UPUCPVyQZ1Ojy0nykXHIRtQVdOIRz4/CL1BxvSEYMweEiY6EpkQywsRmZ0kSUjm1BF1EYNBRspXh1CqbUCUrxv+cXM8F+jaOJYXIhLiwtTRlqxyVDc0CU5D1uy/O05je3YFVA4KLLtzINxUDqIjkYmxvBCRELEB7ujh6wZdswGpx8pExyErdTD/HF7fnAUAePHGOMQGeAhORObA8kJEQkiS1Dr6wqkjuhp1umakfHUIeoOM5P5BmDU4VHQkMhOWFyIS5sIl0ztzKnG+Tic4DVmbxRtPILeyFgEezvjHTVznYk9YXohImJ5+3dAn0APNBhk/HC0VHYesyLascny2Ow8A8PrM/lC7OgpORObE8kJEQrVOHR3m1BF1zLlaHf76zWEAwD0jIjAq2kdwIjI3lhciEuqGfoEAgLRTVSivbhCchqzBc+szUV7diChfNzw1OVZ0HBKA5YWIhAr1csWAUE8YZGDTEU4d0eX9dKwM3x0qhkIC/n3bALg48dwie8TyQkTC8aoj6ojqhib837qjAIB5o6PQP9RTbCAShuWFiISb1jcQkgSk551D8fl60XHIQr36QxZKNA0I93bFwutjRMchgVheiEi4ALUzhrQcorfhcIngNGSJ0s+cbb26aPH0vpwusnMsL0RkEXjVEV1KY7MeT602Xl1026AQjOjJq4vsnUnLy7lz5zBnzhyo1Wqo1WrMmTMH58+fv+xz7rnnHkiS1OY2bNgwU8YkIgswJT4ASoWEw4UanKmsFR2HLMh/t5/GqYpa+HRT4dmpfUTHIQtg0vIye/ZsZGRk4IcffsAPP/yAjIwMzJkz54rPmzx5MkpKSlpvGzduNGVMIrIA3t1UGNHDGwDwPUdfqEXB2Tq8s/UkAOC55D7cjI4AACY7evP48eP44YcfsHv3bgwdOhQA8P7772P48OHIyspCr169LvlclUqFgIAAU0UjIguV3D8IO3Mq8d2hEiwYHy06DlmAl74/hsZmA0b08EZyy55ARCYbeUlLS4NarW4tLgAwbNgwqNVq7Nq167LP3bZtG/z8/BATE4N58+ahvLz8ko9tbGyEVqttcyMi6zQpLgCOSglZZdXILqsWHYcE25pVjtRjZXBQSHjxxjieXUStTFZeSktL4efnd9H9fn5+KC299EZUU6ZMwcqVK7Flyxa88cYb2LdvH8aPH4/GxsZ2H7948eLWNTVqtRqhoTxVlMhaqV0cMTbG+Hvje+75Ytcam/V4cX0mAODekRGI9ncXnIgsSafLywsvvHDRgto/3tLT0wGg3ZYsy/Jl2/OsWbMwbdo0xMfHIzk5GZs2bUJ2djY2bNjQ7uMXLVoEjUbTeisoKOjsX4mILMiF4wI2HCmBLMuC05Ao7+84jTNVdfBzV+FR7ulCf9DpNS8LFizA7bffftnHRERE4PDhwygrK7voaxUVFfD39+/w6wUGBiI8PBw5OTntfl2lUkGlUnX4+xGRZZvQ2w9ODgqcqqhFVlk1YgM8REciMyvR1GNpyyLdZ6f1RjeVyZZnkpXq9E+Ej48PfHyufI398OHDodFosHfvXgwZMgQAsGfPHmg0GowYMaLDr1dVVYWCggIEBnKhFpE9cHd2xNgYX6QeK8PGwyUsL3bo9R+z0dBkwKDw7rixZf8fot8z2ZqX3r17Y/LkyZg3bx52796N3bt3Y968ebjhhhvaXGkUGxuLtWvXAgBqamrwxBNPIC0tDWfOnMG2bduQnJwMHx8fTJ8+3VRRicjCTOvLqSN7dbRIgzUHCwEAf7uhDxfpUrtMus/LypUr0bdvXyQlJSEpKQn9+vXDZ5991uYxWVlZ0Gg0AAClUokjR47gpptuQkxMDO6++27ExMQgLS0N7u5crEVkL/44dUT2QZZl/GPDMcgycNOAIAzgwYt0CSadSPTy8sKKFSsu+5jf/6vKxcUFP/74oykjEZEV4NSRffrpeDl2nz4LJwcFnpx06b3AiHi2ERFZJE4d2ZcmvQGLNx4HANw3KhIh3V0FJyJLxvJCRBaJU0f25fO9+ThdWQtvNyc8OK6H6Dhk4VheiMgiuTs7Yky0LwBg4+ESwWnIlOp0zXjrZ+N2GAuvj4a7M88vostjeSEii8UN6+zDR7+eQWWNDmFerrh9SJjoOGQFWF6IyGL9fuoou6xGdBwyAU1dE/67/RQAIGViDByV/FiiK+NPCRFZrN9PHW04zLOObNHynaegbWhGL393JHNDOuoglhcismjT+gUA4NSRLaqobsSHv5wBADyeFAOlghvSUcewvBCRRbu+tz+njmzUO1tPor5Jj/6hnpjYp+Nn3hGxvBCRRePUkW0qPFeHVXvyAQB/ndSLxwBQp7C8EJHF49SR7Xn755PQ6Q0Y2dMbI3te+bBfot9jeSEii8epI9tScLYOqw8YD19MmchjAKjzWF6IyOK1mTo6wg3rrN2720+h2SBjdLQPEsO7i45DVojlhYisQuvU0eFiTh1ZseLz9fg6vQAA8MiEaMFpyFqxvBCRVZjQ2x9OSk4dWbv3tp9Ck17G8ChvDI7wEh2HrBTLCxFZBQ9nR4yJ4dSRNSvTNuCLfRx1oWvH8kJEVoNTR9btv9tPQ9dswOCI7hgWxVEXunosL0RkNTh1ZL3Kqxuwck8eAOOoC/d1oWvB8kJEVoNTR9brfztz0dhsQEKYJ0ZxXxe6RiwvRGRVOHVkfTR1TVi5u2XUZTxHXejasbwQkVW5nlNHVmfFnjzU6vSIDXDHuF6+ouOQDWB5ISKr4u7siDExxmkHTh1ZvoYmPT76NRcAMH9sD466UJdgeSEiqzO1byAAYCPLi8VbfaAQlTU6BHu6YFq/QNFxyEawvBCR1bm+j3Hq6GR5DbLLqkXHoUvQG2S8v+M0AOD+0ZFwVPIjh7oGf5KIyOp4ODtidHTL1NFhjr5Yqh+OluJMVR08XR0xa3Co6DhkQ1heiMgqcerIssmyjPe2nwIAzB0eAVcnB8GJyJawvBCRVbq+jz8clRJyymuQw6kji5N2qgpHijRwdlTg7uHhouOQjWF5ISKrpHZxxOhoblhnqd5rWety26BQeHdTCU5DtoblhYisFqeOLNOxYi12ZFdAIQHzRkeJjkM2iOWFiKzWxJapo+yyGpws59SRpfiwZV+XqX0DEerlKjgN2SKWFyKyWmoXx9ZzcjYeKRWchgCgoroR6zOKAQD3jYoUnIZsFcsLEVk1Th1ZlhW786DTGw9gTAjrLjoO2SiWFyKyakl9AuColHCitBqnKnjWkUgNTXqs3GM8gPFPIznqQqbD8kJEVk3t6oiRF6aOuGGdUN8dKkZljQ5BamdMiQ8QHYdsGMsLEVm9C1NHvGRaHFmW8cEvxoW6c0dEwIFHAZAJ8aeLiKxeUh9/OCiMU0enOXUkRNrpKpworYaLoxK38ygAMjGWFyKyep6uThjRetURR19E+PCXMwCAGYnB8HR1EhuGbB7LCxHZhGl9jWssNvCSabM7U1mLn0+UAQDu5UJdMgOWFyKyCUl9AqBUSDheokVuZa3oOHbl411nIMvAdb180cO3m+g4ZAdYXojIJnR3c8KIHt4AOHVkTtqGJnydXgAA+BM3pSMzYXkhIpsx7cJVR7xk2mxW7y9ErU6PaL9urbsdE5kaywsR2YykOOPU0bESLc5w6sjkZFnGZ7uNm9LNHR4OSZIEJyJ7YdLy8s9//hMjRoyAq6srPD09O/QcWZbxwgsvICgoCC4uLhg3bhwyMzNNGZOIbITX76aOuOeL6aWdqsLpilq4OSlxc0Kw6DhkR0xaXnQ6HWbOnIkHHnigw8959dVX8eabb2Lp0qXYt28fAgICMHHiRFRX88RYIroynnVkPhdGXaYPDIa7s6PgNGRPTFpeXnzxRTz22GPo27dvhx4vyzKWLFmCZ599Frfccgvi4+PxySefoK6uDqtWrTJlVCKyEZNapo4yi7XIq+LUkamUahqw+Zjx8ui7hoULTkP2xqLWvOTm5qK0tBRJSUmt96lUKowdOxa7du1q9zmNjY3QarVtbkRkv7zcnDA8ilNHpvb53nzoDTKGRHghNsBDdByyMxZVXkpLjZtL+fv7t7nf39+/9Wt/tHjxYqjV6tZbaCi3pSayd5w6Mq0mvQGf780HANw1nKMuZH6dLi8vvPACJEm67C09Pf2aQv1xxbosy5dcxb5o0SJoNJrWW0FBwTW9NhFZv0lx/lAqJBwt0iK/qk50HJuTeqwM5dWN8OnmhMlxPD2azM+hs09YsGABbr/99ss+JiIi4qrCBAQY/ycoLS1FYGBg6/3l5eUXjcZcoFKpoFKprur1iMg2eXdTYViUF349WYUNR0rwwLgeoiPZlBUtC3VvHxwGJweLGsAnO9Hp8uLj4wMfH9NsRBQZGYmAgACkpqYiISEBgPGKpe3bt+OVV14xyWsSkW2a2jcQv56swkaWly51srwau05VQSEBdwwNEx2H7JRJK3N+fj4yMjKQn58PvV6PjIwMZGRkoKbmtyPrY2NjsXbtWgDG6aKFCxfi5Zdfxtq1a3H06FHcc889cHV1xezZs00ZlYhszKS4ACgk4EiRhlNHXWjFbuNalwm9/RHs6SI4DdmrTo+8dMZzzz2HTz75pPXPF0ZTtm7dinHjxgEAsrKyoNFoWh/z17/+FfX19XjwwQdx7tw5DB06FJs3b4a7u7spoxKRjfHppsKwKG/sOlWFjUdLMH8sR1+uVZ2uGav3FwIA5vDyaBJIkmVZFh2iK2m1WqjVamg0Gnh48PI9Inu2Ynce/vbtUfQLUWP9glGi41i9z/fmY9GaI4jwdsWWx8dBoeBxANR1OvP5zZVWRGSzJscbp44OF2pQcJZTR9dClmV8lmZcqHvXsHAWFxKK5YWIbJZPNxWGRho3rOOeL9fmQP55HCvRQuWgwK2JIaLjkJ1jeSEimza1Hzes6woXLo9O7h8ET1cnwWnI3rG8EJFNm9xy1dEhTh1dtaqaRmw4bCx/XKhLloDlhYhsmq+7CkMivQAAm45y9OVqfJVeCJ3egH4havQP9RQdh4jlhYhs37SWs442HGn/jDS6NL1Bxso9vy3UJbIELC9EZPMmxQdAkoBDBedReI5TR52xPbschefqoXZxRHK/INFxiACwvBCRHfBzd8bgCOPU0Q9HOfrSGRcuj56ZGAIXJ6XgNERGLC9EZBd+mzriupeOyq+qw7bsCgDAnZwyIgvC8kJEdmFKy9TRwfzzKD5fLzqOVVi5Nw+yDIyO9kGkj5voOEStWF6IyC74eThjcLhx6oh7vlxZQ5MeX6fzHCOyTCwvRGQ3buhvnDr67lCx4CSWb9PREpyt1SFI7YzxsX6i4xC1wfJCRHZjat9AKBUSDhVqkFtZKzqORbuwUHf20DA4KPlRQZaFP5FEZDd8uqkwsqcPAGB9BkdfLuVokQYH8s/DUSnhtsGhouMQXYTlhYjsyk39jXuVrDtUBFmWBaexTBc2pZscHwg/d2fBaYguxvJCRHYlKc4fKgcFTlfUIrNYKzqOxdHUN+Hbg8ZRKS7UJUvF8kJEdsXd2RHX9/YHAKznwt2LrDlQiPomPXr5u2NwRHfRcYjaxfJCRHYnuWXq6LtDxTAYOHV0gSzL+Gz3hXOMwiBJkuBERO1jeSEiuzOuly/cnR1QomnAvjNnRcexGGmnqnC6ohZuTkrcnBAsOg7RJbG8EJHdcXZUYkp8AABgHaeOWl0YdZk+MBjuzo6C0xBdGssLEdmlmwYYRxY2HimBrtkgOI14JZp6bD5WBgC4iwt1ycKxvBCRXRoW5Q1fdxXO1zVhZ06F6DjCfb63AHqDjCGRXogN8BAdh+iyWF6IyC4pFRJu6Gc8LsDerzrSNRvw+d58AMDc4Rx1IcvH8kJEduvC1NHmzDLU6ZoFpxHnx8xSVFQ3wtddhaQ+AaLjEF0RywsR2a3+IWqEe7uivkmP1Jb1HvbowjlGdwwJg5MDPxbI8vGnlIjsliRJrccFrDlQJDiNGCdKtdh75iyUCgmzh4SJjkPUISwvRGTXbhkYAgDYmVOBMm2D4DTmd2HUJamPPwLUPMeIrAPLCxHZtQgfNwyO6A6DDKw9aF+jL9qGpta/8xwu1CUrwvJCRHZvRsvoyzf7C+3qpOm1B4pQp9Mj2q8bhkd5i45D1GEsL0Rk96b2C4SzowIny2twuFAjOo5Z/P4coznDw3mOEVkVlhcisnsezo6YFGe8RPib/YWC05hH2qkqnCyvgZuTEtN5jhFZGZYXIiIAtyYap47WHypGY7NecBrT4zlGZM1YXoiIAIzo4YMAD2do6pvw8/Fy0XFM6vfnGM0dHiE2DNFVYHkhIoLxuIBbBhqnT1bb+NTRp2l50BtkDI30Qoy/u+g4RJ3G8kJE1GJGy9TRtuwKlFfb5p4v9To9Vu0xnmN036hIwWmIrg7LCxFRix6+3ZAQ5gm9Qca3Nrrny5qDhdDUNyHMyxUTevuLjkN0VVheiIh+Z2ZiKADgi30FNrfni8Eg48NfcgEA94yIgFLBy6PJOrG8EBH9zo0DguDmpMTpilrsyT0rOk6X2nmyEqcqatFN5YCZg0JExyG6aiwvRES/003lgBsHGBfuXlgbYisujLrcNiiUl0eTVWN5ISL6gzuHGk9X/uFoKc7W6gSn6Rony6uxPbsCkmScMiKyZiYtL//85z8xYsQIuLq6wtPTs0PPueeeeyBJUpvbsGHDTBmTiKiN+GA1+garodMb8M3+AtFxusRHv54BAEzs7Y8wb1exYYiukUnLi06nw8yZM/HAAw906nmTJ09GSUlJ623jxo0mSkhE1L7ZLaMvn++1/oW75+t0WH3AuHfNn3h5NNkAB1N+8xdffBEA8PHHH3fqeSqVCgEBASZIRETUMTf2D8I/vj+G3MpapJ2uwogePqIjXbXP0vLQ0GRAn0APDI30Eh2H6JpZ5JqXbdu2wc/PDzExMZg3bx7Ky217q24isjxuKgfc1HJg4UorXrhbr9Pjo11nAAB/GRvF06PJJlhceZkyZQpWrlyJLVu24I033sC+ffswfvx4NDY2tvv4xsZGaLXaNjcioq5w19BwAMaFuyWaesFprs7X+wtwtlaHUC8XTOsbKDoOUZfodHl54YUXLlpQ+8dbenr6VQeaNWsWpk2bhvj4eCQnJ2PTpk3Izs7Ghg0b2n384sWLoVarW2+hoaFX/dpERL/XJ8g4zaI3yPgsLU90nE5r1huwfMdpAMCfR0fBQWlx/14luiqdXvOyYMEC3H777Zd9TERExNXmuUhgYCDCw8ORk5PT7tcXLVqElJSU1j9rtVoWGCLqMveOjMSe3LNYtTcfD4+PhouTUnSkDttwpASF5+rh7eaEmYP4e5FsR6fLi4+PD3x8zLdwraqqCgUFBQgMbH+4U6VSQaVSmS0PEdmXiX38EdLdBYXn6vFtRhHuGBImOlKHyLKMd7edAgDcOzICzo7WU7qIrsSkY4j5+fnIyMhAfn4+9Ho9MjIykJGRgZqamtbHxMbGYu3atQCAmpoaPPHEE0hLS8OZM2ewbds2JCcnw8fHB9OnTzdlVCKidikVUuumbh/9mms1l01vy6rAidJquDkpMWdYhOg4RF3KpOXlueeeQ0JCAp5//nnU1NQgISEBCQkJbdbEZGVlQaPRAACUSiWOHDmCm266CTExMbj77rsRExODtLQ0uLu7mzIqEdElzRwUClcnJbLLavDrySrRca5IlmUs+dk41T57aBjUrjwKgGyLJFvLPyM6SKvVQq1WQ6PRwMPDQ3QcIrIRz687ik/S8jA62gef3TdUdJzL2ppVjns/2gdnRwV2/nU8fN05tU6WrzOf31x6TkTUAfPGRMFBIWFnTiUOFZwXHeeSZFnGktRsAMDc4REsLmSTWF6IiDogpLsrbhwQBABYtu2k4DSXti2rAocKNXBxVOLPY6JExyEyCZYXIqIOenBcD0gS8GNmGXLKqkXHuYgsy1jy04VRl3D4dOOoC9kmlhciog7q6eeOSX2M5669u/2U4DQX+zGztHXUZR5HXciGsbwQEXXCg9f1AACsyyhGXlWt4DS/adIb8MoPWQCAeaMjOepCNo3lhYioE/qFeGJsjC/0BhlvtiyMtQSf781HbmUtvN2c8OexPUTHITIplhciok56clIvAMD6Q8U4XiL+MNjqhib85yfjvi4Lr49GN1WnN08nsiosL0REnRQfrMa0foGQZeD1H7NEx8F/t59GVa0OUT5uuN1Kji8guhYsL0REV+HxiTFQKiT8fKIc+86cFZYjr6oWy3caT47+6+RYOPLkaLID/CknIroKUb7dcFvLSc0vrM+E3mD+zcplWcZz6zKhazZgVE8fTIrzN3sGIhFYXoiIrtLjSTFwd3ZAZrEWX+zLN/vr/5hZiu3ZFXBSKvDSTXGQJMnsGYhEYHkhIrpKPt1UeHxiDADgtR+zcK5WZ7bXrm5owovfHQMA/GVsFKJ8u5nttYlEY3khIroGdw0LR2yAO87XNeFfm06Y7XX//v0xlGgaEObligfH9TTb6xJZApYXIqJr4KBU4KWb4gEAX6YXYGtWuclf86djZfgqvRCSBLw+sz9cnJQmf00iS8LyQkR0jYZEeuFPIyMBAE99cxjn60w3fVRV04in1xwBAPx5dBSGRHqZ7LWILBXLCxFRF/jr5F6I8nVDeXUj/m9dJmS5668+0htkPPLFQVTWNCLGvxsea1lvQ2RvWF6IiLqAs6MSb8zsD6VCwneHivFpWl6Xv8Ybm7Pw68kquDopsXT2QDg7crqI7BPLCxFRF0kI645FU2IBGBfU7jld1WXf++v0AizbZjzJ+pUZ/RDj795l35vI2rC8EBF1oftGRSK5fxCaDTLmfZreJWcfbTlR1rrOZf7YHkjuH3TN35PImrG8EBF1IUmS8OqMfkgM7w5tQzPmfLAXJ8trrvr7/ZhZivmfHYDeIOOWgcF4anKvLkxLZJ1YXoiIupiLkxIf3jMYfQI9UFnTiBnv7sLuTk4hybKMFbvz8MCK/dDpDUjq449XZvTjLrpEYHkhIjIJtYsjPrtvCBLCPKGpb8KcD/bgv9tPdegMJE1dEx7+/CD+9u1RGGRgZmIIlt05kIcuErWQZFNczyeQVquFWq2GRqOBh4eH6DhEZOcamvR4/KtD2HCkBAAQG+CORydEY0Jvfzg5tC0j5+t0+HJfAd7bfgrn6prgoJDweFIvzB8bxREXsnmd+fxmeSEiMjFZlvF1eiH+vuEYqhuaARhHZhLCPBGodkGT3oDcylocKjiP5paRmRj/bvjXjH4YGNZdZHQis2F5YXkhIgt0vk6H93eexlfphaiobmz3MX0CPXDPiAjcMjAYDpwmIjvC8sLyQkQWrFlvwJEiDY4Wa3G2RgelAgjydMHgCC+EermKjkckRGc+vx3MlImIiFo4KBVICOuOBE4JEV0VjkkSERGRVWF5ISIiIqvC8kJERERWheWFiIiIrArLCxEREVkVlhciIiKyKiwvREREZFVYXoiIiMiqsLwQERGRVWF5ISIiIqvC8kJERERWheWFiIiIrArLCxEREVkVmztVWpZlAMajtYmIiMg6XPjcvvA5fjk2V16qq6sBAKGhoYKTEBERUWdVV1dDrVZf9jGS3JGKY0UMBgOKi4vh7u4OSZK69HtrtVqEhoaioKAAHh4eXfq96Td8n82H77V58H02D77P5mGq91mWZVRXVyMoKAgKxeVXtdjcyItCoUBISIhJX8PDw4P/Y5gB32fz4XttHnyfzYPvs3mY4n2+0ojLBVywS0RERFaF5YWIiIisCstLJ6hUKjz//PNQqVSio9g0vs/mw/faPPg+mwffZ/OwhPfZ5hbsEhERkW3jyAsRERFZFZYXIiIisiosL0RERGRVWF6IiIjIqrC8dNCyZcsQGRkJZ2dnJCYmYufOnaIj2ZzFixdj8ODBcHd3h5+fH26++WZkZWWJjmXzFi9eDEmSsHDhQtFRbE5RURHuuusueHt7w9XVFQMGDMD+/ftFx7I5zc3N+Nvf/obIyEi4uLggKioKL730EgwGg+hoVm3Hjh1ITk5GUFAQJEnCt99+2+brsizjhRdeQFBQEFxcXDBu3DhkZmaaJRvLSwd8+eWXWLhwIZ599lkcPHgQo0ePxpQpU5Cfny86mk3Zvn07HnroIezevRupqalobm5GUlISamtrRUezWfv27cPy5cvRr18/0VFszrlz5zBy5Eg4Ojpi06ZNOHbsGN544w14enqKjmZzXnnlFbz33ntYunQpjh8/jldffRWvvfYa3n77bdHRrFptbS369++PpUuXtvv1V199FW+++SaWLl2Kffv2ISAgABMnTmw9Y9CkZLqiIUOGyPPnz29zX2xsrPz0008LSmQfysvLZQDy9u3bRUexSdXV1XJ0dLScmpoqjx07Vn700UdFR7IpTz31lDxq1CjRMezCtGnT5D/96U9t7rvlllvku+66S1Ai2wNAXrt2beufDQaDHBAQIP/rX/9qva+hoUFWq9Xye++9Z/I8HHm5Ap1Oh/379yMpKanN/UlJSdi1a5egVPZBo9EAALy8vAQnsU0PPfQQpk2bhuuvv150FJu0fv16DBo0CDNnzoSfnx8SEhLw/vvvi45lk0aNGoWff/4Z2dnZAIBDhw7hl19+wdSpUwUns125ubkoLS1t89moUqkwduxYs3w22tzBjF2tsrISer0e/v7+be739/dHaWmpoFS2T5ZlpKSkYNSoUYiPjxcdx+Z88cUXOHDgAPbt2yc6is06ffo03n33XaSkpOCZZ57B3r178cgjj0ClUmHu3Lmi49mUp556ChqNBrGxsVAqldDr9fjnP/+JO+64Q3Q0m3Xh86+9z8a8vDyTvz7LSwdJktTmz7IsX3QfdZ0FCxbg8OHD+OWXX0RHsTkFBQV49NFHsXnzZjg7O4uOY7MMBgMGDRqEl19+GQCQkJCAzMxMvPvuuywvXezLL7/EihUrsGrVKsTFxSEjIwMLFy5EUFAQ7r77btHxbJqoz0aWlyvw8fGBUqm8aJSlvLz8osZJXePhhx/G+vXrsWPHDoSEhIiOY3P279+P8vJyJCYmtt6n1+uxY8cOLF26FI2NjVAqlQIT2obAwED06dOnzX29e/fG6tWrBSWyXU8++SSefvpp3H777QCAvn37Ii8vD4sXL2Z5MZGAgAAAxhGYwMDA1vvN9dnINS9X4OTkhMTERKSmpra5PzU1FSNGjBCUyjbJsowFCxZgzZo12LJlCyIjI0VHskkTJkzAkSNHkJGR0XobNGgQ7rzzTmRkZLC4dJGRI0dedKl/dnY2wsPDBSWyXXV1dVAo2n6cKZVKXiptQpGRkQgICGjz2ajT6bB9+3azfDZy5KUDUlJSMGfOHAwaNAjDhw/H8uXLkZ+fj/nz54uOZlMeeughrFq1CuvWrYO7u3vraJdarYaLi4vgdLbD3d39onVEbm5u8Pb25vqiLvTYY49hxIgRePnll3Hbbbdh7969WL58OZYvXy46ms1JTk7GP//5T4SFhSEuLg4HDx7Em2++iT/96U+io1m1mpoanDx5svXPubm5yMjIgJeXF8LCwrBw4UK8/PLLiI6ORnR0NF5++WW4urpi9uzZpg9n8uuZbMQ777wjh4eHy05OTvLAgQN5+a4JAGj39tFHH4mOZvN4qbRpfPfdd3J8fLysUqnk2NhYefny5aIj2SStVis/+uijclhYmOzs7CxHRUXJzz77rNzY2Cg6mlXbunVru7+T7777blmWjZdLP//883JAQICsUqnkMWPGyEeOHDFLNkmWZdn0FYmIiIioa3DNCxEREVkVlhciIiKyKiwvREREZFVYXoiIiMiqsLwQERGRVWF5ISIiIqvC8kJERERWheWFiIiIrArLCxEREVkVlhciIiKyKiwvREREZFVYXoiIiMiq/D9xl8LC0gZOLQAAAABJRU5ErkJggg==", + "image/svg+xml": "\n\n\n \n \n \n \n 2021-08-23T18:55:08.268586\n image/svg+xml\n \n \n Matplotlib v3.4.2, https://matplotlib.org/\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "result_np = result.detach().numpy() # Convert to numpy array\n", + "\n", + "# 2D plot of X and Z\n", + "plt.plot(result_np[:,2], result_np[:,0])\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Euler's method for training" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "ode_train_coeffs = {\"a\": 0., \"b\": 0., \"d\": 0., \"g\": 0., \"w\": 1.}\n", + "\n", + "ode_solver_train = ODESolver(\n", + " ode=ode,\n", + " init_vars=ode_init,\n", + " init_coeffs=ode_train_coeffs,\n", + " dt=dt,\n", + " solver=\"rk4\",\n", + " optimizer=None\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAAGdCAYAAAAfTAk2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAerElEQVR4nO3dfXBU5d3/8c+ShA20yYqk7JIaIFhGQLSFpMZEI3aq4UFRKq0oJdpqqRnFkGSsPNkfDI5JQUsZGx4aim2d+sB0EE1nMDexasQSntIEKU1x2qaEAdYYirsRbQLh/P5g2LvbhADe2az75f2a2T9y7XXOXmdn9Lzn7NnF5TiOIwAAAEP6RXsBAAAAvY3AAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDnx0V5ANJw+fVpHjhxRUlKSXC5XtJcDAAAugOM4amtrU2pqqvr16/kazSUZOEeOHFFaWlq0lwEAAD6DQ4cO6YorruhxziUZOElJSZLOvEHJyclRXg0AALgQwWBQaWlpofN4Ty7JwDn7sVRycjKBAwBAjLmQ20u4yRgAAJhD4AAAAHMIHAAAYA6BAwAAzCFwAACAOQQOAAAwh8ABAADmEDgAAMAcAgcAAJhD4AAAAHMIHAAAYA6BAwAAzCFwAACAOQQOAAAwh8ABAADmEDgAAMAcAgcAAJhD4AAAAHMIHAAAYA6BAwAAzCFwAACAOQQOAAAwh8ABAADmEDgAAMAcAgcAAJhD4AAAAHMIHAAAYA6BAwAAzCFwAACAOQQOAAAwh8ABAADmEDgAAMAcAgcAAJhD4AAAAHMIHAAAYA6BAwAAzCFwAACAOQQOAAAwh8ABAADmEDgAAMAcAgcAAJhD4AAAAHMIHAAAYE6fBM6aNWuUnp6uxMREZWRkaNu2bT3Or6mpUUZGhhITEzVy5EitW7funHNffvlluVwuTZ8+vZdXDQAAYlXEA2fjxo0qKirS4sWLVV9fr9zcXE2ZMkXNzc3dzm9qatLUqVOVm5ur+vp6LVq0SIWFhdq0aVOXuQcPHtRjjz2m3NzcSB8GAACIIS7HcZxIvkBWVpYmTJigtWvXhsbGjBmj6dOnq6ysrMv8+fPnq7KyUo2NjaGxgoIC7d27V7W1taGxzs5OTZw4Ud///ve1bds2ffTRR3r11VcvaE3BYFAej0eBQEDJycmf/eAAAECfuZjzd0Sv4HR0dKiurk55eXlh43l5edq+fXu329TW1naZP2nSJO3Zs0cnT54MjS1btkxf+tKX9OCDD553He3t7QoGg2EPAABgV0QDp7W1VZ2dnfJ6vWHjXq9Xfr+/2238fn+380+dOqXW1lZJ0h//+Edt2LBB69evv6B1lJWVyePxhB5paWmf4WgAAECs6JObjF0uV9jfjuN0GTvf/LPjbW1tmj17ttavX6+UlJQLev2FCxcqEAiEHocOHbrIIwAAALEkPpI7T0lJUVxcXJerNS0tLV2u0pzl8/m6nR8fH6/Bgwdr//79+uc//6lp06aFnj99+rQkKT4+XgcOHNCVV14Ztr3b7Zbb7e6NQwIAADEgoldw+vfvr4yMDFVXV4eNV1dXKycnp9ttsrOzu8zfunWrMjMzlZCQoNGjR2vfvn1qaGgIPe644w594xvfUENDAx8/AQCAyF7BkaSSkhLl5+crMzNT2dnZqqioUHNzswoKCiSd+fjo8OHDev755yWd+cZUeXm5SkpKNGfOHNXW1mrDhg166aWXJEmJiYkaN25c2GtcdtllktRlHAAAXJoiHjgzZ87UsWPHtGzZMh09elTjxo3Tli1bNHz4cEnS0aNHw34TJz09XVu2bFFxcbFWr16t1NRUPfvss5oxY0aklwoAAIyI+O/gfB7xOzgAAMSez83v4AAAAEQDgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABz+iRw1qxZo/T0dCUmJiojI0Pbtm3rcX5NTY0yMjKUmJiokSNHat26dWHPr1+/Xrm5uRo0aJAGDRqkW265Rbt27YrkIQAAgBgS8cDZuHGjioqKtHjxYtXX1ys3N1dTpkxRc3Nzt/Obmpo0depU5ebmqr6+XosWLVJhYaE2bdoUmvP222/r3nvv1VtvvaXa2loNGzZMeXl5Onz4cKQPBwAAxACX4zhOJF8gKytLEyZM0Nq1a0NjY8aM0fTp01VWVtZl/vz581VZWanGxsbQWEFBgfbu3ava2tpuX6Ozs1ODBg1SeXm57rvvvvOuKRgMyuPxKBAIKDk5+TMcFQAA6GsXc/6O6BWcjo4O1dXVKS8vL2w8Ly9P27dv73ab2traLvMnTZqkPXv26OTJk91u88knn+jkyZO6/PLLu32+vb1dwWAw7AEAAOyKaOC0traqs7NTXq83bNzr9crv93e7jd/v73b+qVOn1Nra2u02CxYs0Je//GXdcsst3T5fVlYmj8cTeqSlpX2GowEAALGiT24ydrlcYX87jtNl7HzzuxuXpBUrVuill17SK6+8osTExG73t3DhQgUCgdDj0KFDF3sIAAAghsRHcucpKSmKi4vrcrWmpaWly1Was3w+X7fz4+PjNXjw4LDxZ555RqWlpXrjjTd07bXXnnMdbrdbbrf7Mx4FAACINRG9gtO/f39lZGSouro6bLy6ulo5OTndbpOdnd1l/tatW5WZmamEhITQ2NNPP60nn3xSVVVVyszM7P3FAwCAmBXxj6hKSkr0y1/+Us8995waGxtVXFys5uZmFRQUSDrz8dF/fvOpoKBABw8eVElJiRobG/Xcc89pw4YNeuyxx0JzVqxYoSeeeELPPfecRowYIb/fL7/fr48//jjShwMAAGJARD+ikqSZM2fq2LFjWrZsmY4ePapx48Zpy5YtGj58uCTp6NGjYb+Jk56eri1btqi4uFirV69Wamqqnn32Wc2YMSM0Z82aNero6NC3v/3tsNdasmSJli5dGulDAgAAn3MR/x2czyN+BwcAgNjzufkdHAAAgGggcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGBOnwTOmjVrlJ6ersTERGVkZGjbtm09zq+pqVFGRoYSExM1cuRIrVu3rsucTZs2aezYsXK73Ro7dqw2b94cqeUDAIAYE/HA2bhxo4qKirR48WLV19crNzdXU6ZMUXNzc7fzm5qaNHXqVOXm5qq+vl6LFi1SYWGhNm3aFJpTW1urmTNnKj8/X3v37lV+fr7uvvtu7dy5M9KHAwAAYoDLcRwnki+QlZWlCRMmaO3ataGxMWPGaPr06SorK+syf/78+aqsrFRjY2NorKCgQHv37lVtba0kaebMmQoGg3r99ddDcyZPnqxBgwbppZdeOu+agsGgPB6PAoGAkpOT/y+HF8ZxHH16srPX9gcAQCwbkBAnl8vVa/u7mPN3fK+9ajc6OjpUV1enBQsWhI3n5eVp+/bt3W5TW1urvLy8sLFJkyZpw4YNOnnypBISElRbW6vi4uIuc1atWtXtPtvb29Xe3h76OxgMfoajOb9PT3Zq7P/7n4jsGwCAWPOXZZM0sH9EU+OcIvoRVWtrqzo7O+X1esPGvV6v/H5/t9v4/f5u5586dUqtra09zjnXPsvKyuTxeEKPtLS0z3pIAAAgBvRJVv335SnHcXq8ZNXd/P8ev5h9Lly4UCUlJaG/g8FgRCJnQEKc/rJsUq/vFwCAWDQgIS5qrx3RwElJSVFcXFyXKystLS1drsCc5fP5up0fHx+vwYMH9zjnXPt0u91yu92f9TAumMvlitqlOAAA8L8i+hFV//79lZGRoerq6rDx6upq5eTkdLtNdnZ2l/lbt25VZmamEhISepxzrn0CAIBLS8QvN5SUlCg/P1+ZmZnKzs5WRUWFmpubVVBQIOnMx0eHDx/W888/L+nMN6bKy8tVUlKiOXPmqLa2Vhs2bAj7dtS8efN00003afny5brzzjv12muv6Y033tC7774b6cMBAAAxIOKBM3PmTB07dkzLli3T0aNHNW7cOG3ZskXDhw+XJB09ejTsN3HS09O1ZcsWFRcXa/Xq1UpNTdWzzz6rGTNmhObk5OTo5Zdf1hNPPKEf//jHuvLKK7Vx40ZlZWVF+nAAAEAMiPjv4HweRep3cAAAQORczPmbf4sKAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAnIgGzvHjx5Wfny+PxyOPx6P8/Hx99NFHPW7jOI6WLl2q1NRUDRgwQDfffLP2798fev5f//qXHn30UV111VUaOHCghg0bpsLCQgUCgUgeCgAAiCERDZxZs2apoaFBVVVVqqqqUkNDg/Lz83vcZsWKFVq5cqXKy8u1e/du+Xw+3XrrrWpra5MkHTlyREeOHNEzzzyjffv26de//rWqqqr04IMPRvJQAABADHE5juNEYseNjY0aO3asduzYoaysLEnSjh07lJ2drb/+9a+66qqrumzjOI5SU1NVVFSk+fPnS5La29vl9Xq1fPlyPfTQQ92+1u9+9zvNnj1bJ06cUHx8/HnXFgwG5fF4FAgElJyc/H84SgAA0Fcu5vwdsSs4tbW18ng8obiRpOuvv14ej0fbt2/vdpumpib5/X7l5eWFxtxutyZOnHjObSSFDvRC4gYAANgXsSLw+/0aMmRIl/EhQ4bI7/efcxtJ8nq9YeNer1cHDx7sdptjx47pySefPOfVHenMVaD29vbQ38Fg8LzrBwAAseuir+AsXbpULperx8eePXskSS6Xq8v2juN0O/6f/vv5c20TDAZ12223aezYsVqyZMk591dWVha60dnj8SgtLe1CDhUAAMSoi76CM3fuXN1zzz09zhkxYoTee+89ffDBB12e+/DDD7tcoTnL5/NJOnMlZ+jQoaHxlpaWLtu0tbVp8uTJ+uIXv6jNmzcrISHhnOtZuHChSkpKQn8Hg0EiBwAAwy46cFJSUpSSknLeednZ2QoEAtq1a5euu+46SdLOnTsVCASUk5PT7Tbp6eny+Xyqrq7W+PHjJUkdHR2qqanR8uXLQ/OCwaAmTZokt9utyspKJSYm9rgWt9stt9t9oYcIAABiXMRuMh4zZowmT56sOXPmaMeOHdqxY4fmzJmj22+/PewbVKNHj9bmzZslnfloqqioSKWlpdq8ebP+/Oc/63vf+54GDhyoWbNmSTpz5SYvL08nTpzQhg0bFAwG5ff75ff71dnZGanDAQAAMSSiXzt64YUXVFhYGPpW1B133KHy8vKwOQcOHAj7kb7HH39cn376qR5++GEdP35cWVlZ2rp1q5KSkiRJdXV12rlzpyTpK1/5Sti+mpqaNGLEiAgeEQAAiAUR+x2czzN+BwcAgNjzufgdHAAAgGghcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMwhcAAAgDkEDgAAMIfAAQAA5hA4AADAHAIHAACYQ+AAAABzCBwAAGAOgQMAAMyJaOAcP35c+fn58ng88ng8ys/P10cffdTjNo7jaOnSpUpNTdWAAQN08803a//+/eecO2XKFLlcLr366qu9fwAAACAmRTRwZs2apYaGBlVVVamqqkoNDQ3Kz8/vcZsVK1Zo5cqVKi8v1+7du+Xz+XTrrbeqra2ty9xVq1bJ5XJFavkAACBGxUdqx42NjaqqqtKOHTuUlZUlSVq/fr2ys7N14MABXXXVVV22cRxHq1at0uLFi3XXXXdJkn7zm9/I6/XqxRdf1EMPPRSau3fvXq1cuVK7d+/W0KFDI3UYAAAgBkXsCk5tba08Hk8obiTp+uuvl8fj0fbt27vdpqmpSX6/X3l5eaExt9utiRMnhm3zySef6N5771V5ebl8Pt9519Le3q5gMBj2AAAAdkUscPx+v4YMGdJlfMiQIfL7/efcRpK8Xm/YuNfrDdumuLhYOTk5uvPOOy9oLWVlZaH7gDwej9LS0i70MAAAQAy66MBZunSpXC5Xj489e/ZIUrf3xziOc977Zv77+f/cprKyUm+++aZWrVp1wWteuHChAoFA6HHo0KEL3hYAAMSei74HZ+7cubrnnnt6nDNixAi99957+uCDD7o89+GHH3a5QnPW2Y+b/H5/2H01LS0toW3efPNN/f3vf9dll10Wtu2MGTOUm5urt99+u8t+3W633G53j2sGAAB2XHTgpKSkKCUl5bzzsrOzFQgEtGvXLl133XWSpJ07dyoQCCgnJ6fbbdLT0+Xz+VRdXa3x48dLkjo6OlRTU6Ply5dLkhYsWKAf/OAHYdtdc801+tnPfqZp06Zd7OEAAACDIvYtqjFjxmjy5MmaM2eOfvGLX0iSfvjDH+r2228P+wbV6NGjVVZWpm9961tyuVwqKipSaWmpRo0apVGjRqm0tFQDBw7UrFmzJJ25ytPdjcXDhg1Tenp6pA4HAADEkIgFjiS98MILKiwsDH0r6o477lB5eXnYnAMHDigQCIT+fvzxx/Xpp5/q4Ycf1vHjx5WVlaWtW7cqKSkpkksFAACGuBzHcaK9iL4WDAbl8XgUCASUnJwc7eUAAIALcDHnb/4tKgAAYA6BAwAAzCFwAACAOQQOAAAwh8ABAADmEDgAAMAcAgcAAJhD4AAAAHMIHAAAYA6BAwAAzCFwAACAOQQOAAAwh8ABAADmEDgAAMAcAgcAAJhD4AAAAHMIHAAAYA6BAwAAzCFwAACAOQQOAAAwh8ABAADmEDgAAMAcAgcAAJhD4AAAAHMIHAAAYA6BAwAAzCFwAACAOQQOAAAwh8ABAADmEDgAAMAcAgcAAJhD4AAAAHMIHAAAYA6BAwAAzCFwAACAOQQOAAAwh8ABAADmEDgAAMAcAgcAAJhD4AAAAHMIHAAAYA6BAwAAzImP9gKiwXEcSVIwGIzySgAAwIU6e94+ex7vySUZOG1tbZKktLS0KK8EAABcrLa2Nnk8nh7nuJwLySBjTp8+rSNHjigpKUkul6tX9x0MBpWWlqZDhw4pOTm5V/eN/8X73Dd4n/sO73Xf4H3uG5F6nx3HUVtbm1JTU9WvX8932VySV3D69eunK664IqKvkZyczH88fYD3uW/wPvcd3uu+wfvcNyLxPp/vys1Z3GQMAADMIXAAAIA5BE4vc7vdWrJkidxud7SXYhrvc9/gfe47vNd9g/e5b3we3udL8iZjAABgG1dwAACAOQQOAAAwh8ABAADmEDgAAMAcAqcXrVmzRunp6UpMTFRGRoa2bdsW7SWZU1ZWpq9//etKSkrSkCFDNH36dB04cCDayzKvrKxMLpdLRUVF0V6KOYcPH9bs2bM1ePBgDRw4UF/72tdUV1cX7WWZc+rUKT3xxBNKT0/XgAEDNHLkSC1btkynT5+O9tJi2jvvvKNp06YpNTVVLpdLr776atjzjuNo6dKlSk1N1YABA3TzzTdr//79fbI2AqeXbNy4UUVFRVq8eLHq6+uVm5urKVOmqLm5OdpLM6WmpkaPPPKIduzYoerqap06dUp5eXk6ceJEtJdm1u7du1VRUaFrr7022ksx5/jx47rhhhuUkJCg119/XX/5y1/005/+VJdddlm0l2bO8uXLtW7dOpWXl6uxsVErVqzQ008/rZ///OfRXlpMO3HihL761a+qvLy82+dXrFihlStXqry8XLt375bP59Ott94a+jchI8pBr7juuuucgoKCsLHRo0c7CxYsiNKKLg0tLS2OJKempibaSzGpra3NGTVqlFNdXe1MnDjRmTdvXrSXZMr8+fOdG2+8MdrLuCTcdtttzgMPPBA2dtdddzmzZ8+O0orskeRs3rw59Pfp06cdn8/n/OQnPwmN/fvf/3Y8Ho+zbt26iK+HKzi9oKOjQ3V1dcrLywsbz8vL0/bt26O0qktDIBCQJF1++eVRXolNjzzyiG677Tbdcsst0V6KSZWVlcrMzNR3vvMdDRkyROPHj9f69eujvSyTbrzxRv3hD3/Q+++/L0nau3ev3n33XU2dOjXKK7OrqalJfr8/7Nzodrs1ceLEPjk3XpL/2GZva21tVWdnp7xeb9i41+uV3++P0qrscxxHJSUluvHGGzVu3LhoL8ecl19+WX/605+0e/fuaC/FrH/84x9au3atSkpKtGjRIu3atUuFhYVyu9267777or08U+bPn69AIKDRo0crLi5OnZ2deuqpp3TvvfdGe2lmnT3/dXduPHjwYMRfn8DpRS6XK+xvx3G6jKH3zJ07V++9957efffdaC/FnEOHDmnevHnaunWrEhMTo70cs06fPq3MzEyVlpZKksaPH6/9+/dr7dq1BE4v27hxo37729/qxRdf1NVXX62GhgYVFRUpNTVV999/f7SXZ1q0zo0ETi9ISUlRXFxcl6s1LS0tXcoVvePRRx9VZWWl3nnnHV1xxRXRXo45dXV1amlpUUZGRmiss7NT77zzjsrLy9Xe3q64uLgortCGoUOHauzYsWFjY8aM0aZNm6K0Irt+9KMfacGCBbrnnnskSddcc40OHjyosrIyAidCfD6fpDNXcoYOHRoa76tzI/fg9IL+/fsrIyND1dXVYePV1dXKycmJ0qpschxHc+fO1SuvvKI333xT6enp0V6SSd/85je1b98+NTQ0hB6ZmZn67ne/q4aGBuKml9xwww1dfubg/fff1/Dhw6O0Irs++eQT9esXfsqLi4vja+IRlJ6eLp/PF3Zu7OjoUE1NTZ+cG7mC00tKSkqUn5+vzMxMZWdnq6KiQs3NzSooKIj20kx55JFH9OKLL+q1115TUlJS6KqZx+PRgAEDorw6O5KSkrrc1/SFL3xBgwcP5n6nXlRcXKycnByVlpbq7rvv1q5du1RRUaGKiopoL82cadOm6amnntKwYcN09dVXq76+XitXrtQDDzwQ7aXFtI8//lh/+9vfQn83NTWpoaFBl19+uYYNG6aioiKVlpZq1KhRGjVqlEpLSzVw4EDNmjUr8ouL+Pe0LiGrV692hg8f7vTv39+ZMGECX12OAEndPn71q19Fe2nm8TXxyPj973/vjBs3znG73c7o0aOdioqKaC/JpGAw6MybN88ZNmyYk5iY6IwcOdJZvHix097eHu2lxbS33nqr2/8n33///Y7jnPmq+JIlSxyfz+e43W7npptucvbt29cna3M5juNEPqMAAAD6DvfgAAAAcwgcAABgDoEDAADMIXAAAIA5BA4AADCHwAEAAOYQOAAAwBwCBwAAmEPgAAAAcwgcAABgDoEDAADMIXAAAIA5/x+2AhDIvI7OFAAAAABJRU5ErkJggg==", + "image/svg+xml": "\n\n\n \n \n \n \n 2021-08-23T18:55:09.891956\n image/svg+xml\n \n \n Matplotlib v3.4.2, https://matplotlib.org/\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "results_test = ode_solver_train(1000)\n", + "results_test_np = results_test.detach().numpy() # Convert to numpy array\n", + "\n", + "# 2D plot of X and Z\n", + "plt.plot(results_test_np[:,2], results_test_np[:,0])\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch: 0\t Loss: tensor(8.9032e-06, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.0939, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.2094, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.2346, requires_grad=True), 'g': Parameter containing:\n", + "tensor(0.6090, requires_grad=True), 'w': Parameter containing:\n", + "tensor(1.2538, requires_grad=True)}\n", + "Epoch: 1\t Loss: tensor(4.9188e-08, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(0.0489, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3273, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.1743, requires_grad=True), 'g': Parameter containing:\n", + "tensor(0.5939, requires_grad=True), 'w': Parameter containing:\n", + "tensor(1.1477, requires_grad=True)}\n", + "Epoch: 2\t Loss: tensor(4.3954e-06, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(-0.2185, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3450, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.3001, requires_grad=True), 'g': Parameter containing:\n", + "tensor(0.4604, requires_grad=True), 'w': Parameter containing:\n", + "tensor(1.2030, requires_grad=True)}\n", + "Epoch: 3\t Loss: tensor(3.3049e-06, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(-0.3561, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3772, requires_grad=True), 'd': Parameter containing:\n", + "tensor(0.0100, requires_grad=True), 'g': Parameter containing:\n", + "tensor(0.5225, requires_grad=True), 'w': Parameter containing:\n", + "tensor(1.2189, requires_grad=True)}\n", + "Epoch: 4\t Loss: tensor(7.9197e-07, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(-0.3269, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.4776, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.2438, requires_grad=True), 'g': Parameter containing:\n", + "tensor(0.5106, requires_grad=True), 'w': Parameter containing:\n", + "tensor(1.1436, requires_grad=True)}\n", + "Epoch: 5\t Loss: tensor(3.3845e-06, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(-0.5730, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.2565, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.1418, requires_grad=True), 'g': Parameter containing:\n", + "tensor(0.3832, requires_grad=True), 'w': Parameter containing:\n", + "tensor(1.2320, requires_grad=True)}\n", + "Epoch: 6\t Loss: tensor(1.0648e-06, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(-0.1834, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.5642, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.1710, requires_grad=True), 'g': Parameter containing:\n", + "tensor(0.8034, requires_grad=True), 'w': Parameter containing:\n", + "tensor(1.1777, requires_grad=True)}\n", + "Epoch: 7\t Loss: tensor(3.9116e-06, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(-0.6395, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.3052, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.4333, requires_grad=True), 'g': Parameter containing:\n", + "tensor(0.3324, requires_grad=True), 'w': Parameter containing:\n", + "tensor(1.2096, requires_grad=True)}\n", + "Epoch: 8\t Loss: tensor(8.1171e-06, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(-0.3244, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.5010, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.0407, requires_grad=True), 'g': Parameter containing:\n", + "tensor(0.6752, requires_grad=True), 'w': Parameter containing:\n", + "tensor(1.2112, requires_grad=True)}\n", + "Epoch: 9\t Loss: tensor(8.6279e-08, grad_fn=)\n", + "{'a': Parameter containing:\n", + "tensor(-0.5913, requires_grad=True), 'b': Parameter containing:\n", + "tensor(0.2579, requires_grad=True), 'd': Parameter containing:\n", + "tensor(-0.1300, requires_grad=True), 'g': Parameter containing:\n", + "tensor(0.4061, requires_grad=True), 'w': Parameter containing:\n", + "tensor(1.1630, requires_grad=True)}\n" + ] + } + ], + "source": [ + "ode_solver_train.fit_random_sample(\n", + " result,torch.optim.Adam,\n", + " {\"lr\": 0.1},\n", + " max_epochs=10\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'a': Parameter containing:\n", + " tensor(-0.5913, requires_grad=True),\n", + " 'b': Parameter containing:\n", + " tensor(0.2579, requires_grad=True),\n", + " 'd': Parameter containing:\n", + " tensor(-0.1300, requires_grad=True),\n", + " 'g': Parameter containing:\n", + " tensor(0.4061, requires_grad=True),\n", + " 'w': Parameter containing:\n", + " tensor(1.1630, requires_grad=True)}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ode_solver_train.coeffs\n", + "\n", + "# Awful results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Predictions for nt = 10000" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([[ 0.0000e+00, 0.0000e+00, 0.0000e+00],\n", + " [ 2.0315e-05, 4.0638e-03, 1.0000e-02],\n", + " [ 8.1293e-05, 8.1327e-03, 2.0000e-02],\n", + " ...,\n", + " [-2.5672e+00, -2.0745e+03, 9.9973e+01],\n", + " [-2.3307e+01, -2.0681e+03, 9.9983e+01],\n", + " [-4.3622e+01, -1.9648e+03, 9.9993e+01]], grad_fn=)" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results_test = ode_solver_train(10000)\n", + "results_test" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAisAAAGdCAYAAADT1TPdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAA9hAAAPYQGoP6dpAABkuklEQVR4nO3deXwU9f0/8NfsnZsc5IIAAaMckUNQ5FDwAhVQa6siaqW1VESKaNWKaEW/Qlqv2mpL1V+lVkux3tYDQVQQQY5wyI1AIIEQQg6yOfec3x+7MzszuxsSyGY2yev5eOThZnZ282FQ95X35/35jCCKoggiIiKiKGXQewBEREREzWFYISIioqjGsEJERERRjWGFiIiIohrDChEREUU1hhUiIiKKagwrREREFNUYVoiIiCiqmfQewNnyer0oLS1FQkICBEHQezhERETUAqIoora2FtnZ2TAYmq+ddPiwUlpaipycHL2HQURERGegpKQEPXv2bPacDh9WEhISAPj+sImJiTqPhoiIiFrCbrcjJydH/hxvTocPK9LUT2JiIsMKERFRB9OSFo6INti63W489thjyM3NRUxMDPr27YunnnoKXq9XPkcURSxYsADZ2dmIiYnB+PHjsWvXrkgOi4iIiDqQiIaVP/7xj/j73/+Ol19+GXv27MEzzzyDZ599Fi+99JJ8zjPPPIMXXngBL7/8MjZt2oTMzExcddVVqK2tjeTQiIiIqIOIaFhZv349rr/+ekyaNAl9+vTBz372M0yYMAGbN28G4KuqvPjii5g/fz5uvPFG5Ofn44033kBDQwOWLl0ayaERERFRBxHRsDJ27FisWrUK+/fvBwBs374da9euxbXXXgsAKCoqQllZGSZMmCC/xmq1Yty4cVi3bl3I93Q4HLDb7aovIiIi6rwi2mD7u9/9DjU1Nejfvz+MRiM8Hg8WLlyIW2+9FQBQVlYGAMjIyFC9LiMjA0eOHAn5ngUFBXjyyScjOWwiIiKKIhGtrLz99tt46623sHTpUmzZsgVvvPEGnnvuObzxxhuq87SdwKIohu0OnjdvHmpqauSvkpKSiI2fiIiI9BfRyspDDz2ERx55BFOnTgUAnH/++Thy5AgKCgpw5513IjMzE4CvwpKVlSW/rry8PKjaIrFarbBarZEcNhEREUWRiFZWGhoagrbQNRqN8tLl3NxcZGZmYuXKlfLzTqcTq1evxujRoyM5NCIiIuogIlpZmTJlChYuXIhevXph0KBB2Lp1K1544QX88pe/BOCb/pk7dy4WLVqEvLw85OXlYdGiRYiNjcW0adMiOTQiIiLqICIaVl566SU8/vjjmDVrFsrLy5GdnY27774bv//97+VzHn74YTQ2NmLWrFmorq7GyJEjsWLFihZtv0tERESdnyCKoqj3IM6G3W5HUlISampquN0+ERFRB9Gaz++I9qwQERERnS2GFSIiIgppx9EavL62CF6vvpMwHf6uy0RERBQZU15eCwBIsJlw04gc3cbBygoREREFOWFvkh/vLdP35sIMK0RERKTi8ngxctEq+Xu9l+IwrBAREZHKo+/vUH0vQt+0wrBCREREspKqBrxTeFR1jJUVIiIiigord5/AJc98rfcwgjCsEBEREY7XNGLGvzbrPYyQGFaIiIgIv/xn+KCi92b3DCtERERd3MGTddhz3B72eb3vy8OwQkRE1IU1uTw4UF6n9zCaxR1siYiIurALF36J2iZ3s+d4OQ1EREREejldUIkGrKwQERF1QS6PF6caXHoPo0UYVoiIiLoYURQx8U9rcKiivkXnCxAiPKLmcRqIiIioi7n/7W0tDioAt9snIiKidtTo9ODDbaV6D6NVGFaIiIi6kJteWaf3EFqNPStERERdQG2TC1uKT2HnsfCbv4Wjd88KwwoREVEXcPs/NmJ7yakzei17VoiIiCiiiirqzzioRAOGFSIiok5u4otrzur1ek8DMawQERF1YgfKa+F0e/UexllhWCEiIuqkfjh6Cle+cHZVlWjAsEJERNQJLd1QjOte/k7vYbQJhhUiIqJO6NEPdug9hDbDpctERESdiCiKsDdG/52UW4NhhYiIqBP5v0/24PXvivQeRpviNBAREVEn8d9NJZ0uqAAMK0RERJ2CvcmFh9/7Qe9hRATDChERUSdw3Utr9R5CxDCsEBERdQKHKxv0HkLEMKwQERF1YB9sPYo+j3yq9zAiKuJh5dixY7j99tuRmpqK2NhYDB06FIWFhfLzoihiwYIFyM7ORkxMDMaPH49du3ZFelhEREQd3roDFbj/7e0R/zmCvrcGimxYqa6uxpgxY2A2m/H5559j9+7deP7559GtWzf5nGeeeQYvvPACXn75ZWzatAmZmZm46qqrUFtbG8mhERERdWhNLg+m/b8N7fKzRLFdfkxYEd1n5Y9//CNycnKwZMkS+VifPn3kx6Io4sUXX8T8+fNx4403AgDeeOMNZGRkYOnSpbj77rsjOTwiIqIOyesVcf6CL/QeRruJaGXl448/xogRI3DTTTchPT0dw4YNw2uvvSY/X1RUhLKyMkyYMEE+ZrVaMW7cOKxbty7kezocDtjtdtUXERFRV9Lk9sDlab9yR6eeBjp06BAWL16MvLw8fPHFF5g5cybmzJmDf/3rXwCAsrIyAEBGRobqdRkZGfJzWgUFBUhKSpK/cnJyIvlHICIiiipr9p/EsKdW6j2MdhXRsOL1enHBBRdg0aJFGDZsGO6++27MmDEDixcvVp0naCKbKIpBxyTz5s1DTU2N/FVSUhKx8RMREUWTTYer8PPXN8Lh9uo9lHYV0bCSlZWFgQMHqo4NGDAAxcXFAIDMzEwACKqilJeXB1VbJFarFYmJiaovIiKizq62yYWb/r5e72HoIqJhZcyYMdi3b5/q2P79+9G7d28AQG5uLjIzM7FyZaCc5XQ6sXr1aowePTqSQyMiIuow7E0uDO1iUz9KEV0NdP/992P06NFYtGgRbr75ZmzcuBGvvvoqXn31VQC+6Z+5c+di0aJFyMvLQ15eHhYtWoTY2FhMmzYtkkMjIiLqMOYu2waPV+f1wzqKaFi58MIL8cEHH2DevHl46qmnkJubixdffBG33XabfM7DDz+MxsZGzJo1C9XV1Rg5ciRWrFiBhISESA6NiIioQ1j8zUF8tbdc72HoShBFvbd6OTt2ux1JSUmoqalh/woREXUqK3efwIx/bdZ7GPj5qN546vr8Nn3P1nx+895AREREUcjh9kRFUIkGDCtERERRpsnlwcWLVuk9jKgR0Z4VIiIiap16hxuDnug6W+m3BCsrREREUWRbySm9hxB1GFaIiIiixJHKetzWTndS7kgYVoiIiKLA0g3FGPfsN3oPIyoxrBAREemsuLIBj36wQ+9hRC2GFSIiIh0dPFmHS5/9Wu9hRDWGFSIiIp2Ioogrnl+t9zCiHsMKERGRTg5V1Os9hA6BYYWIiEgH3/54klWVFuKmcERERO3oeE0jHnh7O9YfqtR7KC0m6PzzGVaIiIja0aiCr/QeQqvpfcdjhhUiIqJ2IIoi7n6zUO9hdEgMK0RERBFmb3Lh/mXbsGpvud5DOSN6TwOxwZaIiCjCFn26p8MGlWjAsEJERBRBx041YtmmEr2H0aExrBAREUXItpJTGPOHjtdQG20YVoiIiCLA3uTCDX/9Tu9hdAoMK0RERG1s0+EqDF6wQu9hdBoMK0RERG1o+c7juOnv6/UeRqfCsEJERNRG7E0uzHxri97D6HS4zwoREdFZ8npFbC2pxk8Xs6ISCQwrREREZ+mhd3/Ae1uO6j2MTovTQERERGdhwce7GFQijJUVIiKiM+D1ili+qwz/XHdY76F0egwrRERErSSKIgo+34PXvi3SeyjtQhD0vTsQwwoREVEriKKIO/6xEWsPVOg9lHYjiqKuP589K0RERK1w8GRdlwoq0YCVFSIiohZwe7z4x9oiFHy+V++htDtOAxEREUU5j1fEuGe/wbFTjXoPpUtiWCEiImpGbZMLH20r7dJBRe+eFYYVIiKiMJpcHjzy3g58uuO43kPp0tqtwbagoACCIGDu3LnyMVEUsWDBAmRnZyMmJgbjx4/Hrl272mtIREREYdU0ujDw98sZVKB/z0q7hJVNmzbh1VdfxeDBg1XHn3nmGbzwwgt4+eWXsWnTJmRmZuKqq65CbW1tewyLiIgopH9vOIIhT66AV9/Zj6ih9zRQxMNKXV0dbrvtNrz22mtITk6Wj4uiiBdffBHz58/HjTfeiPz8fLzxxhtoaGjA0qVLIz0sIiKiIB6viG9/PIn5H+zUeyikEPGwcu+992LSpEm48sorVceLiopQVlaGCRMmyMesVivGjRuHdevWhX0/h8MBu92u+iIiIjpbjU4PXllzEHf8Y6PeQ4k6ek8DRbTBdtmyZdiyZQs2bdoU9FxZWRkAICMjQ3U8IyMDR44cCfueBQUFePLJJ9t2oERE1KV5vCKGPLkCTo9X76FQCBGrrJSUlOC+++7DW2+9BZvNFvY8bVoTRbHZBDdv3jzU1NTIXyUlJW02ZiIi6nq+P1SJu9/czKASxSJWWSksLER5eTmGDx8uH/N4PFizZg1efvll7Nu3D4CvwpKVlSWfU15eHlRtUbJarbBarZEaNhERdREer4jNh6sw9dXv9R4KnUbEKitXXHEFduzYgW3btslfI0aMwG233YZt27ahb9++yMzMxMqVK+XXOJ1OrF69GqNHj47UsIiIiAAAS74rwi0MKh1CxCorCQkJyM/PVx2Li4tDamqqfHzu3LlYtGgR8vLykJeXh0WLFiE2NhbTpk2L1LCIiKiLc3u8uO3/bcCGoiq9h0ItpOsOtg8//DAaGxsxa9YsVFdXY+TIkVixYgUSEhL0HBYREXVSGw5VYunGYgaVDkYQ9d7p5SzZ7XYkJSWhpqYGiYmJeg+HiIiilCiKyJ33md7D6JCmj+6DBdcNatP3bM3nN+8NREREnd4/vyvCS18d0HsYdIYYVoiIqNMqqqjH8yv24ZMfeH+fjoxhhYiIOh1RFLHpcDVmL92C8lqH3sOhs8SwQkREnc6H247h/re36z0MaiMMK0RE1Gk0ON24+81CfPtjhd5DoTbEsEJERJ3Cmv0nsXxXGYNKJ8SwQkREHd7hinr8/HXeLbmzYlghIqIOy+MVcetr3+NIZb3eQ6EIYlghIqIO6f99ewgVdU5s5G60nR7DChERdShlNU2oqnfi6U/36D0UaicMK0RE1GFU1ztxccEqvYdB7YxhhYiIop7L48Vv/8t9U7oqhhUiIopqe47bsbvUjo+3l+o9FNIJwwoREUWt7w9VYuqr3+s9DNIZwwoREUWdLcXVuOetQtgb3XoPhaIAwwoREUWNPcfteH/LUbyx/gicbq/ew6EowbBCRES6sze5UNfkxjV//lbvoVAUYlghIiLdjSn4CrUOTvlQaAwrRESkC4fbg3ve2oKx56QxqFCzGFaIiKhd1TncWL3vJCrrHfhqbzm+2luu95AoyjGsEBFRu5rzn634am85DILeI6GOgmGFiIjaxcPvbkdVvUuupHhFnQdEHQbDChERRUxNowuvrTmEKUOy8d/NR/UeDnVQDCtERNTmGpxuON1ePPbhTnzyw3Es+a5I7yHRWRB0nrJjWCEiojZ30cJVqHO4EWsxAgDqnR6dR0RnQ9R5ys6g748nIqLO4nBFPR5+dzsOnaxDnX8pcgNDCrUBVlaIiOisbC85hfREK27/xwYcrW5E4ZFqvYdEbYzTQERE1GEdKK/F9X/9TnXs4Ml6nUZDnRXDChERtVrBZ3uw/0Qtrs7P1Hso1AUwrBARUYscqazHhqIq/GRYD7yy5hAAIDHGrPOoqCtgWCEiombVOdyIt5rw+Ee7sGb/SdQ0uOTnGtlAS+2Aq4GIiCisN9YdRv4TX2D5zuNYs/8kAOC7gxXy825uQ0vtgGGFiIhUDp2swwNvb8PR6gY88fEuAMBv/rNVft7p9sqPeXsfag+cBiIiIrg9Xmw/WoMhPZMw699bsLesFj8cq5GfV24K5lFUUwS917RSlxDRykpBQQEuvPBCJCQkID09HTfccAP27dunOkcURSxYsADZ2dmIiYnB+PHjsWvXrkgOi4iINB79YAd+ungd3ik8ir1ltQCAA+V18vMeRVrxKh7zzsnUHiIaVlavXo17770X33//PVauXAm3240JEyagvj6wBv+ZZ57BCy+8gJdffhmbNm1CZmYmrrrqKtTW1kZyaEREXd47m0tw49++w7FTjfJNBue9vyPkucrKCvtUqL1FdBpo+fLlqu+XLFmC9PR0FBYW4tJLL4UoinjxxRcxf/583HjjjQCAN954AxkZGVi6dCnuvvvuSA6PiKjL2V5yCpX1DlzePwMPvfsDAOC5L/ad5lVqXq+yytKmwyMKqV0bbGtqfPOfKSkpAICioiKUlZVhwoQJ8jlWqxXjxo3DunXrQr6Hw+GA3W5XfRERUXguT6Ah9vq/fodf/nMzjlY3yMfsja5QLwtLWVkR9b7DHXUJ7RZWRFHEAw88gLFjxyI/Px8AUFZWBgDIyMhQnZuRkSE/p1VQUICkpCT5KycnJ7IDJyLqwI6dasQFT63E4x/uVFVEfjwR6EdxtbI8omyw5ZRQ1yDovO6r3cLK7Nmz8cMPP+A///lP0HPabnJRFMN2mM+bNw81NTXyV0lJSUTGS0TUUW0+XIWCz/egzuHGa2sOodbhxpvfH1EFC2W1xaVYitwSHm/oZlvqvETo+/fcLkuXf/Ob3+Djjz/GmjVr0LNnT/l4ZqbvnhJlZWXIysqSj5eXlwdVWyRWqxVWqzWyAyYi6mAOnaxDdrcY2MxGzH17G45WN8LrFVWhxBOm16S1H0TKlUFuD8MKRV5EKyuiKGL27Nl4//338dVXXyE3N1f1fG5uLjIzM7Fy5Ur5mNPpxOrVqzF69OhIDo2IqNPYeawGlz+/Gr96YzMA4Gh1o/+4HcoitdsbCC5n02uingZqXVWGOia9p4EiWlm59957sXTpUnz00UdISEiQ+1CSkpIQExMDQRAwd+5cLFq0CHl5ecjLy8OiRYsQGxuLadOmRXJoREQd2uMf7kSjy4NnfzYY7xb6lh2vPVChOqfJ7YFRkVbCVlZamVuU1RQnKytdQqeeBlq8eDEAYPz48arjS5YswfTp0wEADz/8MBobGzFr1ixUV1dj5MiRWLFiBRISEiI5NCKiDqW4sgG7SmtwdX4mGl0evPn9EQDAry/ti9omd8jXeL3q/j+Xp20qIsrQ42xlvwvRmYhoWGlJmVEQBCxYsAALFiyI5FCIiDocr1eEwb9F7GXPfwOPV8SS6RciLyNePqfe4YbZGL5Eb1RsMetwB+6Q7DqLioiyZ0XZE0Odl97TQLyRIRFRFFrw8S6MWPglSk/5+k+kasaGoio0OgOho8nlRbjb84hQb4evfJ0yZLQ2trCyQu2NYYWIKAqs2FWG19YckivS/1x3GFX1Tryx/rDqPEEAnIqg4auWBBKJsqItipArMwDQECastJZbueyZlZUuoVP3rBARUWjltU1ItJlhMxshiiJ+/WYhAGBYr24Y0SdFPs/hUocBj1dUTeFoKytB0zuKb5tcgbDiOYvN3Dxh9mshihRWVoiI2tnhinqM/cPXmPEv31Jje2OgQbay3qk6VxRF9VJhj6iqbDjcHtVUj1M1vSOqNoJzuNumIhLuPanzYs8KEVEn98WuMvz6X5tRXtsEAFi5+wScHi++/dG31LjWEbg3j8erDiNeUR0s3F6vKpA0uTyqDxLlbrSiqK6CKIOFqteklUUW1R2YuXSZ2gHDChFRG3J7vPhiVxnqHYFqybz3d2DF7hP429cHAUA1beN0e1XBoc7hVoURrxi8C60yILg8omrLe1XjrCboKFcDnU1jrHI1kJPTQNQOGFaIiNrQu4VHcfebhfjFPzfJx6r8UzslVb47HSuXEzc6PaoP/HqHWxUkRAT3oagqLR6vZpM29WtVlRWXMri0zT4rZ9P7QtRSDCtERGfoSGU9RhWswmtrDsnHXv76AABgY1FV0PnSBm3Kz/dGlwcut3opsDKsuD3eoMqKelpIhMur7EXRBhtFM66isuJwq3tbiKIZwwoRUQtU1Dnw168PyFUSAFj8zUEcr2nCws/2yMe0javKpcTSNIwyjDS6PHB61NMz6qXJ6rDi8qhXA7k86mkh5VSPrzlX8V6u0FNCvHEyRTuGFSIiDbfHq1rmCwB/Wrkfz36xD7OXbpGPHa6sD3qtyaD+36oyWEihQBkUGpxuON3qaRxlmGlyeTThxBs8DaQIJMq9VIDwK3eUj71MKxTlGFaIiBREUcTkl9Zi/LPfoLYpsErn3xuKAQDrDlbKx2xmo/zY6w8F2t1kVRUM/3SLqrKi6VnRVlbcHu20j6bSotl3pVEbVsJUXZRhxc2+E4pyDCtE1GWt2nMCc/6zFSdrHfKx8loH9pbVoszehC3Fp5p9vUGRTBr9lRhlWPF6RXVviFxZ0YQTTcVD+b3LK6q+d7rV4cTXYKsOP0rKINLkUv9cCZtkKdoxrBBRp1dR50Dhkeqg4/M/2ImPt5fi9e+K5GNHKhvkx6canEGvCafe6VuqrNzzpNHl0QQNr+qfAODQTOtop4Fcbq8qcARNA3nVG781utS9KKqelTBLlxlWKNoxrBBRp3fHPzbip4vX4bsDFarjZXbfJm2bDwdW7tQ0BqZ+TjW40Bzlh79U0VCurGl0eUL2iWjDiDbQqFYDebXhxKvZZ0X9vTKseMTwO9gqx85pIIp2DCtE1CmUVDVg8kvfYvE3B1XHm1we7DluBwCs2X8y5GuVH/ANzsBmbuHCirTCRznlIjW2avtRlKFAatpVVTg83mZ7VpweUbUrrcstBvW0KBtslY3B2j1Ywu2zwsoKRTuGFSLqMERRxNd7y1FUEbwK570tR7HzmB1/XL5XtVy4uCowrRPursPKD/F6R+AcZZVFtZGbPxAoe0CkkKM81uTyhAwIQeGkucqKNswETQN5wzbYurRBRtlgqxgXb0ZIp6NtHG9vDCtEFHW0y4Yl3/5YgV/8cxNufmW9vPpGclgRYJR7oSirI8pGWuV2+Mp3UlZWVOeIwY2qTa7gykqj5piygiFXVlzhw4lLE0600zzaaSCnW1tZ0WzNH2YHW21gImqO3qvbGVaIKKoUfL4HA36/HMt3Hg96Tlo2fLLWgaPVjarnTtYFgsjxmib5sXL5sV3xuLYpEERChQ4AaPAfd3m8QbvOhnqdV7Nyp8nlCblcWBtGmm2w1SxddrmDlzKH61lxe72aGxl6wjxmWKHoxrBCRO1qY1EVHvtwB8oUgUIiiiJeWX0Iogj8ZdWBoOdLTwUCypEq9VSQcsqmWrGKxx4mrNQrKih1igqK8nij/7F2o7XGEFUUh9urmmYBgKagZcnBlRXtUuWgpcvaSotm2ke7D4syQGl3u1X93DDLmIlC0XsayKTvjyeizqayzoGUOIt8HxytB9/ZLveRPH3D+arnqhVTNsrwEXg+EEIqFJUUALA3uhXnuUIeVz6ua1JP94iiCEEQ0KDoWZH6V7TTUk0helaaXB7V9wDgcHmC9jkRRRGOZnpQtA22vspL+KXLLo96abOyZ8XjFcMuXQ63MogoGrGyQkRt5qNtxzBi4ZeYs2xbyOfLa5vkoLJmf0XQ85WKAFJV71T1iQDqAKPsPwHUVRPl/ijhpoGU1RSXJ7B5W6hpIG1lpcnlgSiKwZUVbahxe4OCQPA+KprwEtRgK6o2fQueBlLfG6hBVVnxtmhTOC4GomjHsEJEp+X1irj/7W342eJ1qkCh9e8NxRBF4H/bS1XBQFKs2HAtVBipqAuEjEaXB3ZF9QNQV1aUYcXrFWFXTgPVKwOKsrLikn+mctUPEAgvDSGmgbS7wmr3TwF8VZTGEBUYh6ba4tQEGKfH0+y0z+lW/2grLdrKSvjt9llNoY6DYYWoC3O6vViz/+RpP7i+L6rEB1uPYfORaiz13yNHy+sVsfNYjfz9vrLaoHPKFQGjzuFWhRNAvYoHAE7Y1X0tpxQhpFJxbr3TraoOVIeprHhFoN7/YV7nUIcpqeG2PsTeKY0udWhqdHqCqiihKisOtzc41IRYqqyd1tFWVpyq1T/aGxmKqiZa1T4rXnVVxsFqCnVQDCtEnYzT7Q2qWITz8Lvb8fPXN2Lmm4XNnvftj4Epm42K3V6VTtY5VNMlIcOKJnxow0hlvbpqo+xLcXm8qHWoqyTyY00FprYpdJ8KEOhVqdNUVqRlyo2KyoocVpzqwNEYpj8l1LGgaSBNgHF5tPf+CW6wVYeZ5m9sqK3uhLvTMlFHwrBC1Il8sasM+U98gTuXbDrtRl81jS58uK0UAPD1vpM4Wt0Q9twD5XXy4x2K6onSsVPqpcTapcUAcELTZ1Jeqw4r2kqLcjpHu5ussn+lpkFbJQndp6J8TrmHivJ75fRQgzN4agjwraQJFQpCNeJqp4FCrf5RTus4g8KIegfb4GkgMWyDrTSGUI+JOhKGFaIodaC8DusOVgRtfhaOKIoo+GwPnB7f1M7K3SeaPX/9QXWD66YwFRMAOFIZWCZ8qsGF6vrgG/yVasKKNrwAQLldHVa0TbJVmspKlWI6p6ZR/TNrGsMHEmXzbFDVxeEOOgcIVFGUwaTJ5YXXKwYFE19lJUQwcQcHGKdHG1aCb26o/t6jCiOAtmm2+X1WQi2fVo6HqCNiWCGKkHJ7E/75XRH2ltlb/dotxdW45s9rMO21DXh2xb4WvWZXqR2HFQ2sX+wqa/b83aXqcW0rPhXyPK9XVN2JGACKKoO3uz/mr6RYjL7/rWjDCxCopJj8W9drw0tlUGUl8H11M5UVe2Po/hMAqNU8F5gG0kwPyQ22wcEkVINtqMqKdr+SphCNuNoGW5cnxLSQJuA0aMaq/NkutwiXcgdbzViV/Sy8BxB1VAwrRBFQ0+jC9X/9Dgv+txvXv/xdqwPLku8Oy79d/+PbomZX4Eg2FvkqIxaT7z/r9Qcrm+1d2evvKRmS0w0AsLM09BjL7E1wuL0wGQRc1CcFgHpre4kUTi7MTQYQCC9KUjgZ1CPJ931t6LDSOzUWgLrhVgouafEWAJppIP/jOIsRQOjKSoLVpHqurklbWQkdVhqcwcEkXIOtNpg0ubxwuLS9MR5Vc2uoBlu3NqyE2ZTO93M9qq3QtWMl6gwYVogUDp2sw1P/240XVu5X7dXRWou/OShv+e5we/HM8pZVRwDfh9U3+8rl750eL/63vfS0r9t8xBdWZl7aFxajAeW1DpRUBQcGiRRWrhuSDcA37RQq3Bz2V1F6pcSiX3q871iIsCJN+1zoDzQnapuC+makykp+diKA4GkgqcE2z/9zlKt6pJ6V3qlxAHxBQAoMUiDpkRwDQN2zIq36yepm830vrfoJqqz49k6pdwaHmFBLklvWnxJcWanVTFlp911xe8Wg1zS4tAFKPVWlxLBCnRHDCpHfrtIaXP/X7/D6d0X4y6ofcfMr60PuFXI6TrcX72wuAQDMu6Y/BAH4am85SqrCN7AqbT5cjdomN1LiLHjkmv4AgFV7y5t9jSiK2FhUDQAYm9cd52UmAAjfDFvvcMubs117fiYEwVed0Da4AsDhCt95vVNj0dMfBo6GmOI5dsoXRAb3TILFaIAoqlf7ONweeSonX66saFcD+X6+FIpUlRV/cMlJjpG3/pamf6TKSo9uvvFJ4cXt8cof5hmJvrCi7VlJsPkqLg0Ot3+HWd9728y+/z2GmgZqcnmCVgg5QmwA1+QKrrbUaio62soKELwHTNA0lOL74CDFvhTqfBhWKOpU1jnw8fZSvLHuML798WS73L6+yeXB3GXbUNvkxoCsRHRPsGL/iTos+nRPq9/rq73lqKx3onuCFXeNzcXofqkAgA+3Hmvh632NsZedl46rBmYAADYcqgrqsVAqqWpERZ0DZqOAwT2T5DAQLqzsO+GrqqQnWJGVFIOcZN+0i3LVj0Rqru2dGieHlVBTPMf8q4l6JsciI8kKAKr7/0hVFIvRgHP8YUQZjlwer1w9yUv3hS1VZcUfSJLjLEi0mQEEQooUWrL9YUXqC1F+6GclqSsr0vWUQkyd062qWKTG+f4MDc7gsNIQchoouIoSquk2ZGUlaNon9EoliTKgsJJCXQHvDdTF7Thag3cLS7DjWA1EAAOzEnHTiBwM9fcxtKc6hxvPLt+LpRuLVashMhKteHDCefjZ8J5h7zdztl7+6gB+LK9DWrwFS381EvtP1OKWV7/Hsk0luG1kb5zfM6nF7yVVVX56QU+YjAb8ZFhPfHegEh9sO4bZl59z2j+DVEW5YkA6+qbFoVdKLIqrGrCpqAqX9U8P+ZptR08B8P392cxG5PfwTbPsKg0dVvYe94UVqQJzTno8iqsacOBkHUb5w5VEmgbKTVOEFU1lpbbJJVczsrvFICsxBiVVjaq7H0v9Kd0TrEhP8AUB5TSQFEwMgu9nAdqly77nk2Mt6BZrRk2jKxBWmtRhBfCFEumD3GIyIDnOIo8VCASAjEQrDpTXocHhkXtDYsxGxFl9/S+NTo88DdMt1oxTDS40uYOXLiurKGajIG/h7/Gq/75DVVa0jbn1IcJRuO95E0LqChhWIqSyzoG1Byqwq9SOPcftOFnrkH9rtJkNSIq1IDPRisxEGzKSbMhKsiE7KQbZ3WKQmWSD2RjZotf+E7V4fsU+fLFLvbx1a/Ep/HtDMa7Jz8TvpwxEVlJMmHdoW9/sK8ej7+9Aqf/DbUBWInqlxKDwSDVO2B146N0f8NmO43jmZ0PQ3f9B11YOlNfhlTUHAQBP35CP5DgLRvZNxU+G9cAHW4/huRX78MYvL2rRe5XXNuGb/ScBADeN6AkAmDgoA4++b8Chk/U4UF6HvIyEsK8vqqjHoZP1MBkEXJKXBkEQMKpvKoqrGrD+UGXYsLK95BSAQLPs+YrKinSDPqV9/obf/oqw8tXechw4EbyRm3IaqEc3XwXmeE0T3B4vTPLKH9/fW1KMGfFWk9wfcrwmEGqk5tr0RCvS4n1/h40uD+odbsRZTXJzbXKsBd39z6sbbH3//XSLNSMpJnRlJSXOgliLEQ1OD+r80zqAr7k2qMHWX3XJSPCNtd7hlvtVYi1GxFj800NOj7zCJiXWglMNLlWDbVKMLzg53B65mTYpxoyKOieaXB654VlS6wgRVk6z+idonxcGFOpioiKs/O1vf8Ozzz6L48ePY9CgQXjxxRdxySWX6D2sVhFFEftP1OHLPSewas8JbC05hWY3Ea1swPYwTwmCrzyf3c0XXnp0i/GFGf/jvt3jEGs5s7+6oop6/GXVj/hw2zGIou9nTRmcjSsHZsAoCFi19wQ+2laKz3eWYd3BSjx30xB5KiISTjU48dQnu/H+Ft8USU5KDAp+Mhhj89IA+Errr689jD99uR9f7zuJq19cg0U3no+JgzLb5OeLoogFH++CyyPi8v7pqvede2Ue/re9FKv3n8TGoipclJty2vf7eFspPF4Rw3p1Q7/uvqmOBJsZY85Jxdf7TuKLXWXNhpVVe3zhcWTfFCT4pzpG9UvF25tLsP5gZdjXyWGlZzcAwLkZCTAZBJxqcOFodSNyUmJV5+8/USefBwDn+Md64KR6GsjrFXGkyldZ6ZMah/QEq1w1OFHrkHtEpJVAUmUjM0kKK8rKiu9xRoINcVaTHCpO1joQZzXJwSQ13oLkON+fXeoXibEY5cpLt1hLiLDi+zBPtJmRYDOhwelBbZNbnoKJs5oQ7w8rgZ4V32vT/dNA9U63XLGItRoRa/ZVVhoUx1PiLDhUUa9qsJWqPMrKSqLNH1bc3qD/D2iXWWsbbH0/s/meFaKuRvew8vbbb2Pu3Ln429/+hjFjxuCVV17BNddcg927d6NXr156D69ZTrcXG4uq8OWeE/hyz4mgHTsHZiXiwj7JGJCViB7JMegW4ytDN7k9qK53oszehLKaJvmfpacaUXqqCU6PFyfsDpywO7A1xN4Xgr9MPjArEQOzEzEgKxGD/H0WoaYYXB4vNhyqwn83l+DTHcflvRauHpSJByacK39gAcCkwVmYcUlfPPzuD9hxrAYz/rUZ00f3waPXDgj6DfFsfbbjOH7/0U5U1DkhCMAvRufiwYnnqoKY1WTEPeP74fL+6bhv2VbsLavF3W8W4ifDemDBlEFIijWf5RjKsPZABSwmAxZMGaS6fr1T43DLhTn494ZivPz1Afwr9/TVlQ/8fSk3DuuhOj5xUCa+3ncSy3eVYfbleWFf/5U0BdQ/EBClaZldpTWoaXTJH9QSl8eLnf7pHqmyYjMbcW5GAnYft2NXqT0orPzo702RgpPU0HqwXL3Kp7zWgSaXb9lyz+QYGAwCsrvF4EhlA45VN8phRZoW6uGvqGT5A8DxU4qwoqisAEBavBXFVQ2oqHOgT1qcvLV+apwV8VaTHIqqG5yIscTIlcnkWDMS/ddAOiZNA0mVnRNwoLbJLVcs4q0mxPvDX12TW9N46xtPvcODBn+1Jc7iC1OALyhIUz7SVJJyu/1uMWYcgbpnRRqfw+WBlFYSrCbUOtxB00DaewEBCF6RxL4U0llkJuBbTvew8sILL+Cuu+7Cr371KwDAiy++iC+++AKLFy9GQUGBzqNTc7q92H+iFpsOV2FjURXW/lihKulaTAaM6ZeKKwZk4IoB6Wc0heL1iqisd6L0VCOO1zTi2CkpxPi+jlY3orLeiUMnfdMFn/xwXH5tWrwFA7IS0TM5FmajgLomN46easSOozWq+fXL+6fj/ivPDduHMSArEe/dMxp/XL4X/1hbhH+uO4ztR0/hb7dd0CbTQuX2Jvz+o11Y7t+07Jz0ePzxp4MxvHdy2Necl5mAj2aPwZ9W/ohX1xzEB1uPYd3BCvzxp4Mx/rzQUyOnU1XvxIL/7QIAzBrfD71SY4POmTmuH5ZtKsGa/Sfxw9FTGOyvXISyr6wWu0rtMBsFTB6crXruyoEZMHywAzuP2XG0ugE9k4N/lr3JJe+VcsWAwJ8pI9GGvmlxOFRRj41FVUGVrv0natHk8iLBZkJff68HAOT3SPSHlRpcnR+oGFXXO+VgIC0RliorZfYm1DncchWiyL9EuUdyjDzl00MKK6caAPiqTdJ0j/TvR5Y/xBy3h6is+INM9wRfWJH6VqRpoJR4CwRBQHKsBeW1DlTVO5HdLUaurCTHWtBNU1mR/pkYY5IrUrVNLrn3Kd4WqKzUOdyqxtv0hEBlRQoJMRYjYixSZSXQYJsqhRVFgEmK9R1zuAKrgaRAqZyuSYwxo9bhDlphFmo1UINmNZD2e6L2pvd2grqGFafTicLCQjzyyCOq4xMmTMC6detCvsbhcMDhCDTl2e2t3x20Jdb+WIF3C0tQ5/DNqR871Yij1Q1BdypNi7fiiv7puGJAOsbmpZ3x9IzEYBDQPcGK7glW+bdkrfLaJuw5Xovd/n6Y3cftOHSyDhV1TtUN55S6xZoxeXAWpl7YS14p0hyLyYDHJw/EqL6peOC/27C1+BQm/2UtXrp1GEafk3ZGfzavV8SyTSUo+HwPapvcMBkE3DO+H2Zffg6sJuNpX281GfHINf1x1cAMPPTOdhyqqMf0JZswbWQvzL92AOKsLb/2oiji0fd34GStA3np8Zg5rl/I83JSYnH9kGy8v/UY/vb1Qfz9juFh3/P9rUcB+FbxSL+BS9LirRjRJwUbi6qwYtcJ/HJsbtDr1+w/CbdXRL/ucfJeIpKL+6XiUEU91h+sDAor20v8VZWe3WAwBH7/ye+RhP9uPqq6EzLgCzeAL3RI1ywp1oy0eCsq6hw4dLJODmXFVYGVQBKpmnJUsYeLVEGRelWklTdlip6VE/ZAg63vmviukRScpGmgNP+1S4nzhZXqBidEUZRXAzXXsyJNAwG+Rla3N1BZkY7XNblR5w8lFqMByf7qXL3DLYcSVWXFFVxZUU0D+cei3GclUXFMIv98/y84JoMAt1cM2WCr7WHRfk/U1egaVioqKuDxeJCRof6fb0ZGBsrKQm8VXlBQgCeffDLiYyuuapBv8qaUaDNheO9kjOiTgtH9UoM+INpDeoIN6Qk2jDu3u3ys0enBvhO+AHOy1gGP14sYiwmZSVYMzEpCXnr8GY3zyoEZ+OQ3l+Dutwqx57gdt/9jAx6+uj/uvrRvq1bmlJ5qxNy3t8mVg8E9k1Bw4/kYlN3yVTaS4b2T8emcS/DMF3ux5LvDWLqhGN/+eBIv3jIUw3ufvq8EAN7fcgzLd5XBZBDwp1uGwmYOH5buGd8P72/1nf/jidqQPScerygvTb7xgh5BzwO+qaCNRVVYvqssZFj5ao+0Cii4R2hU31Qs3VCM9YeC+1a2lfj2VxmSo76Wg7KlFUHqQL+/XOpXiVcd79fdNxVzoDwQVqRt9nsrppF6hFgRVOoPJdlSZcX/z/JaB1weL8z+TeoAyCuBumtWBEkbwqX6m2uT/RWLqnonGl2B++l0868GAnxhxe3xyqtnkmLMqlAg3eAv3qqtrPgCQ7zNhFj/8XqHR1VZiZUbbANLmlMVYaXRFVxFkSopSTEm+Zj0X0miXPEJ7O9S3eDyV1Z847SaDGyepajU5aeBAAR96IVavSCZN28eHnjgAfl7u92OnJycNh/TBb27yb+tx1mNSE+woV/3uLB9IXqLsRgxNKdbRJYc90qNxQezRmP+Bzvx3paj+MPne7G1uBrP3jRE/h9wc1bvP4m5y7aiusGFWIsRv51wHqaP7gPjWYS8GIsRT0wZ5K+y/ICSqkZMffV7/PGng3HjBT2bfW1xZQMWfOyb/rn/qnNPW2nKy0jAxEEZ+GLXCSxefRAv3Dw06Jz1Bytxwu5AUow57IqdiYMy8H+f7Mbmw1WoqHPIK2IAX9j5ep/UrxL8+ov7+vpW9hy3o7reqarcKCsrSgOyEiEIvsBQbm+SG0mlFT/nakLXOenx2FBUhYOKJtsjVYGVQBJtnwoQaKSVKiqpcRa556Tc34h70j8NJE27SH/+k/7pH2nPlRRFZQXwTVtJm8mZjQLiLEZVZUXZA5JgMyHBGpgGkgoScVYT4v0hxt4UeE2c1Yh4/xLleqdb0bOingaS+lOkANWoqKJIwUl51+UkRWVFDiv+ACNVgeL9YUV5s8N4qwkO95nvnEwUKXpPA+m6KVxaWhqMRmNQFaW8vDyo2iKxWq1ITExUfUVC/8xEzLi0L6aN7IXrh/bAqH6pSE+0RWVQaQ82sxHP3TQYC3+SD4vRgC92ncD1L3+HfWXBS10lHq+IP63cj+lLNqK6wYX8HolYft+luGts7lkFFaXR/dKwfO4luCY/Ey6PiAf+ux2vrTkU9vwmlwezlhai1uHG8N7JuPvSvi36ObPGnwMA+GhbacidaN/f4psCmjw4K+yUVs/kWOT3SIRXBL7U3BF5W0k1qhtccuVOq3uCVe4v2VAUqK7UOdzYX+77O9AG1ViLSV6RpKyuSCuBpI3ZJNK5yo3hjii22lf+OYDAxnCiKMphRVoNZDAIcm9KWU0jXB6vvDut1GCrraxUae79I60IqlLc5blbrK+fJcnfrH6qwSn3gMRZjDAZDXIoqVUsRfaFmEBlRZqKibeaAxUU5T4rFpO8GqjJ5ZErKymKBlttZQUIVE2UYUW5Qkh5Trw/VCmneLRTmTHNVPyIuhJdw4rFYsHw4cOxcuVK1fGVK1di9OjROo2KwhEEAbeN7I13Zo5CdpINRRX1uOGv3+GjbcE7s1bUOTB9yUb8edWPEEVg2sheeHfm6JBNrGcrwWbGX6ddIAePhZ/twT/WFoU8d+Gne7DzmB3JsWa8dOswuWn0dIbkdMOYc1Lh8YpB713b5JKbhcNNAUmu9i+N/nynOqB/6Z8CGn9eetgxSTvhrlMsYd55rAai6KtoSJUTJekePMq+lR/Lw1dWAODgSV9AEcXA3ZaVPSvKLfelhnCn2wtBCDTPAoEpodJTTaisc0IUfX0aKf7qRHe5siI12KqngaTzquudqpVAAFSVFXnZsv+YsmdFrqBYAo23ohgISPFWoxwQnB6v3AMTq22wDepZ8coBpltsoMolBScpmHjF4G39pfeSvleK14QVqW+GSG96/5qu+3b7DzzwAP7f//t/eP3117Fnzx7cf//9KC4uxsyZM/UeGoUxJKcbPplzCS7JS0Ojy4P7lm3DI+/9gLIa343r/re9FNf++Vt8+2MFbGYDXrh5CBb95Pxm+0LOlsEgYN61A/Cby30VkP/7ZDde14SKZRuL8eb3RyAIwJ9uGara7bQlpCbcZZuKVZuVfby9FA1OD/p1j8MFvcKvaAKAa87PAgB8d6ACNQ2BVSGBfpXwK5tG9fM1NivDyjbN/ipa0hSXtLS5qt4pT7cEVVYUNymUtr6XPuyVlZXMJBsMgm8VS0W9Q95SPy3eqlrenik32TbJK4HS4q1y71Sav7JSoV0NFCdVVvw9Kw1OnGoMVFZ8/1SEFU1AkHtTmhQVFJsJNrNBruhJjb/Sfi8SqdlXGVbqFZvLpSqm36QAlWAzBd2rSLu83Hee+lhCiIZwaddcSazm+7bePoCoo9C9Z+WWW25BZWUlnnrqKRw/fhz5+fn47LPP0Lt3b72HRs1IibPgn7+4CH9auR8vf30AyzaVYNmmErlPAfAti/3rbRcE/QYfSQ9cdS5cHhF/X30QT32yG5X1Dsy5Ig8fbj2G+R/uBADcd0XeGS13HntOGvJ7JGLnMTveWHcY9191LgBg2Ubf9vq3XtTrtNOE/brH47yMBOw7UYuVe07gZ8N7oqSqAftO1MJoEFRN01oX902BIPimacprm5CeYEPhEV9z7bBe3UK+RmpglqaBfvT3q/RMjgmacshKtCHGbESjy4PiqgY5qKQnWOUPbgAwGw3ISLTheE0TjlU3ylWK7CR1ZSdLsTHcyVr1SiBAXVlxuD3yNgBp/nvyhOpZkVbeKCsrymXLgHK6xSX/fSRYTRAEAfFWE2oaXSjzL6n27edigMVk8IUvf1jxNdj6/szKYJocIqzYzEbYTL7rJo1FG0yU45PEayorRoMQFOjjNKsL4yxG1coh5X9vRJ1ZVMT0WbNm4fDhw3A4HCgsLMSll16q95CoBYwGAQ9OPA9v//piXNTHtwrH5RGRGmfB3Cvz8PHsse0aVADfVNXvrj4PD/iDxF+/Poj+jy/H797bAY9XxC0jcnDfFeE3ZTvde0vVlTfWH0aD040fjp7CjmM1sBgNp23slVxzvn8qaIdvj5wV/v6Vi/qkqKYUtLrFWjAwyzets/5gJURRxObDvtVVF4bZXXegfxroaHUjTjU4FSuBgv9eDAYB/dJ90z0Hy+vkfpU+mmXUQGAqqKS6UdFcq65UBcJKY9BKICAQXJxurzzdZDII8oe6cjXQqXqn6phUWXF5RLmyI1dWFNNAdXIjre+YVHWRXhOvOS6FqlizETFmk/zzJbFmo1zdqPLv+xJjNsLqv0OztCrJZjbAqqmCaANMjNkIk6J3y2I0BN1mI0YzDaTdGkEZbtp5USJRu9K9skId38i+qfjvzFGoaXChzulGZqKtzRpoz4QgCJhzRR56pcTi6U/3oKLOgViLEfdedg7uGdfvrJqkr8nPQu/UfThS2YC3NwW2wJ80OEuuBJzOtedn4cUvf8S3P1agtsmFFf5+l5bc1mB0v1TsKrVj/cFKDMpOQnWDC1aTAflhloAnxZjROzUWRyobsKvULldW8jRTQJJ+3eOx85gdB07WweP/jT1Un1FOSiw2Ha5GSVWDPA0j7bEiyfSHl+M1TUG71wK+D1ppV9c9x32Vn5Q4i/z3I13PUw2uwB4r/qbbGLNRrioU+xueQ/WsSFNOUoCRnlNWVgDftE9VvXIaKDA9JE1P2cwGGAwCYsxG1d4ooYKJ1WSEzWyUm2sNQnD/icXkCydub+BmixZNWNFWVrThxWY2yhWwGLMx6AaIRJ0Fwwq1maRY81lvf9+WbhjWA1OGZONYdSO6a6YyzpTRIGDGJX3x2Ic78eT/dgPw3f7g3stCbyoXSl56PPp2j8Ohk/V48/sj2OSvjrQsrKThtW+LsO5gpbxp4NCcbs32MgzKTsSRygbsPFaD3f7poP5ZoSte0k62ym33e6cEhxWph6WkqkFeQZMdprJSVtOEk3W+cNBdsVwb8PWt1Drc8qoy5XJuZc+KfF8g/yogaUVQRZ0DR6t9YUUKIsppIKlZWVqerK2saCsu0nSTchpImp6SqhoxZqM83QP4p4E00zdWswE2swHSnngWkyFolZjFP/0kNdyajQaYNX+P2oATq5qOE2BW/FIQYzExrFCnFRXTQESRYjQI6JUa2yZBRfKz4T3luxoDwD3j+uGc9JZPdwmCgMn+Rttnlu+DV/Rt+qa9f08oF+amwGgQUFzVgLe+PwIgcO+gcKS+lR+O1mC3v4Jxfpi9ZaQm2wMn61BU4Zsy6p0WPA0khZXiqobAVvuayor0fXltk7zDbXfNiiUpvOz1h5XU+EB1SloN5HR75WXSyYowLG28JldW/CFFqrDYm9RLlIFAhUVqMlZWVpSUDbYSaRlxqOPBlRV1OLEpporkc8wGVci0hqqsNLM6yGQwqFaOxVj4v3PqvFhZIWolm9mIpTNG4v0tx5Aab8Ekf/BojTtH98GS7w7Lv7XPHN+yyky81YQhPZOwpfiU3DR7ZYgdb5WkFUGf+ntkYsxG5KaFngaSVggdUuy1EmrKSBlWvP5dYrU9K2lxVnlLeWk1UnBlxRdI9vpDlPL5GIsRNrMBTS6vfI8iZU+P73E9Svzb/gcabAP7qRhdvspDnKayIpHCS9D+JoodbCU2f9gIVUXRVk1800CB8GAxBk8VWYxGVTgxGwVYTOopyuAQFRiTySjAZFRUVrTj4m641IkwrBCdgQSbGXeO7nPGr0+Nt2LJLy7E698VYdy53ZtdBaR1y4U52OK/G3f/zAR5W/1wtM8Pyk4M21PUOzUWBiEw9WE0COjbPXxlRXmn8T6a3hZpY7hjpxrl+wIpe1aAQDgp9U/LKCsrgK+6UlrTJDfodlNVVnyPpWkUqbKibGSV7jAu7WqrbXKVQoq2N0TZs6I8BgAxmgqJbxoouLKiDDXWUH0tmmOhela0Y9CuylKerw0ryp4Zoo6OYYVIJyP6pGBEn5bdy0jpxgt6YsuRU9h1vAZP33D+aRuG0+KtyEqyyat2mrsRpdVkRK+UWByWN4OLDbkjb/cEq+o393irKWSDcXY3m2pbfuVqIGlszX2fHGeRgwwQWA0EBJYxS6TpH4vJIC/BlkiVFe1GbNJeJ6GmW1o6DWQzhe5ZsaqmeIxhe1bk703q1UBGgxD0mlizchpIXVnRjkHZWyMIvs3wiM6U3pu3c5KTqIMxGw34488G45PfXNLie0FNHuybqhKEwONwlJvFDQ2z2ZwgCKqN4nqnxoYMTZmKqSGjQUCmtmdFE15SNWFFG4CUlZVEbVixKZ8LhA+b2aBotNXsWyKHleCelVjth78/pMRolgubjcH7o1iMmsqKKXRlRRlWtA22ZqMQtJRZGarMRgNMhsDz2jEoqz22FtzVnKg5eoddVlaIuoDfTjgPMRYTBmYlnHbvm0vP7S5v/3/JueGrML1TY/Gjv7clN0QTLqDeKC4ryRZ0KwFtJUU7DZQcGz6sdIvVVlYC/ztLtJnlqSepudb3WBtWjP5/Bk8DmfzTLNK9e6QpGasiFMSYjRAEdVjxVTzUfSxWkyFoxZZFM42j/d4XXgIBUBCgatI1GgSYm+lZUY7JZjaoKk1EHQ3DClEXYDMb5Y3yTuem4TlYf7ASNrMRk87PDnve8N4pcqgJV+HJVIQVaSM5JW1lJSNBXXlRVlZiLeqpFO2W9gm20FWXRMXUj3bXWKmXJS7MEmGbORBWQlVWpECg7GORKijKYBFy6bIpeBpIW2lRhReD+nuTUVBVVkLtwRIYkxGAC0RnSu9pIIYVIlKJsRix+Pbhpz3vqoEZeH7FPhgMQtg9YrIUYaVv9+BVRWmasNIzRR1olJUVbb+LtrKiXNasDCjKvX+09+NJCLEaSBACgSPWYoK9Sdpnpbmwomym9QcdkzosaCsr2mPBlRX1NJBZE07MBsNpe1YCYwqu6kghzCD4brhI1BxOAxFRh3ROejw+u+8SCFDflVlJGVCGhai+pGmmfRI1q3VS4gLfa+8qraysGA2C6rXKKovyPG1lRV66rFgNFOuf2gHUq3Hk1UCW4BBgs6infAB1z0jInhXN6h9fg20gfJg12++bjOpwYjIKqtdrVyRpf772Z0thxWIyyDdqJIpWDCtEdMZO1/9ybkYC7r2sH4qrGjF5cPCUknJqJCcleJpIeePAjKCwothALs4ib60PqPtXlKuGlD0rvi37fR/iyspKjEX5WN2fAmh6QUwhKivSNJBJXW0J2mcl5DSQemmyMrxolzZrw4t0vnRjQ+XYg1crGeXl6VaTkWGFoh7DChFF1EMT+zf7/MNXn4dnv9iH31wWfIPJ3imBik2Pbuowo6yYpMaFr9Aoz0tQVV8U1RTFaiDlyiB1ZSXUNFDwRnFSSNFWNkxGg2rKxaZZDWQxaisrQtDzqnDib+SVmAy+aSKXR7qZYnCoUo4n1GOicNizQkRd2qzx5+CXY3KDfvsHgHMz4+X+isE91bcIUO52q13No2ywTVL0vSgDivKxsuKiPK6sssRagzeFC9V0azEFBxiLotoircrx7b2iCA2hljI308Ni0twbSK60+PtotZvSKWmbf4lOhz0rRNTlhQoqgO8D/R/TR+DHE3WYOChT9ZyycVa70kgZZJR9MfGafUokygqKsiqj3GtFeqyaXgkxDSQ9DlW9sCqWEFs1m8Bpt983BYWV4J4Wk2q1kDrM2EzBU1OBnxU6rBgNgrzrL1E0YaQmoqh2SV53/HJsbshbBMy5Ig+xFmPQrQ/6pAU2rFPeDVoZSpTvF66yEmoayBZiNZCywTYmxHnS1JAyJFhNxhANtsrwIjTfcGvQVloMMBmUq4PUP0siCNDcQDF0qNG77E/RRe9/H1hZIaIO64GrzsX9V+YF7Z57TnoCLCYDRFGUb+QIQHWecupIeYNA5SZyMSFCiLKKEqqPRblHiyTU3ivaaZ/T7rNiUldSTCF6XJThJdzSZQHqoKa983OD01f5sRh5I0SKHgwrRNShhdrmPynGjA9njYHb61VtTCc9V9PowhBFD4yysqK887GysiLd8VkZYKSGXVuIPhb1aiDf80bFWG2afVa000Imzfb7FqMAiyacKPddMWkqT82tBlKeq72ZovIxwwpFC4YVIuqUBoa5G/X/3ZCP9wqPYvqYXPmYMmzEh1nGLC2BVlYspH1aTldZkSokym6QoMqK0aAKStoeFO29gEwGdWVFeysDa4jl1BJjC8KK1WRALYiiA8MKEXUp1w3JxnVD1Hu+CIKA8ed1x7c/VmCSYj8Y5Y630o65yiqF9LxNFVZ8x6whjikF3RvIZIDFqK7GBPWsmJSVF0Gzz4q6smIL05ciAqrQo+2jCXVciTvekh4YVoiIALw87QJU1jlUu/H2UdygUdqvRVl5kfpeQm4ep/jgl/ZuUS7/NBiEoKqG8saFNk0DrtloUC1VNhuCKy1KyoCkrJhoe1asISpA2tcomdnL0iVx6TIRURSIt5qC7sp82Xnd8etL++LCPilyb0xWt0APTOAeQsHBRBkCQlVWAARPAymrHGbN6iCTEFRZsWi+D/feypAjCIKqZyXclv3qZt/AzrjsZSE9MKwQEYVhMhrw6LUDVMesJiOuGpiBvWV2XNY/HQCQFm8Neq2qsqK5I7JE29Bq1kzJmDWrg5QhI9T3qrEbwvezqHtWgjevC3psDOyMG256iDo3Ll0mIupgXvv5CHi9onw/IrPRgO4JVpysdWBAlq+xV1mlkKaLXB51RULVo6K52aG2Z8XXUBu8hX/geQGiolavuumhsrKieS7cnivaKaF6/5JmbSgiag8MK0REZ8CgWSr871+NxLHqRlyS1x2Auuk2PdFXedFOnyiDQpzVpF79Y1BPCxkN0EwLBTfgKil7WLQb6hkNoSso4VcGGSHt46/sqyFqLwwrRERt4NyMBNVdqLOSbJg4KAMWkxGZ/jtGO08XVhTfGzRLlwH1ih+rSdNgaxRUS6NVlRVNP0u4fVasqvsZhZkeYmWFdMCwQkQUAYIg4JU7RqiOzbu2P+Z/sBPTRvYCACQr7m8Ub1Wv/jEI6mqJ0aBtuA2eJlKu2FAGEqOqwRYwCOGmgU6/MojTQKQHhhUionYy7aJeGJCViIH+vpb0hMDKotQ4a4h7AYXfTt9iDF4tpKSdUpIIEKCcFQoXVloSXIjaC8MKEVE7EQQBF/RKlr/PTLLh3sv6wWYyIjnOEvJ8iUEQNI2xRtX3VpMRomIiyBimsuJ738Bj9Y0MW7YyiKi9MawQEenooYn9Vd9PHpyFNftPBu2y2+TyaKaF1E20NrNRNQ1kbqZnRcmiWYEkH+c0EEURhhUioijyl6nD4HB7VbviAsCQnG6qqRkBgiqQ2MwGVYOtcsWPsmICTW5pyXSP8hxtlUbLZBDg5n781MYYVoiIoojBIKiCymdzLsHGokrcMiJHtVzaalbvs9LsnZXNypCjvreP8iaM1jA72CpDzOk2BzMZGVao7TGsEBFFsYHZiao7SN81NhdbiqsxYWAmNh+pko/HmI3wKEKCagM5zdSNV1Sed/rdbJUh5nSVFbPBgCZwO35qWwwrREQdyOOTB8qPz1Ps6xJnNcHh8sjfK0OFdgWP16tsxA1zbyBj6MeG05RWzCYD4Gj2FKJWi1in1OHDh3HXXXchNzcXMTEx6NevH5544gk4nU7VecXFxZgyZQri4uKQlpaGOXPmBJ1DRETB0hNt+OQ3Y/HNg+MBAMP7pMjPqRts1ZURb5j9WML1rCgfn6awctowQ3QmIlZZ2bt3L7xeL1555RWcc8452LlzJ2bMmIH6+no899xzAACPx4NJkyahe/fuWLt2LSorK3HnnXdCFEW89NJLkRoaEVGnkd8jSX48/9oBMArAfVeeqwooyhBiMgiqaSDVTQ3N6nsVBR4Hjp8ujJwuzBCdiYiFlauvvhpXX321/H3fvn2xb98+LF68WA4rK1aswO7du1FSUoLsbN8yveeffx7Tp0/HwoULkZiYGPK9iYgo2HmZCVjyi4sA+KZ6eqfGwuHyom/3OPkcj1dULXE2trKycrqeFRZWKBLatWelpqYGKSmBMuX69euRn58vBxUAmDhxIhwOBwoLC3HZZZcFvYfD4YDDEZgQtdvtkR00EVEHZDAI+HTOJQB8K4X6ZyZgb1ktLu6bGjZQxFkDFRRbmGZb7Q0ctUQuBKIIaLfdfQ4ePIiXXnoJM2fOlI+VlZUhIyNDdV5ycjIsFgvKyspCvk9BQQGSkpLkr5ycnIiOm4ioo4q3mhBv9f1O+vfbh+Oe8f3w7E1DEGcJ/J7q8nhV50uUy6dVS5dP8zOZVTonQeeSWavDyoIFCyAIQrNfmzdvVr2mtLQUV199NW666Sb86le/Uj0X6gKIohj2wsybNw81NTXyV0lJSWv/CEREXU6ftDj87ur+SImzYM6VechMtOHBCefC5QnEizhFWIlVhBVlcOEOtl2TqHPJrNXTQLNnz8bUqVObPadPnz7y49LSUlx22WUYNWoUXn31VdV5mZmZ2LBhg+pYdXU1XC5XUMVFYrVaYbVaWztsIiLy69EtBuvnXQ5BEHC8plE+rlwZFKuovig3jjM3s3U/UaS0OqykpaUhLS2tReceO3YMl112GYYPH44lS5bAYFAn8lGjRmHhwoU4fvw4srKyAPiabq1WK4YPH97aoRERUQtJ1euspBh8eO8YJMeaVRUU5ZRQuGbbUNizQpEQsQbb0tJSjB8/Hr169cJzzz2HkydPys9lZmYCACZMmICBAwfijjvuwLPPPouqqio8+OCDmDFjBlcCERG1k6E53eTHvxjTByaDgKQYs3xMWXE5/TQQ00pnpHfPSsTCyooVK3DgwAEcOHAAPXv2VD0nzX0ZjUZ8+umnmDVrFsaMGYOYmBhMmzZNXtpMRETt64kpgwD47vKcaDPB4xVVVRYTN1Lpkjpcz0pLTZ8+HdOnTz/teb169cInn3wSqWEQEdEZsJmNWPXb8TAZBBw7FehrOd1NCk0GNuBS2+O9gYiIKKTuCb7FDMlxFjxyTX/0TI7B5sPVzb7mdJvGUcek9zQQIzAREZ3WzHH9MHlwNu4e1xfpCVbMuSIv5HncwbZz6rTTQERE1PlkJcVgw6NXQBAE7C6148s9J3DlgHR8uaccAG9kSJHBygoREbWKNCXw/M1D8PxNQ/DCLUPl5063tJnoTPDfKiIiOiNJMWb8dHhPJNrMeGLKQFw3JBs3DM0+/QuJWolhhYiIztovxuTiL7cOw/VDe8BoEHBhn2S9h0RtSO/N/hhWiIiozeSkxGL9vMvx5l0j9R4KdSIMK0RE1KbSE2ywmY14d+YoTL0wB09MGaj3kKiDY1ghIqKIGNEnBX/46WD8ZFgPpMVb8JNhPfQeEnVQXLpMREQR1S3Wgg2PXgmjQcCXe06gtsmt95ColfS+4xMrK0REFHHSzrb/mz0WL9w8BGPPSdN5RNSRMKwQEVG76ZMWhxsv6In/uyEfeenxmH/tAL2HRB0AwwoREbW73LQ4rHxgHO4c3UfvoVAHwLBCRES6sZgM2PL4Vdj6+FV6D4WawX1WiIioS0uJsyA5zoLFt13AHXApJIYVIiKKCtecn4UXpw7Dn6cOhZX3GCIF/ttARERR5fqhPbD7qasx5/Jz9B4K+Yk6L17mPitERBR1jAYBsy/PQ/dEG3YercHbm0v0HhLpiGGFiIiiksVkwB0X98apBie+L6qEySDg4Ml6vYdFOuA0EBERRbVusRasfugy/PfuUXoPhXTCsEJERB1CarwV6x65HG//+mK9h9Ll6L10mdNARETUYWR3i0FWkg3TR/dBdYMTH20r1XtI1A4YVoiIqEMRBAELrhsEj1dE6alGHK5swMlah97DogjiNBAREXVIRoOAd2aOxn9mcFqos2NYISKiDu2c9Hgs+sn5mHR+lt5DoQjhNBAREXV400b2wvVDs7GztAZHKhv0Hg61MVZWiIioU4izmvDNg+Px2s9H6D0UamMMK0RE1GkIgoArB6Rj6YyRGHtOmt7D6TREndcuM6wQEVGnIggCRvdLw99uvwC/Gpur93CoDTCsEBFRp5RoM+OxyQPx8NXn6T0UOksMK0RE1KnNGn8Otj5+ld7DoLPAsEJERJ1ecpwF790zGrMvO0fvoXRIOu+2z6XLRETUNQzvnYzhvZPRLdaMpz/do/dwqBXapbLicDgwdOhQCIKAbdu2qZ4rLi7GlClTEBcXh7S0NMyZMwdOp7M9hkVERF3Qry7pi8W3XaD3MKgV2iWsPPzww8jOzg467vF4MGnSJNTX12Pt2rVYtmwZ3nvvPfz2t79tj2EREVEXdXV+Jv5y6zC9h0EtFPGw8vnnn2PFihV47rnngp5bsWIFdu/ejbfeegvDhg3DlVdeieeffx6vvfYa7HZ7pIdGRERdlCAIuG5INv579ygMyemm93Cins7brEQ2rJw4cQIzZszAm2++idjY2KDn169fj/z8fFXVZeLEiXA4HCgsLAz5ng6HA3a7XfVFRER0Ji7KTcFH947B7Rf30nso1IyIhRVRFDF9+nTMnDkTI0aE3vq4rKwMGRkZqmPJycmwWCwoKysL+ZqCggIkJSXJXzk5OW0+diIi6lqevuF8/P324XoPg8JodVhZsGABBEFo9mvz5s146aWXYLfbMW/evGbfTxCEoGOiKIY8DgDz5s1DTU2N/FVSUtLaPwIREVGQCQMz8MBV5+o9DAqh1UuXZ8+ejalTpzZ7Tp8+ffD000/j+++/h9VqVT03YsQI3HbbbXjjjTeQmZmJDRs2qJ6vrq6Gy+UKqrhIrFZr0HsSERGdLYNBwJwr8nBx31T8edV+fHegUu8hRQ1R551WBDFCdycqLi5W9ZOUlpZi4sSJePfddzFy5Ej07NkTn3/+OSZPnoyjR48iKysLAPD222/jzjvvRHl5ORITE0/7c+x2O5KSklBTU9Oi84mIiFqizyOf6j2EqHH7xb3w9A3nt+l7tubzO2KbwvXqpW5Wio+PBwD069cPPXv2BABMmDABAwcOxB133IFnn30WVVVVePDBBzFjxgwGDyIi0tWyX1+Mqa9+r/cwCDpvt280GvHpp5/CZrNhzJgxuPnmm3HDDTeEXOZMRETUni7um4o1D13Gpc3Qf+lyu22336dPH4SacerVqxc++eST9hoGERFRi/VKjcVH947BvPd/wH82ckGHXngjQyIiotMouHEwFv4kX+9hdFkMK0RERC0w9cJeWDBloN7D6JIYVoiIiFrAaBAwfUwuHp/c9QLLyL6puv58hhUiIqJWuGtsLj6bc4new2hX1w0Jvhlxe2JYISIiaqWB2YnY9vur9B5Gl8GwQkREdAa6xVrw8NXn6T2MLoFhhYiI6AzNGn8O/jrtAr2H0ekxrBAREZ2FSYOzsOq34/QeRqfGsEJERHSW+nWPx7M/G6z3MDothhUiIqI2cNOIHLw+fYTew+iUGFaIiIjayOX9M7Dy/kv1Hkanw7BCRETUhvIyEvDSrcP0HkanwrBCRETUxqYMycbe/7samYk2vYfSKTCsEBERRYDNbMQ7M0chJc6i91A6PIYVIiKiCMlJicU3D43XexgdHsMKERFRBCXazGy6PUsMK0RERBGWl5GAZb++WO9hdFgMK0RERO3g4r6pWD63a92tua0wrBAREbWT/pmJeJsVllZjWCEiImpHF+Wm4Fdjc/UeRofCsEJERNSOBEHAY5MHYjVXCbUYwwoREZEOeqfG4Zmf8uaHLcGwQkREpJObL8zByNwUvYcR9RhWiIiIdPTotQP0HkLUY1ghIiLS0ZCcbtj46BV6DyOqMawQERHpLD3Rhu2/n6D3MKIWwwoREVEUSIo14y+3DtN7GFGJYYWIiChKXDckGzcO66H3MKIOwwoREVEUef7mIbiIK4RUGFaIiIiiiCAIeH36hUiNs+g9lKjBsEJERBRl4q0mfPPQeFhN/JgGGFaIiIiiUoLNjHdnjtZ7GFGBYYWIiChKnd8zCUt/NRLzrumv91B0xbBCREQUxUafk4a7x/XTexi6inhY+fTTTzFy5EjExMQgLS0NN954o+r54uJiTJkyBXFxcUhLS8OcOXPgdDojPSwiIqIO5YkpA/Uegm5MkXzz9957DzNmzMCiRYtw+eWXQxRF7NixQ37e4/Fg0qRJ6N69O9auXYvKykrceeedEEURL730UiSHRkRE1KH8Ykwu7I1u/OnL/XoPpd0JoiiKkXhjt9uNPn364Mknn8Rdd90V8pzPP/8ckydPRklJCbKzswEAy5Ytw/Tp01FeXo7ExMTT/hy73Y6kpCTU1NS06HwiIqKO7K3vj+CxD3e26888/IdJbf6erfn8jtg00JYtW3Ds2DEYDAYMGzYMWVlZuOaaa7Br1y75nPXr1yM/P18OKgAwceJEOBwOFBYWhnxfh8MBu92u+iIiIuoqbr+4t95DaHcRCyuHDh0CACxYsACPPfYYPvnkEyQnJ2PcuHGoqqoCAJSVlSEjI0P1uuTkZFgsFpSVlYV834KCAiQlJclfOTk5kfojEBERRaWFP8nXewjtqtVhZcGCBRAEodmvzZs3w+v1AgDmz5+Pn/70pxg+fDiWLFkCQRDwzjvvyO8nCELQzxBFMeRxAJg3bx5qamrkr5KSktb+EYiIiDq020b2xpCcbnoPo920usF29uzZmDp1arPn9OnTB7W1tQCAgQMD3ctWqxV9+/ZFcXExACAzMxMbNmxQvba6uhoulyuo4qJ8D6vV2tphExERdSqv3TEcFy1apfcw2kWrw0paWhrS0tJOe97w4cNhtVqxb98+jB07FgDgcrlw+PBh9O7tm28bNWoUFi5ciOPHjyMrKwsAsGLFClitVgwfPry1QyMiIuoy0hNt+Py+S3DNn7/VeygRF7GelcTERMycORNPPPEEVqxYgX379uGee+4BANx0000AgAkTJmDgwIG44447sHXrVqxatQoPPvggZsyYwZU9REREpzEgKxF7/+9qvYcRcRHdFO7ZZ5/F1KlTcccdd+DCCy/EkSNH8NVXXyE5ORkAYDQa8emnn8Jms2HMmDG4+eabccMNN+C5556L5LCIiIg6DZvZiEU/OV/vYURUxPZZaS/cZ4WIiLo6t8eLpz/dg3+uOxyR9++0+6wQERFR+zAZDVhw3SCseegyvYcSEQwrREREnUSv1Fg8Prnz3UOIYYWIiKgTGdU3Ve8htDmGFSIiok5kYHYiZlySq/cw2hTDChERUScz75oBSLS1eiu1qMWwQkRE1MkYDAK+enC83sNoMwwrREREnVBavBVrf9c5VgcxrBAREXVSPZNj8f28K/QexlljWCEiIurEMpNseOSa/noP46wwrBAREXVyV/RP13sIZ4VhhYiIqJPLy0jAK3cM13sYZ4xhhYiIqAuYOCgTj00aoPcwzgjDChERURcxeXC23kM4IwwrREREXURmkg1bH79K72G0GsMKERFRF5IcZ+lwgYVhhYiIqItJjrNg4qAMvYfRYgwrREREXdCC6wbpPYQWY1ghIiLqgrKSYrDwJ/l6D6NFGFaIiIi6qGkX9cLHs8foPYzTYlghIiLqogRBwOCe3fD45IF6D6VZDCtERERd3F1jc/UeQrMYVoiIiAiTB2fpPYSwGFaIiIgIf5k6TO8hhMWwQkRERDAYBCyfe4newwiJYYWIiIgAAP0zE7HryYn489Sheg9FhWGFiIiIZHFWE3p0i9F7GCoMK0RERKSSkWiTHy+dMVLHkfiY9B4AERERRZeclFj8/fYL0C3Wgov7puo9HIYVIiIiCnZ1fvQsZeY0EBEREUU1hhUiIiKKagwrREREFNUYVoiIiCiqRTSs7N+/H9dffz3S0tKQmJiIMWPG4Ouvv1adU1xcjClTpiAuLg5paWmYM2cOnE5nJIdFREREHUhEw8qkSZPgdrvx1VdfobCwEEOHDsXkyZNRVlYGAPB4PJg0aRLq6+uxdu1aLFu2DO+99x5++9vfRnJYRERE1IEIoiiKkXjjiooKdO/eHWvWrMEll/juNVBbW4vExER8+eWXuOKKK/D5559j8uTJKCkpQXZ2NgBg2bJlmD59OsrLy5GYmHjan2O325GUlISampoWnU9ERET6a83nd8QqK6mpqRgwYAD+9a9/ob6+Hm63G6+88goyMjIwfPhwAMD69euRn58vBxUAmDhxIhwOBwoLCyM1NCIiIupAIrYpnCAIWLlyJa6//nokJCTAYDAgIyMDy5cvR7du3QAAZWVlyMjIUL0uOTkZFotFnirScjgccDgc8vd2uz1SfwQiIiKKAq2urCxYsACCIDT7tXnzZoiiiFmzZiE9PR3ffvstNm7ciOuvvx6TJ0/G8ePH5fcTBCHoZ4iiGPI4ABQUFCApKUn+ysnJae0fgYiIiDqQVvesVFRUoKKiotlz+vTpg++++w4TJkxAdXW1ai4qLy8Pd911Fx555BH8/ve/x0cffYTt27fLz1dXVyMlJQVfffUVLrvssqD3DlVZycnJYc8KERFRB9KanpVWTwOlpaUhLS3ttOc1NDQAAAwGdfHGYDDA6/UCAEaNGoWFCxfi+PHjyMry3YNgxYoVsFqtcl+LltVqhdVqbe2wiYiIqIOKWIPtqFGjkJycjDvvvBPbt2/H/v378dBDD6GoqAiTJk0CAEyYMAEDBw7EHXfcga1bt2LVqlV48MEHMWPGDFZJiIiICEAEG2zT0tKwfPlyzJ8/H5dffjlcLhcGDRqEjz76CEOGDAEAGI1GfPrpp5g1axbGjBmDmJgYTJs2Dc8991yLf440i8VGWyIioo5D+txuSTdKxPZZaS9Hjx5lky0REVEHVVJSgp49ezZ7TocPK16vF6WlpUhISAi7guhMSc27JSUlnJaKIF7n9sHr3D54ndsHr3P7idS1FkURtbW1yM7ODupv1YrYNFB7MRgMp01kZysxMZH/MbQDXuf2wevcPnid2wevc/uJxLVOSkpq0Xm86zIRERFFNYYVIiIiimoMK82wWq144oknuK9LhPE6tw9e5/bB69w+eJ3bTzRc6w7fYEtERESdGysrREREFNUYVoiIiCiqMawQERFRVGNYISIioqjGsBLG3/72N+Tm5sJms2H48OH49ttv9R5Sh1ZQUIALL7wQCQkJSE9Pxw033IB9+/apzhFFEQsWLEB2djZiYmIwfvx47Nq1S6cRdw4FBQUQBAFz586Vj/E6t41jx47h9ttvR2pqKmJjYzF06FAUFhbKz/M6tw23243HHnsMubm5iImJQd++ffHUU0/B6/XK5/Bat96aNWswZcoUZGdnQxAEfPjhh6rnW3JNHQ4HfvOb3yAtLQ1xcXG47rrrcPTo0cgMWKQgy5YtE81ms/jaa6+Ju3fvFu+77z4xLi5OPHLkiN5D67AmTpwoLlmyRNy5c6e4bds2cdKkSWKvXr3Euro6+Zw//OEPYkJCgvjee++JO3bsEG+55RYxKytLtNvtOo6849q4caPYp08fcfDgweJ9990nH+d1PntVVVVi7969xenTp4sbNmwQi4qKxC+//FI8cOCAfA6vc9t4+umnxdTUVPGTTz4Ri4qKxHfeeUeMj48XX3zxRfkcXuvW++yzz8T58+eL7733nghA/OCDD1TPt+Sazpw5U+zRo4e4cuVKccuWLeJll10mDhkyRHS73W0+XoaVEC666CJx5syZqmP9+/cXH3nkEZ1G1PmUl5eLAMTVq1eLoiiKXq9XzMzMFP/whz/I5zQ1NYlJSUni3//+d72G2WHV1taKeXl54sqVK8Vx48bJYYXXuW387ne/E8eOHRv2eV7ntjNp0iTxl7/8perYjTfeKN5+++2iKPJatwVtWGnJNT116pRoNpvFZcuWyeccO3ZMNBgM4vLly9t8jJwG0nA6nSgsLMSECRNUxydMmIB169bpNKrOp6amBgCQkpICACgqKkJZWZnqulutVowbN47X/Qzce++9mDRpEq688krVcV7ntvHxxx9jxIgRuOmmm5Ceno5hw4bhtddek5/ndW47Y8eOxapVq7B//34AwPbt27F27Vpce+21AHitI6El17SwsBAul0t1TnZ2NvLz8yNy3Tv8jQzbWkVFBTweDzIyMlTHMzIyUFZWptOoOhdRFPHAAw9g7NixyM/PBwD52oa67keOHGn3MXZky5Ytw5YtW7Bp06ag53id28ahQ4ewePFiPPDAA3j00UexceNGzJkzB1arFT//+c95ndvQ7373O9TU1KB///4wGo3weDxYuHAhbr31VgD8dzoSWnJNy8rKYLFYkJycHHROJD4rGVbCEARB9b0oikHH6MzMnj0bP/zwA9auXRv0HK/72SkpKcF9992HFStWwGazhT2P1/nseL1ejBgxAosWLQIADBs2DLt27cLixYvx85//XD6P1/nsvf3223jrrbewdOlSDBo0CNu2bcPcuXORnZ2NO++8Uz6P17rtnck1jdR15zSQRlpaGoxGY1AyLC8vD0qZ1Hq/+c1v8PHHH+Prr79Gz5495eOZmZkAwOt+lgoLC1FeXo7hw4fDZDLBZDJh9erV+Mtf/gKTySRfS17ns5OVlYWBAweqjg0YMADFxcUA+O9zW3rooYfwyCOPYOrUqTj//PNxxx134P7770dBQQEAXutIaMk1zczMhNPpRHV1ddhz2hLDiobFYsHw4cOxcuVK1fGVK1di9OjROo2q4xNFEbNnz8b777+Pr776Crm5uarnc3NzkZmZqbruTqcTq1ev5nVvhSuuuAI7duzAtm3b5K8RI0bgtttuw7Zt29C3b19e5zYwZsyYoKX3+/fvR+/evQHw3+e21NDQAINB/VFlNBrlpcu81m2vJdd0+PDhMJvNqnOOHz+OnTt3Rua6t3nLbicgLV3+xz/+Ie7evVucO3euGBcXJx4+fFjvoXVY99xzj5iUlCR+88034vHjx+WvhoYG+Zw//OEPYlJSkvj++++LO3bsEG+99VYuP2wDytVAosjr3BY2btwomkwmceHCheKPP/4o/vvf/xZjY2PFt956Sz6H17lt3HnnnWKPHj3kpcvvv/++mJaWJj788MPyObzWrVdbWytu3bpV3Lp1qwhAfOGFF8StW7fKW3S05JrOnDlT7Nmzp/jll1+KW7ZsES+//HIuXW5vf/3rX8XevXuLFotFvOCCC+QltnRmAIT8WrJkiXyO1+sVn3jiCTEzM1O0Wq3ipZdeKu7YsUO/QXcS2rDC69w2/ve//4n5+fmi1WoV+/fvL7766quq53md24bdbhfvu+8+sVevXqLNZhP79u0rzp8/X3Q4HPI5vNat9/XXX4f8f/Kdd94pimLLrmljY6M4e/ZsMSUlRYyJiREnT54sFhcXR2S8giiKYtvXa4iIiIjaBntWiIiIKKoxrBAREVFUY1ghIiKiqMawQkRERFGNYYWIiIiiGsMKERERRTWGFSIiIopqDCtEREQU1RhWiIiIKKoxrBAREVFUY1ghIiKiqMawQkRERFHt/wN+sl74EEv7CwAAAABJRU5ErkJggg==", + "image/svg+xml": "\n\n\n \n \n \n \n 2021-08-23T19:00:20.434921\n image/svg+xml\n \n \n Matplotlib v3.4.2, https://matplotlib.org/\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "results_test_np = results_test.detach().numpy() # Convert to numpy array\n", + "\n", + "# 2D plot\n", + "plt.plot(results_test_np[:,2], results_test_np[:,0])\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "import scipy.io\n", + "\n", + "scipy.io.savemat(\"Duffing_fitRandomSample.mat\", {\"x\": results_test_np})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Mean Squared Error Loss" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "ode_solver = ODESolver(\n", + " ode=ode,\n", + " init_vars=ode_init,\n", + " init_coeffs=ode_coeffs,\n", + " dt=dt,\n", + " solver=\"rk4\",\n", + " optimizer=None\n", + ")\n", + "\n", + "result_train = ode_solver(1000)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "ode_solver = ODESolver(\n", + " ode=ode,\n", + " init_vars=ode_init,\n", + " init_coeffs={'a': -0.5913,\n", + " 'b': 0.2579,\n", + " 'd': -0.1300,\n", + " 'g': 0.4061,\n", + " 'w': 1.1630},\n", + " dt=dt,\n", + " solver=\"rk4\",\n", + " optimizer=None\n", + ")\n", + "\n", + "result_test = ode_solver(1000)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor(1.2754, grad_fn=)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "loss = torch.nn.MSELoss()\n", + "loss(result_train,result_test)" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "ff850aa56af9adbd63a8c19b31d292a54ba02470d4346286a8590995083eda9d" + }, + "kernelspec": { + "display_name": "Python 3.9.6 64-bit ('torchTS': conda)", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/ode/README.md b/examples/ode/README.md new file mode 100644 index 00000000..2d88c0cd --- /dev/null +++ b/examples/ode/README.md @@ -0,0 +1,19 @@ +In this folder, we train the ODESolver model on three different ODEs. Our goal is to estimate all the parameters here. + +Lorenz63: + +![alt text](https://wikimedia.org/api/rest_v1/media/math/render/svg/7928004d58943529a7be774575a62ca436a82a7f) + +Parameters to estimate: $\sigma, \rho, \beta$ + + + +SEIR: + +![alt text](https://miro.medium.com/max/1056/1*dXCHv_pSYiMG90efXiFNPQ.png) + +Parameters to estimate: $\alpha, \beta, \gamma$ diff --git a/torchts/nn/models/hybridode.py b/torchts/nn/models/hybridode.py new file mode 100644 index 00000000..66d2b913 --- /dev/null +++ b/torchts/nn/models/hybridode.py @@ -0,0 +1,58 @@ +import torch +from torch import nn + +from torchts.nn.models.ode import ODESolver + + +class HybridODENet(ODESolver): + def __init__( + self, + ode, + dnns, + init_vars, + init_coeffs, + dt, + solver="euler", + outvar=None, + **kwargs, + ): + super().__init__(ode, init_vars, init_coeffs, dt, solver, outvar, **kwargs) + + if ode.keys() != init_vars.keys(): + raise ValueError("Inconsistent keys in ode and init_vars") + + if solver == "euler": + self.solver = self.euler + else: + raise ValueError(f"Unrecognized solver {solver}") + + for name, value in init_coeffs.items(): + self.register_parameter(name, nn.Parameter(torch.tensor(value))) + + self.ode = ode + self.dnns = dnns + self.var_names = ode.keys() + self.init_vars = { + name: torch.tensor(value, device=self.device) + for name, value in init_vars.items() + } + self.coeffs = {name: param for name, param in self.named_parameters()} + self.outvar = self.var_names if outvar is None else outvar + self.dt = dt + + def euler(self, nt): + pred = {name: value.unsqueeze(0) for name, value in self.init_vars.items()} + + for n in range(nt - 1): + # create dictionary containing values from previous time step + prev_val = {var: pred[var][[n]] for var in self.var_names} + + for var in self.var_names: + new_val = ( + prev_val[var] + + self.ode[var](prev_val, self.coeffs, self.dnns) * self.dt + ) + pred[var] = torch.cat([pred[var], new_val]) + + # reformat output to contain desired (observed) variables + return torch.stack([pred[var] for var in self.outvar], dim=1) diff --git a/torchts/utils/data.py b/torchts/utils/data.py index 8f21889a..6b1b4c3f 100644 --- a/torchts/utils/data.py +++ b/torchts/utils/data.py @@ -122,3 +122,13 @@ def sliding_window(tensor, lags, horizon=1, dim=0, step=1): x, y = data[:, [lag - 1 for lag in lags]], data[:, -1] return x, y + + +def generate_ode_dataset(data, num_steps): + n = data.shape[0] + y_2d = data[:num_steps, :] + y = y_2d.view(1, y_2d.shape[0], y_2d.shape[1]) + for i in range(1, n - num_steps): + y_2d = data[i : i + num_steps, :] + y = torch.cat((y, y_2d.view(1, y_2d.shape[0], y_2d.shape[1])), dim=0) + return data[: n - num_steps, :], y