fastai--fastai/nbs/14_callback.schedule.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58bebdcf",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "#| eval: false\n",
    "! [ -e /content ] && pip install -Uqq fastai  # upgrade fastai on colab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba94684d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| default_exp callback.schedule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ade5de8",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "from fastai.basics import *\n",
    "from fastai.callback.tracker import SaveModelCallback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "474bafbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "_all_ = ['SuggestionMethod']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd5fb973",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b100ee6",
   "metadata": {},
   "source": [
    "# Hyperparam schedule\n",
    "\n",
    "> Callback and helper functions to schedule any hyper-parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b14d9997",
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.test_utils import *"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13ab704c",
   "metadata": {},
   "source": [
    "## Annealing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "052b19e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "class _Annealer:\n",
    "    def __init__(self, f, start, end): store_attr('f,start,end')\n",
    "    def __call__(self, pos): return self.f(self.start, self.end, pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c076fad9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def annealer(f):\n",
    "    \"Decorator to make `f` return itself partially applied.\"\n",
    "    @functools.wraps(f)\n",
    "    def _inner(start, end): return _Annealer(f, start, end)\n",
    "    return _inner"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc79396e",
   "metadata": {},
   "source": [
    "This is the decorator we will use for all of our scheduling functions, as it transforms a function taking `(start, end, pos)` to something taking `(start, end)` and return a function depending of `pos`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14096cab",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "#TODO Jeremy, make this pickle\n",
    "#@annealer\n",
    "#def SchedLin(start, end, pos): return start + pos*(end-start)\n",
    "#@annealer\n",
    "#def SchedCos(start, end, pos): return start + (1 + math.cos(math.pi*(1-pos))) * (end-start) / 2\n",
    "#@annealer\n",
    "#def SchedNo (start, end, pos): return start\n",
    "#@annealer\n",
    "#def SchedExp(start, end, pos): return start * (end/start) ** pos\n",
    "#\n",
    "#SchedLin.__doc__ = \"Linear schedule function from `start` to `end`\"\n",
    "#SchedCos.__doc__ = \"Cosine schedule function from `start` to `end`\"\n",
    "#SchedNo .__doc__ = \"Constant schedule function with `start` value\"\n",
    "#SchedExp.__doc__ = \"Exponential schedule function from `start` to `end`\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ffbe3d2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def sched_lin(start, end, pos): return start + pos*(end-start)\n",
    "def sched_cos(start, end, pos): return start + (1 + math.cos(math.pi*(1-pos))) * (end-start) / 2\n",
    "def sched_no (start, end, pos): return start\n",
    "def sched_exp(start, end, pos): return start * (end/start) ** pos\n",
    "\n",
    "def SchedLin(start, end): return _Annealer(sched_lin, start, end)\n",
    "def SchedCos(start, end): return _Annealer(sched_cos, start, end)\n",
    "def SchedNo (start, end): return _Annealer(sched_no,  start, end)\n",
    "def SchedExp(start, end): return _Annealer(sched_exp, start, end)\n",
    "\n",
    "SchedLin.__doc__ = \"Linear schedule function from `start` to `end`\"\n",
    "SchedCos.__doc__ = \"Cosine schedule function from `start` to `end`\"\n",
    "SchedNo .__doc__ = \"Constant schedule function with `start` value\"\n",
    "SchedExp.__doc__ = \"Exponential schedule function from `start` to `end`\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be1db22d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "tst = pickle.dumps(SchedCos(0, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bcd9c050",
   "metadata": {},
   "outputs": [],
   "source": [
    "annealings = \"NO LINEAR COS EXP\".split()\n",
    "p = torch.linspace(0.,1,100)\n",
    "fns = [SchedNo, SchedLin, SchedCos, SchedExp]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1176888",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def SchedPoly(start, end, power):\n",
    "    \"Polynomial schedule (of `power`) function from `start` to `end`\"\n",
    "    def _inner(pos): return start + (end - start) * pos ** power\n",
    "    return _inner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d25bf388",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABfb0lEQVR4nO3dd3hUxdfA8e/spvfeCYEQCKFD6L33qiKiIgoiAooKiqI/RbCggCACAiIoNhQQBZXeO4TeeyA9Ib23nfePjbyAAQLZZFPm8zz7sHvruQHOTubOPSOklCiKoigVl8bYASiKoiglSyV6RVGUCk4lekVRlApOJXpFUZQKTiV6RVGUCs7E2AEUxsXFRfr5+Rk7DEVRlHLjyJEjN6WUroWtK5OJ3s/Pj5CQEGOHoSiKUm4IIa7fa53qulEURangVKJXFEWp4FSiVxRFqeBUolcURangVKJXFEWp4B6Y6IUQVYQQ24UQ54QQZ4QQ4wvZRggh5gohLgshTgohGt+2rocQ4kLBurcNfQGKoijK/RWlRZ8HTJBS1gZaAGOFEEF3bdMTCCh4jQK+BhBCaIH5BeuDgKcK2VdRFEUpQQ8cRy+ljAKiCt6nCiHOAd7A2ds26w8sl/qaxweEEA5CCE/AD7gspbwKIIRYUbDt7fsaRF5GFsten8oN+0z21rRGoEVgigZztNICDVaYSFu00gYTbBFl8xECRVEqsSAvOz7oW8fgx32obCeE8AMaAQfvWuUNhN32ObxgWWHLm9/j2KPQ/zaAr6/vw4QFgEaXRn5+S7yijxJXZ9X9N5YCE+wwkY6YSWfMpBtmOnfMpAcW0gsNFg99fkVRlLKqyIleCGEDrAZek1Km3L26kF3kfZb/d6GUi4HFAMHBwQ89G4rGxgV78xxkujtHrtwg39mP7J7TyfBqQHpuOik5KSRmJZKQlcDNzJtEp0cTnR5NRFoEEWnHyZf5t47lbeNNLcda1HWpS12XutRxqYOdmd3DhqQoilImFCnRCyFM0Sf5n6SUvxeySThQ5bbPPkAkYHaP5SXC0duesAxBXvAsrK7Nw/LnITg0eR66fgiOAffcL1eXS2RaJFeSrnAp8RKXki5xPuE828K2ASAQBDgG0NitMU08mtDcozmOFo4ldRmKoigG9cBEL4QQwLfAOSnlF/fYbC0wrqAPvjmQLKWMEkLEAQFCiGpABDAEGGqY0P/Lpa4vV26Ek3DuGlav74PtH8OBBXBxI/SZDbV6FLqfqcaUqnZVqWpXlU6+nW4tT85O5mz8WU7EneBozFH+vPInKy6sQCAIcg6ilVcr2vm0o55LPbQabUldlqIoSrGIB80ZK4RoA+wGTgG6gsWTAV8AKeXCgi+DeUAPIAN4XkoZUrB/L2AOoAWWSik/flBQwcHB8lGKml09Hsf6hadoGfMTjdd8q18YfgTWjoPYs1D3cej5GVi7PPSxQd/yPxt/lv2R+9kfuZ8TcSfIl/k4WTjRzqcdXat2paVnS0y1po90fEVRlEclhDgipQwudF1ZnBz8URN9YnQ6P085SO1z39Pmh2mY/XtTNy8H9syGXTPAwg56fg51HwNR2C2EokvJSWFvxF62h21nT/geUnNTsTW1paNvR3pX601zz+aqpa8oSqmoNIk+P1/Hold24HttA62eqofTsGfv3CD2HPw5DiJCoGYP6P0F2HsbJOac/BwORB1gY+hGtt/YTmpuKq6WrvSq1osBNQZQw7GGQc6jKIpSmEqT6AF++uAA5teO08zkML5Lv/3vBrp8OLgQtk4DrSl0nQqNnwON4apBZOdnszNsJ+uurmNP+B7yZB4NXRsyKGAQPar1wNLE0mDnUhRFgUqW6P9ecJKEc2EE73iHmvv3obWxKXzDhGuw7lW4tgv82kLfL8HZvxhR3+M0WQmsu7KOVRdXEZoSip2ZHYMCBjG41mCq2FZ58AEURVGK4H6JvsIVNXP0sCIt3xKZm0f63n333tCpGgxbC33nQtQJ+LoV7J0L+XkGjcfJwonn6jzH2gFrWdp9KS29WvLD2R/o/Xtvxm8bz7HYY5TFL1tFUSqOCpfoHdyt0Okg26Uqadu3339jIaDJczD2IPh3gs3/g2+7QswZg8clhKCpR1Nmtp/Jxsc2MrLeSI7EHmHY+mE8888zbLm+BZ3UPfhAiqIoD6nCJXpHD2sAdE07kbZjBzKvCC10Oy8Y8jM8vgySbsCidrD9E8jLLpEY3a3debXxq2x6bBPvNn+XxOxEXt/xOoP+HMRfV/8iT2fY3yoURancKl6id7cCINe/IflJSWQcOVq0HYWAuoNg3GH90Mudn+kTftjhEovVytSKIYFDWDtgLZ+1/QwhBO/sfof+f/Rn3ZV15OvyH3wQRVGUB6hwid7CxhQLG1MybDwQZmakbt3ycAewcoJBi2HoSshO1XflbJgMOeklEzBgojGhV/VerO63mjkd5mBhYsHkPZMZuHYgG0M3qj58RVGKpcIletDfkE26mYN1q1akbdn6aImyZjcYcwCCX4AD8/U3a6/uNHywt9EIDZ2rdmZl35XMaj8LDRom7pzIU38/xcGouwuGKoqiFE2FTPQO7lYkxWRg26UzuZGRZJ8//2gHsrCDPl/A8H9AaGF5P1j7CmQmGTTeu2mEhm5+3VjdbzUft/mYhKwERm4ayejNo7mUeKlEz60oSsVTIRO9o7s1mam5mDRvC0KQumVr8Q7o1xpe3gutx8OxH2FBCzj/j2GCvQ+tRks//36sG7iOicETOXnzJI+ve5xp+6cRnxlf4udXFKViqJCJ3sFDf0M2LdcCy8aNSd3ykP30hTG11D9FO3IrWDnDiqdg5fOQFlf8Yz+Audac5+o8xz8D/2FIrSGsvrSaPmv68OPZH8nV5Zb4+RVFKd8qZKL/d+RNYnQGtl26kH3hAjlhYQ/Yq4i8G8OoHdDpPTj/F8xvCid+hVK4Yepg4cA7zd/h9/6/08CtAZ8d/ozB6wZzKOpQiZ9bUZTyq0ImejsXCzRaQWJ0Orad9fXlU7cWs/vmdlpTaPcmvLQbnGvAmlHw82BIDjfcOe6jun11vu78NXM7ziUzL5MRm0bw9u63uZl5s1TOryhK+VIhE71Gq8HV15YbZxMw8/XFvGZNUjcboPvmbm6B8MJG6PEZhO6B+S3g8BLQlfwTrkIIOvp25I/+f/BS/ZfYGLqRfn/047cLv6knbBVFuUOFTPQAtZp7EB+eRlxYKrbdu5F59Ci5MbGGP5FGCy1Gw5j94NME/p4A3/WGm5cNf65CWJhYMK7ROFb3W01tp9pMOzCN5zc8z9Xkq6VyfkVRyr4HJnohxFIhRKwQ4vQ91r8phDhe8DothMgXQjgVrAsVQpwqWPdo5SgfUUCwOxoTwYX90dj16AFSkrp5c8md0NEPnv0D+s+H2DOwsDXsmWPwImn3Ut2+Oku6LWFa62lcTrrM42sfZ9GJRepmraIoRWrRf4d+isBCSSlnSCkbSikbAu8AO6WUCbdt0rFgfaHlM0uKhY0p1eq5cOFQNCZVq2FWw5/UDRtK9qRCQKNnYOwhqNEFtnwASzpB9KmSPe+t0wsG1BjAnwP+pJNvJ+Ydn8fTfz/NhYQLpXJ+RVHKpgcmeinlLiDhQdsVeAr4pVgRGVBgS0+y0nK5fjoeu+49yDhyhLy4kh8Oia0HPPkjPPE9pETC4g6w7aMSK5J2NxdLF2a2n8nsDrOJyYhhyN9DWHhioWrdK0olZbA+eiGEFfqW/+rbFktgkxDiiBBi1AP2HyWECBFChMQZKBlXqeOEpZ0ZFw5EY9u9G0hJSkl239xOCKgzQN+6rzdYP1/twjZwo/RKGXSp2oU/+v9BV9+uzD8+n2H/DONa8rVSO7+iKGWDIW/G9gX23tVt01pK2RjoCYwVQrS7185SysVSymApZbCrq6tBAtJqNdRs5k7oyZvoPKpiVr06qRs2GuTYRWblBAO/hmdWQ24mLO0O6ydBdlqpnN7RwpHP23/OrPazCEsLY/C6wfx87mdVKE1RKhFDJvoh3NVtI6WMLPgzFlgDNDPg+YoksIUnOp3kUkgsdj26kxESQt5NI4w3r9FFPzKn2Yv6OWu/bglXHjAxigF18+vGmn5rCPYI5tNDn/Ly1pfVuHtFqSQMkuiFEPZAe+DP25ZZCyFs/30PdAMKHblTklx8bHCrasvpnRFYd+0GOp1hSiI8CnNb6DUDnt8AWjP4YQD8ORYyE0vl9K5WrizovIB3m79LSHQIj619jF3hu0rl3IqiGE9Rhlf+AuwHagkhwoUQI4QQo4UQo2/bbCCwSUp5e9F2d2CPEOIEcAj4W0pZwsNeCteoW1WSYjIIT3PAzM+PlPVGCeP/VW0Jo/dCmzfg+C8wvzmcW1cqpxZCMCRwCCt6r8DZ0pmxW8fy2aHPyMnPKZXzK4pS+kRZ7KsNDg6WISGGG3YvdZJfph5EaASd7Q8Sv3AhNXbuwNTNzWDneGRRJ/St+uhTEDRA3+K3KZ24svOz+SLkC34+/zNBzkHMaDcDXzvfUjm3oiiGJYQ4cq9h7BX2ydjbCY2gSU8/EiLTSazZUf/wVEmPqS8qzwbw4nbo/D5cWA/zmupb+aXwBWyuNeed5u8wp+McwlPDGfzXYDaElpGfi6IoBlMpEj1AQLAb9q6WnDiWhVlgIMl//23skP6f1hTaToDRe8A1EP4YDT8+pp+ovBR09u3Mqr6rCHAI4M2db/LJwU9UV46iVCCVJtFrtBoa96hK3I1UMto8TtaJk+TcKJ1EWmSuNeH59dBzBtw4AAtawsHFpVIkzdPGk6U9lvJc0HP8cv4Xhq0fRkRaRImfV1GUkldpEj1ArRYe2DpZcC6jGhJByj8lP0vUQ9NooPkoGHsAqjSH9W/Csp5ws+SnEDTVmDKx6UTmdJzDjZQbPPnXk+yN2Fvi51UUpWRVqkSv1Wpo3q8acZFZJDR/gpSy1H1zNwdf/UNWA76GuPPwdWvYPQvyS76MQWffzqzoswJ3K3de3vIyi04sUqWPFaUcq1SJHqBmMw/cq9lxwb4N6VfDyLpw0dgh3ZsQ0HCovoxCze6wdSp800k/UqeE+dr58mOvH+ldvTfzjs9j/LbxpOaklvh5FUUxvEqX6IVG0GZwAFk5Wq779STlr7+MHdKD2brDkz/A4B8gLQYWd4QtH0JuVome1tLEkk/afMI7zd5hT8Qehv49VNW5V5RyqNIlegCPavbUauFBmE8nojftRZbCzU6DCOoHYw9Cg6dgzxf6mvfX95foKYUQDK09lG+6fUNKTgpD/x7K9hulV7pBUZTiq5SJHqDlAH80JhrO2bYl/fBhY4dTdJaOMGA+PLsG8nNgWQ/4eyJkl2y3SrBHML/2+RU/Oz/Gbx/PNye/UYXRFKWcqLSJ3trBnOCeVbnpUp9zKw8YO5yH598JXt4PzUfr56ld0BIul2wNHw9rD77r8R09q/Vk7rG5TNo1icy8zBI9p6IoxVdpEz1Ao57VcdCmcDzRj/S4FGOH8/DMbaDnZ/oJyk0t9Q9ZrXkZMoo6T8zDszCxYHrb6bzW+DU2hG5g+IbhxGaUwFy8iqIYTKVO9BqthnZ93MnVWrFjYTls1f/Ltzm8tBvaToRTv+mLpJ3988H7PSIhBCPqjWBup7mEJofy1N9PcTb+bImdT1GU4qnUiR7Ap3tzqiXuJTTChNCT5bg+u6kFdP6fvm6OnSf8NgxWPA2p0SV2yg5VOrC853K0QsvwDcPZct1I5Z8VRbmvSp/ohUZDo1aOWKdHsv2Hs2SmlfMaL571YeQ26PIhXNoM85vBsR9LrEhaLada/Nz7ZwIcA3hjxxssO71M3aRVlDKm0id6AMcB/Qg69z1ZablsW36+/CcqrQm0eQ1e3gdudfRlkH8YAImhJXI6F0sXvu32Ld38uvHFkS+YemCqmohcUcoQlegB82rVcPN3pmbiLkJP3uT0zgpSzMulBgz/G3rPgvAQ/cicAwtBl2/wU1mYWPB5u88ZWW8kqy6uYtzWcaTllM68uIqi3F9RZphaKoSIFUIUOg2gEKKDECJZCHG84PX+bet6CCEuCCEuCyHeNmTghmY/cCCex3/Dp6oZe1dd5mZ4BUlSGg00HQljDkDV1rBhEiztAbHnDX8qoWF84/F82OpDDkYd5LkNzxGTHmPw8yiK8nCK0qL/DujxgG12SykbFrymAgghtMB8oCcQBDwlhAgqTrAlya53LzQWFtRP34G5lQmblpwmJyvP2GEZjkMVeHolDFwM8ZdhUVvYOaNEiqQNChjE/M7zCU8N5+l/nuZiYhmuJ6QolcADE72UchfwKAOzmwGXpZRXpZQ5wAqg/yMcp1RobW2x696dnPV/0Plpf5JiMtj+QwXor7+dENDgSX2RtMA+sP0jWNwBIo8Z/FStvVvzfc/vkVLy3PrnOBh10ODnUBSlaAzVR99SCHFCCLFeCFGnYJk3EHbbNuEFywolhBglhAgRQoTExcUZKKyH4/DE4+jS07G7epAWA/y5fCSW41vCHrxjeWPjCk8sgyE/Q0a8viLm5vch17BPuQY6BfJT75/wsPZg9JbRrL+23qDHVxSlaAyR6I8CVaWUDYCvgD8KlotCtr1n81hKuVhKGSylDHZ1dTVAWA/PskkTzPz8SFq1ikbdfPFv5Mr+3y8TfiHRKPGUuMDe+r77Rs/C3i/h61YQusegp/i3bEID1wa8testvj/zvUGPryjKgxU70UspU6SUaQXv/wFMhRAu6FvwVW7b1AeILO75SpIQAvvHBpF55Ag510Lp9FxtHNyt2PjNaVJuVtCaLpYO0G8uDFsLUgff9Ya/Xocsw5WEsDe3Z1HXRXSt2pWZITOZeXimmshEUUpRsRO9EMJDCCEK3jcrOGY8cBgIEEJUE0KYAUOAtcU9X0lzGDAAtFqSf1+NmYUJvV6uj9RJ/l5wkuzMCnRz9m7V2+vH3bccB0e+gwUt4OImgx3eXGvOjHYzGFJrCN+f/Z739rynxtorSikpyvDKX4D9QC0hRLgQYoQQYrQQYnTBJo8Dp4UQJ4C5wBCplweMAzYC54DfpJRnSuYyDMfE1RWbDh1IWvMHMicHB3creoyqS1J0Bpu+OY0uvwK3RM2sofvHMGIzmNvCz0/A6hchPd4gh9dqtExuPplxDcex7uo6Xt32Khm5GQY5tqIo9ybK4qiS4OBgGRISYrTzp+3cSdhLo/GeMxu7HvqRpWf3RLL9x/PUa+9N2yE1KfglpuLKy9bPUbt7Flg4QK/Poc4g/cgdA1h1cRXTDkyjnks95neej725vUGOqyiVlRDiiJQyuLB16snYQli3aYOplxeJv6y4tSyojRcNu/pyamcExzbfMGJ0pcTEHDpOhlE79WPwV72gL5KWEmWQwz9e83FmtZ/F2fizqtSxopQwlegLIbRaHJ58koyDB8m++v9zpLYa6E+NYDf2/36FCwcMk/DKPI+6MGILdPsIrmzVl0A+8r1BiqR1qdqFr7t8TWRaJMPWD+NGSiX4AlUUI1CJ/h4cHhsEpqYk/frrrWVCI+jyXBA+gY5sW36e62cM03dd5mlNoNUr+pu1nvVh3auwvB8kXCv2oZt7Nmdp96Vk5GYwbP0wLiRcMEDAiqLcTiX6ezBxccGuaxeS1vyBLvP/h1ZqTTX0fKkeTt7WbFh8muiryUaMspQ5++uHYfaZA5HH9UXS9s8vdpG0Oi51+K7Hd2g1Wp7f+DzHY48bIlpFUQqom7H3kX7oEDeGPYfnxx/rW/i3r0vOZs3Mo2Sm5TLg9Ua4+toaKUojSY6Av9+AixvAOxj6zwO32sU6ZERaBKM2jSIuM44vO35JS6+WBgpWKQtyc3MJDw8nKyvL2KGUaxYWFvj4+GBqanrH8vvdjFWJ/j6klFzt2xeNpRXVVv72n/WpCVn8PvMIeTk6Br7RGCcvayNEaURSwunVsP4t/QNW7SZCmzfAxOyRD3kz8yajNo8iNDmUme1n0sm3kwEDVozp2rVr2Nra4uzsXPFHrZUQKSXx8fGkpqZSrVq1O9apUTePSAiB45NDyDp1isyTJ/+z3tbJgv7jG6HRCP788hhJMZVsTLgQUO9xfZG0oP6w41NY3B4ijjzyIV0sXVjWfRmBToG8seMN/rn6jwEDVowpKytLJfliEkLg7Oz80L8VqUT/APYDB6Cxtibhhx8LXe/gbkW/8Q3R5Uv+mF0Jkz2AtQs8/i089StkJsGSLrDxXch5tJ+Fvbk933T7hkZujXh799usvrjasPEqRqOSfPE9ys9QJfoH0NrYYD9oECkbNpAbW/hYb2dvG/q/1oj8PB1/fHG0ciZ7gFo9YOwBaPwc7J8HX7eEa7se6VDWptZ83eVrWnu3Zsr+Kfx4tvAvWkVRHkwl+iJweuZpyMsjacWv99zGxceGAa83Ij9f8scXR0mMTi/FCMsQC3voOwee+wsQ8H1fWDcesh5+dJKFiQVfdvySzr6d+ezwZyw5tcTg4SqVixCCCRMm3Po8c+ZMpkyZcuvz4sWLCQwMJDAwkGbNmrFnj2GruRqLSvRFYFa1Kjbt25P466/ocnLuuZ2ztz7Z63SSNbOOVpzpCB9Ftbb6cfetXoGjy2F+C7iw4aEPY6Y1Y2b7mfSq1osvj37JV8e+qliTwSilytzcnN9//52bN2/+Z91ff/3FokWL2LNnD+fPn2fhwoUMHTqU6OhoI0RqWCrRF5Hjs8+QHx9Pyj/3vzno7G3DwAmN0Wg1/PHFUWJCDVfut9wxs9I/UTtyC1g6wi9PwqoRkP7f/2T3Y6Ix4ZM2nzAoYBCLTy5m9pHZKtkrj8TExIRRo0Yxe/bs/6z77LPPmDFjBi4uLgA0btyY5557jvnz55d2mAZnYuwAygvrVq0wq+FP4vIfsO/f/743RBw9rBk0sTF/zjnGn3OO0XtMfbxrOpZitGWMdxMYtQP2zIZdM+Dqduj5OdR9rMhF0rQaLR+0/ABTjSnLziwjOz+bSc0moRGqrVIefbjuDGcjDdsICvKy44O+dR643dixY6lfvz5vvfXWHcvPnDlDkyZN7lgWHBzM99+X/8ly1P+SIhJC4PTMs2SdPUtmEcb427lYMnBCE2wczFk39wRXjxtnesQyw8QMOkyC0bvBsRqsHgG/DNE/eFVEGqHh3ebvMixoGD+f/5lpB6apCUyUh2ZnZ8ewYcOYO3fuA7eVUlaIkUKqRf8Q7Af0J+7LL4lfugyrpk0fuL2NozmDJjZh3bwTbFh0ig7PBBLU2qsUIi3D3GrDiE1wcCFsnaYvktZtKjQeDpoHtzuEEEwMnoiZ1owlp5aQp8tjSsspaDXako9dMZiitLxL0muvvUbjxo15/vnnby0LCgriyJEjdOr0/w/pHT16lKCgIGOEaFCqRf8QNBYWOD79NGnbt5N95UqR9rGwMaX/aw3xqe3E9h/OE7I+VPUva7TQciyM2Q/ejfRTFy7vB/FF+5kKIXi10au83OBl/rj8B//b+z/yi1lvR6lcnJycGDx4MN9+++2tZW+99RaTJk0iPl5frPD48eN89913jBkzxlhhGkxRZphaKoSIFUKcvsf6p4UQJwte+4QQDW5bFyqEOCWEOC6EMH5NAwNwHPoUwtyc+GXLiryPmYUJvcfUJ6CpOwf/vMrOny9U7Jmqisqpmr5IWr+vIOqkfnLyvXMh/8FTNgohGNNwDK80eoV1V9fxzu53yNNV4KkeFYObMGHCHaNv+vXrxwsvvECrVq0IDAzkxRdf5Mcff8TT09OIURrGA2vdCCHaAWnAcill3ULWtwLOSSkThRA9gSlSyuYF60KBYCnlQw2zKCu1bu4l6sMPSV61mhrbtmLi6lrk/aROcmDtVY5uuI5fPWe6jayLqbnqcgD0E5r8PQEu/A1ejfXJ3+M//9wKtfT0UmYfmU13v+582vZTTDWmD95JKXXnzp2jdu3iFb5T9Ar7WRar1o2UcheQcJ/1+6SUiQUfDwA+RQ+3fHIePhyZl0fCjz891H5CI2g5wJ/2Q2tx/XQ8a2YdJS0xu4SiLGfsPGHIT/D4Mki6oa+Zs/0T/ZSGD/BC3ReYGDyRjaEbmbRrkpp0XFHuYug++hHA+ts+S2CTEOKIEGLU/XYUQowSQoQIIULi4sr2CBWzqlWx7dKFxBUr0KU//BOwddt502tMfZJiMlj1WQhxN1JLIMpySAioOwjGHdYPvdz5GSxqB2GHH7jrc3We483gN9l8fTNv7nyT3HyV7BXlXwZL9EKIjugT/aTbFreWUjYGegJjC7qBCiWlXCylDJZSBrs+RHeIsTiPHIEuOZnE31Y+0v5+9VwY9GYThIDfZx1Vwy9vZ+UEgxbD0JWQnQbfdoUNkyHn/l+qw+oM4+1mb7P1xlYm7pyokr2iFDBIohdC1AeWAP2llLfm15NSRhb8GQusAZoZ4nxlgWWDBli1aEHC0qXosh+t+8XFx4bH3w7GydOa9YtOqRE5d6vZTT8yp+kIODBfP6PV1R333eXp2k/zTrN32Ba2jQk7J6hkrygYINELIXyB34FnpZQXb1tuLYSw/fc90A0odOROeeUy+iXy4uJI/v33Rz6Gtb05A99oRECwfkTO5m/PkJujhgreYmEHvWfB8H9AYwLL+8Of4/TlkO9haO2hTG4+me1h23lj5xsq2SuVXlGGV/4C7AdqCSHChRAjhBCjhRCjCzZ5H3AGFtw1jNId2COEOAEcAv6WUj58VasyzKp5cywbNCD+myXI3EdPJiZmWrq+EETLgf5cOhLLmplHSYnPfPCOlYlfa3h5L7QeD8d/0j9ode6ve27+VOBTvNv8XXaE7VDJXqn0ijLq5ikppaeU0lRK6SOl/FZKuVBKubBg/UgppaOUsmHBK7hg+VUpZYOCVx0p5cclfTGlTQiB8+iXyI2MJPmvv4t9rMbdq9J7TH2S4zJZ+UkI4efvOdipcjK1hK5TYeRW/WQnvz4NK4dDWuH3N4YEDmFy88kq2Su32NjY/GfZlClTmDlzJgDDhw/H29ub7ILu2Js3b+Ln5wdAaGgolpaWNGzY8NZr+fLlt45z7NgxhBBs3LjxjuNrtVoaNmxI3bp16du3L0lJSSVzcfehnowtJpsOHTAPDCR+8WJkfvG7XPzqufDE28FY2pmx9svjHNt0Q/Xb3827sb5IWsf34PzfML8pnPhVP4ftXZ4KfEole+WhaLVali5dWug6f39/jh8/fus1bNiwW+t++eUX2rRpwy+//HLHPpaWlhw/fpzTp0/j5ORklGqYKtEXkxACl5dGkXPtGql3fZM/Kgd3Kx6f1ITqjVzZ9/tlNiw+TXameurzDlpTaP8mvLQbnANgzSj4eTAkh/9n09uT/cSdE9U4e+W+XnvtNWbPnk1eXtH/z0kpWbVqFd999x2bNm2655yuLVu2JCKi6IX8DEUVNTMA227dMKvhT9z8Bdh2747QFv9pVzMLE7q/WJcTW8PY9/sVVn56mJ4v1cPZ+7+/elZqboHwwgY49A1s/VDfd9/1Q2jywh1F0p4KfAqd1DH90HTe2vkWn7f/XD1Ba0zr34boU4Y9pkc96Dm92Ifx9fWlTZs2/PDDD/Tt2/eOdVeuXKFhw4a3Pn/11Ve0bduWvXv3Uq1aNfz9/enQoQP//PMPgwYNumPf/Px8tm7dyogRI4od48NSLXoDEFotruPGkXPlCin/rH/wDkU9rhA07OLLgNcbkZuVz6rpIZzbF6m6cu6m0UKL0fqhmD7B+lIK3/WGm5fv2Ozp2k8zqekkttzYop6gVe5r8uTJzJgxA53uzppUd3fdtG3bFtB32wwZMgSAIUOG3NF9k5mZScOGDXF2diYhIYGuXbuW3oUUUC16A7Ht1g3zmjW5OX8+dj17IEwM96P1CnBg8LtN2bz0DNuWnyfiYhLtn6ql6uTczdEPnv1DPypn42RY2Bo6vA0tXwGt/u/jmaBn0EkdM0JmIHYJPmv3GSYa9d+g1Bmg5V2SatSoQcOGDfntt98euG1+fj6rV69m7dq1fPzxx0gpiY+PJzU1FVtb21t99MnJyfTp04f58+fz6quvlsJV/D/VojcQodHgMm4sOaGhJP9172F/j8ra3px+4xvRtLcfFw5Gs/LTw8RHVOI5ae9FCGj0DIw9BDW6wJYpsKTTHd0Ew+oMY0KTCWy6vonJuyerqpdKod59991bo3HuZ8uWLTRo0ICwsDBCQ0O5fv06jz32GH/88ccd29nb2zN37lxmzpxJbjGGYz8KlegNyLZLF8xr1+bmgq+RD3Ejp6g0GkGzvtXpN74h2Rl5rPw0hNM7w1VXTmFsPfRF0gYv11fGXNxBP9FJrv4m2fC6w3mt8WusD13Pu3veVfXsK4mMjAx8fHxuvb744ot7blunTh0aN258x7J/++j/fc2dO5dffvmFgQMH3rHdY489xs8///yfYzZq1IgGDRqwYsUKw1xQET2wTLExlPUyxfeTum0b4WPG4jFtKo5PPFFi58lIyWHr92e5cSaB6o1c6fhMIBbW6uZioTIS9F05J34Bl5rQbx74NgdgyaklfHn0S/pW78u01tPUTFUlSJUpNhyDlylWHo5Nx45YNKjPzXnz0d1jiJUhWNmZ0WdsA1o9VoPQkzdZMe2QesDqXqycYOBCeGY15GbC0u6wfhJkpzGy3kjGNhzLuqvrmLJ/ipqDVqmQVKI3MCEEbm9MIC8mhsSfHq5e/UOfSyNo1NWXx95qgqm5lj+/PM6+3y+Tn6uSVaFqdNGPzGn2IhxcpC+SdnkroxuMZnSD0fxx+Q+m7p+qkr1S4ahEXwKsmzfDum1bbi7+hvyUlBI/n1tVOwZPbkpQGy+ObbrBys9CiI9UN2oLZW4LvWbA8+vBxBx+HAR/jGFMwBBerPciqy+t5pODn6j7HkqFohJ9CXF743V0ycnEf7OkVM5naq6l49OB9Hq5HhnJ2az8JITjW24gdSphFapqSxi9B9q8ASdWIBa04BXzqjxf93l+vfAr0w9NV8leqTBUoi8hFrVrY9enDwk//EBuTGypnbdaA1eG/K85VWo7snfVZf6cc4yUm6oSZqFMLaDLBzBqO9i6I1YO4/VLRxlW43F+Pv8zM0NmqmSvVAgq0Zcg1/GvIvPziZv7Zame18rOjF5j6tPx2UBib6SyYtohzu5RT9Tek2cDeHE7dH4fcWkTE3cv5WmXYJafXc7so7PVz00p91SiL0FmVarg9PTTJP++hqxz50r13EIIglp7MeS9Zrj52bL9x/P8Ne8EqQklNxKoXNOaQtsJMHoPwjWQSYd/50nsWXZ6GV8d+0ol+woiOjqaIUOG4O/vT1BQEL169eLixYucOXOGTp06UbNmTQICApg2bdqtv/OYmBj69OlDgwYNbu1T3hRl4pGlQohYIUShs0MJvblCiMtCiJNCiMa3reshhLhQsO5tQwZeXri8PBqtvT0xn31ulGRh52JJ//GNaPtkTSIvJbFi6kHO7lWt+3tyrQnPr0f0nMHk8Ks8lpbFN6e+YeHxr40dmVJMUkoGDhxIhw4duHLlCmfPnuWTTz4hJiaGfv368fbbb3Px4kVOnDjBvn37WLBgAQDvv/8+Xbt25cSJE5w9e5bp08t2+YbCFKVF/x3Q4z7rewIBBa9RwNcAQggtML9gfRDwlBAiqDjBlkdae3tcxo0j48AB0rZvN0oMQiOo39GHIf9rhksVW7b/cJ51X51Qs1jdi0YDzUehGXuA923r0j81jQUnv2bJ/vL3H1z5f9u3b8fU1JTRo0ffWtawYUMuXrxI69at6datGwBWVlbMmzfvVkKPiorCx8fn1j7169cv3cAN4IHVnKSUu4QQfvfZpD+wXOqbiAeEEA5CCE/AD7gspbwKIIRYUbDt2WJHXc44PjmYxJ9/Jvazz7Fp0wZhZmaUOOxdrRjweiNO74pg/5or/DL1EK0G+lO3nTdCI4wSU5nm4Ivm2d/58PjP6A5M5cuLP2Fy8zzDe32j7+pRHtlnhz7jfMJ5gx4z0CmQSc0m3XP96dOnadKkyX+Wnzlz5j/L/f39SUtLIyUlhbFjx/Lkk08yb948unTpwvPPP4+Xl5dBYy9phuij9wbCbvscXrDsXssrHWFqittbb5Jz/TqJd80+U+qxaAT1Ovgw5P1mePrbs2vFRdbMOkpCVLpR4yqzhEDb6GmmPbODnlpHZiUc4YelrSDyuLEjUwxESokQhTd0hBB0796dq1ev8uKLL3L+/HkaNWpEXFzh01eWVYaoz1rYT0jeZ3nhBxFiFPquH3x9fQ0QVtli07491m3aEPfVPOx698bExcWo8dg5W9L3lQZcOBDNnlWX+PXjQzTp4UeTHlXRmqh79HfT2nryydBt5P31DJ8nnsFkRV+eqj8C2k/Sz2WrPJT7tbxLSp06dVi1alWhy3ft2nXHsqtXr2JjY4OtrS0ATk5ODB06lKFDh9KnTx927drFY489VipxG4Ih/keHA1Vu++wDRN5neaGklIullMFSymBXV1cDhFW2CCFwf3cyuuxsYmfOMnY4gD6mwJaeDP2gBf4NXTn81zV+/egQkZeTjB1amWSiMeGzPj/Q0asNnzg78NuJb2BhG7i+z9ihKUXQqVMnsrOz+eabb24tO3z4MAEBAezZs4ctW7YA+olCXn31Vd566y0Atm3bRkZGBgCpqalcuXKl3DVGDZHo1wLDCkbftACSpZRRwGEgQAhRTQhhBgwp2LbSMq9WDefhw0n+4w8yjh4zdji3WNmZ0W1kXXqPrU9ejo41M4+y/cfzZKWrGZjuZqoxZWanL2nn045pLk78rs2BZT3h74mQnWrs8JT7EEKwZs0aNm/ejL+/P3Xq1GHKlCl4eXnx559/8tFHH1GrVi3q1atH06ZNGTduHABHjhwhODiY+vXr07JlS0aOHEnTpk2NfDUP54FlioUQvwAdABcgBvgAMAWQUi4U+s6teehH5mQAz0spQwr27QXMAbTAUinlx0UJqjyXKX4QXXo6V3r3QevkSLWVKw0yv6wh5Wbnc2jdVU5sC8fC2oTWjwdQs5n7PfswK6vs/GzGbxvPvsh9fGTXgH4n1oGdN/T9EgK6GDu8MkmVKTachy1TrOrRG0HK+vVEvP4G7u//D6ehQ40dTqHiwlLZ+fMFYq6l4F3LgfZP1cLRw9rYYZUpWXlZvLLtFQ5GHeSToJH0Ofgj3LwA9YdAj0/15ZGVW1SiNxxVj74csO3RA6uWLYibPYfc2NKrg/MwXKvY8tibTWg/tBY3w9JYMe0Q+9dcITdbzcT0LwsTC+Z2mkuwRzDvnvuWDT3+B+3ehNOrYH4zOLMGymBDSql8VKI3AiEEnh98gMzOJuaTT40dzj0JjaBuO2+GTmlBzabuHN14nZ8/PMCVY7HqydoCliaWzOs0j4auDXl73//Y7N8cRu3Qd+OsHA6/PgOp0cYOU6nkVKI3EjM/P1zGvEzqhg2k7thh7HDuy8rOjM7Dgxg4sTHmliZsWHSadV+dIDFajb0HsDK1YkGXBdRzqcdbO99ia04sjNwKXT6Ey1v0rfujP6jWvWI0KtEbkfMLL2BWw5/oqVPRpZf9pOlVw4HBk5vS9skAYq6lsGLaIfb9fpmcLMNPhF7eWJta83WXrwlyCWLizonsiNwDbV6D0XvBvS6sHQc/DIDEUCNHqlRGKtEbkTAzw/PDD8mLjCJu7lfGDqdINFoN9TtW4ekPW1CzuQfHNt3gpw8OcP5AVKWf5MTGzIaFXRYS6BjIGzveYFf4LnCpAc/9Bb2/gPAj+ukLD3wNOnWvQyk9KtEbmVWTJjgMeZKE5cvJOFZ2xtY/iJWdGZ2H1eaxSU2wcbRg63fnWD3jCDHXSn7qxLLM1syWRd0WEeAYwGvbX2NPxB59kbSmI2DsAfBrAxvehqU9INawtV6UB9NqtTRs2PDWa/r06eTn59OkSZM7no7t1q0bK1euBMDPz4969erRoEEDunXrRnR0+bvnooZXlgH5aelc7dcXjbkF1db8jsbCwtghPRSpk5w/EMX+P66SmZJDrRYetBzgj7WDubFDM5rk7GRGbhrJ1aSrfNXpK1p5t9KvkBJOrYT1kyAnDdq9pe/iqQRF0srC8EobGxvS0v47n/LBgwcZOXIkR48eZdWqVXz33Xds3LgR0Cf6kJAQXFxcmDx5MmlpacydO7e0Q7+DGl5ZDmltrPGcNo2ca9e4OW+escN5aEIjqN3Ki2c+bEHj7r5cConhxw8OEPLPNfJyKmcXhb25Pd90/YZq9tV4dfur7I/cr18hBNQfDGMPQWAf2P4RLO4AEUeNGm9l17x5c1q1asWUKVOYPHky8+fPL3S7du3acfny5VKOrvgMUdRMMQCb1q1xeOIJ4pcuw7ZbNyzLYc1rM0sTWg6sQVAbL/b/foWDa69xZnckLQb4U7Ope6Urhexg4cA33b5hxKYRvLLtFeZ3nk9zz+b6lTau8MQyqPcE/P0GLOkMLcdBx8mVokha9CefkH3OsF1X5rUD8Zg8+b7bZGZm0rBhw1uf33nnHZ588kkAPv30U6pUqcJrr71GjRo1Ct3/r7/+ol69egaLubSoFn0Z4jbpLUzc3Yl8+x10meV3UhB7Vyt6vFSPAW80wsLGlC3LzrLqsxAiLyUZO7RS52jhyJJuS6hiW4VxW8dxOPrwnRsE9oIxB6DRM7BvLnzdCkL3GCfYSsDS0pLjx4/fev2b5AF27dqFvb09p0//dzK9jh070rBhQ1JSUnjnnXdKM2TDkFKWuVeTJk1kZZW2b588WytQRk2dZuxQDEKXr5Pn9kXKZZP2yHkvbZX/fH1SJkanGzusUncz46bsv6a/bPpjU3ko6lDhG13ZIeWc+lJ+YCflutekzEwu3SBL2NmzZ40dgrS2ti50eVpamgwICJDnzp2TLVu2lH///fetdVWrVpVxcXGlFWKRFPazBELkPXKqatGXMdYtW+L03DASf/qJtN27jR1OsQmNvhTy01Nb0KxvNW6cS+CXDw+ya8VFMlNzjB1eqXG2dGZJ9yV4WnsydutYQqILGWxQvT28vF/fhXPkO1jQAi5uLPVYK6OpU6cyePBgAgMDWbBgAa+//jpZWVnGDstg1KibMkiXnU3o44+Tl5RE9bVrMXF0NHZIBpOenM3hv65xdm8UJmYaGnerSoPOVTA1L1tVPEvKzcybjNg4gqj0KBZ0XkCwR6GDJCA8BP4cB3HnoN5g6DEdrJ1LN1gDKwujbrRa7R197D169GDYsGEMGDCAEydOYGmpvz/y6quv4uzszAcffHDHqJuyQlWvrCCyzp/n2hODse3QHu+5cytcmeCEqHT2r7lC6MmbWNmb0axPNWq38kSjrfi/ZBY52eflwO5Z+peFHfT8HOo+ph+5Uw6VhURfUajhlRWERWAgbq+/TurmLUafZ7YkOHla03tMfQZObIydsyU7frrAL1MPceVoxS+Y5mLpwrfdv8XT2pMxW8f89wbtv0zMoOM78NJOcKgKq0fAiqGQcs+J2hSlUCrRl2FOw5/Dul1bYqd/Rta5c8YOp0R41XBg0JuN6Tm6HkIj2LD4NKs+O0L4+QRjh1ai/k32XtZejN069t7JHsC9DozcAt0+givbYH4LOPK9KpKmFFmREr0QoocQ4oIQ4rIQ4u1C1r8phDhe8DothMgXQjgVrAsVQpwqWFe5+2MektBo8Jo+Ha29PRGvv1EuCp89CiEE1Ru6MuR/zeg0LJCM5Gz+nHOcP+ccIya04pZUcLF0YUn3JXhZezFmyxgORh2898YaLbR6BV7eB571Yd2rsLwfJFwtvYCVcuuBiV4IoQXmAz2BIOApIUTQ7dtIKWdIKRtKKRsC7wA7pZS3N8k6Fqy/R2ekci8mTk54zZhBzvXrRE+dWqG7NTQFT9g+PbUFbZ4I4GZ4Gqumh7B+4SniI//72HpF8G/L3sfWh3Fbx3Eg6sD9d3D2h2Froc8ciDwOC1rBvnmqSJpyX0Vp0TcDLkspr0opc4AVQP/7bP8UUPE6lY3IukVzXMaMIfnPtST9ttLY4ZQ4E1MtDTpX4dlpLWnapxph5xNYMe0Qm5eeITkuw9jhGZyzpbP+oSo7/UNV+yL23X8HjQaCn9c/aFW9PWx6F77tCjFnSydgpdwpSqL3BsJu+xxesOw/hBBW6CcJX33bYglsEkIcEUKMutdJhBCjhBAhQoiQuLi4IoRVubiMeRnrNm2I+egjMk+dMnY4pcLM0oRmfaox7KNWNOrqy9Vjcfz0wUG2/3CO1ISKM8YZ9Mn+227fUtWuKq9se0Vf9fJB7L3hqRXw2Lf6OveL2sH2T/WjdRTlNkVJ9IWN5bpX/0FfYO9d3TatpZSN0Xf9jBVCtCtsRynlYillsJQy2NXVtQhhVS5Cq8VrxueYuLoSPn48eYmJxg6p1FjYmNJqUA2e+aglddt7c/5gND++v59dv1wgLTHb2OEZjKOFI992+xZ/B39e3faqvp79gwgB9R7XF0mrMwB2TofF7fW175X/+LdMcd26dXniiSfIyND/hhgeHk7//v0JCAjA39+f8ePHk5Oj/8LcsWMHffr0ASArK4vAwEBO3dbY+vzzzxk9ejQAUVFRt7YFff2cGjVqUKtWrVvVMO82ZcoUvL29b5VO/ueffwA4deoUw4cPN8h1FyXRhwNVbvvsA9xrfNcQ7uq2kVJGFvwZC6xB3xWkPAITR0e8v/yS/LibRE6YiMyvXP2y1vbmtHuyJs9MbUlgC0/O7I7kx//tZ/dvF0lPrhgJ/99CaAGOAYzfPp5tN7YVbUdrF3hsib6Fn5kE33aBje9CTsXr6iqOf2vdnD59GjMzMxYuXIiUkkGDBjFgwAAuXbrExYsXSUtL49133/3P/hYWFsyZM4cxY8YgpSQiIoJFixbx6af6uZ+/+OILXnzxRQDOnj3LihUrOHPmDBs2bGDMmDHk3+P/7Ouvv36r/k6vXr0AqFevHuHh4dy4caPY112URH8YCBBCVBNCmKFP5mvv3kgIYQ+0B/68bZm1EML23/dAN+C/FYOUIrOsVxf39/9H+r59xM6cZexwjMLWyYKOzwTy9NQW1GzmzqkdEfzw3n72rLxUIRK+vbk933T7hiCnICbsmMCm0E1F37lWT/0EJ42Hwf55+iJp14rwm0El1LZtWy5fvsy2bduwsLDg+eefB/St/tmzZ7N06dJbLf7b9ejRA09PT5YvX87rr7/OlClTcCx4en316tX06NEDgD///JMhQ4Zgbm5OtWrVqFGjBocOHXqoGPv27cuKFSuKeaVFKFMspcwTQowDNgJaYKmU8owQYnTB+oUFmw4ENkkpbx8D6A6sKXiq0wT4WUq5odhRV3KOTzxB9vkLJCxbhnnNmjgMHGDskIzCzsWSTsNq07hHVY78E8rJbWGc2RVBnXbeNOrmi7V9+Z34xM7MjkVdFzFm6xje2vUWubpcelfvXbSdLeyh75dQ93FY+wp83xeaDIeuU/XryoDdv13kZphhR1K5VLGh7eCaRdo2Ly+P9evX06NHD86cOUOTJk3uWG9nZ4evr+89a8/PmTOHZs2aERAQwLPPPgvAtWvXcHR0xNxc/+8uIiKCFi1a3NrHx8eHiIiIQo83b948li9fTnBwMLNmzbr1xREcHMz06dN56623inRd91KkcfRSyn+klDWllP5Syo8Lli28LckjpfxOSjnkrv2uSikbFLzq/LuvUnzu77yNVcsWRL//frmagrAkOLhZ0Xl4EEOntMC/iRsnt4fzw3sFXTpJ5beF/+8ctI3dG/PO7ndYc2nNwx2gWlv9uPtWr8LR5TC/OVxYXzLBlhP/1qMPDg7G19eXESNGIKUstMTIvZYDeHl50alTJ15++eVby6Kiorj9/mJhQ6ELO97LL7/MlStXOH78OJ6enkyYMOHWOjc3NyIji/8ktJp4pJwSJiZ4f/EFoU8OIfyVV6n26wpMvQsdDFVpOLhb0WV4EMG9/Diy4TqndkRwZlckQa09adS9KrZO5WuKRgArUyvmd57Pa9tf4/1975Ory2VwrcFFP4CZFXSbBnUG6ouk/TJEXy+n5+f6fn0jKWrL29D+7aO/XZ06dVi9evUdy1JSUggLC8Pf35/4+PhCj6XRaNBo/r+tbGlpeUfFSx8fH8LC/n/AYnh4OF5eXv85jru7+633L7744h03c7Oysm4VWisOVQKhHDNxdKTKgvnI7GzCRo8mPzXV2CGVCQ5uVnQeVpunP2xBrRYenNmjv2m7/cfzJMeVvwldLE0smdtpLu192jPtwDSWn1n+8AfxbgyjdkCHyXB2LcxrCid/U2UUgM6dO5ORkcHy5fqfa35+PhMmTGD48OFYWVkV+Tg1a9YkNDT01ud+/fqxYsUKsrOzuXbtGpcuXaJZs/+ORYmKirr1fs2aNdStW/fW54sXL97x+VGpRF/Omdeogc9Xc8m+Fkr4q68ic9QY6n/Zu1rS8ZlAnpnWkqA2Xpw/EMVPHxxgy3dnSYwuX+UkzLXmzO4wm65VuzIjZAaLTy5++IOYmEGHSTB6NzhVh99fhJ+fhOTC+40rCyEEa9asYeXKlQQEBFCzZk0sLCz45JNPbm2zdetWfHx8br3279//n+NYW1vj7+9/q1+/Tp06DB48mKCgIHr06MH8+fPRavXluEeOHMm/FXrfeust6tWrR/369dm+fTuzZ8++dczt27fTu3cR783c7xrL4iP1qkzxw0ta8wdR77yD/aBBeH78UYUra2wI6UnZHNt8gzO7IsjL0+HfyJUmPfxw9bU1dmhFlqfL4/2977Pu6jpG1hvJq41efbS/a10+HFwIW6eBxgS6TYXGw/VP3ZaQylCmeM2aNRw5coSPPvqo2MfKzs6mffv27NmzBxOTO3vZH7ZMseqjryAcBg4gNyyMmwsWYOLuhtv48cYOqcyxdjCnzRMBNOlRlRNbwzi1I5wrR+PwreNMkx5V8QpwMHaID2SiMeGjNh9hYWLBklNLyMjNYFKzSWjEQyZojRZajoVavfQF0v56HU7/rh+t4+xfMsFXAgMHDrxnn/7DunHjBtOnT/9Pkn8UqkVfgUgpiX7/fZJWrsJ98mSchj1r7JDKtOzMPE7tCOfE1jCy0nLxrGFP4+5VqVrXucz/RiSlZFbILL4/+z39/fvzYasP0WoecZYuKeHYD7DxPcjPho7vQosxoDVsO7AytOhLi2rRV2JCCDw++ID8pCRiPvkEraMj9n37PHjHSsrc0oTgnn406FyFc3sjObbpBn/PP4mztw2Ne/hSo7FbmZ3xSgjBhOAJWJtas+DEAjLyMvis7WeYak0f5WD6B6xqdIW/J8Dm/8GZ36HfPPAo/o3A291vyKJSNI/SOFct+gpIl51N2MgXyTh2DJ+v5mLbsaOxQyoX8vN0XDocw9GN10mMzsDOxYKGXXwJbOWJqVnZndP2+zPfMzNkJq29WzO7w2wsTYoxHE9KOLMG/nkTspKgzRvQbiKYFP/hs2vXrmFra4uzc9n/jamsklISHx9Pamoq1apVu2OdmjO2EspPTeXG8y+QfeECPgu/xqZ1a2OHVG5IneTayZsc3XidmGspWNiYUr+jD/Xa+2Bh8wgt5lLw+6Xf+XD/hzRwbcC8zvOwM7Mr3gEzEmDD23DyV3AN1LfuqzQt1iFzc3MJDw+/Y6y58vAsLCzw8fHB1PTOf4sq0VdS+UlJXH9uODnXr+P7zWKsmhbvP2plI6Uk6nIyRzdd5/qpeEzMNNRu7UXDzlWwcyn+QyyGtil0E5N2T6KGQw2+7vI1LpYGeCDq0mZY9xqkREDz0dD5f2BmXfzjKganEn0llhcfz/Vhz5EXFUWVJUuwatzI2CGVS/ERaRzffIOLh2KQQI3GrjTs6otb1WK2nA1sb8ReXt/xOm5WbizqughvGwM8LZ2VAlumQMi34OALfeeCv+oOLGtUoq/kcmNiufHcc+TFxlJl8SKsgtWMjo8qLTGLE9vCObM7gtysfLxrOdCwiy9V6zgjNGWj3/l47HHGbh2LhdaCRV0XUcOxhmEOHLpXXyQt4Qo0ega6fQyWDoY5tlJsKtEr5MbGcmP48+RGRVHl66+xbtHc2CGVa9mZeZzdE8nJbWGkJWbj6GFFg85VqNXCAxNT49+4vZR4iZc2v0R2fjbzO8+noVtDwxw4NxN2TId9X4G1K/SeBbXVyK6yQCV6BYC8mze5Pnw4ueER+Hz1FTZt2xg7pHIvP1/H5ZBYjm+5wc2wNCxtTanbzpu67X2wsjMzamwRaRGM2jSK2IxYZrafSfsq7Q138Mhj8OcrEHMKggZArxlg42a44ysPTSV65Za8+HhujBhJ9pUreM/4HLuCSRKU4pFSEnEhkRNbwwg9FY/WREPN5u406FQFZ28bo8UVnxnPmK1juJBwgSmtpjCgxgDDHTw/F/bOgZ2f62/Q9pgO9Z/Uj8tXSp1K9Mod8lNSCBv9MpnHj+Px4RQcn3jC2CFVKInR6ZzYGsaFA9Hk5eqoUtuRBp198Q1yMko/fnpuOq9tf40DUQcY33g8I+qOMOw49rgL+hLI4Yf0D131mQ0OVR68n2JQxU70QogewJfoZ5haIqWcftf6DuinELxWsOh3KeXUouxbGJXoS54uI4Pw8a+Rvns3rq+/jvOoF9VDLAaWlZbL6d0RnNoRTkZyDo4eVtTvVIVazT0wNS/dfvzc/Fze3fsu66+tZ0itIbzd7O1HL5lQGF0+HFoMW6eC0ECXKRA8okSLpCl3KlaiF0JogYtAV/QThR8GnpJSnr1tmw7ARClln4fdtzAq0ZcOmZND5DuTSfn7bxyeGoLHe+8htMa/kVjR5OfpuHwklhNbw4i7kYq5lQlBrb2o19GnVCdD0Ukdc47MYdmZZXT27cz0ttOxMDHw+RND9ePur24H31bQ7ytwMdCoH+W+7pfoi/J12wy4XDAtYA6wAuhfxHMXZ1+lhAkzM7xmfI7zyBEk/bKC8FdeRZdZ/ibmKOu0JhpqNffgiXeCGTSxMT6BThzfGsYP7+5jw6JTRF5KfKT6JQ9LIzS8EfwGk5pOYtuNbby46UUSsxINexJHP3h2DfSfD7Fn9JOT75kN+XmGPY/yUIqS6L2BsNs+hxcsu1tLIcQJIcR6IUSdh9wXIcQoIUSIECIkLi6uCGEphiA0GtwmTsT9vfdI276d6888S25MjLHDqpCEEHjWcKDHqLo8+1FLGnXzJfxiImtmHePXjw9zdm8keTn5JR7HM0HPMKvDLM4lnOPZ9c8SlhL24J0ehhD6cfZjD0PNbvqHrZZ0gqiThj2PUmRFSfSFddze3fw4ClSVUjYAvgL+eIh99QulXCylDJZSBt8+wa5SOpyeeRqf+fPJuXaN0CcGk3n6jLFDqtBsnSxoObAGz33amo7PBCJ1ku0/nOf7d/axf81lUhNKth5M16pdWdJtCcnZyTz9z9OciDth+JPYusOTP8Lg5ZASBYs76Pvwc1Wtm9JWlEQfDtx+C90HuGNacillipQyreD9P4CpEMKlKPsqZYdtp45U/eVnMNFy/ZlnSFm/3tghVXimZlqC2ngx5H/NGPB6I7xqOnBs0w1+eHcf6xeeIvx8Qol16zR0a8gPPX/AxsyGERtHsCl0U4mch6D+MPYgNBgCu2fBorZw40DJnEspVFFuxpqgv6HaGYhAf0N1qJTyzG3beAAxUkophGgGrAKqoh9pc999C6NuxhpX3s2bhL/yKpnHjuE8cgSur72GMMAsN0rRpCZkcXpXBGf3RJKVloujhxV12/sQ2MIDM0vD/z0kZCUwftt4jscd57XGr/FC3RdKbgTW5a36m7XJYdBsFHR+H8yN95xBRWKI4ZW9gDnoE/dSKeXHQojRAFLKhUKIccDLQB6QCbwhpdx3r30fdD6V6I1P5uQQ/cknJK34FetWLfGaNQsTR0djh1Wp5OXmczkkllM7wom9noqpuZZazT2o28EbZy/DJsfs/Gz+t+d/rA9dz6CAQbzX/L1Hm8SkSCdL03fhHFoM9lWg7xyo0blkzlWJqAemlEeWtHo10VM+ROvigvcXs7BqpKpfGkPMtRRO7Qznckgs+Xk6vAIcqNvem+oNXdGaGGasuk7qmH98PotPLibYPZjZHWbjYOFgkGMX6sYB/YNW8Zeg4dPQ7SOwciq581VwKtErxZJ5+gwRr71GbnQ0bm+8gdPzw9XDVUaSmZbDub1RnN4VQWp8FpZ2ZtRp40VQGy+Djcn/6+pfvL/3fTysPZjXeR7V7asb5LiFys2CXZ/Dnjlg5Qy9Z+r79JWHphK9Umz5KSlEvfseqZs3Y9O+PZ6ffIyJs7Oxw6q0dDrJjTPxnN4VwfXT8Qigaj0X6rb3xrd28UstHI89zvjt48nNz+Xz9p/TxruEC+BFnYQ/x0L0SajdD3rN1I/aUYpMJXrFIKSUJP74E7EzZqCxs8Pr00+wadvW2GFVeik3MzmzO5Jz+yLJTM3FzsWCOm29CWzpWawKmpFpkbyy7RUuJ13mjSZvMCxoWMn+Jpefqy9/vGM6mFpA90+h4VBVJK2IVKJXDCrrwkUiJ04k+9IlHJ99Frc3XkdjWfam1qts8vN0XD0ex+mdEUReSkKjFVRv5Eqdtt5413R4pCSdkZvBe3vfY/P1zfTz78f7Ld/HXFv8icLv6+Yl/QQnN/aDfyfoMwccq5bsOSsAlegVg9NlZRE76wsSf/gBs6pV8Zz+qbpRW4YkRKVzdnck5w9EkZ2Rh4O7FUFtvAhs6YGlzcO18nVSx6ITi1hwYgF1nOswp+McPKw9Sijyf0+q009duGUKSAldPoCmL6oiafehEr1SYtIPHCBq8rvkRkfjNHw4rq+MU637MiQvJ5/LR2I5szuC6KspaEwE/o3cqNPGC6+HbOVvu7GNyXsmY641Z1b7WQR7lMKUlEk34K/X4fIWqNJCXyTNtWbJn7ccUoleKVH5aWnEfvY5SStXYurri+fUD7Fu0cLYYSl3iY9I48yeSC4ciCYns6CV37qglW9btFb+1eSrjN82nrDUMCYET+CZ2s+U/AgsKeHkr7DhbchJh/aToPV4KKlx/uWUSvRKqUg/cJCoD94n9/oN7AcNwu3NieohqzIoNyefK0diObsnkqgryWi0gmoNXKnTxgufQMcHjthJzUnlvT3vsS1sGz39ejKl1RSsTK1KPvC0WPjnTTj7B3jUg37zwKthyZ+3nFCJXik1uqwsbs6fT/yy79BaW+M64Q0cHn8cofpWy6SEyHTO7onk/MEostPzsHW2IKi1J4EtPbFxvPe4fCklS08vZe6xuVSzq8YXHb6gukMJjre/3bl18PcESL8JrV/Vt/BNVXehSvRKqcu+dInoD6eSERKCRYP6eEyejGWDBsYOS7mHvNx8rh2/yZk9kURcSEQI8K3rTFArL6rWd0arLfyL+kDUASbtmkRmXiYftPyA3tV7l07AmYmw6T049iM419D33VdtVTrnLqNUoleMQkpJyrp1xMyYQX7cTez798P1jQmYursZOzTlPpLjMji3N4rz+6NIT87B0taUwBae1G7tiaOH9X+2j0mP4a1db3E09iiDaw7mrWZvlfwQzH9d2Q7rXtXftG06Uj+Foblt6Zy7jFGJXjGq/LR04hcvJmHZMjAxwfn553F64QW0Nv9NGkrZocvXceNMAmf3RnL9VDw6ncTT357AVp7UaOKGmcX/V9LM0+Ux99hclp1eRk3HmsxoP6NkSyfcLicdtk6DgwvBzltfJC2ga+mcuwxRiV4pE3LCwoj94gtS129A6+yMy5iXcXziCYTZoz+9qZSO9ORsLhyM5tzeKJJiMjAx1xLQxI3AVp54+tvfGnmzO3w37+55l6z8LCY3n0x///6lVxcp7JC+SNrNC1B/CPT4tFIVSVOJXilTMk+cIHbGTDJCQjD18cFl7Fjs+/VVE5OXA1JKoq+mcG5fJJdDYsnNzsfezZLarTyp1dwTG0dzYtJjeGfPOxyOPkxPv5681/I97MzsSifAvGzYNRP2fAGWjtBrBgQNqBRlFFSiV8ocKSXpu3YR9+Vcss6exax6dVxeHo1dz55qkpNyIicrjytH4zi/P4rIS0kIAVWCnAhs6YlvPSe+v7CMBccX4GblxqdtP6WJe5PSCy76tL5IWtRxCOyjL5Jm51l65zcCleiVMktKSermzdz8ah7Zly5hVrUqzi+9hH3fPghT9UBMeZEUm8H5/VFcOBBNWmI25lYmBAS7Y1I7jY+v/o+I9AheqPsCYxqMKbkJTe6WnwcH5sP2T0BrDt0/1k9aXkFb94aYYaoH8CX6WaKWSCmn37X+aWBSwcc04GUp5YmCdaFAKpAP5N0rkNupRF/5SJ2O1C1buLlwIdlnz2Hi6Ynz8OdwePxxNNbqpm15odNJIs4ncm5/FFePx5Gfq8PBw5JwrzP8Lpbh6+HFJ20+IcAxoPSCir+iL5J2fS9U7wB9vwRHv9I7fykpVqIXQvw772tX9JN9HwaeklKevW2bVsA5KWWiEKInMEVK2bxgXSgQLKW8WdSAVaKvvP7t0on/ZgkZISFo7e1xePJJHJ8eiqm7qk9enmRn5nE5JIbz+6OJvpoMAqIcL3PB5SDdO7ZmeINhmGhKqZtOp4MjS2HzFJD5+rlqm40CTcW5L1TcRN8SfeLuXvD5HQAp5af32N4ROC2l9C74HIpK9MojyDh2jISlS0ndshW0Wux69sTp2WewrF/f2KEpDykpJoPzB6I4tz+SjKRcsrWZJHnfoF+v9jRpULv0RuYkh+snJ7+8GXya6ssouAWWzrlLWHET/eNADynlyILPzwLNpZTj7rH9RCDwtu2vAYmABBZJKRffY79RwCgAX1/fJtevXy/KtSmVQE5YGAk//EDyqtXoMjKwqFsXx6FDsevVE42FYabPU0qH1EnCLySybesREs/mYaozB7tcGrfxJ6ilF/aupVAzR0o4tRLWT4KcNGj3lr5Imkn5HuZb3ET/BND9rkTfTEr5SiHbdgQWAG2klPEFy7yklJFCCDdgM/CKlHLX/c6pWvRKYfLT0kj+808Sf/6FnCtX0NjZYd+vHw5PPIFFLVW6tryJSorh6zU/knveCu/kmggEnv721GzuQY0mblhYl/BN27Q42DAJTq8G97r6MgrejUv2nCWoVLpuhBD1gTVATynlxXscawqQJqWceb9zqkSv3I+UkoxDh0n67TdSN21C5uZiUb8+DgMHYNerF1p7e2OHqDyErde38sXOuTiHV6dpShc0SZZoTAR+9Vyo1dyDqnWd0ZqUYFG88//A329AWgy0HAcdJ5fLImnFTfQm6G/GdgYi0N+MHSqlPHPbNr7ANmCYlHLfbcutAY2UMrXg/WZgqpRyw/3OqRK9UlR5iYmkrF1L0urfyb54EWFqik2nTtj37YN1u3Zo1FO35UJKTgpzjsxh5YWV1Myvz2O8QNZ5MzJTczG3NiGgiTs1m3vgUd2uZPrzM5Ng8/tw9Htwqq5v3fuV8IToBmaI4ZW9gDnoh1culVJ+LIQYDSClXCiEWAI8BvzbsZ4npQwWQlRH38oHMAF+llJ+/KDzqUSvPCwpJdnnzpG05g9S/v6b/IQENHZ22HXvhm2PHlg3b64exCoHjsceZ9qBaVxMvEhrzza8YP8KSacl147HkZerw87FgprNPKjZzL3QAmvFdnWnvkhaYigEvwBdPgSLUnqqt5jUA1NKpSJzc0nfv5/kdX+RtnUruowMtI6O2Hbtim3Xrlg3b6bq65Rhebo8fj73MwtOLCA7P5vngp7j+ZojiDqTxsWD0YSfT0RKcPW1pVZzD2oEu2Ftb8BqmTkZsP1jOLAAbD2hz2yo2d1wxy8hKtErlZYuK4u03btJXb+etB070WVkoLG1xaZDB2w7dcS6TRu0tpWzrG1ZdzPzJrOPzGbtlbW4WrrySqNX6Offj6zUPC4djuHioRjibqQiBPgEOlKzmQfVG7piZmmg39zCQ/RF0uLOQb3B0GM6WDsb5tglQCV6RQF02dmk79tH6qbNpG3fTn5SEpiYYNU0GJt27bFp3w6zatVKb0y3UiQn4k4w4/AMTsSdINApkInBE2nu2RyAhKj0gqQfTcrNLLSmGvzquVCzmTtV6zijNS3mTdy8HNg9S/+ysIOen0Pdx8pkGQWV6BXlLjI/n8wTJ0jbto3UHTvIuXwFAFNvb6xbtcK6dWusWzRH6+Bg3EAVQH8PZmPoRmYfmU1keiStvFrxWuPXqO1c+9b6mGspXDwYzaUjsWSl5WJuZYJ/I1cCmnngFeCA5gFz4d5XzFlYOw4ijkDNntDnC7DzMtDVGYZK9IryALkREaTt3k3a7j1kHDiALj0dhMAiKAirFs2xbtECy0aN1WQpRpadn82K8yv45tQ3JGcn08OvBy83fPmOSU7y83WEn0/k0qEYrh6PIzc7Hyt7MwKauBPQzB23qraP9lubLl/fb7/tY9CaQtep0GR4mWndq0SvKA9B5uaSeeoU6Xv3kXHwIBknTkBuLmi1+sTftClWTRpj2agRJk6VZ2KLsiQ1J5Vlp5fx47kfyc7Ppne13rzU4CWq2lW9Y7vcnHyun4rn4qForp+JR5cnsXe1JKCpOwHB7jh5PcIXd/wVWDceQneDX1voN1c/JNPIVKJXlGLQZWaScfQoGSEhZB4OIfPECWRuLgBmfn5YNmqEZYMGWDZsgHmNGmoYZylKyEpg2ellrDi/ghxdDt39uvNivRcLrY6ZlZ7L1eNxXDocQ8QF/cgdZ29rApq6U6OJO/auD/GQlE6nH3O/+X3Iz4VO70GLl41aJE0lekUxIF12NllnzpB59CgZR4+Refw4+QkJAAgLCyyCgrCsVw+LunWwqFMHs6pV1exZJexm5k2Wn1nOrxd+JSMvgw5VOvB8nedp5Nao0G6a9ORsrhyN5dLhWH1lTcDNz46AYDdqNHHHxrGIwzWTI/RP1V7cAN5N9EXS3IMMeWlFphK9opQgKSW54eFkHj9O5qlTZJ06TdbZs8jsbAA0VlaYBwZiERiIee1ALGrVwrxGDTRWpVDAq5JJzk7mp3M/8fP5n0nOTqa+S32G1RlGJ99OmGoKr52TEp/J5ZBYLh+JJe5GKgjw9LcnINgd/8ZuWNk94JkLKfX1cta/BVkp0HaC/lXKRdJUoleUUibz8si+cpWsM2f0r/PnyT5/Xn+TF0AITH2rYB4QgHmNGpjXCMC8hj9mfn6qIqcBZORmsPbKWpafXU5YahhuVm4MrjmYx2o+houlyz33S4rJ4FJIDJePxJIQmY4Q4F3LkRpN3PBv5IaFzX0KraXH64uknVoJrrWh/3zwKb3pE1WiV5QyQOp05IaHk3XhAtkXL5J98RLZly6Rc/065OfrNxICUx8fzKpXw9yvGmbV/DDzq4aZX1VM3NwQmhIs7lUB5evy2R2xm1/O/8K+yH2YaEzoWKUjjwc8TguvFmjEvX+e8RFpXD4Sy6WQGJJjM9FoBD619Um/WgPXe1fXvLAB/nod0qKhxRjo+C6YlfxvbyrRK0oZpsvJIedaKDlXLpN95SrZV6/oP4eGIrOybm0nLCwwq1IFU19fzHx8MK1SBVMfb/17b280luWv4mJpupp8lVUXV7HuyjqSspPwsvaid/Xe9PXvSzX7avfcT0rJzbC0Wy391PgsNFpBlSCnW0nf/O6ncbOSYfMHcGSZftrCfl9BtXYlen0q0StKOSR1OvKio8kJDSXnxg1yQq+Tc+MGuWE3yAkLv+NLAEDr7Iypl5f+5emJqZcnJh4emHp6YuLujomzs7opDOTk57D1xlb+uPwHB6IOoJM66jrXpbtfd7r5dcPL5t4PQkkpib2eyuWQGC4fjSUtIRuNicC3tj7p+92d9EP36OerTbgKjYdB12lg6VAi16USvaJUMFJK8uLiyI2IIDc8gtyIcHIjIsmNLHhFRf3niwATE0xcXDBxd8PUzR0TV1dM3Fz1f7q6YuLigtbFBRMnp0ozRDQuI45/rv3D31f/5lzCOQDqudSjY5WOdKjSgRoONe75cJWUkpjQFC6HxHLlaCxpiQVJP8iZGo1d/z/p52TAjk9g/3ywcYfeX0BgL4Nfi0r0ilLJSCnJT0oiNzKSvJhY8mKiyY2KJi82lrzYGHJjYsmLi0OXkvLfnYVA6+CA1tkJEydn/Z+OTmidnNA6OmDi5KRf7+iofzk4oDE3YPVIIwlLCWPT9U1svr6ZM/H66Ta8bbxp7dWall4taebZDDuzwksWS11B0j9yV9Kv7YR/YzeqNXDBPPGUvkha7BmoM0hfN8fG1WDxq0SvKEqhdFlZ5N28SV5c3K0/8+MTyIu/SX58PHnxCeQnJJCXkFD4l0IBYWGB1t5e/7KzQ+Ngj9bOHq2tLRp7O7S2dmjtbNHY2qKxsdEvt7VFa2ODxsamzP0GEZsRy67wXewM38mhqENk5GWgERoCnQJp7NaYJu5NaODaAFer/ybqW0n/aEHST8jW9+nXdsK/gRPVMn/D4uB0MLOBnp9BvScMUkZBJXpFUYpN5uaSn5xMfmIieQmJ5Ccl6V+JieSnpJCfnER+UjK65GT954KXzMh44LGFpSUaa2s01lZorK3RWlkXfC54WVnp11lZ6be1tEJjZYnG0vLOzxYW+s8WFggLC4OMUsrV5XIy7iQHog5wJOYIJ+NOkp2vf0bCzcqNus51CXQKpKZjTQIcA/C28UZb8ISslJLY0NRbST81PguNRuBdzQz/vDVUz/wVy1ot9DXv7X2KFachZpjqAXyJfoapJVLK6XetFwXrewEZwHAp5dGi7FsYlegVpeKQubnkp6aiS00lPyUVXVpqwec0/fv0dHRp6ejS0tCl//+f+Rnp+s/pGciMDHSZmfqHkx6CMDNDWFigMTfXfwGYmyPMzREW5mjMLfTvzc3QmBUsNzdHY26m38/MDGF623szM4SZKToTLWFZkVzLiOBqxnUupYUSkRVDrlaSqwVhYoKbvTee9j542HnjZu+Fq60nztaumCTYknQun/CTKaTEZSGExMvsLP5WIVTv0QHrts/BI345FXfOWC36OWO7AuHo54x9Skp59rZtegGvoE/0zYEvpZTNi7JvYVSiVxTlblJKZGYmun9f6RnIzAx0WVnoMjKRWZnoMrPQZWXqt8vK/v9l2VnIrGxkdha67Gxkdg4yKwtdTsGf2VnInFxkdrb+lZNTIteQp4F8DeRpIc3Gm3jnxiQ6NSTbwgOkDsusKzy3cChay4efDOd+ib4oHWPNgMtSyqsFB1sB9AduT9b9geVS/61xQAjhIITwBPyKsK+iKMoDCSEQVlalUjpCSgm5uehycpG5OcicHP0XQG6u/pWTc5/3eci8vFvLsrLTSM1IIjMrlaysdLKz08nNySI/NweL3LN45J4gJ8uRTF118kxsHynJP0hREr03EHbb53D0rfYHbeNdxH0BEEKMAkYB+Pr6FiEsRVGUkiGEADMztGZmQPmfg6AonUGF3Q6+u7/nXtsUZV/9QikXSymDpZTBrq6GG3KkKIpS2RWlRR8OVLntsw8QWcRtzIqwr6IoilKCitKiPwwECCGqCSHMgCHA2ru2WQsME3otgGQpZVQR91UURVFK0ANb9FLKPCHEOGAj+iGSS6WUZ4QQowvWLwT+QT/i5jL64ZXP32/fErkSRVEUpVDqgSlFUZQK4H7DK1Vxa0VRlApOJXpFUZQKTiV6RVGUCq5M9tELIeKA64+4uwtw04DhlAfqmiu+yna9oK75YVWVUhb6EFKZTPTFIYQIudcNiYpKXXPFV9muF9Q1G5LqulEURangVKJXFEWp4Cpiol9s7ACMQF1zxVfZrhfUNRtMheujVxRFUe5UEVv0iqIoym1UolcURangymWiF0L0EEJcEEJcFkK8Xch6IYSYW7D+pBCisTHiNKQiXPPTBdd6UgixTwjRwBhxGtKDrvm27ZoKIfKFEI+XZnwloSjXLIToIIQ4LoQ4I4TYWdoxGloR/m3bCyHWCSFOFFzz88aI01CEEEuFELFCiNP3WG/4/CWlLFcv9FUwrwDV0de7PwEE3bVNL2A9+olPWgAHjR13KVxzK8Cx4H3PynDNt223DX0F1ceNHXcp/D07oJ+K07fgs5ux4y6Fa54MfFbw3hVIAMyMHXsxrrkd0Bg4fY/1Bs9f5bFFf2sOWyllDvDvPLS3uzWHrZTyAPDvHLbl1QOvWUq5T0qZWPDxAPpJXsqzovw9g35S+tVAbGkGV0KKcs1Dgd+llDcApJTl/bqLcs0SsBVCCMAGfaLPK90wDUdKuQv9NdyLwfNXeUz095qf9mG3KU8e9npGoG8RlGcPvGYhhDcwEFhYinGVpKL8PdcEHIUQO4QQR4QQw0otupJRlGueB9RGPzvdKWC8lFJXOuEZhcHzV1GmEixrijOHbXlV5OsRQnREn+jblGhEJa8o1zwHmCSlzNc39sq9olyzCdAE6AxYAvuFEAeklBdLOrgSUpRr7g4cBzoB/sBmIcRuKWVKCcdmLAbPX+Ux0RdnDtvyqkjXI4SoDywBekop40sptpJSlGsOBlYUJHkXoJcQIk9K+UepRGh4Rf23fVNKmQ6kCyF2AQ2A8proi3LNzwPTpb4D+7IQ4hoQCBwqnRBLncHzV3nsuinOHLbl1QOvWQjhC/wOPFuOW3e3e+A1SymrSSn9pJR+wCpgTDlO8lC0f9t/Am2FECZCCCugOXCulOM0pKJc8w30v8EghHAHagFXSzXK0mXw/FXuWvSyGHPYlldFvOb3AWdgQUELN0+W48p/RbzmCqUo1yylPCeE2ACcBHTAEillocP0yoMi/j1PA74TQpxC360xSUpZbssXCyF+AToALkKIcOADwBRKLn+pEgiKoigVXHnsulEURVEegkr0iqIoFZxK9IqiKBWcSvSKoigVnEr0iqIoFZxK9IqiKBWcSvSKoigV3P8BNMHCQanTXs4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for fn, t in zip(fns, annealings):\n",
    "    plt.plot(p, [fn(2, 1e-2)(o) for o in p], label=t)\n",
    "f = SchedPoly(2,1e-2,0.5)\n",
    "plt.plot(p, [f(o) for o in p], label=\"POLY(0.5)\")\n",
    "plt.legend();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe9da96b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"SchedLin\" class=\"doc_header\"><code>SchedLin</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>SchedLin</code>(**`start`**, **`end`**)\n",
       "\n",
       "Linear schedule function from `start` to `end`"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(SchedLin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e20b548",
   "metadata": {},
   "outputs": [],
   "source": [
    "sched = SchedLin(0, 2)\n",
    "test_eq(L(map(sched, [0., 0.25, 0.5, 0.75, 1.])), [0., 0.5, 1., 1.5, 2.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a36ec96a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"SchedCos\" class=\"doc_header\"><code>SchedCos</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>SchedCos</code>(**`start`**, **`end`**)\n",
       "\n",
       "Cosine schedule function from `start` to `end`"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(SchedCos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4af0ce06",
   "metadata": {},
   "outputs": [],
   "source": [
    "sched = SchedCos(0, 2)\n",
    "test_close(L(map(sched, [0., 0.25, 0.5, 0.75, 1.])), [0., 0.29289, 1., 1.70711, 2.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7aaa4731",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"SchedNo\" class=\"doc_header\"><code>SchedNo</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>SchedNo</code>(**`start`**, **`end`**)\n",
       "\n",
       "Constant schedule function with `start` value"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(SchedNo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25bb16d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "sched = SchedNo(0, 2)\n",
    "test_close(L(map(sched, [0., 0.25, 0.5, 0.75, 1.])), [0., 0., 0., 0., 0.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be11a222",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"SchedExp\" class=\"doc_header\"><code>SchedExp</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>SchedExp</code>(**`start`**, **`end`**)\n",
       "\n",
       "Exponential schedule function from `start` to `end`"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(SchedExp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8851d39d",
   "metadata": {},
   "outputs": [],
   "source": [
    "sched = SchedExp(1, 2)\n",
    "test_close(L(map(sched, [0., 0.25, 0.5, 0.75, 1.])), [1., 1.18921, 1.41421, 1.68179, 2.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "657c5d3e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"SchedPoly\" class=\"doc_header\"><code>SchedPoly</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>SchedPoly</code>(**`start`**, **`end`**, **`power`**)\n",
       "\n",
       "Polynomial schedule (of `power`) function from `start` to `end`"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(SchedPoly)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a382b3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "sched = SchedPoly(0, 2, 2)\n",
    "test_close(L(map(sched, [0., 0.25, 0.5, 0.75, 1.])), [0., 0.125, 0.5, 1.125, 2.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9aed2d4c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABOyUlEQVR4nO3dd1hUx9fA8e/Qq4o0pSgiVuxiwd4RW0wssWts0WiaidH80mMSjYmJJhq7xhJjN/bejb1iV0RUBAVBVFD6vH9c4ksM6ioLuyzzeZ59ZPfO3XsuwmF27twzQkqJoiiKYrrMDB2AoiiKkrtUolcURTFxKtEriqKYOJXoFUVRTJxK9IqiKCbOwtABZMfFxUX6+PgYOgxFUZR849ixY3eklK7ZbTPKRO/j48PRo0cNHYaiKEq+IYS49rRtauhGURTFxKlEryiKYuJUolcURTFxRjlGryiK6UhNTSUiIoKkpCRDh2ISbGxs8PLywtLSUud9VKJXFCVXRURE4OjoiI+PD0IIQ4eTr0kpiY2NJSIiglKlSum833OHboQQ3kKInUKI80KIs0KId7NpI4QQvwghQoUQIUKIGlm2tRZCXMzcNlrnyBRFMQlJSUk4OzurJK8HQgicnZ1f+NORLmP0acAHUsoKQF1gmBCi4hNtgoEymY/BwNTMoMyBKZnbKwLds9lXURQTp5K8/rzM9/K5QzdSyiggKvPrB0KI84AncC5Ls1eA+VKreXxQCFFECFEc8AFCpZRhmQEuzmybdV+9SEvP4MsFncHZF0/v0gghMBfmWJhZYGFmgZWZFVbmVthY2GBjboOtpS12FnbYW9rjYOmAo5Ujtha26gdSURST80Jj9EIIH6A6cOiJTZ7AjSzPIzJfy+71Ok9578FonwYoUaLEi4QFgHnyPTbLSyTFXYa4F94dAAthQSHrQhSxLkIR6yI42ThR1KYoRW2K4mrriqudK252brjbueNs64yZUJOWFEXR3dixY5k9ezbm5ub88ssvBAUF/afNl19+ycyZM3F11W5y/e6772jTpk2OjqtzohdCOAArgPeklPef3JzNLvIZr//3RSlnADMAAgICXng1FGHnRLnEn+j3cC7NH24g3cmHjHY/k1aiLqkZqaRmpJKcnkxyWjKP0h/xKPURD9MekpiayIOUBySkJvAg5QHxyfHcS75HfHI81+5f40T0Ce4m3UU+EbaFsMDNzg0PB4/HD29H78cPZxs1JqkoBVlaWhoWFv+fYs+dO8fixYs5e/YskZGRtGjRgkuXLmFubv6ffd9//30+/PBDvcWiU6IXQliiJfk/pJQrs2kSAXhnee4FRAJWT3k9VxRzd+d/lwZwuO9QLNa8AwtexarmG9DyK7B1een3TctII/ZRLHce3SH6YTTRD6O59fAWUYlRRCVEcSjqENEPo//1x8DB0oGShUriU9gH38K+lC5cGt8ivng7emNhpiY7KUpeCQ8Pp3Xr1tSpU4cTJ05QtmxZ5s+fj52dHdu3b+fDDz8kLS2NWrVqMXXqVE6dOsW4ceNYuXIlq1evplu3bty7d4+MjAwqVqxIWFgYV65cYdiwYcTExGBnZ8fMmTMpX748/fr1o2jRopw4cYIaNWowYcKEx3H8817W1taUKlUKPz8/Dh8+TGBgYK5/D56bcYTWLZ0NnJdS/vSUZmuA4Zlj8HWAe1LKKCFEDFBGCFEKuAl0A3roJ/T/Ku3qwPJjETwo3hjHofth57dw8De4tBna/QzlWr/U+1qYWeBu7467vTv++GfbJjU9lZsJN7n+4Do3Htwg/F441+5f49jtY6wPW/+4nZWZFaWLlKaMUxnKOZWjgnMFyjqVpbB14ZeKTVHyk6/WnuVc5JMDAjlT0aMQX7TP/vfyHxcvXmT27NnUr1+f/v3789tvvzF8+HD69evH9u3bKVu2LH369GHq1KkMHz6cEydOALB3714qVarEkSNHSEtLo04dbeR58ODBTJs2jTJlynDo0CHeeustduzYAcClS5fYtm3bf3rqN2/epG7duo+fe3l5cfPmzWzjnTx5MvPnzycgIIAJEybg5OT00t8f0K1HXx/oDZwWQpzMfO1/QAkAKeU0YAPQBggFHgJvZG5LE0IMBzYD5sAcKeXZHEX8DKVd7QEIi0mkqncRCPoW/F+DNcPhz9ehUmcI/h7sX753/zSW5pb4FPbBp7DPf7Y9TH3I1XtXCY0PJTQ+lMt3L3Mg8gBrrqx53MbLwYuKzhXxd/GnknMl/F38sbe013ucilIQeXt7U79+fQB69erFL7/8QsuWLSlVqhRly5YFoG/fvkyZMoX33nsPPz8/zp8/z+HDhxkxYgR79uwhPT2dhg0bkpCQwP79++nSpcvj909OTn78dZcuXbIdjslufe7shneHDh3KZ599hhCCzz77jA8++IA5c+bk6Px1mXWzj+zH2rO2kcCwp2zbgPaHINf5ujoAcCUmQUv0AF41YfBu2Pcz7PkBwnZC8Hio1AnyaAzdztIOfxd//F3+3euIfRTLxbiLnI87z7nYc5yNPcuWa1sAMBNmlC5SmqquVanmWo3qbtXxdvRW4/5Kvva8nnduefL3RgiRbeL9R8OGDdm4cSOWlpa0aNGCfv36kZ6ezo8//khGRgZFihTh5MmT2e5rb599B83Ly4sbN/5/bkpERAQeHh7/aefu7v7460GDBtGuXbtnnZpOTGqwuKSzHRZmgisxCf/eYGEFTUZBxQ6wejisGACnl0Hbn6Cwp2GCBZxtnannWY96nvUev3Y36S5n7pzh9J3ThMSEsPnqZpZfWq61t3GmpntNarrXJKBYAH5F/NTMH0XRwfXr1zlw4ACBgYH8+eefNGjQgPLlyxMeHk5oaCh+fn4sWLCAxo0bA9CoUSP69OlDnz59cHV1JTY2llu3buHv748QglKlSrFs2TK6dOmClJKQkBCqVq36zBg6dOhAjx49GDFiBJGRkVy+fJnatWv/p11UVBTFixcHYNWqVVSqVCnH529Sid7S3IwSRe0Ii0nMvoFbBRiwBQ5Ng+1j4Le60PJrqNEXzIwjYTrZONHQqyENvRoCkCEzuBJ/hRPRJzgRfYKjt48+7vU7WTtRq1gt6hSvQ93idVWPX1GeokKFCsybN48333yTMmXKMHToUGxsbJg7dy5dunR5fDF2yJAhANSpU4fbt2/TqFEjAKpUqYKbm9vj368//viDoUOH8s0335Camkq3bt2em+j9/f3p2rUrFStWxMLCgilTpjwe4hk4cCBDhgwhICCAjz76iJMnTyKEwMfHh+nTp+f4/MWzPr4YSkBAgHzZhUcGzjvK9bhEtrzf+NkN467C2nfg6h7waQjtJ4Fz6Zc6Zl6SUhKZGMmRW0c4cusIh6IOcfvhbQA8HTypW7wuDTwbUKd4HRytHA0craLA+fPnqVChgsGOHx4eTrt27Thz5ozBYtC37L6nQohjUsqA7NqbVI8eoLSbPXsuxZCeITE3e0bvtmgp6LMGjs+HLZ/C1HrQ9BOo+xaYG++3RQiBp4Mnnn6edPTriJSS6w+ucyDyAAciD7A5fDMrLq/AQlhQ1a0qjbwa0cizEaWLlFa9fUUpoIw3o72k0i4OpKRnEHH3ISWdnzNrRQio2RfKtIT1H8DWz+DsKnhlMrgb5qLRixJCULJQSUoWKkm38t1IzUglJCaEv2/+zZ6IPfx87Gd+PvYzng6eNPFuQhPvJtR0r4mlme4lThUlP/Px8TGp3vzLML1E76Yl9ysxCc9P9P8o5AHdFmlJfsNImN4IGn6gPSysczFa/bM0s3x8wfadGu9wK/EWe2/uZfeN3Sy/tJw/zv+Bo5UjTbya0Lxkc+p71MfGwsbQYSuKkotMLtH7umhTLMNiEmlW/gV2FAIqvQa+TWDTaNj9PZxbDR0mg3etXIk1LxSzL0aXsl3oUrYLD1MfciDqADuu72DXjV2sDVuLrYUtjbwaEeQTRAPPBtha2Bo6ZEVR9MzkEr2TvRVF7a3+O8VSV3ZF4bUZ2s1V696D2S21cftmn4BV/r6Byc7SjuYlmtO8RHNSM1I5euso265tY9v1bWwO34ythS1NvJoQXCqYBp4NsDRXwzuKYgpMLtGDdofsleinTLHUVdlW8NZB2PYlHJwCF9dD+1/A9zmzefIJSzNLAj0CCfQI5OM6H3Ps9jE2h29m67WtbAzfiKOVI61KtqKdbztquNdQ8/UVJR8zyd9eXxcHwu68ZI8+K5tC0O4n6LcBhDnM7wBr3oZH8Tl/byNiYWZBneJ1+Dzwc3Z03cGU5lNo7NWYDVc38MbmNwheEcyvJ37l+v3rhg5VUfKt2NhYmjZtioODA8OHD39qu7i4OFq2bEmZMmVo2bIld+/ezfGxTTLRl3az505CCvEPU/Tzhj71YejfUP9dOLFQu9HqQp5UdchzlmaWNPJqxNiGY9nVdRdjG46lVOFSzDo9i7ar2tJnYx9WXFpBYmoOPzEpiolLS0v713MbGxvGjBnDjz/++Mz9xo0bR/Pmzbl8+TLNmzdn3LhxOY7FJBP9PxdkrzztDtmXYWmr3UU7cDvYOcPi7rDsDUiI0d8xjIydpR3tfNsxreU0tnTawns13iM+OZ4vD3xJ06VN+fzvzzkZffKZNUMUxdDCw8MpX748ffv2pUqVKnTu3JmHDx8CsH37dqpXr07lypXp378/ycnJHD58mNdeew3QSgvb2tqSkpJCUlISvr6+AFy5coXWrVtTs2ZNGjZsyIULFwDo168fI0aMoGnTpowaNepfcdjb29OgQQNsbJ49y2316tX07dsX0Aqt/fXXXzn+HpjmGL3bPzNvEqhZMmflPf/DswYM3gV/T4Td47Uiaa2/hypd86xImiG427szoPIA+lfqz6mYU6wKXcXGqxtZFboKvyJ+dC7bmXa+7VS5ZeXZNo6GW6f1+57FKkPws3u9xlCmWFe3b99+XOumePHiREdHv9T7ZGWSPXpvJ1sszQWhLzvz5nnMLaHRSHhzLzj7warBsKgr3IvIneMZESEE1dyq8VW9r9jZdSdfBH6Btbk14w6Po8WyFnyx/wvOxuZaJWpFeSlPlinet28fFy9e/E+Z4j179mBhYZFtmeK9e/f+p0xxtWrVePPNN4mKinp8rKeVKTYkk+zRW5ib4e9RmD2X7vBxcC4eyK089N8Mh2fC9q9gSl1o+SXU7G80RdJyk72lPZ3LdqZz2c6ciz3H0otL2XB1Aysvr6SyS2V6VOhBUMkgNU1T+X/P6XnnFmMoU6wrd3f3xxUso6KicHNzy9H7gYn26AFeq+HJ+aj7nI28l7sHMjOHukPgrQNa7fv1H8DvbeFOaO4e18hUdK7Il/W+ZFuXbYyuPZoHKQ/4eO/HtFzekt9O/sadR3cMHaJSgP1TphjItkwx8J8yxRMnTiQwMPBxmeILFy7g7+9PoUKFHpcpBq3Q4KlTp/QWa4cOHZg3bx4A8+bN45VXXsnxez430Qsh5gghooUQ2RaLEEKMFEKczHycEUKkCyGKZm4LF0Kcztz2cuUoX1L7Kh5YmZux4lj2S3XpnZMP9P4LXpkC0WdhWn3YNxHS056zo2kpZFWInhV6srrjaqa1mIa/iz/TTk2j1fJWfLLvEy7GXTR0iEoB9E+Z4ipVqhAXF/efMsWVK1fGzMzsmWWKq1Sp8q8yxbNnz6Zq1ar4+/uzevVqneLw8fFhxIgR/P7773h5eXHu3DlAK1P8T8Xe0aNHs3XrVsqUKcPWrVsZPXp0js//uWWKhRCNgARgvpTymRXwhRDtgfellM0yn4cDAVLKF+rO5aRMcVZDFx7j0NU4Dn7cHCuLPPzw8uCW1rO/sA6KV9WSf7HKeXd8I3Pt/jUWnlvI6iureZT2iLrF69LXvy/1PeqripoFgCpTrH8vWqb4udlPSrkHiNPx+N2BP3Vsm+s61/QiLjGFXRdzftX6hTgWg9cXQpd5cD8SZjSBHd9AWvJzdzVFJQuV5JO6n7C181beq/EeYfFhDN02lNfWvMaaK2tIzUg1dIiKYtL01s0VQtgBrYEVWV6WwBYhxDEhxODn7D9YCHFUCHE0JkY/c9MblXXFxcGaFccNMBtGCPDvCMMOQ+Wu2nq10xrA9UN5H4uRKGxdmAGVB7Cp0ya+bfAtAJ/s+4Q2K9uw4NwCHqY+NHCEiilSZYr1ezG2PfC3lDJr77++lLIGEAwMyxwGypaUcoaUMkBKGeDq6qqXgCzNzehYzYPt56OJTTBQb9quKLw6FXqtgNRHMCcINo6C5Fya+pkPWJpb0qF0B1Z2WMmU5lPwsPdg/JHxtF7RmhkhM7ifct/QISqKSdFnou/GE8M2UsrIzH+jgVXAf1fCzWWdanqRliFZcyoyrw/9b34ttJk5tQdpa9ZODYQrOw0bk4EJIWjk1Yh5wfNYELyAyq6V+fXErwQtD+LXE78SnxRv6BAVxSToJdELIQoDjYHVWV6zF0I4/vM10ArI889PFYoXoopXYRYcuEZ6hoFv1bd2hDY/wBubwNwKFnSE1cPgUc6LFuV31dyqMaX5FJa1X0agRyAzQ2YStCKIn4/9zN0k9f1RlJzQZXrln8ABoJwQIkIIMUAIMUQIMSRLs1eBLVLKrMVl3IF9QohTwGFgvZRykz6D19WQxqUJu5PIhtNRz2+cF0oGwpC/ocEIOPknTKkD59caOiqjUL5oeX5q8hMrO6yksVdj5p6ZS9CKICYdn6R6+Irykp47vdIQ9DW98h8ZGZJWE/dgLgQb322I2bMWDc9rUae0Xv2t01Cxo9bjd8j5nXCmIiw+jGmnprEpfBN2lnb0qtCLPv59KGRVyNChKToy9PRKY/HPnPiUlBSsrKz44YcfaNas2X/axcXF8frrrxMeHo6Pjw9Lly7FyenfNbv0Pr3SFJiZCYY39ePi7QdsOXfb0OH8W/GqMGgnNP8cLm6EybW0Xr4R/gE2BN8ivoxvPJ6VHVZS36M+00OmE7wimFmnZ6lZOopRe7JMsYuLC2vXruX06dPMmzeP3r17Z7ufKlOcA+2qFMfH2Y7JOy8bX1ldc0ttIfIh+8C1PPw1BBZ2gni10Mc//Jz8mNBkAsvaL6O6W3UmHZ9Em5Vt+PPCn6Smq3n4ytMZS5ni6tWr4+HhAYC/vz9JSUkkJ/93NqAqU5wDFuZmvNXEj49WhLDrYgxNyxvh8IhrWXhjIxyZpS1h+FsgNP8Cag0sEEXSdFG+aHkmN5/MyeiTTDw+ke8Ofce8s/N4u/rbBJcKVkseGrnvD3/PhbgLen3P8kXLM6r2qGe2MbYyxStWrKB69epYW1v/Z5sqU5xDr9bwxLOILT9vu0SGoWfgPI2ZGdQZDMMOgncd2DgS5gbDncuGjsyoVHOrxtyguUxtMRUHSwdG7x1Nt3XdOBh10NChKUbImMoUnz17llGjRjF9+vTcPeksCkyPHrQbqD5oVZYRS0+x8sRNOtf0MnRIT1ekhHaT1ak/YdPHMLU+NBkF9d7RhnoUhBA08GxAPY96rA9bz+QTkxm0ZRD1PevzYc0P8XPyM3SIyhOe1/POLcZSpjgiIoJXX32V+fPnU7p06WzbqDLFetCxmifVvIvw/aYLJCQbeWVJIaBaD62MQtkg2P41zGymzdRRHjMTZrQv3Z41r67hw4APCYkJodPaTnx14CtVHlkBjKNMcXx8PG3btmXs2LGPP11kxyBlik2NmZngi/YViXmQzJSd+aRmvKM7vL4Aui6AhNswoyls+wpSkwwdmVGxNremr39fNry6gR7le/DX5b9ou7Its07PIjm9YBaUUzTGUKZ48uTJhIaGMmbMGKpVq0a1atUej78bvEyxIeh7Hn12Riw9ybpTUWwd0YiSzjlbESZPPboLmz+Fkwu1ZQw7TNZuwFL+49r9a0w4OoGdN3bi6eDJiJojaFmypSqNnMcMPY9elSkugD36f4xqXR4Lc8GYdeeNb7rls9g6Qccp0HsVpKfA3Naw/kNIfmDoyIxOyUIl+aXZL8xsNRM7Szs+2P0BA7cM5NLdS4YOTVHyVIFN9O6FbHineRm2nb/NhtO3DB3OiyvdDIYegDpDtOmYvwVC6DZDR2WU6havy9J2S/mkzidcvHuRLmu78O3Bb7mXnMvLTCpGQZUpLsCJHmBgg1JU9izMF2vOEJeYYuhwXpy1AwR/ry1Qbmmr3WS1aig81HWdmILDwsyCbuW7sa7jOrqU7cLSS0vp8FcHVl1eRYbMMHR4Ji9ffWo2ci/zvSzQid7C3IzxnasQ/zCVr9eeNXQ4L69EHXhzLzT8EE4v1YqkndNtDcuCpohNET6t+ylL2i2hhGMJPt//Ob039uZ87HlDh2aybGxsiI2NVcleD6SUxMbGYmNj80L7FdiLsVn9vPUSk7ZfZnbfAJpXcM+z4+aKqBBYM1ybglm+HbSdoC1tqPyHlJK1YWuZcHQC8cnxdC/fnWHVhuFo5Wjo0ExKamoqERERJCWpWWL6YGNjg5eXF5aW/76f5lkXY1WiB1LSMmj/6z7uPkxh03uNKGpvlWfHzhXpaXBgMuz8DixtIOg7qNZTm5ev/Me95Hv8euJXll5cirOtM6NqjyKoZJCanaPkK2rWzXNYWZjx0+tViX+YykfLQ/L/R0xzC2jwHgzdD27+WhnkBR3hbriBAzNOha0L82ndT1nUdhGutq6M3D2St7a/RcQDA6w1rCi5QCX6TP4ehRkVXJ5t52+z4OA1Q4ejHy5+0G+9NnwTcVSbmXNwKmSkGzoyo1TJpRKL2i5iVK1RHL99nFdXv8rvZ34nLcPI76BWlOfQZYWpOUKIaCFEtvOThBBNhBD3hBAnMx+fZ9nWWghxUQgRKoTI+e1duax/fR+alnPlm/XnOR9lIgtUm5lp1S/fOggl68Om0TCnNUTrt4KgqbAws6BXxV6s7riaQI9AJhybQI/1PTgXe87QoSnKS9OlR/870Po5bfZKKatlPr4GEEKYA1OAYKAi0F0IUTEnweY2IQQ/dKlKYVtL3v7zBInGXgvnRRTxhp7L4NUZEHsZpjeE3T+AquWerWL2xZjUdBI/NfmJmEcxdF/fnZ+O/URSmrqgqOQ/z030Uso9wMtMzK4NhEopw6SUKcBiIOfVeXKZi4M1k16vRlhMAh+tMIHx+qyEgKqvw7Aj2oycnd/AjCYQecLQkRklIQQtS7ZkdcfVvOr3KnPPzKXz2s4cvZV3EwUURR/0NUYfKIQ4JYTYKITwz3zNE7iRpU1E5mvZEkIMFkIcFUIcjYmJ0VNYL6eenwsftS7P+pAoZu29atBYcoWDK3SZC90WwcNYrSLm1s8h9ZGhIzNKhawK8WW9L5nZaibpGem8sfkNvj34rVrKUMk39JHojwMlpZRVgV+BvzJfz25u2lO7x1LKGVLKACllgKurqx7Cypk3G/kSXKkYYzeeZ/8VEy11W76tNnZfvTf8PQmm1oPwfYaOymjVLV6XFR1W0KtCL5ZcXMJra15TC50o+UKOE72U8r6UMiHz6w2ApRDCBa0H752lqRcQmdPj5ZV/xut9XR0YvugEN+JMtPdmWwQ6/AJ91oDMgN/bwrr3IclELkbrmZ2lHaNqj2Je8DwszCwYtGUQYw6MUb17xajlONELIYqJzDtLhBC1M98zFjgClBFClBJCWAHdgDU5PV5ecrC2YEbvmqSlZzBg3hHuJ5nwhUvfxtq8+8DhcOx3+K0uXNpi6KiMVnW36ixvv5w+Ffuw7NIyOq3ppMbuFaOly/TKP4EDQDkhRIQQYoAQYogQYkhmk87AGSHEKeAXoJvUpAHDgc3AeWCplDLfFZTxdXVgWq+ahMUkMnzRCdLSTbgAlpU9BH0LA7aCtSMs6gIrBkFirKEjM0o2FjaMrDWS31v/jhCC/pv7M/7IeDUzRzE6qgSCjhYfvs7olafpE1iSrzr4m/7t8WnJsHeC9rApAm3Gg/9rqozCUzxMfchPx35iycUllC5cmm8bfou/s//zd1QUPVElEPSgW+0SDG7ky/wD15ixJ8zQ4eQ+C2to+j8YvFubg7+8PyzuCfejnr9vAWRnacendT9leovpPEh9QK/1vZh+arq6q1YxCirRv4DRrcvTrkpxxm68wMrjBaQOSrFKMGAbtPoGrmzXSiAfmwdG+EnQGNTzrMfKDitp6dOSyScn88amN7jx4Mbzd1SUXKQS/QswMxNM6FqV+n7OfLQ8hF0Xow0dUt4wt4B6b2sXa4tXgbXvwPwOEGeC9xjoQWHrwoxvNJ5xDcdxJf4Kndd05q/Qv0zr5jslX1GJ/gVZW5gzrVdNyro78tYfxzl+/a6hQ8o7zqW1aZjtJkLkSa1I2oEpqkjaU7T1bcuKDiuo6FyRz/7+jJF7RqrlCxWDUIn+JTjaWPJ7/1q4OVrTd85hztwsQL+8ZmYQ8IZ2o5VvY9j8P5jdCm6rol/ZKe5QnFmtZvFO9XfYfm27KqGgGIRK9C/JzdGGPwbVpZCNJX3mHOby7QeGDilvFfaE7ouh02y4exWmN4Jd4yAtH669m8vMzcwZVGUQ84PnY2VmxYAtA/jt5G/qQq2SZ1SizwHPIrYsHFgHczNBz1mHuHon0dAh5S0hoHJnGHYYKr4Cu8bCjMZw85ihIzNKlV0rs7T9UtqWasvUU1MZsHkAtxJvGTospQBQiT6HSrnYs3BAHdIyJN1nHCx4yR7A3gU6z4buS+BRPMxqAZs/gRRVFuBJ9pb2fNfwO75r8B0X4i7QaU0ndl7faeiwFBOnEr0elCvmyB8D65CSnkG3GQcKZrIHKNcahh2EGn21NWunBsLVPYaOyii1L92epe2X4ungyTs732H8kfGkqrUBlFyiEr2eVCheiEWD6pCaLuk24wBXYhIMHZJh2BSG9hOh7zpAwLz2sPZdSCpAF6x1VLJQSRa2WUiP8j1YcG4BvTf2VuvUKrlCJXo9Kl9MS/Zp6ZLXpx8wneUIX0aphtq8+3pvw/H5MKUuXNxk6KiMjpW5FR/X+ZiJTSZy/f51uq7ryvbr2w0dlmJiVKLXs/LFCrHkzUAszMzoNuMgp27EGzokw7Gy0+6oHbgNbJ3gz9dh+QBINNH6/jnQvGRzlrRfgrejN+/tfE8byslQQzmKfqhEnwv83BxYNiSQQrYW9Jx1iINhBbz6o2dNGLwLmvwPzq2GKbXh9HJVRuEJ3o7eLAheQPfy3VlwbgH9N/XnduJtQ4elmACV6HOJd1E7lr1Zj2KFbegz5zBbzhbwaXQWVtBkFAzZC06lYMUA+LMb3Ltp6MiMipW5Ff+r8z/GNxrPxbsX6bquq1rFSskxlehzUbHCNix7M5AKxQsxZOExlh5Rxa1wqwADtkDQdxC2WyuSdnQOZJhwnf+XEFwqmMVtF+Nk7cSbW99k1ulZZEj1PVJeji4Lj8wRQkQLIc48ZXtPIURI5mO/EKJqlm3hQojTQoiTQogCed+3k70ViwbWob6fCx+tCGHKzlBV3MrMHAKHwVsHwLO6tnTh/A4Qe8XQkRkV3yK+LGq7iKCSQUw6Pon3dr7Hg5QCdge2ohe69Oh/B1o/Y/tVoLGUsgowBpjxxPamUspqTyuIXxDYW1swu28tXqnmwQ+bL/LJX2dMe6UqXRUtpRVJ6/ArRIVoi5P//Qukq9IA/7CztOP7Rt8zuvZo9kbspfv67oTeDTV0WEo+89xEL6XcA8Q9Y/t+KeU/JRwPoi0CrjzBysKMn7tW460mpVl06DpvLjjGwxSV0BACavSBYYegdHPY+hnMbgm3sv0AWSAJIehZoSezg2aTmJpIjw092BKu1vNVdKfvMfoBwMYszyWwRQhxTAgx+Fk7CiEGCyGOCiGOxsTE6Dks42BmJviodXm+6ViJnRej6Tr9ALfuqfVFAShUHLr9AZ3nQvx1rWbOzu+0JQ0VAGq412BJuyWUdSrLB7s/4KdjP5GuSkQrOtBpzVghhA+wTkpZ6RltmgK/AQ2klLGZr3lIKSOFEG7AVuDtzE8Iz2SMa8bq284L0QxfdBxHG0tm9Q2gkmdhQ4dkPB7GwabRELIEXMtDh8ngXcvQURmN1PRUxh0ex9JLS6nvUZ/vG31PYWv181PQ5fqasUKIKsAs4JV/kjyAlDIy899oYBVQWx/HMwVNy7uxfGg9zAR0nX5ATb/Myq4ovDYDeiyD5ARtKGfT/yClgNYQeoKluSWfBX7Gl4FfcujWITVurzxXjhO9EKIEsBLoLaW8lOV1eyGE4z9fA60ANfCaRYXihfhreH3KuDvy5sJjakbOk8q20mbm1BoAB6doK1qF7TJ0VEajU9lOzA2ay8PUh/Tc0JMd13cYOiTFSOkyvfJP4ABQTggRIYQYIIQYIoQYktnkc8AZ+O2JaZTuwD4hxCngMLBeSqmKnTzBzdGGJYPr0qGqNiPnncUneZSixl0fsykEbSdAvw1gZgHzX4HVw7VyyArV3KqxpN0SShUuxbs732VGyAzVWVD+Q6cx+rxWEMbonySlZNruMMZvvoC/RyGm9aqJl5OdocMyLqmPtMVN9v8K9m7aH4AK7QwdlVFISkvii/1fsOHqBlr7tObr+l9ja2Fr6LCUPJTrY/RKzgkhGNqkNLP7BnAt9iHtf93H/lBV/OtfLG2h5dcwcLu22MmSnrCsHySY5iytF2FjYcO4huN4v+b7bA7fTL9N/VSdHOUxleiNTLPy7qwZ3gAXB2t6zT7EjD1X1EfxJ3nW0IqkNf0ULqyHKbXg1JICXyRNCEH/Sv35pdkvhN8Lp/v67py9c9bQYSlGQCV6I1TKxZ5Vw+rTulIxvttwgaELj3M/SZWs/RdzS2g8Et7cC85lYNVgWNQV7qmFO5p4N2FBmwVYmVvRd1NfNodvNnRIioGpRG+kHKwtmNKjBp+2rcDW87d5ZfLfXLhVgBcyeRq38tB/E7T+HsL3aUXSjswq8EXSyjqV5Y82f1ChaAU+3P2hukhbwKlEb8SEEAxs6Mufg+qSkJxGxyl/s/ToDfUL+yQzc6g7RJuK6RUA6z+A39vCnYI9t9zZ1plZQbNo59uOX0/8yif7PiElPcXQYSkGoBJ9PlC7VFHWv9OA6t5OfLQ8hA+WnVJ1crLj5AO9/4JXpkD0WZhWH/b9XKCLpFmbW/Ndg+8YVm0Ya8PWMmjLIOKT4g0dlpLHVKLPJ9wcbVg4sA7vNi/DqhM36aCGcrInBFTvBcMOg18L2PYlzGoGt04bOjKDEUIwpOoQxjcaz5k7Z+i1sRfX7183dFhKHlKJPh8xNxO837IsCwfU4d6jVDpM/psFB6+poZzsOBbTiqR1nQ/3o2BGE9g+BlILbhG54FLBzAqaxf3k+/Tc0JPjt48bOiQlj6hEnw/V93Nh47sNCfR15rO/zjB04XHiH6qx12xVfEUrgVy5C+z9EaY3hOuHDB2VwVR3q84fbf6giHURBm0ZxKZwdbN6QaASfT7l4mDN3H61+KRNBbZfuE3riXvVDVZPY1cUXp0GvVZod9fOCYKNo7SCaQWQdyFtEfJKLpUYuXskc8/MVZ8KTZxK9PmYmZlgUCNfVg6tj52VOT1nH2LsxvMkp6laOdnya6HNzKk9CA5N14qkhW43dFQGUcSmCDNazSDIJ4ifjv3Et4e+VbXtTZhK9Cagsldh1r3TgG61SjB9dxgdp+zn0m21tmi2rB2hzQ/wxkawsIaFr8Ffb2k18AsYa3Nrxjcazxv+b7Dk4hJG7BpBUlrBvYZhylSiNxF2VhaMfa0yM/sEEH0/iXa/7mPW3jAyMtRH8myVDIQh+6DBCDi1WLvR6twaQ0eV58yEGSMCRjC69mh23tjJwC0DuZt09/k7KvmKSvQmpmVFdza914iGfi58s/48PWYd5EbcQ0OHZZwsbaDFFzB4Jzi6w9LesKQ3PCh4xcB6VujJhCYTOB97nj4b+3Az4aahQ1L0SCV6E+TqaM2svgF836kyZ27ep/XEPSw+fF1dcHua4lVh0E5o/jlc2gxTasPJRQWuSFrLki2Z2WomsUmx9NrQi4txFw0dkqInKtGbKCEEr9cqwcZ3G1LFqwijV56m39wjRMY/MnRoxsncEhp+oA3nuJaHv4bCwk5w95qhI8tTNdxrML/1fMyFOf029eNw1GFDh6TogS4rTM0RQkQLIbJdBlBofhFChAohQoQQNbJsay2EuJi5bbQ+A1d0413Ujj8G1uGrDv4cvhpH0M97WHJE9e6fyrWsdqE2+Ae4flCbmXNoeoEqkubn5MfCNgtxt3NnyLYhbAnfYuiQlBzSpUf/O9D6GduDgTKZj8HAVAAhhDkwJXN7RaC7EKJiToJVXo6ZmaBvPR82vdeQih6FGLXiNH3mHCbirhq7z5aZGdQZDMMOQom6sPEjmBsMMZeev6+JKGZfjHnB8/B39ufD3R+y9OJSQ4ek5MBzE72Ucg/wrLlnrwDzpeYgUEQIURyoDYRKKcOklCnA4sy2ioGUdLbnz0F1GfOKP8ev3aXVz3uYfyBczcx5miIltJusOk6DmAtakbQ9P0J6wVgboLB1YWa0mkFDr4aMOTiGqaemqk+C+ZQ+xug9gRtZnkdkvva017MlhBgshDgqhDgaE6OWhsstZmaC3oE+bH6/ETVLOvH56rN0nX6A0Gg17z5bQkC17jD8CJRrAzvGwMymEHnS0JHlCVsLWyY2nUiH0h347eRvfH/kezJkwRnGMhX6SPQim9fkM17PlpRyhpQyQEoZ4OrqqoewlGfxcrJjfv/a/NilKqExCbSZtI+J2y6RkqZ+ibPl4AZd58HrCyEhGmY20ypjppr+xW1LM0vG1B9Drwq9+OP8H3yy7xNSMwrGpxpToY9EHwF4Z3nuBUQ+43XFSAgh6FzTi63vNyaoUjEmbrtMm1/2ciS84N0lqrMK7bUiadW6a7XupzWAa/sNHVWuMxNmfFTrI96u/jbrwtbx/s731V20+Yg+Ev0aoE/m7Ju6wD0pZRRwBCgjhCglhLACumW2VYyMq6M1v3avzpx+ATxKSafLtAN8vDKEew9Vry1btk7a4ia9/4L0FO1C7foPIdm0h7+EEAyuMphP63zKnog9vLX9LRJTEw0dlqID8byLK0KIP4EmgAtwG/gCsASQUk4TQghgMtrMnIfAG1LKo5n7tgEmAubAHCnlt7oEFRAQII8ePfoSp6Pk1MOUNH7eeok5f4fjZGfJJ20r0LGaJ9p/s/IfKYmw4xs4OBUKeUL7SVCmhaGjynXrwtbx6b5PqVC0AlNbTKWITRFDh1TgCSGOSSkDst1mjFfRVaI3vLOR9/hk1RlO3ogn0NeZMR0r4efmYOiwjNeNw7B6ONy5CFW6QeuxWnlkE7brxi4+2PUBJQqVYEbLGbjaqWtrhqQSvfJSMjIkfx65zvcbL/AoNZ2BDX15u5kfdlYWhg7NOKUlw54ftLF7WyetSmbFjtrMHRN1OOoww3cMx9XWlZmtZuLh4GHokAosleiVHLmTkMzYDRdYcTwCzyK2fNauAkH+xdRwztPcOq317qNOQvl20HaCtrShiToZfZK3tr2FvZU9M1vOxKewj6FDKpCelehVrRvluVwcrJnQtSrLhgTiaGPBkIXH6TPnMFdiCuYKTc9VrDIM3A4tvoLQbVqRtOMLTLZIWjW3aswOmk1yWjL9NvXj8t3Lhg5JeYLq0SsvJC09g4UHrzFh6yWSUtPp36AUbzcrg4O1Gs7J1p1QWPsOXPsbfJtoF2udfAwdVa4Iiw9j4JaBpGakMqPlDCo4VzB0SAWKGrpR9O5OQjLfb7zAsmMRuDlaM6p1eV6t7omZmRrO+Y+MDDg2F7Z+ATJdK4dcezCYmRs6Mr27fv86A7cMJCElgWktp1HFtYqhQyowVKJXcs2J63f5cu05Tt2Ip3qJInzR3p9q3kUMHZZxuhcB696Hy1vAqzZ0+BXcyhs6Kr2LTIhkwOYBxCXF8VuL36jpXtPQIRUIaoxeyTXVSzixamg9xneuwo24R3Sc8jcjlp7k9n111+R/FPaCHkvhtZkQGwrTG8LuH0yuSJqHgwe/t/4dNzs3hm4bypFbRwwdUoGnevSK3jxISmXKzivM2XcVC3PB0MalGdTIFxtL0xuiyLGEGK388dmV4F5J69171nj+fvnInUd3GLh5IDcTbjKp2STqedQzdEgmTQ3dKHnqWmwi4zZeYOOZW3gUtuGj1uXpUNVDjd9n58IGWD8CEm5D4HBo+j+wtDV0VHoT+yiWQVsHce3eNSY1m0QDzwaGDslkqUSvGMTBsFjGrDvH2cj7VPUqzCdtK1K7lGnfLfpSHsXD1s/g+Hwo6qv17n1MJyHGJ8UzaOsgwuLDmNh0Ig29Gho6JJOkxugVg6jr68za4Q34sUtVbt9Ppuv0A7y54ChX76hCWP9iW0RL7n3WgMyA39tqF22T7hs6Mr0oYlOEWa1mUbpIad7d+S57IvYYOqQCR/XolTzxKCWdmXvDmLb7CilpGfSsU4J3mpfB2cHa0KEZl5SHsPNbOPgbOBaHdj9D2SBDR6UX95LvMXjrYC7dvcTEJhNp7N3Y0CGZFDV0oxiN6AdJTNx2mSVHbmBrac6Qxr70b1BK1c95UsRRrYxCzHmo3BVajwN7Z0NHlWP3U+4zeEtmsm86kUZejQwdkslQiV4xOqHRDxi38SLbzt/GzdGa91uWpUtNLyzM1WjiY2kpsHeC9rApBMHjoVKnfF8k7Z+e/eW7l1Wy1yOV6BWjdSQ8jnEbL3Ds2l18Xe0Z2aocrSupgmn/cvus1ruPPK6tW9t2AhTK31Uisyb7SU0nqQu0eqASvWLUpJRsPXebHzZf5HJ0AlW9izAqqBz1/FwMHZrxyEjXxu13fAvmltBqDNTom6979/eS7zFoyyCuxF/h12a/Us9TzbPPiRwneiFEa2AS2kpRs6SU457YPhLomfnUAqgAuEop44QQ4cADIB1Ie1ogWalEXzClZ0hWHI9g4tZLRN5LooGfCyODylFVlVT4f7FXYO27EL4XSjXSiqQV9TV0VC8tPimegVsGEn4/nCnNp1CneB1Dh5Rv5SjRCyHMgUtAS7QFv48A3aWU557Svj3wvpSyWebzcCBASnlH14BVoi/YklLT+ePQdabsDCUuMYUgf3c+aFWOsu6Ohg7NOGRkwPF5sPVzrXxCs0+h7tB8WyTtbtJd+m/uT8SDCH5r8Ru1itUydEj5Uk7n0dcGQqWUYVLKFGAx8Moz2ncH/nzxMBVFY2NpzoAGpdg9sgnvtSjD36GxBE3cw3uLT3AtVs3Bx8wMAt6Atw6Cb2PY8gnMbgm3s+17GT0nGydmtZqFh4MHw7YP42T0SUOHZHJ0SfSewI0szyMyX/sPIYQd2iLhK7K8LIEtQohjQojBTzuIEGKwEOKoEOJoTEyMDmEpps7RxpL3WpRl70dNGdzIl01nb9Fswm5GrwjhZvwjQ4dneIU9ofti6DQb7obD9Eawa5w2WyefcbZ1ZlarWY8LoZ25c8bQIZkUXRJ9dld7njbe0x74W0oZl+W1+lLKGkAwMEwIke1cKinlDCllgJQywNVVLTKs/D8neys+Dq7AnpFN6V23JCuP36TpD7v4fPUZbt0r4FUyhYDKnWHYEfDvCLvGwozGEHHM0JG9MFc7V2a1mkVh68IM3jqYC3EXDB2SydAl0UcA3lmeewGRT2nbjSeGbaSUkZn/RgOr0IaCFOWFuRWy4csO/uwc2YRONT1ZdOg6jX7YyVdrzxL9oIAnfHtn6DQLui/RaufMbgGbP9HutM1HitkXY3bQbOwt7Rm8ZTBX4q8YOiSToMvFWAu0i7HNgZtoF2N7SCnPPtGuMHAV8JZSJma+Zg+YSSkfZH69FfhaSrnpWcdUF2MVXdyIe8ivOy6z4vhNLMwEveqW5M3Gvrg52hg6NMNKuqetZnVsrrZsYYdftRk6+ci1+9fot6kfAsHvrX+nRKEShg7J6OljemUbYCLa9Mo5UspvhRBDAKSU0zLb9ANaSym7ZdnPF60XD9q0y0VSym+fdzyV6JUXEX4nkV93hLLqRARWFmb0rKMSPgBX98Cad+DuVajZD1p+DTaFDR2VzkLvhvLG5jewtbBlXut5FHcobuiQjJq6YUopEK7eSeTXHZdZfTISCzNBjzolGNK4NO6FCnDCT3kIu76DA1PAwV0rklYu2NBR6exc7DkGbh6Ik40T84Ln4WKrbqJ7GpXolQIl/E4iU3aGsvLETczNBN1qeTOkcWk8ipjOgh4v7OYxWP02RJ/V6uUEjwf7/JE0T0afZPDWwXg5ejE3aC6FrfPPp5K8pBK9UiBdj33I1N2hLD8WAUDnml4MbexHCWc7A0dmIGkpsO9n2PMDWDtC8PdQuUu+KKNwMOogw7YNo6xTWWYFzcLe0t7QIRkdleiVAu1m/COm7brCkiM3SJeSV6p58FYTP/zcHAwdmmFEn9eKpN08CmWCoN1P2sLlRm7XjV28v/N9qrlVY2qLqdhYFOAhuWyoRK8owO37SczYE8Yfh66RnJZBcKVivNXEj0qeBXAoICMdDk2HHWNAmEOrr6FGP+2uWyO2IWwDo/eOppFXI35u+jOWZpaGDsloqESvKFnEJiQz5++rzN9/jQfJaTQp58pbTfwK5nq2cVe1ImlXd0PJBtDhF3AubeionmnpxaWMOTiGNqXaMLbhWMyEcf9xyisq0StKNu4npbLgwDVm77tKXGIKtXyceKuJH03KuRasevhSwokFsPlTSE+Gpv+DusPA3HhX/Zp1ehaTjk/i9XKv80mdTwrW/9dTqESvKM/wKCWdJUeuM2NPGJH3kihfzJGhTUrTtnLxgrXi1f0oWP8BXFwPHtWhw2QoVsnQUT3VT0d/Yu7ZuQypOoRh1YYZOhyDU4leUXSQkpbBmlORTNt9hdDoBLyL2jK4oS+da3pja5U/SwC/MCnh3F+wYSQ8ugsNRkCjD8HC+BZxl1Ly5YEvWXl5JaNrj6ZnhZ7P38mEqUSvKC8gI0Oy7fxtpu6+wonr8RS1t6JvoA99AkviZG9l6PDyxsM42PQxhCwG1/Ja797b+OrEp2WkMXL3SLZd38Z3Db6jfen2hg7JYFSiV5SXIKXkSPhdpu2+wo4L0dhamvN6LW8GNCiFd9ECMhf/8lZY+x7cvwl1hkDzz8DKuOawJ6cnM2zbMI7dPsakZpMK7GLjKtErSg5dvPWAGXvCWH3yJhJoU7k4gxv6UtmrAEzNTH4A276EI7OgSEltZo5vE0NH9S+JqYm8sekNrt67ysxWM6nmVs3QIeU5legVRU+i7j1i7t/hLDp0nYTkNAJ9nRnUqBRNyrphZmbiMz/C/4Y1b0PcFajeG1p9A7ZFDB3VY7GPYumzsQ/xyfHMaz0PPyc/Q4eUp1SiVxQ9u5+UypLDN5jz91Wi7iVR2tWegQ19ebW6JzaWJnzhNvWRtorV/l/B3lW7q7Z8W0NH9VjEgwh6b+yNmTBjYfDCAlXxUiV6RcklqekZrA+JYubeMM5G3sfZ3opedUvSO7AkLg7GN1NFbyJPaEXSbp8G/1e1ImkOboaOCoCLcRfpt6kfbnZuzA+eX2CKoKlEryi5TErJgSuxzN53le0XorGyMOPVap70b1CKcsUcDR1e7khPhb8nwu7x2gXa1uOgyutGUSTtyK0jDNk6hArOFZjZaia2FqZfufRZiV6nu0GEEK2FEBeFEKFCiNHZbG8ihLgnhDiZ+fhc130VxRQIIajn58LsfrXYNqIxnWt6sfrUTYIm7qH37EPsvBhNRobxdapyxNwSGo2EIfvAuQysehP+6ALxNwwdGbWK1WJco3GExIQwcvdI0jLSDB2SQemylKA52lKCLdHWjz0CdJdSnsvSpgnwoZSy3Yvumx3Vo1dMwd3EFBYdvs68/eFEP0imtKs9b9QvxWs1PLGzMt7yAi8lIx0Oz4TtX4EwgxZfQsAAgxdJW3JhCd8c+oZOZTrxReAXJl0qIac9+tpAqJQyTEqZAiwGXtHx2DnZV1HyNSd7K4Y19WPfqGZMfL0adlYWfPrXGQLH7mDshvPcjH9k6BD1x8wc6g6Btw6AVy3Y8CH83hbuhBo0rNfLv86gyoNYcXkF00OmGzQWQ9Il0XsCWT+LRWS+9qRAIcQpIcRGIYT/C+6LEGKwEOKoEOJoTEyMDmEpSv5gZWFGx+qerBlen+VDAmng58KsfVdp+P0Ohi48xqGwWIzxWtlLcfKB3qvglSnaalZT62mLnaQbbujk7epv06F0B6acnMKqy6uev4MJ0uXzY3afdZ78qTwOlJRSJmQuJP4XUEbHfbUXpZwBzABt6EaHuBQlXxFCEOBTlACfotyMf8SCA9dYfOQ6G8/cokLxQrxRz4cO1Tzy//RMIaB6L/BrCRs+0G62OpuZ/ItVNkA4gi/rfUnso1i+OvAVLrYuNPRqmOdxGJIuPfoIwDvLcy8gMmsDKeV9KWVC5tcbAEshhIsu+ypKQeRZxJbRweU5MLo5416rTEaG5KMVIQSO3c64jRdMY1jH0R1eXwhd52uVMWc0ge1jIDUpz0OxNLNkQpMJlHUqywe7P+Bs7Nk8j8GQdLkYa4F2QbU5cBPtgmoPKeXZLG2KAbellFIIURtYDpQEzJ+3b3bUxViloJFScjAsjnn7w9ly7hYALSu60zfQh8DSzvn/IuLDONj8CZxaBC5ltSJpJerkeRgxD2PouaEnKekp/NH2Dzwdsh1JzpdyPI8+czhmIlriniOl/FYIMQRASjlNCDEcGAqkAY+AEVLK/U/b93nHU4leKchuxj/ij4PXWHzkBnGJKfi5OdC7bkleq+GJo00+XzovdJtWJO1eBNQeDM0/B+u8Xbs3LD6MXht74WzjzMI2C03mhip1w5Si5ENJqemsC4liwYFwTkXcw97KnFdreNIn0Iey7vn4JqzkB7D9a206ZmFvaD8R/JrnaQhHbx1l8NbBVHGtwoyWM7Ayz//lp1WiV5R87uSNeOYfCGddSBQpaRnUKVWU3oElaVWxGFYW+XQVrOsHYfVwiL0M1XpqRdLs8m7d3o1XN/LRno8ILhXMuIbj8v3asyrRK4qJiEtMYenRGyw8eI2Iu49wcbCme21vutcugUeRfHibf2oS7PlBm4Jp5wxtf4SKeXerzT9rzw6sPJB3a7ybZ8fNDSrRK4qJSc+Q7L4UzcKD19l5MRoBNCvvTq+6JWhUxjX/lUyOCoHVw+BWCFToAG1+1Gbt5DIpJV8f/Jrll5bzReAXdC7bOdePmVtUolcUE3Yj7iGLDl9n2dEb3ElIoURRO7rXLkGXAK/8VUEzPQ0O/Ao7x4KlDQSNhWo9cr1IWlpGGsN3DOdg5EF+a/Eb9Tzq5erxcotK9IpSAKSkZbD57C0WHrzGoatxWJoLgvyL0aNOCQJ989EUzTuXtQVOrh8A36bQfhI4lczVQyakJNBnUx+iEqKYHzyfMk5lcvV4uUElekUpYEKjH7Do0A2WH7vB/aQ0fF3s6V67BJ1qelE0PyxwnpEBR2drd9VKCS2+gFqDcrVI2q3EW/RY3wNzM3MWtVmEq51rrh0rN6hErygF1D9TNBcdusbx6/FYmZvRulIxutcuQV3fosbfy4+/Duve1+bfe9eBDr+Ca7lcO9y52HP029SP0oVLM6f1nHxVx14lekVRuHDrPosP32DF8QgeZPbyu9X2plMNL5yNeSxfSghZAptGQ0oiNB4F9d/V6uHngh3Xd/DezvdoUbIFPzb+Md9Mu1SJXlGUxx6lpLP+dBSLD1/n6LW7WJoLWlUsRrfa3tQv7WK8M3YSomHDSDj3l1YcrcNk8KiWK4ead3YePx79MV9Nu1SJXlGUbF2+/YA/D99g5YkI4h+m4uVky+sB3nQO8KJ4YSMdtji/FtZ/AIl3oP47Wg/fUr+xSikZc3AMyy4tY0z9MXT066jX988NKtErivJMSanpbDl3m8WHr7P/SixmApqUc6NrgDfNK7hhaW5kwxeP7sKWT+HEQnD208buS+p3WmRqRipvbXuLo7ePMrPlTAKKZZtDjYZK9Iqi6OxabCJLj95g+bEIbt9PxsXBik41vOhay5vSrnlbgOy5ruyEte9oF21rDdSWMLTWXx2g+yn36bWhF3FJcSxqs4gShUro7b31TSV6RVFeWFp6BrsvxbDkyA12XIgmLUMSUNKJrgHetK1SHHtrI1n3NiVRq3N/aBoU8tSKpJVpqbe3v37/Oj029MDJ2ok/2v5BIatCentvfVKJXlGUHIl+kMSq4zdZcvQGYTGJ2FmZ065KcboEeBNQ0sk4pmneOKwVSbtzEap0g9Zj9VYk7eitowzaOojaxWozpfkULMyM5I9cFirRK4qiF1JKjl+/y9IjEawLiSQxJZ1SLvZ0rulFpxpeFCtsY9gA05Jhz4+w7yewdYI2P0DFjnopo7Dy8kq+2P8FPcr34OM6H+c8Vj1TiV5RFL1LTE5jw+kolh2L4PDVOMwENCzjSpcAL1pUcDfs2re3Tmu9+6iTUL4dtJ0AjsVy/LY/HPmB+efm81ndz+harmvO49Qjfaww1RqYhLZK1Cwp5bgntvcERmU+TQCGSilPZW4LBx4A6UDa0wLJSiV6Rclfwu8ksvxYBCuORxB1L4lCNhZ0qOZBl5reVPEqbJihnfQ0ODAZdo0Fc2sI+lZbtDwHsaRnpPP2jrfZH7mf6S2nU6d43i+H+DQ5SvRCiH/WfW2Jttj3EaC7lPJcljb1gPNSyrtCiGDgSyllncxt4UCAlPKOrgGrRK8o+VN6hmT/lTssPxbBpjO3SE7LoIybA51qevFqdU/cCxlgaOdOqDYz59rf4NsE2k2EoqVe+u0SUhLotaEXd5Lu8GebP/Eu5K23UHMip4k+EC1xB2U+/xhASjn2Ke2dgDNSSs/M5+GoRK8oBc79pFTWh0Sx/FgEx67dxUxAgzKudKrhSZB/sbwd2snIgGNzYOuXINOh2WdQ500we7kYbty/QfcN3XGxcWFhm4U4WBl+2mlOE31noLWUcmDm895AHSnl8Ke0/xAon6X9VeAuIIHpUsoZT9lvMDAYoESJEjWvXbumy7kpipIPhMUksPL4TVYejyDyXhKO1ha0rVKcTjW98nbWzr0IrUja5S3gGQCvTAa3Ci/1VoeiDvHm1jdp4NmASU0nYf6SfzT0JaeJvgsQ9ESiry2lfDubtk2B34AGUsrYzNc8pJSRQgg3YCvwtpRyz7OOqXr0imKaMjIkB8NiWXH8JhvPRPEwJZ0SRe14tbonr9XwpKSzfe4HISWcXgYbR2kLlTf+COq/BxYvXr558YXFfHvoWwZUGsB7Nd/Te6gvIk+GboQQVYBVQLCU8tJT3utLIEFK+eOzjqkSvaKYvsTkNDaducWqEzf5+8odpISAkk68WsOTdpU9KGyXO9UpH0uIgU2j4MwKcPPXeveeNV74bb4+8DXLLi3j+4bf08a3TS4EqpucJnoLtIuxzYGbaBdje0gpz2ZpUwLYAfSRUu7P8ro9YCalfJD59VbgaynlpmcdUyV6RSlYou494q8Tkaw8HsHl6ASszM1oVt6NV2t40rScG1YWuVhr58IGWD8CEm5D4HBo+r8XKpKWmp7KwC0DORt7lnnB8/B39s+9WJ9BH9Mr2wAT0aZXzpFSfiuEGAIgpZwmhJgFdAL+GVhPk1IGCCF80Xr5ABbAIinlt887nkr0ilIwSSk5G3mflcdvsubUTe4kpFDEzpJ2VYrzanVPapTIpfH8R/Gw9XM4Pg+K+mpF0nwa6Lx77KNYuq/vTobMYHG7xbjYuug/xudQN0wpipLvpKVnsDf0DquO32TLuVskpWZQoqgdHat58Ep1z9wpsBa2W5uKeTccAvpDi6/ARrfaNhfiLtB7Q28qOFdgdqvZWObSwihPoxK9oij5WkJyGpvP3OKvkzf5O/QOGRIqexamY3VP2lctjpujHufnpzyEnd/Cwd/AsTi0+xnKBum066armxi5ZyRdy3bls8DP9BeTDlSiVxTFZETfT2LNqUj+OnmTMzfvYyagvp8Lr1TzJMjfHUcbPfWkI45qZRRizkPlrtB6HNg7P3e3n4/9zJwzc/g88HO6lO2in1h0oBK9oigmKTT6AatPakn/RtwjrC3MaFHBnQ7VPGhSzhVrixzObU9Lgb0TtIdNIQgeD5U6PbOMQnpGOsN2DONQ1CHmBM2hulv1nMWgI5XoFUUxaVpVzXhWn7zJupAo4hJTKGRjQZvKxelQzYM6pZwxz8lauLfPar37yONQro1WJK2Qx1Ob30u+R4/1PUhMTWRJuyW427u//LF1pBK9oigFRmp6Bn+H3mHNyUg2n71FYko6bo7WtKviwSvVPF6+yFpGujZuv+NbMLeEVmOgRt+n9u5D74bSY0MPyjiVYW7QXKzMX/yGrBehEr2iKAXSo5R0dlyIZvXJm+y6GENKegY+zna0r+pBh6oelHF/iWUHY6/A2nchfC/4NIQOv2hTMrOx9dpWRuwaQeeynfki8Iscns2zqUSvKEqBd+9hKpvP3mLNqUj2X9Fm7pQv5kj7qh60r+JBCWc73d8sIwNOzIctn0F6KjT7BOq+lW2RtEnHJzHr9KxcvzirEr2iKEoW0Q+S2BASxdqQKI5duwtAVe8itK9SnHZVPHRfKet+JKwbAZc2gmdN6DAZ3Cv+q0nWi7O/t/6dqq5V9X06gEr0iqIoTxVx9yHrQqJYFxLJmZv3EQJqlSxK+6rFCa5cHBcH62e/gZRavZyNH0HSfWj4gfbIUiTtXvI9Xl/3OqkZqSxptyRX7pxViV5RFEUHYTEJj5P+pdsJmAkILO1MuyoetPYvhpP9My6oJsbCptFweim4VoBXpoBXzcebL8ZdpNeGXvi7+DOz1UwszfR756xK9IqiKC/o4q0HrAuJZF1IFFfvJGJhJqjv50LbKsUJqljs6dU1L23Wat4/iNLG7Zt+Alba+P+6sHV8vPdjelXoxajao7Lf/yWpRK8oivKS/im0tjYkkvUhUUTcfYSluaBhGVfaVi5OS393Cj15N27Sfdj2BRydA04+WpG0Uo0AGHd4HH+c/4PxjcYTXCpYb3GqRK8oiqIHUkpCIu6xLiSSDadvcTP+EVbmZjQso/X0W1R8IumH74M1b0NcmDbnvtUYUq3sGLB5ABfiLrCozSL8nPz0EptK9IqiKHompeTkjXjWhUSx8XQUkfeSsDI3o1FZF9pUzpL0Ux7Cru/gwBRwcId2PxPjHUDXdV2xt7Tnz7Z/4mj1EvP5n6ASvaIoSi7KyJCcjIhn/RNJv2EZF4IrF6dlRXcKx53WyihEn4VKnTheqzcDdo+gkVcjJjadmOM6+yrRK4qi5JF/kv6GkCg2ntGGdyzNBQ38XGjr70Lb+0uw3T8BrB1ZUPM1xt/cwvs136d/pf45Oq4+VphqDUxCW2FqlpRy3BPbReb2NsBDoJ+U8rgu+2ZHJXpFUUyBlJJTEffYcDqKDae1C7kWZoLOJR7wYdJknONDGOnrz1aZyKygWdQqVuulj5XTNWPN0daMbQlEoK0Z211KeS5LmzbA22iJvg4wSUpZR5d9s6MSvaIopkZKyZmb91l/OoqNZ6K4EZtAf4tNDLVaTj+PotyzcWR5xzW4ORR7qfd/VqLXZcXd2kColDJMSpkCLAZeeaLNK8B8qTkIFBFCFNdxX0VRFJMnhKCyV2FGB5dn14dNWPdOY2wavsO7tpPoE1mIpNREhi1qzcPEu3o/toUObTyBG1meR6D12p/XxlPHfQEQQgwGBgOUKFFCh7AURVHyJyEEFT0KUdGjEASV4/KtNnTeMpprqWewsLHX+/F0SfTZXQp+crznaW102Vd7UcoZwAzQhm50iEtRFMUklClWiI/6/JZr769Loo8AvLM89wIidWxjpcO+iqIoSi7SZYz+CFBGCFFKCGEFdAPWPNFmDdBHaOoC96SUUTruqyiKouSi5/bopZRpQojhwGa0KZJzpJRnhRBDMrdPAzagzbgJRZte+caz9s2VM1EURVGypW6YUhRFMQE5nV6pKIqi5GMq0SuKopg4legVRVFMnEr0iqIoJs4oL8YKIWKAay+5uwtwR4/h5AfqnE1fQTtfUOf8okpKKV2z22CUiT4nhBBHn3bl2VSpczZ9Be18QZ2zPqmhG0VRFBOnEr2iKIqJM8VEP8PQARiAOmfTV9DOF9Q5643JjdEriqIo/2aKPXpFURQlC5XoFUVRTFy+TPRCiNZCiItCiFAhxOhstgshxC+Z20OEEDUMEac+6XDOPTPPNUQIsV8IUdUQcerT8845S7taQoh0IUTnvIwvN+hyzkKIJkKIk0KIs0KI3Xkdo77p8LNdWAixVghxKvOc3zBEnPoihJgjhIgWQpx5ynb95y8pZb56oJU7vgL4oi1scgqo+ESbNsBGtBWu6gKHDB13HpxzPcAp8+vggnDOWdrtQCuV3dnQcefB/3MR4BxQIvO5m6HjzoNz/h/wfebXrkAcYGXo2HNwzo2AGsCZp2zXe/7Kjz36nCxWnl8995yllPullP+sKnwQbTWv/EzXheXfBlYA0XkZXC7R5Zx7ACullNcBpJT5/bx1OWcJOAohBOCAlujT8jZM/ZFS7kE7h6fRe/7Kj4n+aQuRv2ib/ORFz2cAWo8gP3vuOQshPIFXgWl5GFdu0uX/uSzgJITYJYQ4JoTok2fR5Q5dznkyUAFtGdLTwLtSyoy8Cc8g9J6/dFkz1tjkZLHy/Ern8xFCNEVL9A1yNaLcp8s5TwRGSSnTtc5evqfLOVsANYHmgC1wQAhxUEp5KbeDyyW6nHMQcBJoBpQGtgoh9kop7+dybIai9/yVHxN9ThYrz690Oh8hRBVgFhAspYzNo9hyiy7nHAAszkzyLkAbIUSalPKvPIlQ/3T92b4jpUwEEoUQe4CqQH5N9Lqc8xvAOKkNYIcKIa4C5YHDeRNintN7/sqPQzc5Waw8v3ruOQshSgArgd75uHeX1XPPWUpZSkrpI6X0AZYDb+XjJA+6/WyvBhoKISyEEHZAHeB8HsepT7qc83W0TzAIIdyBckBYnkaZt/Sev/Jdj17mYLHy/ErHc/4ccAZ+y+zhpsl8XPlPx3M2Kbqcs5TyvBBiExACZACzpJTZTtPLD3T8fx4D/C6EOI02rDFKSplvyxcLIf4EmgAuQogI4AvAEnIvf6kSCIqiKCYuPw7dKIqiKC9AJXpFURQTpxK9oiiKiVOJXlEUxcSpRK8oimLiVKJXFEUxcSrRK4qimLj/A7JzuErTLXYWAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "p = torch.linspace(0.,1,100)\n",
    "\n",
    "pows = [0.5,1.,2.]\n",
    "for e in pows:\n",
    "    f = SchedPoly(2, 0, e)\n",
    "    plt.plot(p, [f(o) for o in p], label=f'power {e}')\n",
    "plt.legend();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18507006",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def combine_scheds(pcts, scheds):\n",
    "    \"Combine `scheds` according to `pcts` in one function\"\n",
    "    assert sum(pcts) == 1.\n",
    "    pcts = tensor([0] + L(pcts))\n",
    "    assert torch.all(pcts >= 0)\n",
    "    pcts = torch.cumsum(pcts, 0)\n",
    "    pct_lim = len(pcts) - 2\n",
    "    def _inner(pos):\n",
    "        idx = min((pos >= pcts).nonzero().max(), pct_lim)\n",
    "        actual_pos = (pos-pcts[idx]) / (pcts[idx+1]-pcts[idx])\n",
    "        return scheds[idx](actual_pos.item())\n",
    "    return _inner"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2b085e1",
   "metadata": {},
   "source": [
    "`pcts` must be a list of positive numbers that add up to 1 and is the same length as `scheds`. The generated function will use `scheds[0]` from 0 to `pcts[0]` then `scheds[1]` from `pcts[0]` to `pcts[0]+pcts[1]` and so forth."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ee24fb9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAtFUlEQVR4nO3deXxV1bn/8c+Tk5EACRDCkBAIhHlIgAACDqCo4IQoMmi1VW8pinPrcPVetVo72UpxqlJ/atEKKIqlFVErIsggSZhnwpAQpiRMIfP0/P5I7E0xmBNIss/wvF8vXuScvdbJswivL5u1915LVBVjjDG+K8DpAowxxjQuC3pjjPFxFvTGGOPjLOiNMcbHWdAbY4yPC3S6gNpERUVply5dnC7DGGO8RlpaWq6qtq3tmEcGfZcuXUhNTXW6DGOM8RoiknG2YzZ1Y4wxPs6C3hhjfJwFvTHG+DgLemOM8XEW9MYY4+PcCnoRGSsiO0UkXUQeO0ubUSKyQUS2isjX9elrjDGm8dR5e6WIuIBXgMuBLCBFRBap6rYabSKBV4GxqpopItHu9jXGGNO43LmPfiiQrqp7AURkHjAeqBnWNwMfqWomgKpm16Ov8RKqyuaDp9iXW0BeURl5xeVUVirNQwMJDwmkdbNgOkaGERMZRsuwQETE6ZKNMbgX9DHAgRqvs4BhZ7TpAQSJyDKgBTBLVee42RcAEZkGTAOIi4tzp3bTRDZnneKj9Vl8tuUIh04Vu9UnIiyIXu1b0LtDS/rFRDAsvjWdWjdr5EqNMbVxJ+hrOy07c7eSQGAwcBkQBqwWkTVu9q16U3U2MBsgOTnZdkPxANl5xfz20x18tP4gwYEBXNy9LQ9d0ZOkTpFEhAXRIjQQV4BQUFLO6eJyjheUcuhkEQdPFrE3t4Dth/N4P/UAb6/aD0BMZBgjurVhTJ92XNy9LWHBLmcHaIyfcCfos4BONV7HAodqaZOrqgVAgYgsBxLd7Gs8jKry/77Zx8wvdlFWocwY3Y3pl3SjRWhQre0jmwUT2SyYTq2bkdgp8j+OVVYqu7JP8+3e46zZe4zPth7hg7QsQoMCuKRHWyYMjOXSXtEEB9oNYMY0FneCPgXoLiLxwEFgClVz8jX9HXhZRAKBYKqmZ2YCO9zoazxIcVkFjyzYxKKNh7i0VzRPXtOHLlHh5/x5AQFCr/Yt6dW+JT8e0YWyikrW7jvO51uPsHjLET7bepQ24cFcPzCGWy/ofF7fyxhTuzqDXlXLReQe4DPABbypqltFZHr18ddUdbuILAE2AZXAG6q6BaC2vo00FnOecvNLmDYnlXWZJ3n4yp7cPapbg19QDXIFMDIhipEJUfzvNX1YvjuHD1KzmLN6P2+u3MeY3u2488J4hsW3tou5xjQQ8cTNwZOTk9VWr2xaR04Vc9Prq8jOK2Hm5CSu6t+hSb9/dl4xc1Zn8LdvMzhRWEZy51Y8dHkPhndrY4FvjBtEJE1Vk2s9ZkFvThaWMun11Rw6WcycO4cyKK6VY7UUlVbwQdoBXvkqnaN5JQyLb82j43o5WpMx3uCHgt6ugPm5otIK7vxrKvtzC5l962DHAzUs2MVtw7vw9cOjefraPuzNLeCGV1dx39z1HDxZ5GhtxngrC3o/VlGpzHhvHesyTzBrShIjEqKcLunfQoNc/GRkPMt+MYp7L03gs61HuPQPy5j1r92UlFc4XZ4xXsWC3o/N+nI3S3dk88z4foxr4jl5d4WHBPLzK3qy9BejGNOnHTP/tYurZq3g273HnC7NGK9hQe+nvtmdy0tLdzNxcCy3XtDZ6XLqFBMZxis3D+Kt24dQUl7J5NlreHzhZgpKyp0uzRiPZ0Hvh7Lzinlg/noS2jbnmfF9nS6nXkb3jOaLBy/hpxfFM3dtJuNmrSBl/3GnyzLGo1nQ+5mKSuW+eespKKng1VsG0SzYI/eH/0FhwS6euLoP86cNR1Emvb6a3y/ZQXlFpdOlGeORLOj9zFsr97Fm73GeGd+X7u1aOF3OeRka35ol91/M5OROvLpsD1Nmr+GQ3ZljzPdY0PuRA8cL+ePnu7i0VzQTB8c6XU6DCA8J5Lc3DmDWlCS2H87jqhdXsHTHUafLMsajWND7CVXl8YWbcQUIv7q+n889bTo+KYZ/3ncRMZFh3PnXVF78cjeVlZ73MKAxTrCg9xMfrjvIit25PDq2Jx0jw5wup1HER4Xz4V0jmJAUwwtf7GL6u2mcLi5zuixjHGdB7wdy80t49p/bGNKlFbcM8/xbKc9HaJCLP05K5Klr+/DljmxueHUVB44XOl2WMY6yoPcDf/x8FwUl5fzmhv4EBPjWlE1tRITbR8bzzp1DOZpXzIRXV7Iu84TTZRnjGAt6H7f9cB7zUzK5bXgXEqK9+y6b+hrRLYqFM0YSHhLIlNlr+Ocm2/PG+CcLeh+mqvzqk220DAvi/su6O12OI7q1bc7Cu0eSGBvBPe+t562V+5wuyZgmZ0Hvw77cns3K9GM8cFl3IprVvg2gP2gdHsw7dw7jyr7t+OU/tvH7JTvwxOW5jWksbgW9iIwVkZ0iki4ij9VyfJSInBKRDdW/nqxxbL+IbK5+3xaZbyKl5ZU8t3g73dqGc4sXrGXT2EKDXLx6y2CmDo3j1WV7eOzDzVTY7ZfGT9T5/LuIuIBXgMup2uw7RUQWqeq2M5quUNVrzvIxo1U19/xKNfUxLyWTfbkFvPWTIQS57D9uAK4A4dcT+tG2eTAvLk2noLScmZOT7M/H+Dx3FjoZCqSr6l4AEZkHjAfODHrjIYpKK3hpaTpD41szqmdbp8vxKCLCQ1f0JDwkkN98uoPiskpevnkgoUEup0szptG4cyoTAxyo8Tqr+r0zDReRjSLyqYjUXBJRgc9FJE1Epp3tm4jINBFJFZHUnJwct4o3tXt3TQY5p0v4+eU9fO4J2Ibys0u68ez4vvxr+1F+OieV4jLbzMT4LneCvrakOHNycx3QWVUTgZeAj2scG6mqg4BxwAwRubi2b6Kqs1U1WVWT27a1s9BzVVBSzp+/3sNF3aMY1rWN0+V4tFuHd+H3EwfwTXquhb3xae4EfRbQqcbrWOA/bkhW1TxVza/+ejEQJCJR1a8PVf+eDSykairINJK3V+3neEEpD13ew+lSvMKk5E78/kYLe+Pb3An6FKC7iMSLSDAwBVhUs4GItJfqOQIRGVr9ucdEJFxEWlS/Hw5cAWxpyAGY/3OqqIzXv97DmN7RDHR4k29vclNyJ353Q1XYT3snzfakNT6nzouxqlouIvcAnwEu4E1V3Soi06uPvwZMBO4SkXKgCJiiqioi7YCF1f8GBALvqeqSRhqL33t75X7yist5YIydzdfXpCGdUJRHP9zMPe+t59VbBtndOMZniCc+OJKcnKypqXbLfX0UlJQz8ndLSe7cijd+PMTpcrzW2yv38fQ/tjE+qSMvTErC5QdrAxnfICJpqppc2zHv20fO1Gru2kxOFpZx16gEp0vxaj8ZGU9BaQXPf7aTZsEufj2hv925ZLyeBb0PKCmv4I0V+7iga2sGd7a5+fM1Y3QC+SXl/HnZHiKbBfPo2F5Ol2TMebGg9wEL1x3kSF4xz980wOlSfMYjV/bkZGEZf162h6jmIdx5YbzTJRlzzizovVxFpfLa13voHxPBhQlRTpfjM0Sqtlw8UVDKs//cRlTzYMYn1facoDGez24r8HKLNx9m/7FCZozuZnPJDcwVIPxpShIXdG3Nz9/fyIrd9sS28U4W9F5MVfnLir10jQrnij7tnS7HJ4UGuZh9WzIJ0c256911bDuU53RJxtSbBb0XS804waasU9x+YbxfbBHolJahQbx1+xCahwRy+9trOXSyyOmSjKkXC3ov9saKvUQ2C+LGQTZ33Ng6RITx9h1DKCyp4Pa3UsgrLnO6JGPcZkHvpTKOFfD5tqPcMiyOZsF2Tb0p9GrfktduHcyenHxm/G0d5RWVTpdkjFss6L3UWyv3Exgg3Da8i9Ol+JWRCVE8N6EfK3bn8vQ/ttqWhMYr2KmgFzpVVMb7qQe4NrEj7VqGOl2O35k8JI69OQW8vnwvXaOac4fdY288nAW9F5qfkklhaYU9xOOgR8f2Yl9uAb/6ZBvxUeGM7hXtdEnGnJVN3XiZikplzuoMhsW3pm/HCKfL8VsB1ffY9+7Qkvvmric9O9/pkow5Kwt6L7N0RzZZJ4r4yYguTpfi95oFBzL7tmRCggL46ZxUThXanTjGM1nQe5k5q/fTISKUy/u0c7oUA8REhvHajwaTdaKQe+banTjGM7kV9CIyVkR2iki6iDxWy/FRInJKRDZU/3rS3b7GfenZ+azYncuPLuhMoG2K4TGSu7Tmuev7s2J3Lr/9dIfT5RjzPXVejBURF/AKcDlV+8emiMgiVd12RtMVqnrNOfY1bnhn9X6CXQFMHtKp7samSU0a0olth/N445t99I+NsAXQjEdx57RwKJCuqntVtRSYB4x38/PPp6+p4XRxGQvSsrgmsQNRzUOcLsfU4omrezM0vjWPfriJLQdPOV2OMf/mTtDHAAdqvM6qfu9Mw0Vko4h8KiJ969nX1OGjdQcpKK3gx/aAlMcKcgXwys2DaNUsmJ+9k8bxglKnSzIGcC/oa1st68zHAdcBnVU1EXgJ+LgefasaikwTkVQRSc3JseVga1JV3lmTQWJsBImdIp0ux/yAti1CeO1Hg8nJL+G+ueupqLQnZ43z3An6LKDmpHAscKhmA1XNU9X86q8XA0EiEuVO3xqfMVtVk1U1uW3btvUYgu9bu+846dn53HJBZ6dLMW5I7BTJM9f15Zv0XGZ+scvpcoxxK+hTgO4iEi8iwcAUYFHNBiLSXqp3vRCRodWfe8ydvqZu736bScvQQK4d0NHpUoybpgyNY3JyJ17+Kp1/bTvqdDnGz9UZ9KpaDtwDfAZsB95X1a0iMl1Eplc3mwhsEZGNwIvAFK1Sa9/GGIivys0vYcmWw9w4OJawYJfT5Zh6+OX4vvSLacmD728g41iB0+UYPyaeuPpecnKypqamOl2GR3h1WTq/X7KTfz10CQnRzZ0ux9TTgeOFXPPSN8REhvHR3SMIDbJ/rE3jEJE0VU2u7Zg9dePBKiuV977N5IKurS3kvVSn1s2YOTmRbYfz+OU/7D+zxhkW9B7s6905ZJ0o4kd2EdarXdqrHXeN6sbctQf4aF2W0+UYP2RB78H+tiaTqObBtvG3D/j55T0YFt+aJxZuYdfR006XY/yMBb2HOnKqmKU7jnJTcieCA+3H5O0CXQG8NHUg4SGB3P23dRSWljtdkvEjliAe6v3UA1QqTLF1bXxGdMtQZk1JYk9OPk/+3ebrTdOxoPdAFZXK/JQDXJgQRec24U6XYxrQyIQo7h2dwIK0LD5Ms/l60zQs6D3Q8t05HDxZxNShcU6XYhrB/WOq5uv/5+MtpGfbfL1pfBb0Hmjut1UXYW1zEd/kChBenDqQZsEuZvxtPcVlFU6XZHycBb2HOZpXzJc7spk42C7C+rJ2LUP546REdh49za8+se0ZTOOyJPEwH6QeoKJS7SKsHxjVM5ppF3fl3TWZLNly2OlyjA+zoPcglZXKvJQDjOjWhi5RdhHWH/ziip4kxkbwyIJNZJ0odLoc46Ms6D3Iyj25ZJ0oYopdhPUbwYEBvDh1IJUKD8zbYJuLm0ZhQe9B5q09QKtmQVzZ1y7C+pPObcJ5bkI/UjNO8NLSdKfLMT7Igt5DHMsv4fNtR7hhUCwhgbbCob8ZnxTDDQNjeGnpblL2H3e6HONjLOg9xEfrDlJWYRdh/dkz1/cjtlUzHpi3gVOFZU6XY3yIBb0HUFXmpmQyuHMrurdr4XQ5xiHNQwJ5cepAjuYV8/jCzXjiXhHGO7kV9CIyVkR2iki6iDz2A+2GiEiFiEys8d5+EdksIhtExHYTqUXK/hPszSlgsp3N+72kTpE8eHkPPtl8mA/XHXS6HOMj6gx6EXEBrwDjgD7AVBHpc5Z2v6Nq28AzjVbVpLPtfuLv5qVk0iIkkGsGdHC6FOMBpl/SjWHxrXnq71tsC0LTINw5ox8KpKvqXlUtBeYB42tpdy/wIZDdgPX5vFNFZSzefJhrkzrSLDjQ6XKMB3AFCDMnJ+EKEO6ft4Eyu+XSnCd3gj4GOFDjdVb1e/8mIjHABOC1Wvor8LmIpInItLN9ExGZJiKpIpKak5PjRlm+YdGGgxSXVTJ1iN07b/5Px8gwfn1DfzYcOGm3XJrz5k7QSy3vnXmV6E/Ao6pa2+pMI1V1EFVTPzNE5OLavomqzlbVZFVNbtu2rRtleT9VZe7aA/Tt2JL+sRFOl2M8zDUDOnLjoFheXrqbtAy75dKcO3eCPguoeZUwFjh0RptkYJ6I7AcmAq+KyPUAqnqo+vdsYCFVU0EG2HIwj22H8+yWSnNWT1/Xh5hWYTwwfwP5JbYrlTk37gR9CtBdROJFJBiYAiyq2UBV41W1i6p2ARYAd6vqxyISLiItAEQkHLgC2NKgI/Bic1MyCQ0K4LqkmLobG7/UIjSImZOSOHiiiF8usl2pzLmpM+hVtRy4h6q7abYD76vqVhGZLiLT6+jeDvhGRDYCa4FPVHXJ+RbtCwpLy1m04RBX9e9ARFiQ0+UYD5bcpTV3j0rgg7QsPt1sq1ya+nPrNg9VXQwsPuO92i68oqo/qfH1XiDxPOrzWf/cdJj8knLbRcq45f4x3Vm+O4f/XriZQZ1b0a5lqNMlGS9iT8Y6ZH7KAbq1DSe5cyunSzFeIMgVwMzJSRSXVfDwgk321KypFwt6B+w6epq0jBNMGRKHSG03NRnzfd3aNueJq3qzfFcO767JcLoc40Us6B0wd20mQS7hhkF2EdbUz48u6MwlPdry3OLt7MnJd7oc4yUs6JtYcVkFH607yJV929OmeYjT5RgvIyI8P3EAoUEuHpxvT80a91jQN7ElW45wqqjMLsKacxbdMpTfTOjPpqxT9tSscYsFfRObuzaTuNbNGN61jdOlGC82rn8HbhgYwytfpbPhwEmnyzEezoK+Ce3JyefbfceZMrQTAQF2Edacn6fH96VdixAemr+BotLaVh8xpooFfROan3KAwABh4uBYp0sxPqBlaBB/mJTI3twCfvPpdqfLMR7Mgr6JlJRXsCAtizG92xHdwh52MQ1jRLco7rwwnjmrM/h6l/+s+mrqx4K+iXy29SjHC0qZMtQWMDMN6+Ere9I9ujmPLNhoe82aWlnQN5G/rcmgU+swLu7uH0swm6YTGuTihUlJHMsv5X//bmsGmu+zoG8C6dlVF2GnDo2zi7CmUfSPjeC+y7qzaOMh/rHxzFXEjb+zoG8C731b9STsTYNt2sY0nrtHdSOxUyT/8/EWjuYVO12O8SAW9I2suKyCBWkHuLJve9q2sCdhTeMJdAXwwqRESsorePRDW/jM/B8L+kb2yabD5BWXc/MwexLWNL5ubZvz2NheLNuZw9y1B+ruYPyCBX0j+9u3GXSNCrcnYU2TuW14F0YmtOFXn2wj81ih0+UYD+BW0IvIWBHZKSLpIvLYD7QbIiIVIjKxvn190bZDeazLPMnNw2w5YtN0AgKE5ycm4hLh5x9soKLSpnD8XZ1BLyIu4BVgHNAHmCoifc7S7ndUbTlYr76+6p01+wkNCrCLsKbJdYwM4+nr+pKy/wR/WbHX6XKMw9w5ox8KpKvqXlUtBeYB42tpdy/wIZB9Dn19zqmiMj5ef4jxiTFENLM9YU3Tu2FQDFf2bccLn+9ix5E8p8sxDnIn6GOAmld1sqrf+zcRiQEmAGfuI1tn3xqfMU1EUkUkNSfH+x/lXpCWRVFZBbcO7+x0KcZPiQi/ntCflmGBPDR/I6Xltna9v3In6GubXD5z0u9PwKOqeuYSeu70rXpTdbaqJqtqctu23v30aGWl8u6aDAbFRdIvJsLpcowfa9M8hF9P6M+2w3m8+OVup8sxDgl0o00WUHOSORY489G7ZGBe9QXHKOAqESl3s6/P+SY9l325Bdw/OcnpUozhir7tmTg4lleXpXNp72gGxdmG9P7GnTP6FKC7iMSLSDAwBVhUs4GqxqtqF1XtAiwA7lbVj93p64vmrM6gTXgw4/q3d7oUYwB48to+dIgI4+fvb7S16/1QnUGvquXAPVTdTbMdeF9Vt4rIdBGZfi59z79sz3XgeCFLdxxlytBOhAS6nC7HGKBq7frnbxrAvtwCfmtr1/sdd6ZuUNXFwOIz3jvzwut37/+krr6+bM7q/YgItwyzi7DGs4zoFsUdI+N5c+U+xvRpx0W2kqrfsCdjG1B+STnzUg5wVf8OdIwMc7ocY77nkbE96dY2nIc/2GRr1/sRC/oGtCD1AKeLy7ljZBenSzGmVqFBLmZOTiInv4SnFtna9f7Cgr6BVFYqb63az8C4SAbaXQ3Ggw2IjeTeSxP4eMMhPtl02OlyTBOwoG8gS3dkk3GskDtGxjtdijF1mjE6gcTYCJ74eDPZtna9z7OgbyBvrtxHx4hQxvWzWyqN5wtyBfDHSUkUldra9f7Agr4BbDuUx6o9x7htRBcCXfZHarxDQnRz/ntcL76ytet9nqVSA3h9+R7Cg11MHWKbixjvUnPt+oxjBU6XYxqJBf15OnC8kH9uOszUoXG2SqXxOgEBwh9uSiQwQHhw/gbKK2zhM19kQX+e/t83+wgQuPMiuwhrvFOHiDCevb4f6zJP8vpyW7veF1nQn4fjBaXMS8lkfFIMHSLsASnjva5L7MjVAzow84tdbDl4yulyTAOzoD8Pf121n+KySqZf0tXpUow5LyLCc9f3o03zYB6Yv4HiMlv4zJdY0J+jwtJy/rp6P2N6tyMhuoXT5Rhz3iKbBfP8xETSs/P53ZIdTpdjGpAF/Tl679tMThaWcdcoO5s3vuPiHm35yYguvLVyPyt2e/9Ob6aKBf05KCqt4LWv93BhQhSDO7d2uhxjGtRj43qREN2cX3ywkZOFpU6XYxqABf05eHdNBrn5pTwwprvTpRjT4EKDXPxpchLH8kt5YuEWe2rWB1jQ11NhaTmvL9/DRd2jSO5iZ/PGN/WLieChK3rwyebDfLTuoNPlmPPkVtCLyFgR2Ski6SLyWC3Hx4vIJhHZICKpInJhjWP7RWTzd8casngnfHc2f/9ldjZvfNvPLu7G0C6teWrRVg4cL3S6HHMe6gx6EXEBrwDjgD7AVBHpc0azL4FEVU0C7gDeOOP4aFVNUtXk8y/ZOYWl5bz+9V47mzd+wRUgvDA5EQF7atbLuXNGPxRIV9W9qloKzAPG12ygqvn6fxN54YBPTuq9tXI/xwpsbt74j9hWzXj2+n6kZpzgz8v2OF2OOUfuBH0MUHNpu6zq9/6DiEwQkR3AJ1Sd1X9Hgc9FJE1Epp3tm4jItOppn9ScHM+7retYfgl/XraHMb3b2Z02xq+MT+rItYkd+dOXu1mfecLpcsw5cCfopZb3vnfGrqoLVbUXcD3wbI1DI1V1EFVTPzNE5OLavomqzlbVZFVNbtvW8zYtfvHL3RSVVfDYuF5Ol2JMkxIRfnV9P9q3DOX+eRvILyl3uiRTT+4EfRbQqcbrWODQ2Rqr6nKgm4hEVb8+VP17NrCQqqkgr7I3J5+/fZvJlCGdSIhu7nQ5xjS5iLAgZk5OIutEIU/9favT5Zh6cifoU4DuIhIvIsHAFGBRzQYikiAiUv31ICAYOCYi4SLSovr9cOAKwOt2JP79kp0EBwZwv83NGz82NL41M0Yn8OG6LP6x8aznesYDBdbVQFXLReQe4DPABbypqltFZHr18deAG4HbRKQMKAImq6qKSDtgYfW/AYHAe6q6pJHG0ihS9x9nydYjPDimB9EtQp0uxxhH3XdZd75Jz+XxhZsZGBdJbKtmTpdk3CCe+NRbcnKypqY6f8t9eUUl17z0DScLy1j6i0toFlznv4vG+LzMY4Vc9eIKerZvwfxpF9j2mR5CRNLOdgu7/YR+wNur9rPjyGmevq6Phbwx1eLaNOO5Cf1IyzjBi0vTnS7HuMGC/iwOnSzihS92cWmvaK7s297pcozxKOOTYrhhUAwvL93Nt3uPOV2OqYMF/Vk8vWgrlar88rq+VF9jMMbU8Mz4fsS1bsYD8zfYKpcezoK+Fp9vPcLn245y/2U96NTaLjYZU5vmIYG8NHUQufklPLxgk61y6cEs6M+Qc7qExxduplf7Ftx5oW34bcwP6R8bwWPjevPFtqP8ddV+p8sxZ2FBX4Oq8vCCjZwuLufFqQMJDrQ/HmPqcsfILlzWK5pfL95hG4t7KEuyGv66aj/LdubwxNW96dHO9oE1xh0iwvM3JdI6PJh75663JRI8kAV9tZ1HTvPrT3dwaa9obr2gs9PlGONVWocHM2tKEhnHCnj8o802X+9hLOiB4wWlTHsnlZahQfx+4gC7y8aYczCsaxseurwHizYeYl7Kgbo7mCbj90FfXFbBtDmpHDlVzOzbBhPVPMTpkozxWnePSuCi7lE8tWgr2w7lOV2OqebXQa+qPLJgE6kZJ3hhUhKD4lo5XZIxXi0gQJg5OYnIsCDueW8dp4vLnC7J4MdBr6r85tMdLNp4iEfG9uTqAR2cLskYnxDVPISXpg4k43ghj9l8vUfwy6CvqFQeX7iF2cv3cusFnbnrkm5Ol2SMTxnWtQ0PX9mTTzYd5m27v95xfhf0JeUV3Dt3HXPXZjJjdDeeGW9LHBjTGKZd1JUxvdvx3CfbScuwLQid5FdBvz+3gJv/8i2LNx/hf67uzcNX9rKQN6aRBAQIf7wpkQ6Rodzz3jqO5Zc4XZLfcivoRWSsiOwUkXQReayW4+NFZJOIbKje4PtCd/s2hcpK5c1v9jF21nJ2HT3Ni1MH8l8XdXWiFGP8SkSzIP58y2COFZRy37z1lFdUOl2SX6oz6EXEBbxC1ebefYCpItLnjGZfAomqmgTcAbxRj76NprC0nPkpmVz78jc8889tDO/ahi8evITrEjs2VQnG+L1+MRH86vp+rEw/xh8+3+V0OX7Jnd00hgLpqroXQETmAeOBbd81UNX8Gu3DAXW3b0OprFTWHzhB5vFCMo8VsS83ny+3Z3O6pJwe7ZrzwqREJgyMsakaYxwwKbkT6zNP8trXe0jqFMHYfnaXW1NyJ+hjgJqPuWUBw85sJCITgN8A0cDV9elb3X8aMA0gLi7OjbLO7A83/+VbSsorEYH2LUMZ06cdNw+LI7lzKwt4Yxz29HV92HY4j5+/v5GE6OYkRNt6Uk3FnaCvLSG/d2Osqi6kaiPwi4FngTHu9q3uPxuYDVV7xrpR138WKcLbtw+lbYsQYluFERrkqu9HGGMaUUigiz/fMojrXv6GaXPSWDhjJBFhQU6X5RfcuRibBXSq8ToWOHS2xqq6HOgmIlH17Xu+hndrQ0J0cwt5YzxUx8gwXr1lMJnHC3lg3noqKu1hqqbgTtCnAN1FJF5EgoEpwKKaDUQkQarnRkRkEBAMHHOnrzHGvwyNb81T1/Xlq505zPzCLs42hTqnblS1XETuAT4DXMCbqrpVRKZXH38NuBG4TUTKgCJgslY991xr30YaizHGS/xoWBxbD57i5a/S6dWhBdcMsDvhGpN44joUycnJmpqa6nQZxphGVFJewc1/+Zath07xwc9G0D82wumSvJqIpKlqcm3H/OrJWGOM5wgJdPHajwbTJjyEn85JJTuv2OmSfJYFvTHGMW1bhPCX25LJKy7jp++kUVxW4XRJPsmC3hjjqD4dWzJzchIbD5zkFx9spNLuxGlwFvTGGMdd2bc9j43rxT83HeYFuxOnwbnzwJQxxjS6n13clf25Bbz8VTpxbZoxKblT3Z2MWyzojTEeQUR49vp+ZJ0o4vGPNhMbGcaIhCiny/IJNnVjjPEYQa4AXv3RILq2Dedn76Sx/bBtMN4QLOiNMR6lZWgQb98+lPCQQG5/K4VDJ4ucLsnrWdAbYzxOx8gw3r5jCAUl5fzkrbWcKixzuiSvZkFvjPFIvdq35PXbBrMvt4D/mpNCUandY3+uLOiNMR5rRLco/jR5IKkZJ5jx3jrKbCvCc2JBb4zxaFcP6MCvru/H0h3ZPLJgkz1QdQ7s9kpjjMe7ZVhnThaW8fxnO2kZGsjT1/W1XePqwYLeGOMV7h7VjVNFZcxevpfQIBePjetlYe8mC3pjjFcQEf57XC+KSit4vTrsH7y8h9NleQULemOM1xARfnldX4rLKpj15W6CAwOYMTrB6bI8nltBLyJjgVlU7RL1hqr+9ozjtwCPVr/MB+5S1Y3Vx/YDp4EKoPxsC+MbY4w7AgKE3944gNKKSp7/bCeAhX0d6gx6EXEBrwCXU7XZd4qILFLVbTWa7QMuUdUTIjIOmA0Mq3F8tKrmNmDdxhg/5goQXpiURIAIz3+2k8pK5d7Lujtdlsdy54x+KJCuqnsBRGQeMB74d9Cr6qoa7dcAsQ1ZpDHGnMkVIPzhpkQE+OMXuyivVB4Y090u0NbCnaCPAQ7UeJ3Ff56tn+lO4NMarxX4XEQUeF1VZ9fWSUSmAdMA4uLi3CjLGOPvXAHC8zcl4goQZn25m4KScp64ureF/RncCfra/sRqfWJBREZTFfQX1nh7pKoeEpFo4AsR2aGqy7/3gVX/AMyGqs3B3ajLGGNwBQi/u3EA4SGBvPHNPvJLynluQn9cARb233En6LOAmjsAxAKHzmwkIgOAN4Bxqnrsu/dV9VD179kispCqqaDvBb0xxpyrgADhqWv70CI0kJeWpnO6uJwXJicSEuhyujSP4M4SCClAdxGJF5FgYAqwqGYDEYkDPgJuVdVdNd4PF5EW330NXAFsaajijTHmOyLCz6/oyRNX9eaTzYf58ZtrOVVkq16CG0GvquXAPcBnwHbgfVXdKiLTRWR6dbMngTbAqyKyQURSq99vB3wjIhuBtcAnqrqkwUdhjDHVfnpxV2ZNSSIt4wSTXlvN4VO2nr2oet50eHJysqamptbd0BhjzmJlei4/eyeN5iGBvPHjZPrFRDhdUqMSkbSzPadkq1caY3zSyIQo3v/ZcAIEbnptNUu2HHa6JMdY0BtjfFafji35+J6R9OrQgunvruPlpbvxxFmMxmZBb4zxadEtQpn70wu4Pqkjf/h8F9PfTeN0sX9dpLWgN8b4vNAgFzMnJ/G/1/ThX9uzGf/ySnYdPe10WU3Ggt4Y4xdEhDsvjGfuTy8gr7ic8S+v5P3UA34xlWNBb4zxK0PjW7P4vgsZGBfJIws2cd+8DeT5+FSOBb0xxu9EtwzlnTuH8fCVPVm8+TBXzVrB6j3H6u7opSzojTF+yRUgzBidwAfThxMYIEz9yxqe/PsWCkrKnS6twVnQG2P82qC4Vnx6/8XcMTKed9ZkMHbWcr7ame10WQ3Kgt4Y4/fCgl08eW0f5k8bTpArgNvfSmH6O2kcOukbyydY0BtjTLWh8a359P6LePjKnizblc1lf/yamV/s8vrpHAt6Y4ypISTQxYzRCXzx4CVc2iuaWV/u5pLnl/HumgzKKiqdLu+c2KJmxhjzA9ZlnuA3i7eTsv8EMZFh3DWqGzclx3rcWvc/tKiZBb0xxtRBVVm2M4cXl+5mfeZJ2rUM4bbhXZg6NI7W4cFOlwdY0BtjTINQVVbtOcYrX6Wzas8xggMDuHZAR6YM7URy51aO7lX7Q0HvzlaCxhhjqFpGYWRCFCMToth99DRzVmfw4bosPlyXRWyrMCYMjGFcvw707tDCozYod+uMXkTGArMAF/CGqv72jOO3AI9Wv8wH7lLVje70rY2d0RtjvEV+STmfbz3CwvUHWZmeS6VCh4hQRveK5sKEKAZ3bkW7lqGNXsd5Td2IiAvYBVxO1UbhKcBUVd1Wo80IYLuqnhCRccDTqjrMnb61saA3xnij7NPFLNuRw9Id2azYnUNBaQUAsa3CGBAbQUJ0C3q0a07n1uG0axlCm+YhuAIa5sz/fKduhgLpqrq3+sPmAeOBf4e1qq6q0X4NEOtuX2OM8RXRLUKZNKQTk4Z0orS8kq2HTpGWcYK0jBNsO5THp1uOUPPcOkCgZVgQoYEuQoICaNcilPenD2/wutwJ+hjgQI3XWcCwH2h/J/BpffuKyDRgGkBcXJwbZRljjOcKDgxgYFwrBsa14r8uqnqvuKyCPTn5ZJ0oIvt0CTl5xZwsKqOkrJLi8grCghrnlk13gr62/1fUOt8jIqOpCvoL69tXVWcDs6Fq6saNuowxxquEBrno2zGCvh2bdqNyd4I+C+hU43UscOjMRiIyAHgDGKeqx+rT1xhjTONxZwmEFKC7iMSLSDAwBVhUs4GIxAEfAbeq6q769DXGGNO46jyjV9VyEbkH+IyqWyTfVNWtIjK9+vhrwJNAG+DV6ntHy1U1+Wx9G2ksxhhjamFPxhpjjA/4odsrbfVKY4zxcRb0xhjj4yzojTHGx1nQG2OMj/PIi7EikgNknGP3KCC3AcvxBjZm3+dv4wUbc311VtW2tR3wyKA/HyKSerYrz77Kxuz7/G28YGNuSDZ1Y4wxPs6C3hhjfJwvBv1spwtwgI3Z9/nbeMHG3GB8bo7eGGPMf/LFM3pjjDE1WNAbY4yP88qgF5GxIrJTRNJF5LFajouIvFh9fJOIDHKizobkxphvqR7rJhFZJSKJTtTZkOoac412Q0SkQkQmNmV9jcGdMYvIKBHZICJbReTrpq6xobnxdztCRP4hIhurx3y7E3U2FBF5U0SyRWTLWY43fH6pqlf9omq54z1AVyAY2Aj0OaPNVVRtZyjABcC3TtfdBGMeAbSq/nqcP4y5RrulwGJgotN1N8HPOZKqPZfjql9HO113E4z5ceB31V+3BY4DwU7Xfh5jvhgYBGw5y/EGzy9vPKP/94bjqloKfLfheE3jgTlaZQ0QKSIdmrrQBlTnmFV1laqeqH5Zc4N2b+XOzxngXuBDILspi2sk7oz5ZuAjVc0EUFVvH7c7Y1aghVRtdtGcqqAvb9oyG46qLqdqDGfT4PnljUFf24bjMefQxpvUdzw1N2j3VnWOWURigAnAa01YV2Ny5+fcA2glIstEJE1Ebmuy6hqHO2N+GehN1Takm4H7VbWyacpzRIPnlzt7xnoadzYcd3tTci9xPhu0eyt3xvwn4FFVraje2czbuTPmQGAwcBkQBqwWkTX6n1t4ehN3xnwlsAG4FOgGfCEiK1Q1r5Frc0qD55c3Br07G4772qbk57NBu7dyZ8zJwLzqkI8CrhKRclX9uEkqbHju/t3OVdUCoEBElgOJgLcGvTtjvh34rVZNYKeLyD6gF7C2aUpscg2eX944dePOhuOLgNuqr15fAJxS1cNNXWgDOp8N2r1VnWNW1XhV7aKqXYAFwN1eHPLg3t/tvwMXiUigiDQDhgHbm7jOhuTOmDOp+h8MItIO6AnsbdIqm1aD55fXndGre5uVL6bqynU6UEjVGYHXcnPMtW7Q7lTN58vNMfsUd8asqttFZAmwCagE3lDVWm/T8wZu/pyfBd4Wkc1UTWs8qqpeu3yxiMwFRgFRIpIFPAUEQePlly2BYIwxPs4bp26MMcbUgwW9Mcb4OAt6Y4zxcRb0xhjj4yzojTHGx1nQG2OMj7OgN8YYH/f/AVsRxqSggcbVAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "p = torch.linspace(0.,1,100)\n",
    "f = combine_scheds([0.3,0.7], [SchedCos(0.3,0.6), SchedCos(0.6,0.2)])\n",
    "plt.plot(p, [f(o) for o in p]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94bbb63e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAm4ElEQVR4nO3dd1xVd57/8dfnXpqAggo2ioBdY8cSGxrTNMX0xLRJVdMmszuzk+zOTjK78/vNTDKT2fQRUyeZSZxUoxkTY0wCdsXYC0oTEFDAAoL07/4BZgmiXPRezi2f5+Ph48G99wCfE+Sdr+ee8z5ijEEppZTns1k9gFJKKefQQFdKKS+hga6UUl5CA10ppbyEBrpSSnkJP6u+cUREhImLi7Pq2yullEfasmVLiTEmsrXXLAv0uLg40tLSrPr2SinlkUTk4Nle00MuSinlJTTQlVLKS2igK6WUl9BAV0opL6GBrpRSXqLNQBeRN0XkiIjsOsvrIiIvikiGiOwQkTHOH1MppVRbHFmhvw1ceY7XZwEDmv7MA/5y4WMppZRqrzbPQzfGpIpI3Dk2mQO8Yxp7eDeISLiI9DbGFDprSGWdhgbDP9LyKDx+yupROoYIfjbBzy50DvQjumswMd06Ed01mCB/u9XTKXVOzriwKArIa/Y4v+m5MwJdRObRuIonNjbWCd9audq7Gw7y9NLdAIhYPEwHONvtAQLsNkbFhjOpX3emDohkTGw44gv/QZRHcUagt/a3utVfC2PMImARQGJiot5Zw83lHa3kmS/3MW1gJH+9d5xPBJgxhvoGQ12DoexULXnHKsk7eoo9hWWszyzlhVUHeP7rAyREhHBzYgw3jomiR5cgq8dWCnBOoOcDMc0eRwMFTvi6ykLGGJ74eAc2Ef5ww3CfCHMAkcbDLX52CPK306NLEGP7wnWjowA4UVnLyr2H+WBzHs98uY/nvkrn1nExPHpJf3qHdbJ4euXrnBHoS4FHRWQxMAE4ocfPPd97m3JZl1nK764fTp9wDarTwoL9uWlsNDeNjSar+CRvrMnmg7Q8PtySzx0TYvnZzIGEBftbPabyUdLWPUVF5H1gOhABHAaeBvwBjDELpXHp9jKNZ8JUAvcaY9ps3UpMTDRazuWejpRXccmfUhgZE8bf7p/gM6vz85V3tJKXv8ngwy15dA8N5L+uHcasi3rpfzflEiKyxRiT2OprVt0kWgPdfT3z5T6SUzJZ9fPpxEeEWD2Ox9h16ARPfrKDXYfKuHRIT35/w3AiOwdaPZbyMucKdL1SVP1IeVUtf9twkFnDe2uYt9NFUWEseXgyv5o9hNUHirnqxdVszCq1eizlQzTQ1Y+8vymX8qo65k9LsHoUj+Rnt/HgtASWPDKZ0EA/5r62gVe+zaChQU/qUq6nga5+UFPXwBtrspnUrzsjosOtHsejDendhaWPTWH28N78cUU6P128laraeqvHUl5OA139YMm2Qxwuq2Z+Uj+rR/EKoYF+vDR3NE/OGsznOwq5+81NHK+ssXos5cU00BXQeIn/otQshvTuwrQBEVaP4zVEhAVJ/XjhtlFsyz3OTQvXc8hXahRUh9NAVwB8s+8IGUdOMn9agp5u5wJzRkXxzv3jOVxWxa3J68k7Wmn1SMoLaaArAJJTM4kK78RVI3pbPYrXmpjQnb8/MIGyU7XctmiDhrpyOg10xZaDR9mcc4wHpsbjb9e/Eq40Ijqc9x6cyMnqOg115XT626tYmJJFeLA/t46LaXtjdcEuigrjvQcnUFFTx51vbKS4vNrqkZSX0ED3cRlHTrJyz2HuvjiO4ABnVPsoRwzrE8Zb94zjSFk19769iZPVdVaPpLyABrqPey01i0A/Gz+5uK/Vo/ic0bFdefXOMewtLGf+u2lU1+l56urCaKD7sMNlVXy69RC3JMbQPVQ7R6wwY1APnr1xBGszSnny451Y1a2kvIP+G9uHvbU2h7qGBh6cqpf5W+nGsdEUHD/Fcyv3M7BnZx6arhd2qfOjge6jyqpq+XtTCVds92Crx/F5j17Sn/1HTvLsin0M6BHKpUN7Wj2S8kB6yMVHvb8xl/LqOhZM09WgOxARnr1xBBf1CePxxVtJLyq3eiTlgTTQfVB1XT1vrs1mcv/uDI8Os3oc1aRTgJ3X7k4kJNCP+e+mUVZVa/VIysNooPugz7YWcLismgVawuV2eoUF8codY8g7doonPtqhb5KqdtFA9zENDYbk1EyG9u7ClP5awuWOxsV145dXDOKLXUW8tTbH6nGUB9FA9zGr9h0hs7iC+UlawuXO5k1L4NIhPfnd8r18n3vM6nGUh9BA9zHJKZlEd+3EVcO1hMudiQjP3TyS3uFBPPbeVj2erhyige5D0nKOknbwGA9OTcBPS7jcXliwPy/cNpqisiqeWrLL6nGUB9Dfah+yMCWLrsH+3JwYbfUoykFjYrvy00sGsGRbAUu2HrJ6HOXmNNB9RMaRcr7eqyVcnuiRGf0Y27crv16yS+t21TlpoPuIRalZBPnb+MmkOKtHUe3kZ7fx/K2jMMDPP9hOQ4Oeyqhap4HuA4pONJZw3ZoYQ7eQAKvHUechplswT10zlE05R3l7XY7V4yg3pYHuA95am019g+EBLeHyaDePjWbGoEieXbGP7JIKq8dRbkgD3cuVVdXy9425XDWiDzHdtITLk4kIv79hBP52G7/8SA+9qDNpoHu59zbmcrK6jvnTdHXuDXqFBfH0NcPYnHNMD72oM2ige7HqunreXJPN1AERXBSlJVze4sYxUVwyuAd/XJGuZ72oH9FA92JLth7iSHk187Ui16uICP/vuouwCfznkl1a4KV+oIHupRpLuLIY1qcLk/t3t3oc5WR9wjvxiysGkbK/mGU7Cq0eR7kJhwJdRK4UkXQRyRCRJ1t5PUxElonIdhHZLSL3On9U1R5f7z1MVnEFC5L6aQmXl7r74jhGRofx38t2c6JSu16UA4EuInbgFWAWMBSYKyJDW2z2CLDHGDMSmA48JyJ6wrOFklOziOnWiVkX9bJ6FOUidpvwuxuGc6yylt9/sdfqcZQbcGSFPh7IMMZkGWNqgMXAnBbbGKCzNC4FQ4GjQJ1TJ1UO25xzlC1awuUThvUJ44Ep8SzenMeWg1qz6+sc+W2PAvKaPc5veq65l4EhQAGwE3jcGNPQ8guJyDwRSRORtOLi4vMcWbUlOSWTbiEB3Dw2xupRVAf46cwB9OoSxK+X7KKu/oxfO+VDHAn01g7Atnxb/QpgG9AHGAW8LCJdzvgkYxYZYxKNMYmRkZHtHFU54sDhcr7ee4SfXBxHpwC71eOoDhAS6Mevrx7KnsIy/r4x1+pxlIUcCfR8oPlSL5rGlXhz9wKfmEYZQDYw2DkjqvZYlJpFJ387d1/c1+pRVAeaPbwXU/pH8Kev0ikur7Z6HGURRwJ9MzBAROKb3ui8DVjaYptcYCaAiPQEBgFZzhxUta3wxCmWbDvEreNi6KolXD5FRPjNtcOoqq3XN0h9WJuBboypAx4FVgB7gQ+MMbtFZIGILGja7LfAJBHZCawCnjDGlLhqaNW6t9bm0GDg/inxVo+iLNC/RygPTE3gk+8P6X1IfZRDdzowxiwHlrd4bmGzjwuAy507mmqPE6dqeW9jLlcN760lXD7s0Rn9+XhLPv+1bA+fPjQJm02vQfAlek6bl/ihhCtJS7h8WUigH09cOZjtecf5VG9Z53M00L1AVW09b65tLOEa1kdLuHzd9aOjGBkTzjNf7qOiWi8H8SUa6F5gydZDFJdXsyBJS7gU2GzC09cM5Uh5Na9+l2H1OKoDaaB7uIYGw6LULIZHhTGpn5ZwqUZjYrtyw+goXkvN1opdH6KB7uG+2nOYrJIK5iclaAmX+pFfXjkYmw2eXZFu9Siqg2igezBjDMmpmcR2C+bKYVrCpX6sV1gQD05NYNn2ArbqaYw+QQPdg23OOcbW3OM8OE1LuFTr5if1IyI0kN8t36s3wvABmgIeLDklk+4hAdw8NtrqUZSbCg30418vG8jmnGOs2F1k9TjKxTTQPVR6UTmr9h3hJ5PiCPLXEi51drckRjOgRyh/+GIfNXXaxujNNNA91OkSrrsmagmXOjc/u41/nz2YnNJKFm/WNkZvpoHugQpPnOKzbYe4bbyWcCnHzBjUgwnx3Xhx1QFO6sVGXksD3QO9uSYbg5ZwKceJCP8+ewglJ2t4fbUWoXorDXQPc6KysYTr6hG9ie6qJVzKcaNiwpk9vBevpWZpZ7qX0kD3MH/beJCKmnrmT9PL/FX7/eLyQVTVNfDyNwesHkW5gAa6B6mqreettTlMGxjJ0D5n3OFPqTYlRIZy27gY/r4xl4OlFVaPo5xMA92DfLr1ECUnq1mgFbnqAjw+cwB+duH5r3WV7m000D1EfVMJ14joMC5O0BIudf56dAninknxLNl2iPSicqvHUU6kge4hVu4pIrukgvnT+mkJl7pgC5ISCA3w409faXGXN9FA9wDGGBamZNG3ezBXXqQlXOrChQcHMG9aAiv3HNbiLi+ige4BNmUfZVvecR6YmoBd7xGpnOS+KfF0DwnQVboX0UD3AMmpWVrCpZwuJNCPR2b0Z21GKWszSqweRzmBBrqbSy8q55t9R7hHS7iUC9w+IZbeYUE891W61ut6AQ10N5ecmklwgJ27LtYSLuV8Qf52HrtkAN/nHue79GKrx1EXSAPdjRUcP8XSbQXcNi6W8GAt4VKucXNiNDHdOvHcSl2lezoNdDf2QwnXVC3hUq7jb7fx+MyB7DpUxordh60eR10ADXQ3daKylvc35XLNiN5EhXeyehzl5a4b1YeEiBD+Z+V+Ghp0le6pNNDd1A8lXElawqVcz89u42eXDST9cDmf7yy0ehx1njTQ3VBjCVc20wdFMqS3lnCpjnH18N4M7BnKC1/vp15X6R5JA90Nffx9PiUna7QiV3Uom014fOZAMosr+HxHgdXjqPOgge5m6hsMr6VmMTI6jIkJ3aweR/mYWRf1YnCvzryw6oCu0j2QQ4EuIleKSLqIZIjIk2fZZrqIbBOR3SKS4twxfceK3UXklFYyP0lLuFTHa1ylDyCruIKl2w9ZPY5qpzYDXUTswCvALGAoMFdEhrbYJhx4FbjWGDMMuNn5o3o/YwzJKZnEdQ/mimFawqWsccWwplX61weoq2+wehzVDo6s0McDGcaYLGNMDbAYmNNim9uBT4wxuQDGmCPOHdM3bMg6yvb8E1rCpSxlswk/u3QgOaWVLNmmx9I9iSOBHgXkNXuc3/RccwOBriLynYhsEZG7W/tCIjJPRNJEJK24WC8zbmlhSiYRoQHcpCVcymJXDOvJ0N5dePkbXaV7EkcCvbWlYst3S/yAscBVwBXAr0Vk4BmfZMwiY0yiMSYxMjKy3cN6s72FZaTsL9YSLuUWRISfzhxATmkln+kq3WM4Euj5QEyzx9FAy59wPvClMabCGFMCpAIjnTOib1iUmkVwgJ07J2oJl3IPlw/tyZDeXXj52wxdpXsIRwJ9MzBAROJFJAC4DVjaYpvPgKki4iciwcAEYK9zR/Ve+ccqWbq9gLnjtYRLuY/GM176k11SwTI9L90jtBnoxpg64FFgBY0h/YExZreILBCRBU3b7AW+BHYAm4DXjTG7XDe2d3lzTQ5C4x1klHInlw9tPOPlpVUZel66B3DoPHRjzHJjzEBjTD9jzP9vem6hMWZhs23+aIwZaoy5yBjzvIvm9TrHK2tYvDmXa0f20RIu5XZ+OC+9pIJl23WV7u70SlGLvbv+IJU19cxLSrB6FKVadfq89Je+0atH3Z0GuoWqaut5e10OMwZFMriXlnAp92SzCY9dMoDM4gqWaxOjW9NAt9BHW/IprajRilzl9mZd1IsBPUJ56ZsD2pfuxjTQLVLfYHhtdRYjY8KZEK8lXMq92WzCYzMHsP/wSb7cXWT1OOosNNAt8uWuIg6WVrJgWoKWcCmPcNXw3iREhvDiKl2luysNdAsYY1iYkkl8RAiXawmX8hB2m/DYJf3ZV1TOV3v03qPuSAPdAuuzStl56AQPTI3XEi7lUa4Z0Ye47sG89M0BjNFVurvRQLfAwpQsIkIDuHGMlnApz+Jnt/HIjP7sLijj23QtVXU3GugdbE9BGan7i7l3cryWcCmPdN3oKGK6deKFVRm6SnczGugdbFFqJiEBdu6coCVcyjP52208PL0/2/OOk3qgxOpxVDMa6B0o72gly3YUMnd8LGHB/laPo9R5u3FMNH3CgnhxlR5Ldyca6B3ojTXZWsKlvEKAn42HZvRny8FjrM8stXoc1UQDvYMcq6jhH5vzmDMqij5awqW8wC2J0fTsEsgLqw5YPYpqooHeQd7dcJBTtfXMm6YlXMo7BPrZWZDUj43ZR9mYpat0d6CB3gFO1TSWcF0yuAeDenW2ehylnGbu+FgiQgN56ZsMq0dRaKB3iI+25HG0oob5ujpXXibI3878aQmsyShhy8FjVo/j8zTQXayuvoHXVmczOjac8VrCpbzQHRNj6RYSwEvf6LF0q2mgu9iXu4vIPVrJ/Gn9tIRLeaXgAD8emBrPd+nF7Mg/bvU4Pk0D3YVOl3AlRIRw2dCeVo+jlMvcfXEcYZ38eXGVHku3kga6C63LLGXXoTIenJagJVzKq4UG+nH/lHi+3nuY3QUnrB7HZ2mgu9DClEwiQgO5fnSU1aMo5XI/mRRH5yA/XtYzXiyjge4iuwtOsPpACfdNidMSLuUTwjr5c+/keL7YVUR6UbnV4/gkDXQXSU7JIiTAzh1awqV8yH2T4wgN9NMzXiyige4CeUcr+efOQm6fEEtYJy3hUr4jPDiAuy/uyz93FpJxRFfpHU0D3QXeWJONTbSES/mmB6Ym0MnfrlePWkAD3cmOVtSweHMuc0ZF0TtMS7iU7+kWEsBdE/uybHsBWcUnrR7Hp2igO9m76w9SVdugJVzKpz04LYEAP5ue8dLBNNCd6FRNPX9dn8PMwT0Y2FNLuJTviggN5M4JfVmy7RDZJRVWj+MzNNCd6MPTJVxJ/aweRSnLzUtKwN9u45VvdZXeUTTQnaSxhCuLMbHhjIvravU4SlmuR+cg7pjQl0+3HuJgqa7SO4IGupN8sauIvKOnmJ+kJVxKnbYgqbH2QlfpHcOhQBeRK0UkXUQyROTJc2w3TkTqReQm543o/n5UwjVES7iUOq1HlyBuHx/LJ98fIre00upxvF6bgS4iduAVYBYwFJgrIkPPst0zwApnD+nu1maUsrugjHnTErBpCZdSP/LQ9H7YbMLL3+rVo67myAp9PJBhjMkyxtQAi4E5rWz3GPAxcMSJ83mE5NRMIjsHcv0YLeFSqqWeTav0j3WV7nKOBHoUkNfscX7Tcz8QkSjgemDhub6QiMwTkTQRSSsuLm7vrG5p16GmEq7J8QT6aQmXUq15aHo//HSV7nKOBHprxxBMi8fPA08YY+rP9YWMMYuMMYnGmMTIyEgHR3RvyalZhAb6cfuEWKtHUcpt9ewSxO0TGlfpesaL6zgS6PlATLPH0UBBi20SgcUikgPcBLwqItc5Y0B3lne0kn/uKNASLqUc8FBS0ypdrx51GUcCfTMwQETiRSQAuA1Y2nwDY0y8MSbOGBMHfAQ8bIxZ4uxh3c3rq7Ow24T7JmsJl1Jt6dG0Sv9k6yFy9OpRl2gz0I0xdcCjNJ69shf4wBizW0QWiMgCVw/oro5W1PCPtDyuGxVFr7Agq8dRyiM8NL0f/nbhRe1Ldwk/RzYyxiwHlrd4rtU3QI0x91z4WO7vr+tytIRLqXbq0TmIuyb25Y012Twyoz/9IkOtHsmr6JWi56Gypo53mkq4BmgJl1LtsiCpH0H+dl74WlfpzqaBfh4+TMvnWGUtC6ZrCZdS7dU9NJCfTIpj2Y4C9h/Wuxo5kwZ6O50u4Rrbtyvj4rpZPY5SHmne1ASCdZXudBro7bR8VxH5x04xX4+dK3XeuoYEcN+UeP65s5A9BWVWj+M1NNDbwRjDwu8ySYgM4VIt4VLqgjwwNYEuQX78eWW61aN4DQ30dliTUcKewjLmawmXUhcsrJM/85P68fXeI3yfe8zqcbyCBno7JKdk0aNzINeN1hIupZzhnklxdA8J4LmvdJXuDBroDtqZf4I1GSXcN0VLuJRylpBAPx6a3o+1GaWsyyyxehyPp4HuoOTUTC3hUsoF7pzYl15dgnjuq/0Y07L3T7WHBroDcksrWb6zkDsmxNIlSEu4lHKmIH87j17Sny0Hj/HNPp+7nYJTaaA74PU1TSVcU7SESylXuHVcDH27B/PHFek0NOgq/XxpoLeh9GQ1H6Tlcf3oKHp20RIupVzB327j55cPYl9ROUu3t2znVo7SQG/DX9cf1BIupTrA1cN7M7R3F55bmU5NXYPV43gkDfRzOF3CdemQnvTvoSVcSrmSzSb88spB5B09xeLNuVaP45E00M/hH5vzOF5Zy0PTdXWuVEdIGhjJhPhuvLjqABXVdVaP43E00M+itr6B11dnk9i3K2P7agmXUh1BRHhi1mBKTtbw+upsq8fxOBroZ7F8ZyGHjp9ifpJW5CrVkcbEdmX28F4kp2ZSXF5t9TgeRQO9FcYYFqZk0b9HKDMH97B6HKV8zr9dMZiaugZeWLXf6lE8igZ6K1YfKGFvYRnztIRLKUvER4Rwx4RY3t+UR2bxSavH8Rga6K1YmJJJzy6BzBnVx+pRlPJZj80cQCd/O89+uc/qUTyGBnoLO/KPsy6zlPsmawmXUlaKCA1kQVICK3YfZnPOUavH8Qga6C0kp2bRWUu4lHIL909JoHdYEL/9fI9WAjhAA72Zg6UVfLGzkDsm9qWzlnApZblOAXZ+eeUgduSfYMm2Q1aP4/Y00Jt5bXUWfjYb906Os3oUpVSTOSOjGBkdxrNfplNZoxcbnYsGepOSk9V8mJavJVxKuRmbTfjPq4dSVFZFckqW1eO4NQ30Ju+sy6GmvoF5SXqZv1LuZlxcN64a3pvk1EwKT5yyehy3pYEOVFTX8c6Gg1w6pCf9IkOtHkcp1YonZw2mwcAfvtDTGM9GA53/K+FaoJf5K+W2YroFM39aAp9tK2BTtp7G2BqfD/Ta+gbeWJPNuLiujO3b1epxlFLn8PD0/vQJC+Lppbup19MYz+Dzgf7PHU0lXNN0da6Uu+sUYOdXVw1lb2EZ723SzvSWHAp0EblSRNJFJENEnmzl9TtEZEfTn3UiMtL5ozpfYwlXJgN6hHKJlnAp5RFmD+/FxQndee6rdI5V1Fg9jltpM9BFxA68AswChgJzRWRoi82ygSRjzAjgt8AiZw/qCin7i9lXVK4lXEp5EBHhN9cOo7yqjme05+VHHFmhjwcyjDFZxpgaYDEwp/kGxph1xphjTQ83ANHOHdM1klOymkq4oqweRSnVDoN6dea+yXEs3pzHloP6BulpjgR6FJDX7HF+03Nncz/wRWsviMg8EUkTkbTi4mLHp3SB7XnHWZ9Vyv1T4gnw8/m3EpTyOD+7dCC9w4L41ae7qK3Xm0qDY4He2rGIVt9eFpEZNAb6E629boxZZIxJNMYkRkZGOj6lCyxKzaJzkB9zx2sJl1KeKCTQj99cO4x9ReW8vTbH6nHcgiOBng/ENHscDRS03EhERgCvA3OMMaXOGc81ckoq+GJXIXdqCZdSHu3yoT2ZObgH//P1fg4d1ytIHQn0zcAAEYkXkQDgNmBp8w1EJBb4BLjLGOP294z6oYRrUpzVoyilLsDpN0gbjOGpJbswxrfPTW8z0I0xdcCjwApgL/CBMWa3iCwQkQVNmz0FdAdeFZFtIpLmsokvUHF5NR9uyeeGMVH00BIupTxeTLdgfnH5IFbtO8KyHYVWj2MpP0c2MsYsB5a3eG5hs48fAB5w7miu8c76HGrrG3hwmpZwKeUt7p0cz7Idhfxm6W6m9I+gW0iA1SNZwqdO76ioruOd9Qe5TEu4lPIqdpvw7I0jKK+q5b+X7bZ6HMv4VKAv3pzHiVO1LJiul/kr5W0G9erMIzP6s2RbAav2HrZ6HEv4TKDX1jfwxuosxsd1Y0yslnAp5Y0ent6fQT078++f7PTJWgCfCfTPdxRQcKKKBdP12LlS3irAz8afbx3Jscoafv3ZLqvH6XA+EejGGJJTshjYM5TpA7WESylvNqxPGD+7dCCf7yhk6fYzLpnxaj4R6N/9UMLVT0u4lPIB86clMDo2nP/8dCdFJ6qsHqfD+ESgJ6dk0qtLENeO7GP1KEqpDuBnt/HnW0ZRW2/4+YfbaPCRm2F4faBvyzvOhqyjWsKllI+Jjwjh6WuGsjajlL+kZFo9Tofw+oRblJrZWMI1QUu4lPI1t46L4eoRvfnzyv0+UbPr1YGeXVLBF7uKuGtiX0IDHbooVinlRUSE390wnD7hQfz0/W2cqKy1eiSX8upAf211Fv52G/dMjrN6FKWURboE+fPS3DEcLqviFx9t9+rj6V4b6MXl1Xy0JZ8bx0TTo7OWcCnly0bFhPMfs4ewcs9hXv0uw+pxXMZrA/2v65pKuKbGWz2KUsoN3Ds5jjmj+vDcyv18m37E6nFcwisDvbGEK4crh/UiQUu4lFI0Hk//ww0jGNyrC4+/v5WckgqrR3I6rwz09zflUlZVxzytyFVKNdMpwE7ynWMREea9m0ZZlXe9Sep1gV5b38Aba7KZEN+N0VrCpZRqIbZ7MH+5YwxZxRU8/LfvveoG014X6Mu2F1B4okorcpVSZzWpfwS/v2E4azJK+NWnO73m1nVedXL26RKuQT07M31gpNXjKKXc2M2JMeQdreTFbzKI7RbMo5cMsHqkC+ZVgf5dejHph8v58y0jEdESLqXUuf3LZQPJO3aKP321n7BO/tx1cZzVI10Qrwr0hSmZ9AkL4hot4VJKOUBEePamEZRX1fHrz3bTKcCPm8ZGWz3WefOaY+hbc4+xMfso902Jx9/uNbullHIxf7uNl28fzdQBEfzyo+18vsNzO9S9JvmSU7II6+TP3PFawqWUap8gfzvJd41lbN+uPL54G59uzbd6pPPiFYGeVXySFXsaS7hCtIRLKXUeggP8eOve8UyI78a//GM776zPsXqkdvOKQH9tdbaWcCmlLlhooB9v3jOOy4b25KnPdvPSqgMedUqjxwf6kfIqPv4+n5vHRhMRGmj1OEopDxfkb+cvd4zhhtFRPLdyPz//YDtVtfVWj+UQjz8+8fba0yVcepm/Uso5/Ow2nrtlJHERIfx55X4ySypYdNdYenZx7+ZWj16hn6yu490NB5l1US/iIkKsHkcp5UVEhJ/OHEDyXWPJOFzO1S+tIXV/sdVjnZNHB/riTbmUV9Uxf5pe5q+Uco0rhvXik4cnE9bJn7vf3MRvlu5220MwHhvoNXUNvL46m4sTujMyJtzqcZRSXmxQr858/tgU7pkUx9vrcrjqxdWszSixeqwzeGygL91eQFFZFfOT9Ni5Usr1gvzt/ObaYbx7/3iq6xq44/WNzHsnza161T0y0BsaDItSMxncqzNJWsKllOpAUwdE8vW/JvFvVwxiTUYJl/1PCr/4cDvpReVWj+ZYoIvIlSKSLiIZIvJkK6+LiLzY9PoOERnj/FH/z7fpR9h/+CTzkxK0hEsp1eGC/O08MqM/3/1iOnPHx/LPHYVc8Xwqd72xkSVbD1l24wxp66R5EbED+4HLgHxgMzDXGLOn2TazgceA2cAE4AVjzIRzfd3ExESTlpZ2XkPfsnA9h46f4rt/m669LUopyx2vrOHvG3N5d/1Bisqq8LcLk/pFMD6+G8P6dGFYnzAiQgOcsgAVkS3GmMTWXnPkPPTxQIYxJqvpiy0G5gB7mm0zB3jHNP7fYYOIhItIb2NM4QXOfoYtB4+xKecoT109VMNcKeUWwoMDeGRGfx5K6sfWvOOs2F3E13sOk9LsNEe7TegS5EdYJ3/unNiXB1xw7YwjgR4F5DV7nE/jKrytbaKAHwW6iMwD5gHExp5/ida0gZHcOi7mvD9fKaVcwWYTxvbtyti+XfmP2UMoq6plT0EZewrKKK2o5sSpWspO1RHZ2TVXtTsS6K39G6HlcRpHtsEYswhYBI2HXBz43mcY27cr79w3/nw+VSmlOlSXIH8mJnRnYkL3Dvl+jhyzyAeaL4ejgZaFwY5so5RSyoUcCfTNwAARiReRAOA2YGmLbZYCdzed7TIROOGK4+dKKaXOrs1DLsaYOhF5FFgB2IE3jTG7RWRB0+sLgeU0nuGSAVQC97puZKWUUq1xqG3RGLOcxtBu/tzCZh8b4BHnjqaUUqo99Lw/pZTyEhroSinlJTTQlVLKS2igK6WUl2izy8Vl31ikGDh4np8eAbhfGbFr6T77Bt1n33Ah+9zXGNNqzaxlgX4hRCTtbOU03kr32TfoPvsGV+2zHnJRSikvoYGulFJewlMDfZHVA1hA99k36D77Bpfss0ceQ1dKKXUmT12hK6WUakEDXSmlvIRbB7q73Zy6Iziwz3c07esOEVknIiOtmNOZ2trnZtuNE5F6EbmpI+dzBUf2WUSmi8g2EdktIikdPaOzOfB3O0xElonI9qZ99ujWVhF5U0SOiMius7zu/PwyxrjlHxqrejOBBCAA2A4MbbHNbOALGu+YNBHYaPXcHbDPk4CuTR/P8oV9brbdNzS2ft5k9dwd8HMOp/G+vbFNj3tYPXcH7PN/AM80fRwJHAUCrJ79AvZ5GjAG2HWW152eX+68Qv/h5tTGmBrg9M2pm/vh5tTGmA1AuIj07uhBnajNfTbGrDPGHGt6uIHGu0N5Mkd+zgCPAR8DRzpyOBdxZJ9vBz4xxuQCGGM8fb8d2WcDdBYRAUJpDPS6jh3TeYwxqTTuw9k4Pb/cOdDPduPp9m7jSdq7P/fT+H94T9bmPotIFHA9sBDv4MjPeSDQVUS+E5EtInJ3h03nGo7s88vAEBpvX7kTeNwY09Ax41nC6fnl0A0uLOK0m1N7EIf3R0Rm0BjoU1w6kes5ss/PA08YY+obF28ez5F99gPGAjOBTsB6EdlgjNnv6uFcxJF9vgLYBlwC9ANWishqY0yZi2ezitPzy50D3RdvTu3Q/ojICOB1YJYxprSDZnMVR/Y5EVjcFOYRwGwRqTPGLOmQCZ3P0b/bJcaYCqBCRFKBkYCnBroj+3wv8AfTeIA5Q0SygcHApo4ZscM5Pb/c+ZCLL96cus19FpFY4BPgLg9erTXX5j4bY+KNMXHGmDjgI+BhDw5zcOzv9mfAVBHxE5FgYAKwt4PndCZH9jmXxn+RICI9gUFAVodO2bGcnl9uu0I3Pnhzagf3+SmgO/Bq04q1znhwU52D++xVHNlnY8xeEfkS2AE0AK8bY1o9/c0TOPhz/i3wtojspPFwxBPGGI+t1RWR94HpQISI5ANPA/7guvzSS/+VUspLuPMhF6WUUu2gga6UUl5CA10ppbyEBrpSSnkJDXSllPISGuhKKeUlNNCVUspL/C8L7+KETE+gBQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "p = torch.linspace(0.,1,100)\n",
    "f = combine_scheds([0.3,0.2,0.5], [SchedLin(0.,1.), SchedNo(1.,1.), SchedCos(1., 0.)])\n",
    "plt.plot(p, [f(o) for o in p]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ead0020",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "test_close([f(0.), f(0.15), f(0.3), f(0.4), f(0.5), f(0.7), f(1.)],\n",
    "           [0., 0.5, 1., 1., 1., 0.65451, 0.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5a8a393",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def combined_cos(pct, start, middle, end):\n",
    "    \"Return a scheduler with cosine annealing from `start`→`middle` & `middle`→`end`\"\n",
    "    return combine_scheds([pct,1-pct], [SchedCos(start, middle), SchedCos(middle, end)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a61b8b2e",
   "metadata": {},
   "source": [
    "This is a useful helper function for the [1cycle policy](https://sgugger.github.io/the-1cycle-policy.html). `pct` is used for the `start` to `middle` part, `1-pct` for the `middle` to `end`. Handles floats or collection of floats. For example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ca753c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAmXUlEQVR4nO3dd3RVVd7G8e8vnZBAKCG0hBo6hBJAURxUHEHslaKMldeCgzpjmRmnvfrOOHYBUbF3VNSxd+yIEHqHEFpoIaGFBFL3+0cYVwYjXOAm55bnsxZrcXNOkmcbeNiee87e5pxDRESCX4TXAURExD9U6CIiIUKFLiISIlToIiIhQoUuIhIiorz6xk2bNnVt27b16tuLiASluXPn5jvnkms65lmht23blqysLK++vYhIUDKz9b90TJdcRERChApdRCREqNBFREKECl1EJESo0EVEQsRhC93MnjGzPDNb8gvHzcwmmlm2mS0ys77+jykiIofjywz9OWDYIY4PB9IP/BoHPHbssURE5Egd9j5059w3Ztb2EKecA7zgqtbhnWVmSWbWwjm3xV8h5cht2b2PRbm72V1cxq59pRSXVlAvOpL4mEgS4qJITogjpUEsKQ3jaBAX7XVcEfEDfzxY1ArYWO117oGP/azQzWwcVbN40tLS/PCtpbq8Pft5a/4mPl6ylQUbd/n8eU0TYumQXJ/0lAR6tmpI79RGdGyWQGSE1V5YEfE7fxR6TX/ra9w1wzk3FZgKkJmZqZ01/GR/WQVPfpPDlK/WsK+sgp6tGnLr6Z05sWNTmiTEkBQfQ73oSPaXVVBcWkHh/jLyCkvIKyxh86595GzfS3beXt6Zv5mXZm0AICE2ioHtGjM4vSmDOyXTvml9zFTwIoHMH4WeC6RWe90a2OyHrys+mLFiG3/+91I27drHsO7NuW1YZ9onJ9R4bv3YKOrHRpGcGFvjOZWVjpz8IhZs3MW8DTv5PjufL1bkAdA+uT5n9GjBsB7N6d6ygcpdJAD5o9DfBcab2TRgILBb189rn3OOyTOyeeCzVXROSeSVawYyqEPTY/qaERFGx2YJdGyWwIX9WgOwoaCYr1fl8fHSrUz5KpvJX2aT3iyBizNTObdPK5ITY/0xHBHxAzvcnqJm9iowBGgKbAP+CkQDOOcet6qp2mSq7oQpBq5wzh121a3MzEynxbmOTnFpObe+sYgPFm/hvD6t+Of5PYmLjqz171uwt4RPlm5j+tyNzNuwi6gI4/TuzbnyxHb0TUvSrF2kDpjZXOdcZo3HvNokWoV+dAr3l3HZ07NZmLuLO4Z1YdxJ7T0p0uy8QqbN3shrWRsp3F9ORmoS/3NSe4Z1b06E3kwVqTUq9BCxr7SC3zwzm3kbdjJ5dF+G9WjudSSKSsp5c14uz36/jrX5RaQ3S+DGU9MZ0bOF7pIRqQUq9BBQUl7B1c9n8V12PhNH9uGsjJZeR/ovFZWODxZvYdIXq1mdt5f0ZgncMbwLp3RppksxIn50qELXWi5BoLLSMeHVBXy7Op9/XdAr4MocIDLCODujJZ/cdBKTR/ehvNJx1fNZXDJ1FguP4J54ETl6KvQgMGlGNh8v3cqdI7pycWbq4T/BQxERxpm9WvLpzSdx1zndydm+l3OnfM8dby5iR1Gp1/FEQpoKPcDNWLGNh79YxXl9WnHVie28juOz6MgILju+LV/+fghXn9iON+bmcvL9X/Hyj+uprNQzZSK1QYUewNbmFzFh2gK6tWjAP87rGZTXohPjovnTiG58NGEwXVsk8qe3lzDyyVmszS/yOppIyFGhB6j9ZRVc++JcoiKMxy/tR72Y2r/PvDZ1Sknk1WuO494LerF8yx6GPfwNT3y9hgrN1kX8RoUeoB74dCUrtxXy4CW9SW0c73UcvzAzLu6fyue3/IpfdUrmnx+tYNTUWWzcUex1NJGQoEIPQLNyCnjqu7WMGZjGyZ2beR3H71IaxPHEZf144KIMlm3Zw/BHvuXNublexxIJeir0AFO4v4zfv7GQtMbx/GlEV6/j1Boz44J+rflowmC6tWjA795YyC2vL6CopNzraCJBS4UeYO5+fzmbd+3jwYsziI/xx9ppgS21cTyvjjuOm4am8/b8TZw9+TtWbN3jdSyRoKRCDyAz1+TzWtZGxp3UgX5tGnsdp85ERhg3De3Ey1cNZPe+cs6Z/D1vz9clGJEjpUIPECXlFdz57yWkNY7npqHpXsfxxKCOTflwwolkpCZx82sL+ft7SymrqPQ6lkjQUKEHiKlf55CzvYj/Pad7nSyFG6iaJcbx8tUDuXxQW579fh2XPvWjnjAV8ZEKPQCsyy9i0pfZjOjVgiEheFfLkYqOjOBvZ3fnwYszmL9xF+c++j2rtxV6HUsk4KnQPeac4y/vLiUmMoK/nNnN6zgB5fy+rZk27jiKSys4f8pMvl613etIIgFNhe6xz5Zt45tV27nltE6kNIjzOk7A6ZvWiHfGn0CrRvW48rk5vPLjBq8jiQQsFbqHSssr+ceHy+mQXJ/Ljm/jdZyA1SqpHtOvG8Tg9Kb88e3FPPDpSrxax18kkKnQPfTCD+tYV1DMnSO6ER2pH8WhJMRG8eTYTC7ObM2kGdn8/o1FugNG5CCh/+RKgNpZVMrEL1YzOL0pQzonex0nKERHRvCvC3rRMqkeD3++mp3FpUwZ0zes7woSqU7TQo88/Pkq9paUc+eIbkG5LK5XzKoeQrr73B58uTKPsU/PZs/+Mq9jiQQEFboHcrbv5aUfNzBqQBqdmyd6HScoXXpcGyaO7MO8DTsZNXUW+XtLvI4k4jkVugce+HQVsVER3DS0k9dRgtpZGS156jeZrNm+l5FTZ5G3Z7/XkUQ8pUKvY4tyd/HB4i1cPbg9yYmxXscJekM6N+P5KwawZdc+Ln7iBzbv2ud1JBHPqNDr2H2frKRRfDTXDA6e/UED3cD2TXjhqoEU7C3l4id+0IYZErZU6HXo++x8vl2dzw0ndyQxLtrrOCGlX5tGvHzNQAr3lzNSuyBJmFKh1xHnHPd+vIKWDeO49Dg9RFQberVO4uWrB1K4v4xRT84id6dKXcKLCr2OfLpsGwtzd3PT0E66b7oW9WjVkJeuHsjufVWlvknX1CWMqNDrQGWl46HPVtG+aX3O79vK6zghr1frJF66aiC7isoY86TufpHwoUKvAx8t2cqKrYVMGJpOlB7xrxMZqUk8d2V/8gpLuPRpraku4UHtUssqKh0Pf76Kjs0SOLNXS6/jhJV+bRrz1NhM1hUUM/aZH/VEqYQ8nwrdzIaZ2UozyzazO2o43tDM3jOzhWa21Myu8H/U4PT+os2sztvLTUPTiYzQI/51bVDHpjxxaT9Wbi3kqufmsK+0wutIIrXmsIVuZpHAo8BwoBswyswO3onhBmCZcy4DGAI8YGYxfs4adMorKnnk89V0aZ7IGT1aeB0nbJ3cpRkPX9KHrPU7ueGVeVqlUUKWLzP0AUC2cy7HOVcKTAPOOegcByRa1SpTCcAOoNyvSYPQuws3k5NfxE1D04nQ7NxTI3q14O5zezBjRR63vrGQykqtpy6hx5flc1sBG6u9zgUGHnTOZOBdYDOQCFzinPvZNMjMxgHjANLS0o4mb9CoqHRMnpFN1xYNOL17c6/jCDBmYBt2FZdx3ycrSYqP4a9naaVLCS2+zNBr+hN/8PTmdGAB0BLoDUw2swY/+yTnpjrnMp1zmcnJob0G+HsHZucTTu2o0ggg1w/pwJUntOO5met44pscr+OI+JUvhZ4LpFZ73ZqqmXh1VwBvuSrZwFqgi38iBp+KSsfEGavpnJLIr7tpdh5IzIw7R3TlrIyW3PPRCt6al+t1JBG/8aXQ5wDpZtbuwBudI6m6vFLdBuBUADNLAToDYTv9eX/RZnK2F/HbU3XtPBBFRBj3X9SL49s34bbpi/h61XavI4n4xWEL3TlXDowHPgGWA68755aa2bVmdu2B0+4CBpnZYuAL4HbnXH5thQ5klZWOSTOySW+WwPAemp0HqtioSJ4Y24/0lESuf2kuSzfv9jqSyDEzr3ZPz8zMdFlZWZ5879r0/qLNjH9lPhNH9eHsDD1IFOi27t7PeVO+p9I53r7+BFom1fM6ksghmdlc51xmTcf0pKgfVVY6Jn2RTYfk+ozoqfvOg0HzhnE8c3l/ikoquPK5OXqaVIKaCt2PPl22jZXbChl/Skc9FRpEurZowGOX9iU7by/Xv6QHjyR4qdD9xDnHpBmradsknrO0ZkvQGZyezD/O68l32fn89d2leHUpUuRY+PJgkfhgxoo8lm7ew30X9tKKikHq4v6p5OQX8fjXa2jftD5XD27vdSSRI6JC9wPnHBO/WE3rRvU4t4/WOw9mt53emXX5Rfzfh8tp26Q+Q7uleB1JxGeaSvrBN6vzWZi7mxtO7ki0ZudBLSLCeOiS3vRs1ZDfTpvP8i17vI4k4jO1zzFyzvHI56to2TCOC/q29jqO+EG9mEieHJtJYlwUVz+fRf7eEq8jifhEhX6MZq4pYN6GXVx3ckdiovSfM1SkNIjjybGZFBSVcO2Lcykp1zrqEvjUQMegana+muYN4rg4U7PzUNOrdRL3X5RB1vqd3Pn2Et35IgFPhX4MZuXsYPa6HVw3pAOxUZFex5FacGavlvz21HTemJvLs9+v8zqOyCGp0I/BxC9W0ywxlkv6px7+ZAlaN52azq+7pfB/Hy7nu9VhuUSRBAkV+lGavXYHP+QUcO2vOhAXrdl5KIuIMB68pDcdkutzwyvzWF9Q5HUkkRqp0I/Sw5+vomlCLKMGhPbOS1IlITaKJ8dWrYc07oW5FJWE/Q6LEoBU6Efhx5wCZq4p4LohHagXo9l5uGjTpD6TR/dhdV4ht01fpDdJJeCo0I/CQ5+volliLGMGanYebganJ3P7sC58sHiLtrCTgKNCP0Iz1+QzK6fqzhZdOw9P405qz5m9WnDvxyv4RrsdSQBRoR8B5xwPf7aalAa6dh7OzIx7L+xFp5REbnx1Pht3FHsdSQRQoR+RmWsKmL1uB9cP6ajZeZiLj4niicv64Zzjf16cy/4yPUkq3lOh+8g5xwOfrqR5gzjddy5A1ZukD4/szbIte/iTniSVAKBC99EXy/OYt2EXE4ama3YuPzmlSwoTTk3nzXm5vPzjBq/jSJhTofugstJx/6crade0Phf205ot8t8mnJrOyZ2T+ft7S5m/YafXcSSMqdB98N6izazYWsjNp3XSeufyM/9ZQz2lQRzXvzyPAi23Kx5ROx1GWUUlD362iq4tGnBmzxZex5EAlRQfw2Nj+lFQVMpNry2golLX06XuqdAP442sXNYXFHPr6Z2IiDCv40gA69m6If97dne+XZ3PI5+v8jqOhCEV+iEUlZTz0OeryGzTiJM7N/M6jgSBS/qncmG/1kyckc2XK/O8jiNhRoV+CE98k8P2whL+NKIrZpqdy+GZGXef24MuzRO5+bUFbNq1z+tIEkZU6L9g6+79TP1mDWf2akGftEZex5EgEhcdyWOX9qO8wnHDy/MoLa/0OpKECRX6L3jg05VUVsLtw7p4HUWCULum9bn3wl4s2LiLf3603Os4EiZU6DVYunk30+fl8ptBbUhtHO91HAlSZ/RswRUntOXZ79fx4eItXseRMKBCP4hzjrvfX07DetGMPznd6zgS5P4wvCu9U5O4ffoi1uVrpyOpXT4VupkNM7OVZpZtZnf8wjlDzGyBmS01s6/9G7PuvL9oCz/kFHDLaZ1oGB/tdRwJcjFREUwe3YeICOP6l+dpES+pVYctdDOLBB4FhgPdgFFm1u2gc5KAKcDZzrnuwEX+j1r79paUc/cHy+jesgFjBrbxOo6EiNaN4nnw4gyWbdnDXe8v8zqOhDBfZugDgGznXI5zrhSYBpxz0DmjgbeccxsAnHNBeQPuxC9Ws21PCXed24NIPUQkfnRq1xT+56T2vPzjBt5ZsMnrOBKifCn0VsDGaq9zD3ysuk5AIzP7yszmmtnYmr6QmY0zsywzy9q+PbB2elm1rZBnvlvLJZmp9NVtilILfn96Z/q1acQf31pMzva9XseREORLodc0VT14oYoooB8wAjgd+LOZdfrZJzk31TmX6ZzLTE5OPuKwtaWy0nHnv5dQPzaK24Z19jqOhKjoyAgmjepDdFQEN7wyX9fTxe98KfRcoPqODq2BzTWc87Fzrsg5lw98A2T4J2Lte3HWemav3cEfhnehSUKs13EkhLVMqseDF2ewfMse7v5A19PFv3wp9DlAupm1M7MYYCTw7kHnvAMMNrMoM4sHBgJB8TTF+oIi7vloBSd1StZORFInTulSdT39pVkbeH/RwXMjkaN32EJ3zpUD44FPqCrp151zS83sWjO79sA5y4GPgUXAbOAp59yS2ovtH5WVjlunLyIqwrjn/J5ar0XqzO9P70zftCTueHMx6wt0f7r4h3m1D2JmZqbLysry5Hv/x3Pfr+Vv7y3j3gt6cbFm51LHcncWM2Lid6Q1jmf6dccTG6WtDeXwzGyucy6zpmNh+6To8i17+OdHK/hVp2QuytS2clL3WjeK574Le7F4027u+WiF13EkBIRloRfuL+P6l+fRsF4091+UoUst4plfd2/+03ovnyzd6nUcCXJhV+jOOe54czEbdhQzaVQfkhN1V4t4647hXejZqiG3TV+k9dPlmIRdoT83cx0fLN7Crad3ZmD7Jl7HESE2KpJJo/pQUen47avzKavQ+ulydMKq0L9ckcfdHyxnaNdmjBvc3us4Ij9p27Q+/zi/J3PX7+Shz7QfqRydsCn0eRt2cv3L8+jaIpGHLumtDZ8l4Jyd0ZKR/VN57Os1fLMqsJbGkOAQFoWenVfIlc/NoVmDWJ69fACJcVoWVwLTX8/qTnqzBG55fQF5hfu9jiNBJuQLffW2Qi57ejZRERG8eOVAvQkqAa1eTCSTR/dlb0k5t7y2kMpKb54TkeAU0oU+K6eACx6bSVmF44UrB5DWRNvJSeDrlJLI387qznfZ+Tz29Rqv40gQCdlCf2fBJsY+PZvkxFjevn4Q3Vo28DqSiM8u6Z/Kmb1a8OBnq8hat8PrOBIkQq7QdxSVcsvrC5gwbQG905J467oTtNGzBB0z45/n96RVUj0mTFvAruJSryNJEAiZQq+odLw1L5ehD37Nuws2M/7kjrx41QDtCypBKzEumkmj+pBXuJ/b31yEV+suSfCI8jrAsXDOsb6gmOlzc5k+N5ete/bTOzWJey7oSZfmusQiwS8jNYnbh3Xh7g+W89Ks9Vx2fFuvI0kAC7pCn7NuB098nUPuzmI27iimqLSCCINfdUrmb2d347RuzbUfqISUK09ox/fZ+dz1wXL6tWms94PkFwVdoZeUVbJxRzGpjeM5vkMT2japz6+7p9CiYT2vo4nUiogI4/6LMhj+yLeMf3Ue7994IvExQfdXV+pAWK+HLhJMflhTwOinZnFB39bcf1HQ7PAofqb10EVCwPEdmnDjKelMn5vL2/NzvY4jAUiFLhJEfntKRwa0bcydby9hbb62rpP/pkIXCSJRkRE8PLI30VERjH9lHiXlFV5HkgCiQhcJMi2T6nHfhRks3bxHW9fJf1GhiwSh07qlcPmgqq3rPlu2zes4EiBU6CJB6g9ndKFHqwbcOn0hm7V1naBCFwlaVVvX9aWsvJIJ0+ZTrq3rwp4KXSSItTuwdd2cdTt5+PPVXscRj6nQRYLcOb1bcXFmax79KpvvVud7HUc8pEIXCQF/O7s7HZMTuOm1BWwvLPE6jnhEhS4SAuJjopg8ui+F+8u4+bUF2rouTKnQRUJE5+aJ/P3sqq3rpnyV7XUc8YAKXSSEXNI/lbMzWvLgZ6uYvVZb14UbFbpICDEz/nF+T9Iax/PbV+ezo0hb14UTFbpIiEmIrbqevqOolN+9ruvp4cSnQjezYWa20syyzeyOQ5zX38wqzOxC/0UUkSPVo1VD/nxmV75cuZ0nv83xOo7UkcMWuplFAo8Cw4FuwCgz6/YL5/0L+MTfIUXkyF16XBvO6Nmcez9Zydz1up4eDnyZoQ8Asp1zOc65UmAacE4N590IvAnk+TGfiBwlM+OeC3rRKqke41+Zz05dTw95vhR6K2Bjtde5Bz72EzNrBZwHPH6oL2Rm48wsy8yytm/ffqRZReQINYiL5tHRfSnYW8rv3lio6+khzpdCtxo+dvCfioeB251zh1xt3zk31TmX6ZzLTE5O9jGiiByLnq0b8qcRXZmxIo+pup4e0nzZOjwXSK32ujWw+aBzMoFpZgbQFDjDzMqdc//2R0gROTZjj2/D7LU7uO+TlfRr04j+bRt7HUlqgS8z9DlAupm1M7MYYCTwbvUTnHPtnHNtnXNtgenA9SpzkcBhZvzzgp6kNqrH+FfmUbBX672EosMWunOuHBhP1d0ry4HXnXNLzexaM7u2tgOKiH80iIvm0TF92Vlcxk2vLaBC19NDjjnnzQ81MzPTZWVlefK9RcLZq7M38Ie3FnPz0E5MGJrudRw5QmY21zmXWdMxPSkqEmZG9k/l/D6tePiLVVo/PcSo0EXCjJlx93k9SG+WwG+nzWfLbu1HGipU6CJhKD4miscu7UdJWQXjX5lPmfYjDQkqdJEw1SE5gXsu6MXc9Tu556MVXscRP1Chi4SxszJacvmgtjz93VreX3Tw4yUSbFToImHuj2d0pW9aErdPX0R2XqHXceQYqNBFwlxMVARTxvQjLjqSa1+aR1FJudeR5Cip0EWE5g3jmDS6Dznb93Lb9EV49XyKHBsVuogAMKhDU24f1oUPFm/hqW/Xeh1HjoIKXUR+Mu6k9pzRszn3fLyCmWv00FGwUaGLyE/MjHsvzKBd0/rc+Mp8Nu/SQ0fBRIUuIv8lITaKJy7rR0l5Jde9NJf9ZYfc5kACiApdRH6mQ3ICD13Sm4W5u7nz30v0JmmQUKGLSI1O65bChFPTmT43lxd+WO91HPGBCl1EftGEU9MZ2jWFu95fxqycAq/jyGGo0EXkF0VEGA9dkkFak3iuf3keG3cUex1JDkGFLiKHlBgXzVNjMymrqOSaF7IoLtWTpIFKhS4ih9U+OYFJo/qwalshv39jod4kDVAqdBHxyZDOzbhjeBc+XLyViV9kex1HahDldQARCR7XDG7Piq2FPPT5KtJTEjijZwuvI0k1mqGLiM/MjH+c15O+aUnc8voCFufu9jqSVKNCF5EjEhcdyROXZdKkfizXvJDFtj37vY4kB6jQReSIJSfG8uTYTPbsL+Pq53XnS6BQoYvIUenWsgETR/Zhyebd3PzaAiordeeL11ToInLUhnZL4c4R3fhk6Tb+9bE2mvaa7nIRkWNy5QltWZdfxBPf5NCmSX1GD0zzOlLYUqGLyDExM/56Vjc27izmz+8soUVSHCd3buZ1rLCkSy4icsyiIiOYPLovXZoncsPL81iySbczekGFLiJ+kRAbxbOX96dRfAxXPDdHC3l5QIUuIn7TrEEcz13Rn/1lFVz+7Gx2FpV6HSms+FToZjbMzFaaWbaZ3VHD8TFmtujAr5lmluH/qCISDNJTEnlybCYbd+7jqufnsK9UW9jVlcMWuplFAo8Cw4FuwCgz63bQaWuBXznnegF3AVP9HVREgsdx7ZvwyCW9mb9xF+NfmUd5RaXXkcKCLzP0AUC2cy7HOVcKTAPOqX6Cc26mc27ngZezgNb+jSkiwWZ4zxbcdU4PvliRxx/fXqwld+uAL7cttgI2VnudCww8xPlXAR/VdMDMxgHjANLSdK+qSKi79Lg25BWWMPGL1STFx/CH4V0wM69jhSxfCr2m//o1/lNrZidTVegn1nTcOTeVA5djMjMz9c+1SBi4eWg6u4tLmfpNDg3rRXPDyR29jhSyfCn0XCC12uvWwOaDTzKzXsBTwHDnnHaTFRHgPw8edWf3vjLu+2QlDepFc9lxbbyOFZJ8KfQ5QLqZtQM2ASOB0dVPMLM04C3gMufcKr+nFJGgFhFh3HdRBoX7y/nLO0uIj47kgn56q83fDvumqHOuHBgPfAIsB153zi01s2vN7NoDp/0FaAJMMbMFZpZVa4lFJChFR0bw6Ji+DOrQhFunL+S9hT/7H305RubVO8+ZmZkuK0u9LxJuikvLufyZOczbsJMpY/ry6+7NvY4UVMxsrnMus6ZjelJUROpUfEwUT1+eSY9WDRn/yny+WL7N60ghQ4UuInUuMS6a568YQJcWiVz30jyVup+o0EXEEw3jo3nxyoEqdT9SoYuIZ6qX+rUvzeXjJVu9jhTUVOgi4qn/lHr3lg254ZV5vLNgk9eRgpYKXUQ81zA+mpeuHki/No246bUFvD5n4+E/SX5GhS4iASEhNornrxjAiR2bctubi3jq2xyvIwUdFbqIBIx6MZE89ZtMhvdozt0fLOfej1dolcYjoEIXkYASGxXJ5NF9GTUgjSlfreEPby3Weuo+8mUtFxGROhUZYfzjvB40qR/D5C+z2V5YwqTRfYiPUWUdimboIhKQzIzfn96Zu87pzpcr8xg5dRbbC0u8jhXQVOgiEtAuO74tUy/LZPW2vZw35XtWbyv0OlLAUqGLSMAb2i2FaeOOY39ZJedPmcmXK/O8jhSQVOgiEhQyUpN4Z/wJpDaO56rn5vDUtzm6A+YgKnQRCRqtkuox/brj+XW3qtsaf/f6QvaVVngdK2Co0EUkqMTHRDFlTF9uHtqJtxds4vzHZrKhoNjrWAFBhS4iQSciwpgwNJ1nftOfTTuLOXPSt3y+TKs1qtBFJGid3KUZ7984mNTG8Vz9QhZ/f28pJeXhewlGhS4iQS2tSTxvXjeIywe15dnv13H+lJnkbN/rdSxPqNBFJOjFRUfyt7O7M/WyfmzatY8zJn7LCz+so7IyvO6CUaGLSMj4dffmfDzhJAa0a8Jf3lnK2Gdms3nXPq9j1RkVuoiElOYN43j+iv7833k9mLt+J6c9+DXPz1xHRRjM1lXoIhJyzIwxA9vwyU0n0bdNI/767lIufHwmK7bu8TparVKhi0jISmsSzwtXDuDBizNYl1/EiInf8bd3l7K7uMzraLVChS4iIc3MOL9va2b8bgijB6Txwg/rGHL/l7w4az1lIbbOugpdRMJCo/ox3HVuD96/cTDpKYn8+d9LOO3Br3lv4eaQuRtGhS4iYaVbywa8Nu44nv5NJrFRkdz46nxGTPqODxZtCfo3Ts2r1coyMzNdVlaWJ99bRASgotLxzoJNTJ6RTU5+Ee2T63PtSR04u3dL4qIjvY5XIzOb65zLrPGYCl1Ewl1FpePjJVt59Mtslm3ZQ6P4aEYOSGPMwDRaN4r3Ot5/UaGLiPjAOccPOQU8P3Mdny3bhgMGdWjCBX1bM6xH84DY01SFLiJyhHJ3FvNGVi5vzc9l44591IuO5Fedkjm9RwqndE6hYXy0J7mOudDNbBjwCBAJPOWcu+eg43bg+BlAMXC5c27eob6mCl1EgoFzjjnrdvLuwk18tmwb2/aUEBlh9GzVkEEdmnBc+yZkpCbRsF7dFPwxFbqZRQKrgNOAXGAOMMo5t6zaOWcAN1JV6AOBR5xzAw/1dVXoIhJsKisdizbtZsbybcxcU8CCjbsoP3BnTJsm8fRo1ZAOyQm0axpPmyb1SWkQR5P6MX59g/VQhe7LBaEBQLZzLufAF5sGnAMsq3bOOcALrupfh1lmlmRmLZxzW44xu4hIwIiIMHqnJtE7NYlbgKKScuZt2Mmi3N0s2bSbRbm7+GjxFg6++7F+TCTxsVHEREYQGxXBqAFpXHNSe7/n86XQWwEbq73OpWoWfrhzWgH/VehmNg4YB5CWlnakWUVEAkr92CgGpyczOD35p4+VlFewccc+1hcUsb2whIKiUvL3lrC/rIKS8kpKyytJToytlTy+FLrV8LGDr9P4cg7OuanAVKi65OLD9xYRCSqxUZF0bJZAx2YJdf69fXlSNBdIrfa6NbD5KM4REZFa5EuhzwHSzaydmcUAI4F3DzrnXWCsVTkO2K3r5yIideuwl1ycc+VmNh74hKrbFp9xzi01s2sPHH8c+JCqO1yyqbpt8YraiywiIjXx6bEn59yHVJV29Y89Xu33DrjBv9FERORIaLVFEZEQoUIXEQkRKnQRkRChQhcRCRGerbZoZtuB9Uf56U2BfD/GCQYac3jQmMPDsYy5jXMuuaYDnhX6sTCzrF9anCZUaczhQWMOD7U1Zl1yEREJESp0EZEQEayFPtXrAB7QmMODxhweamXMQXkNXUREfi5YZ+giInIQFbqISIgI6EI3s2FmttLMss3sjhqOm5lNPHB8kZn19SKnP/kw5jEHxrrIzGaaWYYXOf3pcGOudl5/M6swswvrMl9t8GXMZjbEzBaY2VIz+7quM/qbD3+2G5rZe2a28MCYg3rVVjN7xszyzGzJLxz3f3855wLyF1VL9a4B2gMxwEKg20HnnAF8RNWOSccBP3qduw7GPAhodOD3w8NhzNXOm0HVqp8Xep27Dn7OSVTt25t24HUzr3PXwZj/CPzrwO+TgR1AjNfZj2HMJwF9gSW/cNzv/RXIM/SfNqd2zpUC/9mcurqfNqd2zs0CksysRV0H9aPDjtk5N9M5t/PAy1lU7Q4VzHz5OQPcCLwJ5NVluFriy5hHA2855zYAOOeCfdy+jNkBiWZmQAJVhV5etzH9xzn3DVVj+CV+769ALvRf2nj6SM8JJkc6nquo+hc+mB12zGbWCjgPeJzQ4MvPuRPQyMy+MrO5Zja2ztLVDl/GPBnoStX2lYuBCc65yrqJ5wm/95dPG1x4xG+bUwcRn8djZidTVegn1mqi2ufLmB8GbnfOVVRN3oKeL2OOAvoBpwL1gB/MbJZzblVth6slvoz5dGABcArQAfjMzL51zu2p5Wxe8Xt/BXKhh+Pm1D6Nx8x6AU8Bw51zBXWUrbb4MuZMYNqBMm8KnGFm5c65f9dJQv/z9c92vnOuCCgys2+ADCBYC92XMV8B3OOqLjBnm9laoAswu24i1jm/91cgX3IJx82pDztmM0sD3gIuC+LZWnWHHbNzrp1zrq1zri0wHbg+iMscfPuz/Q4w2MyizCweGAgsr+Oc/uTLmDdQ9X8kmFkK0BnIqdOUdcvv/RWwM3QXhptT+zjmvwBNgCkHZqzlLohXqvNxzCHFlzE755ab2cfAIqASeMo5V+Ptb8HAx5/zXcBzZraYqssRtzvngnZZXTN7FRgCNDWzXOCvQDTUXn/p0X8RkRARyJdcRETkCKjQRURChApdRCREqNBFREKECl1EJESo0EVEQoQKXUQkRPw/R6vWzscLciUAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "f = combined_cos(0.25,0.5,1.,0.)\n",
    "plt.plot(p, [f(o) for o in p]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98adfd49",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "test_close([f(0.), f(0.1), f(0.25), f(0.5), f(1.)], [0.5, 0.67275, 1., 0.75, 0.])\n",
    "f = combined_cos(0.25, np.array([0.25,0.5]), np.array([0.5,1.]), np.array([0.,0.]))\n",
    "for a,b in zip([f(0.), f(0.1), f(0.25), f(0.5), f(1.)],\n",
    "               [[0.25,0.5], [0.33638,0.67275], [0.5,1.], [0.375,0.75], [0.,0.]]):\n",
    "    test_close(a,b)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8cafabb",
   "metadata": {},
   "source": [
    "## ParamScheduler -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "333af2cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "@docs\n",
    "class ParamScheduler(Callback):\n",
    "    \"Schedule hyper-parameters according to `scheds`\"\n",
    "    order,run_valid = 60,False\n",
    "\n",
    "    def __init__(self, scheds): self.scheds = scheds\n",
    "    def before_fit(self): self.hps = {p:[] for p in self.scheds.keys()}\n",
    "    def before_batch(self): self._update_val(self.pct_train)\n",
    "\n",
    "    def _update_val(self, pct):\n",
    "        for n,f in self.scheds.items(): self.opt.set_hyper(n, f(pct))\n",
    "\n",
    "    def after_batch(self):\n",
    "        for p in self.scheds.keys(): self.hps[p].append(self.opt.hypers[-1][p])\n",
    "\n",
    "    def after_fit(self):\n",
    "        if hasattr(self.learn, 'recorder') and hasattr(self, 'hps'): self.recorder.hps = self.hps\n",
    "\n",
    "    _docs = {\"before_fit\": \"Initialize container for hyper-parameters\",\n",
    "             \"before_batch\": \"Set the proper hyper-parameters in the optimizer\",\n",
    "             \"after_batch\": \"Record hyper-parameters of this batch\",\n",
    "             \"after_fit\": \"Save the hyper-parameters in the recorder if there is one\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "edfb1f76",
   "metadata": {},
   "source": [
    "`scheds` is a dictionary with one key for each hyper-parameter you want to schedule, with either a scheduler or a list of schedulers as values (in the second case, the list must have the same length as the the number of parameters groups of the optimizer)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d46d60c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>11.929138</td>\n",
       "      <td>4.039281</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn = synth_learner()\n",
    "sched = {'lr': SchedLin(1e-3, 1e-2)}\n",
    "learn.fit(1, cbs=ParamScheduler(sched))\n",
    "n = len(learn.dls.train)\n",
    "test_close(learn.recorder.hps['lr'], [1e-3 + (1e-2-1e-3) * i/n for i in range(n)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88a363c1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>8.085711</td>\n",
       "      <td>5.084560</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#| hide\n",
    "#test discriminative lrs\n",
    "def _splitter(m): return [[m.a], [m.b]]\n",
    "learn = synth_learner(splitter=_splitter)\n",
    "sched = {'lr': combined_cos(0.5, np.array([1e-4,1e-3]), np.array([1e-3,1e-2]), np.array([1e-5,1e-4]))}\n",
    "learn.fit(1, cbs=ParamScheduler(sched))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c698449",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"ParamScheduler.before_fit\" class=\"doc_header\"><code>ParamScheduler.before_fit</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>ParamScheduler.before_fit</code>()\n",
       "\n",
       "Initialize container for hyper-parameters"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(ParamScheduler.before_fit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b25aa5c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"ParamScheduler.before_batch\" class=\"doc_header\"><code>ParamScheduler.before_batch</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>ParamScheduler.before_batch</code>()\n",
       "\n",
       "Set the proper hyper-parameters in the optimizer"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(ParamScheduler.before_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e13615c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"ParamScheduler.after_batch\" class=\"doc_header\"><code>ParamScheduler.after_batch</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>ParamScheduler.after_batch</code>()\n",
       "\n",
       "Record hyper-parameters of this batch"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(ParamScheduler.after_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b262cc2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"ParamScheduler.after_fit\" class=\"doc_header\"><code>ParamScheduler.after_fit</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>ParamScheduler.after_fit</code>()\n",
       "\n",
       "Save the hyper-parameters in the recorder if there is one"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(ParamScheduler.after_fit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e18e1f8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "@patch\n",
    "def fit_one_cycle(self:Learner, n_epoch, lr_max=None, div=25., div_final=1e5, pct_start=0.25, wd=None,\n",
    "                  moms=None, cbs=None, reset_opt=False, start_epoch=0):\n",
    "    \"Fit `self.model` for `n_epoch` using the 1cycle policy.\"\n",
    "    if self.opt is None: self.create_opt()\n",
    "    self.opt.set_hyper('lr', self.lr if lr_max is None else lr_max)\n",
    "    lr_max = np.array([h['lr'] for h in self.opt.hypers])\n",
    "    scheds = {'lr': combined_cos(pct_start, lr_max/div, lr_max, lr_max/div_final),\n",
    "              'mom': combined_cos(pct_start, *(self.moms if moms is None else moms))}\n",
    "    self.fit(n_epoch, cbs=ParamScheduler(scheds)+L(cbs), reset_opt=reset_opt, wd=wd, start_epoch=start_epoch)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7453dd0a",
   "metadata": {},
   "source": [
    "The 1cycle policy was introduced by Leslie N. Smith et al. in [Super-Convergence: Very Fast Training of Neural Networks Using Large Learning Rates](https://arxiv.org/abs/1708.07120). It schedules the learning rate with a cosine annealing from `lr_max/div` to `lr_max` then `lr_max/div_final` (pass an array to `lr_max` if you want to use differential learning rates) and the momentum with cosine annealing according to the values in `moms`. The first phase takes `pct_start` of the training. You can optionally pass additional `cbs` and `reset_opt`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a056ad57",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>19.444899</td>\n",
       "      <td>6.755066</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>9.919473</td>\n",
       "      <td>1.044571</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Integration test: training a few epochs should make the model better\n",
    "learn = synth_learner(lr=1e-2)\n",
    "xb,yb = learn.dls.one_batch()\n",
    "init_loss = learn.loss_func(learn.model(xb), yb)\n",
    "learn.fit_one_cycle(2)\n",
    "xb,yb = learn.dls.one_batch()\n",
    "final_loss = learn.loss_func(learn.model(xb), yb)\n",
    "assert final_loss < init_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24274e32",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Scheduler test\n",
    "lrs,moms = learn.recorder.hps['lr'],learn.recorder.hps['mom']\n",
    "test_close(lrs,  [combined_cos(0.25,1e-2/25,1e-2,1e-7)(i/20) for i in range(20)])\n",
    "test_close(moms, [combined_cos(0.25,0.95,0.85,0.95)(i/20) for i in range(20)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7f5140e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "@patch\n",
    "def plot_sched(self:Recorder, keys=None, figsize=None):\n",
    "    keys = self.hps.keys() if keys is None else L(keys)\n",
    "    rows,cols = (len(keys)+1)//2, min(2, len(keys))\n",
    "    figsize = figsize or (6*cols,4*rows)\n",
    "    _, axs = plt.subplots(rows, cols, figsize=figsize)\n",
    "    axs = axs.flatten() if len(keys) > 1 else L(axs)\n",
    "    for p,ax in zip(keys, axs):\n",
    "        ax.plot(self.hps[p])\n",
    "        ax.set_ylabel(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2363d6ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>17.738113</td>\n",
       "      <td>21.716423</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#| hide\n",
    "#test discriminative lrs\n",
    "def _splitter(m): return [[m.a], [m.b]]\n",
    "learn = synth_learner(splitter=_splitter)\n",
    "learn.fit_one_cycle(1, lr_max=slice(1e-3,1e-2))\n",
    "#n = len(learn.dls.train)\n",
    "#est_close(learn.recorder.hps['lr'], [1e-3 + (1e-2-1e-3) * i/n for i in range(n)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9e1036c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>5.406837</td>\n",
       "      <td>5.305011</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>5.058437</td>\n",
       "      <td>4.899223</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn = synth_learner()\n",
    "learn.fit_one_cycle(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66eb747b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuQAAAD4CAYAAACkL/OsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABUXklEQVR4nO3dd3xUVf7/8dcnvUF6Qg0JEAihQwigYAFUVFzUtYCwIhbEte7qrrru6n636u7q2sWGYkVsKyoWwArSAoSaACEJEEo6IYX08/sjg78YQxJCZu5k5vN8POaRmXvvmXnPTLh8cu+554gxBqWUUkoppZQ1PKwOoJRSSimllDvTglwppZRSSikLaUGulFJKKaWUhbQgV0oppZRSykJakCullFJKKWUhL6sDWCkiIsLExsZaHUMppdpl48aNBcaYSKtzOJLut5VSnVVL+2y3LshjY2NJSUmxOoZSSrWLiOyzOoOj6X5bKdVZtbTP1i4rSimllFJKWUgLcqWUUkoppSykBblSSimllFIW0oJcKaWUUkopC2lBrpRSSimllIXsWpCLyFQR2SUiGSJyXzPrRUSetK3fKiKjWmsrIleKyA4RqReRpCbPd79t+10icoE935tSSimllFIdwW4FuYh4As8AFwKJwEwRSWyy2YVAvO02D3iuDW23A5cD3zV5vURgBjAYmAo8a3sepZRSSimlnJY9j5AnAxnGmExjTDWwGJjeZJvpwGumwVogRES6t9TWGJNmjNnVzOtNBxYbY6qMMVlAhu15lINtyC7i5VVZfJ2eR3ZBObV19VZHUkq5qSMllfzl4526H1JKOTV7TgzUEzjQ6HEOMLYN2/RsY9vmXm9tM8/1EyIyj4aj8cTExLTylOpUbdpfzDUvrqWmzvy4zNtTiAkLIC4iiH6RgcRFBNI3Moi4iEAignwQEQsTK6Vc2frsIhauzsLbU7j/okFWx1FKqWbZsyBvrsoybdymLW3b83oYY14AXgBISkpq7TnVKcgrreSWNzbSLdiP164fS2FZFZkF5WTml5NVUEZWQTnf7c6nutGRqi5+XvRtVKD3jwrinIGRBPi49SSySqkO8ovhPVifVcjz32UyMiaUqUO6WR1JKdWJ7cktJbuwgvMSozv0ee1Z9eQAvRs97gUcauM2Pm1o257XU3ZSXVvPrW9uouR4DR/cciZxEQ1HwpNiw36yXV294dDR47ZCvaFIzyooZ31WER9uPghAsL83s8bGcO34WLoF+1nxdpRSLuRP0xLZllPC797dwsBuXYiLCLQ6klKqE6mvN3y7J5+Fq7L4fk8BPYL9mJQQhadHx53ht2dBvgGIF5E44CANF1xe02SbpcBtIrKYhi4pJcaYwyKS34a2TS0F3hKRx4AeNFwour7D3o1q0d8/3cmG7GKemDGCxB5dT7qdp4fQOyyA3mEBnD0g8ifrjlfXsSXnKIt+yGbBt3t54btMLhnegxsmxDGkZ7C934JSykX5ennyzKxRTHtqFbe8sZEPf30m/j56zb9SqmUV1bV8sOkgr6zOYm9+OVFdfLnn/AFcM7ZPhxbjYMeC3BhTKyK3AV8AnsBCY8wOEZlvW78AWAZcRMMFmBXA3JbaAojIZcBTQCTwqYikGmMusD33EmAnUAvcaoyps9f7U//fextzWLRmHzdOiGP6iJ91228zfx9PxvUNZ1zfcA4UVfDK6mze2bCfDzcfJDkujBsnxDF5UHSH/yNQSrm+XqEBPH71COa+uoEH/reNR68crtevKKWadbjkOIt+2Mfb6/dTcryGoT2DefzqEVw0tDs+XvYZD0WMcd9u1ElJSSYlJcXqGJ3atpwSfrngB5L6hPLa9cl4eXbsL+qxyhreWX+AV3/I5uDR4/QJD+D6M+O4YnQvAn21n7lybyKy0RiT1PqWruN099v/Xb6bJ1bu4R+XDeWasXphv1Lq/9u8v5iFq7NZtu0wxhguGNyN6yfEkdQntEP+gG9pn60VjWq3wrIqbn49hcggX56aObLDi3GArn7e3HRWX+aeGcvnO47w8qosHlq6g0e/3MXMsTHMGR9LjxD/Dn9dpZRrumNyPJsPHOXPS3cwpGdXhvUKsTqSUspCtXX1P9YXm/cfpYuvF9efGcu142PpHRbgsBx6hFyPkLdLbV09v3p5PRv3F/P+/DMY2stxfbw37itm4aosPtt+GBHh4qHduWFCHMN7hzgsg1LOQI+Qt09ReTXTnvweEeHTOyYQEuDTQemUUp1FSUUNb2/Yz2s/ZHOopJI+4QHMPSOWK5J6E2SnM/B6hFx1uIc/S2dNZiGPXjncocU4wOg+oYzuE8qBogoW/ZDN4g0HWLrlEBPjI/jXFcPoHqxHzJVSJxcW6MOzs0dz5YIfuOudVBbOGYOHXpuilFswxvDmuv38Y1kaFdV1nNEvnL9MH8K5HTxqyqmy50ydykV9lHqQl1ZlMWd8H345updlOXqHBfDHaYmsuX8SD1w0iI37ipn6+Pd8vv2wZZmUUp3DiN4hPDgtkW925fPM1xlWx1FKOUBBWRU3vZbCH/+3ndF9Qll2x0TeumkcUxKtHzBCC3J1SnYeOsa9728lOTaMP05LtDoOAF1s/cw/vWMifcIDmP/GJu7/YCsV1bVWR1NKObHZ4/pw6YgePLZiN9/vybc6jlLKjr7elcfUx7/nuz0FPDgtkUVzk1scptnRtCBXbXa0opqb30gh2N+bp2eNxNsOF3GejriIQN6bfwa3nNOPxRsOMO2pVWw/WGJ1LKWUkxIR/nH5UOKjgrhzcSqHjh63OpJSqoNV1tTx56U7mPvKBsIDfVh625lcPyHO6bqpOVdFpZxWXb3h9rc3k1tSxXOzRxPVxTln0PTx8uDeqQm8eeNYKqrquOzZ1bz4XSb19e578bJS6uQCfLx4bvZoqmrq+PWbm6iurbc6klKqg6QdPsYvnl7Fqz9kM/fMWD667UwSujnPUfHGtCBXbfLol7v4fk8B/zd9MKNiQq2O06oz+kXw2Z0TmZQQxd+XpTHnlfXkHau0OpZSygn1iwzi31cOJ/XAUf6xLM3qOEqp01Rfb3jp+0ymP72a4ooaFl2fzEOXDMbP23ln6NWCXLXqs22HefabvcxMjmFmcueZSCM00IcFs0fzj8uGsiG7iKlPfM+KnblWx1JKOaGLbMOnvvpDNku3HLI6jlKqnXKPVTLnlfX87dM0zhoQyed3TuTsAZFWx2qVFuSqRbtzS7n73S2MjAnhz79wjos4T4WIcM3YGD65fSLduvpx42sp/Ol/26msqbM6mlLKydx3YQJJfUK57/2t7MkttTqOUuoUfb79CFMf/44N2UX8/bIhvHjtaMKDfK2O1SZakKuTKjlew82vbyTAx4sFs0fj6+W8p3pa0z8qiA9vPYObJsbx+tp9XPLUKtIOH7M6llLKiXh7evD0NaMI8PFk/hsbKavSkZqU6gwqqmu5/4OtzH9jIz1D/fnk9onMGtunQ6a7dxQtyFWz6usNv30nlQNFFTw3exTRXZ3zIs5T4evlyQMXJ/La9ckcPV7D9GdW88rqLNx5tlqlTpeITBWRXSKSISL3NbM+VEQ+FJGtIrJeRIY0We8pIptF5BPHpT65bsF+PDlzJFkF5dz7/lbdPyjl5LbmHGXak6tYvOEAt5zTjw9uOZP+UUFWxzplWpCrZn267TAr0/P407RExsSGWR2nQ53oUzaxfwT/9/FO5r66gaLyaqtjKdXpiIgn8AxwIZAIzBSRpn3b/gCkGmOGAdcCTzRZfyfgVFdSntEvgnsuGMinWw/z+tp9VsdRSp3ES99ncvmzP1BZU8dbN47j3qkJ+Hh1ztK2c6ZWdvfFjiNEdvHlV+P6WB3FLsKDfHlpThJ/mT6YH/YWcvXza3QUFqVOXTKQYYzJNMZUA4uB6U22SQRWAhhj0oFYEYkGEJFewMXAS46L3Dbzz+rHmNhQFq7KsjqKUqoJYwyPfJ7O3z5NY8qgaD678yzG9wu3OtZp0YJc/Ux1bT3f7spnckKU0w2c35FEhGvHx7JobjIHjx7n6hfW6sQgSp2ansCBRo9zbMsa2wJcDiAiyUAfoJdt3ePA74EWB/8WkXkikiIiKfn5jplR08NDmDasB9mFFWTmlznkNZVSrauvNzy0dAfPfbOXWWNjeHbWKIIDvK2Oddq0IFc/sz6riNKqWqYMirY6ikOM7xfO6zeMpaCsiisXrGFfYbnVkZTqLJr7i71pp+uHgVARSQVuBzYDtSIyDcgzxmxs7UWMMS8YY5KMMUmRkY4bvmxSQhQAX6XnOew1lVInV1tXz+/f38pra/Yx76y+/O3SIS5z4FALcvUzK9Jy8fP24Mz+EVZHcZjRfUJ5+6ZxVFTXctXza8jI0yNiSrVBDtC70eNewE8G8TbGHDPGzDXGjKChD3kkkAWcCfxCRLJp6OoySUTecETotuodFkB8VJAW5Eo5geraeu5YvJn3Nubw2/MGcP+FCZ1qFJXWaEGufsIYw4q0XCb0j8Dfp/MOc9geQ3oGs3jeeOrq4ern17DzkA6LqFQrNgDxIhInIj7ADGBp4w1EJMS2DuBG4DtbkX6/MaaXMSbW1u4rY8xsR4Zvi0mDohrOGlbWWB1FKbdVWVPHvNdTWLbtCH+8eBB3TI53qWIctCBXTezKLSWn+LjbdFdpamC3Liy5eRw+Xh7MfHEtWw4ctTqSUk7LGFML3AZ8QcNIKUuMMTtEZL6IzLdtNgjYISLpNIzGcqc1adtn0sAoausN3+8psDqKUm6prKqW615Zz7e78/nn5UO5cWJfqyPZhRbk6idOTC0/aVCUxUms0zcyiCU3j6ervxezXlrHhuwiqyMp5bSMMcuMMQOMMf2MMX+3LVtgjFlgu7/GGBNvjEkwxlxujClu5jm+McZMc3T2thjdJ5Sufl7abUUpC5RU1DD7pXVsyC7m8atHMDM5xupIdqMFufqJFWl5jOgdQlSXzj8R0OnoHRbAkpvHE9XFl2tfXs8qPTqmlFvy8vTg7IFRfLMrj/p6nSRIKUfJL63i6hcauo8+N2sU00c0HcDJtWhBrn6UV1pJ6oGjTHHjo+ONdQ/2552bx9MnPIDrF23gq/RcqyMppSwwKSGSgrJqth4ssTqKUm7h0NHjXP38GvYVVvDydUmcP7ib1ZHsTgty9aOv0hpOyU5JdM/+482J7OLL2zeNY2B0F25+fSOfbTtsdSSllIOdPSAKD9HhD5VyhH2F5Vy5YA35pVW8dkMyE+MdN9SplbQgVz9akZZLr1B/BkZ3sTqKUwkN9OHNm8YyrFcIt761iQ8351gdSSnlQGGBPoyMCdWzZErZ2Z7cUq5csIaK6lreumkcY2LDrI7kMFqQKwCOV9exKqOAKYOiXW4ooY7Q1c+b165PZmxcOL9dsoW31u23OpJSyoEmJUSx/eAxco9VWh1FKZe0/WAJVz2/BgO8c/N4hvYKtjqSQ2lBrgBYnVFAZU292w532BaBvl68MncMZw+I5A8fbmPhqiyrIymlHOTErJ1fa7cVpTpcSnYRM19YS4CPF+/ePJ4BbnimXgtyBTR0V+ni60VynPucHmoPP29Pnv/VaC4YHM1fPtnJs99kWB1JKeUACd260CPYT/uRK9XBNmQX8auX1xPZxZd3548nNiLQ6kiW0IJcUV9vWJmex9kDI/Hx0l+J1vh6efLMNaP4xfAe/OvzXSzZcMDqSEopOxMRzk2IYlVGAVW1dVbHUcol7M0v48ZFKXQP9uOdm8fTI8Tf6kiW0epLsfVgCfmlVdpd5RR4eXrw6FXDmRgfwR8+3KbjlCvlBiYPiqKiuo51mTpZmFKnq6CsiuteWY+Xh/Dq3GQiu/haHclSWpArVuzMxdNDOGegewwt1FG8PT14ZtYo+kUGccsbG9mdW2p1JKWUHY3vG4Gvl4d2W1HqNB2vruOGRSnkl1bx8nVjiAkPsDqS5bQgV6xIy2VMbCghAT5WR+l0uvp5s3DuGPx8PJn7ygbySnUEBqVclb+PJ2f0C+er9DyM0Vk7lWqPunrDHYs3szXnKE/OGMmI3iFWR3IKdi3IRWSqiOwSkQwRua+Z9SIiT9rWbxWRUa21FZEwEVkuIntsP0Nty71FZJGIbBORNBG5357vzVUcKKog/Uipdlc5DT1D/Fk4ZwxF5dXctCiF49Xav1QpVzVpUDT7iyrYm19udRSlOqW/frKT5TtzeXBaolvMwNlWdivIRcQTeAa4EEgEZopIYpPNLgTibbd5wHNtaHsfsNIYEw+stD0GuBLwNcYMBUYDN4tIrH3enetYmdYw0YUW5KdnaK9gnpw5kq0HS7hz8Wbq6vXomVKu6MTwhzpJkFKn7uVVWbz6QzbXnxnH3DPjrI7jVOx5hDwZyDDGZBpjqoHFwPQm20wHXjMN1gIhItK9lbbTgUW2+4uAS233DRAoIl6AP1ANHLPPW3MdK9Ly6B8V5LbDDHWk8xKj+dPFiXy5M5d/LkuzOo5Syg56hviT0K2L9iNX6hR9vv0wf/t0JxcMjuaBiwdZHcfp2LMg7wk0Hg8ux7asLdu01DbaGHMYwPYzyrb8PaAcOAzsB/5jjPnZpfAiMk9EUkQkJT8/vz3vy2Ucq6xhbWahHh3vQNdPiOO6M2J5aVUWr6/JtjqOUsoOzk2IYkN2MSXHa6yOolSnsGl/MXcuTmV4rxAev3oknh46I3hT9izIm/u0m57HP9k2bWnbVDJQB/QA4oC7RaTvz57EmBeMMUnGmKTISPceVeS73fnU1humDIpqfWPVZn+alsiUQVE8tHSHzuqnlAuanBBFXb3h+z3ufVBHqbbYV1jOjYtSiO7qx0tzkvD38bQ6klOyZ0GeA/Ru9LgXcKiN27TUNtfWrQXbzxMVzzXA58aYGmNMHrAaSOqA9+GyVuzMJSzQh5ExoVZHcSmeHsITM0YyqHtXbntrEzsOlVgdSSnVgUbGhBIS4K3dVpRqRXF5Nde9soF6Y3h17hgigtx7rPGW2LMg3wDEi0iciPgAM4ClTbZZClxrG21lHFBi64bSUtulwBzb/TnAR7b7+4FJtucKBMYB6fZ6c51dTV09X6XnMSkhSk8d2UGgrxcLrxtDsL8317+6gcMlx62OpJTqIJ4ewjkDIvlmV75ewK3USVTW1HHTaykcPHqcl65Nom9kkNWRnJrdCnJjTC1wG/AFkAYsMcbsEJH5IjLfttkyIBPIAF4Eft1SW1ubh4HzRGQPcJ7tMTSMyhIEbKehoH/FGLPVXu+vs0vJLuZYZa32H7ej6K5+vHzdGMqr6rj+1RTKqmqtjqSU6iDnJkRRVF7NlpyjVkdRyunU1xvufncLKfuK+e9VI0iKDbM6ktPzsueTG2OW0VB0N162oNF9A9za1ra25YXA5GaWl9Ew9KFqg5Vpufh4ejAxPsLqKC5tUPeuPDNrFNe/uoHb3trES9cm4eWp83Ep1dmdPSASTw/hq7Q8Rmm3P6V+4pEv0vl062HuvzCBi4d1tzpOp6CVgRsyxrA8LZcz+ocT6GvXv8kUDf9x/3X6EL7Zlc+fP96hM/wp5QJCAnwYHROq/ciVauKNtft4/ttMZo+LYd5ZPxtbQ52EFuRuaG9+GfsKK7S7igNdMzaGm8/uyxtr9/Pyqiyr4yilOsC5CVHsPHxMrxFRyuar9Fwe/Gg7kxKi+PMlgxHRa9TaSgtyN7QireGIzmQd7tCh7r0ggYuGduPvy9L4fPsRq+MopU7TiX3o1+k6/KFS23JKuO2tzST26MpTM0dq98xTpJ+WG1qxM5chPbvSPdjf6ihuxcNDeOyqEYzoHcJd72xmy4GjVkdSSp2G+Kggeob4a7cV5fbyS6u48bUNhAb4sHDOGO0O2w5akLuZwrIqNu4v1u4qFvHz9uTFa5OICPLlljc2UlxebXUkpVQ7iQiTB0WxOqOAypo6q+MoZYnaunpuf3sTJcdreGlOElFd/ayO1ClpQe5mvkrPwxi0ILdQRJAvz80aTUFZNb9Zkkq9jmOsVKd1bkIUx2vqWJtZaHUUpSzx6PLdrM0s4u+XDmVQ965Wx+m0tCB3MyvT8uge7MfgHvqPxkpDewXz4CWJfLMrn2e/ybA6jlKqncb3Dcff21O7rSi3tHxnLs99s5drxsbwy9G9rI7TqWlB7kYqa+r4bk8+kwdF6ZXPTmDW2Bimj+jBY8t380NGgdVxlFLt4OftyZn9w21nH/Vsl3If+wrL+e2SVIb2DObBaYlWx+n0tCB3I2syC6mortPuKk5CRPjHZUPpGxnEHYs3k3us0upISql2mJQQTU7xcfbklVkdRSmHqKypY/4bm/AQ4dlZo/Dz9rQ6UqenBbkbWbEzl0AfT8b3C7c6irIJ9PXiuVmjKK+q4/a3N1NbV291JKXUKTo3IRJAu60ot/HgR9tJO3yMx68eQe+wAKvjuAQtyN2EMYaVaXlMjI/E10v/knUm8dFd+OflQ1mfVcR/vtxtdRylTomITBWRXSKSISL3NbM+VEQ+FJGtIrJeRIbYlvcWka9FJE1EdojInY5P3zG6B/szqHtXLciVW3hnw36WpORwx6T+nJug85l0FC3I3cSOQ8c4cqySKYnaXcUZXTqyJ9eMjWHBt3tZvjPX6jhKtYmIeALPABcCicBMEWnamfQPQKoxZhhwLfCEbXktcLcxZhAwDri1mbadxuSEKDbuK6akosbqKErZzfaDJfzpox1MjI/gzikDrI7jUrQgdxPLd+biIXDuwEiro6iTeHBaIkN6duXuJakcKKqwOo5SbZEMZBhjMo0x1cBiYHqTbRKBlQDGmHQgVkSijTGHjTGbbMtLgTSgp+Oid6xzE6Koqzd8u0dn7VSuqaSihlve3Eh4oA+PXz0CTw8dHKIjaUHuJlak5TIqJpTwIF+ro6iT8PP25NlrRmOAW9/aRFWtTjSinF5P4ECjxzn8vKjeAlwOICLJQB/gJ+OjiUgsMBJYZ6+g9jaidwhhgT58laZnuJTrqa833P1uKkdKKnlm1iitJexAC3I3cLjkODsOHdPuKp1ATHgAj145nK05JfztkzSr4yjVmuYOkTUd++9hIFREUoHbgc00dFdpeAKRIOB94C5jzLFmX0RknoikiEhKfr5zHoH29BDOGRDJN7vzqdPJvpSLee7bvaxIy+OPFycyKibU6jguSQtyN7AireFCIx3usHM4f3A35p3Vl9fX7uOj1INWx1GqJTlA70aPewGHGm9gjDlmjJlrjBlBQx/ySCALQES8aSjG3zTGfHCyFzHGvGCMSTLGJEVGOm+3u0mDojhaUcPm/cVWR1Gqw/yQUcCjX+7ikuE9uHZ8H6vjuCwtyN3Aip25xEUE0i8y0Oooqo1+d8FAxsSGcv8H28jIK7U6jlInswGIF5E4EfEBZgBLG28gIiG2dQA3At8ZY45Jw+xkLwNpxpjHHJraTibGR+LpITrainIZR0oquf3tzfSNDOLhy4fqpIJ2pAW5iyuvqmXN3kImJ+jsnJ2Jt6cHT80chb+3J7e8sYmK6trWGynlYMaYWuA24AsaLspcYozZISLzRWS+bbNBwA4RSadhNJYTwxueCfwKmCQiqbbbRQ5+Cx0q2N+bMbGhWpArl1BTV8+tb23ieE0dC2aPItDXy+pILk0Lchf3/Z58quvqtf94J9Qt2I8nZowkI7+MBz7crtNyK6dkjFlmjBlgjOlnjPm7bdkCY8wC2/01xph4Y0yCMeZyY0yxbfkqY4wYY4YZY0bYbsusfC8dYVJCFOlHSjl49LjVUZQ6Lf9cls7GfcU88sth9I/qYnUcl6cFuYtbvjOPYH9vkvroRRid0YT4CO6aPIAPNx/k7fUHWm+glLLUJNtEKV/rUXLViX269TALV2dx3RmxXDK8h9Vx3IIW5C7MGMM3u/I4Z2AkXp76VXdWt0/qz1kDIvnzxzvYfrDE6jhKqRb0iwwiJixAu62oTisjr4zfv7eFUTEh/OGiQVbHcRtapbmwjLwyCsurObNfhNVR1Gnw8BAev3oE4YE+3PLmRkqO60yASjkrEWFSQhSrMwo4Xq1zCajOpbyqllve2IivtyfPzBqFj5eWiY6in7QLW5tVBMDYvmEWJ1GnKyzQh6evGcXho5Xc8+4W7U+ulBM7NyGKqtp61mUVWh1FqVPy0NIdZOSX8eSMkXQP9rc6jlvRgtyFrcsspFtXP2LCAqyOojrA6D6h3HdhAst35vL62n1Wx1FKncSY2FC8PIT1toMiSnUGS7cc4r2NOdx+bn8mxOuZdUfTgtxFGWNYl1XE2L5hOtyhC7lhQhznDIzk75+msTtXxydXyhkF+HgxuGcwG7K1IFedw4GiCh74YBujYkK4Y3K81XHckhbkLiqroJz80irGxoVbHUV1IBHh31cMJ8jXizve3kxljfZRVcoZJceGsuVAif4bVU6vtq6eOxdvBuCJGSN1EAiL6KfuotZp/3GXFdnFl39fOYz0I6X8+4tdVsdRSjVjTGwY1XX1bM3RkZGUc3vyqww27T/K3y8fSm/t4moZLchd1LrMQiKCfOkbEWh1FGUHkxKimTO+Dy+vyuK73flWx1FKNTEmtuFgiHZbUc5sfVYRT3+1h1+O6sUvdLxxS2lB7oK0/7h7uP+iQQyIDuLud7dQWFZldRylVCOhgT7ERwXphZ3KaZVU1HDX4s3EhAXwf9MHWx3H7WlB7oJyio9zuKSSsXHaXcWV+Xl78sSMkZRU1HDv+1t1KESlnMyYuDA27Sumrl7/bSrnYozh/g+3kldaxRMzRhLk62V1JLenBbkLWpvZMPatXtDp+gZ178q9FyawIi2PN9fttzqOUqqR5NgwSqtqSTt8zOooSv3EkpQDLNt2hHsuGMjw3iFWx1HYuSAXkakisktEMkTkvmbWi4g8aVu/VURGtdZWRMJEZLmI7LH9DG20bpiIrBGRHSKyTUT87Pn+nNW6rCJCA7yJjwqyOopygLlnxDIxPoK/fbqTjDwdClEpZzEmTvuRK+eTkVfGn5fu5Mz+4cyb2NfqOMrGbgW5iHgCzwAXAonATBFJbLLZhUC87TYPeK4Nbe8DVhpj4oGVtseIiBfwBjDfGDMYOAdwyznG12UVkhwXhoeH9h93Bx4ewqNXDifAx4s73k6lqlaHWVPKGfQM8adniL8W5MppVNXWcefizfh5e/DYVSO0TnAi9jxCngxkGGMyjTHVwGJgepNtpgOvmQZrgRAR6d5K2+nAItv9RcCltvvnA1uNMVsAjDGFxhi3q0wOHT3OgaLj2l3FzUR19eORXw5j5+Fj/EeHQlTKaYyJDWV9VrFe46Gcwr8/38WOQ8f41xXDie7qlp0InJY9C/KewIFGj3Nsy9qyTUtto40xhwFsP6NsywcARkS+EJFNIvL75kKJyDwRSRGRlPx81xsubl2Wrf+4jj/uds5LjGb2uBhe/D6LVXsKrI6jlKKh20pBWRXZhRVWR1Fu7tvd+by0KotfjevDeYnRVsdRTdizIG/uPEjTQwQn26YtbZvyAiYAs2w/LxORyT97EmNeMMYkGWOSIiMjW3nKzmddZhFd/bxI6NbV6ijKAg9clEj/qCB+uySVovJqq+Mo5faST4xHrsMfKgsVlFVx95ItDIgO4oGLB1kdRzXDngV5DtC70eNewKE2btNS21xbtxZsP/MaPde3xpgCY0wFsAwYhZtZl1VEclwYntovzC35+3jyxIwRFFdU61CISjmB/lFBhAZ4s177kSuLGGO4590tHKus4cmZI/Hz9rQ6kmqGPQvyDUC8iMSJiA8wA1jaZJulwLW20VbGASW2bigttV0KzLHdnwN8ZLv/BTBMRAJsF3ieDey015tzRnnHKskqKNf+425ucI9gfn9BAst35vL2+gOtN1BK2Y2IkBQbphd2Ksu8sjqbb3bl88eLB+nZcydmt4LcGFML3EZDoZwGLDHG7BCR+SIy37bZMiATyABeBH7dUltbm4eB80RkD3Ce7THGmGLgMRqK+VRgkzHmU3u9P2e0znZKNFknBHJ7N0yIY0L/CP7yyQ4y8sqsjqOUW0uODWNfYQV5xyqtjqLczM5Dx3j4s3SmDIriV+P6WB1HtcCuUzMZY5bRUHQ3Xrag0X0D3NrWtrblhcDP+obb1r1Bw9CHbmldViFBvl4M7qF/Abs7Dw/h0auGc8Hj33HXO5v54JYz8fHSecCUssKJ8cjXZxcxbVgPi9Mod3G8uo7b395ESIA3/7piOCLaldWZ6f/QLmRdZhGj+4Ti5alfq4Jo21CI2w8e49HlOhSiUlYZ3KMr/t6eemGncqi/frqTzIJyHrtqBGGBPlbHUa3Qys1FFJZVsSevTIc7VD9xweBuzEyO4YXvMvkhQ4dCVMoK3p4ejOoTwvrsYqujKDfx+fbDvLVuP/PO6suE+Air46g20ILcRay3HXnRCzpVU3+aNoi4iEB+u2QLxToUolKWSI4NJ/3IMUqOu+UE0sqBDpcc5973tzGsVzB3nzfQ6jiqjbQgdxHrsorw9/ZkWK9gq6MoJxPg48WTM0ZSWF7FHz7cpkMhKmWBMXGhGAOb9ulRcmU/9fUNQxxW19bz+NUj9NqhTkS/KRexNrOQ0X1C8db+46oZQ3oG89vzBvLZ9iN8sOmg1XGUExIRTxH5hYjcISK/PXGzOperGNk7FG9P0fHIlV0tXJ3F6oxCHrwkkb6RQVbHUadAqzcXcLSiml25pYzV4Q5VC+ad1Zfk2DAeWrqDA0U6jbf6mY+B64BwoEujm+oA/j6eDOkZrBd2KrtJO3yMf32+i/MSo5kxpnfrDZRT0YLcBWzILsYYHX9ctczTNhQiwN1LtlBXr11X1E/0MsZcbox5yBjzfyduVodyJcmxYWzNKaGyps7qKMrFVNbUcdfiVLr6e/Pw5UN1iMNOSAtyF7AusxAfLw+G9w6xOopycr3DAnjokkTWZxfx0veZVsdRzuUzETnf6hCubExsGNV19Ww5cNTqKMrF/OvzXezKLeXfVw4jPMjX6jiqHbQgdwHrsooY2TsEP29Pq6OoTuCK0b2YOrgb//lyFzsPHbM6jnIea4EPReS4iBwTkVIR0V+QDpQUGwrABu1HrjrQ93vyWbg6i2vH9+HcgVFWx1HtpAV5J3essoYdh0oY21eHO1RtIyL84/KhhAT48Jt3UvX0uTrhUWA8EGCM6WqM6WKM0Wl/O1BIgA8Do7voeOSqwxSXV3PPu1voHxXE/RcOsjqOOg1akHdyG7OLqTcwTvuPq1MQFujDv64Yxq7cUv7zhc7iqQDYA2w3pzgupohMFZFdIpIhIvc1sz5URD4Uka0isl5EhrS1rSsaExfKpn3Feg2HOm3GGP7w4TaKyqt5/OoR+PvoWfLOTAvyTm5tViHensLImFCro6hO5tyBUcweF8NLq7J0Fk8FcBj4RkTub+uwhyLiCTwDXAgkAjNFJLHJZn8AUo0xw4BrgSdOoa3LGRMbRllVLWmHtTeQOj3vbczhs+1HuPv8gQzpqXOQdHatFuQi4iEi2x0RRp26dZlFDO8Von8Zq3b5w0WD6BsRyD3vbtEZBFUWsBLwoe3DHiYDGcaYTGNMNbAYmN5km0Tb82KMSQdiRSS6jW1dzonRsNbr8IfqNOwrLOfPS3cwNi6Mmyb2tTqO6gCtFuTGmHpgi4jEOCCPOgXlVbVsP1jC2L7aXUW1T4CPF49dPYLc0ioe+kj/7nZnjYY5fAx4tI3DHvYEDjR6nGNb1tgW4HIAEUkG+gC92tgWW7t5IpIiIin5+fltfUtOqXuwP71C/fXCTtVutXX1/OadVDw8hMeuHoGnhw5x6Ara2mWlO7BDRFaKyNITN3sGU63btL+Y2npDcpxe0Knab0TvEG6f1J//pR7i4y2HrI6jLCIiQ0RkM7Cdhv39RhEZ3FqzZpY17Rz9MBAqIqnA7cBmoLaNbRsWGvOCMSbJGJMUGRnZSiTnlxwbxobsIk6xu75SADz7zV427T/K3y4dQs8Qf6vjqA7i1cbtdHIIJ7QuswhPD2F0H+0/rk7Pbef25+td+Tzw4TbGxIbRLdjP6kjK8V4AfmuM+RpARM4BXgTOaKFNDtB4SsBewE/+qjPGHAPm2p5TaOgakwUEtNbWVY2JC+ODzQfJKijX6c3VKdm8v5gnVu5h+ogeTB/R7Akl1Um16Qi5Mebb5m72Dqdati6rkCE9gwnybevfVUo1z8vTg8evHkFNneGed7dQryNAuKPAE8U4gDHmGyCwlTYbgHgRiRMRH2AG8JOzpyISYlsHcCPwna1Ib7WtqxoT29DNULutqFNRXlXLb95JpVtXP/4yfUjrDVSn0mJBfmJiiGZuOmGExSpr6thyoESHO1QdJi4ikD9OG8SqjAIWrcm2Oo5yvEwR+ZOIxNpuf6ThSPZJGWNqgduAL4A0YIkxZoeIzBeR+bbNBtHQBSadhhFV7myprV3emZPpFxlIeKAP6/TCTnUK/vrJTvYVVfDoVcMJ9ve2Oo7qYC0eWjXGtHaFvbLIpv3FVNfV6wWdqkNdkxzDyrQ8Hv4snQn9I4iP1l2AG7mehu6J79PQv/s74LrWGhljlgHLmixb0Oj+GiC+rW3dgYiQFBuqR8hVm32x4wiLNxzglnP6MU4nAnRJOg55J7UuswgPgaRYLchVxxERHv7lUAJ9vbjrnVSqa+utjqQcpx8Nfbo9AG9gMg1FubKDMbFhHCg6zpGSSqujKCeXd6yS+97fyuAeXfnNlAFWx1F2ogV5J7Uuq5DEHl3p6qenrVTHiurixz8vH8qOQ8d4YuVuq+Mox3kTWEjDEIXTbLdLLE3kwn4cj1yPkqsWGGP43Xtbqaiu44kZI/Dx0rLNVek32wlV1daxef9Rxupwh8pOLhjcjauSevHcN3tJ0YLBXeQbYz42xmQZY/aduFkdylUldu9KoI8nG7QfuWrBa2v28e3ufP548SD6R2kXQlemBXkntDWnhKra+h+PsChlDw9eMpieof78ZkkqZVW1VsdR9veQiLwkIjNF5PITN6tDuSovTw9G9dF+5Ork9uSW8o9laZw7MJLZ4/pYHUfZmRbkndC6zEKgYXIJpewlyNeL/141goPFx/nLx24x+IW7mwuMAKbS0FXlEhq6rSg7GRMbxq7cUkoqaqyOopxMZU0ddyxOJdDXi39dMZyGIfyVK9MBrDuhdVlFJHTrQmigT+sbK3UakmLDmH92P579Zi+TEqKZOqSb1ZGU/Qw3xgy1OoQ7GRMbhjGQsq+IyYOirY6jnMgjn6eTdvgYC69LIrKLr9VxlAPoEfJOpqauno37ihmr3VWUg9w1ZQBDewZz3wdbdUQI17ZWRBKtDuFORsaE4O0pemGn+omv0/N4ZXU2150Ry6QE/UPNXWhB3slsO1hCRXUdY3UcUuUgPl4ePD5jBFU19dz9bqrO4um6JgCpIrJLRLaKyDYR2Wp1KFfm5+3J0J7BemGn+lFeaSX3vLuFhG5duO/CBKvjKAfSgryTWZfZsOPWCzqVI/WLDOLBSxJZnVHIS6syrY6j7GMqDRP4nM//7z+uwx7a2Zi4MLYdLKGyps7qKMpi9fWGu5dsoayqlqdmjsTP29PqSMqBtCDvZNZlFdI/KoiIIO1TphxrxpjeXDA4mn9/sYvtB0usjqM6WOOhDnXYQ8dJjg2jps6wef9Rq6Moiy1cncX3ewr407REnSXZDWlB3onU1tWTkq39x5U1RISHLx9GWKAPdy7ezPFqPaKn1OlK6hOGCDr8oZvbfrCERz5P5/zEaGaNjbE6jrKAFuSdSNrhUsqqarW7irJMaKAPj101gr355fzt051Wx1Gq0wsO8GZgdBctyN1YRXUtdyzeTFigD4/8cpgOceim7FqQi8hU2wVCGSJyXzPrRUSetK3fKiKjWmsrImEislxE9th+hjZ5zhgRKRORe+z53qywLqth/PFxekGnstCZ/SOYd1Zf3ly3ny93HLE6jlKd3pjYMDbtK6a2rt7qKMoCf/l4J1kF5fz36hE6nLEbs1tBLiKewDPAhUAiMLOZIbUupOEionhgHvBcG9reB6w0xsQDK22PG/sv8FmHvyEnsDaziNjwAKK7+lkdRbm5u88fwOAeXbn3/a3kHdOhEJU6HWPiwiivrmPn4WNWR1EOtmzbYRZvOMAtZ/fjjH4RVsdRFrLnEfJkIMMYk2mMqQYWA9ObbDMdeM00WAuEiEj3VtpOBxbZ7i8CLj3xZCJyKZAJuNy0gvX1hg3ZRYyN06Pjynq+Xp48MWMkx2vquPvdLToUolKn4cSsy+t1+EO3cvDoce57fyvDe4fwm/MGWB1HWcyeBXlP4ECjxzm2ZW3ZpqW20caYwwC2n1EAIhII3Av8X0uhRGSeiKSISEp+fv4pvSErpR8ppeR4DWP7av9x5Rz6RwXxp2mJfL+ngIWrs6yOo1Sn1S3Yj95h/tqP3I3U1Rt+sziVunrDkzNG4O2pl/S5O3v+BjR3VULTw2gn26YtbZv6P+C/xpiyljYyxrxgjEkyxiRFRka28pTO40T/cZ0QSDmTa5JjOC8xmn99voudh/R0u1LtNSY2jJTsYozRs03u4JmvM1ifXcRfLx1Cn/BAq+MoJ2DPgjwH6N3ocS/gUBu3aaltrq1bC7afebblY4F/iUg2cBfwBxG57bTfhZNYl1lEr1B/eob4Wx1FqR+JCI/8chjBAd7cuXizTm6iVDuNjQujsLyavfnlVkdRdrZxXxFPrNzDpSN6cPmoXlbHUU7CngX5BiBeROJExAeYASxtss1S4FrbaCvjgBJbN5SW2i4F5tjuzwE+AjDGTDTGxBpjYoHHgX8YY56239tzHGMM67X/uHJSYYE+PHbVcPbklfGPZWlWx1GqUxpj60eu3VZc27HKGu54O5UeIX789dIhVsdRTsRuBbkxpha4DfgCSAOWGGN2iMh8EZlv22wZDRdhZgAvAr9uqa2tzcPAeSKyBzjP9tilZeSVUVRerf3HldOaGB/JjRPieG3NPlam5VodR6lOJy4ikIggHzbohZ0uyxjDAx9u58ixSp6YMZIuft5WR1JOxMueT26MWUZD0d142YJG9w1wa1vb2pYXApNbed0/tyOu01qbaes/rhMCKSf2u6kDWb23kN+/t5XP7ppIVBcdnlOpthIRxsSGsV6PkLus9zcd5OMth7jn/AGMigltvYFyK3pZbyfwVXoeMWEBxIQFWB1FqZPy9fLkyRkjKKuq5XfvbtWhEJU6RWNiw8gpPs7hkuNWR1EdLKugnAc/2s7YuDBuOae/1XGUE9KC3MlVVNeyem8hUwZF63S6yunFR3fhjxcP4tvd+Sxak211HKU6leQ4HY/cFVXX1nPn4s14e3rw36tH4Omh/5ern9OC3Ml9v6eA6tp6piRGWR1FqTaZPa4PkxOi+Odn6aQf0aEQlWqrQd27EuTrpRd2uphHl+9ia04Jj/xyKD10pDR1ElqQO7kVO3Pp6uf14xX4Sjk7EeGRK4bR1c+bO99O1aEQlWojTw9hVJ9Q1mZqQe4qvtmVxwvfZTIzOYapQ7pbHUc5MS3InVhdveGr9DzOTYjSWbxUpxIR5Mt/rhzGrtxS/qlDISrVZmfFR5CRV8aBogqro6jTlFNcwV3vpDIwugsPTku0Oo5yclrlObHUA8UUllczeVC01VGUOmXnDIzihglxLFqzj0+3HrY6jlKdwqSEhu6JX+/Ka2VL5cyqauu49c1N1NUZnps9Gn8fT6sjKSenBbkTW5GWh5eHcPaASKujKNUu905NYGRMCPe+v5XM/DKr4yjl9PpGBhEXEchX6VqQd2Z//WQnW3JK+PeVw4mLCLQ6juoEtCB3Yit25jK2bxjB/jp5gOqcfLw8ePqaUXh7Cr9+c5P2J1eqDc4dGMUPewupqK61Oopqhw835/DG2v3cfFZfpg7pZnUc1UloQe6k9hWWsyevjMkJ2l1FdW49Q/z579Uj2JVbyoMfbbc6jlJOb/KgKKpr6/kho9DqKOoU7TpSyv0fbCM5LozfXTDQ6jiqE9GC3EmtSGs4XTlF+48rF3DOwChuO7c/S1JyWJJywOo4qgOJyFQR2SUiGSJyXzPrg0XkYxHZIiI7RGRuo3W/sS3bLiJvi4hO70rDBEFBvl6s1G4rnUppZQ23vLGRLn7ePD1zJF46GIM6Bfrb4qRW7MxlYHQXYsJ1dk7lGu6aMoAz+oXzp/9tJ+2wjk/uCkTEE3gGuBBIBGaKSNPhJG4FdhpjhgPnAI+KiI+I9ATuAJKMMUMAT2CGw8I7MR8vDyb0j+Dr9DyM0RlvOwNjDL9/byv7iip4euZIorrq35bq1GhB7oRKKmpYn12kkwEpl+LpITwxYyTB/t78+s1NlFbWWB1Jnb5kIMMYk2mMqQYWA9ObbGOALtIw1XAQUASc6BztBfiLiBcQABxyTGznN2lQFEeOVbJT/3jtFF5elcVn249w79SBjO0bbnUc1QlpQe6EvtmdR1290eEOlcuJ7OLL09eMYn9RBfe+v1WP/nV+PYHGfZBybMsaexoYREOxvQ240xhTb4w5CPwH2A8cBkqMMV/aP3LncM7AhtG1vtZuK05vQ3YR//wsnQsGR3PTxL5Wx1GdlBbkTmhFWh4RQT6M6BVidRSlOtyJi52WbTvCqz9kWx1HnR5pZlnTv7IuAFKBHsAI4GkR6SoioTQcTY+zrQsUkdnNvojIPBFJEZGU/Pz8jsru1KK6+DG8V7D2I3dy+aVV3PrmJnqH+vPvK4fTcCJIqVOnBbmTqa6t55tdeUxOiMbDQ/9hK9c0b2JfpgyK5h/L0ti8v9jqOKr9coDejR734ufdTuYCH5gGGUAWkABMAbKMMfnGmBrgA+CM5l7EGPOCMSbJGJMUGek+8zKcmxBF6oGjFJZVWR1FNaO2rp473t7Mscoanps9mq5+OkSxaj8tyJ3MhuwiSitrmTxI+48r1+XhITx65XCiu/px65ubKC6vtjqSap8NQLyIxImIDw0XZS5tss1+YDKAiEQDA4FM2/JxIhJg618+GUhzWPJOYHJCNMbAt7vd46xAZ/Po8t2sySzkb5cOZVD3rlbHUZ2cFuROZvnOXHy9PJgQH2F1FKXsKjjAm+dmjaagrJrfLEmlvl77k3c2xpha4DbgCxqK6SXGmB0iMl9E5ts2+ytwhohsA1YC9xpjCowx64D3gE009C33AF5w+JtwYoN7dCWyi692W3FCy3fm8tw3e5mZHMMVo3tZHUe5AC+rA6j/zxjDyvRcJvSPIMBHvxrl+ob2CubBSxL54/+28+w3Gdw2Kd7qSOoUGWOWAcuaLFvQ6P4h4PyTtH0IeMiuATsxDw9h0sAolm0/TE1dPd46rrVT2FdYzm+XpDKkZ1ceuqTpKJ9KtY/+63Yiu3PLOFB0nCmJOrqKch+zxsYwfUQPHlu+mx8yCqyOo5RTOTchitLKWlKy9VoLZ1BZU8ctb2zCQ4TnZo3Gz9vT6kjKRWhB7kRWpOUCMDlB+48r9yEi/OOyofSNDOKOxZvJPVZpdSSlnMaE+Ah8PD34epd2W3EGD360nZ2Hj/Hfq4fTO0wn7lMdRwtyJ7IiLZfhvYJ1hi/ldgJ9vXhu1ijKq+q4/e3N1NbVWx1JKacQ5OvF2L5hrLQdsFHWWbLhAEtScrh9Un8mJeiZbNWxtCB3EnmllaQeOMoUnQxIuan46C788/KhrM8q4j9f7rY6jlJO49yBUezNL2dfYbnVUdzW9oMl/Omj7UzoH8FdUwZYHUe5IC3IncTX6XkYg87OqdzapSN7cs3YGBZ8u5flO/WIoFLAj8PgfqWjrVgi71glN7++kdAAH56YMQJPnSNE2YEW5E5i+c48eob4M6h7F6ujKGWpB6clMqRnV+5eksr+wgqr4yhluT7hgfSNDNSC3ALlVbVcv2gDxRXVvDQnifAgX6sjKRelBbkTqKypY1VGPlMGRem0u8rt+Xl78uw1oxER5r66npKKGqsjKWW5yQlRrMssoryq1uoobqO2rp7b395M2uFSnrlmFEN6BlsdSbkwLcidwOqMAipr6nW4Q6VsYsIDeOFXo9lfVMH8NzZSXasXeSr3dm5CFNV19azSoUEdwhjDnz/ewVfpefxl+mDO1dHPlJ1pQe4EVqTlNlxJHxdudRSlnMbYvuH864phrMks5L4PtmKMzuSp3NeY2DC6+HrxtXZbcYjnv8vkjbX7mX92P2aN7WN1HOUGdDpIi9XXG1ak5XH2gEh8vPTvI6Uau2xkL/YXHue/K3YTGx7IHZN1Jk/lnrw9PThrQCRfpedhjNHujXb08ZZDPPxZOpcM78HvLxhodRzlJrQCtNi2gyXkl1YxJVFPhynVnDsm9+fyUT15bPlu/rf5oNVxlLLMpIQo8kqr2HHomNVRXNaG7CLuXrKFMbGh/PuKYXjoiCrKQbQgt9iKtFw8BM4ZoAW5Us0RER6+fBjj+obx+/e2si6z0OpISlninIGRiMDKNO22Yg9788u46bUUeoX688KvkvDz9rQ6knIjWpBbbPnOXJJiwwgN9LE6ilJOy8fLg+dnJ9ErzJ+b39hIZn6Z1ZGUcrjwIF+G9wrhq11akHe0grIq5r6yAU8RXp2brP8nK4eza0EuIlNFZJeIZIjIfc2sFxF50rZ+q4iMaq2tiISJyHIR2WP7GWpbfp6IbBSRbbafk+z53jpCTnEF6UdKOU8nA1KqVcEB3rx6XTKeIsx9dQOFZVVWR1LK4SYnRLHlwFHyS/X3v6Mcr67jxkUp5JVW8tKcJGLCA6yOpNyQ3QpyEfEEngEuBBKBmSKS2GSzC4F4220e8Fwb2t4HrDTGxAMrbY8BCoBLjDFDgTnA63Z6ax3mxGlHHe5QqbaJCQ/gxTlJHCmpZN7rG6msqbM6klIOdWL4vW/0KHmHqKs33PXOZrbkHOWJGSMZGRNqdSTlpux5hDwZyDDGZBpjqoHFwPQm20wHXjMN1gIhItK9lbbTgUW2+4uASwGMMZuNMYdsy3cAfiLi1FNqrUjLpW9kIHERgVZHUarTGBUTymNXjWDjvmLueXcL9fU6HKJyH4N7dCW6q6/O2tlB/v5pGl/syOVPFydyweBuVsdRbsyeBXlP4ECjxzm2ZW3ZpqW20caYwwC2n81dDflLYLMx5mfn9ERknoikiEhKfn7+KbydjlVaWcPazELtrqJUO1w8rDv3XZjAJ1sP858vd1kdRymHEREmJUTx/Z4CnTDrNL2yOouFq7OYe2Ys10+IszqOcnP2LMibGyuo6aGsk23TlrbNv6jIYOAR4Obm1htjXjDGJBljkiIjI9vylHbx3e4CauqMdldRqp1uPqsvM5NjePabvbyzYb/VcZRymEkJ0ZRV1ZKSXWR1lE7rix1H+MsnO7lgcDR/vLhpb1qlHM+eBXkO0LvR417AoTZu01LbXFu3Fmw/fzxvJyK9gA+Ba40xezvgPdjNirRcQgO8GaX91ZRqFxHhL9MHMzE+ggc+3M6qPTqluHIPZ/YPx8fLg5XabaVdUg8c5c7FmxneK4THrx6Jp441rpyAPQvyDUC8iMSJiA8wA1jaZJulwLW20VbGASW2bigttV1Kw0Wb2H5+BCAiIcCnwP3GmNV2fF+nrbaunq/S8zg3IUp3BEqdBm9PD56dNYr+UUHc8sZGdh0ptTqSUnYX4OPF+L7hfK0F+SnbX1jBDa9uIKqLHy/NScLfR8caV87BbgW5MaYWuA34AkgDlhhjdojIfBGZb9tsGZAJZAAvAr9uqa2tzcPAeSKyBzjP9hjb9v2BP4lIqu3mlLPtpOwrpuR4jfYfV6oDdPHz5uXrxuDn48n1r24gr7TS6khK2d2khCgyC8rJKii3OkqnUVhWxXWvrqfOGF6ZO4aIIKce90G5GbuOQ26MWWaMGWCM6WeM+btt2QJjzALbfWOMudW2fqgxJqWltrblhcaYycaYeNvPItvyvxljAo0xIxrdnPLwwcq0XHw8PZg4wLo+7Eq5kp4h/iycM4ai8mpuXJRCRXWt1ZGUsqtJtuEPdbSVtsk9VsnVL6zlYPFxXvhVEv0ig6yOpNRP6EydDmaMYfnOXMb1CyfI18vqOEq5jKG9gnlq5ki2HSzhjrc36wgUyqX1DgsgPiqIr9JzrY7i9A4UVXDlgjUcPnqcRdcnkxwXZnUkpX5GC3IH25tfTnZhBecNcsreNEp1alMSo/nLLwazIi2P+W/oxEHKtU1KiGJ9VhGllTVWR3Fae/PLuHLBGkqO1/DGjWMZ1zfc6khKNUsLcgdbmdZwNGOy9h9Xyi5+NT6Wv106hK/S87T7inJpkxKiqKkzOsLQSew8dIyrn19DbX09i+eN01k4lVPTgtzBVqTlkti9Kz1C/K2OopTLmj2uD/+5cjg/7C1gzsL1egRRuaTRfULp6uel/cibsWl/MTNeWIO3pwdLbh7PoO5drY6kVIu0IHegwrIqNu4r1smAlHKAK0b34smZI9m8/yizX1rH0YpqqyMp1aG8PD04e2AUX+/Kp76+TXPnuYU1ewuZ/dI6QgN9WHLzePrqBZyqE9CC3IG+3pVPvUGHO1TKQaYN68GC2aNJO1zKjBfWUlBWZXUkpTrUpIRICsqq2HawxOooTuHr9Dyue2U9PUP8effm8fQOC7A6klJtogW5A61MyyW6qy9DeuqpM6UcZUpiNC9fl0R2YTlXP7+G3GM6TrlyHWcPiMJDdPhDgGXbDjPv9RTio4N45+bxRHX1szqSUm2mBbmDHC45zre785k8KBoRnZ1TKUeaGB/JornJHCmp5Krn15BTXGF1JKU6RFigDyNjQt2+IH9vYw63vbWJ4b1CeOumcYQF+lgdSalTogW5A1TV1nHLG5sQ4Poz46yOo5RbGts3nDduHEtxeTVXLVijMxwqlzEpIYptB0vIc9OzP6+tyeaed7dwZv8IXrshma5+3lZHUuqUaUHuAH9euoPUA0d59Krh9I/Si0uUssrImFDenjeOytp6rnp+DXtyS62O1OmJyFQR2SUiGSJyXzPrg0XkYxHZIiI7RGRuo3UhIvKeiKSLSJqIjHdsetdwYtbOr3e531Hy577Zy4Mf7eC8xGhevDaJAB+dcE91TlqQ29lb6/bz9voD/Pqcfkwd0t3qOEq5vcE9gnln3jgEuPqFtWzXi+HaTUQ8gWeAC4FEYKaIJDbZ7FZgpzFmOHAO8KiInOhP8ATwuTEmARgOpDkkuItJ6NaF7sF+btVtxRjDv79I55HP05k+ogfPzhqFn7en1bGUajctyO1o475iHlq6nbMGRHL3+QOtjqOUsomP7sKSm8fj7+3JNS+uZfP+YqsjdVbJQIYxJtMYUw0sBqY32cYAXaTh4pkgoAioFZGuwFnAywDGmGpjzFGHJXchIsKkhCi+31NAVa3rz05bX2/4v4938szXe5mZ3JvHrhqBt6eWM6pz099gO8krreTXb26ke7A/T84YgaeHXsiplDOJjQjknZvHERrow+yX1rEus9DqSJ1RT+BAo8c5tmWNPQ0MAg4B24A7jTH1QF8gH3hFRDaLyEsiEtjci4jIPBFJEZGU/Pz8Dn8TrmBSQhQV1XWsznDtWTsrqmu5590tvPpDNjdOiOMflw3V/1+VS9CC3A6qa+u59c1NHDtey/O/Gk1IgF7trZQz6hUawJKbx9M9xJ85r6znu91a7J2i5iqhpjPUXACkAj2AEcDTtqPjXsAo4DljzEigHPhZH3QAY8wLxpgkY0xSZGRkB0V3LWf2j6BniD9/+t8OistdcxKsrTlHmfbkKj5MPchvzxvAAxcP0lHLlMvQgtwO/vbpTjZkF/PIFcN0ul6lnFx0Vz/emTeOuIggblyUwoebc6yO1JnkAL0bPe5Fw5HwxuYCH5gGGUAWkGBrm2OMWWfb7j0aCnTVDn7enjw7axT5pVXc9U6qS83cWVdveObrDC5/9geO19Tx1o3juGNyvBbjyqVoQd7B3tuYw2tr9nHTxDh+MbyH1XGUUm0QHuTL4pvGMaJ3CL95Zwu/eSeV0soaq2N1BhuAeBGJs12oOQNY2mSb/cBkABGJBgYCmcaYI8ABETlxgc1kYKdjYrum4b1DePCSRL7dnc9TX2VYHadDHDx6nJkvruXfX+zigiHd+PzOsxjfL9zqWEp1OB0fqANtyynhDx9u44x+4dw7NcHqOEqpUxAc4M1bN43lma/38uRXe0jZV8QTM0YyKibU6mhOyxhTKyK3AV8AnsBCY8wOEZlvW78A+Cvwqohso6GLy73GmBMdnW8H3rQV85k0HE1Xp2HW2Bg27Svm8ZW7GRkTwlkDOm8Xn6VbDvHAh9uorzc8euVwLh/VU4+KK5clxrjOaa1TlZSUZFJSUjrkuQrLqrjkqVWICEtvO5PwIN8OeV6llONt3FfEnYtTOVxSyV2T4/n1uf2d8sIxEdlojEmyOocjdeR+21Udr67j0mdWk1daySd3TKRniL/VkU5JaWUND320gw82H2RkTAiPXz2CPuHNXu+rVKfS0j5bu6x0gNq6em57azMF5dUsmD1ai3GlOrnRfcJYdudEpg3rzqPLdzPzhbUcPHrc6lhKtYm/jyfPzR5FTZ3h129u6lRDIaZkF3HhE9/zv9SD3DUlnndvHq/FuHILWpB3gIc/S2dNZiH/vGwoQ3sFWx1HKdUBuvp588SMkfz36uHsPHyMCx//jk+2Nr1eUSnn1DcyiP9cOYwtB47y90+df76lmrp6HvtyF1c9vwYReHf+Gdw1ZQBeOr64chP6m36aPko9yEurspgzvg+/HN3L6jhKqQ522cheLLtjIn0jg7jtrc387t0tlFfVWh1LqVZNHdKdmybG8dqafXyUetDqOCeVXVDOlQvW8ORXGT/+exvdR6/dUO5FC/LTsPPQMe59fyvJsWH8cVrT2aKVUq4iJjyAd+eP5/ZJ/XlvUw7TnlrF1pyjVsdSqlW/n5rAmNhQ7nt/G7tzS62O8xPGGJakHOCiJ78nM7+Mp2aO5NGrhtPFz9vqaEo5nBbk7XS0opqb30gh2N+bp2eN1Gl7lXJx3p4e3H3+QBbfNI6qmjouf/YHnvtmr0uN96xcj7enB09fM4pAXy/mv7GRMic5u3OkpJJb39rE79/bytCewXx+11lcokMFKzemVWQ71NUbbn97M0dKKnlu9miiuvhZHUkp5SBj+4bz2Z1nccHgbjzyeTqzX17HkZJKq2MpdVLRXf14auZIsgvKufe9rVg5utqWA0e5c/FmJjzyFV/uyOXeqQm8ddM4enSykWCU6mg6Dnk7PPrlLr7fU8A/Lx+qYxQr5YaCA7x5+pqRnL0xkj8v3cHUJ77j75cO5aKh3XScZOWUxvcL5/dTE3j4s3RGrw7l+glxDnvt2rp6vtiRy8LVWWzcV0yQrxdzzojlujNi6R0W4LAcSjkzLchP0WfbDvPsN3uZmdybmckxVsdRSllERLgqqTdJfUK5c3Eqt761iYHRXbhhQhy/GNEDP29PqyMq9RM3n9WXjfuK+ceyNIb1CiYpNsyur1dyvIZ3Nuxn0Q/7OHj0ODFhATw4LZErk3ppP3GlmtCJgU5xgonHlu/m+z35LJ43Dl8v/Q9XKQXVtfUs3XKIl77PJP1IKRFBPswe14fZ4/oQYcd5CXRiIHWqSo7X8IunV1FZU8end0y0y+9nVkE5r67O4t2NOVRU1zE2LowbJsQxeVC0U06wpZSjtLTP1oK8HTv2qto6LcaVUj9jjGHN3kJeWpXFV+l5+Hh5cNmIntwwMY4B0V06/PW0IFftsfPQMS57djWj+4Ty+g1jO6RINsbww95CFq7K4qtdeXh7eHDJ8B7MPTOWIT11fg6loOV9tnZZaQctxpVSzRERzugfwRn9I9ibX8bCVVm8vymHd1IOMDE+ghsmxHH2gEjtZ64sldijK3+7dAi/e28rjy3fxe8uSGj3c1XW1LE09RALV2eRfqSU8EAf7pgUz6xxMTrggVKnQAtypZSyg36RQfz9sqHcc/5A3lq/n0U/ZHPdKxuIjwri+glxXDayp/YzV5a5Mqk3G/cV88zXexnZO5QpidEtbl9cXk1mQTnZBeVk2W6ZBeVkFZRRWVNPQrcu/OuKYfxiuF4/oVR7aJcVPfWplHKA6tp6Ptl6iJe+z2Ln4WOEBfowe2wMs8f3afeRRO2yok5HZU0dVyz4gf2FFXxy+0QiuviQXVBhK7jLbAV3w+1oRc2P7Tw9hJiwAOIiAokND2TKoCjG9wvXMz9KtcKyPuQiMhV4AvAEXjLGPNxkvdjWXwRUANcZYza11FZEwoB3gFggG7jKGFNsW3c/cANQB9xhjPmipXy6Y1dKOZoxhrWZRby8KpOV6Q19beef04/fnjfglJ9LC3J1ug4UVXDxk99TXVdPZU39T9Z16+pHXEQgcZGB9I0IbLgfEUjvsACdDE+pdrCkD7mIeALPAOcBOcAGEVlqjNnZaLMLgXjbbSzwHDC2lbb3ASuNMQ+LyH22x/eKSCIwAxgM9ABWiMgAY0ydvd6jUkqdKhFhfL9wxvcLJzO/jFdWZ9NLJ0VRFukdFsDC68bw3sYceob4ExcZ+OOR70Bf7dWqlKPY819bMpBhjMkEEJHFwHSgcUE+HXjNNBymXysiISLSnYaj3ydrOx04x9Z+EfANcK9t+WJjTBWQJSIZtgxr7PgelVKq3fpGBvHXS4dYHUO5uaTYMLuPSa6Uapk9zzn1BA40epxjW9aWbVpqG22MOQxg+xl1Cq+HiMwTkRQRScnPzz+lN6SUUkoppVRHs2dB3tzVHU07rJ9sm7a0bc/rYYx5wRiTZIxJioyMbOUplVJKKaWUsi97FuQ5QO9Gj3sBh9q4TUttc23dWrD9zDuF11NKKaWUUsqp2LMg3wDEi0iciPjQcMHl0ibbLAWulQbjgBJbN5SW2i4F5tjuzwE+arR8hoj4ikgcDReKrrfXm1NKKaWUUqoj2O2iTmNMrYjcBnxBw9CFC40xO0Rkvm39AmAZDUMeZtAw7OHcltranvphYImI3ADsB660tdkhIktouPCzFrhVR1hRSimllFLOTicG0vFslVKdlI5DrpRSnUdL+2wd2V8ppZRSSikLaUGulFJKKaWUhdy6y4qI5AP72tE0Aijo4Diny9kyaZ7WOVsmZ8sDzpfJ2fL0Mca41fit7dxvO9v3Bs6XydnygPNl0jytc7ZMzpbnpPtsty7I20tEUpyt36azZdI8rXO2TM6WB5wvk7PlUW3jjN+bs2VytjzgfJk0T+ucLZOz5WmJdllRSimllFLKQlqQK6WUUkopZSEtyNvnBasDNMPZMmme1jlbJmfLA86XydnyqLZxxu/N2TI5Wx5wvkyap3XOlsnZ8pyU9iFXSimllFLKQnqEXCmllFJKKQtpQa6UUkoppZSFtCBvgYhMFZFdIpIhIvc1s15E5Enb+q0iMsqOWXqLyNcikiYiO0Tkzma2OUdESkQk1XZ70F55Gr1mtohss73ez+azdvBnNLDRe08VkWMicleTbez+GYnIQhHJE5HtjZaFichyEdlj+xl6krYt/s51YJ5/i0i67Tv5UERCTtK2xe+3gzP9WUQONvpuLjpJW0d9Ru80ypItIqknaWuXz0idOmfaZ9tez+n227rPbjaH7rPbl0n32R3JGKO3Zm6AJ7AX6Av4AFuAxCbbXAR8BggwDlhnxzzdgVG2+12A3c3kOQf4xMGfUzYQ0cJ6h31GzXx/R2gYhN+hnxFwFjAK2N5o2b+A+2z37wMeac/vXAfmOR/wst1/pLk8bfl+OzjTn4F72vC9OuQzarL+UeBBR35Gejvl79Cp9tm213O6/bbus5t9bd1nty+T7rM78KZHyE8uGcgwxmQaY6qBxcD0JttMB14zDdYCISLS3R5hjDGHjTGbbPdLgTSgpz1eq4M57DNqYjKw1xjTnplYT4sx5jugqMni6cAi2/1FwKXNNG3L71yH5DHGfGmMqbU9XAv0Ot3XOd1MbeSwz+gEERHgKuDt030dZVdOtc+GTrvf1n12A91nt5KpjXSf3UZakJ9cT+BAo8c5/HxH2pZtOpyIxAIjgXXNrB4vIltE5DMRGWzvLIABvhSRjSIyr5n1lnxGwAxO/o/R0Z8RQLQx5jA0/CcNRDWzjVWf1fU0HBFrTmvfb0e7zXZKduFJThFb8RlNBHKNMXtOst7Rn5FqntPus8Gp9tu6z24b3We3je6zO4gW5CcnzSxrOkZkW7bpUCISBLwP3GWMOdZk9SYaTvcNB54C/mfPLDZnGmNGARcCt4rIWU3WW/EZ+QC/AN5tZrUVn1FbWfFZPQDUAm+eZJPWvt+O9BzQDxgBHKbhlGNTDv+MgJm0fKTFkZ+ROjmn3GeD0+23dZ/dcXSfrfvsDqMF+cnlAL0bPe4FHGrHNh1GRLxp2Km/aYz5oOl6Y8wxY0yZ7f4ywFtEIuyVx/Y6h2w/84APaTg91ZhDPyObC4FNxpjcpius+Ixsck+c9rX9zGtmG0f/Ps0BpgGzjK1jXVNt+H47jDEm1xhTZ4ypB148yWs5+jPyAi4H3jnZNo78jFSLnG6fDc6339Z9dpvpPrsVus/uWFqQn9wGIF5E4mx/vc8AljbZZilwrTQYB5ScOMXV0Wx9ol4G0owxj51km2627RCRZBq+30J75LG9RqCIdDlxn4aLTrY32cxhn1EjJ/3r2NGfUSNLgTm2+3OAj5rZpi2/cx1CRKYC9wK/MMZUnGSbtny/HZmpcT/Vy07yWg77jGymAOnGmJzmVjr6M1Itcqp9Njjfflv32adE99mtZ9J9dkc62dWeevvxavPdNFwh/IBt2Xxgvu2+AM/Y1m8DkuyYZQINp3m2Aqm220VN8twG7KDhKua1wBl2/nz62l5ri+11Lf2MbK8XQMPOOrjRMod+RjT8x3IYqKHh6MANQDiwEthj+xlm27YHsKyl3zk75cmgoV/fid+lBU3znOz7tWOm122/I1tp2GF3t/Izsi1/9cTvTqNtHfIZ6a1d36PT7LNtr+dU++2T/b5a/BnpPrtteXSf3Uoe2/JX6aT7bLEFVEoppZRSSllAu6wopZRSSillIS3IlVJKKaWUspAW5EoppZRSSllIC3KllFJKKaUspAW5UkoppZRSFtKCXCmllFJKKQtpQa6UUkoppZSF/h/myUGbS/tRoAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 864x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot_sched()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f22ba622",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "@patch\n",
    "def fit_flat_cos(self:Learner, n_epoch, lr=None, div_final=1e5, pct_start=0.75, wd=None,\n",
    "                 cbs=None, reset_opt=False, start_epoch=0):\n",
    "    \"Fit `self.model` for `n_epoch` at flat `lr` before a cosine annealing.\"\n",
    "    if self.opt is None: self.create_opt()\n",
    "    self.opt.set_hyper('lr', self.lr if lr is None else lr)\n",
    "    lr = np.array([h['lr'] for h in self.opt.hypers])\n",
    "    scheds = {'lr': combined_cos(pct_start, lr, lr, lr/div_final)}\n",
    "    self.fit(n_epoch, cbs=ParamScheduler(scheds)+L(cbs), reset_opt=reset_opt, wd=wd, start_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97d8cd6d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>10.588930</td>\n",
       "      <td>7.106113</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>8.943380</td>\n",
       "      <td>5.016665</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn = synth_learner()\n",
    "learn.fit_flat_cos(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc75f678",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAD4CAYAAAAkRnsLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAcFUlEQVR4nO3deZCc9X3n8fd3Lh2jY2aYaSF00CMYpi0wh5gRso2NNCyORNYIO4sL7C20m+wqiqESb8WVyEmt145rE5xk7Sw2xgW1ToSXXczu2ovWlkOIEBAwQhoJHci6BklIIwlpdI0upLm++0c/So2bnpmWpp9++vi8qrr66ef5/Z7n+zxq6aN+TnN3REREsqEs6gJERKR4KFRERCRrFCoiIpI1ChUREckahYqIiGRNRdQFRKm+vt7j8XjUZYiIFJQNGzYcc/eGdNNKOlTi8Tjt7e1RlyEiUlDM7L2hpmn3l4iIZI1CRUREskahIiIiWaNQERGRrFGoiIhI1oQaKma20Mx2mlmHmS1PM93M7PFg+hYzmzNSXzN7wMy2mdmAmbWkzO+rQfudZvYbYa6biIh8WGihYmblwBPAImA28JCZzU5ptghoCl5LgScz6PsO8DngtZTlzQYeBG4EFgLfD+YjIiI5EuZ1KnOBDnffA2BmzwGLgV8NarMYeMaT999fa2Y1ZjYViA/V1923B+NSl7cYeM7dLwJ7zawjqOHNbK/Y+90X+B9vDXmatojkOzPuu+Uaro9NiLqSohNmqEwDDgz63AnckUGbaRn2Tbe8tWnm9WvMbCnJX0XMnDlzhFmmd+T0Bb67puOK+opI9Nzh+fUH+Nnv30n9hDFRl1NUwgyVD/2UAFKfCDZUm0z6XsnycPengKcAWlparugJZbfMqGHvX/zmlXQVkTyw7VA3n/v+L/mD597mmd++g/KydP98yJUI80B9JzBj0OfpwKEM22TS90qWJyLCjddM5puLb+KNjuP813/cFXU5RSXMUFkPNJlZo5lVkTyIvjKlzUrg4eAssHlAt7sfzrBvqpXAg2Y2xswaSR78X5fNFRKR4vH51hk8cPt0Hn+5gzU7j0ZdTtEILVTcvQ94FHgR2A487+7bzGyZmS0Lmq0C9gAdwNPAl4brC2BmnzWzTuBjwM/N7MWgzzbgeZInAvw98Ii794e1fiJS+P5s8U0krp7If/jxJjpPno+6nKJgyROvSlNLS4vrLsUipW3vsXPc993XmdVQzfPLPsaYCl2JMBIz2+DuLemm6Yp6ESlpjfXV/NUDN7O5s5v//PPtUZdT8BQqIlLyFt40lX//yUaeefM9Xth0MOpyCppCRUQE+KOFCVrjtXz1J1vZfeRM1OUULIWKiAhQWV7G974wh/FV5fzesxs5d7Ev6pIKkkJFRCQwZdJYHn/wNvZ0nWX5T7ZSyicyXSmFiojIIB+/vp4//HQz/2/zIX60Vvf4u1wKFRGRFL9313W0JWJ882e/YtOBU1GXU1AUKiIiKcrKjG9//hZiE8fyyLMbOXmuJ+qSCoZCRUQkjZrxVTz5r+fQdeYiX/7xJgYGdHwlEwoVEZEh3Dy9hq99Zjav7urie3rcRUYUKiIiw/jiHTO5/9Zr+M4/7uL13ceiLifvKVRERIZhZvz55z5KU2wCv//c2xzu/iDqkvKaQkVEZATjqyr4/hdv52JvP488u5He/oGoS8pbChURkQxcH5vAY791Mxv3n+IvVu2Iupy8pVAREcnQZ265hn/z8Tg/fGMvL257P+py8pJCRUTkMvzJvR+hsb6av31jb9Sl5CWFiojIZaiqKOM3brya9n0nOX2hN+py8o5CRUTkMrUlYvQNOP+0S6cYp1KoiIhcpjkza5g8rpKXdxyNupS8o1AREblMFeVl3HVDA6/uOqrbt6RQqIiIXIG2RIxjZ3vYcrA76lLyikJFROQK3HVDA2WGdoGlUKiIiFyB2uoqbptZyxqFyq9RqIiIXKG2RIytB7s5evpC1KXkDYWKiMgVWtAcA+CVnV0RV5I/FCoiIlfoI1MnMnXyWB1XGUShIiJyhcyM+c0x/ml3Fz19unMxKFREREalLRHjXE8/6/aeiLqUvKBQEREZhU9cfxVVFWXaBRZQqIiIjML4qgrmzbqKNTsVKqBQEREZtbbmBvYeO8feY+eiLiVyChURkVFqS0wBdHU9KFREREZt5lXjua6hWlfXo1AREcmKtkSMt/Ye5+zFvqhLiZRCRUQkC9oSU+jtd17fXdoP7go1VMxsoZntNLMOM1ueZrqZ2ePB9C1mNmekvmZWZ2Yvmdnu4L02GF9pZivMbKuZbTezr4a5biIig7XEa5k4tqLkd4GFFipmVg48ASwCZgMPmdnslGaLgKbgtRR4MoO+y4HV7t4ErA4+AzwAjHH3jwK3A79rZvFw1k5E5NdVlpfxqaYG1uw8invpPrgrzF8qc4EOd9/j7j3Ac8DilDaLgWc8aS1QY2ZTR+i7GFgRDK8A7g+GHag2swpgHNADnA5n1UREPmxBIsbRMxfZdqh0/+kJM1SmAQcGfe4MxmXSZri+U9z9MEDwHgvG/2/gHHAY2A/8tbt/6L4JZrbUzNrNrL2rS3cWFZHsmd/cgJX4g7vCDBVLMy71N+FQbTLpm2ou0A9cAzQCf2hmsz40E/en3L3F3VsaGhpGmKWISObqJ4zh5uk1CpWQdAIzBn2eDhzKsM1wfY8Eu8gI3i/96X0B+Ht373X3o8AbQEsW1kNEJGNtzTE2d57i+NmLUZcSiTBDZT3QZGaNZlYFPAisTGmzEng4OAtsHtAd7NIaru9KYEkwvAR4IRjeD7QF86oG5gE7wlo5EZF02hIx3Ev3wV2hhYq79wGPAi8C24Hn3X2bmS0zs2VBs1XAHqADeBr40nB9gz6PAfeY2W7gnuAzJM8WmwC8QzKU/tbdt4S1fiIi6dx4zSQaJo7h5RK9wWRFmDN391Ukg2PwuB8MGnbgkUz7BuOPA3enGX+W5GnFIiKRKSszFjQ38It33qe3f4DK8tK6xry01lZEJAfaEjHOXOhjw3snoy4l5xQqIiJZdmdTA5XlVpJngSlURESybMKYCuY21ilUREQkOxY0x+g4epYDJ85HXUpOKVRERELQlkje7KPUfq0oVEREQjCrYQKN9dUKFRERyY4FzTHe3HOc8z2l8+AuhYqISEjaEjF6+gb4ZcfxqEvJGYWKiEhI5jbWUV1VXlJX1ytURERCUlVRxp1N9azZUToP7lKoiIiEqC0R43D3BXa8fybqUnJCoSIiEqIFzaV1arFCRUQkRLFJY7lp2iTWKFRERCQb2ppjbNx/kpPneqIuJXQKFRGRkC1IxBhweG138T+4S6EiIhKyW6bXcFV1VUkcV1GoiIiErKzMuKu5gVd3ddE/UNynFitURERyoC0R49T5Xt7eX9wP7lKoiIjkwCebGigvK/4HdylURERyYPK4SlqurVWoiIhIdtz9kRg73j/DwVMfRF1KaBQqIiI5cunBXcV8IaRCRUQkR65rmMCMunEKFRERGT0zo605xhvvHuNCb3/U5YRCoSIikkMLEjEu9A7w5p7ifHCXQkVEJIfmzbqKcZXlvFKku8AUKiIiOTS2spyWeC1v7T0RdSmhUKiIiORYa7yOnUfO0P1Bb9SlZJ1CRUQkx1rjdbjDhveK79eKQkVEJMdunVFDZbmxfl/x3QdMoSIikmPjqsq5adpk1hfhcRWFiohIBObG69jS2V1016soVEREItASr6Onf4DNB05FXUpWKVRERCLQcm0tAO3vFddxFYWKiEgEaquruGHKBNYV2XGVUEPFzBaa2U4z6zCz5Wmmm5k9HkzfYmZzRuprZnVm9pKZ7Q7eawdNu9nM3jSzbWa21czGhrl+IiKj0RqvY+N7J4vqEcOhhYqZlQNPAIuA2cBDZjY7pdkioCl4LQWezKDvcmC1uzcBq4PPmFkF8N+BZe5+IzAfKL4ri0SkaLTG6zhzsY/th09HXUrWhPlLZS7Q4e573L0HeA5YnNJmMfCMJ60Fasxs6gh9FwMrguEVwP3B8KeBLe6+GcDdj7t7cZ1WISJFpbWxDoD2fcWzCyzMUJkGHBj0uTMYl0mb4fpOcffDAMF7LBh/A+Bm9qKZbTSzP0pXlJktNbN2M2vv6uq6gtUSEcmOaTXjmFYzrqguggwzVCzNuNQdh0O1yaRvqgrgTuCLwftnzezuD83E/Sl3b3H3loaGhhFmKSISrtZ4Lev2ncC9OI6rhBkqncCMQZ+nA4cybDNc3yPBLjKC90v3j+4EXnX3Y+5+HlgFzEFEJI+1xOvoOnOR/SfOR11KVoQZKuuBJjNrNLMq4EFgZUqblcDDwVlg84DuYJfWcH1XAkuC4SXAC8Hwi8DNZjY+OGh/F/CrsFZORCQb5gbHVYrl1OLQQsXd+4BHSf5jvx143t23mdkyM1sWNFsF7AE6gKeBLw3XN+jzGHCPme0G7gk+4+4ngW+TDKRNwEZ3/3lY6ycikg3XN0ygZnwl64vkYL0Vy368K9HS0uLt7e1RlyEiJe7frVjPu13nWPOV+VGXkhEz2+DuLemm6Yp6EZGItcbr2HvsHF1nLkZdyqgpVEREIlZM16soVEREInbTNZMZW1nGOoWKiIiMVlVFGbfOqKG9CC6CHDFUzKzMzN7JRTEiIqVqbryObYe6OXuxL+pSRmXEUHH3AWCzmc3MQT0iIiWptbGOAYeNBf58lYoM200FtpnZOuDcpZHufl8oVYmIlJjbZtZSXmas33eCT91QuLeQyjRUvhFqFSIiJW7CmApmT51U8BdBZhQq7v5q2IWIiJS61ngdz771Hj19A1RVFOZ5VMNWbWZnzOx0mtcZMyuep8qIiOSBuY21XOwbYOvB7qhLuWLD/lJx94m5KkREpNTdfm3yIsj1+05w+7W1I7TOT4X5+0pEpAg1TBzDrPrqgr6yXqEiIpJHWuN1rN93koGBwrzZr0JFRCSPtDbW0f1BL7uPno26lCuiUBERySOt8eSxlEI9tVihIiKSR2bWjSc2cYxCRURERs/MaG2sY32BPl5YoSIikmfmxus41H2BzpPnoy7lsilURETyTEtwXKUQb4WvUBERyTOJqycxcUxFQT60S6EiIpJnysuM2+O1BXlcRaEiIpKHWuN17D56lpPneqIu5bIoVERE8lBrPHkfsPYCe2iXQkVEJA/dPH0yVeVlBXe9ikJFRCQPja0s55YZk1lXYMdVFCoiInmqJV7HOwe7+aCnP+pSMqZQERHJU3PjdfQNOG8fKJzjKgoVEZE8NefaWsxg/V6FioiIjNLkcZUkrp5UUAfrFSoiInmsNV7Lxv0n6esfiLqUjChURETyWGu8jvM9/fzq8OmoS8mIQkVEJI9dugiyUE4tVqiIiOSxqyePZUbduIK5Y7FCRUQkz7XG61i/7wTuHnUpIwo1VMxsoZntNLMOM1ueZrqZ2ePB9C1mNmekvmZWZ2Yvmdnu4L02ZZ4zzeysmX0lzHUTEcmVufE6jp/rYc+xc1GXMqLQQsXMyoEngEXAbOAhM5ud0mwR0BS8lgJPZtB3ObDa3ZuA1cHnwb4D/CLrKyQiEpHWxuRxlUK4FX6Yv1TmAh3uvsfde4DngMUpbRYDz3jSWqDGzKaO0HcxsCIYXgHcf2lmZnY/sAfYFs4qiYjk3qz6aq6qrmJ9ARxXCTNUpgEHBn3uDMZl0ma4vlPc/TBA8B4DMLNq4I+BbwxXlJktNbN2M2vv6uq6rBUSEYmCmdESry2IiyDDDBVLMy71KNNQbTLpm+obwHfc/exwjdz9KXdvcfeWhoaGEWYpIpIfWuN17D9xniOnL0RdyrDCDJVOYMagz9OBQxm2Ga7vkWAXGcH70WD8HcBfmtk+4MvAn5jZo6NeCxGRPDC3sTCuVwkzVNYDTWbWaGZVwIPAypQ2K4GHg7PA5gHdwS6t4fquBJYEw0uAFwDc/ZPuHnf3OPA3wJ+7+/fCWz0RkdyZPXUS46vKac/zXWAVYc3Y3fuCXwovAuXAD919m5ktC6b/AFgF3At0AOeBfztc32DWjwHPm9nvAPuBB8JaBxGRfFFRXsacmbWsy/OD9aGFCoC7ryIZHIPH/WDQsAOPZNo3GH8cuHuE5X79CsoVEclrrfE6/mb1Lro/6GXyuMqoy0lLV9SLiBSI1ngt7rBxf/7+WlGoiIgUiNtm1lJRZnl9EaRCRUSkQIyrKuemaZPz+noVhYqISAGZ21jH5gPdXOjtj7qUtBQqIiIFpOXaWnr6B9h6sDvqUtJSqIiIFJB8f2iXQkVEpIDUVlfRPGUiv3z3WNSlpKVQEREpMPObG1i39wRnL/ZFXcqHKFRERArMgkSM3n7n9d35d6d1hYqISIG5/dpaJo6t4OUdR0dunGMKFRGRAlNZXsanbmhgzc4uBgby67n1ChURkQLU1hyj68xFth06HXUpv0ahIiJSgOY3N2BG3u0CU6iIiBSgqyaM4ZbpNby8U6EiIiJZ0JaIsaXzFMfOXoy6lH+mUBERKVBtiRju8MrO/Dm1WKEiIlKgbrxmElMmjWFNHh1XUaiIiBQoM2NBc4zXdnXR2z8QdTmAQkVEpKAtSMQ4c7GP9jx5dr1CRUSkgN15fT1V5WWsyZOzwBQqIiIFrHpMBXfMqsub61UUKiIiBW5Bc4yOo2fZf/x81KUoVERECl1bIgbAyzuORFyJQkVEpODF66uZVV/Ny3lwvYpCRUSkCCxIxFi75zjne6J9cJdCRUSkCLQlYvT0DfBGx/FI61CoiIgUgdZ4HRPGRP/gLoWKiEgRqKoo487r63ll51Hco3twl0JFRKRItCViHO6+wPbDZyKrQaEiIlIk5icaACK9ul6hIiJSJGITx/LRaZMjPa6iUBERKSILEjHe3n+Sk+d6Ilm+QkVEpIi0JWIMOLy6K5oLIRUqIiJF5OZpk6mfUBXZLjCFiohIESkrM+Y3x3h1Vxd9ETy4K9RQMbOFZrbTzDrMbHma6WZmjwfTt5jZnJH6mlmdmb1kZruD99pg/D1mtsHMtgbvbWGum4hIvmpLxOj+oJe3D5zK+bJDCxUzKweeABYBs4GHzGx2SrNFQFPwWgo8mUHf5cBqd28CVgefAY4Bn3H3jwJLgB+FtGoiInntzqZ6Ksoskl1gYf5SmQt0uPsed+8BngMWp7RZDDzjSWuBGjObOkLfxcCKYHgFcD+Au7/t7oeC8duAsWY2JqR1ExHJW5PGVtIar2NNkYXKNODAoM+dwbhM2gzXd4q7HwYI3mNplv1bwNvufjF1gpktNbN2M2vv6or+NtEiImFoS8TY8f4ZDp76IKfLDTNULM241BvSDNUmk77pF2p2I/At4HfTTXf3p9y9xd1bGhoaMpmliEjBWRA8uCvXv1bCDJVOYMagz9OBQxm2Ga7vkWAXGcH7P28xM5sO/BR42N3fzcI6iIgUpOsaqplZN76oQmU90GRmjWZWBTwIrExpsxJ4ODgLbB7QHezSGq7vSpIH4gneXwAwsxrg58BX3f2NENdLRCTvmRltiRhvvHuMC739OVtuaKHi7n3Ao8CLwHbgeXffZmbLzGxZ0GwVsAfoAJ4GvjRc36DPY8A9ZrYbuCf4TND+euA/mtmm4JXueIuISElYkIhxoXeAN9/N3YO7LMr77ketpaXF29vboy5DRCQUF3r7ue3PXuJf3T6db95/U9bma2Yb3L0l3TRdUS8iUqTGVpbzievreXlH7h7cpVARESlibYkYB099wO6jZ3OyPIWKiEgRWxA8uCtXV9crVEREitjUyeP4yNRJChUREcmOtkQDG947Sff53tCXpVARESlybYkY/QPOa7vDvzWVQkVEpMjdOqOW2vGVObm6XqEiIlLkysuMu25o4JVdXfQPhHtqsUJFRKQELEjEOHGuh82dp0JdjkJFRKQE3HVDA+VlFvouMIWKiEgJqBlfxe0za0M/tVihIiJSIhYkYmw7dJojpy+EtgyFiohIiWjLwYO7FCoiIiXihikTmFYzLtRdYAoVEZESYWYsSDTwescxLvaF8+AuhYqISAlpS8Q439PPur0nQpm/QkVEpIR8bFY9YyrKQtsFplARESkh46rK+cIdM5lROz6U+VeEMlcREclb/+kzN4Y2b/1SERGRrFGoiIhI1ihUREQkaxQqIiKSNQoVERHJGoWKiIhkjUJFRESyRqEiIiJZY+7hPq84n5lZF/DeKGZRDxzLUjlhUH2jo/pGR/WNTj7Xd627N6SbUNKhMlpm1u7uLVHXMRTVNzqqb3RU3+jke31D0e4vERHJGoWKiIhkjUJldJ6KuoARqL7RUX2jo/pGJ9/rS0vHVEREJGv0S0VERLJGoSIiIlmjUBmBmS00s51m1mFmy9NMNzN7PJi+xczm5LC2GWa2xsy2m9k2M/uDNG3mm1m3mW0KXl/LVX3B8veZ2dZg2e1ppke5/ZoHbZdNZnbazL6c0ibn28/MfmhmR83snUHj6szsJTPbHbzXDtF32O9riPX9lZntCP4Mf2pmNUP0Hfb7EGJ9Xzezg4P+HO8dom9U2+/Hg2rbZ2abhugb+vYbNXfXa4gXUA68C8wCqoDNwOyUNvcCvwAMmAe8lcP6pgJzguGJwK409c0HfhbhNtwH1A8zPbLtl+bP+n2SF3VFuv2ATwFzgHcGjftLYHkwvBz41hDrMOz3NcT6Pg1UBMPfSldfJt+HEOv7OvCVDL4DkWy/lOn/BfhaVNtvtC/9UhneXKDD3fe4ew/wHLA4pc1i4BlPWgvUmNnUXBTn7ofdfWMwfAbYDkzLxbKzKLLtl+Ju4F13H80dFrLC3V8DTqSMXgysCIZXAPen6ZrJ9zWU+tz9H9y9L/i4Fpie7eVmaojtl4nItt8lZmbA54H/me3l5opCZXjTgAODPnfy4X+0M2kTOjOLA7cBb6WZ/DEz22xmvzCz8B5OnZ4D/2BmG8xsaZrpebH9gAcZ+i9ylNvvkinufhiS/5kAYmna5Mu2/G2Svz7TGen7EKZHg91zPxxi92E+bL9PAkfcffcQ06PcfhlRqAzP0oxLPQc7kzahMrMJwP8Bvuzup1MmbyS5S+cW4LvA/81lbcAn3H0OsAh4xMw+lTI9H7ZfFXAf8L/STI56+12OfNiWfwr0Ac8O0WSk70NYngSuA24FDpPcxZQq8u0HPMTwv1Ki2n4ZU6gMrxOYMejzdODQFbQJjZlVkgyUZ939J6nT3f20u58NhlcBlWZWn6v63P1Q8H4U+CnJXQyDRbr9AouAje5+JHVC1NtvkCOXdgsG70fTtIn6u7gE+JfAFz04AJAqg+9DKNz9iLv3u/sA8PQQy416+1UAnwN+PFSbqLbf5VCoDG890GRmjcH/Zh8EVqa0WQk8HJzFNA/ovrSbImzB/tf/Bmx3928P0ebqoB1mNpfkn/nxHNVXbWYTLw2TPJj7TkqzyLbfIEP+7zDK7ZdiJbAkGF4CvJCmTSbf11CY2ULgj4H73P38EG0y+T6EVd/g43SfHWK5kW2/wL8Adrh7Z7qJUW6/yxL1mQL5/iJ5dtIukmeF/GkwbhmwLBg24Ilg+lagJYe13Uny5/kWYFPwujelvkeBbSTPZFkLfDyH9c0Klrs5qCGvtl+w/PEkQ2LyoHGRbj+SAXcY6CX5v+ffAa4CVgO7g/e6oO01wKrhvq85qq+D5PGIS9/DH6TWN9T3IUf1/Sj4fm0hGRRT82n7BeP/7tL3blDbnG+/0b50mxYREcka7f4SEZGsUaiIiEjWKFRERCRrFCoiIpI1ChUREckahYqIiGSNQkVERLLm/wP3N7iD7LkpUQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot_sched()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "407b8d38",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "@patch\n",
    "def fit_sgdr(self:Learner, n_cycles, cycle_len, lr_max=None, cycle_mult=2, cbs=None, reset_opt=False, wd=None,\n",
    "             start_epoch=0):\n",
    "    \"Fit `self.model` for `n_cycles` of `cycle_len` using SGDR.\"\n",
    "    if self.opt is None: self.create_opt()\n",
    "    self.opt.set_hyper('lr', self.lr if lr_max is None else lr_max)\n",
    "    lr_max = np.array([h['lr'] for h in self.opt.hypers])\n",
    "    n_epoch = cycle_len * (cycle_mult**n_cycles-1)//(cycle_mult-1)\n",
    "    pcts = [cycle_len * cycle_mult**i / n_epoch for i in range(n_cycles)]\n",
    "    scheds = [SchedCos(lr_max, 0) for _ in range(n_cycles)]\n",
    "    scheds = {'lr': combine_scheds(pcts, scheds)}\n",
    "    self.fit(n_epoch, cbs=ParamScheduler(scheds)+L(cbs), reset_opt=reset_opt, wd=wd, start_epoch=start_epoch)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d2a7003",
   "metadata": {},
   "source": [
    "This schedule was introduced by Ilya Loshchilov et al. in [SGDR: Stochastic Gradient Descent with Warm Restarts](https://arxiv.org/abs/1608.03983). It consists of `n_cycles` that are cosine annealings from `lr_max` (defaults to the `Learner` lr) to 0, with a length of `cycle_len * cycle_mult**i` for the `i`-th cycle (first one is `cycle_len`-long, then we multiply the length by `cycle_mult` at each epoch). You can optionally pass additional `cbs` and `reset_opt`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d79d9ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAD4CAYAAAAkRnsLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA7qUlEQVR4nO3dd3xV9fnA8c+TPUkIGUACJJAwAkEZ7o2CoiiOWkdb6fpZW+22rbbValtbu1tbq9XWVuuqo1VUqiIqjooyhDDCCDsQCHvDXd/fH/fcEGPIvOeec8993q9XXnedc+5zb84533ue7xJjDEoppVQ0JDkdgFJKKe/QQkUppVTUaKGilFIqarRQUUopFTVaqCillIqaFKcDcFJhYaEpLy93OgyllIor8+fP326MKWrrtYQuVMrLy5k3b57TYSilVFwRkfXHek3TX0oppaJGCxWllFJRo4WKUkqpqNFCRSmlVNRooaKUUipqbC1UROQCEVkhIvUicksbr4uI3GO9XisiYztaV0SuFJGlIhISkfGttnertfwKETnfzs+mlFLq42wrVEQkGbgXmAxUA9eISHWrxSYDVdbf9cB9nVh3CXA58Far96sGrgZGAhcAf7a2o5RSKkbsvFI5Eag3xqwxxviAJ4GprZaZCjxiwuYA+SLSr711jTF1xpgVbbzfVOBJY8wRY8xaoN7aTtQd9AW4Y/pS9hzy27F5Wzw9byMzl21l/5GA06EkhGDIsH7HAQLBkNOhKBVTdnZ+LAU2tnjcAJzUiWVKO7luW+83p41tfYSIXE/4qoiBAwd2sMm21TXu5bH317N08x7++YWTyEh19wVR077DfOeZWgBSkoSxg3pz1tAiLqrpR3lhtsPRedO/5m7k+/9ZTGZqMqNKezG6LJ/RZXmcUVVEQXaa0+EpZRs7r1Skjedazwh2rGU6s2533g9jzAPGmPHGmPFFRW2OMtChcYMK+N1VxzNv/S5uevxD1/8aPeIPx/fZU8v54hmDOXAkwK9eWcEFf3iLF2s3OxydN23ffwSAq04YQMjAY++v5+tPLuSEu15j2kMf8Mz8BvYejp8rXaU6y84rlQZgQIvHZUDrM9ixlknrxLrdeb+omTK6PzsP+Lj9+aV8/z+L+cUVoxFpq1xzns8q9MYMzGfq8aXcMnk4m3cf4mtPfMhNj3/Iii37+OZ5Q0lKcmf88cgfDCECd1wyEoBAMMSyxr38d8kWXli0mZufXkTav5OYOLKEG84cQk1ZnsMRKxUddhYqc4EqEakANhGuRL+21TLTgZtE5EnC6a09xphGEdnWiXVbmw48LiK/BfoTrvz/IGqfpg3XnVLO9v0+7pm1ij456XzvguF2vl23+a1CJTX56IVp//xMHvu/k7j9uaX88fV6lm/Zx++uOp6c9IQeDi5qfMHQR77vlOQkKwWWz3fPH8aihj1MX7iZp+dv5KXaRs4cWsSNZw/hpMF9HIxaqZ6zLf1ljAkANwGvAHXAU8aYpSJyg4jcYC02A1hDuFL9QeAr7a0LICKXiUgDcArwkoi8Yq2zFHgKWAa8DNxojAna9fkivnleFdeeNJD73lzNo3OOOcaao/yBcBaw5UkOID0lmbuvqOGOi6t5fXkTV/z5f+yw0jaqZwJBQ1py24eXiHD8gHxuv7iad2+ZwHcvGMbSTXu46oE5XHn//1i4cXdsg1UqisSYjqoqvGv8+PEmGqMUB0OGax6Yw+Y9h3j7u+e4Lg22YMMuLv/z//j7507gnGHFbS7z9qptfOHheZwyuA9//+wJmgrrodufX8L0RZtZePukTi1/yBfkX3M38Oc3V7Nt/xGuPXEg3zl/GPlZWqmv3EdE5htjxrf1mvaoj4LkJOGKcaU07DpEXeM+p8P5GH8gnP461i9ngDOqirh9SjWzV27jL2+tiVVonuVvlf7qSGZaMp89rYJZ3z6Lz51awZNzNzLhN7N5et5GEvmHn4o/WqhEybkjShCBV5ZucTqUj/EH205/tfapkwZyUU0/fv3qCuav3xmL0DzL3076qz25GancfnE1L9x0OuV9svjOM7V8+m/v07T3sA1RKhV9WqhESWFOOuMH9ebVZVudDuVjjlbUt5/SEhF+fkUNpfmZfPXxD9l90BeL8DwpfKXS/RRidf9ePHPDqfzsshrmr9/F5D+8zZsrmqIYoVL20EIlis4f2Ze6xr1s3HnQ6VA+wtdG669j6ZWRyp+uHcO2/Ue4+elaTb10kz8YIqUbVyotJSUJ1540kBduOp3CnHQ++/e5/Py/dc0/EpRyIy1UomhSdV/AfSmwyEkoLaVz/+7RZfncOnkEr9Vt5ZH33Nmize18AdOlOpX2VJXk8vxNp/Gpkwbyl9lr+ORf3tN0mHItLVSiaGCfLIb3zeXVpe5KgbXVT6UjnzutnDOqCvn1qyvYdUDTYF0VCIVI60H6q7WM1GTuuqyGe68dy4ot+7j03ndZvmVv1LavVLRooRJlk0b2Zd76nc3DdLhBpKI+pQvNhEWEH1w0gv1HAtz7Rr1doXlWNNJfbblodD+e+tIpBI3hE/e9x+yV26L+Hkr1hBYqUTapuoSQgVl17rla6Wr6K2J43158YmwZj7y33nX1RG7nD5geVdS3Z1RpHs/deBoDCrL4/D/m8tj7mqJU7qGFSpSN7N+L0vxMV6XAIv1UupPj/9akoYjAb2eujHZYntZ6mJZo65eXydM3nMKZVYX84D9L+M2rK7RRhXIFLVSiTESYNLKEt+u3u2bukqP9VLr+y7lfXiafP72C/3y4iSWb9kQ7NM8K16nYe3jlpKfw4HXjuWr8AP74ej13v7xcCxblOC1UbDCpui++QIi3XJLv7kqT4rbccNYQ8rNS+cXLy6MZlqf5A4YUm9JfLaUkJ/Hzy2v49MnhlmE/ebFOCxblKC1UbHBCeW96Z6XyqkuaFnen9VdLeZmp3HROJW+v2u6agtLtujpMS08kJQk/mTqKz51WzkPvruVH05cSCmnBopyhhYoNUpKTOHdECbOWN+ELON9RzR8MkZwkJPdgkMjPnDKIst6Z3P3f5XrC6gR/DNJfLYkIt0+p5vozB/PIe+v54fNL9IpFOUILFZucM6yYfYcD1DU635cgEDRdak7clvSUZL553lCWNe7lrVV6tdIRfxQ7P3aWiHDr5OF8+ewhPP7+Bn7x8oqYvr9SoIWKbUZbM/nVuqBy2xeMzq/mi4/rT3FuOg+9u67nQXlcuJ9K7KcPEBG+e/4wPnXSQO6fvZoHdcRpFWNaqNikrHcmvbNSWdLgfKHiD4ZI7WIflbakpSRx3SmDeGvlNlZtdd8Q/25id5Pi9ogIP546iotG9+OuGXU8M7/BkThUYtJCxSYiwqjSPFdcqUSzI961Jw0iPSVJr1Y6EAiaLnc2jabkJOG3nzyO0ysL+d6ztbzmwtGzlTdpoWKj0WV5rNy6j8N+22c1blc0WyIVZKdx+dhS/r2gQccEa4c/GOpxPVZPpack85fPjGNU/17c+PgC5q3TOXKU/bRQsVFNaT7BkHG8sj5adSoRnzutgiOBEI9/sCFq2/QSYwyBUOwr6tuSnZ7C3z93Iv3yMvjSP+frcDvKds7v9R5WY1XWL3Y4BRYIRvcEN7QklzOqCnnkvXU6t0cbIiMYOJn+aqkgO42/ffYEfMEQX3x4nmtGelDe5I693qP652XQJzuNxQ5X1ocr6qObivn8aRVs3XuEGYsbo7pdL+jsTJuxNKQoh/s+NY76bfv5+hMfEtS+RsomWqjYSESoKctz/ErFFwyRkhTdf/VZQ4sYXJjNQ++s1U52rUQKlWh/5z11elUhd1wyklnLm7j7v3VOh6M8yl17vQfVlIYr6w/5nKus90e5TgXCQ4N87rRyFjXsYcGGXVHddrxrHmvNJemvlj5z8iCmnTKIB99ey1NzNzodjvIg9+31HlNTmkfIwDIHK+v9QRP19BfAFePKyE1P4fH39eTUUiBSp+Ki9FdLt02p5oyqQn743BJqG3Y7HY7yGC1UbDa6LB+AxQ4evHYNbpiVlsKU4/rx3yWNHNDK32ZuTX9FpCQn8Yerx1CUm86XH12gTcNVVLlzr/eQkl7pFOakO9oJ0hewr3f3FWPLOOgLaoV9C34Xp78iCrLT+POnxrJt3xG+8a+FOkioihr37vUeISKMLstzdIKrQMjYNmLuuEG9Ke+TxbMLdCiQCL/L018Rxw3I50eXVDN75Tb++Hq90+Eoj9BCJQZqSvOob9rvWIoonP6y5wQnInxiXBlz1uzUjnWWns5fE0vXnjiQy8eW8vtZK5mtc+WoKHD/Xu8BTlfW+wMhUmw8wV02tgwR9GrF0lynEgeFiohw16U1DCvJ5etPfsjm3YecDknFOffv9R7Q3LPeoU6Qvij3qG+tND+TU4f04dkFDZqbB3yB8Hfgps6P7clMS+a+T4/DHwjxzX8t1I6Rqke0UImBkl4ZlPRKd6wTZLifir0nuE+MK2PjzkPM1UELCYTCVyqxnPmxpyoKs/nx1FG8v3Ynf35D61dU99m614vIBSKyQkTqReSWNl4XEbnHer1WRMZ2tK6IFIjITBFZZd32tp5PFZGHRWSxiNSJyK12frauqinNc6xPQCzmSz9/ZF9y0lN07g7iq06lpcvHljL1+P78ftYq5q/XHweqe2zb60UkGbgXmAxUA9eISHWrxSYDVdbf9cB9nVj3FmCWMaYKmGU9BrgSSDfG1ADjgC+JSLk9n67rakrzWbP9gCOD+UVrkq72ZKWlcFFNP2YsbuSgL7H7rETSX07M/NgTIsJPLx1F//wMvvbEQvYc8jsdkopDdp5pTgTqjTFrjDE+4ElgaqtlpgKPmLA5QL6I9Otg3anAw9b9h4FLrfsGyBaRFCAT8AHOTxBvGV2WhzGwNMYpMGNMuEd9DH41XzGujAO+IC8v2WL7e7lZPKa/InIzUrnn6jFs3XuYH/xnsY7rprrMzr2+FGg5fkeD9Vxnlmlv3RJjTCOAdVtsPf8McABoBDYAvzbGfOwaXkSuF5F5IjJv27bYNaEcVerMMPiBUOz6TJxQ3puBBVkJnwKL1/RXxJiBvfnmxKG8WNvI0wn+v1RdZ+de39ZZrPXPnmMt05l1WzsRCAL9gQrg2yIy+GMbMeYBY8x4Y8z4oqKiDjYZPUW56RTlprN8S2zndo9l81YR4bIxpby3ZgdN+w7b/n5u5Y/T9FdLN5w1hJMHF/DjF5Zp/yPVJXaeaRqAAS0elwGbO7lMe+tutVJkWLdN1vPXAi8bY/zGmCbgXWB8FD5H1FQW5VDftD+m7+lvbt4am1/NF43uhzEkdAosMkpxPKa/IpKThF994jgAvvPMIm0qrjrNzr1+LlAlIhUikgZcDUxvtcx04DqrFdjJwB4rpdXeutOBadb9acDz1v0NwARrW9nAycByuz5cd1QW57C6aX9M89RHT3Cx+dU8tCSXquIcXqpN3LHAAnGe/ooYUJDFbVNGMGfNTv7xv3VOh6PihG17vTEmANwEvALUAU8ZY5aKyA0icoO12AxgDVAPPAh8pb11rXXuBiaKyCpgovUYwq3FcoAlhAulvxtjau36fN1RWZzDviMBmvYdidl7OpHfv7CmHx+s25mwKbDI2F9uHlCysz45fgAThhfzi5eXx/wqW8UnW/d6Y8wMY8xQY8wQY8xd1nP3G2Put+4bY8yN1us1xph57a1rPb/DGHOuMabKut1pPb/fGHOlMWakMabaGPMrOz9bd1QW5wCwOoYHpxOFSqKnwHzNQ9/Hb51KhIhw9+U1ZKYl8+2nFzVfhSl1LPH/UyqORAqV+m2xLFRi/6s50VNgkUm64j39FVHcK4OfXjqKRRt3c9+bq50OR7mcN/b6OFGcm05OekpM0wj+GNepRCRyCswfDJGcJCR74EolYsro/lx8XH/+MGsVdQ7OYqrcTwuVGBIRhhTHtgWYU7MQJnIKzB8MeSL11dqdl4wkLzOV7z1bq2kwdUxaqMRYrJsVOzULYSKnwHzBUFw3Jz6Wguw07rhkJLUNe/jbO2udDke5lPf2fJerLM6had8R9h6OzbhKTg7D3pwC25tYKbBA0Hii5Vdbpozux8TqEn47cyVrYlg3qOKHN/d8F2uurI/R1YrfwY54zSmwpYmVArNzpk2nRQadTEtJ4pZnF2unSPUxWqjEmFOFihMtkRI1BeYLhmJehxVLJb0yuO2iaj5Yt5PHPtjgdDjKZby757vUgN6ZpCUnxayvit/h5q2JmALzBw1pHk1/RVw5vozTKwu5e0Ydm3QKYtWCt/d8F0pJTqKiMDv26a8UZ9IxkRTYq8u2OvL+Tgh4OP0VISL8/PIaQgZuf26JDpGvmmmh4oDK4pyYdYB0qklxRFVxDoP6ZDEzgQoVv8fTXxEDCrL41sShzFrexCsJVm+mjs37e74LDSnOYePOgxz2B21/L6eaFEeICBNHlPDe6h2OzHrpBJ+HW3+19rnTyhnRrxc/mr6UfTFq0ajcLTH2fJepLM4hZGDt9gO2v5cv6FyT4ojzqkvwBUO8tTJ2k6I5KRAMxXwEA6ekJCfx88traNp3hF+/ssLpcJQLaKHigMqi2LUA8wecn9tj/KDe5GelJkwKLNykOHEOreMH5DPtlHIembOehRt3Ox2Oclji7PkuMrgoG5HYFCqR+dKdPMmlJCcxYXgxry9vak7HeZkvaGIy06abfHvSUIpz07n134t1CJcEl1h7vktkpCYzoHdWTCrrnW5SHDGpuoQ9h/zMXbfT0ThiwR9InPRXRG5GKndeMpK6xr089K4O4ZLItFBxSGQWSLv5ApErFWdPcmdUFZGWksRry5o6XjjOBUKJlf6KOH9kX84bUczvZq5is/ZdSViJt+e7RGVxDmu2HyBo8zAXkRFzRZwtVLLTUzhtSB9m1m3xfJ8Gf9AkZKEiIvzo4pEYDD95cZnT4SiHJN6e7xKVRTn4AiE27jxo6/u4qdJ4YnVfNu48xIqt+5wOxVa+QIiUBEt/RQwoyOKmcyr575ItzE6Q1n7qo9xxtklAQ2I0Blj4V7M7TnDnjSgGYOZSb7cCC4S8OfR9Z/3fmYMZXJjNj55fEpO+WMpdEnfPd1isphb2BUOuGYequFcGxw/IZ2adtwuVRE1/RaSnJHPn1JGs23GQB95a43Q4KsYSd893WF5mKkW56bZfqQRclP4CmFhdQm3DHrbs8e4Ak/4ETn9FnFFVxEU1/bj3jXo27LA3xavcxT1nmwQUi1kg3fareWJ1CQCvefhqxaszP3bVbVOqSUkS7nhhqecbZ6ijdM93UKRZsZ0HnM9lI+ZGBpj0cqESCLmrIHdK37wMvnHeUF5f3sRrdd5vSq7CdM93UHlhNvuOBNhxwGfbe/gD7kp/iQgThhfz3uodHPJ5rxI3GDIEtVBp9tnTyqkszuEnLy7TSvsEoXu+gyoKswBYZ+PAkm5qUhwxYXgxRwIh/rd6u9OhRF3zVAMuujp0UmpyEndcPJINOw/yoFbaJwR3nW0STHmfbMDe0Yrd1KQ44sSKArLSknl9ufdSIgGrM6vWqRx1elUhk0f15d4363WWyASge76DBhRkkZwkrNthX6Hic+GVSnpKMqdXFvLG8ibPVeD6XTIsjtv84KIRAPzspTqHI1F2c9fZJsGkJidR1juTddvta3IZcFE/lZYmDC9m857DnutdfzT95b7v3EllvbP48lmVvLS4kf/Vey/tqY7SPd9h5X2ybb1ScVuT4oizh4V713stBeYLOj9/jVt96azBlPXO5I4XlibEFAiJSvd8h1UUZrNu+wHb0kB+lzUpjuibl0F1v1684bFCJRCZaiDFfd+50zJSk7ltSjUrt+7nkffWOx2OsomthYqIXCAiK0SkXkRuaeN1EZF7rNdrRWRsR+uKSIGIzBSRVdZt7xavjRaR90RkqYgsFpEMOz9fNJT3yeKAL8i2/Uds2b4vGHJtKmbC8GLmr9/FnoPemds88gvcjVeHbjCpuoQzqgr5/Wsr2WHTPq+cZdueLyLJwL3AZKAauEZEqlstNhmosv6uB+7rxLq3ALOMMVXALOsxIpICPArcYIwZCZwNuP5sVV4YbgFmV72K38W9u88ZXkzIwOxV3hnNNpL+Skly53futPDw+NUc9AX5zcyVToejbGDnnn8iUG+MWWOM8QFPAlNbLTMVeMSEzQHyRaRfB+tOBR627j8MXGrdnwTUGmMWARhjdhhjXN/bqqK5ULGnXsUfcF+T4ojjB+RTkJ3mqRRYJP2VpumvY6oszuW6UwbxxAcbWLp5j9PhqCizs1ApBTa2eNxgPdeZZdpbt8QY0whg3RZbzw8FjIi8IiILROS7bQUlIteLyDwRmbdtm/O/kEvzM0lJEtbaVFnvxs6PEclJwllDi3hzRZPtk5XFiqa/Oucb5w4lPzOVO19Y5rlm5YnOzj2/rZ9qrfeeYy3TmXVbSwFOBz5l3V4mIud+bCPGPGCMGW+MGV9UVNTBJu2XkpzEwIIs+65UXFyoQDgFtuugn4UbdzsdSlT4tFDplLysVG4+fxgfrN3JjMVbnA5HRZGde34DMKDF4zJgcyeXaW/drVaKDOs2kjtpAGYbY7YbYw4CM4CxxIHywmzbetX7g8aV/VQizqoqIjlJPJMC80daf7k05egmV58wkBH9evGzGXWeHAcuUdl5tpkLVIlIhYikAVcD01stMx24zmoFdjKwx0pptbfudGCadX8a8Lx1/xVgtIhkWZX2ZwFxMVF2eZ9s1u84aEsawK1NiiPyslIZN7C3Z/qrBPRKpdOSk8KV9pt2H9LJvDzEtj3fGBMAbiJ8sq8DnjLGLBWRG0TkBmuxGcAaoB54EPhKe+ta69wNTBSRVcBE6zHGmF3AbwkXSAuBBcaYl+z6fNFUUZjFIX+QrXuj28QyFDIEQsb1LZHOHl7Essa9npi4S+tUuubkwX24qKYf982uZ7OOC+YJtu75xpgZxpihxpghxpi7rOfuN8bcb903xpgbrddrjDHz2lvXen6HMeZcY0yVdbuzxWuPGmNGGmNGGWParKh3o0iz4minwPwhq3e3i9NfAOdYvetnr4z/qxWfpr+67NYLhxMy8MuXlzsdiooCd59tEkRktOJoD9cSL/n94X1z6dsrgzdXON8ar6c0/dV1Zb2zuP6MwTy3cDMLNuxyOhzVQ7rnu0D//EzSkpOi3gIsXk5wIuGmxe+s2h73Y0Jp+qt7vnz2EIpz0/nxC8sIeaR5eaLSPd8FkpOEgX2yop7+iqfmrWcPK2LfkQAL1sf3L9Wj6S/3f+dukp2ewncvGM7Cjbt5ftEmp8NRPaB7vkvYMVpxJP3l1mFaWjqtqpCUJOHNlfGdAtP5VLrv8jGljC7L4xf/XcFBX8DpcFQ3uf9skyAqCrNYv+NgVC/9m09wcTBkSK+MVMYO6h339SqBUPxcHbpNUpJw+5Rqtuw9zP2ztYlxvNI93yXKC7M5EgjRuDd6zWr9cTa44dnDiqhr3MvWKH4HsebX9FePjC8v4OLj+vOX2at16uE4pXu+S1T0if7AkvFUpwJw9lCraXEcX634NP3VY7dMHg5oE+N4FR9nmwRgR18Vf5yNmDuiXy4lvdJ5M477qwRCIVKSBJH4+M7dqDQ/k+vPHMzzCzczP84bbiQiLVRcom+vDNJTotusOF6aFEdEmha/vWp7c+zxxq3TN8ebG84KNzH+yYvaxDje6N7vEklJEvUWYPGW/oJw7/p9hwMs2LDb6VC6xRdw91hr8aJlE+MXaluPQ6vcLH7ONgmgvDC6fVXisdK4uWnxivhMgbl9qoF4cvmYUmpK87j7v8t1FOM4onu/i5QXZrNx56GoTVgVaVIcD/1UIuK9aXFA019Rk5Qk3DalmsY9h3UU4ziie7+LVPTJxhcMRW201uYmxXGWjjl7WHjU4qY4bFrsD4biol9QvDixooALa/py/+zVnhjFOhF0WKiISJKILIlFMIku2i3A4rFOBY42LY7H3vW+YIjUOOkXFC9unTyCYMhoE+M40eHeb4wJAYtEZGAM4kloFVahsj5KlfXxNExLS5GmxfHYX0XrVKJvQEEWXzijgn9/uIlFHpl22ss6u/f3A5aKyCwRmR75szOwRFScm05WWjJrtx+MyvaamxTHWTrmaNPibXHXtDgQNHH3fceDr5w9hMKcNH7y4jJbZkhV0ZPSyeXutDUKBYRPpoOi2Kw4nodhP3tYMU/Na+DDjbs5obzA6XA6zadXKrbIzUjl5knDuOXfi3lpcSNTRvd3OiR1DJ3a+40xs9v6szu4RFRRmBW1DpDxPAz7aZWFJMdh02K/1qnY5srxAxjRrxc/n7Gcw35tYuxW7e79IrJPRPa28bdPRPbGKshEUt4nmw07D0Yl7RO5Uom3OhWAvMxUxg2Mv6bFmv6yT3KScNuUEWzafYi/vbPW6XDUMbR7tjHG5BpjerXxl2uM6RWrIBNJeWE2gZCJygitkX4q8dakOOKsYUUs3byXpn3x05RUK+rtdeqQQiZWl/DnN+rjar9IJLr3u0xFFJsVHx36Pj4LlbOHFQHxNWqxTzs/2u77F47AFwzxm1dWOh2KaoPu/S5THsUh8H1BQ1pyUtyOmFvdrxfFuelx1V8lfKUSn993vKgozGbaKeU8NX8jSzbtcToc1YoWKi5TmJNGTnoK63b0vFlxIM5PcM1Ni1fGT9PigKa/YuKr51bRO0ubGLuR7v0uE25WHJ2BJcNDhsT3v/jsYcXsPRxgYZx0etOh72MjLzOVb00cyvtrd/Lyki1Oh6Na0L3fhcoLo9NXxQv5/dOrIk2L4yMF5ovzq8N4cvUJAxhWkstdM+q0ibGLxPcZx6Mq+mTTsOtQc0V7d/mDobhsTtxSXmYqYwfmx81skJr+ip2U5CRuv7iahl2HeOhdbWLsFrr3u1B5YTbBkGHjzp7Vq/iDobhtTtzS2cOKWbIpPpoWa/ortk6rDDcxvvf1+rgc1dqLdO93oYrCLIAep8C80mfirKHhpsVvrdzucCQd02FaYi/SxPjXr65wOhSFFiquFGlW3NOBJX0Bb/xqHtm/F0W56XExZIs2KY69isJsPndaBU/Pb9Amxi4Q/2ccDyrITiM3I6XHfVUCoRBpHjjBRZoWv+XypsXBkMGY+BxrLd7dNKGSgqw07nxhqTYxdpju/S4kIlREoQWYV9JfAOdYTYs/dHHT4ngeFTre9cpI5ebzhzF33S5erG10OpyEZuveLyIXiMgKEakXkVvaeF1E5B7r9VoRGdvRuiJSICIzRWSVddu71TYHish+EbnZzs9mt/I+2T3uq+L3SPoL4IyhhaQkCa8vd28K7OhMm/F/dRiPPjl+ACP79+LnM+o45NMmxk6x7YwjIsnAvcBkoBq4RkSqWy02Gaiy/q4H7uvEurcAs4wxVcAs63FLvwP+G/UPFGPlhdls3n2II4HuHxw+D3R+jOiVkcr48t684eJCJRDHUw14QXKS8KOLR7J5z2Hun73a6XASlp17/4lAvTFmjTHGBzwJTG21zFTgERM2B8gXkX4drDsVeNi6/zBwaWRjInIpsAZYas9Hip2KwixChh41Kw7P7eGdX80ThhezfMu+qIzgbAdNfznvxIoCpozux/2zV7t2P/E6O/f+UmBji8cN1nOdWaa9dUuMMY0A1m0xgIhkA9+jg1kqReR6EZknIvO2bXNvL+1otADzUp0KhAsVwLVXK76Apr/c4NYLRyACP59R53QoCcnOM05bR1brZhnHWqYz67Z2J/A7Y8z+9hYyxjxgjBlvjBlfVFTUwSadExkCvyctwMITRnmnUBlSlMOAgkzXFip6peIOpfmZ3HDWEF6sbeT9NTucDifh2Ln3NwADWjwuAzZ3cpn21t1qpciwbiNnmJOAX4rIOuAbwPdF5KYefwqH5GelkZ+VytoetADz2jhUIsKEYcW8u3q7K8d6CoS0TsUtvnTmEPrnZXDnC8sIhrSJcSzZuffPBapEpEJE0oCrgemtlpkOXGe1AjsZ2GOltNpbdzowzbo/DXgewBhzhjGm3BhTDvwe+Jkx5k/2fTz7lffJ7tGVihfG/mrtnOHFHPaHeM+Fv0A1/eUemWnJfP+iESxr3MsTH2xwOpyEYtsZxxgTAG4CXgHqgKeMMUtF5AYRucFabAbhivV64EHgK+2ta61zNzBRRFYBE63HnlRR2NNCxTtNiiNOHtyHjNQkV6bAmtNfHko5xrOLavpx8uACfv3qCnYd8DkdTsKwde83xswwxgw1xgwxxtxlPXe/MeZ+674xxtxovV5jjJnX3rrW8zuMMecaY6qs251tvO8dxphf2/nZYqG8Tzab9xzudqrHH/BWRT1ARmoypw0p5PXlTa7rOd2c/kry1ncer0SEOy4Zyb7DAX4zU8cFixXd+12s3BpYcn03Z4H0Wp1KxDnDi2nYdYjV29ptkxFzfk1/uc7wvr34zMmDePz9DSzdrOOCxYIWKi4WaQHW3Z71XmtSHHGO1bTYbb3rfZr+cqVvThxK76w07piu44LFgu79LlYeaVbcjRZgwZAh5NHBDUvzMxneN9d1hYo/qOkvN8rLTOW7F4THBXt+YesGqCradO93sV4ZqfTJTutWZf3RSmNvpmLOGV7MvHW72HvY73QozQIe/87j2ZXjBnBcWR4/m1HH/iMBp8PxNC1UXK6iMJs1PShUvNakOGLC8GICIcNbK90zKoJPOz+6VlJSuNK+ad8R7pm1yulwPE33fpcbXJTNmm5USPs9PrjhmAH59M5K5bVlW50OpVnkO/dqQR7vxgzszVXjB/DQO2tZuXWf0+F4lu79LldZnMP2/T52H+xaO3uvDxmSkpzEhOElvL68qfmzOi2S/krR1l+u9b3Jw8nJSOGHzy3RSnubePOM4yGVxTkA1Dd17Wol0rvbyye4idUl7D0cYO7aj3VVcoTXC3IvKMhO47vnD+eDtTt5buEmp8PxJN37Xa6yKBfoeqHi9ToVgDOHFpKeksTMOnekwHweTzl6xdUnDOC4Afnc9dJy9hxyT0MPr9C93+VKe2eSnpLU5UIlEQY3zEpL4fTKQmYu2+qKVIZfZ36MC0lJwk+njmLHgSP8buZKp8PxHO+ecTwiOUkYXJRDfRcr6xNlcMOJ1SU07DrE8i3OV7wGNP0VN2rK8vj0SYN45L11LNmkPe2jSff+OFBZnNPt9JfXe3efO6IEEZjpglZgkfRXiodm2/SymycNo3dWGj98bgkhHR4/arx9xvGIyqIcNu0+xCFf5weWTJTmrUW56Rw/IN8VhUpkqgERLVTiQV5WKj+4aAQLN+7mMR0eP2q8fcbxiMriHIyhSwMoJlJLpInVJSzetIfGPc7OSe4PhDzd2s6LLhtTyqlD+vDL/y6nae9hp8PxBO+fcTwg0qy4K4WKL4H6TEyqLgHgtTpnxwILhLw3f43XiQh3XVbDkWCIO19c5nQ4nqBHQBwoL8wiSbrWrDgyDLvX018Qnru+ojDb8RSYz6OjQntdRWE2Xz2nkpdqG105+Vu80SMgDqSnJDOoT3aXCpVEaFIcISJMrC7hvdXb2efgAJP+QIi0BLgy9KLrzxpMZXEOP3xuCQd9OuBkT3j/jOMRQ4q61gIs0fpMTKwuwR80zHZwgMlAyJCSAIW4F6WnJHPXpaPYtPsQf3hNB5zsCT0C4kRlcQ7rdhxo7gvRkaP9VBLjXzx2YG8KstMcTYF5dabNRHHS4D5cNX4Af31nrc4S2QOJccbxgMriHPxBw/qdnZtauLlJscf7qUQkJwkTR5Twel0Th/2db3odTf6A1qnEu1svHE7vrFS+92xtp3/AqY/SIyBOdHVgyURqUhxx4eh+7DsS4O1V2x15f69O35xI8rPS+PHUUSzZtJcH3l7jdDhxSY+AODGkKDy1cFcLlURoUhxx6pA+5GelMmNxoyPvH25SnDjft1ddWNOPC0b25fevrepSM34VpoVKnMjNSKVvrwxWd7JQ8SXAKMWtpSYncX51X15bttWRFJhP01+e8eNLR5KZmswtz9bqEC5dpEdAHKks7vzAkoEEHYbdyRSYPxhKmDosryvOzeC2KdXMXbeLf85Z73Q4cUWPgDhSWZzD6qb9nRrm3R8MkSThCuxE4mQKLBAyOpikh1wxtpSzhhbxi5eXs7GTDWSUFipxZUhxDgd8QRr3dDxGUaL27o6kwGY6kALT9Je3iAg/u7wGAW7992JXzNkTD/QIiCOVRZ1vAeYPmISqT2npwtH92O9ACswfDHl+qoFEU5qfyfcvGsE79dt5VNNgnaJHQBzpSrPiRD7BRVJgL9Vujun7+oOGVE1/ec61Jw7krKFF/GzGctZtP+B0OK6XmGedOFWYk0ZeZmqnKuv9wVDC5vebW4HFuCNkIEFTjl4nIvziitGkJgvffnoRQW0N1i49AuKIiHR6Fkh/MLGHYY+kwN6K4VhgvqBJ2KtDr+ubl8FPLh3F/PW7eOAt7RTZHluPABG5QERWiEi9iNzSxusiIvdYr9eKyNiO1hWRAhGZKSKrrNve1vMTRWS+iCy2bifY+dmcUlmU06m+KonevNWJVmCRmR+VN11yXH8urOnL72aupK5xr9PhuJZtR4CIJAP3ApOBauAaEalutdhkoMr6ux64rxPr3gLMMsZUAbOsxwDbgYuNMTXANOCfNn00R1UW57DjgI9dB3ztLudP8MENnUiBBRI45ZgIRISfXlpDr8xUvvXUIo4EnBljzu3s/Fl1IlBvjFljjPEBTwJTWy0zFXjEhM0B8kWkXwfrTgUetu4/DFwKYIz50BgTqZldCmSISLpNn80xQ/vmAlC3pf1fSjoOFUw5LpwCmxWjGSH9mv7yvILsNO6+vIa6xr38+pUVTofjSnYeAaXAxhaPG6znOrNMe+uWGGMaAazb4jbe+wrgQ2PMkdYviMj1IjJPROZt2+bc3BvdNap/LwCWbGp/aG5fgtepAJw6pJC+vTJ4dkGD7e9ljEnYvkGJ5rzqEj5z8iAefHstb67QmSJbs/MIaCsP0LrZxLGW6cy6bb+pyEjgF8CX2nrdGPOAMWa8MWZ8UVFRZzbpKn1y0inNz6S2of1CJTwLYWKf4JKThMvHljJ75Taa9nbcYbQnmmfa1PRXQvjBRSMYVpLLzU8vYtu+j/12TWh2nnUagAEtHpcBrTsOHGuZ9tbdaqXIsG6bfyqISBnwH+A6Y8zqKHwGV6opzevwSsUfDCXUCMXHcsW4MoIhw3MLN9n6Ps1jrWn6KyFkpCZzzzVj2Hc4wM1PL9JBJ1uw8wiYC1SJSIWIpAFXA9NbLTMduM5qBXYysMdKabW37nTCFfFYt88DiEg+8BJwqzHmXRs/l+NqyvJYt+Mgew4dez52f0jTXxCehnnMwHyemd9g6zAbvgScvybRDeubyw8vGsHsldt46N21TofjGrYdAcaYAHAT8ApQBzxljFkqIjeIyA3WYjOANUA98CDwlfbWtda5G5goIquAidZjrOUrgdtEZKH111Z9S9yrKc0DYGk7Vys6C+FRnxhXxsqt+1myyb5moP7mqQb06jCRfPrkQUysLuEXLy/vMHuQKGw96xhjZhhjhhpjhhhj7rKeu98Yc7913xhjbrRerzHGzGtvXev5HcaYc40xVdbtTuv5nxpjso0xx7f482QtWqRQqW2vUAmGSEvRExzAlNH9SUtJ4pn5GzteuJsi6a8ULcgTiojwyytG0yc7na88tqDd7EGi0CMgDvXOTmNAQSaL26ms1ybFR+VlpnL+yL48v2izbX0LEnH6ZhXWOzuNez81hs27D/HtpxYmfP2KHgFxqqY0j8XtXqlonUpLV4wtZfdBP28st+fi9Widil4dJqJxgwr4wUUjeK2uiftme7aNUKfoWSdO1ZTms2HnQXYfbLtnvfaZ+Kgzqooo6ZXOM/Pt6bPiT8Dpm9VHffbUcqaM7sdvXl3Bu/Wxn3nULfQIiFOjy8L1Kse6Wkn0YVpaS04SLhtTxhsrttnSr0DrVFRkNOPBRTl87YkPadxzyOmQHKFHQJwa1b/9QiWg6a+P+cS40nCflQ+j32dF018KIDs9hfs/PY7D/iBfeWxBQo4PpmedOJWXlcqgPlnHrKzX9NfHVRbnMn5Qb/45Z33U58TwBzT9pcIqi3P41ZXH8eGG3dz6bOJNQ6xHQBw7VmW9McYahl1/Nbf2+dMr2LDzILPqtkZ1u5FhWjT9pQAurOnHN88byr8/3JRwFfd6BMSxmtI8GnYdYmerYfCDIYMx2ry1LZOqSyjNz4x6D2hNf6nWvnZuJRcf159fvryCl5dscTqcmNGzThyrOUZlvV/HoTqmlOQkpp06iDlrdrJ0c/R6QEfSX1qQqwgR4VefGM1xA/L55r8WJkyPez0C4tgoq2d9651Vx6Fq31XjB5KVlszf310XtW1GCvJEnm1TfVxGajIPfmYc+Vmp/N8j82wfLdsN9AiIY70yUqkozKa2YfdHnvdrKqZdeVmpfGJcGdMXbo5a8+JAKPyd68yPqrXiXhn8ddp49hzyc91DH3h+KBctVOJcTWnex1qANQ/Drlcqx/TZU8vxBUM8Omd9VLbn0/SXasfI/nk88JnxrN62ny8+PJdDPu82NdYjIM6NLstj857DbN9/9Be3jkPVscFFOUwYXsxj76+Pyhz2mv5SHTm9qpDfXzWGeet3cdPjC5qPU6/RIyDORepVWlbWa0ukzvnC6RVs3+/jhUWt547rusgJQtNfqj0Xje7HT6aOYtbyJr73bK0nB5/UQiXOjezfCxGo3Xi0UNFxqDrn1CF9GFaSy9/eWdvjDmrNV4d6paI68OmTB/GtiUP594JN/PSlOs91jtQjIM7lZqQyom8v3qnf1vycP6B1Kp0hIlx/5mCWb9nHS4sbe7St5vSXfueqE746oZLPn1bBQ++u5c4XlnmqYNEjwAMmjSxh3vpdzfUqPv3V3GmXjilleN9cfvXKiubK9u7QeizVFSLCbVNG8MXTK/jH/9bxw+eWeCYVpkeAB0yq7osx8Nqy8NAjgcgJTvP7HUpOEr43eTjrdxzk8fe73xIsEAwhEt6eUp0hIvzgohF8+ewhPPb+Bm75d23Ux6RzghYqHjCiXy5lvTN51SpUtEd915w9tIhTBvfhntfr2Xe4e30IfDoqtOoGEeG75w/ja+dW8dS8Br7z9KLmH4XxSo8CDxARzh/Zl3dWbWf/kYCmYrpIRLj1wuHsPODjgbfWdGsb4QE89ftWXScifGviUL49MTwA5RcentftHzduoEeBR0yqLsEXDDF7xTZtUtwNo8vyufi4/jz49hq2dmMoDZ0UTfXUV8+t4meX1fBO/XauvP89Nu+Oz0m+tFDxiPHlBRRkp/HK0i3apLibvjNpGMGQ4fevrezyuv6g0WHvVY9de9JA/v7ZE2jYdYhL7303Lgeh1KPAI5KThPNGFPPG8iYOHgn3ENf0V9cM7JPFp04axL/mbmTFln1dWlfTXypazhxaxLNfPpXU5CSuvP89Xl7Ss+busaZHgYdMqu7LviMB3loV7rOiFfVd97Vzq8jLTOUb/1rYpeFbNP2lomlY31z+c+OpDO2byw2PLuBHzy+JynBCsaBnHQ85vaqQrLRkZtU1AdqkuDsKstP4zSePo65xLz99aVmn1wto+ktFWXFuBk9/6RS+cHoFD7+3nsv//D/WbNvvdFgd0qPAQzJSkzlraBGH/Jr+6okJw0u4/szBPDpnAy/Vdi714AuG9PtWUZeWksRtU6r527TxNO45xJQ/vsMz8xtc3QNfjwKPOX9k3+b7mv7qvpsnDeP4Afnc8mwtG3Yc7HD5cJ2KXhkqe5w7ooQZXz+DUaV53Pz0Ij7ztw9Y7dKrFj3reMw5w4qbR8rVHH/3paUk8cdrxoDATU8s6HAIF79eqSib9cvL5In/O5mfTB3JoobdXPD7t/j1KytcNzeLHgUek5eVyilD+gCQmqT/3p4YUJDFrz4xmtqGPfxo+tJ2x2YKNynWQlzZKzlJ+Mwp5bz+7bO5eHR//vRGPRN/N5v/fNjgmp74etbxoP87YzCXjy0lSSvqe+yCUf340lmDeeKDDVz/z2P3dNYrFRVLRbnp/Paq43ny+pPJSU/hm/9axDm/eZNH50Rn0rmeEDdX+Nht/PjxZt68eU6HoVzOGMOjc9ZzxwvLGFyYzV+njWdQn+yPLDPlj29TkpvB3z57gkNRqkQVChlmLW/iT2/Us2jjbopz05l2ajmXHNefAQVZtryniMw3xoxv6zVbf1qJyAUiskJE6kXkljZeFxG5x3q9VkTGdrSuiBSIyEwRWWXd9m7x2q3W8itE5Hw7P5tKHCLhlMM/P38i2/Yf4ZI/vcv/6rd/ZJmApr+UQ5KShInVJTz3lVN5/IsnUVWSw69eWcEZv3yDS+99l7+9s5Yte7o+9FB32XalIiLJwEpgItAAzAWuMcYsa7HMhcBXgQuBk4A/GGNOam9dEfklsNMYc7dV2PQ2xnxPRKqBJ4ATgf7Aa8BQY8wxrwX1SkV11YYdB/niI3NZ1bSfUf3zOHNoIWdWFfG9Z2sZVZrHn64d2/FGlLLZxp0HebG2kRcWbWZZ414ABvXJoqY0j+PK8hldlseo0jyy01O6tf32rlTsLFROAe4wxpxvPb4VwBjz8xbL/AV40xjzhPV4BXA2UH6sdSPLGGMaRaSftf6w1tsXkVesbbx3rBi1UFHdsf9IgH+8u5bZK7exYMPu5jkwLhtTyu+uOt7Z4JRqZfW2/cxctpVFG3dT27CHTdZAleeNKOav07qXrm2vUOleMdU5pcDGFo8bCF+NdLRMaQfrlhhjGgGsgqW4xbbmtLGtjxCR64HrAQYOHNiFj6NUWE56CjdNqOKmCVXsPeznf/U7mLNmx0f6CCnlFkOKchhyVk7z4+37j1DbsJusNHtO/3YWKm0lmFtfFh1rmc6s2533wxjzAPAAhK9UOtimUu3qlZHKBaP6csEoLVBUfCjMSWfC8BLbtm9nRX0DMKDF4zJgcyeXaW/drVbaC+u2qQvvp5RSykZ2FipzgSoRqRCRNOBqYHqrZaYD11mtwE4G9liprfbWnQ5Ms+5PA55v8fzVIpIuIhVAFfCBXR9OKaXUx9mW/jLGBETkJuAVIBl4yBizVERusF6/H5hBuOVXPXAQ+Fx761qbvht4SkS+AGwArrTWWSoiTwHLgABwY3stv5RSSkWfdn7U1l9KKdUljnV+VEoplVi0UFFKKRU1WqgopZSKGi1UlFJKRU1CV9SLyDZgfQ82UQhs73Ap99B47aXx2kvjtVdX4h1kjClq64WELlR6SkTmHasFhBtpvPbSeO2l8dorWvFq+ksppVTUaKGilFIqarRQ6ZkHnA6gizRee2m89tJ47RWVeLVORSmlVNTolYpSSqmo0UJFKaVU1Gih0g0icoGIrBCRehG5xel4WhORh0SkSUSWtHiuQERmisgq67a3kzG2JCIDROQNEakTkaUi8nXreVfGLCIZIvKBiCyy4r3Tet6V8UaISLKIfCgiL1qP3R7vOhFZLCILRWSe9ZxrYxaRfBF5RkSWW/vyKW6NV0SGWd9r5G+viHwjGvFqodJFIpIM3AtMBqqBa0Sk2tmoPuYfwAWtnrsFmGWMqQJmWY/dIgB82xgzAjgZuNH6Tt0a8xFggjHmOOB44AJrPiC3xhvxdaCuxWO3xwtwjjHm+Bb9J9wc8x+Al40xw4HjCH/XrozXGLPC+l6PB8YRnnrkP0QjXmOM/nXhDzgFeKXF41uBW52Oq404y4ElLR6vAPpZ9/sBK5yOsZ3YnwcmxkPMQBawADjJzfESngl1FjABeDEe9glgHVDY6jlXxgz0AtZiNX5ye7ytYpwEvButePVKpetKgY0tHjdYz7ldiQnPqol1W+xwPG0SkXJgDPA+Lo7ZSiUtJDyd9UxjjKvjBX4PfBcItXjOzfECGOBVEZkvItdbz7k15sHANuDvVorxryKSjXvjbelq4Anrfo/j1UKl66SN57RddhSISA7wLPANY8xep+NpjzEmaMKpgzLgRBEZ5XBIxyQiU4AmY8x8p2PpotOMMWMJp5pvFJEznQ6oHSnAWOA+Y8wY4AAuSXW1x5qu/RLg6WhtUwuVrmsABrR4XAZsdiiWrtgqIv0ArNsmh+P5CBFJJVygPGaM+bf1tKtjBjDG7AbeJFyH5dZ4TwMuEZF1wJPABBF5FPfGC4AxZrN120Q4338i7o25AWiwrlgBniFcyLg13ojJwAJjzFbrcY/j1UKl6+YCVSJSYZXyVwPTHY6pM6YD06z70wjXW7iCiAjwN6DOGPPbFi+5MmYRKRKRfOt+JnAesByXxmuMudUYU2aMKSe8v75ujPk0Lo0XQESyRSQ3cp9w3n8JLo3ZGLMF2Cgiw6ynzgWW4dJ4W7iGo6kviEa8TlcSxeMfcCGwElgN/MDpeNqI7wmgEfAT/gX1BaAP4YraVdZtgdNxtoj3dMIpxFpgofV3oVtjBkYDH1rxLgFut553ZbytYj+boxX1ro2XcB3FIutvaeQ4c3nMxwPzrP3iOaC3y+PNAnYAeS2e63G8OkyLUkqpqNH0l1JKqajRQkUppVTUaKGilFIqarRQUUopFTVaqCillIoaLVSUUkpFjRYqSimloub/AUAgsMw6YipfAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#| slow\n",
    "learn = synth_learner()\n",
    "with learn.no_logging(): learn.fit_sgdr(3, 1)\n",
    "test_eq(learn.n_epoch, 7)\n",
    "iters = [k * len(learn.dls.train) for k in [0,1,3,7]]\n",
    "for i in range(3):\n",
    "    n = iters[i+1]-iters[i]\n",
    "    #The start of a cycle can be mixed with the 0 of the previous cycle with rounding errors, so we test at +1\n",
    "    test_close(learn.recorder.lrs[iters[i]+1:iters[i+1]], [SchedCos(learn.lr, 0)(k/n) for k in range(1,n)])\n",
    "\n",
    "learn.recorder.plot_sched()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7a6c562",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "@patch\n",
    "@delegates(Learner.fit_one_cycle)\n",
    "def fine_tune(self:Learner, epochs, base_lr=2e-3, freeze_epochs=1, lr_mult=100,\n",
    "              pct_start=0.3, div=5.0, **kwargs):\n",
    "    \"Fine tune with `Learner.freeze` for `freeze_epochs`, then with `Learner.unfreeze` for `epochs`, using discriminative LR.\"\n",
    "    self.freeze()\n",
    "    self.fit_one_cycle(freeze_epochs, slice(base_lr), pct_start=0.99, **kwargs)\n",
    "    base_lr /= 2\n",
    "    self.unfreeze()\n",
    "    self.fit_one_cycle(epochs, slice(base_lr/lr_mult, base_lr), pct_start=pct_start, div=div, **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e2ee4ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2.428970</td>\n",
       "      <td>1.740237</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2.019952</td>\n",
       "      <td>1.616970</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fine_tune(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23fc3926",
   "metadata": {},
   "source": [
    "## Resume training from checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c054e24a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "class InterruptCallback(Callback):\n",
    "    def __init__(self, epoch):\n",
    "        self._interupt_before = epoch\n",
    "    def before_epoch(self):\n",
    "        if self.epoch == self._interupt_before:\n",
    "            raise CancelFitException"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b99da73f",
   "metadata": {},
   "source": [
    "To enable resuming from checkpoint make sure to save model and optimizer state. This can be done using [SaveModelCallback](https://docs.fast.ai/callback.tracker.html#SaveModelCallback.html) setting `(with_opt=True)`. If training is interrupted define `learn` using the same parameters as before, load model from checkpoint and pass `start_epoch` to `fit` call. The training will be resumed from `start_epoch` with properly scheduled  `lr`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca1a6339",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>18.930223</td>\n",
       "      <td>14.100439</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>17.092665</td>\n",
       "      <td>10.603369</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Better model found at epoch 0 with valid_loss value: 14.100439071655273.\n",
      "Better model found at epoch 1 with valid_loss value: 10.603368759155273.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>11.456764</td>\n",
       "      <td>10.057186</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>10.287196</td>\n",
       "      <td>8.694046</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>9.585465</td>\n",
       "      <td>8.422710</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAD4CAYAAAAdIcpQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAtaklEQVR4nO3deXgV5d3/8fc3e0ISkpAEEkiAQFiSsBoWW1RcWEUjti5o69JaXPs8/VmrWB9ttWqttmq1Lo+2tqitFncECgIFcUP2JQkEQliyL0AWCNnv3x8ZfdIYcw4hyZzl+7quc805c+Y+5zNhrvNl7pm5R4wxKKWUUp3xsTuAUkop16fFQimllENaLJRSSjmkxUIppZRDWiyUUko55Gd3gO4QHR1thgwZYncM5cG2bt1aYYyJ6e3v1W1b9aTT2a49olgMGTKELVu22B1DeTAROWzH9+q2rXrS6WzX2g2llFLKIS0WSimlHNJioZRSyiEtFkoppRzSYqGUUsohp4qFiMwWkRwRyRWRRR28LyLyjPX+LhGZ6KitiFwhIlki0iIi6e0+715r+RwRmXUmK6iUUurMOSwWIuILPAfMAVKABSKS0m6xOUCy9VgIvOBE20zgcmBDu+9LAa4GUoHZwPPW5yillLKJM9dZTAZyjTF5ACLyJpABZLdZJgN41bSOd75RRCJEJA4Y8m1tjTF7rHntvy8DeNMYUw8cFJFcK8MXXVtF1dMamlqoOFH/9aO8pp6KEw34+Qix4YHEhAZZ00AiQvw7+jdXSrk4Z4rFQCC/zesCYIoTywx0sm1H37exg8/6DyKykNa9GBITEx18pOpumYVV3Pd+JocqTlJ1qtHpdv6+QkxoIPERwYyOCyc1PpzU+L6MGBBKoJ/uQIJu28o1OVMsOvpvYPs7Jn3bMs607cr3YYx5CXgJID09Xe/g1IuW7izi7rd3EhkSQMb4eKJDA4kJCyQ6NJDo0ICvnze1GMqq6yivqaf8RD1l1f83PXLsJO9tL+S1ja0XkPr5CMNjQ0mJDyctvi/TkqNJjg31yr0Q3baVK3KmWBQACW1eDwKKnFwmwIm2Xfk+ZYPmFsPvP8rhhfUHmDQkkuevPYuYsMBO24TGhJIUE9rhey0thiPHaskuriarqIqsomo+3V/Bu9sKAYjvG8R5I2M4b0Qs3x3ej7Ag/25fJ6WUc5wpFpuBZBEZChTSevD5mnbLLAXusI5JTAGqjDHFIlLuRNv2lgL/EJEngXhaD5pvcnaFVM+ormvkZ2/u4N97y1gwOZEHL00lwO/Mzrz28RGGRPdhSHQf5o6J+3p+UeUpPt5Xzsc55Xy4s5g3NuXj5yOcNTiS80bGMDctjiHRfc50lZRSp8FhsTDGNInIHcAqwBd4xRiTJSK3WO+/CKwA5gK5QC1wY2dtAURkPvAsEAMsF5EdxphZ1mcvofUAehNwuzGmuVvXWp2WvPIT/OTVLRw+WstvLkvjh1MH9+j3xUcEs2ByIgsmJ9LY3MK2w8dZv6+c9TnlPL4yh8dX5jBpSCTfP2sQF4+NJzTQI8bDVMqlSesJTO4tPT3d6MicPWN9Thk/fWM7/r4+PH/tRKYm9bM1T3HVKd7fXsRbW/PJKz9JsL8vc9IG8P2zBjE1qR8+Pj1zjENEthpj0h0v2b1021Y96XS2a/0vmfpWKzOLue3v2xg5IJyXfngWCVEhdkcirm8wt04fxi3nJbEjv5K3txawdGcR724vZGBEMFemJ3Dd2YOJ7BNgd1SlPIoWC9WhytoG/uf9TFLj+/LPm6cSEuBam4qIMCExkgmJkdw/L4XV2aW8tbWAp9bs48WPD3DVpAR+PG2oSxQ4pTyBa/0CKJfx2xV7OV7byOIfTXa5QtFekL8vl4yL55Jx8ewvreF/N+Tx9y8P89rGw8wbG8fN5w4jJT7c7phKuTUdSFB9wxcHjvLPLfncdM5QUuP72h3ntCT3D+P3V4xjw93n86PvDmFNdilzn/mE617ZxBcHjtodTym3pcVC/Ye6xmbue283iVEh/OzCEXbH6bK4vsHcd3EKny+6kF/MGkl2URULXt7Ida9sYk9xtd3xlHI7WizUf3huXS55FSd5ZH4awQHuP/xG3xB/bj9/OJ/ecwH3zR3NjiPHmfvMJ9z11k6Kq07ZHU8pt6HFQn0tp6SGF9Yf4PIJAzknOcbuON0qyN+Xn5ybxIa7z+cn5ySxdEcR059Yz+Mr91Jd5/zYVkp5Ky0WCmgdeuPed3cRFuTH/8xrPwK954gICeCXc0ez9ufnMSdtAM+vP8D0J9az+PNDNDS12B1PKZelxUIB8PcvD7PtSCX3z0shyguuUUiICuHpqyfw4R3TGNk/jF8tzeLtrQV2x1LKZbn2OZGqV5RU1fG7lTmckxzN/AnfGA3eo40Z1Jd//GQKn+yvsP3qdKVcmRYLxQMfZNLU0sIjl43xyiHBRYRzR3jWMRqlupt2Q3m5lZklfJRdys8uGkFiP73aWSnVMS0WXqylxfD4qr2M7B/Gj6cNtTuOclJTcwuL3tnFe9v1GIvqPdoN5cXW5ZSRV36SP149Hn9f/X+Du2hqMRw+WstbWwsID/LnwtH97Y6kvID+QnixP39ykLi+Qf9x4yHl+oL8fXn5+nRS48O57e/b+DJPhzFRPU+LhZfKLKzii7yj3PCdIbpX4YZCA/34242TGRQZzE2Lt5BZWGV3JOXh9FfCS73y6UH6BPhy9eREu6OoLorqE8BrP55CWJAfN/x1EwcrTtodSXkwLRZeqKSqjqU7i7hyUgJ9g/3tjqPOQHxEMK/dNAVj4Ad//pKSqjq7IykPpcXCCy3+4hAtxnDjd/QMKE8wLCaUxT+aTGVtAze9upnahia7IykPpMXCy5ysb+LvGw8zK3WAXlfhQdIG9uXZayaQXVTNnf/cSUuLsTuS8jBaLLzMO9sKqK5r4qZzdK/C01wwqj+/nDualVkl/GF1jt1xlIfR6yy8SHOL4ZVPDzIhMYKzBkfZHUf1gB9PG8qB8hM8t+4Aw2JCuXziILsjKQ+hexZeZM2eUg4dreWmaUl2R1E9RER48NI0piZFseid3Ww5dMzuSMpDaLHwIn/55CADI4KZlapX/HqyAD8fXvzBWcRHBHHza1sprNQ7Aqozp8XCS+zMr2TToWPc+N0h+OlFeB4vIiSAv9wwifqmFm77+zbqm5rtjqTcnP5qeIm/fHqQsEA/rpqUYHcU1UuGxYTy+yvGsjO/koeX7bE7jnJzWiy8QGHlKZbvLubqyQmEBelFeN5kdlocC89N4rWNh3WUWnVGtFh4gcWfHwLg+u8MsTWHssfds0YyeWgU9767m70l1XbHUW5Ki4WHq2ts5o1NR5idNoBBkXoRnjfy8/XhTwsmEBbkz62vb6OmrtHuSMoNabHwcKuzS6mpa+IaHTDQq8WGB/HcNRM5cqyWX7y1C2P0Cm91erRYeLj3txcyIDyIqUn97I6ibDZ5aBT3zB7JyqwS/v7lEbvjKDfjVLEQkdkikiMiuSKyqIP3RUSesd7fJSITHbUVkSgRWS0i+61ppDXfX0QWi8huEdkjIvd2x4p6o6Mn6vl4XzkZE+Lx9RG74ygXcNO0JM5JjuY3y7LZV1pjdxzlRhwWCxHxBZ4D5gApwAIRSWm32Bwg2XosBF5wou0iYK0xJhlYa70GuAIINMaMAc4CbhaRIV1dQW/24c4imloM8ycMtDuKchE+PsIfrhxHWJAf//XGduoa9foL5Rxn9iwmA7nGmDxjTAPwJpDRbpkM4FXTaiMQISJxDtpmAIut54uBy6znBugjIn5AMNAA6CkcXfDejiJGx4UzakC43VGUC4kNC+KJK8axt6SG367Q6y+Uc5wpFgOB/DavC6x5zizTWdv+xphiAGsaa81/GzgJFANHgN8bY74xwI2ILBSRLSKypby83InV8C4Hyk+wM7+Sy3Wvwu30xrZ9/shYfvTdoSz+4jBrskt75DuUZ3GmWHTU2d3+VIpvW8aZtu1NBpqBeGAo8HMR+cbId8aYl4wx6caY9JiYGAcf6X0+2F6Ij8Cl4+PtjqJOU29t2/fMGcnouHB+8fZOSqv1Dnuqc84UiwKg7RgRg4AiJ5fprG2p1VWFNS2z5l8DrDTGNBpjyoDPgHQnciqLMYb3dhTy3eHR9A8PsjuOclGBfr48u2A8pxqbueutnXo6reqUM8ViM5AsIkNFJAC4GljabpmlwHXWWVFTgSqra6mztkuB663n1wMfWM+PABdYn9UHmArs7eL6eaWth4+Tf+yUHthWDg2PDeO+i1P4ZH8Fr+vptKoTDouFMaYJuANYBewBlhhjskTkFhG5xVpsBZAH5AIvA7d11tZq8xgwQ0T2AzOs19B69lQokElrsfmrMWbXma6oN3l3eyHB/r7MSh1gdxTlBn4wJZFzkqN5dPkeDh89aXcc5aKculOeMWYFrQWh7bwX2zw3wO3OtrXmHwUu7GD+CVpPn1VdUN/UzPJdxcxK7U+fQL0RonJMRHj8+2OZ+dQG7nprJ28uPFuvy1HfoFdwe5h1e8upOtXIZdoFpU5DXN9gHrw0lc2HjvOXT/PsjqNckBYLD/Pe9gKiQwOZNjza7ijKzcyfMJCZKf35/ap9enW3+gYtFh6ksraBdXvLyRgfr3fDU6dNRHj08jGEBvlx55IdNDa32B1JuRD9RfEgy3cX09DcomdBqS6LDg3k0flpZBZW88L6A3bHUS5Ei4UHeW9bIcmxoaTG6/Aequtmp8Vxybh4nv33fvZrd5SyaLHwEEeO1rLl8HHmTxyIiJ7Jos7Mry5JITTQj7vf2UVzi16sp7RYeIz3dxQCkDFeu6DUmYsODeRXl6Sy/Ujl17flVd5Ni4UHMMbw/vZCpiZFMTAi2O44ykNkjI/nglGxPLEqh/xjtXbHUTbTYuEBdhVUkVdxkssnDLI7ivIgIsLDl6Xh6yPc++5uHTvKy2mx8AD/yizBz0d0eA/V7eIjglk0ZxSf5lbw1tYCu+MoG2mxcHPGGFZmFnP2sH70DfG3O47yQNdMTmTy0CgeXpZNmQ5l7rW0WLi5nNIaDh2tZU5anN1RlIfy8REeu3wMdU0tPLQs2+44yiZaLNzcyswSRGBGSn+7oygPlhQTyh3nD2fZrmLW5ZQ5bqA8jhYLN7cys4RJg6OICQu0O4rycDefl8SwmD7c/34mpxqa7Y6jepkWCzd2qOIke0tqmJWmB7ZVzwv08+XR+WMoOH6KZ/693+44qpdpsXBjK7NKAJiVql1QqndMSerHFWcN4uUNeewtqbY7jupFWizc2MrMEsYO6sugyBC7oygv8su5owkP9ueX7+6mRYcC8RpaLNxUcdUpduRX6rUVqtdF9gngvrmj2Xakkjc26327vYUWCzf1UVYpALP1eIWyweUTB3J2Uj8e+9deKk7U2x1H9QItFm7qX5nFJMeGMiwm1O4oyguJCA/PT6OusZnfrthrdxzVC7RYuKGjJ+rZdPCY7lUoWw2LCeUn5yTxzrYCNh86Zncc1cO0WLihNXtKaTHaBaXsd8cFwxkYEcz972fSpLdh9WhaLNzQyswSEqKCSYnTO+Ipe4UE+HH/vBT2ltSw+IvDdsdRPUiLhZuprmvk09wKZqcO0DviKZcwK7U/00fG8NTqfZTqQIMeS4uFm1m3t4zGZqNdUMpliAgPXppKQ3MLjyzfY3cc1UO0WLiZlZklxIYFMiEh0u4oSn1tcL8+3HreMJbuLOLzAxV2x1E9QIuFGznV0Mz6nHJmpQ7Ax0e7oJRruXX6MBKjQnjggywa9WC3x9Fi4UY+3lfOqcZm7YJSLinI35cH5qWQW3aCxZ8fsjuO6mZaLNzIqqwSIkL8mTw0yu4oSnXowtGxTB8Zw9Nr9lNWowe7PYkWCzfR0NTCmj2lXDS6P/6++s+mXJOI8KtLUmloauF3/8qxO47qRk796ojIbBHJEZFcEVnUwfsiIs9Y7+8SkYmO2opIlIisFpH91jSyzXtjReQLEckSkd0iEnSmK+ruvsg7Sk1dE7N14EDl4oZG9+HH5wzlnW0FbD183O44qps4LBYi4gs8B8wBUoAFIpLSbrE5QLL1WAi84ETbRcBaY0wysNZ6jYj4Aa8DtxhjUoHpQGPXV9EzrMwsISTAl2nJ0XZHUcqhO84fzoDwIH69NItmHcbcIzizZzEZyDXG5BljGoA3gYx2y2QAr5pWG4EIEYlz0DYDWGw9XwxcZj2fCewyxuwEMMYcNcZ49T0cW1oMa/eUct6IGIL8fe2Oo5RDfQL9uHfuKHYXVrFkS77dcVQ3cKZYDATa/msXWPOcWaaztv2NMcUA1jTWmj8CMCKySkS2icjdHYUSkYUiskVEtpSXlzuxGu5rd2EVZTX1zEjRO+J5A0/Zti8dF8/koVE8vnIvlbUNdsdRZ8iZYtHRCf3t9yu/bRln2rbnB0wDrrWm80Xkwm98iDEvGWPSjTHpMTExDj7Sva3OLsXXRzh/ZKzjhZXb85RtW0T49SWpVJ1q5KnV++yOo86QM8WiAEho83oQUOTkMp21LbW6qrCmZW0+62NjTIUxphZYAUzEi63ZU0r64Egi+wTYHUWp05ISH861Uwbz+pdHyCmpsTuOOgPOFIvNQLKIDBWRAOBqYGm7ZZYC11lnRU0Fqqyupc7aLgWut55fD3xgPV8FjBWREOtg93lAdhfXz+3lH6tlb0mNdkEpt3XnjBGEBvrx4IdZGKMHu92Vw2JhjGkC7qD1R3wPsMQYkyUit4jILdZiK4A8IBd4Gbits7ZWm8eAGSKyH5hhvcYYcxx4ktZCswPYZoxZfuar6p5WZ7fePvWi0VoslHuK7BPAz2eO4PMDR1mVVWJ3HNVFfs4sZIxZQWtBaDvvxTbPDXC7s22t+UeBbxyLsN57ndbTZ73emj2lJMeGMiS6j91RlOqyayYn8o8vj/Dw8j1MHxmrZ/W5Ib0U2IVV1Tby5cFjXKRdUMrN+fn68MAlKRQcP8WfP8mzO47qAi0WLmz9vjKaW4x2QSmP8J1h0cxJG8Bz6w5QXHXK7jjqNGmxcGGrs0uJDg1gQkKE3VGU6ha/nDuaZmN47F977Y6iTpMWCxfV0NTCxznlXDiqv967QnmMhKgQbj43iQ92FLHl0DG746jToMXCRX158Cg19U16vEJ5nFunD2NAeBAPfphNi44b5Ta0WLioNdmlBPn7MG24DhyoPEtIgB+L5rSOG/X2tgK74ygnabFwQcYY1uwpY9rwGIID9BRD5XkyxsczMTGCx1fmUFPn9YNKuwUtFi4ou7iawspTzNQuKOWhvrpJUsWJep5bd8DuOMoJWixc0JrsMkTg/FE6cKDyXOMSIvjexEG88ulBDh89aXcc5YAWCxe0Zk8pExIiiAkLtDuKUj3q7tkj8fMVHlm+x+4oygEtFi6muOoUuwurmJGit09Vnq9/eBC3nz+cj7JL+Sy3wu44qhNaLFzMmj2tI7XPSNEuKOUdfjxtKAlRwTz0YTZNzS12x1HfQouFi1mTXcqQfiEMiwm1O4pSvSLI35dfzhlNTmkNb2w6Yncc9S20WLiQE/VNfHHgKDNS+iOiV20r7zE7bQBTk6J4cvU+qmr1VFpXpMXChWzYV05Dc4sOHKi8jojwwLzWW7A+vVZvweqKtFi4kDXZpUSG+HPW4Ei7oyjV61Liw7lqUiKvfXGY3DK9Baur0WLhIpqaW1i7t4zzR8Xi56v/LMo7/XzmCIL9ffnNMj2V1tXor5KL2HToGFWnGpmpp8wqLxYdGsh/X5TMx/vKWbe3zO44qg0tFi7io6xSAv18OHeEDhyovNt1Zw8hKboPv1meTaOeSusytFi4AGMMq7NLOSc5mpAAp26LrpTHCvDz4b6LR5NXfpJXvzhsdxxl0WLhAr4aOHCGDhyoFAAXjIrlnORonl6zj6Mn6u2Oo9Bi4RI+yipFBC7UU2aVAr46lTaF2oZmnlytp9K6Ai0WLmB1dinpgyOJDtWBA5X6SnL/MH44dTBvbDpCdlG13XG8nhYLm+UfqyW7uFq7oJTqwP+7aAR9g/15aFkWxugtWO2kxcJma/aUAugos0p1oG+IPz+fOZKNecdYmVlidxyvpsXCZh9llZIcG8rQ6D52R1HKJS2YnMioAWE8smIPdY3NdsfxWlosbFRZ28CmQ8eYmapdUEp9G18f4YFLUig4foo/f5JndxyvpcXCRv/eW0Zzi9GrtpVy4DvDopmTNoDn1h2gpKrO7jheSYuFjT7KKqV/eCBjBva1O4pSLu+Xc0fTbAy//ZeOG2UHLRY2qWtsZsP+cmak9MfHR+9doZQjCVEh3HxuEh/sKGLLoWN2x/E6Wixs8lluBbUNzdoFpdRpuHX6MOL6BvHrD7NobtFTaXuTU8VCRGaLSI6I5IrIog7eFxF5xnp/l4hMdNRWRKJEZLWI7Lemke0+M1FETojIXWeygq7qo6xSwgL9mJrUz+4oSrmNkAA/7p07mszCapZsybc7jldxWCxExBd4DpgDpAALRCSl3WJzgGTrsRB4wYm2i4C1xphkYK31uq2ngH91YZ1cXnOLYe3eUqaPiiXAT3fulDodl4yNY/KQKJ5YlaO3YO1FzvxSTQZyjTF5xpgG4E0go90yGcCrptVGIEJE4hy0zQAWW88XA5d99WEichmQB2R1aa1c3PYjx6k40aBXbSvVBSLCry5NobK2gafW6LhRvcWZYjEQaLu/V2DNc2aZztr2N8YUA1jTWAAR6QPcAzzYWSgRWSgiW0RkS3l5uROr4TpWZ5fi7ytMHxljdxTlgtx52+4tqfF9uWZKIq9tPExOid6CtTc4Uyw6OlWn/ZGlb1vGmbbtPQg8ZYw50dlCxpiXjDHpxpj0mBj3+dE1xrAqq4Szh0UTHuRvdxzlgtx12+5tP58xktBAPx03qpc4UywKgIQ2rwcBRU4u01nbUqurCmv61T0UpwCPi8gh4GfAL0XkDidyuoXcshMcOlqrXVBKnaHIPgHcNXMEn+Ue1XGjeoEzxWIzkCwiQ0UkALgaWNpumaXAddZZUVOBKqtrqbO2S4HrrefXAx8AGGPOMcYMMcYMAZ4GHjXG/KnLa+hiPsq2Bg7Ue1codcYWTE5kdFw4v1mWTW1Dk91xPJrDYmGMaQLuAFYBe4AlxpgsEblFRG6xFltB6wHpXOBl4LbO2lptHgNmiMh+YIb12uN9lF3KuEF9GdA3yO4oSrk9P18fHspIpaiqjufW5dodx6M5dcNnY8wKWgtC23kvtnlugNudbWvNPwpc6OB7f+1MPndRUlXHzvxK7po5wu4oSnmMSUOiuHzCQF7ecJDvn5WgIzj3ED3Jvxct310MwJwxcTYnUcqzLJo7ikA/H369VA929xQtFr1o+a4iRseFMywm1O4oSnmU2LAgfjZjBB/vK//6uKDqXloseklh5Sm2Halk3ljdq1CqJ1x39mBG9A/loQ+zOdWgN0nqblosesmKXa1dUFoslOoZ/r4+PJSRRmHlKV5Yrwe7u5sWi16ybHcxYwb2ZXA/PfimVE+ZmtSPS8fF8+KGPA5VnLQ7jkfRYtEL8o/VsjO/kot1r0KpHnffxaMJ8PXhAT3Y3a20WPSCr86CuljPglKqx/UPD+LOGSPYsK+cFbv1yu7uosWiFyzbVcS4hAgSokLsjqKUV7ju7MGkxIXz0LIsaup0GPPuoMWihx2qOElmYTXzdK9CqV7j5+vDI/PTKKup56nV++2O4xG0WPSwr7qg5urxCqV61YTESBZMTuRvnx8kq6jK7jhuT4tFD1u2q5iJiREMjAi2O4pSXueeWaOIDAng/vczadF7dp8RLRY96ED5CfYUVzNvbLzdUZTySn1D/Lnv4tFsO1LJP/We3WdEi0UPWm5diDdXj1coZZv5EwYyNSmKx/61l/KaervjuC0tFj1o+a5iJg2J1OHIlbKRiPDwZWM41dDMQ8uy7Y7jtrRY9JD9pTXklNZoF5RSLmB4bCi3nz+cD3cWsW5vmeMG6hu0WPSQZbuKEYE5aQPsjqKUAm6dPozk2FD+5/1MTtbrXfVOlxaLHmCMYfnuYqYMjSI2XLuglHIFAX4+PPa9MRRWnuIPH+2zO47b0WLRA3JKa8gtO8HF2gWllEs5a3AUP5jaeu3FzvxKu+O4FS0WPWD5rmJ8BGanaheUUq7m7tmjiAkL5J53dtHY3GJ3HLehxaKbGWNYtquYs4f1IyYs0O44Sql2woP8eSgjjb0lNbz8SZ7dcdyGFotull1czcGKk1w8RruglHJVs1IHMDt1AE+v2U9uWY3dcdyCFotu9s7WQvx9hdl6FpRSLu03l6UREuDLXW/tolmHAnFIi0U3qmts5t3tBcxMHUBUnwC74yilOhETFsiDl6ayI7+Sv3yq3VGOaLHoRquySqisbWTBpES7oyilnHDpuHhmpvTn9x/tI7fshN1xXJoWi270jy+PkBgVwneG9bM7ilLKCSLCw/PTCPb35e63d2p3VCe0WHSTvPITfHnwGFdNSsDHR+yOo5RyUmxYEA9emsq2I5X89bODdsdxWVosusmbm/Px8xGuSB9kdxSl1GnKGB/PRaP788SqHPLKtTuqI1osukF9UzNvby3gwtGxxIbp8B5KuRsR4dH5aQT5+3Lnkp006cV636DFohuszi7l2MkGFkzWA9tKuavY8CAemZ/GjvxK/rQu1+44LkeLRTd4c1M+AyOCOSc5xu4oSqkzMG9sPPMnDOTZf+ey/chxu+O4FKeKhYjMFpEcEckVkUUdvC8i8oz1/i4RmeiorYhEichqEdlvTSOt+TNEZKuI7LamF3THivaUw0dP8mluBVdNSsBXD2wr5fYezEhlQHgQ/++fO3Qo8zYcFgsR8QWeA+YAKcACEUlpt9gcINl6LARecKLtImCtMSYZWGu9BqgALjHGjAGuB17r8tr1gn9uzsdH0APbSnmI8CB//nDlOA4fq+Xh5XvsjuMynNmzmAzkGmPyjDENwJtARrtlMoBXTauNQISIxDlomwEstp4vBi4DMMZsN8YUWfOzgCARcckR+RqbW1iypYALRsUS1zfY7jhKqW4yNakfC89N4o1NR1iTXWp3HJfgTLEYCOS3eV1gzXNmmc7a9jfGFANY09gOvvt7wHZjjEveZX3tnjIqTtRztV6xrZTHuXPGCEbHhXPPO7sor3HJn6Be5Uyx6Kgjvv1ljt+2jDNtO/5SkVTgd8DN3/L+QhHZIiJbysvLnfnIbvfGpiMMCA9i+kg9sK26jyts2woC/Xz549XjOVHfxJ1LdtDi5Vd3O1MsCoCENq8HAUVOLtNZ21Krqwpr+vVd1EVkEPAecJ0x5kBHoYwxLxlj0o0x6TExvf9jXXC8lg37y7kyfRB+vnpSmeo+dm/b6v+M6B/Gg5em8sn+Cp5f792n0zrzK7cZSBaRoSISAFwNLG23zFLgOuusqKlAldW11FnbpbQewMaafgAgIhHAcuBeY8xnXV+1nrVkc2vv2pWTEhwsqZRyZ1dNSiBjfDxPrt7Hl3lH7Y5jG4fFwhjTBNwBrAL2AEuMMVkicouI3GIttgLIA3KBl4HbOmtrtXkMmCEi+4EZ1mus5YcD94vIDuvR0fEM2zRZB7bPTY5hUGSI3XGUUj1IRHhk/hiG9OvDf725naMnvPP4hZ8zCxljVtBaENrOe7HNcwPc7mxba/5R4MIO5j8MPOxMLruszymnpLqOX1+aancUpVQvCA3040/XTOSy5z/jziU7+esNk7xuwFDtbO+Cv31+iJiwQC4c7VI7PEqpHpQSH84D81L4eF85L27o8FCqR9NicZo25h3l09wKFp6ThL8e2FbKq1w7JZF5Y+P4w0f7+Dy3wu44vUp/7U6DMYbfr8qhf3ggPzx7sN1xlFK9TER47HtjSYruw+3/2Eb+sVq7I/UaLRanYf2+crYcPs4dFyQT5O9rdxyllA1CA/14+bp0mlsMC1/bSm2Dd4wfpcXCSS0trXsVgyKDuSpdT5dVypsNie7DMwsmsLekmrvf3kXrOT6eTYuFk1ZmlZBVVM3PLhpBgJ/+2ZTydtNHxnL3rFEs21XM/27IsztOj9NfPSc0txieXL2P4bGhzJ/QflgspZS3uuW8JC4eG8fvVu5lfU6Z4wZuTIuFE97fXkhu2QnunDFC71mhlPqaiPDE98cyakA4P/3HdvaWVNsdqcdosXCgoamFp9fuIzU+nNmpA+yOo5RyMSEBfvzl+nRCAn254ZXNFFedsjtSj9Bi4cCSLfnkHzvFXTNHet0Vm0op58RHBPO3Gydzor6JG/+6meq6RrsjdTstFp2oa2zm2X/v56zBkToMuVKqU6PjwnnhBxPJLTvBra9vpaGpxe5I3UqLRSde33iY0up67po5EhHdq1BKde6c5Bge+95YPss9yqJ3POuUWqcGEvRGJ+qbeH79AaYNj+bsYf3sjqOUchPfP2sQRZWneHL1PmLCAlk0Z5RH/GdTi8W3+OunBzl2soG7Zo20O4pSys389ILhlNXU8b8b8gjw8+HnM93/d0SLRQf2ldbw4scHuGh0f8YnRNgdRynlZkSEhy5No6nZ8Oy/c/H39eG/Lky2O9YZ0WLRzrGTDdy0eAvBAX48lKH3q1BKdY2Pj/Do/DE0Nrde1OvnK9w2fbjdsbpMi0UbDU0t3Pr6Vkqq6/jnwqnERwTbHUkp5cZ8fITHvz+WppYWHl+Zg7+PDz85N8nuWF2ixcJijOFXSzP58uAxnrpqHBMSI+2OpJTyAL4+wh+uGEdTi+GRFXtoaG7htunD3O6gtxYLy+LPD/HGpnxunT6M+RMG2R1HKeVB/Hx9ePqq8fj7CE+syqGsuo4HLkl1q+GDtFgAG/aV89CybGak9OcXHnDWglLK9fj7+vDkleOJCQvk5U8OUnGigSevGkegn3vcG8fri8WB8hPc/o9tjOgfxlNXjdchPZRSPcbHR7jv4hRiw4J4ZMUejp6s56Xr0gkP8rc7mkNefQV3VW0jNy3eQoCvD3++Pp3QQK+vnUqpXvCTc5N4+qrxbDl0nCtf/MItbs/qtcXi+MkGbv/HNgqO1/LiD89iUGSI3ZGUUl7ksgkDeeWGSRRWnmLes5+ybq9r3w/Dq4qFMYZtR45z5z93MOW3a/k0t4JH549h0pAou6MppbzQuSNiWPbTaQyMCObGv23myY9yaG5xzfGkvKLf5WR9Ex/sKOL1jYfJLq4mNNCPq9ITuHZqIqMGhNsdTynlxQb368O7t32H+9/P5Jl/57I9v5I/Xj2BqD4Bdkf7Dx5dLHLLanj1i8O8t62QmvomRseF88j8NDLGD9TjE0oplxHk78sTV4wjfUgk93+QxeynN/Cby9KY5UI3XPPoX8ylO4p4c3M+88bEce3UwUxMjHC7C2GUUt7jqkmJpMb35a63dnLza1uZkzaABy9NJTY8yO5onl0sfjRtKDd8d6jL7c4ppdS3SRvYlw9/Oo2XNuTxx7X7+TS3gvvmjuaqSQm2/mfXow9wR4QEaKFQSrkdf18fbj9/OCv/+xxS4sJZ9O5uLnv+c9bnlNl2QyWPLhZKKeXOkmJCeeMnU3n8+2OpqKnnhr9u5vIXPufjfeW9XjS0WCillAvz8RGuTE9g3V3TeWR+GqVVdVz/yia+98LnfLCjkFMNzb2Tw5mFRGS2iOSISK6ILOrgfRGRZ6z3d4nIREdtRSRKRFaLyH5rGtnmvXut5XNEZNaZrqRSSrm7AD8frp0ymHW/sIpGdT3//eYOJj2yhrve2slnuRU9eo2GwwPcIuILPAfMAAqAzSKy1BiT3WaxOUCy9ZgCvABMcdB2EbDWGPOYVUQWAfeISApwNZAKxANrRGSEMaZ3yqdSSrmwQD9frp0ymAWTEvny4DHe317Iit3FvL21gNiwQCYNjWJCQgQTEiNJjQ8nyL97Bip05myoyUCuMSYPQETeBDKAtsUiA3jVtHaibRSRCBGJA4Z00jYDmG61XwysB+6x5r9pjKkHDopIrpXhi66vplJKeRYfH+HsYf04e1g/HsxIZc2eUlZmlrD9SCXLdxUD4O8rpMT3ZcnNU894dFtnisVAIL/N6wJa9x4cLTPQQdv+xphiAGNMsYjEtvmsjR181n8QkYXAQoDExEQnVkMp96DbtjpdQf6+zBsbz7yx8QCUVdexPb+SHfmVlFTVdcsw6M4Ui45O7G3fMfZtyzjTtivfhzHmJeAlgPT0dNccTEWpLtBtW52p2PAgZqUO6NYrwJ05wF0AJLR5PQgocnKZztqWWl1VWNOvhlx05vuUUkr1ImeKxWYgWUSGikgArQefl7ZbZilwnXVW1FSgyupi6qztUuB66/n1wAdt5l8tIoEiMpTWg+aburh+SimluoHDbihjTJOI3AGsAnyBV4wxWSJyi/X+i8AKYC6QC9QCN3bW1vrox4AlIvJj4AhwhdUmS0SW0HoQvAm4Xc+EUkope4ldl453p/T0dLNlyxa7YygPJiJbjTHpvf29um2rnnQ627Vewa2UUsohLRZKKaUc0mKhlFLKIS0WSimlHPKIA9wiUg4c/pa3o4GKXozjDFfLpHkcG2yMientL3XDbft0uHN+d84O/5ff6e3aI4pFZ0Rkix1nsXTG1TJpHvfk7n8nd87vztmha/m1G0oppZRDWiyUUko55A3F4iW7A3TA1TJpHvfk7n8nd87vztmhC/k9/piFUkqpM+cNexZKKaXOkBYLpZRSDnl0sRCR2SKSIyK51n2+7c5zSER2i8gOEbFldDgReUVEykQks828KBFZLSL7rWmkzXl+LSKF1t9ph4jM7a087sDVtmtHXG2bO10ikiAi60Rkj4hkich/W/Ndfh1EJEhENonITiv7g9b8087uscVCRHyB54A5QAqwQERS7E0FwPnGmPE2nqP9N2B2u3mLgLXGmGRgrfXazjwAT1l/p/HGmBW9mMelufB23Zm/4Vrb3OlqAn5ujBkNTAVut/7m7rAO9cAFxphxwHhgtnXPodPO7rHFApgM5Bpj8owxDcCbQIbNmWxnjNkAHGs3OwNYbD1fDFxmcx717dxuu3a1be50GWOKjTHbrOc1wB5gIG6wDqbVCeulv/UwdCG7JxeLgUB+m9cF1jw7GeAjEdkqIgttztJWf+vOhljTWJvzANwhIrusLgyX2723kStu113hitucQyIyBJgAfImbrIOI+IrIDlpvXb3aGNOl7J5cLKSDeXafJ/xdY8xEWrsQbheRc23O46peAIbRuttcDPzB1jSuxRW3a68gIqHAO8DPjDHVdudxljGm2RgzHhgETBaRtK58jicXiwIgoc3rQUCRTVkAMMYUWdMy4D1auxRcQamIxAFY0zI7wxhjSq0NvAV4Gdf5O7kCl9uuu8iltjlHRMSf1kLxd2PMu9Zst1oHY0wlsJ7W40ennd2Ti8VmIFlEhopIAHA1sNSuMCLSR0TCvnoOzAQyO2/Va5YC11vPrwc+sDHLVxvvV+bjOn8nV+BS2/UZcKltrjMiIsBfgD3GmCfbvOXy6yAiMSISYT0PBi4C9tKV7MYYj30Ac4F9wAHgPpuzJAE7rUeWXXmAN2jt2mmk9X+pPwb60XpGxH5rGmVznteA3cAua6OOs3tbcqWHK23XZ/BvbNs214X802jt6tsF7LAec91hHYCxwHYreybwgDX/tLPrcB9KKaUc8uRuKKWUUt1Ei4VSSimHtFgopZRySIuFUkoph7RYKKWUckiLhVJKKYe0WCillHLo/wMsQqFSVOrQJQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#| slow\n",
    "with tempfile.TemporaryDirectory() as d:\n",
    "    learn1 = synth_learner(path=d, cbs=SaveModelCallback(with_opt=True, fname=\"ckpt\"))\n",
    "    learn1.fit_one_cycle(5, cbs=InterruptCallback(2))\n",
    "    \n",
    "    learn2 = synth_learner(path=d)\n",
    "    learn2 = learn2.load(\"ckpt\")\n",
    "    learn2.fit_one_cycle(5, start_epoch=2)\n",
    "    \n",
    "    fig, axs = plt.subplots(1,2, sharey=True)\n",
    "    axs[0].plot(learn1.recorder.lrs)\n",
    "    axs[1].plot(learn2.recorder.lrs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a273016",
   "metadata": {},
   "source": [
    "## LRFind -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80f31b92",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "@docs\n",
    "class LRFinder(ParamScheduler):\n",
    "    \"Training with exponentially growing learning rate\"\n",
    "    def __init__(self, start_lr=1e-7, end_lr=10, num_it=100, stop_div=True):\n",
    "        if num_it < 6: num_it = 6\n",
    "        self.scheds = {'lr': [SchedExp(s, e) for (s,e) in zip(start_lr,end_lr)\n",
    "                             ] if is_listy(start_lr) else SchedExp(start_lr, end_lr)}\n",
    "        self.num_it,self.stop_div = num_it,stop_div\n",
    "\n",
    "    def before_fit(self):\n",
    "        super().before_fit()\n",
    "        path = self.path/self.model_dir\n",
    "        path.mkdir(parents=True, exist_ok=True)\n",
    "        self.tmp_d = tempfile.TemporaryDirectory(dir=path)\n",
    "        self.tmp_p = Path(self.tmp_d.name).stem\n",
    "        self.learn.save(f'{self.tmp_p}/_tmp')\n",
    "        self.best_loss = float('inf')\n",
    "\n",
    "    def before_batch(self): self._update_val(self.train_iter/self.num_it)\n",
    "\n",
    "    def after_batch(self):\n",
    "        super().after_batch()\n",
    "        if self.smooth_loss < self.best_loss: self.best_loss = self.smooth_loss\n",
    "        if self.smooth_loss > 4*self.best_loss and self.stop_div: raise CancelFitException()\n",
    "        if self.train_iter >= self.num_it: raise CancelFitException()\n",
    "\n",
    "    def before_validate(self): raise CancelValidException()\n",
    "\n",
    "    def after_fit(self):\n",
    "        self.learn.opt.zero_grad() # Needed before detaching the optimizer for future fits\n",
    "        tmp_f = self.path/self.model_dir/self.tmp_p/'_tmp.pth'\n",
    "        if tmp_f.exists():\n",
    "            self.learn.load(f'{self.tmp_p}/_tmp', with_opt=True)\n",
    "            self.tmp_d.cleanup()\n",
    "\n",
    "    _docs = {\"before_fit\": \"Initialize container for hyper-parameters and save the model\",\n",
    "             \"before_batch\": \"Set the proper hyper-parameters in the optimizer\",\n",
    "             \"after_batch\": \"Record hyper-parameters of this batch and potentially stop training\",\n",
    "             \"after_fit\": \"Save the hyper-parameters in the recorder if there is one and load the original model\",\n",
    "             \"before_validate\": \"Skip the validation part of training\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7ee3e3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| cuda\n",
    "from fastai.vision.all import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9060b2b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.086690</td>\n",
       "      <td>0.016682</td>\n",
       "      <td>00:33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "tensor([-5.8191e-04, -2.2443e-03,  0.0000e+00, -1.2517e-03,  0.0000e+00,\n",
       "        -1.4744e-03, -3.6433e-04,  0.0000e+00,  9.3745e-03,  0.0000e+00,\n",
       "         5.1993e-03, -1.5093e-02, -4.0410e-03,  0.0000e+00,  7.1963e-03,\n",
       "        -6.6033e-03, -3.3354e-03, -2.9191e-03, -1.5054e-03, -1.3179e-03,\n",
       "         8.7333e-03, -1.1155e-02, -9.6656e-04,  1.6653e-02,  9.5839e-04,\n",
       "         8.4995e-03, -2.8187e-02,  3.1579e-03, -9.3051e-04, -2.3887e-03,\n",
       "        -7.3557e-04, -1.4501e-02, -6.2110e-03,  1.9949e-03, -7.0233e-03,\n",
       "         1.2792e-02,  0.0000e+00,  1.0687e-03,  0.0000e+00, -4.2413e-04,\n",
       "         2.9628e-03,  7.2686e-03, -9.7241e-03, -4.9941e-04,  1.7408e-02,\n",
       "        -9.2441e-03, -9.7731e-03, -9.9393e-03,  0.0000e+00, -2.1448e-03,\n",
       "         2.7660e-03, -3.1110e-03,  5.9454e-05, -1.4412e-03, -6.1454e-04,\n",
       "        -1.6537e-03,  1.7001e-02,  1.4041e-02, -6.2878e-03,  2.0800e-02,\n",
       "        -1.2900e-02, -1.2626e-02, -2.6591e-03,  3.9685e-03], device='cuda:0')"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| cuda\n",
    "set_seed(99, True)\n",
    "path = untar_data(URLs.PETS)/'images'\n",
    "\n",
    "image_files = get_image_files(path)\n",
    "if sys.platform == \"win32\" and IN_NOTEBOOK:\n",
    "    image_files = random.choices(image_files, k=int(len(image_files)/8))\n",
    "    print(\"Randomly select 1/8 files in NOTEBOOK on Windows to save time\")\n",
    "\n",
    "# pickle can't serializer lamda function.\n",
    "def _label_func(x):\n",
    "    return x[0].isupper()\n",
    "\n",
    "dls = ImageDataLoaders.from_name_func(\n",
    "    path, image_files, valid_pct=0.2,\n",
    "    label_func=_label_func, item_tfms=Resize(224))\n",
    "\n",
    "learn = vision_learner(dls, resnet18)\n",
    "learn.fit(1)\n",
    "learn.opt.state_dict()['state'][1]['grad_avg']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0624d45c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#| slow\n",
    "with tempfile.TemporaryDirectory() as d:\n",
    "    learn = synth_learner(path=Path(d))\n",
    "    init_a,init_b = learn.model.a,learn.model.b\n",
    "    with learn.no_logging(): learn.fit(20, cbs=LRFinder(num_it=100))\n",
    "    assert len(learn.recorder.lrs) <= 100\n",
    "    test_eq(len(learn.recorder.lrs), len(learn.recorder.losses))\n",
    "    #Check stop if diverge\n",
    "    if len(learn.recorder.lrs) < 100: assert learn.recorder.losses[-1] > 4 * min(learn.recorder.losses)\n",
    "    #Test schedule\n",
    "    test_eq(learn.recorder.lrs, [SchedExp(1e-7, 10)(i/100) for i in range_of(learn.recorder.lrs)])\n",
    "    #No validation data\n",
    "    test_eq([len(v) for v in learn.recorder.values], [1 for _ in range_of(learn.recorder.values)])\n",
    "    #Model loaded back properly\n",
    "    test_eq(learn.model.a, init_a)\n",
    "    test_eq(learn.model.b, init_b)\n",
    "    test_eq(learn.opt.state_dict()['state'], [{}, {}])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c274065b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"LRFinder.before_fit\" class=\"doc_header\"><code>LRFinder.before_fit</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>LRFinder.before_fit</code>()\n",
       "\n",
       "Initialize container for hyper-parameters and save the model"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(LRFinder.before_fit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01fddb2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"LRFinder.before_batch\" class=\"doc_header\"><code>LRFinder.before_batch</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>LRFinder.before_batch</code>()\n",
       "\n",
       "Set the proper hyper-parameters in the optimizer"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(LRFinder.before_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99a5d49d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"LRFinder.after_batch\" class=\"doc_header\"><code>LRFinder.after_batch</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>LRFinder.after_batch</code>()\n",
       "\n",
       "Record hyper-parameters of this batch and potentially stop training"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(LRFinder.after_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4503b30e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "<h4 id=\"LRFinder.before_validate\" class=\"doc_header\"><code>LRFinder.before_validate</code><a href=\"\" class=\"source_link\" style=\"float:right\">[source]</a></h4>\n",
       "\n",
       "> <code>LRFinder.before_validate</code>()\n",
       "\n",
       "Skip the validation part of training"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_doc(LRFinder.before_validate)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cfddb75b",
   "metadata": {},
   "source": [
    "### Suggestion Methods"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "015004c2",
   "metadata": {},
   "source": [
    "There are a few methodologies for suggesting a learning rate automatically and these as we will see can further be passed into `lr_find`. Currently four methods are supported, however to write your own it should look like a function that can accept `LRFinder`'s returned `lrs`, `losses`, as well as the `num_it`. \n",
    "Your function should return an `x,y` coordinate that can be plotted, such as below:\n",
    "\n",
    "\n",
    "```python\n",
    "def myfunc(lrs:list, losses:list, num_it:int) -> tuple(float, tuple(float,int)):\n",
    "    ...\n",
    "    return suggestion, (suggestion,loss_idx)\n",
    "```\n",
    "\n",
    "If there are any more parameters to be passed in, you should pass in your `func` as a partial and specify them yourself, such as:\n",
    "\n",
    "```python\n",
    "def myfunc(lrs:list, losses:list, num_it:int, pct_reduction:float) -> tuple(float, tuple(float,int)):\n",
    "    ...\n",
    "    return suggestion, (suggestion,loss_idx)\n",
    "```\n",
    "\n",
    "```python\n",
    "f = partial(myfunc, pct_reduction=.2)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8904b8c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#| hide\n",
    "learn = synth_learner()\n",
    "with learn.no_logging(): learn.fit(20, cbs=LRFinder(num_it=100))\n",
    "\n",
    "lrs,losses = tensor(learn.recorder.lrs[100//10:-5]),tensor(learn.recorder.losses[100//10:-5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3dda6c00",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def valley(lrs:list, losses:list, num_it:int):\n",
    "    \"Suggests a learning rate from the longest valley and returns its index\"\n",
    "    n = len(losses)\n",
    "    max_start, max_end = 0,0\n",
    "\n",
    "    # find the longest valley\n",
    "    lds = [1]*n\n",
    "    for i in range(1,n):\n",
    "        for j in range(0,i):\n",
    "            if (losses[i] < losses[j]) and (lds[i] < lds[j] + 1):\n",
    "                lds[i] = lds[j] + 1\n",
    "            if lds[max_end] < lds[i]:\n",
    "                max_end = i\n",
    "                max_start = max_end - lds[max_end]\n",
    "    \n",
    "    sections = (max_end - max_start) / 3\n",
    "    idx = max_start + int(sections) + int(sections/2)\n",
    "\n",
    "    return float(lrs[idx]), (float(lrs[idx]), losses[idx])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "657ad8d0",
   "metadata": {},
   "source": [
    "The `valley` algorithm was developed by [ESRI](https://forums.fast.ai/t/automated-learning-rate-suggester/44199/30) and takes the steepest slope roughly 2/3 through the longest valley in the LR plot, and is also the default for `Learner.lr_find`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0acc030",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.007585775572806597, (0.007585775572806597, tensor(10.4701)))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| hide\n",
    "valley(lrs, losses, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ffe9e5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def slide(lrs:list, losses:list, num_it:int, lr_diff:int=15, thresh:float=.005, adjust_value:float=1.):\n",
    "    \"Suggests a learning rate following an interval slide rule and returns its index\"\n",
    "    losses = to_np(losses)\n",
    "    loss_grad = np.gradient(losses)\n",
    "\n",
    "    r_idx = -1\n",
    "    l_idx = r_idx - lr_diff\n",
    "    local_min_lr = lrs[l_idx]\n",
    "    while (l_idx >= -len(losses)) and (abs(loss_grad[r_idx] - loss_grad[l_idx]) > thresh):\n",
    "        local_min_lr = lrs[l_idx]\n",
    "        r_idx -= 1\n",
    "        l_idx -= 1\n",
    "    \n",
    "    suggestion = float(local_min_lr) * adjust_value\n",
    "    idx = np.interp(np.log10(suggestion), np.log10(lrs), losses)\n",
    "    return suggestion, (suggestion, idx)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a5a2b85",
   "metadata": {},
   "source": [
    "The `slide` rule is an algorithm developed by Andrew Chang out of Novetta, and is detailed [here](https://forums.fast.ai/t/automated-learning-rate-suggester/44199?u=muellerzr)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a57eeb8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6.309573450380412e-07, (6.309573450380412e-07, 12.832133293151855))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| hide\n",
    "slide(lrs, losses, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1691c8d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def minimum(lrs:list, losses:list, num_it:int):\n",
    "    \"Suggests a learning rate one-tenth the minumum before divergance and returns its index\"\n",
    "    lr_min = lrs[losses.argmin()].item()\n",
    "    loss_idx = losses[min(range(len(lrs)), key=lambda i: abs(lrs[i]-lr_min))]\n",
    "    return lr_min/10, (lr_min, loss_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34a7320a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.10964782238006592, (1.0964782238006592, tensor(7.5869)))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| hide\n",
    "minimum(lrs, losses, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b777f9b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "def steep(lrs:list, losses:list, num_it:int) -> (float, tuple):\n",
    "    \"Suggests a learning rate when the slope is the steepest and returns its index\"\n",
    "    grads = (losses[1:]-losses[:-1]) / (lrs[1:].log()-lrs[:-1].log())\n",
    "    lr_steep = lrs[grads.argmin()].item()\n",
    "    loss_idx = losses[min(range(len(lrs)), key=lambda i: abs(lrs[i]-lr_steep))]\n",
    "    return lr_steep, (lr_steep, loss_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2163af7a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.9054607491852948e-06, (1.9054607491852948e-06, tensor(12.8940)))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#| hide\n",
    "steep(lrs, losses, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0887104e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "@patch\n",
    "def plot_lr_find(self:Recorder, skip_end=5, return_fig=True, suggestions=None, nms=None, **kwargs):\n",
    "    \"Plot the result of an LR Finder test (won't work if you didn't do `learn.lr_find()` before)\"\n",
    "    lrs    = self.lrs    if skip_end==0 else self.lrs   [:-skip_end]\n",
    "    losses = self.losses if skip_end==0 else self.losses[:-skip_end]\n",
    "    fig, ax = plt.subplots(1,1)\n",
    "    ax.plot(lrs, losses)\n",
    "    ax.set_ylabel(\"Loss\")\n",
    "    ax.set_xlabel(\"Learning Rate\")\n",
    "    ax.set_xscale('log')\n",
    "    if suggestions:\n",
    "        colors = plt.rcParams['axes.prop_cycle'].by_key()['color'][1:]\n",
    "        for (val, idx), nm, color in zip(suggestions, nms, colors):\n",
    "            ax.plot(val, idx, 'o', label=nm, c=color)\n",
    "        ax.legend(loc='best')\n",
    "    if return_fig: fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c87361a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "mk_class(\"SuggestionMethod\", **{o.__name__.capitalize():o for o in [valley,slide,minimum,steep]},\n",
    "         doc=\"All possible suggestion methods as convience attributes to get tab-completion and typo-proofing\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7166853f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "@patch\n",
    "def lr_find(self:Learner, start_lr=1e-7, end_lr=10, num_it=100, stop_div=True, show_plot=True, suggest_funcs=(SuggestionMethod.Valley)):\n",
    "    \"Launch a mock training to find a good learning rate and return suggestions based on `suggest_funcs` as a named tuple\"\n",
    "    n_epoch = num_it//len(self.dls.train) + 1\n",
    "    cb=LRFinder(start_lr=start_lr, end_lr=end_lr, num_it=num_it, stop_div=stop_div)\n",
    "    with self.no_logging(): self.fit(n_epoch, cbs=cb)\n",
    "    if suggest_funcs is not None:\n",
    "        lrs, losses = tensor(self.recorder.lrs[num_it//10:-5]), tensor(self.recorder.losses[num_it//10:-5])\n",
    "        nan_idxs = torch.nonzero(torch.isnan(losses.view(-1)))\n",
    "        if len(nan_idxs) > 0:\n",
    "            drop_idx = min(nan_idxs)\n",
    "            lrs = lrs[:drop_idx]\n",
    "            losses = losses[:drop_idx]\n",
    "        _suggestions, nms = [], []\n",
    "        for func in tuplify(suggest_funcs):\n",
    "            nms.append(func.__name__ if not isinstance(func, partial) else func.func.__name__) # deal with partials\n",
    "            _suggestions.append(func(lrs, losses, num_it))\n",
    "        \n",
    "        SuggestedLRs = collections.namedtuple('SuggestedLRs', nms)\n",
    "        lrs, pnts = [], []\n",
    "        for lr, pnt in _suggestions:\n",
    "            lrs.append(lr)\n",
    "            pnts.append(pnt)\n",
    "        if show_plot: self.recorder.plot_lr_find(suggestions=pnts, nms=nms)\n",
    "        return SuggestedLRs(*lrs)\n",
    "\n",
    "    elif show_plot: self.recorder.plot_lr_find()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3999ea8e",
   "metadata": {},
   "source": [
    "First introduced by Leslie N. Smith in [Cyclical Learning Rates for Training Neural Networks](https://arxiv.org/pdf/1506.01186.pdf), the LR Finder trains the model with exponentially growing learning rates from `start_lr` to `end_lr` for `num_it` and stops in case of divergence (unless `stop_div=False`) then plots the losses vs the learning rates with a log scale. \n",
    "\n",
    "A variety of learning rate suggestion algorithms can be passed into the function, by default we use the `valley` paradigm."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a68e2f5c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<style>\n",
       "    /* Turns off some styling */\n",
       "    progress {\n",
       "        /* gets rid of default border in Firefox and Opera. */\n",
       "        border: none;\n",
       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "        background-size: auto;\n",
       "    }\n",
       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "        background: #F44336;\n",
       "    }\n",
       "</style>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minimum/10:\t1.58e-01\n",
      "Steepest point:\t9.12e-03\n",
      "Longest valley:\t1.58e-02\n",
      "Slide interval:\t8.32e-02\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEKCAYAAAAVaT4rAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA260lEQVR4nO3dd3hUVf7H8fdJryRAAiSEkIQOoYXQpYkUpa6CgmBBEMsuZRVk7bg/XV0rKoqiIrJLEbFQZRUEBUFKQgu9SAlJSAim9+T8/pgEIaSSmdxJ5vt6njxJ7tw798MkfHPm3HPPUVprhBBC2A47owMIIYSoXlL4hRDCxkjhF0IIGyOFXwghbIwUfiGEsDFS+IUQwsY4GB2gInx8fHRQUJDRMYQQokaJiIi4rLX2Lb69RhT+oKAg9u7da3QMIYSoUZRS50raLl09QghhY6TwCyGEjZHCL4QQNqZG9PELIWqf3NxcoqOjycrKMjpKjefi4kJAQACOjo4V2l8KvxDCENHR0Xh6ehIUFIRSyug4NZbWmsTERKKjowkODq7QMdLVI4QwRFZWFvXr15eiX0VKKerXr1+pd05S+IGs3HxOXko1OoYQNkeKvnlU9nWUwg98HRnN0He3EZcsfY1CiOutWbOG1157rcx9YmJiGDNmTDUlqjop/EBcchb5BZpfT102OooQojQHV8I7oTDX2/T54MpqOe3IkSP5xz/+UeY+/v7+rFq1qlrymIMUfiA5MxeAX09L4RfCKh1cCWunQ/IFQJs+r51e5eJ/9uxZWrduzZQpUwgNDWXChAls2rSJ3r1706JFC3bv3s3ixYv529/+BsCDDz7I9OnT6dWrFyEhIVeL/dmzZwkNDQVg8eLFjB49mhEjRhAcHMz8+fN5++236dy5Mz169ODKlSsA9O/f/+qMBJcvX6ZoWpqKHl8VUviBpAxT4d9xKhFZilIIK7T5n5Cbef223EzT9io6deoUM2bM4ODBgxw7doxly5axfft23nzzTf71r3/dsH9sbCzbt29n3bp1pb4TiIqKYtmyZezevZtnn30WNzc39u3bR8+ePVmyZEm5map6fHmk8PNniz8uJYvTCekGpxFC3CA5unLbKyE4OJj27dtjZ2dHu3btGDhwIEop2rdvz9mzZ2/Yf/To0djZ2dG2bVsuXbpU4nMOGDAAT09PfH198fLyYsSIEQClPqe5jy+PFH5MhT+ovhsAO6S7Rwjr4xVQue2V4OzsfPVrOzu7q9/b2dmRl5dX5v6l9RBU5DkdHBwoKCgAuGEoZmUzVZYUfkyFP7SxFwF1XeUCrxDWaOAL4Oh6/TZHV9P2GiooKIiIiAiAar8wLIUfU+H3dnPkluY+7DydSH6B9PMLYVU63A0j3gOvJoAyfR7xnml7DTVr1iwWLFhAr169uHy5ehucqiZczAwPD9eWmo9fa03zZ7/n0X4htGpUh+nL97H6r73p2MTbIucTQpgcPXqUNm3aGB2j1ijp9VRKRWitw4vva/Mt/rTsPPILNF6ujvRqVh+A7dLdI4SoxWy+8BeN6PF2dcLHw5nWjTzlAq8Qolaz+cJfNIa/jqtpOtPezX3Yc/YPsnLzjYwlhBAWY/OFP6Wwxe9VWPhvae5DTl4BEef+MDKWEEJYjM0X/qtdPW6mwt8tuB6O9optJ6W7RwhRO9l84U8q1uJ3d3YgLLAu204mGBlLCCEsxuYLf/EWP0Dflr4cjknhclq2UbGEEAaZN28eGRkZRsewKCn8mbk42itcHe2vbuvbwheA7dLdI4TVWH9mPYNXDabDFx0YvGow68+st8h5pPDbgKSMXLxcHa9bwaadfx3quTvxywnp7hHCGqw/s565O+YSmx6LRhObHsvcHXOrXPzT09MZNmwYHTt2JDQ0lJdeeomYmBgGDBjAgAEDAPjhhx/o2bMnYWFhjB07lrS0NAAiIiLo168fXbp0YciQIcTGxgKm6ZZnzpxJr169CA0NZffu3VX7x1uAzRf+lMzcq/37RezsFLc09+GXk5dlmmYhrMC7ke+SlX/9RGZZ+Vm8G/lulZ5348aN+Pv7c+DAAaKiopg5cyb+/v5s2bKFLVu2cPnyZV5++WU2bdpEZGQk4eHhvP322+Tm5jJt2jRWrVpFREQEDz30EM8+++zV501PT2fHjh18+OGHPPTQQ1XKaAkORgcwWnIJhR9M/fxrDsRwNDaVtv51DEgmhCgSlx5Xqe0V1b59e2bNmsWcOXMYPnw4ffr0ue7x3377jSNHjtC7d28AcnJy6NmzJ8ePHycqKopBgwYBkJ+fj5+f39Xjxo8fD0Dfvn1JSUkhKSkJb2/vKmU1J4sVfqXUImA4EK+1Di3c1gn4CHAB8oDHtdaGvg9KyszB18P5hu19WvgAsO1kghR+IQzWyL0RsemxJW6vipYtWxIREcGGDRt4+umnGTx48HWPa60ZNGgQy5cvv277oUOHaNeuHTt37izxeYsvfm5ti8pbsqtnMTC02LbXgZe01p2AFwq/N5RpZk6nG7Y3rONC60ae/CLDOoUw3IywGbjYu1y3zcXehRlhM6r0vDExMbi5uTFx4kRmzZpFZGQknp6epKamAtCjRw9+/fVXTp06BUBGRgYnTpygVatWJCQkXC38ubm5HD58+OrzfvnllwBs374dLy8vvLy8qpTT3CzW4tda/6KUCiq+GShqPnsBMZY6f0UVXdwtSZ8WPnyx4xwZOXm4Odl8r5gQhhkWMgww9fXHpcfRyL0RM8JmXN1+sw4dOsTs2bOxs7PD0dGRBQsWsHPnTm6//Xb8/PzYsmULixcvZvz48WRnm4Z3v/zyy7Rs2ZJVq1Yxffp0kpOTycvLY+bMmbRr1w6AunXr0qtXL1JSUli0aFHV/vEWYNFpmQsL/7prunraAP8DFKZ3G7201udKOXYqMBUgMDCwy7lzJe5WJfkFmmbPbGD6wBY8MajlDY9vO5nAfZ/t5vNJXRnQqoHZzy+ELaut0zL379+fN998k/DwG2ZDtihrnpb5MeDvWusmwN+Bz0rbUWu9UGsdrrUO9/X1tUiY1KyimTlLbvF3DaqHs4OdDOsUQtQq1V34HwC+Kfz6K6BbNZ//OkUzc5bW1ePiaE+PkPpsjIqT2TqFEBWydevWam/tV1Z1F/4YoF/h17cCJ6v5/NcpabqG4h7r34zY5Cw+/vlMdcUSQgiLsljhV0otB3YCrZRS0UqpycDDwFtKqQPAvyjswzdKcmbZLX6AHiH1GdbBjwU/n+JiUmZ1RRNCCIux5Kie8aU81MVS56ys4jNzluaZO9qw+eglXt1wlPn3hlVHNCGEsBibnrLhaou/jK4egMberjzarxnrDsay60xidUQTQgiLsenCX3z1rbI80rcZjb1dmbv2CH+k51g6mhDCCnl4eABw9uxZQkNDDU5z82y68Cdl5ODiaIezg325+7o62fP88LYcjU2h+7828/jSCLYciye/QCZxE6I6JK9dy8lbB3K0TVtO3jqQ5LVrjY5UY9l04U/OzMXb9cbpGkozNLQR38/ow8QeTfntzBUmLd7DC6ujLJhQCAGmoh/7/AvkxcSA1uTFxBD7/AtVLv5z5szhww8/vPr93Llzeemllxg4cCBhYWG0b9+e1atXl/kc+fn5zJ49m65du9KhQwc+/vhjAO67777rjp0wYQJr1qypUl5zsfnCX5Funmu18avDCyPa8tvTAxnfLZDlu89zKj7VQgmFEADx78xDZ10/LbPOyiL+nXlVet5x48ZdnVcHYOXKlUyaNIlvv/2WyMhItmzZwpNPPlnm9OyfffYZXl5e7Nmzhz179vDJJ5/w+++/M2XKFD7//HMAkpOT2bFjB3fccUeV8pqLTRf+pIzcci/slsbJwY7ZQ1rh5uTAm/87YeZkQohr5cXeODNnWdsrqnPnzsTHxxMTE8OBAweoW7cufn5+PPPMM3To0IHbbruNixcvcunSpVKf44cffmDJkiV06tSJ7t27k5iYyMmTJ+nXrx+nTp0iPj6e5cuXc9ddd+HgYB1zfllHCoMkZ+bSpJ7bTR9fz92JqX1DePvHE+w7/wedA+uaMZ0QooiDn5+pm6eE7VU1ZswYVq1aRVxcHOPGjWPp0qUkJCQQERGBo6MjQUFBZBV7t3EtrTXvv/8+Q4YMueGx++67j6VLl7JixQqrmqzNplv8N9PVU9zkW4Kp7+7Evzcek9W6zCwzJ19eUwFAg7/PRLlcPy2zcnGhwd9nVvm5x40bx4oVK1i1ahVjxowhOTmZBg0a4OjoyJYtWyhvgsghQ4awYMECcnNNowRPnDhBeno6AA8++CDz5s0DuDpzpzWo1YVfa311IraSmC7uVq3wuzs7MO3W5vx25grbZHF2s8nIyeOWf//ES2uPGB1FWAGvESPw+79/4uDvD0rh4O+P3//9E68RI6r83O3atSM1NZXGjRvj5+fHhAkT2Lt3L+Hh4SxdupTWrVuXefyUKVNo27YtYWFhhIaG8sgjj5CXlwdAw4YNadOmDZMmTapyTnOy6LTM5hIeHq737t1b6eOe/uYQm49eYvezt93wWE5eAS2f+54nB7Vk2sAWVcqXk1fArW9tpY6LI6v/1htH+1r997RCrqTn4Opoj6tT+UNlS/JNZDRPrDwAwH8nd+eWwhXRrJnW2upWWrJmtXVa5mtlZGTQvn17IiMjLb4YizVPy1ytGng6k5CWTU5ewQ2PVfSu3YpwcrDjuWFtOBKbwps/HK/y89V0efkFDHtvGwPe3MqWY/E39Rxf7Y2mST1XQnzdeWrVAVLKeOdmDeJTs7jt7Z9Zvvu80VGEldi0aROtW7dm2rRpVrcCV60u/P7eLmgNl1JuvDBTkQnaKmNoqB8TewTy8c9n2Hy09BEAlnAqPo3I838QdTGZk5dSSUzLrtbzF7fr9yvEJmeRm1/ApMV7eGLlfpIzKl64L1zJYOeZRMZ2acJbYzsSl5LFK+uOWjBx1WiteeabQ5xOSOfVDUdJypA7uwXcdtttnD9/npkzZxod5Qa1elSPn5crALHJWTeM3jF34Qd4blhbIs8l8eRXB1g/vQ+NvV3N9tylOZ+YweB3fqb4DcQ+Hk60aOBJ50Bvpg9sgYvjzXW53Ix1B2Nxc7Lnp1n9+XTbGT7ceprdv19h48y+eDiX/yv3dWQ0SsFdXQJo7O3KI/2asWDraYaGNmJAa+tbCe3ryItsOhrP+G5NWLHnAh9uPc0zd9TuLgxRs9X6Fj9AbPKN0yknZ5paZeYs/C6O9nwwIYy8fM20ZZHk5t/YxWRuq/dfpEDD/Hs78/F9XXh/fGeeG9aGga0bkpGbz4dbT/Py+uq7QJqXX8DGqFhua9MQL1dHnhzciv9M7kb0H5l8tPV0uccXFGi+joymV7P6V/9wzrytBa0aevLMt4fIzLm5BXGSM3N5dcNR3v6xavdcnIpPZd/5P66ONopJyuSlNYfpFlSPl0e3Z0xYAIt/PcuFKxlVOo8QlmQTLf6YpNK7erzdKj5lQ0UE+7jz2l3t+duyfTzynwjeH98Z9wq0cm+G1prVB2LoFlSP4R38S9znXxuOsvCXM9zS3IehoVUf81yeHacT+SMjl2Ed/jxXr2Y+jOrkzyfbzjC+e2CZ74R2/X6FC1cyeXJQq6vbnB3s+eeodtyz8Dc+3/E7j/dvfsNx8SlZrD8Uy7qDsZy9nM7ANg0Y1akx3YLrsSoimjf+d5wrhZPrNW/gwciOJb9eZcnKzWf8J7tISM2mVUNPxndrwqaj8eRrzRtjO2Bvp3hicEvWHIjh7R9P8M49nSp9DiGqQ61u8bs7O1DHxaHkFn85yy5WxfAO/rw8OpStx+O5Z+FO4ku4xmAOR2NTORWfxshOpRexWYNb0THAi6dWHST6D8u3QtcfjMXD2YF+La9fJ/mpoaYhcW9sPFbm8asiovF0dmBIu0bXbe8eUp+BrRuwYOvp62ZHzcsvYOaKfXR/dTMvrT1CenYePZvVZ8OhOCZ8uov2c//H098cormvB2v+1pvOgd489+0hYm5iUZ0v91wgITWbR/qF4Oxox9y1R9h+6jLP3NGGpvXdAVNj46Fbgvlu/0WiLiZX+hxCVIdaXfgB/L1dS2zxFy3CUsfFMq3xiT2a8ukD4ZxJSOcvH+7geJz55/NZfeAiDnaKO9qX3pJ3crDj/fFhFGiYsWI/eZXofsov0CzddY6Hl+xl2a7zV98llSY3v4CNh+MY1LbhDdcUGnu7MqVPMN/tj2H/haQSj0/LzmPDoViGd/QrcRjoU0Nbk56dxwdbTgGmdzwvrT3Cd/tjmNw7mE1P9GXjzL7MvzeMvc/dxvx7OzOigz/vje/Ml4/0oEOAN/Pu6URegebJlQcoKLwwEp+SxbubTvLFjrOlXhjPySvgo59PE960Lv8Y2po1f7uFddNu4a2xHZnQPfC6fR/r3wxvV0emr9jHgq2nibqYfPVcwvr179+fouHjd9xxB0lJSTfsM3fuXN58881qTmY+tbqrB8DPy6WUPv5cPJ0dcLDgmPtbWzfky6k9eeiLPYx4fzuP9gvh8QHNzXKhtaBAs+5ALH1a+FDPvezuqsD6bvzrzvZMX76PyV/sZc7Q1rT1r1PmMYeik3nuu0MciE6mvrsTPx65xNy1hxnUtiFzhrQmsP6NU138euoyyZm5DCvlD9Fj/Zvz5Z4LvLzuCF892vO6Me+5+QW8sDqKzNx8xnQJKPH4Vo08uSssgCU7z/FAryA2Hb3Ef347x9S+ITdcTHVxtGd4B/8busCa1ndn7oh2PPX1Qd768Tjp2fks332enPwCtIb/W3eEfi19mdAjkFtbN7x63NeR0cQmZ/HaXR2u5g5t7EVo4xuH6dVxceSNMR1584fj/HvjMf690TS0eMHEMLo0rVfKKy7Kc2JXHDtXnybtSjYe9ZzpOaoZLbs3Kv/AKtiwYYNFn98otb/we7tyIPrGt9zJmbnUsUA3T3HtA7xYP/0WXll/lPd+OsXqAzG8NLId/VtVbXRKxPk/uJiUyewhrcrfGRjZ0d/Ust18kjve28awDn6M6RJAUkYOccnZxKdmkZaVR0ZuPimZuWw/dZn67s68O64TIzv6E3Uxha8jo/k6Iprjcams/mvvG65drD8Yi6ezA31alnyzlYezA08ObsXT3xxi9qqDzB7SioZ1XEjPzuPxpZH8fCKBmbe1KLM4/n2QqQ/98aWRHI5JZnDbhswZWvadlcWNDQ9g87FLfLDlNA52irvCAnh8QDOycgv4Zl80q/fF8NDivfxtQHOeHNySvALNh1tP0THAi74VvJHstrYNua1tQ+JTsth+6jLvbT7JQ4v38tWjPWnZ0LNSeYWp6G9Zeoy8HNM71rQr2WxZauo2rErxT09P5+677yY6Opr8/Hyef/756x4PCgpi7969+Pj48Morr7BkyRKaNGmCr68vXbqYVpE9ffo0f/3rX0lISMDNzY1PPvmk3Lt9jVbrC7+/lwtX0nPIys2/rqWdnJGLtxlu3qqIBp4uvDuuM3eHN+H576J48PM93BPehOdHtK3Q8MaSrN5/ERdHOwa1bVj+zoWm9AlhbJcmfLr9DIu2/876g3/ObOjh7ICniwOuTva4OdnzUO9gZtzWgjoupteofYAX7QO8GNy2IRM/28Wcrw/y/vjOV1u/OXkF/O9wHIPaNSxzYZu7w5twJiGNxTvOsu5gDJN6B7P95GWOxKbw2p3tGdctsNRjwdR192DvID7++Qyhjeswb1wn7O0qd7esUorX7+pIl6bnuT3U77qhvk/f3oZZg1vx3LdRzN9yiotJmXQPrseFK5m8OLxdpe/MbVDHhTvDAugaVI+7Fuzg/s92s+qxngTUvfnJAW3RztWnrxb9Ink5BexcfbpKhX/jxo34+/uzfv16wDR98oIFC27YLyIighUrVrBv3z7y8vIICwu7WvinTp3KRx99RIsWLdi1axePP/44P/30001nqg61vvBfO5Y/2Mf96nZzTNBWWb2b+/D9zD68u+kkC34+zc4zibx9d0fCgyr39j83v4ANh+K4rU3DSo8Y8nIzDbGc1DuYE5dSaeDpTMM6LhV+nl7NfZg1pBWvbzxOWGBdHrolmFPxaby09jApWXmMKGe0jL2d4tlhbbm/ZxBv/nCcBVtP4+Jox8L7ujCwTcX+iP1tQHOcHeyZ2D0QN6eb+xX2cnNkat9mJT7maG/Ha3e1J6CuK2/9eILv9l+kjV8dBra5+XdpTeq5sWRyN+7+aCf3f7abrx7tSX0P55t+PluTdqXkay+lba+o9u3bM2vWLObMmcPw4cPp06dPiftt27aNv/zlL7i5mf5gjxw50nT+tDR27NjB2LFjr+6bnW3sDZQVYQOFv3Asf1LmdYX/UmoWnZpU/zTKzg72PDW0NQNaN+DvX+7n7o93Mq5bIH8d0LxCN3xduJLBqohorqTnMKpT45vOUc/diR4h9W/q2Mf6NWPf+ST+teEoR2NT+HbfRVyd7Jk7oi39i43mKU2Tem68O64zj/dvjr2donkDjwqf39PFkScGtbyp7BWllGLawBY0ruvKi2sOM3tIyyrPw9O6UR0+e7ArEz/dxagPfuX1MR3o1cz65yCyBh71nEss8h71qvbHs2XLlkRERLBhwwaefvppBg8eXOq+Jf38CwoK8Pb2Zv/+/VXKUd1q/agev8JiGpP858ielKxcLlzJpHUj4/pauwbV4/sZfbivR1O+2nuB/m9s4fnvorhYwjDDmKRMXll/hP5vbKHP61t4d/NJQhvXoW8pfemWppTirbs7ElDXla8iorkrLIAts/rzYO/gShfHVo08K1X0q9udYQEceGHwdRd6q6JrUD2WT+2Bo70d936yixdXR5GRk2eW567Neo5qhoPT9eXKwcmOnqNKftdWUTExMbi5uTFx4kRmzZpFZGRkifv17duXb7/9lszMTFJTU1lbuORjnTp1CA4O5quvvgJMI80OHDhQpUzVwaZa/EWOxZqGVrb1K3tki6V5ujjy0qhQHu4bwgdbTrN893n+u+scYYF1GdquER2beLNy7wW+23cRgL4tfXmgVxB9WvjQzNfD0Jkg67g4svLRniRl5Nb6i5V2lbyGUJ6wwLpsmN6H1/93jM9/PctPx+N5blhbBrdtKLN7lqKoH9/co3oOHTrE7NmzsbOzw9HRkQULFjBr1qwb9gsLC+Oee+6hU6dONG3a9LouoaVLl/LYY4/x8ssvk5uby7hx4+jYsWOVcllarZ6WuUjY//3IkHaNePXO9gB8seMsL645zG9PD6SRl0s5R1efC1cy+G7fRb6PiuNIbAoALo52jOsayJQ+wXJBsBb67Uwiz38Xxcn4NHqG1OeFEW1pY3CDpLrYwrTM1aky0zLX+hY/3DiW/2hsCvXcnWhYx7ourjWp58a0gS2YNrAF5xLT2X8hiT4tfMsdpy9qrh4h9fl+Rh+W7T7P2z+eYNh725g+sAXTb21h9ncaQhSxkcLvet2kWUdiU2jj52nVb6ub1ne/Og2AqN0c7O24v2cQozo25qW1h5m36SRRF5N5+55OV4fTCmFOtf7iLphm6YwpbPHn5RdwPC6VNo1s4+20qDm83Bx56+6OvDSyHVuPJzB6/q+cijf/VB9C2ETh9/NyJTUrj7TsPM4mppOdV1DulAVCGEEpxQO9glg6pTvJmbmM+WgnR2JSjI5lMTXhGmNNUNnX0SYK/9V5+ZMyOVz4n8hWLqCJmql7SH2+ebwXro72TPj0N47G1r7i7+LiQmJiohT/KtJak5iYiItLxQeq2EwfP5jG8h+NTcXRXtHM13rHjgsBpus8yx/uwbiFvzHh010sf7gHrQy898TcAgICiI6OJiEhwegoNZ6LiwsBASVPblgSGyn8f7b4j8Sm0KKBJ04ONvFmR9RwQT7uLJ/ag3ELd3LvJ7/x5SM9aN6gdhR/R0dHgoODjY5hkyxW/ZRSi5RS8UqpqGLbpymljiulDiulXrfU+a/VyMsFpYpa/CnSzSNqlGAfU8vfzk4x8dPdsqyjqDJLNnsXA0Ov3aCUGgCMAjpordsB1bKSgaO9Hb4ezkRdTCYhNZs2frWjxSRsR4ivB/+d3J2svHzu/fQ34pIts6qbsA0WK/xa61+AK8U2Pwa8prXOLtwn3lLnL87P25VfT10GkBE9okZq1ciTLyZ144/0XCZ8+huXS1ktTIjyVHdHd0ugj1Jql1LqZ6VU1+o6sb+XC9l5pvm8jZ6jR4ib1bGJN589EM7FpExGvL+d3b8Xb1sJUb7qLvwOQF2gBzAbWKlKuX1WKTVVKbVXKbXXHFf9i0b2+Hm54O0mUyCImqt7SH2+eqQXzg52jFu4k3mbTpAva/qKSqjuwh8NfKNNdgMFQIlzC2utF2qtw7XW4b6+FZvjvSxFY/mltS9qg/YBXqyb3odRnRozb9NJ7lqwgx8Ox8kfAFEh1V34vwNuBVBKtQScgMvVceKiFr+M6BG1hYezA+/c04l37ulIQmo2U/8TwcC3trJk51ly8wvKfwJhsyw5nHM5sBNopZSKVkpNBhYBIYVDPFcAD+hqum0vyMc0pXGHAK/qOJ0Q1eYvnQP4eXZ/5t/bmbruTryw+jAPL9krC7yIUtnEfPxFIs//Qecm3lY9K6cQVbV893me/fYQHQK8+fzBrtSVab1tVmnz8dvU7athgXWl6Itab3y3QBZM7MKR2BTGfLSjxOU8hW2zqcIvhK0Y0q4R/3moG/Gp2cxYvk8mQhPXkcIvRC3VPaQ+Tw1pxd5zf7DzTKLRcYQVkcIvRC02NrwJvp7OzP/plNFRhBWRwi9ELebiaM8jfUPYcTqRiHNyl68wkcIvRC13b/dA6rk78b60+kUhKfxC1HJuTg5MviWYrccTOBSdbHQcYQWk8AthA+7v2ZQ6Lg7M33LS6CjCCkjhF8IGeLo4Mql3MP87fInjcalGxxEGk8IvhI14sFcQbk72fPzzaaOjCINJ4RfCRtR1d2J8t0BWH4iR5RttnBR+IWzIlD7B2Cn4dNsZo6MIA0nhF8KG+Hm58pfOjVmx54Is3WjDpPALYWMe6deMnPwCPv/1d6OjCINI4RfCxjTz9WBou0Ys2XmO1Kxco+MIA0jhF8IGPd6/OalZeXyx46zRUYQBpPALYYPaB3gxqG1DPvr5jPT12yAp/ELYqH/c3prM3Hze2yx389oaKfxC2Khmvh6M79aEZbvOcyYhzeg4ohpJ4RfChs0Y2BJnBzte33jc6CiiGknhF8KG+Xo682i/Zmw8HMfeszJfv62oUOFXSrkrpewKv26plBqplHK0bDQhRHWY0ieEhnWcee37Y0ZHEdWkoi3+XwAXpVRjYDMwCVhsqVBCiOrj6mTP4/2bs/fcH9LqtxEVLfxKa50B3Am8r7X+C9DWcrGEENVpbHgA3m6OfPyLzOFjCypc+JVSPYEJwPrCbQ6WiSSEqG5uTg7c36Mpm45e4rSM8Kn1Klr4ZwJPA99qrQ8rpUKALRZLJYSodvf3CsLJ3k5m7rQBFSr8WuuftdYjtdb/LrzIe1lrPd3C2YQQ1cjHw5kxXQL4OvIiCalyN29tVtFRPcuUUnWUUu7AEeC4Umq2ZaMJIarblD4h5OYXyBw+tVxFu3raaq1TgNHABiAQuM9SoYQQxgj2cWdI20b857dzpGfnGR1HWEhFC79j4bj90cBqrXUuoC2WSghhmKn9QkjOzOXLPReMjiIspKKF/2PgLOAO/KKUagqkWCqUEMI4YYF16RpUl8+2/05efoHRcYQFVPTi7nta68Za6zu0yTlggIWzCSEMMrVvMy4mZbL+UKzRUYQFVPTirpdS6m2l1N7Cj7cwtf6FELXQwNYNaObrzsJfzqC19OrWNhXt6lkEpAJ3F36kAJ+XdYBSapFSKl4pFVXCY7OUUlop5VPZwEIIy7OzUzzcJ4TDMSnsOJ1odBxhZhUt/M201i9qrc8UfrwEhJRzzGJgaPGNSqkmwCDgfKWSCiGq1ejOjfFucIhp28fQ4YsODF41mPVn1pd/oLB6FS38mUqpW4q+UUr1BjLLOkBr/QtQ0oxP7wBPIaOChLBqmy9sRPmsIlclotHEpscyd8dcKf61QEUL/6PAB0qps0qps8B84JHKnkwpNRK4qLU+UIF9pxZdU0hISKjsqYQQVfRu5Lvk6evv4M3Kz+LdyHcNSiTMpaKjeg5orTsCHYAOWuvOwK2VOZFSyg14FnihgudcqLUO11qH+/r6VuZUQggziEuPq9R2UXNUagUurXVK4R28AE9U8lzNgGDgQOG7hgAgUinVqJLPI4SoBo3cS/6vWdp2UXNUZelFVZmdtdaHtNYNtNZBWusgIBoI01pL80EIKzQjbAYu9i7XbXOxd2FG2AyDEglzqUrhL/PirFJqObATaKWUilZKTa7CuYQQ1WxYyDDm9pqLn7sfoCjI8WZkwAyGhQwzOpqoojIXU1FKpVJygVeAa1nHaq3Hl/N4UHnhhBDGGhYyjGEhw8gv0Ax4cysHjzlDf6NTiaoqs8WvtfbUWtcp4cNTay0rcAlhI+ztFPf3bMqes38QdTHZ6DiiiqrS1SOEsCFjw5vg6mgvc/XXAlL4hRAV4uXqyF1dGrP6QAyJabJCV00mhV8IUWEP9goiJ6+A5btlxpWaTAq/EKLCmjfwpG9LX5bsPEdOnszVX1NJ4RdCVMqk3kHEp2bzfZTM1V9TSeEXQlRKvxa+hPi6s2j77zJXfw0lhV8IUSl2dopJvYI4EJ1M5Pkko+OImyCFXwhRaXeGBVDHxYFFv/5udBRxE6TwCyEqzd3ZgXHdAtkYFUdMUplLcwgrJIVfCHFT7u/ZFIDFckNXjSOFXwhxUwLqujG8gx/Ldp0nOTPX6DiiEqTwCyFu2tS+IaRl57F01zmjo4hKkMIvhLhp7fy96NPCh89/PUtWbr7RcUQFSeEXQlTJY/2akZCazXf7LhodRVSQFH4hRJX0bFaf+9KPEvD4eI62acvJWweSvHat0bFEGWROfSFElaSsW8e4n/+LXY5pxs68mBhin38BAK8RI4yMJkohLX4hRJXEvzPvatEvorOyiH9nnjGBRLmk8AshqiQvtuTJ2krbLownhV8IUSUOfn6V2i6MJ4VfCFElDf4+E+Xict22fCdnGvx9pjGBRLmk8AshqsRrxAj8/u+fOPj7g1IkedZnYdd7cBxyu9HRRCmk8AshqsxrxAha/LSZNkeP4LRqLWt8O7Bou8zcaa2k8AshzKpL07oMatuQj38+w6WULKPjiBJI4RdCmN3Tt7cmt6CAWV8doKBAVumyNlL4hRBmF+LrwbPD2rLt5GW+2HnW6DiiGCn8QgiLmNg9kIGtG/Dq98c4HpdqdBxxDSn8QgiLUErx7zEdqOPiwIwV+8jOk9k7rYUUfiGExfh4OPP6mA4ci0vl1Q3HjI4jCknhF0JY1K2tG/JQ72AW7zjL2gMxRscRSOEXQlSDp+9oTXjTusz5+iCn4qW/32hS+IUQFudob8f8e8Nwc7Ln0f9Gkp6dZ3QkmyaFXwhRLRp5ufDeuM6cSUjjr8siuZiUaXQkm6W0tszNFUqpRcBwIF5rHVq47Q1gBJADnAYmaa2Tynuu8PBwvXfvXovkFEJUryU7z/LyuqMA3Ns9kMf6NyM5M5eD0ckcik4iIS2bjJx8MrLz8XBxYMotwfRq7mNw6ppJKRWhtQ6/YbsFC39fIA1Yck3hHwz8pLXOU0r9G0BrPae855LCL0TtcjEpk/c3n+SriGjyr7mz183JHn9vV9yc7HF1tOdsYjqXUrLpEVKPJwa1omtQXZRSBiavWaq98BeeNAhYV1T4iz32F2CM1npCec8jhV+I2uns5XRW748hoK4rHZt4Eezjgb3dn4U9KzefZbvO8+HW01xOy8bdyZ5gX3eCfTwY3sGPIe0aGZje+llj4V8LfKm1/m8px04FpgIEBgZ2OXfunMVyCiGsW2ZOPqv3X+RYXCpnLqdzPC6Fy2k5LH+4B92C6xkdz2pZVeFXSj0LhAN36goEkBa/EOJaqVm5jJz/Kxk5eayf3gcfD2ejI1ml0gq/gwFBHsB00XdgRYq+EEIU5+niyAf3hjHn7Z188cwOXHI0HvWc6TmqGS27S/dPeaq18CulhgJzgH5a64zqPLcQonZxuJDB0EwnyDe1H9OuZLNlqWlaCCn+ZbPYOH6l1HJgJ9BKKRWtlJoMzAc8gR+VUvuVUh9Z6vxCiNpt5+rTV4t+kbycAtN2USaLtfi11uNL2PyZpc4nhLAtaVeyK7Vd/Enu3BVC1Ege9Uq+oFvadvEnKfxCiBqp56hmODhdX8LyFHQdEWJQopqj2kf1CCGEORRdwN25+jRpV7Jx8HRkbV463jqbtgZns3ZS+IUQNVbL7o2u/gHQWvPzp7uYt+kEozs3xsvV0eB01ku6eoQQtYJSimfuaENSZi7vbz5pdByrJoVfCFFrhDb24p7wJny+4yxHY1OMjmO1pPALIWqVOUNb4+XqyLPfHqKgQCYHKIkUfiFErVLX3Ymnb29N5PkkVu69YHQcqySFXwhR64zpEkC34Hq8+v0xEtPkhq7ipPALIWodpRSvjA4lPTuPF9ccJi+/wOhIVkUKvxCiVmrR0JNpt7Zg3cFY7lqwg1PxaUZHshpS+IUQtdaM21rw/vjOnLuSwbD3trHwl9OcTkgjPTvP6GiGsuhCLOYiC7EIIaoiPjWLZ76JYtPRS1e3eTo70LKRJ72a1adXMx/Cmnrj7GBvYErzM2QFLnORwi+EqCqtNfsuJHE+MYO4lCzikrPYfyGJg9FJFGio5+7EfyZ3o52/l9FRzUYKvxBClCAlK5ffTify4prD5OZrvnmsF4H13YyOZRalFX7p4xdC2LQ6Lo4MbteIJQ91I6+ggPsW7SIhtXYPAZXCL4QQmEYBLXqwK/Ep2TywaDepWblGR7IYKfxCCFEoLLAuCyaGceJSKnO+PkhN6Aq/GVL4hRDiGv1bNeDJwa3YcCiOryMvGh3HIqTwCyFEMVP7htAtuB4vro7ifGKG0XHMTgq/EEIUY2+neOeeTtjZKWZ+ua/WTfkghV8IIUrQ2NuVl0eHEnk+ide+P0Z2Xr7RkcxGCr8QQpRiVKfGjO0SwKfbf6f/G1v5/Nffycyp+X8A5AYuIYQog9aabScvM/+nU+w+e4W6bo70CKlPpybedGriTZemdXGwt842dGk3cMli60IIUQalFH1b+tK3pS+7ziSybPd5Is//wfdRcQAMbdeIBRPDUEoZnLTipPALIUQFdQ+pT/eQ+gAkpmXz+a9nmb/lFKv3xzC6c2OD01Wcdb4/EUIIK1ffw5m/D2pJWKA3L645THxqltGRKkwKvxBC3CR7O8UbYzuSmZvPs99G1Zg7faXwCyFEFTTz9WDW4Jb8eOQSaw7EGB2nQqTwCyFEFU2+JYTOgd48920UW4/HGx2nXFL4hRCiiuztFPPvDaNxXVcmLd7DB1tOWXW3jxR+IYQwg8bernzzeC+Gd/Dnjf8d57H/RpKcYZ1TO0vhF0IIM3FzcuC9cZ14blgbfjgSR783t7Bo++/k5FnXXD8WK/xKqUVKqXilVNQ12+oppX5USp0s/FzXUucXQggjKKWY0ieEddP60M6/Dv9cd4Qh837h11OXjY52lSVb/IuBocW2/QPYrLVuAWwu/F4IIWqdtv51+O/k7ix6MBwFTP5iD1EXk42OBViw8GutfwGuFNs8Cvii8OsvgNGWOr8QQhhNKcWtrRvy5SM9qefmxNQle7mcZvx6vtXdx99Qax0LUPi5QWk7KqWmKqX2KqX2JiQkVFtAIYQwN19PZxbeH05ieg6P/zfS8D5/q724q7VeqLUO11qH+/r6Gh1HCCGqJLSxF6+P6cDus1d4/rso/kjPMSxLdU/Sdkkp5ae1jlVK+QHWf6eDEEKYyahOjTkWl8qCraf5cu8FWjb0oGtQPR7uE0KQj3u15ajuFv8a4IHCrx8AVlfz+YUQwlBPDWnFqkd7MntIK/y8XPkm8iL3L9pdrWP+LbYQi1JqOdAf8AEuAS8C3wErgUDgPDBWa138AvANZCEWIURtFXHuCuMW/kbv5j589kBX7O3MN69/aQuxWHJUz3ittZ/W2lFrHaC1/kxrnai1Hqi1blH4udyiL4QQtVmXpvV4cUQ7th5PYN6mE38+cHAlvBMKc71Nnw+uNNs5ZSEWIYQw2ITugRyKTub9n04R7OPOcLUdpw0zITfTtEPyBVg73fR1h7urfD6rHdUjhBC2QinFS6Pa0bGJN0+sPEDCd8/+WfSL5GbC5n+a5XzS4hdCCCvg4mjPsind2XI8Hv9vEkveKTnaLOeSFr8QQlgJd2cHhnfwR3kFlLxDadsrSQq/EEJYm4EvgKPr9dscXU3bzUAKvxBCWJsOd8OI98CrCaBMn0e8Z5YLuyB9/EIIYZ063G22Ql+ctPiFEMLGSOEXQggbI4VfCCFsjBR+IYSwMVL4hRDCxlhsdk5zUkolAOcAL+DaRSuv/b7o6+KffYDKrnJc/DwVeawi2UrKWbTN0cw5S3u8sjlLylzZ19TcOUvKZI6c5WWtSM7i22rT76g1/Owt9TtaXtaq/I6WlK+6cjbVWt+4kpXWusZ8AAtL+77o6xI+763qeSryWEWylZSv6Gtz5yzt8crmNMdrau6cZb2O1f2zL29bbfodtYafvaV+R83xs7fm+lT8o6Z19awt4/u1pXw2x3kq8lhFsl37dWmPV0Z5x5X0eGVzXvu1teS89ntz5izv2IrkLL5Nfkcr/7gRv6PlHVuV39FrvzbqZ3+dGtHVUxVKqb26hIUIrE1NyQk1J6vkNL+aklVylq2mtfhvxkKjA1RQTckJNSer5DS/mpJVcpah1rf4hRBCXM8WWvxCCCGuIYVfCCFsjBR+IYSwMTZd+JVSfZRSHymlPlVK7TA6T2mUUnZKqVeUUu8rpR4wOk9plFL9lVLbCl/T/kbnKY9Syl0pFaGUGm50ltIopdoUvp6rlFKPGZ2nNEqp0UqpT5RSq5VSg43OUxalVIhS6jOl1CqjsxRX+Dv5ReFrOcFS56mxhV8ptUgpFa+Uiiq2fahS6rhS6pRS6h9lPYfWepvW+lFgHfCFteYERgGNgVzAPItuWianBtIAF0vlNGNWgDnASsukNNvv6NHC39G7AYsM+zNTzu+01g8DDwL3WCKnGbOe0VpPtlTG4iqZ+U5gVeFrOdJioSp715i1fAB9gTAg6ppt9sBpIARwAg4AbYH2mIr7tR8NrjluJVDHWnMC/wAeKTx2lRXntCs8riGw1Jp/9sBtwDhMhWq4teYsPGYksAO415pzFh73FhBmzT/7a46zyP+lKmZ+GuhUuM8yS2WqsStwaa1/UUoFFdvcDTiltT4DoJRaAYzSWr8KlPh2XikVCCRrrVOsNadSKhrIKfw231pzXuMPwNkSOcFsr+kAwB3Tf7ZMpdQGrXWBteUsfJ41wBql1HpgmTkzmiunUkoBrwHfa60jzZ3RnFmrW2UyY3qnHADsx4I9MjW28JeiMXDhmu+jge7lHDMZ+NxiiUpW2ZzfAO8rpfoAv1gyWDGVyqmUuhMYAngD8y2a7EaVyqq1fhZAKfUgcNncRb8MlX1N+2N6++8MbLBksGIq+zs6DdO7KC+lVHOt9UeWDFdMZV/T+sArQGel1NOFfyCqW2mZ3wPmK6WGUbVpHcpU2wq/KmFbmXeoaa1ftFCWslQqp9Y6A9MfqOpW2ZzfYPojZYRK/+wBtNaLzR+lTJV9TbcCWy0VpgyVzfkepqJlhMpmTQQetVycCikxs9Y6HZhk6ZPX2Iu7pYgGmlzzfQAQY1CWskhO86spWSWn+dWkrEUMzVzbCv8eoIVSKlgp5YTp4t0agzOVRHKaX03JKjnNryZlLWJs5uq4qm2hK+XLgVj+HOI4uXD7HcAJTFfMn5WctStnTcoqOW07qzVnlknahBDCxtS2rh4hhBDlkMIvhBA2Rgq/EELYGCn8QghhY6TwCyGEjZHCL4QQNkYKv6ixlFJp1Xw+s6zZoEzrFiQrpfYppY4ppd6swDGjlVJtzXF+IaTwC1FIKVXm3FVa615mPN02rXVnoDMwXCnVu5z9R2OaSVSIKqttk7QJG6eUagZ8APgCGcDDWutjSqkRwHOY5j5PBCZorS8ppeYC/kAQcFkpdQIIxDRPeiAwT5smIEMplaa19iicMXMucBkIBSKAiVprrZS6A3i78LFIIERrXerUwFrrTKXUfkyzNaKUehiYWpjzFHAf0AnTnPz9lFLPAXcVHn7Dv/NmXzdhW6TFL2qbhcA0rXUXYBbwYeH27UCPwlb2CuCpa47pgmn+9nsLv2+NaXrpbsCLSinHEs7TGZiJqRUeAvRWSrkAHwO3a61vwVSUy6SUqgu04M/ptr/RWnfVWncEjmK6vX8HpnlcZmutO2mtT5fx7xSiXNLiF7WGUsoD6AV8ZVoXBPhzQZgA4EullB+m1vTv1xy6Rmudec3367XW2UC2Uioe04pixZeS3K21ji48735M7xjSgDNa66LnXo6p9V6SPkqpg0Ar4DWtdVzh9lCl1MuY1jTwAP5XyX+nEOWSwi9qEzsgSWvdqYTH3gfe1lqvuaarpkh6sX2zr/k6n5L/n5S0T0lzrJdmm9Z6uFKqJbBdKfWt1no/sBgYrbU+ULhITP8Sji3r3ylEuaSrR9Qa2rR85u9KqbFgWg5QKdWx8GEv4GLh1w9YKMIxIOSaZfbKXXRca30CeBXTwu8AnkBsYffShGt2TS18rLx/pxDlksIvajI3pVT0NR9PYCqWk5VSB4DDmNYxBVML/yul1DZMF17NrrC76HFgo1JqO3AJSK7AoR8BfZVSwcDzwC7gR0x/SIqsAGYXDgFtRun/TiHKJdMyC2FGSikPrXVa4eLjHwAntdbvGJ1LiGtJi18I83q48GLvYUzdSx8bG0eIG0mLXwghbIy0+IUQwsZI4RdCCBsjhV8IIWyMFH4hhLAxUviFEMLGSOEXQggb8/8ISWgMQF6DJgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#| slow\n",
    "with tempfile.TemporaryDirectory() as d:\n",
    "    learn = synth_learner(path=Path(d))\n",
    "    weights_pre_lr_find = L(learn.model.parameters())\n",
    "    lr_min, lr_steep, lr_valley, lr_slide = learn.lr_find(suggest_funcs=(minimum, steep, valley, slide))\n",
    "    weights_post_lr_find = L(learn.model.parameters())\n",
    "test_eq(weights_pre_lr_find, weights_post_lr_find)\n",
    "print(f\"Minimum/10:\\t{lr_min:.2e}\\nSteepest point:\\t{lr_steep:.2e}\\nLongest valley:\\t{lr_valley:.2e}\\nSlide interval:\\t{lr_slide:.2e}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b51e15c",
   "metadata": {},
   "source": [
    "## Export -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "249fa4dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "from nbdev import nbdev_export\n",
    "nbdev_export()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d964594b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "jupytext": {
   "split_at_heading": true
  },
  "kernelspec": {
   "display_name": "python3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}