{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp models.layers.ou_noise"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# OU Noise Layer\n",
    "> Implementation of Ornstein Uhlenbeck Noise."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class OUNoise(object):\n",
    "    \"\"\"\n",
    "    Ornstein-Uhlenbeck Noise\n",
    "    \"\"\"\n",
    "    def __init__(self,\n",
    "                 embedded_action_size,\n",
    "                 ou_mu,\n",
    "                 ou_theta,\n",
    "                 ou_sigma,\n",
    "                 ou_epsilon):\n",
    "        \"\"\"\n",
    "        Initialize OUNoise\n",
    "        \"\"\"\n",
    "        self.embedded_action_size = embedded_action_size\n",
    "        self.ou_mu = ou_mu\n",
    "        self.ou_theta = ou_theta\n",
    "        self.ou_sigma = ou_sigma\n",
    "        self.ou_epsilon = ou_epsilon\n",
    "        self.ou_state = None\n",
    "        self.reset()\n",
    "\n",
    "    def reset(self):\n",
    "        \"\"\"\n",
    "        Reset the OU process state\n",
    "        \"\"\"\n",
    "        self.ou_state = torch.ones(self.embedded_action_size) * self.ou_mu\n",
    "\n",
    "    def evolve_state(self):\n",
    "        \"\"\"\n",
    "        Evolve the OU process state\n",
    "        \"\"\"\n",
    "        self.ou_state += self.ou_theta * (self.ou_mu - self.ou_state) \\\n",
    "            + self.ou_sigma * torch.randn(self.embedded_action_size)\n",
    "\n",
    "    def get_ou_noise(self):\n",
    "        \"\"\"\n",
    "        Get the OU noise for one action\n",
    "        :return OU noise\n",
    "        \"\"\"\n",
    "        self.evolve_state()\n",
    "        return self.ou_state.clone()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 0.1521,  0.2014,  0.1552,  0.0324, -0.3615,  0.2302, -0.1692, -0.1778,\n",
       "         0.0083,  0.0672,  0.0395,  0.0195,  0.2221,  0.0395, -0.1529, -0.2925,\n",
       "         0.0994, -0.0426,  0.0901,  0.2552,  0.2225, -0.0833, -0.2342, -0.1982,\n",
       "         0.2106,  0.1198,  0.2413,  0.3598,  0.0585,  0.0011,  0.2037, -0.0334])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "noise = OUNoise(embedded_action_size = 32,\n",
    "                ou_mu = 0.0,\n",
    "                ou_theta = 0.15,\n",
    "                ou_sigma = 0.2,\n",
    "                ou_epsilon = 1.0,\n",
    ")\n",
    "noise.reset()\n",
    "noise.get_ou_noise()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Author: Sparsh A.\n",
      "\n",
      "Last updated: 2021-12-19 09:15:53\n",
      "\n",
      "Compiler    : GCC 7.5.0\n",
      "OS          : Linux\n",
      "Release     : 5.4.104+\n",
      "Machine     : x86_64\n",
      "Processor   : x86_64\n",
      "CPU cores   : 2\n",
      "Architecture: 64bit\n",
      "\n",
      "torch  : 1.10.0+cu111\n",
      "IPython: 5.5.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "!pip install -q watermark\n",
    "%reload_ext watermark\n",
    "%watermark -a \"Sparsh A.\" -m -iv -u -t -d"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
