{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-09T13:54:24.294222Z",
     "start_time": "2022-11-09T13:54:21.916363Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running typecheck\n",
      "Success: no issues found in 1 source file\n",
      "\n",
      "running flake8\n",
      "flake8 success\n"
     ]
    }
   ],
   "source": [
    "%%checkall\n",
    "from __future__ import annotations\n",
    "import matplotlib as mpl\n",
    "try:\n",
    "    import tkinter\n",
    "except (ImportError, ValueError):\n",
    "    mpl.use('Agg')  # Support running in headless mode\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import sys\n",
    "import tempfile\n",
    "import datetime\n",
    "import pathlib\n",
    "import numpy as np\n",
    "import logging\n",
    "import pandas as pd\n",
    "from typing import Any, Callable\n",
    "\n",
    "\n",
    "SEC_PER_DAY = 3600 * 24\n",
    "_HAS_DISPLAY = None\n",
    "EPOCH = datetime.datetime.utcfromtimestamp(0)\n",
    "DATE_FORMAT = '%Y-%m-%d %H:%M:%S'\n",
    "LOG_FORMAT = '[%(asctime)s.%(msecs)03d %(funcName)s] %(message)s'\n",
    "\n",
    "\n",
    "def has_display() -> bool:\n",
    "    '''\n",
    "    If we are running in unit test mode or on a server, then don't try to draw graphs, etc.\n",
    "    '''\n",
    "    global _HAS_DISPLAY\n",
    "    if _HAS_DISPLAY is not None: return _HAS_DISPLAY\n",
    "    \n",
    "    _HAS_DISPLAY = True\n",
    "    try:\n",
    "        plt.figure()\n",
    "    except tkinter.TclError:\n",
    "        _HAS_DISPLAY = False\n",
    "    return _HAS_DISPLAY\n",
    "\n",
    "\n",
    "def shift_np(array: np.ndarray, n: int, fill_value: Any = None) -> np.ndarray:\n",
    "    '''\n",
    "    Similar to pandas.Series.shift but works on numpy arrays.\n",
    "    \n",
    "    Args:\n",
    "        array: The numpy array to shift\n",
    "        n: Number of places to shift, can be positive or negative\n",
    "        fill_value: After shifting, there will be empty slots left in the array.  If set, fill these with fill_value.\n",
    "          If fill_value is set to None (default), we will fill these with False for boolean arrays, np.nan for floats\n",
    "    '''\n",
    "    if array is None: return None\n",
    "    if len(array) == 0: return array\n",
    "    \n",
    "    if fill_value is None:\n",
    "        fill_value = False if array.dtype == np.dtype(bool) else np.nan\n",
    "\n",
    "    e = np.empty_like(array)\n",
    "    if n >= 0:\n",
    "        e[:n] = fill_value\n",
    "        e[n:] = array[:-n]\n",
    "    else:\n",
    "        e[n:] = fill_value\n",
    "        e[:n] = array[-n:]\n",
    "    return e\n",
    "\n",
    "\n",
    "def set_ipython_defaults(jupyter_multiple_display=True) -> None:\n",
    "    from IPython.core.interactiveshell import InteractiveShell\n",
    "    get_ipython().run_line_magic('matplotlib', 'inline')  # type: ignore # noqa: 89  \n",
    "    \n",
    "    if jupyter_multiple_display:\n",
    "        InteractiveShell.ast_node_interactivity = 'all'\n",
    "    \n",
    "\n",
    "def set_defaults(df_float_sf: int = 9, \n",
    "                 df_display_max_rows: int = 200, \n",
    "                 df_display_max_columns: int = 99,\n",
    "                 np_seterr: str = 'raise',\n",
    "                 mpl_figsize: tuple[int, int] = (8, 6),\n",
    "                 jupyter_multiple_display=True) -> None:\n",
    "    '''\n",
    "    Set some display defaults to make it easier to view dataframes and graphs.\n",
    "    \n",
    "    Args:\n",
    "        df_float_sf: Number of significant figures to show in dataframes (default 4). Set to None to use pandas defaults\n",
    "        df_display_max_rows: Number of rows to display for pandas dataframes when you print them (default 200).  Set to None to use pandas defaults\n",
    "        df_display_max_columns: Number of columns to display for pandas dataframes when you print them (default 99).  Set to None to use pandas defaults\n",
    "        np_seterr: Error mode for numpy warnings.  See numpy seterr function for details.  Set to None to use numpy defaults\n",
    "        mpl_figsize: Default figure size to use when displaying matplotlib plots (default 8,6).  Set to None to use defaults\n",
    "        jupyter_multiple_display: If set, and you have multiple outputs in a Jupyter cell, output will contain all of them. Default True\n",
    "    '''\n",
    "    if df_float_sf is not None: pd.options.display.float_format = ('{:.' + str(df_float_sf) + 'g}').format\n",
    "    if df_display_max_rows is not None: pd.options.display.max_rows = df_display_max_rows\n",
    "    if df_display_max_columns is not None: pd.options.display.max_columns = df_display_max_columns\n",
    "    if mpl_figsize is not None: mpl.rcParams['figure.figsize'] = mpl_figsize\n",
    "    if np_seterr is not None: np.seterr(np_seterr)  # type: ignore\n",
    "    pd.options.mode.chained_assignment = None  # Turn off bogus 'view' warnings from pandas when modifying dataframes\n",
    "    # Display all cell outputs\n",
    "    plt.rcParams.update({'figure.max_open_warning': 100})  # For unit tests, avoid warning when opening more than 20 figures\n",
    "    if in_ipython():\n",
    "        set_ipython_defaults(jupyter_multiple_display)\n",
    "    \n",
    "\n",
    "def str2date(s: np.datetime64 | str | None) -> np.datetime64:\n",
    "    '''Converts a string like \"2008-01-15 15:00:00\" to a numpy datetime64.  If s is not a string, return s back'''\n",
    "    if isinstance(s, str): return np.datetime64(s)\n",
    "    if s is None: return np.datetime64('')  # nat\n",
    "    return s\n",
    "\n",
    "\n",
    "def strtup2date(tup: Any) -> tuple[np.datetime64 | None, np.datetime64 | None]:\n",
    "    '''Converts a string tuple like (\"2008-01-15\", \"2009-01-16\") to a numpy datetime64 tuple.  \n",
    "      If the tuple does not contain strings, return it back unchanged'''\n",
    "    if tup and type(tup) is tuple and isinstance(tup[0], str): return (str2date(tup[0]), str2date(tup[1]))\n",
    "    return tup\n",
    "\n",
    "\n",
    "def remove_dups(lst: list[Any], key_func: Callable[[Any], Any] | None = None) -> list[Any]:\n",
    "    '''\n",
    "    Remove duplicates from a list \n",
    "    Args:\n",
    "        lst: list to remove duplicates from\n",
    "        key_func: A function that takes a list element and converts it to a key for detecting dups\n",
    "        \n",
    "    Return:\n",
    "        A list with duplicates removed.  This is stable in the sense that original list elements will retain their order\n",
    "    \n",
    "    >>> print(remove_dups(['a', 'd', 'a', 'c']))\n",
    "    ['a', 'd', 'c']\n",
    "    >>> print(remove_dups(['a', 'd', 'A']))\n",
    "    ['a', 'd', 'A']\n",
    "    >>> print(remove_dups(['a', 'd', 'A'], key_func = lambda e: e.upper()))\n",
    "    ['a', 'd']\n",
    "    '''\n",
    "    new_list = []\n",
    "    seen: set[Any] = set() \n",
    "    for element in lst:\n",
    "        if key_func:\n",
    "            key = key_func(element)\n",
    "        else:\n",
    "            key = element\n",
    "        if key not in seen:\n",
    "            new_list.append(element)\n",
    "            seen.add(key)\n",
    "    return new_list\n",
    "\n",
    "\n",
    "def np_get_index(array: np.ndarray, value: Any) -> int:\n",
    "    '''Get index of a value in a numpy array.  Returns -1 if the value does not exist.'''\n",
    "    x = np.where(array == value)\n",
    "    if len(x[0]): return x[0][0]\n",
    "    return -1\n",
    "\n",
    "\n",
    "def np_find_closest(a: np.ndarray, v: Any) -> int | np.ndarray:\n",
    "    '''\n",
    "    From https://stackoverflow.com/questions/8914491/finding-the-nearest-value-and-return-the-index-of-array-in-python\n",
    "    Find index of closest value to array v in array a.  Returns an array of the same size as v\n",
    "    a must be sorted\n",
    "    >>> assert(all(np_find_closest(np.array([3, 4, 6]), np.array([4, 2])) == np.array([1, 0])))\n",
    "    '''\n",
    "    idx_ = a.searchsorted(v)\n",
    "    idx = np.clip(idx_, 1, len(a) - 1)\n",
    "    left = a[idx - 1]\n",
    "    right = a[idx]\n",
    "    idx -= v - left < right - v\n",
    "    return idx  # type: ignore\n",
    "\n",
    "\n",
    "def np_rolling_window(a: np.ndarray, window: int) -> np.ndarray:\n",
    "    '''\n",
    "    For applying rolling window functions to a numpy array\n",
    "    See: https://stackoverflow.com/questions/6811183/rolling-window-for-1d-arrays-in-numpy\n",
    "    >>> print(np.std(np_rolling_window(np.array([1, 2, 3, 4]), 2), 1))\n",
    "    [0.5 0.5 0.5]\n",
    "    '''\n",
    "    shape = a.shape[:-1] + (a.shape[-1] - window + 1, window)\n",
    "    strides = a.strides + (a.strides[-1],)\n",
    "    return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides)  # type: ignore\n",
    "\n",
    "\n",
    "def np_round(a: np.ndarray, clip: float):\n",
    "    '''\n",
    "    Round all elements in an array to the nearest clip\n",
    "    \n",
    "    Args:\n",
    "        a: array with elements to round\n",
    "        clip: rounding value\n",
    "    >>> np_round(15.8, 0.25)\n",
    "    15.75\n",
    "    '''\n",
    "        \n",
    "    return np.round(np.array(a, dtype=float) / clip) * clip\n",
    "\n",
    "\n",
    "def np_bucket(a: np.ndarray, buckets: list[Any], default_value=0, side='mid') -> np.ndarray:\n",
    "    '''\n",
    "    Given a numpy array and a sorted list of buckets, assign each element to a bucket.\n",
    "    \n",
    "    Args:\n",
    "        a (np.ndarray): The numpy array of values\n",
    "        buckets: (list) List of buckets\n",
    "        default_value: Used when we cannot assign an element to any bucket if side is 'left' or 'right'\n",
    "        side (str): If set to mid, we use the midpoint between buckets to assign elements\n",
    "            'left', assignment <= element\n",
    "            'right', assignment >= element\n",
    "            Default: 'mid'\n",
    "    Return:\n",
    "        np.ndarray of same length as a\n",
    "        \n",
    "    >>> a = np.array([1, 5, 18, 3, 6, 10, 4])\n",
    "    >>> buckets = [4, 8, 12]\n",
    "    >>> assert np.alltrue(np_bucket(a, buckets, side='left') == np.array([0,  4, 12,  0,  4,  8,  4]))\n",
    "    >>> assert np.alltrue(np_bucket(a, buckets, default_value=25, side='right') == np.array([4,  8, 25,  4,  8, 12,  4])) \n",
    "    >>> assert(np.alltrue(np_bucket(a, buckets) == np.array([4,  4, 12,  4,  8, 12,  4])))\n",
    "    '''\n",
    "    assert side in ['mid', 'left', 'right'], f'unknown side: {side}'\n",
    "    if side == 'mid':\n",
    "        b = [0.5 * (buckets[i + 1] + buckets[i]) for i in range(len(buckets) - 1)]\n",
    "        conditions = [(a < e) for e in b]\n",
    "        ret = np.select(conditions, buckets[:-1], default=buckets[-1])\n",
    "    else:\n",
    "        conditions = [(a < buckets[i]) for i in range(len(buckets))]\n",
    "        if side == 'left':\n",
    "            buckets = buckets[::-1]\n",
    "            conditions = [(a >= buckets[i]) for i in range(len(buckets))]\n",
    "        else:\n",
    "            conditions = [(a <= buckets[i]) for i in range(len(buckets))]\n",
    "        ret = np.select(conditions, buckets, default=default_value)\n",
    "    return ret\n",
    "\n",
    "\n",
    "def np_parse_array(s: str, dtype=float) -> np.ndarray:\n",
    "    '''\n",
    "    Create a 1 or 2 d numpy array from a string that looks like:\n",
    "    [[2. 5. 3. 0. 0.]\n",
    "    [3. 5. 0. 4. 3.]]\n",
    "    or\n",
    "    [2. 5. 3. 0. 8.]  \n",
    "    \n",
    "    >>> x = np_parse_array('[[2. 5. 3. 0. 0.]\\\\n [3. 5. 0. 4. 3.]]')\n",
    "    >>> assert np.allclose(x, np.array([[2., 5., 3., 0., 0.], [3., 5., 0., 4., 3.]]))\n",
    "    >>> x = np_parse_array('[3 4. 5]')\n",
    "    >>> assert np.allclose(x, np.array([3, 4., 5]))\n",
    "    '''\n",
    "    height = s.count(']') - 1\n",
    "    for char in [']', '[', '\\n']:\n",
    "        s = s.replace(char, '')\n",
    "    x = np.fromstring(s, sep=' ', dtype=dtype)\n",
    "    if height > 0:\n",
    "        width = int(len(x) / height)\n",
    "        x = x.reshape(height, width)\n",
    "    return x\n",
    "\n",
    "\n",
    "def np_inc_dates(dates: np.ndarray, num_days: int = 1) -> np.ndarray:\n",
    "    '''\n",
    "    Increment the given date array so each cell gets the next higher value in this array\n",
    "    >>> dates = np.array(['2021-06-01', '2021-06-01', '2021-08-01', '2021-04-01'], dtype='M8[D]')\n",
    "    >>> check = np.array([dates[2], dates[2], np.datetime64('nat'), dates[0]])\n",
    "    >>> assert np.array_equal(np_inc_dates(dates, 1), \n",
    "    ...    np.array(['2021-08-01', '2021-08-01', 'NaT', '2021-06-01'], dtype='M8[D]'), equal_nan=True)\n",
    "    >>> assert np.array_equal(np_inc_dates(dates, 2), \n",
    "    ...    np.array(['NaT', 'NaT', 'NaT', '2021-08-01'], dtype='M8[D]'), equal_nan=True)\n",
    "    >>> assert np.array_equal(np_inc_dates(dates, -1), \n",
    "    ...    np.array(['2021-04-01', '2021-04-01', '2021-06-01', 'NaT'], dtype='M8[D]'), equal_nan=True)\n",
    "    >>> assert np.array_equal(np_inc_dates(dates, -2), \n",
    "    ...    np.array(['NaT', 'NaT', '2021-04-01', 'NaT'], dtype='M8[D]'), equal_nan=True)\n",
    "    '''\n",
    "    uniq_dates = np.unique(dates)\n",
    "    date_values = np.concatenate([uniq_dates, [np.datetime64('nat')]])\n",
    "    indices = np.searchsorted(uniq_dates, dates) + num_days\n",
    "    indices = np.where((indices < 0) | (indices > len(uniq_dates)), len(uniq_dates), indices)\n",
    "    return date_values[indices]\n",
    "\n",
    "\n",
    "def np_uniques(arrays: list[np.ndarray]) -> np.ndarray:\n",
    "    '''\n",
    "    Given a list of numpy arrays that may have different datatype, generate a structured numpy\n",
    "    array with sorted, unique elements from that list\n",
    "    >>> array1 = np.array(['2018-01-02', '2018-01-03', '2018-01-02', '2018-01-03'], dtype='M8[D]')\n",
    "    >>> array2 = np.array(['P', 'P', 'P', 'C'])\n",
    "    >>> x = np_uniques([array1, array2])\n",
    "    >>> assert len(x) == 3\n",
    "    >>> assert x[0][0] == np.datetime64('2018-01-02')\n",
    "    >>> assert x[0][1] ==  'P'\n",
    "    '''\n",
    "    size = len(arrays[0])\n",
    "    cols = len(arrays)\n",
    "    data_ = [tuple([arrays[i][j] for i in range(cols)]) for j in range(size)]\n",
    "    data = np.array(data_, dtype=[('', col.dtype) for col in arrays])\n",
    "    return np.unique(data)\n",
    "\n",
    "\n",
    "def day_of_week_num(a: np.datetime64 | np.ndarray) -> int | np.ndarray:\n",
    "    '''\n",
    "    From https://stackoverflow.com/questions/52398383/finding-day-of-the-week-for-a-datetime64\n",
    "    Get day of week for a numpy array of datetimes \n",
    "    Monday is 0, Sunday is 6\n",
    "    \n",
    "    Args:\n",
    "        a: numpy datetime64 or array of datetime64\n",
    "        \n",
    "    Return:\n",
    "        int or numpy ndarray of int: Monday is 0, Sunday is 6\n",
    "\n",
    "    >>> day_of_week_num(np.datetime64('2015-01-04'))\n",
    "    6\n",
    "    '''\n",
    "    int_date: int = a.astype('datetime64[D]').view('int64')  # type: ignore\n",
    "    ret = (int_date - 4) % 7\n",
    "    # if np.isscalar(ret): ret = ret.item()\n",
    "    return ret\n",
    "\n",
    "\n",
    "def percentile_of_score(a: np.ndarray) -> np.ndarray | None:\n",
    "    '''\n",
    "    For each element in a, find the percentile of a its in.  From stackoverflow.com/a/29989971/5351549\n",
    "    Like scipy.stats.percentileofscore but runs in O(n log(n)) time.\n",
    "    >>> a = np.array([4, 3, 1, 2, 4.1])\n",
    "    >>> percentiles = percentile_of_score(a)\n",
    "    >>> assert(all(np.isclose(np.array([ 75.,  50.,   0.,  25., 100.]), percentiles)))\n",
    "    '''\n",
    "    assert isinstance(a, np.ndarray), f'expected numpy array, got: {a}'\n",
    "    if not len(a): return None\n",
    "    return np.argsort(np.argsort(a)) * 100. / (len(a) - 1)\n",
    "\n",
    "\n",
    "def date_2_num(d: np.datetime64 | np.ndarray) -> int | np.ndarray:\n",
    "    from matplotlib.dates import date2num\n",
    "    return date2num(d)\n",
    "\n",
    "\n",
    "def resample_vwap(df: pd.DataFrame, sampling_frequency: str) -> np.ndarray | None:\n",
    "    '''\n",
    "    Compute weighted average of vwap given higher frequency vwap and volume\n",
    "    '''\n",
    "    if 'v' not in df.columns: return None\n",
    "    sum_1 = df.vwap * df.v\n",
    "    sum_2 = sum_1.resample(sampling_frequency).agg(np.sum)\n",
    "    volume_sum = df.v.resample(sampling_frequency).agg(np.sum)\n",
    "    vwap = sum_2 / volume_sum\n",
    "    return vwap\n",
    "\n",
    "\n",
    "def resample_trade_bars(df, sampling_frequency, resample_funcs=None):\n",
    "    '''Downsample trade bars using sampling frequency\n",
    "    \n",
    "    Args:\n",
    "        df (pd.DataFrame): Must contain an index of numpy datetime64 type which is monotonically increasing\n",
    "        sampling_frequency (str): See pandas frequency strings\n",
    "        resample_funcs (dict of str: int): a dictionary of column name -> resampling function for any columns that are custom defined.  Default None.\n",
    "            If there is no entry for a custom column, defaults to 'last' for that column\n",
    "    Returns:\n",
    "        pd.DataFrame: Resampled dataframe\n",
    "        \n",
    "    >>> import math\n",
    "    >>> df = pd.DataFrame({'date': np.array(['2018-01-08 15:00:00', '2018-01-09 13:30:00', '2018-01-09 15:00:00', '2018-01-11 15:00:00'], dtype = 'M8[ns]'),\n",
    "    ...          'o': np.array([8.9, 9.1, 9.3, 8.6]), \n",
    "    ...          'h': np.array([9.0, 9.3, 9.4, 8.7]), \n",
    "    ...          'l': np.array([8.8, 9.0, 9.2, 8.4]), \n",
    "    ...          'c': np.array([8.95, 9.2, 9.35, 8.5]),\n",
    "    ...          'v': np.array([200, 100, 150, 300]),\n",
    "    ...          'x': np.array([300, 200, 100, 400])\n",
    "    ...         })\n",
    "    >>> df['vwap'] =  0.5 * (df.l + df.h)\n",
    "    >>> df.set_index('date', inplace = True)\n",
    "    >>> df = resample_trade_bars(df, sampling_frequency = 'D', resample_funcs={'x': lambda df, \n",
    "    ...   sampling_frequency: df.x.resample(sampling_frequency).agg(np.mean)})\n",
    "    >>> assert(len(df) == 4)\n",
    "    >>> assert(math.isclose(df.vwap.iloc[1], 9.24))\n",
    "    >>> assert(np.isnan(df.vwap.iloc[2]))\n",
    "    >>> assert(math.isclose(df.l[3], 8.4))\n",
    "    '''\n",
    "    if sampling_frequency is None: return df\n",
    "    \n",
    "    if resample_funcs is None: resample_funcs = {}\n",
    "    if 'vwap' in df.columns: resample_funcs.update({'vwap': resample_vwap})\n",
    "    \n",
    "    funcs = {'o': 'first', 'h': 'max', 'l': 'min', 'c': 'last', 'v': 'sum'}\n",
    "    \n",
    "    agg_dict = {}\n",
    "    \n",
    "    for col in df.columns:\n",
    "        if col in funcs:\n",
    "            agg_dict[col] = funcs[col]\n",
    "            continue\n",
    "        if col not in resample_funcs:\n",
    "            agg_dict[col] = 'last'\n",
    "    \n",
    "    resampled = df.resample(sampling_frequency).agg(agg_dict).dropna(how='all')\n",
    "    \n",
    "    for k, v in resample_funcs.items():\n",
    "        res = v(df, sampling_frequency)\n",
    "        if res is not None: resampled[k] = res\n",
    "            \n",
    "    resampled.reset_index(inplace=True)\n",
    "    return resampled\n",
    "\n",
    "\n",
    "def resample_ts(dates: np.ndarray, values: np.ndarray, sampling_frequency: str) -> tuple[np.ndarray, np.ndarray]:\n",
    "    '''Downsample a tuple of datetimes and value arrays using sampling frequency, using the last value if it does not exist at the bin edge.\n",
    "    See pandas.Series.resample\n",
    "    \n",
    "    Args:\n",
    "        dates: a numpy datetime64 array\n",
    "        values: a numpy array\n",
    "        sampling_frequency: See pandas frequency strings\n",
    "        \n",
    "    Returns:\n",
    "        Resampled tuple of datetime and value arrays\n",
    "    '''\n",
    "    if sampling_frequency is None: return dates, values\n",
    "    s = pd.Series(values, index=dates).resample(sampling_frequency).last()\n",
    "    return s.index.values, s.values\n",
    "\n",
    "\n",
    "def zero_to_nan(array: np.ndarray) -> np.ndarray:\n",
    "    '''Converts any zeros in a numpy array to nans'''\n",
    "    if array is None: return None\n",
    "    return np.where(array == 0, np.nan, array)\n",
    "\n",
    "\n",
    "def nan_to_zero(array: np.ndarray) -> np.ndarray:\n",
    "    '''Converts any nans in a numpy float array to 0'''\n",
    "    if array is None: return None\n",
    "    return np.where(np.isnan(array), 0, array)\n",
    "\n",
    "\n",
    "def monotonically_increasing(array: np.ndarray) -> bool:\n",
    "    '''\n",
    "    Returns True if the array is monotonically_increasing, False otherwise\n",
    "    \n",
    "    >>> monotonically_increasing(np.array(['2018-01-02', '2018-01-03'], dtype = 'M8[D]'))\n",
    "    True\n",
    "    >>> monotonically_increasing(np.array(['2018-01-02', '2018-01-02'], dtype = 'M8[D]'))\n",
    "    False\n",
    "    '''\n",
    "    if not len(array): return False\n",
    "    ret: bool = np.all(np.diff(array).astype(float) > 0).astype(bool)  # type: ignore\n",
    "    return ret\n",
    "\n",
    "\n",
    "def infer_frequency(timestamps: np.ndarray) -> float:\n",
    "    '''Returns most common frequency of date differences as a fraction of days\n",
    "    Args:\n",
    "        timestamps: A numpy array of monotonically increasing datetime64\n",
    "    >>> timestamps = np.array(['2018-01-01 11:00:00', '2018-01-01 11:15:00', '2018-01-01 11:30:00', '2018-01-01 11:35:00'], dtype = 'M8[ns]')\n",
    "    >>> print(round(infer_frequency(timestamps), 8))\n",
    "    0.01041667\n",
    "    '''\n",
    "    if isinstance(timestamps, pd.Series): timestamps = timestamps.values\n",
    "    assert (monotonically_increasing(timestamps))\n",
    "    numeric_dates = date_2_num(timestamps)\n",
    "    diff_dates = np.round(np.diff(numeric_dates), 8)\n",
    "    (values, counts) = np.unique(diff_dates, return_counts=True)\n",
    "    return values[np.argmax(counts)]\n",
    "\n",
    "\n",
    "def series_to_array(series: pd.Series) -> np.ndarray:\n",
    "    '''Convert a pandas series to a numpy array.  If the object is not a pandas Series return it back unchanged'''\n",
    "    if type(series) == pd.Series: return series.values\n",
    "    return series\n",
    "\n",
    "\n",
    "def to_csv(df, file_name: str, index: bool = False, compress: bool = False, *args, **kwargs) -> None:\n",
    "    \"\"\"\n",
    "    Creates a temporary file then renames to the permanent file so we don't have half written files.\n",
    "    Also optionally compresses using the xz algorithm\n",
    "    \"\"\"\n",
    "    compression = None\n",
    "    suffix = ''\n",
    "    if compress:\n",
    "        compression = 'xz'\n",
    "        suffix = '.xz'\n",
    "    df.to_csv(file_name + '.tmp', index=index, compression=compression, *args, **kwargs)\n",
    "    os.rename(file_name + '.tmp', file_name + suffix)\n",
    "    \n",
    "\n",
    "def millis_since_epoch(dt: datetime.datetime) -> float:\n",
    "    \"\"\"\n",
    "    Given a python datetime, return number of milliseconds between the unix epoch and the datetime.\n",
    "    Returns a float since it can contain fractions of milliseconds as well\n",
    "    >>> millis_since_epoch(datetime.datetime(2018, 1, 1))\n",
    "    1514764800000.0\n",
    "    \"\"\"\n",
    "    return (dt - EPOCH).total_seconds() * 1000.0\n",
    "\n",
    "\n",
    "def day_symbol(day_int: int | np.ndarray) -> str | np.ndarray:\n",
    "    day_str = np.select([day_int == 0, day_int == 1, day_int == 2, day_int == 3, day_int == 4, day_int == 5, day_int == 6],\n",
    "                        ['M', 'Tu', 'W', 'Th', 'F', 'Sa', 'Su'], default='')\n",
    "    if day_str.shape == (): day_str = day_str.item()\n",
    "    return day_str\n",
    "\n",
    "\n",
    "def infer_compression(input_filename: str) -> str | None:\n",
    "    \"\"\"\n",
    "    Infers compression for a file from its suffix.  For example, given \"/tmp/hello.gz\", this will return \"gzip\"\n",
    "    >>> infer_compression(\"/tmp/hello.gz\")\n",
    "    'gzip'\n",
    "    >>> infer_compression(\"/tmp/abc.txt\") is None\n",
    "    True\n",
    "    \"\"\"\n",
    "    parts = input_filename.split('.')\n",
    "    if len(parts) <= 1: return None\n",
    "    suffix = parts[-1]\n",
    "    if suffix == 'gz': return 'gzip'\n",
    "    if suffix == 'bz2': return 'bz2'\n",
    "    if suffix == 'zip': return 'zip'\n",
    "    if suffix == 'xz': return 'xz'\n",
    "    return None\n",
    "\n",
    "\n",
    "def touch(fname: str, mode: int = 0o666, dir_fd: int | None = None, **kwargs) -> None:\n",
    "    '''replicate unix touch command, i.e create file if it doesn't exist, otherwise update timestamp'''\n",
    "    flags = os.O_CREAT | os.O_APPEND\n",
    "    with os.fdopen(os.open(fname, flags=flags, mode=mode, dir_fd=dir_fd)) as f:\n",
    "        os.utime(f.fileno() if os.utime in os.supports_fd else fname,\n",
    "                 dir_fd=None if os.supports_fd else dir_fd, **kwargs)\n",
    "        \n",
    "\n",
    "def is_newer(filename: str, ref_filename: str) -> bool:\n",
    "    '''whether filename ctime (modfication time) is newer than ref_filename or either file does not exist\n",
    "    >>> import time\n",
    "    >>> import tempfile\n",
    "    >>> temp_dir = tempfile.gettempdir()\n",
    "    >>> touch(f'{temp_dir}/x.txt')\n",
    "    >>> time.sleep(0.1)\n",
    "    >>> touch(f'{temp_dir}/y.txt')\n",
    "    >>> is_newer(f'{temp_dir}/y.txt', f'{temp_dir}/x.txt')\n",
    "    True\n",
    "    >>> touch(f'{temp_dir}/y.txt')\n",
    "    >>> time.sleep(0.1)\n",
    "    >>> touch(f'{temp_dir}/x.txt')\n",
    "    >>> is_newer(f'{temp_dir}/y.txt', f'{temp_dir}/x.txt')\n",
    "    False\n",
    "    ''' \n",
    "    if not os.path.isfile(filename) or not os.path.isfile(ref_filename): return True\n",
    "    return os.path.getmtime(filename) > os.path.getmtime(ref_filename)\n",
    "\n",
    "\n",
    "def get_empty_np_value(np_dtype: np.dtype) -> Any:\n",
    "    '''\n",
    "    Get empty value for a given numpy datatype\n",
    "    >>> a = np.array(['2018-01-01', '2018-01-03'], dtype = 'M8[D]')\n",
    "    >>> get_empty_np_value(a.dtype)\n",
    "    numpy.datetime64('NaT')\n",
    "    '''\n",
    "    kind = np_dtype.kind\n",
    "    if kind == 'f': return np.nan  # float\n",
    "    if kind == 'b': return False  # bool\n",
    "    if kind == 'i' or kind == 'u': return 0  # signed or unsigned int\n",
    "    if kind == 'M': return np.datetime64('NaT')  # datetime\n",
    "    if kind == 'O' or kind == 'S' or kind == 'U': return ''  # object or string or unicode\n",
    "    raise Exception(f'unknown dtype: {np_dtype}')\n",
    "    \n",
    "\n",
    "def get_temp_dir() -> str:\n",
    "    if os.access('/tmp', os.W_OK):\n",
    "        return '/tmp'\n",
    "    else:\n",
    "        return tempfile.gettempdir()\n",
    "    \n",
    "\n",
    "def linear_interpolate(a1: np.ndarray | float, \n",
    "                       a2: np.ndarray | float, \n",
    "                       x1: np.ndarray | float, \n",
    "                       x2: np.ndarray | float,\n",
    "                       x: np.ndarray | float) -> np.ndarray | float:\n",
    "    '''\n",
    "    >>> assert(linear_interpolate(3, 4, 8, 10, 8.9) == 3.45)\n",
    "    >>> assert(linear_interpolate(3, 3, 8, 10, 8.9) == 3)\n",
    "    >>> assert(np.isnan(linear_interpolate(3, 4, 8, 8, 8.9)))\n",
    "    >>> x = linear_interpolate(\n",
    "    ...    np.array([3., 3.]), \n",
    "    ...    np.array([4., 3.]), \n",
    "    ...    np.array([8., 8.]),\n",
    "    ...    np.array([10, 8.]), \n",
    "    ...    np.array([8.9, 8.]))\n",
    "    >>> assert(np.allclose(x, np.array([3.45, 3.])))\n",
    "    '''\n",
    "    diff = x2 - x1\n",
    "    diff = np.where(diff == 0, 1, diff)\n",
    "    return np.where((a2 == a1), a1, \n",
    "                    np.where(x2 == x1, np.nan, a1 + (a2 - a1) * (x - x1) / diff))\n",
    "\n",
    "\n",
    "def bootstrap_ci(a: np.ndarray, \n",
    "                 ci_level: float = 0.95,\n",
    "                 n: int = 1000, \n",
    "                 func: Callable[[np.ndarray], np.ndarray] = np.mean) -> tuple[float, float]:  # type: ignore\n",
    "    '''\n",
    "    Non parametric bootstrap for confidence intervals\n",
    "    Args:\n",
    "        a: The data to bootstrap from\n",
    "        ci_level: The confidence interval level, e.g. 0.95 for 95%.  Default 0.95\n",
    "        n: Number of boostrap iterations. Default 1000\n",
    "        func: The function to use, e.g np.mean or np.median. Default np.mean\n",
    " \n",
    "        \n",
    "    Return:\n",
    "        A tuple containing the lower and upper ci\n",
    "    >>> np.random.seed(0)\n",
    "    >>> x = np.random.uniform(high=10, size=100000)\n",
    "    >>> assert np.allclose(bootstrap_ci(x), (4.9773159, 5.010328))\n",
    "    '''\n",
    "    simulations = np.full(n, np.nan)\n",
    "    sample_size = len(a)\n",
    "    for c in range(n):\n",
    "        itersample = np.random.choice(a, size=sample_size, replace=True)\n",
    "        simulations[c] = func(itersample)\n",
    "    simulations.sort()\n",
    "    u_pval = (1 + ci_level) / 2.\n",
    "    l_pval = (1 - u_pval)\n",
    "    l_indx = int(np.floor(n * l_pval))\n",
    "    u_indx = int(np.floor(n * u_pval))\n",
    "    return (simulations[l_indx], simulations[u_indx])\n",
    "\n",
    "\n",
    "def _add_stream_handler(logger: logging.Logger, \n",
    "                        log_level: int = logging.INFO, \n",
    "                        formatter: logging.Formatter | None = None) -> None:\n",
    "    if formatter is None: formatter = logging.Formatter(fmt=LOG_FORMAT, datefmt=DATE_FORMAT)\n",
    "    stream_handler = logging.StreamHandler(sys.stdout)\n",
    "    # stream_handler = logging.StreamHandler()\n",
    "    stream_handler.setFormatter(formatter)\n",
    "    stream_handler.setLevel(log_level)\n",
    "    logger.addHandler(stream_handler)\n",
    "\n",
    "\n",
    "def get_main_logger() -> logging.Logger:\n",
    "    # sys.stderr = sys.stdout\n",
    "    main_logger = logging.getLogger('pq')\n",
    "    if len(main_logger.handlers): return main_logger\n",
    "    _add_stream_handler(main_logger)\n",
    "    main_logger.setLevel(logging.INFO)\n",
    "    main_logger.propagate = False\n",
    "    return main_logger\n",
    "\n",
    "\n",
    "def get_child_logger(child_name: str) -> logging.Logger:\n",
    "    _ = get_main_logger()  # Init handlers if needed\n",
    "    full_name = 'pq.' + child_name if child_name else 'pq'\n",
    "    logger = logging.getLogger(full_name)\n",
    "    return logger\n",
    "\n",
    "\n",
    "def in_ipython() -> bool:\n",
    "    '''\n",
    "    Whether we are running in an ipython (or Jupyter) environment\n",
    "    '''\n",
    "    import builtins\n",
    "    return '__IPYTHON__' in vars(builtins)\n",
    "\n",
    "\n",
    "class PQException(Exception):\n",
    "    pass\n",
    "\n",
    "\n",
    "def assert_(condition: bool, msg: str | None = None) -> None:\n",
    "    '''\n",
    "    Like a python assert but raises an exception that is not turned off by \n",
    "    using the python optimization switch\n",
    "    '''\n",
    "    if msg is None: msg = ''\n",
    "    if not condition: raise PQException(msg)\n",
    "\n",
    "\n",
    "class Paths:\n",
    "    '''\n",
    "    Conventions for where to read / write data and reports\n",
    "    '''\n",
    "    def __init__(self, base_path: str | None = None) -> None:\n",
    "        if base_path:\n",
    "            self.base_path = pathlib.Path(base_path)\n",
    "        else:\n",
    "            self.base_path = pathlib.Path.cwd()\n",
    "        # Data paths\n",
    "        self.data_path = self.base_path / 'data'\n",
    "        self.raw_data_path = self.data_path / 'raw'\n",
    "        self.interim_data_path = self.data_path / 'interim'\n",
    "        self.processed_data_path = self.data_path / 'processed'\n",
    "        self.external_data_path = self.data_path / 'external'\n",
    "\n",
    "        # Reports paths\n",
    "        self.reports_path = self.base_path / 'reports'\n",
    "        self.figures_path = self.reports_path / 'figures'\n",
    "        \n",
    "    def create(self) -> None:\n",
    "        default_mode = 0o755\n",
    "        self.data_path.mkdir(mode=default_mode, parents=False, exist_ok=True)\n",
    "        self.raw_data_path.mkdir(mode=default_mode, parents=False, exist_ok=True)\n",
    "        self.interim_data_path.mkdir(mode=default_mode, parents=False, exist_ok=True)\n",
    "        self.processed_data_path.mkdir(mode=default_mode, parents=False, exist_ok=True)\n",
    "        self.external_data_path.mkdir(mode=default_mode, parents=False, exist_ok=True)\n",
    "        self.reports_path.mkdir(mode=default_mode, parents=False, exist_ok=True)\n",
    "        self.figures_path.mkdir(mode=default_mode, parents=False, exist_ok=True)\n",
    "        \n",
    "\n",
    "def get_paths(base_path: str | None = None) -> Paths:\n",
    "    paths = Paths(base_path)\n",
    "    paths.create()\n",
    "    return paths\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    import doctest\n",
    "    doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
