{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-02-01T00:30:17.107381Z",
     "start_time": "2019-02-01T00:30:16.347588Z"
    },
    "run_control": {
     "frozen": false,
     "read_only": false
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import tempfile\n",
    "import numpy as np\n",
    "import datetime\n",
    "import pandas as pd\n",
    "import matplotlib as mpl\n",
    "try:\n",
    "    import tkinter\n",
    "except:\n",
    "    mpl.use('Agg')  # Support running in headless mode\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "SEC_PER_DAY = 3600 * 24\n",
    "\n",
    "_HAS_DISPLAY = None\n",
    "EPOCH = datetime.datetime.utcfromtimestamp(0)\n",
    "\n",
    "class ReasonCode:\n",
    "    '''A class containing constants for predefined order reason codes. Prefer these predefined reason codes if they suit\n",
    "    the reason you are creating your order.  Otherwise, use your own string.\n",
    "    '''\n",
    "    ENTER_LONG = 'enter long'\n",
    "    ENTER_SHORT = 'enter short'\n",
    "    EXIT_LONG = 'exit long'\n",
    "    EXIT_SHORT = 'exit short'\n",
    "    BACKTEST_END = 'backtest end'\n",
    "    ROLL_FUTURE = 'roll future'\n",
    "    NONE = 'none'\n",
    "    \n",
    "    # Used for plotting trades\n",
    "    MARKER_PROPERTIES = {\n",
    "        ENTER_LONG : {'symbol' : 'P', 'color' : 'blue', 'size' : 50},\n",
    "        ENTER_SHORT : {'symbol' : 'P', 'color' : 'red', 'size' : 50},\n",
    "        EXIT_LONG : {'symbol' : 'X', 'color' : 'blue', 'size' : 50},\n",
    "        EXIT_SHORT : {'symbol' : 'X', 'color' : 'red', 'size' : 50},\n",
    "        ROLL_FUTURE : {'symbol' : '>', 'color' : 'green', 'size' : 50},\n",
    "        BACKTEST_END : {'symbol' : '*', 'color' : 'green', 'size' : 50},\n",
    "        NONE : {'symbol' : 'o', 'color' : 'green', 'size' : 50}\n",
    "    }\n",
    " \n",
    "def has_display():\n",
    "    '''\n",
    "    If we are running in unit test mode or on a server, then don't try to draw graphs, etc.\n",
    "    '''\n",
    "    global _HAS_DISPLAY\n",
    "    if _HAS_DISPLAY is not None: return _HAS_DISPLAY\n",
    "    \n",
    "    _HAS_DISPLAY = True\n",
    "    try:\n",
    "        plt.figure()\n",
    "    except:\n",
    "        _HAS_DISPLAY = False\n",
    "    return _HAS_DISPLAY\n",
    "\n",
    "\n",
    "def shift_np(array, n, fill_value = None):\n",
    "    '''\n",
    "    Similar to pandas.Series.shift but works on numpy arrays.\n",
    "    \n",
    "    Args:\n",
    "        array: The numpy array to shift\n",
    "        n: Number of places to shift, can be positive or negative\n",
    "        fill_value: After shifting, there will be empty slots left in the array.  If set, fill these with fill_value.\n",
    "          If fill_value is set to None (default), we will fill these with False for boolean arrays, np.nan for floats\n",
    "    '''\n",
    "    if array is None: return None\n",
    "    if len(array) == 0: return array\n",
    "    \n",
    "    if fill_value is None:\n",
    "        fill_value = False if array.dtype == np.dtype(bool) else np.nan\n",
    "\n",
    "    e = np.empty_like(array)\n",
    "    if n >= 0:\n",
    "        e[:n] = fill_value\n",
    "        e[n:] = array[:-n]\n",
    "    else:\n",
    "        e[n:] = fill_value\n",
    "        e[:n] = array[-n:]\n",
    "    return e\n",
    "\n",
    "def set_defaults(df_float_sf = 4, df_display_max_rows = 200, df_display_max_columns = 99, np_seterr = 'raise', plot_style = 'ggplot', mpl_figsize = (8, 6)):\n",
    "    '''\n",
    "    Set some display defaults to make it easier to view dataframes and graphs.\n",
    "    \n",
    "    Args:\n",
    "        df_float_sf: Number of significant figures to show in dataframes (default 4). Set to None to use pandas defaults\n",
    "        df_display_max_rows: Number of rows to display for pandas dataframes when you print them (default 200).  Set to None to use pandas defaults\n",
    "        df_display_max_columns: Number of columns to display for pandas dataframes when you print them (default 99).  Set to None to use pandas defaults\n",
    "        np_seterr: Error mode for numpy warnings.  See numpy seterr function for details.  Set to None to use numpy defaults\n",
    "        plot_style: Style for matplotlib plots.  Set to None to use default plot style.\n",
    "        mpl_figsize: Default figure size to use when displaying matplotlib plots (default 8,6).  Set to None to use defaults\n",
    "    '''\n",
    "    if df_float_sf is not None: pd.options.display.float_format = ('{:.' + str(df_float_sf) + 'g}').format\n",
    "    if df_display_max_rows is not None: pd.options.display.max_rows = df_display_max_rows\n",
    "    if df_display_max_columns is not None: pd.options.display.max_columns = df_display_max_columns\n",
    "    if plot_style is not None: plt.style.use(plot_style)\n",
    "    if mpl_figsize is not None: mpl.rcParams['figure.figsize'] = mpl_figsize\n",
    "    if np_seterr is not None: np.seterr(np_seterr)\n",
    "    pd.options.mode.chained_assignment = None # Turn off bogus 'view' warnings from pandas when modifying dataframes\n",
    "    try: # This will run if we are in Jupyter\n",
    "        get_ipython().run_line_magic('matplotlib', 'inline')\n",
    "    except:\n",
    "        pass\n",
    "    plt.rcParams.update({'figure.max_open_warning': 100}) # For unit tests, avoid warning when opening more than 20 figures\n",
    "    \n",
    "def str2date(s):\n",
    "    '''Converts a string like \"2008-01-15 15:00:00\" to a numpy datetime64.  If s is not a string, return s back'''\n",
    "    if isinstance(s, str): return np.datetime64(s)\n",
    "    return s\n",
    "\n",
    "def strtup2date(tup):\n",
    "    '''Converts a string tuple like (\"2008-01-15\", \"2009-01-16\") to a numpy datetime64 tuple.  \n",
    "      If the tuple does not contain strings, return it back unchanged'''\n",
    "    if tup and type(tup) is tuple and isinstance(tup[0], str): return (str2date(tup[0]), str2date(tup[1]))\n",
    "    return tup\n",
    "\n",
    "def np_get_index(array, value):\n",
    "    '''Get index of a value in a numpy array.  Returns -1 if the value does not exist.'''\n",
    "    x = np.where(array == value)\n",
    "    if len(x[0]): return x[0][0]\n",
    "    return -1\n",
    "\n",
    "def date_2_num(d):\n",
    "    '''\n",
    "    Adopted from matplotlib.mdates.date2num so we don't have to add a dependency on matplotlib here\n",
    "    '''\n",
    "    extra = d - d.astype('datetime64[s]').astype(d.dtype)\n",
    "    extra = extra.astype('timedelta64[ns]')\n",
    "    t0 = np.datetime64('0001-01-01T00:00:00').astype('datetime64[s]')\n",
    "    dt = (d.astype('datetime64[s]') - t0).astype(np.float64)\n",
    "    dt += extra.astype(np.float64) / 1.0e9\n",
    "    dt = dt / SEC_PER_DAY + 1.0\n",
    "\n",
    "    NaT_int = np.datetime64('NaT').astype(np.int64)\n",
    "    d_int = d.astype(np.int64)\n",
    "    try:\n",
    "        dt[d_int == NaT_int] = np.nan\n",
    "    except TypeError:\n",
    "        if d_int == NaT_int:\n",
    "            dt = np.nan\n",
    "    return dt\n",
    "\n",
    "def resample_vwap(df, sampling_frequency):\n",
    "    '''\n",
    "    Compute weighted average of vwap given higher frequency vwap and volume\n",
    "    '''\n",
    "    if 'v' not in df.columns: return None\n",
    "    sum_1 = df.vwap * df.v\n",
    "    sum_2 = sum_1.resample(sampling_frequency).agg(np.sum)\n",
    "    volume_sum = df.v.resample(sampling_frequency).agg(np.sum)\n",
    "    vwap = sum_2 / volume_sum\n",
    "    return vwap\n",
    "\n",
    "def resample_ohlc(df, sampling_frequency, resample_funcs = None):\n",
    "    '''Downsample OHLCV data using sampling frequency\n",
    "    \n",
    "    Args:\n",
    "        df (pd.DataFrame): Must contain an index of numpy datetime64 type which is monotonically increasing\n",
    "        sampling_frequency (str): See pandas frequency strings\n",
    "        resample_funcs (dict of str : int) : a dictionary of column name -> resampling function for any columns that are custom defined.  Default None.\n",
    "            If there is no entry for a custom column, defaults to 'last' for that column\n",
    "    Returns:\n",
    "        pd.DataFrame: Resampled dataframe\n",
    "        \n",
    "    >>> df = pd.DataFrame({'date' : np.array(['2018-01-08 15:00:00', '2018-01-09 13:30:00', '2018-01-09 15:00:00', '2018-01-11 15:00:00'], dtype = 'M8[ns]'),\n",
    "    ...           'o' : np.array([8.9, 9.1, 9.3, 8.6]), \n",
    "    ...           'h' : np.array([9.0, 9.3, 9.4, 8.7]), \n",
    "    ...           'l' : np.array([8.8, 9.0, 9.2, 8.4]), \n",
    "    ...           'c' : np.array([8.95, 9.2, 9.35, 8.5]),\n",
    "    ...           'v' : np.array([200, 100, 150, 300]),\n",
    "    ...           'x' : np.array([300, 200, 100, 400])\n",
    "    ...          })\n",
    "    >>> df['vwap'] =  0.5 * (df.l + df.h)\n",
    "    >>> df.set_index('date', inplace = True)\n",
    "    >>> resample_ohlc(df, sampling_frequency = 'D', resample_funcs={'x' : lambda df, sampling_frequency : df.x.resample(sampling_frequency).agg(np.mean)})\n",
    "            date   o   h   l    c    v   x  vwap\n",
    "    0 2018-01-08 8.9   9 8.8 8.95  200 300   8.9\n",
    "    1 2018-01-09 9.1 9.4   9 9.35  250 150  9.24\n",
    "    2 2018-01-10 nan nan nan  nan    0 nan   nan\n",
    "    3 2018-01-11 8.6 8.7 8.4  8.5  300 400  8.55\n",
    "    '''\n",
    "    if sampling_frequency is None: return df\n",
    "    \n",
    "    if resample_funcs is None: resample_funcs = {}\n",
    "    if 'vwap' in df.columns: resample_funcs.update({'vwap' : resample_vwap})\n",
    "    \n",
    "    funcs = {'o': 'first', 'h': 'max', 'l': 'min', 'c': 'last', 'v' : 'sum'}\n",
    "    \n",
    "    agg_dict = {}\n",
    "    \n",
    "    for col in df.columns:\n",
    "        if col in funcs:\n",
    "            agg_dict[col] = funcs[col]\n",
    "            continue\n",
    "        if col not in resample_funcs:\n",
    "            agg_dict[col] = 'last'\n",
    "    \n",
    "    resampled = df.resample(sampling_frequency).agg(agg_dict).dropna(how = 'all')\n",
    "    \n",
    "    for k, v in resample_funcs.items():\n",
    "        res = v(df, sampling_frequency)\n",
    "        if res is not None: resampled[k] = res\n",
    "            \n",
    "    resampled.reset_index(inplace = True)\n",
    "    return resampled\n",
    "\n",
    "def resample_ts(dates, values, sampling_frequency):\n",
    "    '''Downsample a pair of dates and values using sampling frequency, using the last value if it does not exist at bin edge.  See pandas.Series.resample\n",
    "    \n",
    "    Args:\n",
    "        dates: a numpy datetime64 array\n",
    "        values: a numpy array\n",
    "        sampling_frequency: See pandas frequency strings\n",
    "    '''\n",
    "    if sampling_frequency is None: return dates, values\n",
    "    s = pd.Series(values, index = dates).resample(sampling_frequency).last()\n",
    "    return s.index.values, s.values\n",
    "\n",
    "def zero_to_nan(array):\n",
    "    '''Converts any zeros in a numpy array to nans'''\n",
    "    if array is None: return None\n",
    "    return np.where(array == 0, np.nan, array)\n",
    "\n",
    "def nan_to_zero(array):\n",
    "    '''Converts any nans in a numpy float array to 0'''\n",
    "    if array is None: return None\n",
    "    return np.where(np.isnan(array), 0, array)\n",
    "\n",
    "def monotonically_increasing(array):\n",
    "    '''\n",
    "    Returns True if the array is monotonically_increasing, False otherwise\n",
    "    \n",
    "    >>> monotonically_increasing(np.array(['2018-01-02', '2018-01-03'], dtype = 'M8[D]'))\n",
    "    True\n",
    "    >>> monotonically_increasing(np.array(['2018-01-02', '2018-01-02'], dtype = 'M8[D]'))\n",
    "    False\n",
    "    '''\n",
    "    if not len(array): return False\n",
    "    return np.all(np.diff(array).astype(np.float) > 0)\n",
    "\n",
    "def infer_frequency(dates):\n",
    "    '''Returns most common frequency of date differences as a fraction of days\n",
    "    Args:\n",
    "        dates: A numpy array of monotonically increasing datetime64\n",
    "    >>> dates = np.array(['2018-01-01 11:00:00', '2018-01-01 11:15:00', '2018-01-01 11:30:00', '2018-01-01 11:35:00'], dtype = 'M8[ns]')\n",
    "    >>> print(round(infer_frequency(dates), 8))\n",
    "    0.01041667\n",
    "    '''\n",
    "    assert(monotonically_increasing(dates))\n",
    "    numeric_dates = date_2_num(dates)\n",
    "    diff_dates = np.round(np.diff(numeric_dates), 8)\n",
    "    (values,counts) = np.unique(diff_dates, return_counts=True)\n",
    "    ind = np.argmax(counts)\n",
    "    return diff_dates[ind]\n",
    "\n",
    "def series_to_array(series):\n",
    "    '''Convert a pandas series to a numpy array.  If the object is not a pandas Series return it back unchanged'''\n",
    "    if type(series) == pd.Series: return series.values\n",
    "    return series\n",
    "\n",
    "def to_csv(df, file_name, index = False, compress = False, *args, **kwargs):\n",
    "    \"\"\"\n",
    "    Creates a temporary file then renames to the permanent file so we don't have half written files.\n",
    "    Also optionally compresses using the xz algorithm\n",
    "    \"\"\"\n",
    "    compression = None\n",
    "    suffix = ''\n",
    "    if compress:\n",
    "        compression = 'xz'\n",
    "        suffix = '.xz'\n",
    "    df.to_csv(file_name + '.tmp', index = index, compression = compression, *args, **kwargs)\n",
    "    os.rename(file_name + '.tmp', file_name + suffix)\n",
    "    \n",
    "def millis_since_epoch(dt):\n",
    "    \"\"\"\n",
    "    Given a python datetime, return number of milliseconds between the unix epoch and the datetime.\n",
    "    Returns a float since it can contain fractions of milliseconds as well\n",
    "    >>> millis_since_epoch(datetime.datetime(2018, 1, 1))\n",
    "    1514764800000.0\n",
    "    \"\"\"\n",
    "    return (dt - EPOCH).total_seconds() * 1000.0\n",
    "\n",
    "def infer_compression(input_filename):\n",
    "    \"\"\"\n",
    "    Infers compression for a file from its suffix.  For example, given \"/tmp/hello.gz\", this will return \"gzip\"\n",
    "    >>> infer_compression(\"/tmp/hello.gz\")\n",
    "    'gzip'\n",
    "    >>> infer_compression(\"/tmp/abc.txt\") is None\n",
    "    True\n",
    "    \"\"\"\n",
    "    parts = input_filename.split('.')\n",
    "    if len(parts) <= 1: return None\n",
    "    suffix = parts[-1]\n",
    "    if suffix == 'gz': return 'gzip'\n",
    "    if suffix == 'bz2': return 'bz2'\n",
    "    if suffix =='zip': return 'zip'\n",
    "    if suffix == 'xz': return 'xz'\n",
    "    return None\n",
    "\n",
    "\n",
    "def touch(fname, mode=0o666, dir_fd=None, **kwargs):\n",
    "    '''replicate unix touch command, i.e create file if it doesn't exist, otherwise update timestamp'''\n",
    "    flags = os.O_CREAT | os.O_APPEND\n",
    "    with os.fdopen(os.open(fname, flags=flags, mode=mode, dir_fd=dir_fd)) as f:\n",
    "        os.utime(f.fileno() if os.utime in os.supports_fd else fname,\n",
    "            dir_fd=None if os.supports_fd else dir_fd, **kwargs)\n",
    "        \n",
    "def is_newer(filename, ref_filename):\n",
    "    '''whether filename ctime (modfication time) is newer than ref_filename or either file does not exist\n",
    "    >>> import time\n",
    "    >>> import tempfile\n",
    "    >>> temp_dir = tempfile.gettempdir()\n",
    "    >>> touch(f'{temp_dir}/x.txt')\n",
    "    >>> time.sleep(0.1)\n",
    "    >>> touch(f'{temp_dir}/y.txt')\n",
    "    >>> is_newer(f'{temp_dir}/y.txt', f'{temp_dir}/x.txt')\n",
    "    True\n",
    "    >>> touch(f'{temp_dir}/y.txt')\n",
    "    >>> time.sleep(0.1)\n",
    "    >>> touch(f'{temp_dir}/x.txt')\n",
    "    >>> is_newer(f'{temp_dir}/y.txt', f'{temp_dir}/x.txt')\n",
    "    False\n",
    "    ''' \n",
    "    if not os.path.isfile(filename) or not os.path.isfile(ref_filename): return True\n",
    "    return os.path.getmtime(filename) > os.path.getmtime(ref_filename)\n",
    "\n",
    "def get_empty_np_value(np_dtype):\n",
    "    '''\n",
    "    Get empty value for a given numpy datatype\n",
    "    >>> a = np.array(['2018-01-01', '2018-01-03'], dtype = 'M8[D]')\n",
    "    >>> get_empty_np_value(a.dtype)\n",
    "    numpy.datetime64('NaT')\n",
    "    '''\n",
    "    kind = np_dtype.kind\n",
    "    if kind == 'f': return np.nan # float\n",
    "    if kind == 'b': return False # bool\n",
    "    if kind ==  'i' or kind == 'u': return 0 # signed or unsigned int\n",
    "    if kind == 'M': return np.datetime64('NaT') # datetime\n",
    "    if kind == 'O' or kind == 'S' or kind == 'U': return '' # object or string or unicode\n",
    "    raise Exception(f'unknown dtype: {dtype}')\n",
    "    \n",
    "FUTURE_CODES_INT = {'F' : 1, 'G' : 2, 'H' : 3, 'J' : 4, 'K' : 5, 'M' : 6, 'N' : 7, 'Q' : 8, 'U' : 9, 'V' : 10, 'X' : 11, 'Z' : 12}\n",
    "FUTURES_CODES_INVERTED = dict([[v,k] for k,v in FUTURE_CODES_INT.items()])\n",
    "\n",
    "FUTURE_CODES_STR = {'F' : 'jan', 'G' : 'feb', 'H' : 'mar', 'J' : 'apr', 'K' : 'may', 'M' : 'jun', 'N' : 'jul', 'Q' : 'aug', 'U' : 'sep', 'V' : 'oct', 'X' : 'nov', 'Z' : 'dec'}\n",
    "\n",
    "def decode_future_code(future_code, as_str = True):\n",
    "    '''\n",
    "    Given a future code such as \"X\", return either the month number (from 1 - 12) or the month abbreviation, such as \"nov\"\n",
    "    \n",
    "    Args:\n",
    "        future_code (str): the one letter future code\n",
    "        as_str (bool, optional): If set, we return the abbreviation, if not, we return the month number\n",
    "        \n",
    "    >>> decode_future_code('X', as_str = False)\n",
    "    11\n",
    "    >>> decode_future_code('X')\n",
    "    'nov'\n",
    "    '''\n",
    "    \n",
    "    if len(future_code) != 1: raise Exception(\"Future code must be a single character\")\n",
    "    if as_str:\n",
    "        if future_code not in FUTURE_CODES_STR: raise Exception(f'unknown future code: {future_code}')\n",
    "        return FUTURE_CODES_STR[future_code]\n",
    "    \n",
    "    if future_code not in FUTURE_CODES_INT: raise Exception(f'unknown future code: {future_code}')\n",
    "    return FUTURE_CODES_INT[future_code]\n",
    "\n",
    "def get_fut_code(month):\n",
    "    '''\n",
    "    Given a month number such as 3 for March, return the future code for it, e.g. H\n",
    "    >>> get_fut_code(3)\n",
    "    'H'\n",
    "    '''\n",
    "    return FUTURES_CODES_INVERTED[month]\n",
    "\n",
    "def get_temp_dir():\n",
    "    if os.access('/tmp', os.W_OK):\n",
    "        return '/tmp'\n",
    "    else:\n",
    "        return tempfile.gettempdir()\n",
    "    \n",
    "def linear_interpolate(a1, a2, x1, x2, x):\n",
    "    '''\n",
    "    >>> linear_interpolate(3, 4, 8, 10, 8.9)\n",
    "    array(3.4500)\n",
    "    '''\n",
    "    return np.where(x2 == x1, np.nan, a1 + (a2 - a1) * (x - x1) / (x2 - x1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "run_control": {
     "frozen": false,
     "read_only": false
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
