{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\nWarm start to choose regularisation strength\n============================================\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Setup\n-----\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "from matplotlib.colors import LogNorm\nimport matplotlib.pyplot as plt\nimport numpy as np\nfrom sklearn.linear_model import Ridge\nfrom sklearn.metrics import r2_score\nfrom sklearn.pipeline import Pipeline\n\nfrom group_lasso import GroupLasso\n\nnp.random.seed(0)\nGroupLasso.LOG_LOSSES = True"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Set dataset parameters\n----------------------\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "group_sizes = [np.random.randint(10, 20) for i in range(50)]\nactive_groups = [np.random.randint(2) for _ in group_sizes]\ngroups = np.concatenate(\n    [size * [i] for i, size in enumerate(group_sizes)]\n).reshape(-1, 1)\nnum_coeffs = sum(group_sizes)\nnum_datapoints = 10000\nnoise_std = 20"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Generate data matrix\n--------------------\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "X = np.random.standard_normal((num_datapoints, num_coeffs))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Generate coefficients\n---------------------\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "w = np.concatenate(\n    [\n        np.random.standard_normal(group_size) * is_active\n        for group_size, is_active in zip(group_sizes, active_groups)\n    ]\n)\nw = w.reshape(-1, 1)\ntrue_coefficient_mask = w != 0\nintercept = 2"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Generate regression targets\n---------------------------\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "y_true = X @ w + intercept\ny = y_true + np.random.randn(*y_true.shape) * noise_std"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Generate estimator and train it\n-------------------------------\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "num_regs = 10\nregularisations = np.logspace(-0.5, 1.5, num_regs)\nweights = np.empty((num_regs, w.shape[0],))\ngl = GroupLasso(\n    groups=groups,\n    group_reg=5,\n    l1_reg=0,\n    frobenius_lipschitz=True,\n    scale_reg=\"inverse_group_size\",\n    subsampling_scheme=1,\n    supress_warning=True,\n    n_iter=1000,\n    tol=1e-3,\n    warm_start=True,  # Warm start to \n)\n\nfor i, group_reg in enumerate(regularisations[::-1]):\n    gl.group_reg = group_reg\n    gl.fit(X, y)\n    weights[-(i+1)] = gl.sparsity_mask_.squeeze()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Visualise chosen covariate groups\n---------------------------------\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "plt.figure()\nplt.pcolormesh(np.arange(w.shape[0]), regularisations, -weights, cmap=\"gray\")\nplt.yscale(\"log\")\nplt.xlabel(\"Covariate number\")\nplt.ylabel(\"Regularisation strength\")\nplt.title(\"Active groups are black and inactive groups are white\")\nplt.show()"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}