{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is an example of how we see \n",
    "# the package work. The functions listed here\n",
    "# are probably the only ones that should be exposed, ie documented.\n",
    "# others should br prepended with a double underscore\n",
    "#  \n",
    "# The cognet directory has the following \"modules\"\n",
    "# which are seprate .py files containing clases and functions\n",
    "# The modules are cognet.py, dataFormatter.py, model.py, util.py, viz.py\n",
    "# we will write the viz.py later.\n",
    "import sys\n",
    "\n",
    "from quasinet.qnet import qdistance\n",
    "from cognet.cognet import cognet as cg\n",
    "from cognet.dataFormatter import dataFormatter\n",
    "from cognet.model import model \n",
    "#import cognet.util\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "yr = '2018'\n",
    "POLEFILE='examples_data/polar_vectors.csv'\n",
    "QPATH='examples_data/gss_'+yr+'.joblib'\n",
    "IMMUTABLE_FILE='examples_data/immutable.csv'\n",
    "GSSDATA = 'examples_data/gss_'+yr+'.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            wrkstat HRS1 HRS2 evwork        wrkslf  wrkgovt OCC10 PRESTG10  \\\n",
      "0  temp not working    e    c    NaN  someone else  private     b        c   \n",
      "1  working fulltime    c    e    NaN  someone else  private     b        d   \n",
      "\n",
      "  PRESTG105PLUS INDUS10  ...    neisafe rlooks rgroomed rweight rhlthend wtss  \\\n",
      "0             c       c  ...  very safe    NaN      NaN     NaN      NaN    e   \n",
      "1             d       c  ...  very safe    NaN      NaN     NaN      NaN    c   \n",
      "\n",
      "  wtssnr wtssall vstrat vpsu  \n",
      "0      e       e   3301    1  \n",
      "1      c       c   3301    1  \n",
      "\n",
      "[2 rows x 1034 columns]\n",
      "(892, 1034)\n"
     ]
    }
   ],
   "source": [
    "# testing dataFormatter\n",
    "data = dataFormatter(samples=GSSDATA)\n",
    "# load the sample data\n",
    "# have option for test/train split\n",
    "# make checks to ensure we will not throw errors at qnet construction \n",
    "print(data.samples[:2])\n",
    "features,samples = data.format_samples('train') # default trains and tests using half\n",
    "print(samples.shape)\n",
    "\n",
    "# format data for Qnet training and fitting\n",
    "data.Qnet_formatter()\n",
    "\n",
    "# set mutable and immutable vars either from list or file\n",
    "im_vars_df = pd.read_csv(IMMUTABLE_FILE, names=['vars'])\n",
    "im_vars_list = im_vars_df.vars.to_list()\n",
    "mutable_vars, immutable_vars = data.mutable_variables(immutable_list=im_vars_list)\n",
    "mutable_vars, immutable_vars = data.mutable_variables(IMMUTABLE_FILE=IMMUTABLE_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# testing model functionality\n",
    "# can either input features and samples directly, or infer from data obj\n",
    "model_ = model()\n",
    "\n",
    "# qnet construction parameters, \n",
    "# choose to either load or fit qnet from scratch\n",
    "test_model_buildqnet = False\n",
    "if test_model_buildqnet:\n",
    "        print(\"fitting\")\n",
    "        model_.fit(data_obj=data,\n",
    "                   min_samples_split=2,\n",
    "                   alpha=0.05,\n",
    "                   max_depth=-1,\n",
    "                   max_feats=-1,\n",
    "                   early_stopping=False,\n",
    "                   verbose=0,\n",
    "                   random_state=None,\n",
    "                   njobs=8)\n",
    "        print(\"fitted\")\n",
    "        model_.export_dot(\"tmp_dot_modelclass.dot\",\n",
    "                        generate_trees=True)\n",
    "        model_.save(\"tmp_nodelclass.joblib\")\n",
    "        #model_.load(\"tmp_nodelclass.joblib\")\n",
    "else:\n",
    "    model_.load(\"examples_data/gss_2018.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1784, 1034)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "featurenames, samples=data.Qnet_formatter()\n",
    "samples.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1034\n",
      "0 pole features not found in sample features\n"
     ]
    }
   ],
   "source": [
    "# testing cognet\n",
    "# set some paramaters in instantiating cognet class \n",
    "# if loading from model obj, no need to use load_data func, otherwise, load_data\n",
    "Cg = cognet()\n",
    "print(len(model_.features))\n",
    "Cg.load_from_model(model_, data, 'all')\n",
    "# produce stats on how many column names actually match\n",
    "stats = Cg.set_poles(POLEFILE,\"R\",\"L\",steps=120, VERBOSE=True)\n",
    "\n",
    "# compute polar distance matrix\n",
    "dmatrix = Cg.polar_separation(nsteps=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'numpy.ndarray' object is not callable",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-27-d2954984ef10>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mreturndict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mrederr\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mr_prob\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrand_err\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mqsample\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrandom_sample\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmask_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandomMaskReconstruction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturndict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m# sample=np.array(samples[1]))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"reconstruction results:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrederr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mr_prob\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrand_err\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_sample\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-24-bcf167f7ba7b>\u001b[0m in \u001b[0;36mrandomMaskReconstruction\u001b[0;34m(self, index, return_dict, sample, feature_names, output_dir, file_name)\u001b[0m\n\u001b[1;32m    879\u001b[0m             \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmakedirs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_dir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    880\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 881\u001b[0;31m         \u001b[0mqs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mqsample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mqnet\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    882\u001b[0m         \u001b[0mdqestim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mqdistance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mqs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mqnet\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mqnet\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    883\u001b[0m         \u001b[0mdactual\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mqdistance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0ms1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mqnet\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mqnet\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: 'numpy.ndarray' object is not callable"
     ]
    }
   ],
   "source": [
    "returndict = {}\n",
    "rederr,r_prob,rand_err,sample,qsample,random_sample,mask_ = Cg.randomMaskReconstruction(index=2, returndict=returndict)# sample=np.array(samples[1]))\n",
    "print(\"reconstruction results:\", rederr, r_prob, rand_err,sample, random_sample, mask_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "from quasinet.qnet import Qnet, qdistance, load_qnet, qdistance_matrix\n",
    "from quasinet.qsampling import qsample, targeted_qsample\n",
    "#from mpi4py.futures import MPIPoolExecutor\n",
    "import sys\n",
    "import subprocess\n",
    "from scipy.stats import entropy\n",
    "import multiprocessing as mp\n",
    "import time\n",
    "from cognet.util import embed_to_pca\n",
    "import pkgutil\n",
    "import os\n",
    "\n",
    "class cognet:\n",
    "    \"\"\"Aggregate related Qnet functions\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        \"\"\"Init\n",
    "        \"\"\"\n",
    "        self.year = None\n",
    "        self.n_jobs = 28\n",
    "        self.qnet = None\n",
    "        self.steps = 120\n",
    "        self.num_qsamples = None\n",
    "        self.all_samples = None\n",
    "        self.samples = None\n",
    "        self.samples_as_strings = None\n",
    "        self.features = None\n",
    "        self.cols = None\n",
    "        self.immutable_vars = None\n",
    "        self.mutable_vars = None\n",
    "        self.poles = None\n",
    "        self.polar_features = None\n",
    "        self.polar_indices = None\n",
    "        self.poles_dict = {}\n",
    "        self.d0 = None\n",
    "        self.qdistance_matrix_file = None\n",
    "        self.dissonance_file = None\n",
    "        self.s_null = None\n",
    "        self.D_null = None\n",
    "        self.mask_prob = 0.5\n",
    "        self.variation_weight = None\n",
    "        self.polar_matrix = None\n",
    "        self.nsamples = None\n",
    "        self.restricted = False\n",
    "    \n",
    "    def load_from_model(self,\n",
    "                        model,\n",
    "                        data_obj,\n",
    "                        key,\n",
    "                        im_vars=None,\n",
    "                        m_vars=None):\n",
    "        \"\"\"load parameters from model object\n",
    "\n",
    "        Args:\n",
    "          model (Class): model obj for loading parameters\n",
    "          data_obj (class): instance of dataformatter class\n",
    "          key (str): 'all', 'train', or 'test', corresponding to sample type\n",
    "          im_vars (list[str], optional): Not implemented yet. Defaults to None.\n",
    "          m_vars (list[str], optional): Not implemented yet. Defaults to None.\n",
    "        \"\"\"\n",
    "        if model is not None:\n",
    "            self.qnet = model.myQnet\n",
    "            # self.cols = np.array(model.features)\n",
    "            featurenames, samples = data_obj.format_samples(key)\n",
    "            samples = pd.DataFrame(samples)\n",
    "            self.cols = np.array(featurenames)\n",
    "            self.features = pd.DataFrame(columns=np.array(featurenames))\n",
    "            if any(x is not None for x in [model.immutable_vars, model.mutable_vars]):\n",
    "                if model.immutable_vars is not None:\n",
    "                    self.immutable_vars = model.immutable_vars\n",
    "                    self.mutable_vars = [x for x in self.features if x not in self.immutable_vars]\n",
    "                elif model.mutable_vars is not None:\n",
    "                    self.mutable_vars = model.mutable_vars\n",
    "                    self.immutable_vars = [x for x in self.features if x not in self.mutable_vars]\n",
    "            else:\n",
    "                self.mutable_vars = self.features\n",
    "            \n",
    "            self.samples = pd.DataFrame(samples).replace(\"nan\",\"\").fillna(\"\")\n",
    "            self.samples.columns = np.array(featurenames)\n",
    "            self.all_samples = self.samples\n",
    "            self.samples_as_strings = self.samples.fillna('').values.astype(str)[:]\n",
    "            self.s_null=['']*len(self.samples_as_strings[0])\n",
    "            self.D_null=self.qnet.predict_distributions(self.s_null)\n",
    "            variation_weight = []\n",
    "            for d in self.D_null:\n",
    "                v=[]\n",
    "                for val in d.values():\n",
    "                    v=np.append(v,val)\n",
    "                variation_weight.append(entropy(v,base=len(v)))\n",
    "            variation_weight = np.nan_to_num(variation_weight) # remove nans\n",
    "            self.variation_weight = variation_weight\n",
    "    \n",
    "    def load_from_dataformatter(self, \n",
    "                                data_obj,\n",
    "                                key):\n",
    "        \"\"\"read in either train or test data, specified by key, from data obj\n",
    "\n",
    "        Args:\n",
    "          data_obj (class): instance of dataformatter class\n",
    "          key (str): 'all', 'train', or 'test', corresponding to sample type\n",
    "          \n",
    "        Returns:\n",
    "          featurenames, samples: formatted arrays\n",
    "        \"\"\"\n",
    "        featurenames, samples = data_obj.format_samples(key)\n",
    "        if any(x is not None for x in [self.features, self.samples]):\n",
    "            print(\"replacing original features/samples with dataformatter data\")\n",
    "        self.cols = featurenames\n",
    "        self.features = pd.DataFrame(columns=self.cols)\n",
    "        self.samples = pd.DataFrame(samples,columns=self.features)\n",
    "        self.all_samples = self.samples\n",
    "        self.samples_as_strings = self.samples[self.cols].fillna('').values.astype(str)[:]\n",
    "        self.s_null=['']*len(self.samples_as_strings[0])\n",
    "        return featurenames, samples\n",
    "\n",
    "    def load_data(self,\n",
    "                  year,\n",
    "                  features_by_year,\n",
    "                  samples,\n",
    "                  qnet):\n",
    "        '''load cols, features, samples, and qnet.\n",
    "\n",
    "        Args:\n",
    "          year (str): to identify cols/features.\n",
    "          features_by_year (str): file containing all features by year of the dataset.\n",
    "          samples (str): file of samples for that year.\n",
    "          Qnet (str): Qnet file location.\n",
    "        '''\n",
    "        self.qnet = load_qnet(qnet)\n",
    "        self.year = year\n",
    "        self.cols = np.array((pd.read_csv(features_by_year,\n",
    "                            keep_default_na=True, \n",
    "                            index_col=0).set_index(\n",
    "                                'year')).loc[int(year)].apply(\n",
    "                                    eval).values[0])\n",
    "        self.features = pd.DataFrame(columns=self.cols)\n",
    "        self.mutable_vars = [x for x in self.cols]\n",
    "        #[self.cols].fillna('').values.astype(str)[:]\n",
    "\n",
    "        self.samples=pd.read_csv(samples)\n",
    "        self.samples = pd.concat([self.samples,self.features], axis=0)\n",
    "        self.all_samples = self.samples\n",
    "        self.samples_as_strings = self.samples[self.cols].fillna('').values.astype(str)[:]\n",
    "        self.s_null=['']*len(self.samples_as_strings[0])\n",
    "        self.D_null=self.qnet.predict_distributions(self.s_null)\n",
    "        variation_weight = []\n",
    "        for d in self.D_null:\n",
    "            v=[]\n",
    "            for val in d.values():\n",
    "                v=np.append(v,val)\n",
    "            variation_weight.append(entropy(v,base=len(v)))\n",
    "        self.variation_weight = variation_weight\n",
    "\n",
    "    def set_immutable_vars(self,\n",
    "                        IMMUTABLE_FILE):\n",
    "        '''set vars to immutable and mutable, \n",
    "        can prob combine this with the load_data func: only set the immutable vars if necessary\n",
    "\n",
    "        Args:\n",
    "          IMMUTABLE_FILE (str): file containing the immutable features/vars\n",
    "        '''\n",
    "        if self.cols is None:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "        self.immutable_vars = pd.read_csv(IMMUTABLE_FILE,index_col=0).transpose()\n",
    "        self.mutable_vars = None\n",
    "        self.mutable_vars = [x for x in self.cols\n",
    "                            if x.upper() not in self.immutable_vars.columns]\n",
    "    \n",
    "    def set_nsamples(self,\n",
    "                    num_samples,\n",
    "                    random=False):\n",
    "        '''select a subset of the samples\n",
    "\n",
    "        Args:\n",
    "          num_samples (int): Set num of samples to subset, default to None, resets to all samples\n",
    "          random (bool): take random sample if true, ordered sample if false\n",
    "        '''\n",
    "        self.samples = self.all_samples\n",
    "        if num_samples is not None:\n",
    "            if all(x is not None for x in [num_samples, self.samples]):\n",
    "                if num_samples > len(self.samples.index):\n",
    "                    string = 'The number of selected samples ({}) ' + \\\n",
    "                        'is greater than the number of samples ({})!'\n",
    "                    string = string.format(num_samples, len(self.samples.index))\n",
    "                    raise ValueError(string)\n",
    "\n",
    "                if num_samples == len(self.samples.index):\n",
    "                    string = 'The number of selected samples ({}) ' + \\\n",
    "                        'is equal to the number of samples ({})!'\n",
    "                    string = string.format(num_samples, len(self.samples.index))\n",
    "                    print(string)\n",
    "                if random:\n",
    "                    self.samples = self.samples.sample(num_samples)\n",
    "                else:\n",
    "                    self.samples = self.samples.iloc[:num_samples]\n",
    "                self.nsamples = num_samples\n",
    "                self.samples_as_strings = self.samples[self.cols].fillna('').values.astype(str)[:]\n",
    "                \n",
    "            elif self.samples is None:\n",
    "                raise ValueError(\"load_data first!\")\n",
    "\n",
    "    def __variation_weight(self,\n",
    "                        index):\n",
    "        \"\"\"\n",
    "        \"\"\"\n",
    "        d_=self.D_null[index]\n",
    "        v=[]\n",
    "        for val in d_.values():\n",
    "            v=np.append(v,val)\n",
    "        return entropy(v,base=len(v))\n",
    "    \n",
    "    def getBaseFrequency(self, \n",
    "                        sample):\n",
    "        '''get frequency of the variables\n",
    "        helper func for qsampling\n",
    "\n",
    "        Args:\n",
    "          sample (list[str]): vector of sample, must have the same num of features as the qnet\n",
    "        '''\n",
    "        MUTABLE=pd.DataFrame(np.zeros(len(self.cols)),index=self.cols).transpose()\n",
    "                \n",
    "        for m in self.mutable_vars:\n",
    "            MUTABLE[m]=1.0\n",
    "        mutable_x=MUTABLE.values[0]\n",
    "        base_frequency=mutable_x/mutable_x.sum()\n",
    "\n",
    "        # commented out for now for testing using smaller qnet\n",
    "        for i in range(len(base_frequency)):\n",
    "            if base_frequency[i]>0.0:\n",
    "                base_frequency[i]= self.variation_weight[i]*base_frequency[i]\n",
    "\n",
    "        return base_frequency/base_frequency.sum()\n",
    "    \n",
    "    def qsampling(self,\n",
    "                sample,\n",
    "                steps,\n",
    "                immutable=False):\n",
    "        '''perturb the sample based on thet qnet distributions and number of steps\n",
    "\n",
    "        Args:\n",
    "          sample (1d array-like): sample vector, must have the same num of features as the qnet\n",
    "          steps (int): number of steps to qsample\n",
    "          immutable (bool): are there variables that are immutable?\n",
    "        '''\n",
    "        if all(x is not None for x in [self.mutable_vars, sample]):\n",
    "            if immutable == True:\n",
    "                return qsample(sample,self.qnet,steps,self.getBaseFrequency(self.samples))\n",
    "            else:\n",
    "                return qsample(sample,self.qnet,steps)\n",
    "        elif self.mutable_vars is None:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "\n",
    "    def set_poles(self,\n",
    "                  POLEFILE,\n",
    "                  pole_1,\n",
    "                  pole_2,\n",
    "                  steps=0,\n",
    "                  mutable=False,\n",
    "                  VERBOSE=False,\n",
    "                  restrict=True,\n",
    "                  nsamples = False,\n",
    "                  random=False):\n",
    "        '''set the poles and samples such that the samples contain features in poles\n",
    "\n",
    "        Args:\n",
    "          steps (int): number of steps to qsample\n",
    "          POLEFILE (str): file containing poles samples and features\n",
    "          pole_1 (str): column name for first pole to use\n",
    "          pole_2 (str): column name for second pole to use\n",
    "          mutable (bool): Whether or not to set poles as the only mutable_vars\n",
    "          VERBOSE (bool): boolean flag prints number of pole features not found in sample features if True\n",
    "          restrict (bool): boolean flag restricts the sample features to polar features if True\n",
    "          random (bool): boolean flag takes random sample of all_samples\n",
    "        '''\n",
    "        invalid_count = 0\n",
    "        if all(x is not None for x in [self.samples, self.qnet]):\n",
    "            poles = pd.read_csv(POLEFILE, index_col=0)\n",
    "            self.poles=poles.transpose()\n",
    "            self.polar_features = pd.concat([self.features, self.poles], axis=0).fillna('')\n",
    "            poles_dict = {}\n",
    "            for column in poles:\n",
    "                p_ = self.polar_features.loc[column][self.cols].fillna('').values.astype(str)[:]\n",
    "                poles_dict[column] = self.qsampling(p_,steps)\n",
    "            self.poles_dict = poles_dict\n",
    "            self.pL = self.poles_dict[pole_1]\n",
    "            self.pR = self.poles_dict[pole_2]\n",
    "            # self.pL = list(poles_dict.values())[0]\n",
    "            # self.pR = list(poles_dict.values())[1]\n",
    "            self.d0 = qdistance(self.pL, self.pR, self.qnet, self.qnet)\n",
    "            \n",
    "            if restrict:\n",
    "                # restrict sample columns to polar columns\n",
    "                cols = [x for x in self.poles.columns if x in self.samples.columns]\n",
    "                self.samples=self.samples[cols]\n",
    "                self.restricted = True\n",
    "                \n",
    "            elif self.restricted:\n",
    "                # if poles had been restricted before, unrestrict it\n",
    "                self.restricted = False\n",
    "                self.samples = self.all_samples\n",
    "                if self.nsamples is not None:\n",
    "                    self.set_nsamples(nsamples, random)\n",
    "\n",
    "            # for x in self.poles.columns:\n",
    "            #     if x not in self.samples.columns:\n",
    "            #         invalid_count += 1\n",
    "            #         self.samples[x]=''\n",
    "\n",
    "            self.samples = pd.concat([self.features,self.samples], axis=0).fillna('')\n",
    "            #self.all_samples = self.samples\n",
    "            self.samples_as_strings = self.samples[self.cols].fillna('').values.astype(str)[:]\n",
    "            \n",
    "            if mutable:\n",
    "                self.mutable_vars=[x for x in self.cols if x in self.poles.columns]\n",
    "        elif self.samples is None:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "\n",
    "        if VERBOSE:\n",
    "            print(\"{} pole features not found in sample features\".format(invalid_count))\n",
    "\n",
    "    def distance(self,\n",
    "                sample1,\n",
    "                sample2,\n",
    "                nsteps1=0,\n",
    "                nsteps2=0):\n",
    "        \"\"\"qsamples each sample set num of steps, then takes qdistance\n",
    "\n",
    "        Args:\n",
    "          sample1 (list[str]): sample vector 1, must have the same num of features as the qnet\n",
    "          sample2 (list[str]): sample vector 2, must have the same num of features as the qnet\n",
    "          nsteps1 (int, optional): number of steps to qsample for sample1\n",
    "          nsteps2 (int, optional): number of steps to qsample for sample2\n",
    "\n",
    "        Returns:\n",
    "          qdistance: float, distance between two samples\n",
    "        \"\"\"\n",
    "        if self.qnet is None:\n",
    "            raise ValueError(\"load qnet first!\")\n",
    "        bp1 = self.getBaseFrequency(sample1)\n",
    "        bp2 = self.getBaseFrequency(sample2)\n",
    "        sample1 = qsample(sample1, self.qnet, nsteps1)#, baseline_prob=bp1)\n",
    "        sample2 = qsample(sample2, self.qnet, nsteps2)#, baseline_prob=bp2)\n",
    "        return qdistance(sample1, sample2, self.qnet, self.qnet)\n",
    "    \n",
    "    def __distfunc(self, \n",
    "                x, \n",
    "                y):\n",
    "        '''Compute distance between two samples\n",
    "\n",
    "        Args:\n",
    "          x (list[str]): first sample\n",
    "          y (list[str]): second sample\n",
    "          \n",
    "        Returns:\n",
    "         d: qdistance\n",
    "        '''\n",
    "        d=qdistance(x,y,self.qnet,self.qnet)\n",
    "        return d\n",
    "    \n",
    "    def polarDistance(self,\n",
    "                    i,\n",
    "                    return_dict=None):\n",
    "        \"\"\"return the distance from a sample to the poles\n",
    "\n",
    "        Args:\n",
    "          i (int): index of sample to take\n",
    "          return_dict (dict): dict used for multiple sample function\n",
    "\n",
    "        Returns:\n",
    "          distances: float, distance from sample to each pole\n",
    "        \"\"\"\n",
    "        samples_as_strings = self.samples[self.cols].fillna('').values.astype(str)[:]\n",
    "        p = samples_as_strings[i]\n",
    "        distances = []\n",
    "        for index, row in self.polar_features[self.cols].iterrows():\n",
    "            row = row.fillna('').values.astype(str)[:]\n",
    "            distances.append(self.distance(p, np.array(row)))\n",
    "        if return_dict is not None:\n",
    "            return_dict[i] = distances\n",
    "        return distances\n",
    "            \n",
    "    def polarDistance_multiple(self,\n",
    "                            outfile):\n",
    "        \"\"\"return the distance from all samples to the poles\n",
    "\n",
    "        Args:\n",
    "          outfile (str): desired output filename and path\n",
    "          \n",
    "        Returns:\n",
    "          return_dict: dictionary containing multiprocessing results\n",
    "        \"\"\"\n",
    "        if all(x is not None for x in [self.samples, self.cols,\n",
    "                                    self.polar_features]):\n",
    "            manager = mp.Manager()\n",
    "            return_dict = manager.dict()\n",
    "            processes = []\n",
    "            \n",
    "            for i in range(len(self.samples)):\n",
    "                p = mp.Process(target=self.polarDistance, args=(i, return_dict))\n",
    "                processes.append(p)\n",
    "\n",
    "            [x.start() for x in processes]\n",
    "            [x.join() for x in processes]\n",
    "\n",
    "            pole_names = []\n",
    "            for index, row in self.polar_features[self.cols].iterrows():\n",
    "                pole_names.append(index)\n",
    "            result=[x for x in return_dict.values()]\n",
    "            result=pd.DataFrame(result,columns=pole_names).to_csv(outfile)\n",
    "            \n",
    "        else:\n",
    "            raise ValueError(\"load data first!\")\n",
    "        return return_dict\n",
    "        \n",
    "    def distfunc_line(self,\n",
    "                    i,\n",
    "                    return_dict=None):\n",
    "        '''compute the dist for a row, or vector of samples\n",
    "\n",
    "        Args:\n",
    "          i (int): row\n",
    "        \n",
    "        Return:\n",
    "          line: float, numpy.ndarray\n",
    "        '''\n",
    "        if all(x is not None for x in [self.samples, self.features]):\n",
    "            w = self.samples.index.size\n",
    "            line = np.zeros(w)\n",
    "            y = self.samples_as_strings[i]\n",
    "            for j in range(w):\n",
    "                # only compute half of the distance matrix\n",
    "                if j > i:\n",
    "                    x = self.samples_as_strings[j]\n",
    "                    line[j] = self.__distfunc(x, y)\n",
    "        else:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "        if return_dict is not None:\n",
    "            return_dict[i] = line\n",
    "        return line\n",
    "    \n",
    "    def distfunc_multiples(self,\n",
    "                        outfile):\n",
    "        \"\"\"compute distance matrix for all samples in the dataset\n",
    "\n",
    "        Args:\n",
    "          outfile (str): desired output filename and path\n",
    "          \n",
    "        Returns:\n",
    "          return_dict: dictionary containing multiprocessing results\n",
    "        \"\"\"\n",
    "        if all(x is not None for x in [self.samples, self.features]):\n",
    "            manager = mp.Manager()\n",
    "            return_dict = manager.dict()\n",
    "            processes = []\n",
    "\n",
    "            for i in range(len(self.samples)):\n",
    "                p = mp.Process(target=self.distfunc_line, args=(i, return_dict))\n",
    "                processes.append(p)\n",
    "            \n",
    "            [x.start() for x in processes]\n",
    "            [x.join() for x in processes]\n",
    "            result=[x for x in return_dict.values()]\n",
    "            columns = [i for i in range(len(self.samples))]\n",
    "            result=pd.DataFrame(result,columns=columns, index=columns).sort_index(ascending=False)\n",
    "            result = result.to_numpy()\n",
    "            result = pd.DataFrame(np.maximum(result, result.transpose()))\n",
    "            result.to_csv(outfile)\n",
    "        else:\n",
    "            raise ValueError(\"load data first!\")\n",
    "        \n",
    "        return result\n",
    "    \n",
    "    def polar_separation(self,\n",
    "                        nsteps=0):\n",
    "        \"\"\"calculates the distance between poles as a qdistance matrix\n",
    "\n",
    "        Args:\n",
    "          nsteps (int, optional): [description]. Defaults to 0.\n",
    "          \n",
    "        Returns:\n",
    "          self.polar_matrix: dictionary containing multiprocessing results\n",
    "        \"\"\"\n",
    "        polar_arraydata = self.polar_features[self.cols].values.astype(str)[:]\n",
    "        samples_ = []\n",
    "        for vector in polar_arraydata:\n",
    "            bp = self.getBaseFrequency(vector)\n",
    "            sample = qsample(vector, self.qnet, nsteps, baseline_prob=bp)\n",
    "            samples_.append(sample)\n",
    "        samples_ = np.array(samples_)\n",
    "        self.polar_matrix = qdistance_matrix(samples_, samples_, self.qnet, self.qnet)\n",
    "        return self.polar_matrix\n",
    "        \n",
    "    def embed(self,\n",
    "            infile,\n",
    "            name_pref,\n",
    "            out_dir,\n",
    "            pca_model=False,\n",
    "            EMBED_BINARY=None):\n",
    "        '''\n",
    "        embed data\n",
    "\n",
    "        Args:\n",
    "          infile (str): input file to be embedded\n",
    "          name_pref (str): preferred name for output file\n",
    "          out_dir (str): output dir for results\n",
    "          pca_model (bool): whether or not to generate PCA model\n",
    "          EMBED_BINARY (os.path.abspath): path to embed binary\n",
    "        '''\n",
    "        if all(x is not None for x in [self.year]):\n",
    "            yr = self.year\n",
    "            PREF = name_pref\n",
    "            FILE = infile\n",
    "\n",
    "            if EMBED_BINARY is None:\n",
    "                EMBED = pkgutil.get_data(\"cognet.bin\", \"__embed__.so\") \n",
    "            else:\n",
    "                EMBED = EMBED_BINARY\n",
    "            DATAFILE = out_dir + 'data_' +yr\n",
    "            EFILE = out_dir + PREF + '_E_' +yr\n",
    "            DFILE = out_dir + PREF + '_D_' +yr\n",
    "\n",
    "            pd.read_csv(FILE, header=None).to_csv(DATAFILE,sep=' ',header=None,index=None)\n",
    "            STR=EMBED+' -f '+DATAFILE+' -E '+EFILE+' -D '+DFILE\n",
    "            subprocess.call(STR,shell=True)\n",
    "            if pca_model:\n",
    "                embed_to_pca(EFILE, EFILE+'_PCA')\n",
    "        elif self.year is None:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "    \n",
    "    def __calc_d0(self,\n",
    "                pole_1,\n",
    "                pole_2):\n",
    "        \"\"\"calculate distance between two poles\n",
    "\n",
    "        Args:\n",
    "          pole_1 (list[str]): a polar vector, must have same number of features as qnet\n",
    "          pole_2 (list[str]): a polar vector, must have same number of features as qnet\n",
    "        \"\"\"\n",
    "        self.pL = self.poles_dict[pole_1]\n",
    "        self.pR = self.poles_dict[pole_2]\n",
    "        self.d0 = qdistance(self.pL, self.pR, self.qnet, self.qnet)\n",
    "        \n",
    "    def ideology(self,\n",
    "                i,\n",
    "                return_dict=None,\n",
    "                pole_1=None,\n",
    "                pole_2=None):\n",
    "        \"\"\"return ideology index (left-leaning or right-leaning) for a singular sample\n",
    "\n",
    "        Args:\n",
    "          i (int): index of sample\n",
    "          pole_1 (int): index of Pole One to calc as base distance. Defaults to 0.\n",
    "          pole_2 (int): index of Pole Two to calc as base distance. Defaults to 1.\n",
    "          return_dict (dict, optional): dict containing results\n",
    "          \n",
    "        Returns:\n",
    "          [ideology_index, dR, dL, self.d0]: which way the sample leans,\n",
    "                                             distance from the right pole,\n",
    "                                             distance from the left pole,\n",
    "                                             and distance between poles, respectively\n",
    "        \"\"\"\n",
    "        if pole_1 is not None or pole_2 is not None:\n",
    "            self.__calc_d0(pole_1, pole_2)\n",
    "            \n",
    "        p = self.samples_as_strings[i]\n",
    "        dR = qdistance(self.pR, p, self.qnet, self.qnet)\n",
    "        dL = qdistance(self.pL, p, self.qnet, self.qnet)\n",
    "        ideology_index = (dR-dL)/self.d0\n",
    "        if return_dict is not None:\n",
    "            return_dict[i] = [ideology_index, dR, dL, self.d0]\n",
    "        return [ideology_index, dR, dL, self.d0]\n",
    "\n",
    "    def dispersion(self,\n",
    "                i,\n",
    "                return_dict=None):\n",
    "        \"\"\"qsamples a sample n times and takes distance matrix \n",
    "        to determine max and std of distances between qsamples\n",
    "\n",
    "        Args:\n",
    "          i (int): index of sample\n",
    "          return_dict (dict): dict containing results\n",
    "\n",
    "        Returns:\n",
    "          list[float]: std and max of the distances btwn qsamples\n",
    "        \"\"\"\n",
    "        p = self.samples_as_strings[i]\n",
    "        Qset = [qsample(p, self.qnet, self.steps) for j in np.arange(self.num_qsamples)]\n",
    "        Qset = np.array(Qset)\n",
    "\n",
    "        matrix = (qdistance_matrix(Qset, Qset, self.qnet, self.qnet))\n",
    "        Q = matrix.max()\n",
    "        Qsd = matrix.std()\n",
    "        if return_dict is not None:\n",
    "            return_dict[i] = [Qsd, Q]\n",
    "        return [Qsd, Q]\n",
    "    \n",
    "    def compute_DLI_samples(self,\n",
    "                        type,\n",
    "                        outfile,\n",
    "                        num_qsamples=40,\n",
    "                        steps=120,\n",
    "                        n_jobs=28,\n",
    "                        pole_1=0,\n",
    "                        pole_2=1):\n",
    "        \"\"\"compute and save ideology index or dispersion for all samples\n",
    "\n",
    "        Args:\n",
    "          num_qsamples (int): number of qsamples to compute\n",
    "          outfile (str): output file for results\n",
    "          type (str): whether to calc dispersion or ideology\n",
    "          steps (int): number of steps to qsample\n",
    "          n_jobs (int, optional): sets the number of jobs for parallelization. Defaults to 28.\n",
    "          pole_1 (int, optional): index of Pole One to calc as base distance. Defaults to 0.\n",
    "          pole_2 (int, optional): index of Pole Two to calc as base distance. Defaults to 1.\n",
    "\n",
    "        Raises:\n",
    "          ValueError: set poles if poles are not set\n",
    "          ValueError: load data if samples or features are not present\n",
    "            \n",
    "        Returns:\n",
    "          Result: dictionary containing multiprocessing results\n",
    "        \"\"\"\n",
    "        if all(x is not None for x in [self.samples, self.features,\n",
    "                                    self.pL, self.pR]):\n",
    "            self.num_qsamples = num_qsamples\n",
    "            self.steps = steps\n",
    "            if pole_1 != 0 or pole_2 != 1:\n",
    "                self.__calc_d0(pole_1, pole_2)\n",
    "            \n",
    "            # testing\n",
    "            # pd.DataFrame(self.samples_as_strings).to_csv('examples_results/class_allsamples_2018.csv')\n",
    "            \n",
    "            manager = mp.Manager()\n",
    "            return_dict = manager.dict()\n",
    "            processes = []\n",
    "\n",
    "            if type == 'ideology':\n",
    "                for i in range(len(self.samples)):\n",
    "                    p = mp.Process(target=self.ideology, args=(i, return_dict))\n",
    "                    processes.append(p)\n",
    "                columns=['ideology', 'dR', 'dL', 'd0']\n",
    "            elif type == 'dispersion':\n",
    "                for i in range(len(self.samples)):\n",
    "                    p = mp.Process(target=self.dispersion, args=(i, return_dict))\n",
    "                    processes.append(p)\n",
    "                columns=['Qsd', 'Qmax']\n",
    "            else:\n",
    "                raise ValueError(\"Type must be either dispersion or ideology!\")\n",
    "            \n",
    "            [x.start() for x in processes]\n",
    "            [x.join() for x in processes]\n",
    "            result=[x for x in return_dict.values()]\n",
    "            result=pd.DataFrame(result,columns=columns).to_csv(outfile)\n",
    "\n",
    "        elif self.pL is None or self.pR is None:\n",
    "            raise ValueError(\"set_poles first!\")\n",
    "        else:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "        return pd.DataFrame(return_dict.copy())\n",
    "\n",
    "    def compute_polar_indices(self,\n",
    "                            num_samples = None,\n",
    "                            polar_comp = False,\n",
    "                            POLEFILE = None,\n",
    "                            steps = 5):\n",
    "        '''set up polar indices for dissonance func\n",
    "\n",
    "        Args:\n",
    "          num_samples (int): subset of samples to take\n",
    "          polar_comp (bool): whether or not to set poles\n",
    "          POLEFILE (None): file containing pole samples and features\n",
    "          steps (int): number of steps to qsample\n",
    "        '''\n",
    "        if all(x is not None for x in [self.samples, self.features, self.poles]):\n",
    "            if num_samples is not None:\n",
    "                self.set_nsamples(num_samples)\n",
    "\n",
    "            # read sample data\n",
    "            if polar_comp:\n",
    "                self.set_poles(self.qnet, steps, POLEFILE)\n",
    "            \n",
    "            polar_features = pd.concat([self.features, self.poles], axis=0)\n",
    "            self.polar_indices=np.where(polar_features[self.cols].fillna('XXXX').values[0]!='XXXX')[0]\n",
    "        \n",
    "        elif self.poles is None:\n",
    "            raise ValueError(\"set_poles first!\")\n",
    "        else:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "\n",
    "    def dissonance(self,\n",
    "                sample_index,\n",
    "                return_dict=None,\n",
    "                MISSING_VAL=0.0):\n",
    "        '''compute dissonance for each sample_index, helper function for all_dissonance\n",
    "        \n",
    "        Args:\n",
    "          sample_index (int): index of the sample to compute dissonance\n",
    "          return_dict (dict): dict containing results\n",
    "          MISSING_VAL (float): default dissonance value\n",
    "          \n",
    "        Returns: \n",
    "          diss: ndarray containing dissonance for sample\n",
    "        '''\n",
    "        if all(x is not None for x in [self.samples, self.features]):\n",
    "            s = self.samples_as_strings[sample_index]\n",
    "            if self.polar_indices is None:\n",
    "                self.polar_indices = range(len(s))\n",
    "\n",
    "            Ds=self.qnet.predict_distributions(s)\n",
    "            \n",
    "            diss=np.ones(len(Ds))*MISSING_VAL\n",
    "            for i in self.polar_indices:\n",
    "                if s[i] != '':\n",
    "                    if s[i] in Ds[i].keys():\n",
    "                        diss[i]=1-Ds[i][s[i]]/np.max(\n",
    "                            list(Ds[i].values())) \n",
    "                    else:\n",
    "                        diss[i]=1.0\n",
    "            if return_dict is not None:\n",
    "                return_dict[sample_index] = diss[self.polar_indices]\n",
    "            return diss[self.polar_indices]\n",
    "        else:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "    \n",
    "    def dissonance_matrix(self,\n",
    "                        output_file='/example_results/DISSONANCE_matrix.csv',\n",
    "                        n_jobs=28):\n",
    "        '''get the dissonance for all samples\n",
    "\n",
    "        Args:\n",
    "          output_file (str): directory and/or file for output\n",
    "          n_jobs (int): number of jobs for pdqm\n",
    "\n",
    "        Returns:\n",
    "          pandas.DataFrame\n",
    "        '''\n",
    "        manager = mp.Manager()\n",
    "        return_dict = manager.dict()\n",
    "        processes = []\n",
    "        \n",
    "        for i in range(len(self.samples)):\n",
    "            p = mp.Process(target=self.dissonance, args=(i, return_dict))\n",
    "            processes.append(p)\n",
    "\n",
    "        [x.start() for x in processes]\n",
    "        [x.join() for x in processes]\n",
    "\n",
    "        result=[x for x in return_dict.values()]\n",
    "        if self.polar_indices is not None:\n",
    "            polar_features = pd.concat([self.features, self.poles], axis=0)\n",
    "            cols = polar_features[self.cols].dropna(axis=1).columns\n",
    "        else:\n",
    "            cols = self.cols\n",
    "        result=pd.DataFrame(result,columns=cols).to_csv(output_file)\n",
    "        \n",
    "        self.dissonance_file = output_file\n",
    "        return pd.DataFrame(return_dict.copy())\n",
    "    \n",
    "    def __choose_one(self,\n",
    "                X):\n",
    "        '''returns a random element of X\n",
    "\n",
    "        Args:\n",
    "          X (1D array-like): vector from which random element is to be chosen\n",
    "        \n",
    "        Returns:\n",
    "          X: random element of sample\n",
    "          None: if X has len 0\n",
    "        '''\n",
    "        X=list(X)\n",
    "        if len(X)>0:\n",
    "            return X[np.random.randint(len(X))]\n",
    "        return None\n",
    "\n",
    "    def getMaskedSample(self,\n",
    "                        s,\n",
    "                        mask_prob=0.5,\n",
    "                        allow_all_mutable=False):\n",
    "        '''inputs a sample and randomly mask elements of the sample\n",
    "\n",
    "        Args:\n",
    "          s (list[str]): vector of sample, must have the same num of features as the qnet\n",
    "          mask_prob (float): float btwn 0 and 1, prob to mask element of sample\n",
    "          allow_all_mutable (bool): whether or not all variables are mutable\n",
    "          \n",
    "        Returns:\n",
    "          s1,base_frequency,MASKrand,\n",
    "          np.where(base_frequency)[0],\n",
    "          np.mean(rnd_match_prob),\n",
    "          np.mean(max_match_prob),\n",
    "          random_sample\n",
    "        '''\n",
    "        if self.samples is not None:\n",
    "            s0=s.copy()\n",
    "            s0=np.array(s0)   \n",
    "            MUTABLE=pd.DataFrame(np.zeros(len(self.cols)),index=self.cols).transpose()\n",
    "            WITHVAL=[x for x in self.cols[np.where(s0)[0]] if x in self.mutable_vars ]\n",
    "            MASKrand=[x for x in WITHVAL if random.random() < mask_prob ]\n",
    "            for m in MASKrand:\n",
    "                MUTABLE[m]=1.0\n",
    "            \n",
    "            mutable_x=MUTABLE.values[0]\n",
    "            base_frequency=mutable_x/mutable_x.sum()\n",
    "\n",
    "            # if np.isnan(base_frequency).any():\n",
    "            #     return np.nan,np.nan,np.nan\n",
    "            #     return self.getMaskedSample(s)\n",
    "\n",
    "            s1=s.copy()\n",
    "            for i in range(len(base_frequency)):\n",
    "                if base_frequency[i]>0.0001:\n",
    "                    s1[i]=''\n",
    "                    \n",
    "            random_sample=np.copy(s)\n",
    "            rnd_match_prob=[]        \n",
    "            max_match_prob=[]        \n",
    "            D=self.qnet.predict_distributions(s)\n",
    "            for i in MASKrand:\n",
    "                random_sample[np.where(\n",
    "                    self.cols==i)[0][0]]=self.__choose_one(\n",
    "                        self.D_null[np.where(self.cols==i)[0][0]].keys())\n",
    "                rnd_match_prob=np.append(rnd_match_prob,1/len(\n",
    "                    self.D_null[np.where(self.cols==i)[0][0]].keys()))\n",
    "                max_match_prob=np.append(\n",
    "                    max_match_prob,np.max(\n",
    "                        list(D[np.where(\n",
    "                            self.cols==i)[0][0]].values())))\n",
    "                \n",
    "            if allow_all_mutable:\n",
    "                for m in mutable_vars:\n",
    "                    MUTABLE[m]=1.0\n",
    "                mutable_x=MUTABLE.values[0]\n",
    "                base_frequency=mutable_x/mutable_x.sum()\n",
    "\n",
    "            return s1,base_frequency,MASKrand,np.where(\n",
    "                base_frequency)[0],np.mean(rnd_match_prob),np.mean(max_match_prob),random_sample\n",
    "        else:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "\n",
    "    def randomMaskReconstruction(self,\n",
    "                                index=None,\n",
    "                                return_dict=None,\n",
    "                                sample=None,\n",
    "                                feature_names=\"feature_names\",\n",
    "                                output_dir=\"recon_results/\",\n",
    "                                file_name=\"recon_tmp.csv\"):\n",
    "        \"\"\"reconstruct the masked sample by qsampling and comparing to original\n",
    "        set self.mask_prob and self.steps if needed\n",
    "\n",
    "        Args:\n",
    "          return_dict (dict): dict containing results. Defaults to None.\n",
    "          sample (list[str], optional): sample vector, must have the same num of features as the qnet. Defaults to None.\n",
    "          index (int): index of sample to take. Defaults to None.\n",
    "\n",
    "        Raises:\n",
    "          ValueError: Neither sample or index were given\n",
    "          ValueError: Both sample and index were given\n",
    "        \"\"\"\n",
    "        if all(x is None for x in [sample, index]):\n",
    "            raise ValueError(\"Must input either sample or index!\")\n",
    "        elif all(x is not None for x in [sample, index]):\n",
    "            raise ValueError(\"Must input either sample or index not both!\")\n",
    "        elif sample is not None:\n",
    "            s=np.array(pd.DataFrame(sample).fillna('').values.astype(str)[:])\n",
    "        elif index is not None:\n",
    "            s=self.samples_as_strings[index]\n",
    "            \n",
    "        s1,bp,mask_,maskindex,rmatch_u,rmatch,random_sample=self.getMaskedSample(s, \n",
    "                                                                        mask_prob=self.mask_prob)\n",
    "        if np.isnan(bp).any():\n",
    "            return_dict[index] = np.nan,np.nan,np.nan\n",
    "            return np.nan,np.nan,np.nan\n",
    "        \n",
    "        if not os.path.exists(output_dir):\n",
    "            os.makedirs(output_dir)\n",
    "\n",
    "        qs=qsample(s1,self.qnet,self.steps,bp)\n",
    "        dqestim=qdistance(s,qs,self.qnet,self.qnet)\n",
    "        dactual=qdistance(s,s1,self.qnet,self.qnet)\n",
    "        qdistance_time_end = time.time()\n",
    "        \n",
    "        cmpf=pd.DataFrame([s,qs,random_sample],columns=self.cols,index=['sample','qsample','random_sample'])[mask_].transpose()\n",
    "        cmpf.index.name= feature_names\n",
    "        file_name = file_name.replace(\"tmp\", str(index))\n",
    "        cmpf.to_csv(output_dir+file_name)\n",
    "        return_dict[index] = (1 - (dqestim/dactual))*100,rmatch_u,rmatch,s,qs\n",
    "        return (1 - (dqestim/dactual))*100,rmatch_u,rmatch,s,qs,random_sample,mask_\n",
    "\n",
    "    def randomMaskReconstruction_multiple(self,\n",
    "                                          out_file):\n",
    "        '''runs and saves the results of the predicted masked sample\n",
    "\n",
    "        Args:\n",
    "          output_file (str): directory and/or file for output\n",
    "          \n",
    "        Returmns:\n",
    "          result.rederr.mean(), result.rand_err.mean(): mean of reconstruction error and random error\n",
    "        '''\n",
    "        manager = mp.Manager()\n",
    "        return_dict = manager.dict()\n",
    "        processes = []\n",
    "        \n",
    "        for i in range(len(self.samples)):\n",
    "            p = mp.Process(target=self.randomMaskReconstruction, args=(i, return_dict))\n",
    "            processes.append(p)\n",
    "\n",
    "        [x.start() for x in processes]\n",
    "        [x.join() for x in processes]\n",
    "        \n",
    "        #result=pd.DataFrame(return_dict.items())[1]#, columns=['sample','rederr','r_prob','rand_err','s','q','r'])\n",
    "        result=[x for x in return_dict.values() if isinstance(x, tuple)]\n",
    "        # #result=pd.DataFrame(result.tolist())\n",
    "        # print(result)\n",
    "        # cmprdf=result[[3,4,5]]\n",
    "        # mask_=result[[6]]\n",
    "        # cmprdf.columns=['s','q','r']#[mask_].transpose()\n",
    "        # cmprdf.to_csv(\"examples_results/CMPF_\"+\"tmp\"+\".csv\")\n",
    "        # print(cmprdf)\n",
    "        # result=result[[0,1,2]]\n",
    "        result=pd.DataFrame(result,columns=['rederr','r_prob','rand_err','sample','qsampled','random_sample','mask_'])\n",
    "        result.rederr=result.rederr.astype(float)\n",
    "\n",
    "        if self.poles is not None:\n",
    "            result.to_csv(out_file)\n",
    "        else:\n",
    "            result.to_csv(out_file)\n",
    "        \n",
    "        return pd.DataFrame(return_dict.copy())\n",
    "    \n",
    "    def dmat_filewriter(self,\n",
    "                        pyfile,\n",
    "                        QNETPATH,\n",
    "                        MPI_SETUP_FILE=\"mpi_setup.sh\",\n",
    "                        MPI_RUN_FILE=\"mpi_run.sh\",\n",
    "                        MPI_LAUNCHER_FILE=\"/project2/ishanu/LAUNCH_UTILITY/launcher_s.sh\",\n",
    "                        YEARS='2016',\n",
    "                        NODES=4,\n",
    "                        T=12,\n",
    "                        num_samples=None,\n",
    "                        OUTFILE='tmp_distmatrix.csv',\n",
    "                        tmp_samplesfile=\"tmp_samples_as_strings.csv\"):\n",
    "        if all(x is not None for x in [self.poles_dict,self.features,\n",
    "                                       self.qnet, self.cols]):\n",
    "            if num_samples is not None:\n",
    "                self.set_nsamples(num_samples)\n",
    "            \n",
    "            tmp_path = \"mpi_tmp/\"\n",
    "            if not os.path.exists(tmp_path):\n",
    "                os.makedirs(tmp_path)\n",
    "            \n",
    "            pd.DataFrame(self.samples_as_strings).to_csv(tmp_path+tmp_samplesfile, header=None, index=None)\n",
    "            \n",
    "            w = self.samples.index.size\n",
    "            with open(tmp_path+pyfile, 'w+') as f:\n",
    "                f.writelines([\"from mpi4py.futures import MPIPoolExecutor\\n\",\n",
    "                              \"import numpy as np\\n\",\n",
    "                              \"import pandas as pd\\n\",\n",
    "                              \"from quasinet.qnet import Qnet, qdistance, load_qnet, qdistance_matrix\\n\",\n",
    "                              \"from quasinet.qsampling import qsample, targeted_qsample\\n\\n\",\n",
    "                              \"qnet=load_qnet(\\'{}\\')\\n\".format(QNETPATH)])\n",
    "\n",
    "                f.writelines([\"w = {}\\n\".format(w),\n",
    "                              \"h = w\\n\",\n",
    "                              \"p_all = pd.read_csv(\\\"{}\\\", header=None)\\n\\n\".format(tmp_samplesfile)])\n",
    "\n",
    "                f.writelines([\"def distfunc(x,y):\\n\",\n",
    "                              \"\\td=qdistance(x,y,qnet,qnet)\\n\",\n",
    "                              \"\\treturn d\\n\\n\"])\n",
    "\n",
    "                f.writelines([\"def dfunc_line(k):\\n\",\n",
    "                              \"\\tline = np.zeros(w)\\n\",\n",
    "                              \"\\ty = np.array(p_all.iloc[k])\\n\",\n",
    "                              \"\\tfor j in range(w):\\n\",\n",
    "                              \"\\t\\tif j > k:\\n\",\n",
    "                              \"\\t\\t\\tx = np.array(p_all.iloc[j])\\n\",\n",
    "                              \"\\t\\t\\tline[j] = distfunc(x, y)\\n\",\n",
    "                              \"\\treturn line\\n\\n\"])\n",
    "\n",
    "                f.writelines([\"if __name__ == '__main__':\\n\",\n",
    "                              \"\\twith MPIPoolExecutor() as executor:\\n\",\n",
    "                              \"\\t\\tresult = executor.map(dfunc_line, range(h))\\n\",\n",
    "                              \"\\tresult = pd.DataFrame(result)\\n\",\n",
    "\t                          \"\\tresult = result.to_numpy()\\n\",\n",
    "                              \"\\tresult = pd.DataFrame(np.maximum(result, result.transpose()))\\n\"\n",
    "                              \"\\tresult.to_csv(\\'{}\\',index=None,header=None)\".format(OUTFILE)])\n",
    "                \n",
    "            with open(tmp_path+MPI_SETUP_FILE, 'w+') as ms:\n",
    "                ms.writelines([\"#!/bin/bash\\n\",\n",
    "                               \"YEAR=$1\\n\\n\",\n",
    "                               \"if [ $# -gt 1 ] ; then\\n\",\n",
    "                               \"\\tNODES=$2\\n\",\n",
    "                               \"else\\n\",\n",
    "                               \"\\tNODES=3\\n\",\n",
    "                               \"fi\\n\",\n",
    "                               \"if [ $# -gt 2 ] ; then\\n\",\n",
    "                               \"\\tNUM=$3\\n\",\n",
    "                               \"else\\n\",\n",
    "                               \"\\tNUM='all'\\n\",\n",
    "                               \"fi\\n\",\n",
    "                               \"if [ $# -gt 3 ] ; then\\n\",\n",
    "                               \"\\tPROG=$4\\n\",\n",
    "                               \"else\\n\",\n",
    "                               \"\\tPROG=$(tty)\\n\",\n",
    "                               \"fi\\n\\n\",\n",
    "                               \"NUMPROC=`expr 28 \\* $NODES`\\n\",\n",
    "                               \"echo \\\"module load midway2\\\" >> $PROG\\n\",\n",
    "                               \"echo \\\"module unload python\\\" >> $PROG\\n\",\n",
    "                               \"echo \\\"module unload openmpi\\\" >> $PROG\\n\",\n",
    "                               \"echo \\\"module load python/anaconda-2020.02\\\" >> $PROG\\n\",\n",
    "                               \"echo \\\"module load mpi4py\\\" >> $PROG\\n\",\n",
    "                               \"echo \\\"date; mpiexec -n \\\"$NUMPROC\\\" python3 -m mpi4py.futures {}; date\\\"  >> $PROG\\n\".format(pyfile),\n",
    "                                ])\n",
    "\n",
    "            with open(tmp_path+MPI_RUN_FILE, 'w+') as mr:\n",
    "                mr.writelines([\"#!/bin/bash\\n\",\n",
    "                               \"YEARS=\\'{}\\'\\n\".format(YEARS),\n",
    "                               \"# nodes requested\\n\",\n",
    "                               \"NODES={}\\n\".format(NODES),\n",
    "                               \"# time requested\\n\",\n",
    "                               \"T={}\\n\".format(T),\n",
    "                               \"NUM=\\'all\\'\\n\",\n",
    "                               \"LAUNCH=\\'{}\\'\\n\\n\".format(MPI_LAUNCHER_FILE),\n",
    "                               \"for yr in `echo $YEARS`\\n\",\n",
    "                               \"do\\n\",\n",
    "                               \"\\techo $yr\\n\",\n",
    "                               \"\\t./{} $yr $NODES $NUM tmp_\\\"$yr\\\"\\n\".format(MPI_SETUP_FILE),\n",
    "                               \"\\t$LAUNCH -P tmp_\\\"$yr\\\" -F -T $T -N \\\"$NODES\\\" -C 28 -p broadwl -J ACRDALL_\\\"$yr\\\" -M 56\\n\",\n",
    "                               \"done\\n\",\n",
    "                               \"rm tmp_\\\"$yr\\\"*\\n\"])\n",
    "            os.system(\"cp {} {}\".format(MPI_LAUNCHER_FILE,tmp_path+'mpi_launcher.sh'))\n",
    "        \n",
    "        else:\n",
    "            raise ValueError(\"load data first!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true,
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1638657521.265449\n",
      "fitting\n",
      "[['temp not working' 'e' 'c' ... 'e' '3301' '1']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '1']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '1']\n",
      " ...\n",
      " ['retired' 'e' 'e' ... 'c' '3301' '2']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '2']\n",
      " ['keeping house' 'e' 'e' ... 'c' '3301' '2']]\n",
      "1638658564.3172483\n",
      "njobs:  1  | elapsed time:  1043.0517992973328\n",
      "1638658564.3179076\n",
      "fitting\n",
      "[['temp not working' 'e' 'c' ... 'e' '3301' '1']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '1']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '1']\n",
      " ...\n",
      " ['retired' 'e' 'e' ... 'c' '3301' '2']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '2']\n",
      " ['keeping house' 'e' 'e' ... 'c' '3301' '2']]\n",
      "1638659168.0808773\n",
      "njobs:  2  | elapsed time:  603.7629697322845\n",
      "1638659168.0825222\n",
      "fitting\n",
      "[['temp not working' 'e' 'c' ... 'e' '3301' '1']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '1']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '1']\n",
      " ...\n",
      " ['retired' 'e' 'e' ... 'c' '3301' '2']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '2']\n",
      " ['keeping house' 'e' 'e' ... 'c' '3301' '2']]\n",
      "1638659580.3828526\n",
      "njobs:  3  | elapsed time:  412.3003304004669\n",
      "1638659580.3831995\n",
      "fitting\n",
      "[['temp not working' 'e' 'c' ... 'e' '3301' '1']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '1']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '1']\n",
      " ...\n",
      " ['retired' 'e' 'e' ... 'c' '3301' '2']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '2']\n",
      " ['keeping house' 'e' 'e' ... 'c' '3301' '2']]\n",
      "1638659894.3004243\n",
      "njobs:  4  | elapsed time:  313.9172248840332\n",
      "1638659894.3011844\n",
      "fitting\n",
      "[['temp not working' 'e' 'c' ... 'e' '3301' '1']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '1']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '1']\n",
      " ...\n",
      " ['retired' 'e' 'e' ... 'c' '3301' '2']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '2']\n",
      " ['keeping house' 'e' 'e' ... 'c' '3301' '2']]\n",
      "1638660151.081877\n",
      "njobs:  5  | elapsed time:  256.78069257736206\n",
      "1638660151.082423\n",
      "fitting\n",
      "[['temp not working' 'e' 'c' ... 'e' '3301' '1']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '1']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '1']\n",
      " ...\n",
      " ['retired' 'e' 'e' ... 'c' '3301' '2']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '2']\n",
      " ['keeping house' 'e' 'e' ... 'c' '3301' '2']]\n",
      "1638660363.4955018\n",
      "njobs:  6  | elapsed time:  212.41307878494263\n",
      "1638660363.4958553\n",
      "fitting\n",
      "[['temp not working' 'e' 'c' ... 'e' '3301' '1']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '1']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '1']\n",
      " ...\n",
      " ['retired' 'e' 'e' ... 'c' '3301' '2']\n",
      " ['working fulltime' 'c' 'e' ... 'c' '3301' '2']\n",
      " ['keeping house' 'e' 'e' ... 'c' '3301' '2']]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/software/python-3.7.0-el7-x86_64/lib/python3.7/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    908\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'supports_timeout'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 909\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    910\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/software/python-3.7.0-el7-x86_64/lib/python3.7/site-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mwrap_future_result\u001b[0;34m(future, timeout)\u001b[0m\n\u001b[1;32m    561\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 562\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    563\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mLokyTimeoutError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/software/python-3.7.0-el7-x86_64/lib/python3.7/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    426\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 427\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_condition\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    428\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/software/python-3.7.0-el7-x86_64/lib/python3.7/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    295\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 296\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    297\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-ba7293d451d5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0mtest_samples\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msamples\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mnsample\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_samples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m     \u001b[0mQnet_\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_samples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m     \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/quasinet/qnet.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    152\u001b[0m             delayed(self._parallel_fit_tree)(\n\u001b[1;32m    153\u001b[0m                 trees[col], X, col)\n\u001b[0;32m--> 154\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mcol\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    155\u001b[0m             )\n\u001b[1;32m    156\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/software/python-3.7.0-el7-x86_64/lib/python3.7/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m   1015\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1016\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieval_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1017\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1018\u001b[0m             \u001b[0;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1019\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/software/python-3.7.0-el7-x86_64/lib/python3.7/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    929\u001b[0m                     \u001b[0;31m# scheduling.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    930\u001b[0m                     \u001b[0mensure_ready\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_managed_backend\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 931\u001b[0;31m                     \u001b[0mbackend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mabort_everything\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mensure_ready\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mensure_ready\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    932\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    933\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexception\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTransportableException\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/software/python-3.7.0-el7-x86_64/lib/python3.7/site-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mabort_everything\u001b[0;34m(self, ensure_ready)\u001b[0m\n\u001b[1;32m    577\u001b[0m         \"\"\"Shutdown the workers and restart a new one with the same parameters\n\u001b[1;32m    578\u001b[0m         \"\"\"\n\u001b[0;32m--> 579\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_workers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshutdown\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkill_workers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    580\u001b[0m         \u001b[0mdelete_folder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_workers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_temp_folder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    581\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_workers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/software/python-3.7.0-el7-x86_64/lib/python3.7/site-packages/joblib/externals/loky/process_executor.py\u001b[0m in \u001b[0;36mshutdown\u001b[0;34m(self, wait, kill_workers)\u001b[0m\n\u001b[1;32m   1099\u001b[0m                     \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1100\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1101\u001b[0;31m                 \u001b[0mqmt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1103\u001b[0m         \u001b[0mcq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_queue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/software/python-3.7.0-el7-x86_64/lib/python3.7/threading.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m   1030\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1031\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1032\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wait_for_tstate_lock\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1033\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1034\u001b[0m             \u001b[0;31m# the behavior of a negative timeout isn't documented, but\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/software/python-3.7.0-el7-x86_64/lib/python3.7/threading.py\u001b[0m in \u001b[0;36m_wait_for_tstate_lock\u001b[0;34m(self, block, timeout)\u001b[0m\n\u001b[1;32m   1046\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlock\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# already determined that the C code is done\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1047\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_stopped\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1048\u001b[0;31m         \u001b[0;32melif\u001b[0m \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblock\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1049\u001b[0m             \u001b[0mlock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrelease\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1050\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from quasinet.qnet import Qnet\n",
    "import time\n",
    "for i in range(1, 10):\n",
    "    nsample = 10\n",
    "\n",
    "    start = time.time()\n",
    "    print(start)\n",
    "    Qnet_ = Qnet(n_jobs=i, feature_names=featurenames)\n",
    "    print(\"fitting\")\n",
    "\n",
    "    test_samples = samples[:nsample]\n",
    "    print(test_samples)\n",
    "    Qnet_.fit(test_samples)\n",
    "    end = time.time()\n",
    "    print(end)\n",
    "    total_time=end-start\n",
    "    print(\"njobs: \",i, \" | elapsed time: \", total_time)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1638660651.9638221\n",
      "fitting\n",
      "1638660690.0304549\n",
      "nsamples:  1  | elapsed time:  38.06663274765015\n",
      "1638660690.0307827\n",
      "fitting\n",
      "1638660899.1668923\n",
      "nsamples:  10  | elapsed time:  209.1361095905304\n",
      "1638660899.1671965\n",
      "fitting\n",
      "1638661693.904029\n",
      "nsamples:  100  | elapsed time:  794.7368323802948\n"
     ]
    }
   ],
   "source": [
    "from quasinet.qnet import Qnet\n",
    "import time\n",
    "for i in range(3):\n",
    "    nsample = 10 ** i\n",
    "\n",
    "    start = time.time()\n",
    "    print(start)\n",
    "    Qnet_ = Qnet(n_jobs=6, feature_names=featurenames)\n",
    "    print(\"fitting\")\n",
    "    test_samples = samples[:nsample]\n",
    "    Qnet_.fit(test_samples)\n",
    "    end = time.time()\n",
    "    print(end)\n",
    "    total_time=end-start\n",
    "    print(\"nsamples: \",nsample, \" | elapsed time: \", total_time)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1034\n"
     ]
    }
   ],
   "source": [
    "# testing cognet\n",
    "# set some paramaters in instantiating cognet class \n",
    "# if loading from model obj, no need to use load_data func, otherwise, load_data\n",
    "Cg = cg()\n",
    "print(len(model_.features))\n",
    "Cg.load_from_model(model_, data, 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "class-computed distance: 0.09605605005536405\n",
      "actual:0.09726189681525353\n"
     ]
    }
   ],
   "source": [
    "# distance calculation for individual samples    \n",
    "# we have a nsteps parameter (for sample 1 and sample2)\n",
    "# which qsamples the sample1 and sample2 if set before\n",
    "# computing distance. Note qsampling must only \n",
    "# change mutable varaibles, so need to compute base-freq\n",
    "distance = Cg.distance(samples[1],samples[3],nsteps1=5, nsteps2=5)\n",
    "print(\"class-computed distance:\", distance)\n",
    "qdistance_ = qdistance(samples[1],samples[3],Cg.qnet,Cg.qnet)\n",
    "print(\"actual:{}\".format(qdistance_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 pole features not found in sample features\n"
     ]
    }
   ],
   "source": [
    "# produce stats on how many column names actually match\n",
    "stats = Cg.set_poles(POLEFILE,\"R\",\"L\",steps=120, VERBOSE=True)\n",
    "\n",
    "# compute polar distance matrix\n",
    "dmatrix = Cg.polar_separation(nsteps=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dissonance: [0. 0. 0. ... 0. 0. 0.]\n",
      "reconstruction results: 60.041315105813254 0.34206349206349207 0.6079032691205803\n",
      "ideology: [0.13563834025518842, 0.09057258009984209, 0.07426032641060697, 0.12026285236567653]\n",
      "Dispersion: [0.04480500841639735, 0.1194594397221057]\n",
      "distance from poles: [0.021635271857159282, 0.022334688414062866]\n"
     ]
    }
   ],
   "source": [
    "#------------------\n",
    "# the following are for single samples\n",
    "\n",
    "# dissonance\n",
    "dissonance_array = Cg.dissonance(1)\n",
    "print(\"dissonance:\", dissonance_array)\n",
    "\n",
    "# random mask and reconstruction\n",
    "returndict = {}\n",
    "rederr,r_prob,rand_err,s,qs,s_rand,mask_ = Cg.randomMaskReconstruction(1, returndict)# sample=np.array(samples[1]))\n",
    "print(\"reconstruction results:\", rederr, r_prob, rand_err)\n",
    "\n",
    "#ideology\n",
    "Cg.num_qsamples = 5\n",
    "ideology_index = Cg.ideology(3,pole_1=\"R\",pole_2=\"L\")\n",
    "print(\"ideology:\", ideology_index)\n",
    "\n",
    "# disperion\n",
    "dispersion_ = Cg.dispersion(3)\n",
    "print(\"Dispersion:\", dispersion_)\n",
    "\n",
    "# compute distance from each pole\n",
    "array_distances = Cg.polarDistance(1, returndict)\n",
    "print(\"distance from poles:\", array_distances)\n",
    "#-------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dissonance array:      0    1    2         3    4    5    6    8         7    9\n",
      "0  0.0  0.0  0.0  0.742767  0.0  0.0  0.0  0.0  0.675424  0.0\n",
      "1  0.0  0.0  0.0  0.000000  0.0  0.0  0.0  0.0  0.000000  0.0\n",
      "----------------------------------------------------------------------\n",
      "\n",
      "reconstruction results            1          0         2          6          3          5          4  \\\n",
      "0  22.131751  21.218220  6.905767  22.586894  20.704863  16.464651  14.335385   \n",
      "1   0.287309   0.310591  0.300113   0.291010   0.292836   0.294539   0.304725   \n",
      "\n",
      "           9          7          8  \n",
      "0  20.479017  27.600058  18.311126  \n",
      "1   0.292954   0.292608   0.299661  \n",
      "----------------------------------------------------------------------\n",
      "\n",
      "ideology indices           0         3         1         2         5         7         4  \\\n",
      "0  0.092942  0.075258  0.052030  0.028209  0.013110  0.113126  0.057992   \n",
      "1  0.153999  0.159029  0.150432  0.165401  0.149743  0.151961  0.147609   \n",
      "2  0.141856  0.149197  0.143635  0.161715  0.148030  0.137181  0.140033   \n",
      "3  0.130650  0.130650  0.130650  0.130650  0.130650  0.130650  0.130650   \n",
      "\n",
      "          6         8         9  \n",
      "0  0.084704  0.070234  0.078314  \n",
      "1  0.155513  0.150783  0.156829  \n",
      "2  0.144446  0.141607  0.146597  \n",
      "3  0.130650  0.130650  0.130650  \n",
      "----------------------------------------------------------------------\n",
      "\n",
      "dispersion array           0         4         7         2         3         9         8  \\\n",
      "0  0.005011  0.005326  0.005597  0.005942  0.005453  0.006428  0.004478   \n",
      "1  0.034881  0.038055  0.040221  0.038154  0.038399  0.049817  0.031655   \n",
      "\n",
      "          5         6         1  \n",
      "0  0.004618  0.004948  0.004813  \n",
      "1  0.032702  0.035061  0.035819  \n",
      "----------------------------------------------------------------------\n",
      "\n",
      "polar distances array {1: [0.13113822150272192, 0.13817617891264253], 0: [0.13341919912248365, 0.1388134557142811], 4: [0.12556483154166867, 0.13489804369935188], 3: [0.139177046204298, 0.1429509323014114], 2: [0.14770140610545116, 0.15568173223424667], 5: [0.13559304485298362, 0.14382642153293623], 7: [0.12226382336294804, 0.13030202309379554], 9: [0.12845960185555608, 0.13830284925478245], 8: [0.1347173139844925, 0.1360870125197645], 6: [0.1368023860565726, 0.13786632263896736]}\n",
      "----------------------------------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# the following are for arrays of samples\n",
    "# multiprocessing suffices\n",
    "\n",
    "# set sammple size to 10\n",
    "count = 0\n",
    "if count == 0:\n",
    "    Cg.set_nsamples(10)\n",
    "    count +=1\n",
    "    \n",
    "# computing polar_indices makes sure that dissonance matrix only takes in polar cols\n",
    "Cg.compute_polar_indices()\n",
    "dissonance_array = Cg.dissonance_matrix(output_file='examples_results/DISSONANCE_matrix.csv')\n",
    "print(\"dissonance array:\", dissonance_array[:2])\n",
    "print('----------------------------------------------------------------------\\n')\n",
    "\n",
    "# random mask and reconstruction\n",
    "recon_df = Cg.randomMaskReconstruction_multiple('examples_results/randomMaskRecon_test.csv')\n",
    "print(\"reconstruction results\", recon_df[:2])\n",
    "print('----------------------------------------------------------------------\\n')\n",
    "\n",
    "# ideology indices\n",
    "ideology_index = Cg.compute_DLI_samples('ideology','examples_results/ideology.csv')\n",
    "print(\"ideology indices\", ideology_index)\n",
    "print('----------------------------------------------------------------------\\n')\n",
    "\n",
    "# dispersion\n",
    "local_dispersion = Cg.compute_DLI_samples('dispersion', 'examples_results/dispersion_test.csv')\n",
    "print(\"dispersion array\", local_dispersion)\n",
    "print('----------------------------------------------------------------------\\n')\n",
    "\n",
    "# polar distances\n",
    "polar_array = Cg.polarDistance_multiple('examples_results/polarDistance_multiple_test.csv')\n",
    "print(\"polar distances array\",polar_array)\n",
    "print('----------------------------------------------------------------------\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1034\n",
      "0 pole features not found in sample features\n",
      "local distance matrix:           0         1         2         3         4         5         6  \\\n",
      "0  0.000000  0.091232  0.134658  0.113836  0.086175  0.120154  0.079885   \n",
      "1  0.091232  0.000000  0.108381  0.114186  0.079264  0.085646  0.086029   \n",
      "2  0.134658  0.108381  0.000000  0.129788  0.120588  0.076976  0.130525   \n",
      "3  0.113836  0.114186  0.129788  0.000000  0.109474  0.105337  0.127201   \n",
      "4  0.086175  0.079264  0.120588  0.109474  0.000000  0.101664  0.093314   \n",
      "5  0.120154  0.085646  0.076976  0.105337  0.101664  0.000000  0.115825   \n",
      "6  0.079885  0.086029  0.130525  0.127201  0.093314  0.115825  0.000000   \n",
      "7  0.079909  0.102618  0.145231  0.117701  0.075187  0.126603  0.094005   \n",
      "8  0.126323  0.098386  0.111161  0.093618  0.124169  0.094520  0.115584   \n",
      "9  0.096016  0.092799  0.139336  0.130243  0.066111  0.124740  0.110569   \n",
      "\n",
      "          7         8         9  \n",
      "0  0.079909  0.126323  0.096016  \n",
      "1  0.102618  0.098386  0.092799  \n",
      "2  0.145231  0.111161  0.139336  \n",
      "3  0.117701  0.093618  0.130243  \n",
      "4  0.075187  0.124169  0.066111  \n",
      "5  0.126603  0.094520  0.124740  \n",
      "6  0.094005  0.115584  0.110569  \n",
      "7  0.000000  0.134604  0.067742  \n",
      "8  0.134604  0.000000  0.136706  \n",
      "9  0.067742  0.136706  0.000000  \n"
     ]
    }
   ],
   "source": [
    "# compute qdistance matrix for small set of samples\n",
    "# set nsamples first to set the number of samples to be included in matrix\n",
    "distance_matrix=Cg.distfunc_multiples(\"examples_results/distfunc_multiples_testing.csv\")\n",
    "print(\"local distance matrix:\", distance_matrix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write files to compute qdistance matrix for large set of samples\n",
    "# execute generated shell script to run mpi parallelization on midway\n",
    "Cg.dmat_filewriter(\"GSS_cognet.py\", \"examples_data/gss_2018.joblib\",\n",
    "                   MPI_SETUP_FILE=\"GSS_mpi_setup.sh\",\n",
    "                   MPI_RUN_FILE=\"GSS_mpi_run.sh\",\n",
    "                   MPI_LAUNCHER_FILE=\"GSS_mpi_launcher.sh\",\n",
    "                   YEARS='2018',NODES=4,T=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## embedding\n",
    "## embed generated Qdist Matrix\n",
    "Cg.year = '2018'\n",
    "Cg.embed('examples_results/distfunc_multiples_testing.csv', 'embed', 'examples_results/',EMBED_BINARY='cognet/cognet/bin/__embed__.so')\n",
    "#pd.read_csv('examples_results/embed_E_2018.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
