{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# This is an example of how we see \n",
    "# the package work. The functions listed here\n",
    "# are probably the only ones that should be exposed, ie documented.\n",
    "# others should br prepended with a double underscore\n",
    "#  \n",
    "# The cognet directory has the following \"modules\"\n",
    "# which are seprate .py files containing clases and functions\n",
    "# The modules are cognet.py, dataFormatter.py, model.py, util.py, viz.py\n",
    "# we will write the viz.py later.\n",
    "import sys\n",
    "\n",
    "from quasinet.qnet import qdistance\n",
    "from cognet.cognet import cognet as cg\n",
    "from cognet.dataFormatter import dataFormatter\n",
    "from cognet.model import model \n",
    "#import cognet.util\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "yr = '2018'\n",
    "POLEFILE='examples_data/polar_vectors.csv'\n",
    "QPATH='examples_data/gss_'+yr+'.joblib'\n",
    "IMMUTABLE_FILE='examples_data/immutable.csv'\n",
    "GSSDATA = 'examples_data/gss_'+yr+'.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            wrkstat HRS1 HRS2 evwork        wrkslf  wrkgovt OCC10 PRESTG10  \\\n",
      "0  temp not working    e    c    NaN  someone else  private     b        c   \n",
      "1  working fulltime    c    e    NaN  someone else  private     b        d   \n",
      "\n",
      "  PRESTG105PLUS INDUS10  ...    neisafe rlooks rgroomed rweight rhlthend wtss  \\\n",
      "0             c       c  ...  very safe    NaN      NaN     NaN      NaN    e   \n",
      "1             d       c  ...  very safe    NaN      NaN     NaN      NaN    c   \n",
      "\n",
      "  wtssnr wtssall vstrat vpsu  \n",
      "0      e       e   3301    1  \n",
      "1      c       c   3301    1  \n",
      "\n",
      "[2 rows x 1034 columns]\n"
     ]
    }
   ],
   "source": [
    "# ---------------------------------\n",
    "# testing dataFormatter\n",
    "data = dataFormatter(samples=GSSDATA,\n",
    "                test_size=0.5)\n",
    "# load the sample data\n",
    "# have option for test/train split\n",
    "# make checks to ensure we will not bark at qnet construction \n",
    "# data.train() returns traininh data\n",
    "# data.test() returns test data\n",
    "print(data.samples[:2])\n",
    "features,samples = data.train()\n",
    "# we can set mutable and immutable vars from list or file\n",
    "im_vars_df = pd.read_csv(IMMUTABLE_FILE, names=['vars'])\n",
    "im_vars_list = im_vars_df.vars.to_list()\n",
    "mutable_vars, immutable_vars = data.mutable_variables(immutable_list=im_vars_list)\n",
    "mutable_vars, immutable_vars = data.mutable_variables(IMMUTABLE_FILE=IMMUTABLE_FILE)\n",
    "# -------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "updating\n"
     ]
    }
   ],
   "source": [
    "# -------------------------------------\n",
    "# testing model functionality\n",
    "# can either input features and samples directly, or infer from data obj\n",
    "model_ = model()\n",
    "\n",
    "# qnet construction parameters\n",
    "test_model_buildqnet = False\n",
    "# infer qnet\n",
    "if test_model_buildqnet:\n",
    "        model_.fit(data_obj=data)\n",
    "        model_.export_dot(\"tmp_dot_modelclass.dot\",\n",
    "                        generate_trees=True)\n",
    "        model_.save(\"tmp_nodelclass.joblib\")\n",
    "        #model_.load(\"tmp_nodelclass.joblib\")\n",
    "else:\n",
    "    model_.load(\"examples_data/gss_2018.joblib\")\n",
    "# -------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1034\n",
      "0.07335480502337259\n",
      "actual:0.0736388684905705\n"
     ]
    }
   ],
   "source": [
    "# -------------------------------------\n",
    "# testing cognet\n",
    "# set some paramaters in instantiating cognet class \n",
    "# if loading from model obj, no need to load_data, otherwise, load_data\n",
    "\n",
    "Cg = cg()\n",
    "print(len(model_.features))\n",
    "Cg.load_from_model(model_, samples_file='examples_data/gss_2018.csv')\n",
    "\n",
    "# distance calculation for individual samples    \n",
    "# we have a nsteps parameter (for sample 1 and sample2)\n",
    "# which qsamples the sample1 and sample2 if set before\n",
    "# computing distance. Note qsampling must only \n",
    "# change mutable varaibles, so need to compute base-freq\n",
    "distance = Cg.distance(samples[1],samples[3],nsteps1=5, nsteps2=5)\n",
    "print(distance)\n",
    "qdistance_ = qdistance(samples[1],samples[3],Cg.qnet,Cg.qnet)\n",
    "print(\"actual:{}\".format(qdistance_))\n",
    "#distance = Cg.distance(data.samples[0],data.samples[1])\n",
    "#distance = Cg.distance(data.test[0],data.test[1])\n",
    "# -------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 pole features not found in sample features\n"
     ]
    }
   ],
   "source": [
    "#------------------\n",
    "# produce stats on how many column names actually match\n",
    "stats = Cg.set_poles(POLEFILE,\"R\",\"L\",steps=120)\n",
    "\n",
    "# compute polar distance matrix\n",
    "dmatrix = Cg.polar_separation(nsteps=0)\n",
    "#------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18.396413669406954 0.3611111111111111 0.555709324408589 ['' '' '' ... '' '' ''] ['' '' '' ... '' '' ''] ['' '' '' ... '' '' ''] ['grass', 'pray', 'pillok', 'polabuse', 'viruses', 'abpoorw']\n",
      "[0.04189684585543724, 0.1127111243657746]\n"
     ]
    }
   ],
   "source": [
    "#------------------\n",
    "# the following are for single samples\n",
    "dissonance_array = Cg.dissonance(1)\n",
    "returndict = {}\n",
    "rederr,r_prob,rand_err,s,qs,s_rand,mask_ = Cg.randomMaskReconstruction(1, returndict)# sample=np.array(samples[1]))\n",
    "print(rederr,r_prob,rand_err,s,qs,s_rand,mask_)\n",
    "#ideology_index = Cg.compute_DLI_sample(3)\n",
    "Cg.num_qsamples = 5\n",
    "ideology_index = Cg.ideology(3,pole_1=\"R\",pole_2=\"L\")\n",
    "\n",
    "# get dispersion of an individual sample\n",
    "Dispersion_ = Cg.dispersion(3)\n",
    "print(Dispersion_)\n",
    "# compute distance from each pole\n",
    "array_distances = Cg.polarDistance(1, returndict)\n",
    "#-------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#-------------------\n",
    "# the following are for arrays of samples\n",
    "# multiprocessing suffices\n",
    "count = 0\n",
    "if count == 0:\n",
    "    Cg.set_nsamples(10)\n",
    "    count +=1\n",
    "# computing polar_indices makes sure that dissonance matrix only takes in polar cols\n",
    "Cg.compute_polar_indices()\n",
    "dissonance_array = Cg.dissonance_matrix(output_file='examples_results/DISSONANCE_matrix.csv')\n",
    "# multiprocessing suffices\n",
    "dataframes,error_array = Cg.randomMaskReconstruction_multiple('examples_results/randomMaskRecon_test.csv')\n",
    "# multiprocessing suffices\n",
    "ideology_index = Cg.compute_DLI_samples('ideology','examples_results/ideology.csv')\n",
    "# multiprocessing suffices\n",
    "local_dispersion = Cg.compute_DLI_samples('dispersion', 'examples_results/dispersion_test.csv')\n",
    "# compute distance from each pole\n",
    "# multiprocessing suffices\n",
    "array_distances = Cg.polarDistance_multiple('examples_results/polarDistance_multiple_test.csv')\n",
    "#-------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#the following must use parallelization\n",
    "# next one must use mpi and hence will not run\n",
    "# with mpi without maybe a seprate script.\n",
    "# But look here: https://stackoverflow.com/questions/25772289/python-multiprocessing-within-mpi\n",
    "distance_matrix=Cg.distfunc_multiples(\"examples_results/distfunc_multiples_testing.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "from quasinet.qnet import Qnet, qdistance, load_qnet, qdistance_matrix\n",
    "from quasinet.qsampling import qsample, targeted_qsample\n",
    "#from mpi4py.futures import MPIPoolExecutor\n",
    "import sys\n",
    "import subprocess\n",
    "from scipy.stats import entropy\n",
    "import multiprocessing as mp\n",
    "import time\n",
    "from cognet.util import embed_to_pca\n",
    "import pkgutil\n",
    "\n",
    "class cognet:\n",
    "    \"\"\"Aggregate related Qnet functions\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        \"\"\"Init\n",
    "        \"\"\"\n",
    "        self.year = None\n",
    "        self.n_jobs = 28\n",
    "        self.qnet = None\n",
    "        self.steps = 120\n",
    "        self.num_qsamples = None\n",
    "        self.all_samples = None\n",
    "        self.samples = None\n",
    "        self.samples_as_strings = None\n",
    "        self.features = None\n",
    "        self.cols = None\n",
    "        self.immutable_vars = None\n",
    "        self.mutable_vars = None\n",
    "        self.poles = None\n",
    "        self.polar_features = None\n",
    "        self.polar_indices = None\n",
    "        self.poles_dict = {}\n",
    "        self.d0 = None\n",
    "        self.qdistance_matrix_file = None\n",
    "        self.dissonance_file = None\n",
    "        self.s_null = None\n",
    "        self.D_null = None\n",
    "        self.mask_prob = 0.5\n",
    "        self.variation_weight = None\n",
    "        self.polar_matrix = None\n",
    "    \n",
    "    def load_from_model(self,\n",
    "                        model,\n",
    "                        samples_file=None,\n",
    "                        im_vars=None,\n",
    "                        m_vars=None):\n",
    "        \"\"\"load parameters from model object\n",
    "\n",
    "        Args:\n",
    "          model (Class): model obj for loading parameters\n",
    "          samples_file (filepath): filepath and name for sample csv\n",
    "          im_vars (list[str], optional): Not implemented yet. Defaults to None.\n",
    "          m_vars (list[str], optional): Not implemented yet. Defaults to None.\n",
    "        \"\"\"\n",
    "        if model is not None:\n",
    "            self.qnet = model.myQnet\n",
    "            self.cols = np.array(model.features)\n",
    "            self.features = pd.DataFrame(columns=self.cols)\n",
    "            if any(x is not None for x in [model.immutable_vars, model.mutable_vars]):\n",
    "                if model.immutable_vars is not None:\n",
    "                    self.immutable_vars = model.immutable_vars\n",
    "                    self.mutable_vars = [x for x in self.features if x not in self.immutable_vars]\n",
    "                elif model.mutable_vars is not None:\n",
    "                    self.mutable_vars = model.mutable_vars\n",
    "                    self.immutable_vars = [x for x in self.features if x not in self.mutable_vars]\n",
    "            else:\n",
    "                self.mutable_vars = self.features\n",
    "            if all(x is None for x in [model.samples, samples_file]):\n",
    "                raise ValueError(\"Please input samples if loading model after loading qnet\")\n",
    "            elif model.samples is not None:\n",
    "                samples = pd.DataFrame(model.samples)\n",
    "            elif samples_file is not None:\n",
    "                samples = pd.read_csv(samples_file)\n",
    "                \n",
    "            self.samples = pd.concat([samples,self.features], axis=0)\n",
    "            self.all_samples = self.samples\n",
    "            self.samples_as_strings = self.samples[self.cols].fillna('').values.astype(str)[:]\n",
    "            self.s_null=['']*len(self.samples_as_strings[0])\n",
    "            self.D_null=self.qnet.predict_distributions(self.s_null)\n",
    "            variation_weight = []\n",
    "            for d in self.D_null:\n",
    "                v=[]\n",
    "                for val in d.values():\n",
    "                    v=np.append(v,val)\n",
    "                variation_weight.append(entropy(v,base=len(v)))\n",
    "            self.variation_weight = variation_weight\n",
    "            \n",
    "    def load_data(self,\n",
    "                year,\n",
    "                features_by_year,\n",
    "                samples,\n",
    "                qnet):\n",
    "        '''load cols, features, samples, and qnet.\n",
    "\n",
    "        Args:\n",
    "          year (str): to identify cols/features.\n",
    "          features_by_year (str): file containing all features by year of the dataset.\n",
    "          samples (str): file of samples for that year.\n",
    "          Qnet (str): Qnet file location.\n",
    "        '''\n",
    "        self.qnet = load_qnet(qnet)\n",
    "        self.year = year\n",
    "        self.cols = np.array((pd.read_csv(features_by_year,\n",
    "                            keep_default_na=True, \n",
    "                            index_col=0).set_index(\n",
    "                                'year')).loc[int(year)].apply(\n",
    "                                    eval).values[0])\n",
    "        self.features = pd.DataFrame(columns=self.cols)\n",
    "        self.mutable_vars = [x for x in self.cols]\n",
    "        #[self.cols].fillna('').values.astype(str)[:]\n",
    "\n",
    "        self.samples=pd.read_csv(samples)\n",
    "        self.samples = pd.concat([self.samples,self.features], axis=0)\n",
    "        self.all_samples = self.samples\n",
    "        self.samples_as_strings = self.samples[self.cols].fillna('').values.astype(str)[:]\n",
    "        self.s_null=['']*len(self.samples_as_strings[0])\n",
    "        self.D_null=self.qnet.predict_distributions(self.s_null)\n",
    "        variation_weight = []\n",
    "        for d in self.D_null:\n",
    "            v=[]\n",
    "            for val in d.values():\n",
    "                v=np.append(v,val)\n",
    "            variation_weight.append(entropy(v,base=len(v)))\n",
    "        self.variation_weight = variation_weight\n",
    "\n",
    "    def set_immutable_vars(self,\n",
    "                        IMMUTABLE_FILE):\n",
    "        '''set vars to immutable and mutable, \n",
    "        can prob combine this with the load_data func: only set the immutable vars if necessary\n",
    "\n",
    "        Args:\n",
    "          IMMUTABLE_FILE (str): file containing the immutable features/vars\n",
    "        '''\n",
    "        if self.cols is None:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "        self.immutable_vars = pd.read_csv(IMMUTABLE_FILE,index_col=0).transpose()\n",
    "        self.mutable_vars = None\n",
    "        self.mutable_vars = [x for x in self.cols\n",
    "                            if x.upper() not in self.immutable_vars.columns]\n",
    "    \n",
    "    def set_nsamples(self,\n",
    "                    num_samples):\n",
    "        '''select a subset of the samples\n",
    "\n",
    "        Args:\n",
    "          num_samples (int): Set num of samples to subset\n",
    "        '''\n",
    "        self.samples = self.all_samples\n",
    "        if all(x is not None for x in [num_samples, self.samples]):\n",
    "            if num_samples > len(self.samples.index):\n",
    "                string = 'The number of selected samples ({}) ' + \\\n",
    "                    'is greater than the number of samples ({})!'\n",
    "                string = string.format(num_samples, len(self.samples.index))\n",
    "                raise ValueError(string)\n",
    "\n",
    "            if num_samples == len(self.samples.index):\n",
    "                string = 'The number of selected samples ({}) ' + \\\n",
    "                    'is equal to the number of samples ({})!'\n",
    "                string = string.format(num_samples, len(self.samples.index))\n",
    "                print(string)\n",
    "            self.samples = self.samples.sample(num_samples)\n",
    "            self.samples_as_strings = self.samples[self.cols].fillna('').values.astype(str)[:]\n",
    "\n",
    "        elif self.samples is None:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "\n",
    "    def __variation_weight(self,\n",
    "                        index):\n",
    "        \"\"\"\n",
    "        \"\"\"\n",
    "        d_=self.D_null[index]\n",
    "        v=[]\n",
    "        for val in d_.values():\n",
    "            v=np.append(v,val)\n",
    "        return entropy(v,base=len(v))\n",
    "    \n",
    "    def getBaseFrequency(self, \n",
    "                        sample):\n",
    "        '''get frequency of the variables\n",
    "        helper func for qsampling\n",
    "\n",
    "        Args:\n",
    "          sample (list[str]): vector of sample, must have the same num of features as the qnet\n",
    "        '''\n",
    "        MUTABLE=pd.DataFrame(np.zeros(len(self.cols)),index=self.cols).transpose()\n",
    "                \n",
    "        for m in self.mutable_vars:\n",
    "            MUTABLE[m]=1.0\n",
    "        mutable_x=MUTABLE.values[0]\n",
    "        base_frequency=mutable_x/mutable_x.sum()\n",
    "\n",
    "        # commented out for now for testing using smaller qnet\n",
    "        for i in range(len(base_frequency)):\n",
    "            if base_frequency[i]>0.0:\n",
    "                base_frequency[i]= self.variation_weight[i]*base_frequency[i]\n",
    "\n",
    "        return base_frequency/base_frequency.sum()\n",
    "    \n",
    "    def qsampling(self,\n",
    "                sample,\n",
    "                steps,\n",
    "                immutable=False):\n",
    "        '''perturb the sample based on thet qnet distributions and number of steps\n",
    "\n",
    "        Args:\n",
    "          sample (1d array-like): sample vector, must have the same num of features as the qnet\n",
    "          steps (int): number of steps to qsample\n",
    "          immutable (bool): are there variables that are immutable?\n",
    "        '''\n",
    "        if all(x is not None for x in [self.mutable_vars, sample]):\n",
    "            if immutable == True:\n",
    "                return qsample(sample,self.qnet,steps,self.getBaseFrequency(self.samples))\n",
    "            else:\n",
    "                return qsample(sample,self.qnet,steps)\n",
    "        elif self.mutable_vars is None:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "\n",
    "    def set_poles(self,\n",
    "                POLEFILE,\n",
    "                pole_1,\n",
    "                pole_2,\n",
    "                steps=0,\n",
    "                mutable=False):\n",
    "        '''set the poles and samples such that the samples contain features in poles\n",
    "\n",
    "        Args:\n",
    "          steps (int): number of steps to qsample\n",
    "          POLEFILE (str): file containing poles samples and features\n",
    "          mutable (boolean): Whether or not to set poles as the only mutable_vars\n",
    "          pole_1 (str): column name for first pole to use\n",
    "          pole_2 (str): column name for second pole to use\n",
    "        '''\n",
    "        invalid_count = 0\n",
    "        if all(x is not None for x in [self.samples, self.qnet]):\n",
    "            poles = pd.read_csv(POLEFILE, index_col=0)\n",
    "            self.poles=poles.transpose()\n",
    "            self.polar_features = pd.concat([self.poles, self.features], axis=0)\n",
    "            poles_dict = {}\n",
    "            for column in poles:\n",
    "                p_ = self.polar_features.loc[column][self.cols].fillna('').values.astype(str)[:]\n",
    "                poles_dict[column] = self.qsampling(p_,steps)\n",
    "            self.poles_dict = poles_dict\n",
    "            self.pL = self.poles_dict[pole_1]\n",
    "            self.pR = self.poles_dict[pole_2]\n",
    "            # self.pL = list(poles_dict.values())[0]\n",
    "            # self.pR = list(poles_dict.values())[1]\n",
    "            self.d0 = qdistance(self.pL, self.pR, self.qnet, self.qnet)\n",
    "            \n",
    "            cols = [x for x in self.poles.columns if x in self.samples.columns]\n",
    "            self.samples=self.samples[cols]\n",
    "        \n",
    "            for x in self.poles.columns:\n",
    "                if x not in self.samples.columns:\n",
    "                    invalid_count += 1\n",
    "                    self.samples[x]=np.nan\n",
    "\n",
    "            self.samples = pd.concat([self.samples,self.features], axis=0)\n",
    "            self.all_samples = self.samples\n",
    "            self.samples_as_strings = self.samples[self.cols].fillna('').values.astype(str)[:]\n",
    "            \n",
    "            if mutable:\n",
    "                self.mutable_vars=[x for x in self.cols if x in self.poles.columns]\n",
    "        elif self.samples is None:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "        \n",
    "        print(\"{} pole features not found in sample features\".format(invalid_count))\n",
    "\n",
    "    def distance(self,\n",
    "                sample1,\n",
    "                sample2,\n",
    "                nsteps1=0,\n",
    "                nsteps2=0):\n",
    "        \"\"\"qsamples each sample set num of steps, then takes qdistance\n",
    "\n",
    "        Args:\n",
    "          sample1 (list[str]): sample vector 1, must have the same num of features as the qnet\n",
    "          sample2 (list[str]): sample vector 2, must have the same num of features as the qnet\n",
    "          nsteps1 (int, optional): number of steps to qsample for sample1\n",
    "          nsteps2 (int, optional): number of steps to qsample for sample2\n",
    "\n",
    "        Returns:\n",
    "          float: qdistance\n",
    "        \"\"\"\n",
    "        if self.qnet is None:\n",
    "            raise ValueError(\"load qnet first!\")\n",
    "        bp1 = self.getBaseFrequency(sample1)\n",
    "        bp2 = self.getBaseFrequency(sample2)\n",
    "        sample1 = qsample(sample1, self.qnet, nsteps1)#, baseline_prob=bp1)\n",
    "        sample2 = qsample(sample2, self.qnet, nsteps2)#, baseline_prob=bp2)\n",
    "        return qdistance(sample1, sample2, self.qnet, self.qnet)\n",
    "    \n",
    "    def __distfunc(self, \n",
    "                x, \n",
    "                y):\n",
    "        '''Compute distance between two samples\n",
    "\n",
    "        Args:\n",
    "          x (list[str]): first sample\n",
    "          y (list[str]): second sample\n",
    "        '''\n",
    "        d=qdistance(x,y,self.qnet,self.qnet)\n",
    "        return d\n",
    "    \n",
    "    def polarDistance(self,\n",
    "                    i,\n",
    "                    return_dict=None):\n",
    "        \"\"\"return the distance from a sample to the poles\n",
    "\n",
    "        Args:\n",
    "            i (int): index of sample to take\n",
    "            return_dict (dict): dict used for multiple sample function\n",
    "\n",
    "        Returns:\n",
    "            [type]: [description]\n",
    "        \"\"\"\n",
    "        samples_as_strings = self.samples[self.cols].fillna('').values.astype(str)[:]\n",
    "        p = samples_as_strings[i]\n",
    "        distances = []\n",
    "        for index, row in self.polar_features[self.cols].iterrows():\n",
    "            row = row.fillna('').values.astype(str)[:]\n",
    "            distances.append(self.distance(p, np.array(row)))\n",
    "        if return_dict is not None:\n",
    "            return_dict[i] = distances\n",
    "        return distances\n",
    "            \n",
    "    def polarDistance_multiple(self,\n",
    "                            outfile):\n",
    "        \"\"\"return the distance from all samples to the poles\n",
    "\n",
    "        Args:\n",
    "          outfile (str): desired output filename and path\n",
    "        \"\"\"\n",
    "        if all(x is not None for x in [self.samples, self.cols,\n",
    "                                    self.polar_features]):\n",
    "            manager = mp.Manager()\n",
    "            return_dict = manager.dict()\n",
    "            processes = []\n",
    "            \n",
    "            for i in range(len(self.samples)):\n",
    "                p = mp.Process(target=self.polarDistance, args=(i, return_dict))\n",
    "                processes.append(p)\n",
    "\n",
    "            [x.start() for x in processes]\n",
    "            [x.join() for x in processes]\n",
    "\n",
    "            pole_names = []\n",
    "            for index, row in self.polar_features[self.cols].iterrows():\n",
    "                pole_names.append(index)\n",
    "            result=[x for x in return_dict.values()]\n",
    "            result=pd.DataFrame(result,columns=pole_names).to_csv(outfile)\n",
    "            \n",
    "        else:\n",
    "            raise ValueError(\"load data first!\")\n",
    "        return return_dict\n",
    "        \n",
    "    def distfunc_line(self,\n",
    "                    i,\n",
    "                    return_dict=None):\n",
    "        '''compute the dist for a row, or vector of samples\n",
    "\n",
    "        Args:\n",
    "          i (int): row\n",
    "        \n",
    "        Return:\n",
    "          numpy.ndarray(float)\n",
    "        '''\n",
    "        if all(x is not None for x in [self.samples, self.features]):\n",
    "            w = self.samples.index.size\n",
    "            line = np.zeros(w)\n",
    "            y = self.samples_as_strings[i]\n",
    "            for j in range(w):\n",
    "                # only compute half of the distance matrix\n",
    "                if j > i:\n",
    "                    x = self.samples_as_strings[j]\n",
    "                    line[j] = self.__distfunc(x, y)\n",
    "        else:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "        if return_dict is not None:\n",
    "            return_dict[i] = line\n",
    "        return line\n",
    "    \n",
    "    def distfunc_multiples(self,\n",
    "                        outfile):\n",
    "        \"\"\"compute distance matrix for all samples in the dataset\n",
    "\n",
    "        Args:\n",
    "          outfile (str): desired output filename and path\n",
    "        \"\"\"\n",
    "        if all(x is not None for x in [self.samples, self.features]):\n",
    "            manager = mp.Manager()\n",
    "            return_dict = manager.dict()\n",
    "            processes = []\n",
    "\n",
    "            for i in range(len(self.samples)):\n",
    "                p = mp.Process(target=self.distfunc_line, args=(i, return_dict))\n",
    "                processes.append(p)\n",
    "            \n",
    "            [x.start() for x in processes]\n",
    "            [x.join() for x in processes]\n",
    "            result=[x for x in return_dict.values()]\n",
    "            columns = [i for i in range(len(self.samples))]\n",
    "            result=pd.DataFrame(result,columns=columns, index=columns).sort_index(ascending=False)\n",
    "            result = result.to_numpy()\n",
    "            result = pd.DataFrame(np.maximum(result, result.transpose()))\n",
    "            result.to_csv(outfile)\n",
    "        else:\n",
    "            raise ValueError(\"load data first!\")\n",
    "        return return_dict\n",
    "    \n",
    "    def polar_separation(self,\n",
    "                        nsteps=0):\n",
    "        \"\"\"returns the distance between poles as a qdistance matrix\n",
    "\n",
    "        Args:\n",
    "          nsteps (int, optional): [description]. Defaults to 0.\n",
    "        \"\"\"\n",
    "        polar_arraydata = self.polar_features[self.cols].fillna('').values.astype(str)[:]\n",
    "        samples_ = []\n",
    "        for vector in polar_arraydata:\n",
    "            bp = self.getBaseFrequency(vector)\n",
    "            sample = qsample(vector, self.qnet, nsteps, baseline_prob=bp)\n",
    "            samples_.append(sample)\n",
    "        samples_ = np.array(samples_)\n",
    "        self.polar_matrix = qdistance_matrix(samples_, samples_, self.qnet, self.qnet)\n",
    "        return self.polar_matrix\n",
    "        \n",
    "    def embed(self,\n",
    "            infile,\n",
    "            name_pref,\n",
    "            out_dir,\n",
    "            pca_model=False,\n",
    "            EMBED_BINARY=None):\n",
    "        '''\n",
    "        embed data\n",
    "\n",
    "        Args:\n",
    "          infile (str): input file to be embedded\n",
    "          name_pref (str): preferred name for output file\n",
    "          out_dir (str): output dir for results\n",
    "          pca_model (bool): whether or not to generate PCA model\n",
    "          EMBED_BINARY (os.path.abspath): path to embed binary\n",
    "        '''\n",
    "        if all(x is not None for x in [self.year]):\n",
    "            yr = self.year\n",
    "            PREF = name_pref\n",
    "            FILE = infile\n",
    "\n",
    "            if EMBED_BINARY is None:\n",
    "                EMBED = pkgutil.get_data(\"cognet.bin\", \"__embed__.so\") \n",
    "            else:\n",
    "                EMBED = EMBED_BINARY\n",
    "            DATAFILE = out_dir + 'data_' +yr\n",
    "            EFILE = out_dir + PREF + '_E_' +yr\n",
    "            DFILE = out_dir + PREF + '_D_' +yr\n",
    "\n",
    "            pd.read_csv(FILE,header=None).to_csv(DATAFILE,sep=' ',header=None,index=None)\n",
    "            STR=EMBED+' -f '+DATAFILE+' -E '+EFILE+' -D '+DFILE\n",
    "            subprocess.call(STR,shell=True)\n",
    "            if pca_model:\n",
    "                embed_to_pca(EFILE, EFILE+'_PCA')\n",
    "        elif self.year is None:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "    \n",
    "    def __calc_d0(self,\n",
    "                pole_1,\n",
    "                pole_2):\n",
    "        \"\"\"calculate distance between two poles\n",
    "\n",
    "        Args:\n",
    "            pole_1 (list[str]): a polar vector, must have same number of features as qnet\n",
    "            pole_2 (list[str]): a polar vector, must have same number of features as qnet\n",
    "        \"\"\"\n",
    "        self.pL = self.poles_dict[pole_1]\n",
    "        self.pR = self.poles_dict[pole_2]\n",
    "        self.d0 = qdistance(self.pL, self.pR, self.qnet, self.qnet)\n",
    "        \n",
    "    def ideology(self,\n",
    "                i,\n",
    "                return_dict=None,\n",
    "                pole_1=None,\n",
    "                pole_2=None):\n",
    "        \"\"\"return ideology index (left-leaning or right-leaning) for a singular sample\n",
    "\n",
    "        Args:\n",
    "          i (int): index of sample\n",
    "          pole_1 (int): index of Pole One to calc as base distance. Defaults to 0.\n",
    "          pole_2 (int): index of Pole Two to calc as base distance. Defaults to 1.\n",
    "          return_dict (dict, optional): dict containing results\n",
    "        \"\"\"\n",
    "        if pole_1 is not None or pole_2 is not None:\n",
    "            self.__calc_d0(pole_1, pole_2)\n",
    "            \n",
    "        p = self.samples_as_strings[i]\n",
    "        dR = qdistance(self.pR, p, self.qnet, self.qnet)\n",
    "        dL = qdistance(self.pL, p, self.qnet, self.qnet)\n",
    "        ideology_index = (dR-dL)/self.d0\n",
    "        if return_dict is not None:\n",
    "            return_dict[i] = [ideology_index, dR, dL, self.d0]\n",
    "        return [ideology_index, dR, dL, self.d0]\n",
    "\n",
    "    def dispersion(self,\n",
    "                i,\n",
    "                return_dict=None):\n",
    "        \"\"\"qsamples a sample n times and takes distance matrix \n",
    "        to determine max and std of distances between qsamples\n",
    "\n",
    "        Args:\n",
    "            i (int): index of sample\n",
    "            return_dict (dict): dict containing results\n",
    "\n",
    "        Returns:\n",
    "            list[float]: std and max of the distances btwn qsamples\n",
    "        \"\"\"\n",
    "        p = self.samples_as_strings[i]\n",
    "        Qset = [qsample(p, self.qnet, self.steps) for j in np.arange(self.num_qsamples)]\n",
    "        Qset = np.array(Qset)\n",
    "\n",
    "        matrix = (qdistance_matrix(Qset, Qset, self.qnet, self.qnet))\n",
    "        Q = matrix.max()\n",
    "        Qsd = matrix.std()\n",
    "        if return_dict is not None:\n",
    "            return_dict[i] = [Qsd, Q]\n",
    "        return [Qsd, Q]\n",
    "    \n",
    "    def compute_DLI_samples(self,\n",
    "                        type,\n",
    "                        outfile,\n",
    "                        num_qsamples=40,\n",
    "                        steps=120,\n",
    "                        n_jobs=28,\n",
    "                        pole_1=0,\n",
    "                        pole_2=1):\n",
    "        \"\"\"compute and save ideology index or dispersion for all samples\n",
    "\n",
    "        Args:\n",
    "          num_qsamples (int): number of qsamples to compute\n",
    "          outfile (str): output file for results\n",
    "          type (str): whether to calc dispersion or ideology\n",
    "          steps (int): number of steps to qsample\n",
    "          n_jobs (int, optional): sets the number of jobs for parallelization. Defaults to 28.\n",
    "          pole_1 (int, optional): index of Pole One to calc as base distance. Defaults to 0.\n",
    "          pole_2 (int, optional): index of Pole Two to calc as base distance. Defaults to 1.\n",
    "\n",
    "        Raises:\n",
    "            ValueError: set poles if poles are not set\n",
    "            ValueError: load data if samples or features are not present\n",
    "        \"\"\"\n",
    "        if all(x is not None for x in [self.samples, self.features,\n",
    "                                    self.pL, self.pR]):\n",
    "            self.num_qsamples = num_qsamples\n",
    "            self.steps = steps\n",
    "            if pole_1 != 0 or pole_2 != 1:\n",
    "                self.__calc_d0(pole_1, pole_2)\n",
    "            \n",
    "            # testing\n",
    "            # pd.DataFrame(self.samples_as_strings).to_csv('examples_results/class_allsamples_2018.csv')\n",
    "            \n",
    "            manager = mp.Manager()\n",
    "            return_dict = manager.dict()\n",
    "            processes = []\n",
    "\n",
    "            if type == 'ideology':\n",
    "                for i in range(len(self.samples)):\n",
    "                    p = mp.Process(target=self.ideology, args=(i, return_dict))\n",
    "                    processes.append(p)\n",
    "                columns=['ideology', 'dR', 'dL', 'd0']\n",
    "            elif type == 'dispersion':\n",
    "                for i in range(len(self.samples)):\n",
    "                    p = mp.Process(target=self.dispersion, args=(i, return_dict))\n",
    "                    processes.append(p)\n",
    "                columns=['Qsd', 'Qmax']\n",
    "            else:\n",
    "                raise ValueError(\"Type must be either dispersion or ideology!\")\n",
    "            \n",
    "            [x.start() for x in processes]\n",
    "            [x.join() for x in processes]\n",
    "            result=[x for x in return_dict.values()]\n",
    "            result=pd.DataFrame(result,columns=columns).to_csv(outfile)\n",
    "\n",
    "        elif self.pL is None or self.pR is None:\n",
    "            raise ValueError(\"set_poles first!\")\n",
    "        else:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "        return result\n",
    "\n",
    "    def compute_polar_indices(self,\n",
    "                            num_samples = None,\n",
    "                            polar_comp = False,\n",
    "                            POLEFILE = None,\n",
    "                            steps = 5):\n",
    "        '''set up polar indices for dissonance func\n",
    "\n",
    "        Args:\n",
    "          num_samples (int): subset of samples to take\n",
    "          polar_comp (bool): whether or not to set poles\n",
    "          POLEFILE (None): file containing pole samples and features\n",
    "          steps (int): number of steps to qsample\n",
    "        '''\n",
    "        if all(x is not None for x in [self.samples, self.features, self.poles]):\n",
    "            if num_samples is not None:\n",
    "                self.set_nsamples(num_samples)\n",
    "\n",
    "            # read sample data\n",
    "            if polar_comp:\n",
    "                self.set_poles(self.qnet, steps, POLEFILE)\n",
    "            \n",
    "            polar_features = pd.concat([self.poles, self.features], axis=0)\n",
    "            self.polar_indices=np.where(polar_features[self.cols].fillna('XXXX').values[0]!='XXXX')[0]\n",
    "        \n",
    "        elif self.poles is None:\n",
    "            raise ValueError(\"set_poles first!\")\n",
    "        else:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "\n",
    "    def dissonance(self,\n",
    "                sample_index,\n",
    "                return_dict=None,\n",
    "                MISSING_VAL=0.0):\n",
    "        '''compute dissonance for each sample_index, helper function for all_dissonance\n",
    "        \n",
    "        Args:\n",
    "          sample_index (int): index of the sample to compute dissonance\n",
    "          return_dict (dict): dict containing results\n",
    "          MISSING_VAL (float): default dissonance value\n",
    "        '''\n",
    "        if all(x is not None for x in [self.samples, self.features, \n",
    "                                    self.poles]):\n",
    "            s = self.samples_as_strings[sample_index]\n",
    "            if self.polar_indices is None:\n",
    "                self.polar_indices = range(len(s))\n",
    "\n",
    "            Ds=self.qnet.predict_distributions(s)\n",
    "            \n",
    "            diss=np.ones(len(Ds))*MISSING_VAL\n",
    "            for i in self.polar_indices:\n",
    "                if s[i] != '':\n",
    "                    if s[i] in Ds[i].keys():\n",
    "                        diss[i]=1-Ds[i][s[i]]/np.max(\n",
    "                            list(Ds[i].values())) \n",
    "                    else:\n",
    "                        diss[i]=1.0\n",
    "            if return_dict is not None:\n",
    "                return_dict[sample_index] = diss[self.polar_indices]\n",
    "            return diss[self.polar_indices]\n",
    "\n",
    "        elif self.poles is None:\n",
    "            raise ValueError(\"set_poles first!\")\n",
    "        else:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "    \n",
    "    def dissonance_matrix(self,\n",
    "                        output_file=None,\n",
    "                        n_jobs=28):\n",
    "        '''get the dissonance for all samples\n",
    "\n",
    "        Args:\n",
    "          output_file (str): directory and/or file for output\n",
    "          n_jobs (int): number of jobs for pdqm\n",
    "        '''\n",
    "        if output_file is None:\n",
    "            output_file = '/example_results/DISSONANCE_matrix.csv'\n",
    "            \n",
    "        manager = mp.Manager()\n",
    "        return_dict = manager.dict()\n",
    "        processes = []\n",
    "        \n",
    "        for i in range(len(self.samples)):\n",
    "            p = mp.Process(target=self.dissonance, args=(i, return_dict))\n",
    "            processes.append(p)\n",
    "\n",
    "        [x.start() for x in processes]\n",
    "        [x.join() for x in processes]\n",
    "\n",
    "        result=[x for x in return_dict.values()]\n",
    "        if self.polar_indices is not None:\n",
    "            polar_features = pd.concat([self.poles, self.features], axis=0)\n",
    "            cols = polar_features[self.cols].dropna(axis=1).columns\n",
    "        else:\n",
    "            cols = self.cols\n",
    "        result=pd.DataFrame(result,columns=cols).to_csv(output_file)\n",
    "        \n",
    "        self.dissonance_file = output_file\n",
    "        return return_dict\n",
    "    \n",
    "    def __choose_one(self,\n",
    "                X):\n",
    "        '''returns a random element of X\n",
    "\n",
    "        Args:\n",
    "          X (1D array-like): vector from which random element is to be chosen\n",
    "        '''\n",
    "        X=list(X)\n",
    "        if len(X)>0:\n",
    "            return X[np.random.randint(len(X))]\n",
    "        return None\n",
    "\n",
    "    def getMaskedSample(self,\n",
    "                        s,\n",
    "                        mask_prob=0.5,\n",
    "                        allow_all_mutable=False):\n",
    "        '''inputs a sample and randomly mask elements of the sample\n",
    "\n",
    "        Args:\n",
    "          s (list[str]): vector of sample, must have the same num of features as the qnet\n",
    "          mask_prob (float): float btwn 0 and 1, prob to mask element of sample\n",
    "          allow_all_mutable (bool): whether or not all variables are mutable\n",
    "        '''\n",
    "        if self.samples is not None:   \n",
    "            MUTABLE=pd.DataFrame(np.zeros(len(self.cols)),index=self.cols).transpose()\n",
    "            WITHVAL=[x for x in self.cols[np.where(s)[0]] if x in self.mutable_vars ]\n",
    "            MASKrand=[x for x in WITHVAL if random.random() < mask_prob ]\n",
    "            for m in MASKrand:\n",
    "                MUTABLE[m]=1.0\n",
    "            \n",
    "            mutable_x=MUTABLE.values[0]\n",
    "            base_frequency=mutable_x/mutable_x.sum()\n",
    "\n",
    "            # if np.isnan(base_frequency).any():\n",
    "            #     return np.nan,np.nan,np.nan\n",
    "            #     return self.getMaskedSample(s)\n",
    "\n",
    "            s1=s.copy()\n",
    "            for i in range(len(base_frequency)):\n",
    "                if base_frequency[i]>0.0001:\n",
    "                    s1[i]=''\n",
    "                    \n",
    "            s_rand=np.copy(s)\n",
    "            rnd_match_prob=[]        \n",
    "            max_match_prob=[]        \n",
    "            D=self.qnet.predict_distributions(s)\n",
    "            for i in MASKrand:\n",
    "                s_rand[np.where(\n",
    "                    self.cols==i)[0][0]]=self.__choose_one(\n",
    "                        self.D_null[np.where(self.cols==i)[0][0]].keys())\n",
    "                rnd_match_prob=np.append(rnd_match_prob,1/len(\n",
    "                    self.D_null[np.where(self.cols==i)[0][0]].keys()))\n",
    "                max_match_prob=np.append(\n",
    "                    max_match_prob,np.max(\n",
    "                        list(D[np.where(\n",
    "                            self.cols==i)[0][0]].values())))\n",
    "                \n",
    "            if allow_all_mutable:\n",
    "                for m in mutable_vars:\n",
    "                    MUTABLE[m]=1.0\n",
    "                mutable_x=MUTABLE.values[0]\n",
    "                base_frequency=mutable_x/mutable_x.sum()\n",
    "\n",
    "            return s1,base_frequency,MASKrand,np.where(\n",
    "                base_frequency)[0],np.mean(rnd_match_prob),np.mean(max_match_prob),s_rand\n",
    "        else:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "\n",
    "    def randomMaskReconstruction(self,\n",
    "                                index=None,\n",
    "                                return_dict=None,\n",
    "                                sample=None):\n",
    "        \"\"\"reconstruct the masked sample by qsampling and comparing to original\n",
    "        set self.mask_prob and self.steps if needed\n",
    "\n",
    "        Args:\n",
    "          return_dict (dict): dict containing results. Defaults to None.\n",
    "          sample (list[str], optional): sample vector, must have the same num of features as the qnet. Defaults to None.\n",
    "          index (int): index of sample to take. Defaults to None.\n",
    "\n",
    "        Raises:\n",
    "          ValueError: Neither sample or index were given\n",
    "          ValueError: Both sample and index were given\n",
    "\n",
    "        Returns:\n",
    "          [type]: [description]\n",
    "        \"\"\"\n",
    "        if all(x is None for x in [sample, index]):\n",
    "            raise ValueError(\"Must input either sample or index!\")\n",
    "        elif all(x is not None for x in [sample, index]):\n",
    "            raise ValueError(\"Must input either sample or index not both!\")\n",
    "        elif sample is not None:\n",
    "            s=np.array(pd.DataFrame(sample).fillna('').values.astype(str)[:])\n",
    "        elif index is not None:\n",
    "            s=self.samples_as_strings[index]\n",
    "            \n",
    "        s1,bp,mask_,maskindex,rmatch_u,rmatch,s_rand=self.getMaskedSample(s, \n",
    "                                                                        mask_prob=self.mask_prob)\n",
    "        if np.isnan(bp).any():\n",
    "            return_dict[index] = np.nan,np.nan,np.nan\n",
    "            return np.nan,np.nan,np.nan\n",
    "\n",
    "        qs=qsample(s1,self.qnet,self.steps,bp)\n",
    "\n",
    "        dqestim=qdistance(s,qs,self.qnet,self.qnet)\n",
    "        dactual=qdistance(s,s1,self.qnet,self.qnet)\n",
    "        qdistance_time_end = time.time()\n",
    "\n",
    "        cmpf=pd.DataFrame([s,qs,s_rand],columns=self.cols,index=['s','q','r'])[mask_].transpose()\n",
    "        cmpf.index.name='gssvar'\n",
    "        cmpf.to_csv('examples_results/CMPF_2018/CMPF-'+str(index)+'.csv')\n",
    "        return_dict[index] = (1 - (dqestim/dactual))*100,rmatch_u,rmatch\n",
    "        return (1 - (dqestim/dactual))*100,rmatch_u,rmatch,s,qs,s_rand,mask_\n",
    "\n",
    "    def randomMaskReconstruction_multiple(self,\n",
    "                                          out_file):\n",
    "        '''runs and saves the results of the predicted masked sample\n",
    "\n",
    "        Args:\n",
    "          output_file (str): directory and/or file for output\n",
    "        '''\n",
    "        manager = mp.Manager()\n",
    "        return_dict = manager.dict()\n",
    "        processes = []\n",
    "        \n",
    "        for i in range(len(self.samples)):\n",
    "            p = mp.Process(target=self.randomMaskReconstruction, args=(i, return_dict))\n",
    "            processes.append(p)\n",
    "\n",
    "        [x.start() for x in processes]\n",
    "        [x.join() for x in processes]\n",
    "        \n",
    "        #result=pd.DataFrame(return_dict.items())[1]#, columns=['sample','rederr','r_prob','rand_err','s','q','r'])\n",
    "        result=[x for x in return_dict.values() if isinstance(x, tuple)]\n",
    "        # #result=pd.DataFrame(result.tolist())\n",
    "        # print(result)\n",
    "        # cmprdf=result[[3,4,5]]\n",
    "        # mask_=result[[6]]\n",
    "        # cmprdf.columns=['s','q','r']#[mask_].transpose()\n",
    "        # cmprdf.to_csv(\"examples_results/CMPF_\"+\"tmp\"+\".csv\")\n",
    "        # print(cmprdf)\n",
    "        # result=result[[0,1,2]]\n",
    "        result=pd.DataFrame(result,columns=['rederr','r_prob','rand_err'])\n",
    "        result.rederr=result.rederr.astype(float)\n",
    "\n",
    "        if self.poles is not None:\n",
    "            result.to_csv(out_file)\n",
    "        else:\n",
    "            result.to_csv(out_file)\n",
    "        \n",
    "        return result.rederr.mean(), result.rand_err.mean()\n",
    "    \n",
    "    def dmat_filewriter(self,\n",
    "                        pyfile,\n",
    "                        QNETPATH,\n",
    "                        MPI_SETUP_FILE=\"mpi_setup.sh\",\n",
    "                        MPI_RUN_FILE=\"mpi_run.sh\",\n",
    "                        YEARS='2016',\n",
    "                        NODES=4,\n",
    "                        T=12,\n",
    "                        num_samples=None,\n",
    "                        OUTFILE='tmp_distmatrix.csv'):\n",
    "        if all(x is not None for x in [self.poles_dict,self.features,\n",
    "                                       self.qnet, self.cols]):\n",
    "            if num_samples is not None:\n",
    "                self.set_nsamples(num_samples)\n",
    "            pd.DataFrame(self.samples_as_strings).to_csv(\"tmp_samples_as_strings.csv\", header=None, index=None)\n",
    "            w = self.samples.index.size\n",
    "            \n",
    "            with open(pyfile, 'w') as f:\n",
    "                f.writelines([\"from mpi4py.futures import MPIPoolExecutor\\n\",\n",
    "                              \"import numpy as np\\n\",\n",
    "                              \"import pandas as pd\\n\",\n",
    "                              \"from quasinet.qnet import Qnet, qdistance, load_qnet, qdistance_matrix\\n\",\n",
    "                              \"from quasinet.qsampling import qsample, targeted_qsample\\n\\n\",\n",
    "                              \"qnet=load_qnet(\\'{}\\')\\n\".format(QNETPATH)])\n",
    "\n",
    "                f.writelines([\"w = {}\\n\".format(w),\n",
    "                              \"h = w\\n\",\n",
    "                              \"p_all = pd.read_csv(\\\"tmp_samples_as_strings.csv\\\")\\n\\n\"])\n",
    "\n",
    "                f.writelines([\"def distfunc(x,y):\\n\",\n",
    "                              \"\\td=qdistance(x,y,qnet,qnet)\\n\",\n",
    "                              \"\\treturn d\\n\\n\"])\n",
    "\n",
    "                f.writelines([\"def dfunc_line(k):\\n\",\n",
    "                              \"\\tline = np.zeros(w)\\n\",\n",
    "                              \"\\ty = p_all[k]\\n\",\n",
    "                              \"\\tfor j in range(w):\\n\",\n",
    "                              \"\\t\\tif j > k:\\n\",\n",
    "                              \"\\t\\t\\tx = p_all[j]\\n\",\n",
    "                              \"\\t\\t\\tline[j] = distfunc(x, y)\\n\",\n",
    "                              \"\\treturn line\\n\\n\"])\n",
    "\n",
    "                f.writelines([\"if __name__ == '__main__':\\n\",\n",
    "                              \"\\twith MPIPoolExecutor() as executor:\\n\",\n",
    "                              \"\\t\\tresult = executor.map(dfunc_line, range(h))\\n\",\n",
    "                              \"\\t\\tpd.DataFrame(result).to_csv(\\'{}\\',index=None,header=None)\".format(OUTFILE)])\n",
    "\n",
    "            with open(MPI_SETUP_FILE, 'w') as ms:\n",
    "                ms.writelines([\"#!/bin/bash\\n\",\n",
    "                               \"YEAR=$1\\n\",\n",
    "                               \"\\tNODES=$2\\n\",\n",
    "                               \"else\\n\",\n",
    "                               \"\\tNODES=3\\n\",\n",
    "                               \"fi\\n\",\n",
    "                               \"if [ $# -gt 2 ] ; then\\n\",\n",
    "                               \"\\tNUM=$3\\n\",\n",
    "                               \"else\\n\",\n",
    "                               \"\\tNUM='all'\\n\",\n",
    "                               \"fi\\n\",\n",
    "                               \"if [ $# -gt 3 ] ; then\\n\",\n",
    "                               \"\\tPROG=$4\\n\",\n",
    "                               \"else\\n\",\n",
    "                               \"\\tPROG=$(tty)\\n\\n\\n\",\n",
    "                               \"fi\\n\",\n",
    "                               \"NUMPROC=`expr 28 \\* $NODES`\\n\",\n",
    "                               \"echo \\\"module load midway2\\\" >> $PROG\\n\",\n",
    "                               \"echo \\\"module unload python\\\" >> $PROG\\n\",\n",
    "                               \"echo \\\"module unload openmpi\\\" >> $PROG\\n\",\n",
    "                               \"echo \\\"module load python/anaconda-2020.02\\\" >> $PROG\\n\",\n",
    "                               \"echo \\\"module load mpi4py\\\" >> $PROG\\n\",\n",
    "                               \"echo \\\"date; mpiexec -n \\\"$NUMPROC\\\" python3 -Xy=\\\"$YEAR\\\" -XN=\\\"$NUM\\\" -m mpi4py.futures {}; date\\\"  >> $PROG\\n\".format(pyfile),\n",
    "                                ])\n",
    "\n",
    "            with open(MPI_RUN_FILE, 'w') as mr:\n",
    "                mr.writelines([\"#!/bin/bash\\n\",\n",
    "                               \"YEARS=\\'{}\\'\\n\".format(YEARS),\n",
    "                               \"# nodes requested\\n\",\n",
    "                               \"NODES={}\\n\".format(NODES),\n",
    "                               \"# time requested\\n\",\n",
    "                               \"T={}\\n\".format(T),\n",
    "                               \"LAUNCH=\\'../../../../LAUNCH_UTILITY/launcher_s.sh\\'\\n\\n\",\n",
    "                               \"for yr in `echo $YEARS`\\n\",\n",
    "                               \"do\\n\",\n",
    "                               \"\\techo $yr\\n\",\n",
    "                               \"\\t./{} $yr $NODES $NUM tmp_\\\"$yr\\\"\\n\".format(MPI_SETUP_FILE),\n",
    "                               \"\\t$LAUNCH -P tmp_\\\"$yr\\\" -F -T $T -N \\\"$NODES\\\" -C 28 -p broadwl -J ../examples_results/ACRDALL_\\\"$yr\\\" -M 56\\n\",\n",
    "                               \"done\\n\",\n",
    "                               \"rm tmp*\\n\"])\n",
    "        else:\n",
    "            raise ValueError(\"load data first!\")\n",
    "        print(\"running\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running\n"
     ]
    }
   ],
   "source": [
    "Cg = cognet()\n",
    "Cg.load_from_model(model_, samples_file='examples_data/gss_2018.csv')\n",
    "Cg.dmat_filewriter('tmp_dmat.py',num_samples=10,QNETPATH=str(QPATH))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pkg' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-edbd9f7e2f4f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m## embedding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mCg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0myear\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'2018'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mCg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'examples_data/DMAT_2018.csv'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'embed'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'examples_results/'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpca_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;31m#pd.read_csv('examples_results/embed_E_2018.csv')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.7/site-packages/cognet/cognet.py\u001b[0m in \u001b[0;36membed\u001b[0;34m(self, infile, name_pref, out_dir, pca_model, EMBED_BINARY)\u001b[0m\n\u001b[1;32m    453\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    454\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mEMBED_BINARY\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 455\u001b[0;31m                 \u001b[0mEMBED\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpkg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"cognet.bin\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"__embed__.so\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    456\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    457\u001b[0m                 \u001b[0mEMBED\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mEMBED_BINARY\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'pkg' is not defined"
     ]
    }
   ],
   "source": [
    "## embedding\n",
    "Cg.year = '2018'\n",
    "Cg.embed('examples_data/DMAT_2018.csv', 'embed', 'examples_results/', pca_model=True)\n",
    "#pd.read_csv('examples_results/embed_E_2018.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## generate pole separation across years\n",
    "#creed2_/GSS/Qnets\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import collections\n",
    "import seaborn as sns \n",
    "\n",
    "# get years \n",
    "features_by_year = pd.read_csv('examples_data/features_by_year_GSS.csv',\n",
    "                               keep_default_na=True, \n",
    "                               index_col=False)\n",
    "qnets_by_year = {x[-11:-7]: x for x in os.listdir('../../creed2_/GSS/Qnets/')}\n",
    "data_by_year = {x[4:8]: x for x in os.listdir('examples_data/processed_data')}\n",
    "polarDist_by_year = {}\n",
    "for year, qnet in qnets_by_year.items():\n",
    "    qnet = '../../creed2_/GSS/Qnets/' + qnet\n",
    "    model_ = model()\n",
    "    model_.load(qnet)\n",
    "    Cg = cognet()\n",
    "    Cg.load_from_model(model_, samples_file='examples_data/processed_data/gss_2018.csv')\n",
    "    # produce stats on how many column names actually match\n",
    "    stats = Cg.set_poles(POLEFILE, steps=120)\n",
    "\n",
    "    # compute polar distance matrix\n",
    "    dmatrix = Cg.polar_separation(nsteps=0)\n",
    "    polarDist_by_year[year]=dmatrix[0][1]\n",
    "polarDist_by_year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_dist = collections.OrderedDict(sorted(polarDist_by_year.items()))\n",
    "df = pd.DataFrame.from_dict(sorted_dist, orient='index')\n",
    "df.reset_index(level=0, inplace=True)\n",
    "df.columns = ['year', 'dist']\n",
    "\n",
    "sns.set(style='darkgrid', rc={'figure.figsize':(20, 8)})\n",
    "sns.lineplot(x='year', y='dist', data=df)\n",
    "    \n",
    "# year = list(sorted_dist.keys())\n",
    "# dist = list(sorted_dist.values())     \n",
    "# plt.plot(year, dist)\n",
    "# plt.show"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
