{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is an example of how we see \n",
    "# the package work. The functions listed here\n",
    "# are probably the only ones that should be exposed, ie documented.\n",
    "# others should br prepended with a double underscore\n",
    "#  \n",
    "# The cognet directory has the following \"modules\"\n",
    "# which are seprate .py files containing clases and functions\n",
    "# The modules are cognet.py, dataFormatter.py, model.py, util.py, viz.py\n",
    "# we will write the viz.py later.\n",
    "import sys\n",
    "\n",
    "from quasinet.qnet import qdistance\n",
    "from cognet.cognet import cognet as cg\n",
    "from cognet.dataFormatter import dataFormatter\n",
    "from cognet.model import model \n",
    "#import cognet.util\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "yr = '2018'\n",
    "POLEFILE='GSS/data/polar_vectors.csv'\n",
    "QPATH='GSS/data/gss_'+yr+'.joblib'\n",
    "IMMUTABLE_FILE='GSS/data/immutable.csv'\n",
    "GSSDATA = 'GSS/data/gss_'+yr+'.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            wrkstat HRS1 HRS2 evwork        wrkslf  wrkgovt OCC10 PRESTG10  \\\n",
      "0  temp not working    e    c    NaN  someone else  private     b        c   \n",
      "1  working fulltime    c    e    NaN  someone else  private     b        d   \n",
      "\n",
      "  PRESTG105PLUS INDUS10  ...    neisafe rlooks rgroomed rweight rhlthend wtss  \\\n",
      "0             c       c  ...  very safe    NaN      NaN     NaN      NaN    e   \n",
      "1             d       c  ...  very safe    NaN      NaN     NaN      NaN    c   \n",
      "\n",
      "  wtssnr wtssall vstrat vpsu  \n",
      "0      e       e   3301    1  \n",
      "1      c       c   3301    1  \n",
      "\n",
      "[2 rows x 1034 columns]\n",
      "(892, 1034)\n"
     ]
    }
   ],
   "source": [
    "# testing dataFormatter\n",
    "data = dataFormatter(samples=GSSDATA)\n",
    "# load the sample data\n",
    "# have option for test/train split\n",
    "# make checks to ensure we will not throw errors at qnet construction \n",
    "print(data.samples[:2])\n",
    "features,samples = data.format_samples('train') # default trains and tests using half\n",
    "all_samples = False\n",
    "if all_samples: # use all samples to train, instead of half\n",
    "    features,samples = data.Qnet_formatter()\n",
    "\n",
    "# format data for Qnet training and fitting\n",
    "print(samples.shape)\n",
    "\n",
    "# set mutable and immutable vars either from list or file\n",
    "im_vars_df = pd.read_csv(IMMUTABLE_FILE, names=['vars'])\n",
    "im_vars_list = im_vars_df.vars.to_list()\n",
    "mutable_vars, immutable_vars = data.mutable_variables(immutable_list=im_vars_list)\n",
    "mutable_vars, immutable_vars = data.mutable_variables(IMMUTABLE_FILE=IMMUTABLE_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# testing model functionality\n",
    "# can either input features and samples directly, or infer from data obj\n",
    "model_ = model()\n",
    "\n",
    "# qnet construction parameters, \n",
    "# choose to either load or fit qnet from scratch\n",
    "# and to either load from url or local repo\n",
    "test_model_buildqnet = False\n",
    "url_load = True\n",
    "if test_model_buildqnet:\n",
    "        print(\"fitting\")\n",
    "        model_.fit(data_obj=data,\n",
    "                   min_samples_split=2,\n",
    "                   alpha=0.05,\n",
    "                   max_depth=-1,\n",
    "                   max_feats=-1,\n",
    "                   early_stopping=False,\n",
    "                   verbose=0,\n",
    "                   random_state=None,\n",
    "                   njobs=8)\n",
    "        print(\"fitted\")\n",
    "        model_.export_dot(\"GSS/results/tmp_dot_modelclass.dot\",\n",
    "                        generate_trees=True)\n",
    "        model_.save(\"GSS/results/tmp_nodelclass.joblib\")\n",
    "        #model_.load(\"tmp_nodelclass.joblib\")\n",
    "else:\n",
    "    if url_load:\n",
    "        QNETFILE = 'https://zenodo.org/record/5781768/files/gss_2018.joblib'\n",
    "    else:\n",
    "        QNETFILE = 'GSS/data/gss_2018.joblib'\n",
    "    model_.load(QNETFILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from quasinet.qnet import Qnet, qdistance, load_qnet, qdistance_matrix\n",
    "from quasinet.qsampling import qsample, targeted_qsample\n",
    "#from mpi4py.futures import MPIPoolExecutor\n",
    "import sys\n",
    "import subprocess\n",
    "from scipy.stats import entropy\n",
    "import multiprocessing as mp\n",
    "import time\n",
    "from cognet.util import embed_to_pca\n",
    "import pkgutil\n",
    "import os\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "\n",
    "class cognet:\n",
    "    \"\"\"Aggregate related Qnet functions\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        \"\"\"Init\n",
    "        \"\"\"\n",
    "        self.year = None\n",
    "        self.n_jobs = 28\n",
    "        self.qnet = None\n",
    "        self.steps = 120\n",
    "        self.num_qsamples = None\n",
    "        self.all_samples = None\n",
    "        self.samples = None\n",
    "        self.samples_as_strings = None\n",
    "        self.features = None\n",
    "        self.cols = None\n",
    "        self.immutable_vars = None\n",
    "        self.mutable_vars = None\n",
    "        self.poles = None\n",
    "        self.polar_features = None\n",
    "        self.polar_indices = None\n",
    "        self.poles_dict = {}\n",
    "        self.d0 = None\n",
    "        self.s_null = None\n",
    "        self.D_null = None\n",
    "        self.mask_prob = 0.5\n",
    "        self.variation_weight = None\n",
    "        self.polar_matrix = None\n",
    "        self.nsamples = None\n",
    "        self.restricted = False\n",
    "    \n",
    "    def load_from_model(self,\n",
    "                        model,\n",
    "                        data_obj,\n",
    "                        key,\n",
    "                        im_vars=None,\n",
    "                        m_vars=None):\n",
    "        \"\"\"load parameters from model object\n",
    "\n",
    "        Args:\n",
    "          model (Class): model obj for loading parameters\n",
    "          data_obj (class): instance of dataformatter class\n",
    "          key (str): 'all', 'train', or 'test', corresponding to sample type\n",
    "          im_vars (list[str], optional): Not implemented yet. Defaults to None.\n",
    "          m_vars (list[str], optional): Not implemented yet. Defaults to None.\n",
    "        \"\"\"\n",
    "        if model is not None:\n",
    "            # inherit atrributes from model object\n",
    "            self.qnet = model.myQnet\n",
    "            featurenames, samples = data_obj.format_samples(key)\n",
    "            samples = pd.DataFrame(samples)\n",
    "            self.cols = np.array(featurenames)\n",
    "            self.features = pd.DataFrame(columns=np.array(featurenames))\n",
    "            \n",
    "            # inherit mutable and immutable variables from model obj\n",
    "            if any(x is not None for x in [model.immutable_vars, model.mutable_vars]):\n",
    "                if model.immutable_vars is not None:\n",
    "                    self.immutable_vars = model.immutable_vars\n",
    "                    self.mutable_vars = [x for x in self.features if x not in self.immutable_vars]\n",
    "                elif model.mutable_vars is not None:\n",
    "                    self.mutable_vars = model.mutable_vars\n",
    "                    self.immutable_vars = [x for x in self.features if x not in self.mutable_vars]\n",
    "            else:\n",
    "                self.mutable_vars = self.features\n",
    "            \n",
    "            # inherit and set class attributes.\n",
    "            self.samples = pd.DataFrame(samples).replace(\"nan\",\"\").fillna(\"\")\n",
    "            self.samples.columns = np.array(featurenames)\n",
    "            self.all_samples = self.samples\n",
    "            self.samples_as_strings = self.samples.fillna('').values.astype(str)[:]\n",
    "            self.s_null=['']*len(self.samples_as_strings[0])\n",
    "            self.D_null=self.qnet.predict_distributions(self.s_null)\n",
    "            variation_weight = []\n",
    "            for d in self.D_null:\n",
    "                v=[]\n",
    "                for val in d.values():\n",
    "                    v=np.append(v,val)\n",
    "                variation_weight.append(entropy(v,base=len(v)))\n",
    "            variation_weight = np.nan_to_num(variation_weight) # remove nans\n",
    "            self.variation_weight = variation_weight\n",
    "    \n",
    "    def load_from_dataformatter(self, \n",
    "                                data_obj,\n",
    "                                key):\n",
    "        \"\"\"read in either train or test data, specified by key, from data obj,\n",
    "        and inherit other attributes.\n",
    "\n",
    "        Args:\n",
    "          data_obj (class): instance of dataformatter class\n",
    "          key (str): 'all', 'train', or 'test', corresponding to sample type\n",
    "          \n",
    "        Returns:\n",
    "          featurenames, samples: formatted arrays\n",
    "        \"\"\"\n",
    "        # inherit attributes from dataformatter object\n",
    "        featurenames, samples = data_obj.format_samples(key)\n",
    "        if any(x is not None for x in [self.features, self.samples]):\n",
    "            print(\"replacing original features/samples with dataformatter data\")\n",
    "        self.cols = featurenames\n",
    "        self.features = pd.DataFrame(columns=self.cols)\n",
    "        self.samples = pd.DataFrame(samples,columns=self.features)\n",
    "        self.all_samples = self.samples\n",
    "        self.samples_as_strings = self.samples[self.cols].fillna('').values.astype(str)[:]\n",
    "        self.s_null=['']*len(self.samples_as_strings[0])\n",
    "        return featurenames, samples\n",
    "\n",
    "    def load_data(self,\n",
    "                  year,\n",
    "                  features_by_year,\n",
    "                  samples,\n",
    "                  Qnet):\n",
    "        '''load cols, features, samples, and qnet.\n",
    "\n",
    "        Args:\n",
    "          year (str): to identify cols/features.\n",
    "          features_by_year (str): file containing all features by year of the dataset.\n",
    "          samples (str): file of samples for that year.\n",
    "          Qnet (str): Qnet file location.\n",
    "        '''\n",
    "        # set attributes from given files and data\n",
    "        self.qnet = load_qnet(qnet)\n",
    "        self.year = year\n",
    "        self.cols = np.array((pd.read_csv(features_by_year,\n",
    "                            keep_default_na=True, \n",
    "                            index_col=0).set_index(\n",
    "                                'year')).loc[int(year)].apply(\n",
    "                                    eval).values[0])\n",
    "        self.features = pd.DataFrame(columns=self.cols)\n",
    "        self.mutable_vars = [x for x in self.cols]\n",
    "        #[self.cols].fillna('').values.astype(str)[:]\n",
    "\n",
    "        # read in samples and initialize related attributes\n",
    "        self.samples=pd.read_csv(samples)\n",
    "        self.samples = pd.concat([self.samples,self.features], axis=0)\n",
    "        self.all_samples = self.samples\n",
    "        self.samples_as_strings = self.samples[self.cols].fillna('').values.astype(str)[:]\n",
    "        self.s_null=['']*len(self.samples_as_strings[0])\n",
    "        self.D_null=self.qnet.predict_distributions(self.s_null)\n",
    "        variation_weight = []\n",
    "        for d in self.D_null:\n",
    "            v=[]\n",
    "            for val in d.values():\n",
    "                v=np.append(v,val)\n",
    "            variation_weight.append(entropy(v,base=len(v)))\n",
    "        self.variation_weight = variation_weight\n",
    "\n",
    "    def set_immutable_vars(self,\n",
    "                        IMMUTABLE_FILE):\n",
    "        '''set vars to immutable and mutable, \n",
    "        can prob combine this with the load_data func: only set the immutable vars if necessary\n",
    "\n",
    "        Args:\n",
    "          IMMUTABLE_FILE (str): file containing the immutable features/vars\n",
    "        '''\n",
    "        # set mutable and immutable variable attributes \n",
    "        if self.cols is None:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "        self.immutable_vars = pd.read_csv(IMMUTABLE_FILE,index_col=0).transpose()\n",
    "        self.mutable_vars = None\n",
    "        self.mutable_vars = [x for x in self.cols\n",
    "                            if x.upper() not in self.immutable_vars.columns]\n",
    "    \n",
    "    def set_nsamples(self,\n",
    "                    num_samples,\n",
    "                    random=False):\n",
    "        '''select a subset of the samples\n",
    "\n",
    "        Args:\n",
    "          num_samples (int): Set num of samples to subset, default to None, resets to all samples\n",
    "          random (bool): take random sample if true, ordered sample if false\n",
    "        '''\n",
    "        # each time function is called, reset samples to use_all_samples\n",
    "        # this allows us to call nsamples numerous times \n",
    "        self.samples = self.all_samples\n",
    "        if self.samples is not None:\n",
    "            # if a greater number of sample is selected than available, raise error\n",
    "            if all(x is not None for x in [num_samples, self.samples]):\n",
    "                if num_samples > len(self.samples.index):\n",
    "                    string = 'The number of selected samples ({}) ' + \\\n",
    "                        'is greater than the number of samples ({})!'\n",
    "                    string = string.format(num_samples, len(self.samples.index))\n",
    "                    raise ValueError(string)\n",
    "\n",
    "                # if the same number of samples is selected as available, print warning\n",
    "                if num_samples == len(self.samples.index):\n",
    "                    string = 'The number of selected samples ({}) ' + \\\n",
    "                        'is equal to the number of samples ({})!'\n",
    "                    string = string.format(num_samples, len(self.samples.index))\n",
    "                    print(string)\n",
    "                    \n",
    "                # if random is true, return random sample, otherwise return an ordered slice\n",
    "                if random:\n",
    "                    self.samples = self.samples.sample(num_samples)\n",
    "                else:\n",
    "                    self.samples = self.samples.iloc[:num_samples]\n",
    "                self.nsamples = num_samples\n",
    "                self.samples_as_strings = self.samples[self.cols].fillna('').values.astype(str)[:]\n",
    "                \n",
    "            elif self.samples is None:\n",
    "                raise ValueError(\"load_data first!\")\n",
    "\n",
    "    def __variation_weight(self,\n",
    "                        index):\n",
    "        \"\"\"\n",
    "        \"\"\"\n",
    "        d_=self.D_null[index]\n",
    "        v=[]\n",
    "        for val in d_.values():\n",
    "            v=np.append(v,val)\n",
    "        return entropy(v,base=len(v))\n",
    "    \n",
    "    def getBaseFrequency(self, \n",
    "                        sample):\n",
    "        '''get frequency of the variables\n",
    "        helper func for qsampling\n",
    "\n",
    "        Args:\n",
    "          sample (list[str]): vector of sample, must have the same num of features as the qnet\n",
    "        '''\n",
    "        # if variable is not mutable, set its base frequency to zero \n",
    "        MUTABLE=pd.DataFrame(np.zeros(len(self.cols)),index=self.cols).transpose()\n",
    "             \n",
    "        for m in self.mutable_vars:\n",
    "            MUTABLE[m]=1.0\n",
    "        mutable_x=MUTABLE.values[0]\n",
    "        base_frequency=mutable_x/mutable_x.sum()\n",
    "        \n",
    "        # otherwise, set base frequency weighted by variation weight\n",
    "        for i in range(len(base_frequency)):\n",
    "            if base_frequency[i]>0.0:\n",
    "                base_frequency[i]= self.variation_weight[i]*base_frequency[i]\n",
    "\n",
    "        return base_frequency/base_frequency.sum()\n",
    "    \n",
    "    def qsampling(self,\n",
    "                sample,\n",
    "                steps,\n",
    "                immutable=False):\n",
    "        '''perturb the sample based on thet qnet distributions and number of steps\n",
    "\n",
    "        Args:\n",
    "          sample (1d array-like): sample vector, must have the same num of features as the qnet\n",
    "          steps (int): number of steps to qsample\n",
    "          immutable (bool): are there variables that are immutable?\n",
    "        '''\n",
    "        # immutable, check that mutable variables have been initialized\n",
    "        if immutable == True:\n",
    "            if all(x is not None for x in [self.mutable_vars, sample]):\n",
    "                return qsample(sample,self.qnet,steps,self.getBaseFrequency(self.samples))\n",
    "            elif self.mutable_vars is None:\n",
    "                raise ValueError(\"set mutable and immutable variables first!\")\n",
    "        else:\n",
    "            return qsample(sample,self.qnet,steps)\n",
    "\n",
    "    def set_poles(self,\n",
    "                  POLEFILE,\n",
    "                  pole_1,\n",
    "                  pole_2,\n",
    "                  steps=0,\n",
    "                  mutable=False,\n",
    "                  VERBOSE=False,\n",
    "                  restrict=True,\n",
    "                  nsamples = False,\n",
    "                  random=False):\n",
    "        '''set the poles and samples such that the samples contain features in poles\n",
    "\n",
    "        Args:\n",
    "          steps (int): number of steps to qsample\n",
    "          POLEFILE (str): file containing poles samples and features\n",
    "          pole_1 (str): column name for first pole\n",
    "          pole_2 (str): column name for second pole\n",
    "          mutable (bool): Whether or not to set poles as the only mutable_vars\n",
    "          VERBOSE (bool): boolean flag prints number of pole features not found in sample features if True\n",
    "          restrict (bool): boolean flag restricts the sample features to polar features if True\n",
    "          random (bool): boolean flag takes random sample of all_samples\n",
    "        '''\n",
    "        invalid_count = 0\n",
    "        if all(x is not None for x in [self.samples, self.qnet]):\n",
    "            # read and set poles\n",
    "            poles = pd.read_csv(POLEFILE, index_col=0)\n",
    "            self.poles=poles.transpose()\n",
    "            self.polar_features = pd.concat([self.features, self.poles], axis=0).fillna('')\n",
    "            poles_dict = {}\n",
    "            for column in poles:\n",
    "                p_ = self.polar_features.loc[column][self.cols].fillna('').values.astype(str)[:]\n",
    "                # qsample poles to qnet\n",
    "                poles_dict[column] = self.qsampling(p_,steps)\n",
    "            self.poles_dict = poles_dict\n",
    "            self.pL = self.poles_dict[pole_1]\n",
    "            self.pR = self.poles_dict[pole_2]\n",
    "            self.d0 = qdistance(self.pL, self.pR, self.qnet, self.qnet)\n",
    "            \n",
    "            # restrict sample columns to polar columns\n",
    "            if restrict:\n",
    "                cols = [x for x in self.poles.columns if x in self.samples.columns]\n",
    "                self.samples=self.samples[cols]\n",
    "                self.restricted = True\n",
    "              \n",
    "            # if poles had been restricted before, unrestrict it and set original\n",
    "            elif self.restricted:\n",
    "                self.restricted = False\n",
    "                self.samples = self.all_samples\n",
    "                if self.nsamples is not None:\n",
    "                    self.set_nsamples(nsamples, random)\n",
    "            \n",
    "            # identify pole features that were excluded due to sample features restriction\n",
    "            if VERBOSE:\n",
    "                for x in self.poles.columns:\n",
    "                    if x not in self.samples.columns:\n",
    "                        invalid_count += 1\n",
    "                        self.samples[x]=''\n",
    "\n",
    "            self.samples = pd.concat([self.features,self.samples], axis=0).fillna('')\n",
    "            self.samples_as_strings = self.samples[self.cols].fillna('').values.astype(str)[:]\n",
    "            \n",
    "            if mutable:\n",
    "                self.mutable_vars=[x for x in self.cols if x in self.poles.columns]\n",
    "        elif self.samples is None:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "\n",
    "        if VERBOSE:\n",
    "            print(\"{} pole features not found in sample features\".format(invalid_count))\n",
    "\n",
    "    def __mpcollector(self, \n",
    "                      processes,\n",
    "                      func, \n",
    "                      cols,\n",
    "                      outfile, \n",
    "                      args=[]):\n",
    "        \"\"\"[summary]\n",
    "\n",
    "        Args:\n",
    "          processes ([type]): [description]\n",
    "          func ([type]): [description]\n",
    "          args (list): list containing arguments for desired function\n",
    "          cols ([type]): [description]\n",
    "          outfile ([type]): [description]\n",
    "\n",
    "        Raises:\n",
    "            ValueError: [description]\n",
    "        \"\"\"\n",
    "\n",
    "        # init mp.Manager and result dict\n",
    "        manager = mp.Manager()\n",
    "        return_dict = manager.dict()\n",
    "\n",
    "        # init mp.Processes for each individual sample\n",
    "        # run once collected processes hit max\n",
    "        if func == \"polar_distance\":\n",
    "            target = self.polar_distance\n",
    "        elif func == \"distance\":\n",
    "            target = self.distfunc_line\n",
    "        elif func == \"ideology\":\n",
    "            target = self.ideology\n",
    "        elif func == \"dispersion\":\n",
    "            target = self.dispersion\n",
    "        elif func == \"dissonance\":\n",
    "            target = self.dissonance\n",
    "        elif func == \"recon\":\n",
    "            target = self.randomMaskReconstruction\n",
    "        \n",
    "        print(args)\n",
    "        max_processes = processes\n",
    "        num_processes = 0\n",
    "        process_list = []\n",
    "        for i in range(len(self.samples)):\n",
    "            params = tuple([i, return_dict] + args)\n",
    "            num_processes += 1\n",
    "            p = mp.Process(target=func,\n",
    "                        args=params)\n",
    "            process_list.append(p)\n",
    "            if num_processes == max_processes:\n",
    "                [x.start() for x in process_list]\n",
    "                [x.join() for x in process_list]\n",
    "                process_list = []\n",
    "                num_processes = 0\n",
    "                \n",
    "        # compute remaining processes\n",
    "        if num_processes != 0:\n",
    "            [x.start() for x in process_list]\n",
    "            [x.join() for x in process_list]\n",
    "            process_list = []\n",
    "            num_processes = 0\n",
    "        \n",
    "        # format and save resulting dict\n",
    "        result=[x for x in return_dict.values()]\n",
    "        pd.DataFrame(result,columns=cols).to_csv(outfile)\n",
    "        return result\n",
    "    \n",
    "    def distance(self,\n",
    "                sample1,\n",
    "                sample2,\n",
    "                nsteps1=0,\n",
    "                nsteps2=0):\n",
    "        \"\"\"qsamples each sample set num of steps, then takes qdistance\n",
    "\n",
    "        Args:\n",
    "          sample1 (list[str]): sample vector 1, must have the same num of features as the qnet\n",
    "          sample2 (list[str]): sample vector 2, must have the same num of features as the qnet\n",
    "          nsteps1 (int, optional): number of steps to qsample for sample1\n",
    "          nsteps2 (int, optional): number of steps to qsample for sample2\n",
    "\n",
    "        Returns:\n",
    "          qdistance: float, distance between two samples\n",
    "        \"\"\"\n",
    "        if self.qnet is None:\n",
    "            raise ValueError(\"load qnet first!\")\n",
    "        #bp1 = self.getBaseFrequency(sample1)\n",
    "        #bp2 = self.getBaseFrequency(sample2)\n",
    "        # qsample samples\n",
    "        sample1 = qsample(sample1, self.qnet, nsteps1)#, baseline_prob=bp1)\n",
    "        sample2 = qsample(sample2, self.qnet, nsteps2)#, baseline_prob=bp2)\n",
    "        return qdistance(sample1, sample2, self.qnet, self.qnet)\n",
    "    \n",
    "    def __distfunc(self, \n",
    "                x, \n",
    "                y):\n",
    "        '''Compute distance between two samples\n",
    "\n",
    "        Args:\n",
    "          x (list[str]): first sample\n",
    "          y (list[str]): second sample\n",
    "          \n",
    "        Returns:\n",
    "         d: qdistance\n",
    "        '''\n",
    "        d=qdistance(x,y,self.qnet,self.qnet)\n",
    "        return d\n",
    "    \n",
    "    def distfunc_line(self,\n",
    "                    i,\n",
    "                    return_dict=None):\n",
    "        '''compute the distance for a single sample from all other samples\n",
    "\n",
    "        Args:\n",
    "          i (int): row\n",
    "          return_dict (dict): dictionary containing multiprocessing results\n",
    "        \n",
    "        Return:\n",
    "          line: float, numpy.ndarray\n",
    "        '''\n",
    "        if all(x is not None for x in [self.samples, self.features]):\n",
    "            w = self.samples.index.size\n",
    "            line = np.zeros(w)\n",
    "            y = self.samples_as_strings[i]\n",
    "            for j in range(w):\n",
    "                # only compute half of the distance matrix\n",
    "                if j > i:\n",
    "                    x = self.samples_as_strings[j]\n",
    "                    line[j] = self.__distfunc(x, y)\n",
    "        else:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "        if return_dict is not None:\n",
    "            return_dict[i] = line\n",
    "        return line\n",
    "    \n",
    "    def distfunc_multiples(self,\n",
    "                           outfile,\n",
    "                           processes=6):\n",
    "        \"\"\"compute distance matrix for all samples in the dataset\n",
    "\n",
    "        Args:\n",
    "          outfile (str): desired output filename and path\n",
    "          \n",
    "        Returns:\n",
    "          return_dict: dictionary containing multiprocessing results\n",
    "        \"\"\"\n",
    "        if all(x is not None for x in [self.samples, self.features]):\n",
    "            cols = [i for i in range(len(self.samples))]\n",
    "            result = self.__mpcollector(processes,\n",
    "                                        self.distfunc_line,\n",
    "                                        cols,\n",
    "                                        outfile)\n",
    "            \"\"\"\n",
    "            # init mp.Manager and result dict\n",
    "            manager = mp.Manager()\n",
    "            return_dict = manager.dict()\n",
    "            processes = []\n",
    "\n",
    "            # init and start mp.Processes for individual samples\n",
    "            for i in range(len(self.samples)):\n",
    "                p = mp.Process(target=self.distfunc_line, args=(i, return_dict))\n",
    "                processes.append(p)\n",
    "            [x.start() for x in processes]\n",
    "            [x.join() for x in processes]\n",
    "            \"\"\"\n",
    "            # format and save resulting dict, and tranpose symmetrical distance matrix\n",
    "            result = pd.DataFrame(result,columns=cols, index=cols).sort_index(ascending=False)\n",
    "            result = result.to_numpy()\n",
    "            result = pd.DataFrame(np.maximum(result, result.transpose()))\n",
    "            result.to_csv(outfile)\n",
    "        else:\n",
    "            raise ValueError(\"load data first!\")\n",
    "        \n",
    "        return result\n",
    "    \n",
    "    def polarDistance(self,\n",
    "                    i,\n",
    "                    return_dict=None):\n",
    "        \"\"\"return the distances from a single sample to the poles\n",
    "\n",
    "        Args:\n",
    "          i (int): index of sample to take\n",
    "          return_dict (dict): dictionary containing multiprocessing results\n",
    "\n",
    "        Returns:\n",
    "          distances: float, distance from sample to each pole\n",
    "        \"\"\"\n",
    "        p = self.samples_as_strings[i]\n",
    "        distances = []\n",
    "        # calculate from each pole to the sample, and append to array\n",
    "        for index, row in self.polar_features[self.cols].iterrows():\n",
    "            row = row.fillna('').values.astype(str)[:]\n",
    "            distances.append(self.distance(p, np.array(row)))\n",
    "        if return_dict is not None:\n",
    "            return_dict[i] = distances\n",
    "        return distances\n",
    "            \n",
    "    def polarDistance_multiple(self,\n",
    "                               outfile,\n",
    "                               processes=6):\n",
    "        \"\"\"return the distance from all samples to the poles\n",
    "\n",
    "        Args:\n",
    "          outfile (str): desired output filename and path\n",
    "          \n",
    "        Returns:\n",
    "          return_dict: dictionary containing multiprocessing results\n",
    "        \"\"\"\n",
    "        if all(x is not None for x in [self.samples, self.cols,\n",
    "                                    self.polar_features]):\n",
    "            pole_names = []\n",
    "            for index, row in self.polar_features[self.cols].iterrows():\n",
    "                pole_names.append(index)\n",
    "            result = self.__mpcollector(processes,\n",
    "                                        self.polarDistance,\n",
    "                                        pole_names,\n",
    "                                        outfile)\n",
    "            \"\"\"\n",
    "            # init mp.Manager and result dict\n",
    "            manager = mp.Manager()\n",
    "            return_dict = manager.dict()\n",
    "            processes = []\n",
    "            \n",
    "            # init and start mp.Processes for individual samples\n",
    "            for i in range(len(self.samples)):\n",
    "                p = mp.Process(target=self.polarDistance, args=(i, return_dict))\n",
    "                processes.append(p)\n",
    "            [x.start() for x in processes]\n",
    "            [x.join() for x in processes]\n",
    "\n",
    "            # format and save resulting dict\n",
    "            pole_names = []\n",
    "            for index, row in self.polar_features[self.cols].iterrows():\n",
    "                pole_names.append(index)\n",
    "            result=[x for x in return_dict.values()]\n",
    "            result=pd.DataFrame(result,columns=pole_names).to_csv(outfile)\n",
    "            \"\"\"\n",
    "            \n",
    "        else:\n",
    "            raise ValueError(\"load data first!\")\n",
    "        return result\n",
    "    \n",
    "    def polar_separation(self,\n",
    "                        nsteps=0):\n",
    "        \"\"\"calculates the distance between poles as a qdistance matrix\n",
    "\n",
    "        Args:\n",
    "          nsteps (int, optional): [description]. Defaults to 0.\n",
    "          \n",
    "        Returns:\n",
    "          self.polar_matrix: dictionary containing multiprocessing results\n",
    "        \"\"\"\n",
    "        # vectorize and qsample poles\n",
    "        polar_arraydata = self.polar_features[self.cols].values.astype(str)[:]\n",
    "        samples_ = []\n",
    "        for vector in polar_arraydata:\n",
    "            bp = self.getBaseFrequency(vector)\n",
    "            sample = qsample(vector, self.qnet, nsteps, baseline_prob=bp)\n",
    "            samples_.append(sample)\n",
    "        samples_ = np.array(samples_)\n",
    "        # calculate distance matrix for poles\n",
    "        self.polar_matrix = qdistance_matrix(samples_, samples_, self.qnet, self.qnet)\n",
    "        return self.polar_matrix\n",
    "        \n",
    "    def embed(self,\n",
    "            infile,\n",
    "            name_pref,\n",
    "            out_dir,\n",
    "            pca_model=False,\n",
    "            EMBED_BINARY=None):\n",
    "        '''\n",
    "        embed data\n",
    "\n",
    "        Args:\n",
    "          infile (str): input file to be embedded\n",
    "          name_pref (str): preferred name for output file\n",
    "          out_dir (str): output dir for results\n",
    "          pca_model (bool): whether or not to generate PCA model\n",
    "          EMBED_BINARY (os.path.abspath): path to embed binary\n",
    "        '''\n",
    "        if all(x is not None for x in [self.year]):\n",
    "            # init file names \n",
    "            yr = self.year\n",
    "            PREF = name_pref\n",
    "            FILE = infile\n",
    "            DATAFILE = out_dir + 'data_' +yr\n",
    "            EFILE = out_dir + PREF + '_E_' +yr\n",
    "            DFILE = out_dir + PREF + '_D_' +yr\n",
    "            \n",
    "            # set embed binary directory\n",
    "            if EMBED_BINARY is None:\n",
    "                EMBED = pkgutil.get_data(\"cognet.bin\", \"__embed__.so\") \n",
    "            else:\n",
    "                EMBED = EMBED_BINARY\n",
    "            \n",
    "            # embed data files\n",
    "            pd.read_csv(FILE, header=None).to_csv(DATAFILE,sep=' ',header=None,index=None)\n",
    "            STR=EMBED+' -f '+DATAFILE+' -E '+EFILE+' -D '+DFILE\n",
    "            subprocess.call(STR,shell=True)\n",
    "            if pca_model:\n",
    "                embed_to_pca(EFILE, EFILE+'_PCA')\n",
    "        elif self.year is None:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "    \n",
    "    def __calc_d0(self,\n",
    "                pole_1,\n",
    "                pole_2):\n",
    "        \"\"\"calculate distance between two poles\n",
    "\n",
    "        Args:\n",
    "          pole_1 (list[str]): a polar vector, must have same number of features as qnet\n",
    "          pole_2 (list[str]): a polar vector, must have same number of features as qnet\n",
    "        \"\"\"\n",
    "        self.pL = self.poles_dict[pole_1]\n",
    "        self.pR = self.poles_dict[pole_2]\n",
    "        self.d0 = qdistance(self.pL, self.pR, self.qnet, self.qnet)\n",
    "        \n",
    "    def ideology(self,\n",
    "                i,\n",
    "                return_dict=None,\n",
    "                pole_1=None,\n",
    "                pole_2=None):\n",
    "        \"\"\"return ideology index (left-leaning or right-leaning) for a singular sample\n",
    "\n",
    "        Args:\n",
    "          i (int): index of sample\n",
    "          pole_1 (int): index of Pole One to calc as base distance. Defaults to 0.\n",
    "          pole_2 (int): index of Pole Two to calc as base distance. Defaults to 1.\n",
    "          return_dict (dict, optional): dict containing results\n",
    "          \n",
    "        Returns:\n",
    "          [ideology_index, dR, dL, self.d0]: which way the sample leans,\n",
    "                                             distance from the right pole,\n",
    "                                             distance from the left pole,\n",
    "                                             and distance between poles, respectively\n",
    "        \"\"\"\n",
    "        # calculate base distance between two poles\n",
    "        if pole_1 is not None or pole_2 is not None:\n",
    "            self.__calc_d0(pole_1, pole_2)\n",
    "        \n",
    "        # calculate distances between sample and the two poles\n",
    "        p = self.samples_as_strings[i]\n",
    "        dR = qdistance(self.pR, p, self.qnet, self.qnet)\n",
    "        dL = qdistance(self.pL, p, self.qnet, self.qnet)\n",
    "        \n",
    "        ideology_index = (dR-dL)/self.d0\n",
    "        if return_dict is not None:\n",
    "            return_dict[i] = [ideology_index, dR, dL, self.d0]\n",
    "        return [ideology_index, dR, dL, self.d0]\n",
    "\n",
    "    def dispersion(self,\n",
    "                   i,\n",
    "                   return_dict=None):\n",
    "        \"\"\"qsamples a sample n times and takes distance matrix \n",
    "        to determine max and std of distances between qsamples\n",
    "\n",
    "        Args:\n",
    "          i (int): index of sample\n",
    "          return_dict (dict): dictionary containing multiprocessing results\n",
    "\n",
    "        Returns:\n",
    "          list[float]: std and max of the distances btwn qsamples\n",
    "        \"\"\"\n",
    "        # qsample sample num_qsample times\n",
    "        p = self.samples_as_strings[i]\n",
    "        Qset = [qsample(p, self.qnet, self.steps) for j in np.arange(self.num_qsamples)]\n",
    "        Qset = np.array(Qset)\n",
    "\n",
    "        # calculate qdistance matrix for qsampled samples\n",
    "        matrix = (qdistance_matrix(Qset, Qset, self.qnet, self.qnet))\n",
    "        Q = matrix.max()\n",
    "        Qsd = matrix.std()\n",
    "        \n",
    "        if return_dict is not None:\n",
    "            return_dict[i] = [Qsd, Q]\n",
    "        return [Qsd, Q]\n",
    "    \n",
    "    def compute_DLI_samples(self,\n",
    "                        type,\n",
    "                        outfile,\n",
    "                        num_qsamples=40,\n",
    "                        steps=120,\n",
    "                        n_jobs=28,\n",
    "                        pole_1=0,\n",
    "                        pole_2=1,\n",
    "                        processes=6):\n",
    "        \"\"\"compute and save ideology index or dispersion for all samples\n",
    "\n",
    "        Args:\n",
    "          num_qsamples (int): number of qsamples to compute\n",
    "          outfile (str): output file for results\n",
    "          type (str): whether to calc dispersion or ideology\n",
    "          steps (int): number of steps to qsample\n",
    "          n_jobs (int, optional): sets the number of jobs for parallelization. Defaults to 28.\n",
    "          pole_1 (int, optional): index of Pole One to calc as base distance. Defaults to 0.\n",
    "          pole_2 (int, optional): index of Pole Two to calc as base distance. Defaults to 1.\n",
    "\n",
    "        Raises:\n",
    "          ValueError: set poles if poles are not set\n",
    "          ValueError: load data if samples or features are not present\n",
    "            \n",
    "        Returns:\n",
    "          Result: dictionary containing multiprocessing results\n",
    "        \"\"\"\n",
    "        if all(x is not None for x in [self.samples, self.features,\n",
    "                                    self.pL, self.pR]):\n",
    "            # init vars\n",
    "            self.num_qsamples = num_qsamples\n",
    "            self.steps = steps\n",
    "            if pole_1 != 0 or pole_2 != 1:\n",
    "                self.__calc_d0(pole_1, pole_2)\n",
    "            \n",
    "            if type == 'ideology':\n",
    "                func_ = self.ideology\n",
    "                cols=['ideology', 'dR', 'dL', 'd0']\n",
    "            elif type == 'dispersion':\n",
    "                func_ = self.dispersion\n",
    "                cols=['Qsd', 'Qmax']\n",
    "            else:\n",
    "                raise ValueError(\"Type must be either dispersion or ideology!\")\n",
    "            \n",
    "            result = self.__mpcollector(processes,\n",
    "                                        func_,\n",
    "                                        cols,\n",
    "                                        outfile)\n",
    "            \n",
    "            \"\"\"\n",
    "            # init mp.Manager and result dict\n",
    "            manager = mp.Manager()\n",
    "            return_dict = manager.dict()\n",
    "            processes = []\n",
    "\n",
    "            # init mp.Processes for each individual sample\n",
    "            # run once collected processes hit max\n",
    "            max_processes = processes\n",
    "            num_processes = 0\n",
    "            for i in range(len(self.samples)):\n",
    "                num_processes += 1\n",
    "                p = mp.Process(target=self.dissonance,\n",
    "                            args=(i, return_dict))\n",
    "                processes.append(p)\n",
    "                if num_processes == max_processes:\n",
    "                    [x.start() for x in processes]\n",
    "                    [x.join() for x in processes]\n",
    "                    processes = []\n",
    "                    num_processes = 0\n",
    "             \n",
    "            # compute remaining processes\n",
    "            if num_processes != 0:\n",
    "                [x.start() for x in processes]\n",
    "                [x.join() for x in processes]\n",
    "                processes = []\n",
    "                num_processes = 0\n",
    "                \n",
    "            # init mp.Processes for individual samples based on type of calulation\n",
    "            if type == 'ideology':\n",
    "                for i in range(len(self.samples)):\n",
    "                    p = mp.Process(target=self.ideology, args=(i, return_dict))\n",
    "                    processes.append(p)\n",
    "                columns=['ideology', 'dR', 'dL', 'd0']\n",
    "            elif type == 'dispersion':\n",
    "                for i in range(len(self.samples)):\n",
    "                    p = mp.Process(target=self.dispersion, args=(i, return_dict))\n",
    "                    processes.append(p)\n",
    "                columns=['Qsd', 'Qmax']\n",
    "            else:\n",
    "                raise ValueError(\"Type must be either dispersion or ideology!\")\n",
    "            \n",
    "            # start mp.Processes for individual samples\n",
    "            [x.start() for x in processes]\n",
    "            [x.join() for x in processes]\n",
    "            \n",
    "            # format and save resulting dict\n",
    "            result=[x for x in return_dict.values()]\n",
    "            result=pd.DataFrame(result,columns=columns).to_csv(outfile)\n",
    "            \"\"\"\n",
    "        elif self.pL is None or self.pR is None:\n",
    "            raise ValueError(\"set_poles first!\")\n",
    "        else:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "        return pd.DataFrame(result, columns=cols)\n",
    "\n",
    "    def compute_polar_indices(self,\n",
    "                              num_samples=None,\n",
    "                              polar_comp=False,\n",
    "                              POLEFILE=None,\n",
    "                              steps=5):\n",
    "        '''set up polar indices for dissonance func\n",
    "\n",
    "        Args:\n",
    "          num_samples (int): subset of samples to take\n",
    "          polar_comp (bool): whether or not to set poles\n",
    "          POLEFILE (None): file containing pole samples and features\n",
    "          steps (int): number of steps to qsample\n",
    "        '''\n",
    "        if all(x is not None for x in [self.samples, self.features, self.poles]):\n",
    "            if num_samples is not None:\n",
    "                self.set_nsamples(num_samples)\n",
    "\n",
    "            if polar_comp:\n",
    "                self.set_poles(self.qnet, steps, POLEFILE)\n",
    "            \n",
    "            # calculate polar indices\n",
    "            polar_features = pd.concat([self.features, self.poles], axis=0)\n",
    "            self.polar_indices=np.where(polar_features[self.cols].fillna('XXXX').values[0]!='XXXX')[0]\n",
    "        \n",
    "        elif self.poles is None:\n",
    "            raise ValueError(\"set_poles first!\")\n",
    "        else:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "\n",
    "    def dissonance(self,\n",
    "                sample_index,\n",
    "                return_dict=None,\n",
    "                MISSING_VAL=0.0):\n",
    "        '''compute dissonance for a single sample, helper function for all_dissonance\n",
    "        \n",
    "        Args:\n",
    "          sample_index (int): index of the sample to compute dissonance\n",
    "          return_dict (dict): dictionary containing multiprocessing results\n",
    "          MISSING_VAL (float): default dissonance value\n",
    "          \n",
    "        Returns: \n",
    "          diss[self.polar_indices]: ndarray containing dissonance for sample\n",
    "        '''\n",
    "        if all(x is not None for x in [self.samples, self.features]):\n",
    "            s = self.samples_as_strings[sample_index]\n",
    "            if self.polar_indices is None:\n",
    "                self.polar_indices = range(len(s))\n",
    "\n",
    "            # init vars and calculate dissonance for sample\n",
    "            Ds=self.qnet.predict_distributions(s)\n",
    "            diss=np.ones(len(Ds))*MISSING_VAL\n",
    "            for i in self.polar_indices:\n",
    "                if s[i] != '':\n",
    "                    if s[i] in Ds[i].keys():\n",
    "                        diss[i]=1-Ds[i][s[i]]/np.max(\n",
    "                            list(Ds[i].values())) \n",
    "                    else:\n",
    "                        diss[i]=1.0\n",
    "            if return_dict is not None:\n",
    "                return_dict[sample_index] = diss[self.polar_indices]\n",
    "            return diss[self.polar_indices]\n",
    "        else:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "    \n",
    "    def dissonance_matrix(self,\n",
    "                        outfile='/example_results/DISSONANCE_matrix.csv',\n",
    "                        processes=6):\n",
    "        '''get the dissonance for all samples\n",
    "\n",
    "        Args:\n",
    "          output_file (str): directory and/or file for output\n",
    "          processes (int): max number of processes. Defaults to 6.\n",
    "\n",
    "        Returns:\n",
    "          pandas.DataFrame\n",
    "        '''\n",
    "        # set columns\n",
    "        if self.polar_indices is not None:\n",
    "            polar_features = pd.concat([self.features, self.poles], axis=0)\n",
    "            cols = polar_features[self.cols].dropna(axis=1).columns\n",
    "        else:\n",
    "            cols = self.cols\n",
    "        \n",
    "        result = self.__mpcollector(processes,\n",
    "                                    self.dissonance,\n",
    "                                    cols,\n",
    "                                    outfile)\n",
    "        return pd.DataFrame(result, columns=cols)\n",
    "        \n",
    "        \"\"\"\n",
    "        # init mp.Manager and result dict\n",
    "        manager = mp.Manager()\n",
    "        return_dict = manager.dict()\n",
    "        processes = []\n",
    "        \n",
    "        # init mp.Processes for each individual sample\n",
    "        # run once collected processes hit max\n",
    "        max_processes = processes\n",
    "        num_processes = 0\n",
    "        for i in range(len(self.samples)):\n",
    "            num_processes += 1\n",
    "            p = mp.Process(target=self.dissonance,\n",
    "                           args=(i, return_dict))\n",
    "            processes.append(p)\n",
    "            if num_processes == max_processes:\n",
    "                [x.start() for x in processes]\n",
    "                [x.join() for x in processes]\n",
    "                processes = []\n",
    "                num_processes = 0\n",
    "                \n",
    "        # compute remaining processes\n",
    "        if num_processes != 0:\n",
    "            [x.start() for x in processes]\n",
    "            [x.join() for x in processes]\n",
    "            processes = []\n",
    "            num_processes = 0\n",
    "\n",
    "        # format and save resulting dict\n",
    "        result=[x for x in return_dict.values()]\n",
    "        if self.polar_indices is not None:\n",
    "            polar_features = pd.concat([self.features, self.poles], axis=0)\n",
    "            cols = polar_features[self.cols].dropna(axis=1).columns\n",
    "        else:\n",
    "            cols = self.cols\n",
    "        result=pd.DataFrame(result,columns=cols).to_csv(output_file)\n",
    "        return pd.DataFrame(return_dict.copy())\n",
    "        \"\"\"\n",
    "    \n",
    "    def __choose_one(self,\n",
    "                X):\n",
    "        '''returns a random element of X\n",
    "\n",
    "        Args:\n",
    "          X (1D array-like): vector from which random element is to be chosen\n",
    "        \n",
    "        Returns:\n",
    "          X: random element of sample\n",
    "          None: if X has len 0\n",
    "        '''\n",
    "        X=list(X)\n",
    "        if len(X)>0:\n",
    "            return X[np.random.randint(len(X))]\n",
    "        return None\n",
    "\n",
    "    def getMaskedSample(self,\n",
    "                        s,\n",
    "                        mask_prob=0.5,\n",
    "                        allow_all_mutable=False):\n",
    "        '''inputs a sample and randomly mask elements of the sample\n",
    "\n",
    "        Args:\n",
    "          s (list[str]): vector of sample, must have the same num of features as the qnet.\n",
    "          mask_prob (float): float btwn 0 and 1, prob to mask element of sample. Defaults to 0.5\n",
    "          allow_all_mutable (bool): whether or not all variables are mutable. Defaults to False.\n",
    "          \n",
    "        Returns:\n",
    "          s1,\n",
    "          base_frequency,\n",
    "          MASKrand,\n",
    "          np.where(base_frequency)[0],\n",
    "          np.mean(rnd_match_prob),\n",
    "          np.mean(max_match_prob),\n",
    "          random_sample\n",
    "        '''\n",
    "        if self.samples is not None:\n",
    "            # init random mutable variable masking\n",
    "            s0=s.copy()\n",
    "            s0=np.array(s0)   \n",
    "            # double check, because code seems to imply that masking happens in order,\n",
    "            # i.e. limited to the first 100 features, if there are only 100 mutable features\n",
    "            MUTABLE=pd.DataFrame(np.zeros(len(self.cols)),index=self.cols).transpose()\n",
    "            WITHVAL=[x for x in self.cols[np.where(s0)[0]] if x in self.mutable_vars ]\n",
    "            MASKrand=[x for x in WITHVAL if random.random() < mask_prob ]\n",
    "            for m in MASKrand:\n",
    "                MUTABLE[m]=1.0\n",
    "            \n",
    "            mutable_x=MUTABLE.values[0]\n",
    "            base_frequency=mutable_x/mutable_x.sum()\n",
    "\n",
    "            # if np.isnan(base_frequency).any():\n",
    "            #     return np.nan,np.nan,np.nan\n",
    "            #     return self.getMaskedSample(s)\n",
    "\n",
    "            # mask sample according to masking (base_frequency)\n",
    "            s1=s.copy()\n",
    "            for i in range(len(base_frequency)):\n",
    "                if base_frequency[i]>0.0001:\n",
    "                    s1[i]=''\n",
    "                \n",
    "            # create a random sample to test reconstruction effectiveness\n",
    "            random_sample=np.copy(s)\n",
    "            rnd_match_prob=[]        \n",
    "            max_match_prob=[]        \n",
    "            D=self.qnet.predict_distributions(s)\n",
    "            for i in MASKrand:\n",
    "                random_sample[np.where(\n",
    "                    self.cols==i)[0][0]]=self.__choose_one(\n",
    "                        self.D_null[np.where(self.cols==i)[0][0]].keys())\n",
    "                rnd_match_prob=np.append(rnd_match_prob,1/len(\n",
    "                    self.D_null[np.where(self.cols==i)[0][0]].keys()))\n",
    "                max_match_prob=np.append(\n",
    "                    max_match_prob,np.max(\n",
    "                        list(D[np.where(\n",
    "                            self.cols==i)[0][0]].values())))\n",
    "            \n",
    "            # calculate base_frequency if all variables are mutable\n",
    "            if allow_all_mutable:\n",
    "                WITHVAL=[x for x in self.cols[np.where(s0)[0]]]\n",
    "                MASKrand=[x for x in WITHVAL if random.random() < mask_prob ]\n",
    "                for m in MASKrand:\n",
    "                    MUTABLE[m]=1.0\n",
    "                mutable_x=MUTABLE.values[0]\n",
    "                base_frequency=mutable_x/mutable_x.sum()\n",
    "                s1=s.copy()\n",
    "                for i in range(len(base_frequency)):\n",
    "                    if base_frequency[i]>0.0001:\n",
    "                        s1[i]=''\n",
    "\n",
    "            return s1,base_frequency,MASKrand,np.where(\n",
    "                base_frequency)[0],np.mean(rnd_match_prob),np.mean(max_match_prob),random_sample\n",
    "        else:\n",
    "            raise ValueError(\"load_data first!\")\n",
    "\n",
    "    def randomMaskReconstruction(self,\n",
    "                                index,\n",
    "                                return_dict=None,\n",
    "                                sample=None,\n",
    "                                index_colname=\"feature_names\",\n",
    "                                output_dir=\"recon_results/\",\n",
    "                                file_name=\"recon_tmp.csv\",\n",
    "                                mask_prob=0.5,\n",
    "                                allow_all_mutable=False,\n",
    "                                save_samples=False):\n",
    "        \"\"\"reconstruct the masked sample by qsampling and comparing to original\n",
    "        set self.mask_prob and self.steps if needed\n",
    "\n",
    "        Args:\n",
    "          index (int): index of sample to take.\n",
    "          return_dict (dict): dictionary containing multiprocessing results. Defaults to None.\n",
    "          sample (list[str], optional): sample vector, must have the same num of features as the qnet. Defaults to None.\n",
    "          index_colname (str): column name for index. Defaults to \"feature_names\"\n",
    "          output_dir (str): directory name for output files. Defaults to \"recon_results/\".\n",
    "          file_name (str): base file name for output files Defaults to \"recon_tmp.csv\".\n",
    "          mask_prob (float): float btwn 0 and 1, prob to mask element of sample. Defaults to 0.5\n",
    "          allow_all_mutable (bool): whether or not all variables are mutable. Defaults to False.\n",
    "\n",
    "        Raises:\n",
    "          ValueError: Neither sample or index were given\n",
    "          ValueError: Both sample and index were given\n",
    "          \n",
    "        Returns:\n",
    "          return_dict[index]:(1 - (dqestim/dactual))*100,\n",
    "                            rmatch_u,\n",
    "                            rmatch,\n",
    "                            s,\n",
    "                            qs,\n",
    "                            random_sample,\n",
    "                            mask_\n",
    "        \"\"\"\n",
    "        if all(x is None for x in [sample, index]):\n",
    "            raise ValueError(\"Must input either sample or index!\")\n",
    "        elif all(x is not None for x in [sample, index]):\n",
    "            raise ValueError(\"Must input either sample or index not both!\")\n",
    "        elif sample is not None:\n",
    "            s=np.array(pd.DataFrame(sample).fillna('').values.astype(str)[:])\n",
    "        elif index is not None:\n",
    "            s=self.samples_as_strings[index]\n",
    "        \n",
    "        # calculate masked sample and get variables\n",
    "        s1,bp,mask_,maskindex,rmatch_u,rmatch,random_sample=self.getMaskedSample(s, \n",
    "                                                                        mask_prob=mask_prob,\n",
    "                                                                        allow_all_mutable=allow_all_mutable)\n",
    "        # if base_frequency is nan, set return_dict to nans\n",
    "        if np.isnan(bp).any():\n",
    "            return_dict[index] = np.nan,np.nan,np.nan\n",
    "            return np.nan,np.nan,np.nan\n",
    "        \n",
    "        # make directories\n",
    "        if not os.path.exists(output_dir):\n",
    "            os.mkdir(output_dir)\n",
    "\n",
    "        # qsample sample and calculate distances between original and qsampled \n",
    "        qs=qsample(s1,self.qnet,self.steps,bp)\n",
    "        dqestim=qdistance(s,qs,self.qnet,self.qnet)\n",
    "        dactual=qdistance(s,s1,self.qnet,self.qnet)\n",
    "        \n",
    "        # format and save sample and qsample statistics and values\n",
    "        cmpf=pd.DataFrame([s,qs,random_sample],\n",
    "                          columns=self.cols,\n",
    "                          index=['sample','qsampled','random_sample'])[mask_].transpose()\n",
    "        cmpf.index.name= index_colname\n",
    "        file_name = file_name.replace(\"tmp\", str(index))\n",
    "        cmpf.to_csv(output_dir+file_name)\n",
    "        if return_dict is not None:\n",
    "            if save_samples:\n",
    "                return_dict[index] = (1 - (dqestim/dactual))*100,rmatch_u,rmatch,s,qs,random_sample,mask_\n",
    "            else:\n",
    "                return_dict[index] = (1 - (dqestim/dactual))*100,rmatch_u,rmatch,mask_\n",
    "        return return_dict[index]\n",
    "\n",
    "    def randomMaskReconstruction_multiple(self,\n",
    "                                          outfile,\n",
    "                                          processes=6,\n",
    "                                          save_samples=False,\n",
    "                                          index_colname=\"feature_names\",\n",
    "                                          output_dir=\"recon_results/\",\n",
    "                                          file_name=\"recon_tmp.csv\",\n",
    "                                          mask_prob=0.5,\n",
    "                                          allow_all_mutable=False):\n",
    "        '''runs and saves the results of the predicted masked sample\n",
    "\n",
    "        Args:\n",
    "          output_file (str): directory and/or file for output.\n",
    "          processes (int): max number of processes. Defaults to 6.\n",
    "          save_samples (boolean): whether or not to save the generated qsamples, random samples, etc. Defaults to False.\n",
    "          index_colname=\"feature_names\",\n",
    "          output_dir=\"recon_results/\",\n",
    "          file_name=\"recon_tmp.csv\",\n",
    "          mask_prob (float): float btwn 0 and 1, prob to mask element of sample. Defaults to 0.5\n",
    "          allow_all_mutable (bool): whether or not all variables are mutable. Defaults to False.\n",
    "          \n",
    "        Returns:\n",
    "          result.rederr.mean(), result.rand_err.mean(): mean of reconstruction error and random error\n",
    "        '''\n",
    "        # set columns\n",
    "        if save_samples:\n",
    "            cols = ['rederr','r_prob','rand_err','sample','qsampled','random_sample','mask_']\n",
    "        else:\n",
    "            cols = ['rederr','r_prob','rand_err','mask_']\n",
    "            \n",
    "        args=[None, index_colname, output_dir,\n",
    "              file_name, mask_prob, allow_all_mutable]\n",
    "        \n",
    "        result = self.__mpcollector(processes,\n",
    "                                    self.randomMaskReconstruction,\n",
    "                                    cols,\n",
    "                                    outfile,\n",
    "                                    args=args)\n",
    "        \"\"\"\n",
    "        # init mp.Manager and result dict\n",
    "        manager = mp.Manager()\n",
    "        return_dict = manager.dict()\n",
    "        processes = []\n",
    "        \n",
    "        # init mp.Processes for each individual sample\n",
    "        # run once collected processes hit max\n",
    "        max_processes = processes\n",
    "        num_processes = 0\n",
    "        for i in range(len(self.samples)):\n",
    "            num_processes += 1\n",
    "            p = mp.Process(target=self.randomMaskReconstruction,\n",
    "                           args=(i, \n",
    "                                 return_dict,\n",
    "                                 None,\n",
    "                                 index_colname,\n",
    "                                 output_dir,\n",
    "                                 file_name,\n",
    "                                 mask_prob,\n",
    "                                 allow_all_mutable))\n",
    "            processes.append(p)\n",
    "            if num_processes == max_processes:\n",
    "                [x.start() for x in processes]\n",
    "                [x.join() for x in processes]\n",
    "                processes = []\n",
    "                num_processes = 0\n",
    "                \n",
    "        # compute remaining processes\n",
    "        if num_processes != 0:\n",
    "            [x.start() for x in processes]\n",
    "            [x.join() for x in processes]\n",
    "            processes = []\n",
    "            num_processes = 0\n",
    "        \n",
    "        # format and save resulting dict\n",
    "        result=[x for x in return_dict.values() if isinstance(x, tuple)]\n",
    "        result_cols = ['rederr','r_prob','rand_err','mask_']\n",
    "        if save_samples:\n",
    "            result_cols = ['rederr','r_prob','rand_err','sample','qsampled','random_sample','mask_']\n",
    "        result=pd.DataFrame(result,columns=result_cols)\n",
    "        result.rederr=result.rederr.astype(float)\n",
    "        \"\"\"\n",
    "        return result\n",
    "    \n",
    "    def dmat_filewriter(self,\n",
    "                        QNETPATH,\n",
    "                        pyfile=\"cognet_qdistmatrix.py\",\n",
    "                        MPI_SETUP_FILE=\"mpi_setup.sh\",\n",
    "                        MPI_RUN_FILE=\"mpi_run.sh\",\n",
    "                        MPI_LAUNCHER_FILE=\"../launcher.sh\",\n",
    "                        YEARS='2016',\n",
    "                        NODES=4,\n",
    "                        T=12,\n",
    "                        num_samples=None,\n",
    "                        OUTFILE='tmp_distmatrix.csv',\n",
    "                        tmp_samplesfile=\"tmp_samples_as_strings.csv\"):\n",
    "        \"\"\"generate files to compute qdistance matrix using mpi parallelization\n",
    "\n",
    "        Args:\n",
    "          QNETPATH (str): Qnet filepath\n",
    "          pyfile (str, optional): Name of generated python file. Defaults to \"cognet_qdistmatrix.py\".\n",
    "          MPI_SETUP_FILE (str, optional): Name of mpi setup script. Defaults to \"mpi_setup.sh\".\n",
    "          MPI_RUN_FILE (str, optional): Name of mpi run script. Defaults to \"mpi_run.sh\".\n",
    "          MPI_LAUNCHER_FILE (str, optional): Launcher script filepath. Defaults to \"launcher.sh\".\n",
    "          YEARS (str, optional): If looping by year, not currently implemented. Defaults to '2016'.\n",
    "          NODES (int, optional): Number of nodes to use. Defaults to 4.\n",
    "          T (int, optional): Number of hours to reserve nodes for. Defaults to 12.\n",
    "          num_samples ([type], optional): How many samples to take. Defaults to None.\n",
    "          OUTFILE (str, optional): CSV File to write computed qdist matrix. Defaults to 'tmp_distmatrix.csv'.\n",
    "          tmp_samplesfile (str, optional): CSV File to write samples as strings. Defaults to \"tmp_samples_as_strings.csv\".\n",
    "\n",
    "        Raises:\n",
    "            ValueError: load data if qnet, features, or samples are not present]\n",
    "        \"\"\"\n",
    "        if all(x is not None for x in [self.samples,self.features,\n",
    "                                       self.qnet, self.cols]):\n",
    "            if num_samples is not None:\n",
    "                self.set_nsamples(num_samples)\n",
    "            \n",
    "            # init and make tmp dir \n",
    "            tmp_path = \"mpi_tmp/\"\n",
    "            if not os.path.exists(tmp_path):\n",
    "                os.makedirs(tmp_path)\n",
    "            \n",
    "            pd.DataFrame(self.samples_as_strings).to_csv(tmp_path+tmp_samplesfile, header=None, index=None)\n",
    "            \n",
    "            w = self.samples.index.size\n",
    "            \n",
    "            # writing python file\n",
    "            with open(tmp_path+pyfile, 'w+') as f:\n",
    "                f.writelines([\"from mpi4py.futures import MPIPoolExecutor\\n\",\n",
    "                              \"import numpy as np\\n\",\n",
    "                              \"import pandas as pd\\n\",\n",
    "                              \"from quasinet.qnet import Qnet, qdistance, load_qnet, qdistance_matrix\\n\",\n",
    "                              \"from quasinet.qsampling import qsample, targeted_qsample\\n\\n\",\n",
    "                              \"qnet=load_qnet(\\'{}\\')\\n\".format(QNETPATH)])\n",
    "\n",
    "                f.writelines([\"w = {}\\n\".format(w),\n",
    "                              \"h = w\\n\",\n",
    "                              \"p_all = pd.read_csv(\\\"{}\\\", header=None).values.astype(str)[:]\\n\\n\".format(tmp_samplesfile)])\n",
    "\n",
    "                f.writelines([\"def distfunc(x,y):\\n\",\n",
    "                              \"\\td=qdistance(x,y,qnet,qnet)\\n\",\n",
    "                              \"\\treturn d\\n\\n\"])\n",
    "\n",
    "                f.writelines([\"def dfunc_line(k):\\n\",\n",
    "                              \"\\tline = np.zeros(w)\\n\",\n",
    "                              \"\\ty = p_all[k]\\n\",\n",
    "                              \"\\tfor j in range(w):\\n\",\n",
    "                              \"\\t\\tif j > k:\\n\",\n",
    "                              \"\\t\\t\\tx = p_all[j]\\n\",\n",
    "                              \"\\t\\t\\tline[j] = distfunc(x, y)\\n\",\n",
    "                              \"\\treturn line\\n\\n\"])\n",
    "\n",
    "                f.writelines([\"if __name__ == '__main__':\\n\",\n",
    "                              \"\\twith MPIPoolExecutor() as executor:\\n\",\n",
    "                              \"\\t\\tresult = executor.map(dfunc_line, range(h))\\n\",\n",
    "                              \"\\tresult = pd.DataFrame(result)\\n\",\n",
    "\t                          \"\\tresult = result.to_numpy()\\n\",\n",
    "                              \"\\tresult = pd.DataFrame(np.maximum(result, result.transpose()))\\n\"\n",
    "                              \"\\tresult.to_csv(\\'{}\\',index=None,header=None)\".format(OUTFILE)])\n",
    "            \n",
    "            # writing MPI setup file\n",
    "            with open(tmp_path+MPI_SETUP_FILE, 'w+') as ms:\n",
    "                ms.writelines([\"#!/bin/bash\\n\",\n",
    "                               \"YEAR=$1\\n\\n\",\n",
    "                               \"if [ $# -gt 1 ] ; then\\n\",\n",
    "                               \"\\tNODES=$2\\n\",\n",
    "                               \"else\\n\",\n",
    "                               \"\\tNODES=3\\n\",\n",
    "                               \"fi\\n\",\n",
    "                               \"if [ $# -gt 2 ] ; then\\n\",\n",
    "                               \"\\tNUM=$3\\n\",\n",
    "                               \"else\\n\",\n",
    "                               \"\\tNUM='all'\\n\",\n",
    "                               \"fi\\n\",\n",
    "                               \"if [ $# -gt 3 ] ; then\\n\",\n",
    "                               \"\\tPROG=$4\\n\",\n",
    "                               \"else\\n\",\n",
    "                               \"\\tPROG=$(tty)\\n\",\n",
    "                               \"fi\\n\\n\",\n",
    "                               \"NUMPROC=`expr 28 \\* $NODES`\\n\",\n",
    "                               \"echo \\\"module load midway2\\\" >> $PROG\\n\",\n",
    "                               \"echo \\\"module unload python\\\" >> $PROG\\n\",\n",
    "                               \"echo \\\"module unload openmpi\\\" >> $PROG\\n\",\n",
    "                               \"echo \\\"module load python/anaconda-2020.02\\\" >> $PROG\\n\",\n",
    "                               \"echo \\\"module load mpi4py\\\" >> $PROG\\n\",\n",
    "                               \"echo \\\"date; mpiexec -n \\\"$NUMPROC\\\" python3 -m mpi4py.futures {}; date\\\"  >> $PROG\\n\".format(pyfile),\n",
    "                                ])\n",
    "\n",
    "            # writing MPI run file\n",
    "            with open(tmp_path+MPI_RUN_FILE, 'w+') as mr:\n",
    "                mr.writelines([\"#!/bin/bash\\n\",\n",
    "                               \"YEARS=\\'{}\\'\\n\".format(YEARS),\n",
    "                               \"# nodes requested\\n\",\n",
    "                               \"NODES={}\\n\".format(NODES),\n",
    "                               \"# time requested\\n\",\n",
    "                               \"T={}\\n\".format(T),\n",
    "                               \"NUM=\\'all\\'\\n\",\n",
    "                               \"LAUNCH=\\'{}\\'\\n\\n\".format(MPI_LAUNCHER_FILE),\n",
    "                               \"for yr in `echo $YEARS`\\n\",\n",
    "                               \"do\\n\",\n",
    "                               \"\\techo $yr\\n\",\n",
    "                               \"\\t./{} $yr $NODES $NUM tmp_\\\"$yr\\\"\\n\".format(MPI_SETUP_FILE),\n",
    "                               \"\\t$LAUNCH -P tmp_\\\"$yr\\\" -F -T $T -N \\\"$NODES\\\" -C 28 -p broadwl -J MPI_TMP_\\\"$yr\\\" -M 56\\n\",\n",
    "                               \"done\\n\",\n",
    "                               \"rm tmp_\\\"$yr\\\"*\\n\"])\n",
    "            os.system(\"cp {} {}\".format(MPI_LAUNCHER_FILE,tmp_path+'mpi_launcher.sh'))\n",
    "        \n",
    "        else:\n",
    "            raise ValueError(\"load data first!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1034\n"
     ]
    }
   ],
   "source": [
    "# testing cognet\n",
    "# set some paramaters in instantiating cognet class \n",
    "# if loading from model obj, no need to use load_data func, otherwise, load_data\n",
    "Cg = cognet()\n",
    "print(len(model_.features))\n",
    "Cg.load_from_model(model_, data, 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "class-computed distance: 0.12318448800408259\n",
      "actual:0.12351443752539294\n"
     ]
    }
   ],
   "source": [
    "# distance calculation for individual samples    \n",
    "# we have a nsteps parameter (for sample 1 and sample2)\n",
    "# which qsamples the sample1 and sample2 if set before\n",
    "# computing distance. Note qsampling must only \n",
    "# change mutable varaibles, so need to compute base-freq\n",
    "distance = Cg.distance(samples[1],samples[3],nsteps1=5, nsteps2=5)\n",
    "print(\"class-computed distance:\", distance)\n",
    "qdistance_ = qdistance(samples[1],samples[3],Cg.qnet,Cg.qnet)\n",
    "print(\"actual:{}\".format(qdistance_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 pole features not found in sample features\n"
     ]
    }
   ],
   "source": [
    "# produce stats on how many column names actually match\n",
    "stats = Cg.set_poles(POLEFILE,\"R\",\"L\",steps=120, VERBOSE=True)\n",
    "\n",
    "# compute polar distance matrix\n",
    "dmatrix = Cg.polar_separation(nsteps=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dissonance: [0. 0. 0. ... 0. 0. 0.]\n",
      "reconstruction results: 14.489811228903315 0.27539682539682536 0.5691305791213414\n",
      "ideology: [0.17788964536320437, 0.10332081969850498, 0.07945569888115739, 0.13415688568393752]\n",
      "Dispersion: [0.04398053116781898, 0.11784592549498388]\n",
      "distance from poles: [0.021635271857159282, 0.022334688414062873]\n"
     ]
    }
   ],
   "source": [
    "#------------------\n",
    "# the following are for single samples\n",
    "\n",
    "# dissonance\n",
    "dissonance_array = Cg.dissonance(1)\n",
    "print(\"dissonance:\", dissonance_array)\n",
    "\n",
    "# random mask and reconstruction\n",
    "returndict = {}\n",
    "rederr,r_prob,rand_err,s,qs,s_rand,mask_ = Cg.randomMaskReconstruction(index=1, \n",
    "                                                                       return_dict=returndict,\n",
    "                                                                       index_colname=\"feature_names\",\n",
    "                                                                       output_dir=\"GSS/results/recon_results/\",\n",
    "                                                                       file_name=\"recon_tmp.csv\",\n",
    "                                                                       save_samples=True)# sample=np.array(samples[1]))\n",
    "print(\"reconstruction results:\", rederr, r_prob, rand_err)\n",
    "\n",
    "#ideology\n",
    "Cg.num_qsamples = 5\n",
    "ideology_index = Cg.ideology(3,pole_1=\"R\",pole_2=\"L\")\n",
    "print(\"ideology:\", ideology_index)\n",
    "\n",
    "# disperion\n",
    "dispersion_ = Cg.dispersion(3)\n",
    "print(\"Dispersion:\", dispersion_)\n",
    "\n",
    "# compute distance from each pole\n",
    "array_distances = Cg.polarDistance(1, returndict)\n",
    "print(\"distance from poles:\", array_distances)\n",
    "#-------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n",
      "dissonance array:    spkcom  colcom  libcom  spkmil  colmil  libmil  libhomo  libmslm  gunlaw  \\\n",
      "0     0.0     0.0     0.0     0.0     0.0     0.0      0.0      0.0     0.0   \n",
      "1     0.0     0.0     0.0     0.0     0.0     0.0      0.0      0.0     0.0   \n",
      "\n",
      "      grass  ...  shotgun  rowngun  viruses  intmil  abpoorw  godchnge  \\\n",
      "0  0.000000  ...      0.0      0.0      0.0     0.0      0.0  0.853822   \n",
      "1  0.612712  ...      0.0      0.0      0.0     0.0      0.0  0.000000   \n",
      "\n",
      "   prayfreq  religcon  religint  comfort  \n",
      "0       0.0       0.0  0.827436      0.0  \n",
      "1       0.0       0.0  0.000000      0.0  \n",
      "\n",
      "[2 rows x 35 columns]\n",
      "----------------------------------------------------------------------\n",
      "\n",
      "[None, 'feature_names', 'recon_results/', 'recon_tmp.csv', 0.5, False]\n",
      "reconstruction results [(20.040551655105265, 0.28853729068395007, 0.6522447701969935, ['wrkstat', 'HRS1', 'HRS2', 'wrkslf', 'OCC10', 'SPHRS1', 'COHRS1', 'COHRS2', 'PAIND10', 'MAIND10', 'sibs', 'age', 'educ', 'degree', 'MAJOR1', 'sex', 'RES16', 'MOBILE16', 'hompop', 'babies', 'teens', 'adults', 'earnrs', 'region', 'PRES12', 'natenvir', 'natheal', 'natcity', 'natcrime', 'natrace', 'natarms', 'nataid', 'natsoc', 'natpark', 'natenrgy', 'eqwlth', 'tax', 'spkath', 'libath', 'spkrac', 'colrac', 'colcom', 'spkmil', 'colmil', 'libmil', 'spkhomo', 'colmslm', 'libmslm', 'cappun', 'courts', 'relig', 'denom', 'pray', 'prayer', 'bible', 'raclive', 'affrmact', 'happy', 'health', 'life', 'socommun', 'partfull', 'joblose', 'jobfind', 'class', 'rank', 'wksub', 'WKSUB1', 'WKSUBS1', 'getahead', 'abnomore', 'absingle', 'pillok', 'premarsx', 'teensex', 'homosex', 'spanking', 'SUICIDE2', 'SUICIDE3', 'SUICIDE4', 'owngun', 'pistol', 'shotgun', 'HUNT1', 'news', 'tvhours', 'phone', 'fechld', 'fepresch', 'RACDIF1', 'RACDIF3', 'helppoor', 'helpsick', 'numcong', 'wlthhsps', 'workblks', 'liveblks', 'marblk', 'marwht', 'racwork', 'discaff', 'fejobaff', 'vigversn', 'CLOSETO1', 'CLOSETO2', 'CLOSETO3', 'CLOSETO4', 'CLOSETO5', 'SEETALK2', 'SEETALK3', 'MYPROBS1', 'MYPROBS2', 'SEVERE2', 'SEVERE3', 'DANGOTH3', 'DANGOTH5', 'DANGSLF1', 'DANGSLF5', 'othlang', 'OTHLANG1', 'spklang', 'compuse', 'emailmin', 'wwwhr', 'wrktype', 'yearsjob', 'wrksched', 'mustwork', 'wrkhome', 'famvswk', 'hrsrelax', 'learnnew', 'workfast', 'overwork', 'knowwhat', 'myskills', 'respect', 'safefrst', 'teamsafe', 'safehlth', 'wksmooth', 'hlpequip', 'haveinfo', 'supcares', 'condemnd', 'promtefr', 'cowrkhlp', 'handmove', 'fairearn', 'rincblls', 'trynewjb', 'wkharsex', 'HEALTH1', 'slpprblm', 'SATJOB1', 'knowschd', 'hyperten', 'arthrtis', 'diabetes', 'depress', 'height', 'ntwkhard', 'lifenow', 'LIFEIN5', 'disrspct', 'threaten', 'quallife', 'hlthmntl', 'satsoc', 'actssoc', 'physacts', 'emoprobs', 'fatigue', 'abmoral', 'ABSTATE1', 'ABSTATE2', 'ABHELP4', 'abinspay', 'numpets', 'stockval', 'deptperf', 'extraval', 'numemps', 'newsfrom', 'scifrom', 'seeksci', 'nextgen', 'toofast', 'advfront', 'scibnfts', 'scistudy', 'ODDS2', 'hotcore', 'viruses', 'bigbang', 'condrift', 'evolved', 'earthsun', 'solarrev', 'majorcol', 'hsmath', 'hsbio', 'nanowill', 'intintl', 'inteduc', 'intsci', 'intecon', 'intmed', 'intenvir', 'intmil', 'visnhist', 'viszoo', 'SCINEWS3', 'scienthe', 'scientod', 'buyvalue', 'numorg', 'empinput', 'slfmangd', 'esopnot', 'ratetone', 'posslq', 'trcourts', 'intcntct', 'hubbywrk', 'SATFAM7', 'stress', 'HOMOSEX1', 'concong', 'conbiz', 'conchurh', 'concourt', 'conschls', 'churhpow', 'afterlif', 'heaven', 'hell', 'miracles', 'madenkid', 'parelkid', 'padenkid', 'religkid', 'denkid', 'attendpa', 'prayfreq', 'relscrpt', 'relmarry', 'ancestrs', 'mywaygod', 'relgeneq', 'vistholy', 'makefrnd', 'comfort', 'christns', 'muslims', 'hindus', 'buddhsts', 'jews', 'nukegen', 'watergen', 'genegen', 'numwomen', 'HIVTEST1', 'GENETST1', 'GENEGOO2', 'maleornt', 'GENESELF2', 'GENEABRT2', 'realinc', 'realrinc', 'ETH1', 'ETH2', 'RACECEN1', 'mnthsusa', 'vetyears', 'DWELOWN16', 'wordb', 'wordd', 'worde', 'wordg', 'wordsum', 'OLD1', 'GENDER2', 'OLD2', 'RELATE4', 'MAR4', 'OLD8', 'OLD11', 'RELHHD3', 'RELHHD4', 'hefinfo', 'respnum', 'HHTYPE1', 'famgen', 'rvisitor', 'visitors', 'RELHH2', 'RELSP1', 'RELSP4', 'ISCO88', 'MAISCO88', 'ISCO08', 'PAISCO08', 'SEI10', 'SEI10INC', 'PASEI10', 'PASEI10EDUC', 'PASEI10INC', 'MASEI10EDUC', 'SPSEI10EDUC', 'COSEI10', 'COPRES10', 'uswary', 'zodiac', 'WHOELSE1', 'WHOELSE3', 'WHOELSE6', 'feelevel', 'lngthinv', 'intethn', 'mode', 'ballot', 'sampcode', 'phase', 'huadd', 'wtssnr', 'wtssall', 'vstrat', 'vpsu']), (18.640859711160385, 0.30026412692932414, 0.6624007417612711, ['wrkstat', 'PRESTG10', 'INDUS10', 'marital', 'martype', 'divorce', 'spwrksta', 'spwrkslf', 'SPPRES10', 'SPPRES105PLUS', 'SPIND10', 'COOCC10', 'mawrkslf', 'MAOCC10', 'MAPRES105PLUS', 'MAIND10', 'childs', 'paeduc', 'coeduc', 'degree', 'spdeg', 'MAJOR1', 'dipged', 'sex', 'REG16', 'FAMILY16', 'INCOM16', 'hompop', 'babies', 'adults', 'unrelat', 'income', 'INCOME16', 'RINCOM16', 'region', 'srcbelt', 'partyid', 'VOTE12', 'polviews', 'natspac', 'natheal', 'natdrug', 'nateduc', 'natrace', 'natpark', 'cappun', 'grass', 'relig', 'fund', 'attend', 'reliten', 'RELIG16', 'FUND16', 'sprel', 'spfund', 'prayer', 'bible', 'raclive', 'wrkwayup', 'happy', 'helpful', 'fair', 'conclerg', 'coneduc', 'contv', 'conjudge', 'conarmy', 'popular', 'thnkself', 'socrel', 'socommun', 'socbar', 'jobfind', 'satfin', 'wksubs', 'WKSUBS1', 'wksup', 'union', 'kidssol', 'fepol', 'premarsx', 'teensex', 'pornlaw', 'SUICIDE1', 'SUICIDE2', 'SUICIDE4', 'polhitok', 'polabuse', 'polmurdr', 'polescap', 'tvhours', 'phone', 'coop', 'comprend', 'random', 'fechld', 'fepresch', 'RACDIF1', 'RACDIF3', 'RACDIF4', 'helppoor', 'helpnot', 'helpblk', 'reborn', 'savesoul', 'wlthwhts', 'wlthhsps', 'workblks', 'workhsps', 'intlblks', 'intlhsps', 'liveblks', 'marblk', 'MARAsian', 'marwht', 'yousup', 'spwksup', 'fejobaff', 'discaffm', 'vigversn', 'CLOSETO1', 'CLOSETO2', 'SEETALK1', 'SEETALK2', 'MYPROBS4', 'SEVERE1', 'SEVERE3', 'DANGOTH4', 'DANGSLF1', 'DANGSLF4', 'relpersn', 'compuse', 'wwwhr', 'wrktype', 'wrksched', 'mustwork', 'wrkhome', 'wkvsfam', 'workdiff', 'safetywk', 'safefrst', 'promteok', 'opdevel', 'condemnd', 'promtefr', 'cowrkhlp', 'manvsemp', 'hvylift', 'rincblls', 'JOBFIND1', 'trynewjb', 'wkageism', 'wkracism', 'wkharsex', 'HEALTH1', 'painarms', 'hurtatwk', 'spvtrfair', 'phyeffrt', 'STRESS12', 'hyperten', 'diabetes', 'weight', 'ntwkhard', 'LIFEIN5', 'quallife', 'hlthphys', 'hlthmntl', 'satsoc', 'fatigue', 'ratepain', 'abmelegl', 'abmoral', 'ABSTATE2', 'ABHELP1', 'ABHELP4', 'ABMEDGOV1', 'abinspay', 'numpets', 'WORKFOR1', 'stockval', 'stockops', 'extrapay', 'deptperf', 'EXTR2017', 'numemps', 'scifrom', 'astrolgy', 'scibnfts', 'scistudy', 'exptext', 'ODDS2', 'hotcore', 'radioact', 'lasers', 'viruses', 'BIGBANG1', 'condrift', 'earthsun', 'solarrev', 'colsci', 'hschem', 'hsphys', 'intsci', 'intecon', 'intspace', 'intenvir', 'visnhist', 'viszoo', 'vissci', 'scientod', 'scientbe', 'compwage', 'empinput', 'slfmangd', 'emptrain', 'wealth', 'defpensn', 'company', 'marcohab', 'trcourts', 'trbigbus', 'intcntct', 'goodlife', 'meovrwrk', 'stress', 'hapunhap', 'abpoorw', 'conbiz', 'conschls', 'clergvte', 'churhpow', 'afterlif', 'hell', 'miracles', 'godmeans', 'attendma', 'prayfreq', 'relactiv', 'feelrel', 'religcon', 'vistholy', 'comfort', 'relgenbar', 'govvsrel', 'difrel', 'relrlvnt', 'christns', 'muslims', 'buddhsts', 'jews', 'LETIN1A', 'partners', 'sexsex', 'numwomen', 'evpaidsx', 'evstray', 'evidu', 'evcrack', 'hivtest', 'HIVTEST1', 'sexornt', 'sexbirth', 'realrinc', 'ETH1', 'vetyears', 'dwelling', 'wordb', 'wordd', 'worde', 'wordg', 'wordj', 'wordsum', 'GENDER1', 'OLD1', 'RELATE2', 'OLD2', 'RELATE3', 'GENDER3', 'MAR3', 'OLD4', 'MAR4', 'OLD5', 'OLD6', 'OLD7', 'RELHHD3', 'hefinfo', 'rvisitor', 'visitors', 'RELHH2', 'RELHH3', 'RELSP1', 'RELSP3', 'RELSP4', 'PAISCO88', 'MAISCO88', 'SPISCO88', 'ISCO08', 'MAISCO08', 'SEI10EDUC', 'PASEI10EDUC', 'MASEI10', 'MASEI10INC', 'SPSEI10EDUC', 'COSEI10INC', 'COPRES105PLUS', 'cohort', 'WHOELSE2', 'WHOELSE6', 'intid', 'feelevel', 'intage', 'intethn', 'intsex', 'version', 'issp', 'sampcode', 'respond', 'wtss'])]\n",
      "----------------------------------------------------------------------\n",
      "\n",
      "[]\n",
      "ideology indices     ideology        dR        dL        d0\n",
      "0   0.033709  0.149266  0.144744  0.134157\n",
      "1   0.032159  0.145952  0.141637  0.134157\n",
      "2   0.022888  0.161017  0.157946  0.134157\n",
      "3   0.115046  0.161827  0.146393  0.134157\n",
      "4   0.057458  0.145560  0.137851  0.134157\n",
      "..       ...       ...       ...       ...\n",
      "95  0.003390  0.144445  0.143991  0.134157\n",
      "96 -0.029410  0.163562  0.167508  0.134157\n",
      "97  0.095597  0.163497  0.150672  0.134157\n",
      "98 -0.000343  0.143215  0.143261  0.134157\n",
      "99  0.026518  0.145982  0.142424  0.134157\n",
      "\n",
      "[100 rows x 4 columns]\n",
      "----------------------------------------------------------------------\n",
      "\n",
      "[]\n",
      "dispersion array          Qsd      Qmax\n",
      "0   0.005914  0.039197\n",
      "1   0.004557  0.031976\n",
      "2   0.005235  0.035182\n",
      "3   0.004911  0.034308\n",
      "4   0.005503  0.039232\n",
      "..       ...       ...\n",
      "95  0.004806  0.034165\n",
      "96  0.004811  0.033436\n",
      "97  0.004935  0.033293\n",
      "98  0.005737  0.040946\n",
      "99  0.006439  0.045415\n",
      "\n",
      "[100 rows x 2 columns]\n",
      "----------------------------------------------------------------------\n",
      "\n",
      "[]\n",
      "polar distances array [[0.13341919912248365, 0.13881345571428114], [0.13113822150272195, 0.13817617891264256], [0.14770140610545113, 0.15568173223424667], [0.13917704620429802, 0.1429509323014114], [0.12556483154166861, 0.13489804369935188], [0.1355930448529836, 0.1438264215329362], [0.13680238605657258, 0.13786632263896725], [0.12226382336294804, 0.13030202309379554], [0.13471731398449246, 0.13608701251976443], [0.13529321115293955, 0.1383942312930219], [0.12952114779543863, 0.1377400733121509], [0.12845960185555613, 0.13830284925478242], [0.14416041655100553, 0.1409398303441535], [0.12887300115363878, 0.13562920921916827], [0.1291665006061314, 0.13815339101680477], [0.1206308512220135, 0.12898833203076665], [0.13731795384510326, 0.13877869963024078], [0.13413853197128714, 0.14359141875188106], [0.12731761317219525, 0.1298926805209884], [0.1332959577195029, 0.14367317291956813], [0.1307970573155431, 0.13629093242499435], [0.12961792624535967, 0.14051870882679848], [0.13406534432818912, 0.13946100085803947], [0.12507545900452283, 0.13452395748548285], [0.13711617963150308, 0.13241855040542652], [0.13050291661247823, 0.1380352060409075], [0.13799083373086227, 0.1382559017796013], [0.1298641671752478, 0.12896481311907448], [0.12572321505025325, 0.13421629127950535], [0.13325193451293868, 0.13591150822145276], [0.12763211288177806, 0.12886079551720567], [0.13575805320817202, 0.13426379452337017], [0.13682566511869906, 0.1410447668459687], [0.1298811250937132, 0.13782629313775724], [0.13730306322548672, 0.145532522099398], [0.12516339492956144, 0.13507367376428464], [0.1367857697170047, 0.13561492774034045], [0.13138126500718486, 0.13467929669493242], [0.1254756687377542, 0.13414634876164422], [0.13149605053220356, 0.13284535130178], [0.13760820498244755, 0.14025151570570132], [0.12583119599734469, 0.13420662337771522], [0.12909481167966386, 0.12977071506421217], [0.12281402028906661, 0.13063991073199463], [0.12643774710097094, 0.13477402545594794], [0.12572690075762052, 0.13057549646466352], [0.1282710188712311, 0.1335077955321847], [0.13351666058162692, 0.13166856794431364], [0.12943402094965015, 0.12975361820321807], [0.13193526851307588, 0.13983914776332834], [0.1317438924869327, 0.1423104468893211], [0.12973089876439628, 0.12924567000819198], [0.1400919042019024, 0.1340797856178071], [0.1377723263411739, 0.13853223333195708], [0.14096138403956643, 0.1427016946098263], [0.1295992579642452, 0.13304861118669162], [0.13501979055063743, 0.14168394789540303], [0.12662240991263107, 0.12394034967427527], [0.13174480208671532, 0.13458112981336817], [0.13579307175907251, 0.14350684182453813], [0.12655390743608458, 0.12896968563300196], [0.1351272360426095, 0.14210351780472533], [0.14105012274481352, 0.14435185074090293], [0.14128492227627135, 0.13462035840110306], [0.13701810161028422, 0.14118722304824247], [0.12356066890401264, 0.13221710452485372], [0.13076758270012814, 0.1279984043648313], [0.13046545463046258, 0.14019548179412708], [0.13717553494430143, 0.14615447890169514], [0.13262239595476052, 0.1288375221465133], [0.1246577681084187, 0.1315194196003797], [0.14285924390922555, 0.1463608372593915], [0.13676653302726996, 0.13522489115643124], [0.12628700999501924, 0.12868357292140964], [0.1411856990484497, 0.13517968155351592], [0.13407720966681075, 0.14173877761336498], [0.12638636676014436, 0.1309853595605741], [0.1318626514757429, 0.13948714581958754], [0.13097271528374216, 0.13997512563875422], [0.1254062815435287, 0.1342081671930648], [0.14566573019192616, 0.14880228423430764], [0.1274686510081863, 0.1342335478044023], [0.1316742111189293, 0.1364952523703068], [0.13338242466327, 0.13138197538993976], [0.13304972585577401, 0.13662223214374808], [0.14071902149415302, 0.14657857355620396], [0.13730755316635793, 0.14237906861281108], [0.13571696550575932, 0.1348949385052787], [0.14502684704983193, 0.1526228968633893], [0.12465171346215222, 0.12979968508883252], [0.13737335622300284, 0.14102648676760582], [0.13089708069929665, 0.1289190381768661], [0.12697103217746158, 0.13301610815414303], [0.12674685398721555, 0.12744780666485794], [0.13777173256687097, 0.13896194813789947], [0.13369420716348307, 0.13258434382229642], [0.16289861120496532, 0.1576582092217143], [0.12926298133383157, 0.12804109538047484], [0.13717707773370005, 0.14697860625855994], [0.1320616069111307, 0.1365918481935005]]\n",
      "----------------------------------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# the following are for arrays of samples\n",
    "# multiprocessing suffices\n",
    "\n",
    "# set sammple size\n",
    "Cg.set_nsamples(100)\n",
    "    \n",
    "# computing polar_indices makes sure that dissonance matrix only takes in polar cols\n",
    "Cg.compute_polar_indices()\n",
    "dissonance_array = Cg.dissonance_matrix(outfile='GSS/results/DISSONANCE_matrix.csv')\n",
    "print(\"dissonance array:\", dissonance_array[:2])\n",
    "print('----------------------------------------------------------------------\\n')\n",
    "\n",
    "# random mask and reconstruction\n",
    "recon_df = Cg.randomMaskReconstruction_multiple('GSS/results/randomMaskRecon_test.csv')\n",
    "print(\"reconstruction results\", recon_df[:2])\n",
    "print('----------------------------------------------------------------------\\n')\n",
    "\n",
    "# ideology indices\n",
    "ideology_index = Cg.compute_DLI_samples('ideology','GSS/results/ideology.csv')\n",
    "print(\"ideology indices\", ideology_index)\n",
    "print('----------------------------------------------------------------------\\n')\n",
    "\n",
    "# dispersion\n",
    "local_dispersion = Cg.compute_DLI_samples('dispersion', 'GSS/results/dispersion_test.csv')\n",
    "print(\"dispersion array\", local_dispersion)\n",
    "print('----------------------------------------------------------------------\\n')\n",
    "\n",
    "# polar distances\n",
    "polar_array = Cg.polarDistance_multiple('GSS/results/polarDistance_multiple_test.csv')\n",
    "print(\"polar distances array\",polar_array)\n",
    "print('----------------------------------------------------------------------\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute qdistance matrix for small set of samples\n",
    "# set nsamples first to set the number of samples to be included in matrix\n",
    "distance_matrix=Cg.distfunc_multiples(\"examples_results/distfunc_multiples_testing.csv\")\n",
    "print(\"local distance matrix:\", distance_matrix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write files to compute qdistance matrix for large set of samples\n",
    "# execute generated shell script to run mpi parallelization on midway\n",
    "Cg.dmat_filewriter(\"GSS_cognet.py\", \"examples_data/gss_2018.joblib\",\n",
    "                   MPI_SETUP_FILE=\"GSS_mpi_setup.sh\",\n",
    "                   MPI_RUN_FILE=\"GSS_mpi_run.sh\",\n",
    "                   MPI_LAUNCHER_FILE=\"GSS_mpi_launcher.sh\",\n",
    "                   YEARS='2018',NODES=4,T=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## embedding\n",
    "## embed generated Qdist Matrix\n",
    "Cg.year = '2018'\n",
    "Cg.embed('examples_results/distfunc_multiples_testing.csv', 'embed', 'examples_results/',EMBED_BINARY='cognet/cognet/bin/__embed__.so')\n",
    "#pd.read_csv('examples_results/embed_E_2018.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
