#!/usr/bin/env python
import subprocess
import nose
import numpy
import operator
import random

@nose.plugins.attrib.attr('slow')
def testValuesByNodeSimulationsReport():
    """
    >>> command = "python src/get_simulations_background.py -t values_by_node -f test/test_data/simulations/binary/globalpops/"
    >>> output = subprocess.check_output(command.split(), stderr=subprocess.PIPE)
    >>> output = output.splitlines()

    >>> print output[0]
    genotype name window whole_gene_nsnps network_gene network_continent network_distance_definition cont_str n_datapoints degree closeness betweenness

    >>> output.sort()

    #NOTE: the blankline at the end is generated by subprocess.check_output, not by the script
    >>> for line in output[:5]:
    ...     print line #doctest: +NORMALIZE_WHITESPACE 
    0001111111  neutral_sim window_35   360 neutral_sim glob    1   ASN 21  0   0.0323  0.0
    0001111111  neutral_sim_ASN window_35   360 neutral_sim ASN 1   ASN 21  0   0.0833  0.0
    0010011111  neutral_sim window_35   360 neutral_sim glob    1   ASN 181 2   0.0932  22.0
    0010011111  neutral_sim_ASN window_35   360 neutral_sim ASN 1   ASN 181 1   0.0909  0.0
    0010101111  neutral_sim window_35   360 neutral_sim glob    1   ASN 1   1   0.0333  0.0


    """

def testDebugSimulationsBackgroundResampling():
    """
    Check that the individuals subsampled are always the same.

    >>> command = "python src/get_simulations_background.py  -f ./test/test_data/simulations/binary/globalpops_allpops/ --random_seed 2 -n 50 -t individuals_line"

    >>> output = subprocess.check_output(command.split(), stderr=subprocess.PIPE)

    #NOTE: the blankline at the end is generated by subprocess.check_output, not by the script
    >>> output_lines = output.split('\\n')

    >>> for line in output_lines: #doctest: +NORMALIZE_WHITESPACE 
    ...     print line[0:100]
    <BLANKLINE>
    -----neutral_sim EUR  hap0_pop1 EUR pop1  hap1_pop1 EUR pop1  hap2_pop1 EUR pop1  hap3_pop1 EUR pop1
    -----neutral_sim EUR  hap0_pop1 EUR pop1  hap1_pop1 EUR pop1  hap2_pop1 EUR pop1  hap3_pop1 EUR pop1
    -----neutral_sim EUR  hap0_pop1 EUR pop1  hap1_pop1 EUR pop1  hap2_pop1 EUR pop1  hap3_pop1 EUR pop1
    <BLANKLINE>

    >>> output_lines = output_lines[1:-1]
    >>> output_fields = [line.split() for line in output_lines]

    # Check that all the columns, except for the ones starting with "--", have the same values for all the output lines.
    # This means that all the selected individuals are the same, for each line.

    # Select 500 random columns:
    >>> min_len = min(len(output_fields[0]), len(output_fields[1]), len(output_fields[2]))
    >>> random_columns = random.sample(xrange(min_len), 500)
    >>> fields_getter = operator.itemgetter(*random_columns)
    >>> sim1 = fields_getter(output_fields[0])
    >>> sim2 = fields_getter(output_fields[1])
    >>> sim3 = fields_getter(output_fields[2])
    >>> for column_id in xrange(len(sim1)):
    ...     if not sim1[column_id].startswith("----"):
    ...        if not sim1[column_id] == sim2[column_id] == sim3[column_id]:
    ...             print "not matching values", sim1[column_id], sim2[column_id], sim3[column_id]
    """


def testValuesByGeneSimulationsReport():
    """
    >>> command = "python src/get_simulations_background.py -f test/test_data/simulations/binary/globalpops/"
    >>> output = subprocess.check_output(command.split(), stderr=subprocess.PIPE)
    >>> output = output.splitlines()
    >>> print output[0]
    gene continent distance_definition n_snps central_snp chromosome central_snp_position upstream_position downstream_position region_size distance_from_upstream_margin distance_from_downstream_margin n_datapoints av_datapoints_per_node n_vertices n_edges n_components av_path_length var_path_length diameter av_w_path_length av_w_path_length_inv av_degree var_degree median_degree max_degree av_w_degree av_w_degree_inv av_closeness var_closeness median_closeness max_closeness density av_betweenness var_betweenness median_betweenness max_betweenness window whole_gene_nsnps
    >>> output = output[1:]
    >>> output.sort(key=lambda x:x[-3:])

    >>> outputvalues = [v.split() for v in output]
    >>> [(v[13]) for v in outputvalues] == ['58.7742', '68.9091', '47.6667', '41.0', '15.6522', '15.0', '12.0', '10.0', '8.8889', '6.0', '17.1429']
    True

    >>> mean_av_datapoints = numpy.mean([float(v[13]) for v in outputvalues])
    >>> print round(mean_av_datapoints, 4)
    27.3667

    >>> for line in output:
    ...     print line  #doctest: +NORMALIZE_WHITESPACE
    neutral_sim glob 1 10 NA NA nan nan nan nan nan nan 1822 58.7742 31 30 6 3.7374 3.2124 9 0.4461 63.1098 1.9355 2.1249 2.0 6 0.1043 721.1816 0.0842 0.0008 0.098 0.1095 0.0645 24.5484 1928.5269 6.0 155.1667 window_35 360
    neutral_sim EUR 1 10 NA NA nan nan nan nan nan nan 758 68.9091 11 9 2 1.6667 0.2308 2 0.7175 3.921 1.6364 2.2314 1.0 6 0.3411 105.8128 0.162 0.0009 0.1818 0.2 0.1636 1.6364 18.595 0.0 15.0 window_35 360
    neutral_sim ASN 1 10 NA NA nan nan nan nan nan nan 572 47.6667 12 5 7 1.1667 0.1667 2 0.1809 13.7025 0.8333 0.3056 1.0 2 0.0775 70.2283 0.0911 0.0 0.0909 0.1 0.0758 0.0833 0.0764 0.0 1.0 window_35 360
    neutral_sim AFR 1 10 NA NA nan nan nan nan nan nan 492 41.0 12 10 3 2.7838 2.5075 6 0.7313 32.4049 1.6667 0.7222 2.0 3 0.1651 232.1307 0.1635 0.002 0.1792 0.2075 0.1515 5.5 45.4583 1.75 16.0 window_35 360
    neutral_sim glob 1 10 NA NA nan nan nan nan nan nan 360 15.6522 23 20 6 2.935 2.0121 7 0.4047 41.6849 1.7391 2.1059 2.0 7 0.1104 104.895 0.0884 0.0008 0.1038 0.117 0.0791 10.3478 311.6616 1.0 84.0 window_35 361
    neutral_sim EUR 1 10 NA NA nan nan nan nan nan nan 120 15.0 8 4 4 1.8 0.6222 3 0.4667 9.2108 1.0 1.0 1.0 3 0.1323 14.0625 0.1874 0.0024 0.2154 0.2414 0.1429 1.0 3.25 0.0 5.0 window_35 361
    neutral_sim ASN 1 10 NA NA nan nan nan nan nan nan 120 12.0 10 7 3 2.0 1.0588 4 0.3551 16.6272 1.4 0.64 1.0 3 0.1292 38.7697 0.1539 0.0009 0.1699 0.1875 0.1556 1.8 6.96 0.0 7.0 window_35 361
    neutral_sim AFR 1 10 NA NA nan nan nan nan nan nan 120 10.0 12 12 1 2.8182 1.9049 6 0.5909 18.9151 2.0 1.3333 2.0 5 0.2194 62.9627 0.3739 0.0073 0.3798 0.55 0.1818 10.0 137.1667 10.0 41.5 window_35 361
    neutral_sim glob 1 10 NA NA nan nan nan nan nan nan 240 8.8889 27 21 8 2.7582 1.8965 6 0.1165 99.4246 1.5556 0.9877 1.0 4 0.0481 141.2267 0.051 0.0001 0.0433 0.0645 0.0598 5.9259 99.013 0.0 30.5 window_37 383
    neutral_sim EUR 1 10 NA NA nan nan nan nan nan nan 120 6.0 20 18 4 2.8182 1.8516 6 0.2301 51.2801 1.8 1.06 2.0 4 0.0825 89.0 0.0911 0.0006 0.1033 0.1152 0.0947 8.0 117.075 3.0 30.5 window_37 383
    neutral_sim ASN 1 10 NA NA nan nan nan nan nan nan 120 17.1429 7 3 4 1.0 0.0 1 0.3167 3.7221 0.8571 0.1224 1.0 1 0.1357 18.08 0.1633 0.0001 0.1667 0.1667 0.1429 0.0 0.0 0.0 0.0 window_37 383


    """

@nose.plugins.attrib.attr('slow')
def testValuesByGeneSimulationsBigWindowReport():
    """
    >>> command = "python src/get_simulations_background.py -f test/test_data/simulations/binary/globalpops/ -w 30"
    >>> output = subprocess.check_output(command.split(), stderr=subprocess.PIPE)
    >>> print output[:1000] #doctest: +NORMALIZE_WHITESPACE
    gene continent distance_definition n_snps central_snp chromosome central_snp_position upstream_position downstream_position region_size distance_from_upstream_margin distance_from_downstream_margin n_datapoints av_datapoints_per_node n_vertices n_edges n_components av_path_length var_path_length diameter av_w_path_length av_w_path_length_inv av_degree var_degree median_degree max_degree av_w_degree av_w_degree_inv av_closeness var_closeness median_closeness max_closeness density av_betweenness var_betweenness median_betweenness max_betweenness window whole_gene_nsnps
    neutral_sim glob 1 30 NA NA nan nan nan nan nan nan 360 4.9315 73 20 53 1.9535 1.2359 5 0.0408 128.0379 0.5479 0.5491 0.0 4 0.0076 95.3552 0.0139 0.0 0.0137 0.0151 0.0076 0.5616 6.027 0.0 17.0 window_11 371
    neutral_sim EUR 1 30 NA NA nan nan nan nan nan nan 120 8.5714 14 3 11 1.0 0.0 1 0.1806 9.972 0.4286 0.2449 0.0 1 0.0387 20.5179 0.0738 0.0 0.0714 0.0769 0.033 0.0 0.0 0.0 0.0 window_11 371
    neutral_sim ASN 1 30 NA NA nan

    """
def testDebugReportEvenWindowIndividuals():
    """
    >>> command = "python src/get_simulations_background.py -f test/test_data/simulations/binary/globalpops/ -w 6 -t individuals "
    >>> output = subprocess.check_output(command.split(), stderr=subprocess.PIPE)
    >>> print output[:200] #doctest: +NORMALIZE_WHITESPACE
    <BLANKLINE>
    -----
    neutral_sim EUR
    hap1_pop1 EUR pop1
    hap2_pop1 EUR pop1
    hap3_pop1 EUR pop1
    hap4_pop1 EUR pop1
    hap5_pop1 EUR pop1
    hap6_pop1 EUR pop1
    hap7_pop1 EUR pop1
    hap8_pop1 EUR pop1
    hap9_pop1 EUR pop1
    hap10_

    """

def testDebugReportEvenWindow():
    """
    >>> command = "python src/get_simulations_background.py -f test/test_data/simulations/binary/globalpops/ -w 6"
    >>> output = subprocess.check_output(command.split(), stderr=subprocess.PIPE)
    >>> print output[:1000] #doctest: +NORMALIZE_WHITESPACE
    gene continent distance_definition n_snps central_snp chromosome central_snp_position upstream_position downstream_position region_size distance_from_upstream_margin distance_from_downstream_margin n_datapoints av_datapoints_per_node n_vertices n_edges n_components av_path_length var_path_length diameter av_w_path_length av_w_path_length_inv av_degree var_degree median_degree max_degree av_w_degree av_w_degree_inv av_closeness var_closeness median_closeness max_closeness density av_betweenness var_betweenness median_betweenness max_betweenness window whole_gene_nsnps
    neutral_sim glob 1 6 NA NA nan nan nan nan nan nan 360 30.0 12 10 4 2.1111 0.7873 4 0.4461 12.7717 1.6667 1.8889 1.5 4 0.2139 34.6374 0.1774 0.0031 0.2019 0.2292 0.1515 3.3333 23.9815 0.6667 14.0 window_58 359
    neutral_sim EUR 1 6 NA NA nan nan nan nan nan nan 120 20.0 6 4 2 1.6 0.2667 2 0.4333 9.1389 1.3333 1.5556 1.0 4 0.1806 17.4565 0.3675 0.0098 0.3846 0.5 0.2667 1.0 5.0 0.0 6.0 window_58 359
    neutral_sim ASN 1 6 NA NA n


    """
def testDebugReportOddWindow():
    """

    >>> import glob
    >>> for testfile in sorted(glob.glob("test/test_data/simulations/binary/globalpops/*")): print testfile
    test/test_data/simulations/binary/globalpops/bestfit_neutral_replica_1.hap-0.binary.gz
    test/test_data/simulations/binary/globalpops/bestfit_neutral_replica_10.hap-0.binary.gz
    test/test_data/simulations/binary/globalpops/bestfit_neutral_replica_2.hap-0.binary.gz

    >>> command = "python src/get_simulations_background.py -f test/test_data/simulations/binary/globalpops/ -w 5"
    >>> output = subprocess.check_output(command.split(), stderr=subprocess.PIPE)
    >>> print output[:1000] #doctest: +NORMALIZE_WHITESPACE
    gene continent distance_definition n_snps central_snp chromosome central_snp_position upstream_position downstream_position region_size distance_from_upstream_margin distance_from_downstream_margin n_datapoints av_datapoints_per_node n_vertices n_edges n_components av_path_length var_path_length diameter av_w_path_length av_w_path_length_inv av_degree var_degree median_degree max_degree av_w_degree av_w_degree_inv av_closeness var_closeness median_closeness max_closeness density av_betweenness var_betweenness median_betweenness max_betweenness window whole_gene_nsnps
    neutral_sim glob 1 5 NA NA nan nan nan nan nan nan 360 40.0 9 6 3 2.1905 1.0619 4 0.9257 7.8792 1.3333 1.3333 1.0 4 0.283 15.3591 0.2261 0.0042 0.25 0.2963 0.1667 2.7778 18.1728 0.0 12.0 window_70 359
    neutral_sim EUR 1 5 NA NA nan nan nan nan nan nan 120 24.0 5 3 2 1.5 0.3 2 1.0833 2.0809 1.2 0.96 1.0 3 0.4333 9.5351 0.38 0.0096 0.4 0.5 0.3 0.6 1.44 0.0 3.0 window_70 359
    neutral_sim ASN 1 5 NA NA nan nan nan nan nan nan 12

    """
#@nose.plugins.attrib.attr('slow')
def testValuesByNodeSimulationsReportDifferentNIndividuals():
    """

    NOTE: this test may fail in some systems. I believe it depends on the random_seed, and on the version of the libraries installed.

    >>> command = "python src/get_simulations_background.py -t values_by_gene -f test/test_data/simulations/binary/globalpops/ -i 30 --random_seed 2" 
    >>> output = subprocess.check_output(command.split(), stderr=subprocess.PIPE)

    >>> output = output.splitlines()
    >>> print output[0]
    gene continent distance_definition n_snps central_snp chromosome central_snp_position upstream_position downstream_position region_size distance_from_upstream_margin distance_from_downstream_margin n_datapoints av_datapoints_per_node n_vertices n_edges n_components av_path_length var_path_length diameter av_w_path_length av_w_path_length_inv av_degree var_degree median_degree max_degree av_w_degree av_w_degree_inv av_closeness var_closeness median_closeness max_closeness density av_betweenness var_betweenness median_betweenness max_betweenness window whole_gene_nsnps

    >>> output = output[1:]
    >>> output.sort(key=lambda x:x[-3:])

    >>> outputvalues = [v.split() for v in output]
    >>> print sorted([(float(v[13])) for v in outputvalues])
    [2.0, 2.1429, 2.3077, 2.5, 3.0, 3.75, 4.2857, 4.2857, 7.5, 7.5, 8.8889, 10.0, 15.6522, 58.7742]


    NOTE: oldvalues in henikoff:
    [2.1429, 2.3077, 2.5, 2.7273, 3.0, 3.3333, 4.2857, 4.2857, 5.0, 5.0, 6.0, 8.8889, 15.6522, 58.7742]
    [1.875, 2.1429, 2.5, 3.0, 3.0, 3.3333, 4.2857, 4.2857, 5.0, 6.0, 6.0, 8.8889, 15.6522, 58.7742]
    NOTE: oldvalues in laptop:
    [10.0, 15.6522, 2.0, 2.5, 3.0, 3.0, 3.3333, 3.3333, 3.3333, 3.75, 5.0, 58.7742, 6.0, 8.8889]

    >>> mean_av_datapoints = numpy.mean([float(v[13]) for v in outputvalues])
    >>> print round(mean_av_datapoints, 4)
    9.4705

    NOTE: on henikoff, before fixing the order of lines in output (commit 1634:a4aeff4bd7d9)
    9.1832

    NOTE: on laptop:
    8.9099

    #NOTE: the blankline at the end is generated by subprocess.check_output, not by the script
    >>> for line in output[:5]:
    ...     print line  #doctest: +NORMALIZE_WHITESPACE 
    neutral_sim glob 1 10 NA NA nan nan nan nan nan nan 1822 58.7742 31 30 6 3.7374 3.2124 9 0.4461 63.1098 1.9355 2.1249 2.0 6 0.1043 721.1816 0.0842 0.0008 0.098 0.1095 0.0645 24.5484 1928.5269 6.0 155.1667 window_35 360
    neutral_sim global_sub 1 10 NA NA nan nan nan nan nan nan 30 2.3077 13 5 8 1.4444 0.2778 2 0.2333 12.3333 0.7692 0.7929 1.0 3 0.0692 14.8901 0.0868 0.0001 0.0902 0.1 0.0641 0.3077 0.6746 0.0 3.0 window_35 360
    neutral_sim EUR 1 10 NA NA nan nan nan nan nan nan 30 7.5 4 2 2 1.3333 0.3333 2 0.6222 2.8718 1.0 0.5 1.0 2 0.2333 11.25 0.4018 0.0085 0.4286 0.5 0.3333 0.25 0.1875 0.0 1.0 window_35 360
    neutral_sim ASN 1 10 NA NA nan nan nan nan nan nan 30 3.75 8 3 5 1.25 0.25 2 0.175 9.8214 0.75 0.4375 1.0 2 0.0583 12.8125 0.1441 0.0003 0.1429 0.1667 0.1071 0.125 0.1094 0.0 1.0 window_35 360
    neutral_sim AFR 1 10 NA NA nan nan nan nan nan nan 30 7.5 4 0 4 nan 0.0 0 nan nan 0.0 0.0 0.0 0 0.0 0.0 0.25 0.0 0.25 0.25 0.0 0.0 0.0 0.0 0.0 window_35 360


    """

