pyMez.Code.DataHandlers.GeneralModels API documentation
Description: Module that contains general data models and functions for handling them

Top

# pyMez.Code.DataHandlers.GeneralModels module

Module that contains general data models and functions for handling them

Show source ≡

    #-----------------------------------------------------------------------------
    # Name:        GeneralModels
    # Purpose:     To create base classes
    # Author:      Aric Sanders
    # Created:     2/24/2016
    # License:     MIT License
    #-----------------------------------------------------------------------------
    """ Module that contains general data models and functions for handling them """
    #Todo: add a handler for when self.data contains a string with the data_delimiter in it

    #-----------------------------------------------------------------------------
    # Standard Imports
    from types import *
    import os
    import pickle
    #-----------------------------------------------------------------------------
    # Third Party Imports
    try:
        from pyMez.Code.Utils.Alias import *
        METHOD_ALIASES=1
    except:
        print("The module pyMez.Code.Utils.Alias was not found")
        METHOD_ALIASES=0
        pass
    try:
        from pyMez.Code.Utils.Names import auto_name
        DEFAULT_FILE_NAME=None
    except:
        print("The function auto_name in pyMez.Code.Utils.Names was not found")
        print("Setting Default file name to New_Data_Table.txt")
        DEFAULT_FILE_NAME='New_Data_Table.txt'
        pass
    try:
        import numpy as np
    except:
        np.ndarray='np.ndarray'
        print("Numpy was not imported")
        pass
    #-----------------------------------------------------------------------------
    # Module Constants
    TESTS_DIRECTORY=os.path.join(os.path.dirname(os.path.realpath(__file__)),'Tests')
    #-----------------------------------------------------------------------------
    # Module Functions
    def print_comparison(var_1,var_2):
        """If var_1==var_2 prints True, else Prints false and a string representation of the 2 vars"""
        print(var_1==var_2)
        if var_1!=var_2:
            print("The value of variable 1 is :")
            print(var_1)
            print("-"*80)
            print("The value of variable 2 is :")
            print(var_2)
            print("-"*80)

    def check_arg_type(arg,arg_type):
        "Checks argument and prints out a statement if arg is not type"
        if type(arg) is arg_type:
            return
        else:
            print("{0} was not {1}".format(arg,arg_type))

    def string_list_collapse(list_of_strings,string_delimiter='\n'):
        """ Makes a list of strings a single string"""
        check_arg_type(list_of_strings,ListType)
        if string_delimiter is None:
            string_delimiter=""
        out_string=''
        for index,item in enumerate(list_of_strings):
            if index is len(list_of_strings)-1:
                out_string=out_string+item
            else:
                out_string=out_string+item+string_delimiter
        return out_string

    def list_to_string(row_list,data_delimiter=None,row_formatter_string=None,begin=None,end=None):
        """Given a list of values returns a string, if row_formatter is specifed
         it uses it as a template, else uses data delimiter. Inserts data_delimiter between each list element. An optional
        begin and end wrap the resultant string. (i.e ['1','2','3']-> 'begin+'1'+','+'2'+','+'3'+'end') end defaults
        to \n to have nothing at the end use ''
        """
        check_arg_type(row_list,ListType)
        if data_delimiter is None:
            data_delimiter=','
        string_out=""
        if row_formatter_string is None:
            for index,item in enumerate(row_list):
                if index is len(row_list)-1:
                    string_out=string_out+str(item)
                else:
                    string_out=string_out+str(item)+data_delimiter
        else:
            string_out=row_formatter_string.format(*row_list,delimiter=data_delimiter)
        if end is None:
            end="\n"
        if begin is None:
            begin=""
        return begin+string_out+end

    def list_list_to_string(list_lists,data_delimiter=None,row_formatter_string=None,line_begin=None,line_end=None):
        """Repeatedly calls list to string on each element of a list and string adds the result
        . ie coverts a list of lists to a string. If line end is None the value defaults to "\n", for no seperator use ''
        """
        if line_end is None:
            line_end="\n"
        check_arg_type(list_lists,ListType)
        string_out=""
        for index,row in enumerate(list_lists):
            if index==len(list_lists)-1:
                if line_end is "\n":
                    last_end=""
                else:
                    last_end=re.sub("\n","",line_end,count=1)
                string_out=string_out+list_to_string(row,data_delimiter=data_delimiter,
                                                 row_formatter_string=row_formatter_string,
                                                 begin=line_begin,end=last_end)
            else:
                string_out=string_out+list_to_string(row,data_delimiter=data_delimiter,
                                                 row_formatter_string=row_formatter_string,
                                                 begin=line_begin,end=line_end)
        return string_out

    def line_comment_string(comment,comment_begin=None,comment_end=None):
        "Creates a comment optionally wrapped with comment_begin and comment_end, meant for a single string comment "
        check_arg_type(comment,StringType)
        string_out=""
        if comment_begin is None:
            if comment_end is None:
                string_out=comment
            else:
                string_out=comment+comment_end
        else:
            if comment_end is None:
                string_out=comment_begin+comment
            else:
                string_out=comment_begin+comment+comment_end
        return string_out

    def line_list_comment_string(comment_list,comment_begin=None,comment_end=None,block=False):
        """Creates a string with each line wrapped in comment_begin and comment_end, by repeatedly calling
        line_comment_string,
        or the full string wrapped with block_comment_begin and block_comment_end if block is set to True. Meant
        to deal with a list of comment strings"""
        check_arg_type(comment_list,ListType)
        string_out=""
        if block:
            string_out=comment_begin+string_list_collapse(comment_list,string_delimiter='\n')+comment_end
        else:
            for item in comment_list:
                string_out=string_out+line_comment_string(item,comment_begin=comment_begin,comment_end=comment_end)
        return string_out

    def ensure_string(input_object,  list_delimiter="",  end_if_list=""):
        """Returns a string given an object. If the object is a string just returns it,
        if it is a list of strings, returns a collapsed version. If is another type of object returns str(object).
          If all else fails it returns an empty string"""
        string_out=""
        try:
            if type(input_object) in StringTypes:
                string_out=input_object
            elif type(input_object) in [ListType,np.ndarray]:
                if type(input_object[0]) in [ListType,np.ndarray]:
                    string_out=list_list_to_string(input_object,data_delimiter=list_delimiter,end=end_if_list)
                else:
                    string_out=list_to_string(input_object,data_delimiter=list_delimiter,end=end_if_list)
            else:
                string_out=str(input_object)
        except:
            pass
        return string_out

    def strip_tokens(string_list,*remove_tokens):
        """Strips all tokens in the list remove_tokens from a list of strings
        Returns the list with less elements if the tokens contained "\n". Now newline characters are returned in the list
        elements. Meant to reverse the action of adding tokens to a list of strings"""
        temp_string=string_list_collapse(string_list,string_delimiter="")
        remove_list=[]
        for token in remove_tokens:
            if token:
                # we can't strip endlines f
                if token=="\n":
                    print("Warning \\n is in the remove tokens")
                    break
                remove_list.append(token)
                # print remove_list
        try:
            for item in remove_list:
                temp_string=temp_string.replace(item,"")
                #print temp_string
        except:
            print("Strip Tokens Did not work")
            pass
        # spliting using "\n" seems to give an extra empty element at the end always
        new_string_list=temp_string.splitlines()
        # now we add endlines back in for consistency
        for index,line in enumerate(new_string_list):
            new_string_list[index]=line+'\n'
        return new_string_list

    def strip_begin_end_tokens(string_list,begin_token=None,end_token=None):
        """Strips out tokens at the begining and ending of a list of strings. Meant to reverse the
        action of "begin_data_token", etc. This does not work with the end_token's because of where the \n is."""
        check_arg_type(string_list,ListType)
        out_list=string_list
        # check the first line to see if it is equal to begin token, if so remove it
        # if any token is None ignore it
        # replace it and leave the line alone otherwize
        if begin_token is None or begin_token is "":
            pass
        else:
            #print("The {0} var is {1}".format('string_list',string_list))
            #print("The {0} var is {1}".format('string_list[0]',string_list[0]))
            #print_comparison(string_list[0],begin_token)
            if string_list[0]==begin_token:
                out_list.pop(0)
            else:
               out_list[0]=out_list[0].replace(begin_token,"")

        if end_token is None or end_token is "":
            pass
        else:
            #print_comparison(string_list[-1],end_token)
            if out_list[-1]==end_token:
                out_list.pop(-1)
            else:
                out_list[-1]=out_list[-1].replace(end_token,"")
        # print("The {0} var is {1}".format('out_list',out_list))
        return out_list

    def strip_line_tokens(string,begin_token=None,end_token=None):
        """Strips a begin and end token if present from an inputted string, meant to remove line_comments"""
        check_arg_type(string,StringType)
        string_out=string
        try:
            if begin_token is not None:
                string_out=string_out.replace(begin_token,'')
            if end_token is not None:
                string_out=string_out.replace(end_token,'')
        except:
            pass
        return string_out

    def strip_all_line_tokens(string_list,begin_token=None,end_token=None):
        """Strips all line tokens from a list of strings, meant  to reverse the action of line_list_comment_string
        with block=false"""
        check_arg_type(string_list,ListType)
        stripped_list=[]
        for row in string_list:
            check_arg_type(row,StringType)
            stripped_list.append(strip_line_tokens(row,begin_token=begin_token,end_token=end_token))
        return stripped_list

    def split_row(row_string,delimiter=None,escape_character=None):
        """Splits a row given a delimiter, and ignores any delimiters after an escape character
        returns a list. If the string is unsplit returns a list of length 1"""
        check_arg_type(row_string,StringType)
        if delimiter is None:
            row_list=[row_string]
            return row_list
        if escape_character is None:
            row_list=row_string.split(delimiter)
        else:
            temp_row_string=row_string.replace(escape_character+delimiter,'TempPlaceHolder')
            temp_row_list=temp_row_string.split(delimiter)
            for item in temp_row_list:
                item.replace('TempPlaceHolder',escape_character+delimiter)
            row_list=temp_row_list
        return row_list

    def split_all_rows(row_list,delimiter=None,escape_character=None):
        """Splits all rows in a list of rows and returns a 2d list """
        if type(row_list) is not ListType:
            print("Split row argument (%s) was not a list"%str(row_list))
            return row_list
        out_list=[]
        for row in row_list:
            out_list.append(split_row(row,delimiter=delimiter,escape_character=escape_character))
        return out_list

    def convert_row(row_list_strings,column_types=None):
        """Converts a row list of strings to native
        python types using a column types list"""

        if column_types is None or len(row_list_strings) != len(column_types):
            #print("Convert row could not convert {0} using {1}".format(row_list_strings,column_types))
            return row_list_strings
        else:
            out_row=row_list_strings
            for index,column_type in enumerate(column_types):
                if re.match('int',column_type,re.IGNORECASE):
                    out_row[index]=int(row_list_strings[index])
                elif re.match('float',column_type,re.IGNORECASE):
                    out_row[index]=float(row_list_strings[index])
                elif re.match('str|char',column_type,re.IGNORECASE):
                    #print_comparison(row_list_strings[index],out_row[index])
                    out_row[index]=str(row_list_strings[index])
                elif re.match('com',column_type,re.IGNORECASE):
                    out_row[index]=complex(row_list_strings[index])
                elif re.match('list',column_type,re.IGNORECASE):
                    out_row[index]=list(row_list_strings[index])
                elif re.match('dict',column_type,re.IGNORECASE):
                    out_row[index]=dict(row_list_strings[index])
                else:
                    out_row[index]=row_list_strings[index]
        return out_row

    def convert_all_rows(list_rows,column_types=None):
        "Converts all the rows (list of strings) in a list of rows using column types "
        check_arg_type(list_rows,ListType)
        out_list=[]
        for index,row in enumerate(list_rows):
            out_list.append(convert_row(row,column_types))
        return out_list

    def insert_inline_comment(list_of_strings,comment="",line_number=None,string_position=None,begin_token='(*',end_token='*)'):
        "Inserts an inline comment in a list of strings, location is determined by line_number and string_position"
        if line_number is None or string_position is None:
            print("inline comment must have both line number and string position")
            return
        if begin_token is None or end_token is None:
            print("inline comment must have both a begin and end token")
            return
        inline_comment=begin_token+comment+end_token
        #make sure there are no end lines
        inline_comment=inline_comment.replace('\n','')
        if string_position in [-1,'EOL','eol']:
            list_of_strings[line_number]=list_of_strings[line_number]+inline_comment
        else:
            list_of_strings[line_number]=list_of_strings[line_number][:string_position]+inline_comment+list_of_strings[line_number][string_position:]
        return list_of_strings

    def collect_inline_comments(list_of_strings,begin_token=None,end_token=None):
        """Reads a list of strings and returns all of the inline comments in a list.
        Output form is ['comment',line_number,string_location] returns None  if there are none or tokens are set to None"""
        if begin_token in [None] and end_token in [None]:
            return None
        match=re.compile('{0}(?P<inline_comments>.+){1}'.format(re.escape(begin_token),re.escape(end_token)))
        inline_comment_list=[]
        for index,line in enumerate(list_of_strings):
            comment_match=re.search(match,line)
            if comment_match:
                inline_comment_list.append([comment_match.group('inline_comments'),index,comment_match.start()])
        if inline_comment_list:
            return inline_comment_list
        else:
            return None

    def strip_inline_comments(list_of_strings,begin_token='(*',end_token='*)'):
        "Removes inline coments from a list of strings"
        if begin_token in [None] and end_token in [None]:
            return list_of_strings
        match=re.compile('{0}(?P<inline_comments>.+){1}'.format(re.escape(begin_token),re.escape(end_token)))
        out_list=[]
        for index,line in enumerate(list_of_strings):
            out_list.append(re.sub(match,'',line))
        return out_list

    def read_schema(file_path,format=None):
        """Reads in a schema and returns it as a python dictionary, the default format is a single string"""
        if format in[None,'python','pickle']:
            schema=pickle.load(open(file_path,'rb'))
        elif format in ['txt','text','.txt']:
            #Todo fix the other formats
            schema={}
            in_file=open(file_path,'r')
            in_lines=[]
            for line in in_file:
                in_lines.append(line)
                schema[line.split(":")[0]]=line.split(":")[1].replace("\\n","\n")
            #in_dictionary=dict(*str(in_dictionary).split(","))
        return schema
    #-----------------------------------------------------------------------------
    # Module Classes
    class AsciiDataTable():
        """ An AsciiDatable is a generalized model of a data table with optional header,
        column names,rectangular array of data, and footer """
        def __init__(self,file_path=None,**options):
            " Initializes the AsciiDataTable class "
            # This is a general pattern for adding a lot of options
            defaults={"data_delimiter":None,
                      "column_names_delimiter":None,
                      "specific_descriptor":'Data',
                      "general_descriptor":'Table',
                      "directory":None,
                      "extension":'txt',
                      "comment_begin":None,
                      "comment_end":None,
                      "inline_comment_begin":None,
                      "inline_comment_end":None,
                      "block_comment_begin":None,
                      "block_comment_end":None,
                      "footer_begin_line":None,
                      "footer_end_line":None,
                      "header_begin_line":None,
                      "header_end_line":None,
                      "column_names_begin_line":None,
                      "column_names_end_line":None,
                      "data_begin_line":None,
                      "data_end_line":None,
                      "footer_begin_token":None,
                      "footer_end_token":None,
                      "header_begin_token":None,
                      "header_end_token":None,
                      "column_names_begin_token":None,
                      "column_names_end_token":None,
                      "data_begin_token":None,
                      "data_end_token":None,
                      "metadata_delimiter":None,
                      "metadata_key_value_delimiter":None,
                      "header_line_types":None,
                      "column_types":None,
                      "column_descriptions":None,
                      "column_units":None,
                      "footer_line_types":None,
                      "header":None,
                      "column_names":None,
                      "data":None,
                      "footer":None,
                      "inline_comments":None,
                      "row_begin_token":None,
                      "row_end_token":None,
                      "row_formatter_string":None,
                      "empty_value":None,
                      "escape_character":None,
                      "data_table_element_separator":'\n',
                      "treat_header_as_comment":None,
                      "treat_footer_as_comment":None,
                      "metadata":None
                      }
            #some of the options have the abiltiy to confilct with each other, so there has to be a
            #built-in way to determine the precedence of each option, for import lines first, then begin and then end
            self.options={}
            for key,value in defaults.iteritems():
                self.options[key]=value
            for key,value in options.iteritems():
                self.options[key]=value
            self.elements=['header','column_names','data','footer','inline_comments']
            #Define Method Aliases if they are available
            #unqualified exec is not allowed in function '__init__' because it contains a nested function with free variables
            # This is because __init__ has nested functions
            # if METHOD_ALIASES:
            #     for command in alias(self):
            #         exec(command)
            if file_path is None:
                #create a new data table
                if DEFAULT_FILE_NAME is None:
                    self.name=auto_name(self.options["specific_descriptor"],
                                        self.options["general_descriptor"],
                                        self.options["directory"],
                                        self.options["extension"])
                    if self.options['directory'] is None:
                        self.path=self.name
                    else:
                        self.path=os.path.join(self.options["directory"],self.name)
                #Now we see if the table has been defined in the options
                # We should reset the self.options versions to None after this so as to not recreate or we
                # can use it as a cache and add a method reset_table which either redoes the below or reloads the saved file
                self.header=self.options["header"]
                self.column_names=self.options["column_names"]
                self.data=self.options["data"]
                self.footer=self.options["footer"]
                self.metadata=self.options["metadata"]
                self.inline_comments=self.options["inline_comments"]
                self.initial_state=[self.options["header"],self.options["column_names"],
                                    self.options["data"],self.options["footer"],
                                    self.options["inline_comments"]]
                #print "I got here {0}".format(self.path)
                [self.options["header"],self.options["column_names"],
                                    self.options["data"],self.options["footer"],
                                    self.options["inline_comments"]]=[None for i in self.elements]

                #if you are validating the model, you have to skip the updating until it hsa been parsed
                try:
                    if self.options["validate"]:
                        del self.options["validate"]
                        pass
                except KeyError:
                    self.update_model()

            else:
                # open the file and read it in as lines
                # do we parse it here?
                # once parsed we should end up with the major components
                # if we are given options we should use them, if not try to autodetect them?
                # we can just return an error right now and then have an __autoload__ method
                # we can assume it is in ascii or utf-8
                # set any attribute that has no options to None
                import_table=[]
                for item in self.elements:
                    if len(filter(lambda x: None!=x,self.get_options_by_element(item).values()))==0:
                        self.__dict__[item]=None
                        #elements.remove(item)
                    elif item not in ['inline_comments']:
                        self.__dict__[item]=[]
                        import_row=[self.options['%s_begin_line'%item],
                                    self.options['%s_end_line'%item],
                                    self.options['%s_begin_token'%item],
                                    self.options['%s_end_token'%item]]
                        import_table.append(import_row)
                    elif item in ['inline_comments']:
                        self.inline_comments=self.options['inline_comments']
                file_in=open(file_path,'r')
                # in order to parse the file we need to know line #'s, once we deduce them we use __parse__
                self.lines=[]
                for line in file_in:
                    self.lines.append(line)
                file_in.close()
                self.path=file_path
                if self.lines_defined():
                    self.__parse__()
                else:
                    import_table[0][0]=0
                    import_table[-1][1]=None
                    # This is to make sure the lines inbetween the data table's elements are accounted for
                    if self.options['data_table_element_separator'] is None:
                        inner_element_spacing=0
                    else:
                        inner_element_spacing=self.options['data_table_element_separator'].count('\n')-1
                    #print import_table
                    self.update_import_options(import_table=import_table)
                    #self.get_options()
                    if self.lines_defined():
                        #print("%s says %s"%('self.lines_defined()',str(self.lines_defined())))
                        self.__parse__()
                    row_zero=[import_table[i][0] for i in range(len(import_table))]
                    for index,item in enumerate(row_zero):
                        #print import_table
                        #print index,item
                        if index>0:
                            #print("Row Zero Loop Returns index={0}, item={1}".format(index,item))
                            if item is not None:
                                import_table[index-1][1]=item+inner_element_spacing
                                #print import_table
                                self.update_import_options(import_table)
                                #print self.lines_defined()
                    if self.lines_defined():
                            self.__parse__()
                    else:
                        row_one=[import_table[i][1] for i in range(len(import_table))]
                        for index,item in enumerate(row_one):
                            #print("Row One Loop Returns index={0}, item={1}".format(index,item))
                            if index<(len(row_one)-1):
                                #print((index+1)<len(row_one))
                                #print("Row One Loop Returns index={0}, item={1}".format(index,item))
                                if item is not None:
                                    #print import_table
                                    import_table[index+1][0]=item-inner_element_spacing
                                    self.update_import_options(import_table)
                        if self.lines_defined():
                            self.__parse__()
                        else:
                            row_two=[import_table[i][2] for i in range(len(import_table))]
                            for index,item in enumerate(row_two):
                                if item is not None:
                                    import_table[index][0]=self.find_line(item)
                            for index,item in enumerate(row_zero):
                                if index>0:
                                    if item is not None:
                                        import_table[index-1][1]=item++inner_element_spacing
                                        self.update_import_options(import_table)
                            if self.lines_defined():
                                self.__parse__()
                            else:
                                row_three=[import_table[i][3] for i in range(len(import_table))]
                                for index,item in enumerate(row_three):
                                    if item is not None:
                                        import_table[index][1]=self.find_line(item)
                                for index,item in enumerate(row_one):
                                    if index<(len(row_one)-1):
                                        if item is not None:
                                            import_table[index+1][0]=item-inner_element_spacing
                                self.update_import_options(import_table)
                                if self.lines_defined():
                                    self.__parse__()
                                else:
                                    print("FAILED to import file!")
                                    raise

        def find_line(self,begin_token):
            """Finds the first line that has begin token in it"""
            for index,line in enumerate(self.lines):
                if re.match(begin_token,line):
                    return index

        def update_import_options(self,import_table):
            """Updates the options in the import table"""
            for index,element in enumerate(self.elements):
                if self.__dict__[element] is not None:
                    [self.options['%s_begin_line'%element],
                                    self.options['%s_end_line'%element],
                                    self.options['%s_begin_token'%element],
                                    self.options['%s_end_token'%element]]=import_table[index][:]
                    #self.get_options_by_element(element)

        def lines_defined(self):
            """If begin_line and end_line for all elements that are None are defined returns True"""
            truth_table=[]
            last_element=""
            output=False
            for index,element in enumerate(self.elements):
                if element not in ['inline_comments'] and self.__dict__[element] is not None:
                    try:
                        last_element=element
                        if not None in [self.options['%s_begin_line'%element],self.options['%s_end_line'%element]]:
                            truth_table.append(True)
                        else:
                             truth_table.append(False)
                    except:
                        return False
            #print truth_table
            # The last_line of the last element is fine to be none
            if truth_table[-1] is False:
                if self.options['%s_begin_line'%last_element] is not None:
                    truth_table[-1]=True
            if False in truth_table:
                output=False
            else:
                output=True
            #print output
            return output

        def __parse__(self):
            """Parses self.lines into its components once all the relevant begin and end lines have been set. It assumes
             that the self.__dict__[self.element[i]]=None for elements that are not defined"""
            # Collect the inline comments if they are not already defined
            if self.inline_comments is None:
               self.inline_comments=collect_inline_comments(self.lines,begin_token=self.options["inline_comment_begin"],
                                                                 end_token=self.options["inline_comment_end"])
            # Strip the inline comments
            self.lines=strip_inline_comments(self.lines,
                                                 begin_token=self.options['inline_comment_begin'],
                                                 end_token=self.options['inline_comment_end'])
            # Define each major element that are not inline comments by their line numbers
            for index,element in enumerate(self.elements):
                if self.__dict__[element] is not None and element not in ['inline_comments']:
                    try:
                        if not None in [self.options['%s_begin_line'%element]]:
                            content_list=self.lines[
                                                self.options['%s_begin_line'%element]:self.options['%s_end_line'%element]]
                            self.__dict__[element]=content_list
                            # print("The result of parsing is self.{0} = {1}".format(element,content_list))
                    except:
                        raise
            # Remove any defined begin and end tokens
            for index,element in enumerate(self.elements):
                if self.__dict__[element] is not None and element not in ["inline_comments"]:
                            for index,line in enumerate(self.__dict__[element]):
                                self.__dict__[element][index]=line

                            content_list=strip_tokens(self.__dict__[element],
                                                      *[self.options['%s_begin_token'%element],
                                                        self.options['%s_end_token'%element]])
                            self.__dict__[element]=content_list
                            # print("The result of parsing is self.{0} = {1}".format(element,content_list))
            # parse the header
            if self.header is not None:
                #print("The {0} variable is {1}".format('self.header',self.header))
                remove_tokens=[self.options['block_comment_begin'],self.options['block_comment_end']]
                temp_header=self.header
                if self.options["treat_header_as_comment"] is None or self.options["treat_header_as_comment"]:
                    remove_tokens=[self.options["block_comment_begin"],
                                                        self.options["block_comment_end"]]
                    if self.options["data_table_element_separator"] is not None and self.options['comment_end'] != None:
                        remove_tokens.append(self.options['comment_end']+self.options["data_table_element_separator"])
                    temp_header=strip_tokens(temp_header,
                                                       *remove_tokens)
                    temp_header=strip_all_line_tokens(temp_header,
                                                      self.options['comment_begin'],self.options['comment_end'])
                else:
                    pass

                #print("The {0} variable is {1}".format('temp_header',temp_header))
                #temp_header=strip_tokens(temp_header,*remove_tokens)
                # print("The {0} variable is {1}".format('temp_header',temp_header))
                for index,line in enumerate(temp_header):
                    temp_header[index]=line.replace('\n',"")

                self.header=temp_header
                # print("The {0} variable is {1}".format('self.header',self.header))
            # parse the column_names
            if self.column_names is not None:
                #print("The result of parsing is self.{0} = {1}".format('column_names',self.column_names))
                self.column_names=strip_all_line_tokens(self.column_names,begin_token=self.options['column_names_begin_token'],
                                                  end_token=self.options['column_names_end_token'])
                # print("The result of parsing is self.{0} = {1}".format('column_names',self.column_names))
                self.column_names=split_all_rows(self.column_names,delimiter=self.options["column_names_delimiter"],
                                            escape_character=self.options["escape_character"])
                # print("The result of parsing is self.{0} = {1}".format('column_names',self.column_names))
                self.column_names=self.column_names[0]
                for index,line in enumerate(self.column_names):
                    self.column_names[index]=line.replace('\n',"")
                # print("The result of parsing is self.{0} = {1}".format('column_names',self.column_names))
            # parse the data
            if self.data is not None:
                self.data=strip_all_line_tokens(self.data,begin_token=self.options["row_begin_token"],
                                                end_token=self.options["row_end_token"])
                #print("The result of parsing is self.{0} = {1}".format('data',self.data))
                self.data=split_all_rows(self.data,delimiter=self.options["data_delimiter"],
                                         escape_character=self.options["escape_character"])
                #print("The result of parsing is self.{0} = {1}".format('data',self.data))
                self.data=convert_all_rows(self.data,self.options["column_types"])
                #print("The result of parsing is self.{0} = {1}".format('data',self.data))
            # parse the footer
            if self.footer is not None:
                #print("The {0} variable is {1}".format('self.footer',self.footer))
                remove_tokens=[self.options['block_comment_begin'],self.options['block_comment_end']]
                temp_footer=self.footer
                if self.options["treat_footer_as_comment"] is None or self.options["treat_footer_as_comment"]:
                    temp_footer=strip_tokens(temp_footer,
                                                       *[self.options["block_comment_begin"],
                                                       self.options["block_comment_end"]])
                    temp_footer=strip_all_line_tokens(temp_footer,
                                                      self.options['comment_begin'],self.options['comment_end'])
                else:
                    pass

                #print("The {0} variable is {1}".format('temp_footer',temp_footer))
                #temp_footer=strip_tokens(temp_footer,*remove_tokens)
                #print("The {0} variable is {1}".format('temp_footer',temp_footer))
                for index,line in enumerate(temp_footer):
                    temp_footer[index]=line.replace('\n',"")

                self.footer=temp_footer
                # print("The {0} variable is {1}".format('self.footer',self.footer))

        def get_options_by_element(self,element_name):
            """ returns a dictionary
             of all the options that have to do with element. Element must be header,column_names,data, or footer"""
            keys_regarding_element=filter(lambda x: re.search(element_name,str(x),re.IGNORECASE),self.options.keys())
            out_dictionary={key:self.options[key] for key in keys_regarding_element}
            #print out_dictionary
            return out_dictionary

        def __str__(self):
            "Controls the str output of AsciiDataTable"
            self.string=self.build_string()
            return self.string

        def update_index(self):
            """ Updates the index column if it exits, otherwise exits quietly
            """
            if 'index' not in self.column_names:
                return
            else:
                try:
                    #This should be 0 but just in case
                    index_column_number=self.column_names.index('index')
                    for i in range(len(self.data)):
                        self.data[i][index_column_number]=i
                except:
                    pass

        def update_model(self):
            """Updates the model after a change has been made. If you add anything to the attributes of the model,
            or change this updates the values. If the model has an index column it will make sure the numbers are correct.
            In addition, it will update the options dictionary to reflect added rows, changes in deliminators etc.  """
            if self.column_names is not None and 'index' in self.column_names:
               self.update_index()
            #make sure there are no "\n" characters in the element lists (if so replace them with "") for data this is
            # done on import
            list_types=["header","column_names","footer"]
            for element in list_types:
                if self.__dict__[element] is not None:
                    for index,item in enumerate(self.__dict__[element]):
                        self.__dict__[element][index]=item.replace("\n","")
            self.update_column_names()
            if self.data is not None:
                self.data=convert_all_rows(self.data,self.options["column_types"])
            self.string=self.build_string()
            self.lines=self.string.splitlines()

        def update_column_names(self):
            """Update column names adds the value x# for any column that exists in self.data that is not named"""
            if self.data is None:
                return
            elif type(self.column_names) is StringType:
                self.column_names=split_row(self.column_names,self.options["column_names_delimiter"])
            elif self.column_names is None:
                column_names=[]
                for index,column in enumerate(self.data[0]):
                    column_names.append("x"+str(index))
                self.column_names=column_names
                return
            elif len(self.column_names)==len(self.data[0]):
                return
            elif len(self.column_names) < len(self.data[0]):
                for index in range(len(self.column_names),len(self.data[0])):
                    self.column_names.append("x"+str(index))
                return

        def save(self,path=None,**temp_options):
            """" Saves the file, to save in another ascii format specify elements in temp_options, the options
            specified do not permanently change the object's options. If path is supplied it saves the file to that path
            otherwise uses the object's attribute path to define the saving location """
            original_options=self.options
            for key,value in temp_options.iteritems():
                self.options[key]=value
            out_string=self.build_string(**temp_options)
            if path is None:
                path=self.path
            file_out=open(path,'w')
            file_out.write(out_string)
            file_out.close()
            self.options=original_options

        def build_string(self,**temp_options):
            """Builds a string representation of the data table based on self.options, or temp_options.
            Passing temp_options does not permanently change the model"""
            # store the original options to be put back after the string is made
            original_options=self.options
            for key,value in temp_options.iteritems():
                self.options[key]=value
            section_end=0
            next_section_begin=0
            if self.options['data_table_element_separator'] is None:
                inner_element_spacing=0
            else:
                inner_element_spacing=self.options['data_table_element_separator'].count('\n')-1
            string_out=""
            between_section=""
            if self.options['data_table_element_separator'] is not None:
                between_section=self.options['data_table_element_separator']
            if self.header is None:
                self.options['header_begin_line']=self.options['header_end_line']=None
                pass
            else:
                self.options["header_begin_line"]=0
                if self.data is None and self.column_names is None and self.footer is None:
                    string_out=self.get_header_string()
                    self.options["header_end_line"]=None
                else:
                    string_out=self.get_header_string()+between_section

                    last_header_line=self.get_header_string().count('\n')+1
                    self.options["header_end_line"]=last_header_line
                    next_section_begin=last_header_line+inner_element_spacing

            if self.column_names is None:
                self.options['column_names_begin_line']=self.options['column_names_end_line']=None
                pass
            else:
                self.options["column_names_begin_line"]=next_section_begin
                if self.data is None and self.footer is None:
                    self.options["column_names_end_line"]=None
                    string_out=string_out+self.get_column_names_string()
                else:
                    string_out=string_out+self.get_column_names_string()+between_section
                    last_column_names_line=self.get_column_names_string().count('\n')+\
                                           self.options["column_names_begin_line"]+1
                    self.options["column_names_end_line"]=last_column_names_line
                    next_section_begin=last_column_names_line+inner_element_spacing
            if self.data is None:
                self.options['data_begin_line']=self.options['data_end_line']=None
                pass
            else:
                self.options["data_begin_line"]=next_section_begin
                if self.footer is None:
                    self.options["data_end_line"]=None
                    string_out=string_out+self.get_data_string()
                else:
                    string_out=string_out+self.get_data_string()+between_section
                    last_data_line=self.get_data_string().count("\n")+\
                                    self.options["data_begin_line"]+1
                    self.options["data_end_line"]=last_data_line
                    next_section_begin=last_data_line+inner_element_spacing
            if self.footer is None:
                self.options['footer_begin_line']=self.options['footer_end_line']=None
                pass
            else:
                self.options["footer_begin_line"]=next_section_begin
                string_out=string_out+self.get_footer_string()
                self.options['footer_end_line']=None
            # set the options back after the string has been made
            if self.inline_comments is None:
                pass
            else:
                lines=string_out.splitlines()
                for comment in self.inline_comments:
                    lines=insert_inline_comment(lines,comment=comment[0],line_number=comment[1],
                                                string_position=comment[2],
                                                begin_token=self.options['inline_comment_begin'],
                                                end_token=self.options['inline_comment_end'])
                string_out=string_list_collapse(lines,string_delimiter='\n')
            self.options=original_options
            return string_out

        def get_header_string(self):
            """Returns the header using options in self.options. If block comment is specified, and the header is a
            list it will block comment out the header. If comment_begin and comment_end are specified it will use
            those to represent each line of the header. If header_begin_token and/or header_end_token are specified it
             will wrap the header in those.
            """
            string_out=""
            header_begin=""
            header_end=""
            if self.options["header_begin_token"] is None:
                header_begin=""
            else:
                header_begin=self.options["header_begin_token"]
            if self.options["header_end_token"] is None:
                header_end=""
            else:
                header_end=self.options["header_end_token"]
            # This writes the header
            if self.header is None:
                string_out= ""
            elif self.options["header_line_types"] is not None:
                for index,line in enumerate(self.options["header_line_types"]):
                    if index == len(self.options["header_line_types"])-1:
                        end=''
                    else:
                        end='\n'
                    if line in ['header','header_line','normal']:
                        string_out=string_out+self.header[index]+end
                    elif line in ['line_comment','comment']:
                        string_out=string_out+line_comment_string(self.header[index],
                                                   comment_begin=self.options["comment_begin"],
                                                   comment_end=self.options["comment_end"])+end
                    elif line in ['block_comment','block']:
                        if index-1<0:
                            block_comment_begin=index
                            block_comment_end=index+2
                            continue
                        elif self.options["header_line_types"][index-1] not in ['block_comment','block']:
                            block_comment_begin=index
                            block_comment_end=index+2
                            continue
                        else:
                            if index+1>len(self.options["header_line_types"])-1:
                                string_out=string_out+line_list_comment_string(self.header[block_comment_begin:],
                                                                               comment_begin=self.options['block_comment_begin'],
                                                                                 comment_end=self.options['block_comment_end'],
                                                                               block=True)+end
                            elif self.options["header_line_types"][index+1] in ['block_comment','block']:
                                block_comment_end+=1
                            else:
                                string_out=string_out+\
                                           line_list_comment_string(self.header[block_comment_begin:block_comment_end],
                                                                    comment_begin=self.options['block_comment_begin'],
                                                                    comment_end=self.options['block_comment_end'],
                                                                    block=True)+end
                    else:
                        string_out=string_out+line
            elif self.options['treat_header_as_comment'] in [None,True] and self.options["header_line_types"] in [None]:
                # Just happens if the user has set self.header manually
                if type(self.header) is StringType:
                    string_out=line_comment_string(self.header,
                                                   comment_begin=self.options["comment_begin"],
                                                   comment_end=self.options["comment_end"])
                    #string_out=re.sub('\n','',string_out,count=1)
                elif type(self.header) is ListType:
                    if self.options['block_comment_begin'] is None:
                        if self.options['comment_begin'] is None:
                            string_out=string_list_collapse(self.header)

                        else:
                            string_out=line_list_comment_string(self.header,comment_begin=self.options['comment_begin'],
                                                            comment_end=self.options['comment_end'])
                            lines_out=string_out.splitlines()

                            # if re.search('\n',self.options['comment_end']):
                            #     string_out=re.sub('\n','',string_out,count=1)
                            #self.options["header_line_types"]=["line_comment" for line in self.header]
                    else:
                        string_out=line_list_comment_string(self.header,comment_begin=self.options['block_comment_begin'],
                                                            comment_end=self.options['block_comment_end'],block=True)
                        #self.options["header_line_types"]=["block_comment" for line in self.header]
            else:
                string_out=ensure_string(self.header,list_delimiter="\n",end_if_list="")
            return header_begin+string_out+header_end

        def get_column_names_string(self):
            "Returns the column names as a string using options"
            string_out=""
            # This writes the column_names
            column_name_begin=""
            column_name_end=""
            if self.options["column_names_begin_token"] is None:
                column_name_begin=""
            else:
                column_name_begin=self.options["column_names_begin_token"]
            if self.options["column_names_end_token"] is None:
                column_name_end=""
            else:
                column_name_end=self.options["column_names_end_token"]

            if self.column_names is None:
                string_out=""
            else:
                if type(self.column_names) is StringType:
                    string_out=self.column_names

                elif type(self.column_names) is ListType:
                    string_out=list_to_string(self.column_names,
                                              data_delimiter=self.options["column_names_delimiter"],end="")
                    #print("{0} is {1}".format('string_out',string_out))
                else:

                    string_out=ensure_string(self.column_names)

            #print column_name_begin,string_out,column_name_end
            return column_name_begin+string_out+column_name_end

        def get_data_string(self):
            "Returns the data as a string"
            #Todo:refactor to cut out unused lines
            string_out=""
            if self.data is None:
                string_out= ""
            else:
                if type(self.data) is StringType:
                    if self.options['data_begin_token'] is None:
                           if self.options['data_end_token'] is None:
                               string_out=self.data
                           else:
                               if re.search(self.options['data_end_token'],self.data):
                                   string_out=self.data
                               else:
                                   string_out=self.data+self.options['data_end_token']
                    else:
                            if self.options['data_end_token'] is None:
                                if re.match(self.options['data_begin_token'],self.data):
                                    string_out=self.data
                                else:
                                    string_out=self.options['data_begin_token']+self.data
                elif type(self.data) in [ListType,np.ndarray]:
                    try:
                            #If the first row is a string, we should strip all the tokens and add them back in
                            if type(self.data[0]) is StringType:
                                if self.options['data_begin_token'] is None:
                                    string_out=string_list_collapse(self.data)
                                else:
                                    if re.match(self.options['data_begin_token'],self.data[0]):
                                        if self.options['data_end_token'] is None:
                                            string_out=string_list_collapse(self.data)
                                        else:
                                            if re.search(self.options['data_end_token'],self.data[-1]):
                                                string_out=string_list_collapse(self.data)
                                            else:
                                                string_out=string_list_collapse(self.data)+self.options['data_end_token']
                                    else:
                                        if self.options['data_end_token'] is None:
                                            string_out=self.options['data_begin_token']+string_list_collapse(self.data)
                                        else:
                                            if re.search(self.options['data_end_token'],self.data[-1]):
                                                string_out=self.options['data_begin_token']+string_list_collapse(self.data)
                                            else:
                                                string_out=self.options['data_begin_token']+\
                                                           string_list_collapse(self.data)+\
                                                           self.options['data_end_token']

                            elif type(self.data[0]) in [ListType,np.ndarray]:
                                prefix=""
                                if self.options['data_begin_token'] is None:
                                    if self.options['data_end_token'] is None:
                                        string_out=list_list_to_string(self.data,data_delimiter=self.options['data_delimiter'],
                                                                       row_formatter_string=self.options['row_formatter_string'],
                                                                       line_begin=self.options["row_begin_token"],
                                                                       line_end=self.options["row_end_token"])
                                else:
                                    if self.options['data_end_token'] is None:
                                        string_out=self.options['data_begin_token']+\
                                                   list_list_to_string(self.data,
                                                                       data_delimiter=self.options['data_delimiter'],
                                                                       row_formatter_string=self.options['row_formatter_string'],
                                                                       line_begin=self.options["row_begin_token"],
                                                                       line_end=self.options["row_end_token"])
                                    else:
                                        string_out=self.options['data_begin_token']+\
                                                   list_list_to_string(self.data,
                                                                       data_delimiter=self.options['data_delimiter'],
                                                                       row_formatter_string=\
                                                                       self.options['row_formatter_string'],
                                                                       line_begin=self.options["row_begin_token"],
                                                                       line_end=self.options["row_end_token"])+\
                                                                        self.options['data_end_token']
                            else:
                                string_out=list_to_string(self.data,
                                                          data_delimiter=self.options['data_delimiter'],
                                                          row_formatter_string=self.options['row_formatter_string'],
                                                          begin=self.options["row_begin_token"],
                                                          end=self.options["row_end_token"])

                    except IndexError:
                        pass
                else:
                    string_out=ensure_string(self.data)
            return string_out

        def get_footer_string(self):
            """Returns the footer using options in self.options. If block comment is specified, and the footer is a
            list it will block comment out the footer. If comment_begin and comment_end are specified it will use
            those to represent each line of the footer. If footer_begin_token and/or footer_end_token are specified it
             will wrap the footer in those.
            """
            string_out=""
            footer_begin=""
            footer_end=""
            if self.options["footer_begin_token"] is None:
                footer_begin=""
            else:
                footer_begin=self.options["footer_begin_token"]
            if self.options["footer_end_token"] is None:
                footer_end=""
            else:
                footer_end=self.options["footer_end_token"]
            # This writes the footer
            if self.footer is None:
                string_out= ""
            elif self.options["footer_line_types"] is not None:
                for index,line in enumerate(self.options["footer_line_types"]):
                    if line in ['footer','footer_line','normal']:
                        string_out=string_out+self.footer[index]+'\n'
                    elif line in ['line_comment','comment']:
                        string_out=string_out+line_comment_string(self.footer[index],
                                                   comment_begin=self.options["comment_begin"],
                                                   comment_end=self.options["comment_end"])
                    elif line in ['block_comment','block']:
                        if index-1<0:
                            block_comment_begin=index
                            continue
                        elif self.options["footer_line_types"][index-1] not in ['block_comment','block']:
                            block_comment_begin=index
                            continue
                        else:
                            if index+1>len(self.options["footer_line_types"])-1:
                                string_out=string_out+line_list_comment_string(self.footer[block_comment_begin:],
                                                                               comment_begin=self.options['block_comment_begin'],
                                                                                 comment_end=self.options['block_comment_end'],
                                                                               block=True)
                            elif self.options["footer_line_types"][index+1] in ['block_comment','block']:
                                block_comment_end+=1
                            else:
                                string_out=string_out+line_list_comment_string(self.footer[block_comment_begin:block_comment_end],
                                                                               comment_begin=self.options['block_comment_begin'],
                                                                                 comment_end=self.options['block_comment_end'],
                                                                               block=True)
                    else:
                        string_out=string_out+line

            elif self.options['treat_footer_as_comment'] is None or self.options['treat_footer_as_comment'] is True:
                if type(self.footer) is StringType:
                    string_out=line_comment_string(self.footer,
                                                   comment_begin=self.options["comment_begin"],
                                                   comment_end=self.options["comment_end"])
                elif type(self.footer) is ListType:
                    if self.options['block_comment_begin'] is None:
                        string_out=line_list_comment_string(self.footer,comment_begin=self.options['comment_begin'],
                                                            comment_end=self.options['comment_end'])
                    else:
                        string_out=line_list_comment_string(self.footer,comment_begin=self.options['block_comment_begin'],
                                                            comment_end=self.options['block_comment_end'],block=True)
                else:
                    try:
                        string_out=str(self.footer)
                    except:raise
            else:
                string_out=ensure_string(self.footer,list_delimiter="\n",end_if_list="")
            return footer_begin+string_out+footer_end
        # our current definition of add is not reversible !!!!!!!!!
        # def __radd__(self, other):
        #     "Controls the behavior of radd to use the sum function it is required"
        #     if other==0:
        #         return self
        #     else:
        #         return self.__add__(other)

        def __add__(self, other):
            """Controls the behavior of the addition operator, if column_names are equal it adds rows at the end
            and increments any column named index. If the column_names and number of roows are different it adds columns
            to the table and fills the non-defined rows with self.options['empty_character'] which is None by default. If
            the rows are equal it adds the columns to the table ignoring any columns that are the same. If the headers or
            footers are different it appends them to the left most object."""
            if self==other:
                return
            if self.column_names is other.column_names:
                for row in other.data:
                    self.add_row(row)
            elif len(self.data)==len(other.data):
                for index,column in enumerate(other.column_names):
                    if column in self.column_names:
                        pass
                    else:
                        if other.column_types is not None:
                            column_type=other.column_types[index]
                        else:
                            column_type='string'
                        self.add_column(column_name=column,
                                        column_type=column_type,column_data=other.get_column(column_name=column))
            else:
                for column in other.column_names:
                    self.add_column(column)
                for row in other.data:
                    data=[self.options['empty_value'] for i in self.column_names]
                    self.add_row(data.append(row))
            if self.header is not other.header:
                for line in other.header:
                    if line is None:
                        pass
                    else:
                        self.header.append(line)
            if self.footer is not other.footer:
                for line in other.footer:
                    if line is None:
                        pass
                    else:
                        self.footer.append(line)
            return self

        def is_valid(self):
            """Returns True if ascii table conforms to its specification given by its own options"""
            options={}
            for key,value in self.options.iteritems():
                options[key]=value
                # print("self.options[{0}] is {1} ".format(key,value))
            for element in self.elements:
                if self.__dict__[element] is None:
                    options[element]=None
                else:
                    options[element]=[]
            options["validate"]=True
            newtable=AsciiDataTable(None,**options)
            lines=self.build_string().splitlines()
            for index,line in enumerate(lines):
                lines[index]=line+"\n"
            newtable.lines=lines
            newtable.__parse__()
            # print newtable.data
            # print newtable.column_names
            # print newtable
            #print_comparison(newtable.footer,None)
            newtable.update_model()
            # The new table rows are not being coerced into the right format
            #print newtable
            #newtable.update_model()
            #print newtable.options
            #print self.options
            #print newtable.data
            # print newtable.options==self.options
            # for option_key,option_value in newtable.options.iteritems():
            #     print("New Table Option {0} is {1} ".format(option_key,option_value))
            #     print("self.options[{0}] is {1} ".format(option_key,self.options[option_key]))
            #     print_comparison(option_value,self.options[option_key])
            # #print self
            return self==newtable
            # create a clone and then parse the clone and compare it to the
            # original. If they are equal then it is valid
            #self.add_inline_comments()

        def __eq__(self, other):
            """Defines what being equal means for the AsciiDataTable Class"""
            compare_elements=['options','header','column_names','data','footer']
            truth_table=[]
            output=False
            for item in compare_elements:
                if self.__dict__[item]==other.__dict__[item]:
                    truth_table.append(True)
                else:
                    truth_table.append(False)
            if False in truth_table:
                output=False
            else:
                output=True
            #print(truth_table)
            return output

        def __ne__(self,other):
            """Defines what being not equal means for the AsciiDataTable Class"""
            compare_elements=['options','header','column_names','data','footer']
            truth_table=[]
            output=True
            for item in compare_elements:
                if self.__dict__[item]==other.__dict__[item]:
                    truth_table.append(True)
                else:
                    truth_table.append(False)
            if False in truth_table:
                output=True
            else:
                output=False
            return output

        def add_row(self,row_data):
            """Adds a single row given row_data which can be an ordered list/tuple or a dictionary with
            column names as keys"""
            if len(row_data) not in [len(self.column_names),len(self.column_names)]:
                print(" could not add the row, dimensions do not match")
                return
            if type(row_data) in [ListType,np.ndarray]:
                self.data.append(row_data)
            elif type(row_data) in [DictionaryType]:
                data_list=[row_data[column_name] for column_name in self.column_names]
                self.data.append(data_list)

        def remove_row(self,row_index):
            """Removes the row specified by row_index and updates the model. Note index is relative to the
            data attribute so to remove the first row use row_index=0 and the last data row is row_index=-1"""
            self.data.pop(row_index)
            self.update_model()

        def add_column(self,column_name=None,column_type=None,column_data=None):
            """Adds a column with column_name, and column_type. If column data is supplied and it's length is the
            same as data(same number of rows) then it is added, else self.options['empty_character'] is added in each
            spot in the preceding rows"""
            original_column_names=self.column_names
            try:
                self.column_names.append(column_name)
                self.options["column_types"].append(column_type)
                if len(column_data) is len(self.data):
                    for index,row in enumerate(self.data):
                        row.append(column_data[index])
                else:
                    for index,row in enumerate(self.data):
                        row.append(self.options['empty_value'])
                        if column_data is not None:
                            for item in column_data:
                                empty_row=[self.options['empty_value'] for column in original_column_names]
                                self.add_row(empty_row.append(item))
            except:
                self.column_names=original_column_names
                print("Could not add columns")
                pass
        def remove_column(self,column_name=None,column_index=None):
            """Rmoves the column specified by column_name or column_index and updates the model. The column is removed from
            column_names, data and if present column_types, column_descriptions and row formatter"""
            pass
            #Todo:Add remove column functionality

        def add_index(self):
            """Adds a column with name index and values that are 0 referenced indices, does nothing if there is
            already a column with name index, always inserts it at the 0 position"""
            if 'index' in self.column_names:
                print("Add Index passed")
                pass
            else:
                self.column_names.insert(0,'index')
                for index,row in enumerate(self.data):
                    self.data[index].insert(0,index)
                if self.options['column_types']:
                    self.options['column_types'].insert(0,'int')
                if self.options['row_formatter_string']:
                    temp_formatter_list=self.options['row_formatter_string'].split("{delimiter}")
                    iterated_row_formatter_list=[temp_formatter_list[i].replace(str(i),str(i+1))
                                                 for i in range(len(temp_formatter_list))]
                    new_formatter=string_list_collapse(iterated_row_formatter_list,string_delimiter="{delimiter}")
                    self.options['row_formatter_string']='{0}{delimiter}'+new_formatter

        def move_footer_to_header(self):
            """Moves the DataTable's footer to the header and updates the model"""
            # check to see if the footer is defined
            if self.footer is None:
                return
            try:
              for item in self.footer:
                  self.header.append(item)
            except:
              self.header=ensure_string(self.header)+ensure_string(self.footer)
            self.footer=None

        # # This actually operated on self.lines before parsing and not directly on self.header
        # def add_comment(self,comment):
        #     "Adds a line comment to the header"
        #     self.header.append(comment)
        #     self.options["header_line_types"].insert('line_comment',len(self.header)-1)

        def add_inline_comment(self,comment="",line_number=None,string_position=None):
            "Adds an inline in the specified location"
            try:
                self.inline_comments.append([comment,line_number,string_position])
            except:pass

        # def add_block_comment(self,comment,element=None,location=None):
        #     "Adds a block comment in the specified location"
        #     pass

        def get_options(self):
            "Prints the option list"
            for key,value in self.options.iteritems():
                print("{0} = {1}".format(key,value))
        def get_row(self,row_index=None):
            """Returns the row as a list specified by row_index"""
            if row_index is None:
                return
            else:
                return self.data[row_index]

        def get_column(self,column_name=None,column_index=None):
            """Returns a column as a list given a column name or column index"""
            if column_name is None:
                if column_index is None:
                    return
                else:
                    column_selector=column_index
            else:
                column_selector=self.column_names.index(column_name)
            out_list=[self.data[i][column_selector] for i in range(len(self.data))]
            return out_list

        def get_data_dictionary_list(self,use_row_formatter_string=True):
            """Returns a python list with a row dictionary of form {column_name:data_column}"""
            try:
                if self.options["row_formatter_string"] is None:
                    use_row_formatter_string=False
                if use_row_formatter_string:
                    list_formatter=[item.replace(str(index),"0")
                                    for index,item in enumerate(self.options["row_formatter_string"].split("{delimiter}"))]
                else:
                    list_formatter=["{0}" for i in self.column_names]
                #print self.column_names
                #print self.data
                #print list_formatter
                #print len(self.column_names)==len(self.data[0])
                #print len(list_formatter)==len(self.data[0])
                #print type(self.data[0])
                out_list=[{self.column_names[i]:list_formatter[i].format(value) for i,value in enumerate(line)}
                          for line in self.data]
                return out_list
            except:
                print("Could not form a data_dictionary_list, check that row_formatter_string is properly defined")
                #print(out_list)
                raise

        def save_schema(self,path=None,format=None):
            """Saves the tables options as a text file or pickled dictionary (default).
            If no name is supplied, autonames it and saves"""
            if path is None:
                path=auto_name(self.name.replace('.'+self.options["extension"],""),'Schema',self.options["directory"],'txt')
            if format in [None,'python','pickle']:
                pickle.dump(self.options,open(path,'wb'))
            elif format in ['txt','text','.txt']:
                file_out=open(path,'w')
                keys=self.options.keys()
                keys.sort()
                for key in keys:
                    out_key=str(key).replace("\n","\\n")
                    out_value=str(self.options[key]).replace("\n","\\n")
                    file_out.write("{0} : {1} \n".format(out_key,out_value))
                file_out.close()
        def change_unit_prefix(self,column_selector=None,old_prefix=None,new_prefix=None,unit='Hz'):
            """Changes the prefix of the units of the column specified by column_selector (column name or index)
            example usage is self.change_unit_prefix(column_selector='Frequency',old_prefix=None,new_prefix='G',unit='Hz')
            to change a column from Hz to GHz. It updates the data values, column_descriptions, and column_units if they
            exist, see http://www.nist.gov/pml/wmd/metric/prefixes.cfm for possible prefixes"""

            multipliers={"yotta":10.**24,"Y":10.**24,"zetta":10.**21,"Z":10.**21,"exa":10.**18,"E":10.**18,"peta":10.**15,
                         "P":10.**15,"tera":10.**12,"T":10.**12,"giga":10.**9,"G":10.**9,"mega":10.**6,"M":10.**6,
                         "kilo":10.**3,"k":10.**3,"hecto":10.**2,"h":10.**2,"deka":10.,"da":10.,None:1.,"":1.,
                         "deci":10.**-1,"d":10.**-1,"centi":10.**-2,"c":10.**-2,"milli":10.**-3,"m":10.**-3,
                         "micro":10.**-6,"mu":10.**-6,u"\u00B5":10.**-6,"nano":10.**-9,
                         "n":10.**-9,"pico":10.**-12,"p":10.**-12,"femto":10.**-15,
                         "f":10.**-15,"atto":10.**-18,"a":10.**-18,"zepto":10.**-21,"z":10.**-21,
                         "yocto":10.**-24,"y":10.**-24}
            # change column name into column index
            try:
                if old_prefix is None:
                    old_prefix=""
                if new_prefix is None:
                    new_prefix=""
                old_unit=old_prefix+unit
                new_unit=new_prefix+unit
                if column_selector in self.column_names:
                    column_selector=self.column_names.index(column_selector)
                for index,row in enumerate(self.data):
                    if type(self.data[index][column_selector]) in [FloatType,LongType]:
                        #print "{0:e}".format(multipliers[old_prefix]/multipliers[new_prefix])
                        self.data[index][column_selector]=\
                        (multipliers[old_prefix]/multipliers[new_prefix])*self.data[index][column_selector]
                    elif type(self.data[index][column_selector]) in [StringType,IntType]:
                        self.data[index][column_selector]=\
                        str((multipliers[old_prefix]/multipliers[new_prefix])*float(self.data[index][column_selector]))
                    else:
                        print type(self.data[index][column_selector])
                        raise
                if self.options["column_descriptions"] is not None:
                    old=self.options["column_descriptions"][column_selector]
                    self.options["column_descriptions"][column_selector]=old.replace(old_unit,new_unit)
                if self.options["column_units"] is not None:
                    old=self.options["column_units"][column_selector]
                    self.options["column_units"][column_selector]=old.replace(old_unit,new_unit)
                if re.search(old_unit,self.column_names[column_selector]):
                    old=self.column_names[column_selector]
                    self.column_names[column_selector]=old.replace(old_unit,new_unit)
            except:
                print("Could not change the unit prefix of column {0}".format(column_selector))
                raise

    class AsciiDataTableCollection():
        """A collection of multiple AsciiDataTables"""
        def __init__(self,file_path=None,table_names=None,**options):
            # The primary attritbute should be self.data_tables=list of tables
            # the dictionary should be in the form {"table_name":AsciiDataTable}
            # since there is no way to know how many tables or their options,
            # you can't read them in without **options-> self.tables
            # which can be passed as an option. Each table is an independent entity
            # but can have options set by global_options
            if file_path is None:
                pass
            else:
                if table_names is None and options["table_names"] is None:
                    raise
                else:
                    file_in=open(file_path,'r')
                    # in order to parse the file we need to know line #'s, once we deduce them we use __parse__
                    self.lines=[]
                    for line in file_in:
                        self.lines.append(line)
                    file_in.close()
                    self.path=file_path

        def build_string(self,**temp_options):
            """Builds the string for the table collection using the temp_options"""
            out_string=""
            for table in self.tables:
                out_string=out_string+table.build_string()
            return out_string

    #-----------------------------------------------------------------------------
    # Module Scripts
    def test_AsciiDataTable():
        options={"column_names":["a","b","c"],"data":[[0,1,2],[2,3,4]],"data_delimiter":'\t',
                 "header":['Hello There\n',"My Darling"],"column_names_begin_token":'!',"comment_begin":'#',
                 "directory":TESTS_DIRECTORY}
        new_table=AsciiDataTable(file_path=None,**options)
        print new_table.data
        #print dir(new_table)
        for key,value in new_table.options.iteritems():
            print("{0} = {1}".format(key,value))
        print new_table.get_header_string()
        print new_table.get_data_string()
        print new_table.build_string()
        print new_table.path
        new_table.save()
        print("Test add_index")
        new_table.add_index()
        new_table.add_index()
        new_table.add_index()
        print new_table
        new_table.data[1][0]=4
        print new_table
        new_table.update_index()
        print new_table

    def test_open_existing_AsciiDataTable():
        # To make this controlled we should create a data table and then save it.
        # after saving it we should open it with the options
        options={"data_delimiter":',',
                      "column_names_delimiter":"{column_names_delimiter}",
                      "specific_descriptor":'Data',
                      "general_descriptor":'Table',
                      "directory":None,
                      "extension":'txt',
                      "comment_begin":"{comment_begin}",
                      "comment_end":"{comment_end}",
                      "inline_comment_begin":"{inline_comment_begin}",
                      "inline_comment_end":"{inline_comment_end}",
                      "block_comment_begin":"{block_comment_begin}\n",
                      "block_comment_end":"\n{block_comment_end}",
                      "footer_begin_token":"{footer_begin_token}\n",
                      "footer_end_token":"\n{footer_end_token}",
                      "header_begin_token":"{header_begin_token}\n",
                      "header_end_token":"\n{header_end_token}",
                      "column_names_begin_token":"{column_names_begin_token}",
                      "column_names_end_token":"{column_names_end_token}",
                      "data_begin_token":"{data_begin_token}\n",
                      "data_end_token":"\n{data_end_token}",
                      "metadata_delimiter":"{metadata_delimiter}",
                      "header":["self.header[0]","self.header[1]","self.header[2]","self.header[3]","","self.header[4]"],
                      "column_names":["column_names[0]","column_names[1]","column_names[2]"],
                      "data":[["data[0][0]","data[1][0]","data[2][0]",'1.0'],["data[0][1]","data[1][1]","data[2][1]",2.0]],
                      "footer":["self.footer[0]","self.footer[1]"],
                      "inline_comments":[["inline_comments[0][0]",2,-1]],
                      "empty_value":None,
                      "data_table_element_separator":'\n{data_table_element_separator}\n',
                      "treat_header_as_comment":None,
                      "treat_footer_as_comment":None,
                      "header_line_types":["block_comment","block_comment","line_comment","header","header","line_comment"],
                      "column_types":["str","string","STR","float"],
                      "row_formatter_string":"{0}{delimiter}{1}{delimiter}{2}{delimiter}{3:.2f}"

                      }
        os.chdir(TESTS_DIRECTORY)
        new_table=AsciiDataTable(None,**options)
        table_string_1=str(new_table)
        print("Printing the lines representation of the table with line numbers")
        print("-"*80+"\n")
        for index,line in enumerate(table_string_1.splitlines()):
            print("{0} {1}".format(index,line))
        new_table.save()
        file_path=new_table.path
        new_options={}
        for key,value in new_table.options.iteritems():
            new_options[key]=value
        new_table_1=AsciiDataTable(file_path=file_path,**new_options)
        table_string_2=str(new_table_1)
        print("Printing the lines representation of the opened table with line numbers")
        print("-"*80+"\n")
        for index,line in enumerate(table_string_2.splitlines()):
            print("{0} {1}".format(index,line))
        #print new_table_1
        print("Printing the line by line equality of the table strings")
        print("-"*80)
        for index,line in enumerate(table_string_2.splitlines()):
            print("{0} {1}".format(index,line==table_string_1.splitlines()[index]))
        print("The assertion that the built table is equal to the opened table is {0}".format(new_table==new_table_1))
        # new_table.get_options_by_element('columns')
        # print new_table.lines
        # print new_table.header
        # print new_table.get_header_string()
        # print new_table.column_names
        # print new_table.get_column_names_string()
        # print new_table.data
        # print new_table.get_data_string()
        # print new_table.footer
        # print new_table.get_footer_string()
        # print new_table_1
        # temp_options={"data_delimiter":',','column_names_delimiter':',',
        #          "column_names_begin_token":'#',"comment_begin":'#',"comment_end":'\n',
        #          "directory":TESTS_DIRECTORY,'header_begin_token':'BEGIN HEADER\n',
        #               'header_end_token':'END HEADER','data_begin_token':'BEGIN DATA\n','data_end_token':"END DATA"}
        # new_table.save('new_test_table.txt',**temp_options)

    def test_AsciiDataTable_equality():
        options={"column_names":["a","b","c"],"data":[[0,1,2],[2,3,4]],"data_delimiter":'\t',
                 "header":['Hello There',"My Darling"],"column_names_begin_token":'!',"comment_begin":'#',
                 "directory":TESTS_DIRECTORY,"treat_header_as_comment":False}
        new_table=AsciiDataTable(file_path=None,**options)
        print new_table.data
        new_table_2=AsciiDataTable()
        new_table_2.options=new_table.options
        new_table_2.header=new_table.header
        new_table_2.column_names=new_table.column_names
        new_table_2.data=[[0,1,2],[2,3,4]]
        print new_table==new_table_2
        print new_table!=new_table_2
        new_table_2.data[0][0]=9

        print new_table.data==new_table_2.data
        print new_table==new_table_2
        print new_table_2
        print new_table

    def test_inline_comments():
        options={"column_names":["a","b","c"],"data":[[0,1,2],[2,3,4]],"data_delimiter":'\t',
                 "header":['Hello There\n',"My Darling"],"column_names_begin_token":'!',"comment_begin":'#',
                 "directory":TESTS_DIRECTORY,'inline_comment_begin':'(*','inline_comment_end':'*)',
                 'inline_comments':[["My Inline Comment",0,5]]}
        new_table=AsciiDataTable(**options)
        print new_table

    def test_add_row():
        options={"column_names":["a","b","c"],"data":[[0,1,2],[2,3,4]],"data_delimiter":'\t',
                 "header":['Hello There\n',"My Darling"],"column_names_begin_token":'!',"comment_begin":'#',
                 "directory":TESTS_DIRECTORY}
        new_table=AsciiDataTable(**options)
        print "Table before add row"
        print new_table
        print "Add the row 0,1,3"
        new_table.add_row([0,1,3])
        print new_table

    def test_add_index():
        options={"column_names":["a","b","c"],"data":[[0,1,2],[2,3,4]],"data_delimiter":'\t',
                 "header":['Hello There\n',"My Darling"],"column_names_begin_token":'!',"comment_begin":'#',
                 "directory":TESTS_DIRECTORY,'treat_header_as_comment':False}
        new_table=AsciiDataTable(**options)
        print "Table before add row"
        print "*"*80
        print new_table
        print "Add the row 0,1,3"
        print "*"*80
        new_table.add_row([0,1,3])
        print new_table
        print "Add an index"
        print "*"*80
        new_table.add_index()
        print new_table
        print "Now Get the index column"
        print new_table.get_column(column_name='index')

    def show_structure_script():
        """ Shows a table elements by substituting the names Explicitly
        :return: None
        """
        os.chdir(TESTS_DIRECTORY)
        options={"data_delimiter":'{data_delimiter}',
                      "column_names_delimiter":"{column_names_delimiter}",
                      "specific_descriptor":'Data',
                      "general_descriptor":'Table',
                      "directory":None,
                      "extension":'txt',
                      "comment_begin":"{comment_begin}",
                      "comment_end":"{comment_end}\n",
                      "inline_comment_begin":"{inline_comment_begin}",
                      "inline_comment_end":"{inline_comment_end}",
                      "block_comment_begin":"{block_comment_begin}\n",
                      "block_comment_end":"\n{block_comment_end}",
                      "footer_begin_token":"{footer_begin_token}\n",
                      "footer_end_token":"\n{footer_end_token}",
                      "header_begin_token":"{header_begin_token}\n",
                      "header_end_token":"\n{header_end_token}",
                      "column_names_begin_token":"{column_names_begin_token}",
                      "column_names_end_token":"{column_names_end_token}",
                      "data_begin_token":"{data_begin_token}\n",
                      "data_end_token":"\n{data_end_token}",
                      "metadata_delimiter":"{metadata_delimiter}",
                      "header":["self.header[0]","self.header[1]","self.header[2]","self.header[3]","self.header{4]"],
                      "column_names":["column_names[0]","column_names[1]","column_names[2]"],
                      "data":[["data[0][0]","data[1][0]","data[2][0]"],["data[0][1]","data[1][1]","data[2][1]"]],
                      "footer":["self.footer[0]","self.footer[1]"],
                      "inline_comments":[["inline_comments[0][0]",2,-1]],
                      "row_formatter_string":None,
                      "row_begin_token":"{row_begin_token}",
                      "row_end_token":"{row_end_token}\n",
                      "empty_value":None,
                      "data_table_element_separator":'\n{data_table_element_separator}\n',
                      "treat_header_as_comment":None,
                      "treat_footer_as_comment":None
                      }
        new_table=AsciiDataTable(**options)
        #new_table.options["block_comment_begin"]=new_table.options["block_comment_end"]=None
        new_table.options["header_line_types"]=["header","header","line_comment","block_comment","block_comment"]
        print("Printing the string representation of the table")
        print("-"*80)
        print new_table
        test_string=str(new_table)
        test_lines=test_string.splitlines()
        print("Printing the lines representation of the table with line numbers")
        print("-"*80)
        for index,line in enumerate(test_lines):
            print("{0} {1}".format(index,line))

        for item in new_table.elements:
            if item is 'inline_comments':
                pass
            else:
                begin_line=new_table.options["%s_begin_line"%item]
                end_line=new_table.options["%s_end_line"%item]
                print("-"*80)
                print("The result of self.lines[{0}:{1}] is :".format(begin_line,end_line))
                for line in test_lines[begin_line:end_line]:
                    print line
                print("-"*80)
        # new_table.footer=None
        # print("Printing the string representation of the table")
        # print("-"*80)
        # print new_table
        # test_string=str(new_table)
        # test_lines=test_string.splitlines()
        # print("Printing the lines representation of the table with line numbers")
        # print("-"*80)
        # for index,line in enumerate(test_lines):
        #     print("{0} {1}".format(index,line))
        #
        # for item in new_table.elements:
        #     if item is 'inline_comments':
        #         pass
        #     else:
        #         begin_line=new_table.options["%s_begin_line"%item]
        #         end_line=new_table.options["%s_end_line"%item]
        #         print("-"*80)
        #         print("The result of self.lines[{0}:{1}] is :".format(begin_line,end_line))
        #         for line in test_lines[begin_line:end_line]:
        #             print line
        #         print("-"*80)
        #     #
        # end_line=6
        # print("-"*80)
        # print("The result of self.lines[:{0}] is :".format(end_line))
        # for line in test_lines[:end_line]:
        #     print line
        #print new_table.get_options()

    def test_save_schema():
        "tests the save schema method of the Ascii Data Table"
        os.chdir(TESTS_DIRECTORY)
        options={"data_delimiter":'{data_delimiter}',
                      "column_names_delimiter":"{column_names_delimiter}",
                      "specific_descriptor":'Data',
                      "general_descriptor":'Table',
                      "directory":None,
                      "extension":'txt',
                      "comment_begin":"{comment_begin}",
                      "comment_end":"{comment_end}",
                      "inline_comment_begin":"{inline_comment_begin}",
                      "inline_comment_end":"{inline_comment_end}",
                      "block_comment_begin":"{block_comment_begin}\n",
                      "block_comment_end":"\n{block_comment_end}",
                      "footer_begin_token":"{footer_begin_token}\n",
                      "footer_end_token":"\n{footer_end_token}",
                      "header_begin_token":"{header_begin_token}\n",
                      "header_end_token":"\n{header_end_token}",
                      "column_names_begin_token":"{column_names_begin_token}",
                      "column_names_end_token":"{column_names_end_token}",
                      "data_begin_token":"{data_begin_token}\n",
                      "data_end_token":"\n{data_end_token}",
                      "metadata_delimiter":"{metadata_delimiter}",
                      "header":["self.header[0]","self.header[1]","self.header[2]","self.header[3]","","self.header[4]"],
                      "column_names":["column_names[0]","column_names[1]","column_names[2]"],
                      "data":[["data[0][0]","data[1][0]","data[2][0]",'1.0'],["data[0][1]","data[1][1]","data[2][1]",2.0]],
                      "footer":["self.footer[0]","self.footer[1]"],
                      "inline_comments":[["inline_comments[0][0]",2,-1]],
                      "empty_value":None,
                      "data_table_element_separator":'\n{data_table_element_separator}\n',
                      "treat_header_as_comment":None,
                      "treat_footer_as_comment":None,
                      "header_line_types":["block_comment","block_comment","line_comment","header","header","line_comment"],
                      "column_types":["str","string","STR","float"],
                      "row_formatter_string":"{0}{delimiter}{1}{delimiter}{2}{delimiter}{3:.2f}"

                      }
        new_table=AsciiDataTable(None,**options)
        print(" New Table is:")
        new_table.__parse__()
        print new_table
        print type(new_table.data[0][3])
        print new_table.get_data_dictionary_list()
        new_table.add_index()
        print new_table.get_data_dictionary_list(False)
        #print("-"*80)
        #new_table.save()
        #new_table.save_schema()
        options_2=new_table.options
        #new_table_2=AsciiDataTable(new_table.path,**options_2)
        #print_comparison(new_table.header,new_table_2.header)

    def test_read_schema():
        """ Tests the read_schema function
        """
        os.chdir(TESTS_DIRECTORY)
        file_path="Data_Table_20160301_031_Schema_20160301_001.txt"
        schema=read_schema(file_path)
        print schema

    def test_change_unit_prefix():
        """Tests the change_unit_prefix method of the AsciiDataTable Class
        """
        options={"column_names":["Frequency","b","c"],"column_names_delimiter":",","data":[[0.1*10**10,1,2],[2*10**10,3,4]],"data_delimiter":'\t',
                 "header":['Hello There',"My Darling"],"column_names_begin_token":'#',"comment_begin":'!',
                 "comment_end":"\n",
                 "directory":TESTS_DIRECTORY,
                 "column_units":["Hz",None,None],
                 "column_descriptions":["Frequency in Hz",None,None],
                 "column_types":['float','float','float'],
                 "row_formatter_string":"{0:.2e}{delimiter}{1}{delimiter}{2}",
                 "treat_header_as_comment":True}
        new_table=AsciiDataTable(None,**options)
        print(" After creation the table looks like")
        print("*"*80)
        print new_table
        print new_table.options["column_units"]
        print new_table.options["column_descriptions"]
        new_table.change_unit_prefix(column_selector='Frequency',old_prefix=None,new_prefix='G',unit='Hz')
        print(" After running change_unit_prefix(column_selector='Frequency',old_prefix=None,new_prefix='G',unit='Hz')  "
              "the table is ")
        print new_table
        print("The options column_units and column_descriptions are:")
        print new_table.options["column_units"]
        print new_table.options["column_descriptions"]
        print new_table.is_valid()

    #-----------------------------------------------------------------------------
    # Module Runner
    if __name__ == '__main__':
        #test_AsciiDataTable()
        #test_open_existing_AsciiDataTable()
        test_AsciiDataTable_equality()
        #test_inline_comments()
        #test_add_row()
        #test_add_index()
        #show_structure_script()
        #test_save_schema()
        #test_read_schema()
        #test_change_unit_prefix()

## Module variables

var DEFAULT_FILE_NAME

var METHOD_ALIASES

var StringTypes

var TESTS_DIRECTORY

## Functions

def check_arg_type(

arg, arg_type)

Checks argument and prints out a statement if arg is not type

Show source ≡

    def check_arg_type(arg,arg_type):
        "Checks argument and prints out a statement if arg is not type"
        if type(arg) is arg_type:
            return
        else:
            print("{0} was not {1}".format(arg,arg_type))

def collect_inline_comments(

list_of_strings, begin_token=None, end_token=None)

Reads a list of strings and returns all of the inline comments in a list. Output form is ['comment',line_number,string_location] returns None if there are none or tokens are set to None

Show source ≡

    def collect_inline_comments(list_of_strings,begin_token=None,end_token=None):
        """Reads a list of strings and returns all of the inline comments in a list.
        Output form is ['comment',line_number,string_location] returns None  if there are none or tokens are set to None"""
        if begin_token in [None] and end_token in [None]:
            return None
        match=re.compile('{0}(?P<inline_comments>.+){1}'.format(re.escape(begin_token),re.escape(end_token)))
        inline_comment_list=[]
        for index,line in enumerate(list_of_strings):
            comment_match=re.search(match,line)
            if comment_match:
                inline_comment_list.append([comment_match.group('inline_comments'),index,comment_match.start()])
        if inline_comment_list:
            return inline_comment_list
        else:
            return None

def convert_all_rows(

list_rows, column_types=None)

Converts all the rows (list of strings) in a list of rows using column types

Show source ≡

    def convert_all_rows(list_rows,column_types=None):
        "Converts all the rows (list of strings) in a list of rows using column types "
        check_arg_type(list_rows,ListType)
        out_list=[]
        for index,row in enumerate(list_rows):
            out_list.append(convert_row(row,column_types))
        return out_list

def convert_row(

row_list_strings, column_types=None)

Converts a row list of strings to native python types using a column types list

Show source ≡

    def convert_row(row_list_strings,column_types=None):
        """Converts a row list of strings to native
        python types using a column types list"""

        if column_types is None or len(row_list_strings) != len(column_types):
            #print("Convert row could not convert {0} using {1}".format(row_list_strings,column_types))
            return row_list_strings
        else:
            out_row=row_list_strings
            for index,column_type in enumerate(column_types):
                if re.match('int',column_type,re.IGNORECASE):
                    out_row[index]=int(row_list_strings[index])
                elif re.match('float',column_type,re.IGNORECASE):
                    out_row[index]=float(row_list_strings[index])
                elif re.match('str|char',column_type,re.IGNORECASE):
                    #print_comparison(row_list_strings[index],out_row[index])
                    out_row[index]=str(row_list_strings[index])
                elif re.match('com',column_type,re.IGNORECASE):
                    out_row[index]=complex(row_list_strings[index])
                elif re.match('list',column_type,re.IGNORECASE):
                    out_row[index]=list(row_list_strings[index])
                elif re.match('dict',column_type,re.IGNORECASE):
                    out_row[index]=dict(row_list_strings[index])
                else:
                    out_row[index]=row_list_strings[index]
        return out_row

def ensure_string(

input_object, list_delimiter='', end_if_list='')

Returns a string given an object. If the object is a string just returns it, if it is a list of strings, returns a collapsed version. If is another type of object returns str(object). If all else fails it returns an empty string

Show source ≡

    def ensure_string(input_object,  list_delimiter="",  end_if_list=""):
        """Returns a string given an object. If the object is a string just returns it,
        if it is a list of strings, returns a collapsed version. If is another type of object returns str(object).
          If all else fails it returns an empty string"""
        string_out=""
        try:
            if type(input_object) in StringTypes:
                string_out=input_object
            elif type(input_object) in [ListType,np.ndarray]:
                if type(input_object[0]) in [ListType,np.ndarray]:
                    string_out=list_list_to_string(input_object,data_delimiter=list_delimiter,end=end_if_list)
                else:
                    string_out=list_to_string(input_object,data_delimiter=list_delimiter,end=end_if_list)
            else:
                string_out=str(input_object)
        except:
            pass
        return string_out

def insert_inline_comment(

list_of_strings, comment='', line_number=None, string_position=None, begin_token='(*', end_token='*)')

Inserts an inline comment in a list of strings, location is determined by line_number and string_position

Show source ≡

    def insert_inline_comment(list_of_strings,comment="",line_number=None,string_position=None,begin_token='(*',end_token='*)'):
        "Inserts an inline comment in a list of strings, location is determined by line_number and string_position"
        if line_number is None or string_position is None:
            print("inline comment must have both line number and string position")
            return
        if begin_token is None or end_token is None:
            print("inline comment must have both a begin and end token")
            return
        inline_comment=begin_token+comment+end_token
        #make sure there are no end lines
        inline_comment=inline_comment.replace('\n','')
        if string_position in [-1,'EOL','eol']:
            list_of_strings[line_number]=list_of_strings[line_number]+inline_comment
        else:
            list_of_strings[line_number]=list_of_strings[line_number][:string_position]+inline_comment+list_of_strings[line_number][string_position:]
        return list_of_strings

def line_comment_string(

comment, comment_begin=None, comment_end=None)

Creates a comment optionally wrapped with comment_begin and comment_end, meant for a single string comment

Show source ≡

    def line_comment_string(comment,comment_begin=None,comment_end=None):
        "Creates a comment optionally wrapped with comment_begin and comment_end, meant for a single string comment "
        check_arg_type(comment,StringType)
        string_out=""
        if comment_begin is None:
            if comment_end is None:
                string_out=comment
            else:
                string_out=comment+comment_end
        else:
            if comment_end is None:
                string_out=comment_begin+comment
            else:
                string_out=comment_begin+comment+comment_end
        return string_out

def line_list_comment_string(

comment_list, comment_begin=None, comment_end=None, block=False)

Creates a string with each line wrapped in comment_begin and comment_end, by repeatedly calling line_comment_string, or the full string wrapped with block_comment_begin and block_comment_end if block is set to True. Meant to deal with a list of comment strings

Show source ≡

    def line_list_comment_string(comment_list,comment_begin=None,comment_end=None,block=False):
        """Creates a string with each line wrapped in comment_begin and comment_end, by repeatedly calling
        line_comment_string,
        or the full string wrapped with block_comment_begin and block_comment_end if block is set to True. Meant
        to deal with a list of comment strings"""
        check_arg_type(comment_list,ListType)
        string_out=""
        if block:
            string_out=comment_begin+string_list_collapse(comment_list,string_delimiter='\n')+comment_end
        else:
            for item in comment_list:
                string_out=string_out+line_comment_string(item,comment_begin=comment_begin,comment_end=comment_end)
        return string_out

def list_list_to_string(

list_lists, data_delimiter=None, row_formatter_string=None, line_begin=None, line_end=None)

Repeatedly calls list to string on each element of a list and string adds the result . ie coverts a list of lists to a string. If line end is None the value defaults to " ", for no seperator use ''

Show source ≡

    def list_list_to_string(list_lists,data_delimiter=None,row_formatter_string=None,line_begin=None,line_end=None):
        """Repeatedly calls list to string on each element of a list and string adds the result
        . ie coverts a list of lists to a string. If line end is None the value defaults to "\n", for no seperator use ''
        """
        if line_end is None:
            line_end="\n"
        check_arg_type(list_lists,ListType)
        string_out=""
        for index,row in enumerate(list_lists):
            if index==len(list_lists)-1:
                if line_end is "\n":
                    last_end=""
                else:
                    last_end=re.sub("\n","",line_end,count=1)
                string_out=string_out+list_to_string(row,data_delimiter=data_delimiter,
                                                 row_formatter_string=row_formatter_string,
                                                 begin=line_begin,end=last_end)
            else:
                string_out=string_out+list_to_string(row,data_delimiter=data_delimiter,
                                                 row_formatter_string=row_formatter_string,
                                                 begin=line_begin,end=line_end)
        return string_out

def list_to_string(

row_list, data_delimiter=None, row_formatter_string=None, begin=None, end=None)

Given a list of values returns a string, if row_formatter is specifed it uses it as a template, else uses data delimiter. Inserts data_delimiter between each list element. An optional begin and end wrap the resultant string. (i.e ['1','2','3']-> 'begin+'1'+','+'2'+','+'3'+'end') end defaults to to have nothing at the end use ''

Show source ≡

    def list_to_string(row_list,data_delimiter=None,row_formatter_string=None,begin=None,end=None):
        """Given a list of values returns a string, if row_formatter is specifed
         it uses it as a template, else uses data delimiter. Inserts data_delimiter between each list element. An optional
        begin and end wrap the resultant string. (i.e ['1','2','3']-> 'begin+'1'+','+'2'+','+'3'+'end') end defaults
        to \n to have nothing at the end use ''
        """
        check_arg_type(row_list,ListType)
        if data_delimiter is None:
            data_delimiter=','
        string_out=""
        if row_formatter_string is None:
            for index,item in enumerate(row_list):
                if index is len(row_list)-1:
                    string_out=string_out+str(item)
                else:
                    string_out=string_out+str(item)+data_delimiter
        else:
            string_out=row_formatter_string.format(*row_list,delimiter=data_delimiter)
        if end is None:
            end="\n"
        if begin is None:
            begin=""
        return begin+string_out+end

def print_comparison(

var_1, var_2)

If var_1==var_2 prints True, else Prints false and a string representation of the 2 vars

Show source ≡

    def print_comparison(var_1,var_2):
        """If var_1==var_2 prints True, else Prints false and a string representation of the 2 vars"""
        print(var_1==var_2)
        if var_1!=var_2:
            print("The value of variable 1 is :")
            print(var_1)
            print("-"*80)
            print("The value of variable 2 is :")
            print(var_2)
            print("-"*80)

def read_schema(

file_path, format=None)

Reads in a schema and returns it as a python dictionary, the default format is a single string

Show source ≡

    def read_schema(file_path,format=None):
        """Reads in a schema and returns it as a python dictionary, the default format is a single string"""
        if format in[None,'python','pickle']:
            schema=pickle.load(open(file_path,'rb'))
        elif format in ['txt','text','.txt']:
            #Todo fix the other formats
            schema={}
            in_file=open(file_path,'r')
            in_lines=[]
            for line in in_file:
                in_lines.append(line)
                schema[line.split(":")[0]]=line.split(":")[1].replace("\\n","\n")
            #in_dictionary=dict(*str(in_dictionary).split(","))
        return schema

def show_structure_script(

)

Shows a table elements by substituting the names Explicitly :return: None

Show source ≡

    def show_structure_script():
        """ Shows a table elements by substituting the names Explicitly
        :return: None
        """
        os.chdir(TESTS_DIRECTORY)
        options={"data_delimiter":'{data_delimiter}',
                      "column_names_delimiter":"{column_names_delimiter}",
                      "specific_descriptor":'Data',
                      "general_descriptor":'Table',
                      "directory":None,
                      "extension":'txt',
                      "comment_begin":"{comment_begin}",
                      "comment_end":"{comment_end}\n",
                      "inline_comment_begin":"{inline_comment_begin}",
                      "inline_comment_end":"{inline_comment_end}",
                      "block_comment_begin":"{block_comment_begin}\n",
                      "block_comment_end":"\n{block_comment_end}",
                      "footer_begin_token":"{footer_begin_token}\n",
                      "footer_end_token":"\n{footer_end_token}",
                      "header_begin_token":"{header_begin_token}\n",
                      "header_end_token":"\n{header_end_token}",
                      "column_names_begin_token":"{column_names_begin_token}",
                      "column_names_end_token":"{column_names_end_token}",
                      "data_begin_token":"{data_begin_token}\n",
                      "data_end_token":"\n{data_end_token}",
                      "metadata_delimiter":"{metadata_delimiter}",
                      "header":["self.header[0]","self.header[1]","self.header[2]","self.header[3]","self.header{4]"],
                      "column_names":["column_names[0]","column_names[1]","column_names[2]"],
                      "data":[["data[0][0]","data[1][0]","data[2][0]"],["data[0][1]","data[1][1]","data[2][1]"]],
                      "footer":["self.footer[0]","self.footer[1]"],
                      "inline_comments":[["inline_comments[0][0]",2,-1]],
                      "row_formatter_string":None,
                      "row_begin_token":"{row_begin_token}",
                      "row_end_token":"{row_end_token}\n",
                      "empty_value":None,
                      "data_table_element_separator":'\n{data_table_element_separator}\n',
                      "treat_header_as_comment":None,
                      "treat_footer_as_comment":None
                      }
        new_table=AsciiDataTable(**options)
        #new_table.options["block_comment_begin"]=new_table.options["block_comment_end"]=None
        new_table.options["header_line_types"]=["header","header","line_comment","block_comment","block_comment"]
        print("Printing the string representation of the table")
        print("-"*80)
        print new_table
        test_string=str(new_table)
        test_lines=test_string.splitlines()
        print("Printing the lines representation of the table with line numbers")
        print("-"*80)
        for index,line in enumerate(test_lines):
            print("{0} {1}".format(index,line))

        for item in new_table.elements:
            if item is 'inline_comments':
                pass
            else:
                begin_line=new_table.options["%s_begin_line"%item]
                end_line=new_table.options["%s_end_line"%item]
                print("-"*80)
                print("The result of self.lines[{0}:{1}] is :".format(begin_line,end_line))
                for line in test_lines[begin_line:end_line]:
                    print line
                print("-"*80)

def split_all_rows(

row_list, delimiter=None, escape_character=None)

Splits all rows in a list of rows and returns a 2d list

Show source ≡

    def split_all_rows(row_list,delimiter=None,escape_character=None):
        """Splits all rows in a list of rows and returns a 2d list """
        if type(row_list) is not ListType:
            print("Split row argument (%s) was not a list"%str(row_list))
            return row_list
        out_list=[]
        for row in row_list:
            out_list.append(split_row(row,delimiter=delimiter,escape_character=escape_character))
        return out_list

def split_row(

row_string, delimiter=None, escape_character=None)

Splits a row given a delimiter, and ignores any delimiters after an escape character returns a list. If the string is unsplit returns a list of length 1

Show source ≡

    def split_row(row_string,delimiter=None,escape_character=None):
        """Splits a row given a delimiter, and ignores any delimiters after an escape character
        returns a list. If the string is unsplit returns a list of length 1"""
        check_arg_type(row_string,StringType)
        if delimiter is None:
            row_list=[row_string]
            return row_list
        if escape_character is None:
            row_list=row_string.split(delimiter)
        else:
            temp_row_string=row_string.replace(escape_character+delimiter,'TempPlaceHolder')
            temp_row_list=temp_row_string.split(delimiter)
            for item in temp_row_list:
                item.replace('TempPlaceHolder',escape_character+delimiter)
            row_list=temp_row_list
        return row_list

def string_list_collapse(

list_of_strings, string_delimiter='\n')

Makes a list of strings a single string

Show source ≡

    def string_list_collapse(list_of_strings,string_delimiter='\n'):
        """ Makes a list of strings a single string"""
        check_arg_type(list_of_strings,ListType)
        if string_delimiter is None:
            string_delimiter=""
        out_string=''
        for index,item in enumerate(list_of_strings):
            if index is len(list_of_strings)-1:
                out_string=out_string+item
            else:
                out_string=out_string+item+string_delimiter
        return out_string

def strip_all_line_tokens(

string_list, begin_token=None, end_token=None)

Strips all line tokens from a list of strings, meant to reverse the action of line_list_comment_string with block=false

Show source ≡

    def strip_all_line_tokens(string_list,begin_token=None,end_token=None):
        """Strips all line tokens from a list of strings, meant  to reverse the action of line_list_comment_string
        with block=false"""
        check_arg_type(string_list,ListType)
        stripped_list=[]
        for row in string_list:
            check_arg_type(row,StringType)
            stripped_list.append(strip_line_tokens(row,begin_token=begin_token,end_token=end_token))
        return stripped_list

def strip_begin_end_tokens(

string_list, begin_token=None, end_token=None)

Strips out tokens at the begining and ending of a list of strings. Meant to reverse the action of "begin_data_token", etc. This does not work with the end_token's because of where the is.

Show source ≡

    def strip_begin_end_tokens(string_list,begin_token=None,end_token=None):
        """Strips out tokens at the begining and ending of a list of strings. Meant to reverse the
        action of "begin_data_token", etc. This does not work with the end_token's because of where the \n is."""
        check_arg_type(string_list,ListType)
        out_list=string_list
        # check the first line to see if it is equal to begin token, if so remove it
        # if any token is None ignore it
        # replace it and leave the line alone otherwize
        if begin_token is None or begin_token is "":
            pass
        else:
            #print("The {0} var is {1}".format('string_list',string_list))
            #print("The {0} var is {1}".format('string_list[0]',string_list[0]))
            #print_comparison(string_list[0],begin_token)
            if string_list[0]==begin_token:
                out_list.pop(0)
            else:
               out_list[0]=out_list[0].replace(begin_token,"")

        if end_token is None or end_token is "":
            pass
        else:
            #print_comparison(string_list[-1],end_token)
            if out_list[-1]==end_token:
                out_list.pop(-1)
            else:
                out_list[-1]=out_list[-1].replace(end_token,"")
        # print("The {0} var is {1}".format('out_list',out_list))
        return out_list

def strip_inline_comments(

list_of_strings, begin_token='(*', end_token='*)')

Removes inline coments from a list of strings

Show source ≡

    def strip_inline_comments(list_of_strings,begin_token='(*',end_token='*)'):
        "Removes inline coments from a list of strings"
        if begin_token in [None] and end_token in [None]:
            return list_of_strings
        match=re.compile('{0}(?P<inline_comments>.+){1}'.format(re.escape(begin_token),re.escape(end_token)))
        out_list=[]
        for index,line in enumerate(list_of_strings):
            out_list.append(re.sub(match,'',line))
        return out_list

def strip_line_tokens(

string, begin_token=None, end_token=None)

Strips a begin and end token if present from an inputted string, meant to remove line_comments

Show source ≡

    def strip_line_tokens(string,begin_token=None,end_token=None):
        """Strips a begin and end token if present from an inputted string, meant to remove line_comments"""
        check_arg_type(string,StringType)
        string_out=string
        try:
            if begin_token is not None:
                string_out=string_out.replace(begin_token,'')
            if end_token is not None:
                string_out=string_out.replace(end_token,'')
        except:
            pass
        return string_out

def strip_tokens(

string_list, *remove_tokens)

Strips all tokens in the list remove_tokens from a list of strings Returns the list with less elements if the tokens contained " ". Now newline characters are returned in the list elements. Meant to reverse the action of adding tokens to a list of strings

Show source ≡

    def strip_tokens(string_list,*remove_tokens):
        """Strips all tokens in the list remove_tokens from a list of strings
        Returns the list with less elements if the tokens contained "\n". Now newline characters are returned in the list
        elements. Meant to reverse the action of adding tokens to a list of strings"""
        temp_string=string_list_collapse(string_list,string_delimiter="")
        remove_list=[]
        for token in remove_tokens:
            if token:
                # we can't strip endlines f
                if token=="\n":
                    print("Warning \\n is in the remove tokens")
                    break
                remove_list.append(token)
                # print remove_list
        try:
            for item in remove_list:
                temp_string=temp_string.replace(item,"")
                #print temp_string
        except:
            print("Strip Tokens Did not work")
            pass
        # spliting using "\n" seems to give an extra empty element at the end always
        new_string_list=temp_string.splitlines()
        # now we add endlines back in for consistency
        for index,line in enumerate(new_string_list):
            new_string_list[index]=line+'\n'
        return new_string_list

def test_AsciiDataTable(

)

Show source ≡

    def test_AsciiDataTable():
        options={"column_names":["a","b","c"],"data":[[0,1,2],[2,3,4]],"data_delimiter":'\t',
                 "header":['Hello There\n',"My Darling"],"column_names_begin_token":'!',"comment_begin":'#',
                 "directory":TESTS_DIRECTORY}
        new_table=AsciiDataTable(file_path=None,**options)
        print new_table.data
        #print dir(new_table)
        for key,value in new_table.options.iteritems():
            print("{0} = {1}".format(key,value))
        print new_table.get_header_string()
        print new_table.get_data_string()
        print new_table.build_string()
        print new_table.path
        new_table.save()
        print("Test add_index")
        new_table.add_index()
        new_table.add_index()
        new_table.add_index()
        print new_table
        new_table.data[1][0]=4
        print new_table
        new_table.update_index()
        print new_table

def test_AsciiDataTable_equality(

)

Show source ≡

    def test_AsciiDataTable_equality():
        options={"column_names":["a","b","c"],"data":[[0,1,2],[2,3,4]],"data_delimiter":'\t',
                 "header":['Hello There',"My Darling"],"column_names_begin_token":'!',"comment_begin":'#',
                 "directory":TESTS_DIRECTORY,"treat_header_as_comment":False}
        new_table=AsciiDataTable(file_path=None,**options)
        print new_table.data
        new_table_2=AsciiDataTable()
        new_table_2.options=new_table.options
        new_table_2.header=new_table.header
        new_table_2.column_names=new_table.column_names
        new_table_2.data=[[0,1,2],[2,3,4]]
        print new_table==new_table_2
        print new_table!=new_table_2
        new_table_2.data[0][0]=9

        print new_table.data==new_table_2.data
        print new_table==new_table_2
        print new_table_2
        print new_table

def test_add_index(

)

Show source ≡

    def test_add_index():
        options={"column_names":["a","b","c"],"data":[[0,1,2],[2,3,4]],"data_delimiter":'\t',
                 "header":['Hello There\n',"My Darling"],"column_names_begin_token":'!',"comment_begin":'#',
                 "directory":TESTS_DIRECTORY,'treat_header_as_comment':False}
        new_table=AsciiDataTable(**options)
        print "Table before add row"
        print "*"*80
        print new_table
        print "Add the row 0,1,3"
        print "*"*80
        new_table.add_row([0,1,3])
        print new_table
        print "Add an index"
        print "*"*80
        new_table.add_index()
        print new_table
        print "Now Get the index column"
        print new_table.get_column(column_name='index')

def test_add_row(

)

Show source ≡

    def test_add_row():
        options={"column_names":["a","b","c"],"data":[[0,1,2],[2,3,4]],"data_delimiter":'\t',
                 "header":['Hello There\n',"My Darling"],"column_names_begin_token":'!',"comment_begin":'#',
                 "directory":TESTS_DIRECTORY}
        new_table=AsciiDataTable(**options)
        print "Table before add row"
        print new_table
        print "Add the row 0,1,3"
        new_table.add_row([0,1,3])
        print new_table

def test_change_unit_prefix(

)

Tests the change_unit_prefix method of the AsciiDataTable Class

Show source ≡

    def test_change_unit_prefix():
        """Tests the change_unit_prefix method of the AsciiDataTable Class
        """
        options={"column_names":["Frequency","b","c"],"column_names_delimiter":",","data":[[0.1*10**10,1,2],[2*10**10,3,4]],"data_delimiter":'\t',
                 "header":['Hello There',"My Darling"],"column_names_begin_token":'#',"comment_begin":'!',
                 "comment_end":"\n",
                 "directory":TESTS_DIRECTORY,
                 "column_units":["Hz",None,None],
                 "column_descriptions":["Frequency in Hz",None,None],
                 "column_types":['float','float','float'],
                 "row_formatter_string":"{0:.2e}{delimiter}{1}{delimiter}{2}",
                 "treat_header_as_comment":True}
        new_table=AsciiDataTable(None,**options)
        print(" After creation the table looks like")
        print("*"*80)
        print new_table
        print new_table.options["column_units"]
        print new_table.options["column_descriptions"]
        new_table.change_unit_prefix(column_selector='Frequency',old_prefix=None,new_prefix='G',unit='Hz')
        print(" After running change_unit_prefix(column_selector='Frequency',old_prefix=None,new_prefix='G',unit='Hz')  "
              "the table is ")
        print new_table
        print("The options column_units and column_descriptions are:")
        print new_table.options["column_units"]
        print new_table.options["column_descriptions"]
        print new_table.is_valid()

def test_inline_comments(

)

Show source ≡

    def test_inline_comments():
        options={"column_names":["a","b","c"],"data":[[0,1,2],[2,3,4]],"data_delimiter":'\t',
                 "header":['Hello There\n',"My Darling"],"column_names_begin_token":'!',"comment_begin":'#',
                 "directory":TESTS_DIRECTORY,'inline_comment_begin':'(*','inline_comment_end':'*)',
                 'inline_comments':[["My Inline Comment",0,5]]}
        new_table=AsciiDataTable(**options)
        print new_table

def test_open_existing_AsciiDataTable(

)

Show source ≡

    def test_open_existing_AsciiDataTable():
        # To make this controlled we should create a data table and then save it.
        # after saving it we should open it with the options
        options={"data_delimiter":',',
                      "column_names_delimiter":"{column_names_delimiter}",
                      "specific_descriptor":'Data',
                      "general_descriptor":'Table',
                      "directory":None,
                      "extension":'txt',
                      "comment_begin":"{comment_begin}",
                      "comment_end":"{comment_end}",
                      "inline_comment_begin":"{inline_comment_begin}",
                      "inline_comment_end":"{inline_comment_end}",
                      "block_comment_begin":"{block_comment_begin}\n",
                      "block_comment_end":"\n{block_comment_end}",
                      "footer_begin_token":"{footer_begin_token}\n",
                      "footer_end_token":"\n{footer_end_token}",
                      "header_begin_token":"{header_begin_token}\n",
                      "header_end_token":"\n{header_end_token}",
                      "column_names_begin_token":"{column_names_begin_token}",
                      "column_names_end_token":"{column_names_end_token}",
                      "data_begin_token":"{data_begin_token}\n",
                      "data_end_token":"\n{data_end_token}",
                      "metadata_delimiter":"{metadata_delimiter}",
                      "header":["self.header[0]","self.header[1]","self.header[2]","self.header[3]","","self.header[4]"],
                      "column_names":["column_names[0]","column_names[1]","column_names[2]"],
                      "data":[["data[0][0]","data[1][0]","data[2][0]",'1.0'],["data[0][1]","data[1][1]","data[2][1]",2.0]],
                      "footer":["self.footer[0]","self.footer[1]"],
                      "inline_comments":[["inline_comments[0][0]",2,-1]],
                      "empty_value":None,
                      "data_table_element_separator":'\n{data_table_element_separator}\n',
                      "treat_header_as_comment":None,
                      "treat_footer_as_comment":None,
                      "header_line_types":["block_comment","block_comment","line_comment","header","header","line_comment"],
                      "column_types":["str","string","STR","float"],
                      "row_formatter_string":"{0}{delimiter}{1}{delimiter}{2}{delimiter}{3:.2f}"

                      }
        os.chdir(TESTS_DIRECTORY)
        new_table=AsciiDataTable(None,**options)
        table_string_1=str(new_table)
        print("Printing the lines representation of the table with line numbers")
        print("-"*80+"\n")
        for index,line in enumerate(table_string_1.splitlines()):
            print("{0} {1}".format(index,line))
        new_table.save()
        file_path=new_table.path
        new_options={}
        for key,value in new_table.options.iteritems():
            new_options[key]=value
        new_table_1=AsciiDataTable(file_path=file_path,**new_options)
        table_string_2=str(new_table_1)
        print("Printing the lines representation of the opened table with line numbers")
        print("-"*80+"\n")
        for index,line in enumerate(table_string_2.splitlines()):
            print("{0} {1}".format(index,line))
        #print new_table_1
        print("Printing the line by line equality of the table strings")
        print("-"*80)
        for index,line in enumerate(table_string_2.splitlines()):
            print("{0} {1}".format(index,line==table_string_1.splitlines()[index]))
        print("The assertion that the built table is equal to the opened table is {0}".format(new_table==new_table_1))

def test_read_schema(

)

Tests the read_schema function

Show source ≡

    def test_read_schema():
        """ Tests the read_schema function
        """
        os.chdir(TESTS_DIRECTORY)
        file_path="Data_Table_20160301_031_Schema_20160301_001.txt"
        schema=read_schema(file_path)
        print schema

def test_save_schema(

)

tests the save schema method of the Ascii Data Table

Show source ≡

    def test_save_schema():
        "tests the save schema method of the Ascii Data Table"
        os.chdir(TESTS_DIRECTORY)
        options={"data_delimiter":'{data_delimiter}',
                      "column_names_delimiter":"{column_names_delimiter}",
                      "specific_descriptor":'Data',
                      "general_descriptor":'Table',
                      "directory":None,
                      "extension":'txt',
                      "comment_begin":"{comment_begin}",
                      "comment_end":"{comment_end}",
                      "inline_comment_begin":"{inline_comment_begin}",
                      "inline_comment_end":"{inline_comment_end}",
                      "block_comment_begin":"{block_comment_begin}\n",
                      "block_comment_end":"\n{block_comment_end}",
                      "footer_begin_token":"{footer_begin_token}\n",
                      "footer_end_token":"\n{footer_end_token}",
                      "header_begin_token":"{header_begin_token}\n",
                      "header_end_token":"\n{header_end_token}",
                      "column_names_begin_token":"{column_names_begin_token}",
                      "column_names_end_token":"{column_names_end_token}",
                      "data_begin_token":"{data_begin_token}\n",
                      "data_end_token":"\n{data_end_token}",
                      "metadata_delimiter":"{metadata_delimiter}",
                      "header":["self.header[0]","self.header[1]","self.header[2]","self.header[3]","","self.header[4]"],
                      "column_names":["column_names[0]","column_names[1]","column_names[2]"],
                      "data":[["data[0][0]","data[1][0]","data[2][0]",'1.0'],["data[0][1]","data[1][1]","data[2][1]",2.0]],
                      "footer":["self.footer[0]","self.footer[1]"],
                      "inline_comments":[["inline_comments[0][0]",2,-1]],
                      "empty_value":None,
                      "data_table_element_separator":'\n{data_table_element_separator}\n',
                      "treat_header_as_comment":None,
                      "treat_footer_as_comment":None,
                      "header_line_types":["block_comment","block_comment","line_comment","header","header","line_comment"],
                      "column_types":["str","string","STR","float"],
                      "row_formatter_string":"{0}{delimiter}{1}{delimiter}{2}{delimiter}{3:.2f}"

                      }
        new_table=AsciiDataTable(None,**options)
        print(" New Table is:")
        new_table.__parse__()
        print new_table
        print type(new_table.data[0][3])
        print new_table.get_data_dictionary_list()
        new_table.add_index()
        print new_table.get_data_dictionary_list(False)
        #print("-"*80)
        #new_table.save()
        #new_table.save_schema()
        options_2=new_table.options

## Classes

class AsciiDataTable

An AsciiDatable is a generalized model of a data table with optional header, column names,rectangular array of data, and footer

Show source ≡

    class AsciiDataTable():
        """ An AsciiDatable is a generalized model of a data table with optional header,
        column names,rectangular array of data, and footer """
        def __init__(self,file_path=None,**options):
            " Initializes the AsciiDataTable class "
            # This is a general pattern for adding a lot of options
            defaults={"data_delimiter":None,
                      "column_names_delimiter":None,
                      "specific_descriptor":'Data',
                      "general_descriptor":'Table',
                      "directory":None,
                      "extension":'txt',
                      "comment_begin":None,
                      "comment_end":None,
                      "inline_comment_begin":None,
                      "inline_comment_end":None,
                      "block_comment_begin":None,
                      "block_comment_end":None,
                      "footer_begin_line":None,
                      "footer_end_line":None,
                      "header_begin_line":None,
                      "header_end_line":None,
                      "column_names_begin_line":None,
                      "column_names_end_line":None,
                      "data_begin_line":None,
                      "data_end_line":None,
                      "footer_begin_token":None,
                      "footer_end_token":None,
                      "header_begin_token":None,
                      "header_end_token":None,
                      "column_names_begin_token":None,
                      "column_names_end_token":None,
                      "data_begin_token":None,
                      "data_end_token":None,
                      "metadata_delimiter":None,
                      "metadata_key_value_delimiter":None,
                      "header_line_types":None,
                      "column_types":None,
                      "column_descriptions":None,
                      "column_units":None,
                      "footer_line_types":None,
                      "header":None,
                      "column_names":None,
                      "data":None,
                      "footer":None,
                      "inline_comments":None,
                      "row_begin_token":None,
                      "row_end_token":None,
                      "row_formatter_string":None,
                      "empty_value":None,
                      "escape_character":None,
                      "data_table_element_separator":'\n',
                      "treat_header_as_comment":None,
                      "treat_footer_as_comment":None,
                      "metadata":None
                      }
            #some of the options have the abiltiy to confilct with each other, so there has to be a
            #built-in way to determine the precedence of each option, for import lines first, then begin and then end
            self.options={}
            for key,value in defaults.iteritems():
                self.options[key]=value
            for key,value in options.iteritems():
                self.options[key]=value
            self.elements=['header','column_names','data','footer','inline_comments']
            #Define Method Aliases if they are available
            #unqualified exec is not allowed in function '__init__' because it contains a nested function with free variables
            # This is because __init__ has nested functions
            # if METHOD_ALIASES:
            #     for command in alias(self):
            #         exec(command)
            if file_path is None:
                #create a new data table
                if DEFAULT_FILE_NAME is None:
                    self.name=auto_name(self.options["specific_descriptor"],
                                        self.options["general_descriptor"],
                                        self.options["directory"],
                                        self.options["extension"])
                    if self.options['directory'] is None:
                        self.path=self.name
                    else:
                        self.path=os.path.join(self.options["directory"],self.name)
                #Now we see if the table has been defined in the options
                # We should reset the self.options versions to None after this so as to not recreate or we
                # can use it as a cache and add a method reset_table which either redoes the below or reloads the saved file
                self.header=self.options["header"]
                self.column_names=self.options["column_names"]
                self.data=self.options["data"]
                self.footer=self.options["footer"]
                self.metadata=self.options["metadata"]
                self.inline_comments=self.options["inline_comments"]
                self.initial_state=[self.options["header"],self.options["column_names"],
                                    self.options["data"],self.options["footer"],
                                    self.options["inline_comments"]]
                #print "I got here {0}".format(self.path)
                [self.options["header"],self.options["column_names"],
                                    self.options["data"],self.options["footer"],
                                    self.options["inline_comments"]]=[None for i in self.elements]

                #if you are validating the model, you have to skip the updating until it hsa been parsed
                try:
                    if self.options["validate"]:
                        del self.options["validate"]
                        pass
                except KeyError:
                    self.update_model()

            else:
                # open the file and read it in as lines
                # do we parse it here?
                # once parsed we should end up with the major components
                # if we are given options we should use them, if not try to autodetect them?
                # we can just return an error right now and then have an __autoload__ method
                # we can assume it is in ascii or utf-8
                # set any attribute that has no options to None
                import_table=[]
                for item in self.elements:
                    if len(filter(lambda x: None!=x,self.get_options_by_element(item).values()))==0:
                        self.__dict__[item]=None
                        #elements.remove(item)
                    elif item not in ['inline_comments']:
                        self.__dict__[item]=[]
                        import_row=[self.options['%s_begin_line'%item],
                                    self.options['%s_end_line'%item],
                                    self.options['%s_begin_token'%item],
                                    self.options['%s_end_token'%item]]
                        import_table.append(import_row)
                    elif item in ['inline_comments']:
                        self.inline_comments=self.options['inline_comments']
                file_in=open(file_path,'r')
                # in order to parse the file we need to know line #'s, once we deduce them we use __parse__
                self.lines=[]
                for line in file_in:
                    self.lines.append(line)
                file_in.close()
                self.path=file_path
                if self.lines_defined():
                    self.__parse__()
                else:
                    import_table[0][0]=0
                    import_table[-1][1]=None
                    # This is to make sure the lines inbetween the data table's elements are accounted for
                    if self.options['data_table_element_separator'] is None:
                        inner_element_spacing=0
                    else:
                        inner_element_spacing=self.options['data_table_element_separator'].count('\n')-1
                    #print import_table
                    self.update_import_options(import_table=import_table)
                    #self.get_options()
                    if self.lines_defined():
                        #print("%s says %s"%('self.lines_defined()',str(self.lines_defined())))
                        self.__parse__()
                    row_zero=[import_table[i][0] for i in range(len(import_table))]
                    for index,item in enumerate(row_zero):
                        #print import_table
                        #print index,item
                        if index>0:
                            #print("Row Zero Loop Returns index={0}, item={1}".format(index,item))
                            if item is not None:
                                import_table[index-1][1]=item+inner_element_spacing
                                #print import_table
                                self.update_import_options(import_table)
                                #print self.lines_defined()
                    if self.lines_defined():
                            self.__parse__()
                    else:
                        row_one=[import_table[i][1] for i in range(len(import_table))]
                        for index,item in enumerate(row_one):
                            #print("Row One Loop Returns index={0}, item={1}".format(index,item))
                            if index<(len(row_one)-1):
                                #print((index+1)<len(row_one))
                                #print("Row One Loop Returns index={0}, item={1}".format(index,item))
                                if item is not None:
                                    #print import_table
                                    import_table[index+1][0]=item-inner_element_spacing
                                    self.update_import_options(import_table)
                        if self.lines_defined():
                            self.__parse__()
                        else:
                            row_two=[import_table[i][2] for i in range(len(import_table))]
                            for index,item in enumerate(row_two):
                                if item is not None:
                                    import_table[index][0]=self.find_line(item)
                            for index,item in enumerate(row_zero):
                                if index>0:
                                    if item is not None:
                                        import_table[index-1][1]=item++inner_element_spacing
                                        self.update_import_options(import_table)
                            if self.lines_defined():
                                self.__parse__()
                            else:
                                row_three=[import_table[i][3] for i in range(len(import_table))]
                                for index,item in enumerate(row_three):
                                    if item is not None:
                                        import_table[index][1]=self.find_line(item)
                                for index,item in enumerate(row_one):
                                    if index<(len(row_one)-1):
                                        if item is not None:
                                            import_table[index+1][0]=item-inner_element_spacing
                                self.update_import_options(import_table)
                                if self.lines_defined():
                                    self.__parse__()
                                else:
                                    print("FAILED to import file!")
                                    raise

        def find_line(self,begin_token):
            """Finds the first line that has begin token in it"""
            for index,line in enumerate(self.lines):
                if re.match(begin_token,line):
                    return index

        def update_import_options(self,import_table):
            """Updates the options in the import table"""
            for index,element in enumerate(self.elements):
                if self.__dict__[element] is not None:
                    [self.options['%s_begin_line'%element],
                                    self.options['%s_end_line'%element],
                                    self.options['%s_begin_token'%element],
                                    self.options['%s_end_token'%element]]=import_table[index][:]
                    #self.get_options_by_element(element)

        def lines_defined(self):
            """If begin_line and end_line for all elements that are None are defined returns True"""
            truth_table=[]
            last_element=""
            output=False
            for index,element in enumerate(self.elements):
                if element not in ['inline_comments'] and self.__dict__[element] is not None:
                    try:
                        last_element=element
                        if not None in [self.options['%s_begin_line'%element],self.options['%s_end_line'%element]]:
                            truth_table.append(True)
                        else:
                             truth_table.append(False)
                    except:
                        return False
            #print truth_table
            # The last_line of the last element is fine to be none
            if truth_table[-1] is False:
                if self.options['%s_begin_line'%last_element] is not None:
                    truth_table[-1]=True
            if False in truth_table:
                output=False
            else:
                output=True
            #print output
            return output

        def __parse__(self):
            """Parses self.lines into its components once all the relevant begin and end lines have been set. It assumes
             that the self.__dict__[self.element[i]]=None for elements that are not defined"""
            # Collect the inline comments if they are not already defined
            if self.inline_comments is None:
               self.inline_comments=collect_inline_comments(self.lines,begin_token=self.options["inline_comment_begin"],
                                                                 end_token=self.options["inline_comment_end"])
            # Strip the inline comments
            self.lines=strip_inline_comments(self.lines,
                                                 begin_token=self.options['inline_comment_begin'],
                                                 end_token=self.options['inline_comment_end'])
            # Define each major element that are not inline comments by their line numbers
            for index,element in enumerate(self.elements):
                if self.__dict__[element] is not None and element not in ['inline_comments']:
                    try:
                        if not None in [self.options['%s_begin_line'%element]]:
                            content_list=self.lines[
                                                self.options['%s_begin_line'%element]:self.options['%s_end_line'%element]]
                            self.__dict__[element]=content_list
                            # print("The result of parsing is self.{0} = {1}".format(element,content_list))
                    except:
                        raise
            # Remove any defined begin and end tokens
            for index,element in enumerate(self.elements):
                if self.__dict__[element] is not None and element not in ["inline_comments"]:
                            for index,line in enumerate(self.__dict__[element]):
                                self.__dict__[element][index]=line

                            content_list=strip_tokens(self.__dict__[element],
                                                      *[self.options['%s_begin_token'%element],
                                                        self.options['%s_end_token'%element]])
                            self.__dict__[element]=content_list
                            # print("The result of parsing is self.{0} = {1}".format(element,content_list))
            # parse the header
            if self.header is not None:
                #print("The {0} variable is {1}".format('self.header',self.header))
                remove_tokens=[self.options['block_comment_begin'],self.options['block_comment_end']]
                temp_header=self.header
                if self.options["treat_header_as_comment"] is None or self.options["treat_header_as_comment"]:
                    remove_tokens=[self.options["block_comment_begin"],
                                                        self.options["block_comment_end"]]
                    if self.options["data_table_element_separator"] is not None and self.options['comment_end'] != None:
                        remove_tokens.append(self.options['comment_end']+self.options["data_table_element_separator"])
                    temp_header=strip_tokens(temp_header,
                                                       *remove_tokens)
                    temp_header=strip_all_line_tokens(temp_header,
                                                      self.options['comment_begin'],self.options['comment_end'])
                else:
                    pass

                #print("The {0} variable is {1}".format('temp_header',temp_header))
                #temp_header=strip_tokens(temp_header,*remove_tokens)
                # print("The {0} variable is {1}".format('temp_header',temp_header))
                for index,line in enumerate(temp_header):
                    temp_header[index]=line.replace('\n',"")

                self.header=temp_header
                # print("The {0} variable is {1}".format('self.header',self.header))
            # parse the column_names
            if self.column_names is not None:
                #print("The result of parsing is self.{0} = {1}".format('column_names',self.column_names))
                self.column_names=strip_all_line_tokens(self.column_names,begin_token=self.options['column_names_begin_token'],
                                                  end_token=self.options['column_names_end_token'])
                # print("The result of parsing is self.{0} = {1}".format('column_names',self.column_names))
                self.column_names=split_all_rows(self.column_names,delimiter=self.options["column_names_delimiter"],
                                            escape_character=self.options["escape_character"])
                # print("The result of parsing is self.{0} = {1}".format('column_names',self.column_names))
                self.column_names=self.column_names[0]
                for index,line in enumerate(self.column_names):
                    self.column_names[index]=line.replace('\n',"")
                # print("The result of parsing is self.{0} = {1}".format('column_names',self.column_names))
            # parse the data
            if self.data is not None:
                self.data=strip_all_line_tokens(self.data,begin_token=self.options["row_begin_token"],
                                                end_token=self.options["row_end_token"])
                #print("The result of parsing is self.{0} = {1}".format('data',self.data))
                self.data=split_all_rows(self.data,delimiter=self.options["data_delimiter"],
                                         escape_character=self.options["escape_character"])
                #print("The result of parsing is self.{0} = {1}".format('data',self.data))
                self.data=convert_all_rows(self.data,self.options["column_types"])
                #print("The result of parsing is self.{0} = {1}".format('data',self.data))
            # parse the footer
            if self.footer is not None:
                #print("The {0} variable is {1}".format('self.footer',self.footer))
                remove_tokens=[self.options['block_comment_begin'],self.options['block_comment_end']]
                temp_footer=self.footer
                if self.options["treat_footer_as_comment"] is None or self.options["treat_footer_as_comment"]:
                    temp_footer=strip_tokens(temp_footer,
                                                       *[self.options["block_comment_begin"],
                                                       self.options["block_comment_end"]])
                    temp_footer=strip_all_line_tokens(temp_footer,
                                                      self.options['comment_begin'],self.options['comment_end'])
                else:
                    pass

                #print("The {0} variable is {1}".format('temp_footer',temp_footer))
                #temp_footer=strip_tokens(temp_footer,*remove_tokens)
                #print("The {0} variable is {1}".format('temp_footer',temp_footer))
                for index,line in enumerate(temp_footer):
                    temp_footer[index]=line.replace('\n',"")

                self.footer=temp_footer
                # print("The {0} variable is {1}".format('self.footer',self.footer))

        def get_options_by_element(self,element_name):
            """ returns a dictionary
             of all the options that have to do with element. Element must be header,column_names,data, or footer"""
            keys_regarding_element=filter(lambda x: re.search(element_name,str(x),re.IGNORECASE),self.options.keys())
            out_dictionary={key:self.options[key] for key in keys_regarding_element}
            #print out_dictionary
            return out_dictionary

        def __str__(self):
            "Controls the str output of AsciiDataTable"
            self.string=self.build_string()
            return self.string

        def update_index(self):
            """ Updates the index column if it exits, otherwise exits quietly
            """
            if 'index' not in self.column_names:
                return
            else:
                try:
                    #This should be 0 but just in case
                    index_column_number=self.column_names.index('index')
                    for i in range(len(self.data)):
                        self.data[i][index_column_number]=i
                except:
                    pass

        def update_model(self):
            """Updates the model after a change has been made. If you add anything to the attributes of the model,
            or change this updates the values. If the model has an index column it will make sure the numbers are correct.
            In addition, it will update the options dictionary to reflect added rows, changes in deliminators etc.  """
            if self.column_names is not None and 'index' in self.column_names:
               self.update_index()
            #make sure there are no "\n" characters in the element lists (if so replace them with "") for data this is
            # done on import
            list_types=["header","column_names","footer"]
            for element in list_types:
                if self.__dict__[element] is not None:
                    for index,item in enumerate(self.__dict__[element]):
                        self.__dict__[element][index]=item.replace("\n","")
            self.update_column_names()
            if self.data is not None:
                self.data=convert_all_rows(self.data,self.options["column_types"])
            self.string=self.build_string()
            self.lines=self.string.splitlines()

        def update_column_names(self):
            """Update column names adds the value x# for any column that exists in self.data that is not named"""
            if self.data is None:
                return
            elif type(self.column_names) is StringType:
                self.column_names=split_row(self.column_names,self.options["column_names_delimiter"])
            elif self.column_names is None:
                column_names=[]
                for index,column in enumerate(self.data[0]):
                    column_names.append("x"+str(index))
                self.column_names=column_names
                return
            elif len(self.column_names)==len(self.data[0]):
                return
            elif len(self.column_names) < len(self.data[0]):
                for index in range(len(self.column_names),len(self.data[0])):
                    self.column_names.append("x"+str(index))
                return

        def save(self,path=None,**temp_options):
            """" Saves the file, to save in another ascii format specify elements in temp_options, the options
            specified do not permanently change the object's options. If path is supplied it saves the file to that path
            otherwise uses the object's attribute path to define the saving location """
            original_options=self.options
            for key,value in temp_options.iteritems():
                self.options[key]=value
            out_string=self.build_string(**temp_options)
            if path is None:
                path=self.path
            file_out=open(path,'w')
            file_out.write(out_string)
            file_out.close()
            self.options=original_options

        def build_string(self,**temp_options):
            """Builds a string representation of the data table based on self.options, or temp_options.
            Passing temp_options does not permanently change the model"""
            # store the original options to be put back after the string is made
            original_options=self.options
            for key,value in temp_options.iteritems():
                self.options[key]=value
            section_end=0
            next_section_begin=0
            if self.options['data_table_element_separator'] is None:
                inner_element_spacing=0
            else:
                inner_element_spacing=self.options['data_table_element_separator'].count('\n')-1
            string_out=""
            between_section=""
            if self.options['data_table_element_separator'] is not None:
                between_section=self.options['data_table_element_separator']
            if self.header is None:
                self.options['header_begin_line']=self.options['header_end_line']=None
                pass
            else:
                self.options["header_begin_line"]=0
                if self.data is None and self.column_names is None and self.footer is None:
                    string_out=self.get_header_string()
                    self.options["header_end_line"]=None
                else:
                    string_out=self.get_header_string()+between_section

                    last_header_line=self.get_header_string().count('\n')+1
                    self.options["header_end_line"]=last_header_line
                    next_section_begin=last_header_line+inner_element_spacing

            if self.column_names is None:
                self.options['column_names_begin_line']=self.options['column_names_end_line']=None
                pass
            else:
                self.options["column_names_begin_line"]=next_section_begin
                if self.data is None and self.footer is None:
                    self.options["column_names_end_line"]=None
                    string_out=string_out+self.get_column_names_string()
                else:
                    string_out=string_out+self.get_column_names_string()+between_section
                    last_column_names_line=self.get_column_names_string().count('\n')+\
                                           self.options["column_names_begin_line"]+1
                    self.options["column_names_end_line"]=last_column_names_line
                    next_section_begin=last_column_names_line+inner_element_spacing
            if self.data is None:
                self.options['data_begin_line']=self.options['data_end_line']=None
                pass
            else:
                self.options["data_begin_line"]=next_section_begin
                if self.footer is None:
                    self.options["data_end_line"]=None
                    string_out=string_out+self.get_data_string()
                else:
                    string_out=string_out+self.get_data_string()+between_section
                    last_data_line=self.get_data_string().count("\n")+\
                                    self.options["data_begin_line"]+1
                    self.options["data_end_line"]=last_data_line
                    next_section_begin=last_data_line+inner_element_spacing
            if self.footer is None:
                self.options['footer_begin_line']=self.options['footer_end_line']=None
                pass
            else:
                self.options["footer_begin_line"]=next_section_begin
                string_out=string_out+self.get_footer_string()
                self.options['footer_end_line']=None
            # set the options back after the string has been made
            if self.inline_comments is None:
                pass
            else:
                lines=string_out.splitlines()
                for comment in self.inline_comments:
                    lines=insert_inline_comment(lines,comment=comment[0],line_number=comment[1],
                                                string_position=comment[2],
                                                begin_token=self.options['inline_comment_begin'],
                                                end_token=self.options['inline_comment_end'])
                string_out=string_list_collapse(lines,string_delimiter='\n')
            self.options=original_options
            return string_out

        def get_header_string(self):
            """Returns the header using options in self.options. If block comment is specified, and the header is a
            list it will block comment out the header. If comment_begin and comment_end are specified it will use
            those to represent each line of the header. If header_begin_token and/or header_end_token are specified it
             will wrap the header in those.
            """
            string_out=""
            header_begin=""
            header_end=""
            if self.options["header_begin_token"] is None:
                header_begin=""
            else:
                header_begin=self.options["header_begin_token"]
            if self.options["header_end_token"] is None:
                header_end=""
            else:
                header_end=self.options["header_end_token"]
            # This writes the header
            if self.header is None:
                string_out= ""
            elif self.options["header_line_types"] is not None:
                for index,line in enumerate(self.options["header_line_types"]):
                    if index == len(self.options["header_line_types"])-1:
                        end=''
                    else:
                        end='\n'
                    if line in ['header','header_line','normal']:
                        string_out=string_out+self.header[index]+end
                    elif line in ['line_comment','comment']:
                        string_out=string_out+line_comment_string(self.header[index],
                                                   comment_begin=self.options["comment_begin"],
                                                   comment_end=self.options["comment_end"])+end
                    elif line in ['block_comment','block']:
                        if index-1<0:
                            block_comment_begin=index
                            block_comment_end=index+2
                            continue
                        elif self.options["header_line_types"][index-1] not in ['block_comment','block']:
                            block_comment_begin=index
                            block_comment_end=index+2
                            continue
                        else:
                            if index+1>len(self.options["header_line_types"])-1:
                                string_out=string_out+line_list_comment_string(self.header[block_comment_begin:],
                                                                               comment_begin=self.options['block_comment_begin'],
                                                                                 comment_end=self.options['block_comment_end'],
                                                                               block=True)+end
                            elif self.options["header_line_types"][index+1] in ['block_comment','block']:
                                block_comment_end+=1
                            else:
                                string_out=string_out+\
                                           line_list_comment_string(self.header[block_comment_begin:block_comment_end],
                                                                    comment_begin=self.options['block_comment_begin'],
                                                                    comment_end=self.options['block_comment_end'],
                                                                    block=True)+end
                    else:
                        string_out=string_out+line
            elif self.options['treat_header_as_comment'] in [None,True] and self.options["header_line_types"] in [None]:
                # Just happens if the user has set self.header manually
                if type(self.header) is StringType:
                    string_out=line_comment_string(self.header,
                                                   comment_begin=self.options["comment_begin"],
                                                   comment_end=self.options["comment_end"])
                    #string_out=re.sub('\n','',string_out,count=1)
                elif type(self.header) is ListType:
                    if self.options['block_comment_begin'] is None:
                        if self.options['comment_begin'] is None:
                            string_out=string_list_collapse(self.header)

                        else:
                            string_out=line_list_comment_string(self.header,comment_begin=self.options['comment_begin'],
                                                            comment_end=self.options['comment_end'])
                            lines_out=string_out.splitlines()

                            # if re.search('\n',self.options['comment_end']):
                            #     string_out=re.sub('\n','',string_out,count=1)
                            #self.options["header_line_types"]=["line_comment" for line in self.header]
                    else:
                        string_out=line_list_comment_string(self.header,comment_begin=self.options['block_comment_begin'],
                                                            comment_end=self.options['block_comment_end'],block=True)
                        #self.options["header_line_types"]=["block_comment" for line in self.header]
            else:
                string_out=ensure_string(self.header,list_delimiter="\n",end_if_list="")
            return header_begin+string_out+header_end

        def get_column_names_string(self):
            "Returns the column names as a string using options"
            string_out=""
            # This writes the column_names
            column_name_begin=""
            column_name_end=""
            if self.options["column_names_begin_token"] is None:
                column_name_begin=""
            else:
                column_name_begin=self.options["column_names_begin_token"]
            if self.options["column_names_end_token"] is None:
                column_name_end=""
            else:
                column_name_end=self.options["column_names_end_token"]

            if self.column_names is None:
                string_out=""
            else:
                if type(self.column_names) is StringType:
                    string_out=self.column_names

                elif type(self.column_names) is ListType:
                    string_out=list_to_string(self.column_names,
                                              data_delimiter=self.options["column_names_delimiter"],end="")
                    #print("{0} is {1}".format('string_out',string_out))
                else:

                    string_out=ensure_string(self.column_names)

            #print column_name_begin,string_out,column_name_end
            return column_name_begin+string_out+column_name_end

        def get_data_string(self):
            "Returns the data as a string"
            #Todo:refactor to cut out unused lines
            string_out=""
            if self.data is None:
                string_out= ""
            else:
                if type(self.data) is StringType:
                    if self.options['data_begin_token'] is None:
                           if self.options['data_end_token'] is None:
                               string_out=self.data
                           else:
                               if re.search(self.options['data_end_token'],self.data):
                                   string_out=self.data
                               else:
                                   string_out=self.data+self.options['data_end_token']
                    else:
                            if self.options['data_end_token'] is None:
                                if re.match(self.options['data_begin_token'],self.data):
                                    string_out=self.data
                                else:
                                    string_out=self.options['data_begin_token']+self.data
                elif type(self.data) in [ListType,np.ndarray]:
                    try:
                            #If the first row is a string, we should strip all the tokens and add them back in
                            if type(self.data[0]) is StringType:
                                if self.options['data_begin_token'] is None:
                                    string_out=string_list_collapse(self.data)
                                else:
                                    if re.match(self.options['data_begin_token'],self.data[0]):
                                        if self.options['data_end_token'] is None:
                                            string_out=string_list_collapse(self.data)
                                        else:
                                            if re.search(self.options['data_end_token'],self.data[-1]):
                                                string_out=string_list_collapse(self.data)
                                            else:
                                                string_out=string_list_collapse(self.data)+self.options['data_end_token']
                                    else:
                                        if self.options['data_end_token'] is None:
                                            string_out=self.options['data_begin_token']+string_list_collapse(self.data)
                                        else:
                                            if re.search(self.options['data_end_token'],self.data[-1]):
                                                string_out=self.options['data_begin_token']+string_list_collapse(self.data)
                                            else:
                                                string_out=self.options['data_begin_token']+\
                                                           string_list_collapse(self.data)+\
                                                           self.options['data_end_token']

                            elif type(self.data[0]) in [ListType,np.ndarray]:
                                prefix=""
                                if self.options['data_begin_token'] is None:
                                    if self.options['data_end_token'] is None:
                                        string_out=list_list_to_string(self.data,data_delimiter=self.options['data_delimiter'],
                                                                       row_formatter_string=self.options['row_formatter_string'],
                                                                       line_begin=self.options["row_begin_token"],
                                                                       line_end=self.options["row_end_token"])
                                else:
                                    if self.options['data_end_token'] is None:
                                        string_out=self.options['data_begin_token']+\
                                                   list_list_to_string(self.data,
                                                                       data_delimiter=self.options['data_delimiter'],
                                                                       row_formatter_string=self.options['row_formatter_string'],
                                                                       line_begin=self.options["row_begin_token"],
                                                                       line_end=self.options["row_end_token"])
                                    else:
                                        string_out=self.options['data_begin_token']+\
                                                   list_list_to_string(self.data,
                                                                       data_delimiter=self.options['data_delimiter'],
                                                                       row_formatter_string=\
                                                                       self.options['row_formatter_string'],
                                                                       line_begin=self.options["row_begin_token"],
                                                                       line_end=self.options["row_end_token"])+\
                                                                        self.options['data_end_token']
                            else:
                                string_out=list_to_string(self.data,
                                                          data_delimiter=self.options['data_delimiter'],
                                                          row_formatter_string=self.options['row_formatter_string'],
                                                          begin=self.options["row_begin_token"],
                                                          end=self.options["row_end_token"])

                    except IndexError:
                        pass
                else:
                    string_out=ensure_string(self.data)
            return string_out

        def get_footer_string(self):
            """Returns the footer using options in self.options. If block comment is specified, and the footer is a
            list it will block comment out the footer. If comment_begin and comment_end are specified it will use
            those to represent each line of the footer. If footer_begin_token and/or footer_end_token are specified it
             will wrap the footer in those.
            """
            string_out=""
            footer_begin=""
            footer_end=""
            if self.options["footer_begin_token"] is None:
                footer_begin=""
            else:
                footer_begin=self.options["footer_begin_token"]
            if self.options["footer_end_token"] is None:
                footer_end=""
            else:
                footer_end=self.options["footer_end_token"]
            # This writes the footer
            if self.footer is None:
                string_out= ""
            elif self.options["footer_line_types"] is not None:
                for index,line in enumerate(self.options["footer_line_types"]):
                    if line in ['footer','footer_line','normal']:
                        string_out=string_out+self.footer[index]+'\n'
                    elif line in ['line_comment','comment']:
                        string_out=string_out+line_comment_string(self.footer[index],
                                                   comment_begin=self.options["comment_begin"],
                                                   comment_end=self.options["comment_end"])
                    elif line in ['block_comment','block']:
                        if index-1<0:
                            block_comment_begin=index
                            continue
                        elif self.options["footer_line_types"][index-1] not in ['block_comment','block']:
                            block_comment_begin=index
                            continue
                        else:
                            if index+1>len(self.options["footer_line_types"])-1:
                                string_out=string_out+line_list_comment_string(self.footer[block_comment_begin:],
                                                                               comment_begin=self.options['block_comment_begin'],
                                                                                 comment_end=self.options['block_comment_end'],
                                                                               block=True)
                            elif self.options["footer_line_types"][index+1] in ['block_comment','block']:
                                block_comment_end+=1
                            else:
                                string_out=string_out+line_list_comment_string(self.footer[block_comment_begin:block_comment_end],
                                                                               comment_begin=self.options['block_comment_begin'],
                                                                                 comment_end=self.options['block_comment_end'],
                                                                               block=True)
                    else:
                        string_out=string_out+line

            elif self.options['treat_footer_as_comment'] is None or self.options['treat_footer_as_comment'] is True:
                if type(self.footer) is StringType:
                    string_out=line_comment_string(self.footer,
                                                   comment_begin=self.options["comment_begin"],
                                                   comment_end=self.options["comment_end"])
                elif type(self.footer) is ListType:
                    if self.options['block_comment_begin'] is None:
                        string_out=line_list_comment_string(self.footer,comment_begin=self.options['comment_begin'],
                                                            comment_end=self.options['comment_end'])
                    else:
                        string_out=line_list_comment_string(self.footer,comment_begin=self.options['block_comment_begin'],
                                                            comment_end=self.options['block_comment_end'],block=True)
                else:
                    try:
                        string_out=str(self.footer)
                    except:raise
            else:
                string_out=ensure_string(self.footer,list_delimiter="\n",end_if_list="")
            return footer_begin+string_out+footer_end
        # our current definition of add is not reversible !!!!!!!!!
        # def __radd__(self, other):
        #     "Controls the behavior of radd to use the sum function it is required"
        #     if other==0:
        #         return self
        #     else:
        #         return self.__add__(other)

        def __add__(self, other):
            """Controls the behavior of the addition operator, if column_names are equal it adds rows at the end
            and increments any column named index. If the column_names and number of roows are different it adds columns
            to the table and fills the non-defined rows with self.options['empty_character'] which is None by default. If
            the rows are equal it adds the columns to the table ignoring any columns that are the same. If the headers or
            footers are different it appends them to the left most object."""
            if self==other:
                return
            if self.column_names is other.column_names:
                for row in other.data:
                    self.add_row(row)
            elif len(self.data)==len(other.data):
                for index,column in enumerate(other.column_names):
                    if column in self.column_names:
                        pass
                    else:
                        if other.column_types is not None:
                            column_type=other.column_types[index]
                        else:
                            column_type='string'
                        self.add_column(column_name=column,
                                        column_type=column_type,column_data=other.get_column(column_name=column))
            else:
                for column in other.column_names:
                    self.add_column(column)
                for row in other.data:
                    data=[self.options['empty_value'] for i in self.column_names]
                    self.add_row(data.append(row))
            if self.header is not other.header:
                for line in other.header:
                    if line is None:
                        pass
                    else:
                        self.header.append(line)
            if self.footer is not other.footer:
                for line in other.footer:
                    if line is None:
                        pass
                    else:
                        self.footer.append(line)
            return self

        def is_valid(self):
            """Returns True if ascii table conforms to its specification given by its own options"""
            options={}
            for key,value in self.options.iteritems():
                options[key]=value
                # print("self.options[{0}] is {1} ".format(key,value))
            for element in self.elements:
                if self.__dict__[element] is None:
                    options[element]=None
                else:
                    options[element]=[]
            options["validate"]=True
            newtable=AsciiDataTable(None,**options)
            lines=self.build_string().splitlines()
            for index,line in enumerate(lines):
                lines[index]=line+"\n"
            newtable.lines=lines
            newtable.__parse__()
            # print newtable.data
            # print newtable.column_names
            # print newtable
            #print_comparison(newtable.footer,None)
            newtable.update_model()
            # The new table rows are not being coerced into the right format
            #print newtable
            #newtable.update_model()
            #print newtable.options
            #print self.options
            #print newtable.data
            # print newtable.options==self.options
            # for option_key,option_value in newtable.options.iteritems():
            #     print("New Table Option {0} is {1} ".format(option_key,option_value))
            #     print("self.options[{0}] is {1} ".format(option_key,self.options[option_key]))
            #     print_comparison(option_value,self.options[option_key])
            # #print self
            return self==newtable
            # create a clone and then parse the clone and compare it to the
            # original. If they are equal then it is valid
            #self.add_inline_comments()

        def __eq__(self, other):
            """Defines what being equal means for the AsciiDataTable Class"""
            compare_elements=['options','header','column_names','data','footer']
            truth_table=[]
            output=False
            for item in compare_elements:
                if self.__dict__[item]==other.__dict__[item]:
                    truth_table.append(True)
                else:
                    truth_table.append(False)
            if False in truth_table:
                output=False
            else:
                output=True
            #print(truth_table)
            return output

        def __ne__(self,other):
            """Defines what being not equal means for the AsciiDataTable Class"""
            compare_elements=['options','header','column_names','data','footer']
            truth_table=[]
            output=True
            for item in compare_elements:
                if self.__dict__[item]==other.__dict__[item]:
                    truth_table.append(True)
                else:
                    truth_table.append(False)
            if False in truth_table:
                output=True
            else:
                output=False
            return output

        def add_row(self,row_data):
            """Adds a single row given row_data which can be an ordered list/tuple or a dictionary with
            column names as keys"""
            if len(row_data) not in [len(self.column_names),len(self.column_names)]:
                print(" could not add the row, dimensions do not match")
                return
            if type(row_data) in [ListType,np.ndarray]:
                self.data.append(row_data)
            elif type(row_data) in [DictionaryType]:
                data_list=[row_data[column_name] for column_name in self.column_names]
                self.data.append(data_list)

        def remove_row(self,row_index):
            """Removes the row specified by row_index and updates the model. Note index is relative to the
            data attribute so to remove the first row use row_index=0 and the last data row is row_index=-1"""
            self.data.pop(row_index)
            self.update_model()

        def add_column(self,column_name=None,column_type=None,column_data=None):
            """Adds a column with column_name, and column_type. If column data is supplied and it's length is the
            same as data(same number of rows) then it is added, else self.options['empty_character'] is added in each
            spot in the preceding rows"""
            original_column_names=self.column_names
            try:
                self.column_names.append(column_name)
                self.options["column_types"].append(column_type)
                if len(column_data) is len(self.data):
                    for index,row in enumerate(self.data):
                        row.append(column_data[index])
                else:
                    for index,row in enumerate(self.data):
                        row.append(self.options['empty_value'])
                        if column_data is not None:
                            for item in column_data:
                                empty_row=[self.options['empty_value'] for column in original_column_names]
                                self.add_row(empty_row.append(item))
            except:
                self.column_names=original_column_names
                print("Could not add columns")
                pass
        def remove_column(self,column_name=None,column_index=None):
            """Rmoves the column specified by column_name or column_index and updates the model. The column is removed from
            column_names, data and if present column_types, column_descriptions and row formatter"""
            pass
            #Todo:Add remove column functionality

        def add_index(self):
            """Adds a column with name index and values that are 0 referenced indices, does nothing if there is
            already a column with name index, always inserts it at the 0 position"""
            if 'index' in self.column_names:
                print("Add Index passed")
                pass
            else:
                self.column_names.insert(0,'index')
                for index,row in enumerate(self.data):
                    self.data[index].insert(0,index)
                if self.options['column_types']:
                    self.options['column_types'].insert(0,'int')
                if self.options['row_formatter_string']:
                    temp_formatter_list=self.options['row_formatter_string'].split("{delimiter}")
                    iterated_row_formatter_list=[temp_formatter_list[i].replace(str(i),str(i+1))
                                                 for i in range(len(temp_formatter_list))]
                    new_formatter=string_list_collapse(iterated_row_formatter_list,string_delimiter="{delimiter}")
                    self.options['row_formatter_string']='{0}{delimiter}'+new_formatter

        def move_footer_to_header(self):
            """Moves the DataTable's footer to the header and updates the model"""
            # check to see if the footer is defined
            if self.footer is None:
                return
            try:
              for item in self.footer:
                  self.header.append(item)
            except:
              self.header=ensure_string(self.header)+ensure_string(self.footer)
            self.footer=None

        # # This actually operated on self.lines before parsing and not directly on self.header
        # def add_comment(self,comment):
        #     "Adds a line comment to the header"
        #     self.header.append(comment)
        #     self.options["header_line_types"].insert('line_comment',len(self.header)-1)

        def add_inline_comment(self,comment="",line_number=None,string_position=None):
            "Adds an inline in the specified location"
            try:
                self.inline_comments.append([comment,line_number,string_position])
            except:pass

        # def add_block_comment(self,comment,element=None,location=None):
        #     "Adds a block comment in the specified location"
        #     pass

        def get_options(self):
            "Prints the option list"
            for key,value in self.options.iteritems():
                print("{0} = {1}".format(key,value))
        def get_row(self,row_index=None):
            """Returns the row as a list specified by row_index"""
            if row_index is None:
                return
            else:
                return self.data[row_index]

        def get_column(self,column_name=None,column_index=None):
            """Returns a column as a list given a column name or column index"""
            if column_name is None:
                if column_index is None:
                    return
                else:
                    column_selector=column_index
            else:
                column_selector=self.column_names.index(column_name)
            out_list=[self.data[i][column_selector] for i in range(len(self.data))]
            return out_list

        def get_data_dictionary_list(self,use_row_formatter_string=True):
            """Returns a python list with a row dictionary of form {column_name:data_column}"""
            try:
                if self.options["row_formatter_string"] is None:
                    use_row_formatter_string=False
                if use_row_formatter_string:
                    list_formatter=[item.replace(str(index),"0")
                                    for index,item in enumerate(self.options["row_formatter_string"].split("{delimiter}"))]
                else:
                    list_formatter=["{0}" for i in self.column_names]
                #print self.column_names
                #print self.data
                #print list_formatter
                #print len(self.column_names)==len(self.data[0])
                #print len(list_formatter)==len(self.data[0])
                #print type(self.data[0])
                out_list=[{self.column_names[i]:list_formatter[i].format(value) for i,value in enumerate(line)}
                          for line in self.data]
                return out_list
            except:
                print("Could not form a data_dictionary_list, check that row_formatter_string is properly defined")
                #print(out_list)
                raise

        def save_schema(self,path=None,format=None):
            """Saves the tables options as a text file or pickled dictionary (default).
            If no name is supplied, autonames it and saves"""
            if path is None:
                path=auto_name(self.name.replace('.'+self.options["extension"],""),'Schema',self.options["directory"],'txt')
            if format in [None,'python','pickle']:
                pickle.dump(self.options,open(path,'wb'))
            elif format in ['txt','text','.txt']:
                file_out=open(path,'w')
                keys=self.options.keys()
                keys.sort()
                for key in keys:
                    out_key=str(key).replace("\n","\\n")
                    out_value=str(self.options[key]).replace("\n","\\n")
                    file_out.write("{0} : {1} \n".format(out_key,out_value))
                file_out.close()
        def change_unit_prefix(self,column_selector=None,old_prefix=None,new_prefix=None,unit='Hz'):
            """Changes the prefix of the units of the column specified by column_selector (column name or index)
            example usage is self.change_unit_prefix(column_selector='Frequency',old_prefix=None,new_prefix='G',unit='Hz')
            to change a column from Hz to GHz. It updates the data values, column_descriptions, and column_units if they
            exist, see http://www.nist.gov/pml/wmd/metric/prefixes.cfm for possible prefixes"""

            multipliers={"yotta":10.**24,"Y":10.**24,"zetta":10.**21,"Z":10.**21,"exa":10.**18,"E":10.**18,"peta":10.**15,
                         "P":10.**15,"tera":10.**12,"T":10.**12,"giga":10.**9,"G":10.**9,"mega":10.**6,"M":10.**6,
                         "kilo":10.**3,"k":10.**3,"hecto":10.**2,"h":10.**2,"deka":10.,"da":10.,None:1.,"":1.,
                         "deci":10.**-1,"d":10.**-1,"centi":10.**-2,"c":10.**-2,"milli":10.**-3,"m":10.**-3,
                         "micro":10.**-6,"mu":10.**-6,u"\u00B5":10.**-6,"nano":10.**-9,
                         "n":10.**-9,"pico":10.**-12,"p":10.**-12,"femto":10.**-15,
                         "f":10.**-15,"atto":10.**-18,"a":10.**-18,"zepto":10.**-21,"z":10.**-21,
                         "yocto":10.**-24,"y":10.**-24}
            # change column name into column index
            try:
                if old_prefix is None:
                    old_prefix=""
                if new_prefix is None:
                    new_prefix=""
                old_unit=old_prefix+unit
                new_unit=new_prefix+unit
                if column_selector in self.column_names:
                    column_selector=self.column_names.index(column_selector)
                for index,row in enumerate(self.data):
                    if type(self.data[index][column_selector]) in [FloatType,LongType]:
                        #print "{0:e}".format(multipliers[old_prefix]/multipliers[new_prefix])
                        self.data[index][column_selector]=\
                        (multipliers[old_prefix]/multipliers[new_prefix])*self.data[index][column_selector]
                    elif type(self.data[index][column_selector]) in [StringType,IntType]:
                        self.data[index][column_selector]=\
                        str((multipliers[old_prefix]/multipliers[new_prefix])*float(self.data[index][column_selector]))
                    else:
                        print type(self.data[index][column_selector])
                        raise
                if self.options["column_descriptions"] is not None:
                    old=self.options["column_descriptions"][column_selector]
                    self.options["column_descriptions"][column_selector]=old.replace(old_unit,new_unit)
                if self.options["column_units"] is not None:
                    old=self.options["column_units"][column_selector]
                    self.options["column_units"][column_selector]=old.replace(old_unit,new_unit)
                if re.search(old_unit,self.column_names[column_selector]):
                    old=self.column_names[column_selector]
                    self.column_names[column_selector]=old.replace(old_unit,new_unit)
            except:
                print("Could not change the unit prefix of column {0}".format(column_selector))
                raise

### Ancestors (in MRO)

- AsciiDataTable

### Instance variables

var elements

var options

### Methods

def __init__(

self, file_path=None, **options)

Initializes the AsciiDataTable class

Show source ≡

    def __init__(self,file_path=None,**options):
        " Initializes the AsciiDataTable class "
        # This is a general pattern for adding a lot of options
        defaults={"data_delimiter":None,
                  "column_names_delimiter":None,
                  "specific_descriptor":'Data',
                  "general_descriptor":'Table',
                  "directory":None,
                  "extension":'txt',
                  "comment_begin":None,
                  "comment_end":None,
                  "inline_comment_begin":None,
                  "inline_comment_end":None,
                  "block_comment_begin":None,
                  "block_comment_end":None,
                  "footer_begin_line":None,
                  "footer_end_line":None,
                  "header_begin_line":None,
                  "header_end_line":None,
                  "column_names_begin_line":None,
                  "column_names_end_line":None,
                  "data_begin_line":None,
                  "data_end_line":None,
                  "footer_begin_token":None,
                  "footer_end_token":None,
                  "header_begin_token":None,
                  "header_end_token":None,
                  "column_names_begin_token":None,
                  "column_names_end_token":None,
                  "data_begin_token":None,
                  "data_end_token":None,
                  "metadata_delimiter":None,
                  "metadata_key_value_delimiter":None,
                  "header_line_types":None,
                  "column_types":None,
                  "column_descriptions":None,
                  "column_units":None,
                  "footer_line_types":None,
                  "header":None,
                  "column_names":None,
                  "data":None,
                  "footer":None,
                  "inline_comments":None,
                  "row_begin_token":None,
                  "row_end_token":None,
                  "row_formatter_string":None,
                  "empty_value":None,
                  "escape_character":None,
                  "data_table_element_separator":'\n',
                  "treat_header_as_comment":None,
                  "treat_footer_as_comment":None,
                  "metadata":None
                  }
        #some of the options have the abiltiy to confilct with each other, so there has to be a
        #built-in way to determine the precedence of each option, for import lines first, then begin and then end
        self.options={}
        for key,value in defaults.iteritems():
            self.options[key]=value
        for key,value in options.iteritems():
            self.options[key]=value
        self.elements=['header','column_names','data','footer','inline_comments']
        #Define Method Aliases if they are available
        #unqualified exec is not allowed in function '__init__' because it contains a nested function with free variables
        # This is because __init__ has nested functions
        # if METHOD_ALIASES:
        #     for command in alias(self):
        #         exec(command)
        if file_path is None:
            #create a new data table
            if DEFAULT_FILE_NAME is None:
                self.name=auto_name(self.options["specific_descriptor"],
                                    self.options["general_descriptor"],
                                    self.options["directory"],
                                    self.options["extension"])
                if self.options['directory'] is None:
                    self.path=self.name
                else:
                    self.path=os.path.join(self.options["directory"],self.name)
            #Now we see if the table has been defined in the options
            # We should reset the self.options versions to None after this so as to not recreate or we
            # can use it as a cache and add a method reset_table which either redoes the below or reloads the saved file
            self.header=self.options["header"]
            self.column_names=self.options["column_names"]
            self.data=self.options["data"]
            self.footer=self.options["footer"]
            self.metadata=self.options["metadata"]
            self.inline_comments=self.options["inline_comments"]
            self.initial_state=[self.options["header"],self.options["column_names"],
                                self.options["data"],self.options["footer"],
                                self.options["inline_comments"]]
            #print "I got here {0}".format(self.path)
            [self.options["header"],self.options["column_names"],
                                self.options["data"],self.options["footer"],
                                self.options["inline_comments"]]=[None for i in self.elements]
            #if you are validating the model, you have to skip the updating until it hsa been parsed
            try:
                if self.options["validate"]:
                    del self.options["validate"]
                    pass
            except KeyError:
                self.update_model()
        else:
            # open the file and read it in as lines
            # do we parse it here?
            # once parsed we should end up with the major components
            # if we are given options we should use them, if not try to autodetect them?
            # we can just return an error right now and then have an __autoload__ method
            # we can assume it is in ascii or utf-8
            # set any attribute that has no options to None
            import_table=[]
            for item in self.elements:
                if len(filter(lambda x: None!=x,self.get_options_by_element(item).values()))==0:
                    self.__dict__[item]=None
                    #elements.remove(item)
                elif item not in ['inline_comments']:
                    self.__dict__[item]=[]
                    import_row=[self.options['%s_begin_line'%item],
                                self.options['%s_end_line'%item],
                                self.options['%s_begin_token'%item],
                                self.options['%s_end_token'%item]]
                    import_table.append(import_row)
                elif item in ['inline_comments']:
                    self.inline_comments=self.options['inline_comments']
            file_in=open(file_path,'r')
            # in order to parse the file we need to know line #'s, once we deduce them we use __parse__
            self.lines=[]
            for line in file_in:
                self.lines.append(line)
            file_in.close()
            self.path=file_path
            if self.lines_defined():
                self.__parse__()
            else:
                import_table[0][0]=0
                import_table[-1][1]=None
                # This is to make sure the lines inbetween the data table's elements are accounted for
                if self.options['data_table_element_separator'] is None:
                    inner_element_spacing=0
                else:
                    inner_element_spacing=self.options['data_table_element_separator'].count('\n')-1
                #print import_table
                self.update_import_options(import_table=import_table)
                #self.get_options()
                if self.lines_defined():
                    #print("%s says %s"%('self.lines_defined()',str(self.lines_defined())))
                    self.__parse__()
                row_zero=[import_table[i][0] for i in range(len(import_table))]
                for index,item in enumerate(row_zero):
                    #print import_table
                    #print index,item
                    if index>0:
                        #print("Row Zero Loop Returns index={0}, item={1}".format(index,item))
                        if item is not None:
                            import_table[index-1][1]=item+inner_element_spacing
                            #print import_table
                            self.update_import_options(import_table)
                            #print self.lines_defined()
                if self.lines_defined():
                        self.__parse__()
                else:
                    row_one=[import_table[i][1] for i in range(len(import_table))]
                    for index,item in enumerate(row_one):
                        #print("Row One Loop Returns index={0}, item={1}".format(index,item))
                        if index<(len(row_one)-1):
                            #print((index+1)<len(row_one))
                            #print("Row One Loop Returns index={0}, item={1}".format(index,item))
                            if item is not None:
                                #print import_table
                                import_table[index+1][0]=item-inner_element_spacing
                                self.update_import_options(import_table)
                    if self.lines_defined():
                        self.__parse__()
                    else:
                        row_two=[import_table[i][2] for i in range(len(import_table))]
                        for index,item in enumerate(row_two):
                            if item is not None:
                                import_table[index][0]=self.find_line(item)
                        for index,item in enumerate(row_zero):
                            if index>0:
                                if item is not None:
                                    import_table[index-1][1]=item++inner_element_spacing
                                    self.update_import_options(import_table)
                        if self.lines_defined():
                            self.__parse__()
                        else:
                            row_three=[import_table[i][3] for i in range(len(import_table))]
                            for index,item in enumerate(row_three):
                                if item is not None:
                                    import_table[index][1]=self.find_line(item)
                            for index,item in enumerate(row_one):
                                if index<(len(row_one)-1):
                                    if item is not None:
                                        import_table[index+1][0]=item-inner_element_spacing
                            self.update_import_options(import_table)
                            if self.lines_defined():
                                self.__parse__()
                            else:
                                print("FAILED to import file!")
                                raise

def add_column(

self, column_name=None, column_type=None, column_data=None)

Adds a column with column_name, and column_type. If column data is supplied and it's length is the same as data(same number of rows) then it is added, else self.options['empty_character'] is added in each spot in the preceding rows

Show source ≡

    def add_column(self,column_name=None,column_type=None,column_data=None):
        """Adds a column with column_name, and column_type. If column data is supplied and it's length is the
        same as data(same number of rows) then it is added, else self.options['empty_character'] is added in each
        spot in the preceding rows"""
        original_column_names=self.column_names
        try:
            self.column_names.append(column_name)
            self.options["column_types"].append(column_type)
            if len(column_data) is len(self.data):
                for index,row in enumerate(self.data):
                    row.append(column_data[index])
            else:
                for index,row in enumerate(self.data):
                    row.append(self.options['empty_value'])
                    if column_data is not None:
                        for item in column_data:
                            empty_row=[self.options['empty_value'] for column in original_column_names]
                            self.add_row(empty_row.append(item))
        except:
            self.column_names=original_column_names
            print("Could not add columns")
            pass

def add_index(

self)

Adds a column with name index and values that are 0 referenced indices, does nothing if there is already a column with name index, always inserts it at the 0 position

Show source ≡

    def add_index(self):
        """Adds a column with name index and values that are 0 referenced indices, does nothing if there is
        already a column with name index, always inserts it at the 0 position"""
        if 'index' in self.column_names:
            print("Add Index passed")
            pass
        else:
            self.column_names.insert(0,'index')
            for index,row in enumerate(self.data):
                self.data[index].insert(0,index)
            if self.options['column_types']:
                self.options['column_types'].insert(0,'int')
            if self.options['row_formatter_string']:
                temp_formatter_list=self.options['row_formatter_string'].split("{delimiter}")
                iterated_row_formatter_list=[temp_formatter_list[i].replace(str(i),str(i+1))
                                             for i in range(len(temp_formatter_list))]
                new_formatter=string_list_collapse(iterated_row_formatter_list,string_delimiter="{delimiter}")
                self.options['row_formatter_string']='{0}{delimiter}'+new_formatter

def add_inline_comment(

self, comment='', line_number=None, string_position=None)

Adds an inline in the specified location

Show source ≡

    def add_inline_comment(self,comment="",line_number=None,string_position=None):
        "Adds an inline in the specified location"
        try:
            self.inline_comments.append([comment,line_number,string_position])
        except:pass

def add_row(

self, row_data)

Adds a single row given row_data which can be an ordered list/tuple or a dictionary with column names as keys

Show source ≡

    def add_row(self,row_data):
        """Adds a single row given row_data which can be an ordered list/tuple or a dictionary with
        column names as keys"""
        if len(row_data) not in [len(self.column_names),len(self.column_names)]:
            print(" could not add the row, dimensions do not match")
            return
        if type(row_data) in [ListType,np.ndarray]:
            self.data.append(row_data)
        elif type(row_data) in [DictionaryType]:
            data_list=[row_data[column_name] for column_name in self.column_names]
            self.data.append(data_list)

def build_string(

self, **temp_options)

Builds a string representation of the data table based on self.options, or temp_options. Passing temp_options does not permanently change the model

Show source ≡

    def build_string(self,**temp_options):
        """Builds a string representation of the data table based on self.options, or temp_options.
        Passing temp_options does not permanently change the model"""
        # store the original options to be put back after the string is made
        original_options=self.options
        for key,value in temp_options.iteritems():
            self.options[key]=value
        section_end=0
        next_section_begin=0
        if self.options['data_table_element_separator'] is None:
            inner_element_spacing=0
        else:
            inner_element_spacing=self.options['data_table_element_separator'].count('\n')-1
        string_out=""
        between_section=""
        if self.options['data_table_element_separator'] is not None:
            between_section=self.options['data_table_element_separator']
        if self.header is None:
            self.options['header_begin_line']=self.options['header_end_line']=None
            pass
        else:
            self.options["header_begin_line"]=0
            if self.data is None and self.column_names is None and self.footer is None:
                string_out=self.get_header_string()
                self.options["header_end_line"]=None
            else:
                string_out=self.get_header_string()+between_section
                last_header_line=self.get_header_string().count('\n')+1
                self.options["header_end_line"]=last_header_line
                next_section_begin=last_header_line+inner_element_spacing
        if self.column_names is None:
            self.options['column_names_begin_line']=self.options['column_names_end_line']=None
            pass
        else:
            self.options["column_names_begin_line"]=next_section_begin
            if self.data is None and self.footer is None:
                self.options["column_names_end_line"]=None
                string_out=string_out+self.get_column_names_string()
            else:
                string_out=string_out+self.get_column_names_string()+between_section
                last_column_names_line=self.get_column_names_string().count('\n')+\
                                       self.options["column_names_begin_line"]+1
                self.options["column_names_end_line"]=last_column_names_line
                next_section_begin=last_column_names_line+inner_element_spacing
        if self.data is None:
            self.options['data_begin_line']=self.options['data_end_line']=None
            pass
        else:
            self.options["data_begin_line"]=next_section_begin
            if self.footer is None:
                self.options["data_end_line"]=None
                string_out=string_out+self.get_data_string()
            else:
                string_out=string_out+self.get_data_string()+between_section
                last_data_line=self.get_data_string().count("\n")+\
                                self.options["data_begin_line"]+1
                self.options["data_end_line"]=last_data_line
                next_section_begin=last_data_line+inner_element_spacing
        if self.footer is None:
            self.options['footer_begin_line']=self.options['footer_end_line']=None
            pass
        else:
            self.options["footer_begin_line"]=next_section_begin
            string_out=string_out+self.get_footer_string()
            self.options['footer_end_line']=None
        # set the options back after the string has been made
        if self.inline_comments is None:
            pass
        else:
            lines=string_out.splitlines()
            for comment in self.inline_comments:
                lines=insert_inline_comment(lines,comment=comment[0],line_number=comment[1],
                                            string_position=comment[2],
                                            begin_token=self.options['inline_comment_begin'],
                                            end_token=self.options['inline_comment_end'])
            string_out=string_list_collapse(lines,string_delimiter='\n')
        self.options=original_options
        return string_out

def change_unit_prefix(

self, column_selector=None, old_prefix=None, new_prefix=None, unit='Hz')

Changes the prefix of the units of the column specified by column_selector (column name or index) example usage is self.change_unit_prefix(column_selector='Frequency',old_prefix=None,new_prefix='G',unit='Hz') to change a column from Hz to GHz. It updates the data values, column_descriptions, and column_units if they exist, see http://www.nist.gov/pml/wmd/metric/prefixes.cfm for possible prefixes

Show source ≡

    def change_unit_prefix(self,column_selector=None,old_prefix=None,new_prefix=None,unit='Hz'):
        """Changes the prefix of the units of the column specified by column_selector (column name or index)
        example usage is self.change_unit_prefix(column_selector='Frequency',old_prefix=None,new_prefix='G',unit='Hz')
        to change a column from Hz to GHz. It updates the data values, column_descriptions, and column_units if they
        exist, see http://www.nist.gov/pml/wmd/metric/prefixes.cfm for possible prefixes"""
        multipliers={"yotta":10.**24,"Y":10.**24,"zetta":10.**21,"Z":10.**21,"exa":10.**18,"E":10.**18,"peta":10.**15,
                     "P":10.**15,"tera":10.**12,"T":10.**12,"giga":10.**9,"G":10.**9,"mega":10.**6,"M":10.**6,
                     "kilo":10.**3,"k":10.**3,"hecto":10.**2,"h":10.**2,"deka":10.,"da":10.,None:1.,"":1.,
                     "deci":10.**-1,"d":10.**-1,"centi":10.**-2,"c":10.**-2,"milli":10.**-3,"m":10.**-3,
                     "micro":10.**-6,"mu":10.**-6,u"\u00B5":10.**-6,"nano":10.**-9,
                     "n":10.**-9,"pico":10.**-12,"p":10.**-12,"femto":10.**-15,
                     "f":10.**-15,"atto":10.**-18,"a":10.**-18,"zepto":10.**-21,"z":10.**-21,
                     "yocto":10.**-24,"y":10.**-24}
        # change column name into column index
        try:
            if old_prefix is None:
                old_prefix=""
            if new_prefix is None:
                new_prefix=""
            old_unit=old_prefix+unit
            new_unit=new_prefix+unit
            if column_selector in self.column_names:
                column_selector=self.column_names.index(column_selector)
            for index,row in enumerate(self.data):
                if type(self.data[index][column_selector]) in [FloatType,LongType]:
                    #print "{0:e}".format(multipliers[old_prefix]/multipliers[new_prefix])
                    self.data[index][column_selector]=\
                    (multipliers[old_prefix]/multipliers[new_prefix])*self.data[index][column_selector]
                elif type(self.data[index][column_selector]) in [StringType,IntType]:
                    self.data[index][column_selector]=\
                    str((multipliers[old_prefix]/multipliers[new_prefix])*float(self.data[index][column_selector]))
                else:
                    print type(self.data[index][column_selector])
                    raise
            if self.options["column_descriptions"] is not None:
                old=self.options["column_descriptions"][column_selector]
                self.options["column_descriptions"][column_selector]=old.replace(old_unit,new_unit)
            if self.options["column_units"] is not None:
                old=self.options["column_units"][column_selector]
                self.options["column_units"][column_selector]=old.replace(old_unit,new_unit)
            if re.search(old_unit,self.column_names[column_selector]):
                old=self.column_names[column_selector]
                self.column_names[column_selector]=old.replace(old_unit,new_unit)
        except:
            print("Could not change the unit prefix of column {0}".format(column_selector))
            raise

def find_line(

self, begin_token)

Finds the first line that has begin token in it

Show source ≡

    def find_line(self,begin_token):
        """Finds the first line that has begin token in it"""
        for index,line in enumerate(self.lines):
            if re.match(begin_token,line):
                return index

def get_column(

self, column_name=None, column_index=None)

Returns a column as a list given a column name or column index

Show source ≡

    def get_column(self,column_name=None,column_index=None):
        """Returns a column as a list given a column name or column index"""
        if column_name is None:
            if column_index is None:
                return
            else:
                column_selector=column_index
        else:
            column_selector=self.column_names.index(column_name)
        out_list=[self.data[i][column_selector] for i in range(len(self.data))]
        return out_list

def get_column_names_string(

self)

Returns the column names as a string using options

Show source ≡

    def get_column_names_string(self):
        "Returns the column names as a string using options"
        string_out=""
        # This writes the column_names
        column_name_begin=""
        column_name_end=""
        if self.options["column_names_begin_token"] is None:
            column_name_begin=""
        else:
            column_name_begin=self.options["column_names_begin_token"]
        if self.options["column_names_end_token"] is None:
            column_name_end=""
        else:
            column_name_end=self.options["column_names_end_token"]
        if self.column_names is None:
            string_out=""
        else:
            if type(self.column_names) is StringType:
                string_out=self.column_names
            elif type(self.column_names) is ListType:
                string_out=list_to_string(self.column_names,
                                          data_delimiter=self.options["column_names_delimiter"],end="")
                #print("{0} is {1}".format('string_out',string_out))
            else:
                string_out=ensure_string(self.column_names)
        #print column_name_begin,string_out,column_name_end
        return column_name_begin+string_out+column_name_end

def get_data_dictionary_list(

self, use_row_formatter_string=True)

Returns a python list with a row dictionary of form {column_name:data_column}

Show source ≡

    def get_data_dictionary_list(self,use_row_formatter_string=True):
        """Returns a python list with a row dictionary of form {column_name:data_column}"""
        try:
            if self.options["row_formatter_string"] is None:
                use_row_formatter_string=False
            if use_row_formatter_string:
                list_formatter=[item.replace(str(index),"0")
                                for index,item in enumerate(self.options["row_formatter_string"].split("{delimiter}"))]
            else:
                list_formatter=["{0}" for i in self.column_names]
            #print self.column_names
            #print self.data
            #print list_formatter
            #print len(self.column_names)==len(self.data[0])
            #print len(list_formatter)==len(self.data[0])
            #print type(self.data[0])
            out_list=[{self.column_names[i]:list_formatter[i].format(value) for i,value in enumerate(line)}
                      for line in self.data]
            return out_list
        except:
            print("Could not form a data_dictionary_list, check that row_formatter_string is properly defined")
            #print(out_list)
            raise

def get_data_string(

self)

Returns the data as a string

Show source ≡

    def get_data_string(self):
        "Returns the data as a string"
        #Todo:refactor to cut out unused lines
        string_out=""
        if self.data is None:
            string_out= ""
        else:
            if type(self.data) is StringType:
                if self.options['data_begin_token'] is None:
                       if self.options['data_end_token'] is None:
                           string_out=self.data
                       else:
                           if re.search(self.options['data_end_token'],self.data):
                               string_out=self.data
                           else:
                               string_out=self.data+self.options['data_end_token']
                else:
                        if self.options['data_end_token'] is None:
                            if re.match(self.options['data_begin_token'],self.data):
                                string_out=self.data
                            else:
                                string_out=self.options['data_begin_token']+self.data
            elif type(self.data) in [ListType,np.ndarray]:
                try:
                        #If the first row is a string, we should strip all the tokens and add them back in
                        if type(self.data[0]) is StringType:
                            if self.options['data_begin_token'] is None:
                                string_out=string_list_collapse(self.data)
                            else:
                                if re.match(self.options['data_begin_token'],self.data[0]):
                                    if self.options['data_end_token'] is None:
                                        string_out=string_list_collapse(self.data)
                                    else:
                                        if re.search(self.options['data_end_token'],self.data[-1]):
                                            string_out=string_list_collapse(self.data)
                                        else:
                                            string_out=string_list_collapse(self.data)+self.options['data_end_token']
                                else:
                                    if self.options['data_end_token'] is None:
                                        string_out=self.options['data_begin_token']+string_list_collapse(self.data)
                                    else:
                                        if re.search(self.options['data_end_token'],self.data[-1]):
                                            string_out=self.options['data_begin_token']+string_list_collapse(self.data)
                                        else:
                                            string_out=self.options['data_begin_token']+\
                                                       string_list_collapse(self.data)+\
                                                       self.options['data_end_token']
                        elif type(self.data[0]) in [ListType,np.ndarray]:
                            prefix=""
                            if self.options['data_begin_token'] is None:
                                if self.options['data_end_token'] is None:
                                    string_out=list_list_to_string(self.data,data_delimiter=self.options['data_delimiter'],
                                                                   row_formatter_string=self.options['row_formatter_string'],
                                                                   line_begin=self.options["row_begin_token"],
                                                                   line_end=self.options["row_end_token"])
                            else:
                                if self.options['data_end_token'] is None:
                                    string_out=self.options['data_begin_token']+\
                                               list_list_to_string(self.data,
                                                                   data_delimiter=self.options['data_delimiter'],
                                                                   row_formatter_string=self.options['row_formatter_string'],
                                                                   line_begin=self.options["row_begin_token"],
                                                                   line_end=self.options["row_end_token"])
                                else:
                                    string_out=self.options['data_begin_token']+\
                                               list_list_to_string(self.data,
                                                                   data_delimiter=self.options['data_delimiter'],
                                                                   row_formatter_string=\
                                                                   self.options['row_formatter_string'],
                                                                   line_begin=self.options["row_begin_token"],
                                                                   line_end=self.options["row_end_token"])+\
                                                                    self.options['data_end_token']
                        else:
                            string_out=list_to_string(self.data,
                                                      data_delimiter=self.options['data_delimiter'],
                                                      row_formatter_string=self.options['row_formatter_string'],
                                                      begin=self.options["row_begin_token"],
                                                      end=self.options["row_end_token"])
                except IndexError:
                    pass
            else:
                string_out=ensure_string(self.data)
        return string_out

Returns the footer using options in self.options. If block comment is specified, and the footer is a list it will block comment out the footer. If comment_begin and comment_end are specified it will use those to represent each line of the footer. If footer_begin_token and/or footer_end_token are specified it will wrap the footer in those.

Show source ≡

def get_header_string(

self)

Returns the header using options in self.options. If block comment is specified, and the header is a list it will block comment out the header. If comment_begin and comment_end are specified it will use those to represent each line of the header. If header_begin_token and/or header_end_token are specified it will wrap the header in those.

Show source ≡

    def get_header_string(self):
        """Returns the header using options in self.options. If block comment is specified, and the header is a
        list it will block comment out the header. If comment_begin and comment_end are specified it will use
        those to represent each line of the header. If header_begin_token and/or header_end_token are specified it
         will wrap the header in those.
        """
        string_out=""
        header_begin=""
        header_end=""
        if self.options["header_begin_token"] is None:
            header_begin=""
        else:
            header_begin=self.options["header_begin_token"]
        if self.options["header_end_token"] is None:
            header_end=""
        else:
            header_end=self.options["header_end_token"]
        # This writes the header
        if self.header is None:
            string_out= ""
        elif self.options["header_line_types"] is not None:
            for index,line in enumerate(self.options["header_line_types"]):
                if index == len(self.options["header_line_types"])-1:
                    end=''
                else:
                    end='\n'
                if line in ['header','header_line','normal']:
                    string_out=string_out+self.header[index]+end
                elif line in ['line_comment','comment']:
                    string_out=string_out+line_comment_string(self.header[index],
                                               comment_begin=self.options["comment_begin"],
                                               comment_end=self.options["comment_end"])+end
                elif line in ['block_comment','block']:
                    if index-1<0:
                        block_comment_begin=index
                        block_comment_end=index+2
                        continue
                    elif self.options["header_line_types"][index-1] not in ['block_comment','block']:
                        block_comment_begin=index
                        block_comment_end=index+2
                        continue
                    else:
                        if index+1>len(self.options["header_line_types"])-1:
                            string_out=string_out+line_list_comment_string(self.header[block_comment_begin:],
                                                                           comment_begin=self.options['block_comment_begin'],
                                                                             comment_end=self.options['block_comment_end'],
                                                                           block=True)+end
                        elif self.options["header_line_types"][index+1] in ['block_comment','block']:
                            block_comment_end+=1
                        else:
                            string_out=string_out+\
                                       line_list_comment_string(self.header[block_comment_begin:block_comment_end],
                                                                comment_begin=self.options['block_comment_begin'],
                                                                comment_end=self.options['block_comment_end'],
                                                                block=True)+end
                else:
                    string_out=string_out+line
        elif self.options['treat_header_as_comment'] in [None,True] and self.options["header_line_types"] in [None]:
            # Just happens if the user has set self.header manually
            if type(self.header) is StringType:
                string_out=line_comment_string(self.header,
                                               comment_begin=self.options["comment_begin"],
                                               comment_end=self.options["comment_end"])
                #string_out=re.sub('\n','',string_out,count=1)
            elif type(self.header) is ListType:
                if self.options['block_comment_begin'] is None:
                    if self.options['comment_begin'] is None:
                        string_out=string_list_collapse(self.header)
                    else:
                        string_out=line_list_comment_string(self.header,comment_begin=self.options['comment_begin'],
                                                        comment_end=self.options['comment_end'])
                        lines_out=string_out.splitlines()
                        # if re.search('\n',self.options['comment_end']):
                        #     string_out=re.sub('\n','',string_out,count=1)
                        #self.options["header_line_types"]=["line_comment" for line in self.header]
                else:
                    string_out=line_list_comment_string(self.header,comment_begin=self.options['block_comment_begin'],
                                                        comment_end=self.options['block_comment_end'],block=True)
                    #self.options["header_line_types"]=["block_comment" for line in self.header]
        else:
            string_out=ensure_string(self.header,list_delimiter="\n",end_if_list="")
        return header_begin+string_out+header_end

def get_options(

self)

Prints the option list

Show source ≡

    def get_options(self):
        "Prints the option list"
        for key,value in self.options.iteritems():
            print("{0} = {1}".format(key,value))

def get_options_by_element(

self, element_name)

returns a dictionary of all the options that have to do with element. Element must be header,column_names,data, or footer

Show source ≡

    def get_options_by_element(self,element_name):
        """ returns a dictionary
         of all the options that have to do with element. Element must be header,column_names,data, or footer"""
        keys_regarding_element=filter(lambda x: re.search(element_name,str(x),re.IGNORECASE),self.options.keys())
        out_dictionary={key:self.options[key] for key in keys_regarding_element}
        #print out_dictionary
        return out_dictionary

def get_row(

self, row_index=None)

Returns the row as a list specified by row_index

Show source ≡

    def get_row(self,row_index=None):
        """Returns the row as a list specified by row_index"""
        if row_index is None:
            return
        else:
            return self.data[row_index]

def is_valid(

self)

Returns True if ascii table conforms to its specification given by its own options

Show source ≡

    def is_valid(self):
        """Returns True if ascii table conforms to its specification given by its own options"""
        options={}
        for key,value in self.options.iteritems():
            options[key]=value
            # print("self.options[{0}] is {1} ".format(key,value))
        for element in self.elements:
            if self.__dict__[element] is None:
                options[element]=None
            else:
                options[element]=[]
        options["validate"]=True
        newtable=AsciiDataTable(None,**options)
        lines=self.build_string().splitlines()
        for index,line in enumerate(lines):
            lines[index]=line+"\n"
        newtable.lines=lines
        newtable.__parse__()
        # print newtable.data
        # print newtable.column_names
        # print newtable
        #print_comparison(newtable.footer,None)
        newtable.update_model()
        # The new table rows are not being coerced into the right format
        #print newtable
        #newtable.update_model()
        #print newtable.options
        #print self.options
        #print newtable.data
        # print newtable.options==self.options
        # for option_key,option_value in newtable.options.iteritems():
        #     print("New Table Option {0} is {1} ".format(option_key,option_value))
        #     print("self.options[{0}] is {1} ".format(option_key,self.options[option_key]))
        #     print_comparison(option_value,self.options[option_key])
        # #print self
        return self==newtable

def lines_defined(

self)

If begin_line and end_line for all elements that are None are defined returns True

Show source ≡

    def lines_defined(self):
        """If begin_line and end_line for all elements that are None are defined returns True"""
        truth_table=[]
        last_element=""
        output=False
        for index,element in enumerate(self.elements):
            if element not in ['inline_comments'] and self.__dict__[element] is not None:
                try:
                    last_element=element
                    if not None in [self.options['%s_begin_line'%element],self.options['%s_end_line'%element]]:
                        truth_table.append(True)
                    else:
                         truth_table.append(False)
                except:
                    return False
        #print truth_table
        # The last_line of the last element is fine to be none
        if truth_table[-1] is False:
            if self.options['%s_begin_line'%last_element] is not None:
                truth_table[-1]=True
        if False in truth_table:
            output=False
        else:
            output=True
        #print output
        return output

Moves the DataTable's footer to the header and updates the model

Show source ≡

def remove_column(

self, column_name=None, column_index=None)

Rmoves the column specified by column_name or column_index and updates the model. The column is removed from column_names, data and if present column_types, column_descriptions and row formatter

Show source ≡

    def remove_column(self,column_name=None,column_index=None):
        """Rmoves the column specified by column_name or column_index and updates the model. The column is removed from
        column_names, data and if present column_types, column_descriptions and row formatter"""
        pass

def remove_row(

self, row_index)

Removes the row specified by row_index and updates the model. Note index is relative to the data attribute so to remove the first row use row_index=0 and the last data row is row_index=-1

Show source ≡

    def remove_row(self,row_index):
        """Removes the row specified by row_index and updates the model. Note index is relative to the
        data attribute so to remove the first row use row_index=0 and the last data row is row_index=-1"""
        self.data.pop(row_index)
        self.update_model()

def save(

self, path=None, **temp_options)

" Saves the file, to save in another ascii format specify elements in temp_options, the options specified do not permanently change the object's options. If path is supplied it saves the file to that path otherwise uses the object's attribute path to define the saving location

Show source ≡

    def save(self,path=None,**temp_options):
        """" Saves the file, to save in another ascii format specify elements in temp_options, the options
        specified do not permanently change the object's options. If path is supplied it saves the file to that path
        otherwise uses the object's attribute path to define the saving location """
        original_options=self.options
        for key,value in temp_options.iteritems():
            self.options[key]=value
        out_string=self.build_string(**temp_options)
        if path is None:
            path=self.path
        file_out=open(path,'w')
        file_out.write(out_string)
        file_out.close()
        self.options=original_options

def save_schema(

self, path=None, format=None)

Saves the tables options as a text file or pickled dictionary (default). If no name is supplied, autonames it and saves

Show source ≡

    def save_schema(self,path=None,format=None):
        """Saves the tables options as a text file or pickled dictionary (default).
        If no name is supplied, autonames it and saves"""
        if path is None:
            path=auto_name(self.name.replace('.'+self.options["extension"],""),'Schema',self.options["directory"],'txt')
        if format in [None,'python','pickle']:
            pickle.dump(self.options,open(path,'wb'))
        elif format in ['txt','text','.txt']:
            file_out=open(path,'w')
            keys=self.options.keys()
            keys.sort()
            for key in keys:
                out_key=str(key).replace("\n","\\n")
                out_value=str(self.options[key]).replace("\n","\\n")
                file_out.write("{0} : {1} \n".format(out_key,out_value))
            file_out.close()

def update_column_names(

self)

Update column names adds the value x# for any column that exists in self.data that is not named

Show source ≡

    def update_column_names(self):
        """Update column names adds the value x# for any column that exists in self.data that is not named"""
        if self.data is None:
            return
        elif type(self.column_names) is StringType:
            self.column_names=split_row(self.column_names,self.options["column_names_delimiter"])
        elif self.column_names is None:
            column_names=[]
            for index,column in enumerate(self.data[0]):
                column_names.append("x"+str(index))
            self.column_names=column_names
            return
        elif len(self.column_names)==len(self.data[0]):
            return
        elif len(self.column_names) < len(self.data[0]):
            for index in range(len(self.column_names),len(self.data[0])):
                self.column_names.append("x"+str(index))
            return

def update_import_options(

self, import_table)

Updates the options in the import table

Show source ≡

    def update_import_options(self,import_table):
        """Updates the options in the import table"""
        for index,element in enumerate(self.elements):
            if self.__dict__[element] is not None:
                [self.options['%s_begin_line'%element],
                                self.options['%s_end_line'%element],
                                self.options['%s_begin_token'%element],
                                self.options['%s_end_token'%element]]=import_table[index][:]

def update_index(

self)

Updates the index column if it exits, otherwise exits quietly

Show source ≡

    def update_index(self):
        """ Updates the index column if it exits, otherwise exits quietly
        """
        if 'index' not in self.column_names:
            return
        else:
            try:
                #This should be 0 but just in case
                index_column_number=self.column_names.index('index')
                for i in range(len(self.data)):
                    self.data[i][index_column_number]=i
            except:
                pass

def update_model(

self)

Updates the model after a change has been made. If you add anything to the attributes of the model, or change this updates the values. If the model has an index column it will make sure the numbers are correct. In addition, it will update the options dictionary to reflect added rows, changes in deliminators etc.

Show source ≡

    def update_model(self):
        """Updates the model after a change has been made. If you add anything to the attributes of the model,
        or change this updates the values. If the model has an index column it will make sure the numbers are correct.
        In addition, it will update the options dictionary to reflect added rows, changes in deliminators etc.  """
        if self.column_names is not None and 'index' in self.column_names:
           self.update_index()
        #make sure there are no "\n" characters in the element lists (if so replace them with "") for data this is
        # done on import
        list_types=["header","column_names","footer"]
        for element in list_types:
            if self.__dict__[element] is not None:
                for index,item in enumerate(self.__dict__[element]):
                    self.__dict__[element][index]=item.replace("\n","")
        self.update_column_names()
        if self.data is not None:
            self.data=convert_all_rows(self.data,self.options["column_types"])
        self.string=self.build_string()
        self.lines=self.string.splitlines()

class AsciiDataTableCollection

A collection of multiple AsciiDataTables

Show source ≡

    class AsciiDataTableCollection():
        """A collection of multiple AsciiDataTables"""
        def __init__(self,file_path=None,table_names=None,**options):
            # The primary attritbute should be self.data_tables=list of tables
            # the dictionary should be in the form {"table_name":AsciiDataTable}
            # since there is no way to know how many tables or their options,
            # you can't read them in without **options-> self.tables
            # which can be passed as an option. Each table is an independent entity
            # but can have options set by global_options
            if file_path is None:
                pass
            else:
                if table_names is None and options["table_names"] is None:
                    raise
                else:
                    file_in=open(file_path,'r')
                    # in order to parse the file we need to know line #'s, once we deduce them we use __parse__
                    self.lines=[]
                    for line in file_in:
                        self.lines.append(line)
                    file_in.close()
                    self.path=file_path

        def build_string(self,**temp_options):
            """Builds the string for the table collection using the temp_options"""
            out_string=""
            for table in self.tables:
                out_string=out_string+table.build_string()
            return out_string

### Ancestors (in MRO)

- AsciiDataTableCollection

### Methods

def __init__(

self, file_path=None, table_names=None, **options)

Show source ≡

    def __init__(self,file_path=None,table_names=None,**options):
        # The primary attritbute should be self.data_tables=list of tables
        # the dictionary should be in the form {"table_name":AsciiDataTable}
        # since there is no way to know how many tables or their options,
        # you can't read them in without **options-> self.tables
        # which can be passed as an option. Each table is an independent entity
        # but can have options set by global_options
        if file_path is None:
            pass
        else:
            if table_names is None and options["table_names"] is None:
                raise
            else:
                file_in=open(file_path,'r')
                # in order to parse the file we need to know line #'s, once we deduce them we use __parse__
                self.lines=[]
                for line in file_in:
                    self.lines.append(line)
                file_in.close()
                self.path=file_path

def build_string(

self, **temp_options)

Builds the string for the table collection using the temp_options

Show source ≡

    def build_string(self,**temp_options):
        """Builds the string for the table collection using the temp_options"""
        out_string=""
        for table in self.tables:
            out_string=out_string+table.build_string()
        return out_string