"""
.. include:: ../docs/sql_converter.md
"""
from pyaddons import __version__, __name__ as pkg_name
from pyaddons import gzopen
from simple_progress import progress
from io import UnsupportedOperation
import pprint as pp
# import sqlparse
import re
import sys
import os
import argparse
import warnings
import json


# The program description for the --help on the command line
PROG_DESC = """Parse SQL create table commands out of an SQL script. Note this
is only really been tested against MySQL"""

# I have put -- and # as single line comments
SQL_COMMENT = re.compile(r'^(?:--|#)')

# So any infile comments can be stripped
SQL_INLINE_COMMENT = re.compile('''(?:#|--)(?=(?:[^'"]|'[^']*'|"[^"]*")*$)''')

# Opening statements at the begining of lines
SQL_SET_OPEN = re.compile(r'^/\*\s*!')
STATEMENT_OPEN = re.compile(r'^\w+')

# This is used as a split delimiter
SQL_SET_DELIMITER = re.compile(r'''(\*/\s*;)(?=(?:[^'"]|'[^']*'|"[^"]*")*$)''')
# SQL_SET_LINE_END = re.compile(r'\*/\s*;$')

# Statement delimiters and line endings
SQL_STATEMENT_DELIMITER = re.compile(
    '''(;)(?=(?:[^'"`]|'[^']*'|"[^"]*"|`[^`]*`)*$)''')

# The SQL column delimiter so we can split on commas
# Statement delimiters and line endings
SQL_COLUMN_DELIMITER = re.compile(
    r''',(?=(?:[^'"`|]|'[^']*'|"[^"]*"|`[^`]*`|\|[^|]*\|)*$)''')

# This will replace brackets with pipes
SQL_BRACKET_REPLACE = re.compile(
     r'''[()](?=(?:[^'"`]|'[^']*'|"[^"]*"|`[^`]*`)*$)''')
# SQL_BRACKET_REPLACE = re.compile(
#   r''',([()]=(?:[^'"`]|'[^']*'|"[^"]*"|`[^`]*`)*$)''')

# SQL_STATEMENT_LINE_END = re.compile(r';$')

# Regexps to recognise create table and load data local infile statements
CREATE_TABLE_REGEXP = re.compile(
    r'''^CREATE\s+TABLE\s+["'`]?
    (?P<TABLENAME>.+?)["'`]?(?!\s+LIKE)\s*\(''',
    re.VERBOSE | re.IGNORECASE)

LOAD_DATA_LOCAL_REGEXP = re.compile(
    r'''LOAD\s+DATA\s+LOCAL\s+INFILE\s+["'`]?
    (:P<FILENAME>.+?)["'`]?
    \s+INTO\s+TABLE["'`]?(:P<TABLENAME>.+?)["'`]?''',
    re.VERBOSE | re.IGNORECASE)

# To extract the options that are listed after the column definitions
CT_OPTIONS_REGEXP = re.compile(r'^(?P<REMAIN>.+)\){1}?(?P<OPTIONS>.*?);$')

# Can be used to sub out newlines, if needed
NEW_LINE_REGEXP = re.compile(r'[\r\n]]+')

# This is to recognise a constraint with a key after it
KEY_CONSTRAINT = re.compile(
    r'^CONSTRAINT\s+(PRIMARY|UNIQUE|FOREIGN)', re.IGNORECASE)

# This is to recognise a constraint with a name after it
NAME_CONSTRAINT = re.compile(
    r'^CONSTRAINT\s+`?(.+)`?\s+', re.IGNORECASE)

# SQL_SET_LINE = re.compile(r'^/\*\s*!.*\*/\s*;')
# SQL_SET_START = re.compile(r'^/\*\s*!.*(?!\*/\s*;)')
# SQL_SET_END = re.compile(r'(?!^/\*!).*\*/\s*;')
#
# https://stackoverflow.com/questions/2785755
# SQL_DEFAULT_DELIMITER = re.compile(
#     ''';(?=(?:[^'"`]|'[^']*'|"[^"]*"|`[^`]*`)*$)''')
# IGNORE = re.compile(r'^(?:INSERT|CREATE\t+VIEW|UPDATE)', re.IGNORECASE)


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class SqlStatementTokeniser(object):
    """
    Parse SQL statements out of an SQL file. They are returned as text strings
    of complete statements
    """
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __init__(self, infile):
        """
        Parameters
        ----------
        infile : str
            The SQL file to parse
        """
        self._infile = infile
        self._infobj = None

        # Statements can either span several lines or all be on a single line
        # if there are a lot of statements on a single line then we will need
        # to hold or buffer up statements that occur after the first one. This
        # is because we are using an iterator to return 1 statement at a time
        # the iterator will attempt to return from this buffer first if it is
        # empty, then the iterator will extract a statement from the file
        self._buffer = []

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __enter__(self):
        """
        Entry point for the context manager

        Returns
        -------
        self : :obj:`MysqlToSqlite`
            Returns `self`
        """
        # Open the file when we enter the context manager
        self.open()
        return self

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __exit__(self, exc_type, exc_val, exc_tb):
        """
        Exit point for the context manager

        Parameters
        ----------
        exc_type
        exc_val
        exc_tb
        """
        # Close the file when we leave the context manager
        self.close()

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __next__(self):
        """
        Return the next statement in thr SQL file

        Returns
        -------
        sql_statement : str
            An SQL statement string
        """
        return self._get_statement()

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __iter__(self):
        """
        Entry point for the iterator

        Returns
        -------
        self : :obj:`SqlStatementTokeniser`
            Return self for iterating
        """
        return self

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def open(self):
        """
        Open the file. This is agnstic to compressed files
        """
        try:
            # Attempt to open as a file
            self._infobj = gzopen.gzip_fh(self._infile)
        except UnsupportedOperation:
            # Input is STDIN
            self._infobj = self._infile

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def close(self):
        """
        Close the file
        """
        try:
            self._infobj.close()
        except AttributeError:
            # Just in ase it is None
            pass

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _get_statement(self):
        """
        Return the next statement in the SQL file

        Returns
        -------
        sql_statement : str
            An SQL statement string
        """
        # The loop is broken by a statement return or a StopIteration
        while True:
            # Extract a line and strip lead/lagging whitespace and newlines
            line = self._next_line()

            # Ignore blank lines
            if line == '':
                continue

            # If it is a comment line move on
            if SQL_COMMENT.match(line):
                continue

            # I am not sure if inline comments are a thing in SQL, but just in
            # case we use a split to remove them
            line = SQL_INLINE_COMMENT.split(line)[0].strip()

            # Now look for opening statement blocks
            if SQL_SET_OPEN.match(line):
                # Loops for a setting statement (as in MySQLDump)
                return self._process_sql_set(line)
            elif STATEMENT_OPEN.match(line):
                # Look for an alphanumeric to indicate another statement
                return self._process_sql_statement(line)
            else:
                # If we do not understand it then warn
                warnings.warn("can't process: {0}".format(line))

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _next_line(self):
        """
        Depending on the structure of the SQL. A statement may all be on one
        line, or multiple statements on a single line or a statement may span
        over several lines. If occurring on the same line, successive
        statements will be buffered and returned after calls to next(). This
        function ensures the line gets served up in the correct order

        Returns
        -------
        line : str
            The next line in the sequence
        """
        try:
            # First attempt to return from the buffer
            return self._buffer.pop().strip()
        except IndexError:
            # pop on an empty list, so now we go to the file
            pass

        return next(self._infobj).strip()

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _add_to_buffer(self, lines):
        """
        Add some lines to the buffer. At present this is implemented with lists
        but because of the algorithm a FILO buffer might be more appropriate

        Parameters
        ----------
        lines : list or str
            Lines to add to the buffer
        """
        # Lines always get added to the start of the buffer
        self._buffer = lines + self._buffer

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _process_sql_statement(self, line):
        """
        Detect a standard SQL statement, i.e.:
        DROP TABLE IF EXISTS `phenotype_ontology_accession`;

        Parameters
        ----------
        line : str
            A line from the SQL file

        Returns
        -------
        sql_statement : str
            An SQL statement string
        """
        return self._process_unit(line, SQL_STATEMENT_DELIMITER)

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _process_sql_set(self, line):
        """
        Detect `SET` statements, i.e.:
        /*!40101 SET character_set_client = utf8 */;

        Parameters
        ----------
        line : str
            A line from the SQL file

        Returns
        -------
        sql_statement : str
            An SQL statement string
        """
        return self._process_unit(line, SQL_SET_DELIMITER)

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _process_unit(self, line, delimiter):
        """
        Generalised statement finder. The statements are identified by
        splitting on statement end delimiters, capturing whole statements and
        adding any remainers to a buffer for the next call

        Parameters
        ----------
        line : str
            A line from the SQL file
        delimiter : `re.Pattern`
            A compiled regexp for a delimiter split. The regexp should not
            split on protected delimiters

        Returns
        -------
        sql_statement : str
            An SQL statement string
        """
        # There are several outcomes
        # 1. We have a partial statement
        # 2. We have a complete statement encompasing the while line
        # 3. We have a complete statement with some other code after it
        line, remainder, hanging_statement = self._process_line(line,
                                                                delimiter)

        if hanging_statement is False and len(remainder) == 0:
            # Single line statement with whole line
            return line
        elif hanging_statement is False:
            # Single line statement with some trailing stuff
            self._add_to_buffer(remainder)
            return line

        # Anything here is a statement that spans multiple lines. Hanging
        # statement must be True, so we keep going through lines until
        # we get to the end of the statement
        statement_line = line
        while True:
            line = self._next_line()
            line, remainder, hanging_statement = self._process_line(line,
                                                                    delimiter)
            if hanging_statement is False and len(remainder) == 0:
                # Single line statement with whole line
                return '{0} {1}'.format(statement_line, line)
            elif hanging_statement is False:
                # Single line statement with some trailing stuff
                self._add_to_buffer(remainder)
                return '{0} {1}'.format(statement_line, line)

            # Another hanging statement
            statement_line = '{0} {1}'.format(statement_line, line)

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _process_line(self, line, delimiter):
        """
        Utility function, to split on the delimiter and then recombine the
        delimiters into the string

        Parameters
        ----------
        line : str
            A line from the SQL file
        delimiter : `re.Pattern`
            A compiled regexp for a delimiter split. The regexp should not
            split on protected delimiters

        Returns
        -------
        line : str
            The statement line (or partial statement line is hanging_statement
            is True
        remainder : list or str
            The Remainder of the statement if the statement does not consume
            the whole line, if it does, this will be length 0
        hanging_statement
            Indicates is line is a complete statement or just part of a larger
            statement
        """
        # 1. We have a partial statement
        # 2. We have a complete statement encompasing the while line
        # 3. We have a complete statement with some other code after it
        lines = delimiter.split(line)

        line_with_delimiter = []
        hanging_statement = True
        for start_idx in range(0, len(lines), 2):
            try:
                line_with_delimiter.append("{0}{1}".format(lines[start_idx],
                                                           lines[start_idx+1]))
            except IndexError:
                line_with_delimiter.append(lines[start_idx])
                break
            hanging_statement = False

        line = line_with_delimiter.pop(0)
        return line, line_with_delimiter, hanging_statement


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class CreateTableParser(object):
    """
    Parse create table statements into python data structures. This is not
    everything, see here:
    https://dev.mysql.com/doc/refman/8.0/en/create-table.html
    """
    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __init__(self, infile):
        """

        """
        self._infile = infile
        self.create_table_tokens = []
        self.load_data_tokens = {}

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def parse(self):
        """
        Parse the SQL into python

        """
        # Initialise the tokeniser with the input file
        with SqlStatementTokeniser(self._infile) as tokeniser:
            # Get each statement token
            for token in tokeniser:
                proc_token = token
                # Initialise an empty create table statement
                ct_statement = {'name': None, 'sql': token, 'load_data': None}
                is_create = is_create_table(token)
                if is_create:
                    proc_token = CREATE_TABLE_REGEXP.sub('', proc_token)
                    ct_statement['name'] = is_create

                    #
                    options = CT_OPTIONS_REGEXP.search(token)

                    try:
                        options_sql = options.group('OPTIONS')
                        proc_token = CT_OPTIONS_REGEXP.sub(r'\1', proc_token)
                    except AttributeError:
                        options_sql = ''

                    # ct_statement['proc_token'] = proc_token

                    columns = SQL_BRACKET_REPLACE.sub('|', proc_token)
                    columns_sql = SQL_COLUMN_DELIMITER.split(columns)

                    # pp.pprint(ct_statement)
                    parsed_cols, parsed_keys = self._parse_definitions(
                        columns_sql)
                    ct_statement['columns'] = parsed_cols
                    ct_statement['keys'] = parsed_keys
                    ct_statement['options'] = self._parse_table_options(
                        options_sql)

                    self.create_table_tokens.append(ct_statement)
                    continue

                # is_load_data = is_load_data_local(token)
                # if is_load_data:
                try:
                    load_data_spec = parse_load_data(token)
                    self.load_data_tokens[load_data_spec['table_name']] = \
                        load_data_spec
                except ValueError as e:
                    pass
                    # print(str(e))

        if len(self.create_table_tokens) == 0:
            raise ValueError("no create table statements")

        # Now loop through any load data and assign it to a create table
        for ct in self.create_table_tokens:
            try:
                ct['load_data'] = self.load_data_tokens[ct['name']]
            except KeyError:
                pass

        return self.create_table_tokens

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _parse_table_options(self, options_sql):
        """
        Parse the table options

        Parameters
        ----------
        options

        Returns
        -------

        """
        options = {}
        # These all mean the same
        charset_regexp = re.compile(
            r'''
            (?:DEFAULT\s+)?
            (?P<charset>CHARACTER\s+SET|CHARSET)
            \s*=?\s*
            (?P<value>\w+)
            \b
            ''',
            re.IGNORECASE | re.VERBOSE)

        charset_match = charset_regexp.search(options_sql)
        options_sql = charset_regexp.sub(r' ', options_sql)
        try:
            options['charset'] = charset_match.group('value')
        except (AttributeError, TypeError, IndexError):
            pass

        options_sql = options_sql.strip()

        other_opt_regexp = re.compile(
            r'''
            \b(?P<option>[A-Za-z_]+)\s*=\s*(?P<value>[\w,`]+)\b
            ''',
            re.VERBOSE
        )
        for i in other_opt_regexp.finditer(options_sql):
            options[i.group('option').lower()] = i.group('value')

        options_sql = other_opt_regexp.sub(r' ', options_sql)
        options['remainder'] = options_sql.strip()
        return options

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _parse_definitions(self, definitions):
        """
        This parses all the definitions within the CREATE TABLE statement, so
        here could be column definitions of INDEX definitions

        Parameters
        ----------
        columns : list of str
            Each element is either a column definition or an index definition

        Returns
        -------
        """
        columns = []
        keys = []
        for d in definitions:
            try:
                keys.append(self._parse_all_keys(d))
            except ValueError:
                columns.append(self._parse_column(d))

        # Now we assess the columns for the presence of keys that have been
        # defined in the column definitions
        for c in columns:
            try:
                if c['primary_key'] is True:
                    keys.append({"type": 'primary',
                                 "columns": [(c.name, None)]})
            except KeyError:
                pass

            try:
                if c['unique'] is True:
                    keys.append({"type": 'index',
                                 "unique": True,
                                 "columns": [(c.name, None)]})
            except KeyError:
                pass

        return columns, keys

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _parse_column(self, definition):
        """
        Parse a column definition

        Parameters
        ----------
        definition

        Returns
        -------

        """
        column = {'name': None, 'type': None, 'definition': None,
                  'remainder': None}
        # this is to extract the column names from the start of the definition
        col_name_regexp = re.compile(r'^`?([a-zA-Z0-9_]+)`?\s+')

        # Make sure no leading or lagging whitespace
        definition = definition.strip()
        column['definition'] = definition

        # Extract the column name
        col_name_match = col_name_regexp.match(definition)
        definition = col_name_regexp.sub('', definition, count=1)

        column['name'] = col_name_match.group(1)

        # Now extract the type, this should be the next in the string, these
        # maybe single values like INT or they may have values associated with
        # them like VARCHAR(10), DECIMAL(10,2)
        # I am not sure if this is allowed in SQL but make sure that there is
        # no space between the type and value bracket, so make sure that:
        # DECIMAL(10,2) is not represented as DECIMAL  (10,2)
        definition = re.sub(r'^([a-zA-Z_]+)\s+(\|)', r'\1\2', definition)
        # print(definition)
        col_type_regexp = re.compile(r'''
                ^(?P<type>[a-zA-Z_]+)
                (?:\|\s*(?P<values>[a-zA-Z0123456789,_`"'-]+)\s*\|)?\s*
                ''', re.VERBOSE)
        col_type_match = col_type_regexp.match(definition)
        column['type'] = col_type_match.group('type')

        try:
            type_values = col_type_match.group('values')
            column['type_values'] = \
                [i.strip(r'`"\' ') for i in SQL_COLUMN_DELIMITER.split(
                    type_values)]
        except (IndexError, TypeError):
            # TypeError: type_values is None
            column['type_values'] = []

        definition = col_type_regexp.sub('', definition)

        # Now match optional flags that have values associated with them
        opt_values = re.compile(
            r'''
            (?P<option>CHARACTER\s+SET|COLLATE|DEFAULT|COLUMN_FORMAT|STORAGE|COMMENT)
            \s+
            (?P<value>[`'"_-a-zA-Z0-9]+)''', re.IGNORECASE | re.VERBOSE)

        options = {}
        for reg in opt_values.finditer(definition):
            try:
                option = re.sub(r'\s+', '_', reg.group('option').lower())
            except IndexError:
                continue
            options[option] = reg.group('value').strip(r'`"\' ')
        definition = opt_values.sub('', definition)

        flag_options = re.compile(
            r'''
            \b
            (?P<flag>UNSIGNED|ZEROFILL|BINARY|(?:NOT\s+)?NULL|AUTO_?INCREMENT|PRIMARY\s+KEY|UNIQUE)
            \b
            ''', re.IGNORECASE | re.VERBOSE)

        for reg in flag_options.finditer(definition):
            try:
                option = re.sub(r'\s+', '_', reg.group('flag').lower())
            except IndexError:
                continue
            options[option] = True
        definition = flag_options.sub('', definition)

        try:
            if options['default'] == 'NULL':
                options['default'] = None
        except KeyError:
            pass

        column['options'] = options
        column['remainder'] = definition.strip()

        return column

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _parse_all_keys(self, definition):
        """
        Parse the index definition

        Parameters
        ----------

        Returns
        -------

        Raises
        ------
        ValueError
            If there is no index to parse out of the definition
        """
        definition = definition.strip()

        constraint = False
        constraint_name = None

        # If it is a constraint followed by a key then, contrain is True but
        # there are no names after it
        if KEY_CONSTRAINT.match(definition):
            # Here we do not want to lose the KEY information, so it gets put
            # back in
            definition = KEY_CONSTRAINT.sub(r'\1', definition)
            constraint = True
            constraint_name = None

        # If we get here we test for a constraint with some sort of name
        name_constraint = NAME_CONSTRAINT.match(definition)
        if name_constraint:
            definition = NAME_CONSTRAINT.sub('', definition)
            # If that exists then constraint is True and we also have a name
            constraint = True
            constraint_name = name_constraint.group(1)

        try:
            return self._parse_index(definition)
        except ValueError:
            pass

        try:
            return self._parse_primary_key(definition)
        except ValueError:
            pass

        try:
            return self._parse_fulltext_key(definition)
        except ValueError:
            pass

        try:
            return self._parse_foreign_key(definition)
        except ValueError:
            raise

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _parse_foreign_key(self, definition):
        """
        Parse a foreign key:
        FOREIGN KEY [index_name] (index_col_name,...) reference_definition
          reference_definition:
            REFERENCES tbl_name (index_col_name,...)
              [MATCH FULL | MATCH PARTIAL | MATCH SIMPLE]
              [ON DELETE reference_option]
              [ON UPDATE reference_option]

        Parameters
        ----------
        definition

        Returns
        -------

        """
        idx_entry = {'type': 'foreign', 'definition': definition}
        index_regexp = re.compile(
            r'''^(?P<idx>FOREIGN\s+KEY)
                \s+
                (?:`?(?P<name>[A-Za-z_]+?)`?\s+)?
                (?:\|(?P<cols>[, \w`]+)\|)   # Will split and test for ASC/DESC
                \s*
                REFERENCES\s+`?(?P<references>[A-Za-z_]+?)`?
                \s*
                (?:\|(?P<refcols>[, \w`]+)\|)   # Will split and test for ASC/DESC
                (?:\s+MATCH\s+(?P<match_type>FULL|PARTIAL|SIMPLE))
                (?:
                (?:\s+ON\s+DELETE\s+
                (?P<on_delete_option>\w+))
                |
                (?:\s+ON\s+UPDATE\s+
                (?P<on_update_option>\w+))
                )''', re.IGNORECASE | re.VERBOSE)

        # Do the regexp match
        regexp_match = index_regexp.match(definition)

        try:
            idx_type = regexp_match.group('idx')
        except (IndexError, AttributeError) as e:
            # NoneType - not an index entry
            raise ValueError("not an index: {0}".format(definition)) from e

        for i in ['name', 'references', 'match_type', 'on_delete_option',
                  'on_update_option']:
            try:
                idx_entry[i] = regexp_match.group(i)
            except IndexError:
                idx_entry[i] = None

        try:
            # Parse the index columns
            idx_entry['columns'] = self._parse_index_cols(
                regexp_match.group('cols'))
        except IndexError:
            idx_entry['columns'] = []

        try:
            # Parse the index columns
            idx_entry['reference_columns'] = self._parse_index_cols(
                regexp_match.group('refcols'))
        except IndexError:
            idx_entry['reference_columns'] = []

        return idx_entry

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _parse_fulltext_key(self, definition):
        """
        Parse a full text index or key:
        FULLTEXT		[index_name] (index_col_name,...) [index_option] ...
        FULLTEXT INDEX	[index_name] (index_col_name,...) [index_option] ...
        FULLTEXT KEY	[index_name] (index_col_name,...) [index_option] ...
        SPATIAL		    [index_name] (index_col_name,...) [index_option] ...
        SPATIAL INDEX	[index_name] (index_col_name,...) [index_option] ...
        SPATIAL KEY		[index_name] (index_col_name,...) [index_option] ...

        Parameters
        ----------
        definition

        Returns
        -------

        """
        index_regexp = re.compile(
            r'''^(?P<idx>FULLTEXT|SPATIAL)(?:\s+INDEX|KEY)?
                \s+
                (?:`?(?P<name>[A-Za-z_]+?)`?\s+)?
                (?:USING\s+(?P<mode>BTREE|HASH)\s+)?
                (?:\|(?P<cols>[, \w`]+)\|)   # Will split and test for ASC/DESC
                \s*
                (?:KEY_BLOCK_SIZE\s+=\s+(?P<key_block_size>\d+)|
                WITH\s+PARSER\s+(?P<parser>\w+)|
                COMMENT\s+(?P<comment>\w+))?
                ''', re.IGNORECASE | re.VERBOSE)

        idx_entry = self._parse_general_key(definition, index_regexp,
                                            'fulltext',
                                            ['name', 'mode', 'key_block_size',
                                             'parser', 'comment'])
        if idx_entry['type'] == None:
            idx_entry['type'] = 'spatial'

        return idx_entry

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _parse_primary_key(self, definition):
        """
        Attempt to parse a primary key definition:
        PRIMARY KEY [index_type] (index_col_name,...) [index_option] ...

        Parameters
        ----------
        definition

        Returns
        -------

        """
        pk_regexp = re.compile(
            r'''^(?P<idx>PRIMARY\s+KEY)
                \s+
                (?:USING\s+(?P<mode>BTREE|HASH)\s+)?
                (?:\|(?P<cols>[, \w`]+)\|)   # Will split and test for ASC/DESC
                \s*
                (?:KEY_BLOCK_SIZE\s+=\s+(?P<key_block_size>\d+)|
                WITH\s+PARSER\s+(?P<parser>\w+)|
                COMMENT\s+(?P<comment>\w+))?
                ''', re.IGNORECASE | re.VERBOSE)

        idx_entry = self._parse_general_key(definition, pk_regexp, 'primary',
                                            ['mode', 'key_block_size',
                                             'parser', 'comment'])

        return idx_entry

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _parse_index(self, definition):
        """
        Parse any INDEX, KEY, UNIQUE, UNIQUE INDEX, UNIQUE KEY attributes. In
        MySQL, these are followed by:
        [index_name] [index_type] (index_col_name,...) [index_option]

        Parameters
        ----------
        definition

        Returns
        -------

        """
        index_regexp = re.compile(
            r'''^(?P<idx>INDEX|KEY|UNIQUE\s+KEY|UNIQUE\s+INDEX|UNIQUE)
                \s+
                (?:`?(?P<name>[A-Za-z_]+?)`?\s+)?
                (?:USING\s+(?P<mode>BTREE|HASH)\s+)?
                (?:\|(?P<cols>[, \w`]+)\|)   # Will split and test for ASC/DESC
                \s*
                (?:KEY_BLOCK_SIZE\s+=\s+(?P<key_block_size>\d+)|
                WITH\s+PARSER\s+(?P<parser>\w+)|
                COMMENT\s+(?P<comment>\w+))?
                ''', re.IGNORECASE | re.VERBOSE)

        idx_entry = self._parse_general_key(definition, index_regexp, 'unique',
                                            ['name', 'mode', 'key_block_size',
                                             'parser', 'comment'])
        if idx_entry['type'] == 'unique':
            idx_entry['unique'] = True
        else:
            idx_entry['unique'] = False

        idx_entry['type'] = 'index'
        return idx_entry

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _parse_general_key(self, definition, regexp, required_idx_type,
                           fields):
        """

        Parameters
        ----------
        regexp_match
        fields

        Returns
        -------

        """
        idx_entry = {'type': None, 'definition':definition}

        regexp_match = regexp.match(definition)

        try:
            idx_type = regexp_match.group('idx')
            if idx_type.lower().startswith(required_idx_type):
                idx_entry['type'] = required_idx_type
        except (IndexError, AttributeError) as e:
            # NoneType - not an index entry
            raise ValueError("not an index: {0}".format(definition)) from e

        for i in fields:
            try:
                idx_entry[i] = regexp_match.group(i)
            except IndexError:
                pass

        try:
            # Parse the index columns
            idx_entry['columns'] = self._parse_index_cols(
                regexp_match.group('cols'))
        except IndexError:
            idx_entry['columns'] = []

        return idx_entry

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _parse_index_cols(self, cols):
        """

        Parameters
        ----------
        cols

        Returns
        -------

        """
        idx_cols = []
        backtick = re.compile(r'`')
        # print("Columns: {0}".format(cols))
        order = re.compile(r'^(?P<name>\w+)(?:\s+(?P<order>ASC|DESC))?$')
        for i in SQL_COLUMN_DELIMITER.split(cols):
            i = backtick.sub('', i.strip())
            # print(i)
            order_match = order.match(i)
            # print(order_match)
            idx_cols.append((order_match.group('name'),
                             order_match.group('order')))

        return idx_cols

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _get_table_name(self):
        pass


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def parse_load_data(statement):
    """
    Parse a load data local statement. Note this does not deal with all of the
    options listed below, just the most common ones I have come across:
    LOAD DATA
    [LOW_PRIORITY | CONCURRENT] [LOCAL]
    INFILE 'file_name'
    [REPLACE | IGNORE]
    INTO TABLE tbl_name
    [PARTITION (partition_name [, partition_name] ...)]
    [CHARACTER SET charset_name]
    [{FIELDS | COLUMNS}
        [TERMINATED BY 'string']
        [[OPTIONALLY] ENCLOSED BY 'char']
        [ESCAPED BY 'char']
    ]
    [LINES
        [STARTING BY 'string']
        [TERMINATED BY 'string']
    ]
    [IGNORE number {LINES | ROWS}]
    [(col_name_or_user_var
        [, col_name_or_user_var] ...)]
    [SET col_name={expr | DEFAULT},
        [, col_name={expr | DEFAULT}] ...]

    Parameters
    ----------
    statement

    Returns
    -------

    """
    statement = statement.strip()
    # print(statement)
    ld_regexp = re.compile(r'''
    ^LOAD\s+DATA\s+
    (?:(?P<load_type>LOW_PRIORITY|CONCURRENT)\s+)?
    (?:(?P<is_local>LOCAL)\s+)?
    INFILE\s+["'](?P<infile>[\w\/.-]+)["']
    \s+
    (?:(?P<load_action>REPLACE|IGNORE)\s+)?
    INTO\s+TABLE\s+
    (?:`?(?P<table_name>[A-Za-z_-]+)`?)
    \s+
    (?:CHARACTER\s+SET\s+(?P<charset>[A-Za-z_-]+)\s+)?
    (?:
       (?:FIELDS|COLUMNS)\s+
       (?:TERMINATED\s+BY\s+(?P<col_terminator>[\w|.\t\s@,_"'-]+?)\s+)?
       (?:(?:OPTIONALLY\s+)?ENCLOSED\s+BY\s+(?P<col_enclosed>[\w|.\t\s@,_"'-]+?)\s+)?
       (?:ESCAPED\s+BY\s+(?P<col_escape>[\w|.\t\s,_"@'-]+?)\s+)?
    )?
    (?:LINES\s+
       (?:STARTING\s+BY\s+(?P<line_start>[\w|.\t\s@,_"'-]+?)\s+)?
       (?:TERMINATED\s+BY\s+(?P<line_terminate>[\w|.\t\s@,_"'-]+)\s+)?
    )?
    (?:IGNORE\s+(?P<ignore_lines>\d+)\s+(?:LINES|ROWS)\s+)?
    ''', re.IGNORECASE | re.VERBOSE)

    load_data = {'infile': None, 'table_name': None, 'is_local': False}

    ld_match = ld_regexp.match(statement)

    try:
        # print(ld_match.groups())
        load_data['infile'] = ld_match.group('infile')
        load_data['table_name'] = ld_match.group('table_name')
    except (AttributeError, TypeError, IndexError) as e:
        raise ValueError("not a load data statement: '{0}'".format(
            statement)) from e

    try:
        if ld_match.group('is_local'):
            load_data['is_local'] = True
    except IndexError:
        pass

    # Now the rest
    for i in ['load_type', 'load_action', 'charset', 'col_terminator',
              'col_enclosed', 'col_escape', 'line_start', 'line_terminate',
              'ignore_lines']:
        try:
            if ld_match.group(i):
                load_data[i] = ld_match.group(i)
        except IndexError:
            pass

    statement = ld_regexp.sub(' ', statement).strip()

    # Now see if we have any columns defined that will control how the input
    # file contents are mapped to the table
    colmap_regexp = re.compile(r'''
    \(\s*(?P<col_order>[\w,@-_]+)\s*\)
    \s+
    SET\s+(?P<col_map>.+)$    
    ''', re.IGNORECASE | re.VERBOSE)
    colmap_match = colmap_regexp.match(statement)

    try:
        col_order = SQL_COLUMN_DELIMITER.split(colmap_match.group('col_order'))
        col_map = SQL_BRACKET_REPLACE.sub(r'|', colmap_match.group('col_map'))
        col_map = [i.split('=') for i in SQL_COLUMN_DELIMITER.split(col_map)]
        # pp.pprint(col_map)
        load_data['col_order'] = [i.strip() for i in col_order]
        load_data['col_map'] = [(i[0].strip(), i[1].strip()) for i in col_map]

        file_header = []
        for co in load_data['col_order']:
            # search_string = re.compile(r'\b{0}\b'.format(co))
            search_string = re.compile(r'{0}'.format(co))

            found = False
            for col, mapping in load_data['col_map']:
                if search_string.search(mapping):
                    if found is True:
                        raise ValueError("multiple mappings for '{0}'".format(
                            co
                        ))
                    found = True
                    file_header.append(col)
            if found is False:
                file_header.append(None)

        load_data['file_header'] = file_header
    except (AttributeError, TypeError, IndexError) as e:
        pass

    statement = colmap_regexp.sub(' ', statement)
    load_data['remainder'] = statement.strip()
    return load_data


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def is_create_table(statement):
    """
    Determine if the SQL statement is a create table statement. Note that this
    does not consider CREATE TABLE XXX LIKE as a create table statement

    Parameters
    ----------
    statement : str
        The statement to check

    Returns
    -------
    is_create_table : bool
        `True` if the statement is a create table statement `False` if not.
    """
    statement = statement.strip()
    statement = NEW_LINE_REGEXP.sub('', statement)
    match = CREATE_TABLE_REGEXP.match(statement)
    if match:
        return match.group(1)
    return False


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def is_load_data_local(statement):
    """
    Determine if a statement is a load data local infile statement


    Parameters
    ----------
    statement : str
        The statement to check

    Returns
    -------
    is_load_data_local : bool
        `True` if the statement is a load data local infile statement
        `False` if not.
    """
    statement = statement.strip()
    statement = NEW_LINE_REGEXP.sub('', statement)
    match = LOAD_DATA_LOCAL_REGEXP.match(statement)
    if match:
        return match.group(1)
    return False


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def init_cmd_line_args(description=PROG_DESC):
    """
    Initialise the command line arguments. This does not parse them, it just
    sets them up

    Parameters
    ----------
    description : str or NoneType, optional, default: NoneType
        The program description, if NoneType it will default to the main
        program description

    Returns
    -------
    parser : :obj:`argparse.ArgumentParser`
        The parser object with all the arguments setup
    """
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument('-i', '--infile', default=sys.stdin,
                        help="The input file if not present then, it is "
                             "assumed that input is from STDIN")
    parser.add_argument('-o', '--outfile', default=sys.stdout,
                        help="The output file name, if not present then it is"
                             " assumed output is to STDOUT")
    parser.add_argument('-f', '--out-format', type=str, default='json',
                        choices=['json'])
    parser.add_argument('-v', '--verbose', action='store_true',
                        help="turn on verbose output")

    return parser


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def parse_cmd_line_args(parser):
    """

    Parameters
    ----------
    parser : :obj:`argparse.ArgumentParser`
        The parser object with all the arguments setup

    Returns
    -------
    args : :obj:`Namespace`
        A container object with the argument inside it
    """
    return parser.parse_args()


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def process_sql_file(infile, outfile, out_format='json'):
    """
    Process the SQLfile and extract the create table statements out of it

    Parameters
    ----------
    infile : str
        The input SQL file
    outfile : str or NoneType, optional, default: NoneType
        The output file, if `NoneType` then output will be to STDOUT
    out_format : str, optional, default: json
        The output file format, at present the only valid option is JSON, but
        other formats will be supported in future
    """
    try:
        outfile_obj = open(os.path.realpath(os.path.expanduser(outfile)), 'wt')
    except TypeError:
        # Output to STDOUT
        outfile_obj = outfile

    try:
        if out_format == 'json':
            ctp = CreateTableParser(infile)
            json.dump(ctp.parse(), outfile_obj, indent=4)
        else:
            raise ValueError("unknown output format: '{0}'".format(out_format))
    finally:
        try:
            # Make sure the outfile is closed
            outfile_obj.close()
        except Exception:
            # There so I can see what error to catch
            raise


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def main():
    """
    Main entry point for the script
    """
    parser = init_cmd_line_args()
    args = parse_cmd_line_args(parser)

    msg = progress.Msg(verbose=args.verbose, file=sys.stderr)
    msg.msg("= SQL converter ({0} v{1}) =".format(pkg_name, __version__))
    msg.prefix = '[info] '
    msg.msg_atts(args)

    process_sql_file(args.infile, args.outfile)


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
if __name__ == '__main__':
    main()

# # @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# class MysqlToSqlite(object):
#     """
#     Take a MySQL dumpfile and attempt to convert it into SQLite syntax. Also
#     see here:
#     https://gist.github.com/esperlu/943776#file-mysql2sqlite-sh
#     """
#
#     # Multiline/Trigger comments, these will be deleted
#     MULTI_COMMENT = re.compile(r"/\*.*\*/")
#
#     # Single line comments
#     SINGLE_COMMENT = re.compile(r"#.+?\n")
#
#     # New line characters
#     NEW_LINE = re.compile(r'\n')
#
#     # Backticks that are used in MySQL dump files
#     BACKTICKS = re.compile(r'`(.+?)`')
#
#     # Group capture to auto_incrment so I can remove the _ i.e. r.sub(r'\1\2',s)
#     # I will also have to substitute a PRIMARY KEY statement in here as any
#     # column that is AUTOINCREMENT has to eb defined as primary key on the
#     # same line
#     AUTO_INC = re.compile(r'(auto)_(increment)(?:\s+PRIMARY\s+KEY)?',
#                           re.IGNORECASE)
#
#     # This removes all the crap after the table dininition that mysql uses
#     # this is used in a findall
#     ENGINE = re.compile(r"ENGINE.*", re.IGNORECASE)
#
#     ENGINE_FIND = re.compile(r"create table .+?\)\s*\)", re.IGNORECASE)
#
#     # Capture of the table name
#     # TABLE_NAME = re.compile(r"create table\s+(.+?)\s*\(\s*(.*)\)",
#     #                         re.IGNORECASE)
#     TABLE_NAME = re.compile(r"""create table\s+(?P<NAME>.+?)\s*\(\s*(?P<DEF>.*)\)""",
#                             re.IGNORECASE)
#
#     # Compiled so I can add in the IF NOT EXISTS if required
#     IF_NOT_EXISTS = re.compile(r"^(create table)", re.IGNORECASE)
#
#     # This is so I can remove all key definitions to get at the column names
#     PRIMARY_KEY = re.compile(r",\s*PRIMARY\s+KEY\s*\(.+?\)", re.IGNORECASE)
#
#     # This was the old way I used to grab key/index columns. It remains in
#     # there as it is used in other parts of the code to determine the column
#     # names
#     UNIQUE_KEY = re.compile(
#         r",\s*(?P<KT>UNIQUE|FULLTEXT)?\s+(?P<KIDX>KEY|INDEX){1}\s*(?P<KN>\w+)\s*\((?P<COLS>.*)\)",
#         re.IGNORECASE)
#
#     # The Dynamic key has the potential key columns added to it during
#     # processing, this makes the regexp more specific and easier to handle
#     DYNAMIC_KEY = r',?\s*(?P<KT>UNIQUE|FULLTEXT)?\s+(?P<KIDX>KEY|INDEX){1}\s*(?P<KN>\w+)\s*\((?P<COLS>%s)\s*\),?'
#
#     # COLNAME = re.compile(r",\s*(\w+)|^(\w+)")
#     COLNAME = re.compile(r"(?:,\s*|^)(?P<COLNAME>\w+)")
#
#     UNSIGNED = re.compile(r"\s+unsigned\s+")
#
#     # To match 'set' MySQL types
#     SET = re.compile(r"(?:set|enum)\((.+?)\)")
#
#     # Ints
#     INTS = re.compile(r"(small|tiny|big)?int\(\d+\)", re.IGNORECASE)
#
#     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#     def __init__(self, dumpfile, if_not_exists=False):
#         # We use gzopen to open it as they usually are compressed
#         self._sql_fo =  gzopen.gzip_fh(dumpfile)
#
#         # I use sqlparse to parse the file into separate create table statements
#         # from those I can extract the table name and I also want to change the
#         # engine from MyISAM to InnoDB
#         self._parsed = sqlparse.parse(self._sql_fo.read())
#         self._cur_idx = 0
#         # pp.pprint(parsed)
#
#     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#     def __enter__(self):
#         """
#         Entry point for the context manager
#
#         Returns
#         -------
#         self : :obj:`MysqlToSqlite`
#             Returns `self`
#         """
#         return self
#
#     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#     def __exit__(self, exc_type, exc_val, exc_tb):
#         """
#         Exit point for the context manager
#
#         Parameters
#         ----------
#         exc_type
#         exc_val
#         exc_tb
#         """
#         pass
#
#     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#     def __next__(self):
#         """
#
#         Returns
#         -------
#
#         """
#         return self._get_create_table()
#
#     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#     def __iter__(self):
#         return self
#
#     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#     def _get_create_table(self):
#         """
#
#         Returns
#         -------
#
#         """
#         table = None
#         stmt = None
#         while not table:
#             try:
#                 stmt = self._parsed[self._cur_idx]
#                 self._cur_idx += 1
#             except IndexError as e:
#                 raise StopIteration("no more statements") from e
#
#             stmt = str(stmt).strip()
#             table = MysqlToSqlite.TABLE_NAME.match(stmt)
#
#         # Will hold the SQL read in from the file
#         stmt = MysqlToSqlite.ENGINE.sub('', stmt)
#         stmt = MysqlToSqlite.MULTI_COMMENT.sub('', stmt)
#         stmt = MysqlToSqlite.SINGLE_COMMENT.sub('', stmt)
#         stmt = MysqlToSqlite.NEW_LINE.sub('', stmt)
#         stmt = MysqlToSqlite.BACKTICKS.sub(r'\1', stmt)
#         stmt = MysqlToSqlite.INTS.sub(r'INTEGER', stmt)
#
#         # Attempt to parse out the table name and the column defitions as a
#         # single chunk
#         tm = MysqlToSqlite.TABLE_NAME.match(stmt)
#         table_name = tm.group('NAME')
#         column_defs = tm.group('DEF')
#
#         # Remove any unsigned key words and replce with a space
#         stmt = MysqlToSqlite.UNSIGNED.sub(r' ', stmt)
#         pp.pprint(table_name)
#         pp.pprint(column_defs)
#         # We first remove primary key and unique key statements
#         column_defs = MysqlToSqlite.PRIMARY_KEY.sub(r"", column_defs)
#         column_defs = MysqlToSqlite.UNIQUE_KEY.sub(r"", column_defs)
#         pp.pprint(table_name)
#         pp.pprint(column_defs)
#
#         # This extracts only the column names from the complete column
#         # definitions
#         column_names = [k.group('COLNAME')
#                         for k in MysqlToSqlite.COLNAME.finditer(column_defs)]
#
#         index = []
#
#         pp.pprint(table_name)
#         pp.pprint(column_defs)
#         pp.pprint(column_names)
#
#         return stmt
#
#
#     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#     def parse_create_table(self, sql, if_not_exists=False):
#         """
#         Parse a MySQL create table command into an SQLite one. This works well
#         with properly formated SQL from a MySQL dump file but is NOT
#         comprehensive
#
#         Parameters
#         ----------
#
#         ctsql : :obj:`str`
#             A string containing a MySQL create table statement
#
#         Returns
#         -------
#
#         An SQLite formatted create table statement
#         """
#
#         # print(sql)
#         # Do we want to add if not exists in front of create table
#         # statements and create index statements
#         if if_not_exists is False:
#             if_not_exists = ""
#         else:
#             if_not_exists = " IF NOT EXISTS "
#
#         sql = sql.strip()
#
#         # Will hold the SQL read in from the file
#         sql = MysqlToSqlite.ENGINE.sub('', sql)
#         sql = MysqlToSqlite.MULTI_COMMENT.sub('', sql)
#         sql = MysqlToSqlite.SINGLE_COMMENT.sub('', sql)
#         sql = MysqlToSqlite.NEW_LINE.sub('', sql)
#         sql = MysqlToSqlite.BACKTICKS.sub(r'\1', sql)
#         sql = MysqlToSqlite.INTS.sub(r'INTEGER', sql)
#
#         # print(sql)
#         tm = MysqlToSqlite.TABLE_NAME.match(sql)
#         table_name = tm.groups()[0]
#         column_defs = tm.groups()[1]
#
#         sql = MysqlToSqlite.UNSIGNED.sub(r' ', sql)
#
#         column_defs = MysqlToSqlite.PRIMARY_KEY.sub(r"", column_defs)
#         column_defs = MysqlToSqlite.UNIQUE_KEY.sub(r"", column_defs)
#         column_defs = [k.groups()[0] if k.groups()[0] is not None else
#                        k.groups()[1] for k in MysqlToSqlite.COLNAME.finditer(
#             column_defs)]
#
#         index = []
#
#         # Create REGEX for the index columns
#         index_cols = r"(?:{0})+".format(r"|".join(
#             [r"{0}\s*(:?\(\d+\))?,?".format(i) for i in column_defs]))
#
#         # Add the index columns to the dynamic key regexp
#         keyreg = MysqlToSqlite.DYNAMIC_KEY % index_cols
#         keyreg = re.compile(keyreg)
#         # print(sql)
#         # Now find indexes and remove from the create table statement and
#         # create separate create index statements
#         for k in keyreg.finditer(sql):
#             # remove any index length values such as column(10). These are used
#             # when creating indexes on text fields in MySQL but SQLite does note
#             # seem to recognise them
#             idxcols = re.sub(r'\(\d+\)', '', k.group('COLS'))
#             idx = "CREATE INDEX %s%s_%s ON %s (%s)" % (if_not_exists,
#                                                        table_name,
#                                                        k.group('KN'),
#                                                        table_name,
#                                                        idxcols)
#             index.append(idx)
#             # sql = MysqlToSqlite.UNIQUE_KEY.sub(r"", sql)
#             sql = keyreg.sub(r"", sql)
#
#         # Make sets into VARCHARs with the length of the max set member
#         # length. To do that I use index replacment, this is why opperate in
#         # reverse so that if there is more than 1 enum type then the
#         # coordinates are maintained for the ones the come first
#         for k in reversed(list(MysqlToSqlite.SET.finditer(sql))):
#             varchar_max = max([len(x) for x in re.split(r"','|'",
#                                                         k.group(1))[1:-1]])
#             sql = sql[:k.start()] + "VARCHAR(%d)" % (varchar_max) + sql[
#                                                                     k.end():]
#
#         # if we match auto increment then we must also
#         if MysqlToSqlite.AUTO_INC.search(sql):
#             # Remove any primary key definitions that may be at the end of
#             # the create table syntax and add in the one inline with
#             # autoincrment note that it is possible that there is already
#             # one there in the mysql definition, this will be removed
#             # (see auto_inc regex above)
#             sql = MysqlToSqlite.PRIMARY_KEY.sub(r"", sql)
#             sql = MysqlToSqlite.AUTO_INC.sub(r'PRIMARY KEY \1\2', sql)
#
#         # Add in the if not exists if that is required
#         sql = MysqlToSqlite.IF_NOT_EXISTS.sub((r'\1 %s' % if_not_exists),
#                                               sql)
#
#         # Return the components that will be needed to create the table in SQL
#         return table_name, column_defs, sql, index
#
#     # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#     def parse_dump_no_data(self, sql_file, if_not_exists=False):
#         """
#         *** WARNING THIS IS BROKEN ***
#         TODO: Fix this to use sqlparse
#         Parse a MySQL dum file that contains no data (only create table
#         statments. Can except either a file name (i.e. string), or a file
#         handle
#
#         Parameters
#         ----------
#
#         sql_file : :obj:`str` or `FileObject`
#             A route to the MySQL dump file
#         if_no_exists : :obj:`bool`
#             Do you want add IF NOT EXISTS statments in from of the CREATE TABLE
#             statments
#         """
#
#         # Do we want to add if not exists in front of create table
#         # statements and create index statements
#         if if_not_exists is False:
#             if_not_exists = ""
#         else:
#             if_not_exists = " IF NOT EXISTS "
#
#         table_sql = {}
#
#         # Will hold the SQL read in from the file
#         sql = ""
#
#         # Slurp the MySQL sql file into memory, this is potentially bad news if
#         # the file contains data as well! I need to handle this better
#         if isinstance(sql_file, str) is True:
#             try:
#                 sql = open(sql_file).read()
#             except IOError:
#                 raise IOError(
#                     "[FATAL] Can't open sql file '{0}'".format(sql_file))
#         else:
#             # assume a file handle
#             sql = sql_file.read()
#
#         # Remove all the new line characters
#         sql = MysqlToSqlite.MULTI_COMMENT.sub('', sql)
#         sql = MysqlToSqlite.SINGLE_COMMENT.sub('', sql)
#         sql = MysqlToSqlite.NEW_LINE.sub('', sql)
#         sql = MysqlToSqlite.BACKTICKS.sub(r'\1', sql)
#         sql = MysqlToSqlite.INTS.sub(r'INTEGER', sql)
#
#         # Id the tables through the engine statement (not great)
#         sql_tables = re.findall(MysqlToSqlite.ENGINE, sql)
#
#         for s in sql_tables:
#             # As well as formating the MySQL sql for sqlite
#             # I want to parse the table and column names out of it
#             tm = MysqlToSqlite.TABLE_NAME.match(s)
#             table_name = tm.groups()[0]
#             column_defs = tm.groups()[1]
#
#             table_sql[table_name] = {'create': "", 'index': [], 'columns': []}
#             s = MysqlToSqlite.UNSIGNED.sub(r' ', s)
#
#             column_defs = MysqlToSqlite.PRIMARY_KEY.sub(r"", column_defs)
#             column_defs = MysqlToSqlite.UNIQUE_KEY.sub(r"", column_defs)
#             table_sql[table_name]['columns'] = [
#                 k.groups()[0] if k.groups()[0] is not None else k.groups()[1]
#                 for k in self.colname.finditer(column_defs)]
#
#             for k in self.uniqkey_reg.finditer(s):
#                 idx = "CREATE INDEX %s%s_%s ON %s (%s)" % (
#                 if_not_exists, table_name, k.group(3), table_name, k.group(4))
#                 table_sql[table_name]['index'].append(idx)
#
#             s = MysqlToSqlite.UNIQUE_KEY.sub(r"", s)
#
#             for k in reversed(list(self.set_reg.finditer(s))):
#                 varchar_max = max(
#                     [len(x) for x in re.split(r"','|'", k.group(1))[1:-1]])
#                 s = s[:k.start()] + "VARCHAR(%d)" % (varchar_max) + s[k.end():]
#
#             # if we match auto increment then we must also
#             if self.auto_inc.search(s):
#                 # Remove any primary key definitions that may be at the end of
#                 # the create table syntax and add in the one inline with
#                 # autoincrment note that it is possible that there is already
#                 # one there in the mysql definition, this will be removed
#                 # (see auto_inc regex above)
#                 s = self.primkey_reg.sub(r"", s)
#                 s = self.auto_inc.sub(r'PRIMARY KEY \1\2', s)
#
#             # Add in the if not exists if that is required
#             s = self.if_not_exists.sub((r'\1 %s' % if_not_exists), s)
#
#             table_sql[table_name]['create'] = s
#
#         return table_sql
