ewa.ply.ctokens

Package ewa :: Package ply :: Module ctokens |
[hide private]
[frames] | no frames]

# Source Code for Module ewa.ply.ctokens

      1  # ----------------------------------------------------------------------
      2  # ctokens.py
      3  #
      4  # Token specifications for symbols in ANSI C and C++.  This file is
      5  # meant to be used as a library in other tokenizers.
      6  # ----------------------------------------------------------------------
      7
      8  # Reserved words
      9
     10  tokens = [
     11      # Literals (identifier, integer constant, float constant, string constant, char const)
     12      'ID', 'TYPEID', 'ICONST', 'FCONST', 'SCONST', 'CCONST',
     13
     14      # Operators (+,-,*,/,%,|,&,~,^,<<,>>, ||, &&, !, <, <=, >, >=, ==, !=)
     15      'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD',
     16      'OR', 'AND', 'NOT', 'XOR', 'LSHIFT', 'RSHIFT',
     17      'LOR', 'LAND', 'LNOT',
     18      'LT', 'LE', 'GT', 'GE', 'EQ', 'NE',
     19
     20      # Assignment (=, *=, /=, %=, +=, -=, <<=, >>=, &=, ^=, |=)
     21      'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL',
     22      'LSHIFTEQUAL','RSHIFTEQUAL', 'ANDEQUAL', 'XOREQUAL', 'OREQUAL',
     23
     24      # Increment/decrement (++,--)
     25      'PLUSPLUS', 'MINUSMINUS',
     26
     27      # Structure dereference (->)
     28      'ARROW',
     29
     30      # Ternary operator (?)
     31      'TERNARY',
     32
     33      # Delimeters ( ) [ ] { } , . ; :
     34      'LPAREN', 'RPAREN',
     35      'LBRACKET', 'RBRACKET',
     36      'LBRACE', 'RBRACE',
     37      'COMMA', 'PERIOD', 'SEMI', 'COLON',
     38
     39      # Ellipsis (...)
     40      'ELLIPSIS',
     41  ]
     42
     43  # Operators
     44  t_PLUS             = r'\+'
     45  t_MINUS            = r'-'
     46  t_TIMES            = r'\*'
     47  t_DIVIDE           = r'/'
     48  t_MODULO           = r'%'
     49  t_OR               = r'\|'
     50  t_AND              = r'&'
     51  t_NOT              = r'~'
     52  t_XOR              = r'\^'
     53  t_LSHIFT           = r'<<'
     54  t_RSHIFT           = r'>>'
     55  t_LOR              = r'\|\|'
     56  t_LAND             = r'&&'
     57  t_LNOT             = r'!'
     58  t_LT               = r'<'
     59  t_GT               = r'>'
     60  t_LE               = r'<='
     61  t_GE               = r'>='
     62  t_EQ               = r'=='
     63  t_NE               = r'!='
     64
     65  # Assignment operators
     66
     67  t_EQUALS           = r'='
     68  t_TIMESEQUAL       = r'\*='
     69  t_DIVEQUAL         = r'/='
     70  t_MODEQUAL         = r'%='
     71  t_PLUSEQUAL        = r'\+='
     72  t_MINUSEQUAL       = r'-='
     73  t_LSHIFTEQUAL      = r'<<='
     74  t_RSHIFTEQUAL      = r'>>='
     75  t_ANDEQUAL         = r'&='
     76  t_OREQUAL          = r'\|='
     77  t_XOREQUAL         = r'^='
     78
     79  # Increment/decrement
     80  t_INCREMENT        = r'\+\+'
     81  t_DECREMENT        = r'--'
     82
     83  # ->
     84  t_ARROW            = r'->'
     85
     86  # ?
     87  t_TERNARY          = r'\?'
     88
     89  # Delimeters
     90  t_LPAREN           = r'\('
     91  t_RPAREN           = r'\)'
     92  t_LBRACKET         = r'\['
     93  t_RBRACKET         = r'\]'
     94  t_LBRACE           = r'\{'
     95  t_RBRACE           = r'\}'
     96  t_COMMA            = r','
     97  t_PERIOD           = r'\.'
     98  t_SEMI             = r';'
     99  t_COLON            = r':'
    100  t_ELLIPSIS         = r'\.\.\.'
    101
    102  # Identifiers
    103  t_ID = r'[A-Za-z_][A-Za-z0-9_]*'
    104
    105  # Integer literal
    106  t_INTEGER = r'\d+([uU]|[lL]|[uU][lL]|[lL][uU])?'
    107
    108  # Floating literal
    109  t_FLOAT = r'((\d+)(\.\d+)(e(\+|-)?(\d+))? | (\d+)e(\+|-)?(\d+))([lL]|[fF])?'
    110
    111  # String literal
    112  t_STRING = r'\"([^\\\n]|(\\.))*?\"'
    113
    114  # Character constant 'c' or L'c'
    115  t_CHARACTER = r'(L)?\'([^\\\n]|(\\.))*?\''
    116
    117  # Comment (C-Style)
    118 -def t_COMMENT(t):
    119      r'/\*(.|\n)*?\*/'
    120      t.lexer.lineno += t.value.count('\n')
    121      return t
    122
    123  # Comment (C++-Style)
    124 -def t_CPPCOMMENT(t):
    125      r'//.*\n'
    126      t.lexer.lineno += 1
    127      return t
    128