pywurfl.ql

Package pywurfl :: Module ql |
[hide private]
[frames] | no frames]

# Source Code for Module pywurfl.ql

      1  # pywurfl QL - Wireless Universal Resource File Query Language in Python
      2  # Copyright (C) 2006-2008 Armand Lynch
      3  #
      4  # This library is free software; you can redistribute it and/or modify it
      5  # under the terms of the GNU Lesser General Public License as published by the
      6  # Free Software Foundation; either version 2.1 of the License, or (at your
      7  # option) any later version.
      8  #
      9  # This library is distributed in the hope that it will be useful, but WITHOUT
     10  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
     11  # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
     12  # details.
     13  #
     14  # You should have received a copy of the GNU Lesser General Public License
     15  # along with this library; if not, write to the Free Software Foundation, Inc.,
     16  # 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
     17  #
     18  # Armand Lynch <lyncha@users.sourceforge.net>
     19
     20  __doc__ = \
     21  """
     22  pywurfl Query Language
     23
     24  pywurfl QL is a WURFL query language that looks very similar to SQL.
     25
     26  Language Definition
     27  ===================
     28
     29  Select statement
     30  ================
     31
     32      select (device|id|ua)
     33      ---------------------
     34
     35      The select statement consists of the keyword 'select' followed by the
     36      select type which can be one of these keywords: 'device', 'ua', 'id'.
     37      The select statement is the first statement in all queries.
     38
     39      device
     40      ------
     41      When 'select' is followed by the keyword 'device', a device object will
     42      be returned for each device that matches the 'where' expression
     43      (see below).
     44
     45      ua
     46      --
     47      When 'select' is followed by the keyword 'ua', an user-agent string
     48      will be returned for each device that matches the 'where' expression
     49      (see below).
     50
     51      id
     52      --
     53      When 'select' is followed by the keyword 'id', a WURFL id string will be
     54      returned for each device that matches the 'where' expression (see below).
     55
     56
     57  Where statement
     58  ===============
     59
     60      where condition
     61      ---------------
     62      where condition and/or condition
     63      --------------------------------
     64      where any/all and/or condition
     65      ------------------------------
     66
     67      The where statement follows a select statement and can consist of the
     68      following elements: 'where condition', 'any statement', 'all statement'.
     69
     70      Where condition
     71      ---------------
     72      A where condition consists of a capability name followed by a test
     73      operator followed by a value. For example, "ringtone = true".
     74
     75      Any statement
     76      -------------
     77      An any statement consists of the keyword 'any' followed by a
     78      parenthesized, comma delimited list of capability names, followed by
     79      a test operator and then followed by a value. All capabilities
     80      listed in an any statement will be 'ored' together. There must be a
     81      minimum of two capabilities listed.
     82
     83      For example: "any(ringtone_mp3, ringtone_wav) = true".
     84
     85      All statement
     86      -------------
     87      An all statement consists of the keyword 'all' followed by a
     88      parenthesized, comma delimited list of capability names, followed by
     89      a test operator and then followed by a value. All capabilities
     90      listed in an all statement will be 'anded' together. There must be a
     91      minimum of two capabilities listed.
     92
     93      For example: "all(ringtone_mp3, ringtone_wav) = true".
     94
     95      Test operators
     96      --------------
     97      The following are the test operators that the query language can
     98      recognize::
     99
    100          = != < > >= <=
    101
    102      Comparing strings follow Python's rules.
    103
    104      Values
    105      ------
    106      Test values can be integers, strings in quotes and the tokens
    107      "true" or "false" for boolean tests.
    108
    109
    110  Binary operators
    111  ================
    112
    113      There are two binary operators defined in the language "and" and "or".
    114      They can be used between any where statement tests and follow
    115      conventional precedence rules::
    116
    117        ringtone=true or ringtone_mp3=false and preferred_markup="wml_1_1"
    118                                  -- becomes --
    119        (ringtone=true or (ringtone_mp3=false and preferred_markup="wml_1_1"))
    120
    121
    122  Example Queries
    123  ===============
    124
    125      select id where ringtone=true
    126
    127      select id where ringtone=false and ringtone_mp3=true
    128
    129      select id where rows > 3
    130
    131      select id where all(ringtone_mp3, ringtone_aac, ringtone_qcelp)=true
    132
    133      select ua where preferred_markup = "wml_1_1"
    134
    135
    136  EBNF
    137  ====
    138
    139  query := select_statement where_statement
    140
    141  select_statement := 'select' ('device' | 'id' | 'ua')
    142
    143  where_statement := 'where' + where_expression
    144
    145  where_expression := where_test (boolop where_test)*
    146
    147  where_test := (any_statement | all_statement | expr_test)
    148
    149  any_statement := 'any' '(' expr_list ')' operator expr
    150
    151  all_statement := 'all' '(' expr_list ')' operator expr
    152
    153  capability := alphanums ('_' alphanums)*
    154
    155  expr_test := expr operator expr
    156
    157  expr_list := expr (',' expr)*
    158
    159  expr := types attributes_methods_concat | capability attributes_methods_concat
    160
    161  attributes_methods_concat := ('.' method '(' method_args? ')')*
    162
    163  method_args := (method_arg (',' method_arg)*)
    164
    165  method_arg := (types | expr)
    166
    167  method := ('_' alphanums)*
    168
    169  operator := ('='|'!='|'<'|'>'|'>='|'<=')
    170
    171  types := (<quote> string <quote> | integer | boolean)
    172
    173  boolean := ('true' | 'false')
    174
    175  boolop := ('and' | 'or')
    176  """
    177
    178  import re
    179  import operator
    180
    181  from pyparsing import (CaselessKeyword, Forward, Group, ParseException,
    182                         QuotedString, StringEnd, Suppress, Word, ZeroOrMore,
    183                         alphanums, alphas, nums, oneOf, delimitedList)
    184
    185  from pywurfl.exceptions import WURFLException
    186
    187
    188  __author__ = "Armand Lynch <lyncha@users.sourceforge.net>"
    189  __contributors__ = "Gabriele Fantini <gabriele.fantini@staff.dada.net>"
    190  __copyright__ = "Copyright 2006-2008, Armand Lynch"
    191  __license__ = "LGPL"
    192  __url__ = "http://celljam.net/"
    193  __all__ = ['QueryLanguageError', 'QL']
    194
    195
    196 -class QueryLanguageError(WURFLException):
    197      """Base exception class for pywurfl.ql"""
    198      pass
    199
    200
    201 -def _toNum(s, l, toks):
    202      """Convert to pywurfl number type"""
    203      n = toks[0]
    204      try:
    205          return TypeNum(int(n))
    206      except ValueError, e:
    207          return TypeNum(float(n))
    208
    209
    210 -def _toBool(s, l, toks):
    211      """Convert to pywurfl boolean type"""
    212      val = toks[0]
    213      if val.lower() == 'true':
    214          return TypeBool(True)
    215      elif val.lower() == 'false':
    216          return TypeBool(False)
    217      else:
    218          raise QueryLanguageError("Invalid boolean value '%s'" % val)
    219
    220
    221 -def _toStr(s, l, toks):
    222      """Convert to pywurfl string type"""
    223      val = toks[0]
    224      return TypeStr(val)
    225
    226
    227 -class _Type:
    228 -    def __init__(self, py_value):
    229          self.py_value = py_value
    230
    231 -    def __getattr__(self, method):
    232          return getattr(self.py_value, method)
    233
    234
    235 -class TypeNone(_Type):
    236      pass
    237
    238
    239 -class TypeNum(_Type):
    240      pass
    241
    242
    243 -class TypeStr(_Type):
    244 -    def substr(self, begin, end):
    245          try:
    246              return self.py_value[begin:end]
    247          except IndexError, e:
    248              return None
    249
    250 -    def _match(self, regex, num=0, flags=0):
    251          if re.compile(regex, flags).match(self.py_value, num) is None:
    252              return False
    253          else:
    254              return True
    255
    256 -    def match(self, regex, num=0):
    257          return self._match(regex, num)
    258
    259 -    def imatch(self, regex, num=0):
    260          return self._match(regex, num, re.IGNORECASE)
    261
    262
    263 -class TypeBool(_Type):
    264      pass
    265
    266
    267 -class TypeList(_Type):
    268 -    def getitem(self, i):
    269          try:
    270              return self.__getitem__(i)
    271          except IndexError, e:
    272              return None
    273
    274 -def define_language():
    275      """
    276      Defines the pywurfl query language.
    277
    278      @rtype: pyparsing.ParserElement
    279      @return: The definition of the pywurfl query language.
    280      """
    281
    282      # Data types to bind to python objects
    283      integer = Word(nums).setParseAction(_toNum)
    284      boolean = (CaselessKeyword("true") | CaselessKeyword("false")).setParseAction(_toBool)
    285      string = (QuotedString("'") | QuotedString('"')).setParseAction(_toStr)
    286      types = (integer | boolean | string)('value')
    287
    288      capability = Word(alphas, alphanums + '_')('capability')
    289
    290      # Select statement
    291      select_token = CaselessKeyword("select")
    292      ua_token = CaselessKeyword("ua")
    293      id_token = CaselessKeyword("id")
    294      device_token = CaselessKeyword("device")
    295      select_type = (device_token | ua_token | id_token)("type")
    296      select_clause = select_token + select_type
    297      select_statement = Group(select_clause)("select")
    298
    299      expr = Forward()
    300
    301      # class methods
    302      method_arg = (types | Group(expr))
    303      method_args = Group(ZeroOrMore(delimitedList(method_arg)))('method_args')
    304
    305      # class attribute
    306      attribute = Word(alphas + '_', alphanums + '_')("attribute")
    307      attribute_call = (attribute + Suppress('(') + method_args +
    308                        Suppress(')'))("attribute_call")
    309      # To support method and attribute list like .lower().upper()
    310      attribute_concat = Group(ZeroOrMore(Group(Suppress('.') + (attribute_call | attribute))))('attribute_concat')
    311
    312      expr << Group(types + attribute_concat | capability + attribute_concat)('expr')
    313
    314      binop = oneOf("= != < > >= <=", caseless=True)("operator")
    315      and_ = CaselessKeyword("and")
    316      or_ = CaselessKeyword("or")
    317
    318      expr_list = (expr + ZeroOrMore(Suppress(',') + expr))
    319
    320      # Any test
    321      any_token = CaselessKeyword("any")
    322      any_expr_list = expr_list("any_expr_list")
    323      any_statement = (any_token + Suppress('(') + any_expr_list + Suppress(')') +
    324                       binop + expr("rexpr"))('any_statement')
    325
    326      # All test
    327      all_token = CaselessKeyword("all")
    328      all_expr_list = expr_list("all_expr_list")
    329      all_statement = (all_token + Suppress('(') + all_expr_list + Suppress(')') +
    330                       binop + expr("rexpr"))('all_statement')
    331
    332      # Capability test
    333      expr_test = expr('lexpr') + binop + expr('rexpr')
    334
    335      # WHERE statement
    336      boolop = (and_ | or_)('boolop')
    337      where_token = CaselessKeyword("where")
    338
    339      where_test = (all_statement | any_statement | expr_test)('where_test')
    340      where_expression = Forward()
    341      where_expression << Group(where_test + ZeroOrMore(boolop + where_expression))('where_expression')
    342
    343      #where_expression << (Group(where_test + ZeroOrMore(boolop +
    344      #                                                   where_expression) +
    345      #                           StringEnd())('where'))
    346      #where_expression = (Group(where_test + ZeroOrMore(boolop + where_test) +
    347      #                    StringEnd())('where'))
    348
    349      where_statement = where_token + where_expression
    350
    351      # Mon Jan  1 12:35:56 EST 2007
    352      # If there isn't a concrete end to the string pyparsing will not parse
    353      # query correctly
    354      return select_statement + where_statement + '*' + StringEnd()
    355
    356
    357 -def get_operators():
    358      """
    359      Returns a dictionary of operator mappings for the query language.
    360
    361      @rtype: dict
    362      """
    363
    364      def and_(func1, func2):
    365          """
    366          Return an 'anding' function that is a closure over func1 and func2.
    367          """
    368          def and_tester(value):
    369              """Tests a device by 'anding' the two following functions:"""
    370              return func1(value) and func2(value)
    371          return and_tester
    372
    373      def or_(func1, func2):
    374          """
    375          Return an 'oring' function that is a closure over func1 and func2.
    376          """
    377          def or_tester(value):
    378              """Tests a device by 'oring' the two following functions:"""
    379              return func1(value) or func2(value)
    380          return or_tester
    381
    382      return {'=':operator.eq, '!=':operator.ne, '<':operator.lt,
    383              '>':operator.gt, '>=':operator.ge, '<=':operator.le,
    384              'and':and_, 'or':or_}
    385
    386
    387  ops = get_operators()
    388
    389
    390 -def expr_test(lexpr, op, rexpr):
    391      """
    392      Returns an exp test function.
    393
    394      @param lexpr: An expr
    395      @type lexpr: expr
    396      @param op: A binary test operator
    397      @type op: string
    398      @param rexpr: An expr
    399      @type rexpr: expr
    400
    401      @rtype: function
    402      """
    403
    404      def expr_tester(devobj):
    405
    406          def evaluate(expression):
    407              value = None
    408              if expression.keys() == ['expr']:
    409                  expression = expression.expr
    410              # check wheather the expression is a capability or not
    411              if 'capability' in expression.keys():
    412                  capability = expression.capability
    413                  try:
    414                      py_value = getattr(devobj, capability)
    415                  except AttributeError, e:
    416                      raise QueryLanguageError("Invalid capability '%s'" %
    417                                               capability)
    418
    419                  if isinstance(py_value, bool):
    420                      value = TypeBool(py_value)
    421                  elif isinstance(py_value, int):
    422                      value = TypeNum(py_value)
    423                  elif isinstance(py_value, str):
    424                      value = TypeStr(py_value)
    425                  else:
    426                      raise QueryLanguageError("Unknown type '%s'" %
    427                                               py_value.__class__)
    428              else:
    429                  value = expression.value
    430
    431              for attribute in expression.attribute_concat:
    432                  py_value = None
    433                  if 'attribute_call' in attribute.keys():
    434                      method_name = attribute.attribute_call.attribute
    435                      method_args = []
    436                      for method_arg in attribute.attribute_call.method_args:
    437                          method_arg_value = None
    438                          try:
    439                              method_arg_value = evaluate(method_arg.expression)
    440                          except AttributeError, e:
    441                              method_arg_value = method_arg
    442
    443                          method_args.append(method_arg_value.py_value)
    444
    445                      try:
    446                          attr = getattr(value, method_name)
    447                          py_value = attr(*method_args)
    448                      except (AttributeError, TypeError), e:
    449                          msg = "'%s' object has no callable attribute '%s'"
    450                          raise QueryLanguageError(msg %
    451                                                   (type(value.py_value).__name__,
    452                                                    method_name))
    453                  elif 'attribute' in attribute.keys():
    454                      try:
    455                          py_value = getattr(value, attribute.attribute)
    456                      except AttributeError, e:
    457                          raise QueryLanguageError(str(e))
    458                      if callable(py_value):
    459                          msg = "'%s' object has no attribute '%s'"
    460                          raise QueryLanguageError(msg %
    461                                                   (type(value.py_value).__name__,
    462                                                    attribute.attribute))
    463                  else:
    464                      raise QueryLanguageError('query syntax error')
    465
    466                  if isinstance(py_value, bool):
    467                      value = TypeBool(py_value)
    468                  elif py_value is None:
    469                      value = TypeNone(py_value)
    470                  elif isinstance(py_value, int):
    471                      value = TypeNum(py_value)
    472                  elif isinstance(py_value, str):
    473                      value = TypeStr(py_value)
    474                  elif isinstance(py_value, (list, tuple)):
    475                      value = TypeList(py_value)
    476                  else:
    477                      raise QueryLanguageError("Unknown type '%s'" %
    478                                               py_value.__class__)
    479
    480              return value
    481
    482          lvalue = evaluate(lexpr)
    483          rvalue = evaluate(rexpr)
    484          return ops[op](lvalue.py_value, rvalue.py_value)
    485
    486      return expr_tester
    487
    488
    489 -def combine_funcs(funcs):
    490      """
    491      Combines a list of functions with binary operators.
    492
    493      @param funcs: A python list of function objects with descriptions of
    494                    binary operators interspersed.
    495
    496                    For example [func1, 'and', func2, 'or', func3]
    497      @type funcs: list
    498      @rtype: function
    499      """
    500
    501      while len(funcs) > 1:
    502          try:
    503              f_index = funcs.index('and')
    504              op = ops['and']
    505          except ValueError:
    506              try:
    507                  f_index = funcs.index('or')
    508                  op = ops['or']
    509              except ValueError:
    510                  break
    511          combined = op(funcs[f_index - 1], funcs[f_index + 1])
    512          funcs = funcs[:f_index-1] + [combined] + funcs[f_index + 2:]
    513      return funcs[0]
    514
    515
    516 -def reduce_funcs(func, seq):
    517      """
    518      Reduces a sequence of function objects to one function object by applying
    519      a binary function recursively to the sequence::
    520
    521          In:
    522              func = and
    523              seq = [func1, func2, func3, func4]
    524          Out:
    525              and(func1, and(func2, and(func3, func4)))
    526
    527      @param func: A function that acts as a binary operator.
    528      @type func: function
    529      @param seq: An ordered sequence of function objects
    530      @type seq: list
    531      @rtype: function
    532      """
    533
    534      if seq[1:]:
    535          return func(seq[0], reduce_funcs(func, seq[1:]))
    536      else:
    537          return seq[0]
    538
    539
    540 -def reduce_statement(exp):
    541      """
    542      Produces a function that represents the "any" or "all" expression passed
    543      in by exp::
    544
    545          In:
    546              any(ringtone_mp3, ringtone_awb) = true
    547          Out:
    548              ((ringtone_mp3 = true) or (ringtone_awb = true))
    549
    550      @param exp: The result from parsing an 'any' or 'all' statement.
    551      @type exp: pyparsing.ParseResults
    552      @rtype: function
    553      """
    554
    555      funcs = []
    556      if exp.any_statement:
    557          for expr in exp.any_statement.any_expr_list:
    558              funcs.append(expr_test(expr, exp.operator, exp.rexpr))
    559          return reduce_funcs(ops['or'], funcs)
    560      elif exp.all_statement:
    561          for expr in exp.all_statement.all_expr_list:
    562              funcs.append(expr_test(expr, exp.operator, exp.rexpr))
    563          return reduce_funcs(ops['and'], funcs)
    564
    565
    566 -def test_generator(ql_result):
    567      """
    568      Produces a function that encapsulates all the tests from a where
    569      statement and takes a Device class or object as a parameter::
    570
    571          In (a result object from the following query):
    572            select id where ringtone=true and any(ringtone_mp3, ringtone_awb)=true
    573
    574          Out:
    575            def func(devobj):
    576                if (devobj.ringtone == True and
    577                    (devobj.ringtone_mp3 == True or
    578                     devobj.ringtone_awb == True)):
    579                    return True
    580                else:
    581                    return False
    582            return func
    583
    584      @param ql_result: The result from calling pyparsing.parseString()
    585      @rtype: function
    586      """
    587
    588      funcs = []
    589      where_test = ql_result.where_expression
    590      while where_test:
    591          if where_test.any_statement or where_test.all_statement:
    592              func = reduce_statement(where_test)
    593          else:
    594              func = expr_test(where_test.lexpr, where_test.operator,
    595                               where_test.rexpr)
    596
    597          boolop = where_test.boolop
    598          if boolop:
    599              funcs.extend([func, boolop])
    600          else:
    601              funcs.append(func)
    602          where_test = where_test.where_expression
    603      return combine_funcs(funcs)
    604
    605
    606 -def QL(devices):
    607      """
    608      Return a function that can run queries against the WURFL.
    609
    610      @param devices: The device class hierarchy from pywurfl
    611      @type devices: pywurfl.Devices
    612      @rtype: function
    613      """
    614
    615      language = define_language()
    616
    617      def query(qstr, instance=True):
    618          """
    619          Return a generator that filters the pywurfl.Devices instance by the
    620          query string provided in qstr.
    621
    622          @param qstr: A query string that follows the pywurfl.ql language
    623                       syntax.
    624          @type qstr: string
    625          @param instance: Used to select that you want an instance instead of a
    626                           class.
    627          @type instance: boolean
    628          @rtype: generator
    629          """
    630          qstr = qstr.replace('\n', ' ').replace('\r', ' ') + '*'
    631          try:
    632              qres = language.parseString(qstr)
    633              tester = test_generator(qres)
    634              if qres.select.type == 'ua':
    635                  return (x.devua for x in devices.devids.itervalues()
    636                          if tester(x))
    637              elif qres.select.type == 'id':
    638                  return (x.devid for x in devices.devids.itervalues()
    639                          if tester(x))
    640              else:
    641                  if instance:
    642                      return (x() for x in devices.devids.itervalues()
    643                              if tester(x))
    644                  else:
    645                      return (x for x in devices.devids.itervalues()
    646                              if tester(x))
    647          except ParseException, exception:
    648              raise QueryLanguageError(str(exception))
    649      setattr(devices, 'query', query)
    650      return query
    651