pywurfl.ql

Package pywurfl :: Module ql |
[hide private]
[frames] | no frames]

# Source Code for Module pywurfl.ql

      1  # pywurfl QL - Wireless Universal Resource File Query Language in Python
      2  # Copyright (C) 2006-2010 Armand Lynch
      3  #
      4  # This library is free software; you can redistribute it and/or modify it
      5  # under the terms of the GNU Lesser General Public License as published by the
      6  # Free Software Foundation; either version 2.1 of the License, or (at your
      7  # option) any later version.
      8  #
      9  # This library is distributed in the hope that it will be useful, but WITHOUT
     10  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
     11  # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
     12  # details.
     13  #
     14  # You should have received a copy of the GNU Lesser General Public License
     15  # along with this library; if not, write to the Free Software Foundation, Inc.,
     16  # 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
     17  #
     18  # Armand Lynch <lyncha@users.sourceforge.net>
     19
     20  __doc__ = \
     21  """
     22  pywurfl Query Language
     23
     24  pywurfl QL is a WURFL query language that looks very similar to SQL.
     25
     26  Language Definition
     27  ===================
     28
     29  Select statement
     30  ================
     31
     32      select (device|id|ua)
     33      ---------------------
     34
     35      The select statement consists of the keyword 'select' followed by the
     36      select type which can be one of these keywords: 'device', 'ua', 'id'.
     37      The select statement is the first statement in all queries.
     38
     39      device
     40      ------
     41      When 'select' is followed by the keyword 'device', a device object will
     42      be returned for each device that matches the 'where' expression
     43      (see below).
     44
     45      ua
     46      --
     47      When 'select' is followed by the keyword 'ua', an user-agent string
     48      will be returned for each device that matches the 'where' expression
     49      (see below).
     50
     51      id
     52      --
     53      When 'select' is followed by the keyword 'id', a WURFL id string will be
     54      returned for each device that matches the 'where' expression (see below).
     55
     56
     57  Where statement
     58  ===============
     59
     60      where condition
     61      ---------------
     62      where condition and/or condition
     63      --------------------------------
     64      where any/all and/or condition
     65      ------------------------------
     66
     67      The where statement follows a select statement and can consist of the
     68      following elements: 'where condition', 'any statement', 'all statement'.
     69
     70      Where condition
     71      ---------------
     72      A where condition consists of a capability name followed by a test
     73      operator followed by a value. For example, "ringtone = true".
     74
     75      Any statement
     76      -------------
     77      An any statement consists of the keyword 'any' followed by a
     78      parenthesized, comma delimited list of capability names, followed by
     79      a test operator and then followed by a value. All capabilities
     80      listed in an any statement will be 'ored' together. There must be a
     81      minimum of two capabilities listed.
     82
     83      For example: "any(ringtone_mp3, ringtone_wav) = true".
     84
     85      All statement
     86      -------------
     87      An all statement consists of the keyword 'all' followed by a
     88      parenthesized, comma delimited list of capability names, followed by
     89      a test operator and then followed by a value. All capabilities
     90      listed in an all statement will be 'anded' together. There must be a
     91      minimum of two capabilities listed.
     92
     93      For example: "all(ringtone_mp3, ringtone_wav) = true".
     94
     95      Test operators
     96      --------------
     97      The following are the test operators that the query language can
     98      recognize::
     99
    100          = != < > >= <=
    101
    102      Comparing strings follow Python's rules.
    103
    104      Values
    105      ------
    106      Test values can be integers, strings in quotes and the tokens
    107      "true" or "false" for boolean tests.
    108
    109
    110  Binary operators
    111  ================
    112
    113      There are two binary operators defined in the language "and" and "or".
    114      They can be used between any where statement tests and follow
    115      conventional precedence rules::
    116
    117        ringtone=true or ringtone_mp3=false and preferred_markup="wml_1_1"
    118                                  -- becomes --
    119        (ringtone=true or (ringtone_mp3=false and preferred_markup="wml_1_1"))
    120
    121
    122  Example Queries
    123  ===============
    124
    125      select id where ringtone=true
    126
    127      select id where ringtone=false and ringtone_mp3=true
    128
    129      select id where rows > 3
    130
    131      select id where all(ringtone_mp3, ringtone_aac, ringtone_qcelp)=true
    132
    133      select ua where preferred_markup = "wml_1_1"
    134
    135
    136  EBNF
    137  ====
    138
    139  query ::= select_statement where_statement
    140
    141  select_statement ::= 'select' ('device' | 'id' | 'ua')
    142
    143  where_statement ::= 'where' + where_expression
    144
    145  where_expression ::= where_test (boolop where_test)*
    146
    147  where_test ::= (any_statement | all_statement | expr_test)
    148
    149  any_statement ::= 'any' '(' expr_list ')' operator expr
    150
    151  all_statement ::= 'all' '(' expr_list ')' operator expr
    152
    153  capability ::= alphanums ('_' alphanums)*
    154
    155  expr_test ::= expr operator expr
    156
    157  expr_list ::= expr (',' expr)*
    158
    159  expr ::= types attributes_methods_concat | capability attributes_methods_concat
    160
    161  attributes_methods_concat ::= ('.' method '(' method_args? ')')*
    162
    163  method_args ::= (method_arg (',' method_arg)*)
    164
    165  method_arg ::= (types | expr)
    166
    167  method ::= ('_' alphanums)*
    168
    169  operator ::= ('='|'!='|'<'|'>'|'>='|'<=')
    170
    171  types ::= (<quote> string <quote> | integer | boolean)
    172
    173  boolean ::= ('true' | 'false')
    174
    175  boolop ::= ('and' | 'or')
    176  """
    177
    178  import re
    179  import operator
    180
    181  from pyparsing import (CaselessKeyword, Forward, Group, ParseException,
    182                         QuotedString, StringEnd, Suppress, Word, ZeroOrMore,
    183                         alphanums, alphas, nums, oneOf, delimitedList)
    184
    185  from pywurfl.exceptions import WURFLException
    186
    187
    188  __author__ = "Armand Lynch <lyncha@users.sourceforge.net>"
    189  __contributors__ = "Gabriele Fantini <gabriele.fantini@staff.dada.net>"
    190  __copyright__ = "Copyright 2006-2010, Armand Lynch"
    191  __license__ = "LGPL"
    192  __url__ = "http://celljam.net/"
    193  __all__ = ['QueryLanguageError', 'QL']
    194
    195
    196 -class QueryLanguageError(WURFLException):
    197      """Base exception class for pywurfl.ql"""
    198      pass
    199
    200
    201 -def _toNum(s, l, toks):
    202      """Convert to pywurfl number type"""
    203      n = toks[0]
    204      try:
    205          return TypeNum(int(n))
    206      except ValueError, e:
    207          return TypeNum(float(n))
    208
    209
    210 -def _toBool(s, l, toks):
    211      """Convert to pywurfl boolean type"""
    212      val = toks[0]
    213      if val.lower() == 'true':
    214          return TypeBool(True)
    215      elif val.lower() == 'false':
    216          return TypeBool(False)
    217      else:
    218          raise QueryLanguageError("Invalid boolean value '%s'" % val)
    219
    220
    221 -def _toStr(s, l, toks):
    222      """Convert to pywurfl string type"""
    223      val = toks[0]
    224      return TypeStr(val)
    225
    226
    227 -class _Type:
    228 -    def __init__(self, py_value):
    229          self.py_value = py_value
    230
    231 -    def __getattr__(self, method):
    232          return getattr(self.py_value, method)
    233
    234
    235 -class TypeNone(_Type):
    236      pass
    237
    238
    239 -class TypeNum(_Type):
    240      pass
    241
    242
    243 -class TypeStr(_Type):
    244 -    def substr(self, begin, end):
    245          try:
    246              return self.py_value[begin:end]
    247          except IndexError, e:
    248              return None
    249
    250 -    def _match(self, regex, num=0, flags=0):
    251          if re.compile(regex, flags).match(self.py_value, num) is None:
    252              return False
    253          else:
    254              return True
    255
    256 -    def match(self, regex, num=0):
    257          return self._match(regex, num)
    258
    259 -    def imatch(self, regex, num=0):
    260          return self._match(regex, num, re.IGNORECASE)
    261
    262
    263 -class TypeBool(_Type):
    264      pass
    265
    266
    267 -class TypeList(_Type):
    268 -    def getitem(self, i):
    269          try:
    270              return self.__getitem__(i)
    271          except IndexError, e:
    272              return None
    273
    274 -def define_language():
    275      """
    276      Defines the pywurfl query language.
    277
    278      @rtype: pyparsing.ParserElement
    279      @return: The definition of the pywurfl query language.
    280      """
    281
    282      # Data types to bind to python objects
    283      integer = Word(nums).setParseAction(_toNum)
    284      boolean = (CaselessKeyword("true") | CaselessKeyword("false")).setParseAction(_toBool)
    285      string = (QuotedString("'") | QuotedString('"')).setParseAction(_toStr)
    286      types = (integer | boolean | string)('value')
    287
    288      capability = Word(alphas, alphanums + '_')('capability')
    289
    290      # Select statement
    291      select_token = CaselessKeyword("select")
    292      ua_token = CaselessKeyword("ua")
    293      id_token = CaselessKeyword("id")
    294      device_token = CaselessKeyword("device")
    295      select_type = (device_token | ua_token | id_token)("type")
    296      select_statement = (select_token + select_type)("select")
    297
    298      expr = Forward()
    299
    300      # class methods
    301      method_arg = (types | Group(expr))
    302      method_args = Group(ZeroOrMore(delimitedList(method_arg)))('method_args')
    303
    304      # class attribute
    305      attribute = Word(alphas + '_', alphanums + '_')("attribute")
    306      attribute_call = (attribute + Suppress('(') + method_args +
    307                        Suppress(')'))("attribute_call")
    308      # To support method and attribute list like .lower().upper()
    309      attribute_concat = Group(ZeroOrMore(Group(Suppress('.') + (attribute_call | attribute))))('attribute_concat')
    310
    311      expr << Group(types + attribute_concat | capability + attribute_concat)('expr')
    312
    313      binop = oneOf("= != < > >= <=", caseless=True)("operator")
    314      and_ = CaselessKeyword("and")
    315      or_ = CaselessKeyword("or")
    316
    317      expr_list = (expr + ZeroOrMore(Suppress(',') + expr))
    318
    319      # Any test
    320      any_token = CaselessKeyword("any")
    321      any_expr_list = expr_list("any_expr_list")
    322      any_statement = (any_token + Suppress('(') + any_expr_list + Suppress(')') +
    323                       binop + expr("rexpr"))('any_statement')
    324
    325      # All test
    326      all_token = CaselessKeyword("all")
    327      all_expr_list = expr_list("all_expr_list")
    328      all_statement = (all_token + Suppress('(') + all_expr_list + Suppress(')') +
    329                       binop + expr("rexpr"))('all_statement')
    330
    331      # Capability test
    332      expr_test = expr('lexpr') + binop + expr('rexpr')
    333
    334      # WHERE statement
    335      boolop = (and_ | or_)('boolop')
    336      where_token = CaselessKeyword("where")
    337
    338      where_test = (all_statement | any_statement | expr_test)('where_test')
    339      where_expression = Forward()
    340      where_expression << Group(where_test + ZeroOrMore(boolop + where_expression))('where_expression')
    341
    342      where_statement = where_token + where_expression
    343
    344      # Mon Jan  1 12:35:56 EST 2007
    345      # If there isn't a concrete end to the string pyparsing will not parse
    346      # query correctly
    347      return select_statement + where_statement + '*' + StringEnd()
    348
    349
    350 -def get_operators():
    351      """
    352      Returns a dictionary of operator mappings for the query language.
    353
    354      @rtype: dict
    355      """
    356
    357      def and_(func1, func2):
    358          """
    359          Return an 'anding' function that is a closure over func1 and func2.
    360          """
    361          def and_tester(value):
    362              """Tests a device by 'anding' the two following functions:"""
    363              return func1(value) and func2(value)
    364          return and_tester
    365
    366      def or_(func1, func2):
    367          """
    368          Return an 'oring' function that is a closure over func1 and func2.
    369          """
    370          def or_tester(value):
    371              """Tests a device by 'oring' the two following functions:"""
    372              return func1(value) or func2(value)
    373          return or_tester
    374
    375      return {'=':operator.eq, '!=':operator.ne, '<':operator.lt,
    376              '>':operator.gt, '>=':operator.ge, '<=':operator.le,
    377              'and':and_, 'or':or_}
    378
    379
    380  ops = get_operators()
    381
    382
    383 -def expr_test(lexpr, op, rexpr):
    384      """
    385      Returns an exp test function.
    386
    387      @param lexpr: An expr
    388      @type lexpr: expr
    389      @param op: A binary test operator
    390      @type op: string
    391      @param rexpr: An expr
    392      @type rexpr: expr
    393
    394      @rtype: function
    395      """
    396
    397      def expr_tester(devobj):
    398          lvalue = _evaluate(devobj, lexpr)
    399          rvalue = _evaluate(devobj, rexpr)
    400          return ops[op](lvalue.py_value, rvalue.py_value)
    401
    402      return expr_tester
    403
    404
    405 -def _evaluate(devobj, expression):
    406      """
    407      Evaluate an expression with respect to a device object
    408      """
    409      value = None
    410      if expression.keys() == ['expr']:
    411          expression = expression.expr
    412      # check wheather the expression is a capability or not
    413      if 'capability' in expression.keys():
    414          capability = expression.capability
    415          try:
    416              py_value = getattr(devobj, capability)
    417          except AttributeError, e:
    418              raise QueryLanguageError("Invalid capability '%s'" %
    419                                       capability)
    420
    421          if isinstance(py_value, bool):
    422              value = TypeBool(py_value)
    423          elif isinstance(py_value, int):
    424              value = TypeNum(py_value)
    425          elif isinstance(py_value, basestring):
    426              value = TypeStr(py_value)
    427          else:
    428              raise QueryLanguageError("Unknown type '%s'" %
    429                                       py_value.__class__)
    430      else:
    431          value = expression.value
    432
    433      for attribute in expression.attribute_concat:
    434          py_value = None
    435          if 'attribute_call' in attribute.keys():
    436              method_name = attribute.attribute_call.attribute
    437              method_args = []
    438              for method_arg in attribute.attribute_call.method_args:
    439                  method_arg_value = None
    440                  try:
    441                      method_arg_value = _evaluate(devobj, method_arg.expr)
    442                  except AttributeError, e:
    443                      method_arg_value = method_arg
    444
    445                  method_args.append(method_arg_value.py_value)
    446
    447              try:
    448                  attr = getattr(value, method_name)
    449                  py_value = attr(*method_args)
    450              except (AttributeError, TypeError), e:
    451                  msg = "'%s' object has no callable attribute '%s'"
    452                  raise QueryLanguageError(msg %
    453                                           (type(value.py_value).__name__,
    454                                            method_name))
    455          elif 'attribute' in attribute.keys():
    456              try:
    457                  py_value = getattr(value, attribute.attribute)
    458              except AttributeError, e:
    459                  raise QueryLanguageError(str(e))
    460              if callable(py_value):
    461                  msg = "'%s' object has no attribute '%s'"
    462                  raise QueryLanguageError(msg %
    463                                           (type(value.py_value).__name__,
    464                                            attribute.attribute))
    465          else:
    466              raise QueryLanguageError('query syntax error')
    467
    468          if isinstance(py_value, bool):
    469              value = TypeBool(py_value)
    470          elif py_value is None:
    471              value = TypeNone(py_value)
    472          elif isinstance(py_value, int):
    473              value = TypeNum(py_value)
    474          elif isinstance(py_value, basestring):
    475              value = TypeStr(py_value)
    476          elif isinstance(py_value, (list, tuple)):
    477              value = TypeList(py_value)
    478          else:
    479              raise QueryLanguageError("Unknown type '%s'" %
    480                                       py_value.__class__)
    481
    482      return value
    483
    484
    485 -def combine_funcs(funcs):
    486      """
    487      Combines a list of functions with binary operators.
    488
    489      @param funcs: A python list of function objects with descriptions of
    490                    binary operators interspersed.
    491
    492                    For example [func1, 'and', func2, 'or', func3]
    493      @type funcs: list
    494      @rtype: function
    495      """
    496
    497      while len(funcs) > 1:
    498          try:
    499              f_index = funcs.index('and')
    500              op = ops['and']
    501          except ValueError:
    502              try:
    503                  f_index = funcs.index('or')
    504                  op = ops['or']
    505              except ValueError:
    506                  break
    507          combined = op(funcs[f_index - 1], funcs[f_index + 1])
    508          funcs = funcs[:f_index-1] + [combined] + funcs[f_index + 2:]
    509      return funcs[0]
    510
    511
    512 -def reduce_funcs(func, seq):
    513      """
    514      Reduces a sequence of function objects to one function object by applying
    515      a binary function recursively to the sequence::
    516
    517          In:
    518              func = and
    519              seq = [func1, func2, func3, func4]
    520          Out:
    521              and(func1, and(func2, and(func3, func4)))
    522
    523      @param func: A function that acts as a binary operator.
    524      @type func: function
    525      @param seq: An ordered sequence of function objects
    526      @type seq: list
    527      @rtype: function
    528      """
    529
    530      if seq[1:]:
    531          return func(seq[0], reduce_funcs(func, seq[1:]))
    532      else:
    533          return seq[0]
    534
    535
    536 -def reduce_statement(exp):
    537      """
    538      Produces a function that represents the "any" or "all" expression passed
    539      in by exp::
    540
    541          In:
    542              any(ringtone_mp3, ringtone_awb) = true
    543          Out:
    544              ((ringtone_mp3 = true) or (ringtone_awb = true))
    545
    546      @param exp: The result from parsing an 'any' or 'all' statement.
    547      @type exp: pyparsing.ParseResults
    548      @rtype: function
    549      """
    550
    551      funcs = []
    552      if exp.any_statement:
    553          for expr in exp.any_statement.any_expr_list:
    554              funcs.append(expr_test(expr, exp.operator, exp.rexpr))
    555          func = ops['or']
    556      elif exp.all_statement:
    557          for expr in exp.all_statement.all_expr_list:
    558              funcs.append(expr_test(expr, exp.operator, exp.rexpr))
    559          func = ops['and']
    560      return reduce_funcs(func, funcs)
    561
    562
    563 -def test_generator(ql_result):
    564      """
    565      Produces a function that encapsulates all the tests from a where
    566      statement and takes a Device class or object as a parameter::
    567
    568          In (a result object from the following query):
    569            select id where ringtone=true and any(ringtone_mp3, ringtone_awb)=true
    570
    571          Out:
    572            def func(devobj):
    573                if (devobj.ringtone == True and
    574                    (devobj.ringtone_mp3 == True or
    575                     devobj.ringtone_awb == True)):
    576                    return True
    577                else:
    578                    return False
    579            return func
    580
    581      @param ql_result: The result from calling pyparsing.parseString()
    582      @rtype: function
    583      """
    584
    585      funcs = []
    586      where_test = ql_result.where_expression
    587      while where_test:
    588          if where_test.any_statement or where_test.all_statement:
    589              func = reduce_statement(where_test)
    590          else:
    591              func = expr_test(where_test.lexpr, where_test.operator,
    592                               where_test.rexpr)
    593
    594          boolop = where_test.boolop
    595          if boolop:
    596              funcs.extend([func, boolop])
    597          else:
    598              funcs.append(func)
    599          where_test = where_test.where_expression
    600      return combine_funcs(funcs)
    601
    602
    603 -def QL(devices):
    604      """
    605      Return a function that can run queries against the WURFL.
    606
    607      @param devices: The device class hierarchy from pywurfl
    608      @type devices: pywurfl.Devices
    609      @rtype: function
    610      """
    611
    612      language = define_language()
    613
    614      def query(qstr, instance=True):
    615          """
    616          Return a generator that filters the pywurfl.Devices instance by the
    617          query string provided in qstr.
    618
    619          @param qstr: A query string that follows the pywurfl.ql language
    620                       syntax.
    621          @type qstr: string
    622          @param instance: Used to select that you want an instance instead of a
    623                           class.
    624          @type instance: boolean
    625          @rtype: generator
    626          """
    627          if isinstance(qstr, str):
    628              raise UnicodeError(u"query must be a unicode string")
    629          qstr = qstr.replace('\n', ' ').replace('\r', ' ') + '*'
    630          try:
    631              qres = language.parseString(qstr)
    632              tester = test_generator(qres)
    633              if qres.select.type == 'ua':
    634                  return (x.devua for x in devices.devids.itervalues()
    635                          if tester(x))
    636              elif qres.select.type == 'id':
    637                  return (x.devid for x in devices.devids.itervalues()
    638                          if tester(x))
    639              else:
    640                  if instance:
    641                      return (x() for x in devices.devids.itervalues()
    642                              if tester(x))
    643                  else:
    644                      return (x for x in devices.devids.itervalues()
    645                              if tester(x))
    646          except ParseException, exception:
    647              raise QueryLanguageError(str(exception))
    648      setattr(devices, 'query', query)
    649      return query
    650