Coverage for gramex\config.py: 89%

# Coverage for gramex\config.py : 89%

## 403 statements 359 run 44 missing 0 excluded 15 partial

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1'''

2 Manages YAML config files as layered configurations with imports.

3

4:class:PathConfig loads YAML files from a path::

5

6 pc = PathConfig('/path/to/file.yaml')

7

8 This can be reloaded via the ``+`` operator. ``+pc`` reloads the YAML file

9(but only if it is newer than before.)

10

11:class:ChainConfig chains multiple YAML files into a single config. For example

12 this merges ``base.yaml`` and ``next.yaml`` in sequence::

13

14 cc = ChainConfig()

15 cc['base'] = PathConfig('base.yaml')

16 cc['next'] = PathConfig('next.yaml')

17

18 To get the merged file, use ``+cc``. This updates the PathConfig files and

19 merges the YAMLs.

20'''

21

22 import os

23 import re

24 import csv

25 import six

26 import sys

27 import yaml

28 import string

29 import socket

30 import inspect

31 import logging

32 import datetime

33 import dateutil.tz

34 import dateutil.parser

35 from pathlib import Path

36 from copy import deepcopy

37 from random import choice

38 from fnmatch import fnmatch

39 from six import string_types

40 from collections import OrderedDict

41 from pydoc import locate as _locate, ErrorDuringImport

42 from yaml import Loader, MappingNode

43 from json import loads, JSONEncoder, JSONDecoder

44 from yaml.constructor import ConstructorError

45 from orderedattrdict import AttrDict, DefaultAttrDict

46 from errno import EACCES, EPERM

47

48 ERROR_SHARING_VIOLATION = 32 # from winerror.ERROR_SHARING_VIOLATION

49

50# gramex.config.app_log is the default logger used by all of gramex

51# If it's not there, create one.

52 logging.basicConfig()

53 app_log = logging.getLogger('gramex')

54

55# app_log_extra has additional parameters that may be used by the logger

56 app_log_extra = {'port': 'PORT'}

57 app_log = logging.LoggerAdapter(app_log, app_log_extra)

58

59# sqlalchemy.create_engine requires an encoding= that must be an str across

60# Python 2 and Python 3. Expose this for other modules to use

61 str_utf8 = str('utf-8') # noqa

62

63

64 def walk(node):

65 '''

66 Bottom-up recursive walk through a data structure yielding a (key, value,

67 node) tuple for every entry. ``node[key] == value`` is true in every entry.

68

69 For example::

70

71 >>> list(walk([{'x': 1}]))

72 [

73 ('x', 1, {'x': 1}), # leaf: key, value, node

74 (0, {'x': 1}, [{'x': 1}]) # parent: index, value, node

75 ]

76

77 Circular linkage can lead to a RuntimeError::

78

79 >>> x = {}

80 >>> x['x'] = x

81 >>> list(walk(x))

82 ...

83 RuntimeError: maximum recursion depth exceeded

84 '''

85 if hasattr(node, 'items'):

86 for key, value in list(node.items()):

87 for item in walk(value):

88 yield item

89 yield key, value, node

90 elif isinstance(node, list):

91 for index, value in enumerate(node):

92 for item in walk(value):

93 yield item

94 yield index, value, node

95

96

97 def merge(old, new, mode='overwrite', warn=None, _path=''):

98 '''

99 Update old dict with new dict recursively.

100

101 >>> merge({'a': {'x': 1}}, {'a': {'y': 2}})

102 {'a': {'x': 1, 'y': 2}}

103

104 If ``new`` is a list, convert into a dict with random keys.

105

106 If ``mode='overwrite'``, the old dict is overwritten (default).

107 If ``mode='setdefault'``, the old dict values are updated only if missing.

108

109 ``warn=`` is an optional list of key paths. Any conflict on dictionaries

110 matching any of these paths is logged as a warning. For example,

111 ``warn=['url.*', 'watch.*']`` warns if any url: sub-key or watch: sub-key

112 has a conflict.

113 '''

114 for key in new:

115 if key in old and hasattr(old[key], 'items') and hasattr(new[key], 'items'):

116 path_key = _path + ('.' if _path else '') + six.text_type(key)

117 if warn is not None:

118 for pattern in warn:

119 if fnmatch(path_key, pattern): 119 ↛ 120line 119 didn't jump to line 120, because the condition on line 119 was never true

120 app_log.warning('Duplicate key: %s', path_key)

121 break

122 merge(old=old[key], new=new[key], mode=mode, warn=warn, _path=path_key)

123 elif mode == 'overwrite' or key not in old:

124 old[key] = deepcopy(new[key])

125 return old

126

127

128 class ChainConfig(AttrDict):

129 '''

130 An AttrDict that manages multiple configurations as layers.

131

132 >>> config = ChainConfig([

133 ... ('base', PathConfig('gramex.yaml')),

134 ... ('app1', PathConfig('app.yaml')),

135 ... ('app2', AttrDict())

136 ... ])

137

138 Any dict-compatible values are allowed. ``+config`` returns the merged values.

139 '''

140

141 def __pos__(self):

142 '''+config returns layers merged in order, removing null keys'''

143 conf = AttrDict()

144 for name, config in self.items():

145 if hasattr(config, '__pos__'):

146 config.__pos__()

147 merge(old=conf, new=config, mode='overwrite')

148

149 # Remove keys where the value is None

150 for key, value, node in list(walk(conf)):

151 if value is None:

152 del node[key]

153

154 return conf

155

156

157# Paths that users have already been warned about. Don't warn them again

158 _warned_paths = set()

159# Get the directory where gramex is located. This is the same as the directory

160# where this file (config.py) is located.

161 _gramex_path = os.path.dirname(os.path.abspath(__file__))

162

163

164 def setup_variables():

165 '''Initialise variables'''

166 variables = DefaultAttrDict(str)

167 # Load all environment variables

168 variables.update(os.environ)

169 # GRAMEXPATH is the Gramex root directory

170 variables['GRAMEXPATH'] = _gramex_path

171 # GRAMEXAPPS is the Gramex apps directory

172 variables['GRAMEXAPPS'] = os.path.join(_gramex_path, 'apps')

173 # GRAMEXHOST is the hostname

174 variables['GRAMEXHOST'] = socket.gethostname()

175 # GRAMEXDATA varies based on OS

176 if 'GRAMEXDATA' not in variables: 176 ↛ 189line 176 didn't jump to line 189, because the condition on line 176 was never false

177 if sys.platform.startswith('linux') or sys.platform == 'cygwin': 177 ↛ 178line 177 didn't jump to line 178, because the condition on line 177 was never true

178 variables['GRAMEXDATA'] = os.path.expanduser('~/.config/gramexdata')

179 elif sys.platform == 'win32': 179 ↛ 181line 179 didn't jump to line 181, because the condition on line 179 was never false

180 variables['GRAMEXDATA'] = os.path.join(variables['LOCALAPPDATA'], 'Gramex Data')

181 elif sys.platform == 'darwin':

182 variables['GRAMEXDATA'] = os.path.expanduser(

183 '~/Library/Application Support/Gramex Data')

184 else:

185 variables['GRAMEXDATA'] = os.path.abspath('.')

186 app_log.warning('$GRAMEXDATA set to %s for OS %s', variables['GRAMEXDATA'],

187 sys.platform)

188

189 return variables

190

191

192 variables = setup_variables()

193

194

195 def _substitute_variable(val):

196 '''

197 If val contains a ${VAR} or $VAR and VAR is in the variables global,

198 substitute it.

199

200 Direct variables are substituted as-is. For example, $x will return

201 variables['x'] without converting it to a string. Otherwise, treat it as a

202 string tempate. So "/$x/" will return "/1/" if x=1.

203 '''

204 if not isinstance(val, string_types):

205 return val

206 if val.startswith('$') and val[1:] in variables:

207 return variables[val[1:]]

208 else:

209 try:

210 return string.Template(val).substitute(variables)

211 except ValueError:

212 raise ValueError('Use $$ instead of $ in %s' % val)

213

214

215 def _calc_value(val, key):

216 '''

217 Calculate the value to assign to this key.

218

219 If ``val`` is not a dictionary that has a ``function`` key, return it as-is.

220

221 If it has a function key, call that function (with specified args, kwargs,

222 etc) and allow the ``key`` parameter as an argument.

223

224 If the function is a generator, the first value is used.

225 '''

226 if hasattr(val, 'get') and val.get('function'):

227 from .transforms import build_transform

228 function = build_transform(val, vars={'key': None}, filename='config:%s' % key)

229 for result in function(key):

230 if result is not None:

231 return result

232 return val.get('default')

233 else:

234 return _substitute_variable(val)

235

236

237 _valid_key_chars = string.ascii_letters + string.digits

238

239

240 def random_string(size, chars=_valid_key_chars):

241 '''Return random string of length size using chars (which defaults to alphanumeric)'''

242 return ''.join(choice(chars) for index in range(size)) # nosec - ok for non-cryptographic use

243

244

245 RANDOM_KEY = r'$*'

246

247

248 def _from_yaml(loader, node):

249 '''

250 Load mapping as AttrDict, preserving order. Raise error on duplicate keys

251 '''

252 # Based on yaml.constructor.SafeConstructor.construct_mapping()

253 attrdict = AttrDict()

254 yield attrdict

255 if not isinstance(node, MappingNode): 255 ↛ 256line 255 didn't jump to line 256, because the condition on line 255 was never true

256 raise ConstructorError(

257 None, None, 'expected a mapping node, but found %s' % node.id, node.start_mark)

258 loader.flatten_mapping(node)

259 for key_node, value_node in node.value:

260 key = loader.construct_object(key_node, deep=False)

261 if isinstance(key, six.string_types) and RANDOM_KEY in key:

262 # With k=5 there's a <0.1% chance of collision even for 1mn uses.

263 # (1 - decimal.Decimal(62 ** -5)) ** 1000000 ~ 0.999

264 key = key.replace(RANDOM_KEY, random_string(5))

265 try:

266 hash(key)

267 except TypeError as exc:

268 raise ConstructorError(

269 'while constructing a mapping', node.start_mark,

270 'found unacceptable key (%s)' % exc, key_node.start_mark)

271 if key in attrdict:

272 raise ConstructorError(

273 'while constructing a mapping', node.start_mark,

274 'found duplicate key (%s)' % key, key_node.start_mark)

275 attrdict[key] = loader.construct_object(value_node, deep=False)

276

277

278 class ConfigYAMLLoader(Loader):

279 '''

280 A YAML loader that loads a YAML file into an ordered AttrDict. Usage::

281

282 >>> attrdict = yaml.load(yaml_string, Loader=ConfigYAMLLoader)

283

284 If there are duplicate keys, this raises an error.

285 '''

286 def __init__(self, *args, **kwargs):

287 super(ConfigYAMLLoader, self).__init__(*args, **kwargs)

288 self.add_constructor(u'tag:yaml.org,2002:map', _from_yaml)

289 self.add_constructor(u'tag:yaml.org,2002:omap', _from_yaml)

290

291

292 def _yaml_open(path, default=AttrDict(), **kwargs):

293 '''

294 Load a YAML path.Path as AttrDict. Replace ${VAR} or $VAR with variables.

295 Defines special variables $YAMLPATH as the absolute path of the YAML file,

296 and $YAMLURL as the path relative to current directory. These can be

297 overridden via keyward arguments (e.g. ``YAMLURL=...``)

298

299 If key has " if ", include it only if the condition (eval-ed in Python) is

300 true.

301

302 If the path is missing, or YAML has a parse error, or the YAML is not a

303 dict, returns the default value.

304 '''

305 path = path.absolute()

306 if not path.exists():

307 if path not in _warned_paths:

308 app_log.warning('Missing config: %s', path)

309 _warned_paths.add(path)

310 return default

311 app_log.debug('Loading config: %s', path)

312 with path.open(encoding='utf-8') as handle:

313 try:

314 result = yaml.load(handle, Loader=ConfigYAMLLoader) # nosec

315 except Exception:

316 app_log.exception('Config error: %s', path)

317 return default

318 if not isinstance(result, AttrDict):

319 if result is not None:

320 app_log.warning('Config is not a dict: %s', path)

321 return default

322

323 # Variables based on YAML file location

324 yaml_path = str(path.parent)

325 kwargs.setdefault('YAMLPATH', yaml_path) # Path to YAML folder

326 kwargs.setdefault('YAMLFILE', str(path)) # Path to YAML file

327 # $YAMLURL defaults to the relative URL from cwd to YAML folder.

328 try:

329 yamlurl = os.path.relpath(yaml_path)

330 except ValueError:

331 # If YAML is in a different drive, this fails. So don't set YAMLURL.

332 # Impact: $YAMLURL is undefined for imports from a different drive.

333 pass

334 else:

335 kwargs.setdefault('YAMLURL', yamlurl)

336 # Typically, we use /$YAMLURL/url - so strip the slashes. Replace backslashes

337 if isinstance(kwargs.get('YAMLURL'), string_types):

338 kwargs['YAMLURL'] = kwargs['YAMLURL'].replace('\\', '/').strip('/')

339 variables.update(kwargs)

340

341 # Update context with the variables section.

342 # key: value sets key = value

343 # key: {function: fn} sets key = fn(key)

344 # key: {default: value} sets key = value if it's not already set

345 # key: {default: {function: fn}} sets key = fn(key) if it's not already set

346 if 'variables' in result:

347 for key, val in result['variables'].items():

348 if hasattr(val, 'get') and 'default' in val and 'function' not in val:

349 variables.setdefault(key, _calc_value(val['default'], key))

350 else:

351 variables[key] = _calc_value(val, key)

352 del result['variables']

353

354 # Evaluate conditionals. "x if cond: y" becomes "x: y" if cond evals to True

355 remove, replace = [], []

356 frozen_vars = dict(variables)

357 for key, value, node in walk(result):

358 if isinstance(key, string_types) and ' if ' in key:

359 # Evaluate conditional

360 base, expr = key.split(' if ', 2)

361 try:

362 condition = eval(expr, globals(), frozen_vars) # nosec - any Python expr is OK

363 except Exception:

364 condition = False

365 app_log.exception('Failed condition evaluation: %s', key)

366 if condition:

367 replace.append((node, key, base))

368 else:

369 remove.append((node, key))

370 for node, key in remove:

371 del node[key]

372 for node, key, base in replace:

373 node[base] = node.pop(key)

374

375 # Substitute variables

376 for key, value, node in walk(result):

377 if isinstance(value, string_types):

378 # Backward compatibility: before v1.0.4, we used {.} for {YAMLPATH}

379 value = value.replace('{.}', '$YAMLPATH')

380 # Substitute with variables in context, defaulting to ''

381 node[key] = _substitute_variable(value)

382 return result

383

384

385 def _pathstat(path):

386 '''

387 Return a path stat object, which has 2 attributes/keys: ``.path`` is the

388 same as the ``path`` parameter. ``stat`` is the result of ``os.stat``. If

389 path is missing, ``stat`` has ``st_mtime`` and ``st_size`` set to ``0``.

390 '''

391 # If path doesn't exist, create a dummy stat structure with

392 # safe defaults (old mtime, 0 filesize, etc)

393 stat = path.stat() if path.exists() else AttrDict(st_mtime=0, st_size=0)

394 return AttrDict(path=path, stat=stat)

395

396

397 def _add_ns(config, namespace, prefix):

398 '''

399 Given a YAML config (basically a dict), add prefix to specified namespaces.

400

401 For example::

402

403 >>> _add_ns({'x': 1}, '*', 'a')

404 {'a.x': 1}

405 >>> _add_ns({'x': {'y': 1}}, ['*', 'x'], 'a')

406 {'a.x': {'a.y': 1}}

407 '''

408 if not isinstance(namespace, list):

409 namespace = [namespace]

410 # Sort in descending order of key depth. So "x.y" is before "x" is before "*"

411 namespace = sorted(namespace, key=lambda ns: -1 if ns == '*' else ns.count('.'), reverse=True)

412 prefix += ':'

413 for keypath in namespace:

414 if keypath == '*':

415 el = config

416 else:

417 el = objectpath(config, keypath, default={})

418 if isinstance(el, dict): 418 ↛ 413line 418 didn't jump to line 413, because the condition on line 418 was never false

419 for subkey in list(el.keys()):

420 if subkey not in {'import'}: 420 ↛ 419line 420 didn't jump to line 419, because the condition on line 420 was never false

421 el[prefix + subkey] = el.pop(subkey)

422 return config

423

424

425 def load_imports(config, source, warn=None):

426 '''

427 Post-process a config for imports.

428

429 ``config`` is the data to process. ``source`` is the path where it was

430 loaded from.

431

432 If ``config`` has an ``import:`` key, treat all values below that as YAML

433 files (specified relative to ``source``) and import them in sequence.

434

435 Return a list of imported paths as :func:_pathstat objects. (This includes

436 ``source``.)

437

438 For example, if the ``source`` is ``base.yaml`` (which has the below

439 configuration) and is loaded into ``config``::

440

441 app:

442 port: 20

443 start: true

444 path: /

445 import: update*.yaml # Can be any glob, e.g. */gramex.yaml

446

447 ... and ``update.yaml`` looks like this::

448

449 app:

450 port: 30

451 new: yes

452

453 ... then after this function is called, ``config`` looks like this::

454

455 app:

456 port: 20 # From base.yaml. NOT updated by update.yaml

457 start: true # From base.yaml

458 new: yes # From update.yaml

459 path: / # From base.yaml

460

461 The ``import:`` keys are deleted. The return value contains :func:_pathstat

462 values for ``base.yaml`` and ``update.yaml`` in that order.

463

464 Multiple ``import:`` values can be specified as a dictionary::

465

466 import:

467 first-app: app1/*.yaml

468 next-app: app2/*.yaml

469

470 To import sub-keys as namespaces, use::

471

472 import:

473 app: {path: */gramex.yaml, namespace: 'url'}

474

475 This prefixes all keys under ``url:``. Here are more examples::

476

477 namespace: True # Add namespace to all top-level keys

478 namespace: url # Add namespace to url.*

479 namespace: log.loggers # Add namespace to log.loggers.*

480 namespace: [True, url] # Add namespace to top level keys and url.*

481

482 By default, the prefix is the relative path of the imported YAML file

483 (relative to the importer).

484

485 ``warn=`` is an optional list of key paths. Any conflict on dictionaries

486 matching any of these paths is logged as a warning. For example,

487 ``warn=['url.*', 'watch.*']`` warns if any url: sub-key or watch: sub-key

488 has a conflict.

489 '''

490 imported_paths = [_pathstat(source)]

491 root = source.absolute().parent

492 for key, value, node in list(walk(config)):

493 if isinstance(key, six.string_types) and key.startswith('import.merge'):

494 # Strip the top level key(s) from import.merge values

495 if isinstance(value, dict):

496 for name, conf in value.items():

497 node[name] = conf

498 elif value: 498 ↛ 499line 498 didn't jump to line 499, because the condition on line 498 was never true

499 raise ValueError('import.merge: must be dict, not %s at %s' % (

500 repr(value), source))

501 # Delete the import key

502 del node[key]

503 elif key == 'import':

504 # Convert "import: path" to "import: {app: path}"

505 if isinstance(value, six.string_types):

506 value = {'apps': value}

507 # Allow "import: [path, path]" to "import: {app0: path, app1: path}"

508 elif isinstance(value, list):

509 value = OrderedDict((('app%d' % i, conf) for i, conf in enumerate(value)))

510 # By now, import: should be a dict

511 elif not isinstance(value, dict): 511 ↛ 512line 511 didn't jump to line 512, because the condition on line 511 was never true

512 raise ValueError('import: must be string/list/dict, not %s at %s' % (

513 repr(value), source))

514 # If already a dict with a single import via 'path', convert to dict of apps

515 if 'path' in value:

516 value = {'app': value}

517 for name, conf in value.items():

518 if not isinstance(conf, dict):

519 conf = AttrDict(path=conf)

520 if 'path' not in conf: 520 ↛ 521line 520 didn't jump to line 521, because the condition on line 520 was never true

521 raise ValueError('import: has no conf at %s' % source)

522 paths = conf.pop('path')

523 paths = paths if isinstance(paths, list) else [paths]

524 globbed_paths = []

525 for path in paths:

526 globbed_paths += sorted(root.glob(path)) if '*' in path else [Path(path)]

527 ns = conf.pop('namespace', None)

528 for path in globbed_paths:

529 abspath = root.joinpath(path)

530 new_conf = _yaml_open(abspath, **conf)

531 if ns is not None:

532 prefix = Path(path).as_posix()

533 new_conf = _add_ns(new_conf, ns, name + ':' + prefix)

534 imported_paths += load_imports(new_conf, source=abspath)

535 merge(old=node, new=new_conf, mode='setdefault', warn=warn)

536 # Delete the import key

537 del node[key]

538 return imported_paths

539

540

541 class PathConfig(AttrDict):

542 '''

543 An ``AttrDict`` that is loaded from a path as a YAML file. For e.g.,

544 ``conf = PathConfig(path)`` loads the YAML file at ``path`` as an AttrDict.

545 ``+conf`` reloads the path if required.

546

547 ``warn=`` is an optional list of key paths. Any conflict on dictionaries

548 matching any of these paths is logged as a warning. For example,

549 ``warn=['url.*', 'watch.*']`` warns if any url: sub-key or watch: sub-key

550 has a conflict.

551

552 Like http://configure.readthedocs.org/ but supports imports not inheritance.

553 This lets us import YAML files in the middle of a YAML structure::

554

555 key:

556 import:

557 conf1: file1.yaml # Import file1.yaml here

558 conf2: file2.yaml # Import file2.yaml here

559

560 Each ``PathConfig`` object has an ``__info__`` attribute with the following

561 keys:

562

563 __info__.path

564 The path that this instance syncs with, stored as a ``pathlib.Path``

565 __info__.warn

566 The keys to warn in case about in case of an import merge conflict

567 __info__.imports

568 A list of imported files, stored as an ``AttrDict`` with 2 attributes:

569

570 path

571 The path that was imported, stored as a ``pathlib.Path``

572 stat

573 The ``os.stat()`` information about this file (or ``None`` if the

574 file is missing.)

575 '''

576 duplicate_warn = None

577

578 def __init__(self, path, warn=None):

579 super(PathConfig, self).__init__()

580 if warn is None: 580 ↛ 582line 580 didn't jump to line 582, because the condition on line 580 was never false

581 warn = self.duplicate_warn

582 self.__info__ = AttrDict(path=Path(path), imports=[], warn=warn)

583 self.__pos__()

584

585 def __pos__(self):

586 '''+config reloads this config (if it has a path)'''

587 path = self.__info__.path

588

589 # We must reload the layer if nothing has been imported...

590 reload = not self.__info__.imports

591 # ... or if an imported file is deleted / updated

592 for imp in self.__info__.imports:

593 exists = imp.path.exists()

594 if not exists and imp.stat is not None:

595 reload = True

596 app_log.info('No config found: %s', imp.path)

597 break

598 if exists and (imp.path.stat().st_mtime > imp.stat.st_mtime or

599 imp.path.stat().st_size != imp.stat.st_size):

600 reload = True

601 app_log.info('Updated config: %s', imp.path)

602 break

603 if reload:

604 self.clear()

605 self.update(_yaml_open(path))

606 self.__info__.imports = load_imports(self, source=path, warn=self.__info__.warn)

607 return self

608

609

610 def locate(path, modules=[], forceload=0):

611 '''

612 Locate an object by name or dotted path.

613

614 For example, ``locate('str')`` returns the ``str`` built-in.

615 ``locate('gramex.handlers.FileHandler')`` returns the class

616 ``gramex.handlers.FileHandler``.

617

618 ``modules`` is a list of modules to search for the path in first. So

619 ``locate('FileHandler', modules=[gramex.handlers])`` will return

620 ``gramex.handlers.FileHandler``.

621

622 If importing raises an Exception, log it and return None.

623 '''

624 try:

625 for module_name in modules:

626 module = _locate(module_name, forceload)

627 if hasattr(module, path):

628 return getattr(module, path)

629 return _locate(path, forceload)

630 except ErrorDuringImport:

631 app_log.exception('Exception when importing %s', path)

632 return None

633

634

635 _checked_old_certs = []

636

637

638 class CustomJSONEncoder(JSONEncoder):

639 '''

640 Encodes object to JSON, additionally converting datetime into ISO 8601 format

641 '''

642 def default(self, obj):

643 import numpy as np

644

645 if hasattr(obj, 'to_dict'):

646 # Slow but reliable. Handles conversion of numpy objects, mixed types, etc.

647 return loads(obj.to_json(orient='records', date_format='iso'),

648 object_pairs_hook=OrderedDict)

649 elif isinstance(obj, datetime.datetime):

650 # Use local timezone if no timezone is specified

651 if obj.tzinfo is None:

652 obj = obj.replace(tzinfo=dateutil.tz.tzlocal())

653 return obj.isoformat()

654 elif isinstance(obj, np.datetime64):

655 obj = obj.item()

656 if (isinstance(obj, datetime.datetime) and obj.tzinfo is None):

657 obj = obj.replace(tzinfo=dateutil.tz.tzlocal())

658 return obj.isoformat()

659 elif isinstance(obj, np.integer):

660 return int(obj)

661 elif isinstance(obj, np.floating):

662 return float(obj)

663 elif isinstance(obj, np.ndarray):

664 return obj.tolist()

665 elif isinstance(obj, np.bool_):

666 return bool(obj)

667 elif isinstance(obj, np.bytes_): 667 ↛ 669line 667 didn't jump to line 669, because the condition on line 667 was never false

668 return obj.decode('utf-8')

669 return super(CustomJSONEncoder, self).default(obj)

670

671

672 class CustomJSONDecoder(JSONDecoder):

673 '''

674 Decodes JSON string, converting ISO 8601 datetime to datetime

675 '''

676 # Check if a string might be a datetime. Handles variants like:

677 # 2001-02-03T04:05:06Z

678 # 2001-02-03T04:05:06+000

679 # 2001-02-03T04:05:06.000+0000

680 re_datetimeval = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}')

681 re_datetimestr = re.compile(r'"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}')

682

683 def __init__(self, *args, **kwargs):

684 self.old_object_pairs_hook = kwargs.get('object_pairs_hook')

685 kwargs['object_pairs_hook'] = self.convert

686 super(CustomJSONDecoder, self).__init__(*args, **kwargs)

687

688 def decode(self, obj):

689 if self.re_datetimestr.match(obj):

690 return dateutil.parser.parse(obj[1:-1])

691 return super(CustomJSONDecoder, self).decode(obj)

692

693 def convert(self, obj):

694 for index, (key, val) in enumerate(obj):

695 if isinstance(val, six.string_types) and self.re_datetimeval.match(val):

696 obj[index] = (key, dateutil.parser.parse(val))

697 if callable(self.old_object_pairs_hook):

698 return self.old_object_pairs_hook(obj)

699 return dict(obj)

700

701

702 def check_old_certs():

703 '''

704 The latest SSL certificates from certifi don't work for Google Auth. Do

705 a one-time check to access accounts.google.com. If it throws an SSL

706 error, switch to old SSL certificates. See

707 https://github.com/tornadoweb/tornado/issues/1534

708 '''

709 if not _checked_old_certs:

710 _checked_old_certs.append(True)

711

712 import ssl

713 from tornado.httpclient import HTTPClient, AsyncHTTPClient

714

715 # Use HTTPClient to check instead of AsyncHTTPClient because it's synchronous.

716 _client = HTTPClient()

717 try:

718 # Use accounts.google.com because we know it fails with new certifi certificates

719 # cdn.redhat.com is another site that fails.

720 _client.fetch("https://accounts.google.com/")

721 except ssl.SSLError:

722 try:

723 import certifi # noqa: late import to minimise dependencies

724 AsyncHTTPClient.configure(None, defaults=dict(ca_certs=certifi.old_where()))

725 app_log.warning('Using old SSL certificates for compatibility')

726 except ImportError:

727 pass

728 try:

729 _client.fetch("https://accounts.google.com/")

730 except ssl.SSLError:

731 app_log.error('Gramex cannot connect to HTTPS sites. Auth may fail')

732 except Exception:

733 # Ignore any other kind of exception

734 app_log.warning('Gramex has no direct Internet connection')

735 _client.close()

736

737

738 def objectpath(node, keypath, default=None):

739 '''

740 Traverse down a dot-separated object path into dict items or object attrs.

741 For example, ``objectpath(handler, 'request.headers.User-Agent')`` returns

742 ``handler.request.headers['User-Agent']``. Dictionary access is preferred.

743 Returns ``None`` if the path is not found.

744 '''

745 for key in keypath.split('.'):

746 if hasattr(node, '__getitem__'): 746 ↛ 749line 746 didn't jump to line 749, because the condition on line 746 was never false

747 node = node.get(key)

748 else:

749 node = getattr(node, key, None)

750 if node is None:

751 return default

752 return node

753

754

755 def recursive_encode(data, encoding='utf-8'):

756 '''

757 Convert all Unicode values into UTF-8 encoded byte strings in-place

758 '''

759 for key, value, node in walk(data):

760 if isinstance(key, six.text_type):

761 newkey = key.encode(encoding)

762 node[newkey] = node.pop(key)

763 key = newkey

764 if isinstance(value, six.text_type):

765 node[key] = value.encode(encoding)

766

767

768 class TimedRotatingCSVHandler(logging.handlers.TimedRotatingFileHandler):

769 '''

770 Same as logging.handlers.TimedRotatingFileHandler, but writes to a CSV.

771 The constructor accepts an additional ``keys`` list as input that has

772 column keys. When ``.emit()`` is called, it expects an object with the

773 same keys as ``keys``.

774 '''

775 def __init__(self, *args, **kwargs):

776 self.keys = kwargs.pop('keys')

777 super(TimedRotatingCSVHandler, self).__init__(*args, **kwargs)

778

779 def _open(self):

780 stream = super(TimedRotatingCSVHandler, self)._open()

781 self.writer = csv.DictWriter(stream, fieldnames=self.keys, lineterminator='\n')

782 return stream

783

784 def emit(self, record):

785 try:

786 # From logging.handlers.BaseRotatingHandler

787 if self.shouldRollover(record):

788 self.doRollover()

789 # From logging.handlers.StreamHandler

790 if self.stream is None:

791 self.stream = self._open()

792 except (KeyboardInterrupt, SystemExit):

793 raise

794 except Exception as e:

795 # On Windows, multiple processes cannot rotate the same file.

796 # Ignore this, and just re-open the stream.

797 # On Linux, this needs to be tested.

798 if e.errno == EPERM or e.errno == EACCES:

799 if getattr(e, 'winerror', None) == ERROR_SHARING_VIOLATION:

800 self.stream = self._open()

801 else:

802 return self.handleError(record)

803 try:

804 # Write the CSV record instead of the formatted record

805 self.writer.writerow(record.msg)

806 self.stream.flush()

807 except Exception:

808 self.handleError(record)

809

810

811 def ioloop_running(loop):

812 '''Returns whether the Tornado ioloop is running on not'''

813 # Python 2.7 and Tornado < 5.0 use this

814 if hasattr(loop, '_running'): 814 ↛ 815line 814 didn't jump to line 815, because the condition on line 814 was never true

815 return loop._running

816 # Python 3 on Tornado >= 5.0 delegates to asyncio

817 if hasattr(loop, 'asyncio_loop'): 817 ↛ 819line 817 didn't jump to line 819, because the condition on line 817 was never false

818 return loop.asyncio_loop.is_running()

819 raise NotImplementedError('Cannot determine tornado.ioloop is running')

820

821

822 def used_kwargs(method, kwargs, ignore_keywords=False):

823 '''

824 Splits kwargs into those used by method, and those that are not.

825

826 Returns a tuple of (used, rest). *used* is a dict subset of kwargs with only

827 keys used by method. *rest* has the remaining kwargs keys.

828

829 If the method uses ``**kwargs`` (keywords), it uses all keys. To ignore this

830 and return only named arguments, use ``ignore_keywords=True``.

831 '''

832 argspec = inspect.getargspec(method)

833 # If method uses **kwargs, return all kwargs (unless you ignore **kwargs)

834 if argspec.keywords and not ignore_keywords:

835 used, rest = kwargs, {}

836 else:

837 # Split kwargs into 2 dicts -- used and rest

838 used, rest = {}, {}

839 for key, val in kwargs.items():

840 target = used if key in set(argspec.args) else rest

841 target[key] = val

842 return used, rest