Coverage for gramex\handlers\basehandler.py: 80%

# Coverage for gramex\handlers\basehandler.py : 80%

## 515 statements 428 run 87 missing 0 excluded 29 partial

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1 import io

2 import os

3 import six

4 import time

5 import logging

6 import datetime

7 import mimetypes

8 import traceback

9 import tornado.gen

10 import gramex.cache

11 from binascii import b2a_base64, hexlify

12 from orderedattrdict import AttrDict

13 from six.moves.urllib_parse import urlparse, urlsplit, urljoin, urlencode

14 from tornado.web import RequestHandler, HTTPError, MissingArgumentError, decode_signed_value

15 from tornado.websocket import WebSocketHandler

16 from gramex import conf, __version__

17 from gramex.config import merge, objectpath, app_log

18 from gramex.transforms import build_transform, CacheLoader

19 from gramex.http import UNAUTHORIZED, FORBIDDEN, BAD_REQUEST

20 from gramex.cache import get_store

21# We don't use these, but these stores used to be defined here. Programs may import these

22 from gramex.cache import KeyStore, JSONStore, HDF5Store, SQLiteStore, RedisStore # noqa

23

24 server_header = 'Gramex/%s' % __version__

25 session_store_cache = {}

26 _missing = object()

27 _arg_default = object()

28

29

30 class BaseMixin(object):

31 @classmethod

32 def setup(cls, transform={}, redirect={}, auth=None, log=None, set_xsrf=None,

33 error=None, xsrf_cookies=None, **kwargs):

34 '''

35 One-time setup for all request handlers. This is called only when

36 gramex.yaml is parsed / changed.

37 '''

38 cls._on_init_methods = []

39 cls._on_finish_methods = []

40 cls._set_xsrf = set_xsrf

41

42 cls.kwargs = cls.conf.get('kwargs', AttrDict())

43

44 cls.setup_transform(transform)

45 cls.setup_redirect(redirect)

46 # Note: call setup_session before setup_auth to ensure that

47 # override_user is run before authorize

48 cls.setup_session(conf.app.get('session'))

49 cls.setup_auth(auth)

50 cls.setup_error(error)

51 cls.setup_xsrf(xsrf_cookies)

52 cls.setup_log()

53

54 # app.settings.debug enables debugging exceptions using pdb

55 if conf.app.settings.get('debug', False): 55 ↛ 56line 55 didn't jump to line 56, because the condition on line 55 was never true

56 cls.log_exception = cls.debug_exception

57

58 # A list of special keys for BaseHandler. Can be extended by other classes.

59 special_keys = ['transform', 'redirect', 'auth', 'log', 'set_xsrf',

60 'error', 'xsrf_cookies', 'headers']

61

62 @classmethod

63 def clear_special_keys(cls, kwargs, *args):

64 '''

65 Remove keys handled by BaseHandler that may interfere with setup().

66 This should be called explicitly in setup() where required.

67 '''

68 for special_key in cls.special_keys:

69 kwargs.pop(special_key, None)

70 for special_key in args:

71 kwargs.pop(special_key, None)

72 return kwargs

73

74 @classmethod

75 def setup_default_kwargs(cls):

76 '''

77 Use default config from handlers.<Class>.* and handlers.BaseHandler.

78 Called by gramex.services.url().

79 '''

80 c = cls.conf.setdefault('kwargs', {})

81 merge(c, objectpath(conf, 'handlers.' + cls.conf.handler, {}), mode='setdefault')

82 merge(c, objectpath(conf, 'handlers.BaseHandler', {}), mode='setdefault')

83

84 @classmethod

85 def setup_transform(cls, transform):

86 cls.transform = {}

87 for pattern, trans in transform.items():

88 cls.transform[pattern] = {

89 'function': build_transform(

90 trans, vars=AttrDict((('content', None), ('handler', None))),

91 filename='url:%s' % cls.name),

92 'headers': trans.get('headers', {}),

93 'encoding': trans.get('encoding'),

94 }

95

96 @staticmethod

97 def _purge_keys(data):

98 '''

99 Returns session keys to be deleted. These are either None values or

100 those with expired keys based on _t.

101 setup_session makes the session store call this method.

102 Until v1.20 (31 Jul 2017) no _t keys were set.

103 From v1.23 (31 Oct 2017) these are cleared.

104 '''

105 now = time.time()

106 week = 7 * 24 * 60 * 60

107 keys = []

108 # When using sqlitedict, fetching keys may fail if DB is locked. Try later

109 try:

110 items = list(data.items())

111 except Exception:

112 items = []

113 for key, val in items:

114 # Purge already cleared / removed sessions

115 if val is None:

116 keys.append(key)

117 elif isinstance(val, dict): 117 ↛ 127line 117 didn't jump to line 127, because the condition on line 117 was never false

118 # If the session has expired, remove it

119 if val.get('_t', 0) < now:

120 keys.append(key)

121 # If the session is inactive, remove it after a week.

122 # If we remove immediately, then we may lose WIP sessions.

123 # For example, people who opened a login page where _next_url was set

124 elif '_i' in val and '_l' in val and val['_i'] + val['_l'] < now - week: 124 ↛ 125line 124 didn't jump to line 125, because the condition on line 124 was never true

125 keys.append(key)

126 else:

127 app_log.warning('Store key: %s has value type %s (not dict)', key, type(val))

128 return keys

129

130 @classmethod

131 def setup_session(cls, session_conf):

132 '''handler.session returns the session object. It is saved on finish.'''

133 if session_conf is None: 133 ↛ 134line 133 didn't jump to line 134, because the condition on line 133 was never true

134 return

135 key = store_type, store_path = session_conf.get('type'), session_conf.get('path')

136 if key not in session_store_cache:

137 session_store_cache[key] = get_store(

138 type=store_type,

139 path=store_path,

140 flush=session_conf.get('flush'),

141 purge=session_conf.get('purge'),

142 purge_keys=cls._purge_keys

143 )

144 cls._session_store = session_store_cache[key]

145 cls.session = property(cls.get_session)

146 cls._session_expiry = session_conf.get('expiry')

147 cls._session_cookie = {

148 key: session_conf[key] for key in ('domain', 'httponly', 'secure')

149 if key in session_conf

150 }

151 cls._on_finish_methods.append(cls.save_session)

152 cls._on_init_methods.append(cls.override_user)

153 cls._on_finish_methods.append(cls.set_last_visited)

154

155 @classmethod

156 def setup_redirect(cls, redirect):

157 '''

158 Any handler can have a ``redirect:`` kwarg that looks like this::

159

160 redirect:

161 query: next # If the URL has a ?next=..., redirect to that page next

162 header: X-Next # Else if the header has an X-Next=... redirect to that

163 url: ... # Else redirect to this URL

164

165 Only these 3 keys are allowed. All are optional, and checked in the

166 order specified. So, for example::

167

168 redirect:

169 header: X-Next # Checks the X-Next header first

170 query: next # If it's missing, uses the ?next=

171

172 You can also specify a string for redirect. ``redirect: ...`` is the same

173 as ``redirect: {url: ...}``.

174

175 When any BaseHandler subclass calls ``self.save_redirect_page()``, it

176 stores the redirect URL in ``session['_next_url']``. The URL is

177 calculated relative to the handler's URL.

178

179 After that, when the subclass calls ``self.redirect_next()``, it

180 redirects to ``session['_next_url']`` and clears the value. (If the

181 ``_next_url`` was not stored, we redirect to the home page ``/``.)

182

183 Only some handlers implement redirection. But they all implement it in

184 this same consistent way.

185 '''

186 # Ensure that redirect is a dictionary before proceeding.

187 if isinstance(redirect, six.string_types): 187 ↛ 188line 187 didn't jump to line 188, because the condition on line 187 was never true

188 redirect = {'url': redirect}

189 if not isinstance(redirect, dict): 189 ↛ 190line 189 didn't jump to line 190, because the condition on line 189 was never true

190 app_log.error('url:%s.redirect must be a URL or a dict, not %s',

191 cls.name, repr(redirect))

192 return

193

194 cls.redirects = []

195 add = cls.redirects.append

196 for key, value in redirect.items():

197 if key == 'query':

198 add(lambda h, v=value: h.get_argument(v, None))

199 elif key == 'header':

200 add(lambda h, v=value: h.request.headers.get(v))

201 elif key == 'url': 201 ↛ 196line 201 didn't jump to line 196, because the condition on line 201 was never false

202 add(lambda h, v=value: v)

203

204 # redirect.external=False disallows external URLs

205 if not redirect.get('external', False): 205 ↛ exitline 205 didn't return from function 'setup_redirect', because the condition on line 205 was never false

206 def no_external(method):

207 def redirect_method(handler):

208 next_uri = method(handler)

209 if next_uri is not None:

210 target = urlparse(next_uri)

211 if not target.scheme and not target.netloc:

212 return next_uri

213 req = handler.request

214 if req.protocol == target.scheme and req.host == target.netloc:

215 return next_uri

216 app_log.error('Not redirecting to external url: %s', next_uri)

217 return redirect_method

218 cls.redirects = [no_external(method) for method in cls.redirects]

219

220 @classmethod

221 def setup_auth(cls, auth):

222 # auth: if there's no auth: in handler, default to app.auth

223 if auth is None:

224 auth = conf.app.get('auth')

225 # Treat True as an empty dict, i.e. auth: {}

226 if auth is True:

227 auth = AttrDict()

228 # Set up the auth

229 if isinstance(auth, dict):

230 cls._login_url = auth.get('login_url', None)

231 cls._on_init_methods.append(cls.authorize)

232 cls.permissions = []

233 # Add check for condition

234 if auth.get('condition'):

235 cls.permissions.append(

236 build_transform(auth['condition'], vars=AttrDict(handler=None),

237 filename='url:%s.auth.permission' % cls.name))

238 # Add check for membership

239 memberships = auth.get('membership', [])

240 if not isinstance(memberships, list):

241 memberships = [memberships]

242 if len(memberships):

243 cls.permissions.append(check_membership(memberships))

244 elif auth: 244 ↛ 245line 244 didn't jump to line 245, because the condition on line 244 was never true

245 app_log.error('url:%s.auth is not a dict', cls.name)

246

247 @classmethod

248 def setup_log(cls):

249 '''

250 Logs access requests to gramex.requests as a CSV file.

251 '''

252 logger = logging.getLogger('gramex.requests')

253 keys = objectpath(conf, 'log.handlers.requests.keys', [])

254 log_info = build_log_info(keys)

255 cls.log_request = lambda handler: logger.info(log_info(handler))

256

257 @classmethod

258 def _error_fn(cls, error_code, error_config):

259 template_kwargs = {}

260 if 'autoescape' in error_config:

261 if not error_config['autoescape']: 261 ↛ 264line 261 didn't jump to line 264, because the condition on line 261 was never false

262 template_kwargs['autoescape'] = None

263 else:

264 app_log.error('url:%s.error.%d.autoescape can only be false', cls.name, error_code)

265 if 'whitespace' in error_config:

266 template_kwargs['whitespace'] = error_config['whitespace']

267

268 def error(*args, **kwargs):

269 tmpl = gramex.cache.open(error_config['path'], 'template', **template_kwargs)

270 return tmpl.generate(*args, **kwargs)

271

272 return error

273

274 @classmethod

275 def setup_error(cls, error):

276 '''

277 Sample configuration::

278

279 error:

280 404:

281 path: template.json # Use a template

282 autoescape: false # with no autoescape

283 whitespace: single # as a single line

284 headers:

285 Content-Type: application/json

286 500:

287 function: module.fn

288 args: [=status_code, =kwargs, =handler]

289 '''

290 if not error:

291 return

292 if not isinstance(error, dict): 292 ↛ 293line 292 didn't jump to line 293, because the condition on line 292 was never true

293 return app_log.error('url:%s.error is not a dict', cls.name)

294 # Compile all errors handlers

295 cls.error = {}

296 for error_code, error_config in error.items():

297 try:

298 error_code = int(error_code)

299 if error_code < 100 or error_code > 1000: 299 ↛ 300line 299 didn't jump to line 300, because the condition on line 299 was never true

300 raise ValueError()

301 except ValueError:

302 app_log.error('url.%s.error code %s is not a number (100 - 1000)',

303 cls.name, error_code)

304 continue

305 if not isinstance(error_config, dict): 305 ↛ 306line 305 didn't jump to line 306, because the condition on line 305 was never true

306 return app_log.error('url:%s.error.%d is not a dict', cls.name, error_code)

307 # Make a copy of the original. When we add headers, etc, it shouldn't affect original

308 error_config = AttrDict(error_config)

309 error_path, error_function = error_config.get('path'), error_config.get('function')

310 if error_function:

311 if error_path: 311 ↛ 315line 311 didn't jump to line 315, because the condition on line 311 was never false

312 error_config.pop('path')

313 app_log.warning('url.%s.error.%d has function: AND path:. Ignoring path:',

314 cls.name, error_code)

315 cls.error[error_code] = {'function': build_transform(

316 error_config,

317 vars=AttrDict((('status_code', None), ('kwargs', None), ('handler', None))),

318 filename='url:%s.error.%d' % (cls.name, error_code)

319 )}

320 elif error_path: 320 ↛ 327line 320 didn't jump to line 327, because the condition on line 320 was never false

321 encoding = error_config.get('encoding', 'utf-8')

322 cls.error[error_code] = {'function': cls._error_fn(error_code, error_config)}

323 mime_type, encoding = mimetypes.guess_type(error_path, strict=False)

324 if mime_type: 324 ↛ 330line 324 didn't jump to line 330, because the condition on line 324 was never false

325 error_config.setdefault('headers', {}).setdefault('Content-Type', mime_type)

326 else:

327 app_log.error('url.%s.error.%d must have a path or function key',

328 cls.name, error_code)

329 # Add the error configuration for reference

330 if error_code in cls.error: 330 ↛ 296line 330 didn't jump to line 296, because the condition on line 330 was never false

331 cls.error[error_code]['conf'] = error_config

332 cls._write_error, cls.write_error = cls.write_error, cls._write_custom_error

333

334 @classmethod

335 def setup_xsrf(cls, xsrf_cookies):

336 '''

337 Sample configuration::

338

339 xsrf_cookies: false # Disables xsrf_cookies

340 xsrf_cookies: true # or anything other than false keeps it enabled

341 '''

342 cls.check_xsrf_cookie = cls.noop if xsrf_cookies is False else cls.xsrf_ajax

343

344 def xsrf_ajax(self):

345 '''

346 TODO: explain things clearly.

347 Same as Tornado's check_xsrf_cookie() -- but is ignored for AJAX requests

348 '''

349 ajax = self.request.headers.get('X-Requested-With', '').lower() == 'xmlhttprequest'

350 if not ajax:

351 return super(BaseHandler, self).check_xsrf_cookie()

352

353 def noop(self):

354 '''Does nothing. Used when overriding functions or providing a dummy operation'''

355 pass

356

357 def save_redirect_page(self):

358 '''

359 Loop through all redirect: methods and save the first available redirect

360 page against the session. Defaults to previously set value, else ``/``.

361

362 See :py:func:`setup_redirect`

363 '''

364 for method in self.redirects:

365 next_url = method(self)

366 if next_url:

367 self.session['_next_url'] = urljoin(self.request.uri, next_url)

368 return

369 self.session.setdefault('_next_url', '/')

370

371 def redirect_next(self):

372 '''

373 Redirect the user ``session['_next_url']``. If it does not exist,

374 set it up first. Then redirect.

375

376 See :py:func:`setup_redirect`

377 '''

378 if '_next_url' not in self.session:

379 self.save_redirect_page()

380 self.redirect(self.session.pop('_next_url', '/'))

381

382 @tornado.gen.coroutine

383 def _cached_get(self, *args, **kwargs):

384 cached = self.cachefile.get()

385 if cached is not None:

386 self.set_status(cached['status'])

387 self._write_headers(cached['headers'])

388 self.write(cached['body'])

389 else:

390 self.cachefile.wrap(self)

391 yield self.original_get(*args, **kwargs)

392

393 def _write_headers(self, headers):

394 '''Write headers from a list of pairs that may be duplicated'''

395 headers_written = set()

396 for name, value in headers:

397 # If value is explicitly False or None, clear header.

398 # This gives a way to clear pre-set headers like the Server header

399 if value is False or value is None: 399 ↛ 400line 399 didn't jump to line 400, because the condition on line 399 was never true

400 self.clear_header(name)

401 elif name in headers_written: 401 ↛ 402line 401 didn't jump to line 402, because the condition on line 401 was never true

402 self.add_header(name, value)

403 else:

404 self.set_header(name, value)

405 headers_written.add(name)

406

407 def debug_exception(self, typ, value, tb):

408 super(BaseHandler, self).log_exception(typ, value, tb)

409 import ipdb as pdb # noqa

410 pdb.post_mortem(tb)

411

412 def _write_custom_error(self, status_code, **kwargs):

413 if status_code in self.error:

414 try:

415 result = self.error[status_code]['function'](

416 status_code=status_code, kwargs=kwargs, handler=self)

417 headers = self.error[status_code].get('conf', {}).get('headers', {})

418 self._write_headers(headers.items())

419 # result may be a generator / list from build_transform,

420 # or a str/bytes/unicode from Template.generate. Handle both

421 if isinstance(result, (six.string_types, six.binary_type)):

422 self.write(result)

423 else:

424 for item in result:

425 self.write(item)

426 return

427 except Exception:

428 app_log.exception('url:%s.error.%d error handler raised an exception:',

429 self.name, status_code)

430 # If error was not written, use the default error

431 self._write_error(status_code, **kwargs)

432

433 @property

434 def session(self):

435 '''

436 By default, session is not implemented. You need to specify a

437 ``session:`` section in ``gramex.yaml`` to activate it. It is replaced by

438 the ``get_session`` method as a property.

439 '''

440 raise NotImplementedError('Specify a session: section in gramex.yaml')

441

442 def _set_new_session_id(self, expires_days):

443 '''Sets a new random session ID as the sid: cookie. Returns a bytes object'''

444 session_id = b2a_base64(os.urandom(24))[:-1]

445 kwargs = dict(self._session_cookie)

446 kwargs['expires_days'] = expires_days

447 # Use Secure cookies on HTTPS to prevent leakage into HTTP

448 if self.request.protocol == 'https': 448 ↛ 449line 448 didn't jump to line 449, because the condition on line 448 was never true

449 kwargs['secure'] = True

450 # Websockets cannot set cookies. They raise a RuntimeError. Ignore those.

451 try:

452 self.set_secure_cookie('sid', session_id, **kwargs)

453 except RuntimeError:

454 pass

455 return session_id

456

457 def get_session(self, expires_days=None, new=False):

458 '''

459 Return the session object for the cookie "sid" value.

460 If no "sid" cookie exists, set up a new one.

461 If no session object exists for the sid, create it.

462 By default, the session object contains a "id" holding the "sid" value.

463

464 The session is a dict. You must ensure that it is JSON serializable.

465

466 Sessions use these pre-defined timing keys (values are timestamps):

467

468 - ``_t`` is the expiry time of the session

469 - ``_l`` is the last time the user accessed a page. Updated by

470 :py:func:`BaseHandler.set_last_visited`

471 - ``_i`` is the inactive expiry duration in seconds, i.e. if ``now > _l +

472 _i``, the session has expired.

473

474 ``new=`` creates a new session to avoid session fixation.

475 https://www.owasp.org/index.php/Session_fixation.

476 :py:func:`gramex.handlers.authhandler.AuthHandler.set_user` uses it.

477 When the user logs in:

478

479 - If no old session exists, it returns a new session object.

480 - If an old session exists, it creates a new "sid" and new session

481 object, copying all old contents, but updates the "id" and expiry (_t).

482 '''

483 if expires_days is None:

484 expires_days = self._session_expiry

485 created_new_sid = False

486 if getattr(self, '_session', None) is None:

487 # Populate self._session based on the sid. If there's no sid cookie,

488 # generate one and create an associated session object

489 session_id = self.get_secure_cookie('sid', max_age_days=9999999)

490 # If there's no session id cookie "sid", create a random 32-char cookie

491 if session_id is None:

492 session_id = self._set_new_session_id(expires_days)

493 created_new_sid = True

494 # Convert bytes session to unicode before using

495 session_id = session_id.decode('ascii')

496 # If there's no stored session associated with it, create it

497 expires = time.time() + expires_days * 24 * 60 * 60

498 self._session = self._session_store.load(session_id, {'_t': expires})

499 # Overwrite id to the session ID even if a handler has changed it

500 self._session['id'] = session_id

501 # At this point, the "sid" cookie and self._session exist and are synced

502 s = self._session

503 old_sid = s['id']

504 # If session has expiry keys _i and _l defined, check for expiry. Not otherwise

505 if '_i' in s and '_l' in s and time.time() > s['_l'] + s['_i']:

506 new = True

507 s.clear()

508 if new and not created_new_sid:

509 new_sid = self._set_new_session_id(expires_days).decode('ascii')

510 # Update expiry and new SID on session

511 s.update(id=new_sid, _t=time.time() + expires_days * 24 * 60 * 60)

512 # Delete old contents. No _t also means expired

513 self._session_store.dump(old_sid, {})

514

515 return s

516

517 def save_session(self):

518 '''Persist the session object as a JSON'''

519 if getattr(self, '_session', None) is not None: 519 ↛ exitline 519 didn't return from function 'save_session', because the condition on line 519 was never false

520 self._session_store.dump(self._session['id'], self._session)

521

522 def otp(self, expire=60):

523 '''

524 Return a one-time password valid for ``expire`` seconds. When the

525 X-Gramex-OTP header

526 '''

527 user = self.current_user

528 if not user: 528 ↛ 529line 528 didn't jump to line 529, because the condition on line 528 was never true

529 raise HTTPError(UNAUTHORIZED)

530 nbits = 16

531 otp = hexlify(os.urandom(nbits)).decode('ascii')

532 self._session_store.dump('otp:' + otp, {'user': user, '_t': time.time() + expire})

533 return otp

534

535 def override_user(self):

536 '''

537 Use ``X-Gramex-User`` HTTP header to override current user for the session.

538 Use ``X-Gramex-OTP`` HTTP header to set user based on OTP.

539 ``?gramex-otp=`` is a synonym for X-Gramex-OTP.

540 '''

541 headers = self.request.headers

542 cipher = headers.get('X-Gramex-User')

543 if cipher:

544 import json

545 try:

546 user = json.loads(decode_signed_value(

547 conf.app.settings['cookie_secret'], 'user', cipher,

548 max_age_days=self._session_expiry))

549 except Exception:

550 reason = '%s: invalid X-Gramex-User: %s' % (self.name, cipher)

551 raise HTTPError(BAD_REQUEST, reason=reason)

552 else:

553 app_log.debug('%s: Overriding user to %r', self.name, user)

554 self.session['user'] = user

555 return

556 otp = headers.get('X-Gramex-OTP') or self.get_argument('gramex-otp', None)

557 if otp:

558 otp_data = self._session_store.load('otp:' + otp, None)

559 if not isinstance(otp_data, dict) or '_t' not in otp_data or 'user' not in otp_data:

560 reason = '%s: invalid X-Gramex-OTP: %s' % (self.name, otp)

561 raise HTTPError(BAD_REQUEST, reason=reason)

562 elif otp_data['_t'] < time.time():

563 reason = '%s: expired X-Gramex-OTP: %s' % (self.name, otp)

564 raise HTTPError(BAD_REQUEST, reason=reason)

565 self._session_store.dump('otp:' + otp, None)

566 self.session['user'] = otp_data['user']

567

568 def set_last_visited(self):

569 '''

570 This method is called by :py:func:`BaseHandler.prepare` when any user

571 accesses a page. It updates the last visited time in the ``_l`` session

572 key. It does this only if the ``_i`` key exists.

573 '''

574 # For efficiency reasons, don't call get_session every time. Check

575 # session only if there's a valid sid cookie (with possibly long expiry)

576 if self.get_secure_cookie('sid', max_age_days=9999999):

577 session = self.get_session()

578 if '_i' in session:

579 session['_l'] = time.time()

580

581

582 class BaseHandler(RequestHandler, BaseMixin):

583 '''

584 BaseHandler provides auth, caching and other services common to all request

585 handlers. All RequestHandlers must inherit from BaseHandler.

586 '''

587 def initialize(self, **kwargs):

588 # self.request.arguments does not handle unicode keys well.

589 # In Py2, it returns a str (not unicode). In Py3, it returns latin-1 unicode.

590 # Convert this to proper unicode using UTF-8 and store in self.args

591 self.args = {}

592 for k in self.request.arguments:

593 key = (k if isinstance(k, six.binary_type) else k.encode('latin-1')).decode('utf-8')

594 # Invalid unicode (e.g. ?x=%f4) throws HTTPError. This disrupts even

595 # error handlers. So if there's invalid unicode, log & continue.

596 try:

597 self.args[key] = self.get_arguments(k)

598 except HTTPError:

599 app_log.exception('Invalid URL argument %s' % k)

600

601 self._session, self._session_json = None, 'null'

602 if self.cache:

603 self.cachefile = self.cache()

604 self.original_get = self.get

605 self.get = self._cached_get

606 if self._set_xsrf:

607 self.xsrf_token

608

609 # Set the method to the ?x-http-method-overrride argument or the

610 # X-HTTP-Method-Override header if they exist

611 if 'x-http-method-override' in self.args: 611 ↛ 612line 611 didn't jump to line 612, because the condition on line 611 was never true

612 self.request.method = self.args.pop('x-http-method-override')[0].upper()

613 elif 'X-HTTP-Method-Override' in self.request.headers: 613 ↛ 614line 613 didn't jump to line 614, because the condition on line 613 was never true

614 self.request.method = self.request.headers['X-HTTP-Method-Override'].upper()

615

616 def get_arg(self, name, default=_arg_default, first=False):

617 '''

618 Returns the value of the argument with the given name. Similar to

619 ``.get_argument`` but uses ``self.args`` instead.

620

621 If default is not provided, the argument is considered to be

622 required, and we raise a `MissingArgumentError` if it is missing.

623

624 If the argument is repeated, we return the last value. If ``first=True``

625 is passed, we return the first value.

626

627 ``self.args`` is always UTF-8 decoded unicode. Whitespaces are stripped.

628 '''

629 if name not in self.args:

630 if default is _arg_default:

631 raise MissingArgumentError(name)

632 return default

633 return self.args[name][0 if first else -1]

634

635 def prepare(self):

636 for method in self._on_init_methods:

637 method(self)

638

639 def set_default_headers(self):

640 # Only set BaseHandler headers.

641 # Don't set headers for the specific class. Those are overrides handled

642 # by the respective classes, not the default headers.

643 headers = [('Server', server_header)]

644 headers += list(objectpath(conf, 'handlers.BaseHandler.headers', {}).items())

645 self._write_headers(headers)

646

647 def on_finish(self):

648 # Loop through class-level callbacks

649 for callback in self._on_finish_methods:

650 callback(self)

651

652 def get_current_user(self):

653 '''Return the ``user`` key from the session as an AttrDict if it exists.'''

654 result = self.session.get('user')

655 return AttrDict(result) if isinstance(result, dict) else result

656

657 def log_exception(self, typ, value, tb):

658 '''Store the exception value for logging'''

659 super(BaseHandler, self).log_exception(typ, value, tb)

660 # _exception is stored for use by log_request. Sample error string:

661 # ZeroDivisionError: integer division or modulo by zero

662 self._exception = traceback.format_exception_only(typ, value)[0].strip()

663

664 def authorize(self):

665 if not self.current_user:

666 # Redirect non-AJAX requests GET/HEAD to login URL (if it's a string)

667 ajax = self.request.headers.get('X-Requested-With', '').lower() == 'xmlhttprequest'

668 if self.request.method in ('GET', 'HEAD') and not ajax:

669 url = self.get_login_url() if self._login_url is None else self._login_url

670 # If login_url is a string, redirect

671 if isinstance(url, six.string_types):

672 if '?' not in url:

673 if urlsplit(url).scheme:

674 # if login url is absolute, make next absolute too

675 next_url = self.request.full_url()

676 else:

677 next_url = self.request.uri

678 next_key = 'next'

679 if isinstance(self.conf.kwargs.auth, dict):

680 next_key = self.conf.kwargs.auth.get('query', 'next')

681 url += '?' + urlencode({next_key: next_url})

682 self.redirect(url)

683 return

684 # Else, send a 401 header

685 raise HTTPError(UNAUTHORIZED)

686

687 # If the user doesn't have permissions, show 403 (with template)

688 for permit_generator in self.permissions:

689 for result in permit_generator(self):

690 if not result:

691 template = self.conf.kwargs.auth.get('template')

692 if template:

693 self.set_status(FORBIDDEN)

694 self.render(template)

695 raise HTTPError(FORBIDDEN)

696

697 def argparse(self, *args, **kwargs):

698 '''

699 Parse URL query parameters and return an AttrDict. For example::

700

701 args = handler.argparse('x', 'y')

702 args.x # is the last value of ?x=value

703 args.y # is the last value of ?y=value

704

705 A missing ``?x=`` or ``?y=`` raises a HTTP 400 error mentioning the

706 missing key.

707

708 For optional arguments, use::

709

710 args = handler.argparse(z={'default': ''})

711 args.z # returns '' if ?z= is missing

712

713 You can convert the value to a type::

714

715 args = handler.argparse(limit={'type': int, 'default': 100})

716 args.limit # returns ?limit= as an integer

717

718 You can restrict the choice of values. If the query parameter is not in

719 choices, we raise a HTTP 400 error mentioning the invalid key & value::

720

721 args = handler.argparse(gender={'choices': ['M', 'F']})

722 args.gender # returns ?gender= which will be 'M' or 'F'

723

724 You can retrieve multiple values as a list::

725

726 args = handler.argparse(cols={'nargs': '*', 'default': []})

727 args.cols # returns an array with all ?col= values

728

729 ``type:`` conversion and ``choices:`` apply to each value in the list.

730

731 To return all arguments as a list, pass ``list`` as the first parameter::

732

733 args = handler.argparse(list, 'x', 'y')

734 args.x # ?x=1 sets args.x to ['1'], not '1'

735 args.y # Similarly for ?y=1

736

737 To handle unicode arguments and return all arguments as ``str`` or

738 ``unicode`` or ``bytes``, pass the type as the first parameter::

739

740 args = handler.argparse(str, 'x', 'y')

741 args = handler.argparse(bytes, 'x', 'y')

742 args = handler.argparse(unicode, 'x', 'y')

743

744 By default, all arguments are added as str in PY3 and unicode in PY2.

745

746 There are the full list of parameters you can pass to each keyword

747 argument:

748

749 - name: Name of the URL query parameter to read. Defaults to the key

750 - required: Whether or not the query parameter may be omitted

751 - default: The value produced if the argument is missing. Implies required=False

752 - nargs: The number of parameters that should be returned. '*' or '+'

753 return all values as a list.

754 - type: Python type to which the parameter should be converted (e.g. `int`)

755 - choices: A container of the allowable values for the argument (after type conversion)

756

757 You can combine all these options. For example::

758

759 args = handler.argparse(

760 'name', # Raise error if ?name= is missing

761 department={'name': 'dept'}, # ?dept= is mapped to args.department

762 org={'default': 'Gramener'}, # If ?org= is missing, defaults to Gramener

763 age={'type': int}, # Convert ?age= to an integer

764 married={'type': bool}, # Convert ?married to a boolean

765 alias={'nargs': '*'}, # Convert all ?alias= to a list

766 gender={'choices': ['M', 'F']}, # Raise error if gender is not M or F

767 )

768 '''

769 result = AttrDict()

770

771 args_type = six.text_type

772 if len(args) > 0 and args[0] in (six.text_type, six.binary_type, list, None):

773 args_type, args = args[0], args[1:]

774

775 for key in args:

776 result[key] = self.get_argument(key, None)

777 if result[key] is None:

778 raise HTTPError(BAD_REQUEST, reason='%s: missing ?%s=' % (key, key))

779 for key, config in kwargs.items():

780 name = config.get('name', key)

781 val = self.args.get(name, [])

782

783 # default: set if query is missing

784 # required: check if query is defined at all

785 if len(val) == 0:

786 if 'default' in config:

787 result[key] = config['default']

788 continue

789 if config.get('required', False):

790 raise HTTPError(BAD_REQUEST, reason='%s: missing ?%s=' % (key, name))

791

792 # nargs: select the subset of items

793 nargs = config.get('nargs', None)

794 if isinstance(nargs, int):

795 val = val[:nargs]

796 if len(val) < nargs:

797 val += [''] * (nargs - len(val))

798 elif nargs not in ('*', '+', None): 798 ↛ 799line 798 didn't jump to line 799, because the condition on line 798 was never true

799 raise ValueError('%s: invalid nargs %s' % (key, nargs))

800

801 # convert to specified type

802 newtype = config.get('type', None)

803 if newtype is not None:

804 newval = []

805 for v in val:

806 try:

807 newval.append(newtype(v))

808 except ValueError:

809 reason = "%s: type error ?%s=%s to %r" % (key, name, v, newtype)

810 raise HTTPError(BAD_REQUEST, reason=reason)

811 val = newval

812

813 # choices: check valid items

814 choices = config.get('choices', None)

815 if isinstance(choices, (list, dict, set)):

816 choices = set(choices)

817 for v in val:

818 if v not in choices:

819 reason = '%s: invalid choice ?%s=%s' % (key, name, v)

820 raise HTTPError(BAD_REQUEST, reason=reason)

821

822 # Set the final value

823 if nargs is None:

824 if len(val) > 0:

825 result[key] = val[-1]

826 else:

827 result[key] = val

828

829 # Parse remaining keys

830 if args_type is list:

831 for key, val in self.args.items():

832 if key not in args and key not in kwargs: 832 ↛ 831line 832 didn't jump to line 831, because the condition on line 832 was never false

833 result[key] = val

834 elif args_type in (six.string_types, six.binary_type): 834 ↛ 835line 834 didn't jump to line 835, because the condition on line 834 was never true

835 for key, val in self.args.items():

836 if key not in args and key not in kwargs:

837 result[key] = args_type(val[0])

838

839 return result

840

841 def create_template_loader(self, template_path):

842 settings = self.application.settings

843 return CacheLoader(template_path, autoescape=settings['autoescape'],

844 whitespace=settings.get('template_whitespace', None))

845

846

847 class BaseWebSocketHandler(WebSocketHandler, BaseMixin):

848 def initialize(self, **kwargs):

849 self._session, self._session_json = None, 'null'

850 if self.cache:

851 self.cachefile = self.cache()

852 self.original_get = self.get

853 self.get = self._cached_get

854 if self._set_xsrf:

855 self.xsrf_token

856

857 @tornado.gen.coroutine

858 def get(self, *args, **kwargs):

859 for method in self._on_init_methods:

860 method(self)

861 super(BaseWebSocketHandler, self).get(*args, **kwargs)

862

863 def on_close(self):

864 # Loop through class-level callbacks

865 for callback in self._on_finish_methods:

866 callback(self)

867

868 def get_current_user(self):

869 '''Return the ``user`` key from the session as an AttrDict if it exists.'''

870 result = self.session.get('user')

871 return AttrDict(result) if isinstance(result, dict) else result

872

873 def authorize(self):

874 '''If a valid user isn't logged in, send a message and close connection'''

875 if not self.current_user:

876 raise HTTPError(UNAUTHORIZED)

877 for permit_generator in self.permissions:

878 for result in permit_generator(self):

879 if not result:

880 raise HTTPError(FORBIDDEN)

881

882

883 class SetupFailedHandler(RequestHandler, BaseMixin):

884 '''

885 Reports that the setup() operation has failed.

886

887 Used by gramex.services.init() when setting up URLs. If it's not able to set

888 up a handler, it replaces it with this handler.

889 '''

890 def get(self):

891 six.reraise(*self.exc_info)

892

893

894 def check_membership(memberships):

895 '''

896 Return a generator that checks all memberships for a user, and yields True if

897 any membership is allowed, else False

898 '''

899 # Pre-process memberships into an array of {objectpath: set(values)}

900 conds = [{

901 keypath: set(values) if isinstance(values, list) else {values}

902 for keypath, values in cond.items()

903 } for cond in memberships]

904

905 def allowed(self):

906 user = self.current_user

907 for cond in conds:

908 if _check_condition(cond, user):

909 yield True

910 break

911 else:

912 yield False

913

914 return allowed

915

916

917 def _check_condition(condition, user):

918 '''

919 A condition is a dictionary of {keypath: values}. Extract the keypath from

920 the user. Check if the value is in the values list. If not, this condition

921 fails.

922 '''

923 for keypath, values in condition.items():

924 node = objectpath(user, keypath)

925 # If nothing exists at keypath, the check fails

926 if node is None:

927 return False

928 # If the value is a list, it must overlap with values

929 elif isinstance(node, list):

930 if not set(node) & values:

931 return False

932 # If the value is not a list, it must be present in values

933 elif node not in values:

934 return False

935 return True

936

937

938 handle_cache = {}

939

940

941 def _handle(path):

942 '''Returns a cached append-binary handle to path'''

943 if path not in handle_cache:

944 # In Python 2, csv writerow writes byte string. In PY3, it's to a unicode string.

945 # Open file handles accordingly

946 handle_cache[path] = open(path, 'ab') if six.PY2 else io.open(path, 'a', encoding='utf-8')

947 return handle_cache[path]

948

949

950 def build_log_info(keys, *vars):

951 '''

952 Creates a ``handler.method(vars)`` that returns a dictionary of computed

953 values. ``keys`` defines what keys are returned in the dictionary. The values

954 are computed using the formulas in the code.

955 '''

956 # Define direct keys. These can be used as-is

957 direct_vars = {

958 'name': 'handler.name',

959 'class': 'handler.__class__.__name__',

960 'time': 'round(time.time() * 1000, 0)',

961 'datetime': 'datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%SZ")',

962 'method': 'handler.request.method',

963 'uri': 'handler.request.uri',

964 'ip': 'handler.request.remote_ip',

965 'status': 'handler.get_status()',

966 'duration': 'round(handler.request.request_time() * 1000, 0)',

967 'port': 'conf.app.listen.port',

968 # TODO: get_content_size() is not available in RequestHandler

969 # 'size': 'handler.get_content_size()',

970 'user': '(handler.current_user or {}).get("id", "")',

971 'session': 'handler.session.get("id", "")',

972 'error': 'getattr(handler, "_exception", "")',

973 }

974 # Define object keys for us as key.value. E.g. cookies.sid, user.email, etc

975 object_vars = {

976 'args': 'handler.get_argument("{val}", "")',

977 'request': 'getattr(handler.request, "{val}", "")',

978 'headers': 'handler.request.headers.get("{val}", "")',

979 'cookies': 'handler.request.cookies["{val}"].value ' +

980 'if "{val}" in handler.request.cookies else ""',

981 'user': '(handler.current_user or {{}}).get("{val}", "")',

982 'env': 'os.environ.get("{val}", "")',

983 }

984 vals = []

985 for key in keys:

986 if key in vars:

987 vals.append('"{}": {},'.format(key, key))

988 continue

989 if key in direct_vars:

990 vals.append('"{}": {},'.format(key, direct_vars[key]))

991 continue

992 if '.' in key: 992 ↛ 997line 992 didn't jump to line 997, because the condition on line 992 was never false

993 prefix, value = key.split('.', 2)

994 if prefix in object_vars: 994 ↛ 997line 994 didn't jump to line 997, because the condition on line 994 was never false

995 vals.append('"{}": {},'.format(key, object_vars[prefix].format(val=value)))

996 continue

997 app_log.error('Skipping unknown key %s', key)

998 code = compile('def fn(handler, %s):\n\treturn {%s}' % (', '.join(vars), ' '.join(vals)),

999 filename='log', mode='exec')

1000 context = {'os': os, 'time': time, 'datetime': datetime, 'conf': conf, 'AttrDict': AttrDict}

1001 # The code is constructed entirely by this function. Using exec is safe

1002 exec(code, context) # nosec

1003 return context['fn']