Coverage for gramex\handlers\formhandler.py: 91%

# Coverage for gramex\handlers\formhandler.py : 91%

## 172 statements 160 run 12 missing 0 excluded 12 partial

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1 import six

2 import json

3 import tornado.gen

4 import gramex.cache

5 import gramex.data

6 import pandas as pd

7 from orderedattrdict import AttrDict

8 from tornado.web import HTTPError

9 from gramex import conf as gramex_conf

10 from gramex.http import BAD_REQUEST, INTERNAL_SERVER_ERROR

11 from gramex.transforms import build_transform

12 from gramex.config import merge, app_log, objectpath, CustomJSONEncoder

13 from .basehandler import BaseHandler

14

15

16 def namespaced_args(args, namespace):

17 '''

18 Filter from handler.args the keys relevant for namespace, i.e. those

19 prefixed with the namespace, or those without a prefix.

20 '''

21 if not namespace.endswith(':'): 21 ↛ 23line 21 didn't jump to line 23, because the condition on line 21 was never false

22 namespace += ':'

23 result = {}

24 for key, val in args.items():

25 if key.startswith(namespace): 25 ↛ 26line 25 didn't jump to line 26, because the condition on line 25 was never true

26 result[key[len(namespace):]] = val

27 elif ':' not in key: 27 ↛ 24line 27 didn't jump to line 24, because the condition on line 27 was never false

28 result[key] = val

29 return result

30

31

32 class FormHandler(BaseHandler):

33 # Else there should be at least 1 key that has a url: sub-key. The data spec is at that level

34 # Data spec is (url, engine, table, ext, ...) which goes directly to filter

35 # It also has

36 # default: which is interpreted as argument defaults

37 # keys: defines the primary key columns

38

39 # FormHandler function kwargs and the parameters they accept:

40 function_vars = {

41 'modify': {'data': None, 'key': None, 'handler': None},

42 'prepare': {'args': None, 'key': None, 'handler': None},

43 'queryfunction': {'args': None, 'key': None, 'handler': None},

44 }

45 data_filter_method = staticmethod(gramex.data.filter)

46

47 @classmethod

48 def setup(cls, **kwargs):

49 super(FormHandler, cls).setup(**kwargs)

50 conf_kwargs = merge(AttrDict(kwargs),

51 objectpath(gramex_conf, 'handlers.FormHandler', {}),

52 'setdefault')

53 cls.headers = conf_kwargs.pop('headers', {})

54 # Top level formats: key is special. Don't treat it as data

55 cls.formats = conf_kwargs.pop('formats', {})

56 default_config = conf_kwargs.pop('default', None)

57 # Remove other known special keys from dataset configuration

58 cls.clear_special_keys(conf_kwargs)

59 # If top level has url: then data spec is at top level. Else it's a set of sub-keys

60 if 'url' in conf_kwargs:

61 cls.datasets = AttrDict(data=conf_kwargs)

62 cls.single = True

63 else:

64 if 'modify' in conf_kwargs:

65 cls.modify_all = staticmethod(build_transform(

66 conf={'function': conf_kwargs.pop('modify', None)},

67 vars=cls.function_vars['modify'],

68 filename='%s.%s' % (cls.name, 'modify'), iter=False))

69 cls.datasets = conf_kwargs

70 cls.single = False

71 # Apply defaults to each key

72 if isinstance(default_config, dict): 72 ↛ 77line 72 didn't jump to line 77, because the condition on line 72 was never false

73 for key in cls.datasets:

74 config = cls.datasets[key].get('default', {})

75 cls.datasets[key]['default'] = merge(config, default_config, mode='setdefault')

76 # Ensure that each dataset is a dict with a url: key at least

77 for key, dataset in list(cls.datasets.items()):

78 if not isinstance(dataset, dict): 78 ↛ 79line 78 didn't jump to line 79, because the condition on line 78 was never true

79 app_log.error('%s: %s: must be a dict, not %r' % (cls.name, key, dataset))

80 del cls.datasets[key]

81 elif 'url' not in dataset: 81 ↛ 82line 81 didn't jump to line 82, because the condition on line 81 was never true

82 app_log.error('%s: %s: does not have a url: key' % (cls.name, key))

83 del cls.datasets[key]

84 # Ensure that id: is a list -- if it exists

85 if 'id' in dataset and not isinstance(dataset['id'], list):

86 dataset['id'] = [dataset['id']]

87 # Convert function: into a data = transform(data) function

88 conf = {

89 'function': dataset.pop('function', None),

90 'args': dataset.pop('args', None),

91 'kwargs': dataset.pop('kwargs', None)

92 }

93 if conf['function'] is not None:

94 fn_name = '%s.%s.transform' % (cls.name, key)

95 dataset['transform'] = build_transform(

96 conf, vars={'data': None, 'handler': None}, filename=fn_name, iter=False)

97 # Convert modify: and prepare: into a data = modify(data) function

98 for fn, fn_vars in cls.function_vars.items():

99 if fn in dataset:

100 dataset[fn] = build_transform(

101 conf={'function': dataset[fn]},

102 vars=fn_vars,

103 filename='%s.%s.%s' % (cls.name, key, fn), iter=False)

104

105 def _options(self, dataset, args, path_args, path_kwargs, key):

106 """For each dataset, prepare the arguments."""

107 if self.request.body:

108 content_type = self.request.headers.get('Content-Type', '')

109 if content_type == 'application/json':

110 args.update(json.loads(self.request.body))

111 filter_kwargs = AttrDict(dataset)

112 filter_kwargs.pop('modify', None)

113 prepare = filter_kwargs.pop('prepare', None)

114 queryfunction = filter_kwargs.pop('queryfunction', None)

115 filter_kwargs['transform_kwargs'] = {'handler': self}

116 # Use default arguments

117 defaults = {

118 k: v if isinstance(v, list) else [v]

119 for k, v in filter_kwargs.pop('default', {}).items()

120 }

121 # /(.*)/(.*) become 2 path arguments _0 and _1

122 defaults.update({'_%d' % k: [v] for k, v in enumerate(path_args)})

123 # /(?P<x>\d+)/(?P<y>\d+) become 2 keyword arguments x and y

124 defaults.update({k: [v] for k, v in path_kwargs.items()})

125 args = merge(namespaced_args(args, key), defaults, mode='setdefault')

126 if callable(prepare):

127 result = prepare(args=args, key=key, handler=self)

128 if result is not None: 128 ↛ 129line 128 didn't jump to line 129, because the condition on line 128 was never true

129 args = result

130 if callable(queryfunction):

131 filter_kwargs['query'] = queryfunction(args=args, key=key, handler=self)

132 return AttrDict(

133 fmt=args.pop('_format', ['json'])[0],

134 download=args.pop('_download', [''])[0],

135 args=args,

136 meta_header=args.pop('_meta', [''])[0],

137 filter_kwargs=filter_kwargs,

138 )

139

140 @tornado.gen.coroutine

141 def get(self, *path_args, **path_kwargs):

142 meta, futures = AttrDict(), AttrDict()

143 for key, dataset in self.datasets.items():

144 meta[key] = AttrDict()

145 opt = self._options(dataset, self.args, path_args, path_kwargs, key)

146 opt.filter_kwargs.pop('id', None)

147 # Run query in a separate threadthread

148 futures[key] = gramex.service.threadpool.submit(

149 self.data_filter_method, args=opt.args, meta=meta[key], **opt.filter_kwargs)

150 result = AttrDict()

151 for key, val in futures.items():

152 try:

153 result[key] = yield val

154 except ValueError as e:

155 app_log.exception('%s: filter failed' % self.name)

156 raise HTTPError(BAD_REQUEST, reason=e.args[0])

157 except Exception as e:

158 app_log.exception('%s: filter failed' % self.name)

159 raise HTTPError(INTERNAL_SERVER_ERROR, reason=repr(e))

160 modify = self.datasets[key].get('modify', None)

161 if callable(modify):

162 result[key] = modify(data=result[key], key=key, handler=self)

163

164 # modify the result for multiple datasets

165 if hasattr(self, 'modify_all'):

166 result = self.modify_all(data=result, key=None, handler=self)

167

168 format_options = self.set_format(opt.fmt, meta)

169 format_options['args'] = opt.args

170 params = {k: v[0] for k, v in opt.args.items() if len(v) > 0}

171 for key, val in format_options.items():

172 if isinstance(val, six.text_type):

173 format_options[key] = val.format(**params)

174 # In PY2, the values are binary. TODO: ensure that format values are in Unicode

175 elif isinstance(val, six.binary_type): 175 ↛ 176line 175 didn't jump to line 176, because the condition on line 175 was never true

176 format_options[key] = val.decode('utf-8').format(**params)

177 if opt.download:

178 self.set_header('Content-Disposition', 'attachment;filename=%s' % opt.download)

179 if opt.meta_header:

180 self.set_meta_headers(meta)

181 result = result['data'] if self.single else result

182 # If modify has changed the content type from a dataframe, write it as-is

183 if isinstance(result, (pd.DataFrame, dict)):

184 self.write(gramex.data.download(result, **format_options))

185 else:

186 self.write(result)

187

188 @tornado.gen.coroutine

189 def update(self, method, *path_args, **path_kwargs):

190 if self.redirects:

191 self.save_redirect_page()

192 meta, result = AttrDict(), AttrDict()

193 # For each dataset

194 for key, dataset in self.datasets.items():

195 meta[key] = AttrDict()

196 opt = self._options(dataset, self.args, path_args, path_kwargs, key)

197 if 'id' not in opt.filter_kwargs:

198 raise HTTPError(BAD_REQUEST, reason='%s: missing id: <col> for %s' % (

199 self.name, self.request.method))

200 missing_args = [col for col in opt.filter_kwargs['id'] if col not in opt.args]

201 if method != gramex.data.insert and len(missing_args) > 0:

202 raise HTTPError(BAD_REQUEST, reason='%s: missing column(s) in URL query: %s' % (

203 self.name, ', '.join(missing_args)))

204 # Execute the query. This returns the count of records updated

205 result[key] = method(meta=meta[key], args=opt.args, **opt.filter_kwargs)

206 for key, val in result.items():

207 modify = self.datasets[key].get('modify', None)

208 if callable(modify):

209 meta[key]['modify'] = modify(data=result[key], key=key, handler=self)

210 self.set_header('Count-%s' % key, val)

211 # modify the result for multiple datasets

212 if hasattr(self, 'modify_all'): 212 ↛ 213line 212 didn't jump to line 213, because the condition on line 212 was never true

213 meta['modify'] = self.modify_all(data=result, key=None, handler=self)

214 if opt.meta_header: 214 ↛ 215line 214 didn't jump to line 215, because the condition on line 214 was never true

215 self.set_meta_headers(meta)

216 if self.redirects:

217 self.redirect_next()

218 else:

219 self.set_format(opt.fmt, meta)

220 self.set_header('Cache-Control', 'no-cache, no-store')

221 self.write(json.dumps(meta, indent=2, cls=CustomJSONEncoder))

222

223 @tornado.gen.coroutine

224 def delete(self, *path_args, **path_kwargs):

225 yield self.update(gramex.data.delete, *path_args, **path_kwargs)

226

227 @tornado.gen.coroutine

228 def post(self, *path_args, **path_kwargs):

229 yield self.update(gramex.data.insert, *path_args, **path_kwargs)

230

231 @tornado.gen.coroutine

232 def put(self, *path_args, **path_kwargs):

233 yield self.update(gramex.data.update, *path_args, **path_kwargs)

234

235 def set_format(self, fmt, meta):

236 # Identify format to render in. The default format, json, is defined in

237 # the base gramex.yaml under handlers.FormHandler.formats

238 if fmt in self.formats: 238 ↛ 241line 238 didn't jump to line 241, because the condition on line 238 was never false

239 fmt = dict(self.formats[fmt])

240 else:

241 app_log.error('%s: _format=%s unknown. Using _format=json' % (self.name, fmt))

242 fmt = dict(self.formats['json'])

243

244 # Set up default headers, and over-ride with headers for the format

245 for key, val in self.headers.items():

246 self.set_header(key, val)

247 for key, val in fmt.pop('headers', {}).items():

248 self.set_header(key, val)

249

250 if fmt['format'] in {'template', 'pptx', 'vega', 'vega-lite', 'vegam'}:

251 fmt['handler'] = self

252 if fmt['format'] in {'template'}: 252 ↛ 253line 252 didn't jump to line 253, because the condition on line 252 was never true

253 fmt['meta'] = meta['data'] if self.single else meta

254

255 return fmt

256

257 def set_meta_headers(self, meta):

258 '''Add FH-<dataset>-<key>: JSON(value) for each key: value in meta'''

259 prefix = 'FH-{}-{}'

260 for dataset, metadata in meta.items():

261 for key, value in metadata.items():

262 string_value = json.dumps(value, separators=(',', ':'),

263 ensure_ascii=True, cls=CustomJSONEncoder)

264 self.set_header(prefix.format(dataset, key), string_value)