Coverage for gramex\apps\nlg\nlgapp.py: 16%

# Coverage for gramex\apps\nlg\nlgapp.py : 16%

## 148 statements 28 run 120 missing 0 excluded 1 partial

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1# vim:fenc=utf-8

2

3"""

4 Module for gramex exposure. This shouldn't be imported anywhere, only for use

5 with gramex.

6"""

7 import json

8 import os

9 import os.path as op

10 from six.moves.urllib import parse

11

12 import pandas as pd

13 from tornado.template import Template

14

15 from gramex.config import variables

16 from gramex.apps.nlg import grammar

17 from gramex.apps.nlg import templatize

18 from gramex.apps.nlg import nlgutils as utils

19

20 DATAFILE_EXTS = {'.csv', '.xls', '.xlsx', '.tsv'}

21

22 grx_data_dir = variables['GRAMEXDATA']

23 nlg_path = op.join(grx_data_dir, 'nlg')

24

25 if not op.isdir(nlg_path): 25 ↛ 26line 25 didn't jump to line 26, because the condition on line 25 was never true

26 os.mkdir(nlg_path)

27

28

29 def render_live_template(handler):

30 """Given a narrative ID and df records, render the template."""

31 orgdf = get_original_df(handler)

32 nrid = handler.args['nrid'][0]

33 if not nrid.endswith('.json'):

34 nrid += '.json'

35 data = json.loads(handler.args['data'][0])

36 df = pd.DataFrame.from_records(data)

37 nrpath = op.join(nlg_path, handler.current_user.email, nrid)

38 with open(nrpath, 'r') as fout: # noqa: No encoding for json

39 templates = json.load(fout)

40 narratives = []

41 style = json.loads(handler.args['style'][0])

42 for t in templates['config']:

43 tmpl = utils.add_html_styling(t['template'], style)

44 s = Template(tmpl).generate(df=df, fh_args=t.get('fh_args', {}),

45 G=grammar, U=utils, orgdf=orgdf)

46 rendered = s.decode('utf8')

47 narratives.append(rendered)

48 return '\n'.join(narratives)

49

50

51 def get_original_df(handler):

52 """Get the original dataframe which was uploaded to the webapp."""

53 data_dir = op.join(nlg_path, handler.current_user.email)

54 with open(op.join(data_dir, 'meta.cfg'), 'r') as fout: # noqa: No encoding for json

55 meta = json.load(fout)

56 dataset_path = op.join(data_dir, meta['dsid'])

57 return pd.read_csv(dataset_path, encoding='utf-8')

58

59

60 def render_template(handler):

61 """Render a set of templates against a dataframe and formhandler actions on it."""

62 orgdf = get_original_df(handler)

63 payload = parse.parse_qsl(handler.request.body.decode("utf8"))

64 if not payload:

65 payload = json.loads(handler.request.body.decode("utf8"))

66 fh_args = payload['args']

67 templates = payload['template']

68 df = pd.DataFrame.from_records(payload['data'])

69 else:

70 payload = dict(payload)

71 fh_args = json.loads(payload.get("args", {}))

72 templates = json.loads(payload["template"])

73 df = pd.read_json(payload["data"], orient="records")

74 # fh_args = {k: [x.lstrip('-') for x in v] for k, v in fh_args.items()}

75 resp = []

76 for t in templates:

77 rendered = Template(t).generate(

78 orgdf=orgdf, df=df, fh_args=fh_args, G=grammar, U=utils).decode('utf8')

79 rendered = rendered.replace('-', '')

80 # grmerr = utils.check_grammar(rendered)

81 resp.append({'text': rendered}) # , 'grmerr': grmerr})

82 return json.dumps(resp)

83

84

85 def process_template(handler):

86 """Process English text in the context of a df and formhandler arguments

87 to templatize it."""

88 payload = parse.parse_qsl(handler.request.body.decode("utf8"))

89 payload = dict(payload)

90 text = json.loads(payload["text"])

91 df = pd.read_json(payload["data"], orient="records")

92 args = json.loads(payload.get("args", {}))

93 if args is None:

94 args = {}

95 resp = []

96 for t in text:

97 # grammar_errors = yield utils.check_grammar(t)

98 replacements, t, infl = templatize(t, args.copy(), df)

99 resp.append({

100 "text": t, "tokenmap": replacements, 'inflections': infl,

101 "fh_args": args, "setFHArgs": False,

102 # "grmerr": json.loads(grammar_errors.decode('utf8'))['matches']

103 })

104 return json.dumps(resp)

105

106

107 def read_current_config(handler):

108 """Read the current data and narrative IDs written to the session file."""

109 meta_path = op.join(nlg_path, handler.current_user.email, 'meta.cfg')

110 with open(meta_path, 'r') as fout: # noqa: No encoding for json

111 meta = json.load(fout)

112 return meta

113

114

115 def get_dataset_files(handler):

116 """Get all filenames uploaded by the user.

117

118 Parameters

119 ----------

120 handler : tornado.RequestHandler

121

122 Returns

123 -------

124 list

125 List of filenames.

126 """

127

128 user_dir = op.join(nlg_path, handler.current_user.email)

129 if op.isdir(user_dir):

130 files = [f for f in os.listdir(user_dir) if op.splitext(f)[-1].lower() in DATAFILE_EXTS]

131 else:

132 files = []

133 return files

134

135

136 def get_narrative_config_files(handler):

137 """Get list of narrative config files generated by the user.

138

139 Parameters

140 ----------

141 handler : tornado.RequestHandler

142

143 Returns

144 -------

145 list

146 List of narrative configurations.

147 """

148 # TODO: current_user.email needs to be replaced with a more suitable user ID

149 user_dir = op.join(nlg_path, handler.current_user.email)

150 if op.isdir(user_dir):

151 return [f for f in os.listdir(user_dir) if f.endswith('.json')]

152 return []

153

154

155 def download_config(handler):

156 """Download the current narrative config as JSON."""

157 payload = {}

158 payload['config'] = json.loads(parse.unquote(handler.args['config'][0]))

159 payload['data'] = json.loads(parse.unquote(handler.args.get('data', [None])[0]))

160 payload['name'] = parse.unquote(handler.args['name'][0])

161 return json.dumps(payload, indent=4)

162

163

164 def save_config(handler):

165 """Save the current narrative config.

166 (to $GRAMEXDATA/{{ handler.current_user.email }})"""

167 payload = {}

168 payload['config'] = json.loads(parse.unquote(handler.args['config'][0]))

169 payload['name'] = parse.unquote(handler.args['name'][0])

170 nname = payload['name']

171 if not nname.endswith('.json'):

172 nname += '.json'

173 payload['dataset'] = parse.unquote(handler.args['dataset'][0])

174 fpath = op.join(nlg_path, handler.current_user.email, nname)

175 with open(fpath, 'w') as fout: # noqa: No encoding for json

176 json.dump(payload, fout, indent=4)

177

178

179 def get_gramopts(handler):

180 """Find all Grammar and token inflection options from the NLG library.

181

182 Primarily used for creating the select box in the template settings dialog."""

183 funcs = {}

184 for attrname in dir(grammar):

185 obj = getattr(grammar, attrname)

186 if getattr(obj, 'gramopt', False):

187 funcs[obj.fe_name] = {'source': obj.source, 'func_name': attrname}

188 return funcs

189

190

191 def init_form(handler):

192 """Process input from the landing page and write the current session config."""

193 meta = {}

194 # prioritize files first

195 data_dir = op.join(nlg_path, handler.current_user.email)

196 if not op.isdir(data_dir):

197 os.makedirs(data_dir)

198

199 # handle dataset

200 data_file = handler.request.files.get('data-file', [{}])[0]

201 if data_file:

202 # TODO: Unix filenames may not be valid Windows filenames.

203 outpath = op.join(data_dir, data_file['filename'])

204 with open(outpath, 'wb') as fout:

205 fout.write(data_file['body'])

206 else:

207 dataset = handler.args['dataset'][0]

208 outpath = op.join(data_dir, dataset)

209 # shutil.copy(outpath, fh_fpath)

210 meta['dsid'] = op.basename(outpath)

211

212 # handle config

213 config_name = handler.get_argument('narrative', '')

214 if config_name:

215 config_path = op.join(data_dir, config_name)

216 # shutil.copy(config_path, op.join(local_data_dir, 'config.json'))

217 meta['nrid'] = op.basename(config_path)

218

219 # write meta config

220 with open(op.join(data_dir, 'meta.cfg'), 'w') as fout: # NOQA: no encoding for JSON

221 json.dump(meta, fout, indent=4)

222

223

224 def edit_narrative(handler):

225 """Set the handler's narrative and dataset ID to the current session."""

226 user_dir = op.join(nlg_path, handler.current_user.email)

227 dataset_name = handler.args.get('dsid', [''])[0]

228 narrative_name = handler.args.get('nrid', [''])[0] + '.json'

229 with open(op.join(user_dir, 'meta.cfg'), 'w') as fout: # NOQA: no encoding for JSON

230 json.dump({'dsid': dataset_name, 'nrid': narrative_name}, fout, indent=4)

231

232

233 def get_init_config(handler):

234 """Get the initial default configuration for the current user."""

235 user_dir = op.join(nlg_path, handler.current_user.email)

236 metapath = op.join(user_dir, 'meta.cfg')

237 if op.isfile(metapath):

238 with open(metapath, 'r') as fout: # NOQA: no encoding for JSON

239 meta = json.load(fout)

240 config_file = op.join(user_dir, meta.get('nrid', ''))

241 if op.isfile(config_file):

242 with open(config_file, 'r') as fout: # NOQA: no encoding for JSON

243 meta['config'] = json.load(fout)

244 return meta

245 return {}