Coverage for gramex\__init__.py: 79%

# Coverage for gramex\__init__.py : 79%

## 142 statements 115 run 27 missing 0 excluded 7 partial

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1'''

2 Gramex {__version__} Copyright (c) 2017 by Gramener

3

4 Start the Gramex server on port 9988 at the current directory.

5 If no gramex.yaml exists, show the guide (https://learn.gramener.com/guide/)

6

7 Options

8 --listen.port=9090 Starts Gramex at port 9090

9 --browser Open the browser after startup

10 --settings.debug Enable serving tracebacks and autoreload

11 --settings.xsrf_cookies=false Disable XSRF cookies (only for testing)

12 --settings.cookie_secret=... Change cookie encryption key

13

14 Helper applications

15 gramex init Add Gramex project scaffolding to current dir

16 gramex service Windows service setup

17 gramex mail Send email from command line

18 gramex license See Gramex license, accept or reject it

19

20 Installation commands. Run without arguments to see help

21 gramex install Install an app

22 gramex update Update an app

23 gramex setup Run make, npm install, bower install etc on app

24 gramex run Run an installed app

25 gramex uninstall Uninstall an app

26'''

27

28 import os

29 import sys

30 import json

31 import yaml

32 import logging

33 import logging.config

34 import tornado.ioloop

35 from pathlib import Path

36 from copy import deepcopy

37 from orderedattrdict import AttrDict

38 from gramex.config import ChainConfig, PathConfig, app_log, variables, setup_variables

39 from gramex.config import ioloop_running

40

41 paths = AttrDict() # Paths where configurations are stored

42 conf = AttrDict() # Final merged configurations

43 config_layers = ChainConfig() # Loads all configurations. init() updates it

44

45 paths['source'] = Path(__file__).absolute().parent # Where gramex source code is

46 paths['base'] = Path('.') # Where gramex is run from

47

48 callbacks = {} # Services callbacks

49

50# Populate __version__ from release.json

51 with (paths['source'] / 'release.json').open() as _release_file:

52 release = json.load(_release_file, object_pairs_hook=AttrDict)

53 __version__ = release.version

54

55 _sys_path = list(sys.path) # Preserve original sys.path

56

57

58# List of URLs to warn about in case of duplicates

59 PathConfig.duplicate_warn = [

60 'url.*',

61 'cache.*',

62 'schedule.*',

63 'watch.*',

64 'email.*',

65 'alert.*',

66 'sms.*',

67 'log.loggers.*', 'log.handlers.*', 'log.formatters.*',

68]

69

70

71 def parse_command_line(commands):

72 '''

73 Parse command line arguments. For example:

74

75 gramex cmd1 cmd2 --a=1 2 -b x --c --p.q=4

76

77 returns:

78

79 {"_": ["cmd1", "cmd2"], "a": [1, 2], "b": "x", "c": True, "p": {"q": [4]}}

80

81 Values are parsed as YAML. Arguments with '.' are split into subgroups. For

82 example, ``gramex --listen.port 80`` returns ``{"listen": {"port": 80}}``.

83 '''

84 group = '_'

85 args = AttrDict({group: []})

86 for arg in commands:

87 if arg.startswith('-'):

88 group, value = arg.lstrip('-'), 'True'

89 if '=' in group:

90 group, value = group.split('=', 1)

91 else:

92 value = arg

93

94 value = yaml.safe_load(value)

95 base = args

96 keys = group.split('.')

97 for key in keys[:-1]:

98 base = base.setdefault(key, AttrDict())

99

100 # Add the key to the base.

101 # If it's already there, make it a list.

102 # If it's already a list, append to it.

103 if keys[-1] not in base or base[keys[-1]] is True:

104 base[keys[-1]] = value

105 elif not isinstance(base[keys[-1]], list):

106 base[keys[-1]] = [base[keys[-1]], value]

107 else:

108 base[keys[-1]].append(value)

109

110 return args

111

112

113 def callback_commandline(commands):

114 '''

115 Find what method should be run based on the command line programs. This

116 refactoring allows us to test gramex.commandline() to see if it processes

117 the command line correctly, without actually running the commands.

118

119 Returns a callback method and kwargs for the callback method.

120 '''

121 # Set logging config at startup. (Services may override this.)

122 log_config = (+PathConfig(paths['source'] / 'gramex.yaml')).get('log', AttrDict())

123 log_config.root.level = logging.INFO

124 from . import services

125 services.log(log_config)

126

127 # args has all optional command line args as a dict of values / lists.

128 # cmd has all positional arguments as a list.

129 args = parse_command_line(commands)

130 cmd = args.pop('_')

131

132 # If --help or -V --version is specified, print a message and end

133 if args.get('V') is True or args.get('version') is True:

134 return console, {'msg': 'Gramex %s' % __version__}

135 if args.get('help') is True:

136 return console, {'msg': __doc__.strip().format(**globals())}

137

138 # Any positional argument is treated as a gramex command

139 if len(cmd) > 0:

140 kwargs = {'cmd': cmd, 'args': args}

141 base_command = cmd.pop(0).lower()

142 method = 'install' if base_command == 'update' else base_command

143 if method in {

144 'install', 'uninstall', 'setup', 'run', 'service', 'init',

145 'mail', 'license',

146 }:

147 import gramex.install

148 return getattr(gramex.install, method), kwargs

149 raise NotImplementedError('Unknown gramex command: %s' % base_command)

150

151 # Use current dir as base (where gramex is run from) if there's a gramex.yaml.

152 if not os.path.isfile('gramex.yaml'):

153 return console, {'msg': 'No gramex.yaml. See https://learn.gramener.com/guide/'}

154

155 app_log.info('Gramex %s | %s | Python %s', __version__, os.getcwd(),

156 sys.version.replace('\n', ' '))

157 return init, {'cmd': AttrDict(app=args)}

158

159

160 def commandline(args=None):

161 '''

162 Run Gramex from the command line. Called via:

163

164 - setup.py console_scripts when running gramex

165 - __main__.py when running python -m gramex

166 '''

167 callback, kwargs = callback_commandline(sys.argv[1:] if args is None else args)

168 callback(**kwargs)

169

170

171 def gramex_update(url):

172 '''If a newer version of gramex is available, logs a warning'''

173 import time

174 import requests

175 import platform

176 from . import services

177

178 if not services.info.eventlog: 178 ↛ 179line 178 didn't jump to line 179, because the condition on line 178 was never true

179 return app_log.error('eventlog: service is not running. So Gramex update is disabled')

180

181 query = services.info.eventlog.query

182 update = query('SELECT * FROM events WHERE event="update" ORDER BY time DESC LIMIT 1')

183 delay = 24 * 60 * 60 # Wait for one day before updates

184 if update and time.time() < update[0]['time'] + delay:

185 return app_log.debug('Gramex update ran recently. Deferring check.')

186

187 meta = {

188 'dir': variables.get('GRAMEXDATA'),

189 'uname': platform.uname(),

190 }

191 if update:

192 events = query('SELECT * FROM events WHERE time > ? ORDER BY time',

193 (update[0]['time'], ))

194 else:

195 events = query('SELECT * FROM events')

196 logs = [dict(log, **meta) for log in events]

197

198 r = requests.post(url, data=json.dumps(logs))

199 r.raise_for_status()

200 update = r.json()

201 version = update['version']

202 if version > __version__: 202 ↛ 203line 202 didn't jump to line 203, because the condition on line 202 was never true

203 app_log.error('Gramex %s is available. See https://learn.gramener.com/guide/', version)

204 elif version < __version__:

205 app_log.warning('Gramex update: your version %s is ahead of the stable %s',

206 __version__, version)

207 else:

208 app_log.debug('Gramex version %s is up to date', __version__)

209 services.info.eventlog.add('update', update)

210 return {'logs': logs, 'response': update}

211

212

213 def console(msg):

214 '''Write message to console'''

215 print(msg) # noqa

216

217

218 def init(force_reload=False, **kwargs):

219 '''

220 Update Gramex configurations and start / restart the instance.

221

222 ``gramex.init()`` can be called any time to refresh configuration files.

223 ``gramex.init(key=val)`` adds ``val`` as a configuration layer named

224 ``key``. If ``val`` is a Path, it is converted into a PathConfig. (If it is

225 Path directory, use ``gramex.yaml``.)

226

227 Services are re-initialised if their configurations have changed. Service

228 callbacks are always re-run (even if the configuration hasn't changed.)

229 '''

230 # Reset variables

231 variables.clear()

232 variables.update(setup_variables())

233

234 # Initialise configuration layers with provided configurations

235 # AttrDicts are updated as-is. Paths are converted to PathConfig

236 paths.update(kwargs)

237 for key, val in paths.items():

238 if isinstance(val, Path):

239 if val.is_dir(): 239 ↛ 241line 239 didn't jump to line 241, because the condition on line 239 was never false

240 val = val / 'gramex.yaml'

241 val = PathConfig(val)

242 config_layers[key] = val

243

244 # Locate all config files

245 config_files = set()

246 for path_config in config_layers.values():

247 if hasattr(path_config, '__info__'):

248 for pathinfo in path_config.__info__.imports:

249 config_files.add(pathinfo.path)

250 config_files = list(config_files)

251

252 # Add config file folders to sys.path

253 sys.path[:] = _sys_path + [str(path.absolute().parent) for path in config_files]

254

255 from . import services

256 globals()['service'] = services.info # gramex.service = gramex.services.info

257

258 # Override final configurations

259 final_config = +config_layers

260 # --settings.debug => log.root.level = True

261 if final_config.app.get('settings', {}).get('debug', False): 261 ↛ 262line 261 didn't jump to line 262, because the condition on line 261 was never true

262 final_config.log.root.level = logging.DEBUG

263

264 # Set up a watch on config files (including imported files)

265 if final_config.app.get('watch', True): 265 ↛ 271line 265 didn't jump to line 271, because the condition on line 265 was never false

266 from services import watcher

267 watcher.watch('gramex-reconfig', paths=config_files, on_modified=lambda event: init()) 267 ↛ exitline 267 didn't run the lambda on line 267

268

269 # Run all valid services. (The "+" before config_chain merges the chain)

270 # Services may return callbacks to be run at the end

271 for key, val in final_config.items():

272 if key not in conf or conf[key] != val or force_reload:

273 if hasattr(services, key):

274 app_log.debug('Loading service: %s', key)

275 conf[key] = deepcopy(val)

276 callback = getattr(services, key)(conf[key])

277 if callable(callback):

278 callbacks[key] = callback

279 else:

280 app_log.error('No service named %s', key)

281

282 # Run the callbacks. Specifically, the app service starts the Tornado ioloop

283 for key in (+config_layers).keys():

284 if key in callbacks:

285 app_log.debug('Running callback: %s', key)

286 callbacks[key]()

287

288

289 def shutdown():

290 '''Shut down this instance'''

291 ioloop = tornado.ioloop.IOLoop.current()

292 if ioloop_running(ioloop): 292 ↛ exitline 292 didn't return from function 'shutdown', because the condition on line 292 was never false

293 app_log.info('Shutting down Gramex...')

294 ioloop.stop()