Coverage for /Users/rik/github/cgse/libs/cgse-common/src/egse/settings.py: 66%

1"""

2 The Settings class handles user and configuration settings that are provided in

3 a [`YAML`](http://yaml.org) file.

4

5 The idea is that settings are grouped by components or any arbitrary grouping that makes sense for

6 the application or for the user. Settings are also modular and provided by each package by means

7 of entry-points. The Settings class can read from different YAML files.

8

9 By default, settings are loaded from a file called `settings.yaml`, but this can be changed in the entry-point

10 definition.

11

12 The yaml configuration files are provided as entry points by the packages that specified an entry-point group

13'_cgse.settings_' in the `pyproject.toml`. The Settings dictionary (attrdict) is constructed from the configuration

14 YAML files from each of the packages. Settings can be overwritten by the next package configuration file. So,

15 make sure the group names in each package configuration file are unique.

16

17 The YAML file is read and the configuration parameters for the given group are

18 available as instance variables of the returned class.

19

20 The intended use is as follows:

21

22```python

23 from egse.settings import Settings

24

25 dsi_settings = Settings.load("DSI")

26

27 if 0x000C <= dsi_settings.RMAP_BASE_ADDRESS <= 0x00FF:

28 ... # do something here

29 else:

30 raise RMAPError("Attempt to access outside the RMAP memory map.")

31```

32

33 The above code reads the settings from the default YAML file for a group called `DSI`.

34 The settings will then be available as variables of the returned class, in this case

35`dsi_settings`. The returned class is and behaves also like a dictionary, so you can

36 check if a configuration parameter is defined like this:

37

38```python

39 if "DSI_FEE_IP_ADDRESS" not in dsi_settings:

40 # define the IP address of the DSI

41```

42 The YAML section for the above code looks like this:

43

44```text

45 DSI:

46

47 # DSI Specific Settings

48

49 DSI_FEE_IP_ADDRESS 10.33.178.144 # IP address of the DSI EtherSpaceLink interface

50 LINK_SPEED: 100 # SpW link speed used for both up- and downlink

51

52 # RMAP Specific Settings

53

54 RMAP_BASE_ADDRESS: 0x00000000 # The start of the RMAP memory map managed by the FEE

55 RMAP_MEMORY_SIZE: 4096 # The size of the RMAP memory map managed by the FEE

56```

57

58 When you want to read settings from another YAML file, specify the `filename=` keyword.

59 If that file is located at a specific location, also use the `location=` keyword.

60

61 my_settings = Settings.load(filename="user.yaml", location="/Users/JohnDoe")

62

63 The above code will read the YAML file from the given location and not from the entry-points.

64

65---

66

67"""

68

69 from __future__ import annotations

70

71 import logging

72 import re

73 from pathlib import Path

74 from typing import Any

75

76 import yaml # This module is provided by the pip package PyYaml - pip install pyyaml

77

78 from egse.env import get_local_settings_env_name

79 from egse.env import get_local_settings_path

80 from egse.log import logger

81 from egse.system import attrdict

82 from egse.system import recursive_dict_update

83

84 _HERE = Path(__file__).resolve().parent

85

86

87 class SettingsError(Exception):

88 """A settings-specific error."""

89

90 pass

91

92

93# Fix the problem: YAML loads 5e-6 as string and not a number

94# https://stackoverflow.com/questions/30458977/yaml-loads-5e-6-as-string-and-not-a-number

95

96 SAFE_LOADER = yaml.SafeLoader

97 SAFE_LOADER.add_implicit_resolver(

98 "tag:yaml.org,2002:float",

99 re.compile(

100 """^(?:

101 [-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)?

102 |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)

103 |\\.[0-9_]+(?:[eE][-+][0-9]+)?

104 |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]*

105 |[-+]?\\.(?:inf|Inf|INF)

106 |\\.(?:nan|NaN|NAN))$""",

107 re.X,

108 ),

109 list("-+0123456789."),

110)

111

112

113 def load_settings_file(path: Path, filename: str, force: bool = False) -> attrdict:

114 """

115 Loads the YAML configuration file that is located at `path / filename`.

116

117 Args:

118 path (PATH): the folder where the YAML file is located

119 filename (str): the name of the YAML configuration file

120 force (bool): force reloading, i.e. don't use the cached information

121

122 Raises:

123 SettingsError: when the configuration file doesn't exist or cannot be found or \

124 when there was an error reading the configuration file.

125

126 Returns:

127 A dictionary (attrdict) with all the settings from the given file.

128

129 Note:

130 in case of an empty configuration file, and empty dictionary \

131 is returned and a warning message is issued.

132 """

133 try:

134 yaml_document = read_configuration_file(path / filename, force=force)

135 settings = attrdict({name: value for name, value in yaml_document.items()})

136 except FileNotFoundError as exc:

137 raise SettingsError(f"The Settings YAML file '{filename}' is not found at {path! s}. ") from exc

138

139 if not settings: 139 ↛ 140line 139 didn't jump to line 140 because the condition on line 139 was never true

140 logger.warning(

141 f"The Settings YAML file '{filename}' at {path! s} is empty. "

142 f"No local settings were loaded, an empty dictionary is returned."

143 )

144

145 return settings

146

147

148 def load_global_settings(entry_point: str = "cgse.settings", force: bool = False) -> attrdict:

149 """

150 Loads the settings that are defined by the given entry_point. The entry-points are defined in the

151 `pyproject.toml` files of the packages that export their global settings.

152

153 Args:

154 entry_point (str): the name of the entry-point group [default: 'cgse.settings']

155 force (bool): force reloading the settings, i.e. ignore the cache

156

157 Returns:

158 A dictionary (attrdict) containing a collection of all the settings exported by the packages \

159 through the given entry-point.

160

161 """

162 from egse.plugin import get_file_infos

163

164 ep_settings = get_file_infos(entry_point)

165

166 global_settings = attrdict(label="Settings")

167

168 for ep_name, (path, filename) in ep_settings.items():

169 settings = load_settings_file(path, filename, force)

170 recursive_dict_update(global_settings, settings)

171

172 return global_settings

173

174

175 def load_local_settings(force: bool = False) -> attrdict:

176 """

177 Loads the local settings file that is defined from the environment variable *PROJECT*_LOCAL_SETTINGS (where

178 *PROJECT* is the name of your project, defined in the environment variable of the same name).

179

180 This function might return an empty dictionary when

181

182 - the local settings YAML file is empty

183 - the local settings environment variable is not defined.

184

185 in both cases a warning message is logged.

186

187 Raises:

188 SettingsError: when the local settings YAML file is not found. Check the *PROJECT*_LOCAL_SETTINGS \

189 environment variable.

190

191 Returns:

192 A dictionary (attrdict) with all local settings.

193

194 """

195 local_settings = attrdict()

196

197 local_settings_path = get_local_settings_path()

198

199 if local_settings_path: 199 ↛ 203line 199 didn't jump to line 203 because the condition on line 199 was always true

200 path = Path(local_settings_path)

201 local_settings = load_settings_file(path.parent, path.name, force)

202

203 return local_settings

204

205

206 def read_configuration_file(filename: Path, *, force=False) -> dict:

207 """

208 Read the YAML input configuration file. The configuration file is only read

209 once and memoized as load optimization.

210

211 Args:

212 filename (Path): the fully qualified filename of the YAML file

213 force (bool): force reloading the file, even when it was memoized

214

215 Raises:

216 SettingsError: when there was an error reading the YAML file.

217

218 Returns:

219 a dictionary containing all the configuration settings from the YAML file.

220 """

221 filename = str(filename)

222

223 if force or not Settings.is_memoized(filename):

224 logger.debug(f"Parsing YAML configuration file {filename}.")

225

226 with open(filename, "r") as stream:

227 try:

228 yaml_document = yaml.load(stream, Loader=SAFE_LOADER)

229 except yaml.YAMLError as exc:

230 logger.error(exc)

231 raise SettingsError(f"Error loading YAML document {filename}") from exc

232

233 Settings.add_memoized(filename, yaml_document)

234

235 return Settings.get_memoized(filename) or {}

236

237

238 class Settings:

239 """

240 The Settings class provides a load() method that loads configuration settings for a group

241 into a dynamically created class as instance variables.

242 """

243

244 __memoized_yaml = {} # Memoized settings yaml files

245 __profile = False # Used for profiling methods and functions

246

247 LOG_FORMAT_DEFAULT = "%(levelname)s:%(module)s:%(lineno)d:%(message)s"

248 LOG_FORMAT_FULL = "%(asctime)23s:%(levelname)8s:%(lineno)5d:%(name)-20s: %(message)s"

249 LOG_FORMAT_THREAD = "%(asctime)23s:%(levelname)7s:%(lineno)5d:%(name)-20s(%(threadName)-15s): %(message)s"

250 LOG_FORMAT_PROCESS = (

251 "%(asctime)23s:%(levelname)7s:%(lineno)5d:%(name)20s.%(funcName)-31s(%(processName)-20s): %(message)s"

252 )

253 LOG_FORMAT_DATE = "%d/%m/%Y %H:%M:%S"

254

255 @classmethod

256 def get_memoized_locations(cls) -> list:

257 return list(cls.__memoized_yaml.keys())

258

259 @classmethod

260 def is_memoized(cls, filename: str) -> bool:

261 return filename in cls.__memoized_yaml

262

263 @classmethod

264 def add_memoized(cls, filename: str, yaml_document: Any):

265 cls.__memoized_yaml[filename] = yaml_document

266

267 @classmethod

268 def get_memoized(cls, filename: str):

269 return cls.__memoized_yaml.get(filename)

270

271 @classmethod

272 def clear_memoized(cls):

273 cls.__memoized_yaml.clear()

274

275 @classmethod

276 def set_profiling(cls, flag):

277 cls.__profile = flag

278

279 @classmethod

280 def profiling(cls):

281 return cls.__profile

282

283 @staticmethod

284 def _load_all(

285 entry_point: str = "cgse.settings", add_local_settings: bool = False, force: bool = False

286 ) -> attrdict:

287 """

288 Loads all settings from all package with the entry point 'cgse.settings'

289 """

290 global_settings = load_global_settings(entry_point, force)

291

292 # Load the LOCAL settings YAML file

293

294 if add_local_settings:

295 local_settings = load_local_settings(force)

296 recursive_dict_update(global_settings, local_settings)

297

298 return global_settings

299

300 @staticmethod

301 def _load_group(

302 group_name: str, entry_point: str = "cgse.settings", add_local_settings: bool = False, force: bool = False

303 ) -> attrdict:

304 global_settings = load_global_settings(entry_point, force)

305

306 group_settings = attrdict(label=group_name)

307

308 if group_name in global_settings: 308 ↛ 313line 308 didn't jump to line 313 because the condition on line 308 was always true

309 group_settings = attrdict(

310 {name: value for name, value in global_settings[group_name].items()}, label=group_name

311 )

312

313 if add_local_settings: 313 ↛ 318line 313 didn't jump to line 318 because the condition on line 313 was always true

314 local_settings = load_local_settings(force)

315 if group_name in local_settings: 315 ↛ 316line 315 didn't jump to line 316 because the condition on line 315 was never true

316 recursive_dict_update(group_settings, local_settings[group_name])

317

318 if not group_settings: 318 ↛ 319line 318 didn't jump to line 319 because the condition on line 318 was never true

319 raise SettingsError(f"Group name '{group_name}' is not defined in the global nor in the local settings.")

320

321 return group_settings

322

323 @staticmethod

324 def _load_one(location: str, filename: str, force=False) -> attrdict:

325 return load_settings_file(Path(location).expanduser(), filename, force)

326

327 @classmethod

328 def load(

329 cls, group_name=None, filename="settings.yaml", location=None, *, add_local_settings=True, force=False

330 ) -> attrdict:

331 """

332 Load the settings for the given group. When no group is provided, the

333 complete configuration is returned.

334

335 The Settings are loaded from entry-points that are defined in each of the

336 packages that provide a Settings file.

337

338 If a location is explicitly provided, the Settings will be loaded from that

339 location, using the given filename or the default (which is settings.yaml).

340

341 Args:

342 group_name (str): the name of one of the main groups from the YAML file

343 filename (str): the name of the YAML file to read [default=settings.yaml]

344 location (str, Path): the path to the location of the YAML file

345 force (bool): force reloading the file

346 add_local_settings (bool): update the Settings with site specific local settings

347

348 Returns:

349 a dynamically created class with the configuration parameters as instance variables.

350

351 Raises:

352 SettingsError: when the group is not defined in the YAML file.

353 """

354 if group_name: 354 ↛ 356line 354 didn't jump to line 356 because the condition on line 354 was always true

355 return cls._load_group(group_name, add_local_settings=add_local_settings, force=force)

356 elif location:

357 return cls._load_one(location=location, filename=filename, force=force)

358 else:

359 return cls._load_all(add_local_settings=add_local_settings, force=force)

360

361 @classmethod

362 def to_string(cls):

363 """

364 Returns a simple string representation of the cached configuration of this Settings class.

365 """

366 memoized = cls.__memoized_yaml

367

368 msg = ""

369 for key in memoized.keys():

370 msg += f"YAML file: {key}\n"

371 for field in memoized[key].keys():

372 length = 60

373 line = str(memoized[key][field])

374 trunc = line[:length]

375 if len(line) > length:

376 trunc += " ..."

377 msg += f" {field}: {trunc}\n"

378

379 return msg.rstrip()

380

381

382 def main(args: list | None = None): # pragma: no cover

383 # We provide convenience to inspect the settings by calling this module directly from Python.

384 #

385 # python -m egse.settings

386 #

387 # Use the '--help' option to see what your choices are.

388

389 logging.basicConfig(level=20)

390

391 import argparse

392

393 parser = argparse.ArgumentParser(

394 description=(

395 f"Print out the default Settings, updated with local settings if the "

396 f"{get_local_settings_env_name()} environment variable is set."

397 ),

398 )

399 parser.add_argument("--local", action="store_true", help="print only the local settings.")

400 parser.add_argument(

401 "--global", action="store_true", help="print only the global settings, don't include local settings."

402 )

403 parser.add_argument("--group", help="print only settings for this group")

404 args = parser.parse_args(args or [])

405

406 # The following import will activate the pretty printing of the AttributeDict

407 # through the __rich__ method.

408

409 from rich import print

410

411 if args.local:

412 location = get_local_settings_path()

413 if location:

414 location = str(Path(location).expanduser().resolve())

415 settings = Settings.load(filename=location)

416 print(settings)

417 print(f"Loaded from [purple]{location}.")

418 else:

419 print("[red]No local settings defined.")

420 else:

421 # if the global option is given we don't want to include local settings

422 add_local_settings = False if vars(args)["global"] else True

423

424 if args.group:

425 settings = Settings.load(args.group, add_local_settings=add_local_settings)

426 else:

427 settings = Settings.load(add_local_settings=add_local_settings)

428 print(settings)

429 print("[blue]Memoized locations:")

430 locations = Settings.get_memoized_locations()

431 print([str(loc) for loc in locations])

432

433

434 def get_site_id() -> str:

435 site = Settings.load("SITE")

436 return site.ID

437

438

439# ignore_m_warning('egse.settings')

440

441 if __name__ == "__main__": 441 ↛ 442line 441 didn't jump to line 442 because the condition on line 441 was never true

442 main()