Coverage for /Users/rik/github/cgse/libs/cgse-common/src/egse/plugin.py: 31%

1"""

2 This module provides function to load plugins and settings from entry-points.

3"""

4

5 __all__ = [

6 "load_plugins_ep",

7 "load_plugins_fn",

8 "get_file_infos",

9 "entry_points",

10 "HierarchicalEntryPoints",

11]

12

13 import importlib.util

14 import os

15 import sys

16 import textwrap

17 import traceback

18 import types

19 from functools import lru_cache

20 from pathlib import Path

21

22 import click

23 import rich

24

25 from egse.log import logger

26 from egse.system import type_name

27

28 if sys.version_info >= (3, 12): # Use the standard library version (Python 3.10+) 28 ↛ 33line 28 didn't jump to line 33 because the condition on line 28 was always true

29 from importlib.metadata import entry_points as lib_entry_points

30 from importlib.metadata import EntryPoint

31 else:

32 # Fall back to the backport for Python 3.9

33 from importlib_metadata import entry_points as lib_entry_points

34 from importlib_metadata import EntryPoint

35

36

37 HERE = Path(__file__).parent

38

39

40 def entry_points(group: str) -> set[EntryPoint]:

41 """

42 Returns a set with all entry-points for the given group name.

43

44 When the name is not known as an entry-point group, an empty set will be returned.

45 """

46

47 try:

48 x = lib_entry_points().select(group=group)

49 return {ep for ep in x} # use of set here to remove duplicates

50 except KeyError:

51 return set()

52

53

54 def load_plugins_ep(entry_point: str) -> dict:

55 """

56 Returns a dictionary with plugins loaded. The keys are the names of the entry-points,

57 the values are the loaded modules or objects.

58

59 Note:

60 When an entry point cannot be loaded, an error is logged and the value for that

61 entry point in the returned dictionary will be None.

62 """

63 eps = {}

64 for ep in entry_points(entry_point):

65 try:

66 eps[ep.name] = ep.load()

67 except Exception as exc:

68 eps[ep.name] = None

69 logger.error(f"Couldn't load entry point {entry_point}: {type_name(exc)} – {exc}")

70

71 return eps

72

73

74@lru_cache

75 def load_plugins_fn(pattern: str, package_name: str = None) -> dict[str, types.ModuleType]:

76 """

77 Returns a dictionary with plugins loaded for the filenames that match the given pattern.

78 The keys are the names of the modules, the values are the loaded modules.

79

80 If no package_name is provided, the pattern is relative to the location of this module

81 (which is in the top-level module `egse`).

82

83 If the pattern results in two or more modules with the same name, a warning will be logged

84 and only the last imported module will be returned in the dictionary.

85

86 Plugins are usually located in the `egse.plugins` module. Remember that `egse.plugins`

87 is a namespace and external packages can also deliver plugin modules in that location.

88

89 Note:

90 When a plugin cannot be loaded, an error is logged.

91

92 This function uses an LRU cache to avoid reloading modules. If you need to reload,

93 use `load_plugins_fn.cache_clear()` to reset the cache.

94

95 Raises:

96 ImportError when the given package_name cannot be imported as a module.

97

98 Examples:

99

100 # Loading the InfluxDB plugin for time series metrics

101 >>> influxdb = load_plugins_fn("influxdb.py", "egse.plugins.metrics")

102

103 # Loading the HDF5 storage plugin for PLATO

104 >>> hdf5 = load_plugins_fn("hdf5.py", "egse.plugins.storage")

105

106 # Loading all plugins

107 >>> x = load_plugins_fn("**/*.py", "egse.plugins")

108

109 """

110 loaded_modules = {}

111 failed_modules = {} # we keep failed modules for housekeeping only

112

113 if package_name is None:

114 package_path = [HERE]

115 else:

116 try:

117 package = importlib.import_module(package_name)

118 except ImportError as exc:

119 raise ImportError(f"Cannot import package '{package_name}': {exc}")

120

121 if hasattr(package, "__path__"):

122 package_path = package.__path__

123 else:

124 package_path = [package.__file__.replace("__init__.py", "")]

125

126 # rich.print(package_path)

127

128 for path_entry in map(Path, package_path):

129 # rich.print(path_entry)

130 for plugin_file in path_entry.rglob(pattern):

131 # rich.print(" ", plugin_file)

132 module_name = plugin_file.stem

133 try:

134 spec = importlib.util.spec_from_file_location(module_name, plugin_file)

135 module = importlib.util.module_from_spec(spec)

136 spec.loader.exec_module(module)

137 if module_name in loaded_modules:

138 logger.warning(

139 f"Overwriting module '{module_name}' from {loaded_modules[module_name].__file__} "

140 f"with {plugin_file}."

141 )

142 loaded_modules[module_name] = module

143 except Exception as exc:

144 error_msg = f"Couldn't load module '{module_name}' from {plugin_file}: {type_name(exc)} – {exc}"

145 logger.error(error_msg)

146 if module_name in failed_modules:

147 logger.warning(f"Overwriting previously unloaded module '{module_name}' with {plugin_file}.")

148 failed_modules[module_name] = error_msg

149

150 return loaded_modules

151

152

153 def get_file_infos(entry_point: str) -> dict[str, tuple[Path, str]]:

154 """

155 Returns a dictionary with location and filename of all the entries found for

156 the given entry-point name.

157

158 The entry-points are interpreted as follows: `<name> = "<module>:<filename>"` where

159

160 - `<name>` is the name of the entry-point given in the pyproject.toml file

161 - `<module>` is a valid module name that can be imported and from which the location can be determined.

162 - `<filename>` is the name of the target file, e.g. a YAML file

163

164 As an example, for the `cgse-common` settings, the following entry in the `pyproject.toml`:

165

166 [project.entry-points."cgse.settings"]

167 cgse-common = "cgse_common:settings.yaml"

168

169 Note that the module name for this entry point has an underscore instead of a dash.

170

171 Return:

172 A dictionary with the entry point name as the key and a tuple (location, filename) as the value.

173 """

174 from egse.system import get_module_location

175

176 eps = dict()

177

178 for ep in entry_points(entry_point):

179 try:

180 path = get_module_location(ep.module)

181

182 if path is None: 182 ↛ 183line 182 didn't jump to line 183 because the condition on line 182 was never true

183 logger.error(

184 f"The entry-point '{ep.name}' is ill defined. The module part doesn't exist or is a "

185 f"namespace. No settings are loaded for this entry-point."

186 )

187 else:

188 eps[ep.name] = (path, ep.attr)

189

190 except Exception as exc:

191 logger.error(f"The entry point '{ep.name}' is ill defined: {exc}")

192

193 return eps

194

195

196 class HierarchicalEntryPoints:

197 def __init__(self, base_group: str):

198 self.base_group = base_group

199 self._discover_groups()

200

201 def _discover_groups(self):

202 """Discover all groups that match the base group pattern."""

203 all_eps = lib_entry_points()

204 # print(f"{type(all_eps) = }")

205

206 self.groups = {}

207 self.flat_entries = []

208

209 for ep in all_eps:

210 # rich.print(f"{type(ep) = }, {dir(ep) = }")

211 if ep.group == self.base_group or ep.group.startswith(f"{self.base_group}."):

212 # print(f"{ep.group = }, {ep = }")

213 if ep.group in self.groups:

214 self.groups[ep.group].append(ep)

215 else:

216 self.groups[ep.group] = [ep]

217 self.flat_entries.append(ep)

218

219 def get_all_entry_points(self) -> list:

220 """Get all entry points as a flat list."""

221 return self.flat_entries

222

223 def get_by_subgroup(self, subgroup=None) -> list:

224 """Get entry points from a specific subgroup."""

225 if subgroup is None:

226 return self.groups.get(self.base_group, [])

227

228 full_group = f"{self.base_group}.{subgroup}"

229 return self.groups.get(full_group, [])

230

231 def get_all_groups(self) -> list:

232 """Get all discovered group names."""

233 return list(self.groups.keys())

234

235 def get_by_type(self, entry_type) -> list:

236 """Get entry points by type (assuming type is the subgroup name)."""

237 return self.get_by_subgroup(entry_type)

238

239

240# The following code was adapted from the inspiring package click-plugins

241# at https://github.com/click-contrib/click-plugins/

242

243

244 def handle_click_plugins(plugins):

245 def decorator(group):

246 if not isinstance(group, click.Group):

247 raise TypeError("Plugins can only be attached to an instance of click.Group()")

248

249 for entry_point in plugins or ():

250 try:

251 group.add_command(entry_point.load())

252 except Exception:

253 # Catch this so a busted plugin doesn't take down the CLI.

254 # Handled by registering a dummy command that does nothing

255 # other than explain the error.

256 group.add_command(BrokenCommand(entry_point.name))

257

258 return group

259

260 return decorator

261

262

263# This class is filtered and will not be included in the API docs, see `mkdocs.yml`.

264

265

266 class BrokenCommand(click.Command):

267 """

268 Rather than completely crash the CLI when a broken plugin is loaded, this

269 class provides a modified help message informing the user that the plugin is

270 broken, and they should contact the owner. If the user executes the plugin

271 or specifies `--help` a traceback is reported showing the exception the

272 plugin loader encountered.

273 """

274

275 def __init__(self, name):

276 """

277 Define the special help messages after instantiating a `click.Command()`.

278 """

279

280 click.Command.__init__(self, name)

281

282 util_name = os.path.basename(sys.argv and sys.argv[0] or __file__)

283 icon = "\u2020"

284

285 self.help = textwrap.dedent(

286 f"""\

287 Warning: entry point could not be loaded. Contact its author for help.

288

289 {traceback.format_exc()}

290 """

291 )

292

293 self.short_help = f"{icon} Warning: could not load plugin. See `{util_name} {self.name} --help`."

294

295 def invoke(self, ctx):

296 """

297 Print the traceback instead of doing nothing.

298 """

299

300 rich.print()

301 rich.print(self.help)

302 ctx.exit(1)

303

304 def parse_args(self, ctx, args):

305 return args