Coverage for src/configuraptor/core.py: 100%

1"""

2 Contains most of the loading logic.

3"""

4

5 import dataclasses as dc

6 import types

7 import typing

8 import warnings

9 from collections import ChainMap

10 from pathlib import Path

11

12 from typeguard import TypeCheckError

13 from typeguard import check_type as _check_type

14

15 from . import loaders

16 from .errors import ConfigErrorInvalidType, ConfigErrorMissingKey

17 from .helpers import camel_to_snake

18

19# T is a reusable typevar

20 T = typing.TypeVar("T")

21# t_typelike is anything that can be type hinted

22 T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union

23# t_data is anything that can be fed to _load_data

24 T_data = str | Path | dict[str, typing.Any]

25# c = a config class instance, can be any (user-defined) class

26 C = typing.TypeVar("C")

27# type c is a config class

28 Type_C = typing.Type[C]

29

30

31 def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:

32 """

33 If a key contains a dot, traverse the raw dict until the right key was found.

34

35 Example:

36 key = some.nested.key

37 raw = {"some": {"nested": {"key": {"with": "data"}}}}

38 -> {"with": "data"}

39 """

40 parts = key.split(".")

41 while parts:

42 raw = raw[parts.pop(0)]

43

44 return raw

45

46

47 def _guess_key(clsname: str) -> str:

48 """

49 If no key is manually defined for `load_into`, \

50 the class' name is converted to snake_case to use as the default key.

51 """

52 return camel_to_snake(clsname)

53

54

55 def _load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]:

56 """

57 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.

58

59 E.g. class Tool will be mapped to key tool.

60 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}

61 """

62 if isinstance(data, str):

63 data = Path(data)

64 if isinstance(data, Path):

65 with data.open("rb") as f:

66 loader = loaders.get(data.suffix)

67 data = loader(f)

68

69 if not data:

70 return {}

71

72 if key is None:

73 # try to guess key by grabbing the first one or using the class name

74 if len(data) == 1:

75 key = list(data.keys())[0]

76 elif classname is not None:

77 key = _guess_key(classname)

78

79 if key:

80 return _data_for_nested_key(key, data)

81 else:

82 # no key found, just return all data

83 return data

84

85

86 def check_type(value: typing.Any, expected_type: T_typelike) -> bool:

87 """

88 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.).

89

90 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError

91 """

92 try:

93 _check_type(value, expected_type)

94 return True

95 except TypeCheckError:

96 return False

97

98

99 def ensure_types(data: dict[str, T], annotations: dict[str, type]) -> dict[str, T | None]:

100 """

101 Make sure all values in 'data' are in line with the ones stored in 'annotations'.

102

103 If an annotated key in missing from data, it will be filled with None for convenience.

104 """

105 # custom object to use instead of None, since typing.Optional can be None!

106 # cast to T to make mypy happy

107 notfound = typing.cast(T, object())

108

109 final: dict[str, T | None] = {}

110 for key, _type in annotations.items():

111 compare = data.get(key, notfound)

112 if compare is notfound: # pragma: nocover

113 warnings.warn(

114 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"

115 )

116 # skip!

117 continue

118 if not check_type(compare, _type):

119 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)

120

121 final[key] = compare

122 return final

123

124

125 def convert_config(items: dict[str, T]) -> dict[str, T]:

126 """

127 Converts the config dict (from toml) or 'overwrites' dict in two ways.

128

129 1. removes any items where the value is None, since in that case the default should be used;

130 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.

131 """

132 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}

133

134

135 Type = typing.Type[typing.Any]

136 T_Type = typing.TypeVar("T_Type", bound=Type)

137

138

139 def is_builtin_type(_type: Type) -> bool:

140 """

141 Returns whether _type is one of the builtin types.

142 """

143 return _type.__module__ in ("__builtin__", "builtins")

144

145

146# def is_builtin_class_instance(obj: typing.Any) -> bool:

147# return is_builtin_type(obj.__class__)

148

149

150 def is_from_types_or_typing(_type: Type) -> bool:

151 """

152 Returns whether _type is one of the stlib typing/types types.

153

154 e.g. types.UnionType or typing.Union

155 """

156 return _type.__module__ in ("types", "typing")

157

158

159 def is_from_other_toml_supported_module(_type: Type) -> bool:

160 """

161 Besides builtins, toml also supports 'datetime' and 'math' types, \

162 so this returns whether _type is a type from these stdlib modules.

163 """

164 return _type.__module__ in ("datetime", "math")

165

166

167 def is_parameterized(_type: Type) -> bool:

168 """

169 Returns whether _type is a parameterized type.

170

171 Examples:

172 list[str] -> True

173 str -> False

174 """

175 return typing.get_origin(_type) is not None

176

177

178 def is_custom_class(_type: Type) -> bool:

179 """

180 Tries to guess if _type is a builtin or a custom (user-defined) class.

181

182 Other logic in this module depends on knowing that.

183 """

184 return (

185 type(_type) is type

186 and not is_builtin_type(_type)

187 and not is_from_other_toml_supported_module(_type)

188 and not is_from_types_or_typing(_type)

189 )

190

191

192 def is_optional(_type: Type | None) -> bool:

193 """

194 Tries to guess if _type could be optional.

195

196 Examples:

197 None -> True

198 NoneType -> True

199 typing.Union[str, None] -> True

200 str | None -> True

201 list[str | None] -> False

202 list[str] -> False

203 """

204 if _type and is_parameterized(_type) and typing.get_origin(_type) in (dict, list):

205 # e.g. list[str]

206 # will crash issubclass to test it first here

207 return False

208

209 return (

210 _type is None

211 or issubclass(types.NoneType, _type)

212 or issubclass(types.NoneType, type(_type)) # no type # Nonetype

213 or type(None) in typing.get_args(_type) # union with Nonetype

214 )

215

216

217 def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]:

218 """

219 Get Field info for a dataclass cls.

220 """

221 fields = getattr(cls, "__dataclass_fields__", {})

222 return fields.get(key)

223

224

225 def load_recursive(cls: Type, data: dict[str, T], annotations: dict[str, Type]) -> dict[str, T]:

226 """

227 For all annotations (recursively gathered from parents with `all_annotations`), \

228 try to resolve the tree of annotations.

229

230 Uses `load_into_recurse`, not itself directly.

231

232 Example:

233 class First:

234 key: str

235

236 class Second:

237 other: First

238

239 # step 1

240 cls = Second

241 data = {"second": {"other": {"key": "anything"}}}

242 annotations: {"other": First}

243

244 # step 1.5

245 data = {"other": {"key": "anything"}

246 annotations: {"other": First}

247

248 # step 2

249 cls = First

250 data = {"key": "anything"}

251 annotations: {"key": str}

252

253 """

254 updated = {}

255 for _key, _type in annotations.items():

256 if _key in data:

257 value: typing.Any = data[_key] # value can change so define it as any instead of T

258 if is_parameterized(_type):

259 origin = typing.get_origin(_type)

260 arguments = typing.get_args(_type)

261 if origin is list and arguments and is_custom_class(arguments[0]):

262 subtype = arguments[0]

263 value = [load_into_recurse(subtype, subvalue) for subvalue in value]

264

265 elif origin is dict and arguments and is_custom_class(arguments[1]):

266 # e.g. dict[str, Point]

267 subkeytype, subvaluetype = arguments

268 # subkey(type) is not a custom class, so don't try to convert it:

269 value = {subkey: load_into_recurse(subvaluetype, subvalue) for subkey, subvalue in value.items()}

270 # elif origin is dict:

271 # keep data the same

272 elif origin is typing.Union and arguments:

273 for arg in arguments:

274 if is_custom_class(arg):

275 value = load_into_recurse(arg, value)

276 else:

277 # print(_type, arg, value)

278 ...

279

280 # todo: other parameterized/unions/typing.Optional

281

282 elif is_custom_class(_type):

283 # type must be C (custom class) at this point

284 value = load_into_recurse(

285 # make mypy and pycharm happy by telling it _type is of type C...

286 # actually just passing _type as first arg!

287 typing.cast(Type_C[typing.Any], _type),

288 value,

289 )

290

291 elif _key in cls.__dict__:

292 # property has default, use that instead.

293 value = cls.__dict__[_key]

294 elif is_optional(_type):

295 # type is optional and not found in __dict__ -> default is None

296 value = None

297 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:

298 # could have a default factory

299 # todo: do something with field.default?

300 value = field.default_factory()

301 else:

302 # todo: exception group?

303 raise ConfigErrorMissingKey(_key, cls, _type)

304

305 updated[_key] = value

306

307 return updated

308

309

310 def _all_annotations(cls: Type) -> ChainMap[str, Type]:

311 """

312 Returns a dictionary-like ChainMap that includes annotations for all \

313 attributes defined in cls or inherited from superclasses.

314 """

315 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__))

316

317

318 def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, Type]:

319 """

320 Wrapper around `_all_annotations` that filters away any keys in _except.

321

322 It also flattens the ChainMap to a regular dict.

323 """

324 if _except is None:

325 _except = set()

326

327 _all = _all_annotations(cls)

328 return {k: v for k, v in _all.items() if k not in _except}

329

330

331 def _check_and_convert_data(

332 cls: typing.Type[C],

333 data: dict[str, typing.Any],

334 _except: typing.Iterable[str],

335) -> dict[str, typing.Any]:

336 """

337 Based on class annotations, this prepares the data for `load_into_recurse`.

338

339 1. convert config-keys to python compatible config_keys

340 2. loads custom class type annotations with the same logic (see also `load_recursive`)

341 3. ensures the annotated types match the actual types after loading the config file.

342 """

343 annotations = all_annotations(cls, _except=_except)

344

345 to_load = convert_config(data)

346 to_load = load_recursive(cls, to_load, annotations)

347 to_load = ensure_types(to_load, annotations)

348 return to_load

349

350

351 def load_into_recurse(

352 cls: typing.Type[C],

353 data: dict[str, typing.Any],

354 init: dict[str, typing.Any] = None,

355) -> C:

356 """

357 Loads an instance of `cls` filled with `data`.

358

359 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).

360 `init` can be used to optionally pass extra __init__ arguments. \

361 NOTE: This will overwrite a config key with the same name!

362 """

363 if init is None:

364 init = {}

365

366 # fixme: cls.__init__ can set other keys than the name is in kwargs!!

367

368 if dc.is_dataclass(cls):

369 to_load = _check_and_convert_data(cls, data, init.keys())

370 to_load |= init # add extra init variables (should not happen for a dataclass but whatev)

371

372 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)

373 inst = typing.cast(C, cls(**to_load))

374 else:

375 inst = cls(**init)

376 to_load = _check_and_convert_data(cls, data, inst.__dict__.keys())

377 inst.__dict__.update(**to_load)

378

379 return inst

380

381

382 def load_into_existing(

383 inst: C,

384 cls: typing.Type[C],

385 data: dict[str, typing.Any],

386 init: dict[str, typing.Any] = None,

387) -> C:

388 """

389 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \

390 and thus does not support init.

391

392 """

393 if init is not None:

394 raise ValueError("Can not init an existing instance!")

395

396 existing_data = inst.__dict__

397

398 annotations = all_annotations(cls, _except=existing_data.keys())

399 to_load = convert_config(data)

400 to_load = load_recursive(cls, to_load, annotations)

401 to_load = ensure_types(to_load, annotations)

402

403 inst.__dict__.update(**to_load)

404

405 return inst

406

407

408 def load_into_class(

409 cls: typing.Type[C],

410 data: T_data,

411 /,

412 key: str = None,

413 init: dict[str, typing.Any] = None,

414) -> C:

415 """

416 Shortcut for _load_data + load_into_recurse.

417 """

418 to_load = _load_data(data, key, cls.__name__)

419 return load_into_recurse(cls, to_load, init=init)

420

421

422 def load_into_instance(

423 inst: C,

424 data: T_data,

425 /,

426 key: str = None,

427 init: dict[str, typing.Any] = None,

428) -> C:

429 """

430 Shortcut for _load_data + load_into_existing.

431 """

432 cls = inst.__class__

433 to_load = _load_data(data, key, cls.__name__)

434 return load_into_existing(inst, cls, to_load, init=init)

435

436

437 def load_into(

438 cls: typing.Type[C] | C,

439 data: T_data,

440 /,

441 key: str = None,

442 init: dict[str, typing.Any] = None,

443) -> C:

444 """

445 Load your config into a class (instance).

446

447 Args:

448 cls: either a class or an existing instance of that class.

449 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)

450 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')

451 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)

452

453 """

454 if not isinstance(cls, type):

455 return load_into_instance(cls, data, key=key, init=init)

456

457 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'

458 _cls = typing.cast(typing.Type[C], cls)

459 return load_into_class(_cls, data, key=key, init=init)