Coverage for src/configuraptor/core.py: 100%

1"""

2 Contains most of the loading logic.

3"""

4

5 import dataclasses as dc

6 import math

7 import types

8 import typing

9 import warnings

10 from collections import ChainMap

11 from pathlib import Path

12

13 from typeguard import TypeCheckError

14 from typeguard import check_type as _check_type

15

16 from . import loaders

17 from .errors import ConfigErrorInvalidType, ConfigErrorMissingKey

18 from .helpers import camel_to_snake

19 from .postpone import Postponed

20

21# T is a reusable typevar

22 T = typing.TypeVar("T")

23# t_typelike is anything that can be type hinted

24 T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union

25# t_data is anything that can be fed to _load_data

26 T_data = str | Path | dict[str, typing.Any]

27# c = a config class instance, can be any (user-defined) class

28 C = typing.TypeVar("C")

29# type c is a config class

30 Type_C = typing.Type[C]

31

32

33 def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:

34 """

35 If a key contains a dot, traverse the raw dict until the right key was found.

36

37 Example:

38 key = some.nested.key

39 raw = {"some": {"nested": {"key": {"with": "data"}}}}

40 -> {"with": "data"}

41 """

42 parts = key.split(".")

43 while parts:

44 raw = raw[parts.pop(0)]

45

46 return raw

47

48

49 def _guess_key(clsname: str) -> str:

50 """

51 If no key is manually defined for `load_into`, \

52 the class' name is converted to snake_case to use as the default key.

53 """

54 return camel_to_snake(clsname)

55

56

57 def __load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]:

58 """

59 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.

60

61 E.g. class Tool will be mapped to key tool.

62 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}

63 """

64 if isinstance(data, str):

65 data = Path(data)

66 if isinstance(data, Path):

67 with data.open("rb") as f:

68 loader = loaders.get(data.suffix)

69 data = loader(f)

70

71 if not data:

72 return {}

73

74 if key is None:

75 # try to guess key by grabbing the first one or using the class name

76 if len(data) == 1:

77 key = list(data.keys())[0]

78 elif classname is not None:

79 key = _guess_key(classname)

80

81 if key:

82 data = _data_for_nested_key(key, data)

83

84 if not data:

85 raise ValueError("No data found!")

86

87 if not isinstance(data, dict):

88 raise ValueError("Data is not a dict!")

89

90 return data

91

92

93 def _load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]:

94 """

95 Wrapper around __load_data that retries with key="" if anything goes wrong.

96 """

97 try:

98 return __load_data(data, key, classname)

99 except Exception as e:

100 if key != "":

101 return __load_data(data, "", classname)

102 else: # pragma: no cover

103 # key already was "", just return data!

104 # (will probably not happen but fallback)

105 return {}

106

107

108 def check_type(value: typing.Any, expected_type: T_typelike) -> bool:

109 """

110 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.).

111

112 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError

113 """

114 try:

115 _check_type(value, expected_type)

116 return True

117 except TypeCheckError:

118 return False

119

120

121 def ensure_types(data: dict[str, T], annotations: dict[str, type]) -> dict[str, T | None]:

122 """

123 Make sure all values in 'data' are in line with the ones stored in 'annotations'.

124

125 If an annotated key in missing from data, it will be filled with None for convenience.

126

127 TODO: python 3.11 exception groups to throw multiple errors at once!

128 """

129 # custom object to use instead of None, since typing.Optional can be None!

130 # cast to T to make mypy happy

131 notfound = typing.cast(T, object())

132 postponed = Postponed()

133

134 final: dict[str, T | None] = {}

135 for key, _type in annotations.items():

136 compare = data.get(key, notfound)

137 if compare is notfound: # pragma: nocover

138 warnings.warn(

139 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"

140 )

141 # skip!

142 continue

143

144 if compare is postponed:

145 # don't do anything with this item!

146 continue

147

148 if not check_type(compare, _type):

149 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)

150

151 final[key] = compare

152

153 return final

154

155

156 def convert_config(items: dict[str, T]) -> dict[str, T]:

157 """

158 Converts the config dict (from toml) or 'overwrites' dict in two ways.

159

160 1. removes any items where the value is None, since in that case the default should be used;

161 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.

162 """

163 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}

164

165

166 Type = typing.Type[typing.Any]

167 T_Type = typing.TypeVar("T_Type", bound=Type)

168

169

170 def is_builtin_type(_type: Type) -> bool:

171 """

172 Returns whether _type is one of the builtin types.

173 """

174 return _type.__module__ in ("__builtin__", "builtins")

175

176

177# def is_builtin_class_instance(obj: typing.Any) -> bool:

178# return is_builtin_type(obj.__class__)

179

180

181 def is_from_types_or_typing(_type: Type) -> bool:

182 """

183 Returns whether _type is one of the stlib typing/types types.

184

185 e.g. types.UnionType or typing.Union

186 """

187 return _type.__module__ in ("types", "typing")

188

189

190 def is_from_other_toml_supported_module(_type: Type) -> bool:

191 """

192 Besides builtins, toml also supports 'datetime' and 'math' types, \

193 so this returns whether _type is a type from these stdlib modules.

194 """

195 return _type.__module__ in ("datetime", "math")

196

197

198 def is_parameterized(_type: Type) -> bool:

199 """

200 Returns whether _type is a parameterized type.

201

202 Examples:

203 list[str] -> True

204 str -> False

205 """

206 return typing.get_origin(_type) is not None

207

208

209 def is_custom_class(_type: Type) -> bool:

210 """

211 Tries to guess if _type is a builtin or a custom (user-defined) class.

212

213 Other logic in this module depends on knowing that.

214 """

215 return (

216 type(_type) is type

217 and not is_builtin_type(_type)

218 and not is_from_other_toml_supported_module(_type)

219 and not is_from_types_or_typing(_type)

220 )

221

222

223 def instance_of_custom_class(var: typing.Any) -> bool:

224 """

225 Calls `is_custom_class` on an instance of a (possibly custom) class.

226 """

227 return is_custom_class(var.__class__)

228

229

230 def is_optional(_type: Type | typing.Any) -> bool:

231 """

232 Tries to guess if _type could be optional.

233

234 Examples:

235 None -> True

236 NoneType -> True

237 typing.Union[str, None] -> True

238 str | None -> True

239 list[str | None] -> False

240 list[str] -> False

241 """

242 if _type and (is_parameterized(_type) and typing.get_origin(_type) in (dict, list)) or (_type is math.nan):

243 # e.g. list[str]

244 # will crash issubclass to test it first here

245 return False

246

247 return (

248 _type is None

249 or types.NoneType in typing.get_args(_type) # union with Nonetype

250 or issubclass(types.NoneType, _type)

251 or issubclass(types.NoneType, type(_type)) # no type # Nonetype

252 )

253

254

255 def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]:

256 """

257 Get Field info for a dataclass cls.

258 """

259 fields = getattr(cls, "__dataclass_fields__", {})

260 return fields.get(key)

261

262

263 def load_recursive(cls: Type, data: dict[str, T], annotations: dict[str, Type]) -> dict[str, T]:

264 """

265 For all annotations (recursively gathered from parents with `all_annotations`), \

266 try to resolve the tree of annotations.

267

268 Uses `load_into_recurse`, not itself directly.

269

270 Example:

271 class First:

272 key: str

273

274 class Second:

275 other: First

276

277 # step 1

278 cls = Second

279 data = {"second": {"other": {"key": "anything"}}}

280 annotations: {"other": First}

281

282 # step 1.5

283 data = {"other": {"key": "anything"}

284 annotations: {"other": First}

285

286 # step 2

287 cls = First

288 data = {"key": "anything"}

289 annotations: {"key": str}

290

291

292 TODO: python 3.11 exception groups to throw multiple errors at once!

293 """

294 updated = {}

295

296 for _key, _type in annotations.items():

297 if _key in data:

298 value: typing.Any = data[_key] # value can change so define it as any instead of T

299 if is_parameterized(_type):

300 origin = typing.get_origin(_type)

301 arguments = typing.get_args(_type)

302 if origin is list and arguments and is_custom_class(arguments[0]):

303 subtype = arguments[0]

304 value = [_load_into_recurse(subtype, subvalue) for subvalue in value]

305

306 elif origin is dict and arguments and is_custom_class(arguments[1]):

307 # e.g. dict[str, Point]

308 subkeytype, subvaluetype = arguments

309 # subkey(type) is not a custom class, so don't try to convert it:

310 value = {subkey: _load_into_recurse(subvaluetype, subvalue) for subkey, subvalue in value.items()}

311 # elif origin is dict:

312 # keep data the same

313 elif origin is typing.Union and arguments:

314 for arg in arguments:

315 if is_custom_class(arg):

316 value = _load_into_recurse(arg, value)

317 else:

318 # print(_type, arg, value)

319 ...

320

321 # todo: other parameterized/unions/typing.Optional

322

323 elif is_custom_class(_type):

324 # type must be C (custom class) at this point

325 value = _load_into_recurse(

326 # make mypy and pycharm happy by telling it _type is of type C...

327 # actually just passing _type as first arg!

328 typing.cast(Type_C[typing.Any], _type),

329 value,

330 )

331

332 elif _key in cls.__dict__:

333 # property has default, use that instead.

334 value = cls.__dict__[_key]

335 elif is_optional(_type):

336 # type is optional and not found in __dict__ -> default is None

337 value = None

338 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:

339 # could have a default factory

340 # todo: do something with field.default?

341 value = field.default_factory()

342 else:

343 raise ConfigErrorMissingKey(_key, cls, _type)

344

345 updated[_key] = value

346

347 return updated

348

349

350 def _all_annotations(cls: Type) -> ChainMap[str, Type]:

351 """

352 Returns a dictionary-like ChainMap that includes annotations for all \

353 attributes defined in cls or inherited from superclasses.

354 """

355 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__))

356

357

358 def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, Type]:

359 """

360 Wrapper around `_all_annotations` that filters away any keys in _except.

361

362 It also flattens the ChainMap to a regular dict.

363 """

364 if _except is None:

365 _except = set()

366

367 _all = _all_annotations(cls)

368 return {k: v for k, v in _all.items() if k not in _except}

369

370

371 def check_and_convert_data(

372 cls: typing.Type[C],

373 data: dict[str, typing.Any],

374 _except: typing.Iterable[str],

375 strict: bool = True,

376) -> dict[str, typing.Any]:

377 """

378 Based on class annotations, this prepares the data for `load_into_recurse`.

379

380 1. convert config-keys to python compatible config_keys

381 2. loads custom class type annotations with the same logic (see also `load_recursive`)

382 3. ensures the annotated types match the actual types after loading the config file.

383 """

384 annotations = all_annotations(cls, _except=_except)

385

386 to_load = convert_config(data)

387 to_load = load_recursive(cls, to_load, annotations)

388 if strict:

389 to_load = ensure_types(to_load, annotations)

390

391 return to_load

392

393

394 def _load_into_recurse(

395 cls: typing.Type[C],

396 data: dict[str, typing.Any],

397 init: dict[str, typing.Any] = None,

398 strict: bool = True,

399) -> C:

400 """

401 Loads an instance of `cls` filled with `data`.

402

403 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).

404 `init` can be used to optionally pass extra __init__ arguments. \

405 NOTE: This will overwrite a config key with the same name!

406 """

407 if init is None:

408 init = {}

409

410 # fixme: cls.__init__ can set other keys than the name is in kwargs!!

411

412 if dc.is_dataclass(cls):

413 to_load = check_and_convert_data(cls, data, init.keys(), strict=strict)

414 to_load |= init # add extra init variables (should not happen for a dataclass but whatev)

415

416 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)

417 inst = typing.cast(C, cls(**to_load))

418 else:

419 inst = cls(**init)

420 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict)

421 inst.__dict__.update(**to_load)

422

423 return inst

424

425

426 def _load_into_instance(

427 inst: C,

428 cls: typing.Type[C],

429 data: dict[str, typing.Any],

430 init: dict[str, typing.Any] = None,

431 strict: bool = True,

432) -> C:

433 """

434 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \

435 and thus does not support init.

436

437 """

438 if init is not None:

439 raise ValueError("Can not init an existing instance!")

440

441 existing_data = inst.__dict__

442

443 to_load = check_and_convert_data(cls, data, _except=existing_data.keys(), strict=strict)

444

445 inst.__dict__.update(**to_load)

446

447 return inst

448

449

450 def load_into_class(

451 cls: typing.Type[C],

452 data: T_data,

453 /,

454 key: str = None,

455 init: dict[str, typing.Any] = None,

456 strict: bool = True,

457) -> C:

458 """

459 Shortcut for _load_data + load_into_recurse.

460 """

461 to_load = _load_data(data, key, cls.__name__)

462 return _load_into_recurse(cls, to_load, init=init, strict=strict)

463

464

465 def load_into_instance(

466 inst: C,

467 data: T_data,

468 /,

469 key: str = None,

470 init: dict[str, typing.Any] = None,

471 strict: bool = True,

472) -> C:

473 """

474 Shortcut for _load_data + load_into_existing.

475 """

476 cls = inst.__class__

477 to_load = _load_data(data, key, cls.__name__)

478 return _load_into_instance(inst, cls, to_load, init=init, strict=strict)

479

480

481 def load_into(

482 cls: typing.Type[C],

483 data: T_data,

484 /,

485 key: str = None,

486 init: dict[str, typing.Any] = None,

487 strict: bool = True,

488) -> C:

489 """

490 Load your config into a class (instance).

491

492 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only

493 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`.

494

495 Args:

496 cls: either a class or an existing instance of that class.

497 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)

498 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')

499 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)

500 strict: enable type checks or allow anything?

501

502 """

503 if not isinstance(cls, type):

504 # would not be supported according to mypy, but you can still load_into(instance)

505 return load_into_instance(cls, data, key=key, init=init, strict=strict)

506

507 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'

508 # _cls = typing.cast(typing.Type[C], cls)

509 return load_into_class(cls, data, key=key, init=init, strict=strict)