Coverage for src/configuraptor/core.py: 100%

1"""

2 Contains most of the loading logic.

3"""

4

5 import dataclasses as dc

6 import math

7 import types

8 import typing

9 import warnings

10 from collections import ChainMap

11 from pathlib import Path

12

13 from typeguard import TypeCheckError

14 from typeguard import check_type as _check_type

15

16 from . import loaders

17 from .errors import ConfigErrorInvalidType, ConfigErrorMissingKey

18 from .helpers import camel_to_snake

19 from .postpone import Postponed

20

21# T is a reusable typevar

22 T = typing.TypeVar("T")

23# t_typelike is anything that can be type hinted

24 T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union

25# t_data is anything that can be fed to _load_data

26 T_data = str | Path | dict[str, typing.Any]

27# c = a config class instance, can be any (user-defined) class

28 C = typing.TypeVar("C")

29# type c is a config class

30 Type_C = typing.Type[C]

31

32

33 def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:

34 """

35 If a key contains a dot, traverse the raw dict until the right key was found.

36

37 Example:

38 key = some.nested.key

39 raw = {"some": {"nested": {"key": {"with": "data"}}}}

40 -> {"with": "data"}

41 """

42 parts = key.split(".")

43 while parts:

44 raw = raw[parts.pop(0)]

45

46 return raw

47

48

49 def _guess_key(clsname: str) -> str:

50 """

51 If no key is manually defined for `load_into`, \

52 the class' name is converted to snake_case to use as the default key.

53 """

54 return camel_to_snake(clsname)

55

56

57 def __load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]:

58 """

59 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.

60

61 E.g. class Tool will be mapped to key tool.

62 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}

63 """

64 if isinstance(data, str):

65 data = Path(data)

66 if isinstance(data, Path):

67 with data.open("rb") as f:

68 loader = loaders.get(data.suffix)

69 data = loader(f)

70

71 if not data:

72 return {}

73

74 if key is None:

75 # try to guess key by grabbing the first one or using the class name

76 if len(data) == 1:

77 key = list(data.keys())[0]

78 elif classname is not None:

79 key = _guess_key(classname)

80

81 if key:

82 data = _data_for_nested_key(key, data)

83

84 if not data:

85 raise ValueError("No data found!")

86

87 if not isinstance(data, dict):

88 raise ValueError("Data is not a dict!")

89

90 return data

91

92

93 def _load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]:

94 """

95 Wrapper around __load_data that retries with key="" if anything goes wrong.

96 """

97 try:

98 return __load_data(data, key, classname)

99 except Exception as e:

100 if key != "":

101 return __load_data(data, "", classname)

102 else: # pragma: no cover

103 warnings.warn(f"Data could not be loaded: {e}", source=e)

104 # key already was "", just return data!

105 # (will probably not happen but fallback)

106 return {}

107

108

109 def check_type(value: typing.Any, expected_type: T_typelike) -> bool:

110 """

111 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.).

112

113 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError

114 """

115 try:

116 _check_type(value, expected_type)

117 return True

118 except TypeCheckError:

119 return False

120

121

122 def ensure_types(data: dict[str, T], annotations: dict[str, type]) -> dict[str, T | None]:

123 """

124 Make sure all values in 'data' are in line with the ones stored in 'annotations'.

125

126 If an annotated key in missing from data, it will be filled with None for convenience.

127

128 TODO: python 3.11 exception groups to throw multiple errors at once!

129 """

130 # custom object to use instead of None, since typing.Optional can be None!

131 # cast to T to make mypy happy

132 notfound = typing.cast(T, object())

133 postponed = Postponed()

134

135 final: dict[str, T | None] = {}

136 for key, _type in annotations.items():

137 compare = data.get(key, notfound)

138 if compare is notfound: # pragma: nocover

139 warnings.warn(

140 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"

141 )

142 # skip!

143 continue

144

145 if compare is postponed:

146 # don't do anything with this item!

147 continue

148

149 if not check_type(compare, _type):

150 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)

151

152 final[key] = compare

153

154 return final

155

156

157 def convert_config(items: dict[str, T]) -> dict[str, T]:

158 """

159 Converts the config dict (from toml) or 'overwrites' dict in two ways.

160

161 1. removes any items where the value is None, since in that case the default should be used;

162 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.

163 """

164 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}

165

166

167 Type = typing.Type[typing.Any]

168 T_Type = typing.TypeVar("T_Type", bound=Type)

169

170

171 def is_builtin_type(_type: Type) -> bool:

172 """

173 Returns whether _type is one of the builtin types.

174 """

175 return _type.__module__ in ("__builtin__", "builtins")

176

177

178# def is_builtin_class_instance(obj: typing.Any) -> bool:

179# return is_builtin_type(obj.__class__)

180

181

182 def is_from_types_or_typing(_type: Type) -> bool:

183 """

184 Returns whether _type is one of the stlib typing/types types.

185

186 e.g. types.UnionType or typing.Union

187 """

188 return _type.__module__ in ("types", "typing")

189

190

191 def is_from_other_toml_supported_module(_type: Type) -> bool:

192 """

193 Besides builtins, toml also supports 'datetime' and 'math' types, \

194 so this returns whether _type is a type from these stdlib modules.

195 """

196 return _type.__module__ in ("datetime", "math")

197

198

199 def is_parameterized(_type: Type) -> bool:

200 """

201 Returns whether _type is a parameterized type.

202

203 Examples:

204 list[str] -> True

205 str -> False

206 """

207 return typing.get_origin(_type) is not None

208

209

210 def is_custom_class(_type: Type) -> bool:

211 """

212 Tries to guess if _type is a builtin or a custom (user-defined) class.

213

214 Other logic in this module depends on knowing that.

215 """

216 return (

217 type(_type) is type

218 and not is_builtin_type(_type)

219 and not is_from_other_toml_supported_module(_type)

220 and not is_from_types_or_typing(_type)

221 )

222

223

224 def instance_of_custom_class(var: typing.Any) -> bool:

225 """

226 Calls `is_custom_class` on an instance of a (possibly custom) class.

227 """

228 return is_custom_class(var.__class__)

229

230

231 def is_optional(_type: Type | typing.Any) -> bool:

232 """

233 Tries to guess if _type could be optional.

234

235 Examples:

236 None -> True

237 NoneType -> True

238 typing.Union[str, None] -> True

239 str | None -> True

240 list[str | None] -> False

241 list[str] -> False

242 """

243 if _type and (is_parameterized(_type) and typing.get_origin(_type) in (dict, list)) or (_type is math.nan):

244 # e.g. list[str]

245 # will crash issubclass to test it first here

246 return False

247

248 return (

249 _type is None

250 or types.NoneType in typing.get_args(_type) # union with Nonetype

251 or issubclass(types.NoneType, _type)

252 or issubclass(types.NoneType, type(_type)) # no type # Nonetype

253 )

254

255

256 def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]:

257 """

258 Get Field info for a dataclass cls.

259 """

260 fields = getattr(cls, "__dataclass_fields__", {})

261 return fields.get(key)

262

263

264 def load_recursive(cls: Type, data: dict[str, T], annotations: dict[str, Type]) -> dict[str, T]:

265 """

266 For all annotations (recursively gathered from parents with `all_annotations`), \

267 try to resolve the tree of annotations.

268

269 Uses `load_into_recurse`, not itself directly.

270

271 Example:

272 class First:

273 key: str

274

275 class Second:

276 other: First

277

278 # step 1

279 cls = Second

280 data = {"second": {"other": {"key": "anything"}}}

281 annotations: {"other": First}

282

283 # step 1.5

284 data = {"other": {"key": "anything"}

285 annotations: {"other": First}

286

287 # step 2

288 cls = First

289 data = {"key": "anything"}

290 annotations: {"key": str}

291

292

293 TODO: python 3.11 exception groups to throw multiple errors at once!

294 """

295 updated = {}

296

297 for _key, _type in annotations.items():

298 if _key in data:

299 value: typing.Any = data[_key] # value can change so define it as any instead of T

300 if is_parameterized(_type):

301 origin = typing.get_origin(_type)

302 arguments = typing.get_args(_type)

303 if origin is list and arguments and is_custom_class(arguments[0]):

304 subtype = arguments[0]

305 value = [_load_into_recurse(subtype, subvalue) for subvalue in value]

306

307 elif origin is dict and arguments and is_custom_class(arguments[1]):

308 # e.g. dict[str, Point]

309 subkeytype, subvaluetype = arguments

310 # subkey(type) is not a custom class, so don't try to convert it:

311 value = {subkey: _load_into_recurse(subvaluetype, subvalue) for subkey, subvalue in value.items()}

312 # elif origin is dict:

313 # keep data the same

314 elif origin is typing.Union and arguments:

315 for arg in arguments:

316 if is_custom_class(arg):

317 value = _load_into_recurse(arg, value)

318 else:

319 # print(_type, arg, value)

320 ...

321

322 # todo: other parameterized/unions/typing.Optional

323

324 elif is_custom_class(_type):

325 # type must be C (custom class) at this point

326 value = _load_into_recurse(

327 # make mypy and pycharm happy by telling it _type is of type C...

328 # actually just passing _type as first arg!

329 typing.cast(Type_C[typing.Any], _type),

330 value,

331 )

332

333 elif _key in cls.__dict__:

334 # property has default, use that instead.

335 value = cls.__dict__[_key]

336 elif is_optional(_type):

337 # type is optional and not found in __dict__ -> default is None

338 value = None

339 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:

340 # could have a default factory

341 # todo: do something with field.default?

342 value = field.default_factory()

343 else:

344 raise ConfigErrorMissingKey(_key, cls, _type)

345

346 updated[_key] = value

347

348 return updated

349

350

351 def _all_annotations(cls: Type) -> ChainMap[str, Type]:

352 """

353 Returns a dictionary-like ChainMap that includes annotations for all \

354 attributes defined in cls or inherited from superclasses.

355 """

356 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__))

357

358

359 def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, Type]:

360 """

361 Wrapper around `_all_annotations` that filters away any keys in _except.

362

363 It also flattens the ChainMap to a regular dict.

364 """

365 if _except is None:

366 _except = set()

367

368 _all = _all_annotations(cls)

369 return {k: v for k, v in _all.items() if k not in _except}

370

371

372 def check_and_convert_data(

373 cls: typing.Type[C],

374 data: dict[str, typing.Any],

375 _except: typing.Iterable[str],

376 strict: bool = True,

377) -> dict[str, typing.Any]:

378 """

379 Based on class annotations, this prepares the data for `load_into_recurse`.

380

381 1. convert config-keys to python compatible config_keys

382 2. loads custom class type annotations with the same logic (see also `load_recursive`)

383 3. ensures the annotated types match the actual types after loading the config file.

384 """

385 annotations = all_annotations(cls, _except=_except)

386

387 to_load = convert_config(data)

388 to_load = load_recursive(cls, to_load, annotations)

389 if strict:

390 to_load = ensure_types(to_load, annotations)

391

392 return to_load

393

394

395 T_init_list = list[typing.Any]

396 T_init_dict = dict[str, typing.Any]

397 T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None

398

399

400@typing.no_type_check # (mypy doesn't understand 'match' fully yet)

401 def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]:

402 """

403 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple.

404 """

405 if not init:

406 return [], {}

407

408 args: T_init_list = []

409 kwargs: T_init_dict = {}

410 match init:

411 case (args, kwargs):

412 return args, kwargs

413 case [*args]:

414 return args, {}

415 case {**kwargs}:

416 return [], kwargs

417 case _:

418 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.")

419

420

421 def _load_into_recurse(

422 cls: typing.Type[C],

423 data: dict[str, typing.Any],

424 init: T_init = None,

425 strict: bool = True,

426) -> C:

427 """

428 Loads an instance of `cls` filled with `data`.

429

430 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).

431 `init` can be used to optionally pass extra __init__ arguments. \

432 NOTE: This will overwrite a config key with the same name!

433 """

434 init_args, init_kwargs = _split_init(init)

435

436 if dc.is_dataclass(cls):

437 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict)

438 if init:

439 raise ValueError("Init is not allowed for dataclasses!")

440

441 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)

442 inst = typing.cast(C, cls(**to_load))

443 else:

444 inst = cls(*init_args, **init_kwargs)

445 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict)

446 inst.__dict__.update(**to_load)

447

448 return inst

449

450

451 def _load_into_instance(

452 inst: C,

453 cls: typing.Type[C],

454 data: dict[str, typing.Any],

455 init: T_init = None,

456 strict: bool = True,

457) -> C:

458 """

459 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \

460 and thus does not support init.

461

462 """

463 if init is not None:

464 raise ValueError("Can not init an existing instance!")

465

466 existing_data = inst.__dict__

467

468 to_load = check_and_convert_data(cls, data, _except=existing_data.keys(), strict=strict)

469

470 inst.__dict__.update(**to_load)

471

472 return inst

473

474

475 def load_into_class(

476 cls: typing.Type[C],

477 data: T_data,

478 /,

479 key: str = None,

480 init: T_init = None,

481 strict: bool = True,

482) -> C:

483 """

484 Shortcut for _load_data + load_into_recurse.

485 """

486 to_load = _load_data(data, key, cls.__name__)

487 return _load_into_recurse(cls, to_load, init=init, strict=strict)

488

489

490 def load_into_instance(

491 inst: C,

492 data: T_data,

493 /,

494 key: str = None,

495 init: T_init = None,

496 strict: bool = True,

497) -> C:

498 """

499 Shortcut for _load_data + load_into_existing.

500 """

501 cls = inst.__class__

502 to_load = _load_data(data, key, cls.__name__)

503 return _load_into_instance(inst, cls, to_load, init=init, strict=strict)

504

505

506 def load_into(

507 cls: typing.Type[C],

508 data: T_data,

509 /,

510 key: str = None,

511 init: T_init = None,

512 strict: bool = True,

513) -> C:

514 """

515 Load your config into a class (instance).

516

517 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only

518 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`.

519

520 Args:

521 cls: either a class or an existing instance of that class.

522 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)

523 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')

524 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)

525 strict: enable type checks or allow anything?

526

527 """

528 if not isinstance(cls, type):

529 # would not be supported according to mypy, but you can still load_into(instance)

530 return load_into_instance(cls, data, key=key, init=init, strict=strict)

531

532 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'

533 # _cls = typing.cast(typing.Type[C], cls)

534 return load_into_class(cls, data, key=key, init=init, strict=strict)