Coverage for src/configuraptor/core.py: 100%

1"""

2 Contains most of the loading logic.

3"""

4

5 import dataclasses as dc

6 import math

7 import types

8 import typing

9 import warnings

10 from collections import ChainMap

11 from pathlib import Path

12

13 from typeguard import TypeCheckError

14 from typeguard import check_type as _check_type

15

16 from . import loaders

17 from .errors import ConfigErrorInvalidType, ConfigErrorMissingKey, ConfigErrorCouldNotConvert

18 from .helpers import camel_to_snake

19 from .postpone import Postponed

20

21# T is a reusable typevar

22 T = typing.TypeVar("T")

23# t_typelike is anything that can be type hinted

24 T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union

25# t_data is anything that can be fed to _load_data

26 T_data = str | Path | dict[str, typing.Any]

27# c = a config class instance, can be any (user-defined) class

28 C = typing.TypeVar("C")

29# type c is a config class

30 Type_C = typing.Type[C]

31

32

33 def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:

34 """

35 If a key contains a dot, traverse the raw dict until the right key was found.

36

37 Example:

38 key = some.nested.key

39 raw = {"some": {"nested": {"key": {"with": "data"}}}}

40 -> {"with": "data"}

41 """

42 parts = key.split(".")

43 while parts:

44 raw = raw[parts.pop(0)]

45

46 return raw

47

48

49 def _guess_key(clsname: str) -> str:

50 """

51 If no key is manually defined for `load_into`, \

52 the class' name is converted to snake_case to use as the default key.

53 """

54 return camel_to_snake(clsname)

55

56

57 def __load_data(

58 data: T_data, key: str = None, classname: str = None, lower_keys: bool = False

59) -> dict[str, typing.Any]:

60 """

61 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.

62

63 E.g. class Tool will be mapped to key tool.

64 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}

65 """

66 if isinstance(data, str):

67 data = Path(data)

68 if isinstance(data, Path):

69 with data.open("rb") as f:

70 loader = loaders.get(data.suffix or data.name)

71 data = loader(f)

72

73 if not data:

74 return {}

75

76 if key is None:

77 # try to guess key by grabbing the first one or using the class name

78 if len(data) == 1:

79 key = list(data.keys())[0]

80 elif classname is not None:

81 key = _guess_key(classname)

82

83 if key:

84 data = _data_for_nested_key(key, data)

85

86 if not data:

87 raise ValueError("No data found!")

88

89 if not isinstance(data, dict):

90 raise ValueError("Data is not a dict!")

91

92 if lower_keys:

93 data = {k.lower(): v for k, v in data.items()}

94

95 return data

96

97

98 def _load_data(data: T_data, key: str = None, classname: str = None, lower_keys: bool = False) -> dict[str, typing.Any]:

99 """

100 Wrapper around __load_data that retries with key="" if anything goes wrong.

101 """

102 try:

103 return __load_data(data, key, classname, lower_keys=lower_keys)

104 except Exception as e:

105 if key != "":

106 return __load_data(data, "", classname, lower_keys=lower_keys)

107 else: # pragma: no cover

108 warnings.warn(f"Data could not be loaded: {e}", source=e)

109 # key already was "", just return data!

110 # (will probably not happen but fallback)

111 return {}

112

113

114 def check_type(value: typing.Any, expected_type: T_typelike) -> bool:

115 """

116 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.).

117

118 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError

119 """

120 try:

121 _check_type(value, expected_type)

122 return True

123 except TypeCheckError:

124 return False

125

126

127 F = typing.TypeVar("F")

128

129

130 def str_to_bool(value: str) -> bool:

131 """

132 Used by convert_between, usually for .env loads.

133

134 Example:

135 SOME_VALUE=TRUE -> True

136 SOME_VALUE=1 -> True

137 SOME_VALUE=Yes -> True

138

139 SOME_VALUE -> None

140 SOME_VALUE=NOpe -> False

141

142 SOME_VALUE=Unrelated -> Error

143 """

144 if not value:

145 return False

146

147 first_letter = value[0].lower()

148 # yes, true, 1

149 if first_letter in {"y", "t", "1"}:

150 return True

151 elif first_letter in {"n", "f", "0"}:

152 return False

153 else:

154 raise ValueError("Not booly.")

155

156

157 def convert_between(from_value: F, from_type: typing.Type[F], to_type: type[T]) -> T:

158 """

159 Convert a value between types.

160 """

161 if from_type is str and to_type is bool:

162 return str_to_bool(from_value)

163 # default: just convert type:

164 return to_type(from_value)

165

166

167 def ensure_types(

168 data: dict[str, T], annotations: dict[str, type[T]], convert_types: bool = False

169) -> dict[str, T | None]:

170 """

171 Make sure all values in 'data' are in line with the ones stored in 'annotations'.

172

173 If an annotated key in missing from data, it will be filled with None for convenience.

174

175 TODO: python 3.11 exception groups to throw multiple errors at once!

176 """

177 # custom object to use instead of None, since typing.Optional can be None!

178 # cast to T to make mypy happy

179 notfound = typing.cast(T, object())

180 postponed = Postponed()

181

182 final: dict[str, T | None] = {}

183 for key, _type in annotations.items():

184 compare = data.get(key, notfound)

185 if compare is notfound: # pragma: nocover

186 warnings.warn(

187 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"

188 )

189 # skip!

190 continue

191

192 if compare is postponed:

193 # don't do anything with this item!

194 continue

195

196 if not check_type(compare, _type):

197 if convert_types:

198 try:

199 compare = convert_between(compare, type(compare), _type)

200 except (TypeError, ValueError) as e:

201 raise ConfigErrorCouldNotConvert(type(compare), _type, compare) from e

202 else:

203 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)

204

205 final[key] = compare

206

207 return final

208

209

210 def convert_config(items: dict[str, T]) -> dict[str, T]:

211 """

212 Converts the config dict (from toml) or 'overwrites' dict in two ways.

213

214 1. removes any items where the value is None, since in that case the default should be used;

215 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.

216 """

217 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}

218

219

220 Type = typing.Type[typing.Any]

221 T_Type = typing.TypeVar("T_Type", bound=Type)

222

223

224 def is_builtin_type(_type: Type) -> bool:

225 """

226 Returns whether _type is one of the builtin types.

227 """

228 return _type.__module__ in ("__builtin__", "builtins")

229

230

231# def is_builtin_class_instance(obj: typing.Any) -> bool:

232# return is_builtin_type(obj.__class__)

233

234

235 def is_from_types_or_typing(_type: Type) -> bool:

236 """

237 Returns whether _type is one of the stlib typing/types types.

238

239 e.g. types.UnionType or typing.Union

240 """

241 return _type.__module__ in ("types", "typing")

242

243

244 def is_from_other_toml_supported_module(_type: Type) -> bool:

245 """

246 Besides builtins, toml also supports 'datetime' and 'math' types, \

247 so this returns whether _type is a type from these stdlib modules.

248 """

249 return _type.__module__ in ("datetime", "math")

250

251

252 def is_parameterized(_type: Type) -> bool:

253 """

254 Returns whether _type is a parameterized type.

255

256 Examples:

257 list[str] -> True

258 str -> False

259 """

260 return typing.get_origin(_type) is not None

261

262

263 def is_custom_class(_type: Type) -> bool:

264 """

265 Tries to guess if _type is a builtin or a custom (user-defined) class.

266

267 Other logic in this module depends on knowing that.

268 """

269 return (

270 type(_type) is type

271 and not is_builtin_type(_type)

272 and not is_from_other_toml_supported_module(_type)

273 and not is_from_types_or_typing(_type)

274 )

275

276

277 def instance_of_custom_class(var: typing.Any) -> bool:

278 """

279 Calls `is_custom_class` on an instance of a (possibly custom) class.

280 """

281 return is_custom_class(var.__class__)

282

283

284 def is_optional(_type: Type | typing.Any) -> bool:

285 """

286 Tries to guess if _type could be optional.

287

288 Examples:

289 None -> True

290 NoneType -> True

291 typing.Union[str, None] -> True

292 str | None -> True

293 list[str | None] -> False

294 list[str] -> False

295 """

296 if _type and (is_parameterized(_type) and typing.get_origin(_type) in (dict, list)) or (_type is math.nan):

297 # e.g. list[str]

298 # will crash issubclass to test it first here

299 return False

300

301 return (

302 _type is None

303 or types.NoneType in typing.get_args(_type) # union with Nonetype

304 or issubclass(types.NoneType, _type)

305 or issubclass(types.NoneType, type(_type)) # no type # Nonetype

306 )

307

308

309 def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]:

310 """

311 Get Field info for a dataclass cls.

312 """

313 fields = getattr(cls, "__dataclass_fields__", {})

314 return fields.get(key)

315

316

317 def load_recursive(cls: Type, data: dict[str, T], annotations: dict[str, Type]) -> dict[str, T]:

318 """

319 For all annotations (recursively gathered from parents with `all_annotations`), \

320 try to resolve the tree of annotations.

321

322 Uses `load_into_recurse`, not itself directly.

323

324 Example:

325 class First:

326 key: str

327

328 class Second:

329 other: First

330

331 # step 1

332 cls = Second

333 data = {"second": {"other": {"key": "anything"}}}

334 annotations: {"other": First}

335

336 # step 1.5

337 data = {"other": {"key": "anything"}

338 annotations: {"other": First}

339

340 # step 2

341 cls = First

342 data = {"key": "anything"}

343 annotations: {"key": str}

344

345

346 TODO: python 3.11 exception groups to throw multiple errors at once!

347 """

348 updated = {}

349

350 for _key, _type in annotations.items():

351 if _key in data:

352 value: typing.Any = data[_key] # value can change so define it as any instead of T

353 if is_parameterized(_type):

354 origin = typing.get_origin(_type)

355 arguments = typing.get_args(_type)

356 if origin is list and arguments and is_custom_class(arguments[0]):

357 subtype = arguments[0]

358 value = [_load_into_recurse(subtype, subvalue) for subvalue in value]

359

360 elif origin is dict and arguments and is_custom_class(arguments[1]):

361 # e.g. dict[str, Point]

362 subkeytype, subvaluetype = arguments

363 # subkey(type) is not a custom class, so don't try to convert it:

364 value = {subkey: _load_into_recurse(subvaluetype, subvalue) for subkey, subvalue in value.items()}

365 # elif origin is dict:

366 # keep data the same

367 elif origin is typing.Union and arguments:

368 for arg in arguments:

369 if is_custom_class(arg):

370 value = _load_into_recurse(arg, value)

371 else:

372 # print(_type, arg, value)

373 ...

374

375 # todo: other parameterized/unions/typing.Optional

376

377 elif is_custom_class(_type):

378 # type must be C (custom class) at this point

379 value = _load_into_recurse(

380 # make mypy and pycharm happy by telling it _type is of type C...

381 # actually just passing _type as first arg!

382 typing.cast(Type_C[typing.Any], _type),

383 value,

384 )

385

386 elif _key in cls.__dict__:

387 # property has default, use that instead.

388 value = cls.__dict__[_key]

389 elif is_optional(_type):

390 # type is optional and not found in __dict__ -> default is None

391 value = None

392 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:

393 # could have a default factory

394 # todo: do something with field.default?

395 value = field.default_factory()

396 else:

397 raise ConfigErrorMissingKey(_key, cls, _type)

398

399 updated[_key] = value

400

401 return updated

402

403

404 def _all_annotations(cls: Type) -> ChainMap[str, Type]:

405 """

406 Returns a dictionary-like ChainMap that includes annotations for all \

407 attributes defined in cls or inherited from superclasses.

408 """

409 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__))

410

411

412 def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, type[object]]:

413 """

414 Wrapper around `_all_annotations` that filters away any keys in _except.

415

416 It also flattens the ChainMap to a regular dict.

417 """

418 if _except is None:

419 _except = set()

420

421 _all = _all_annotations(cls)

422 return {k: v for k, v in _all.items() if k not in _except}

423

424

425 def check_and_convert_data(

426 cls: typing.Type[C],

427 data: dict[str, typing.Any],

428 _except: typing.Iterable[str],

429 strict: bool = True,

430 convert_types: bool = False,

431) -> dict[str, typing.Any]:

432 """

433 Based on class annotations, this prepares the data for `load_into_recurse`.

434

435 1. convert config-keys to python compatible config_keys

436 2. loads custom class type annotations with the same logic (see also `load_recursive`)

437 3. ensures the annotated types match the actual types after loading the config file.

438 """

439 annotations = all_annotations(cls, _except=_except)

440

441 to_load = convert_config(data)

442 to_load = load_recursive(cls, to_load, annotations)

443 if strict:

444 to_load = ensure_types(to_load, annotations, convert_types=convert_types)

445

446 return to_load

447

448

449 T_init_list = list[typing.Any]

450 T_init_dict = dict[str, typing.Any]

451 T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None

452

453

454@typing.no_type_check # (mypy doesn't understand 'match' fully yet)

455 def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]:

456 """

457 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple.

458 """

459 if not init:

460 return [], {}

461

462 args: T_init_list = []

463 kwargs: T_init_dict = {}

464 match init:

465 case (args, kwargs):

466 return args, kwargs

467 case [*args]:

468 return args, {}

469 case {**kwargs}:

470 return [], kwargs

471 case _:

472 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.")

473

474

475 def _load_into_recurse(

476 cls: typing.Type[C],

477 data: dict[str, typing.Any],

478 init: T_init = None,

479 strict: bool = True,

480 convert_types: bool = False,

481) -> C:

482 """

483 Loads an instance of `cls` filled with `data`.

484

485 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).

486 `init` can be used to optionally pass extra __init__ arguments. \

487 NOTE: This will overwrite a config key with the same name!

488 """

489 init_args, init_kwargs = _split_init(init)

490

491 if dc.is_dataclass(cls):

492 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict, convert_types=convert_types)

493 if init:

494 raise ValueError("Init is not allowed for dataclasses!")

495

496 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)

497 inst = typing.cast(C, cls(**to_load))

498 else:

499 inst = cls(*init_args, **init_kwargs)

500 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict, convert_types=convert_types)

501 inst.__dict__.update(**to_load)

502

503 return inst

504

505

506 def _load_into_instance(

507 inst: C,

508 cls: typing.Type[C],

509 data: dict[str, typing.Any],

510 init: T_init = None,

511 strict: bool = True,

512 convert_types: bool = False,

513) -> C:

514 """

515 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \

516 and thus does not support init.

517

518 """

519 if init is not None:

520 raise ValueError("Can not init an existing instance!")

521

522 existing_data = inst.__dict__

523

524 to_load = check_and_convert_data(

525 cls, data, _except=existing_data.keys(), strict=strict, convert_types=convert_types

526 )

527

528 inst.__dict__.update(**to_load)

529

530 return inst

531

532

533 def load_into_class(

534 cls: typing.Type[C],

535 data: T_data,

536 /,

537 key: str = None,

538 init: T_init = None,

539 strict: bool = True,

540 lower_keys: bool = False,

541 convert_types: bool = False,

542) -> C:

543 """

544 Shortcut for _load_data + load_into_recurse.

545 """

546 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys)

547 return _load_into_recurse(cls, to_load, init=init, strict=strict, convert_types=convert_types)

548

549

550 def load_into_instance(

551 inst: C,

552 data: T_data,

553 /,

554 key: str = None,

555 init: T_init = None,

556 strict: bool = True,

557 lower_keys: bool = False,

558 convert_types: bool = False,

559) -> C:

560 """

561 Shortcut for _load_data + load_into_existing.

562 """

563 cls = inst.__class__

564 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys)

565 return _load_into_instance(inst, cls, to_load, init=init, strict=strict, convert_types=convert_types)

566

567

568 def load_into(

569 cls: typing.Type[C],

570 data: T_data,

571 /,

572 key: str = None,

573 init: T_init = None,

574 strict: bool = True,

575 lower_keys: bool = False,

576 convert_types: bool = False,

577) -> C:

578 """

579 Load your config into a class (instance).

580

581 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only

582 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`.

583

584 Args:

585 cls: either a class or an existing instance of that class.

586 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)

587 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')

588 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)

589 strict: enable type checks or allow anything?

590

591 """

592 if not isinstance(cls, type):

593 # would not be supported according to mypy, but you can still load_into(instance)

594 return load_into_instance(

595 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types

596 )

597

598 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'

599 # _cls = typing.cast(typing.Type[C], cls)

600 return load_into_class(

601 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types

602 )