Coverage for src/configuraptor/core.py: 100%

1"""

2 Contains most of the loading logic.

3"""

4

5 import dataclasses as dc

6 import math

7 import types

8 import typing

9 import warnings

10 from collections import ChainMap

11 from pathlib import Path

12

13 from typeguard import TypeCheckError

14 from typeguard import check_type as _check_type

15

16 from . import loaders

17 from .errors import (

18 ConfigErrorCouldNotConvert,

19 ConfigErrorInvalidType,

20 ConfigErrorMissingKey,

21)

22 from .helpers import camel_to_snake

23 from .postpone import Postponed

24

25# T is a reusable typevar

26 T = typing.TypeVar("T")

27# t_typelike is anything that can be type hinted

28 T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union

29# t_data is anything that can be fed to _load_data

30 T_data = str | Path | dict[str, typing.Any]

31# c = a config class instance, can be any (user-defined) class

32 C = typing.TypeVar("C")

33# type c is a config class

34 Type_C = typing.Type[C]

35

36

37 def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:

38 """

39 If a key contains a dot, traverse the raw dict until the right key was found.

40

41 Example:

42 key = some.nested.key

43 raw = {"some": {"nested": {"key": {"with": "data"}}}}

44 -> {"with": "data"}

45 """

46 parts = key.split(".")

47 while parts:

48 raw = raw[parts.pop(0)]

49

50 return raw

51

52

53 def _guess_key(clsname: str) -> str:

54 """

55 If no key is manually defined for `load_into`, \

56 the class' name is converted to snake_case to use as the default key.

57 """

58 return camel_to_snake(clsname)

59

60

61 def __load_data(

62 data: T_data, key: str = None, classname: str = None, lower_keys: bool = False

63) -> dict[str, typing.Any]:

64 """

65 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.

66

67 E.g. class Tool will be mapped to key tool.

68 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}

69 """

70 if isinstance(data, str):

71 data = Path(data)

72 if isinstance(data, Path):

73 with data.open("rb") as f:

74 loader = loaders.get(data.suffix or data.name)

75 data = loader(f, data.resolve())

76

77 if not data:

78 return {}

79

80 if key is None:

81 # try to guess key by grabbing the first one or using the class name

82 if len(data) == 1:

83 key = list(data.keys())[0]

84 elif classname is not None:

85 key = _guess_key(classname)

86

87 if key:

88 data = _data_for_nested_key(key, data)

89

90 if not data:

91 raise ValueError("No data found!")

92

93 if not isinstance(data, dict):

94 raise ValueError("Data is not a dict!")

95

96 if lower_keys:

97 data = {k.lower(): v for k, v in data.items()}

98

99 return data

100

101

102 def _load_data(data: T_data, key: str = None, classname: str = None, lower_keys: bool = False) -> dict[str, typing.Any]:

103 """

104 Wrapper around __load_data that retries with key="" if anything goes wrong.

105 """

106 try:

107 return __load_data(data, key, classname, lower_keys=lower_keys)

108 except Exception as e:

109 if key != "":

110 return __load_data(data, "", classname, lower_keys=lower_keys)

111 else: # pragma: no cover

112 warnings.warn(f"Data could not be loaded: {e}", source=e)

113 # key already was "", just return data!

114 # (will probably not happen but fallback)

115 return {}

116

117

118 def check_type(value: typing.Any, expected_type: T_typelike) -> bool:

119 """

120 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.).

121

122 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError

123 """

124 try:

125 _check_type(value, expected_type)

126 return True

127 except TypeCheckError:

128 return False

129

130

131 F = typing.TypeVar("F")

132

133

134 def str_to_bool(value: str) -> bool:

135 """

136 Used by convert_between, usually for .env loads.

137

138 Example:

139 SOME_VALUE=TRUE -> True

140 SOME_VALUE=1 -> True

141 SOME_VALUE=Yes -> True

142

143 SOME_VALUE -> None

144 SOME_VALUE=NOpe -> False

145

146 SOME_VALUE=Unrelated -> Error

147 """

148 if not value:

149 return False

150

151 first_letter = value[0].lower()

152 # yes, true, 1

153 if first_letter in {"y", "t", "1"}:

154 return True

155 elif first_letter in {"n", "f", "0"}:

156 return False

157 else:

158 raise ValueError("Not booly.")

159

160

161 def str_to_none(value: str) -> typing.Optional[str]:

162 """

163 Convert a string value of null/none to None, or keep the original string otherwise.

164 """

165 if value.lower() in {"", "null", "none"}:

166 return None

167 else:

168 return value

169

170

171 def convert_between(from_value: F, from_type: typing.Type[F], to_type: type[T]) -> T:

172 """

173 Convert a value between types.

174 """

175 if from_type is str:

176 if to_type is bool:

177 return str_to_bool(from_value) # type: ignore

178 elif to_type is None or to_type is types.NoneType: # noqa: E721

179 return str_to_none(from_value) # type: ignore

180 # default: just convert type:

181 return to_type(from_value) # type: ignore

182

183

184 def ensure_types(

185 data: dict[str, T], annotations: dict[str, type[T]], convert_types: bool = False

186) -> dict[str, T | None]:

187 """

188 Make sure all values in 'data' are in line with the ones stored in 'annotations'.

189

190 If an annotated key in missing from data, it will be filled with None for convenience.

191

192 TODO: python 3.11 exception groups to throw multiple errors at once!

193 """

194 # custom object to use instead of None, since typing.Optional can be None!

195 # cast to T to make mypy happy

196 notfound = typing.cast(T, object())

197 postponed = Postponed()

198

199 final: dict[str, T | None] = {}

200 for key, _type in annotations.items():

201 compare = data.get(key, notfound)

202 if compare is notfound: # pragma: nocover

203 warnings.warn(

204 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"

205 )

206 # skip!

207 continue

208

209 if compare is postponed:

210 # don't do anything with this item!

211 continue

212

213 if not check_type(compare, _type):

214 if convert_types:

215 try:

216 compare = convert_between(compare, type(compare), _type)

217 except (TypeError, ValueError) as e:

218 raise ConfigErrorCouldNotConvert(type(compare), _type, compare) from e

219 else:

220 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)

221

222 final[key] = compare

223

224 return final

225

226

227 def convert_config(items: dict[str, T]) -> dict[str, T]:

228 """

229 Converts the config dict (from toml) or 'overwrites' dict in two ways.

230

231 1. removes any items where the value is None, since in that case the default should be used;

232 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.

233 """

234 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}

235

236

237 Type = typing.Type[typing.Any]

238 T_Type = typing.TypeVar("T_Type", bound=Type)

239

240

241 def is_builtin_type(_type: Type) -> bool:

242 """

243 Returns whether _type is one of the builtin types.

244 """

245 return _type.__module__ in ("__builtin__", "builtins")

246

247

248# def is_builtin_class_instance(obj: typing.Any) -> bool:

249# return is_builtin_type(obj.__class__)

250

251

252 def is_from_types_or_typing(_type: Type) -> bool:

253 """

254 Returns whether _type is one of the stlib typing/types types.

255

256 e.g. types.UnionType or typing.Union

257 """

258 return _type.__module__ in ("types", "typing")

259

260

261 def is_from_other_toml_supported_module(_type: Type) -> bool:

262 """

263 Besides builtins, toml also supports 'datetime' and 'math' types, \

264 so this returns whether _type is a type from these stdlib modules.

265 """

266 return _type.__module__ in ("datetime", "math")

267

268

269 def is_parameterized(_type: Type) -> bool:

270 """

271 Returns whether _type is a parameterized type.

272

273 Examples:

274 list[str] -> True

275 str -> False

276 """

277 return typing.get_origin(_type) is not None

278

279

280 def is_custom_class(_type: Type) -> bool:

281 """

282 Tries to guess if _type is a builtin or a custom (user-defined) class.

283

284 Other logic in this module depends on knowing that.

285 """

286 return (

287 type(_type) is type

288 and not is_builtin_type(_type)

289 and not is_from_other_toml_supported_module(_type)

290 and not is_from_types_or_typing(_type)

291 )

292

293

294 def instance_of_custom_class(var: typing.Any) -> bool:

295 """

296 Calls `is_custom_class` on an instance of a (possibly custom) class.

297 """

298 return is_custom_class(var.__class__)

299

300

301 def is_optional(_type: Type | typing.Any) -> bool:

302 """

303 Tries to guess if _type could be optional.

304

305 Examples:

306 None -> True

307 NoneType -> True

308 typing.Union[str, None] -> True

309 str | None -> True

310 list[str | None] -> False

311 list[str] -> False

312 """

313 if _type and (is_parameterized(_type) and typing.get_origin(_type) in (dict, list)) or (_type is math.nan):

314 # e.g. list[str]

315 # will crash issubclass to test it first here

316 return False

317

318 return (

319 _type is None

320 or types.NoneType in typing.get_args(_type) # union with Nonetype

321 or issubclass(types.NoneType, _type)

322 or issubclass(types.NoneType, type(_type)) # no type # Nonetype

323 )

324

325

326 def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]:

327 """

328 Get Field info for a dataclass cls.

329 """

330 fields = getattr(cls, "__dataclass_fields__", {})

331 return fields.get(key)

332

333

334 def load_recursive(

335 cls: Type, data: dict[str, T], annotations: dict[str, Type], convert_types: bool = False

336) -> dict[str, T]:

337 """

338 For all annotations (recursively gathered from parents with `all_annotations`), \

339 try to resolve the tree of annotations.

340

341 Uses `load_into_recurse`, not itself directly.

342

343 Example:

344 class First:

345 key: str

346

347 class Second:

348 other: First

349

350 # step 1

351 cls = Second

352 data = {"second": {"other": {"key": "anything"}}}

353 annotations: {"other": First}

354

355 # step 1.5

356 data = {"other": {"key": "anything"}

357 annotations: {"other": First}

358

359 # step 2

360 cls = First

361 data = {"key": "anything"}

362 annotations: {"key": str}

363

364

365 TODO: python 3.11 exception groups to throw multiple errors at once!

366 """

367 updated = {}

368

369 for _key, _type in annotations.items():

370 if _key in data:

371 value: typing.Any = data[_key] # value can change so define it as any instead of T

372 if is_parameterized(_type):

373 origin = typing.get_origin(_type)

374 arguments = typing.get_args(_type)

375 if origin is list and arguments and is_custom_class(arguments[0]):

376 subtype = arguments[0]

377 value = [_load_into_recurse(subtype, subvalue, convert_types=convert_types) for subvalue in value]

378

379 elif origin is dict and arguments and is_custom_class(arguments[1]):

380 # e.g. dict[str, Point]

381 subkeytype, subvaluetype = arguments

382 # subkey(type) is not a custom class, so don't try to convert it:

383 value = {

384 subkey: _load_into_recurse(subvaluetype, subvalue, convert_types=convert_types)

385 for subkey, subvalue in value.items()

386 }

387 # elif origin is dict:

388 # keep data the same

389 elif origin is typing.Union and arguments:

390 for arg in arguments:

391 if is_custom_class(arg):

392 value = _load_into_recurse(arg, value, convert_types=convert_types)

393 else:

394 # print(_type, arg, value)

395 ...

396

397 # todo: other parameterized/unions/typing.Optional

398

399 elif is_custom_class(_type):

400 # type must be C (custom class) at this point

401 value = _load_into_recurse(

402 # make mypy and pycharm happy by telling it _type is of type C...

403 # actually just passing _type as first arg!

404 typing.cast(Type_C[typing.Any], _type),

405 value,

406 convert_types=convert_types,

407 )

408

409 elif _key in cls.__dict__:

410 # property has default, use that instead.

411 value = cls.__dict__[_key]

412 elif is_optional(_type):

413 # type is optional and not found in __dict__ -> default is None

414 value = None

415 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:

416 # could have a default factory

417 # todo: do something with field.default?

418 value = field.default_factory()

419 else:

420 raise ConfigErrorMissingKey(_key, cls, _type)

421

422 updated[_key] = value

423

424 return updated

425

426

427 def _all_annotations(cls: Type) -> ChainMap[str, Type]:

428 """

429 Returns a dictionary-like ChainMap that includes annotations for all \

430 attributes defined in cls or inherited from superclasses.

431 """

432 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__))

433

434

435 def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, type[object]]:

436 """

437 Wrapper around `_all_annotations` that filters away any keys in _except.

438

439 It also flattens the ChainMap to a regular dict.

440 """

441 if _except is None:

442 _except = set()

443

444 _all = _all_annotations(cls)

445 return {k: v for k, v in _all.items() if k not in _except}

446

447

448 def check_and_convert_data(

449 cls: typing.Type[C],

450 data: dict[str, typing.Any],

451 _except: typing.Iterable[str],

452 strict: bool = True,

453 convert_types: bool = False,

454) -> dict[str, typing.Any]:

455 """

456 Based on class annotations, this prepares the data for `load_into_recurse`.

457

458 1. convert config-keys to python compatible config_keys

459 2. loads custom class type annotations with the same logic (see also `load_recursive`)

460 3. ensures the annotated types match the actual types after loading the config file.

461 """

462 annotations = all_annotations(cls, _except=_except)

463

464 to_load = convert_config(data)

465 to_load = load_recursive(cls, to_load, annotations, convert_types=convert_types)

466 if strict:

467 to_load = ensure_types(to_load, annotations, convert_types=convert_types)

468

469 return to_load

470

471

472 T_init_list = list[typing.Any]

473 T_init_dict = dict[str, typing.Any]

474 T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None

475

476

477@typing.no_type_check # (mypy doesn't understand 'match' fully yet)

478 def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]:

479 """

480 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple.

481 """

482 if not init:

483 return [], {}

484

485 args: T_init_list = []

486 kwargs: T_init_dict = {}

487 match init:

488 case (args, kwargs):

489 return args, kwargs

490 case [*args]:

491 return args, {}

492 case {**kwargs}:

493 return [], kwargs

494 case _:

495 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.")

496

497

498 def _load_into_recurse(

499 cls: typing.Type[C],

500 data: dict[str, typing.Any],

501 init: T_init = None,

502 strict: bool = True,

503 convert_types: bool = False,

504) -> C:

505 """

506 Loads an instance of `cls` filled with `data`.

507

508 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).

509 `init` can be used to optionally pass extra __init__ arguments. \

510 NOTE: This will overwrite a config key with the same name!

511 """

512 init_args, init_kwargs = _split_init(init)

513

514 if dc.is_dataclass(cls):

515 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict, convert_types=convert_types)

516 if init:

517 raise ValueError("Init is not allowed for dataclasses!")

518

519 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)

520 inst = typing.cast(C, cls(**to_load))

521 else:

522 inst = cls(*init_args, **init_kwargs)

523 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict, convert_types=convert_types)

524 inst.__dict__.update(**to_load)

525

526 return inst

527

528

529 def _load_into_instance(

530 inst: C,

531 cls: typing.Type[C],

532 data: dict[str, typing.Any],

533 init: T_init = None,

534 strict: bool = True,

535 convert_types: bool = False,

536) -> C:

537 """

538 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \

539 and thus does not support init.

540

541 """

542 if init is not None:

543 raise ValueError("Can not init an existing instance!")

544

545 existing_data = inst.__dict__

546

547 to_load = check_and_convert_data(

548 cls, data, _except=existing_data.keys(), strict=strict, convert_types=convert_types

549 )

550

551 inst.__dict__.update(**to_load)

552

553 return inst

554

555

556 def load_into_class(

557 cls: typing.Type[C],

558 data: T_data,

559 /,

560 key: str = None,

561 init: T_init = None,

562 strict: bool = True,

563 lower_keys: bool = False,

564 convert_types: bool = False,

565) -> C:

566 """

567 Shortcut for _load_data + load_into_recurse.

568 """

569 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys)

570 return _load_into_recurse(cls, to_load, init=init, strict=strict, convert_types=convert_types)

571

572

573 def load_into_instance(

574 inst: C,

575 data: T_data,

576 /,

577 key: str = None,

578 init: T_init = None,

579 strict: bool = True,

580 lower_keys: bool = False,

581 convert_types: bool = False,

582) -> C:

583 """

584 Shortcut for _load_data + load_into_existing.

585 """

586 cls = inst.__class__

587 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys)

588 return _load_into_instance(inst, cls, to_load, init=init, strict=strict, convert_types=convert_types)

589

590

591 def load_into(

592 cls: typing.Type[C],

593 data: T_data,

594 /,

595 key: str = None,

596 init: T_init = None,

597 strict: bool = True,

598 lower_keys: bool = False,

599 convert_types: bool = False,

600) -> C:

601 """

602 Load your config into a class (instance).

603

604 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only

605 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`.

606

607 Args:

608 cls: either a class or an existing instance of that class.

609 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)

610 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')

611 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)

612 strict: enable type checks or allow anything?

613 lower_keys: should the config keys be lowercased? (for .env)

614 convert_types: should the types be converted to the annotated type if not yet matching? (for .env)

615

616 """

617 if not isinstance(cls, type):

618 # would not be supported according to mypy, but you can still load_into(instance)

619 return load_into_instance(

620 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types

621 )

622

623 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'

624 # _cls = typing.cast(typing.Type[C], cls)

625 return load_into_class(

626 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types

627 )