Coverage for src/configuraptor/core.py: 100%

1"""

2 Contains most of the loading logic.

3"""

4

5 import dataclasses as dc

6 import math

7 import types

8 import typing

9 import warnings

10 from collections import ChainMap

11 from pathlib import Path

12

13 from typeguard import TypeCheckError

14 from typeguard import check_type as _check_type

15

16 from . import loaders

17 from .errors import ConfigErrorInvalidType, ConfigErrorMissingKey

18 from .helpers import camel_to_snake

19 from .postpone import Postponed

20

21# T is a reusable typevar

22 T = typing.TypeVar("T")

23# t_typelike is anything that can be type hinted

24 T_typelike: typing.TypeAlias = type | types.UnionType # | typing.Union

25# t_data is anything that can be fed to _load_data

26 T_data = str | Path | dict[str, typing.Any]

27# c = a config class instance, can be any (user-defined) class

28 C = typing.TypeVar("C")

29# type c is a config class

30 Type_C = typing.Type[C]

31

32

33 def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:

34 """

35 If a key contains a dot, traverse the raw dict until the right key was found.

36

37 Example:

38 key = some.nested.key

39 raw = {"some": {"nested": {"key": {"with": "data"}}}}

40 -> {"with": "data"}

41 """

42 parts = key.split(".")

43 while parts:

44 raw = raw[parts.pop(0)]

45

46 return raw

47

48

49 def _guess_key(clsname: str) -> str:

50 """

51 If no key is manually defined for `load_into`, \

52 the class' name is converted to snake_case to use as the default key.

53 """

54 return camel_to_snake(clsname)

55

56

57 def _load_data(data: T_data, key: str = None, classname: str = None) -> dict[str, typing.Any]:

58 """

59 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.

60

61 E.g. class Tool will be mapped to key tool.

62 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}

63 """

64 if isinstance(data, str):

65 data = Path(data)

66 if isinstance(data, Path):

67 with data.open("rb") as f:

68 loader = loaders.get(data.suffix)

69 data = loader(f)

70

71 if not data:

72 return {}

73

74 if key is None:

75 # try to guess key by grabbing the first one or using the class name

76 if len(data) == 1:

77 key = list(data.keys())[0]

78 elif classname is not None:

79 key = _guess_key(classname)

80

81 if key:

82 return _data_for_nested_key(key, data)

83 else:

84 # no key found, just return all data

85 return data

86

87

88 def check_type(value: typing.Any, expected_type: T_typelike) -> bool:

89 """

90 Given a variable, check if it matches 'expected_type' (which can be a Union, parameterized generic etc.).

91

92 Based on typeguard but this returns a boolean instead of returning the value or throwing a TypeCheckError

93 """

94 try:

95 _check_type(value, expected_type)

96 return True

97 except TypeCheckError:

98 return False

99

100

101 def ensure_types(data: dict[str, T], annotations: dict[str, type]) -> dict[str, T | None]:

102 """

103 Make sure all values in 'data' are in line with the ones stored in 'annotations'.

104

105 If an annotated key in missing from data, it will be filled with None for convenience.

106

107 TODO: python 3.11 exception groups to throw multiple errors at once!

108 """

109 # custom object to use instead of None, since typing.Optional can be None!

110 # cast to T to make mypy happy

111 notfound = typing.cast(T, object())

112 postponed = Postponed()

113

114 final: dict[str, T | None] = {}

115 for key, _type in annotations.items():

116 compare = data.get(key, notfound)

117 if compare is notfound: # pragma: nocover

118 warnings.warn(

119 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"

120 )

121 # skip!

122 continue

123

124 if compare is postponed:

125 # don't do anything with this item!

126 continue

127

128 if not check_type(compare, _type):

129 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)

130

131 final[key] = compare

132

133 return final

134

135

136 def convert_config(items: dict[str, T]) -> dict[str, T]:

137 """

138 Converts the config dict (from toml) or 'overwrites' dict in two ways.

139

140 1. removes any items where the value is None, since in that case the default should be used;

141 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.

142 """

143 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}

144

145

146 Type = typing.Type[typing.Any]

147 T_Type = typing.TypeVar("T_Type", bound=Type)

148

149

150 def is_builtin_type(_type: Type) -> bool:

151 """

152 Returns whether _type is one of the builtin types.

153 """

154 return _type.__module__ in ("__builtin__", "builtins")

155

156

157# def is_builtin_class_instance(obj: typing.Any) -> bool:

158# return is_builtin_type(obj.__class__)

159

160

161 def is_from_types_or_typing(_type: Type) -> bool:

162 """

163 Returns whether _type is one of the stlib typing/types types.

164

165 e.g. types.UnionType or typing.Union

166 """

167 return _type.__module__ in ("types", "typing")

168

169

170 def is_from_other_toml_supported_module(_type: Type) -> bool:

171 """

172 Besides builtins, toml also supports 'datetime' and 'math' types, \

173 so this returns whether _type is a type from these stdlib modules.

174 """

175 return _type.__module__ in ("datetime", "math")

176

177

178 def is_parameterized(_type: Type) -> bool:

179 """

180 Returns whether _type is a parameterized type.

181

182 Examples:

183 list[str] -> True

184 str -> False

185 """

186 return typing.get_origin(_type) is not None

187

188

189 def is_custom_class(_type: Type) -> bool:

190 """

191 Tries to guess if _type is a builtin or a custom (user-defined) class.

192

193 Other logic in this module depends on knowing that.

194 """

195 return (

196 type(_type) is type

197 and not is_builtin_type(_type)

198 and not is_from_other_toml_supported_module(_type)

199 and not is_from_types_or_typing(_type)

200 )

201

202

203 def instance_of_custom_class(var: typing.Any) -> bool:

204 """

205 Calls `is_custom_class` on an instance of a (possibly custom) class.

206 """

207 return is_custom_class(var.__class__)

208

209

210 def is_optional(_type: Type | typing.Any) -> bool:

211 """

212 Tries to guess if _type could be optional.

213

214 Examples:

215 None -> True

216 NoneType -> True

217 typing.Union[str, None] -> True

218 str | None -> True

219 list[str | None] -> False

220 list[str] -> False

221 """

222 if _type and (is_parameterized(_type) and typing.get_origin(_type) in (dict, list)) or (_type is math.nan):

223 # e.g. list[str]

224 # will crash issubclass to test it first here

225 return False

226

227 return (

228 _type is None

229 or types.NoneType in typing.get_args(_type) # union with Nonetype

230 or issubclass(types.NoneType, _type)

231 or issubclass(types.NoneType, type(_type)) # no type # Nonetype

232 )

233

234

235 def dataclass_field(cls: Type, key: str) -> typing.Optional[dc.Field[typing.Any]]:

236 """

237 Get Field info for a dataclass cls.

238 """

239 fields = getattr(cls, "__dataclass_fields__", {})

240 return fields.get(key)

241

242

243 def load_recursive(cls: Type, data: dict[str, T], annotations: dict[str, Type]) -> dict[str, T]:

244 """

245 For all annotations (recursively gathered from parents with `all_annotations`), \

246 try to resolve the tree of annotations.

247

248 Uses `load_into_recurse`, not itself directly.

249

250 Example:

251 class First:

252 key: str

253

254 class Second:

255 other: First

256

257 # step 1

258 cls = Second

259 data = {"second": {"other": {"key": "anything"}}}

260 annotations: {"other": First}

261

262 # step 1.5

263 data = {"other": {"key": "anything"}

264 annotations: {"other": First}

265

266 # step 2

267 cls = First

268 data = {"key": "anything"}

269 annotations: {"key": str}

270

271

272 TODO: python 3.11 exception groups to throw multiple errors at once!

273 """

274 updated = {}

275

276 for _key, _type in annotations.items():

277 if _key in data:

278 value: typing.Any = data[_key] # value can change so define it as any instead of T

279 if is_parameterized(_type):

280 origin = typing.get_origin(_type)

281 arguments = typing.get_args(_type)

282 if origin is list and arguments and is_custom_class(arguments[0]):

283 subtype = arguments[0]

284 value = [_load_into_recurse(subtype, subvalue) for subvalue in value]

285

286 elif origin is dict and arguments and is_custom_class(arguments[1]):

287 # e.g. dict[str, Point]

288 subkeytype, subvaluetype = arguments

289 # subkey(type) is not a custom class, so don't try to convert it:

290 value = {subkey: _load_into_recurse(subvaluetype, subvalue) for subkey, subvalue in value.items()}

291 # elif origin is dict:

292 # keep data the same

293 elif origin is typing.Union and arguments:

294 for arg in arguments:

295 if is_custom_class(arg):

296 value = _load_into_recurse(arg, value)

297 else:

298 # print(_type, arg, value)

299 ...

300

301 # todo: other parameterized/unions/typing.Optional

302

303 elif is_custom_class(_type):

304 # type must be C (custom class) at this point

305 value = _load_into_recurse(

306 # make mypy and pycharm happy by telling it _type is of type C...

307 # actually just passing _type as first arg!

308 typing.cast(Type_C[typing.Any], _type),

309 value,

310 )

311

312 elif _key in cls.__dict__:

313 # property has default, use that instead.

314 value = cls.__dict__[_key]

315 elif is_optional(_type):

316 # type is optional and not found in __dict__ -> default is None

317 value = None

318 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:

319 # could have a default factory

320 # todo: do something with field.default?

321 value = field.default_factory()

322 else:

323 raise ConfigErrorMissingKey(_key, cls, _type)

324

325 updated[_key] = value

326

327 return updated

328

329

330 def _all_annotations(cls: Type) -> ChainMap[str, Type]:

331 """

332 Returns a dictionary-like ChainMap that includes annotations for all \

333 attributes defined in cls or inherited from superclasses.

334 """

335 return ChainMap(*(c.__annotations__ for c in getattr(cls, "__mro__", []) if "__annotations__" in c.__dict__))

336

337

338 def all_annotations(cls: Type, _except: typing.Iterable[str] = None) -> dict[str, Type]:

339 """

340 Wrapper around `_all_annotations` that filters away any keys in _except.

341

342 It also flattens the ChainMap to a regular dict.

343 """

344 if _except is None:

345 _except = set()

346

347 _all = _all_annotations(cls)

348 return {k: v for k, v in _all.items() if k not in _except}

349

350

351 def check_and_convert_data(

352 cls: typing.Type[C],

353 data: dict[str, typing.Any],

354 _except: typing.Iterable[str],

355 strict: bool = True,

356) -> dict[str, typing.Any]:

357 """

358 Based on class annotations, this prepares the data for `load_into_recurse`.

359

360 1. convert config-keys to python compatible config_keys

361 2. loads custom class type annotations with the same logic (see also `load_recursive`)

362 3. ensures the annotated types match the actual types after loading the config file.

363 """

364 annotations = all_annotations(cls, _except=_except)

365

366 to_load = convert_config(data)

367 to_load = load_recursive(cls, to_load, annotations)

368 if strict:

369 to_load = ensure_types(to_load, annotations)

370

371 return to_load

372

373

374 def _load_into_recurse(

375 cls: typing.Type[C],

376 data: dict[str, typing.Any],

377 init: dict[str, typing.Any] = None,

378 strict: bool = True,

379) -> C:

380 """

381 Loads an instance of `cls` filled with `data`.

382

383 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).

384 `init` can be used to optionally pass extra __init__ arguments. \

385 NOTE: This will overwrite a config key with the same name!

386 """

387 if init is None:

388 init = {}

389

390 # fixme: cls.__init__ can set other keys than the name is in kwargs!!

391

392 if dc.is_dataclass(cls):

393 to_load = check_and_convert_data(cls, data, init.keys(), strict=strict)

394 to_load |= init # add extra init variables (should not happen for a dataclass but whatev)

395

396 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)

397 inst = typing.cast(C, cls(**to_load))

398 else:

399 inst = cls(**init)

400 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict)

401 inst.__dict__.update(**to_load)

402

403 return inst

404

405

406 def _load_into_instance(

407 inst: C,

408 cls: typing.Type[C],

409 data: dict[str, typing.Any],

410 init: dict[str, typing.Any] = None,

411 strict: bool = True,

412) -> C:

413 """

414 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \

415 and thus does not support init.

416

417 """

418 if init is not None:

419 raise ValueError("Can not init an existing instance!")

420

421 existing_data = inst.__dict__

422

423 to_load = check_and_convert_data(cls, data, _except=existing_data.keys(), strict=strict)

424

425 inst.__dict__.update(**to_load)

426

427 return inst

428

429

430 def load_into_class(

431 cls: typing.Type[C],

432 data: T_data,

433 /,

434 key: str = None,

435 init: dict[str, typing.Any] = None,

436 strict: bool = True,

437) -> C:

438 """

439 Shortcut for _load_data + load_into_recurse.

440 """

441 to_load = _load_data(data, key, cls.__name__)

442 return _load_into_recurse(cls, to_load, init=init, strict=strict)

443

444

445 def load_into_instance(

446 inst: C,

447 data: T_data,

448 /,

449 key: str = None,

450 init: dict[str, typing.Any] = None,

451 strict: bool = True,

452) -> C:

453 """

454 Shortcut for _load_data + load_into_existing.

455 """

456 cls = inst.__class__

457 to_load = _load_data(data, key, cls.__name__)

458 return _load_into_instance(inst, cls, to_load, init=init, strict=strict)

459

460

461 def load_into(

462 cls: typing.Type[C],

463 data: T_data,

464 /,

465 key: str = None,

466 init: dict[str, typing.Any] = None,

467 strict: bool = True,

468) -> C:

469 """

470 Load your config into a class (instance).

471

472 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only

473 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`.

474

475 Args:

476 cls: either a class or an existing instance of that class.

477 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)

478 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')

479 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)

480 strict: enable type checks or allow anything?

481

482 """

483 if not isinstance(cls, type):

484 # would not be supported according to mypy, but you can still load_into(instance)

485 return load_into_instance(cls, data, key=key, init=init, strict=strict)

486

487 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'

488 # _cls = typing.cast(typing.Type[C], cls)

489 return load_into_class(cls, data, key=key, init=init, strict=strict)