Coverage for src/configuraptor/core.py: 100%

1"""

2 Contains most of the loading logic.

3"""

4

5 import dataclasses as dc

6 import typing

7 import warnings

8 from pathlib import Path

9

10 from . import loaders

11 from .abs import C, T, T_data, Type_C

12 from .binary_config import BinaryConfig

13 from .errors import (

14 ConfigErrorCouldNotConvert,

15 ConfigErrorInvalidType,

16 ConfigErrorMissingKey,

17)

18 from .helpers import (

19 all_annotations,

20 camel_to_snake,

21 check_type,

22 dataclass_field,

23 find_pyproject_toml,

24 is_custom_class,

25 is_optional,

26 is_parameterized,

27)

28 from .postpone import Postponed

29 from .type_converters import CONVERTERS

30

31

32 def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:

33 """

34 If a key contains a dot, traverse the raw dict until the right key was found.

35

36 Example:

37 key = some.nested.key

38 raw = {"some": {"nested": {"key": {"with": "data"}}}}

39 -> {"with": "data"}

40 """

41 parts = key.split(".")

42 while parts:

43 key = parts.pop(0)

44 if key not in raw:

45 return {}

46

47 raw = raw[key]

48

49 return raw

50

51

52 def _guess_key(clsname: str) -> str:

53 """

54 If no key is manually defined for `load_into`, \

55 the class' name is converted to snake_case to use as the default key.

56 """

57 return camel_to_snake(clsname)

58

59

60 def __load_data(

61 data: T_data,

62 key: str = None,

63 classname: str = None,

64 lower_keys: bool = False,

65 allow_types: tuple[type, ...] = (dict,),

66) -> dict[str, typing.Any]:

67 """

68 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.

69

70 E.g. class Tool will be mapped to key tool.

71 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}

72 """

73 if isinstance(data, list):

74 if not data:

75 raise ValueError("Empty list passed!")

76

77 final_data: dict[str, typing.Any] = {}

78 for source in data:

79 final_data |= _load_data(source, key=key, classname=classname, lower_keys=True, allow_types=allow_types)

80

81 return final_data

82

83 if isinstance(data, str):

84 data = Path(data)

85 if isinstance(data, Path):

86 with data.open("rb") as f:

87 loader = loaders.get(data.suffix or data.name)

88 data = loader(f, data.resolve())

89 if not data:

90 return {}

91

92 if key is None:

93 # try to guess key by grabbing the first one or using the class name

94 if len(data) == 1:

95 key = next(iter(data.keys()))

96 elif classname is not None:

97 key = _guess_key(classname)

98

99 if key:

100 data = _data_for_nested_key(key, data)

101

102 if not data:

103 raise ValueError("No data found!")

104

105 if not isinstance(data, allow_types):

106 raise ValueError(f"Data should be one of {allow_types} but it is {type(data)}!")

107

108 if lower_keys and isinstance(data, dict):

109 data = {k.lower(): v for k, v in data.items()}

110

111 return data

112

113

114 def _load_data(

115 data: T_data,

116 key: str = None,

117 classname: str = None,

118 lower_keys: bool = False,

119 allow_types: tuple[type, ...] = (dict,),

120) -> dict[str, typing.Any]:

121 """

122 Wrapper around __load_data that retries with key="" if anything goes wrong.

123 """

124 if data is None:

125 # try to load pyproject.toml

126 data = find_pyproject_toml()

127

128 try:

129 return __load_data(data, key, classname, lower_keys=lower_keys, allow_types=allow_types)

130 except Exception as e:

131 if key != "":

132 return __load_data(data, "", classname, lower_keys=lower_keys, allow_types=allow_types)

133 else: # pragma: no cover

134 warnings.warn(f"Data could not be loaded: {e}", source=e)

135 # key already was "", just return data!

136 # (will probably not happen but fallback)

137 return {}

138

139

140 F = typing.TypeVar("F")

141

142

143 def convert_between(from_value: F, from_type: typing.Type[F], to_type: type[T]) -> T:

144 """

145 Convert a value between types.

146 """

147 if converter := CONVERTERS.get((from_type, to_type)):

148 return typing.cast(T, converter(from_value))

149

150 # default: just convert type:

151 return to_type(from_value) # type: ignore

152

153

154 def ensure_types(

155 data: dict[str, T], annotations: dict[str, type[T]], convert_types: bool = False

156) -> dict[str, T | None]:

157 """

158 Make sure all values in 'data' are in line with the ones stored in 'annotations'.

159

160 If an annotated key in missing from data, it will be filled with None for convenience.

161

162 TODO: python 3.11 exception groups to throw multiple errors at once!

163 """

164 # custom object to use instead of None, since typing.Optional can be None!

165 # cast to T to make mypy happy

166 notfound = typing.cast(T, object())

167 postponed = Postponed()

168

169 final: dict[str, T | None] = {}

170 for key, _type in annotations.items():

171 compare = data.get(key, notfound)

172 if compare is notfound: # pragma: nocover

173 warnings.warn(

174 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"

175 )

176 # skip!

177 continue

178

179 if compare is postponed:

180 # don't do anything with this item!

181 continue

182

183 if not check_type(compare, _type):

184 if convert_types:

185 try:

186 compare = convert_between(compare, type(compare), _type)

187 except (TypeError, ValueError) as e:

188 raise ConfigErrorCouldNotConvert(type(compare), _type, compare) from e

189 else:

190 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)

191

192 final[key] = compare

193

194 return final

195

196

197 def convert_config(items: dict[str, T]) -> dict[str, T]:

198 """

199 Converts the config dict (from toml) or 'overwrites' dict in two ways.

200

201 1. removes any items where the value is None, since in that case the default should be used;

202 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.

203 """

204 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}

205

206

207 Type = typing.Type[typing.Any]

208 T_Type = typing.TypeVar("T_Type", bound=Type)

209

210

211 def load_recursive(

212 cls: Type, data: dict[str, T], annotations: dict[str, Type], convert_types: bool = False

213) -> dict[str, T]:

214 """

215 For all annotations (recursively gathered from parents with `all_annotations`), \

216 try to resolve the tree of annotations.

217

218 Uses `load_into_recurse`, not itself directly.

219

220 Example:

221 class First:

222 key: str

223

224 class Second:

225 other: First

226

227 # step 1

228 cls = Second

229 data = {"second": {"other": {"key": "anything"}}}

230 annotations: {"other": First}

231

232 # step 1.5

233 data = {"other": {"key": "anything"}

234 annotations: {"other": First}

235

236 # step 2

237 cls = First

238 data = {"key": "anything"}

239 annotations: {"key": str}

240

241

242 TODO: python 3.11 exception groups to throw multiple errors at once!

243 """

244 updated = {}

245

246 for _key, _type in annotations.items():

247 if _key in data:

248 value: typing.Any = data[_key] # value can change so define it as any instead of T

249 if is_parameterized(_type):

250 origin = typing.get_origin(_type)

251 arguments = typing.get_args(_type)

252 if origin is list and arguments and is_custom_class(arguments[0]):

253 subtype = arguments[0]

254 value = [_load_into_recurse(subtype, subvalue, convert_types=convert_types) for subvalue in value]

255

256 elif origin is dict and arguments and is_custom_class(arguments[1]):

257 # e.g. dict[str, Point]

258 subkeytype, subvaluetype = arguments

259 # subkey(type) is not a custom class, so don't try to convert it:

260 value = {

261 subkey: _load_into_recurse(subvaluetype, subvalue, convert_types=convert_types)

262 for subkey, subvalue in value.items()

263 }

264 # elif origin is dict:

265 # keep data the same

266 elif origin is typing.Union and arguments:

267 for arg in arguments:

268 if is_custom_class(arg):

269 value = _load_into_recurse(arg, value, convert_types=convert_types)

270 else:

271 # print(_type, arg, value)

272 ...

273

274 # todo: other parameterized/unions/typing.Optional

275

276 elif is_custom_class(_type):

277 # type must be C (custom class) at this point

278 value = _load_into_recurse(

279 # make mypy and pycharm happy by telling it _type is of type C...

280 # actually just passing _type as first arg!

281 typing.cast(Type_C[typing.Any], _type),

282 value,

283 convert_types=convert_types,

284 )

285

286 elif _key in cls.__dict__:

287 # property has default, use that instead.

288 value = cls.__dict__[_key]

289 elif is_optional(_type):

290 # type is optional and not found in __dict__ -> default is None

291 value = None

292 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:

293 # could have a default factory

294 # todo: do something with field.default?

295 value = field.default_factory()

296 else:

297 raise ConfigErrorMissingKey(_key, cls, _type)

298

299 updated[_key] = value

300

301 return updated

302

303

304 def check_and_convert_data(

305 cls: typing.Type[C],

306 data: dict[str, typing.Any],

307 _except: typing.Iterable[str],

308 strict: bool = True,

309 convert_types: bool = False,

310) -> dict[str, typing.Any]:

311 """

312 Based on class annotations, this prepares the data for `load_into_recurse`.

313

314 1. convert config-keys to python compatible config_keys

315 2. loads custom class type annotations with the same logic (see also `load_recursive`)

316 3. ensures the annotated types match the actual types after loading the config file.

317 """

318 annotations = all_annotations(cls, _except=_except)

319

320 to_load = convert_config(data)

321 to_load = load_recursive(cls, to_load, annotations, convert_types=convert_types)

322 if strict:

323 to_load = ensure_types(to_load, annotations, convert_types=convert_types)

324

325 return to_load

326

327

328 T_init_list = list[typing.Any]

329 T_init_dict = dict[str, typing.Any]

330 T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None

331

332

333@typing.no_type_check # (mypy doesn't understand 'match' fully yet)

334 def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]:

335 """

336 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple.

337 """

338 if not init:

339 return [], {}

340

341 args: T_init_list = []

342 kwargs: T_init_dict = {}

343 match init:

344 case (args, kwargs):

345 return args, kwargs

346 case [*args]:

347 return args, {}

348 case {**kwargs}:

349 return [], kwargs

350 case _:

351 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.")

352

353

354 def _load_into_recurse(

355 cls: typing.Type[C],

356 data: dict[str, typing.Any],

357 init: T_init = None,

358 strict: bool = True,

359 convert_types: bool = False,

360) -> C:

361 """

362 Loads an instance of `cls` filled with `data`.

363

364 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).

365 `init` can be used to optionally pass extra __init__ arguments. \

366 NOTE: This will overwrite a config key with the same name!

367 """

368 init_args, init_kwargs = _split_init(init)

369

370 if issubclass(cls, BinaryConfig):

371 if not isinstance(data, (bytes, dict)): # pragma: no cover

372 raise NotImplementedError("BinaryConfig can only deal with `bytes` or a dict of bytes as input.")

373 inst = typing.cast(C, cls._parse_into(data))

374 elif dc.is_dataclass(cls):

375 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict, convert_types=convert_types)

376 if init:

377 raise ValueError("Init is not allowed for dataclasses!")

378

379 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)

380 inst = typing.cast(C, cls(**to_load))

381 else:

382 inst = cls(*init_args, **init_kwargs)

383 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict, convert_types=convert_types)

384 inst.__dict__.update(**to_load)

385

386 return inst

387

388

389 def _load_into_instance(

390 inst: C,

391 cls: typing.Type[C],

392 data: dict[str, typing.Any],

393 init: T_init = None,

394 strict: bool = True,

395 convert_types: bool = False,

396) -> C:

397 """

398 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \

399 and thus does not support init.

400

401 """

402 if init is not None:

403 raise ValueError("Can not init an existing instance!")

404

405 existing_data = inst.__dict__

406

407 to_load = check_and_convert_data(

408 cls, data, _except=existing_data.keys(), strict=strict, convert_types=convert_types

409 )

410

411 inst.__dict__.update(**to_load)

412

413 return inst

414

415

416 def load_into_class(

417 cls: typing.Type[C],

418 data: T_data,

419 /,

420 key: str = None,

421 init: T_init = None,

422 strict: bool = True,

423 lower_keys: bool = False,

424 convert_types: bool = False,

425) -> C:

426 """

427 Shortcut for _load_data + load_into_recurse.

428 """

429 allow_types = (dict, bytes) if issubclass(cls, BinaryConfig) else (dict,)

430 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys, allow_types=allow_types)

431 return _load_into_recurse(cls, to_load, init=init, strict=strict, convert_types=convert_types)

432

433

434 def load_into_instance(

435 inst: C,

436 data: T_data,

437 /,

438 key: str = None,

439 init: T_init = None,

440 strict: bool = True,

441 lower_keys: bool = False,

442 convert_types: bool = False,

443) -> C:

444 """

445 Shortcut for _load_data + load_into_existing.

446 """

447 cls = inst.__class__

448 allow_types = (dict, bytes) if issubclass(cls, BinaryConfig) else (dict,)

449 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys, allow_types=allow_types)

450 return _load_into_instance(inst, cls, to_load, init=init, strict=strict, convert_types=convert_types)

451

452

453 def load_into(

454 cls: typing.Type[C],

455 data: T_data = None,

456 /,

457 key: str = None,

458 init: T_init = None,

459 strict: bool = True,

460 lower_keys: bool = False,

461 convert_types: bool = False,

462) -> C:

463 """

464 Load your config into a class (instance).

465

466 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only

467 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`.

468

469 Args:

470 cls: either a class or an existing instance of that class.

471 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)

472 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')

473 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)

474 strict: enable type checks or allow anything?

475 lower_keys: should the config keys be lowercased? (for .env)

476 convert_types: should the types be converted to the annotated type if not yet matching? (for .env)

477

478 """

479 if not isinstance(cls, type):

480 # would not be supported according to mypy, but you can still load_into(instance)

481 return load_into_instance(

482 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types

483 )

484

485 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'

486 # _cls = typing.cast(typing.Type[C], cls)

487 return load_into_class(

488 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types

489 )