Coverage for src/configuraptor/core.py: 100%

1"""

2 Contains most of the loading logic.

3"""

4

5 import dataclasses as dc

6 import typing

7 import warnings

8 from pathlib import Path

9

10 from . import loaders

11 from .abs import C, T, T_data, Type_C

12 from .binary_config import BinaryConfig

13 from .errors import (

14 ConfigErrorCouldNotConvert,

15 ConfigErrorInvalidType,

16 ConfigErrorMissingKey,

17)

18 from .helpers import (

19 all_annotations,

20 camel_to_snake,

21 check_type,

22 dataclass_field,

23 find_pyproject_toml,

24 is_custom_class,

25 is_optional,

26 is_parameterized,

27)

28 from .postpone import Postponed

29 from .type_converters import CONVERTERS

30

31

32 def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:

33 """

34 If a key contains a dot, traverse the raw dict until the right key was found.

35

36 Example:

37 key = some.nested.key

38 raw = {"some": {"nested": {"key": {"with": "data"}}}}

39 -> {"with": "data"}

40 """

41 parts = key.split(".")

42 while parts:

43 key = parts.pop(0)

44 if key not in raw:

45 return {}

46

47 raw = raw[key]

48

49 return raw

50

51

52 def _guess_key(clsname: str) -> str:

53 """

54 If no key is manually defined for `load_into`, \

55 the class' name is converted to snake_case to use as the default key.

56 """

57 return camel_to_snake(clsname)

58

59

60 def __load_data(

61 data: T_data,

62 key: str = None,

63 classname: str = None,

64 lower_keys: bool = False,

65 allow_types: tuple[type, ...] = (dict,),

66) -> dict[str, typing.Any]:

67 """

68 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.

69

70 E.g. class Tool will be mapped to key tool.

71 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}

72 """

73 if isinstance(data, bytes):

74 # instantly return, don't modify

75 # bytes as inputs -> bytes as output

76 # but since `T_data` is re-used, that's kind of hard to type for mypy.

77 return data # type: ignore

78

79 if isinstance(data, list):

80 if not data:

81 raise ValueError("Empty list passed!")

82

83 final_data: dict[str, typing.Any] = {}

84 for source in data:

85 final_data |= _load_data(source, key=key, classname=classname, lower_keys=True, allow_types=allow_types)

86

87 return final_data

88

89 if isinstance(data, str):

90 data = Path(data)

91

92 if isinstance(data, Path):

93 with data.open("rb") as f:

94 loader = loaders.get(data.suffix or data.name)

95 data = loader(f, data.resolve())

96

97 if not data:

98 return {}

99

100 if key is None:

101 # try to guess key by grabbing the first one or using the class name

102 if len(data) == 1:

103 key = next(iter(data.keys()))

104 elif classname is not None:

105 key = _guess_key(classname)

106

107 if key:

108 data = _data_for_nested_key(key, data)

109

110 if not data:

111 raise ValueError("No data found!")

112

113 if not isinstance(data, allow_types):

114 raise ValueError(f"Data should be one of {allow_types} but it is {type(data)}!")

115

116 if lower_keys and isinstance(data, dict):

117 data = {k.lower(): v for k, v in data.items()}

118

119 return data

120

121

122 def _load_data(

123 data: T_data,

124 key: str = None,

125 classname: str = None,

126 lower_keys: bool = False,

127 allow_types: tuple[type, ...] = (dict,),

128) -> dict[str, typing.Any]:

129 """

130 Wrapper around __load_data that retries with key="" if anything goes wrong.

131 """

132 if data is None:

133 # try to load pyproject.toml

134 data = find_pyproject_toml()

135

136 try:

137 return __load_data(data, key, classname, lower_keys=lower_keys, allow_types=allow_types)

138 except Exception as e:

139 if key != "":

140 return __load_data(data, "", classname, lower_keys=lower_keys, allow_types=allow_types)

141 else: # pragma: no cover

142 warnings.warn(f"Data could not be loaded: {e}", source=e)

143 # key already was "", just return data!

144 # (will probably not happen but fallback)

145 return {}

146

147

148 F = typing.TypeVar("F")

149

150

151 def convert_between(from_value: F, from_type: typing.Type[F], to_type: type[T]) -> T:

152 """

153 Convert a value between types.

154 """

155 if converter := CONVERTERS.get((from_type, to_type)):

156 return typing.cast(T, converter(from_value))

157

158 # default: just convert type:

159 return to_type(from_value) # type: ignore

160

161

162 def ensure_types(

163 data: dict[str, T], annotations: dict[str, type[T]], convert_types: bool = False

164) -> dict[str, T | None]:

165 """

166 Make sure all values in 'data' are in line with the ones stored in 'annotations'.

167

168 If an annotated key in missing from data, it will be filled with None for convenience.

169

170 TODO: python 3.11 exception groups to throw multiple errors at once!

171 """

172 # custom object to use instead of None, since typing.Optional can be None!

173 # cast to T to make mypy happy

174 notfound = typing.cast(T, object())

175 postponed = Postponed()

176

177 final: dict[str, T | None] = {}

178 for key, _type in annotations.items():

179 compare = data.get(key, notfound)

180 if compare is notfound: # pragma: nocover

181 warnings.warn(

182 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"

183 )

184 # skip!

185 continue

186

187 if compare is postponed:

188 # don't do anything with this item!

189 continue

190

191 if not check_type(compare, _type):

192 if convert_types:

193 try:

194 compare = convert_between(compare, type(compare), _type)

195 except (TypeError, ValueError) as e:

196 raise ConfigErrorCouldNotConvert(type(compare), _type, compare) from e

197 else:

198 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)

199

200 final[key] = compare

201

202 return final

203

204

205 def convert_config(items: dict[str, T]) -> dict[str, T]:

206 """

207 Converts the config dict (from toml) or 'overwrites' dict in two ways.

208

209 1. removes any items where the value is None, since in that case the default should be used;

210 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.

211 """

212 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}

213

214

215 Type = typing.Type[typing.Any]

216 T_Type = typing.TypeVar("T_Type", bound=Type)

217

218

219 def load_recursive(

220 cls: Type, data: dict[str, T], annotations: dict[str, Type], convert_types: bool = False

221) -> dict[str, T]:

222 """

223 For all annotations (recursively gathered from parents with `all_annotations`), \

224 try to resolve the tree of annotations.

225

226 Uses `load_into_recurse`, not itself directly.

227

228 Example:

229 class First:

230 key: str

231

232 class Second:

233 other: First

234

235 # step 1

236 cls = Second

237 data = {"second": {"other": {"key": "anything"}}}

238 annotations: {"other": First}

239

240 # step 1.5

241 data = {"other": {"key": "anything"}

242 annotations: {"other": First}

243

244 # step 2

245 cls = First

246 data = {"key": "anything"}

247 annotations: {"key": str}

248

249

250 TODO: python 3.11 exception groups to throw multiple errors at once!

251 """

252 updated = {}

253

254 for _key, _type in annotations.items():

255 if _key in data:

256 value: typing.Any = data[_key] # value can change so define it as any instead of T

257 if is_parameterized(_type):

258 origin = typing.get_origin(_type)

259 arguments = typing.get_args(_type)

260 if origin is list and arguments and is_custom_class(arguments[0]):

261 subtype = arguments[0]

262 value = [_load_into_recurse(subtype, subvalue, convert_types=convert_types) for subvalue in value]

263

264 elif origin is dict and arguments and is_custom_class(arguments[1]):

265 # e.g. dict[str, Point]

266 subkeytype, subvaluetype = arguments

267 # subkey(type) is not a custom class, so don't try to convert it:

268 value = {

269 subkey: _load_into_recurse(subvaluetype, subvalue, convert_types=convert_types)

270 for subkey, subvalue in value.items()

271 }

272 # elif origin is dict:

273 # keep data the same

274 elif origin is typing.Union and arguments:

275 for arg in arguments:

276 if is_custom_class(arg):

277 value = _load_into_recurse(arg, value, convert_types=convert_types)

278 else:

279 # print(_type, arg, value)

280 ...

281

282 # todo: other parameterized/unions/typing.Optional

283

284 elif is_custom_class(_type):

285 # type must be C (custom class) at this point

286 value = _load_into_recurse(

287 # make mypy and pycharm happy by telling it _type is of type C...

288 # actually just passing _type as first arg!

289 typing.cast(Type_C[typing.Any], _type),

290 value,

291 convert_types=convert_types,

292 )

293

294 elif _key in cls.__dict__:

295 # property has default, use that instead.

296 value = cls.__dict__[_key]

297 elif is_optional(_type):

298 # type is optional and not found in __dict__ -> default is None

299 value = None

300 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:

301 # could have a default factory

302 # todo: do something with field.default?

303 value = field.default_factory()

304 else:

305 raise ConfigErrorMissingKey(_key, cls, _type)

306

307 updated[_key] = value

308

309 return updated

310

311

312 def check_and_convert_data(

313 cls: typing.Type[C],

314 data: dict[str, typing.Any],

315 _except: typing.Iterable[str],

316 strict: bool = True,

317 convert_types: bool = False,

318) -> dict[str, typing.Any]:

319 """

320 Based on class annotations, this prepares the data for `load_into_recurse`.

321

322 1. convert config-keys to python compatible config_keys

323 2. loads custom class type annotations with the same logic (see also `load_recursive`)

324 3. ensures the annotated types match the actual types after loading the config file.

325 """

326 annotations = all_annotations(cls, _except=_except)

327

328 to_load = convert_config(data)

329 to_load = load_recursive(cls, to_load, annotations, convert_types=convert_types)

330 if strict:

331 to_load = ensure_types(to_load, annotations, convert_types=convert_types)

332

333 return to_load

334

335

336 T_init_list = list[typing.Any]

337 T_init_dict = dict[str, typing.Any]

338 T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None

339

340

341@typing.no_type_check # (mypy doesn't understand 'match' fully yet)

342 def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]:

343 """

344 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple.

345 """

346 if not init:

347 return [], {}

348

349 args: T_init_list = []

350 kwargs: T_init_dict = {}

351 match init:

352 case (args, kwargs):

353 return args, kwargs

354 case [*args]:

355 return args, {}

356 case {**kwargs}:

357 return [], kwargs

358 case _:

359 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.")

360

361

362 def _load_into_recurse(

363 cls: typing.Type[C],

364 data: dict[str, typing.Any] | bytes,

365 init: T_init = None,

366 strict: bool = True,

367 convert_types: bool = False,

368) -> C:

369 """

370 Loads an instance of `cls` filled with `data`.

371

372 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).

373 `init` can be used to optionally pass extra __init__ arguments. \

374 NOTE: This will overwrite a config key with the same name!

375 """

376 init_args, init_kwargs = _split_init(init)

377

378 if isinstance(data, bytes) or issubclass(cls, BinaryConfig):

379 if not isinstance(data, (bytes, dict)): # pragma: no cover

380 raise NotImplementedError("BinaryConfig can only deal with `bytes` or a dict of bytes as input.")

381 elif not issubclass(cls, BinaryConfig): # pragma: no cover

382 raise NotImplementedError("Only BinaryConfig can be used with `bytes` (or a dict of bytes) as input.")

383

384 inst = typing.cast(C, cls._parse_into(data))

385 elif dc.is_dataclass(cls):

386 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict, convert_types=convert_types)

387 if init:

388 raise ValueError("Init is not allowed for dataclasses!")

389

390 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)

391 inst = typing.cast(C, cls(**to_load))

392 else:

393 inst = cls(*init_args, **init_kwargs)

394 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict, convert_types=convert_types)

395 inst.__dict__.update(**to_load)

396

397 return inst

398

399

400 def _load_into_instance(

401 inst: C,

402 cls: typing.Type[C],

403 data: dict[str, typing.Any],

404 init: T_init = None,

405 strict: bool = True,

406 convert_types: bool = False,

407) -> C:

408 """

409 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \

410 and thus does not support init.

411

412 """

413 if init is not None:

414 raise ValueError("Can not init an existing instance!")

415

416 existing_data = inst.__dict__

417

418 to_load = check_and_convert_data(

419 cls, data, _except=existing_data.keys(), strict=strict, convert_types=convert_types

420 )

421

422 inst.__dict__.update(**to_load)

423

424 return inst

425

426

427 def load_into_class(

428 cls: typing.Type[C],

429 data: T_data,

430 /,

431 key: str = None,

432 init: T_init = None,

433 strict: bool = True,

434 lower_keys: bool = False,

435 convert_types: bool = False,

436) -> C:

437 """

438 Shortcut for _load_data + load_into_recurse.

439 """

440 allow_types = (dict, bytes) if issubclass(cls, BinaryConfig) else (dict,)

441 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys, allow_types=allow_types)

442 return _load_into_recurse(cls, to_load, init=init, strict=strict, convert_types=convert_types)

443

444

445 def load_into_instance(

446 inst: C,

447 data: T_data,

448 /,

449 key: str = None,

450 init: T_init = None,

451 strict: bool = True,

452 lower_keys: bool = False,

453 convert_types: bool = False,

454) -> C:

455 """

456 Shortcut for _load_data + load_into_existing.

457 """

458 cls = inst.__class__

459 allow_types = (dict, bytes) if issubclass(cls, BinaryConfig) else (dict,)

460 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys, allow_types=allow_types)

461 return _load_into_instance(inst, cls, to_load, init=init, strict=strict, convert_types=convert_types)

462

463

464 def load_into(

465 cls: typing.Type[C],

466 data: T_data = None,

467 /,

468 key: str = None,

469 init: T_init = None,

470 strict: bool = True,

471 lower_keys: bool = False,

472 convert_types: bool = False,

473) -> C:

474 """

475 Load your config into a class (instance).

476

477 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only

478 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`.

479

480 Args:

481 cls: either a class or an existing instance of that class.

482 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)

483 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')

484 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)

485 strict: enable type checks or allow anything?

486 lower_keys: should the config keys be lowercased? (for .env)

487 convert_types: should the types be converted to the annotated type if not yet matching? (for .env)

488

489 """

490 if not isinstance(cls, type):

491 # would not be supported according to mypy, but you can still load_into(instance)

492 return load_into_instance(

493 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types

494 )

495

496 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'

497 # _cls = typing.cast(typing.Type[C], cls)

498 return load_into_class(

499 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types

500 )