Coverage for src/configuraptor/core.py: 100%

1"""

2 Contains most of the loading logic.

3"""

4

5 import dataclasses as dc

6 import typing

7 import warnings

8 from pathlib import Path

9

10 from . import loaders

11 from .abs import C, T, T_data, Type_C

12 from .binary_config import BinaryConfig

13 from .errors import (

14 ConfigErrorCouldNotConvert,

15 ConfigErrorInvalidType,

16 ConfigErrorMissingKey,

17)

18 from .helpers import (

19 all_annotations,

20 camel_to_snake,

21 check_type,

22 dataclass_field,

23 find_pyproject_toml,

24 is_custom_class,

25 is_optional,

26 is_parameterized,

27)

28 from .postpone import Postponed

29 from .type_converters import CONVERTERS

30

31

32 def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:

33 """

34 If a key contains a dot, traverse the raw dict until the right key was found.

35

36 Example:

37 key = some.nested.key

38 raw = {"some": {"nested": {"key": {"with": "data"}}}}

39 -> {"with": "data"}

40 """

41 parts = key.split(".")

42 while parts:

43 key = parts.pop(0)

44 if key not in raw:

45 return {}

46

47 raw = raw[key]

48

49 return raw

50

51

52 def _guess_key(clsname: str) -> str:

53 """

54 If no key is manually defined for `load_into`, \

55 the class' name is converted to snake_case to use as the default key.

56 """

57 return camel_to_snake(clsname)

58

59

60 def __load_data(

61 data: T_data, key: str = None, classname: str = None, lower_keys: bool = False

62) -> dict[str, typing.Any]:

63 """

64 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.

65

66 E.g. class Tool will be mapped to key tool.

67 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}

68 """

69 if isinstance(data, list):

70 if not data:

71 raise ValueError("Empty list passed!")

72

73 final_data: dict[str, typing.Any] = {}

74 for source in data:

75 final_data |= _load_data(source, key=key, classname=classname, lower_keys=True)

76

77 return final_data

78

79 if isinstance(data, str):

80 data = Path(data)

81 if isinstance(data, Path):

82 with data.open("rb") as f:

83 loader = loaders.get(data.suffix or data.name)

84 data = loader(f, data.resolve())

85 if not data:

86 return {}

87

88 if key is None:

89 # try to guess key by grabbing the first one or using the class name

90 if len(data) == 1:

91 key = next(iter(data.keys()))

92 elif classname is not None:

93 key = _guess_key(classname)

94

95 if key:

96 data = _data_for_nested_key(key, data)

97

98 if not data:

99 raise ValueError("No data found!")

100

101 if not isinstance(data, dict):

102 raise ValueError("Data is not a dict!")

103

104 if lower_keys:

105 data = {k.lower(): v for k, v in data.items()}

106

107 return data

108

109

110 def _load_data(data: T_data, key: str = None, classname: str = None, lower_keys: bool = False) -> dict[str, typing.Any]:

111 """

112 Wrapper around __load_data that retries with key="" if anything goes wrong.

113 """

114 if data is None:

115 # try to load pyproject.toml

116 data = find_pyproject_toml()

117

118 try:

119 return __load_data(data, key, classname, lower_keys=lower_keys)

120 except Exception as e:

121 if key != "":

122 return __load_data(data, "", classname, lower_keys=lower_keys)

123 else: # pragma: no cover

124 warnings.warn(f"Data could not be loaded: {e}", source=e)

125 # key already was "", just return data!

126 # (will probably not happen but fallback)

127 return {}

128

129

130 F = typing.TypeVar("F")

131

132

133 def convert_between(from_value: F, from_type: typing.Type[F], to_type: type[T]) -> T:

134 """

135 Convert a value between types.

136 """

137 if converter := CONVERTERS.get((from_type, to_type)):

138 return typing.cast(T, converter(from_value))

139

140 # default: just convert type:

141 return to_type(from_value) # type: ignore

142

143

144 def ensure_types(

145 data: dict[str, T], annotations: dict[str, type[T]], convert_types: bool = False

146) -> dict[str, T | None]:

147 """

148 Make sure all values in 'data' are in line with the ones stored in 'annotations'.

149

150 If an annotated key in missing from data, it will be filled with None for convenience.

151

152 TODO: python 3.11 exception groups to throw multiple errors at once!

153 """

154 # custom object to use instead of None, since typing.Optional can be None!

155 # cast to T to make mypy happy

156 notfound = typing.cast(T, object())

157 postponed = Postponed()

158

159 final: dict[str, T | None] = {}

160 for key, _type in annotations.items():

161 compare = data.get(key, notfound)

162 if compare is notfound: # pragma: nocover

163 warnings.warn(

164 "This should not happen since " "`load_recursive` already fills `data` " "based on `annotations`"

165 )

166 # skip!

167 continue

168

169 if compare is postponed:

170 # don't do anything with this item!

171 continue

172

173 if not check_type(compare, _type):

174 if convert_types:

175 try:

176 compare = convert_between(compare, type(compare), _type)

177 except (TypeError, ValueError) as e:

178 raise ConfigErrorCouldNotConvert(type(compare), _type, compare) from e

179 else:

180 raise ConfigErrorInvalidType(key, value=compare, expected_type=_type)

181

182 final[key] = compare

183

184 return final

185

186

187 def convert_config(items: dict[str, T]) -> dict[str, T]:

188 """

189 Converts the config dict (from toml) or 'overwrites' dict in two ways.

190

191 1. removes any items where the value is None, since in that case the default should be used;

192 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.

193 """

194 return {k.replace("-", "_").replace(".", "_"): v for k, v in items.items() if v is not None}

195

196

197 Type = typing.Type[typing.Any]

198 T_Type = typing.TypeVar("T_Type", bound=Type)

199

200

201 def load_recursive(

202 cls: Type, data: dict[str, T], annotations: dict[str, Type], convert_types: bool = False

203) -> dict[str, T]:

204 """

205 For all annotations (recursively gathered from parents with `all_annotations`), \

206 try to resolve the tree of annotations.

207

208 Uses `load_into_recurse`, not itself directly.

209

210 Example:

211 class First:

212 key: str

213

214 class Second:

215 other: First

216

217 # step 1

218 cls = Second

219 data = {"second": {"other": {"key": "anything"}}}

220 annotations: {"other": First}

221

222 # step 1.5

223 data = {"other": {"key": "anything"}

224 annotations: {"other": First}

225

226 # step 2

227 cls = First

228 data = {"key": "anything"}

229 annotations: {"key": str}

230

231

232 TODO: python 3.11 exception groups to throw multiple errors at once!

233 """

234 updated = {}

235

236 for _key, _type in annotations.items():

237 if _key in data:

238 value: typing.Any = data[_key] # value can change so define it as any instead of T

239 if is_parameterized(_type):

240 origin = typing.get_origin(_type)

241 arguments = typing.get_args(_type)

242 if origin is list and arguments and is_custom_class(arguments[0]):

243 subtype = arguments[0]

244 value = [_load_into_recurse(subtype, subvalue, convert_types=convert_types) for subvalue in value]

245

246 elif origin is dict and arguments and is_custom_class(arguments[1]):

247 # e.g. dict[str, Point]

248 subkeytype, subvaluetype = arguments

249 # subkey(type) is not a custom class, so don't try to convert it:

250 value = {

251 subkey: _load_into_recurse(subvaluetype, subvalue, convert_types=convert_types)

252 for subkey, subvalue in value.items()

253 }

254 # elif origin is dict:

255 # keep data the same

256 elif origin is typing.Union and arguments:

257 for arg in arguments:

258 if is_custom_class(arg):

259 value = _load_into_recurse(arg, value, convert_types=convert_types)

260 else:

261 # print(_type, arg, value)

262 ...

263

264 # todo: other parameterized/unions/typing.Optional

265

266 elif is_custom_class(_type):

267 # type must be C (custom class) at this point

268 value = _load_into_recurse(

269 # make mypy and pycharm happy by telling it _type is of type C...

270 # actually just passing _type as first arg!

271 typing.cast(Type_C[typing.Any], _type),

272 value,

273 convert_types=convert_types,

274 )

275

276 elif _key in cls.__dict__:

277 # property has default, use that instead.

278 value = cls.__dict__[_key]

279 elif is_optional(_type):

280 # type is optional and not found in __dict__ -> default is None

281 value = None

282 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:

283 # could have a default factory

284 # todo: do something with field.default?

285 value = field.default_factory()

286 else:

287 raise ConfigErrorMissingKey(_key, cls, _type)

288

289 updated[_key] = value

290

291 return updated

292

293

294 def check_and_convert_data(

295 cls: typing.Type[C],

296 data: dict[str, typing.Any],

297 _except: typing.Iterable[str],

298 strict: bool = True,

299 convert_types: bool = False,

300) -> dict[str, typing.Any]:

301 """

302 Based on class annotations, this prepares the data for `load_into_recurse`.

303

304 1. convert config-keys to python compatible config_keys

305 2. loads custom class type annotations with the same logic (see also `load_recursive`)

306 3. ensures the annotated types match the actual types after loading the config file.

307 """

308 annotations = all_annotations(cls, _except=_except)

309

310 to_load = convert_config(data)

311 to_load = load_recursive(cls, to_load, annotations, convert_types=convert_types)

312 if strict:

313 to_load = ensure_types(to_load, annotations, convert_types=convert_types)

314

315 return to_load

316

317

318 T_init_list = list[typing.Any]

319 T_init_dict = dict[str, typing.Any]

320 T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None

321

322

323@typing.no_type_check # (mypy doesn't understand 'match' fully yet)

324 def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]:

325 """

326 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple.

327 """

328 if not init:

329 return [], {}

330

331 args: T_init_list = []

332 kwargs: T_init_dict = {}

333 match init:

334 case (args, kwargs):

335 return args, kwargs

336 case [*args]:

337 return args, {}

338 case {**kwargs}:

339 return [], kwargs

340 case _:

341 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.")

342

343

344 def _load_into_recurse(

345 cls: typing.Type[C],

346 data: dict[str, typing.Any],

347 init: T_init = None,

348 strict: bool = True,

349 convert_types: bool = False,

350) -> C:

351 """

352 Loads an instance of `cls` filled with `data`.

353

354 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).

355 `init` can be used to optionally pass extra __init__ arguments. \

356 NOTE: This will overwrite a config key with the same name!

357 """

358 init_args, init_kwargs = _split_init(init)

359

360 if issubclass(cls, BinaryConfig):

361 # todo: init?

362

363 if not isinstance(data, (bytes, dict)): # pragma: no cover

364 raise NotImplementedError("BinaryConfig can only deal with `bytes` or a dict of bytes as input.")

365 inst = typing.cast(C, cls._parse_into(data))

366 elif dc.is_dataclass(cls):

367 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict, convert_types=convert_types)

368 if init:

369 raise ValueError("Init is not allowed for dataclasses!")

370

371 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)

372 inst = typing.cast(C, cls(**to_load))

373 else:

374 inst = cls(*init_args, **init_kwargs)

375 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict, convert_types=convert_types)

376 inst.__dict__.update(**to_load)

377

378 return inst

379

380

381 def _load_into_instance(

382 inst: C,

383 cls: typing.Type[C],

384 data: dict[str, typing.Any],

385 init: T_init = None,

386 strict: bool = True,

387 convert_types: bool = False,

388) -> C:

389 """

390 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \

391 and thus does not support init.

392

393 """

394 if init is not None:

395 raise ValueError("Can not init an existing instance!")

396

397 existing_data = inst.__dict__

398

399 to_load = check_and_convert_data(

400 cls, data, _except=existing_data.keys(), strict=strict, convert_types=convert_types

401 )

402

403 inst.__dict__.update(**to_load)

404

405 return inst

406

407

408 def load_into_class(

409 cls: typing.Type[C],

410 data: T_data,

411 /,

412 key: str = None,

413 init: T_init = None,

414 strict: bool = True,

415 lower_keys: bool = False,

416 convert_types: bool = False,

417) -> C:

418 """

419 Shortcut for _load_data + load_into_recurse.

420 """

421 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys)

422 return _load_into_recurse(cls, to_load, init=init, strict=strict, convert_types=convert_types)

423

424

425 def load_into_instance(

426 inst: C,

427 data: T_data,

428 /,

429 key: str = None,

430 init: T_init = None,

431 strict: bool = True,

432 lower_keys: bool = False,

433 convert_types: bool = False,

434) -> C:

435 """

436 Shortcut for _load_data + load_into_existing.

437 """

438 cls = inst.__class__

439 to_load = _load_data(data, key, cls.__name__, lower_keys=lower_keys)

440 return _load_into_instance(inst, cls, to_load, init=init, strict=strict, convert_types=convert_types)

441

442

443 def load_into(

444 cls: typing.Type[C],

445 data: T_data = None,

446 /,

447 key: str = None,

448 init: T_init = None,

449 strict: bool = True,

450 lower_keys: bool = False,

451 convert_types: bool = False,

452) -> C:

453 """

454 Load your config into a class (instance).

455

456 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only

457 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`.

458

459 Args:

460 cls: either a class or an existing instance of that class.

461 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)

462 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')

463 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)

464 strict: enable type checks or allow anything?

465 lower_keys: should the config keys be lowercased? (for .env)

466 convert_types: should the types be converted to the annotated type if not yet matching? (for .env)

467

468 """

469 if not isinstance(cls, type):

470 # would not be supported according to mypy, but you can still load_into(instance)

471 return load_into_instance(

472 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types

473 )

474

475 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'

476 # _cls = typing.cast(typing.Type[C], cls)

477 return load_into_class(

478 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types

479 )