Coverage for src/configuraptor/core.py: 84%

1"""

2 Contains most of the loading logic.

3"""

4

5 import dataclasses as dc

6 import io

7 import os

8 import typing

9 import warnings

10 from pathlib import Path

11 from typing import Any, Type

12

13 import requests

14

15 from . import loaders

16 from .abs import AnyType, C, T, T_data, Type_C

17 from .alias import Alias, has_alias

18 from .binary_config import BinaryConfig

19 from .errors import (

20 ConfigErrorCouldNotConvert,

21 ConfigErrorInvalidType,

22 ConfigErrorMissingKey,

23)

24 from .helpers import (

25 all_annotations,

26 camel_to_snake,

27 check_type,

28 dataclass_field,

29 find_pyproject_toml,

30 is_custom_class,

31 is_optional,

32 is_parameterized,

33)

34 from .postpone import Postponed

35 from .type_converters import CONVERTERS

36

37

38 def _data_for_nested_key(key: str, raw: dict[str, typing.Any]) -> dict[str, typing.Any]:

39 """

40 If a key contains a dot, traverse the raw dict until the right key was found.

41

42 Example:

43 key = some.nested.key

44 raw = {"some": {"nested": {"key": {"with": "data"}}}}

45 -> {"with": "data"}

46 """

47 parts = key.split(".")

48 while parts:

49 key = parts.pop(0)

50 if key not in raw:

51 return {}

52

53 raw = raw[key]

54

55 return raw

56

57

58 def _guess_key(clsname: str) -> str:

59 """

60 If no key is manually defined for `load_into`, \

61 the class' name is converted to snake_case to use as the default key.

62 """

63 return camel_to_snake(clsname)

64

65

66 def _from_mock_url(url: str) -> str:

67 """

68 Pytest only: when starting a url with mock:// it is expected to just be json afterwards.

69 """

70 return url.removeprefix("mock://")

71

72

73 def guess_filetype_for_url(url: str, response: requests.Response = None) -> str:

74 """

75 Based on the url (which may have an extension) and the requests response \

76 (which may have a content-type), try to guess the right filetype (-> loader, e.g. json or yaml).

77

78 Falls back to JSON if none can be found.

79 """

80 url = url.split("?")[0]

81 if url_extension := os.path.splitext(url)[1].lower():

82 return url_extension.strip(".")

83

84 if response and (content_type_header := response.headers.get("content-type", "").split(";")[0].strip()):

85 content_type = content_type_header.split("/")[-1]

86 if content_type != "plain":

87 return content_type

88

89 # If both methods fail, default to JSON

90 return "json"

91

92

93 def from_url(url: str, _dummy: bool = False) -> tuple[io.BytesIO, str]:

94 """

95 Load data as bytes into a file-like object and return the file type.

96

97 This can be used by __load_data:

98 > loader = loaders.get(filetype)

99 > # dev/null exists but always returns b''

100 > data = loader(contents, Path("/dev/null"))

101 """

102 if url.startswith("mock://"):

103 data = _from_mock_url(url)

104 resp = None

105 elif _dummy:

106 resp = None

107 data = "{}"

108 else:

109 resp = requests.get(url, timeout=10)

110 data = resp.text

111

112 filetype = guess_filetype_for_url(url, resp)

113 return io.BytesIO(data.encode()), filetype

114

115

116 def _load_data(

117 data: T_data,

118 key: str = None,

119 classname: str = None,

120 lower_keys: bool = False,

121 allow_types: tuple[type, ...] = (dict,),

122) -> dict[str, typing.Any]:

123 """

124 Tries to load the right data from a filename/path or dict, based on a manual key or a classname.

125

126 E.g. class Tool will be mapped to key tool.

127 It also deals with nested keys (tool.extra -> {"tool": {"extra": ...}}

128 """

129 if isinstance(data, bytes):

130 # instantly return, don't modify

131 # bytes as inputs -> bytes as output

132 # but since `T_data` is re-used, that's kind of hard to type for mypy.

133 return data # type: ignore

134

135 if isinstance(data, list):

136 if not data:

137 raise ValueError("Empty list passed!")

138

139 final_data: dict[str, typing.Any] = {}

140 for source in data:

141 final_data |= load_data(source, key=key, classname=classname, lower_keys=True, allow_types=allow_types)

142

143 return final_data

144

145 if isinstance(data, str):

146 if data.startswith(("http://", "https://", "mock://")):

147 contents, filetype = from_url(data)

148

149 loader = loaders.get(filetype)

150 # dev/null exists but always returns b''

151 data = loader(contents, Path("/dev/null"))

152 else:

153 data = Path(data)

154

155 if isinstance(data, Path):

156 with data.open("rb") as f:

157 loader = loaders.get(data.suffix or data.name)

158 data = loader(f, data.resolve())

159

160 if not data:

161 return {}

162

163 if key is None:

164 # try to guess key by grabbing the first one or using the class name

165 if len(data) == 1:

166 key = next(iter(data.keys()))

167 elif classname is not None:

168 key = _guess_key(classname)

169

170 if key:

171 data = _data_for_nested_key(key, data)

172

173 if not data:

174 raise ValueError("No data found!")

175

176 if not isinstance(data, allow_types):

177 raise ValueError(f"Data should be one of {allow_types} but it is {type(data)}!")

178

179 if lower_keys and isinstance(data, dict):

180 data = {k.lower(): v for k, v in data.items()}

181

182 return typing.cast(dict[str, typing.Any], data)

183

184

185 def load_data(

186 data: T_data,

187 key: str = None,

188 classname: str = None,

189 lower_keys: bool = False,

190 allow_types: tuple[type, ...] = (dict,),

191) -> dict[str, typing.Any]:

192 """

193 Wrapper around __load_data that retries with key="" if anything goes wrong.

194 """

195 if data is None:

196 # try to load pyproject.toml

197 data = find_pyproject_toml()

198

199 try:

200 return _load_data(data, key, classname, lower_keys=lower_keys, allow_types=allow_types)

201 except Exception as e:

202 # sourcery skip: remove-unnecessary-else, simplify-empty-collection-comparison, swap-if-else-branches

203 # @sourcery: `key != ""` is NOT the same as `not key`

204 if key != "":

205 return _load_data(data, "", classname, lower_keys=lower_keys, allow_types=allow_types)

206 else: # pragma: no cover

207 warnings.warn(f"Data could not be loaded: {e}", source=e)

208 # key already was "", just return data!

209 # (will probably not happen but fallback)

210 return {}

211

212

213 F = typing.TypeVar("F")

214

215

216 def convert_between(from_value: F, from_type: Type[F], to_type: Type[T]) -> T:

217 """

218 Convert a value between types.

219 """

220 if converter := CONVERTERS.get((from_type, to_type)):

221 return typing.cast(T, converter(from_value))

222

223 # default: just convert type:

224 return to_type(from_value) # type: ignore

225

226

227 def check_and_convert_type(value: Any, _type: Type[T], convert_types: bool, key: str = "variable") -> T:

228 """

229 Checks if the given value matches the specified type. If it does, the value is returned as is.

230

231 Args:

232 value (Any): The value to be checked and potentially converted.

233 _type (Type[T]): The expected type for the value.

234 convert_types (bool): If True, allows type conversion if the types do not match.

235 key (str, optional): The name or key associated with the variable (used in error messages).

236 Defaults to "variable".

237

238 Returns:

239 T: The value, potentially converted to the expected type.

240

241 Raises:

242 ConfigErrorInvalidType: If the type does not match, and type conversion is not allowed.

243 ConfigErrorCouldNotConvert: If type conversion fails.

244 """

245 if check_type(value, _type):

246 # type matches

247 return value

248

249 if isinstance(value, Alias):

250 if is_optional(_type):

251 return typing.cast(T, None)

252 else:

253 # unresolved alias, error should've already been thrown for parent but lets do it again:

254 raise ConfigErrorInvalidType(value.to, value=value, expected_type=_type)

255

256 if not convert_types:

257 # type does not match and should not be converted

258 raise ConfigErrorInvalidType(key, value=value, expected_type=_type)

259

260 # else: type does not match, try to convert it

261 try:

262 return convert_between(value, type(value), _type)

263 except (TypeError, ValueError) as e:

264 raise ConfigErrorCouldNotConvert(type(value), _type, value) from e

265

266

267 def ensure_types(

268 data: dict[str, T], annotations: dict[str, type[T]], convert_types: bool = False

269) -> dict[str, T | None]:

270 """

271 Make sure all values in 'data' are in line with the ones stored in 'annotations'.

272

273 If an annotated key in missing from data, it will be filled with None for convenience.

274

275 TODO: python 3.11 exception groups to throw multiple errors at once!

276 """

277 # custom object to use instead of None, since typing.Optional can be None!

278 # cast to T to make mypy happy

279 notfound = typing.cast(T, object())

280

281 final: dict[str, T | None] = {}

282 for key, _type in annotations.items():

283 compare = data.get(key, notfound)

284 if compare is notfound: # pragma: nocover

285 warnings.warn("This should not happen since `load_recursive` already fills `data` based on `annotations`")

286 # skip!

287 continue

288

289 if isinstance(compare, Postponed):

290 # don't do anything with this item!

291 continue

292

293 if isinstance(compare, Alias):

294 related_data = data.get(compare.to, notfound)

295 if related_data is not notfound:

296 if isinstance(related_data, Postponed):

297 # also continue alias for postponed items

298 continue

299

300 # original key set, update alias

301 compare = related_data

302

303 compare = check_and_convert_type(compare, _type, convert_types, key)

304

305 final[key] = compare

306

307 return final

308

309

310 def convert_key(key: str) -> str:

311 """

312 Replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.

313 """

314 return key.replace("-", "_").replace(".", "_")

315

316

317 def convert_config(items: dict[str, T]) -> dict[str, T]:

318 """

319 Converts the config dict (from toml) or 'overwrites' dict in two ways.

320

321 1. removes any items where the value is None, since in that case the default should be used;

322 2. replaces '-' and '.' in keys with '_' so it can be mapped to the Config properties.

323 """

324 return {convert_key(k): v for k, v in items.items() if v is not None}

325

326

327 def load_recursive(

328 cls: AnyType, data: dict[str, T], annotations: dict[str, AnyType], convert_types: bool = False

329) -> dict[str, T]:

330 """

331 For all annotations (recursively gathered from parents with `all_annotations`), \

332 try to resolve the tree of annotations.

333

334 Uses `load_into_recurse`, not itself directly.

335

336 Example:

337 class First:

338 key: str

339

340 class Second:

341 other: First

342

343 # step 1

344 cls = Second

345 data = {"second": {"other": {"key": "anything"}}}

346 annotations: {"other": First}

347

348 # step 1.5

349 data = {"other": {"key": "anything"}

350 annotations: {"other": First}

351

352 # step 2

353 cls = First

354 data = {"key": "anything"}

355 annotations: {"key": str}

356

357

358 TODO: python 3.11 exception groups to throw multiple errors at once!

359 """

360 updated = {}

361

362 for _key, _type in annotations.items():

363 if _key in data:

364 value: typing.Any = data[_key] # value can change so define it as any instead of T

365 if is_parameterized(_type):

366 origin = typing.get_origin(_type)

367 arguments = typing.get_args(_type)

368 if origin is list and arguments and is_custom_class(arguments[0]):

369 subtype = arguments[0]

370 value = [_load_into_recurse(subtype, subvalue, convert_types=convert_types) for subvalue in value]

371

372 elif origin is dict and arguments and is_custom_class(arguments[1]):

373 # e.g. dict[str, Point]

374 subkeytype, subvaluetype = arguments

375 # subkey(type) is not a custom class, so don't try to convert it:

376 value = {

377 subkey: _load_into_recurse(subvaluetype, subvalue, convert_types=convert_types)

378 for subkey, subvalue in value.items()

379 }

380 # elif origin is dict:

381 # keep data the same

382 elif origin is typing.Union and arguments:

383 for arg in arguments:

384 if is_custom_class(arg):

385 value = _load_into_recurse(arg, value, convert_types=convert_types)

386

387 # todo: other parameterized/unions/typing.Optional

388

389 elif is_custom_class(_type):

390 # type must be C (custom class) at this point

391 value = _load_into_recurse(

392 # make mypy and pycharm happy by telling it _type is of type C...

393 # actually just passing _type as first arg!

394 typing.cast(Type_C[typing.Any], _type),

395 value,

396 convert_types=convert_types,

397 )

398 elif value := has_alias(cls, _key, data):

399 # value updated by alias

400 ...

401 elif _key in cls.__dict__:

402 # property has default, use that instead.

403 value = cls.__dict__[_key]

404 elif is_optional(_type):

405 # type is optional and not found in __dict__ -> default is None

406 value = None

407 elif dc.is_dataclass(cls) and (field := dataclass_field(cls, _key)) and field.default_factory is not dc.MISSING:

408 # could have a default factory

409 # todo: do something with field.default?

410 value = field.default_factory()

411 else:

412 raise ConfigErrorMissingKey(_key, cls, _type)

413

414 updated[_key] = value

415

416 return updated

417

418

419 def check_and_convert_data(

420 cls: typing.Type[C],

421 data: dict[str, typing.Any],

422 _except: typing.Iterable[str],

423 strict: bool = True,

424 convert_types: bool = False,

425) -> dict[str, typing.Any]:

426 """

427 Based on class annotations, this prepares the data for `load_into_recurse`.

428

429 1. convert config-keys to python compatible config_keys

430 2. loads custom class type annotations with the same logic (see also `load_recursive`)

431 3. ensures the annotated types match the actual types after loading the config file.

432 """

433 annotations = all_annotations(cls, _except=_except)

434

435 to_load = convert_config(data)

436 to_load = load_recursive(cls, to_load, annotations, convert_types=convert_types)

437

438 if strict:

439 to_load = ensure_types(to_load, annotations, convert_types=convert_types)

440

441 return to_load

442

443

444 T_init_list = list[typing.Any]

445 T_init_dict = dict[str, typing.Any]

446 T_init = tuple[T_init_list, T_init_dict] | T_init_list | T_init_dict | None

447

448

449@typing.no_type_check # (mypy doesn't understand 'match' fully yet)

450 def _split_init(init: T_init) -> tuple[T_init_list, T_init_dict]:

451 """

452 Accept a tuple, a dict or a list of (arg, kwarg), {kwargs: ...}, [args] respectively and turn them all into a tuple.

453 """

454 if not init:

455 return [], {}

456

457 args: T_init_list = []

458 kwargs: T_init_dict = {}

459 match init:

460 case (args, kwargs):

461 return args, kwargs

462 case [*args]:

463 return args, {}

464 case {**kwargs}:

465 return [], kwargs

466 case _:

467 raise ValueError("Init must be either a tuple of list and dict, a list or a dict.")

468

469

470 def _load_into_recurse(

471 cls: typing.Type[C],

472 data: dict[str, typing.Any] | bytes,

473 init: T_init = None,

474 strict: bool = True,

475 convert_types: bool = False,

476) -> C:

477 """

478 Loads an instance of `cls` filled with `data`.

479

480 Uses `load_recursive` to load any fillable annotated properties (see that method for an example).

481 `init` can be used to optionally pass extra __init__ arguments. \

482 NOTE: This will overwrite a config key with the same name!

483 """

484 init_args, init_kwargs = _split_init(init)

485

486 if isinstance(data, bytes) or issubclass(cls, BinaryConfig):

487 if not isinstance(data, (bytes, dict)): # pragma: no cover

488 raise NotImplementedError("BinaryConfig can only deal with `bytes` or a dict of bytes as input.")

489 elif not issubclass(cls, BinaryConfig): # pragma: no cover

490 raise NotImplementedError("Only BinaryConfig can be used with `bytes` (or a dict of bytes) as input.")

491

492 inst = typing.cast(C, cls._parse_into(data))

493 elif dc.is_dataclass(cls):

494 to_load = check_and_convert_data(cls, data, init_kwargs.keys(), strict=strict, convert_types=convert_types)

495 if init:

496 raise ValueError("Init is not allowed for dataclasses!")

497

498 # ensure mypy inst is an instance of the cls type (and not a fictuous `DataclassInstance`)

499 inst = typing.cast(C, cls(**to_load))

500 elif isinstance(data, cls):

501 # already the right type! (e.g. Pathlib)

502 inst = typing.cast(C, data)

503 else:

504 inst = cls(*init_args, **init_kwargs)

505 to_load = check_and_convert_data(cls, data, inst.__dict__.keys(), strict=strict, convert_types=convert_types)

506 inst.__dict__.update(**to_load)

507

508 return inst

509

510

511 def _load_into_instance(

512 inst: C,

513 cls: typing.Type[C],

514 data: dict[str, typing.Any],

515 init: T_init = None,

516 strict: bool = True,

517 convert_types: bool = False,

518) -> C:

519 """

520 Similar to `load_into_recurse` but uses an existing instance of a class (so after __init__) \

521 and thus does not support init.

522

523 """

524 if init is not None:

525 raise ValueError("Can not init an existing instance!")

526

527 existing_data = inst.__dict__

528

529 to_load = check_and_convert_data(

530 cls, data, _except=existing_data.keys(), strict=strict, convert_types=convert_types

531 )

532

533 inst.__dict__.update(**to_load)

534

535 return inst

536

537

538 def load_into_class(

539 cls: typing.Type[C],

540 data: T_data,

541 /,

542 key: str = None,

543 init: T_init = None,

544 strict: bool = True,

545 lower_keys: bool = False,

546 convert_types: bool = False,

547) -> C:

548 """

549 Shortcut for _load_data + load_into_recurse.

550 """

551 allow_types = (dict, bytes) if issubclass(cls, BinaryConfig) else (dict,)

552 to_load = load_data(data, key, cls.__name__, lower_keys=lower_keys, allow_types=allow_types)

553 return _load_into_recurse(cls, to_load, init=init, strict=strict, convert_types=convert_types)

554

555

556 def load_into_instance(

557 inst: C,

558 data: T_data,

559 /,

560 key: str = None,

561 init: T_init = None,

562 strict: bool = True,

563 lower_keys: bool = False,

564 convert_types: bool = False,

565) -> C:

566 """

567 Shortcut for _load_data + load_into_existing.

568 """

569 cls = inst.__class__

570 allow_types = (dict, bytes) if issubclass(cls, BinaryConfig) else (dict,)

571 to_load = load_data(data, key, cls.__name__, lower_keys=lower_keys, allow_types=allow_types)

572 return _load_into_instance(inst, cls, to_load, init=init, strict=strict, convert_types=convert_types)

573

574

575 def load_into(

576 cls: typing.Type[C],

577 data: T_data = None,

578 /,

579 key: str = None,

580 init: T_init = None,

581 strict: bool = True,

582 lower_keys: bool = False,

583 convert_types: bool = False,

584) -> C:

585 """

586 Load your config into a class (instance).

587

588 Supports both a class or an instance as first argument, but that's hard to explain to mypy, so officially only

589 classes are supported, and if you want to `load_into` an instance, you should use `load_into_instance`.

590

591 Args:

592 cls: either a class or an existing instance of that class.

593 data: can be a dictionary or a path to a file to load (as pathlib.Path or str)

594 key: optional (nested) dictionary key to load data from (e.g. 'tool.su6.specific')

595 init: optional data to pass to your cls' __init__ method (only if cls is not an instance already)

596 strict: enable type checks or allow anything?

597 lower_keys: should the config keys be lowercased? (for .env)

598 convert_types: should the types be converted to the annotated type if not yet matching? (for .env)

599

600 """

601 if not isinstance(cls, type):

602 # would not be supported according to mypy, but you can still load_into(instance)

603 return load_into_instance(

604 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types

605 )

606

607 # make mypy and pycharm happy by telling it cls is of type C and not just 'type'

608 # _cls = typing.cast(typing.Type[C], cls)

609 return load_into_class(

610 cls, data, key=key, init=init, strict=strict, lower_keys=lower_keys, convert_types=convert_types

611 )