Coverage for src/typedal/config.py: 100%

1"""

2 TypeDAL can be configured by a combination of pyproject.toml (static), env (dynamic) and code (programmic).

3"""

4

5 import os

6 import re

7 import typing

8 import warnings

9 from collections import defaultdict

10 from pathlib import Path

11 from typing import Any, Optional

12

13 import tomli

14 from configuraptor import TypedConfig, alias

15 from configuraptor.helpers import find_pyproject_toml

16 from dotenv import dotenv_values, find_dotenv

17

18 from .types import AnyDict

19

20 if typing.TYPE_CHECKING: # pragma: no cover

21 from edwh_migrate import Config as MigrateConfig

22 from pydal2sql.typer_support import Config as P2SConfig

23

24

25 class TypeDALConfig(TypedConfig):

26 """

27 Unified config for TypeDAL runtime behavior and migration utilities.

28 """

29

30 # typedal:

31 database: str

32 dialect: str

33 folder: str = "databases"

34 caching: bool = True

35 pool_size: int = 0

36 pyproject: str

37 connection: str = "default"

38

39 # pydal2sql:

40 input: str = ""

41 output: str = ""

42 noop: bool = False

43 magic: bool = True

44 tables: Optional[list[str]] = None

45 function: str = "define_tables"

46

47 # edwh-migrate:

48 # migrate uri = database

49 database_to_restore: Optional[str]

50 migrate_cat_command: Optional[str]

51 schema_version: Optional[str]

52 redis_host: Optional[str]

53 migrate_table: str = "typedal_implemented_features"

54 flag_location: str

55 create_flag_location: bool = True

56 schema: str = "public"

57

58 # typedal (depends on properties above)

59 migrate: bool = True

60 fake_migrate: bool = False

61

62 # aliases:

63 db_uri: str = alias("database")

64 db_type: str = alias("dialect")

65 db_folder: str = alias("folder")

66

67 # repr set by @beautify (by inheriting from TypedConfig)

68

69 def to_pydal2sql(self) -> "P2SConfig":

70 """

71 Convert the config to the format required by pydal2sql.

72 """

73 from pydal2sql.typer_support import Config, get_pydal2sql_config

74

75 if self.pyproject: # pragma: no cover

76 project = Path(self.pyproject).read_text()

77

78 if "[tool.typedal]" not in project and "[tool.pydal2sql]" in project:

79 # no typedal config, but existing p2s config:

80 return get_pydal2sql_config(self.pyproject)

81

82 return Config.load(

83 {

84 "db_type": self.dialect,

85 "format": "edwh-migrate",

86 "tables": self.tables,

87 "magic": self.magic,

88 "function": self.function,

89 "input": self.input,

90 "output": self.output,

91 "pyproject": self.pyproject,

92 }

93 )

94

95 def to_migrate(self) -> "MigrateConfig":

96 """

97 Convert the config to the format required by edwh-migrate.

98 """

99 from edwh_migrate import Config, get_config

100

101 if self.pyproject: # pragma: no cover

102 project = Path(self.pyproject).read_text()

103

104 if "[tool.typedal]" not in project and "[tool.migrate]" in project:

105 # no typedal config, but existing p2s config:

106 return get_config()

107

108 return Config.load(

109 {

110 "migrate_uri": self.database,

111 "schema_version": self.schema_version,

112 "redis_host": self.redis_host,

113 "migrate_cat_command": self.migrate_cat_command,

114 "database_to_restore": self.database_to_restore,

115 "migrate_table": self.migrate_table,

116 "flag_location": self.flag_location,

117 "create_flag_location": self.create_flag_location,

118 "schema": self.schema,

119 "db_folder": self.folder,

120 "migrations_file": self.output,

121 }

122 )

123

124

125 def _load_toml(path: str | bool | Path | None = True) -> tuple[str, AnyDict]:

126 """

127 Path can be a file, a directory, a bool or None.

128

129 If it is True or None, the default logic is used.

130 If it is False, no data is loaded.

131 if it is a directory, the pyproject.toml will be searched there.

132 If it is a path, that file will be used.

133 """

134 if path is False:

135 toml_path = None

136 elif path in (True, None):

137 toml_path = find_pyproject_toml()

138 elif (_p := Path(str(path))) and _p.is_file():

139 toml_path = _p

140 else:

141 toml_path = find_pyproject_toml(str(path))

142

143 if not toml_path:

144 # nothing to load

145 return "", {}

146

147 try:

148 with open(toml_path, "rb") as f:

149 data = tomli.load(f)

150

151 return str(toml_path) or "", typing.cast(AnyDict, data["tool"]["typedal"])

152 except Exception as e:

153 warnings.warn(f"Could not load typedal config toml: {e}", source=e)

154 return str(toml_path) or "", {}

155

156

157 def _load_dotenv(path: str | bool | None = True) -> tuple[str, AnyDict]:

158 fallback_data = {k.lower().removeprefix("typedal_"): v for k, v in os.environ.items()}

159 if path is False:

160 dotenv_path = None

161 fallback_data = {}

162 elif path in (True, None):

163 dotenv_path = find_dotenv(usecwd=True)

164 elif Path(str(path)).is_file():

165 dotenv_path = str(path)

166 else:

167 dotenv_path = str(Path(str(path)) / ".env")

168

169 if not dotenv_path:

170 return "", fallback_data

171

172 # 1. find everything with TYPEDAL_ prefix

173 # 2. remove that prefix

174 # 3. format values if possible

175 data = dotenv_values(dotenv_path)

176 data |= os.environ # higher prio than .env

177

178 typedal_data = {k.lower().removeprefix("typedal_"): v for k, v in data.items()}

179

180 return dotenv_path, typedal_data

181

182

183 DB_ALIASES = {

184 "postgresql": "postgres",

185 "psql": "postgres",

186 "sqlite3": "sqlite",

187}

188

189

190 def get_db_for_alias(db_name: str) -> str:

191 """

192 Convert a db dialect alias to the standard name.

193 """

194 return DB_ALIASES.get(db_name, db_name)

195

196

197 DEFAULTS: dict[str, Any | typing.Callable[[AnyDict], Any]] = {

198 "database": lambda data: data.get("db_uri") or "sqlite:memory",

199 "dialect": lambda data: (

200 get_db_for_alias(data["database"].split(":")[0]) if ":" in data["database"] else data.get("db_type")

201 ),

202 "migrate": lambda data: not (data.get("input") or data.get("output")),

203 "folder": lambda data: data.get("db_folder"),

204 "flag_location": lambda data: (

205 f"{db_folder}/flags" if (db_folder := (data.get("folder") or data.get("db_folder"))) else "/flags"

206 ),

207 "pool_size": lambda data: 1 if data.get("dialect", "sqlite") == "sqlite" else 3,

208}

209

210

211 def _fill_defaults(data: AnyDict, prop: str, fallback: Any = None) -> None:

212 default = DEFAULTS.get(prop, fallback)

213 if callable(default):

214 default = default(data)

215 data[prop] = default

216

217

218 def fill_defaults(data: AnyDict, prop: str) -> None:

219 """

220 Fill missing property defaults with (calculated) sane defaults.

221 """

222 if data.get(prop, None) is None:

223 _fill_defaults(data, prop)

224

225

226 TRANSFORMS: dict[str, typing.Callable[[AnyDict], Any]] = {

227 "database": lambda data: (

228 data["database"]

229 if (":" in data["database"] or not data.get("dialect"))

230 else (data["dialect"] + "://" + data["database"])

231 )

232}

233

234

235 def transform(data: AnyDict, prop: str) -> bool:

236 """

237 After the user has chosen a value, possibly transform it.

238 """

239 if fn := TRANSFORMS.get(prop):

240 data[prop] = fn(data)

241 return True

242 return False

243

244

245 def expand_posix_vars(posix_expr: str, context: dict[str, str]) -> str:

246 """

247 Replace case-insensitive POSIX and Docker Compose-like environment variables in a string with their values.

248

249 Args:

250 posix_expr (str): The input string containing case-insensitive POSIX or Docker Compose-like variables.

251 context (dict): A dictionary containing variable names and their respective values.

252

253 Returns:

254 str: The string with replaced variable values.

255

256 See Also:

257 https://stackoverflow.com/questions/386934/how-to-evaluate-environment-variables-into-a-string-in-python

258 and ChatGPT

259 """

260 env = defaultdict(lambda: "")

261 for key, value in context.items():

262 env[key.lower()] = value

263

264 # Regular expression to match "${VAR:default}" pattern

265 pattern = r"\$\{([^}]+)\}"

266

267 def replace_var(match: re.Match[Any]) -> str:

268 var_with_default = match.group(1)

269 var_name, default_value = var_with_default.split(":") if ":" in var_with_default else (var_with_default, "")

270 return env.get(var_name.lower(), default_value)

271

272 return re.sub(pattern, replace_var, posix_expr)

273

274

275 def expand_env_vars_into_toml_values(toml: AnyDict, env: AnyDict) -> None:

276 """

277 Recursively expands POSIX/Docker Compose-like environment variables in a TOML dictionary.

278

279 This function traverses a TOML dictionary and expands POSIX/Docker Compose-like

280 environment variables (${VAR:default}) using values provided in the 'env' dictionary.

281 It performs in-place modification of the 'toml' dictionary.

282

283 Args:

284 toml (dict): A TOML dictionary with string values possibly containing environment variables.

285 env (dict): A dictionary containing environment variable names and their respective values.

286

287 Returns:

288 None: The function modifies the 'toml' dictionary in place.

289

290 Notes:

291 The function recursively traverses the 'toml' dictionary. If a value is a string or a list of strings,

292 it attempts to substitute any environment variables found within those strings using the 'env' dictionary.

293

294 Example:

295 toml_data = {

296 'key1': 'This has ${ENV_VAR:default}',

297 'key2': ['String with ${ANOTHER_VAR}', 'Another ${YET_ANOTHER_VAR}']

298 }

299 environment = {

300 'ENV_VAR': 'replaced_value',

301 'ANOTHER_VAR': 'value_1',

302 'YET_ANOTHER_VAR': 'value_2'

303 }

304

305 expand_env_vars_into_toml_values(toml_data, environment)

306 # 'toml_data' will be modified in place:

307 # {

308 # 'key1': 'This has replaced_value',

309 # 'key2': ['String with value_1', 'Another value_2']

310 # }

311 """

312 if not toml or not env:

313 return

314

315 for key, var in toml.items():

316 if isinstance(var, dict):

317 expand_env_vars_into_toml_values(var, env)

318 elif isinstance(var, list):

319 toml[key] = [expand_posix_vars(_, env) for _ in var if isinstance(_, str)]

320 elif isinstance(var, str):

321 toml[key] = expand_posix_vars(var, env)

322 else:

323 # nothing to substitute

324 continue

325

326

327 def load_config(

328 connection_name: Optional[str] = None,

329 _use_pyproject: bool | str | None = True,

330 _use_env: bool | str | None = True,

331 **fallback: Any,

332) -> TypeDALConfig:

333 """

334 Combines multiple sources of config into one config instance.

335 """

336 # load toml data

337 # load .env data

338 # combine and fill with fallback values

339 # load typedal config or fail

340 toml_path, toml = _load_toml(_use_pyproject)

341 dotenv_path, dotenv = _load_dotenv(_use_env)

342

343 expand_env_vars_into_toml_values(toml, dotenv)

344

345 connection_name = connection_name or dotenv.get("connection", "") or toml.get("default", "")

346 connection: AnyDict = (toml.get(connection_name) if connection_name else toml) or {}

347

348 combined = connection | dotenv | fallback

349 combined = {k.replace("-", "_"): v for k, v in combined.items()}

350

351 combined["pyproject"] = toml_path

352 combined["connection"] = connection_name

353

354 for prop in TypeDALConfig.__annotations__:

355 fill_defaults(combined, prop)

356

357 for prop in TypeDALConfig.__annotations__:

358 transform(combined, prop)

359

360 return TypeDALConfig.load(combined, convert_types=True)