Coverage for src/hatch_ci/tools.py: 87%

1# see https://pypi.org/project/setuptools-github

2# copy of setuptools_github.tools

3 from __future__ import annotations

4

5 import ast

6 import json

7 import re

8 from pathlib import Path

9 from typing import Any

10

11 from . import scm

12

13

14 class ToolsError(Exception):

15 pass

16

17

18 class ValidationError(ToolsError):

19 pass

20

21

22 class InvalidVersionError(ToolsError):

23 pass

24

25

26 class MissingVariableError(ToolsError):

27 pass

28

29

30 class AbortExecutionError(Exception):

31 @staticmethod

32 def _strip(txt):

33 txt = txt or ""

34 txt = txt[1:] if txt.startswith("\n") else txt

35 txt = indent(txt, pre="")

36 return txt[:-1] if txt.endswith("\n") else txt

37

38 def __init__(

39 self, message: str, explain: str | None = None, hint: str | None = None

40 ):

41 self.message = message.strip()

42 self._explain = explain

43 self._hint = hint

44

45 @property

46 def explain(self):

47 return self._strip(self._explain)

48

49 @property

50 def hint(self):

51 return self._strip(self._hint)

52

53 def __str__(self):

54 result = [self.message]

55 if self.explain:

56 result.append(indent("\n" + self.explain, pre=" " * 2)[2:])

57 if self.hint:

58 result.extend(["\nhint:", indent("\n" + self.hint, pre=" " * 2)[2:]])

59 return "".join(result)

60

61

62 def urmtree(path: Path):

63 "universal (win|*nix) rmtree"

64 from os import name

65 from shutil import rmtree

66 from stat import S_IWUSR

67

68 if name == "nt": 68 ↛ 69line 68 didn't jump to line 69, because the condition on line 68 was never true

69 for p in path.rglob("*"):

70 p.chmod(S_IWUSR)

71 rmtree(path, ignore_errors=True)

72 if path.exists(): 72 ↛ 73line 72 didn't jump to line 73, because the condition on line 72 was never true

73 raise RuntimeError(f"cannot remove {path=}")

74

75

76 def indent(txt: str, pre: str = " " * 2) -> str:

77 "simple text indentation"

78

79 from textwrap import dedent

80

81 txt = dedent(txt)

82 if txt.endswith("\n"):

83 last_eol = "\n"

84 txt = txt[:-1]

85 else:

86 last_eol = ""

87

88 result = pre + txt.replace("\n", "\n" + pre) + last_eol

89 return result if result.strip() else result.strip()

90

91

92 def list_of_paths(paths: str | Path | list[str | Path] | None) -> list[Path]:

93 if not paths:

94 return []

95 return [Path(s) for s in ([paths] if isinstance(paths, (str, Path)) else paths)]

96

97

98 def lstrip(txt: str, left: str) -> str:

99 return txt[len(left) :] if txt.startswith(left) else txt

100

101

102 def loadmod(path: Path) -> Any:

103 from importlib.util import module_from_spec, spec_from_file_location

104

105 module = None

106 spec = spec_from_file_location(Path(path).name, Path(path))

107 if spec:

108 module = module_from_spec(spec)

109 if module and spec and spec.loader:

110 spec.loader.exec_module(module)

111 return module

112

113

114 def apply_fixers(txt: str, fixers: dict[str, str] | None = None) -> str:

115 result = txt

116 for src, dst in (fixers or {}).items():

117 if src.startswith("re:"):

118 result = re.sub(src[3:], dst, result)

119 else:

120 result = result.replace(src, dst)

121 return result

122

123

124 def get_module_var(

125 path: Path | str, var: str = "__version__", abort=True

126) -> str | None:

127 """extract from a python module in path the module level <var> variable

128

129 Args:

130 path (str,Path): python module file to parse using ast (no code-execution)

131 var (str): module level variable name to extract

132 abort (bool): raise MissingVariable if var is not present

133

134 Returns:

135 None or str: the variable value if found or None

136

137 Raises:

138 MissingVariable: if the var is not found and abort is True

139

140 Notes:

141 this uses ast to parse path, so it doesn't load the module

142 """

143

144 class V(ast.NodeVisitor):

145 def __init__(self, keys):

146 self.keys = keys

147 self.result = {}

148

149 def visit_Module(self, node): # noqa: N802

150 # we extract the module level variables

151 for subnode in ast.iter_child_nodes(node):

152 if not isinstance(subnode, ast.Assign): 152 ↛ 153line 152 didn't jump to line 153, because the condition on line 152 was never true

153 continue

154 for target in subnode.targets:

155 if target.id not in self.keys:

156 continue

157 if not isinstance(subnode.value, (ast.Num, ast.Str, ast.Constant)):

158 raise ValidationError(

159 f"cannot extract non Constant variable "

160 f"{target.id} ({type(subnode.value)})"

161 )

162 if isinstance(subnode.value, ast.Str):

163 value = subnode.value.s

164 elif isinstance(subnode.value, ast.Num): 164 ↛ 167line 164 didn't jump to line 167, because the condition on line 164 was never false

165 value = subnode.value.n

166 else:

167 value = subnode.value.value

168 if target.id in self.result:

169 raise ValidationError(

170 f"found multiple repeated variables {target.id}"

171 )

172 self.result[target.id] = value

173 return self.generic_visit(node)

174

175 v = V({var})

176 path = Path(path)

177 if path.exists():

178 tree = ast.parse(Path(path).read_text())

179 v.visit(tree)

180 if var not in v.result and abort:

181 raise MissingVariableError(f"cannot find {var} in {path}", path, var)

182 return v.result.get(var, None)

183

184

185 def set_module_var(

186 path: str | Path, var: str, value: Any, create: bool = True

187) -> tuple[Any, str]:

188 """replace var in path with value

189

190 Args:

191 path (str,Path): python module file to parse

192 var (str): module level variable name to extract

193 value (None or Any): if not None replace var in version_file

194 create (bool): create path if not present

195

196 Returns:

197 (str, str) the (<previous-var-value|None>, <the new text>)

198 """

199

200 # validate the var

201 get_module_var(path, var, abort=False)

202

203 # module level var

204 expr = re.compile(f"^{var}\\s*=\\s*['\\\"](?P<value>[^\\\"']*)['\\\"]")

205 fixed = None

206 lines = []

207

208 src = Path(path)

209 if not src.exists() and create:

210 src.parent.mkdir(parents=True, exist_ok=True)

211 src.touch()

212

213 input_lines = src.read_text().split("\n")

214 for line in input_lines:

215 if fixed is not None:

216 lines.append(line)

217 continue

218 match = expr.search(line)

219 if match:

220 fixed = match.group("value")

221 if value is not None: 221 ↛ 224line 221 didn't jump to line 224, because the condition on line 221 was never false

222 x, y = match.span(1)

223 line = line[:x] + value + line[y:]

224 lines.append(line)

225 txt = "\n".join(lines)

226 if (fixed is None) and create:

227 if txt and txt[-1] != "\n":

228 txt += "\n"

229 txt += f'{var} = "{value}"'

230

231 with Path(path).open("w") as fp:

232 fp.write(txt)

233 return fixed, txt

234

235

236 def bump_version(version: str, mode: str) -> str:

237 """given a version str will bump it according to mode

238

239 Arguments:

240 version: text in the N.M.O form

241 mode: major, minor or micro

242

243 Returns:

244 increased text

245

246 >>> bump_version("1.0.3", "micro")

247 "1.0.4"

248 >>> bump_version("1.0.3", "minor")

249 "1.1.0"

250 """

251 newver = [int(n) for n in version.split(".")]

252 if mode == "major":

253 newver[-3] += 1

254 newver[-2] = 0

255 newver[-1] = 0

256 elif mode == "minor":

257 newver[-2] += 1

258 newver[-1] = 0

259 elif mode == "micro":

260 newver[-1] += 1

261 return ".".join(str(v) for v in newver)

262

263

264 def get_data(

265 version_file: str | Path,

266 github_dump: str | None = None,

267 record_path: Path | None = None,

268 abort: bool = True,

269) -> tuple[dict[str, str | None], dict[str, Any]]:

270 """extracts version information from github_dump and updates version_file in-place

271

272 Args:

273 version_file (str, Path): path to a file with a __version__ variable

274 github_dump (str): the os.getenv("GITHUB_DUMP") value

275 record: pull data from a _build.py file

276

277 Returns:

278 dict[str,str|None]: a dict with the current config

279 """

280 result = {

281 "version": get_module_var(version_file, "__version__"),

282 "current": get_module_var(version_file, "__version__"),

283 "branch": None,

284 "hash": None,

285 "build": None,

286 "runid": None,

287 "workflow": None,

288 }

289

290 path = Path(version_file)

291 repo = scm.lookup(path)

292 record = record_path.exists() if record_path else None

293

294 if not (repo or github_dump or record): 294 ↛ 295line 294 didn't jump to line 295, because the condition on line 294 was never true

295 if abort:

296 raise scm.InvalidGitRepoError(

297 f"cannot figure out settings (no repo in {path}, "

298 f"a GITHUB_DUMP or a _build.py file)"

299 )

300 return result, {}

301

302 dirty = False

303 if github_dump:

304 gdata = json.loads(github_dump) if isinstance(github_dump, str) else github_dump

305 elif record_path and record_path.exists(): 305 ↛ 306line 305 didn't jump to line 306, because the condition on line 305 was never true

306 mod = loadmod(record_path)

307 gdata = {k: getattr(mod, k) for k in dir(mod) if not k.startswith("_")}

308 elif repo: 308 ↛ 317line 308 didn't jump to line 317, because the condition on line 308 was never false

309 gdata = {

310 "ref": repo.head.name,

311 "sha": repo.head.target.hex[:7],

312 "run_number": 0,

313 "run_id": 0,

314 }

315 dirty = repo.dirty()

316 else:

317 raise RuntimeError("un-reacheable code")

318

319 expr = re.compile(r"/(?P<what>beta|release)/(?P<version>\d+([.]\d+)*)$")

320 expr1 = re.compile(r"(?P<version>\d+([.]\d+)*)(?P<num>b\d+)?$")

321

322 result["branch"] = lstrip(gdata["ref"], "refs/heads/")

323 result["hash"] = gdata["sha"] + ("*" if dirty else "")

324 result["build"] = gdata["run_number"]

325 result["runid"] = gdata["run_id"]

326 result["workflow"] = result["branch"]

327

328 current = result["current"]

329 if match := expr.search(gdata["ref"]):

330 # setuptools double calls the update_version,

331 # this fixes the issue

332 match1 = expr1.search(current or "")

333 if not match1: 333 ↛ 334line 333 didn't jump to line 334, because the condition on line 333 was never true

334 raise InvalidVersionError(f"cannot parse current version '{current}'")

335 if match1.group("version") != match.group("version"):

336 raise InvalidVersionError(

337 f"building package for {current} from '{gdata['ref']}' "

338 f"branch ({match.groupdict()} mismatch {match1.groupdict()})"

339 )

340 if match.group("what") == "beta":

341 result["version"] = f"{match1.group('version')}b{gdata['run_number']}"

342 result["workflow"] = "beta"

343 else:

344 result["workflow"] = "tags"

345 return result, gdata

346

347

348 def update_version(

349 version_file: str | Path, github_dump: str | None = None, abort: bool = True

350) -> str | None:

351 """extracts version information from github_dump and updates version_file in-place

352

353 Args:

354 version_file (str, Path): path to a file with a __version__ variable

355 github_dump (str): the os.getenv("GITHUB_DUMP") value

356

357 Returns:

358 str: the new version for the package

359 """

360

361 data = get_data(version_file, github_dump, abort=abort)[0]

362 set_module_var(version_file, "__version__", data["version"])

363 set_module_var(version_file, "__hash__", data["hash"])

364 return data["version"]

365

366

367 def process(

368 version_file: str | Path,

369 github_dump: str | None = None,

370 paths: str | Path | list[str | Path] | None = None,

371 fixers: dict[str, str] | None = None,

372 record: str | Path = "_build.py",

373 abort: bool = True,

374) -> dict[str, str | None]:

375 """get version from github_dump and updates version_file/paths

376

377 Args:

378 version_file (str, Path): path to a file with __version__ variable

379 github_dump (str): the os.getenv("GITHUB_DUMP") value

380 paths (str, Path): path(s) to files jinja2 processeable

381 fixers (dict[str,str]): fixer dictionary

382 record: set to True will generate a _build.py sibling of version_file

383

384 Returns:

385 str: the new version for the package

386

387 Example:

388 {'branch': 'beta/0.3.1',

389 'build': 0,

390 'current': '0.3.1',

391 'hash': 'c9e484a*',

392 'version': '0.3.1b0',

393 'runid': 0

394 }

395 """

396 from argparse import Namespace

397 from functools import partial

398 from urllib.parse import quote

399

400 from jinja2 import Environment

401

402 class Context(Namespace):

403 def items(self):

404 for name, value in self.__dict__.items():

405 if name.startswith("_"): 405 ↛ 406line 405 didn't jump to line 406, because the condition on line 405 was never true

406 continue

407 yield (name, value)

408

409 record_path = (Path(version_file).parent / record).absolute() if record else None

410 data, gdata = get_data(version_file, github_dump, record_path, abort)

411 set_module_var(version_file, "__version__", data["version"])

412 set_module_var(version_file, "__hash__", data["hash"])

413

414 env = Environment(autoescape=True)

415 env.filters["urlquote"] = partial(quote, safe="")

416 for path in list_of_paths(paths):

417 txt = apply_fixers(path.read_text(), fixers)

418 tmpl = env.from_string(txt)

419 path.write_text(tmpl.render(ctx=Context(**data)))

420

421 if record_path: 421 ↛ 429line 421 didn't jump to line 429, because the condition on line 421 was never false

422 record_path.parent.mkdir(parents=True, exist_ok=True)

423 with record_path.open("w") as fp:

424 print("# autogenerate build file", file=fp)

425 for key, value in sorted((gdata or {}).items()):

426 value = f"'{value}'" if isinstance(value, str) else value

427 print(f"{key} = {value}", file=fp)

428

429 return data