Coverage for /Applications/PyCharm.app/Contents/plugins/python-ce/helpers/pycharm/teamcity/pytest_plugin.py: 35%

1# coding=utf-8

2"""

3 Aaron Buchanan

4 Nov. 2012

5

6 Plug-in for py.test for reporting to TeamCity server

7 Report results to TeamCity during test execution for immediate reporting

8 when using TeamCity.

9

10 This should be installed as a py.test plugin and will be automatically enabled by running

11 tests under TeamCity build.

12"""

13

14 import os

15 import re

16 import sys

17 import traceback

18 from datetime import timedelta

19

20 from teamcity import diff_tools

21 from teamcity import is_running_under_teamcity

22 from teamcity.common import convert_error_to_string, dump_test_stderr, dump_test_stdout

23 from teamcity.messages import TeamcityServiceMessages

24

25 diff_tools.patch_unittest_diff()

26 _ASSERTION_FAILURE_KEY = '_teamcity_assertion_failure'

27

28

29 def pytest_addoption(parser):

30 group = parser.getgroup("terminal reporting", "reporting", after="general")

31

32 group._addoption('--teamcity', action="count",

33 dest="teamcity", default=0, help="force output of JetBrains TeamCity service messages")

34 group._addoption('--no-teamcity', action="count",

35 dest="no_teamcity", default=0, help="disable output of JetBrains TeamCity service messages")

36 parser.addoption('--jb-swapdiff', action="store_true", dest="swapdiff", default=False, help="Swap actual/expected in diff")

37

38 kwargs = {"help": "skip output of passed tests for JetBrains TeamCity service messages"}

39 kwargs.update({"type": "bool"})

40

41 parser.addini("skippassedoutput", **kwargs)

42 parser.addini("swapdiff", **kwargs)

43

44

45 def pytest_configure(config):

46 if config.option.no_teamcity >= 1: 46 ↛ 47line 46 didn't jump to line 47 because the condition on line 46 was never true

47 enabled = False

48 elif config.option.teamcity >= 1: 48 ↛ 49line 48 didn't jump to line 49 because the condition on line 48 was never true

49 enabled = True

50 else:

51 enabled = is_running_under_teamcity()

52

53 if enabled: 53 ↛ exitline 53 didn't return from function 'pytest_configure' because the condition on line 53 was always true

54 output_capture_enabled = getattr(config.option, 'capture', 'fd') != 'no'

55 coverage_controller = _get_coverage_controller(config)

56 skip_passed_output = bool(config.getini('skippassedoutput'))

57

58 config.option.verbose = 2 # don't truncate assert explanations

59 config._teamcityReporting = EchoTeamCityMessages(

60 output_capture_enabled,

61 coverage_controller,

62 skip_passed_output,

63 bool(config.getini('swapdiff') or config.option.swapdiff)

64 )

65 config.pluginmanager.register(config._teamcityReporting)

66

67

68 def pytest_unconfigure(config):

69 teamcity_reporting = getattr(config, '_teamcityReporting', None)

70 if teamcity_reporting:

71 del config._teamcityReporting

72 config.pluginmanager.unregister(teamcity_reporting)

73

74

75 def _get_coverage_controller(config):

76 cov_plugin = config.pluginmanager.getplugin('_cov')

77 if not cov_plugin: 77 ↛ 78line 77 didn't jump to line 78 because the condition on line 77 was never true

78 return None

79

80 return cov_plugin.cov_controller

81

82

83 class EchoTeamCityMessages(object):

84 def __init__(self, output_capture_enabled, coverage_controller, skip_passed_output, swap_diff):

85 self.coverage_controller = coverage_controller

86 self.output_capture_enabled = output_capture_enabled

87 self.skip_passed_output = skip_passed_output

88

89 self.teamcity = TeamcityServiceMessages()

90 self.test_start_reported_mark = set()

91 self.current_test_item = None

92

93 self.max_reported_output_size = 1 * 1024 * 1024

94 self.reported_output_chunk_size = 50000

95 self.swap_diff = swap_diff

96

97 def get_id_from_location(self, location):

98 if type(location) is not tuple or len(location) != 3 or not hasattr(location[2], "startswith"): 98 ↛ 99line 98 didn't jump to line 99 because the condition on line 98 was never true

99 return None

100

101 def convert_file_to_id(filename):

102 filename = re.sub(r"\.pyc?$", "", filename)

103 return filename.replace(os.sep, ".").replace("/", ".")

104

105 def add_prefix_to_filename_id(filename_id, prefix):

106 dot_location = filename_id.rfind('.')

107 if dot_location <= 0 or dot_location >= len(filename_id) - 1:

108 return None

109

110 return filename_id[:dot_location + 1] + prefix + filename_id[dot_location + 1:]

111

112 pylint_prefix = '[pylint] '

113 if location[2].startswith(pylint_prefix): 113 ↛ 114line 113 didn't jump to line 114 because the condition on line 113 was never true

114 id_from_file = convert_file_to_id(location[2][len(pylint_prefix):])

115 return id_from_file + ".Pylint"

116

117 if location[2] == "PEP8-check": 117 ↛ 118line 117 didn't jump to line 118 because the condition on line 117 was never true

118 id_from_file = convert_file_to_id(location[0])

119 return id_from_file + ".PEP8"

120

121 return None

122

123 def format_test_id(self, nodeid, location):

124 id_from_location = self.get_id_from_location(location)

125

126 if id_from_location is not None: 126 ↛ 127line 126 didn't jump to line 127 because the condition on line 126 was never true

127 return id_from_location

128

129 test_id = nodeid

130

131 if test_id:

132 if test_id.find("::") < 0:

133 test_id += "::top_level"

134 else:

135 test_id = "top_level"

136

137 first_bracket = test_id.find("[")

138 if first_bracket > 0: 138 ↛ 140line 138 didn't jump to line 140 because the condition on line 138 was never true

139 # [] -> (), make it look like nose parameterized tests

140 params = "(" + test_id[first_bracket + 1:]

141 if params.endswith("]"):

142 params = params[:-1] + ")"

143 test_id = test_id[:first_bracket]

144 if test_id.endswith("::"):

145 test_id = test_id[:-2]

146 else:

147 params = ""

148

149 test_id = test_id.replace("::()::", "::")

150 test_id = re.sub(r"\.pyc?::", r"::", test_id)

151 test_id = test_id.replace(".", "_").replace(os.sep, ".").replace("/", ".").replace('::', '.')

152

153 if params: 153 ↛ 154line 153 didn't jump to line 154 because the condition on line 153 was never true

154 params = params.replace(".", "_")

155 test_id += params

156

157 return test_id

158

159 def format_location(self, location):

160 if type(location) is tuple and len(location) == 3: 160 ↛ 162line 160 didn't jump to line 162 because the condition on line 160 was always true

161 return "%s:%s (%s)" % (str(location[0]), str(location[1]), str(location[2]))

162 return str(location)

163

164 def pytest_collection_finish(self, session):

165 self.teamcity.testCount(len(session.items))

166

167 def pytest_runtest_logstart(self, nodeid, location):

168 # test name fetched from location passed as metainfo to PyCharm

169 # it will be used to run specific test

170 # See IDEA-176950, PY-31836

171 test_name = location[2]

172 if test_name: 172 ↛ 174line 172 didn't jump to line 174 because the condition on line 172 was always true

173 test_name = str(test_name).split(".")[-1]

174 self.ensure_test_start_reported(self.format_test_id(nodeid, location), test_name)

175

176 def pytest_runtest_protocol(self, item):

177 self.current_test_item = item

178 return None # continue to next hook

179

180 def ensure_test_start_reported(self, test_id, metainfo=None):

181 if test_id not in self.test_start_reported_mark:

182 if self.output_capture_enabled: 182 ↛ 185line 182 didn't jump to line 185 because the condition on line 182 was always true

183 capture_standard_output = "false"

184 else:

185 capture_standard_output = "true"

186 self.teamcity.testStarted(test_id, flowId=test_id, captureStandardOutput=capture_standard_output, metainfo=metainfo)

187 self.test_start_reported_mark.add(test_id)

188

189 def report_has_output(self, report):

190 for (secname, data) in report.sections:

191 if report.when in secname and ('stdout' in secname or 'stderr' in secname): 191 ↛ 192line 191 didn't jump to line 192 because the condition on line 191 was never true

192 return True

193 return False

194

195 def report_test_output(self, report, test_id):

196 for (secname, data) in report.sections:

197 # https://github.com/JetBrains/teamcity-messages/issues/112

198 # CollectReport didn't have 'when' property, but now it has.

199 # But we still need output on 'collect' state

200 if hasattr(report, "when") and report.when not in secname and report.when != 'collect': 200 ↛ 201line 200 didn't jump to line 201 because the condition on line 200 was never true

201 continue

202 if not data: 202 ↛ 203line 202 didn't jump to line 203 because the condition on line 202 was never true

203 continue

204

205 if 'stdout' in secname: 205 ↛ 206line 205 didn't jump to line 206 because the condition on line 205 was never true

206 dump_test_stdout(self.teamcity, test_id, test_id, data)

207 elif 'stderr' in secname:

208 dump_test_stderr(self.teamcity, test_id, test_id, data)

209

210 def report_test_finished(self, test_id, duration=None):

211 self.teamcity.testFinished(test_id, testDuration=duration, flowId=test_id)

212 self.test_start_reported_mark.remove(test_id)

213

214 def report_test_failure(self, test_id, report, message=None, report_output=True):

215 if hasattr(report, 'duration'): 215 ↛ 218line 215 didn't jump to line 218 because the condition on line 215 was always true

216 duration = timedelta(seconds=report.duration)

217 else:

218 duration = None

219

220 if message is None: 220 ↛ 223line 220 didn't jump to line 223 because the condition on line 220 was always true

221 message = self.format_location(report.location)

222

223 self.ensure_test_start_reported(test_id)

224 if report_output: 224 ↛ 227line 224 didn't jump to line 227 because the condition on line 224 was always true

225 self.report_test_output(report, test_id)

226

227 diff_error = None

228 try:

229 err_message = str(report.longrepr.reprcrash.message)

230 diff_name = diff_tools.EqualsAssertionError.__name__

231 # There is a string like "foo.bar.DiffError: [serialized_data]"

232 if diff_name in err_message: 232 ↛ 233line 232 didn't jump to line 233 because the condition on line 232 was never true

233 serialized_data = err_message[err_message.index(diff_name) + len(diff_name) + 1:]

234 diff_error = diff_tools.deserialize_error(serialized_data)

235

236 assertion_tuple = getattr(self.current_test_item, _ASSERTION_FAILURE_KEY, None)

237 if assertion_tuple: 237 ↛ 238line 237 didn't jump to line 238 because the condition on line 237 was never true

238 op, left, right = assertion_tuple

239 if self.swap_diff:

240 left, right = right, left

241 diff_error = diff_tools.EqualsAssertionError(expected=right, actual=left)

242 except Exception:

243 pass

244

245 if not diff_error: 245 ↛ 249line 245 didn't jump to line 249 because the condition on line 245 was always true

246 from .jb_local_exc_store import get_exception

247 diff_error = get_exception()

248

249 if diff_error: 249 ↛ 251line 249 didn't jump to line 251 because the condition on line 249 was never true

250 # Cut everything after postfix: it is internal view of DiffError

251 strace = str(report.longrepr)

252 data_postfix = "_ _ _ _ _"

253 # Error message in pytest must be in "file.py:22 AssertionError" format

254 # This message goes to strace

255 # With custom error we must add real exception class explicitly

256 if data_postfix in strace:

257 strace = strace[0: strace.index(data_postfix)].strip()

258 if strace.endswith(":") and diff_error.real_exception:

259 strace += " " + type(diff_error.real_exception).__name__

260 self.teamcity.testFailed(test_id, diff_error.msg or message, strace,

261 flowId=test_id,

262 comparison_failure=diff_error

263 )

264 else:

265 self.teamcity.testFailed(test_id, message, str(report.longrepr), flowId=test_id)

266 self.report_test_finished(test_id, duration)

267

268 def report_test_skip(self, test_id, report):

269 if type(report.longrepr) is tuple and len(report.longrepr) == 3:

270 reason = report.longrepr[2]

271 else:

272 reason = str(report.longrepr)

273

274 if hasattr(report, 'duration'):

275 duration = timedelta(seconds=report.duration)

276 else:

277 duration = None

278

279 self.ensure_test_start_reported(test_id)

280 self.report_test_output(report, test_id)

281 self.teamcity.testIgnored(test_id, reason, flowId=test_id)

282 self.report_test_finished(test_id, duration)

283

284 def pytest_assertrepr_compare(self, config, op, left, right):

285 setattr(self.current_test_item, _ASSERTION_FAILURE_KEY, (op, left, right))

286

287 def pytest_runtest_logreport(self, report):

288 """

289 :type report: _pytest.runner.TestReport

290 """

291 test_id = self.format_test_id(report.nodeid, report.location)

292

293 duration = timedelta(seconds=report.duration)

294

295 if report.passed:

296 # Do not report passed setup/teardown if no output

297 if report.when == 'call': 297 ↛ 298line 297 didn't jump to line 298 because the condition on line 297 was never true

298 self.ensure_test_start_reported(test_id)

299 if not self.skip_passed_output:

300 self.report_test_output(report, test_id)

301 self.report_test_finished(test_id, duration)

302 else:

303 if self.report_has_output(report) and not self.skip_passed_output: 303 ↛ 304line 303 didn't jump to line 304 because the condition on line 303 was never true

304 block_name = "test " + report.when

305 self.teamcity.blockOpened(block_name, flowId=test_id)

306 self.report_test_output(report, test_id)

307 self.teamcity.blockClosed(block_name, flowId=test_id)

308 elif report.failed: 308 ↛ 321line 308 didn't jump to line 321 because the condition on line 308 was always true

309 if report.when == 'call': 309 ↛ 311line 309 didn't jump to line 311 because the condition on line 309 was always true

310 self.report_test_failure(test_id, report)

311 elif report.when == 'setup':

312 if self.report_has_output(report):

313 self.teamcity.blockOpened("test setup", flowId=test_id)

314 self.report_test_output(report, test_id)

315 self.teamcity.blockClosed("test setup", flowId=test_id)

316

317 self.report_test_failure(test_id, report, message="test setup failed", report_output=False)

318 elif report.when == 'teardown':

319 # Report failed teardown as a separate test as original test is already finished

320 self.report_test_failure(test_id + "_teardown", report)

321 elif report.skipped:

322 self.report_test_skip(test_id, report)

323

324 def pytest_collectreport(self, report):

325 test_id = self.format_test_id(report.nodeid, report.location) + "_collect"

326

327 if report.failed: 327 ↛ 328line 327 didn't jump to line 328 because the condition on line 327 was never true

328 self.report_test_failure(test_id, report)

329 elif report.skipped: 329 ↛ 330line 329 didn't jump to line 330 because the condition on line 329 was never true

330 self.report_test_skip(test_id, report)

331

332 def pytest_terminal_summary(self):

333 if self.coverage_controller is not None:

334 try:

335 self._report_coverage()

336 except Exception:

337 tb = traceback.format_exc()

338 self.teamcity.customMessage("Coverage statistics reporting failed", "ERROR", errorDetails=tb)

339

340 def _report_coverage(self):

341 from coverage.misc import NotPython

342 from coverage.results import Numbers

343

344 class _Reporter(object):

345 def __init__(self, coverage, config):

346 try:

347 from coverage.report import Reporter

348 except ImportError:

349 # Support for coverage >= 5.0.1.

350 from coverage.report import get_analysis_to_report

351

352 class Reporter(object):

353

354 def __init__(self, coverage, config):

355 self.coverage = coverage

356 self.config = config

357 self._file_reporters = []

358

359 def find_file_reporters(self, morfs):

360 return [fr for fr, _ in get_analysis_to_report(self.coverage, morfs)]

361

362 self._reporter = Reporter(coverage, config)

363

364 def find_file_reporters(self, morfs):

365 self.file_reporters = self._reporter.find_file_reporters(morfs)

366

367 def __getattr__(self, name):

368 return getattr(self._reporter, name)

369

370 class _CoverageReporter(_Reporter):

371 def __init__(self, coverage, config, messages):

372 super(_CoverageReporter, self).__init__(coverage, config)

373

374 if hasattr(coverage, 'data'):

375 self.branches = coverage.data.has_arcs()

376 else:

377 self.branches = coverage.get_data().has_arcs()

378 self.messages = messages

379

380 def report(self, morfs, outfile=None):

381 if hasattr(self, 'find_code_units'):

382 self.find_code_units(morfs)

383 else:

384 self.find_file_reporters(morfs)

385

386 total = Numbers()

387

388 if hasattr(self, 'code_units'):

389 units = self.code_units

390 else:

391 units = self.file_reporters

392

393 for cu in units:

394 try:

395 analysis = self.coverage._analyze(cu)

396 nums = analysis.numbers

397 total += nums

398 except KeyboardInterrupt:

399 raise

400 except Exception:

401 if self.config.ignore_errors:

402 continue

403

404 err = sys.exc_info()

405 typ, msg = err[:2]

406 if typ is NotPython and not cu.should_be_python():

407 continue

408

409 test_id = cu.name

410 details = convert_error_to_string(err)

411

412 self.messages.testStarted(test_id, flowId=test_id)

413 self.messages.testFailed(test_id, message="Coverage analysis failed", details=details, flowId=test_id)

414 self.messages.testFinished(test_id, flowId=test_id)

415

416 if total.n_files > 0:

417 covered = total.n_executed

418 total_statements = total.n_statements

419

420 if self.branches:

421 covered += total.n_executed_branches

422 total_statements += total.n_branches

423

424 self.messages.buildStatisticLinesCovered(covered)

425 self.messages.buildStatisticTotalLines(total_statements)

426 self.messages.buildStatisticLinesUncovered(total_statements - covered)

427 reporter = _CoverageReporter(

428 self.coverage_controller.cov,

429 self.coverage_controller.cov.config,

430 self.teamcity,

431 )

432 reporter.report(None)