Coverage for testrail_api_reporter/engines/results_reporter.py: 12%

1# -*- coding: utf-8 -*-

2""" Module for reporting results to TestRails from xml report results, obtained by pytest """

3

4 import datetime

5 from os.path import exists

6

7 from requests.exceptions import ReadTimeout

8 from testrail_api import TestRailAPI, StatusCodeError

9 from xmltodict import parse

10

11 from ..utils.logger_config import setup_logger, DEFAULT_LOGGING_LEVEL

12 from ..utils.reporter_utils import format_error, init_get_cases_process

13

14

15 class TestRailResultsReporter:

16 """Reporter to TestRails from xml report results, obtained by pytest"""

17

18 def __init__(

19 self,

20 url: str,

21 email: str,

22 password: str,

23 project_id: int,

24 xml_report="junit-report.xml",

25 suite_id=None,

26 logger=None,

27 log_level=DEFAULT_LOGGING_LEVEL,

28 ):

29 """

30 Default init

31

32 :param url: url of TestRail, string, required

33 :param email: email of TestRail user with proper access rights, string, required

34 :param password: password of TestRail user with proper access rights, string, required

35 :param project_id: project id, integer, required

36 :param xml_report: filename (maybe with path) of xml test report

37 :param suite_id: suite id, integer, optional, if no suite-management is activated

38 :param logger: logger object, optional

39 :param log_level: logging level, optional, by default is 'logging.DEBUG'

40 """

41 if not logger:

42 self.___logger = setup_logger(

43 name="TestRailResultsReporter", log_file="TestRailResultsReporter.log", level=log_level

44 )

45 else:

46 self.___logger = logger

47 self.___logger.debug("Initializing TestRail Results Reporter")

48 if url is None or email is None or password is None:

49 raise ValueError("No TestRails credentials are provided!")

50 self.__api = TestRailAPI(url, email, password)

51 self.__xml_report = xml_report if self.__check_report_exists(xml_report=xml_report) else None

52 self.__project_id = project_id if self.__check_project(project_id=project_id) else None

53 self.__suite_id = suite_id if self.__check_suite(suite_id=suite_id) else None

54 self.__at_section = self.__ensure_automation_section() if self.__project_id else None

55 self.__check_section(section_id=self.__at_section)

56 self.__timestamp = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")

57

58 def __xml_to_dict(self, filename="junit-report.xml"):

59 """

60 Converts xml file to python dict

61 :param filename: filename, string, maybe with path

62 :return: dict with a list of cases

63 """

64 if not self.__check_report_exists(xml_report=self.__xml_report):

65 return None

66 with open(filename, "r", encoding="utf-8") as file:

67 xml = file.read()

68

69 list_of_cases = []

70

71 parsed_xml = parse(xml)

72 self.__timestamp = parsed_xml["testsuites"]["testsuite"]["@timestamp"].split(".")[0]

73

74 cases = parsed_xml["testsuites"]["testsuite"]["testcase"]

75 cases = cases if isinstance(cases, list) else [cases]

76 for item in cases:

77 status = 1

78 if "failure" in item.keys():

79 status = 5

80 if "skipped" in item.keys():

81 if item["skipped"]["@type"] == "pytest.xfail":

82 status = 5

83 else:

84 status = 7

85 list_of_cases.append(

86 {

87 "automation_id": f'{item["@classname"]}.{item["@name"]}',

88 "time": item["@time"],

89 "status": status,

90 "comment": (

91 f'{item["failure"]["@message"]} : ' f'{item["failure"]["#text"]}'

92 if "failure" in item.keys()

93 else ""

94 ),

95 }

96 )

97 self.___logger.debug("Found test run at %s, found %s test results", self.__timestamp, len(list_of_cases))

98 return list_of_cases

99

100 @staticmethod

101 def __search_for_item(searched_value, list_to_seek, field):

102 """

103 Item seeker by value within a list of dicts

104

105 :param searched_value: value what we're looking for

106 :param list_to_seek: a list where we perform the search

107 :param field: field of a list dict

108 :return: element

109 """

110 for item in list_to_seek:

111 if item[field] == searched_value:

112 return item

113

114 return [element for element in list_to_seek if element[field] == searched_value]

115

116 def __ensure_automation_section(self, title="pytest"):

117 """

118 Service function, checks that special (default) placeholder for automation non-classified tests exists

119

120 :param title: title for default folder, string

121 :return: id of a section

122 """

123 first_run = True

124 item_id = None

125 criteria = None

126 response = None

127 while criteria is not None or first_run:

128 if first_run:

129 try:

130 response = self.__api.sections.get_sections(project_id=self.__project_id, suite_id=self.__suite_id)

131 except Exception as error:

132 self.___logger.error(

133 "Get sections failed. Please validate your settings!\nError%s", format_error(error)

134 )

135 self.__self_check()

136 return None

137 first_run = False

138 elif response["_links"]["next"] is not None: # pylint: disable=E1136

139 offset = int(response["_links"]["next"].split("&offset=")[1].split("&")[0]) # pylint: disable=E1136

140 response = self.__api.sections.get_sections(

141 project_id=self.__project_id, suite_id=self.__suite_id, offset=offset

142 )

143 sections = response["sections"]

144 for item in sections:

145 if item["name"] == title:

146 item_id = item["id"]

147 criteria = response["_links"]["next"]

148 if not item_id:

149 try:

150 item_id = self.__api.sections.add_section(

151 project_id=self.__project_id, suite_id=self.__suite_id, name=title

152 )["id"]

153 except Exception as error:

154 self.___logger.error("Can't add section. Something nasty happened.\nError%s", format_error(error))

155 self.__self_check()

156 return None

157 self.___logger.debug("No default automation folder is found, created new one with name '%s'", title)

158 return item_id

159

160 def __enrich_with_tc_num(self, xml_dict_list, tc_dict_list):

161 """

162 Add a test case id to case result

163

164 :param xml_dict_list: list of dict, with test cases, obtained from xml report

165 :param tc_dict_list: list of dict, with test cases, obtained from TestRails

166 :return: enriched list of dict with test cases

167 """

168 enriched_list = []

169 missed_tests_counter = 0

170 for item in xml_dict_list:

171 cases = self.__search_for_item(

172 searched_value=item["automation_id"], list_to_seek=tc_dict_list, field="custom_automation_id"

173 )

174 if not cases:

175 try:

176 cases = [

177 self.__api.cases.add_case(

178 section_id=self.__at_section,

179 title=item["automation_id"],

180 custom_automation_id=item["automation_id"],

181 )

182 ]

183 except Exception as error:

184 self.___logger.error(

185 "Add case failed. Please validate your settings!\nError: %s", format_error(error)

186 )

187 self.__self_check()

188 return None

189 missed_tests_counter = missed_tests_counter + 1

190 cases = cases if isinstance(cases, list) else [cases]

191 for case in cases:

192 comment = item["message"] if "failure" in item.keys() else ""

193 elapsed = item["time"].split(".")[0]

194 elapsed = 1 if elapsed == 0 else elapsed

195 enriched_list.append(

196 {

197 "case_id": case["id"], # type: ignore

198 "status_id": item["status"],

199 "comment": comment,

200 "elapsed": elapsed,

201 "attachments": [],

202 }

203 )

204 if missed_tests_counter:

205 self.___logger.debug("Missed %s test cases, they was automatically created", missed_tests_counter)

206 self.___logger.debug("Found %s test cases in TestRails", len(enriched_list))

207 return enriched_list

208

209 def ___handle_read_timeout(self, retry, retries, error):

210 if retry < retries:

211 retry += 1

212 self.___logger.debug("Timeout error, retrying %s/%s...", retry, retries)

213 return retry, True

214 raise ValueError(f"Get cases failed. Please validate your settings!nError{format_error(error)}") from error

215

216 # pylint: disable=R0912

217 def __get_all_auto_cases(self, retries=3):

218 """

219 Collects all test cases from TestRails with non-empty automation_id

220

221 :param retries: number of retries, integer

222 :return: list of dict with cases

223 """

224 cases_list, first_run, criteria, response, retry = init_get_cases_process()

225 while criteria is not None or first_run:

226 if first_run:

227 try:

228 response = self.__api.cases.get_cases(project_id=self.__project_id, suite_id=self.__suite_id)

229 except (ReadTimeout, StatusCodeError) as error:

230 if (

231 isinstance(error, StatusCodeError)

232 and error.status_code == 504 # type: ignore # pylint: disable=no-member

233 or isinstance(error, ReadTimeout)

234 ):

235 retry, should_continue = self.___handle_read_timeout(retry, retries, error)

236 if should_continue:

237 continue

238 except Exception as error:

239 self.___logger.error(

240 "Get cases failed. Please validate your settings!\nError%s", format_error(error)

241 )

242 self.__self_check()

243 return None

244 first_run = False

245 retry = 0

246 elif response["_links"]["next"] is not None: # pylint: disable=E1136

247 offset = int(response["_links"]["next"].split("&offset=")[1].split("&")[0]) # pylint: disable=E1136

248 try:

249 response = self.__api.cases.get_cases(

250 project_id=self.__project_id, suite_id=self.__suite_id, offset=offset

251 )

252 except (ReadTimeout, StatusCodeError) as error:

253 if (

254 isinstance(error, StatusCodeError)

255 and error.status_code == 504 # type: ignore # pylint: disable=no-member

256 or isinstance(error, ReadTimeout)

257 ):

258 retry, should_continue = self.___handle_read_timeout(retry, retries, error)

259 if should_continue:

260 continue

261 retry = 0

262 cases = response["cases"]

263 for item in cases:

264 if item["custom_automation_id"] is not None:

265 cases_list.append({"id": item["id"], "custom_automation_id": item["custom_automation_id"]})

266 criteria = response["_links"]["next"]

267 self.___logger.debug("Found %s test cases in TestRails with automation_id", len(cases_list))

268 return cases_list

269

270 def __prepare_payload(self):

271 """

272 Prepares payload from xml report for sending to TestRails

273

274 :return: payload in proper format (list of dicts)

275 """

276 parsed_xml = self.__xml_to_dict(filename=self.__xml_report)

277 parsed_cases = self.__get_all_auto_cases()

278 if not parsed_xml:

279 self.___logger.error("Preparation of payload failed, aborted")

280 return None

281 payload = self.__enrich_with_tc_num(xml_dict_list=parsed_xml, tc_dict_list=parsed_cases)

282 return payload

283

284 def __prepare_title(self, environment=None, timestamp=None):

285 """

286 Format test run name based on input string (most probably environment) and timestamp

287

288 :param environment: some string run identifier

289 :param timestamp: custom timestamp

290 :return: string of prepared string for AT run name

291 """

292 if timestamp is None:

293 timestamp = self.__timestamp

294 title = f"AT run {timestamp}"

295 if environment:

296 title = f"{title} on {environment}"

297 return title

298

299 def send_results(

300 self,

301 run_id=None,

302 environment=None,

303 title=None,

304 timestamp=None,

305 close_run=True,

306 run_name=None,

307 delete_old_run=False,

308 ):

309 """

310 Send results to TestRail

311

312 :param run_id: specific run id, if any, optional

313 :param environment: custom name pattern for run name, optional

314 :param title: custom title, if provided, will be used as priority, optional

315 :param timestamp: custom timestamp, optional

316 :param close_run: close or not run, True or False

317 :param run_name: name of test run, will be used if provided at top priority

318 :param delete_old_run: delete or not previous run if old one exists with the same name

319 :return: run id where results were submitted

320 """

321 if (

322 not self.__project_id

323 or not self.__at_section

324 or not self.__check_report_exists(xml_report=self.__xml_report)

325 ):

326 self.___logger.error("Error! Please specify all required params!")

327 self.__self_check()

328 return True

329 title = self.__prepare_title(environment, timestamp) if not title else title

330 title = run_name if run_name else title

331 payload = self.__prepare_payload()

332 run_id = self.__prepare_runs(

333 cases=payload, title=title, run_id=run_id, run_name=run_name, delete_run=delete_old_run

334 )

335 retval = self.__add_results(run_id=run_id, results=payload)

336 if close_run:

337 self.__close_run(run_id=run_id, title=title)

338 self.___logger.debug("%s results were added to test run '%s', cases updated. Done", len(payload), title)

339 return retval

340

341 def set_project_id(self, project_id):

342 """

343 Set project id

344

345 :param project_id: project id, integer

346 """

347 self.__project_id = project_id if self.__check_project(project_id=project_id) else None

348

349 def set_suite_id(self, suite_id):

350 """

351 Set suite id

352

353 :param suite_id: suite id, integer

354 """

355 if self.__check_project():

356 self.__suite_id = suite_id if self.__check_suite(suite_id=suite_id) else None

357

358 def set_xml_filename(self, xml_filename):

359 """

360 Set xml filename

361

362 :param xml_filename: filename of xml report, string

363 """

364 self.__xml_report = xml_filename if self.__check_report_exists(xml_report=xml_filename) else None

365

366 def set_at_report_section(self, section_name):

367 """

368 Set section name for AT a report

369

370 :param section_name: name of a section, string

371 """

372 if self.__check_project() and self.__check_suite():

373 self.__at_section = self.__ensure_automation_section(title=section_name)

374

375 def set_timestamp(self, new_timestamp):

376 """

377 Set timestamp

378

379 :param new_timestamp: timestamp, string

380 """

381 self.__timestamp = new_timestamp

382

383 def __check_project(self, project_id=None):

384 """

385 Check that the project exists

386

387 :param project_id: project id, integer

388 :return: True or False

389 """

390 retval = True

391 try:

392 self.__api.projects.get_project(project_id=project_id)

393 except Exception as error:

394 self.___logger.error("No such project is found, please set valid project ID.\nError%s", format_error(error))

395 retval = False

396 return retval

397

398 def __check_suite(self, suite_id=None):

399 """

400 Check that the suite exists

401

402 :param suite_id: id of suite, integer

403 :return: True or False

404 """

405 retval = True

406 try:

407 self.__api.suites.get_suite(suite_id=suite_id)

408 except Exception as error:

409 self.___logger.error("No such suite is found, please set valid suite ID.\nError%s", format_error(error))

410 retval = False

411 return retval

412

413 def __check_section(self, section_id=None):

414 """

415 Check that the section exists

416

417 :param section_id: id of suite, integer

418 :return: True or False

419 """

420 retval = True

421 try:

422 self.__api.sections.get_section(section_id=section_id)

423 except Exception as error:

424 self.___logger.error("No such section is found, please set valid section ID.\nError%s", format_error(error))

425 retval = False

426 return retval

427

428 def __check_report_exists(self, xml_report=None):

429 """

430 Check that the xml report exists

431

432 :param xml_report: filename of the xml report, maybe with path

433 :return: True or False

434 """

435 retval = False

436 if not xml_report:

437 xml_report = self.__xml_report

438 if xml_report:

439 if exists(xml_report):

440 retval = True

441 if not retval:

442 self.___logger.error("Please specify correct path.\nError 404: No XML file found")

443 return retval

444

445 def __check_run_exists(self, run_id=None):

446 """

447 Check that the run exists

448

449 :param run_id: id of suite, integer

450 :return: True or False

451 """

452 retval = True

453 try:

454 self.__api.runs.get_run(run_id=run_id)

455 except Exception as error:

456 self.___logger.error("No such run is found, please set valid run ID.\nError%s", format_error(error))

457 retval = False

458 return retval

459

460 def __self_check(self):

461 """

462 Health checker, calls checks

463 """

464 self.__check_project(project_id=self.__project_id)

465 self.__check_suite(suite_id=self.__suite_id)

466 self.__check_section(section_id=self.__at_section)

467 self.__check_report_exists(xml_report=self.__xml_report)

468

469 def __search_for_run_by_name(self, title=None):

470 """

471 Search run by name

472

473 :param title: name of the run

474 :return: id, integer

475 """

476 retval = None

477 first_run = True

478 criteria = None

479 while criteria is not None or first_run:

480 response = None

481 if first_run:

482 try:

483 response = self.__api.runs.get_runs(project_id=self.__project_id, suite_id=self.__suite_id)

484 except Exception as error:

485 self.___logger.error("Can't get run list. Something nasty happened.\nError%s", format_error(error))

486 break

487 first_run = False

488 elif response["_links"]["next"] is not None: # pylint: disable=E1136

489 offset = int(response["_links"]["next"].split("&offset=")[1].split("&")[0]) # pylint: disable=E1136

490 response = self.__api.runs.get_runs(

491 project_id=self.__project_id, suite_id=self.__suite_id, offset=offset

492 )

493 if response["runs"]["name"] == title:

494 retval = response["runs"]["id"]

495 break

496 criteria = response["_links"]["next"]

497 return retval

498

499 def __delete_run(self, run_id=None):

500 """

501 Delete run

502

503 :param run_id: run id, integer

504 :return: True if deleted, False in case of error

505 """

506 retval = True

507 try:

508 self.__api.runs.delete_run(run_id=run_id)

509 except Exception as error:

510 self.___logger.error("Can't delete run. Something nasty happened.\nError%s", format_error(error))

511 retval = False

512 return retval

513

514 def __add_run(self, title, cases_list=None, include_all=False):

515 """

516 Add a run

517

518 :param title: title of run

519 :param cases_list: test cases, which will be added to run

520 :param include_all: every existing testcases may be included, by default - False

521 :return: id of run, integer

522 """

523 retval = None

524 self.___logger.debug("Creating new test run '%s'", title)

525 try:

526 retval = self.__api.runs.add_run(

527 project_id=self.__project_id,

528 suite_id=self.__suite_id,

529 name=title,

530 include_all=include_all,

531 case_ids=cases_list,

532 )["id"]

533 except Exception as error:

534 self.___logger.error("Can't add run. Something nasty happened.\nError%s", format_error(error))

535 self.__self_check()

536 return retval

537

538 def __add_results(self, run_id=None, results=None):

539 """

540 Add results for test cases to TestRail

541

542 :param run_id: run id

543 :param results: payload (list of dicts)

544 :return: run id or False in case of error

545 """

546 retval = False

547 try:

548 self.__api.results.add_results_for_cases(run_id=run_id, results=results)

549 return run_id

550 except Exception as error:

551 self.___logger.error("Add results failed. Please validate your settings!\nError%s", format_error(error))

552 self.__self_check()

553 self.__check_run_exists(run_id=run_id)

554 return retval

555

556 def __prepare_runs(self, cases=None, title=None, run_id=None, run_name=None, delete_run=False):

557 """

558 Prepare run for submitting

559

560 :param cases: list of cases (list of dicts)

561 :param title: title of test run (which will be submitted)

562 :param run_id: run id

563 :param run_name:

564 :param delete_run: delete existing run or not (True or False), will be checked via run_name

565 :return: run id

566 """

567 cases_list = []

568 for item in cases:

569 cases_list.append(item["case_id"])

570 if run_name:

571 run_id = self.__search_for_run_by_name(title=run_name)

572 if not run_id:

573 self.___logger.debug("No run has been found by given name")

574 if delete_run and run_id:

575 self.__delete_run(run_id=run_id)

576 run_id = None

577 if not run_id:

578 run_id = self.__add_run(title=title, cases_list=cases_list, include_all=False)

579 return run_id

580

581 def __close_run(self, title=None, run_id=None):

582 """

583 Closes run

584

585 :param title: title of test run

586 :param run_id: run id, integer

587 :return: True or False

588 """

589 retval = True

590 try:

591 self.__api.runs.close_run(run_id=run_id)

592 self.___logger.debug("Test run '%s' is closed", title)

593 except Exception as error:

594 self.___logger.error("Can't close run. Something nasty happened.\nError%s", format_error(error))

595 retval = False

596 return retval