Coverage for tests/testspec.py: 66%

1"""TestSpec testing framework."""

2

3 from collections.abc import Callable

4 from enum import Enum

5 import traceback

6 from typing import Any, NamedTuple, Optional, Sequence

7 import unittest

8

9

10# Sentinel value used to indicate that no expected value is set

11 NO_EXPECTED_VALUE = object()

12"""

13 A sentinel value used to indicate that no expected value is set.

14"""

15

16

17 class TestSpec(NamedTuple):

18 """A generic unit test specification class.

19

20 It allow tests to be specified declaratively while providing a large amount

21 of flexibility.

22

23 Args:

24 name (str):

25 Identifying name for the test.

26 action (Callable[..., Any]):

27 A reference to a callable function or method to be invoked for the test.

28 args (Sequence[Any], default = []):

29 Sequence of positional arguments to be passed to the `action` function or method.

30 kwargs (dict[str, Any], default = {}):

31 Dictionary containing keyword arguments to be passed to the `action` function or method.

32 expected (Any, default=NO_EXPECTED_VALUE ):

33 Expected value (if any) that is expected to be returned by the `action` function or method.

34 If there is no expected value, the special class NoExpectedValue is used to flag it.

35 This is used so that the specific return value of None can be distinguished from no

36 particular value or any value at all is expected to be returned from the function or method.

37 obj: Optional[Any] = None

38 validate_obj: Optional[Callable[[Any], bool]] = None

39 validate_result: Optional[Callable[[Any], bool]] = None

40 exception: Optional[type[Exception]] = None

41 exception_tag: Optional[str] = None

42 display_on_fail: Optional[Callable[[], str]] = None

43 """

44 name: str

45 action: Callable[..., Any]

46 args: Optional[list[Any]] = None

47 kwargs: Optional[dict[str, Any]] = None

48 expected: Any = NO_EXPECTED_VALUE

49 obj: Optional[Any] = None

50 validate_obj: Optional[Callable[[Any], bool]] = None

51 validate_result: Optional[Callable[[Any], bool]] = None

52 exception: Optional[type[Exception]] = None

53 exception_tag: Optional[str | Enum] = None

54 display_on_fail: Optional[Callable[[], str]] = None

55

56

57 def run_tests_list(test_case: unittest.TestCase, test_specs: Sequence[TestSpec]) -> None:

58 """Run a list of tests based on the provided TestSpec entries.

59

60 This function iterates over the list of TestSpec entries and runs each test using

61 the `run_test` function. It allows for a clean and organized way to execute multiple tests.

62

63 Args:

64 test_case (unittest.TestCase): The test case instance that will run the tests.

65 test_specs (list[TestSpec]): A list of TestSpec entries, each representing a test to be run.

66 """

67 for spec in test_specs:

68 run_test(test_case, spec)

69

70

71 def run_test(test_case: unittest.TestCase, spec: TestSpec) -> None: # pylint: disable=too-many-branches

72 """Run a single test based on the provided TestSpec entry.

73 This function executes the action specified in the entry, checks the result against

74 the expected value, and reports any errors.

75

76 Args:

77 test_case (unittest.TestCase): The test case instance that will run the test.

78 spec (TestSpec): The test configuration entry containing all necessary information for the test.

79 """

80 with test_case.subTest(msg=spec.name):

81 test_description: str = f"{spec.name}"

82 errors: list[str] = []

83 try:

84 # Use empty list/dict if the spec field is None

85 pos_args = spec.args if spec.args is not None else []

86 kw_args = spec.kwargs if spec.kwargs is not None else {}

87 found: Any = spec.action(*pos_args, **kw_args)

88 if spec.exception: 88 ↛ 89line 88 didn't jump to line 89 because the condition on line 88 was never true

89 errors.append("returned result instead of raising exception")

90

91 else:

92 if spec.validate_result and not spec.validate_result(found): 92 ↛ 93line 92 didn't jump to line 93 because the condition on line 92 was never true

93 errors.append(f"failed result validation: found={found}")

94 if spec.validate_obj and not spec.validate_obj(spec.obj): 94 ↛ 95line 94 didn't jump to line 95 because the condition on line 94 was never true

95 errors.append(f"failed object validation: obj={spec.obj}")

96 if spec.expected is not NO_EXPECTED_VALUE and spec.expected != found: 96 ↛ 97line 96 didn't jump to line 97 because the condition on line 96 was never true

97 errors.append(f"expected={spec.expected}, found={found}")

98 if isinstance(spec.display_on_fail, Callable):

99 errors.append(spec.display_on_fail())

100 except Exception as err: # pylint: disable=broad-exception-caught

101 if spec.exception is None: 101 ↛ 102line 101 didn't jump to line 102 because the condition on line 101 was never true

102 errors.append(f"Did not expect exception. Caught exception {repr(err)}")

103 errors.append("stacktrace = ")

104 errors.append("\n".join(traceback.format_tb(tb=err.__traceback__)))

105

106 elif not isinstance(err, spec.exception): 106 ↛ 107line 106 didn't jump to line 107 because the condition on line 106 was never true

107 errors.append(

108 f"Unexpected exception type: expected={spec.exception}, "

109 f"found = {type(err)}"

110 )

111 elif spec.exception_tag:

112 # Case 1: The expected tag is an Enum member.

113 # This requires the exception object to have a 'tag_code' attribute.

114 if isinstance(spec.exception_tag, Enum): 114 ↛ 126line 114 didn't jump to line 126 because the condition on line 114 was always true

115 if not hasattr(err, 'tag_code'): 115 ↛ 116line 115 didn't jump to line 116 because the condition on line 115 was never true

116 errors.append(

117 "Exception is missing the 'tag_code' attribute required for Enum tag validation.")

118 else:

119 actual_tag = getattr(err, 'tag_code')

120 if actual_tag != spec.exception_tag: 120 ↛ 121line 120 didn't jump to line 121 because the condition on line 120 was never true

121 errors.append(f"Unexpected exception tag: expected={spec.exception_tag}, "

122 f"found={actual_tag}")

123 # Case 2: The expected tag is a string.

124 # This performs a substring search in the exception's string representation.

125 else:

126 if str(spec.exception_tag) not in str(err):

127 errors.append(

128 f"Correct exception type, but tag '{spec.exception_tag}' "

129 f"not found in exception message: {repr(err)}"

130 )

131 if errors: 131 ↛ 132line 131 didn't jump to line 132 because the condition on line 131 was never true

132 test_case.fail(msg=test_description + ": " + "\n".join(errors))