Coverage for tests / goldtest.py: 100.000%

1# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 

2# For details: https://github.com/coveragepy/coveragepy/blob/main/NOTICE.txt 

3 

4"""A test base class for tests based on gold file comparison.""" 

5 

6from __future__ import annotations 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

7 

8import difflib 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

9import filecmp 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

10import fnmatch 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

11import os 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

12import os.path 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

13import re 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

14import xml.etree.ElementTree 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

15 

16from collections.abc import Iterable 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

17 

18from tests.coveragetest import TESTS_DIR 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

19from tests.helpers import os_sep 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

20 

21 

22def gold_path(path: str) -> str: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

23 """Get a path to a gold file for comparison.""" 

24 return os.path.join(TESTS_DIR, "gold", path) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

25 

26 

27def compare( 66 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%

28 expected_dir: str, 

29 actual_dir: str, 

30 file_pattern: str | None = None, 

31 actual_extra: bool = False, 

32 scrubs: list[tuple[str, str]] | None = None, 

33) -> None: 

34 """Compare files matching `file_pattern` in `expected_dir` and `actual_dir`. 

35 

36 `actual_extra` true means `actual_dir` can have extra files in it 

37 without triggering an assertion. 

38 

39 `scrubs` is a list of pairs: regexes to find and replace to scrub the 

40 files of unimportant differences. 

41 

42 If a comparison fails, a message will be written to stdout, the original 

43 unscrubbed output of the test will be written to an "/actual/" directory 

44 alongside the "/gold/" directory, and an assertion will be raised. 

45 

46 """ 

47 __tracebackhide__ = True # pytest, please don't show me this function. 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

48 assert os_sep("/gold/") in expected_dir 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

49 assert os.path.exists(actual_dir) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

50 os.makedirs(expected_dir, exist_ok=True) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

51 

52 dc = filecmp.dircmp(expected_dir, actual_dir) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

53 diff_files = _fnmatch_list(dc.diff_files, file_pattern) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

54 expected_only = _fnmatch_list(dc.left_only, file_pattern) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

55 actual_only = _fnmatch_list(dc.right_only, file_pattern) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

56 

57 def save_mismatch(f: str) -> None: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

58 """Save a mismatched result to tests/actual.""" 

59 save_path = expected_dir.replace(os_sep("/gold/"), os_sep("/actual/")) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

60 os.makedirs(save_path, exist_ok=True) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

61 save_file = os.path.join(save_path, f) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

62 with open(save_file, "w", encoding="utf-8") as savef: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

63 with open(os.path.join(actual_dir, f), encoding="utf-8") as readf: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

64 savef.write(readf.read()) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

65 print(os_sep(f"Saved actual output to '{save_file}': see tests/gold/README.rst")) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

66 

67 # filecmp only compares in binary mode, but we want text mode. So 

68 # look through the list of different files, and compare them 

69 # ourselves. 

70 text_diff = [] 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

71 for f in diff_files: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

72 expected_file = os.path.join(expected_dir, f) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

73 with open(expected_file, encoding="utf-8") as fobj: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

74 expected = fobj.read() 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

75 if expected_file.endswith(".xml"): 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

76 expected = canonicalize_xml(expected) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

77 

78 actual_file = os.path.join(actual_dir, f) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

79 with open(actual_file, encoding="utf-8") as fobj: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

80 actual = fobj.read() 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

81 if actual_file.endswith(".xml"): 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

82 actual = canonicalize_xml(actual) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

83 

84 if scrubs: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

85 expected = scrub(expected, scrubs) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

86 actual = scrub(actual, scrubs) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

87 if expected != actual: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

88 text_diff.append(f"{expected_file} != {actual_file}") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

89 expected_lines = expected.splitlines() 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

90 actual_lines = actual.splitlines() 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

91 print(f":::: diff '{expected_file}' and '{actual_file}'") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

92 print("\n".join(difflib.Differ().compare(expected_lines, actual_lines))) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

93 print(f":::: end diff '{expected_file}' and '{actual_file}'") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

94 print(f"==== expected output in '{os.path.abspath(expected_dir)}'") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

95 print(f"==== actual output in '{os.path.abspath(actual_dir)}'") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

96 save_mismatch(f) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

97 

98 if not actual_extra: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

99 for f in actual_only: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

100 save_mismatch(f) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

101 

102 assert not text_diff, "Files differ: " + "\n".join(text_diff) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

103 

104 assert not expected_only, f"Files in {os.path.abspath(expected_dir)} only: {expected_only}" 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

105 if not actual_extra: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

106 assert not actual_only, f"Files in {os.path.abspath(actual_dir)} only: {actual_only}" 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

107 

108 

109def contains(filename: str, *strlist: str) -> None: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

110 """Check that the file contains all of a list of strings. 

111 

112 An assert will be raised if one of the arguments in `strlist` is 

113 missing in `filename`. 

114 

115 """ 

116 __tracebackhide__ = True # pytest, please don't show me this function. 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

117 with open(filename, encoding="utf-8") as fobj: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

118 text = fobj.read() 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

119 for s in strlist: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

120 assert s in text, f"Missing content in {filename}: {s!r}" 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

121 

122 

123def contains_rx(filename: str, *rxlist: str) -> None: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

124 """Check that the file has lines that re.search all of the regexes. 

125 

126 An assert will be raised if one of the regexes in `rxlist` doesn't match 

127 any lines in `filename`. 

128 

129 """ 

130 __tracebackhide__ = True # pytest, please don't show me this function. 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

131 with open(filename, encoding="utf-8") as fobj: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

132 lines = fobj.readlines() 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

133 for rx in rxlist: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

134 assert any(re.search(rx, line) for line in lines), f"Missing regex in {filename}: r{rx!r}" 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

135 

136 

137def contains_any(filename: str, *strlist: str) -> None: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

138 """Check that the file contains at least one of a list of strings. 

139 

140 An assert will be raised if none of the arguments in `strlist` is in 

141 `filename`. 

142 

143 """ 

144 __tracebackhide__ = True # pytest, please don't show me this function. 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

145 with open(filename, encoding="utf-8") as fobj: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

146 text = fobj.read() 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

147 for s in strlist: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

148 if s in text: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

149 return 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

150 

151 assert False, f"Missing content in {filename}: {strlist[0]!r} [1 of {len(strlist)}]" 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

152 

153 

154def doesnt_contain(filename: str, *strlist: str) -> None: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

155 """Check that the file contains none of a list of strings. 

156 

157 An assert will be raised if any of the strings in `strlist` appears in 

158 `filename`. 

159 

160 """ 

161 __tracebackhide__ = True # pytest, please don't show me this function. 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

162 with open(filename, encoding="utf-8") as fobj: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

163 text = fobj.read() 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

164 for s in strlist: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

165 assert s not in text, f"Forbidden content in {filename}: {s!r}" 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

166 

167 

168# Helpers 

169 

170 

171def canonicalize_xml(xtext: str) -> str: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

172 """Canonicalize some XML text.""" 

173 root = xml.etree.ElementTree.fromstring(xtext) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

174 for node in root.iter(): 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

175 node.attrib = dict(sorted(node.items())) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

176 return xml.etree.ElementTree.tostring(root).decode("utf-8") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

177 

178 

179def _fnmatch_list(files: list[str], file_pattern: str | None) -> list[str]: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

180 """Filter the list of `files` to only those that match `file_pattern`. 

181 If `file_pattern` is None, then return the entire list of files. 

182 Returns a list of the filtered files. 

183 """ 

184 if file_pattern: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

185 files = [f for f in files if fnmatch.fnmatch(f, file_pattern)] 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

186 return files 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

187 

188 

189def scrub(strdata: str, scrubs: Iterable[tuple[str, str]]) -> str: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

190 """Scrub uninteresting data from the payload in `strdata`. 

191 `scrubs` is a list of (find, replace) pairs of regexes that are used on 

192 `strdata`. A string is returned. 

193 """ 

194 for rx_find, rx_replace in scrubs: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

195 strdata = re.sub(rx_find, rx_replace, strdata) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

196 return strdata 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()