Coverage for tests / test_goldtest.py: 100.000%

1# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 

2# For details: https://github.com/coveragepy/coveragepy/blob/main/NOTICE.txt 

3 

4"""Tests of the helpers in goldtest.py""" 

5 

6from __future__ import annotations 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

7 

8import os.path 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

9import re 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

10 

11import pytest 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

12 

13from tests.coveragetest import CoverageTest, TESTS_DIR 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

14from tests.goldtest import compare, gold_path 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

15from tests.goldtest import contains, contains_any, contains_rx, doesnt_contain 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

16from tests.helpers import os_sep, re_line, remove_tree 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

17 

18GOOD_GETTY = """\ 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

19Four score and seven years ago our fathers brought forth upon this continent, a 

20new nation, conceived in Liberty, and dedicated to the proposition that all men 

21are created equal. 

2211/19/9999, Gettysburg, Pennsylvania 

23""" 

24 

25BAD_GETTY = """\ 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

26Five score and seven years ago our fathers brought forth upon this continent, a 

27new nation, conceived in Liberty, and dedicated to the proposition that all men 

28are created equal. 

29333/4444/55555, Gabcdef, Pennsylvania 

30""" 

31 

32SCRUBS = [ 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

33 # Numbers don't matter when comparing. 

34 (r"\d+", "D"), 

35 (r"G\w+", "Gxxx"), 

36] 

37 

38 

39def path_regex(path: str) -> str: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

40 """Convert a file path into a regex that will match that path on any OS.""" 

41 return re.sub(r"[/\\]", r"[/\\\\]", path.replace(".", "[.]")) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

42 

43 

44ACTUAL_DIR = os.path.join(TESTS_DIR, "actual/testing") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

45ACTUAL_GETTY_FILE = os.path.join(ACTUAL_DIR, "getty/gettysburg.txt") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

46GOLD_GETTY_FILE = os.path.join(TESTS_DIR, "gold/testing/getty/gettysburg.txt") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

47GOLD_GETTY_FILE_RX = path_regex(GOLD_GETTY_FILE) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

48 

49GOLD_PATH_RX = path_regex("/tests/gold/testing/getty/gettysburg.txt") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

50OUT_PATH_RX = path_regex("out/gettysburg.txt") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

51 

52 

53@pytest.mark.xdist_group(name="compare_test") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

54class CompareTest(CoverageTest): 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

55 """Tests of goldtest.py:compare()""" 

56 

57 def setUp(self) -> None: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

58 super().setUp() 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

59 self.addCleanup(remove_tree, ACTUAL_DIR) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

60 

61 def test_good(self) -> None: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

62 self.make_file("out/gettysburg.txt", GOOD_GETTY) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

63 compare(gold_path("testing/getty"), "out", scrubs=SCRUBS) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

64 self.assert_doesnt_exist(ACTUAL_GETTY_FILE) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

65 

66 def test_bad(self) -> None: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

67 self.make_file("out/gettysburg.txt", BAD_GETTY) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

68 

69 # compare() raises an assertion. 

70 msg = rf"Files differ: .*{GOLD_PATH_RX} != {OUT_PATH_RX}" 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

71 with pytest.raises(AssertionError, match=msg): 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

72 compare(gold_path("testing/getty"), "out", scrubs=SCRUBS) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

73 

74 # Stdout has a description of the diff. The diff shows the scrubbed content. 

75 stdout = self.stdout() 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

76 assert "- Four score" in stdout 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

77 assert "+ Five score" in stdout 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

78 assert re_line(rf"^:::: diff '.*{GOLD_PATH_RX}' and '{OUT_PATH_RX}'", stdout) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

79 assert re_line(rf"^:::: end diff '.*{GOLD_PATH_RX}' and '{OUT_PATH_RX}'", stdout) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

80 assert os_sep( 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

81 f"Saved actual output to '{ACTUAL_GETTY_FILE}': see tests/gold/README.rst" 

82 ) in os_sep(stdout) 

83 assert " D/D/D, Gxxx, Pennsylvania" in stdout 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

84 

85 # The actual file was saved. 

86 with open(ACTUAL_GETTY_FILE, encoding="utf-8") as f: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

87 saved = f.read() 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

88 assert saved == BAD_GETTY 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

89 

90 def test_good_needs_scrubs(self) -> None: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

91 # Comparing the "good" result without scrubbing the variable parts will fail. 

92 self.make_file("out/gettysburg.txt", GOOD_GETTY) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

93 

94 # compare() raises an assertion. 

95 msg = rf"Files differ: .*{GOLD_PATH_RX} != {OUT_PATH_RX}" 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

96 with pytest.raises(AssertionError, match=msg): 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

97 compare(gold_path("testing/getty"), "out") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

98 

99 stdout = self.stdout() 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

100 assert "- 11/19/1863, Gettysburg, Pennsylvania" in stdout 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

101 assert "+ 11/19/9999, Gettysburg, Pennsylvania" in stdout 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

102 

103 def test_actual_extra(self) -> None: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

104 self.make_file("out/gettysburg.txt", GOOD_GETTY) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

105 self.make_file("out/another.more", "hi") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

106 

107 # Extra files in the output are ok with actual_extra=True. 

108 compare(gold_path("testing/getty"), "out", scrubs=SCRUBS, actual_extra=True) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

109 

110 # But not without it: 

111 # (test output is in files like /tmp/pytest-of-user/pytest-0/popen-gw3/t76/out) 

112 msg = r"Files in .*[/\\]t\d+[/\\]out only: \['another.more'\]" 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

113 with pytest.raises(AssertionError, match=msg): 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

114 compare(gold_path("testing/getty"), "out", scrubs=SCRUBS) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

115 self.assert_exists(os.path.join(TESTS_DIR, "actual/testing/getty/another.more")) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

116 

117 # But only the files matching the file_pattern are considered. 

118 compare(gold_path("testing/getty"), "out", file_pattern="*.txt", scrubs=SCRUBS) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

119 

120 def test_xml_good(self) -> None: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

121 self.make_file( 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

122 "out/output.xml", 

123 """\ 

124 <?xml version="1.0" ?> 

125 <the_root c="three" b="222" a="one"> 

126 <also z="nine" x="seven" y="888"> 

127 Goodie 

128 </also> 

129 </the_root> 

130 """, 

131 ) 

132 compare(gold_path("testing/xml"), "out", scrubs=SCRUBS) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

133 

134 def test_xml_bad(self) -> None: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

135 self.make_file( 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

136 "out/output.xml", 

137 """\ 

138 <?xml version="1.0" ?> 

139 <the_root c="nine" b="2" a="one"> 

140 <also z="three" x="seven" y="8"> 

141 Goodbye 

142 </also> 

143 </the_root> 

144 """, 

145 ) 

146 

147 # compare() raises an exception. 

148 gold_rx = path_regex(gold_path("testing/xml/output.xml")) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

149 out_rx = path_regex("out/output.xml") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

150 msg = rf"Files differ: .*{gold_rx} != {out_rx}" 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

151 with pytest.raises(AssertionError, match=msg): 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

152 compare(gold_path("testing/xml"), "out", scrubs=SCRUBS) 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

153 

154 # Stdout has a description of the diff. The diff shows the 

155 # canonicalized and scrubbed content. 

156 stdout = self.stdout() 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

157 assert '- <the_root a="one" b="D" c="three">' in stdout 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

158 assert '+ <the_root a="one" b="D" c="nine">' in stdout 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

159 

160 

161class ContainsTest(CoverageTest): 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

162 """Tests of the various "contains" functions in goldtest.py""" 

163 

164 run_in_temp_dir = False 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

165 

166 def test_contains(self) -> None: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

167 contains(GOLD_GETTY_FILE, "Four", "fathers", "dedicated") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

168 msg = rf"Missing content in {GOLD_GETTY_FILE_RX}: 'xyzzy'" 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

169 with pytest.raises(AssertionError, match=msg): 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

170 contains(GOLD_GETTY_FILE, "Four", "fathers", "xyzzy", "dedicated") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

171 

172 def test_contains_rx(self) -> None: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

173 contains_rx(GOLD_GETTY_FILE, r"Fo.r", r"f[abc]thers", "dedi[cdef]ated") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

174 msg = rf"Missing regex in {GOLD_GETTY_FILE_RX}: r'm\[opq\]thers'" 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

175 with pytest.raises(AssertionError, match=msg): 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

176 contains_rx(GOLD_GETTY_FILE, r"Fo.r", r"m[opq]thers") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

177 

178 def test_contains_any(self) -> None: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

179 contains_any(GOLD_GETTY_FILE, "Five", "Four", "Three") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

180 msg = rf"Missing content in {GOLD_GETTY_FILE_RX}: 'One' \[1 of 3\]" 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

181 with pytest.raises(AssertionError, match=msg): 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

182 contains_any(GOLD_GETTY_FILE, "One", "Two", "Three") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

183 

184 def test_doesnt_contain(self) -> None: 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

185 doesnt_contain(GOLD_GETTY_FILE, "One", "Two", "Three") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

186 msg = rf"Forbidden content in {GOLD_GETTY_FILE_RX}: 'Four'" 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

187 with pytest.raises(AssertionError, match=msg): 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()

188 doesnt_contain(GOLD_GETTY_FILE, "Three", "Four", "Five") 69 ctx1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#$%'()