Coverage for coverage / data.py: 95.513%

110 statements  

« prev     ^ index     » next       coverage.py v7.12.1a0.dev1, created at 2025-11-30 17:57 +0000

1# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 

2# For details: https://github.com/coveragepy/coveragepy/blob/main/NOTICE.txt 

3 

4"""Coverage data for coverage.py. 

5 

6This file had the 4.x JSON data support, which is now gone. This file still 

7has storage-agnostic helpers, and is kept to avoid changing too many imports. 

8CoverageData is now defined in sqldata.py, and imported here to keep the 

9imports working. 

10 

11""" 

12 

13from __future__ import annotations 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

14 

15import functools 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

16import glob 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

17import hashlib 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

18import os.path 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

19from collections.abc import Iterable 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

20from typing import Callable 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

21 

22from coverage.exceptions import CoverageException, NoDataError 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

23from coverage.files import PathAliases 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

24from coverage.misc import Hasher, file_be_gone, human_sorted, plural 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

25from coverage.sqldata import CoverageData as CoverageData # pylint: disable=useless-import-alias 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

26 

27 

28def line_counts(data: CoverageData, fullpath: bool = False) -> dict[str, int]: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

29 """Return a dict summarizing the line coverage data. 

30 

31 Keys are based on the file names, and values are the number of executed 

32 lines. If `fullpath` is true, then the keys are the full pathnames of 

33 the files, otherwise they are the basenames of the files. 

34 

35 Returns a dict mapping file names to counts of lines. 

36 

37 """ 

38 summ = {} 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

39 filename_fn: Callable[[str], str] 

40 if fullpath: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

41 # pylint: disable=unnecessary-lambda-assignment 

42 filename_fn = lambda f: f 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

43 else: 

44 filename_fn = os.path.basename 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

45 for filename in data.measured_files(): 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

46 lines = data.lines(filename) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

47 assert lines is not None 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

48 summ[filename_fn(filename)] = len(lines) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

49 return summ 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

50 

51 

52def add_data_to_hash(data: CoverageData, filename: str, hasher: Hasher) -> None: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

53 """Contribute `filename`'s data to the `hasher`. 

54 

55 `hasher` is a `coverage.misc.Hasher` instance to be updated with 

56 the file's data. It should only get the results data, not the run 

57 data. 

58 

59 """ 

60 if data.has_arcs(): 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

61 hasher.update(sorted(data.arcs(filename) or [])) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

62 else: 

63 hasher.update(sorted_lines(data, filename)) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

64 hasher.update(data.file_tracer(filename)) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

65 

66 

67def combinable_files(data_file: str, data_paths: Iterable[str] | None = None) -> list[str]: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

68 """Make a list of data files to be combined. 

69 

70 `data_file` is a path to a data file. `data_paths` is a list of files or 

71 directories of files. 

72 

73 Returns a list of absolute file paths. 

74 """ 

75 data_dir, local = os.path.split(os.path.abspath(data_file)) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

76 

77 data_paths = data_paths or [data_dir] 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

78 files_to_combine = [] 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

79 for p in data_paths: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

80 if os.path.isfile(p): 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

81 files_to_combine.append(os.path.abspath(p)) 1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQR6STUVWX7YZ501234

82 elif os.path.isdir(p): 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

83 pattern = glob.escape(os.path.join(os.path.abspath(p), local)) + ".*" 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

84 files_to_combine.extend(glob.glob(pattern)) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

85 else: 

86 raise NoDataError(f"Couldn't combine from non-existent path '{p}'") 1abcdefghijklmnopqrstuvwxyzABCDEF8GHIJKL!MNOPQR6STUVWX7YZ01234

87 

88 # SQLite might have made journal files alongside our database files. 

89 # We never want to combine those. 

90 files_to_combine = [fnm for fnm in files_to_combine if not fnm.endswith("-journal")] 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

91 

92 # Sorting isn't usually needed, since it shouldn't matter what order files 

93 # are combined, but sorting makes tests more predictable, and makes 

94 # debugging more understandable when things go wrong. 

95 return sorted(files_to_combine) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

96 

97 

98def combine_parallel_data( 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%

99 data: CoverageData, 

100 aliases: PathAliases | None = None, 

101 data_paths: Iterable[str] | None = None, 

102 strict: bool = False, 

103 keep: bool = False, 

104 message: Callable[[str], None] | None = None, 

105) -> None: 

106 """Combine a number of data files together. 

107 

108 `data` is a CoverageData. 

109 

110 Treat `data.filename` as a file prefix, and combine the data from all 

111 of the data files starting with that prefix plus a dot. 

112 

113 If `aliases` is provided, it's a `PathAliases` object that is used to 

114 re-map paths to match the local machine's. 

115 

116 If `data_paths` is provided, it is a list of directories or files to 

117 combine. Directories are searched for files that start with 

118 `data.filename` plus dot as a prefix, and those files are combined. 

119 

120 If `data_paths` is not provided, then the directory portion of 

121 `data.filename` is used as the directory to search for data files. 

122 

123 Unless `keep` is True every data file found and combined is then deleted 

124 from disk. If a file cannot be read, a warning will be issued, and the 

125 file will not be deleted. 

126 

127 If `strict` is true, and no files are found to combine, an error is 

128 raised. 

129 

130 `message` is a function to use for printing messages to the user. 

131 

132 """ 

133 files_to_combine = combinable_files(data.base_filename(), data_paths) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

134 

135 if strict and not files_to_combine: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

136 raise NoDataError("No data to combine") 1abcdefghijklmnopqrstuvwxyzABCDEFGH9IJKLMNOPQRST#UV)WXYZ01%234

137 

138 if aliases is None: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

139 map_path = None 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJKL!MN$OPQR6ST#UVWX7YZ501234

140 else: 

141 map_path = functools.cache(aliases.map) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

142 

143 file_hashes = set() 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

144 combined_any = False 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

145 

146 for f in files_to_combine: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

147 if f == data.data_filename(): 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

148 # Sometimes we are combining into a file which is one of the 

149 # parallel files. Skip that file. 

150 if data._debug.should("dataio"): 150 ↛ 151line 150 didn't jump to line 151 because the condition on line 150 was never true1abcdefghijklmnopqrstuvwxyzABCDEFGHIJ'KLMN$OP(QRSTUVWXYZ501%234

151 data._debug.write(f"Skipping combining ourself: {f!r}") 

152 continue 1abcdefghijklmnopqrstuvwxyzABCDEFGHIJ'KLMN$OP(QRSTUVWXYZ501%234

153 

154 try: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

155 rel_file_name = os.path.relpath(f) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

156 except ValueError: 

157 # ValueError can be raised under Windows when os.getcwd() returns a 

158 # folder from a different drive than the drive of f, in which case 

159 # we print the original value of f instead of its relative path 

160 rel_file_name = f 

161 

162 with open(f, "rb") as fobj: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

163 hasher = hashlib.new("sha3_256", usedforsecurity=False) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

164 hasher.update(fobj.read()) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

165 sha = hasher.digest() 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

166 combine_this_one = sha not in file_hashes 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

167 

168 delete_this_one = not keep 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

169 if combine_this_one: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

170 if data._debug.should("dataio"): 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

171 data._debug.write(f"Combining data file {f!r}") 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJKL!MNOPQR6ST#UVWX7YZ501234

172 file_hashes.add(sha) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

173 try: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

174 new_data = CoverageData(f, debug=data._debug) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

175 new_data.read() 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

176 except CoverageException as exc: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

177 if data._warn: 177 ↛ 181line 177 didn't jump to line 181 because the condition on line 177 was always true1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

178 # The CoverageException has the file name in it, so just 

179 # use the message as the warning. 

180 data._warn(str(exc)) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

181 if message: 181 ↛ 183line 181 didn't jump to line 183 because the condition on line 181 was always true1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

182 message(f"Couldn't combine data file {rel_file_name}: {exc}") 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

183 delete_this_one = False 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

184 else: 

185 data.update(new_data, map_path=map_path) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

186 combined_any = True 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

187 if message: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

188 message(f"Combined data file {rel_file_name}") 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

189 else: 

190 if message: 190 ↛ 193line 190 didn't jump to line 193 because the condition on line 190 was always true1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

191 message(f"Skipping duplicate data {rel_file_name}") 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

192 

193 if delete_this_one: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

194 if data._debug.should("dataio"): 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

195 data._debug.write(f"Deleting data file {f!r}") 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJKL!MNOPQR6ST#UVWX7YZ501234

196 file_be_gone(f) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

197 

198 if strict and not combined_any: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

199 raise NoDataError("No usable data files") 1abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234

200 

201 

202def debug_data_file(filename: str) -> None: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

203 """Implementation of 'coverage debug data'.""" 

204 data = CoverageData(filename) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

205 filename = data.data_filename() 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

206 print(f"path: {filename}") 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

207 if not os.path.exists(filename): 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

208 print("No data collected: file doesn't exist") 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

209 return 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

210 data.read() 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

211 print(f"has_arcs: {data.has_arcs()!r}") 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

212 summary = line_counts(data, fullpath=True) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

213 filenames = human_sorted(summary.keys()) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

214 nfiles = len(filenames) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

215 print(f"{nfiles} file{plural(nfiles)}:") 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

216 for f in filenames: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

217 line = f"{f}: {summary[f]} line{plural(summary[f])}" 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

218 plugin = data.file_tracer(f) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

219 if plugin: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

220 line += f" [{plugin}]" 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

221 print(line) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

222 

223 

224def sorted_lines(data: CoverageData, filename: str) -> list[int]: 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

225 """Get the sorted lines for a file, for tests.""" 

226 lines = data.lines(filename) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234

227 return sorted(lines or []) 1abcdefghijklmnopqrstuvwxyzABCDEF8GH9IJ'KL!MN$OP(QR6ST#UV)WX7YZ501%234