CHANGES.rst

@@ -22,6 +22,23 @@ upgrading your version of coverage.py.
 
 .. scriv-start-here
 
+.. _changes_7-5-3:
+
+Version 7.5.3 — 2024-05-28
+--------------------------
+
+- Performance improvements for combining data files, especially when measuring
+  line coverage. A few different quadratic behaviors were eliminated. In one
+  extreme case of combining 700+ data files, the time dropped from more than
+  three hours to seven minutes.  Thanks for Kraken Tech for funding the fix.
+
+- Performance improvements for generating HTML reports, with a side benefit of
+  reducing memory use, closing `issue 1791`_.  Thanks to Daniel Diniz for
+  helping to diagnose the problem.
+
+.. _issue 1791: https://github.com/nedbat/coveragepy/issues/1791
+
+
 .. _changes_7-5-2:
 
 Version 7.5.2 — 2024-05-24

coverage/control.py

@@ -998,7 +998,7 @@ def _prepare_data_for_reporting(self) -> None:
         if self.config.paths:
             mapped_data = CoverageData(warn=self._warn, debug=self._debug, no_disk=True)
             if self._data is not None:
-                mapped_data.update(self._data, aliases=self._make_aliases())
+                mapped_data.update(self._data, map_path=self._make_aliases().map)
             self._data = mapped_data
 
     def report(

coverage/data.py

@@ -12,6 +12,7 @@
 
 from __future__ import annotations
 
+import functools
 import glob
 import hashlib
 import os.path
@@ -134,6 +135,11 @@ def combine_parallel_data(
     if strict and not files_to_combine:
         raise NoDataError("No data to combine")
 
+    if aliases is None:
+        map_path = None
+    else:
+        map_path = functools.lru_cache(maxsize=None)(aliases.map)
+
     file_hashes = set()
     combined_any = False
 
@@ -176,7 +182,7 @@ def combine_parallel_data(
                     message(f"Couldn't combine data file {rel_file_name}: {exc}")
                 delete_this_one = False
             else:
-                data.update(new_data, aliases=aliases)
+                data.update(new_data, map_path=map_path)
                 combined_any = True
                 if message:
                     message(f"Combined data file {rel_file_name}")

coverage/phystokens.py

@@ -6,7 +6,6 @@
 from __future__ import annotations
 
 import ast
-import functools
 import io
 import keyword
 import re
@@ -163,20 +162,15 @@ def source_token_lines(source: str) -> TSourceTokenLines:
         yield line
 
 
-@functools.lru_cache(maxsize=100)
 def generate_tokens(text: str) -> TokenInfos:
-    """A cached version of `tokenize.generate_tokens`.
+    """A helper around `tokenize.generate_tokens`.
 
-    When reporting, coverage.py tokenizes files twice, once to find the
-    structure of the file, and once to syntax-color it.  Tokenizing is
-    expensive, and easily cached.
+    Originally this was used to cache the results, but it didn't seem to make
+    reporting go faster, and caused issues with using too much memory.
 
-    Unfortunately, the HTML report code tokenizes all the files the first time
-    before then tokenizing them a second time, so we cache many.  Ideally we'd
-    rearrange the code to tokenize each file twice before moving onto the next.
     """
     readline = io.StringIO(text).readline
-    return list(tokenize.generate_tokens(readline))
+    return tokenize.generate_tokens(readline)
 
 
 def source_encoding(source: bytes) -> str:

coverage/sqldata.py

@@ -21,13 +21,12 @@
 import zlib
 
 from typing import (
-    cast, Any, Collection, Mapping,
+    cast, Any, Callable, Collection, Mapping,
     Sequence,
 )
 
 from coverage.debug import NoDebugging, auto_repr
 from coverage.exceptions import CoverageException, DataError
-from coverage.files import PathAliases
 from coverage.misc import file_be_gone, isolate_module
 from coverage.numbits import numbits_to_nums, numbits_union, nums_to_numbits
 from coverage.sqlitedb import SqliteDb
@@ -647,12 +646,16 @@ def purge_files(self, filenames: Collection[str]) -> None:
                     continue
                 con.execute_void(sql, (file_id,))
 
-    def update(self, other_data: CoverageData, aliases: PathAliases | None = None) -> None:
-        """Update this data with data from several other :class:`CoverageData` instances.
+    def update(
+        self,
+        other_data: CoverageData,
+        map_path: Callable[[str], str] | None = None,
+    ) -> None:
+        """Update this data with data from another :class:`CoverageData`.
 
-        If `aliases` is provided, it's a `PathAliases` object that is used to
-        re-map paths to match the local machine's.  Note: `aliases` is None
-        only when called directly from the test suite.
+        If `map_path` is provided, it's a function that re-map paths to match
+        the local machine's.  Note: `map_path` is None only when called
+        directly from the test suite.
 
         """
         if self._debug.should("dataop"):
@@ -664,7 +667,7 @@ def update(self, other_data: CoverageData, aliases: PathAliases | None = None) -
         if self._has_arcs and other_data._has_lines:
             raise DataError("Can't combine line data with arc data")
 
-        aliases = aliases or PathAliases()
+        map_path = map_path or (lambda p: p)
 
         # Force the database we're writing to to exist before we start nesting contexts.
         self._start_using()
@@ -674,7 +677,7 @@ def update(self, other_data: CoverageData, aliases: PathAliases | None = None) -
         with other_data._connect() as con:
             # Get files data.
             with con.execute("select path from file") as cur:
-                files = {path: aliases.map(path) for (path,) in cur}
+                files = {path: map_path(path) for (path,) in cur}
 
             # Get contexts data.
             with con.execute("select context from context") as cur:
@@ -729,7 +732,7 @@ def update(self, other_data: CoverageData, aliases: PathAliases | None = None) -
                 "inner join file on file.id = tracer.file_id",
             ) as cur:
                 this_tracers.update({
-                    aliases.map(path): tracer
+                    map_path(path): tracer
                     for path, tracer in cur
                 })
 
@@ -767,27 +770,15 @@ def update(self, other_data: CoverageData, aliases: PathAliases | None = None) -
             # Prepare arc and line rows to be inserted by converting the file
             # and context strings with integer ids. Then use the efficient
             # `executemany()` to insert all rows at once.
-            arc_rows = (
-                (file_ids[file], context_ids[context], fromno, tono)
-                for file, context, fromno, tono in arcs
-            )
-
-            # Get line data.
-            with con.execute(
-                "select file.path, context.context, line_bits.numbits " +
-                "from line_bits " +
-                "inner join file on file.id = line_bits.file_id " +
-                "inner join context on context.id = line_bits.context_id",
-            ) as cur:
-                for path, context, numbits in cur:
-                    key = (aliases.map(path), context)
-                    if key in lines:
-                        numbits = numbits_union(lines[key], numbits)
-                    lines[key] = numbits
 
             if arcs:
                 self._choose_lines_or_arcs(arcs=True)
 
+                arc_rows = (
+                    (file_ids[file], context_ids[context], fromno, tono)
+                    for file, context, fromno, tono in arcs
+                )
+
                 # Write the combined data.
                 con.executemany_void(
                     "insert or ignore into arc " +
@@ -797,15 +788,25 @@ def update(self, other_data: CoverageData, aliases: PathAliases | None = None) -
 
             if lines:
                 self._choose_lines_or_arcs(lines=True)
-                con.execute_void("delete from line_bits")
+
+                for (file, context), numbits in lines.items():
+                    with con.execute(
+                        "select numbits from line_bits where file_id = ? and context_id = ?",
+                        (file_ids[file], context_ids[context]),
+                    ) as cur:
+                        existing = list(cur)
+                    if existing:
+                        lines[(file, context)] = numbits_union(numbits, existing[0][0])
+
                 con.executemany_void(
-                    "insert into line_bits " +
+                    "insert or replace into line_bits " +
                     "(file_id, context_id, numbits) values (?, ?, ?)",
                     [
                         (file_ids[file], context_ids[context], numbits)
                         for (file, context), numbits in lines.items()
                     ],
                 )
+
             con.executemany_void(
                 "insert or ignore into tracer (file_id, tracer) values (?, ?)",
                 ((file_ids[filename], tracer) for filename, tracer in tracer_map.items()),

coverage/version.py

@@ -8,7 +8,7 @@
 
 # version_info: same semantics as sys.version_info.
 # _dev: the .devN suffix if any.
-version_info = (7, 5, 2, "final", 0)
+version_info = (7, 5, 3, "final", 0)
 _dev = 0
 
 

doc/conf.py

@@ -67,11 +67,11 @@
 # @@@ editable
 copyright = "2009–2024, Ned Batchelder" # pylint: disable=redefined-builtin
 # The short X.Y.Z version.
-version = "7.5.2"
+version = "7.5.3"
 # The full version, including alpha/beta/rc tags.
-release = "7.5.2"
+release = "7.5.3"
 # The date of release, in "monthname day, year" format.
-release_date = "May 24, 2024"
+release_date = "May 28, 2024"
 # @@@ end
 
 rst_epilog = f"""

doc/sample_html/class_index.html

@@ -56,8 +56,8 @@ <h2>
                 <a class="button current">Classes</a>
         </h2>
         <p class="text">
-            <a class="nav" href="https://coverage.readthedocs.io/en/7.5.2">coverage.py v7.5.2</a>,
-            created at 2024-05-24 16:52 -0400
+            <a class="nav" href="https://coverage.readthedocs.io/en/7.5.3">coverage.py v7.5.3</a>,
+            created at 2024-05-28 09:37 -0400
         </p>
     </div>
 </header>
@@ -537,8 +537,8 @@ <h2>
 <footer>
     <div class="content">
         <p>
-            <a class="nav" href="https://coverage.readthedocs.io/en/7.5.2">coverage.py v7.5.2</a>,
-            created at 2024-05-24 16:52 -0400
+            <a class="nav" href="https://coverage.readthedocs.io/en/7.5.3">coverage.py v7.5.3</a>,
+            created at 2024-05-28 09:37 -0400
         </p>
     </div>
     <aside class="hidden">

doc/sample_html/function_index.html

@@ -56,8 +56,8 @@ <h2>
                 <a class="button" href="class_index.html">Classes</a>
         </h2>
         <p class="text">
-            <a class="nav" href="https://coverage.readthedocs.io/en/7.5.2">coverage.py v7.5.2</a>,
-            created at 2024-05-24 16:52 -0400
+            <a class="nav" href="https://coverage.readthedocs.io/en/7.5.3">coverage.py v7.5.3</a>,
+            created at 2024-05-28 09:37 -0400
         </p>
     </div>
 </header>
@@ -2377,8 +2377,8 @@ <h2>
 <footer>
     <div class="content">
         <p>
-            <a class="nav" href="https://coverage.readthedocs.io/en/7.5.2">coverage.py v7.5.2</a>,
-            created at 2024-05-24 16:52 -0400
+            <a class="nav" href="https://coverage.readthedocs.io/en/7.5.3">coverage.py v7.5.3</a>,
+            created at 2024-05-28 09:37 -0400
         </p>
     </div>
     <aside class="hidden">

doc/sample_html/index.html

@@ -55,8 +55,8 @@ <h2>
                 <a class="button" href="class_index.html">Classes</a>
         </h2>
         <p class="text">
-            <a class="nav" href="https://coverage.readthedocs.io/en/7.5.2">coverage.py v7.5.2</a>,
-            created at 2024-05-24 16:52 -0400
+            <a class="nav" href="https://coverage.readthedocs.io/en/7.5.3">coverage.py v7.5.3</a>,
+            created at 2024-05-28 09:37 -0400
         </p>
     </div>
 </header>
@@ -175,8 +175,8 @@ <h2>
 <footer>
     <div class="content">
         <p>
-            <a class="nav" href="https://coverage.readthedocs.io/en/7.5.2">coverage.py v7.5.2</a>,
-            created at 2024-05-24 16:52 -0400
+            <a class="nav" href="https://coverage.readthedocs.io/en/7.5.3">coverage.py v7.5.3</a>,
+            created at 2024-05-28 09:37 -0400
         </p>
     </div>
     <aside class="hidden">

doc/sample_html/status.json

@@ -1 +1 @@
-{"note":"This file is an internal implementation detail to speed up HTML report generation. Its format can change at any time. You might be looking for the JSON report: https://coverage.rtfd.io/cmd.html#cmd-json","format":5,"version":"7.5.2","globals":"b6d62a4c6ebf32368f078323bb24aaf8","files":{"z_7b071bdc2a35fa80___init___py":{"hash":"70a508cdcdeb999b005ef6bbb19ef352","index":{"url":"z_7b071bdc2a35fa80___init___py.html","file":"cogapp/__init__.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":1,"n_excluded":0,"n_missing":0,"n_branches":0,"n_partial_branches":0,"n_missing_branches":0}}},"z_7b071bdc2a35fa80___main___py":{"hash":"6d9d0d551879aa3e73791f40c5739845","index":{"url":"z_7b071bdc2a35fa80___main___py.html","file":"cogapp/__main__.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":3,"n_excluded":0,"n_missing":3,"n_branches":0,"n_partial_branches":0,"n_missing_branches":0}}},"z_7b071bdc2a35fa80_cogapp_py":{"hash":"5ba0c64e49e07207b0c428615ecf9962","index":{"url":"z_7b071bdc2a35fa80_cogapp_py.html","file":"cogapp/cogapp.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":483,"n_excluded":1,"n_missing":228,"n_branches":208,"n_partial_branches":28,"n_missing_branches":140}}},"z_7b071bdc2a35fa80_makefiles_py":{"hash":"eaf4689c0c47697806b20a0a782f9e2a","index":{"url":"z_7b071bdc2a35fa80_makefiles_py.html","file":"cogapp/makefiles.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":22,"n_excluded":0,"n_missing":18,"n_branches":14,"n_partial_branches":0,"n_missing_branches":14}}},"z_7b071bdc2a35fa80_test_cogapp_py":{"hash":"10bdcf9d7378f460315a89e0163729dc","index":{"url":"z_7b071bdc2a35fa80_test_cogapp_py.html","file":"cogapp/test_cogapp.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":854,"n_excluded":2,"n_missing":598,"n_branches":24,"n_partial_branches":1,"n_missing_branches":21}}},"z_7b071bdc2a35fa80_test_makefiles_py":{"hash":"a4a125d4209ab0e413c7c49768fd322f","index":{"url":"z_7b071bdc2a35fa80_test_makefiles_py.html","file":"cogapp/test_makefiles.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":68,"n_excluded":0,"n_missing":51,"n_branches":6,"n_partial_branches":0,"n_missing_branches":6}}},"z_7b071bdc2a35fa80_test_whiteutils_py":{"hash":"59819ec39ae83287b478821e619c36df","index":{"url":"z_7b071bdc2a35fa80_test_whiteutils_py.html","file":"cogapp/test_whiteutils.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":68,"n_excluded":0,"n_missing":50,"n_branches":0,"n_partial_branches":0,"n_missing_branches":0}}},"z_7b071bdc2a35fa80_utils_py":{"hash":"e49fdb51d89dd0fe016bf8a9e73315fa","index":{"url":"z_7b071bdc2a35fa80_utils_py.html","file":"cogapp/utils.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":37,"n_excluded":0,"n_missing":8,"n_branches":6,"n_partial_branches":2,"n_missing_branches":2}}},"z_7b071bdc2a35fa80_whiteutils_py":{"hash":"828c0e3a8398ba557c1f936ae3093939","index":{"url":"z_7b071bdc2a35fa80_whiteutils_py.html","file":"cogapp/whiteutils.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":44,"n_excluded":0,"n_missing":5,"n_branches":34,"n_partial_branches":4,"n_missing_branches":4}}}}}
+{"note":"This file is an internal implementation detail to speed up HTML report generation. Its format can change at any time. You might be looking for the JSON report: https://coverage.rtfd.io/cmd.html#cmd-json","format":5,"version":"7.5.3","globals":"b6d62a4c6ebf32368f078323bb24aaf8","files":{"z_7b071bdc2a35fa80___init___py":{"hash":"70a508cdcdeb999b005ef6bbb19ef352","index":{"url":"z_7b071bdc2a35fa80___init___py.html","file":"cogapp/__init__.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":1,"n_excluded":0,"n_missing":0,"n_branches":0,"n_partial_branches":0,"n_missing_branches":0}}},"z_7b071bdc2a35fa80___main___py":{"hash":"6d9d0d551879aa3e73791f40c5739845","index":{"url":"z_7b071bdc2a35fa80___main___py.html","file":"cogapp/__main__.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":3,"n_excluded":0,"n_missing":3,"n_branches":0,"n_partial_branches":0,"n_missing_branches":0}}},"z_7b071bdc2a35fa80_cogapp_py":{"hash":"5ba0c64e49e07207b0c428615ecf9962","index":{"url":"z_7b071bdc2a35fa80_cogapp_py.html","file":"cogapp/cogapp.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":483,"n_excluded":1,"n_missing":228,"n_branches":208,"n_partial_branches":28,"n_missing_branches":140}}},"z_7b071bdc2a35fa80_makefiles_py":{"hash":"eaf4689c0c47697806b20a0a782f9e2a","index":{"url":"z_7b071bdc2a35fa80_makefiles_py.html","file":"cogapp/makefiles.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":22,"n_excluded":0,"n_missing":18,"n_branches":14,"n_partial_branches":0,"n_missing_branches":14}}},"z_7b071bdc2a35fa80_test_cogapp_py":{"hash":"10bdcf9d7378f460315a89e0163729dc","index":{"url":"z_7b071bdc2a35fa80_test_cogapp_py.html","file":"cogapp/test_cogapp.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":854,"n_excluded":2,"n_missing":598,"n_branches":24,"n_partial_branches":1,"n_missing_branches":21}}},"z_7b071bdc2a35fa80_test_makefiles_py":{"hash":"a4a125d4209ab0e413c7c49768fd322f","index":{"url":"z_7b071bdc2a35fa80_test_makefiles_py.html","file":"cogapp/test_makefiles.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":68,"n_excluded":0,"n_missing":51,"n_branches":6,"n_partial_branches":0,"n_missing_branches":6}}},"z_7b071bdc2a35fa80_test_whiteutils_py":{"hash":"59819ec39ae83287b478821e619c36df","index":{"url":"z_7b071bdc2a35fa80_test_whiteutils_py.html","file":"cogapp/test_whiteutils.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":68,"n_excluded":0,"n_missing":50,"n_branches":0,"n_partial_branches":0,"n_missing_branches":0}}},"z_7b071bdc2a35fa80_utils_py":{"hash":"e49fdb51d89dd0fe016bf8a9e73315fa","index":{"url":"z_7b071bdc2a35fa80_utils_py.html","file":"cogapp/utils.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":37,"n_excluded":0,"n_missing":8,"n_branches":6,"n_partial_branches":2,"n_missing_branches":2}}},"z_7b071bdc2a35fa80_whiteutils_py":{"hash":"828c0e3a8398ba557c1f936ae3093939","index":{"url":"z_7b071bdc2a35fa80_whiteutils_py.html","file":"cogapp/whiteutils.py","description":"","nums":{"precision":2,"n_files":1,"n_statements":44,"n_excluded":0,"n_missing":5,"n_branches":34,"n_partial_branches":4,"n_missing_branches":4}}}}}

doc/sample_html/z_7b071bdc2a35fa80___init___py.html

@@ -66,8 +66,8 @@ <h2>
             <a id="indexLink" class="nav" href="index.html">&Hat; index</a> &nbsp; &nbsp;
             <a id="nextFileLink" class="nav" href="z_7b071bdc2a35fa80___main___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
-            <a class="nav" href="https://coverage.readthedocs.io/en/7.5.2">coverage.py v7.5.2</a>,
-            created at 2024-05-24 16:52 -0400
+            <a class="nav" href="https://coverage.readthedocs.io/en/7.5.3">coverage.py v7.5.3</a>,
+            created at 2024-05-28 09:37 -0400
         </p>
         <aside class="hidden">
             <button type="button" class="button_next_chunk" data-shortcut="j"></button>
@@ -97,8 +97,8 @@ <h2>
             <a class="nav" href="index.html">&Hat; index</a> &nbsp; &nbsp;
             <a class="nav" href="z_7b071bdc2a35fa80___main___py.html">&#xbb; next</a>
             &nbsp; &nbsp; &nbsp;
-            <a class="nav" href="https://coverage.readthedocs.io/en/7.5.2">coverage.py v7.5.2</a>,
-            created at 2024-05-24 16:52 -0400
+            <a class="nav" href="https://coverage.readthedocs.io/en/7.5.3">coverage.py v7.5.3</a>,
+            created at 2024-05-28 09:37 -0400
         </p>
     </div>
 </footer>