Add summary statistics to the coverage report (#5474)

The current implementation of ``sphinx.ext.coverage`` outputs which
methods,classes, and functions are documented.
This commit adds a short summary of this report in terms of
``documented objects / total number of objects``,
both per module and total.

The purpose of this is to support
a currently not mainstream but relevant use-case:
a coverage report on the number of objects that are documented.

By having the statistics on the report or on the stdout,
a regex expression can capture the coverage percentage
(e.g. ``re.search(r'TOTAL.*?([0-9.]{4,6}\%)', d).group(1)``)
and use it e.g. in another report, a status badge, etc.

Two options were added to the configuration to allow a table
to be printed in the report and/or to stdout.

Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com>

CHANGES

@@ -22,6 +22,8 @@ Features added
 
 * #11526: Support ``os.PathLike`` types and ``pathlib.Path`` objects
   in many more places.
+* #5474: coverage: Print summary statistics tables.
+  Patch by Jorge Leitao.
 
 Bugs fixed
 ----------

doc/conf.py

@@ -9,8 +9,9 @@
 extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo',
               'sphinx.ext.autosummary', 'sphinx.ext.extlinks',
               'sphinx.ext.intersphinx',
-              'sphinx.ext.viewcode', 'sphinx.ext.inheritance_diagram']
-
+              'sphinx.ext.viewcode', 'sphinx.ext.inheritance_diagram',
+              'sphinx.ext.coverage']
+coverage_statistics_to_report = coverage_statistics_to_stdout = True
 templates_path = ['_templates']
 exclude_patterns = ['_build']
 

doc/usage/extensions/coverage.rst

@@ -30,6 +30,8 @@ should check:
    of a Python object, that Python object is excluded from the documentation
    coverage report.
 
+   .. _Python regular expressions: https://docs.python.org/library/re
+
    .. versionadded:: 2.1
 
 .. confval:: coverage_c_path
@@ -58,4 +60,40 @@ should check:
 
    .. versionadded:: 3.1
 
-.. _Python regular expressions: https://docs.python.org/library/re
+.. confval:: coverage_statistics_to_report
+
+   Print a tabluar report of the coverage statistics to the coverage report.
+   ``True`` by default.
+
+   Example output:
+
+   .. code-block:: text
+
+      +-----------------------+----------+--------------+
+      | Module                | Coverage | Undocumented |
+      +=======================+==========+==============+
+      | package.foo_module    | 100.00%  | 0            |
+      +-----------------------+----------+--------------+
+      | package.bar_module    | 83.33%   | 1            |
+      +-----------------------+----------+--------------+
+
+   .. versionadded:: 7.2
+
+.. confval:: coverage_statistics_to_stdout
+
+   Print a tabluar report of the coverage statistics to standard output.
+   ``False`` by default.
+
+   Example output:
+
+   .. code-block:: text
+
+      +-----------------------+----------+--------------+
+      | Module                | Coverage | Undocumented |
+      +=======================+==========+==============+
+      | package.foo_module    | 100.00%  | 0            |
+      +-----------------------+----------+--------------+
+      | package.bar_module    | 83.33%   | 1            |
+      +-----------------------+----------+--------------+
+
+   .. versionadded:: 7.2

sphinx/ext/coverage.py

@@ -10,9 +10,10 @@
 import inspect
 import pickle
 import re
+import sys
 from importlib import import_module
 from os import path
-from typing import IO, Any
+from typing import IO, TYPE_CHECKING, Any, TextIO
 
 import sphinx
 from sphinx.application import Sphinx
@@ -22,13 +23,16 @@
 from sphinx.util.console import red  # type: ignore
 from sphinx.util.inspect import safe_getattr
 
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
 logger = logging.getLogger(__name__)
 
 
 # utility
 def write_header(f: IO[str], text: str, char: str = '-') -> None:
     f.write(text + '\n')
-    f.write(char * len(text) + '\n')
+    f.write(char * len(text) + '\n\n')
 
 
 def compile_regex_list(name: str, exps: str) -> list[re.Pattern[str]]:
@@ -41,6 +45,25 @@ def compile_regex_list(name: str, exps: str) -> list[re.Pattern[str]]:
     return lst
 
 
+def _write_table(table: list[list[str]]) -> Iterator[str]:
+    sizes = [max(len(x[column]) for x in table) + 1 for column in range(len(table[0]))]
+
+    yield _add_line(sizes, '-')
+    yield from _add_row(sizes, table[0], '=')
+
+    for row in table[1:]:
+        yield from _add_row(sizes, row, '-')
+
+
+def _add_line(sizes: list[int], separator: str) -> str:
+    return '+' + ''.join((separator * (size + 1)) + '+' for size in sizes)
+
+
+def _add_row(col_widths: list[int], columns: list[str], separator: str) -> Iterator[str]:
+    yield ''.join(f'| {column: <{col_widths[i]}}' for i, column in enumerate(columns)) + '|'
+    yield _add_line(col_widths, separator)
+
+
 class CoverageBuilder(Builder):
     """
     Evaluates coverage of code in the documentation.
@@ -80,6 +103,8 @@ def get_outdated_docs(self) -> str:
 
     def write(self, *ignored: Any) -> None:
         self.py_undoc: dict[str, dict[str, Any]] = {}
+        self.py_undocumented: dict[str, set[str]] = {}
+        self.py_documented: dict[str, set[str]] = {}
         self.build_py_coverage()
         self.write_py_coverage()
 
@@ -142,6 +167,7 @@ def build_py_coverage(self) -> None:
         skip_undoc = self.config.coverage_skip_undoc_in_source
 
         for mod_name in modules:
+            print(mod_name)
             ignore = False
             for exp in self.mod_ignorexps:
                 if exp.match(mod_name):
@@ -157,6 +183,9 @@ def build_py_coverage(self) -> None:
                 self.py_undoc[mod_name] = {'error': err}
                 continue
 
+            documented_objects: set[str] = set()
+            undocumented_objects: set[str] = set()
+
             funcs = []
             classes: dict[str, list[str]] = {}
 
@@ -185,6 +214,9 @@ def build_py_coverage(self) -> None:
                             if skip_undoc and not obj.__doc__:
                                 continue
                             funcs.append(name)
+                            undocumented_objects.add(full_name)
+                    else:
+                        documented_objects.add(full_name)
                 elif inspect.isclass(obj):
                     for exp in self.cls_ignorexps:
                         if exp.match(name):
@@ -220,18 +252,63 @@ def build_py_coverage(self) -> None:
                                 continue
                             if full_attr_name not in objects:
                                 attrs.append(attr_name)
+                                undocumented_objects.add(full_attr_name)
+                            else:
+                                documented_objects.add(full_attr_name)
+
                         if attrs:
                             # some attributes are undocumented
                             classes[name] = attrs
 
             self.py_undoc[mod_name] = {'funcs': funcs, 'classes': classes}
+            self.py_undocumented[mod_name] = undocumented_objects
+            self.py_documented[mod_name] = documented_objects
+
+    def _write_py_statistics(self, op: TextIO) -> None:
+        """ Outputs the table of ``op``."""
+        all_modules = set(self.py_documented.keys()).union(
+            set(self.py_undocumented.keys()))
+        all_objects: set[str] = set()
+        all_documented_objects: set[str] = set()
+        for module in all_modules:
+            all_module_objects = self.py_documented[module].union(self.py_undocumented[module])
+            all_objects = all_objects.union(all_module_objects)
+            all_documented_objects = all_documented_objects.union(self.py_documented[module])
+
+        # prepare tabular
+        table = [['Module', 'Coverage', 'Undocumented']]
+        for module in all_modules:
+            module_objects = self.py_documented[module].union(self.py_undocumented[module])
+            if len(module_objects):
+                value = 100.0 * len(self.py_documented[module]) / len(module_objects)
+            else:
+                value = 100.0
+
+            table.append([module, '%.2f%%' % value, '%d' % len(self.py_undocumented[module])])
+        table.append([
+            'TOTAL',
+            f'{100 * len(all_documented_objects) / len(all_objects):.2f}%',
+            f'{len(all_objects) - len(all_documented_objects)}',
+        ])
+
+        for line in _write_table(table):
+            op.write(f'{line}\n')
 
     def write_py_coverage(self) -> None:
         output_file = path.join(self.outdir, 'python.txt')
         failed = []
         with open(output_file, 'w', encoding="utf-8") as op:
             if self.config.coverage_write_headline:
                 write_header(op, 'Undocumented Python objects', '=')
+
+            if self.config.coverage_statistics_to_stdout:
+                self._write_py_statistics(sys.stdout)
+
+            if self.config.coverage_statistics_to_report:
+                write_header(op, 'Statistics')
+                self._write_py_statistics(op)
+                op.write('\n')
+
             keys = sorted(self.py_undoc.keys())
             for name in keys:
                 undoc = self.py_undoc[name]
@@ -297,7 +374,8 @@ def finish(self) -> None:
         # dump the coverage data to a pickle file too
         picklepath = path.join(self.outdir, 'undoc.pickle')
         with open(picklepath, 'wb') as dumpfile:
-            pickle.dump((self.py_undoc, self.c_undoc), dumpfile)
+            pickle.dump((self.py_undoc, self.c_undoc,
+                         self.py_undocumented, self.py_documented), dumpfile)
 
 
 def setup(app: Sphinx) -> dict[str, Any]:
@@ -310,6 +388,8 @@ def setup(app: Sphinx) -> dict[str, Any]:
     app.add_config_value('coverage_c_regexes', {}, False)
     app.add_config_value('coverage_ignore_c_items', {}, False)
     app.add_config_value('coverage_write_headline', True, False)
+    app.add_config_value('coverage_statistics_to_report', True, False, (bool,))
+    app.add_config_value('coverage_statistics_to_stdout', True, False, (bool,))
     app.add_config_value('coverage_skip_undoc_in_source', False, False)
     app.add_config_value('coverage_show_missing_items', False, False)
     return {'version': sphinx.__display_version__, 'parallel_read_safe': True}

tests/test_ext_coverage.py

@@ -28,7 +28,7 @@ def test_build(app, status, warning):
     assert 'api.h' in c_undoc
     assert ' * Py_SphinxTest' in c_undoc
 
-    undoc_py, undoc_c = pickle.loads((app.outdir / 'undoc.pickle').read_bytes())
+    undoc_py, undoc_c, py_undocumented, py_documented = pickle.loads((app.outdir / 'undoc.pickle').read_bytes())
     assert len(undoc_c) == 1
     # the key is the full path to the header file, which isn't testable
     assert list(undoc_c.values())[0] == {('function', 'Py_SphinxTest')}
@@ -47,10 +47,24 @@ def test_build(app, status, warning):
 def test_coverage_ignore_pyobjects(app, status, warning):
     app.builder.build_all()
     actual = (app.outdir / 'python.txt').read_text(encoding='utf8')
-    expected = '''Undocumented Python objects
+    expected = '''\
+Undocumented Python objects
 ===========================
+
+Statistics
+----------
+
++----------------------+----------+--------------+
+| Module               | Coverage | Undocumented |
++======================+==========+==============+
+| coverage_not_ignored | 0.00%    | 2            |
++----------------------+----------+--------------+
+| TOTAL                | 0.00%    | 2            |
++----------------------+----------+--------------+
+
 coverage_not_ignored
 --------------------
+
 Classes:
  * Documented -- missing methods: