Write .patch files to add @example() decorators for failing test cases

hypothesis-python/.coveragerc

@@ -12,6 +12,9 @@ omit =
     **/utils/terminal.py
 
 [report]
+fail_under = 100
+show_missing = True
+skip_covered = True
 exclude_lines =
     pragma: no cover
     raise NotImplementedError

hypothesis-python/RELEASE.rst

@@ -0,0 +1,10 @@
+RELEASE_TYPE: minor
+
+Sick of adding :obj:`@example() <hypothesis.example>`\ s by hand?
+Our Pytest plugin now writes ``.patch`` files to insert them for you, making
+`this workflow <https://blog.nelhage.com/post/property-testing-like-afl/>`__
+easier than ever before.
+
+Note that you'll need :pypi:`LibCST` (via :ref:`codemods`), and that
+:obj:`@example().via() <hypothesis.example.via>` requires :pep:`614`
+(Python 3.9 or later).

hypothesis-python/docs/changes.rst

@@ -9216,8 +9216,8 @@ This changes only the formatting of our docstrings and should have no user-visib
 what arguments are valid, and additional validation logic to raise a
 clear error early (instead of e.g. silently ignoring a bad argument).
 Categories may be specified as the Unicode 'general category'
-(eg ``u'Nd'``), or as the 'major category' (eg ``[u'N', u'Lu']``
-is equivalent to ``[u'Nd', u'Nl', u'No', u'Lu']``).
+(eg ``'Nd'``), or as the 'major category' (eg ``['N', 'Lu']``
+is equivalent to ``['Nd', 'Nl', 'No', 'Lu']``).
 
 In previous versions, general categories were supported and all other
 input was silently ignored.  Now, major categories are supported in

hypothesis-python/docs/settings.rst

@@ -240,7 +240,7 @@ If this variable is not defined the Hypothesis defined defaults will be loaded.
     >>> settings.register_profile("ci", max_examples=1000)
     >>> settings.register_profile("dev", max_examples=10)
     >>> settings.register_profile("debug", max_examples=10, verbosity=Verbosity.verbose)
-    >>> settings.load_profile(os.getenv(u"HYPOTHESIS_PROFILE", "default"))
+    >>> settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default"))
 
 If you are using the hypothesis pytest plugin and your profiles are registered
 by your conftest you can load one with the command line option ``--hypothesis-profile``.

hypothesis-python/scripts/other-tests.sh

@@ -46,7 +46,9 @@ pip uninstall -y lark
 if [ "$(python -c $'import platform, sys; print(sys.version_info.releaselevel == \'final\' and platform.python_implementation() not in ("PyPy", "GraalVM"))')" = "True" ] ; then
   pip install ".[codemods,cli]"
   $PYTEST tests/codemods/
-  pip uninstall -y libcst click
+  pip install "$(grep 'black==' ../requirements/coverage.txt)"
+  $PYTEST tests/patching/
+  pip uninstall -y libcst
 
   if [ "$(python -c 'import sys; print(sys.version_info[:2] == (3, 7))')" = "True" ] ; then
     # Per NEP-29, this is the last version to support Python 3.7
@@ -63,7 +65,6 @@ if [ "$(python -c $'import platform, sys; print(sys.version_info.releaselevel ==
       $PYTEST tests/numpy
   esac
 
-  pip install "$(grep 'black==' ../requirements/coverage.txt)"
   $PYTEST tests/ghostwriter/
   pip uninstall -y black numpy
 fi

hypothesis-python/src/_hypothesis_pytestplugin.py

@@ -20,6 +20,7 @@
 """
 
 import base64
+import json
 import sys
 from inspect import signature
 
@@ -59,6 +60,7 @@
 """
 
 STATS_KEY = "_hypothesis_stats"
+FAILING_EXAMPLES_KEY = "_hypothesis_failing_examples"
 
 
 class StoringReporter:
@@ -298,7 +300,12 @@ def pytest_runtest_makereport(item, call):
             report.sections.append(
                 ("Hypothesis", "\n".join(item.hypothesis_report_information))
             )
-        if hasattr(item, "hypothesis_statistics") and report.when == "teardown":
+        if report.when != "teardown":
+            return
+
+        terminalreporter = item.config.pluginmanager.getplugin("terminalreporter")
+
+        if hasattr(item, "hypothesis_statistics"):
             stats = item.hypothesis_statistics
             stats_base64 = base64.b64encode(stats.encode()).decode()
 
@@ -314,10 +321,7 @@ def pytest_runtest_makereport(item, call):
                 xml.add_global_property(name, stats_base64)
 
             # If there's a terminal report, include our summary stats for each test
-            terminalreporter = item.config.pluginmanager.getplugin("terminalreporter")
             if terminalreporter is not None:
-                # ideally, we would store this on terminalreporter.config.stash, but
-                # pytest-xdist doesn't copy that back to the controller
                 report.__dict__[STATS_KEY] = stats
 
             # If there's an HTML report, include our summary stats for each test
@@ -327,14 +331,49 @@ def pytest_runtest_makereport(item, call):
                     pytest_html.extras.text(stats, name="Hypothesis stats")
                 ]
 
+        # This doesn't intrinsically have anything to do with the terminalreporter;
+        # we're just cargo-culting a way to get strings back to a single function
+        # even if the test were distributed with pytest-xdist.
+        failing_examples = getattr(item, FAILING_EXAMPLES_KEY, None)
+        if failing_examples and terminalreporter is not None:
+            try:
+                from hypothesis.extra.patching import get_patch_for
+            except ImportError:
+                return
+            # We'll save this as a triple of [filename, hunk_before, hunk_after].
+            triple = get_patch_for(item.obj, failing_examples)
+            if triple is not None:
+                report.__dict__[FAILING_EXAMPLES_KEY] = json.dumps(triple)
+
     def pytest_terminal_summary(terminalreporter):
-        if terminalreporter.config.getoption(PRINT_STATISTICS_OPTION):
+        failing_examples = []
+        print_stats = terminalreporter.config.getoption(PRINT_STATISTICS_OPTION)
+        if print_stats:
             terminalreporter.section("Hypothesis Statistics")
-            for reports in terminalreporter.stats.values():
-                for report in reports:
-                    stats = report.__dict__.get(STATS_KEY)
-                    if stats:
-                        terminalreporter.write_line(stats + "\n\n")
+        for reports in terminalreporter.stats.values():
+            for report in reports:
+                stats = report.__dict__.get(STATS_KEY)
+                if stats and print_stats:
+                    terminalreporter.write_line(stats + "\n\n")
+                fex = report.__dict__.get(FAILING_EXAMPLES_KEY)
+                if fex:
+                    failing_examples.append(json.loads(fex))
+
+        if failing_examples:
+            # This must have been imported already to write the failing examples
+            from hypothesis.extra.patching import gc_patches, make_patch, save_patch
+
+            patch = make_patch(failing_examples)
+            try:
+                gc_patches()
+                fname = save_patch(patch)
+            except Exception:
+                # fail gracefully if we hit any filesystem or permissions problems
+                return
+            terminalreporter.section("Hypothesis")
+            terminalreporter.write_line(
+                f"`git apply {fname}` to add failing examples to your code."
+            )
 
     def pytest_collection_modifyitems(items):
         if "hypothesis" not in sys.modules:

hypothesis-python/src/hypothesis/core.py

@@ -79,6 +79,7 @@
 from hypothesis.internal.conjecture.shrinker import sort_key
 from hypothesis.internal.entropy import deterministic_PRNG
 from hypothesis.internal.escalation import (
+    current_pytest_item,
     escalate_hypothesis_internal_error,
     format_exception,
     get_interesting_origin,
@@ -787,10 +788,12 @@ def run(data):
                                     kwargs,
                                     force_split=True,
                                     arg_slices=argslices,
+                                    leading_comment=(
+                                        "# " + context.data.slice_comments[(0, 0)]
+                                        if (0, 0) in context.data.slice_comments
+                                        else None
+                                    ),
                                 )
-                            if (0, 0) in context.data.slice_comments:
-                                printer.break_()
-                                printer.text("# " + context.data.slice_comments[(0, 0)])
                             report(printer.getvalue())
                         return test(*args, **kwargs)
 
@@ -1028,10 +1031,15 @@ def add_note(exc, note):
 
 def _raise_to_user(errors_to_report, settings, target_lines, trailer=""):
     """Helper function for attaching notes and grouping multiple errors."""
-    if settings.verbosity >= Verbosity.normal:
-        for fragments, err in errors_to_report:
-            for note in fragments:
-                add_note(err, note)
+    failing_prefix = "Falsifying example: "
+    ls = []
+    for fragments, err in errors_to_report:
+        for note in fragments:
+            add_note(err, note)
+            if note.startswith(failing_prefix):
+                ls.append(note[len(failing_prefix) :])
+    if current_pytest_item.value:
+        current_pytest_item.value._hypothesis_failing_examples = ls
 
     if len(errors_to_report) == 1:
         _, the_error_hypothesis_found = errors_to_report[0]

hypothesis-python/src/hypothesis/extra/_patching.py

@@ -0,0 +1,186 @@
+# This file is part of Hypothesis, which may be found at
+# https://github.com/HypothesisWorks/hypothesis/
+#
+# Copyright the Hypothesis Authors.
+# Individual contributors are listed in AUTHORS.rst and the git log.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at https://mozilla.org/MPL/2.0/.
+
+"""
+Write patches which add @example() decorators for discovered test cases.
+
+Requires `hypothesis[codemods,ghostwriter]` installed, i.e. black and libcst.
+
+This module is used by Hypothesis' builtin pytest plugin for failing examples
+discovered during testing, and by HypoFuzz for _covering_ examples discovered
+during fuzzing.
+"""
+
+import difflib
+import hashlib
+import inspect
+import re
+import sys
+from contextlib import suppress
+from datetime import date, datetime, timedelta, timezone
+from pathlib import Path
+
+import libcst as cst
+from libcst.codemod import CodemodContext, VisitorBasedCodemodCommand
+
+from hypothesis.configuration import storage_directory
+from hypothesis.extra.codemods import _native_parser
+from hypothesis.version import __version__
+
+try:
+    import black
+except ImportError:
+    black = None  # type: ignore
+
+HEADER = f"""\
+From HEAD Mon Sep 17 00:00:00 2001
+From: Hypothesis {__version__} <no-reply@hypothesis.works>
+Date: {{when:%a, %d %b %Y %H:%M:%S}}
+Subject: [PATCH] {{msg}}
+
+---
+"""
+_space_only_re = re.compile("^ +$", re.MULTILINE)
+_leading_space_re = re.compile("(^[ ]*)(?:[^ \n])", re.MULTILINE)
+
+
+def dedent(text):
+    # Simplified textwrap.dedent, for valid Python source code only
+    text = _space_only_re.sub("", text)
+    prefix = min(_leading_space_re.findall(text), key=len)
+    return re.sub(r"(?m)^" + prefix, "", text), prefix
+
+
+def indent(text: str, prefix: str) -> str:
+    return "".join(prefix + line for line in text.splitlines(keepends=True))
+
+
+class AddExamplesCodemod(VisitorBasedCodemodCommand):
+    DESCRIPTION = "Add explicit examples to failing tests."
+
+    @classmethod
+    def refactor(cls, code: str, fn_examples: dict) -> str:
+        """Add @example() decorator(s) for failing test(s).
+
+        `code` is the source code of the module where the test functions are defined.
+        `fn_examples` is a dict of function name to list-of-failing-examples.
+        """
+        dedented, prefix = dedent(code)
+        with _native_parser():
+            mod = cst.parse_module(dedented)
+        modded = cls(CodemodContext(), fn_examples, prefix).transform_module(mod).code
+        return indent(modded, prefix=prefix)
+
+    def __init__(self, context, fn_examples, prefix="", via="discovered failure"):
+        assert fn_examples, "This codemod does nothing without fn_examples."
+        super().__init__(context)
+
+        # Codemod the failing examples to Call nodes usable as decorators
+        self.via = via
+        self.line_length = 88 - len(prefix)  # to match Black's default formatting
+        self.fn_examples = {
+            k: tuple(self.__call_node_to_example_dec(ex) for ex in nodes)
+            for k, nodes in fn_examples.items()
+        }
+
+    def __call_node_to_example_dec(self, node):
+        node = node.with_changes(
+            func=cst.Name("example"),
+            args=[a.with_changes(comma=cst.MaybeSentinel.DEFAULT) for a in node.args]
+            if black
+            else node.args,
+        )
+        # Note: calling a method on a decorator requires PEP-614, i.e. Python 3.9+,
+        # but plumbing two cases through doesn't seem worth the trouble :-/
+        via = cst.Call(
+            func=cst.Attribute(node, cst.Name("via")),
+            args=[cst.Arg(cst.SimpleString(repr(self.via)))],
+        )
+        if black:  # pragma: no branch
+            pretty = black.format_str(
+                cst.Module([]).code_for_node(via),
+                mode=black.FileMode(line_length=self.line_length),
+            )
+            via = cst.parse_expression(pretty.strip())
+        return cst.Decorator(via)
+
+    def leave_FunctionDef(self, _, updated_node):
+        return updated_node.with_changes(
+            # TODO: improve logic for where in the list to insert this decorator
+            decorators=updated_node.decorators
+            + self.fn_examples.get(updated_node.name.value, ())
+        )
+
+
+def get_patch_for(func, failing_examples):
+    # Skip this if we're unable to find the location or source of this function.
+    try:
+        fname = Path(sys.modules[func.__module__].__file__).relative_to(Path.cwd())
+        before = inspect.getsource(func)
+    except Exception:
+        return None
+
+    # The printed examples might include object reprs which are invalid syntax,
+    # so we parse here and skip over those.  If _none_ are valid, there's no patch.
+    call_nodes = []
+    for ex in failing_examples:
+        with suppress(Exception):
+            node = cst.parse_expression(ex)
+            assert isinstance(node, cst.Call), node
+            call_nodes.append(node)
+    if not call_nodes:
+        return None
+
+    # Do the codemod and return a triple containing location and replacement info.
+    after = AddExamplesCodemod.refactor(
+        before,
+        fn_examples={func.__name__: call_nodes},
+    )
+    return (str(fname), before, after)
+
+
+def make_patch(triples, *, msg="Hypothesis: add failing examples", when=None):
+    """Create a patch for (fname, before, after) triples."""
+    assert triples, "attempted to create empty patch"
+    when = when or datetime.now(tz=timezone.utc)
+
+    by_fname = {}
+    for fname, before, after in triples:
+        by_fname.setdefault(Path(fname), []).append((before, after))
+
+    diffs = [HEADER.format(msg=msg, when=when)]
+    for fname, changes in sorted(by_fname.items()):
+        source_before = source_after = fname.read_text(encoding="utf-8")
+        for before, after in changes:
+            source_after = source_after.replace(before, after, 1)
+        ud = difflib.unified_diff(
+            source_before.splitlines(keepends=True),
+            source_after.splitlines(keepends=True),
+            fromfile=str(fname),
+            tofile=str(fname),
+        )
+        diffs.append("".join(ud))
+    return "".join(diffs)
+
+
+def save_patch(patch: str) -> Path:  # pragma: no cover
+    today = date.today().isoformat()
+    hash = hashlib.sha1(patch.encode()).hexdigest()[:8]
+    fname = Path(storage_directory("patches", f"{today}--{hash}.patch"))
+    fname.parent.mkdir(parents=True, exist_ok=True)
+    fname.write_text(patch, encoding="utf-8")
+    return fname.relative_to(Path.cwd())
+
+
+def gc_patches():  # pragma: no cover
+    cutoff = date.today() - timedelta(days=7)
+    for fname in Path(storage_directory("patches")).glob("????-??-??--????????.patch"):
+        if date.fromisoformat(fname.stem.split("--")[0]) < cutoff:
+            fname.unlink()