diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..57ddcd3387 --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,10 @@ +RELEASE_TYPE: minor + +Sick of adding :obj:`@example() `\ s by hand? +Our Pytest plugin now writes ``.patch`` files to insert them for you, making +`this workflow `__ +easier than ever before. + +Note that you'll need :pypi:`LibCST` (via :ref:`codemods`), and that +:obj:`@example().via() ` requires :pep:`614` +(Python 3.9 or later). diff --git a/hypothesis-python/scripts/other-tests.sh b/hypothesis-python/scripts/other-tests.sh index 13c2dde821..4162aac845 100644 --- a/hypothesis-python/scripts/other-tests.sh +++ b/hypothesis-python/scripts/other-tests.sh @@ -46,7 +46,9 @@ pip uninstall -y lark if [ "$(python -c $'import platform, sys; print(sys.version_info.releaselevel == \'final\' and platform.python_implementation() not in ("PyPy", "GraalVM"))')" = "True" ] ; then pip install ".[codemods,cli]" $PYTEST tests/codemods/ - pip uninstall -y libcst click + pip install "$(grep 'black==' ../requirements/coverage.txt)" + $PYTEST tests/patching/ + pip uninstall -y libcst if [ "$(python -c 'import sys; print(sys.version_info[:2] == (3, 7))')" = "True" ] ; then # Per NEP-29, this is the last version to support Python 3.7 @@ -63,7 +65,6 @@ if [ "$(python -c $'import platform, sys; print(sys.version_info.releaselevel == $PYTEST tests/numpy esac - pip install "$(grep 'black==' ../requirements/coverage.txt)" $PYTEST tests/ghostwriter/ pip uninstall -y black numpy fi diff --git a/hypothesis-python/src/_hypothesis_pytestplugin.py b/hypothesis-python/src/_hypothesis_pytestplugin.py index afafd52ce4..2ff62a29df 100644 --- a/hypothesis-python/src/_hypothesis_pytestplugin.py +++ b/hypothesis-python/src/_hypothesis_pytestplugin.py @@ -20,6 +20,7 @@ """ import base64 +import json import sys from inspect import signature @@ -59,6 +60,7 @@ """ STATS_KEY = "_hypothesis_stats" +FAILING_EXAMPLES_KEY = "_hypothesis_failing_examples" class StoringReporter: @@ -298,7 +300,12 @@ def pytest_runtest_makereport(item, call): report.sections.append( ("Hypothesis", "\n".join(item.hypothesis_report_information)) ) - if hasattr(item, "hypothesis_statistics") and report.when == "teardown": + if report.when != "teardown": + return + + terminalreporter = item.config.pluginmanager.getplugin("terminalreporter") + + if hasattr(item, "hypothesis_statistics"): stats = item.hypothesis_statistics stats_base64 = base64.b64encode(stats.encode()).decode() @@ -314,10 +321,7 @@ def pytest_runtest_makereport(item, call): xml.add_global_property(name, stats_base64) # If there's a terminal report, include our summary stats for each test - terminalreporter = item.config.pluginmanager.getplugin("terminalreporter") if terminalreporter is not None: - # ideally, we would store this on terminalreporter.config.stash, but - # pytest-xdist doesn't copy that back to the controller report.__dict__[STATS_KEY] = stats # If there's an HTML report, include our summary stats for each test @@ -327,14 +331,49 @@ def pytest_runtest_makereport(item, call): pytest_html.extras.text(stats, name="Hypothesis stats") ] + # This doesn't intrinsically have anything to do with the terminalreporter; + # we're just cargo-culting a way to get strings back to a single function + # even if the test were distributed with pytest-xdist. + failing_examples = getattr(item, FAILING_EXAMPLES_KEY, None) + if failing_examples and terminalreporter is not None: + try: + from hypothesis.extra.patching import get_patch_for + except ImportError: + return + # We'll save this as a triple of [filename, hunk_before, hunk_after]. + triple = get_patch_for(item.obj, failing_examples) + if triple is not None: + report.__dict__[FAILING_EXAMPLES_KEY] = json.dumps(triple) + def pytest_terminal_summary(terminalreporter): - if terminalreporter.config.getoption(PRINT_STATISTICS_OPTION): + failing_examples = [] + print_stats = terminalreporter.config.getoption(PRINT_STATISTICS_OPTION) + if print_stats: terminalreporter.section("Hypothesis Statistics") - for reports in terminalreporter.stats.values(): - for report in reports: - stats = report.__dict__.get(STATS_KEY) - if stats: - terminalreporter.write_line(stats + "\n\n") + for reports in terminalreporter.stats.values(): + for report in reports: + stats = report.__dict__.get(STATS_KEY) + if stats and print_stats: + terminalreporter.write_line(stats + "\n\n") + fex = report.__dict__.get(FAILING_EXAMPLES_KEY) + if fex: + failing_examples.append(json.loads(fex)) + + if failing_examples: + # This must have been imported already to write the failing examples + from hypothesis.extra.patching import gc_patches, make_patch, save_patch + + patch = make_patch(failing_examples) + try: + gc_patches() + fname = save_patch(patch) + except Exception: + # fail gracefully if we hit any filesystem or permissions problems + return + terminalreporter.section("Hypothesis") + terminalreporter.write_line( + f"`git apply {fname}` to add failing examples to your code." + ) def pytest_collection_modifyitems(items): if "hypothesis" not in sys.modules: diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py index 6186fe67f7..07cf9f897b 100644 --- a/hypothesis-python/src/hypothesis/core.py +++ b/hypothesis-python/src/hypothesis/core.py @@ -79,6 +79,7 @@ from hypothesis.internal.conjecture.shrinker import sort_key from hypothesis.internal.entropy import deterministic_PRNG from hypothesis.internal.escalation import ( + current_pytest_item, escalate_hypothesis_internal_error, format_exception, get_interesting_origin, @@ -1030,10 +1031,15 @@ def add_note(exc, note): def _raise_to_user(errors_to_report, settings, target_lines, trailer=""): """Helper function for attaching notes and grouping multiple errors.""" - if settings.verbosity >= Verbosity.normal: - for fragments, err in errors_to_report: - for note in fragments: - add_note(err, note) + failing_prefix = "Falsifying example: " + ls = [] + for fragments, err in errors_to_report: + for note in fragments: + add_note(err, note) + if note.startswith(failing_prefix): + ls.append(note[len(failing_prefix) :]) + if current_pytest_item.value: + current_pytest_item.value._hypothesis_failing_examples = ls if len(errors_to_report) == 1: _, the_error_hypothesis_found = errors_to_report[0] diff --git a/hypothesis-python/src/hypothesis/extra/_patching.py b/hypothesis-python/src/hypothesis/extra/_patching.py new file mode 100644 index 0000000000..f2dc74705c --- /dev/null +++ b/hypothesis-python/src/hypothesis/extra/_patching.py @@ -0,0 +1,186 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +""" +Write patches which add @example() decorators for discovered test cases. + +Requires `hypothesis[codemods,ghostwriter]` installed, i.e. black and libcst. + +This module is used by Hypothesis' builtin pytest plugin for failing examples +discovered during testing, and by HypoFuzz for _covering_ examples discovered +during fuzzing. +""" + +import difflib +import hashlib +import inspect +import re +import sys +from contextlib import suppress +from datetime import date, datetime, timedelta, timezone +from pathlib import Path + +import libcst as cst +from libcst.codemod import CodemodContext, VisitorBasedCodemodCommand + +from hypothesis.configuration import storage_directory +from hypothesis.extra.codemods import _native_parser +from hypothesis.version import __version__ + +try: + import black +except ImportError: + black = None # type: ignore + +HEADER = f"""\ +From HEAD Mon Sep 17 00:00:00 2001 +From: Hypothesis {__version__} +Date: {{when:%a, %d %b %Y %H:%M:%S}} +Subject: [PATCH] {{msg}} + +--- +""" +_space_only_re = re.compile("^ +$", re.MULTILINE) +_leading_space_re = re.compile("(^[ ]*)(?:[^ \n])", re.MULTILINE) + + +def dedent(text): + # Simplified textwrap.dedent, for valid Python source code only + text = _space_only_re.sub("", text) + prefix = min(_leading_space_re.findall(text), key=len) + return re.sub(r"(?m)^" + prefix, "", text), prefix + + +def indent(text: str, prefix: str) -> str: + return "".join(prefix + line for line in text.splitlines(keepends=True)) + + +class AddExamplesCodemod(VisitorBasedCodemodCommand): + DESCRIPTION = "Add explicit examples to failing tests." + + @classmethod + def refactor(cls, code: str, fn_examples: dict) -> str: + """Add @example() decorator(s) for failing test(s). + + `code` is the source code of the module where the test functions are defined. + `fn_examples` is a dict of function name to list-of-failing-examples. + """ + dedented, prefix = dedent(code) + with _native_parser(): + mod = cst.parse_module(dedented) + modded = cls(CodemodContext(), fn_examples, prefix).transform_module(mod).code + return indent(modded, prefix=prefix) + + def __init__(self, context, fn_examples, prefix="", via="discovered failure"): + assert fn_examples, "This codemod does nothing without fn_examples." + super().__init__(context) + + # Codemod the failing examples to Call nodes usable as decorators + self.via = via + self.line_length = 88 - len(prefix) # to match Black's default formatting + self.fn_examples = { + k: tuple(self.__call_node_to_example_dec(ex) for ex in nodes) + for k, nodes in fn_examples.items() + } + + def __call_node_to_example_dec(self, node): + node = node.with_changes( + func=cst.Name("example"), + args=[a.with_changes(comma=cst.MaybeSentinel.DEFAULT) for a in node.args] + if black + else node.args, + ) + # Note: calling a method on a decorator requires PEP-614, i.e. Python 3.9+, + # but plumbing two cases through doesn't seem worth the trouble :-/ + via = cst.Call( + func=cst.Attribute(node, cst.Name("via")), + args=[cst.Arg(cst.SimpleString(repr(self.via)))], + ) + if black: # pragma: no branch + pretty = black.format_str( + cst.Module([]).code_for_node(via), + mode=black.FileMode(line_length=self.line_length), + ) + via = cst.parse_expression(pretty.strip()) + return cst.Decorator(via) + + def leave_FunctionDef(self, _, updated_node): + return updated_node.with_changes( + # TODO: improve logic for where in the list to insert this decorator + decorators=updated_node.decorators + + self.fn_examples.get(updated_node.name.value, ()) + ) + + +def get_patch_for(func, failing_examples): + # Skip this if we're unable to find the location or source of this function. + try: + fname = Path(sys.modules[func.__module__].__file__).relative_to(Path.cwd()) + before = inspect.getsource(func) + except Exception: + return None + + # The printed examples might include object reprs which are invalid syntax, + # so we parse here and skip over those. If _none_ are valid, there's no patch. + call_nodes = [] + for ex in failing_examples: + with suppress(Exception): + node = cst.parse_expression(ex) + assert isinstance(node, cst.Call), node + call_nodes.append(node) + if not call_nodes: + return None + + # Do the codemod and return a triple containing location and replacement info. + after = AddExamplesCodemod.refactor( + before, + fn_examples={func.__name__: call_nodes}, + ) + return (str(fname), before, after) + + +def make_patch(triples, *, msg="Hypothesis: add failing examples", when=None): + """Create a patch for (fname, before, after) triples.""" + assert triples, "attempted to create empty patch" + when = when or datetime.now(tz=timezone.utc) + + by_fname = {} + for fname, before, after in triples: + by_fname.setdefault(Path(fname), []).append((before, after)) + + diffs = [HEADER.format(msg=msg, when=when)] + for fname, changes in sorted(by_fname.items()): + source_before = source_after = fname.read_text(encoding="utf-8") + for before, after in changes: + source_after = source_after.replace(before, after, 1) + ud = difflib.unified_diff( + source_before.splitlines(keepends=True), + source_after.splitlines(keepends=True), + fromfile=str(fname), + tofile=str(fname), + ) + diffs.append("".join(ud)) + return "".join(diffs) + + +def save_patch(patch: str) -> Path: # pragma: no cover + today = date.today().isoformat() + hash = hashlib.sha1(patch.encode()).hexdigest()[:8] + fname = Path(storage_directory("patches", f"{today}--{hash}.patch")) + fname.parent.mkdir(parents=True, exist_ok=True) + fname.write_text(patch, encoding="utf-8") + return fname.relative_to(Path.cwd()) + + +def gc_patches(): # pragma: no cover + cutoff = date.today() - timedelta(days=7) + for fname in Path(storage_directory("patches")).glob("????-??-??--????????.patch"): + if date.fromisoformat(fname.stem.split("--")[0]) < cutoff: + fname.unlink() diff --git a/hypothesis-python/tests/patching/__init__.py b/hypothesis-python/tests/patching/__init__.py new file mode 100644 index 0000000000..fcb1ac6538 --- /dev/null +++ b/hypothesis-python/tests/patching/__init__.py @@ -0,0 +1,9 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. diff --git a/hypothesis-python/tests/patching/callables.py b/hypothesis-python/tests/patching/callables.py new file mode 100644 index 0000000000..1686fe59c5 --- /dev/null +++ b/hypothesis-python/tests/patching/callables.py @@ -0,0 +1,32 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +"""A stable file for which we can write patches. Don't move stuff around!""" + +from pathlib import Path + +from hypothesis import example, given, strategies as st + +WHERE = Path(__file__).relative_to(Path.cwd()) + + +@given(st.integers()) +def fn(x): + """A trivial test function.""" + + +class Cases: + @example(n=0, label="whatever") + @given(st.integers(), st.text()) + def mth(self, n, label): + """Indented method with existing example decorator.""" + + +# TODO: test function for insertion-order logic, once I get that set up. diff --git a/hypothesis-python/tests/patching/test_patching.py b/hypothesis-python/tests/patching/test_patching.py new file mode 100644 index 0000000000..c0b7910c30 --- /dev/null +++ b/hypothesis-python/tests/patching/test_patching.py @@ -0,0 +1,123 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import re +from copy import deepcopy +from datetime import datetime +from pathlib import Path + +import pytest + +from hypothesis.extra._patching import HEADER, get_patch_for, indent, make_patch + +from .callables import WHERE, Cases, fn + +SIMPLE = ( + fn, + "fn(\n x=1,\n)", + indent('@example(x=1).via("discovered failure")', prefix="+"), +) +CASES = ( + Cases.mth, + 'mth(\n n=100,\n label="a long label which forces a newline",\n)', + indent( + '@example(n=100, label="a long label which forces a newline")' + + '.via(\n "discovered failure"\n)', + prefix="+ ", + ), +) + + +def strip_trailing_whitespace(s): + """Patches have whitespace-only lines; strip that out.""" + return re.sub(r" +$", "", s, flags=re.MULTILINE) + + +@pytest.mark.parametrize( + "tst, example, expected", + [ + pytest.param(*SIMPLE, id="simple"), + pytest.param(*CASES, id="cases"), + ], +) +def test_adds_simple_patch(tst, example, expected): + where, before, after = get_patch_for(tst, [example]) + assert Path(where) == WHERE + added = set(after.splitlines()) - set(before.splitlines()) + assert added == {line.lstrip("+") for line in expected.splitlines()} + + +SIMPLE_PATCH_BODY = f'''\ +--- {WHERE} ++++ {WHERE} +@@ -18,6 +18,7 @@ + + + @given(st.integers()) +{{0}} + def fn(x): + """A trivial test function.""" + +''' +CASES_PATCH_BODY = f'''\ +--- {WHERE} ++++ {WHERE} +@@ -25,6 +25,9 @@ + class Cases: + @example(n=0, label="whatever") + @given(st.integers(), st.text()) +{{0}} + def mth(self, n, label): + """Indented method with existing example decorator.""" + +''' + + +@pytest.mark.parametrize( + "tst, example, expected, body", + [ + pytest.param(*SIMPLE, SIMPLE_PATCH_BODY, id="simple"), + pytest.param(*CASES, CASES_PATCH_BODY, id="cases"), + ], +) +def test_make_full_patch(tst, example, expected, body): + when = datetime.now() + msg = "a message from the test" + expected = HEADER.format(when=when, msg=msg) + body.format(expected) + + triple = get_patch_for(tst, [example]) + got = make_patch([triple], when=when, msg=msg) + stripped = strip_trailing_whitespace(got) + + assert stripped.splitlines() == expected.splitlines() + + +@pytest.mark.parametrize("n", [0, 1, 2]) +def test_invalid_syntax_cases_dropped(n): + tst, example, expected = SIMPLE + example_ls = [example] * n + example_ls.insert(-1, "fn(\n x=<__main__.Cls object at 0x>,\n)") + + got = get_patch_for(tst, example_ls) + if n == 0: + assert got is None, "no valid examples, and hence no patch" + return + where, _, after = got + + assert Path(where) == WHERE + assert after.count(expected.lstrip("+")) == n + + +def test_irretrievable_callable(): + # Check that we return None instead of raising an exception + tst = deepcopy(fn) + tst.__module__ = "this.does.not.exist" + triple = get_patch_for(tst, [SIMPLE[1]]) + assert triple is None diff --git a/hypothesis-python/tox.ini b/hypothesis-python/tox.ini index aa368c35e8..3a194fee6d 100644 --- a/hypothesis-python/tox.ini +++ b/hypothesis-python/tox.ini @@ -171,7 +171,7 @@ ignore_errors = true commands = python -bb -X dev -m coverage run --rcfile=.coveragerc --source=hypothesis -m pytest -n0 --ff {posargs} \ tests/cover tests/conjecture tests/datetime tests/numpy tests/pandas tests/lark \ - tests/redis tests/dpcontracts tests/codemods tests/typing_extensions + tests/redis tests/dpcontracts tests/codemods tests/typing_extensions tests/patching python -m coverage report python scripts/validate_branch_check.py