diff --git a/hypothesis-python/.coveragerc b/hypothesis-python/.coveragerc index 4cd8a529e4..654a330c9c 100644 --- a/hypothesis-python/.coveragerc +++ b/hypothesis-python/.coveragerc @@ -12,6 +12,9 @@ omit = **/utils/terminal.py [report] +fail_under = 100 +show_missing = True +skip_covered = True exclude_lines = pragma: no cover raise NotImplementedError diff --git a/hypothesis-python/RELEASE.rst b/hypothesis-python/RELEASE.rst new file mode 100644 index 0000000000..57ddcd3387 --- /dev/null +++ b/hypothesis-python/RELEASE.rst @@ -0,0 +1,10 @@ +RELEASE_TYPE: minor + +Sick of adding :obj:`@example() `\ s by hand? +Our Pytest plugin now writes ``.patch`` files to insert them for you, making +`this workflow `__ +easier than ever before. + +Note that you'll need :pypi:`LibCST` (via :ref:`codemods`), and that +:obj:`@example().via() ` requires :pep:`614` +(Python 3.9 or later). diff --git a/hypothesis-python/docs/changes.rst b/hypothesis-python/docs/changes.rst index b7644de9f6..2d1c8107f5 100644 --- a/hypothesis-python/docs/changes.rst +++ b/hypothesis-python/docs/changes.rst @@ -9216,8 +9216,8 @@ This changes only the formatting of our docstrings and should have no user-visib what arguments are valid, and additional validation logic to raise a clear error early (instead of e.g. silently ignoring a bad argument). Categories may be specified as the Unicode 'general category' -(eg ``u'Nd'``), or as the 'major category' (eg ``[u'N', u'Lu']`` -is equivalent to ``[u'Nd', u'Nl', u'No', u'Lu']``). +(eg ``'Nd'``), or as the 'major category' (eg ``['N', 'Lu']`` +is equivalent to ``['Nd', 'Nl', 'No', 'Lu']``). In previous versions, general categories were supported and all other input was silently ignored. Now, major categories are supported in diff --git a/hypothesis-python/docs/settings.rst b/hypothesis-python/docs/settings.rst index 6e3719039c..f578d7f3a6 100644 --- a/hypothesis-python/docs/settings.rst +++ b/hypothesis-python/docs/settings.rst @@ -240,7 +240,7 @@ If this variable is not defined the Hypothesis defined defaults will be loaded. >>> settings.register_profile("ci", max_examples=1000) >>> settings.register_profile("dev", max_examples=10) >>> settings.register_profile("debug", max_examples=10, verbosity=Verbosity.verbose) - >>> settings.load_profile(os.getenv(u"HYPOTHESIS_PROFILE", "default")) + >>> settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "default")) If you are using the hypothesis pytest plugin and your profiles are registered by your conftest you can load one with the command line option ``--hypothesis-profile``. diff --git a/hypothesis-python/scripts/other-tests.sh b/hypothesis-python/scripts/other-tests.sh index 13c2dde821..4162aac845 100644 --- a/hypothesis-python/scripts/other-tests.sh +++ b/hypothesis-python/scripts/other-tests.sh @@ -46,7 +46,9 @@ pip uninstall -y lark if [ "$(python -c $'import platform, sys; print(sys.version_info.releaselevel == \'final\' and platform.python_implementation() not in ("PyPy", "GraalVM"))')" = "True" ] ; then pip install ".[codemods,cli]" $PYTEST tests/codemods/ - pip uninstall -y libcst click + pip install "$(grep 'black==' ../requirements/coverage.txt)" + $PYTEST tests/patching/ + pip uninstall -y libcst if [ "$(python -c 'import sys; print(sys.version_info[:2] == (3, 7))')" = "True" ] ; then # Per NEP-29, this is the last version to support Python 3.7 @@ -63,7 +65,6 @@ if [ "$(python -c $'import platform, sys; print(sys.version_info.releaselevel == $PYTEST tests/numpy esac - pip install "$(grep 'black==' ../requirements/coverage.txt)" $PYTEST tests/ghostwriter/ pip uninstall -y black numpy fi diff --git a/hypothesis-python/src/_hypothesis_pytestplugin.py b/hypothesis-python/src/_hypothesis_pytestplugin.py index afafd52ce4..2ff62a29df 100644 --- a/hypothesis-python/src/_hypothesis_pytestplugin.py +++ b/hypothesis-python/src/_hypothesis_pytestplugin.py @@ -20,6 +20,7 @@ """ import base64 +import json import sys from inspect import signature @@ -59,6 +60,7 @@ """ STATS_KEY = "_hypothesis_stats" +FAILING_EXAMPLES_KEY = "_hypothesis_failing_examples" class StoringReporter: @@ -298,7 +300,12 @@ def pytest_runtest_makereport(item, call): report.sections.append( ("Hypothesis", "\n".join(item.hypothesis_report_information)) ) - if hasattr(item, "hypothesis_statistics") and report.when == "teardown": + if report.when != "teardown": + return + + terminalreporter = item.config.pluginmanager.getplugin("terminalreporter") + + if hasattr(item, "hypothesis_statistics"): stats = item.hypothesis_statistics stats_base64 = base64.b64encode(stats.encode()).decode() @@ -314,10 +321,7 @@ def pytest_runtest_makereport(item, call): xml.add_global_property(name, stats_base64) # If there's a terminal report, include our summary stats for each test - terminalreporter = item.config.pluginmanager.getplugin("terminalreporter") if terminalreporter is not None: - # ideally, we would store this on terminalreporter.config.stash, but - # pytest-xdist doesn't copy that back to the controller report.__dict__[STATS_KEY] = stats # If there's an HTML report, include our summary stats for each test @@ -327,14 +331,49 @@ def pytest_runtest_makereport(item, call): pytest_html.extras.text(stats, name="Hypothesis stats") ] + # This doesn't intrinsically have anything to do with the terminalreporter; + # we're just cargo-culting a way to get strings back to a single function + # even if the test were distributed with pytest-xdist. + failing_examples = getattr(item, FAILING_EXAMPLES_KEY, None) + if failing_examples and terminalreporter is not None: + try: + from hypothesis.extra.patching import get_patch_for + except ImportError: + return + # We'll save this as a triple of [filename, hunk_before, hunk_after]. + triple = get_patch_for(item.obj, failing_examples) + if triple is not None: + report.__dict__[FAILING_EXAMPLES_KEY] = json.dumps(triple) + def pytest_terminal_summary(terminalreporter): - if terminalreporter.config.getoption(PRINT_STATISTICS_OPTION): + failing_examples = [] + print_stats = terminalreporter.config.getoption(PRINT_STATISTICS_OPTION) + if print_stats: terminalreporter.section("Hypothesis Statistics") - for reports in terminalreporter.stats.values(): - for report in reports: - stats = report.__dict__.get(STATS_KEY) - if stats: - terminalreporter.write_line(stats + "\n\n") + for reports in terminalreporter.stats.values(): + for report in reports: + stats = report.__dict__.get(STATS_KEY) + if stats and print_stats: + terminalreporter.write_line(stats + "\n\n") + fex = report.__dict__.get(FAILING_EXAMPLES_KEY) + if fex: + failing_examples.append(json.loads(fex)) + + if failing_examples: + # This must have been imported already to write the failing examples + from hypothesis.extra.patching import gc_patches, make_patch, save_patch + + patch = make_patch(failing_examples) + try: + gc_patches() + fname = save_patch(patch) + except Exception: + # fail gracefully if we hit any filesystem or permissions problems + return + terminalreporter.section("Hypothesis") + terminalreporter.write_line( + f"`git apply {fname}` to add failing examples to your code." + ) def pytest_collection_modifyitems(items): if "hypothesis" not in sys.modules: diff --git a/hypothesis-python/src/hypothesis/core.py b/hypothesis-python/src/hypothesis/core.py index 84409b5471..07cf9f897b 100644 --- a/hypothesis-python/src/hypothesis/core.py +++ b/hypothesis-python/src/hypothesis/core.py @@ -79,6 +79,7 @@ from hypothesis.internal.conjecture.shrinker import sort_key from hypothesis.internal.entropy import deterministic_PRNG from hypothesis.internal.escalation import ( + current_pytest_item, escalate_hypothesis_internal_error, format_exception, get_interesting_origin, @@ -787,10 +788,12 @@ def run(data): kwargs, force_split=True, arg_slices=argslices, + leading_comment=( + "# " + context.data.slice_comments[(0, 0)] + if (0, 0) in context.data.slice_comments + else None + ), ) - if (0, 0) in context.data.slice_comments: - printer.break_() - printer.text("# " + context.data.slice_comments[(0, 0)]) report(printer.getvalue()) return test(*args, **kwargs) @@ -1028,10 +1031,15 @@ def add_note(exc, note): def _raise_to_user(errors_to_report, settings, target_lines, trailer=""): """Helper function for attaching notes and grouping multiple errors.""" - if settings.verbosity >= Verbosity.normal: - for fragments, err in errors_to_report: - for note in fragments: - add_note(err, note) + failing_prefix = "Falsifying example: " + ls = [] + for fragments, err in errors_to_report: + for note in fragments: + add_note(err, note) + if note.startswith(failing_prefix): + ls.append(note[len(failing_prefix) :]) + if current_pytest_item.value: + current_pytest_item.value._hypothesis_failing_examples = ls if len(errors_to_report) == 1: _, the_error_hypothesis_found = errors_to_report[0] diff --git a/hypothesis-python/src/hypothesis/extra/_patching.py b/hypothesis-python/src/hypothesis/extra/_patching.py new file mode 100644 index 0000000000..f2dc74705c --- /dev/null +++ b/hypothesis-python/src/hypothesis/extra/_patching.py @@ -0,0 +1,186 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +""" +Write patches which add @example() decorators for discovered test cases. + +Requires `hypothesis[codemods,ghostwriter]` installed, i.e. black and libcst. + +This module is used by Hypothesis' builtin pytest plugin for failing examples +discovered during testing, and by HypoFuzz for _covering_ examples discovered +during fuzzing. +""" + +import difflib +import hashlib +import inspect +import re +import sys +from contextlib import suppress +from datetime import date, datetime, timedelta, timezone +from pathlib import Path + +import libcst as cst +from libcst.codemod import CodemodContext, VisitorBasedCodemodCommand + +from hypothesis.configuration import storage_directory +from hypothesis.extra.codemods import _native_parser +from hypothesis.version import __version__ + +try: + import black +except ImportError: + black = None # type: ignore + +HEADER = f"""\ +From HEAD Mon Sep 17 00:00:00 2001 +From: Hypothesis {__version__} +Date: {{when:%a, %d %b %Y %H:%M:%S}} +Subject: [PATCH] {{msg}} + +--- +""" +_space_only_re = re.compile("^ +$", re.MULTILINE) +_leading_space_re = re.compile("(^[ ]*)(?:[^ \n])", re.MULTILINE) + + +def dedent(text): + # Simplified textwrap.dedent, for valid Python source code only + text = _space_only_re.sub("", text) + prefix = min(_leading_space_re.findall(text), key=len) + return re.sub(r"(?m)^" + prefix, "", text), prefix + + +def indent(text: str, prefix: str) -> str: + return "".join(prefix + line for line in text.splitlines(keepends=True)) + + +class AddExamplesCodemod(VisitorBasedCodemodCommand): + DESCRIPTION = "Add explicit examples to failing tests." + + @classmethod + def refactor(cls, code: str, fn_examples: dict) -> str: + """Add @example() decorator(s) for failing test(s). + + `code` is the source code of the module where the test functions are defined. + `fn_examples` is a dict of function name to list-of-failing-examples. + """ + dedented, prefix = dedent(code) + with _native_parser(): + mod = cst.parse_module(dedented) + modded = cls(CodemodContext(), fn_examples, prefix).transform_module(mod).code + return indent(modded, prefix=prefix) + + def __init__(self, context, fn_examples, prefix="", via="discovered failure"): + assert fn_examples, "This codemod does nothing without fn_examples." + super().__init__(context) + + # Codemod the failing examples to Call nodes usable as decorators + self.via = via + self.line_length = 88 - len(prefix) # to match Black's default formatting + self.fn_examples = { + k: tuple(self.__call_node_to_example_dec(ex) for ex in nodes) + for k, nodes in fn_examples.items() + } + + def __call_node_to_example_dec(self, node): + node = node.with_changes( + func=cst.Name("example"), + args=[a.with_changes(comma=cst.MaybeSentinel.DEFAULT) for a in node.args] + if black + else node.args, + ) + # Note: calling a method on a decorator requires PEP-614, i.e. Python 3.9+, + # but plumbing two cases through doesn't seem worth the trouble :-/ + via = cst.Call( + func=cst.Attribute(node, cst.Name("via")), + args=[cst.Arg(cst.SimpleString(repr(self.via)))], + ) + if black: # pragma: no branch + pretty = black.format_str( + cst.Module([]).code_for_node(via), + mode=black.FileMode(line_length=self.line_length), + ) + via = cst.parse_expression(pretty.strip()) + return cst.Decorator(via) + + def leave_FunctionDef(self, _, updated_node): + return updated_node.with_changes( + # TODO: improve logic for where in the list to insert this decorator + decorators=updated_node.decorators + + self.fn_examples.get(updated_node.name.value, ()) + ) + + +def get_patch_for(func, failing_examples): + # Skip this if we're unable to find the location or source of this function. + try: + fname = Path(sys.modules[func.__module__].__file__).relative_to(Path.cwd()) + before = inspect.getsource(func) + except Exception: + return None + + # The printed examples might include object reprs which are invalid syntax, + # so we parse here and skip over those. If _none_ are valid, there's no patch. + call_nodes = [] + for ex in failing_examples: + with suppress(Exception): + node = cst.parse_expression(ex) + assert isinstance(node, cst.Call), node + call_nodes.append(node) + if not call_nodes: + return None + + # Do the codemod and return a triple containing location and replacement info. + after = AddExamplesCodemod.refactor( + before, + fn_examples={func.__name__: call_nodes}, + ) + return (str(fname), before, after) + + +def make_patch(triples, *, msg="Hypothesis: add failing examples", when=None): + """Create a patch for (fname, before, after) triples.""" + assert triples, "attempted to create empty patch" + when = when or datetime.now(tz=timezone.utc) + + by_fname = {} + for fname, before, after in triples: + by_fname.setdefault(Path(fname), []).append((before, after)) + + diffs = [HEADER.format(msg=msg, when=when)] + for fname, changes in sorted(by_fname.items()): + source_before = source_after = fname.read_text(encoding="utf-8") + for before, after in changes: + source_after = source_after.replace(before, after, 1) + ud = difflib.unified_diff( + source_before.splitlines(keepends=True), + source_after.splitlines(keepends=True), + fromfile=str(fname), + tofile=str(fname), + ) + diffs.append("".join(ud)) + return "".join(diffs) + + +def save_patch(patch: str) -> Path: # pragma: no cover + today = date.today().isoformat() + hash = hashlib.sha1(patch.encode()).hexdigest()[:8] + fname = Path(storage_directory("patches", f"{today}--{hash}.patch")) + fname.parent.mkdir(parents=True, exist_ok=True) + fname.write_text(patch, encoding="utf-8") + return fname.relative_to(Path.cwd()) + + +def gc_patches(): # pragma: no cover + cutoff = date.today() - timedelta(days=7) + for fname in Path(storage_directory("patches")).glob("????-??-??--????????.patch"): + if date.fromisoformat(fname.stem.split("--")[0]) < cutoff: + fname.unlink() diff --git a/hypothesis-python/src/hypothesis/extra/codemods.py b/hypothesis-python/src/hypothesis/extra/codemods.py index 1de665328c..b5dcae72bb 100644 --- a/hypothesis-python/src/hypothesis/extra/codemods.py +++ b/hypothesis-python/src/hypothesis/extra/codemods.py @@ -47,6 +47,8 @@ import functools import importlib +import os +from contextlib import contextmanager from inspect import Parameter, signature from typing import List @@ -55,6 +57,20 @@ from libcst.codemod import VisitorBasedCodemodCommand +@contextmanager +def _native_parser(): + # Only the native parser supports Python 3.9 and later, but for now it's + # only active if you set an environment variable. Very well then: + var = os.environ.get("LIBCST_PARSER_TYPE") + try: + os.environ["LIBCST_PARSER_TYPE"] = "native" + yield + finally: + os.environ.pop("LIBCST_PARSER_TYPE") + if var is not None: # pragma: no cover + os.environ["LIBCST_PARSER_TYPE"] = var + + def refactor(code: str) -> str: """Update a source code string from deprecated to modern Hypothesis APIs. @@ -64,7 +80,8 @@ def refactor(code: str) -> str: We recommend using the CLI, but if you want a Python function here it is. """ context = cst.codemod.CodemodContext() - mod = cst.parse_module(code) + with _native_parser(): + mod = cst.parse_module(code) transforms: List[VisitorBasedCodemodCommand] = [ HypothesisFixPositionalKeywonlyArgs(context), HypothesisFixComplexMinMagnitude(context), diff --git a/hypothesis-python/src/hypothesis/internal/charmap.py b/hypothesis-python/src/hypothesis/internal/charmap.py index 148f438223..80e94e2846 100644 --- a/hypothesis-python/src/hypothesis/internal/charmap.py +++ b/hypothesis-python/src/hypothesis/internal/charmap.py @@ -336,7 +336,7 @@ def query( >>> query(min_codepoint=0, max_codepoint=128, include_categories=['Lu']) ((65, 90),) >>> query(min_codepoint=0, max_codepoint=128, include_categories=['Lu'], - ... include_characters=u'☃') + ... include_characters='☃') ((65, 90), (9731, 9731)) """ if min_codepoint is None: diff --git a/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py b/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py index 071e993c8a..4503804afd 100644 --- a/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py +++ b/hypothesis-python/src/hypothesis/internal/conjecture/shrinker.py @@ -546,10 +546,11 @@ def explain(self): ) # Turns out this was a variable-length part, so grab the infix... - if ( - result.status == Status.OVERRUN - or len(buf_attempt_fixed) != len(result.buffer) - or not result.buffer.endswith(buffer[end:]) + if result.status == Status.OVERRUN: + continue # pragma: no cover + if not ( + len(buf_attempt_fixed) == len(result.buffer) + and result.buffer.endswith(buffer[end:]) ): for ex, res in zip(shrink_target.examples, result.examples): assert ex.start == res.start @@ -612,8 +613,6 @@ def explain(self): # This *can't* be a shrink because none of the components were. assert shrink_target is self.shrink_target if result.status == Status.VALID: - # TODO: cover this branch. - # I might need to save or retrieve passing chunks too??? self.shrink_target.slice_comments[ (0, 0) ] = "The test sometimes passed when commented parts were varied together." diff --git a/hypothesis-python/src/hypothesis/strategies/_internal/numbers.py b/hypothesis-python/src/hypothesis/strategies/_internal/numbers.py index bd506e9d15..825e91de2e 100644 --- a/hypothesis-python/src/hypothesis/strategies/_internal/numbers.py +++ b/hypothesis-python/src/hypothesis/strategies/_internal/numbers.py @@ -438,7 +438,7 @@ def floats( if math.copysign(1.0, -0.0) == 1.0: # pragma: no cover raise FloatingPointError( - "You Python install can't represent -0.0, which is required by the " + "Your Python install can't represent -0.0, which is required by the " "IEEE-754 floating-point specification. This is probably because it was " "compiled with an unsafe option like -ffast-math; for a more detailed " "explanation see https://simonbyrne.github.io/notes/fastmath/" diff --git a/hypothesis-python/src/hypothesis/vendor/pretty.py b/hypothesis-python/src/hypothesis/vendor/pretty.py index 06928e8a43..3b8a5847c6 100644 --- a/hypothesis-python/src/hypothesis/vendor/pretty.py +++ b/hypothesis-python/src/hypothesis/vendor/pretty.py @@ -352,7 +352,16 @@ def getvalue(self): self.flush() return self.output.getvalue() - def repr_call(self, func_name, args, kwargs, *, force_split=None, arg_slices=None): + def repr_call( + self, + func_name, + args, + kwargs, + *, + force_split=None, + arg_slices=None, + leading_comment=None, + ): """Helper function to represent a function call. - func_name, args, and kwargs should all be pretty obvious. @@ -372,7 +381,7 @@ def repr_call(self, func_name, args, kwargs, *, force_split=None, arg_slices=Non if v in self.slice_comments } - if any(k in comments for k, _ in all_args): + if leading_comment or any(k in comments for k, _ in all_args): # We have to split one arg per line in order to leave comments on them. force_split = True if force_split is None: @@ -388,6 +397,9 @@ def repr_call(self, func_name, args, kwargs, *, force_split=None, arg_slices=Non with self.group(indent=4, open="(", close=""): for i, (k, v) in enumerate(all_args): if force_split: + if i == 0 and leading_comment: + self.break_() + self.text(leading_comment) self.break_() else: self.breakable(" " if i else "") diff --git a/hypothesis-python/tests/conjecture/test_inquisitor.py b/hypothesis-python/tests/conjecture/test_inquisitor.py index 6326fa7e04..177251068c 100644 --- a/hypothesis-python/tests/conjecture/test_inquisitor.py +++ b/hypothesis-python/tests/conjecture/test_inquisitor.py @@ -29,13 +29,13 @@ def _new(): @fails_with_output( """ Falsifying example: test_inquisitor_comments_basic_fail_if_either( + # The test always failed when commented parts were varied together. a=False, # or any other generated value b=True, c=[], # or any other generated value d=True, e=False, # or any other generated value ) -# The test always failed when commented parts were varied together. """ ) @given(st.booleans(), st.booleans(), st.lists(st.none()), st.booleans(), st.booleans()) @@ -46,11 +46,11 @@ def test_inquisitor_comments_basic_fail_if_either(a, b, c, d, e): @fails_with_output( """ Falsifying example: test_inquisitor_comments_basic_fail_if_not_all( + # The test sometimes passed when commented parts were varied together. a='', # or any other generated value b='', # or any other generated value c='', # or any other generated value ) -# The test sometimes passed when commented parts were varied together. """ ) @given(st.text(), st.text(), st.text()) diff --git a/hypothesis-python/tests/patching/__init__.py b/hypothesis-python/tests/patching/__init__.py new file mode 100644 index 0000000000..fcb1ac6538 --- /dev/null +++ b/hypothesis-python/tests/patching/__init__.py @@ -0,0 +1,9 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. diff --git a/hypothesis-python/tests/patching/callables.py b/hypothesis-python/tests/patching/callables.py new file mode 100644 index 0000000000..1686fe59c5 --- /dev/null +++ b/hypothesis-python/tests/patching/callables.py @@ -0,0 +1,32 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +"""A stable file for which we can write patches. Don't move stuff around!""" + +from pathlib import Path + +from hypothesis import example, given, strategies as st + +WHERE = Path(__file__).relative_to(Path.cwd()) + + +@given(st.integers()) +def fn(x): + """A trivial test function.""" + + +class Cases: + @example(n=0, label="whatever") + @given(st.integers(), st.text()) + def mth(self, n, label): + """Indented method with existing example decorator.""" + + +# TODO: test function for insertion-order logic, once I get that set up. diff --git a/hypothesis-python/tests/patching/test_patching.py b/hypothesis-python/tests/patching/test_patching.py new file mode 100644 index 0000000000..c0b7910c30 --- /dev/null +++ b/hypothesis-python/tests/patching/test_patching.py @@ -0,0 +1,123 @@ +# This file is part of Hypothesis, which may be found at +# https://github.com/HypothesisWorks/hypothesis/ +# +# Copyright the Hypothesis Authors. +# Individual contributors are listed in AUTHORS.rst and the git log. +# +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at https://mozilla.org/MPL/2.0/. + +import re +from copy import deepcopy +from datetime import datetime +from pathlib import Path + +import pytest + +from hypothesis.extra._patching import HEADER, get_patch_for, indent, make_patch + +from .callables import WHERE, Cases, fn + +SIMPLE = ( + fn, + "fn(\n x=1,\n)", + indent('@example(x=1).via("discovered failure")', prefix="+"), +) +CASES = ( + Cases.mth, + 'mth(\n n=100,\n label="a long label which forces a newline",\n)', + indent( + '@example(n=100, label="a long label which forces a newline")' + + '.via(\n "discovered failure"\n)', + prefix="+ ", + ), +) + + +def strip_trailing_whitespace(s): + """Patches have whitespace-only lines; strip that out.""" + return re.sub(r" +$", "", s, flags=re.MULTILINE) + + +@pytest.mark.parametrize( + "tst, example, expected", + [ + pytest.param(*SIMPLE, id="simple"), + pytest.param(*CASES, id="cases"), + ], +) +def test_adds_simple_patch(tst, example, expected): + where, before, after = get_patch_for(tst, [example]) + assert Path(where) == WHERE + added = set(after.splitlines()) - set(before.splitlines()) + assert added == {line.lstrip("+") for line in expected.splitlines()} + + +SIMPLE_PATCH_BODY = f'''\ +--- {WHERE} ++++ {WHERE} +@@ -18,6 +18,7 @@ + + + @given(st.integers()) +{{0}} + def fn(x): + """A trivial test function.""" + +''' +CASES_PATCH_BODY = f'''\ +--- {WHERE} ++++ {WHERE} +@@ -25,6 +25,9 @@ + class Cases: + @example(n=0, label="whatever") + @given(st.integers(), st.text()) +{{0}} + def mth(self, n, label): + """Indented method with existing example decorator.""" + +''' + + +@pytest.mark.parametrize( + "tst, example, expected, body", + [ + pytest.param(*SIMPLE, SIMPLE_PATCH_BODY, id="simple"), + pytest.param(*CASES, CASES_PATCH_BODY, id="cases"), + ], +) +def test_make_full_patch(tst, example, expected, body): + when = datetime.now() + msg = "a message from the test" + expected = HEADER.format(when=when, msg=msg) + body.format(expected) + + triple = get_patch_for(tst, [example]) + got = make_patch([triple], when=when, msg=msg) + stripped = strip_trailing_whitespace(got) + + assert stripped.splitlines() == expected.splitlines() + + +@pytest.mark.parametrize("n", [0, 1, 2]) +def test_invalid_syntax_cases_dropped(n): + tst, example, expected = SIMPLE + example_ls = [example] * n + example_ls.insert(-1, "fn(\n x=<__main__.Cls object at 0x>,\n)") + + got = get_patch_for(tst, example_ls) + if n == 0: + assert got is None, "no valid examples, and hence no patch" + return + where, _, after = got + + assert Path(where) == WHERE + assert after.count(expected.lstrip("+")) == n + + +def test_irretrievable_callable(): + # Check that we return None instead of raising an exception + tst = deepcopy(fn) + tst.__module__ = "this.does.not.exist" + triple = get_patch_for(tst, [SIMPLE[1]]) + assert triple is None diff --git a/hypothesis-python/tox.ini b/hypothesis-python/tox.ini index d179b30cee..3a194fee6d 100644 --- a/hypothesis-python/tox.ini +++ b/hypothesis-python/tox.ini @@ -157,7 +157,6 @@ deps = allowlist_externals = rm setenv= - PYTHONDEVMODE=1 HYPOTHESIS_INTERNAL_COVERAGE=true commands_pre = rm -f branch-check @@ -171,8 +170,9 @@ commands_pre = ignore_errors = true commands = python -bb -X dev -m coverage run --rcfile=.coveragerc --source=hypothesis -m pytest -n0 --ff {posargs} \ - tests/cover tests/conjecture tests/datetime tests/numpy tests/pandas tests/lark tests/redis tests/dpcontracts tests/codemods tests/typing_extensions - python -m coverage report -m --fail-under=100 --show-missing --skip-covered + tests/cover tests/conjecture tests/datetime tests/numpy tests/pandas tests/lark \ + tests/redis tests/dpcontracts tests/codemods tests/typing_extensions tests/patching + python -m coverage report python scripts/validate_branch_check.py @@ -180,14 +180,13 @@ commands = deps = -r../requirements/coverage.txt setenv= - PYTHONDEVMODE=1 HYPOTHESIS_INTERNAL_COVERAGE=true commands_pre = python -m coverage erase ignore_errors = true commands = python -bb -X dev -m coverage run --rcfile=.coveragerc --source=hypothesis.internal.conjecture -m pytest -n0 --strict-markers tests/conjecture - python -m coverage report -m --fail-under=100 --show-missing --skip-covered + python -m coverage report [testenv:examples3] diff --git a/requirements/coverage.txt b/requirements/coverage.txt index 8ce2358caa..62e078c4b8 100644 --- a/requirements/coverage.txt +++ b/requirements/coverage.txt @@ -14,7 +14,7 @@ click==8.1.3 # via # -r requirements/coverage.in # black -coverage==7.2.3 +coverage==7.2.4 # via -r requirements/coverage.in dpcontracts==0.6.0 # via -r requirements/coverage.in @@ -24,7 +24,7 @@ exceptiongroup==1.1.1 ; python_version < "3.11" # pytest execnet==1.9.0 # via pytest-xdist -fakeredis==2.11.0 +fakeredis==2.11.2 # via -r requirements/coverage.in iniconfig==2.0.0 # via pytest @@ -44,13 +44,13 @@ packaging==23.1 # via # black # pytest -pandas==2.0.0 +pandas==2.0.1 # via -r requirements/coverage.in pathspec==0.11.1 # via black pexpect==4.8.0 # via -r requirements/test.in -platformdirs==3.2.0 +platformdirs==3.5.0 # via black pluggy==1.0.0 # via pytest diff --git a/requirements/tools.txt b/requirements/tools.txt index 421f1ead73..ac851e345d 100644 --- a/requirements/tools.txt +++ b/requirements/tools.txt @@ -52,7 +52,7 @@ colorama==0.4.6 # via tox com2ann==0.3.0 # via shed -coverage==7.2.3 +coverage==7.2.4 # via -r requirements/tools.in cryptography==40.0.2 # via @@ -140,7 +140,7 @@ importlib-metadata==6.6.0 # twine iniconfig==2.0.0 # via pytest -ipython==8.12.0 +ipython==8.13.1 # via -r requirements/tools.in isort==5.12.0 # via shed @@ -205,7 +205,7 @@ pip-tools==6.13.0 # via -r requirements/tools.in pkginfo==1.9.6 # via twine -platformdirs==3.2.0 +platformdirs==3.5.0 # via # black # tox @@ -240,13 +240,13 @@ pyproject-api==1.5.1 # via tox pyproject-hooks==1.0.0 # via build -pyright==1.1.304 +pyright==1.1.305 # via -r requirements/tools.in pytest==7.3.1 # via -r requirements/tools.in python-dateutil==2.8.2 # via -r requirements/tools.in -pyupgrade==3.3.1 +pyupgrade==3.3.2 # via shed pyyaml==6.0 # via @@ -254,7 +254,7 @@ pyyaml==6.0 # libcst readme-renderer==37.3 # via twine -requests==2.28.2 +requests==2.29.0 # via # -r requirements/tools.in # requests-toolbelt @@ -266,7 +266,7 @@ restructuredtext-lint==1.4.0 # via -r requirements/tools.in rfc3986==2.0.0 # via twine -rich==13.3.4 +rich==13.3.5 # via # bandit # twine @@ -289,7 +289,7 @@ sortedcontainers==2.4.0 # via hypothesis (hypothesis-python/setup.py) soupsieve==2.4.1 # via beautifulsoup4 -sphinx==6.1.3 +sphinx==6.2.1 # via # -r requirements/tools.in # sphinx-codeautolink @@ -338,7 +338,7 @@ tomli==2.0.1 # pyproject-hooks # pytest # tox -tox==4.4.12 +tox==4.5.1 # via -r requirements/tools.in traitlets==5.9.0 # via @@ -370,7 +370,7 @@ urllib3==1.26.15 # via # requests # twine -virtualenv==20.22.0 +virtualenv==20.23.0 # via tox wcwidth==0.2.6 # via prompt-toolkit @@ -382,9 +382,9 @@ zipp==3.15.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -pip==23.1.1 +pip==23.1.2 # via pip-tools -setuptools==67.7.1 +setuptools==67.7.2 # via # nodeenv # pip-tools