Merge pull request #3691 from Zac-HD/enable-defaultencodingwarning

Enable `PYTHONWARNDEFAULTENCODING=1` in CI

.github/workflows/main.yml

@@ -61,7 +61,7 @@ jobs:
           - check-py39-nose
           - check-py39-pytest46
           - check-py39-pytest54
-          - check-pytest62
+          # - check-pytest62
           - check-django42
           - check-django41
           - check-django32

hypothesis-python/RELEASE.rst

@@ -0,0 +1,4 @@
+RELEASE_TYPE: patch
+
+This patch updates some internal code for selftests.
+There is no user-visible change.

hypothesis-python/docs/conf.py

@@ -9,13 +9,14 @@
 # obtain one at https://mozilla.org/MPL/2.0/.
 
 import datetime
-import os
 import sys
 import types
+from pathlib import Path
 
 import sphinx_rtd_theme
 
-sys.path.append(os.path.join(os.path.dirname(__file__), "..", "src"))
+root = Path(__file__).parent.parent
+sys.path.append(str(root / "src"))
 
 
 autodoc_member_order = "bysource"
@@ -44,16 +45,14 @@
 copyright = f"2013-{datetime.datetime.utcnow().year}, {author}"
 
 _d = {}
-with open(
-    os.path.join(os.path.dirname(__file__), "..", "src", "hypothesis", "version.py")
-) as f:
-    exec(f.read(), _d)
-    version = _d["__version__"]
-    release = _d["__version__"]
+_version_file = root.joinpath("src", "hypothesis", "version.py")
+exec(_version_file.read_text(encoding="utf-8"), _d)
+version = _d["__version__"]
+release = _d["__version__"]
 
 
 def setup(app):
-    if os.path.isfile(os.path.join(os.path.dirname(__file__), "..", "RELEASE.rst")):
+    if root.joinpath("RELEASE.rst").is_file():
         app.tags.add("has_release_file")
 
     # patch in mock array_api namespace so we can autodoc it

hypothesis-python/scripts/basic-test.sh

@@ -54,7 +54,7 @@ if [ "$(python -c $'import platform, sys; print(sys.version_info.releaselevel ==
     pip install "$(grep 'numpy==' ../requirements/coverage.txt)"
   fi
 
-  pip install "$(grep 'black==' ../requirements/coverage.txt)"
+  pip install "$(grep -E 'black(==| @)' ../requirements/coverage.txt)"
   $PYTEST tests/ghostwriter/
   pip uninstall -y black numpy
 fi

hypothesis-python/scripts/other-tests.sh

@@ -43,7 +43,7 @@ pip uninstall -y lark
 if [ "$(python -c $'import platform, sys; print(sys.version_info.releaselevel == \'final\' and platform.python_implementation() not in ("PyPy", "GraalVM"))')" = "True" ] ; then
   pip install ".[codemods,cli]"
   $PYTEST tests/codemods/
-  pip install "$(grep 'black==' ../requirements/coverage.txt)"
+  pip install "$(grep -E 'black(==| @)' ../requirements/coverage.txt)"
   if [ "$(python -c 'import sys; print(sys.version_info[:2] >= (3, 9))')" = "True" ] ; then
     $PYTEST tests/patching/
   fi

hypothesis-python/scripts/validate_branch_check.py

@@ -13,7 +13,7 @@
 from collections import defaultdict
 
 if __name__ == "__main__":
-    with open("branch-check") as i:
+    with open("branch-check", encoding="utf-8") as i:
         data = [json.loads(l) for l in i]
 
     checks = defaultdict(set)

hypothesis-python/setup.py

@@ -8,9 +8,9 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
-import os
 import sys
 import warnings
+from pathlib import Path
 
 import setuptools
 
@@ -25,11 +25,10 @@
 
 
 def local_file(name):
-    return os.path.relpath(os.path.join(os.path.dirname(__file__), name))
+    return Path(__file__).parent.joinpath(name).relative_to(Path.cwd())
 
 
-SOURCE = local_file("src")
-README = local_file("README.rst")
+SOURCE = str(local_file("src"))
 
 setuptools_version = tuple(map(int, setuptools.__version__.split(".")[:1]))
 
@@ -44,10 +43,7 @@ def local_file(name):
 
 # Assignment to placate pyflakes. The actual version is from the exec that follows.
 __version__ = None
-
-with open(local_file("src/hypothesis/version.py")) as o:
-    exec(o.read())
-
+exec(local_file("src/hypothesis/version.py").read_text(encoding="utf-8"))
 assert __version__ is not None
 
 
@@ -131,7 +127,7 @@ def local_file(name):
         "pytest11": ["hypothesispytest = _hypothesis_pytestplugin"],
         "console_scripts": ["hypothesis = hypothesis.extra.cli:main"],
     },
-    long_description=open(README).read(),
+    long_description=local_file("README.rst").read_text(encoding="utf-8"),
     long_description_content_type="text/x-rst",
     keywords="python testing fuzzing property-based-testing",
 )

hypothesis-python/src/hypothesis/extra/cli.py

@@ -161,14 +161,14 @@ def describe_close_matches(
 
     def _refactor(func, fname):
         try:
-            with open(fname) as f:
+            with open(fname, encoding="utf-8") as f:
                 oldcode = f.read()
         except (OSError, UnicodeError) as err:
             # Permissions or encoding issue, or file deleted, etc.
             return f"skipping {fname!r} due to {err}"
         newcode = func(oldcode)
         if newcode != oldcode:
-            with open(fname, mode="w") as f:
+            with open(fname, mode="w", encoding="utf-8") as f:
                 f.write(newcode)
 
     @main.command()  # type: ignore  # Click adds the .command attribute

hypothesis-python/src/hypothesis/internal/conjecture/shrinking/dfas.py

@@ -8,26 +8,29 @@
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
 
+"""
+This is a module for learning new DFAs that help normalize test
+functions. That is, given a test function that sometimes shrinks
+to one thing and sometimes another, this module is designed to
+help learn new DFA-based shrink passes that will cause it to
+always shrink to the same thing.
+"""
+
 import hashlib
 import math
 from itertools import islice
+from pathlib import Path
 
 from hypothesis import HealthCheck, settings
 from hypothesis.errors import HypothesisException
 from hypothesis.internal.conjecture.data import ConjectureResult, Status
 from hypothesis.internal.conjecture.dfa.lstar import LStar
 from hypothesis.internal.conjecture.shrinking.learned_dfas import (
     SHRINKING_DFAS,
-    __file__ as learned_dfa_file,
+    __file__ as _learned_dfa_file,
 )
 
-"""
-This is a module for learning new DFAs that help normalize test
-functions. That is, given a test function that sometimes shrinks
-to one thing and sometimes another, this module is designed to
-help learn new DFA-based shrink passes that will cause it to
-always shrink to the same thing.
-"""
+learned_dfa_file = Path(_learned_dfa_file)
 
 
 class FailedToNormalise(HypothesisException):
@@ -38,8 +41,7 @@ def update_learned_dfas():
     """Write any modifications to the SHRINKING_DFAS dictionary
     back to the learned DFAs file."""
 
-    with open(learned_dfa_file) as i:
-        source = i.read()
+    source = learned_dfa_file.read_text(encoding="utf-8")
 
     lines = source.splitlines()
 
@@ -60,8 +62,7 @@ def update_learned_dfas():
     new_source = "\n".join(lines) + "\n"
 
     if new_source != source:
-        with open(learned_dfa_file, "w") as o:
-            o.write(new_source)
+        learned_dfa_file.write_text(new_source, encoding="utf-8")
 
 
 def learn_a_new_dfa(runner, u, v, predicate):

hypothesis-python/src/hypothesis/internal/coverage.py

@@ -61,7 +61,7 @@ def record_branch(name, value):
         if key in written:
             return
         written.add(key)
-        with open("branch-check", "a") as log:
+        with open("branch-check", mode="a", encoding="utf-8") as log:
             log.write(json.dumps({"name": name, "value": value}) + "\n")
 
     description_stack = []

hypothesis-python/tests/conjecture/test_shrinking_dfas.py

@@ -36,41 +36,35 @@ def preserving_dfas():
         dfas.SHRINKING_DFAS.update(original)
         dfas.update_learned_dfas()
     assert TEST_DFA_NAME not in dfas.SHRINKING_DFAS
-    with open(dfas.learned_dfa_file) as i:
-        assert TEST_DFA_NAME not in i.read()
+    assert TEST_DFA_NAME not in dfas.learned_dfa_file.read_text(encoding="utf-8")
 
 
 def test_updating_the_file_makes_no_changes_normally():
-    with open(dfas.learned_dfa_file) as i:
-        source1 = i.read()
+    source1 = dfas.learned_dfa_file.read_text(encoding="utf-8")
 
     dfas.update_learned_dfas()
 
-    with open(dfas.learned_dfa_file) as i:
-        source2 = i.read()
+    source2 = dfas.learned_dfa_file.read_text(encoding="utf-8")
 
     assert source1 == source2
 
 
 def test_updating_the_file_include_new_shrinkers():
     with preserving_dfas():
-        with open(dfas.learned_dfa_file) as i:
-            source1 = i.read()
+        source1 = dfas.learned_dfa_file.read_text(encoding="utf-8")
 
         dfas.SHRINKING_DFAS[TEST_DFA_NAME] = "hello"
 
         dfas.update_learned_dfas()
 
-        with open(dfas.learned_dfa_file) as i:
-            source2 = i.read()
+        source2 = dfas.learned_dfa_file.read_text(encoding="utf-8")
 
         assert source1 != source2
         assert repr(TEST_DFA_NAME) in source2
 
     assert TEST_DFA_NAME not in dfas.SHRINKING_DFAS
 
-    with open(dfas.learned_dfa_file) as i:
-        assert "test name" not in i.read()
+    assert "test name" not in dfas.learned_dfa_file.read_text(encoding="utf-8")
 
 
 def called_by_shrinker():

hypothesis-python/tests/ghostwriter/test_expected_output.py

@@ -45,8 +45,8 @@ def update_recorded_outputs(request):
 def get_recorded(name, actual=""):
     file_ = pathlib.Path(__file__).parent / "recorded" / f"{name}.txt"
     if actual:
-        file_.write_text(actual)
-    return file_.read_text()
+        file_.write_text(actual, encoding="utf-8")
+    return file_.read_text(encoding="utf-8")
 
 
 def timsort(seq: Sequence[int]) -> Sequence[int]:

hypothesis-python/tests/ghostwriter/test_ghostwriter.py

@@ -463,7 +463,8 @@ def temp_script_file():
             def say_hello():
                 print("Hello world!")
             """
-        )
+        ),
+        encoding="utf-8",
     )
     yield p
     p.unlink()
@@ -483,7 +484,8 @@ def temp_script_file_with_py_function():
             def py():
                 print('A function named "py" has been called')
             """
-        )
+        ),
+        encoding="utf-8",
     )
     yield p
     p.unlink()

hypothesis-python/tests/numpy/__init__.py

@@ -7,3 +7,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public License,
 # v. 2.0. If a copy of the MPL was not distributed with this file, You can
 # obtain one at https://mozilla.org/MPL/2.0/.
+
+try:
+    EncodingWarning
+except NameError:
+    pass
+else:
+    # Work around https://github.com/numpy/numpy/issues/24115
+    import warnings
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", EncodingWarning)
+        import numpy.testing  # noqa

hypothesis-python/tests/patching/test_patching.py

@@ -199,6 +199,6 @@ def test_pytest_reports_patch_file_location(pytester):
     print(f"{pattern=}")
     print(f"result.stdout=\n{indent(str(result.stdout), '    ')}")
     fname = re.search(pattern, str(result.stdout)).group(1)
-    patch = Path(pytester.path).joinpath(fname).read_text()
+    patch = Path(pytester.path).joinpath(fname).read_text(encoding="utf-8")
     print(patch)
     assert ADDED_LINES in patch

hypothesis-python/tests/pytest/test_seeding.py

@@ -67,10 +67,10 @@ def test_runs_repeatably_when_seed_is_set(seed, testdir):
 
 if os.path.exists(RECORD_EXAMPLES):
     target = None
-    with open(RECORD_EXAMPLES, 'r') as i:
+    with open(RECORD_EXAMPLES, "r", encoding="utf-8") as i:
         seen = set(map(int, i.read().strip().split("\\n")))
 else:
-    target = open(RECORD_EXAMPLES, 'w')
+    target = open(RECORD_EXAMPLES, "w", encoding="utf-8")
 
 @given(st.integers())
 def test_failure(i):

hypothesis-python/tox.ini

@@ -15,7 +15,7 @@ passenv=
     # Allow CI builds (or user builds) to force coloured terminal output.
     PY_COLORS
 setenv=
-    # PYTHONWARNDEFAULTENCODING=1  # TODO: enable this
+    PYTHONWARNDEFAULTENCODING=1
     brief: HYPOTHESIS_PROFILE=speedy
 commands =
     full: bash scripts/basic-test.sh
@@ -97,6 +97,8 @@ commands =
 deps =
     -r../requirements/test.txt
     pandas~=2.0.0
+setenv=
+    PYTHONWARNDEFAULTENCODING=1
 commands =
     python -bb -X dev -m pytest tests/pandas -n auto
 # Adding a new pandas?  See comment above!
@@ -113,6 +115,8 @@ commands =
     python -bb -X dev -m tests.django.manage test tests.django
 
 [testenv:django42]
+setenv=
+    PYTHONWARNDEFAULTENCODING=1
 commands =
     pip install django~=4.2.0
     python -bb -X dev -m tests.django.manage test tests.django
@@ -140,6 +144,8 @@ commands=
 [testenv:pytest62]
 deps =
     -r../requirements/test.txt
+setenv=
+    PYTHONWARNDEFAULTENCODING=0
 commands=
     pip install pytest==6.2.5 pytest-xdist
     python -bb -X dev -m pytest tests/pytest tests/cover/test_testdecorators.py
@@ -157,6 +163,7 @@ deps =
 allowlist_externals =
     rm
 setenv=
+    PYTHONWARNDEFAULTENCODING=1
     HYPOTHESIS_INTERNAL_COVERAGE=true
 commands_pre =
     rm -f branch-check
@@ -180,6 +187,7 @@ commands =
 deps =
     -r../requirements/coverage.txt
 setenv=
+    PYTHONWARNDEFAULTENCODING=1
     HYPOTHESIS_INTERNAL_COVERAGE=true
 commands_pre =
     python -m coverage erase

requirements/coverage.in

@@ -1,4 +1,4 @@
-black
+git+https://github.com/psf/black.git@eedfc3832290b3a32825b3c0f2dfa3f3d7ee9d1c
 click
 coverage
 dpcontracts