teststubtest.py

from __future__ import annotations

import contextlib
import inspect
import io
import os
import re
import sys
import tempfile
import textwrap
import unittest
from typing import Any, Callable, Iterator

import mypy.stubtest
from mypy.stubtest import parse_options, test_stubs
from mypy.test.data import root_dir


@contextlib.contextmanager
def use_tmp_dir(mod_name: str) -> Iterator[str]:
    current = os.getcwd()
    current_syspath = sys.path.copy()
    with tempfile.TemporaryDirectory() as tmp:
        try:
            os.chdir(tmp)
            if sys.path[0] != tmp:
                sys.path.insert(0, tmp)
            yield tmp
        finally:
            sys.path = current_syspath.copy()
            if mod_name in sys.modules:
                del sys.modules[mod_name]

            os.chdir(current)


TEST_MODULE_NAME = "test_module"


stubtest_typing_stub = """
Any = object()

class _SpecialForm:
    def __getitem__(self, typeargs: Any) -> object: ...

Callable: _SpecialForm = ...
Generic: _SpecialForm = ...
Protocol: _SpecialForm = ...
Union: _SpecialForm = ...

class TypeVar:
    def __init__(self, name, covariant: bool = ..., contravariant: bool = ...) -> None: ...

class ParamSpec:
    def __init__(self, name: str) -> None: ...

AnyStr = TypeVar("AnyStr", str, bytes)
_T = TypeVar("_T")
_T_co = TypeVar("_T_co", covariant=True)
_K = TypeVar("_K")
_V = TypeVar("_V")
_S = TypeVar("_S", contravariant=True)
_R = TypeVar("_R", covariant=True)

class Coroutine(Generic[_T_co, _S, _R]): ...
class Iterable(Generic[_T_co]): ...
class Iterator(Iterable[_T_co]): ...
class Mapping(Generic[_K, _V]): ...
class Match(Generic[AnyStr]): ...
class Sequence(Iterable[_T_co]): ...
class Tuple(Sequence[_T_co]): ...
class NamedTuple(tuple[Any, ...]): ...
def overload(func: _T) -> _T: ...
def type_check_only(func: _T) -> _T: ...
def final(func: _T) -> _T: ...
"""

stubtest_builtins_stub = """
from typing import Generic, Mapping, Sequence, TypeVar, overload

T = TypeVar('T')
T_co = TypeVar('T_co', covariant=True)
KT = TypeVar('KT')
VT = TypeVar('VT')

class object:
    __module__: str
    def __init__(self) -> None: pass
    def __repr__(self) -> str: pass
class type: ...

class tuple(Sequence[T_co], Generic[T_co]):
    def __ge__(self, __other: tuple[T_co, ...]) -> bool: pass

class dict(Mapping[KT, VT]): ...

class function: pass
class ellipsis: pass

class int: ...
class float: ...
class bool(int): ...
class str: ...
class bytes: ...

class list(Sequence[T]): ...

def property(f: T) -> T: ...
def classmethod(f: T) -> T: ...
def staticmethod(f: T) -> T: ...
"""

stubtest_enum_stub = """
import sys
from typing import Any, TypeVar, Iterator

_T = TypeVar('_T')

class EnumMeta(type):
    def __len__(self) -> int: pass
    def __iter__(self: type[_T]) -> Iterator[_T]: pass
    def __reversed__(self: type[_T]) -> Iterator[_T]: pass
    def __getitem__(self: type[_T], name: str) -> _T: pass

class Enum(metaclass=EnumMeta):
    def __new__(cls: type[_T], value: object) -> _T: pass
    def __repr__(self) -> str: pass
    def __str__(self) -> str: pass
    def __format__(self, format_spec: str) -> str: pass
    def __hash__(self) -> Any: pass
    def __reduce_ex__(self, proto: Any) -> Any: pass
    name: str
    value: Any

class Flag(Enum):
    def __or__(self: _T, other: _T) -> _T: pass
    def __and__(self: _T, other: _T) -> _T: pass
    def __xor__(self: _T, other: _T) -> _T: pass
    def __invert__(self: _T) -> _T: pass
    if sys.version_info >= (3, 11):
        __ror__ = __or__
        __rand__ = __and__
        __rxor__ = __xor__
"""


def run_stubtest(
    stub: str, runtime: str, options: list[str], config_file: str | None = None
) -> str:
    with use_tmp_dir(TEST_MODULE_NAME) as tmp_dir:
        with open("builtins.pyi", "w") as f:
            f.write(stubtest_builtins_stub)
        with open("typing.pyi", "w") as f:
            f.write(stubtest_typing_stub)
        with open("enum.pyi", "w") as f:
            f.write(stubtest_enum_stub)
        with open(f"{TEST_MODULE_NAME}.pyi", "w") as f:
            f.write(stub)
        with open(f"{TEST_MODULE_NAME}.py", "w") as f:
            f.write(runtime)
        if config_file:
            with open(f"{TEST_MODULE_NAME}_config.ini", "w") as f:
                f.write(config_file)
            options = options + ["--mypy-config-file", f"{TEST_MODULE_NAME}_config.ini"]
        output = io.StringIO()
        with contextlib.redirect_stdout(output):
            test_stubs(parse_options([TEST_MODULE_NAME] + options), use_builtins_fixtures=True)
        return remove_color_code(
            output.getvalue()
            # remove cwd as it's not available from outside
            .replace(os.path.realpath(tmp_dir) + os.sep, "").replace(tmp_dir + os.sep, "")
        )


class Case:
    def __init__(self, stub: str, runtime: str, error: str | None) -> None:
        self.stub = stub
        self.runtime = runtime
        self.error = error


def collect_cases(fn: Callable[..., Iterator[Case]]) -> Callable[..., None]:
    """run_stubtest used to be slow, so we used this decorator to combine cases.

    If you're reading this and bored, feel free to refactor this and make it more like
    other mypy tests.

    """

    def test(*args: Any, **kwargs: Any) -> None:
        cases = list(fn(*args, **kwargs))
        expected_errors = set()
        for c in cases:
            if c.error is None:
                continue
            expected_error = c.error
            if expected_error == "":
                expected_error = TEST_MODULE_NAME
            elif not expected_error.startswith(f"{TEST_MODULE_NAME}."):
                expected_error = f"{TEST_MODULE_NAME}.{expected_error}"
            assert expected_error not in expected_errors, (
                "collect_cases merges cases into a single stubtest invocation; we already "
                "expect an error for {}".format(expected_error)
            )
            expected_errors.add(expected_error)
        output = run_stubtest(
            stub="\n\n".join(textwrap.dedent(c.stub.lstrip("\n")) for c in cases),
            runtime="\n\n".join(textwrap.dedent(c.runtime.lstrip("\n")) for c in cases),
            options=["--generate-allowlist"],
        )

        actual_errors = set(output.splitlines())
        if actual_errors != expected_errors:
            output = run_stubtest(
                stub="\n\n".join(textwrap.dedent(c.stub.lstrip("\n")) for c in cases),
                runtime="\n\n".join(textwrap.dedent(c.runtime.lstrip("\n")) for c in cases),
                options=[],
            )
            assert actual_errors == expected_errors, output

    return test


class StubtestUnit(unittest.TestCase):
    @collect_cases
    def test_basic_good(self) -> Iterator[Case]:
        yield Case(
            stub="def f(number: int, text: str) -> None: ...",
            runtime="def f(number, text): pass",
            error=None,
        )
        yield Case(
            stub="""
            class X:
                def f(self, number: int, text: str) -> None: ...
            """,
            runtime="""
            class X:
                def f(self, number, text): pass
            """,
            error=None,
        )

    @collect_cases
    def test_types(self) -> Iterator[Case]:
        yield Case(
            stub="def mistyped_class() -> None: ...",
            runtime="class mistyped_class: pass",
            error="mistyped_class",
        )
        yield Case(
            stub="class mistyped_fn: ...", runtime="def mistyped_fn(): pass", error="mistyped_fn"
        )
        yield Case(
            stub="""
            class X:
                def mistyped_var(self) -> int: ...
            """,
            runtime="""
            class X:
                mistyped_var = 1
            """,
            error="X.mistyped_var",
        )

    @collect_cases
    def test_coroutines(self) -> Iterator[Case]:
        yield Case(stub="def bar() -> int: ...", runtime="async def bar(): return 5", error="bar")
        # Don't error for this one -- we get false positives otherwise
        yield Case(stub="async def foo() -> int: ...", runtime="def foo(): return 5", error=None)
        yield Case(stub="def baz() -> int: ...", runtime="def baz(): return 5", error=None)
        yield Case(
            stub="async def bingo() -> int: ...", runtime="async def bingo(): return 5", error=None
        )

    @collect_cases
    def test_arg_name(self) -> Iterator[Case]:
        yield Case(
            stub="def bad(number: int, text: str) -> None: ...",
            runtime="def bad(num, text) -> None: pass",
            error="bad",
        )
        yield Case(
            stub="def good_posonly(__number: int, text: str) -> None: ...",
            runtime="def good_posonly(num, /, text): pass",
            error=None,
        )
        yield Case(
            stub="def bad_posonly(__number: int, text: str) -> None: ...",
            runtime="def bad_posonly(flag, /, text): pass",
            error="bad_posonly",
        )
        yield Case(
            stub="""
            class BadMethod:
                def f(self, number: int, text: str) -> None: ...
            """,
            runtime="""
            class BadMethod:
                def f(self, n, text): pass
            """,
            error="BadMethod.f",
        )
        yield Case(
            stub="""
            class GoodDunder:
                def __exit__(self, t, v, tb) -> None: ...
            """,
            runtime="""
            class GoodDunder:
                def __exit__(self, exc_type, exc_val, exc_tb): pass
            """,
            error=None,
        )

    @collect_cases
    def test_arg_kind(self) -> Iterator[Case]:
        yield Case(
            stub="def runtime_kwonly(number: int, text: str) -> None: ...",
            runtime="def runtime_kwonly(number, *, text): pass",
            error="runtime_kwonly",
        )
        yield Case(
            stub="def stub_kwonly(number: int, *, text: str) -> None: ...",
            runtime="def stub_kwonly(number, text): pass",
            error="stub_kwonly",
        )
        yield Case(
            stub="def stub_posonly(__number: int, text: str) -> None: ...",
            runtime="def stub_posonly(number, text): pass",
            error="stub_posonly",
        )
        yield Case(
            stub="def good_posonly(__number: int, text: str) -> None: ...",
            runtime="def good_posonly(number, /, text): pass",
            error=None,
        )
        yield Case(
            stub="def runtime_posonly(number: int, text: str) -> None: ...",
            runtime="def runtime_posonly(number, /, text): pass",
            error="runtime_posonly",
        )
        yield Case(
            stub="def stub_posonly_570(number: int, /, text: str) -> None: ...",
            runtime="def stub_posonly_570(number, text): pass",
            error="stub_posonly_570",
        )

    @collect_cases
    def test_private_parameters(self) -> Iterator[Case]:
        # Private parameters can optionally be omitted.
        yield Case(
            stub="def priv_pos_arg_missing() -> None: ...",
            runtime="def priv_pos_arg_missing(_p1=None): pass",
            error=None,
        )
        yield Case(
            stub="def multi_priv_args() -> None: ...",
            runtime="def multi_priv_args(_p='', _q=''): pass",
            error=None,
        )
        yield Case(
            stub="def priv_kwarg_missing() -> None: ...",
            runtime="def priv_kwarg_missing(*, _p2=''): pass",
            error=None,
        )
        # But if they are included, they must be correct.
        yield Case(
            stub="def priv_pos_arg_wrong(_p: int = ...) -> None: ...",
            runtime="def priv_pos_arg_wrong(_p=None): pass",
            error="priv_pos_arg_wrong",
        )
        yield Case(
            stub="def priv_kwarg_wrong(*, _p: int = ...) -> None: ...",
            runtime="def priv_kwarg_wrong(*, _p=None): pass",
            error="priv_kwarg_wrong",
        )
        # Private parameters must have a default and start with exactly one
        # underscore.
        yield Case(
            stub="def pos_arg_no_default() -> None: ...",
            runtime="def pos_arg_no_default(_np): pass",
            error="pos_arg_no_default",
        )
        yield Case(
            stub="def kwarg_no_default() -> None: ...",
            runtime="def kwarg_no_default(*, _np): pass",
            error="kwarg_no_default",
        )
        yield Case(
            stub="def double_underscore_pos_arg() -> None: ...",
            runtime="def double_underscore_pos_arg(__np = None): pass",
            error="double_underscore_pos_arg",
        )
        yield Case(
            stub="def double_underscore_kwarg() -> None: ...",
            runtime="def double_underscore_kwarg(*, __np = None): pass",
            error="double_underscore_kwarg",
        )
        # But spot parameters that are accidentally not marked kw-only and
        # vice-versa.
        yield Case(
            stub="def priv_arg_is_kwonly(_p=...) -> None: ...",
            runtime="def priv_arg_is_kwonly(*, _p=''): pass",
            error="priv_arg_is_kwonly",
        )
        yield Case(
            stub="def priv_arg_is_positional(*, _p=...) -> None: ...",
            runtime="def priv_arg_is_positional(_p=''): pass",
            error="priv_arg_is_positional",
        )
        # Private parameters not at the end of the parameter list must be
        # included so that users can pass the following arguments using
        # positional syntax.
        yield Case(
            stub="def priv_args_not_at_end(*, q='') -> None: ...",
            runtime="def priv_args_not_at_end(_p='', q=''): pass",
            error="priv_args_not_at_end",
        )

    @collect_cases
    def test_default_presence(self) -> Iterator[Case]:
        yield Case(
            stub="def f1(text: str = ...) -> None: ...",
            runtime="def f1(text = 'asdf'): pass",
            error=None,
        )
        yield Case(
            stub="def f2(text: str = ...) -> None: ...", runtime="def f2(text): pass", error="f2"
        )
        yield Case(
            stub="def f3(text: str) -> None: ...",
            runtime="def f3(text = 'asdf'): pass",
            error="f3",
        )
        yield Case(
            stub="def f4(text: str = ...) -> None: ...",
            runtime="def f4(text = None): pass",
            error="f4",
        )
        yield Case(
            stub="def f5(data: bytes = ...) -> None: ...",
            runtime="def f5(data = 'asdf'): pass",
            error="f5",
        )
        yield Case(
            stub="""
            from typing import TypeVar
            _T = TypeVar("_T", bound=str)
            def f6(text: _T = ...) -> None: ...
            """,
            runtime="def f6(text = None): pass",
            error="f6",
        )

    @collect_cases
    def test_default_value(self) -> Iterator[Case]:
        yield Case(
            stub="def f1(text: str = 'x') -> None: ...",
            runtime="def f1(text = 'y'): pass",
            error="f1",
        )
        yield Case(
            stub='def f2(text: bytes = b"x\'") -> None: ...',
            runtime='def f2(text = b"x\'"): pass',
            error=None,
        )
        yield Case(
            stub='def f3(text: bytes = b"y\'") -> None: ...',
            runtime='def f3(text = b"x\'"): pass',
            error="f3",
        )
        yield Case(
            stub="def f4(text: object = 1) -> None: ...",
            runtime="def f4(text = 1.0): pass",
            error="f4",
        )
        yield Case(
            stub="def f5(text: object = True) -> None: ...",
            runtime="def f5(text = 1): pass",
            error="f5",
        )
        yield Case(
            stub="def f6(text: object = True) -> None: ...",
            runtime="def f6(text = True): pass",
            error=None,
        )
        yield Case(
            stub="def f7(text: object = not True) -> None: ...",
            runtime="def f7(text = False): pass",
            error=None,
        )
        yield Case(
            stub="def f8(text: object = not True) -> None: ...",
            runtime="def f8(text = True): pass",
            error="f8",
        )
        yield Case(
            stub="def f9(text: object = {1: 2}) -> None: ...",
            runtime="def f9(text = {1: 3}): pass",
            error="f9",
        )
        yield Case(
            stub="def f10(text: object = [1, 2]) -> None: ...",
            runtime="def f10(text = [1, 2]): pass",
            error=None,
        )

        # Simulate "<unrepresentable>"
        yield Case(
            stub="def f11() -> None: ...",
            runtime="""
            def f11(text=None) -> None: pass
            f11.__text_signature__ = "(text=<unrepresentable>)"
            """,
            error="f11",
        )

    @collect_cases
    def test_static_class_method(self) -> Iterator[Case]:
        yield Case(
            stub="""
            class Good:
                @classmethod
                def f(cls, number: int, text: str) -> None: ...
            """,
            runtime="""
            class Good:
                @classmethod
                def f(cls, number, text): pass
            """,
            error=None,
        )
        yield Case(
            stub="""
            class Bad1:
                def f(cls, number: int, text: str) -> None: ...
            """,
            runtime="""
            class Bad1:
                @classmethod
                def f(cls, number, text): pass
            """,
            error="Bad1.f",
        )
        yield Case(
            stub="""
            class Bad2:
                @classmethod
                def f(cls, number: int, text: str) -> None: ...
            """,
            runtime="""
            class Bad2:
                @staticmethod
                def f(self, number, text): pass
            """,
            error="Bad2.f",
        )
        yield Case(
            stub="""
            class Bad3:
                @staticmethod
                def f(cls, number: int, text: str) -> None: ...
            """,
            runtime="""
            class Bad3:
                @classmethod
                def f(self, number, text): pass
            """,
            error="Bad3.f",
        )
        yield Case(
            stub="""
            class GoodNew:
                def __new__(cls, *args, **kwargs): ...
            """,
            runtime="""
            class GoodNew:
                def __new__(cls, *args, **kwargs): pass
            """,
            error=None,
        )

    @collect_cases
    def test_arg_mismatch(self) -> Iterator[Case]:
        yield Case(
            stub="def f1(a, *, b, c) -> None: ...", runtime="def f1(a, *, b, c): pass", error=None
        )
        yield Case(
            stub="def f2(a, *, b) -> None: ...", runtime="def f2(a, *, b, c): pass", error="f2"
        )
        yield Case(
            stub="def f3(a, *, b, c) -> None: ...", runtime="def f3(a, *, b): pass", error="f3"
        )
        yield Case(
            stub="def f4(a, *, b, c) -> None: ...", runtime="def f4(a, b, *, c): pass", error="f4"
        )
        yield Case(
            stub="def f5(a, b, *, c) -> None: ...", runtime="def f5(a, *, b, c): pass", error="f5"
        )

    @collect_cases
    def test_varargs_varkwargs(self) -> Iterator[Case]:
        yield Case(
            stub="def f1(*args, **kwargs) -> None: ...",
            runtime="def f1(*args, **kwargs): pass",
            error=None,
        )
        yield Case(
            stub="def f2(*args, **kwargs) -> None: ...",
            runtime="def f2(**kwargs): pass",
            error="f2",
        )
        yield Case(
            stub="def g1(a, b, c, d) -> None: ...", runtime="def g1(a, *args): pass", error=None
        )
        yield Case(
            stub="def g2(a, b, c, d, *args) -> None: ...", runtime="def g2(a): pass", error="g2"
        )
        yield Case(
            stub="def g3(a, b, c, d, *args) -> None: ...",
            runtime="def g3(a, *args): pass",
            error=None,
        )
        yield Case(
            stub="def h1(a) -> None: ...", runtime="def h1(a, b, c, d, *args): pass", error="h1"
        )
        yield Case(
            stub="def h2(a, *args) -> None: ...", runtime="def h2(a, b, c, d): pass", error="h2"
        )
        yield Case(
            stub="def h3(a, *args) -> None: ...",
            runtime="def h3(a, b, c, d, *args): pass",
            error="h3",
        )
        yield Case(
            stub="def j1(a: int, *args) -> None: ...", runtime="def j1(a): pass", error="j1"
        )
        yield Case(
            stub="def j2(a: int) -> None: ...", runtime="def j2(a, *args): pass", error="j2"
        )
        yield Case(
            stub="def j3(a, b, c) -> None: ...", runtime="def j3(a, *args, c): pass", error="j3"
        )
        yield Case(stub="def k1(a, **kwargs) -> None: ...", runtime="def k1(a): pass", error="k1")
        yield Case(
            # In theory an error, but led to worse results in practice
            stub="def k2(a) -> None: ...",
            runtime="def k2(a, **kwargs): pass",
            error=None,
        )
        yield Case(
            stub="def k3(a, b) -> None: ...", runtime="def k3(a, **kwargs): pass", error="k3"
        )
        yield Case(
            stub="def k4(a, *, b) -> None: ...", runtime="def k4(a, **kwargs): pass", error=None
        )
        yield Case(
            stub="def k5(a, *, b) -> None: ...",
            runtime="def k5(a, *, b, c, **kwargs): pass",
            error="k5",
        )
        yield Case(
            stub="def k6(a, *, b, **kwargs) -> None: ...",
            runtime="def k6(a, *, b, c, **kwargs): pass",
            error="k6",
        )

    @collect_cases
    def test_overload(self) -> Iterator[Case]:
        yield Case(
            stub="""
            from typing import overload

            @overload
            def f1(a: int, *, c: int = ...) -> int: ...
            @overload
            def f1(a: int, b: int, c: int = ...) -> str: ...
            """,
            runtime="def f1(a, b = 0, c = 0): pass",
            error=None,
        )
        yield Case(
            stub="""
            @overload
            def f2(a: int, *, c: int = ...) -> int: ...
            @overload
            def f2(a: int, b: int, c: int = ...) -> str: ...
            """,
            runtime="def f2(a, b, c = 0): pass",
            error="f2",
        )
        yield Case(
            stub="""
            @overload
            def f3(a: int) -> int: ...
            @overload
            def f3(a: int, b: str) -> str: ...
            """,
            runtime="def f3(a, b = None): pass",
            error="f3",
        )
        yield Case(
            stub="""
            @overload
            def f4(a: int, *args, b: int, **kwargs) -> int: ...
            @overload
            def f4(a: str, *args, b: int, **kwargs) -> str: ...
            """,
            runtime="def f4(a, *args, b, **kwargs): pass",
            error=None,
        )
        yield Case(
            stub="""
            @overload
            def f5(__a: int) -> int: ...
            @overload
            def f5(__b: str) -> str: ...
            """,
            runtime="def f5(x, /): pass",
            error=None,
        )
        yield Case(
            stub="""
            from typing import final
            from typing_extensions import deprecated
            class Foo:
                @overload
                @final
                def f6(self, __a: int) -> int: ...
                @overload
                @deprecated("evil")
                def f6(self, __b: str) -> str: ...
            """,
            runtime="""
            class Foo:
                def f6(self, x, /): pass
            """,
            error=None,
        )
        yield Case(
            stub="""
            @overload
            def f7(a: int, /) -> int: ...
            @overload
            def f7(b: str, /) -> str: ...
            """,
            runtime="def f7(x, /): pass",
            error=None,
        )
        yield Case(
            stub="""
            @overload
            def f8(a: int, c: int = 0, /) -> int: ...
            @overload
            def f8(b: str, d: int, /) -> str: ...
            """,
            runtime="def f8(x, y, /): pass",
            error="f8",
        )
        yield Case(
            stub="""
            @overload
            def f9(a: int, c: int = 0, /) -> int: ...
            @overload
            def f9(b: str, d: int, /) -> str: ...
            """,
            runtime="def f9(x, y=0, /): pass",
            error=None,
        )
        yield Case(
            stub="""
            class Bar:
                @overload
                def f1(self) -> int: ...
                @overload
                def f1(self, a: int, /) -> int: ...

                @overload
                def f2(self, a: int, /) -> int: ...
                @overload
                def f2(self, a: str, /) -> int: ...
            """,
            runtime="""
            class Bar:
                def f1(self, *a) -> int: ...
                def f2(self, *a) -> int: ...
            """,
            error=None,
        )

    @collect_cases
    def test_property(self) -> Iterator[Case]:
        yield Case(
            stub="""
            class Good:
                @property
                def read_only_attr(self) -> int: ...
            """,
            runtime="""
            class Good:
                @property
                def read_only_attr(self): return 1
            """,
            error=None,
        )
        yield Case(
            stub="""
            class Bad:
                @property
                def f(self) -> int: ...
            """,
            runtime="""
            class Bad:
                def f(self) -> int: return 1
            """,
            error="Bad.f",
        )
        yield Case(
            stub="""
            class GoodReadOnly:
                @property
                def f(self) -> int: ...
            """,
            runtime="""
            class GoodReadOnly:
                f = 1
            """,
            error=None,
        )
        yield Case(
            stub="""
            class BadReadOnly:
                @property
                def f(self) -> str: ...
            """,
            runtime="""
            class BadReadOnly:
                f = 1
            """,
            error="BadReadOnly.f",
        )
        yield Case(
            stub="""
            class Y:
                @property
                def read_only_attr(self) -> int: ...
                @read_only_attr.setter
                def read_only_attr(self, val: int) -> None: ...
            """,
            runtime="""
            class Y:
                @property
                def read_only_attr(self): return 5
            """,
            error="Y.read_only_attr",
        )
        yield Case(
            stub="""
            class Z:
                @property
                def read_write_attr(self) -> int: ...
                @read_write_attr.setter
                def read_write_attr(self, val: int) -> None: ...
            """,
            runtime="""
            class Z:
                @property
                def read_write_attr(self): return self._val
                @read_write_attr.setter
                def read_write_attr(self, val): self._val = val
            """,
            error=None,
        )
        yield Case(
            stub="""
            class FineAndDandy:
                @property
                def attr(self) -> int: ...
            """,
            runtime="""
            class _EvilDescriptor:
                def __get__(self, instance, ownerclass=None):
                    if instance is None:
                        raise AttributeError('no')
                    return 42
                def __set__(self, instance, value):
                    raise AttributeError('no')

            class FineAndDandy:
                attr = _EvilDescriptor()
            """,
            error=None,
        )

    @collect_cases
    def test_var(self) -> Iterator[Case]:
        yield Case(stub="x1: int", runtime="x1 = 5", error=None)
        yield Case(stub="x2: str", runtime="x2 = 5", error="x2")
        yield Case("from typing import Tuple", "", None)  # dummy case
        yield Case(
            stub="""
            x3: Tuple[int, int]
            """,
            runtime="x3 = (1, 3)",
            error=None,
        )
        yield Case(
            stub="""
            x4: Tuple[int, int]
            """,
            runtime="x4 = (1, 3, 5)",
            error="x4",
        )
        yield Case(stub="x5: int", runtime="def x5(a, b): pass", error="x5")
        yield Case(
            stub="def foo(a: int, b: int) -> None: ...\nx6 = foo",
            runtime="def foo(a, b): pass\ndef x6(c, d): pass",
            error="x6",
        )
        yield Case(
            stub="""
            class X:
                f: int
            """,
            runtime="""
            class X:
                def __init__(self):
                    self.f = "asdf"
            """,
            error=None,
        )
        yield Case(
            stub="""
            class Y:
                read_only_attr: int
            """,
            runtime="""
            class Y:
                @property
                def read_only_attr(self): return 5
            """,
            error="Y.read_only_attr",
        )
        yield Case(
            stub="""
            class Z:
                read_write_attr: int
            """,
            runtime="""
            class Z:
                @property
                def read_write_attr(self): return self._val
                @read_write_attr.setter
                def read_write_attr(self, val): self._val = val
            """,
            error=None,
        )

    @collect_cases
    def test_type_alias(self) -> Iterator[Case]:
        yield Case(
            stub="""
            import collections.abc
            import re
            import typing
            from typing import Callable, Dict, Generic, Iterable, List, Match, Tuple, TypeVar, Union
            """,
            runtime="""
            import collections.abc
            import re
            from typing import Callable, Dict, Generic, Iterable, List, Match, Tuple, TypeVar, Union
            """,
            error=None,
        )
        yield Case(
            stub="""
            class X:
                def f(self) -> None: ...
            Y = X
            """,
            runtime="""
            class X:
                def f(self) -> None: ...
            class Y: ...
            """,
            error="Y.f",
        )
        yield Case(stub="A = Tuple[int, str]", runtime="A = (int, str)", error="A")
        # Error if an alias isn't present at runtime...
        yield Case(stub="B = str", runtime="", error="B")
        # ... but only if the alias isn't private
        yield Case(stub="_C = int", runtime="", error=None)
        yield Case(
            stub="""
            D = tuple[str, str]
            E = Tuple[int, int, int]
            F = Tuple[str, int]
            """,
            runtime="""
            D = Tuple[str, str]
            E = Tuple[int, int, int]
            F = List[str]
            """,
            error="F",
        )
        yield Case(
            stub="""
            G = str | int
            H = Union[str, bool]
            I = str | int
            """,
            runtime="""
            G = Union[str, int]
            H = Union[str, bool]
            I = str
            """,
            error="I",
        )
        yield Case(
            stub="""
            K = dict[str, str]
            L = Dict[int, int]
            KK = collections.abc.Iterable[str]
            LL = typing.Iterable[str]
            """,
            runtime="""
            K = Dict[str, str]
            L = Dict[int, int]
            KK = Iterable[str]
            LL = Iterable[str]
            """,
            error=None,
        )
        yield Case(
            stub="""
            _T = TypeVar("_T")
            class _Spam(Generic[_T]):
                def foo(self) -> None: ...
            IntFood = _Spam[int]
            """,
            runtime="""
            _T = TypeVar("_T")
            class _Bacon(Generic[_T]):
                def foo(self, arg): pass
            IntFood = _Bacon[int]
            """,
            error="IntFood.foo",
        )
        yield Case(stub="StrList = list[str]", runtime="StrList = ['foo', 'bar']", error="StrList")
        yield Case(
            stub="""
            N = typing.Callable[[str], bool]
            O = collections.abc.Callable[[int], str]
            P = typing.Callable[[str], bool]
            """,
            runtime="""
            N = Callable[[str], bool]
            O = Callable[[int], str]
            P = int
            """,
            error="P",
        )
        yield Case(
            stub="""
            class Foo:
                class Bar: ...
            BarAlias = Foo.Bar
            """,
            runtime="""
            class Foo:
                class Bar: pass
            BarAlias = Foo.Bar
            """,
            error=None,
        )
        yield Case(
            stub="""
            from io import StringIO
            StringIOAlias = StringIO
            """,
            runtime="""
            from _io import StringIO
            StringIOAlias = StringIO
            """,
            error=None,
        )
        yield Case(stub="M = Match[str]", runtime="M = Match[str]", error=None)
        yield Case(
            stub="""
            class Baz:
                def fizz(self) -> None: ...
            BazAlias = Baz
            """,
            runtime="""
            class Baz:
                def fizz(self): pass
            BazAlias = Baz
            Baz.__name__ = Baz.__qualname__ = Baz.__module__ = "New"
            """,
            error=None,
        )
        yield Case(
            stub="""
            class FooBar:
                __module__: None  # type: ignore
                def fizz(self) -> None: ...
            FooBarAlias = FooBar
            """,
            runtime="""
            class FooBar:
                def fizz(self): pass
            FooBarAlias = FooBar
            FooBar.__module__ = None
            """,
            error=None,
        )
        if sys.version_info >= (3, 10):
            yield Case(
                stub="""
                Q = Dict[str, str]
                R = dict[int, int]
                S = Tuple[int, int]
                T = tuple[str, str]
                U = int | str
                V = Union[int, str]
                W = typing.Callable[[str], bool]
                Z = collections.abc.Callable[[str], bool]
                QQ = typing.Iterable[str]
                RR = collections.abc.Iterable[str]
                MM = typing.Match[str]
                MMM = re.Match[str]
                """,
                runtime="""
                Q = dict[str, str]
                R = dict[int, int]
                S = tuple[int, int]
                T = tuple[str, str]
                U = int | str
                V = int | str
                W = collections.abc.Callable[[str], bool]
                Z = collections.abc.Callable[[str], bool]
                QQ = collections.abc.Iterable[str]
                RR = collections.abc.Iterable[str]
                MM = re.Match[str]
                MMM = re.Match[str]
                """,
                error=None,
            )

    @collect_cases
    def test_enum(self) -> Iterator[Case]:
        yield Case(stub="import enum", runtime="import enum", error=None)
        yield Case(
            stub="""
            class X(enum.Enum):
                a: int
                b: str
                c: str
            """,
            runtime="""
            class X(enum.Enum):
                a = 1
                b = "asdf"
                c = 2
            """,
            error="X.c",
        )
        yield Case(
            stub="""
            class Flags1(enum.Flag):
                a: int
                b: int
            def foo(x: Flags1 = ...) -> None: ...
            """,
            runtime="""
            class Flags1(enum.Flag):
                a = 1
                b = 2
            def foo(x=Flags1.a|Flags1.b): pass
            """,
            error=None,
        )
        yield Case(
            stub="""
            class Flags2(enum.Flag):
                a: int
                b: int
            def bar(x: Flags2 | None = None) -> None: ...
            """,
            runtime="""
            class Flags2(enum.Flag):
                a = 1
                b = 2
            def bar(x=Flags2.a|Flags2.b): pass
            """,
            error="bar",
        )
        yield Case(
            stub="""
            class Flags3(enum.Flag):
                a: int
                b: int
            def baz(x: Flags3 | None = ...) -> None: ...
            """,
            runtime="""
            class Flags3(enum.Flag):
                a = 1
                b = 2
            def baz(x=Flags3(0)): pass
            """,
            error=None,
        )
        yield Case(
            runtime="""
            import enum
            class SomeObject: ...

            class WeirdEnum(enum.Enum):
                a = SomeObject()
                b = SomeObject()
            """,
            stub="""
            import enum
            class SomeObject: ...
            class WeirdEnum(enum.Enum):
                _value_: SomeObject
                a = ...
                b = ...
            """,
            error=None,
        )
        yield Case(
            stub="""
            class Flags4(enum.Flag):
                a: int
                b: int
            def spam(x: Flags4 | None = None) -> None: ...
            """,
            runtime="""
            class Flags4(enum.Flag):
                a = 1
                b = 2
            def spam(x=Flags4(0)): pass
            """,
            error="spam",
        )
        yield Case(
            stub="""
            from typing_extensions import Final, Literal
            class BytesEnum(bytes, enum.Enum):
                a: bytes
            FOO: Literal[BytesEnum.a]
            BAR: Final = BytesEnum.a
            BAZ: BytesEnum
            EGGS: bytes
            """,
            runtime="""
            class BytesEnum(bytes, enum.Enum):
                a = b'foo'
            FOO = BytesEnum.a
            BAR = BytesEnum.a
            BAZ = BytesEnum.a
            EGGS = BytesEnum.a
            """,
            error=None,
        )

    @collect_cases
    def test_decorator(self) -> Iterator[Case]:
        yield Case(
            stub="""
            from typing import Any, Callable
            def decorator(f: Callable[[], int]) -> Callable[..., Any]: ...
            @decorator
            def f() -> Any: ...
            """,
            runtime="""
            def decorator(f): return f
            @decorator
            def f(): return 3
            """,
            error=None,
        )

    @collect_cases
    def test_all_at_runtime_not_stub(self) -> Iterator[Case]:
        yield Case(
            stub="Z: int",
            runtime="""
            __all__ = []
            Z = 5""",
            error=None,
        )

    @collect_cases
    def test_all_in_stub_not_at_runtime(self) -> Iterator[Case]:
        yield Case(stub="__all__ = ()", runtime="", error="__all__")

    @collect_cases
    def test_all_in_stub_different_to_all_at_runtime(self) -> Iterator[Case]:
        # We *should* emit an error with the module name itself + __all__,
        # if the stub *does* define __all__,
        # but the stub's __all__ is inconsistent with the runtime's __all__
        yield Case(
            stub="""
            __all__ = ['foo']
            foo: str
            """,
            runtime="""
            __all__ = []
            foo = 'foo'
            """,
            error="__all__",
        )

    @collect_cases
    def test_missing(self) -> Iterator[Case]:
        yield Case(stub="x = 5", runtime="", error="x")
        yield Case(stub="def f(): ...", runtime="", error="f")
        yield Case(stub="class X: ...", runtime="", error="X")
        yield Case(
            stub="""
            from typing import overload
            @overload
            def h(x: int): ...
            @overload
            def h(x: str): ...
            """,
            runtime="",
            error="h",
        )
        yield Case(stub="", runtime="__all__ = []", error=None)  # dummy case
        yield Case(stub="", runtime="__all__ += ['y']\ny = 5", error="y")
        yield Case(stub="", runtime="__all__ += ['g']\ndef g(): pass", error="g")
        # Here we should only check that runtime has B, since the stub explicitly re-exports it
        yield Case(
            stub="from mystery import A, B as B, C as D  # type: ignore", runtime="", error="B"
        )
        yield Case(
            stub="class Y: ...",
            runtime="__all__ += ['Y']\nclass Y:\n  def __or__(self, other): return self|other",
            error="Y.__or__",
        )
        yield Case(
            stub="class Z: ...",
            runtime="__all__ += ['Z']\nclass Z:\n  def __reduce__(self): return (Z,)",
            error=None,
        )

    @collect_cases
    def test_missing_no_runtime_all(self) -> Iterator[Case]:
        yield Case(stub="", runtime="import sys", error=None)
        yield Case(stub="", runtime="def g(): ...", error="g")
        yield Case(stub="", runtime="CONSTANT = 0", error="CONSTANT")
        yield Case(stub="", runtime="import re; constant = re.compile('foo')", error="constant")
        yield Case(stub="", runtime="from json.scanner import NUMBER_RE", error=None)
        yield Case(stub="", runtime="from string import ascii_letters", error=None)

    @collect_cases
    def test_missing_no_runtime_all_terrible(self) -> Iterator[Case]:
        yield Case(
            stub="",
            runtime="""
import sys
import types
import __future__
_m = types.SimpleNamespace()
_m.annotations = __future__.annotations
sys.modules["_terrible_stubtest_test_module"] = _m

from _terrible_stubtest_test_module import *
assert annotations
""",
            error=None,
        )

    @collect_cases
    def test_non_public_1(self) -> Iterator[Case]:
        yield Case(
            stub="__all__: list[str]", runtime="", error=f"{TEST_MODULE_NAME}.__all__"
        )  # dummy case
        yield Case(stub="_f: int", runtime="def _f(): ...", error="_f")

    @collect_cases
    def test_non_public_2(self) -> Iterator[Case]:
        yield Case(stub="__all__: list[str] = ['f']", runtime="__all__ = ['f']", error=None)
        yield Case(stub="f: int", runtime="def f(): ...", error="f")
        yield Case(stub="g: int", runtime="def g(): ...", error="g")

    @collect_cases
    def test_dunders(self) -> Iterator[Case]:
        yield Case(
            stub="class A:\n  def __init__(self, a: int, b: int) -> None: ...",
            runtime="class A:\n  def __init__(self, a, bx): pass",
            error="A.__init__",
        )
        yield Case(
            stub="class B:\n  def __call__(self, c: int, d: int) -> None: ...",
            runtime="class B:\n  def __call__(self, c, dx): pass",
            error="B.__call__",
        )
        yield Case(
            stub=(
                "class C:\n"
                "  def __init_subclass__(\n"
                "    cls, e: int = ..., **kwargs: int\n"
                "  ) -> None: ...\n"
            ),
            runtime="class C:\n  def __init_subclass__(cls, e=1, **kwargs): pass",
            error=None,
        )
        if sys.version_info >= (3, 9):
            yield Case(
                stub="class D:\n  def __class_getitem__(cls, type: type) -> type: ...",
                runtime="class D:\n  def __class_getitem__(cls, type): ...",
                error=None,
            )

    @collect_cases
    def test_not_subclassable(self) -> Iterator[Case]:
        yield Case(
            stub="class CanBeSubclassed: ...", runtime="class CanBeSubclassed: ...", error=None
        )
        yield Case(
            stub="class CannotBeSubclassed:\n  def __init_subclass__(cls) -> None: ...",
            runtime="class CannotBeSubclassed:\n  def __init_subclass__(cls): raise TypeError",
            error="CannotBeSubclassed",
        )

    @collect_cases
    def test_has_runtime_final_decorator(self) -> Iterator[Case]:
        yield Case(
            stub="from typing_extensions import final",
            runtime="""
            import functools
            from typing_extensions import final
            """,
            error=None,
        )
        yield Case(
            stub="""
            @final
            class A: ...
            """,
            runtime="""
            @final
            class A: ...
            """,
            error=None,
        )
        yield Case(  # Runtime can miss `@final` decorator
            stub="""
            @final
            class B: ...
            """,
            runtime="""
            class B: ...
            """,
            error=None,
        )
        yield Case(  # Stub cannot miss `@final` decorator
            stub="""
            class C: ...
            """,
            runtime="""
            @final
            class C: ...
            """,
            error="C",
        )
        yield Case(
            stub="""
            class D:
                @final
                def foo(self) -> None: ...
                @final
                @staticmethod
                def bar() -> None: ...
                @staticmethod
                @final
                def bar2() -> None: ...
                @final
                @classmethod
                def baz(cls) -> None: ...
                @classmethod
                @final
                def baz2(cls) -> None: ...
                @property
                @final
                def eggs(self) -> int: ...
                @final
                @property
                def eggs2(self) -> int: ...
                @final
                def ham(self, obj: int) -> int: ...
            """,
            runtime="""
            class D:
                @final
                def foo(self): pass
                @final
                @staticmethod
                def bar(): pass
                @staticmethod
                @final
                def bar2(): pass
                @final
                @classmethod
                def baz(cls): pass
                @classmethod
                @final
                def baz2(cls): pass
                @property
                @final
                def eggs(self): return 42
                @final
                @property
                def eggs2(self): pass
                @final
                @functools.lru_cache()
                def ham(self, obj): return obj * 2
            """,
            error=None,
        )
        # Stub methods are allowed to have @final even if the runtime doesn't...
        yield Case(
            stub="""
            class E:
                @final
                def foo(self) -> None: ...
                @final
                @staticmethod
                def bar() -> None: ...
                @staticmethod
                @final
                def bar2() -> None: ...
                @final
                @classmethod
                def baz(cls) -> None: ...
                @classmethod
                @final
                def baz2(cls) -> None: ...
                @property
                @final
                def eggs(self) -> int: ...
                @final
                @property
                def eggs2(self) -> int: ...
                @final
                def ham(self, obj: int) -> int: ...
            """,
            runtime="""
            class E:
                def foo(self): pass
                @staticmethod
                def bar(): pass
                @staticmethod
                def bar2(): pass
                @classmethod
                def baz(cls): pass
                @classmethod
                def baz2(cls): pass
                @property
                def eggs(self): return 42
                @property
                def eggs2(self): return 42
                @functools.lru_cache()
                def ham(self, obj): return obj * 2
            """,
            error=None,
        )
        # ...But if the runtime has @final, the stub must have it as well
        yield Case(
            stub="""
            class F:
                def foo(self) -> None: ...
            """,
            runtime="""
            class F:
                @final
                def foo(self): pass
            """,
            error="F.foo",
        )
        yield Case(
            stub="""
            class G:
                @staticmethod
                def foo() -> None: ...
            """,
            runtime="""
            class G:
                @final
                @staticmethod
                def foo(): pass
            """,
            error="G.foo",
        )
        yield Case(
            stub="""
            class H:
                @staticmethod
                def foo() -> None: ...
            """,
            runtime="""
            class H:
                @staticmethod
                @final
                def foo(): pass
            """,
            error="H.foo",
        )
        yield Case(
            stub="""
            class I:
                @classmethod
                def foo(cls) -> None: ...
            """,
            runtime="""
            class I:
                @final
                @classmethod
                def foo(cls): pass
            """,
            error="I.foo",
        )
        yield Case(
            stub="""
            class J:
                @classmethod
                def foo(cls) -> None: ...
            """,
            runtime="""
            class J:
                @classmethod
                @final
                def foo(cls): pass
            """,
            error="J.foo",
        )
        yield Case(
            stub="""
            class K:
                @property
                def foo(self) -> int: ...
            """,
            runtime="""
            class K:
                @property
                @final
                def foo(self): return 42
            """,
            error="K.foo",
        )
        # This test wouldn't pass,
        # because the runtime can't set __final__ on instances of builtins.property,
        # so stubtest has non way of knowing that the runtime was decorated with @final:
        #
        # yield Case(
        #     stub="""
        #     class K2:
        #         @property
        #         def foo(self) -> int: ...
        #     """,
        #     runtime="""
        #     class K2:
        #         @final
        #         @property
        #         def foo(self): return 42
        #     """,
        #     error="K2.foo",
        # )
        yield Case(
            stub="""
            class L:
                def foo(self, obj: int) -> int: ...
            """,
            runtime="""
            class L:
                @final
                @functools.lru_cache()
                def foo(self, obj): return obj * 2
            """,
            error="L.foo",
        )

    @collect_cases
    def test_name_mangling(self) -> Iterator[Case]:
        yield Case(
            stub="""
            class X:
                def __mangle_good(self, text: str) -> None: ...
                def __mangle_bad(self, number: int) -> None: ...
            """,
            runtime="""
            class X:
                def __mangle_good(self, text): pass
                def __mangle_bad(self, text): pass
            """,
            error="X.__mangle_bad",
        )
        yield Case(
            stub="""
            class Klass:
                class __Mangled1:
                    class __Mangled2:
                        def __mangle_good(self, text: str) -> None: ...
                        def __mangle_bad(self, number: int) -> None: ...
            """,
            runtime="""
            class Klass:
                class __Mangled1:
                    class __Mangled2:
                        def __mangle_good(self, text): pass
                        def __mangle_bad(self, text): pass
            """,
            error="Klass.__Mangled1.__Mangled2.__mangle_bad",
        )
        yield Case(
            stub="""
            class __Dunder__:
                def __mangle_good(self, text: str) -> None: ...
                def __mangle_bad(self, number: int) -> None: ...
            """,
            runtime="""
            class __Dunder__:
                def __mangle_good(self, text): pass
                def __mangle_bad(self, text): pass
            """,
            error="__Dunder__.__mangle_bad",
        )
        yield Case(
            stub="""
            class _Private:
                def __mangle_good(self, text: str) -> None: ...
                def __mangle_bad(self, number: int) -> None: ...
            """,
            runtime="""
            class _Private:
                def __mangle_good(self, text): pass
                def __mangle_bad(self, text): pass
            """,
            error="_Private.__mangle_bad",
        )

    @collect_cases
    def test_mro(self) -> Iterator[Case]:
        yield Case(
            stub="""
            class A:
                def foo(self, x: int) -> None: ...
            class B(A):
                pass
            class C(A):
                pass
            """,
            runtime="""
            class A:
                def foo(self, x: int) -> None: ...
            class B(A):
                def foo(self, x: int) -> None: ...
            class C(A):
                def foo(self, y: int) -> None: ...
            """,
            error="C.foo",
        )
        yield Case(
            stub="""
            class X: ...
            """,
            runtime="""
            class X:
                def __init__(self, x): pass
            """,
            error="X.__init__",
        )

    @collect_cases
    def test_good_literal(self) -> Iterator[Case]:
        yield Case(
            stub=r"""
            from typing_extensions import Literal

            import enum
            class Color(enum.Enum):
                RED: int

            NUM: Literal[1]
            CHAR: Literal['a']
            FLAG: Literal[True]
            NON: Literal[None]
            BYT1: Literal[b'abc']
            BYT2: Literal[b'\x90']
            ENUM: Literal[Color.RED]
            """,
            runtime=r"""
            import enum
            class Color(enum.Enum):
                RED = 3

            NUM = 1
            CHAR = 'a'
            NON = None
            FLAG = True
            BYT1 = b"abc"
            BYT2 = b'\x90'
            ENUM = Color.RED
            """,
            error=None,
        )

    @collect_cases
    def test_bad_literal(self) -> Iterator[Case]:
        yield Case("from typing_extensions import Literal", "", None)  # dummy case
        yield Case(
            stub="INT_FLOAT_MISMATCH: Literal[1]",
            runtime="INT_FLOAT_MISMATCH = 1.0",
            error="INT_FLOAT_MISMATCH",
        )
        yield Case(stub="WRONG_INT: Literal[1]", runtime="WRONG_INT = 2", error="WRONG_INT")
        yield Case(stub="WRONG_STR: Literal['a']", runtime="WRONG_STR = 'b'", error="WRONG_STR")
        yield Case(
            stub="BYTES_STR_MISMATCH: Literal[b'value']",
            runtime="BYTES_STR_MISMATCH = 'value'",
            error="BYTES_STR_MISMATCH",
        )
        yield Case(
            stub="STR_BYTES_MISMATCH: Literal['value']",
            runtime="STR_BYTES_MISMATCH = b'value'",
            error="STR_BYTES_MISMATCH",
        )
        yield Case(
            stub="WRONG_BYTES: Literal[b'abc']",
            runtime="WRONG_BYTES = b'xyz'",
            error="WRONG_BYTES",
        )
        yield Case(
            stub="WRONG_BOOL_1: Literal[True]",
            runtime="WRONG_BOOL_1 = False",
            error="WRONG_BOOL_1",
        )
        yield Case(
            stub="WRONG_BOOL_2: Literal[False]",
            runtime="WRONG_BOOL_2 = True",
            error="WRONG_BOOL_2",
        )

    @collect_cases
    def test_special_subtype(self) -> Iterator[Case]:
        yield Case(
            stub="""
            b1: bool
            b2: bool
            b3: bool
            """,
            runtime="""
            b1 = 0
            b2 = 1
            b3 = 2
            """,
            error="b3",
        )
        yield Case(
            stub="""
            from typing_extensions import TypedDict

            class _Options(TypedDict):
                a: str
                b: int

            opt1: _Options
            opt2: _Options
            opt3: _Options
            """,
            runtime="""
            opt1 = {"a": "3.", "b": 14}
            opt2 = {"some": "stuff"}  # false negative
            opt3 = 0
            """,
            error="opt3",
        )

    @collect_cases
    def test_runtime_typing_objects(self) -> Iterator[Case]:
        yield Case(
            stub="from typing_extensions import Protocol, TypedDict",
            runtime="from typing_extensions import Protocol, TypedDict",
            error=None,
        )
        yield Case(
            stub="""
            class X(Protocol):
                bar: int
                def foo(self, x: int, y: bytes = ...) -> str: ...
            """,
            runtime="""
            class X(Protocol):
                bar: int
                def foo(self, x: int, y: bytes = ...) -> str: ...
            """,
            error=None,
        )
        yield Case(
            stub="""
            class Y(TypedDict):
                a: int
            """,
            runtime="""
            class Y(TypedDict):
                a: int
            """,
            error=None,
        )

    @collect_cases
    def test_named_tuple(self) -> Iterator[Case]:
        yield Case(
            stub="from typing import NamedTuple",
            runtime="from typing import NamedTuple",
            error=None,
        )
        yield Case(
            stub="""
            class X1(NamedTuple):
                bar: int
                foo: str = ...
            """,
            runtime="""
            class X1(NamedTuple):
                bar: int
                foo: str = 'a'
            """,
            error=None,
        )
        yield Case(
            stub="""
            class X2(NamedTuple):
                bar: int
                foo: str
            """,
            runtime="""
            class X2(NamedTuple):
                bar: int
                foo: str = 'a'
            """,
            # `__new__` will miss a default value for a `foo` parameter,
            # but we don't generate special errors for `foo` missing `...` part.
            error="X2.__new__",
        )

    @collect_cases
    def test_named_tuple_typing_and_collections(self) -> Iterator[Case]:
        yield Case(
            stub="from typing import NamedTuple",
            runtime="from collections import namedtuple",
            error=None,
        )
        yield Case(
            stub="""
            class X1(NamedTuple):
                bar: int
                foo: str = ...
            """,
            runtime="""
            X1 = namedtuple('X1', ['bar', 'foo'], defaults=['a'])
            """,
            error=None,
        )
        yield Case(
            stub="""
            class X2(NamedTuple):
                bar: int
                foo: str
            """,
            runtime="""
            X2 = namedtuple('X1', ['bar', 'foo'], defaults=['a'])
            """,
            error="X2.__new__",
        )

    @collect_cases
    def test_type_var(self) -> Iterator[Case]:
        yield Case(
            stub="from typing import TypeVar", runtime="from typing import TypeVar", error=None
        )
        yield Case(stub="A = TypeVar('A')", runtime="A = TypeVar('A')", error=None)
        yield Case(stub="B = TypeVar('B')", runtime="B = 5", error="B")
        if sys.version_info >= (3, 10):
            yield Case(
                stub="from typing import ParamSpec",
                runtime="from typing import ParamSpec",
                error=None,
            )
            yield Case(stub="C = ParamSpec('C')", runtime="C = ParamSpec('C')", error=None)

    @collect_cases
    def test_metaclass_match(self) -> Iterator[Case]:
        yield Case(stub="class Meta(type): ...", runtime="class Meta(type): ...", error=None)
        yield Case(stub="class A0: ...", runtime="class A0: ...", error=None)
        yield Case(
            stub="class A1(metaclass=Meta): ...",
            runtime="class A1(metaclass=Meta): ...",
            error=None,
        )
        yield Case(stub="class A2: ...", runtime="class A2(metaclass=Meta): ...", error="A2")
        yield Case(stub="class A3(metaclass=Meta): ...", runtime="class A3: ...", error="A3")

        # Explicit `type` metaclass can always be added in any part:
        yield Case(
            stub="class T1(metaclass=type): ...",
            runtime="class T1(metaclass=type): ...",
            error=None,
        )
        yield Case(stub="class T2: ...", runtime="class T2(metaclass=type): ...", error=None)
        yield Case(stub="class T3(metaclass=type): ...", runtime="class T3: ...", error=None)

        # Explicit check that `_protected` names are also supported:
        yield Case(stub="class _P1(type): ...", runtime="class _P1(type): ...", error=None)
        yield Case(stub="class P2: ...", runtime="class P2(metaclass=_P1): ...", error="P2")

        # With inheritance:
        yield Case(
            stub="""
            class I1(metaclass=Meta): ...
            class S1(I1): ...
            """,
            runtime="""
            class I1(metaclass=Meta): ...
            class S1(I1): ...
            """,
            error=None,
        )
        yield Case(
            stub="""
            class I2(metaclass=Meta): ...
            class S2: ...  # missing inheritance
            """,
            runtime="""
            class I2(metaclass=Meta): ...
            class S2(I2): ...
            """,
            error="S2",
        )

    @collect_cases
    def test_metaclass_abcmeta(self) -> Iterator[Case]:
        # Handling abstract metaclasses is special:
        yield Case(stub="from abc import ABCMeta", runtime="from abc import ABCMeta", error=None)
        yield Case(
            stub="class A1(metaclass=ABCMeta): ...",
            runtime="class A1(metaclass=ABCMeta): ...",
            error=None,
        )
        # Stubs cannot miss abstract metaclass:
        yield Case(stub="class A2: ...", runtime="class A2(metaclass=ABCMeta): ...", error="A2")
        # But, stubs can add extra abstract metaclass, this might be a typing hack:
        yield Case(stub="class A3(metaclass=ABCMeta): ...", runtime="class A3: ...", error=None)

    @collect_cases
    def test_abstract_methods(self) -> Iterator[Case]:
        yield Case(
            stub="""
            from abc import abstractmethod
            from typing import overload
            """,
            runtime="from abc import abstractmethod",
            error=None,
        )
        yield Case(
            stub="""
            class A1:
                def some(self) -> None: ...
            """,
            runtime="""
            class A1:
                @abstractmethod
                def some(self) -> None: ...
            """,
            error="A1.some",
        )
        yield Case(
            stub="""
            class A2:
                @abstractmethod
                def some(self) -> None: ...
            """,
            runtime="""
            class A2:
                @abstractmethod
                def some(self) -> None: ...
            """,
            error=None,
        )
        yield Case(
            stub="""
            class A3:
                @overload
                def some(self, other: int) -> str: ...
                @overload
                def some(self, other: str) -> int: ...
            """,
            runtime="""
            class A3:
                @abstractmethod
                def some(self, other) -> None: ...
            """,
            error="A3.some",
        )
        yield Case(
            stub="""
            class A4:
                @overload
                @abstractmethod
                def some(self, other: int) -> str: ...
                @overload
                @abstractmethod
                def some(self, other: str) -> int: ...
            """,
            runtime="""
            class A4:
                @abstractmethod
                def some(self, other) -> None: ...
            """,
            error=None,
        )
        yield Case(
            stub="""
            class A5:
                @abstractmethod
                @overload
                def some(self, other: int) -> str: ...
                @abstractmethod
                @overload
                def some(self, other: str) -> int: ...
            """,
            runtime="""
            class A5:
                @abstractmethod
                def some(self, other) -> None: ...
            """,
            error=None,
        )
        # Runtime can miss `@abstractmethod`:
        yield Case(
            stub="""
            class A6:
                @abstractmethod
                def some(self) -> None: ...
            """,
            runtime="""
            class A6:
                def some(self) -> None: ...
            """,
            error=None,
        )

    @collect_cases
    def test_abstract_properties(self) -> Iterator[Case]:
        # TODO: test abstract properties with setters
        yield Case(
            stub="from abc import abstractmethod",
            runtime="from abc import abstractmethod",
            error=None,
        )
        # Ensure that `@property` also can be abstract:
        yield Case(
            stub="""
            class AP1:
                @property
                def some(self) -> int: ...
            """,
            runtime="""
            class AP1:
                @property
                @abstractmethod
                def some(self) -> int: ...
            """,
            error="AP1.some",
        )
        yield Case(
            stub="""
            class AP1_2:
                def some(self) -> int: ...  # missing `@property` decorator
            """,
            runtime="""
            class AP1_2:
                @property
                @abstractmethod
                def some(self) -> int: ...
            """,
            error="AP1_2.some",
        )
        yield Case(
            stub="""
            class AP2:
                @property
                @abstractmethod
                def some(self) -> int: ...
            """,
            runtime="""
            class AP2:
                @property
                @abstractmethod
                def some(self) -> int: ...
            """,
            error=None,
        )
        # Runtime can miss `@abstractmethod`:
        yield Case(
            stub="""
            class AP3:
                @property
                @abstractmethod
                def some(self) -> int: ...
            """,
            runtime="""
            class AP3:
                @property
                def some(self) -> int: ...
            """,
            error=None,
        )

    @collect_cases
    def test_type_check_only(self) -> Iterator[Case]:
        yield Case(
            stub="from typing import type_check_only, overload",
            runtime="from typing import overload",
            error=None,
        )
        # You can have public types that are only defined in stubs
        # with `@type_check_only`:
        yield Case(
            stub="""
            @type_check_only
            class A1: ...
            """,
            runtime="",
            error=None,
        )
        # Having `@type_check_only` on a type that exists at runtime is an error
        yield Case(
            stub="""
            @type_check_only
            class A2: ...
            """,
            runtime="class A2: ...",
            error="A2",
        )
        # The same is true for NamedTuples and TypedDicts:
        yield Case(
            stub="from typing_extensions import NamedTuple, TypedDict",
            runtime="from typing_extensions import NamedTuple, TypedDict",
            error=None,
        )
        yield Case(
            stub="""
            @type_check_only
            class NT1(NamedTuple): ...
            """,
            runtime="class NT1(NamedTuple): ...",
            error="NT1",
        )
        yield Case(
            stub="""
            @type_check_only
            class TD1(TypedDict): ...
            """,
            runtime="class TD1(TypedDict): ...",
            error="TD1",
        )
        # The same is true for functions:
        yield Case(
            stub="""
            @type_check_only
            def func1() -> None: ...
            """,
            runtime="",
            error=None,
        )
        yield Case(
            stub="""
            @type_check_only
            def func2() -> None: ...
            """,
            runtime="def func2() -> None: ...",
            error="func2",
        )


def remove_color_code(s: str) -> str:
    return re.sub("\\x1b.*?m", "", s)  # this works!


class StubtestMiscUnit(unittest.TestCase):
    def test_output(self) -> None:
        output = run_stubtest(
            stub="def bad(number: int, text: str) -> None: ...",
            runtime="def bad(num, text): pass",
            options=[],
        )
        expected = (
            f'error: {TEST_MODULE_NAME}.bad is inconsistent, stub argument "number" differs '
            'from runtime argument "num"\n'
            f"Stub: in file {TEST_MODULE_NAME}.pyi:1\n"
            "def (number: builtins.int, text: builtins.str)\n"
            f"Runtime: in file {TEST_MODULE_NAME}.py:1\ndef (num, text)\n\n"
            "Found 1 error (checked 1 module)\n"
        )
        assert output == expected

        output = run_stubtest(
            stub="def bad(number: int, text: str) -> None: ...",
            runtime="def bad(num, text): pass",
            options=["--concise"],
        )
        expected = (
            "{}.bad is inconsistent, "
            'stub argument "number" differs from runtime argument "num"\n'.format(TEST_MODULE_NAME)
        )
        assert output == expected

    def test_ignore_flags(self) -> None:
        output = run_stubtest(
            stub="", runtime="__all__ = ['f']\ndef f(): pass", options=["--ignore-missing-stub"]
        )
        assert output == "Success: no issues found in 1 module\n"

        output = run_stubtest(stub="", runtime="def f(): pass", options=["--ignore-missing-stub"])
        assert output == "Success: no issues found in 1 module\n"

        output = run_stubtest(
            stub="def f(__a): ...", runtime="def f(a): pass", options=["--ignore-positional-only"]
        )
        assert output == "Success: no issues found in 1 module\n"

    def test_allowlist(self) -> None:
        # Can't use this as a context because Windows
        allowlist = tempfile.NamedTemporaryFile(mode="w+", delete=False)
        try:
            with allowlist:
                allowlist.write(f"{TEST_MODULE_NAME}.bad  # comment\n# comment")

            output = run_stubtest(
                stub="def bad(number: int, text: str) -> None: ...",
                runtime="def bad(asdf, text): pass",
                options=["--allowlist", allowlist.name],
            )
            assert output == "Success: no issues found in 1 module\n"

            # test unused entry detection
            output = run_stubtest(stub="", runtime="", options=["--allowlist", allowlist.name])
            assert output == (
                f"note: unused allowlist entry {TEST_MODULE_NAME}.bad\n"
                "Found 1 error (checked 1 module)\n"
            )

            output = run_stubtest(
                stub="",
                runtime="",
                options=["--allowlist", allowlist.name, "--ignore-unused-allowlist"],
            )
            assert output == "Success: no issues found in 1 module\n"

            # test regex matching
            with open(allowlist.name, mode="w+") as f:
                f.write(f"{TEST_MODULE_NAME}.b.*\n")
                f.write("(unused_missing)?\n")
                f.write("unused.*\n")

            output = run_stubtest(
                stub=textwrap.dedent(
                    """
                    def good() -> None: ...
                    def bad(number: int) -> None: ...
                    def also_bad(number: int) -> None: ...
                    """.lstrip(
                        "\n"
                    )
                ),
                runtime=textwrap.dedent(
                    """
                    def good(): pass
                    def bad(asdf): pass
                    def also_bad(asdf): pass
                    """.lstrip(
                        "\n"
                    )
                ),
                options=["--allowlist", allowlist.name, "--generate-allowlist"],
            )
            assert output == (
                f"note: unused allowlist entry unused.*\n{TEST_MODULE_NAME}.also_bad\n"
            )
        finally:
            os.unlink(allowlist.name)

    def test_mypy_build(self) -> None:
        output = run_stubtest(stub="+", runtime="", options=[])
        assert output == (
            "error: not checking stubs due to failed mypy compile:\n{}.pyi:1: "
            "error: invalid syntax  [syntax]\n".format(TEST_MODULE_NAME)
        )

        output = run_stubtest(stub="def f(): ...\ndef f(): ...", runtime="", options=[])
        assert output == (
            "error: not checking stubs due to mypy build errors:\n{}.pyi:2: "
            'error: Name "f" already defined on line 1  [no-redef]\n'.format(TEST_MODULE_NAME)
        )

    def test_missing_stubs(self) -> None:
        output = io.StringIO()
        with contextlib.redirect_stdout(output):
            test_stubs(parse_options(["not_a_module"]))
        assert remove_color_code(output.getvalue()) == (
            "error: not_a_module failed to find stubs\n"
            "Stub:\nMISSING\nRuntime:\nN/A\n\n"
            "Found 1 error (checked 1 module)\n"
        )

    def test_only_py(self) -> None:
        # in this case, stubtest will check the py against itself
        # this is useful to support packages with a mix of stubs and inline types
        with use_tmp_dir(TEST_MODULE_NAME):
            with open(f"{TEST_MODULE_NAME}.py", "w") as f:
                f.write("a = 1")
            output = io.StringIO()
            with contextlib.redirect_stdout(output):
                test_stubs(parse_options([TEST_MODULE_NAME]))
            output_str = remove_color_code(output.getvalue())
            assert output_str == "Success: no issues found in 1 module\n"

    def test_get_typeshed_stdlib_modules(self) -> None:
        stdlib = mypy.stubtest.get_typeshed_stdlib_modules(None, (3, 7))
        assert "builtins" in stdlib
        assert "os" in stdlib
        assert "os.path" in stdlib
        assert "asyncio" in stdlib
        assert "graphlib" not in stdlib
        assert "formatter" in stdlib
        assert "contextvars" in stdlib  # 3.7+
        assert "importlib.metadata" not in stdlib

        stdlib = mypy.stubtest.get_typeshed_stdlib_modules(None, (3, 10))
        assert "graphlib" in stdlib
        assert "formatter" not in stdlib
        assert "importlib.metadata" in stdlib

    def test_signature(self) -> None:
        def f(a: int, b: int, *, c: int, d: int = 0, **kwargs: Any) -> None:
            pass

        assert (
            str(mypy.stubtest.Signature.from_inspect_signature(inspect.signature(f)))
            == "def (a, b, *, c, d = ..., **kwargs)"
        )

    def test_builtin_signature_with_unrepresentable_default(self) -> None:
        sig = mypy.stubtest.safe_inspect_signature(bytes.hex)
        assert sig is not None
        assert (
            str(mypy.stubtest.Signature.from_inspect_signature(sig))
            == "def (self, sep = ..., bytes_per_sep = ...)"
        )

    def test_config_file(self) -> None:
        runtime = "temp = 5\n"
        stub = "from decimal import Decimal\ntemp: Decimal\n"
        config_file = f"[mypy]\nplugins={root_dir}/test-data/unit/plugins/decimal_to_int.py\n"
        output = run_stubtest(stub=stub, runtime=runtime, options=[])
        assert output == (
            f"error: {TEST_MODULE_NAME}.temp variable differs from runtime type Literal[5]\n"
            f"Stub: in file {TEST_MODULE_NAME}.pyi:2\n_decimal.Decimal\nRuntime:\n5\n\n"
            "Found 1 error (checked 1 module)\n"
        )
        output = run_stubtest(stub=stub, runtime=runtime, options=[], config_file=config_file)
        assert output == "Success: no issues found in 1 module\n"

    def test_no_modules(self) -> None:
        output = io.StringIO()
        with contextlib.redirect_stdout(output):
            test_stubs(parse_options([]))
        assert remove_color_code(output.getvalue()) == "error: no modules to check\n"

    def test_module_and_typeshed(self) -> None:
        output = io.StringIO()
        with contextlib.redirect_stdout(output):
            test_stubs(parse_options(["--check-typeshed", "some_module"]))
        assert remove_color_code(output.getvalue()) == (
            "error: cannot pass both --check-typeshed and a list of modules\n"
        )