-
-
Notifications
You must be signed in to change notification settings - Fork 2.6k
/
test_stepwise.py
357 lines (286 loc) · 10.3 KB
/
test_stepwise.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
from pathlib import Path
import pytest
from _pytest.cacheprovider import Cache
from _pytest.monkeypatch import MonkeyPatch
from _pytest.pytester import Pytester
from _pytest.stepwise import STEPWISE_CACHE_DIR
@pytest.fixture
def stepwise_pytester(pytester: Pytester) -> Pytester:
# Rather than having to modify our testfile between tests, we introduce
# a flag for whether or not the second test should fail.
pytester.makeconftest(
"""
def pytest_addoption(parser):
group = parser.getgroup('general')
group.addoption('--fail', action='store_true', dest='fail')
group.addoption('--fail-last', action='store_true', dest='fail_last')
"""
)
# Create a simple test suite.
pytester.makepyfile(
test_a="""
def test_success_before_fail():
assert 1
def test_fail_on_flag(request):
assert not request.config.getvalue('fail')
def test_success_after_fail():
assert 1
def test_fail_last_on_flag(request):
assert not request.config.getvalue('fail_last')
def test_success_after_last_fail():
assert 1
"""
)
pytester.makepyfile(
test_b="""
def test_success():
assert 1
"""
)
# customize cache directory so we don't use the tox's cache directory, which makes tests in this module flaky
pytester.makeini(
"""
[pytest]
cache_dir = .cache
"""
)
return pytester
@pytest.fixture
def error_pytester(pytester: Pytester) -> Pytester:
pytester.makepyfile(
test_a="""
def test_error(nonexisting_fixture):
assert 1
def test_success_after_fail():
assert 1
"""
)
return pytester
@pytest.fixture
def broken_pytester(pytester: Pytester) -> Pytester:
pytester.makepyfile(
working_testfile="def test_proper(): assert 1", broken_testfile="foobar"
)
return pytester
def _strip_resource_warnings(lines):
# Strip unreliable ResourceWarnings, so no-output assertions on stderr can work.
# (https://github.com/pytest-dev/pytest/issues/5088)
return [
x
for x in lines
if not x.startswith(("Exception ignored in:", "ResourceWarning"))
]
def test_run_without_stepwise(stepwise_pytester: Pytester) -> None:
result = stepwise_pytester.runpytest("-v", "--strict-markers", "--fail")
result.stdout.fnmatch_lines(["*test_success_before_fail PASSED*"])
result.stdout.fnmatch_lines(["*test_fail_on_flag FAILED*"])
result.stdout.fnmatch_lines(["*test_success_after_fail PASSED*"])
def test_stepwise_output_summary(pytester: Pytester) -> None:
pytester.makepyfile(
"""
import pytest
@pytest.mark.parametrize("expected", [True, True, True, True, False])
def test_data(expected):
assert expected
"""
)
result = pytester.runpytest("-v", "--stepwise")
result.stdout.fnmatch_lines(["stepwise: no previously failed tests, not skipping."])
result = pytester.runpytest("-v", "--stepwise")
result.stdout.fnmatch_lines(
["stepwise: skipping 4 already passed items.", "*1 failed, 4 deselected*"]
)
def test_fail_and_continue_with_stepwise(stepwise_pytester: Pytester) -> None:
# Run the tests with a failing second test.
result = stepwise_pytester.runpytest(
"-v", "--strict-markers", "--stepwise", "--fail"
)
assert _strip_resource_warnings(result.stderr.lines) == []
stdout = result.stdout.str()
# Make sure we stop after first failing test.
assert "test_success_before_fail PASSED" in stdout
assert "test_fail_on_flag FAILED" in stdout
assert "test_success_after_fail" not in stdout
# "Fix" the test that failed in the last run and run it again.
result = stepwise_pytester.runpytest("-v", "--strict-markers", "--stepwise")
assert _strip_resource_warnings(result.stderr.lines) == []
stdout = result.stdout.str()
# Make sure the latest failing test runs and then continues.
assert "test_success_before_fail" not in stdout
assert "test_fail_on_flag PASSED" in stdout
assert "test_success_after_fail PASSED" in stdout
@pytest.mark.parametrize("stepwise_skip", ["--stepwise-skip", "--sw-skip"])
def test_run_with_skip_option(stepwise_pytester: Pytester, stepwise_skip: str) -> None:
result = stepwise_pytester.runpytest(
"-v",
"--strict-markers",
"--stepwise",
stepwise_skip,
"--fail",
"--fail-last",
)
assert _strip_resource_warnings(result.stderr.lines) == []
stdout = result.stdout.str()
# Make sure first fail is ignore and second fail stops the test run.
assert "test_fail_on_flag FAILED" in stdout
assert "test_success_after_fail PASSED" in stdout
assert "test_fail_last_on_flag FAILED" in stdout
assert "test_success_after_last_fail" not in stdout
def test_fail_on_errors(error_pytester: Pytester) -> None:
result = error_pytester.runpytest("-v", "--strict-markers", "--stepwise")
assert _strip_resource_warnings(result.stderr.lines) == []
stdout = result.stdout.str()
assert "test_error ERROR" in stdout
assert "test_success_after_fail" not in stdout
def test_change_testfile(stepwise_pytester: Pytester) -> None:
result = stepwise_pytester.runpytest(
"-v", "--strict-markers", "--stepwise", "--fail", "test_a.py"
)
assert _strip_resource_warnings(result.stderr.lines) == []
stdout = result.stdout.str()
assert "test_fail_on_flag FAILED" in stdout
# Make sure the second test run starts from the beginning, since the
# test to continue from does not exist in testfile_b.
result = stepwise_pytester.runpytest(
"-v", "--strict-markers", "--stepwise", "test_b.py"
)
assert _strip_resource_warnings(result.stderr.lines) == []
stdout = result.stdout.str()
assert "test_success PASSED" in stdout
@pytest.mark.parametrize("broken_first", [True, False])
def test_stop_on_collection_errors(
broken_pytester: Pytester, broken_first: bool
) -> None:
"""Stop during collection errors. Broken test first or broken test last
actually surfaced a bug (#5444), so we test both situations."""
files = ["working_testfile.py", "broken_testfile.py"]
if broken_first:
files.reverse()
result = broken_pytester.runpytest("-v", "--strict-markers", "--stepwise", *files)
result.stdout.fnmatch_lines("*error during collection*")
def test_xfail_handling(pytester: Pytester, monkeypatch: MonkeyPatch) -> None:
"""Ensure normal xfail is ignored, and strict xfail interrupts the session in sw mode
(#5547)
"""
monkeypatch.setattr("sys.dont_write_bytecode", True)
contents = """
import pytest
def test_a(): pass
@pytest.mark.xfail(strict={strict})
def test_b(): assert {assert_value}
def test_c(): pass
def test_d(): pass
"""
pytester.makepyfile(contents.format(assert_value="0", strict="False"))
result = pytester.runpytest("--sw", "-v")
result.stdout.fnmatch_lines(
[
"*::test_a PASSED *",
"*::test_b XFAIL *",
"*::test_c PASSED *",
"*::test_d PASSED *",
"* 3 passed, 1 xfailed in *",
]
)
pytester.makepyfile(contents.format(assert_value="1", strict="True"))
result = pytester.runpytest("--sw", "-v")
result.stdout.fnmatch_lines(
[
"*::test_a PASSED *",
"*::test_b FAILED *",
"* Interrupted*",
"* 1 failed, 1 passed in *",
]
)
pytester.makepyfile(contents.format(assert_value="0", strict="True"))
result = pytester.runpytest("--sw", "-v")
result.stdout.fnmatch_lines(
[
"*::test_b XFAIL *",
"*::test_c PASSED *",
"*::test_d PASSED *",
"* 2 passed, 1 deselected, 1 xfailed in *",
]
)
def test_stepwise_skip_is_independent(pytester: Pytester) -> None:
pytester.makepyfile(
"""
def test_one():
assert False
def test_two():
assert False
def test_three():
assert False
"""
)
result = pytester.runpytest("--tb", "no", "--stepwise-skip")
result.assert_outcomes(failed=2)
result.stdout.fnmatch_lines(
[
"FAILED test_stepwise_skip_is_independent.py::test_one - assert False",
"FAILED test_stepwise_skip_is_independent.py::test_two - assert False",
"*Interrupted: Test failed, continuing from this test next run.*",
]
)
def test_sw_skip_help(pytester: Pytester) -> None:
result = pytester.runpytest("-h")
result.stdout.fnmatch_lines("*Implicitly enables --stepwise.")
def test_stepwise_xdist_dont_store_lastfailed(pytester: Pytester) -> None:
pytester.makefile(
ext=".ini",
pytest=f"[pytest]\ncache_dir = {pytester.path}\n",
)
pytester.makepyfile(
conftest="""
import pytest
@pytest.hookimpl(tryfirst=True)
def pytest_configure(config) -> None:
config.workerinput = True
"""
)
pytester.makepyfile(
test_one="""
def test_one():
assert False
"""
)
result = pytester.runpytest("--stepwise")
assert result.ret == pytest.ExitCode.INTERRUPTED
stepwise_cache_file = (
pytester.path / Cache._CACHE_PREFIX_VALUES / STEPWISE_CACHE_DIR
)
assert not Path(stepwise_cache_file).exists()
def test_disabled_stepwise_xdist_dont_clear_cache(pytester: Pytester) -> None:
pytester.makefile(
ext=".ini",
pytest=f"[pytest]\ncache_dir = {pytester.path}\n",
)
stepwise_cache_file = (
pytester.path / Cache._CACHE_PREFIX_VALUES / STEPWISE_CACHE_DIR
)
stepwise_cache_dir = stepwise_cache_file.parent
stepwise_cache_dir.mkdir(exist_ok=True, parents=True)
stepwise_cache_file_relative = f"{Cache._CACHE_PREFIX_VALUES}/{STEPWISE_CACHE_DIR}"
expected_value = '"test_one.py::test_one"'
content = {f"{stepwise_cache_file_relative}": expected_value}
pytester.makefile(ext="", **content)
pytester.makepyfile(
conftest="""
import pytest
@pytest.hookimpl(tryfirst=True)
def pytest_configure(config) -> None:
config.workerinput = True
"""
)
pytester.makepyfile(
test_one="""
def test_one():
assert True
"""
)
result = pytester.runpytest()
assert result.ret == 0
assert Path(stepwise_cache_file).exists()
with stepwise_cache_file.open() as file_handle:
observed_value = file_handle.readlines()
assert [expected_value] == observed_value