bpo-29512: Add test.bisect, bisect failing tests (#2452) · python/cpython@84d9d14 (original) (raw)
``
1
`+
#!/usr/bin/env python3
`
``
2
`+
"""
`
``
3
`+
Command line tool to bisect failing CPython tests.
`
``
4
+
``
5
`+
Find the test_os test method which alters the environment:
`
``
6
+
``
7
`+
./python -m test.bisect --fail-env-changed test_os
`
``
8
+
``
9
`+
Find a reference leak in "test_os", write the list of failing tests into the
`
``
10
`+
"bisect" file:
`
``
11
+
``
12
`+
./python -m test.bisect -o bisect -R 3:3 test_os
`
``
13
+
``
14
`+
Load an existing list of tests from a file using -i option:
`
``
15
+
``
16
`+
./python -m test --list-cases -m FileTests test_os > tests
`
``
17
`+
./python -m test.bisect -i tests test_os
`
``
18
`+
"""
`
``
19
+
``
20
`+
import argparse
`
``
21
`+
import datetime
`
``
22
`+
import os.path
`
``
23
`+
import math
`
``
24
`+
import random
`
``
25
`+
import subprocess
`
``
26
`+
import sys
`
``
27
`+
import tempfile
`
``
28
`+
import time
`
``
29
+
``
30
+
``
31
`+
def write_tests(filename, tests):
`
``
32
`+
with open(filename, "w") as fp:
`
``
33
`+
for name in tests:
`
``
34
`+
print(name, file=fp)
`
``
35
`+
fp.flush()
`
``
36
+
``
37
+
``
38
`+
def write_output(filename, tests):
`
``
39
`+
if not filename:
`
``
40
`+
return
`
``
41
`+
print("Write %s tests into %s" % (len(tests), filename))
`
``
42
`+
write_tests(filename, tests)
`
``
43
`+
return filename
`
``
44
+
``
45
+
``
46
`+
def format_shell_args(args):
`
``
47
`+
return ' '.join(args)
`
``
48
+
``
49
+
``
50
`+
def list_cases(args):
`
``
51
`+
cmd = [sys.executable, '-m', 'test', '--list-cases']
`
``
52
`+
cmd.extend(args.test_args)
`
``
53
`+
proc = subprocess.run(cmd,
`
``
54
`+
stdout=subprocess.PIPE,
`
``
55
`+
universal_newlines=True)
`
``
56
`+
exitcode = proc.returncode
`
``
57
`+
if exitcode:
`
``
58
`+
cmd = format_shell_args(cmd)
`
``
59
`+
print("Failed to list tests: %s failed with exit code %s"
`
``
60
`+
% (cmd, exitcode))
`
``
61
`+
sys.exit(exitcode)
`
``
62
`+
tests = proc.stdout.splitlines()
`
``
63
`+
return tests
`
``
64
+
``
65
+
``
66
`+
def run_tests(args, tests, huntrleaks=None):
`
``
67
`+
tmp = tempfile.mktemp()
`
``
68
`+
try:
`
``
69
`+
write_tests(tmp, tests)
`
``
70
+
``
71
`+
cmd = [sys.executable, '-m', 'test', '--matchfile', tmp]
`
``
72
`+
cmd.extend(args.test_args)
`
``
73
`+
print("+ %s" % format_shell_args(cmd))
`
``
74
`+
proc = subprocess.run(cmd)
`
``
75
`+
return proc.returncode
`
``
76
`+
finally:
`
``
77
`+
if os.path.exists(tmp):
`
``
78
`+
os.unlink(tmp)
`
``
79
+
``
80
+
``
81
`+
def parse_args():
`
``
82
`+
parser = argparse.ArgumentParser()
`
``
83
`+
parser.add_argument('-i', '--input',
`
``
84
`+
help='Test names produced by --list-tests written '
`
``
85
`+
'into a file. If not set, run --list-tests')
`
``
86
`+
parser.add_argument('-o', '--output',
`
``
87
`+
help='Result of the bisection')
`
``
88
`+
parser.add_argument('-n', '--max-tests', type=int, default=1,
`
``
89
`+
help='Maximum number of tests to stop the bisection '
`
``
90
`+
'(default: 1)')
`
``
91
`+
parser.add_argument('-N', '--max-iter', type=int, default=100,
`
``
92
`+
help='Maximum number of bisection iterations '
`
``
93
`+
'(default: 100)')
`
``
94
`+
FIXME: document that following arguments are test arguments
`
``
95
+
``
96
`+
args, test_args = parser.parse_known_args()
`
``
97
`+
args.test_args = test_args
`
``
98
`+
return args
`
``
99
+
``
100
+
``
101
`+
def main():
`
``
102
`+
args = parse_args()
`
``
103
+
``
104
`+
if args.input:
`
``
105
`+
with open(args.input) as fp:
`
``
106
`+
tests = [line.strip() for line in fp]
`
``
107
`+
else:
`
``
108
`+
tests = list_cases(args)
`
``
109
+
``
110
`+
print("Start bisection with %s tests" % len(tests))
`
``
111
`+
print("Test arguments: %s" % format_shell_args(args.test_args))
`
``
112
`+
print("Bisection will stop when getting %s or less tests "
`
``
113
`+
"(-n/--max-tests option), or after %s iterations "
`
``
114
`+
"(-N/--max-iter option)"
`
``
115
`+
% (args.max_tests, args.max_iter))
`
``
116
`+
output = write_output(args.output, tests)
`
``
117
`+
print()
`
``
118
+
``
119
`+
start_time = time.monotonic()
`
``
120
`+
iteration = 1
`
``
121
`+
try:
`
``
122
`+
while len(tests) > args.max_tests and iteration <= args.max_iter:
`
``
123
`+
ntest = len(tests)
`
``
124
`+
ntest = max(ntest // 2, 1)
`
``
125
`+
subtests = random.sample(tests, ntest)
`
``
126
+
``
127
`+
print("[+] Iteration %s: run %s tests/%s"
`
``
128
`+
% (iteration, len(subtests), len(tests)))
`
``
129
`+
print()
`
``
130
+
``
131
`+
exitcode = run_tests(args, subtests)
`
``
132
+
``
133
`+
print("ran %s tests/%s" % (ntest, len(tests)))
`
``
134
`+
print("exit", exitcode)
`
``
135
`+
if exitcode:
`
``
136
`+
print("Tests failed: use this new subtest")
`
``
137
`+
tests = subtests
`
``
138
`+
output = write_output(args.output, tests)
`
``
139
`+
else:
`
``
140
`+
print("Tests succeeded: skip this subtest, try a new subbset")
`
``
141
`+
print()
`
``
142
`+
iteration += 1
`
``
143
`+
except KeyboardInterrupt:
`
``
144
`+
print()
`
``
145
`+
print("Bisection interrupted!")
`
``
146
`+
print()
`
``
147
+
``
148
`+
print("Tests (%s):" % len(tests))
`
``
149
`+
for test in tests:
`
``
150
`+
print("* %s" % test)
`
``
151
`+
print()
`
``
152
+
``
153
`+
if output:
`
``
154
`+
print("Output written into %s" % output)
`
``
155
+
``
156
`+
dt = math.ceil(time.monotonic() - start_time)
`
``
157
`+
if len(tests) <= args.max_tests:
`
``
158
`+
print("Bisection completed in %s iterations and %s"
`
``
159
`+
% (iteration, datetime.timedelta(seconds=dt)))
`
``
160
`+
sys.exit(1)
`
``
161
`+
else:
`
``
162
`+
print("Bisection failed after %s iterations and %s"
`
``
163
`+
% (iteration, datetime.timedelta(seconds=dt)))
`
``
164
+
``
165
+
``
166
`+
if name == "main":
`
``
167
`+
main()
`