bpo-30283: Backport regrtest features from master to 2.7 (#1516) · python/cpython@453a685 (original) (raw)
`@@ -27,7 +27,7 @@
`
27
27
`-w/--verbose2 -- re-run failed tests in verbose mode
`
28
28
`-W/--verbose3 -- re-run failed tests in verbose mode immediately
`
29
29
`-q/--quiet -- no output unless one or more tests fail
`
30
``
`-
-S/--slow -- print the slowest 10 tests
`
``
30
`+
-S/--slowest -- print the slowest 10 tests
`
31
31
` --header -- print header with interpreter info
`
32
32
``
33
33
`Selecting tests
`
62
62
`-P/--pgo -- enable Profile Guided Optimization training
`
63
63
`--testdir -- execute test files in the specified directory
`
64
64
` (instead of the Python stdlib test suite)
`
``
65
`+
--list-tests -- only write the name of tests that will be run,
`
``
66
`+
don't execute them
`
65
67
``
66
68
``
67
69
`Additional Option Details:
`
158
160
`"""
`
159
161
``
160
162
`import StringIO
`
``
163
`+
import datetime
`
161
164
`import getopt
`
162
165
`import json
`
163
166
`import os
`
226
229
`INTERRUPTED = -4
`
227
230
`CHILD_ERROR = -5 # error in a child process
`
228
231
``
``
232
`+
Minimum duration of a test to display its duration or to mention that
`
``
233
`+
the test is running in background
`
``
234
`+
PROGRESS_MIN_TIME = 30.0 # seconds
`
``
235
+
``
236
`+
Display the running tests if nothing happened last N seconds
`
``
237
`+
PROGRESS_UPDATE = 30.0 # seconds
`
``
238
+
229
239
`from test import test_support
`
230
240
``
231
241
`RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', 'bsddb',
`
`@@ -241,6 +251,32 @@ def usage(code, msg=''):
`
241
251
`sys.exit(code)
`
242
252
``
243
253
``
``
254
`+
def format_duration(seconds):
`
``
255
`+
if seconds < 1.0:
`
``
256
`+
return '%.0f ms' % (seconds * 1e3)
`
``
257
`+
if seconds < 60.0:
`
``
258
`+
return '%.0f sec' % seconds
`
``
259
+
``
260
`+
minutes, seconds = divmod(seconds, 60.0)
`
``
261
`+
return '%.0f min %.0f sec' % (minutes, seconds)
`
``
262
+
``
263
+
``
264
`+
_FORMAT_TEST_RESULT = {
`
``
265
`+
PASSED: '%s passed',
`
``
266
`+
FAILED: '%s failed',
`
``
267
`+
ENV_CHANGED: '%s failed (env changed)',
`
``
268
`+
SKIPPED: '%s skipped',
`
``
269
`+
RESOURCE_DENIED: '%s skipped (resource denied)',
`
``
270
`+
INTERRUPTED: '%s interrupted',
`
``
271
`+
CHILD_ERROR: '%s crashed',
`
``
272
`+
}
`
``
273
+
``
274
+
``
275
`+
def format_test_result(test_name, result):
`
``
276
`+
fmt = _FORMAT_TEST_RESULT.get(result, "%s")
`
``
277
`+
return fmt % test_name
`
``
278
+
``
279
+
244
280
`def main(tests=None, testdir=None, verbose=0, quiet=False,
`
245
281
`exclude=False, single=False, randomize=False, fromfile=None,
`
246
282
`findleaks=False, use_resources=None, trace=False, coverdir='coverage',
`
`@@ -269,16 +305,18 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
`
269
305
` directly to set the values that would normally be set by flags
`
270
306
` on the command line.
`
271
307
` """
`
``
308
`+
regrtest_start_time = time.time()
`
272
309
``
273
310
`test_support.record_original_stdout(sys.stdout)
`
274
311
`try:
`
275
312
`opts, args = getopt.getopt(sys.argv[1:], 'hvqxsSrf:lu:t:TD:NLR:FwWM:j:PGm:',
`
276
313
` ['help', 'verbose', 'verbose2', 'verbose3', 'quiet',
`
277
``
`-
'exclude', 'single', 'slow', 'randomize', 'fromfile=', 'findleaks',
`
``
314
`+
'exclude', 'single', 'slow', 'slowest', 'randomize', 'fromfile=',
`
``
315
`+
'findleaks',
`
278
316
`'use=', 'threshold=', 'trace', 'coverdir=', 'nocoverdir',
`
279
317
`'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=',
`
280
318
`'multiprocess=', 'slaveargs=', 'forever', 'header', 'pgo',
`
281
``
`-
'failfast', 'match=', 'testdir='])
`
``
319
`+
'failfast', 'match=', 'testdir=', 'list-tests'])
`
282
320
`except getopt.error, msg:
`
283
321
`usage(2, msg)
`
284
322
``
`@@ -288,6 +326,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
`
288
326
`if use_resources is None:
`
289
327
`use_resources = []
`
290
328
`slaveargs = None
`
``
329
`+
list_tests = False
`
291
330
`for o, a in opts:
`
292
331
`if o in ('-h', '--help'):
`
293
332
`usage(0)
`
`@@ -306,7 +345,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
`
306
345
`exclude = True
`
307
346
`elif o in ('-s', '--single'):
`
308
347
`single = True
`
309
``
`-
elif o in ('-S', '--slow'):
`
``
348
`+
elif o in ('-S', '--slow', '--slowest'):
`
310
349
`print_slow = True
`
311
350
`elif o in ('-r', '--randomize'):
`
312
351
`randomize = True
`
`@@ -373,8 +412,10 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
`
373
412
`slaveargs = a
`
374
413
`elif o in ('-P', '--pgo'):
`
375
414
`pgo = True
`
376
``
`-
elif o in ('--testdir'):
`
``
415
`+
elif o == '--testdir':
`
377
416
`testdir = a
`
``
417
`+
elif o == '--list-tests':
`
``
418
`+
list_tests = True
`
378
419
`else:
`
379
420
`print >>sys.stderr, ("No handler for option {}. Please "
`
380
421
`"report this as a bug at http://bugs.python.org.").format(o)
`
`@@ -482,6 +523,12 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
`
482
523
`random.seed(random_seed)
`
483
524
`print "Using random seed", random_seed
`
484
525
`random.shuffle(selected)
`
``
526
+
``
527
`+
if list_tests:
`
``
528
`+
for name in selected:
`
``
529
`+
print(name)
`
``
530
`+
sys.exit(0)
`
``
531
+
485
532
`if trace:
`
486
533
`import trace
`
487
534
`tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix],
`
`@@ -525,13 +572,27 @@ def test_forever(tests=list(selected)):
`
525
572
`test_count = '/{}'.format(len(selected))
`
526
573
`test_count_width = len(test_count) - 1
`
527
574
``
``
575
`+
def display_progress(test_index, test):
`
``
576
`+
"[ 51/405/1] test_tcl"
`
``
577
`+
fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}"
`
``
578
`+
line = fmt.format(test_count_width, test_index, test_count,
`
``
579
`+
len(bad), test)
`
``
580
+
``
581
`+
add the timestamp prefix: "0:01:05 "
`
``
582
`+
test_time = time.time() - regrtest_start_time
`
``
583
`+
test_time = datetime.timedelta(seconds=int(test_time))
`
``
584
`+
line = "%s %s" % (test_time, line)
`
``
585
+
``
586
`+
print(line)
`
``
587
`+
sys.stdout.flush()
`
``
588
+
528
589
`if use_mp:
`
529
590
`try:
`
530
591
`from threading import Thread
`
531
592
`except ImportError:
`
532
593
`print "Multiprocess option requires thread support"
`
533
594
`sys.exit(2)
`
534
``
`-
from Queue import Queue
`
``
595
`+
from Queue import Queue, Empty
`
535
596
`from subprocess import Popen, PIPE
`
536
597
`debug_output_pat = re.compile(r"[\d+ refs]$")
`
537
598
`output = Queue()
`
`@@ -551,63 +612,106 @@ def tests_and_args():
`
551
612
`# required to spawn a new process with PGO flag on/off
`
552
613
`if pgo:
`
553
614
`base_cmd = base_cmd + ['--pgo']
`
554
``
`-
def work():
`
555
``
`-
A worker thread.
`
556
``
`-
try:
`
557
``
`-
while True:
`
558
``
`-
try:
`
559
``
`-
test, args_tuple = next(pending)
`
560
``
`-
except StopIteration:
`
561
``
`-
output.put((None, None, None, None))
`
562
``
`-
return
`
563
``
`-
-E is needed by some tests, e.g. test_import
`
564
``
`-
args = base_cmd + ['--slaveargs', json.dumps(args_tuple)]
`
565
``
`-
if testdir:
`
566
``
`-
args.extend(('--testdir', testdir))
`
``
615
+
``
616
`+
class MultiprocessThread(Thread):
`
``
617
`+
current_test = None
`
``
618
`+
start_time = None
`
``
619
+
``
620
`+
def runtest(self):
`
``
621
`+
try:
`
``
622
`+
test, args_tuple = next(pending)
`
``
623
`+
except StopIteration:
`
``
624
`+
output.put((None, None, None, None))
`
``
625
`+
return True
`
``
626
+
``
627
`+
-E is needed by some tests, e.g. test_import
`
``
628
`+
args = base_cmd + ['--slaveargs', json.dumps(args_tuple)]
`
``
629
`+
if testdir:
`
``
630
`+
args.extend(('--testdir', testdir))
`
``
631
`+
try:
`
``
632
`+
self.start_time = time.time()
`
``
633
`+
self.current_test = test
`
567
634
`popen = Popen(args,
`
568
635
`stdout=PIPE, stderr=PIPE,
`
569
636
`universal_newlines=True,
`
570
637
`close_fds=(os.name != 'nt'))
`
571
638
`stdout, stderr = popen.communicate()
`
572
639
`retcode = popen.wait()
`
``
640
`+
finally:
`
``
641
`+
self.current_test = None
`
573
642
``
574
``
`-
Strip last refcount output line if it exists, since it
`
575
``
`-
comes from the shutdown of the interpreter in the subcommand.
`
576
``
`-
stderr = debug_output_pat.sub("", stderr)
`
``
643
`+
Strip last refcount output line if it exists, since it
`
``
644
`+
comes from the shutdown of the interpreter in the subcommand.
`
``
645
`+
stderr = debug_output_pat.sub("", stderr)
`
577
646
``
578
``
`-
if retcode == 0:
`
579
``
`-
stdout, _, result = stdout.strip().rpartition("\n")
`
580
``
`-
if not result:
`
581
``
`-
output.put((None, None, None, None))
`
582
``
`-
return
`
``
647
`+
if retcode == 0:
`
``
648
`+
stdout, _, result = stdout.strip().rpartition("\n")
`
``
649
`+
if not result:
`
``
650
`+
output.put((None, None, None, None))
`
``
651
`+
return True
`
583
652
``
584
``
`-
result = json.loads(result)
`
585
``
`-
else:
`
586
``
`-
result = (CHILD_ERROR, "Exit code %s" % retcode)
`
``
653
`+
result = json.loads(result)
`
``
654
`+
else:
`
``
655
`+
result = (CHILD_ERROR, "Exit code %s" % retcode)
`
587
656
``
588
``
`-
output.put((test, stdout.rstrip(), stderr.rstrip(), result))
`
589
``
`-
except BaseException:
`
590
``
`-
output.put((None, None, None, None))
`
591
``
`-
raise
`
``
657
`+
output.put((test, stdout.rstrip(), stderr.rstrip(), result))
`
``
658
`+
return False
`
592
659
``
593
``
`-
workers = [Thread(target=work) for i in range(use_mp)]
`
``
660
`+
def run(self):
`
``
661
`+
try:
`
``
662
`+
stop = False
`
``
663
`+
while not stop:
`
``
664
`+
stop = self.runtest()
`
``
665
`+
except BaseException:
`
``
666
`+
output.put((None, None, None, None))
`
``
667
`+
raise
`
``
668
+
``
669
`+
workers = [MultiprocessThread() for i in range(use_mp)]
`
``
670
`+
print("Run tests in parallel using %s child processes"
`
``
671
`+
% len(workers))
`
594
672
`for worker in workers:
`
595
673
`worker.start()
`
596
674
``
``
675
`+
def get_running(workers):
`
``
676
`+
running = []
`
``
677
`+
for worker in workers:
`
``
678
`+
current_test = worker.current_test
`
``
679
`+
if not current_test:
`
``
680
`+
continue
`
``
681
`+
dt = time.time() - worker.start_time
`
``
682
`+
if dt >= PROGRESS_MIN_TIME:
`
``
683
`+
running.append('%s (%.0f sec)' % (current_test, dt))
`
``
684
`+
return running
`
``
685
+
597
686
`finished = 0
`
598
687
`test_index = 1
`
``
688
`+
get_timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME)
`
599
689
`try:
`
600
690
`while finished < use_mp:
`
601
``
`-
test, stdout, stderr, result = output.get()
`
``
691
`+
try:
`
``
692
`+
item = output.get(timeout=get_timeout)
`
``
693
`+
except Empty:
`
``
694
`+
running = get_running(workers)
`
``
695
`+
if running and not pgo:
`
``
696
`+
print('running: %s' % ', '.join(running))
`
``
697
`+
continue
`
``
698
+
``
699
`+
test, stdout, stderr, result = item
`
602
700
`if test is None:
`
603
701
`finished += 1
`
604
702
`continue
`
605
703
`accumulate_result(test, result)
`
606
704
`if not quiet:
`
607
``
`-
fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}"
`
608
``
`-
print(fmt.format(
`
609
``
`-
test_count_width, test_index, test_count,
`
610
``
`-
len(bad), test))
`
``
705
`+
ok, test_time = result
`
``
706
`+
text = format_test_result(test, ok)
`
``
707
`+
if (ok not in (CHILD_ERROR, INTERRUPTED)
`
``
708
`+
and test_time >= PROGRESS_MIN_TIME
`
``
709
`+
and not pgo):
`
``
710
`+
text += ' (%.0f sec)' % test_time
`
``
711
`+
running = get_running(workers)
`
``
712
`+
if running and not pgo:
`
``
713
`+
text += ' -- running: %s' % ', '.join(running)
`
``
714
`+
display_progress(test_index, text)
`
611
715
``
612
716
`if stdout:
`
613
717
`print stdout
`
`@@ -627,18 +731,22 @@ def work():
`
627
731
`for worker in workers:
`
628
732
`worker.join()
`
629
733
`else:
`
``
734
`+
print("Run tests sequentially")
`
``
735
+
``
736
`+
previous_test = None
`
630
737
`for test_index, test in enumerate(tests, 1):
`
631
738
`if not quiet:
`
632
``
`-
fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}"
`
633
``
`-
print(fmt.format(
`
634
``
`-
test_count_width, test_index, test_count, len(bad), test))
`
635
``
`-
sys.stdout.flush()
`
``
739
`+
text = test
`
``
740
`+
if previous_test:
`
``
741
`+
text = '%s -- %s' % (text, previous_test)
`
``
742
`+
display_progress(test_index, text)
`
636
743
`if trace:
`
637
744
`# If we're tracing code coverage, then we don't exit with status
`
638
745
`# if on a false return value from main.
`
639
746
`tracer.runctx('runtest(test, verbose, quiet, testdir=testdir)',
`
640
747
`globals=globals(), locals=vars())
`
641
748
`else:
`
``
749
`+
start_time = time.time()
`
642
750
`try:
`
643
751
`result = runtest(test, verbose, quiet, huntrleaks, None, pgo,
`
644
752
`failfast=failfast,
`
`@@ -655,6 +763,16 @@ def work():
`
655
763
`break
`
656
764
`except:
`
657
765
`raise
`
``
766
+
``
767
`+
previous_test = format_test_result(test, result[0])
`
``
768
`+
test_time = time.time() - start_time
`
``
769
`+
if test_time >= PROGRESS_MIN_TIME:
`
``
770
`+
previous_test = "%s in %s" % (previous_test,
`
``
771
`+
format_duration(test_time))
`
``
772
`+
elif result[0] == PASSED:
`
``
773
`+
be quiet: say nothing if the test passed shortly
`
``
774
`+
previous_test = None
`
``
775
+
658
776
`if findleaks:
`
659
777
`gc.collect()
`
660
778
`if gc.garbage:
`
`@@ -683,8 +801,8 @@ def work():
`
683
801
`if print_slow:
`
684
802
`test_times.sort(reverse=True)
`
685
803
`print "10 slowest tests:"
`
686
``
`-
for time, test in test_times[:10]:
`
687
``
`-
print "%s: %.1fs" % (test, time)
`
``
804
`+
for test_time, test in test_times[:10]:
`
``
805
`+
print("- %s: %.1fs" % (test, test_time))
`
688
806
`if bad and not pgo:
`
689
807
`print count(len(bad), "test"), "failed:"
`
690
808
`printlist(bad)
`
`@@ -745,6 +863,18 @@ def work():
`
745
863
`if runleaks:
`
746
864
`os.system("leaks %d" % os.getpid())
`
747
865
``
``
866
`+
`
``
867
`+
duration = time.time() - regrtest_start_time
`
``
868
`+
print("Total duration: %s" % format_duration(duration))
`
``
869
+
``
870
`+
if bad:
`
``
871
`+
result = "FAILURE"
`
``
872
`+
elif interrupted:
`
``
873
`+
result = "INTERRUPTED"
`
``
874
`+
else:
`
``
875
`+
result = "SUCCESS"
`
``
876
`+
print("Tests result: %s" % result)
`
``
877
+
748
878
`sys.exit(len(bad) > 0 or interrupted)
`
749
879
``
750
880
``