bpo-30764: regrtest: add --fail-env-changed option (#2402) · python/cpython@1f33857 (original) (raw)
`@@ -377,19 +377,19 @@ def parse_executed_tests(self, output):
`
377
377
`return list(match.group(1) for match in parser)
`
378
378
``
379
379
`def check_executed_tests(self, output, tests, skipped=(), failed=(),
`
380
``
`-
omitted=(), randomize=False, interrupted=False):
`
``
380
`+
env_changed=(), omitted=(),
`
``
381
`+
randomize=False, interrupted=False,
`
``
382
`+
fail_env_changed=False):
`
381
383
`if isinstance(tests, str):
`
382
384
`tests = [tests]
`
383
385
`if isinstance(skipped, str):
`
384
386
`skipped = [skipped]
`
385
387
`if isinstance(failed, str):
`
386
388
`failed = [failed]
`
``
389
`+
if isinstance(env_changed, str):
`
``
390
`+
env_changed = [env_changed]
`
387
391
`if isinstance(omitted, str):
`
388
392
`omitted = [omitted]
`
389
``
`-
ntest = len(tests)
`
390
``
`-
nskipped = len(skipped)
`
391
``
`-
nfailed = len(failed)
`
392
``
`-
nomitted = len(omitted)
`
393
393
``
394
394
`executed = self.parse_executed_tests(output)
`
395
395
`if randomize:
`
`@@ -415,11 +415,17 @@ def list_regex(line_format, tests):
`
415
415
`regex = list_regex('%s test%s failed', failed)
`
416
416
`self.check_line(output, regex)
`
417
417
``
``
418
`+
if env_changed:
`
``
419
`+
regex = list_regex('%s test%s altered the execution environment',
`
``
420
`+
env_changed)
`
``
421
`+
self.check_line(output, regex)
`
``
422
+
418
423
`if omitted:
`
419
424
`regex = list_regex('%s test%s omitted', omitted)
`
420
425
`self.check_line(output, regex)
`
421
426
``
422
``
`-
good = ntest - nskipped - nfailed - nomitted
`
``
427
`+
good = (len(tests) - len(skipped) - len(failed)
`
``
428
`+
- len(omitted) - len(env_changed))
`
423
429
`if good:
`
424
430
`regex = r'%s test%s OK.$' % (good, plural(good))
`
425
431
`if not skipped and not failed and good > 1:
`
`@@ -429,10 +435,12 @@ def list_regex(line_format, tests):
`
429
435
`if interrupted:
`
430
436
`self.check_line(output, 'Test suite interrupted by signal SIGINT.')
`
431
437
``
432
``
`-
if nfailed:
`
``
438
`+
if failed:
`
433
439
`result = 'FAILURE'
`
434
440
`elif interrupted:
`
435
441
`result = 'INTERRUPTED'
`
``
442
`+
elif fail_env_changed and env_changed:
`
``
443
`+
result = 'ENV CHANGED'
`
436
444
`else:
`
437
445
`result = 'SUCCESS'
`
438
446
`self.check_line(output, 'Tests result: %s' % result)
`
`@@ -604,7 +612,7 @@ def test_failing(self):
`
604
612
`test_failing = self.create_test('failing', code=code)
`
605
613
`tests = [test_ok, test_failing]
`
606
614
``
607
``
`-
output = self.run_tests(*tests, exitcode=1)
`
``
615
`+
output = self.run_tests(*tests, exitcode=2)
`
608
616
`self.check_executed_tests(output, tests, failed=test_failing)
`
609
617
``
610
618
`def test_resources(self):
`
`@@ -703,7 +711,7 @@ def test_fromfile(self):
`
703
711
`def test_interrupted(self):
`
704
712
`code = TEST_INTERRUPTED
`
705
713
`test = self.create_test('sigint', code=code)
`
706
``
`-
output = self.run_tests(test, exitcode=1)
`
``
714
`+
output = self.run_tests(test, exitcode=130)
`
707
715
`self.check_executed_tests(output, test, omitted=test,
`
708
716
`interrupted=True)
`
709
717
``
`@@ -732,7 +740,7 @@ def test_slow_interrupted(self):
`
732
740
`args = ("--slowest", "-j2", test)
`
733
741
`else:
`
734
742
`args = ("--slowest", test)
`
735
``
`-
output = self.run_tests(*args, exitcode=1)
`
``
743
`+
output = self.run_tests(*args, exitcode=130)
`
736
744
`self.check_executed_tests(output, test,
`
737
745
`omitted=test, interrupted=True)
`
738
746
``
`@@ -772,7 +780,7 @@ def test_run(self):
`
772
780
` builtins.dict['RUN'] = 1
`
773
781
` """)
`
774
782
`test = self.create_test('forever', code=code)
`
775
``
`-
output = self.run_tests('--forever', test, exitcode=1)
`
``
783
`+
output = self.run_tests('--forever', test, exitcode=2)
`
776
784
`self.check_executed_tests(output, [test]*3, failed=test)
`
777
785
``
778
786
`def test_list_tests(self):
`
`@@ -815,7 +823,7 @@ def test_crashed(self):
`
815
823
`ok_test = self.create_test(name="ok")
`
816
824
``
817
825
`tests = [crash_test, ok_test]
`
818
``
`-
output = self.run_tests("-j2", *tests, exitcode=1)
`
``
826
`+
output = self.run_tests("-j2", *tests, exitcode=2)
`
819
827
`self.check_executed_tests(output, tests, failed=crash_test,
`
820
828
`randomize=True)
`
821
829
``
`@@ -864,6 +872,25 @@ def test_method4(self):
`
864
872
`subset = ['test_method1', 'test_method3']
`
865
873
`self.assertEqual(methods, subset)
`
866
874
``
``
875
`+
def test_env_changed(self):
`
``
876
`+
code = textwrap.dedent("""
`
``
877
`+
import unittest
`
``
878
+
``
879
`+
class Tests(unittest.TestCase):
`
``
880
`+
def test_env_changed(self):
`
``
881
`+
open("env_changed", "w").close()
`
``
882
`+
""")
`
``
883
`+
testname = self.create_test(code=code)
`
``
884
+
``
885
`+
don't fail by default
`
``
886
`+
output = self.run_tests(testname)
`
``
887
`+
self.check_executed_tests(output, [testname], env_changed=testname)
`
``
888
+
``
889
`+
fail with --fail-env-changed
`
``
890
`+
output = self.run_tests("--fail-env-changed", testname, exitcode=3)
`
``
891
`+
self.check_executed_tests(output, [testname], env_changed=testname,
`
``
892
`+
fail_env_changed=True)
`
``
893
+
867
894
``
868
895
`if name == 'main':
`
869
896
`unittest.main()
`