bpo-36044: Reduce number of unit tests run for PGO build (GH-14702) · python/cpython@2406672 (original) (raw)

7 files changed

lines changed

Original file line number Diff line number Diff line change
@@ -264,7 +264,9 @@ def _create_parser():
264 264 help='only write the name of test cases that will be run'
265 265 ' , don\'t execute them')
266 266 group.add_argument('-P', '--pgo', dest='pgo', action='store_true',
267 -help='enable Profile Guided Optimization training')
267 +help='enable Profile Guided Optimization (PGO) training')
268 +group.add_argument('--pgo-extended', action='store_true',
269 +help='enable extended PGO training (slower training)')
268 270 group.add_argument('--fail-env-changed', action='store_true',
269 271 help='if a test file alters the environment, mark '
270 272 'the test as failed')
@@ -344,6 +346,8 @@ def _parse_args(args, **kwargs):
344 346 parser.error("-G/--failfast needs either -v or -W")
345 347 if ns.pgo and (ns.verbose or ns.verbose2 or ns.verbose3):
346 348 parser.error("--pgo/-v don't go together!")
349 +if ns.pgo_extended:
350 +ns.pgo = True # pgo_extended implies pgo
347 351
348 352 if ns.nowindows:
349 353 print("Warning: the --nowindows (-n) option is deprecated. "
Original file line number Diff line number Diff line change
@@ -17,6 +17,7 @@
17 17 INTERRUPTED, CHILD_ERROR, TEST_DID_NOT_RUN,
18 18 PROGRESS_MIN_TIME, format_test_result, is_failed)
19 19 from test.libregrtest.setup import setup_tests
20 +from test.libregrtest.pgo import setup_pgo_tests
20 21 from test.libregrtest.utils import removepy, count, format_duration, printlist
21 22 from test import support
22 23
@@ -214,6 +215,9 @@ def find_tests(self, tests):
214 215
215 216 removepy(self.tests)
216 217
218 +# add default PGO tests if no tests are specified
219 +setup_pgo_tests(self.ns)
220 +
217 221 stdtests = STDTESTS[:]
218 222 nottests = NOTTESTS.copy()
219 223 if self.ns.exclude:
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
1 +# Set of tests run by default if --pgo is specified. The tests below were
2 +# chosen based on the following criteria: either they exercise a commonly used
3 +# C extension module or type, or they run some relatively typical Python code.
4 +# Long running tests should be avoided because the PGO instrumented executable
5 +# runs slowly.
6 +PGO_TESTS = [
7 +'test_array',
8 +'test_base64',
9 +'test_binascii',
10 +'test_binop',
11 +'test_bisect',
12 +'test_bytes',
13 +'test_cmath',
14 +'test_codecs',
15 +'test_collections',
16 +'test_complex',
17 +'test_dataclasses',
18 +'test_datetime',
19 +'test_decimal',
20 +'test_difflib',
21 +'test_embed',
22 +'test_float',
23 +'test_fstring',
24 +'test_functools',
25 +'test_generators',
26 +'test_hashlib',
27 +'test_heapq',
28 +'test_int',
29 +'test_itertools',
30 +'test_json',
31 +'test_long',
32 +'test_math',
33 +'test_memoryview',
34 +'test_operator',
35 +'test_ordered_dict',
36 +'test_pickle',
37 +'test_pprint',
38 +'test_re',
39 +'test_set',
40 +'test_statistics',
41 +'test_struct',
42 +'test_tabnanny',
43 +'test_time',
44 +'test_unicode',
45 +'test_xml_etree',
46 +'test_xml_etree_c',
47 +]
48 +
49 +def setup_pgo_tests(ns):
50 +if not ns.args and not ns.pgo_extended:
51 +# run default set of tests for PGO training
52 +ns.args = PGO_TESTS[:]
Original file line number Diff line number Diff line change
@@ -255,9 +255,10 @@ TCLTK_INCLUDES= @TCLTK_INCLUDES@
255 255 TCLTK_LIBS= @TCLTK_LIBS@
256 256
257 257 # The task to run while instrumented when building the profile-opt target.
258 -# We exclude unittests with -x that take a rediculious amount of time to
259 -# run in the instrumented training build or do not provide much value.
260 -PROFILE_TASK=-m test.regrtest --pgo
258 +# To speed up profile generation, we don't run the full unit test suite
259 +# by default. The default is "-m test --pgo". To run more tests, use
260 +# PROFILE_TASK="-m test --pgo-extended"
261 +PROFILE_TASK= @PROFILE_TASK@
261 262
262 263 # report files for gcov / lcov coverage report
263 264 COVERAGE_INFO= $(abs_builddir)/coverage.info
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
1 +Reduce the number of unit tests run for the PGO generation task. This
2 +speeds up the task by a factor of about 15x. Running the full unit test
3 +suite is slow. This change may result in a slightly less optimized build
4 +since not as many code branches will be executed. If you are willing to
5 +wait for the much slower build, the old behavior can be restored using
6 +'./configure [..] PROFILE_TASK="-m test --pgo-extended"'. We make no
7 +guarantees as to which PGO task set produces a faster build. Users who
8 +care should run their own relevant benchmarks as results can depend on
9 +the environment, workload, and compiler tool chain.
Original file line number Diff line number Diff line change
@@ -686,6 +686,7 @@ target_vendor
686 686 target_cpu
687 687 target
688 688 LLVM_AR
689 +PROFILE_TASK
689 690 DEF_MAKE_RULE
690 691 DEF_MAKE_ALL_RULE
691 692 ABIFLAGS
@@ -856,6 +857,7 @@ LDFLAGS
856 857 LIBS
857 858 CPPFLAGS
858 859 CPP
860 +PROFILE_TASK
859 861 PKG_CONFIG
860 862 PKG_CONFIG_PATH
861 863 PKG_CONFIG_LIBDIR'
@@ -1559,6 +1561,8 @@ Some influential environment variables:
1559 1561 CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if
1560 1562 you have headers in a nonstandard directory
1561 1563 CPP C preprocessor
1564 + PROFILE_TASK
1565 + Python args for PGO generation task
1562 1566 PKG_CONFIG path to pkg-config utility
1563 1567 PKG_CONFIG_PATH
1564 1568 directories to add to pkg-config's search path
@@ -6426,6 +6430,16 @@ else
6426 6430 DEF_MAKE_RULE="all"
6427 6431 fi
6428 6432
6433 +
6434 +{ asecho"as_echo "asecho"as_me:${as_lineno-$LINENO}: checking PROFILE_TASK" >&5
6435 +$as_echo_n "checking PROFILE_TASK... " >&6; }
6436 +if test -z "$PROFILE_TASK"
6437 +then
6438 + PROFILE_TASK='-m test --pgo'
6439 +fi
6440 +{ asecho"as_echo "asecho"as_me:${as_lineno-$LINENO}: result: $PROFILE_TASK" >&5
6441 +$as_echo "$PROFILE_TASK" >&6; }
6442 +
6429 6443 # Make llvm-relatec checks work on systems where llvm tools are not installed with their
6430 6444 # normal names in the default $PATH (ie: Ubuntu). They exist under the
6431 6445 # non-suffixed name in their versioned llvm directory.
Original file line number Diff line number Diff line change
@@ -1293,6 +1293,14 @@ else
1293 1293 DEF_MAKE_RULE="all"
1294 1294 fi
1295 1295
1296 +AC_ARG_VAR(PROFILE_TASK, Python args for PGO generation task)
1297 +AC_MSG_CHECKING(PROFILE_TASK)
1298 +if test -z "$PROFILE_TASK"
1299 +then
1300 + PROFILE_TASK='-m test --pgo'
1301 +fi
1302 +AC_MSG_RESULT($PROFILE_TASK)
1303 +
1296 1304 # Make llvm-relatec checks work on systems where llvm tools are not installed with their
1297 1305 # normal names in the default $PATH (ie: Ubuntu). They exist under the
1298 1306 # non-suffixed name in their versioned llvm directory.