bpo-30283: Backport regrtest features from master to 2.7 (#1516) · python/cpython@453a685 (original) (raw)

`@@ -27,7 +27,7 @@

27

`-w/--verbose2 -- re-run failed tests in verbose mode

28

`-W/--verbose3 -- re-run failed tests in verbose mode immediately

29

`-q/--quiet -- no output unless one or more tests fail

30

-S/--slow -- print the slowest 10 tests

30

-S/--slowest -- print the slowest 10 tests

31

` --header -- print header with interpreter info

32

33

`Selecting tests

62

`-P/--pgo -- enable Profile Guided Optimization training

63

`--testdir -- execute test files in the specified directory

64

` (instead of the Python stdlib test suite)

65

--list-tests -- only write the name of tests that will be run,

66

don't execute them

65

67

66

68

67

69

`Additional Option Details:

158

160

`"""

159

161

160

162

`import StringIO

163

import datetime

161

164

`import getopt

162

165

`import json

163

166

`import os

226

229

`INTERRUPTED = -4

227

230

`CHILD_ERROR = -5 # error in a child process

228

231

232

Minimum duration of a test to display its duration or to mention that

233

the test is running in background

234

PROGRESS_MIN_TIME = 30.0 # seconds

235

+

236

Display the running tests if nothing happened last N seconds

237

PROGRESS_UPDATE = 30.0 # seconds

238

+

229

239

`from test import test_support

230

240

231

241

`RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network', 'bsddb',

`@@ -241,6 +251,32 @@ def usage(code, msg=''):

241

251

`sys.exit(code)

242

252

243

253

254

def format_duration(seconds):

255

if seconds < 1.0:

256

return '%.0f ms' % (seconds * 1e3)

257

if seconds < 60.0:

258

return '%.0f sec' % seconds

259

+

260

minutes, seconds = divmod(seconds, 60.0)

261

return '%.0f min %.0f sec' % (minutes, seconds)

262

+

263

+

264

_FORMAT_TEST_RESULT = {

265

PASSED: '%s passed',

266

FAILED: '%s failed',

267

ENV_CHANGED: '%s failed (env changed)',

268

SKIPPED: '%s skipped',

269

RESOURCE_DENIED: '%s skipped (resource denied)',

270

INTERRUPTED: '%s interrupted',

271

CHILD_ERROR: '%s crashed',

272

}

273

+

274

+

275

def format_test_result(test_name, result):

276

fmt = _FORMAT_TEST_RESULT.get(result, "%s")

277

return fmt % test_name

278

+

279

+

244

280

`def main(tests=None, testdir=None, verbose=0, quiet=False,

245

281

`exclude=False, single=False, randomize=False, fromfile=None,

246

282

`findleaks=False, use_resources=None, trace=False, coverdir='coverage',

`@@ -269,16 +305,18 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,

269

305

` directly to set the values that would normally be set by flags

270

306

` on the command line.

271

307

` """

308

regrtest_start_time = time.time()

272

309

273

310

`test_support.record_original_stdout(sys.stdout)

274

311

`try:

275

312

`opts, args = getopt.getopt(sys.argv[1:], 'hvqxsSrf:lu:t:TD:NLR:FwWM:j:PGm:',

276

313

` ['help', 'verbose', 'verbose2', 'verbose3', 'quiet',

277

'exclude', 'single', 'slow', 'randomize', 'fromfile=', 'findleaks',

314

'exclude', 'single', 'slow', 'slowest', 'randomize', 'fromfile=',

315

'findleaks',

278

316

`'use=', 'threshold=', 'trace', 'coverdir=', 'nocoverdir',

279

317

`'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=',

280

318

`'multiprocess=', 'slaveargs=', 'forever', 'header', 'pgo',

281

'failfast', 'match=', 'testdir='])

319

'failfast', 'match=', 'testdir=', 'list-tests'])

282

320

`except getopt.error, msg:

283

321

`usage(2, msg)

284

322

`@@ -288,6 +326,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,

288

326

`if use_resources is None:

289

327

`use_resources = []

290

328

`slaveargs = None

329

list_tests = False

291

330

`for o, a in opts:

292

331

`if o in ('-h', '--help'):

293

332

`usage(0)

`@@ -306,7 +345,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,

306

345

`exclude = True

307

346

`elif o in ('-s', '--single'):

308

347

`single = True

309

elif o in ('-S', '--slow'):

348

elif o in ('-S', '--slow', '--slowest'):

310

349

`print_slow = True

311

350

`elif o in ('-r', '--randomize'):

312

351

`randomize = True

`@@ -373,8 +412,10 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,

373

412

`slaveargs = a

374

413

`elif o in ('-P', '--pgo'):

375

414

`pgo = True

376

elif o in ('--testdir'):

415

elif o == '--testdir':

377

416

`testdir = a

417

elif o == '--list-tests':

418

list_tests = True

378

419

`else:

379

420

`print >>sys.stderr, ("No handler for option {}. Please "

380

421

`"report this as a bug at http://bugs.python.org.").format(o)

`@@ -482,6 +523,12 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,

482

523

`random.seed(random_seed)

483

524

`print "Using random seed", random_seed

484

525

`random.shuffle(selected)

526

+

527

if list_tests:

528

for name in selected:

529

print(name)

530

sys.exit(0)

531

+

485

532

`if trace:

486

533

`import trace

487

534

`tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix],

`@@ -525,13 +572,27 @@ def test_forever(tests=list(selected)):

525

572

`test_count = '/{}'.format(len(selected))

526

573

`test_count_width = len(test_count) - 1

527

574

575

def display_progress(test_index, test):

576

"[ 51/405/1] test_tcl"

577

fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}"

578

line = fmt.format(test_count_width, test_index, test_count,

579

len(bad), test)

580

+

581

add the timestamp prefix: "0:01:05 "

582

test_time = time.time() - regrtest_start_time

583

test_time = datetime.timedelta(seconds=int(test_time))

584

line = "%s %s" % (test_time, line)

585

+

586

print(line)

587

sys.stdout.flush()

588

+

528

589

`if use_mp:

529

590

`try:

530

591

`from threading import Thread

531

592

`except ImportError:

532

593

`print "Multiprocess option requires thread support"

533

594

`sys.exit(2)

534

from Queue import Queue

595

from Queue import Queue, Empty

535

596

`from subprocess import Popen, PIPE

536

597

`debug_output_pat = re.compile(r"[\d+ refs]$")

537

598

`output = Queue()

`@@ -551,63 +612,106 @@ def tests_and_args():

551

612

`# required to spawn a new process with PGO flag on/off

552

613

`if pgo:

553

614

`base_cmd = base_cmd + ['--pgo']

554

def work():

555

A worker thread.

556

try:

557

while True:

558

try:

559

test, args_tuple = next(pending)

560

except StopIteration:

561

output.put((None, None, None, None))

562

return

563

-E is needed by some tests, e.g. test_import

564

args = base_cmd + ['--slaveargs', json.dumps(args_tuple)]

565

if testdir:

566

args.extend(('--testdir', testdir))

615

+

616

class MultiprocessThread(Thread):

617

current_test = None

618

start_time = None

619

+

620

def runtest(self):

621

try:

622

test, args_tuple = next(pending)

623

except StopIteration:

624

output.put((None, None, None, None))

625

return True

626

+

627

-E is needed by some tests, e.g. test_import

628

args = base_cmd + ['--slaveargs', json.dumps(args_tuple)]

629

if testdir:

630

args.extend(('--testdir', testdir))

631

try:

632

self.start_time = time.time()

633

self.current_test = test

567

634

`popen = Popen(args,

568

635

`stdout=PIPE, stderr=PIPE,

569

636

`universal_newlines=True,

570

637

`close_fds=(os.name != 'nt'))

571

638

`stdout, stderr = popen.communicate()

572

639

`retcode = popen.wait()

640

finally:

641

self.current_test = None

573

642

574

Strip last refcount output line if it exists, since it

575

comes from the shutdown of the interpreter in the subcommand.

576

stderr = debug_output_pat.sub("", stderr)

643

Strip last refcount output line if it exists, since it

644

comes from the shutdown of the interpreter in the subcommand.

645

stderr = debug_output_pat.sub("", stderr)

577

646

578

if retcode == 0:

579

stdout, _, result = stdout.strip().rpartition("\n")

580

if not result:

581

output.put((None, None, None, None))

582

return

647

if retcode == 0:

648

stdout, _, result = stdout.strip().rpartition("\n")

649

if not result:

650

output.put((None, None, None, None))

651

return True

583

652

584

result = json.loads(result)

585

else:

586

result = (CHILD_ERROR, "Exit code %s" % retcode)

653

result = json.loads(result)

654

else:

655

result = (CHILD_ERROR, "Exit code %s" % retcode)

587

656

588

output.put((test, stdout.rstrip(), stderr.rstrip(), result))

589

except BaseException:

590

output.put((None, None, None, None))

591

raise

657

output.put((test, stdout.rstrip(), stderr.rstrip(), result))

658

return False

592

659

593

workers = [Thread(target=work) for i in range(use_mp)]

660

def run(self):

661

try:

662

stop = False

663

while not stop:

664

stop = self.runtest()

665

except BaseException:

666

output.put((None, None, None, None))

667

raise

668

+

669

workers = [MultiprocessThread() for i in range(use_mp)]

670

print("Run tests in parallel using %s child processes"

671

% len(workers))

594

672

`for worker in workers:

595

673

`worker.start()

596

674

675

def get_running(workers):

676

running = []

677

for worker in workers:

678

current_test = worker.current_test

679

if not current_test:

680

continue

681

dt = time.time() - worker.start_time

682

if dt >= PROGRESS_MIN_TIME:

683

running.append('%s (%.0f sec)' % (current_test, dt))

684

return running

685

+

597

686

`finished = 0

598

687

`test_index = 1

688

get_timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME)

599

689

`try:

600

690

`while finished < use_mp:

601

test, stdout, stderr, result = output.get()

691

try:

692

item = output.get(timeout=get_timeout)

693

except Empty:

694

running = get_running(workers)

695

if running and not pgo:

696

print('running: %s' % ', '.join(running))

697

continue

698

+

699

test, stdout, stderr, result = item

602

700

`if test is None:

603

701

`finished += 1

604

702

`continue

605

703

`accumulate_result(test, result)

606

704

`if not quiet:

607

fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}"

608

print(fmt.format(

609

test_count_width, test_index, test_count,

610

len(bad), test))

705

ok, test_time = result

706

text = format_test_result(test, ok)

707

if (ok not in (CHILD_ERROR, INTERRUPTED)

708

and test_time >= PROGRESS_MIN_TIME

709

and not pgo):

710

text += ' (%.0f sec)' % test_time

711

running = get_running(workers)

712

if running and not pgo:

713

text += ' -- running: %s' % ', '.join(running)

714

display_progress(test_index, text)

611

715

612

716

`if stdout:

613

717

`print stdout

`@@ -627,18 +731,22 @@ def work():

627

731

`for worker in workers:

628

732

`worker.join()

629

733

`else:

734

print("Run tests sequentially")

735

+

736

previous_test = None

630

737

`for test_index, test in enumerate(tests, 1):

631

738

`if not quiet:

632

fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}"

633

print(fmt.format(

634

test_count_width, test_index, test_count, len(bad), test))

635

sys.stdout.flush()

739

text = test

740

if previous_test:

741

text = '%s -- %s' % (text, previous_test)

742

display_progress(test_index, text)

636

743

`if trace:

637

744

`# If we're tracing code coverage, then we don't exit with status

638

745

`# if on a false return value from main.

639

746

`tracer.runctx('runtest(test, verbose, quiet, testdir=testdir)',

640

747

`globals=globals(), locals=vars())

641

748

`else:

749

start_time = time.time()

642

750

`try:

643

751

`result = runtest(test, verbose, quiet, huntrleaks, None, pgo,

644

752

`failfast=failfast,

`@@ -655,6 +763,16 @@ def work():

655

763

`break

656

764

`except:

657

765

`raise

766

+

767

previous_test = format_test_result(test, result[0])

768

test_time = time.time() - start_time

769

if test_time >= PROGRESS_MIN_TIME:

770

previous_test = "%s in %s" % (previous_test,

771

format_duration(test_time))

772

elif result[0] == PASSED:

773

be quiet: say nothing if the test passed shortly

774

previous_test = None

775

+

658

776

`if findleaks:

659

777

`gc.collect()

660

778

`if gc.garbage:

`@@ -683,8 +801,8 @@ def work():

683

801

`if print_slow:

684

802

`test_times.sort(reverse=True)

685

803

`print "10 slowest tests:"

686

for time, test in test_times[:10]:

687

print "%s: %.1fs" % (test, time)

804

for test_time, test in test_times[:10]:

805

print("- %s: %.1fs" % (test, test_time))

688

806

`if bad and not pgo:

689

807

`print count(len(bad), "test"), "failed:"

690

808

`printlist(bad)

`@@ -745,6 +863,18 @@ def work():

745

863

`if runleaks:

746

864

`os.system("leaks %d" % os.getpid())

747

865

866

867

duration = time.time() - regrtest_start_time

868

print("Total duration: %s" % format_duration(duration))

869

+

870

if bad:

871

result = "FAILURE"

872

elif interrupted:

873

result = "INTERRUPTED"

874

else:

875

result = "SUCCESS"

876

print("Tests result: %s" % result)

877

+

748

878

`sys.exit(len(bad) > 0 or interrupted)

749

879

750

880