Revert "bpo-30406: Make async and await proper keywords (GH-1669)" (G… · python/cpython@a90df50 (original) (raw)
`@@ -234,7 +234,7 @@ def compat(self, token, iterable):
`
234
234
`for tok in iterable:
`
235
235
`toknum, tokval = tok[:2]
`
236
236
``
237
``
`-
if toknum in (NAME, NUMBER):
`
``
237
`+
if toknum in (NAME, NUMBER, ASYNC, AWAIT):
`
238
238
`tokval += ' '
`
239
239
``
240
240
`if toknum == INDENT:
`
`@@ -380,6 +380,12 @@ def generate_tokens(readline):
`
380
380
`contline = None
`
381
381
`indents = [0]
`
382
382
``
``
383
`+
'stashed' and 'async_*' are used for async/await parsing
`
``
384
`+
stashed = None
`
``
385
`+
async_def = False
`
``
386
`+
async_def_indent = 0
`
``
387
`+
async_def_nl = False
`
``
388
+
383
389
`while 1: # loop over lines in stream
`
384
390
`try:
`
385
391
`line = readline()
`
`@@ -420,6 +426,10 @@ def generate_tokens(readline):
`
420
426
`pos = pos + 1
`
421
427
`if pos == max: break
`
422
428
``
``
429
`+
if stashed:
`
``
430
`+
yield stashed
`
``
431
`+
stashed = None
`
``
432
+
423
433
`if line[pos] in '#\r\n': # skip comments or blank lines
`
424
434
`if line[pos] == '#':
`
425
435
`comment_token = line[pos:].rstrip('\r\n')
`
`@@ -443,8 +453,18 @@ def generate_tokens(readline):
`
443
453
` ("", lnum, pos, line))
`
444
454
`indents = indents[:-1]
`
445
455
``
``
456
`+
if async_def and async_def_indent >= indents[-1]:
`
``
457
`+
async_def = False
`
``
458
`+
async_def_nl = False
`
``
459
`+
async_def_indent = 0
`
``
460
+
446
461
`yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
`
447
462
``
``
463
`+
if async_def and async_def_nl and async_def_indent >= indents[-1]:
`
``
464
`+
async_def = False
`
``
465
`+
async_def_nl = False
`
``
466
`+
async_def_indent = 0
`
``
467
+
448
468
`else: # continued statement
`
449
469
`if not line:
`
450
470
`raise TokenError("EOF in multi-line statement", (lnum, 0))
`
`@@ -464,17 +484,28 @@ def generate_tokens(readline):
`
464
484
`newline = NEWLINE
`
465
485
`if parenlev > 0:
`
466
486
`newline = NL
`
``
487
`+
elif async_def:
`
``
488
`+
async_def_nl = True
`
``
489
`+
if stashed:
`
``
490
`+
yield stashed
`
``
491
`+
stashed = None
`
467
492
`yield (newline, token, spos, epos, line)
`
468
493
``
469
494
`elif initial == '#':
`
470
495
`assert not token.endswith("\n")
`
``
496
`+
if stashed:
`
``
497
`+
yield stashed
`
``
498
`+
stashed = None
`
471
499
`yield (COMMENT, token, spos, epos, line)
`
472
500
`elif token in triple_quoted:
`
473
501
`endprog = endprogs[token]
`
474
502
`endmatch = endprog.match(line, pos)
`
475
503
`if endmatch: # all on one line
`
476
504
`pos = endmatch.end(0)
`
477
505
`token = line[start:pos]
`
``
506
`+
if stashed:
`
``
507
`+
yield stashed
`
``
508
`+
stashed = None
`
478
509
`yield (STRING, token, spos, (lnum, pos), line)
`
479
510
`else:
`
480
511
`strstart = (lnum, start) # multiple lines
`
`@@ -492,22 +523,63 @@ def generate_tokens(readline):
`
492
523
`contline = line
`
493
524
`break
`
494
525
`else: # ordinary string
`
``
526
`+
if stashed:
`
``
527
`+
yield stashed
`
``
528
`+
stashed = None
`
495
529
`yield (STRING, token, spos, epos, line)
`
496
530
`elif initial in namechars: # ordinary name
`
497
``
`-
yield (NAME, token, spos, epos, line)
`
``
531
`+
if token in ('async', 'await'):
`
``
532
`+
if async_def:
`
``
533
`+
yield (ASYNC if token == 'async' else AWAIT,
`
``
534
`+
token, spos, epos, line)
`
``
535
`+
continue
`
``
536
+
``
537
`+
tok = (NAME, token, spos, epos, line)
`
``
538
`+
if token == 'async' and not stashed:
`
``
539
`+
stashed = tok
`
``
540
`+
continue
`
``
541
+
``
542
`+
if token == 'def':
`
``
543
`+
if (stashed
`
``
544
`+
and stashed[0] == NAME
`
``
545
`+
and stashed[1] == 'async'):
`
``
546
+
``
547
`+
async_def = True
`
``
548
`+
async_def_indent = indents[-1]
`
``
549
+
``
550
`+
yield (ASYNC, stashed[1],
`
``
551
`+
stashed[2], stashed[3],
`
``
552
`+
stashed[4])
`
``
553
`+
stashed = None
`
``
554
+
``
555
`+
if stashed:
`
``
556
`+
yield stashed
`
``
557
`+
stashed = None
`
``
558
+
``
559
`+
yield tok
`
498
560
`elif initial == '\': # continued stmt
`
499
561
`# This yield is new; needed for better idempotency:
`
``
562
`+
if stashed:
`
``
563
`+
yield stashed
`
``
564
`+
stashed = None
`
500
565
`yield (NL, token, spos, (lnum, pos), line)
`
501
566
`continued = 1
`
502
567
`else:
`
503
568
`if initial in '([{': parenlev = parenlev + 1
`
504
569
`elif initial in ')]}': parenlev = parenlev - 1
`
``
570
`+
if stashed:
`
``
571
`+
yield stashed
`
``
572
`+
stashed = None
`
505
573
`yield (OP, token, spos, epos, line)
`
506
574
`else:
`
507
575
`yield (ERRORTOKEN, line[pos],
`
508
576
` (lnum, pos), (lnum, pos+1), line)
`
509
577
`pos = pos + 1
`
510
578
``
``
579
`+
if stashed:
`
``
580
`+
yield stashed
`
``
581
`+
stashed = None
`
``
582
+
511
583
`for indent in indents[1:]: # pop remaining indent levels
`
512
584
`yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
`
513
585
`yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
`