bpo-30377: Simplify handling of COMMENT and NL in tokenize.py (#1607) · python/cpython@c471ca4 (original) (raw)

`@@ -39,20 +39,22 @@ def test_basic(self):

`

39

39

` """)

`

40

40

`self.check_tokenize("if False:\n"

`

41

41

`" # NL\n"

`

``

42

`+

" \n"

`

42

43

`" True = False # NEWLINE\n", """\

`

43

44

` NAME 'if' (1, 0) (1, 2)

`

44

45

` NAME 'False' (1, 3) (1, 8)

`

45

46

` OP ':' (1, 8) (1, 9)

`

46

47

` NEWLINE '\n' (1, 9) (1, 10)

`

47

48

` COMMENT '# NL' (2, 4) (2, 8)

`

48

49

` NL '\n' (2, 8) (2, 9)

`

49

``

`-

INDENT ' ' (3, 0) (3, 4)

`

50

``

`-

NAME 'True' (3, 4) (3, 8)

`

51

``

`-

OP '=' (3, 9) (3, 10)

`

52

``

`-

NAME 'False' (3, 11) (3, 16)

`

53

``

`-

COMMENT '# NEWLINE' (3, 17) (3, 26)

`

54

``

`-

NEWLINE '\n' (3, 26) (3, 27)

`

55

``

`-

DEDENT '' (4, 0) (4, 0)

`

``

50

`+

NL '\n' (3, 4) (3, 5)

`

``

51

`+

INDENT ' ' (4, 0) (4, 4)

`

``

52

`+

NAME 'True' (4, 4) (4, 8)

`

``

53

`+

OP '=' (4, 9) (4, 10)

`

``

54

`+

NAME 'False' (4, 11) (4, 16)

`

``

55

`+

COMMENT '# NEWLINE' (4, 17) (4, 26)

`

``

56

`+

NEWLINE '\n' (4, 26) (4, 27)

`

``

57

`+

DEDENT '' (5, 0) (5, 0)

`

56

58

` """)

`

57

59

`indent_error_file = b"""\

`

58

60

`def k(x):

`