(original) (raw)

--- Parser/tokenizer.c.orig Tue Feb 4 21:54:46 2003 +++ Parser/tokenizer.c Tue Feb 4 23:00:17 2003 @@ -506,14 +506,14 @@ /* Fetch a byte from TOK, using the string buffer. */ static int buf_getc(struct tok_state *tok) { - return *tok->str++; + return Py_CHARMASK(*tok->str++); } /* Unfetch a byte from TOK, using the string buffer. */ static void buf_ungetc(int c, struct tok_state *tok) { tok->str--; - assert(*tok->str == c); /* tok->cur may point to read-only segment */ + assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */ } /* Set the readline function for TOK to ENC. For the string-based