[issue3672] Ill-formed surrogates not treated as errors during encoding/decoding - Code Review (original) (raw)

OLD

NEW

1 /*

3 Unicode implementation based on original code by Fredrik Lundh,

4 modified by Marc-Andre Lemburg mal@lemburg.com according to the

5 Unicode Integration Proposal (see file Misc/unicode.txt).

7 Major speed upgrades to the method implementations at the Reykjavik

8 NeedForSpeed sprint, by Fredrik Lundh and Andrew Dalke.

10 Copyright (c) Corporation for National Research Initiatives.

(...skipping 136 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...

147 0, 0, 0, 0, 0, 0, 0, 0,

148 0, 0, 0, 0, 0, 0, 0, 0,

149 0, 0, 0, 0, 0, 0, 0, 0,

150 0, 0, 0, 0, 0, 0, 0, 0,

151 0, 0, 0, 0, 0, 0, 0, 0,

152 0, 0, 0, 0, 0, 0, 0, 0,

153 0, 0, 0, 0, 0, 0, 0, 0,

154 0, 0, 0, 0, 0, 0, 0, 0

155 };

156

157 static PyObject *unicode_encode_call_errorhandler(const char *errors,

158 PyObject **errorHandler,

159 const char *encoding, const ch ar *reason,

160 const Py_UNICODE *unicode, Py_ ssize_t size, PyObject **exceptionObject,

161 Py_ssize_t startpos, Py_ssize_ t endpos,

162 Py_ssize_t *newpos);

163

157 /* Same for linebreaks */

164 /* Same for linebreaks */

158 static unsigned char ascii_linebreak[] = {

165 static unsigned char ascii_linebreak[] = {

159 0, 0, 0, 0, 0, 0, 0, 0,

166 0, 0, 0, 0, 0, 0, 0, 0,

160 /* 0x000A, * LINE FEED */

167 /* 0x000A, * LINE FEED */

161 /* 0x000D, * CARRIAGE RETURN */

168 /* 0x000D, * CARRIAGE RETURN */

162 0, 0, 1, 0, 0, 1, 0, 0,

169 0, 0, 1, 0, 0, 1, 0, 0,

163 0, 0, 0, 0, 0, 0, 0, 0,

170 0, 0, 0, 0, 0, 0, 0, 0,

164 /* 0x001C, * FILE SEPARATOR */

171 /* 0x001C, * FILE SEPARATOR */

165 /* 0x001D, * GROUP SEPARATOR */

172 /* 0x001D, * GROUP SEPARATOR */

166 /* 0x001E, * RECORD SEPARATOR */

173 /* 0x001E, * RECORD SEPARATOR */

(...skipping 2040 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...

2207

2214

2208 case 3:

2215 case 3:

2209 if ((s[1] & 0xc0) != 0x80 ||

2216 if ((s[1] & 0xc0) != 0x80 ||

2210 (s[2] & 0xc0) != 0x80) {

2217 (s[2] & 0xc0) != 0x80) {

2211 errmsg = "invalid data";

2218 errmsg = "invalid data";

2212 startinpos = s-starts;

2219 startinpos = s-starts;

2213 endinpos = startinpos+3;

2220 endinpos = startinpos+3;

2214 goto utf8Error;

2221 goto utf8Error;

2215 }

2222 }

2216 ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f);

2223 ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f);

2217 if (ch < 0x0800) {

2224 if (ch < 0x0800 || (ch >= 0xd800 && ch <= 0xDFFF)) {

2218 /* Note: UTF-8 encodings of surrogates are considered

2219 legal UTF-8 sequences;

2220

2221 XXX For wide builds (UCS-4) we should probably try

2222 to recombine the surrogates into a single code

2223 unit.

2224 */

2225 errmsg = "illegal encoding";

2226 startinpos = s-starts;

2227 endinpos = startinpos+3;

2228 goto utf8Error;

2229 }

2230 else

2231 *p++ = (Py_UNICODE)ch;

2232 break;

2233

2234 case 4:

(...skipping 86 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...

2321 const char *errors)

2322 {

2323 #define MAX_SHORT_UNICHARS 300 /* largest size we'll do on the stack */

2324

2325 Py_ssize_t i; /* index into s of next input byte */

2326 PyObject *result; /* result string object */

2327 char *p; /* next free byte in output buffer */

2328 Py_ssize_t nallocated; /* number of result bytes allocated */

2329 Py_ssize_t nneeded; /* number of result bytes needed */

2330 char stackbuf[MAX_SHORT_UNICHARS * 4];

2331 PyObject *errorHandler = NULL;

2331

2332

2332 assert(s != NULL);

2333 assert(s != NULL);

2333 assert(size >= 0);

2334 assert(size >= 0);

2334

2335

2335 if (size <= MAX_SHORT_UNICHARS) {

2336 if (size <= MAX_SHORT_UNICHARS) {

2336 /* Write into the stack buffer; nallocated can't overflow.

2337 /* Write into the stack buffer; nallocated can't overflow.

2337 * At the end, we'll allocate exactly as much heap space as it

2338 * At the end, we'll allocate exactly as much heap space as it

2338 * turns out we need.

2339 * turns out we need.

2339 */

2340 */

2340 nallocated = Py_SAFE_DOWNCAST(sizeof(stackbuf), size_t, int);

2341 nallocated = Py_SAFE_DOWNCAST(sizeof(stackbuf), size_t, int);

(...skipping 19 matching lines...) Expand all Loading...

2360 *p++ = (char) ch;

2361 *p++ = (char) ch;

2361

2362

2362 else if (ch < 0x0800) {

2363 else if (ch < 0x0800) {

2363 /* Encode Latin-1 */

2364 /* Encode Latin-1 */

2364 *p++ = (char)(0xc0 | (ch >> 6));

2365 *p++ = (char)(0xc0 | (ch >> 6));

2365 *p++ = (char)(0x80 | (ch & 0x3f));

2366 *p++ = (char)(0x80 | (ch & 0x3f));

2366 }

2367 }

2367 else {

2368 else {

2368 /* Encode UCS2 Unicode ordinals */

2369 /* Encode UCS2 Unicode ordinals */

2369 if (ch < 0x10000) {

2370 if (ch < 0x10000) {

2371 #ifndef Py_UNICODE_WIDE

2370 /* Special case: check for high surrogate */

2372 /* Special case: check for high surrogate */

2371 if (0xD800 <= ch && ch <= 0xDBFF && i != size) {

2373 if (0xD800 <= ch && ch <= 0xDBFF && i != size) {

2372 Py_UCS4 ch2 = s[i];

2374 Py_UCS4 ch2 = s[i];

2373 /* Check for low surrogate and combine the two to

2375 /* Check for low surrogate and combine the two to

2374 form a UCS4 value */

2376 form a UCS4 value */

2375 if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {

2377 if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {

2376 ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000;

2378 ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000;

2377 i++;

2379 i++;

2378 goto encodeUCS4;

2380 goto encodeUCS4;

2379 }

2381 }

2380 /* Fall through: handles isolated high surrogates */

2382 /* Fall through: handles isolated high surrogates */

2381 }

2383 }

2384 #endif

2385 if (ch >= 0xd800 && ch <= 0xdfff) {

2386 PyObject *exc = NULL;

2387 Py_ssize_t newpos;

2388 PyObject *rep;

2389 char *prep;

2390 int k;

2391 rep = unicode_encode_call_errorhandler

2392 (errors, &errorHandler, "utf-8", "surrogates not allowed ",·

2393 s, size, &exc, i-1, i, &newpos);

2394 if (!rep)

2395 goto error;

2396 /* Implementation limitations: only support error handler th at return

2397 bytes, and only support up to four replacement bytes. */

2398 if (!PyBytes_Check(rep)) {

2399 PyErr_SetString(PyExc_TypeError, "error handler should h ave returned bytes");

2400 Py_DECREF(rep);

2401 goto error;

2402 }

2403 if (PyBytes_Size(rep) > 4) {

2404 PyErr_SetString(PyExc_TypeError, "error handler returned too many bytes");

2405 Py_DECREF(rep);

2406 goto error;

2407 }

2408 prep = PyBytes_AsString(rep);

2409 for(k = PyBytes_Size(rep); k > 0; k--)

2410 *p++ = *prep++;

2411 Py_DECREF(rep);

2412 continue;

2413 ····················

2414 }

2382 *p++ = (char)(0xe0 | (ch >> 12));

2415 *p++ = (char)(0xe0 | (ch >> 12));

2383 *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));

2416 *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));

2384 *p++ = (char)(0x80 | (ch & 0x3f));

2417 *p++ = (char)(0x80 | (ch & 0x3f));

2385 continue;

2418 continue;

2386 }

2419 }

2387 encodeUCS4:

2420 encodeUCS4:

2388 /* Encode UCS4 Unicode ordinals */

2421 /* Encode UCS4 Unicode ordinals */

2389 *p++ = (char)(0xf0 | (ch >> 18));

2422 *p++ = (char)(0xf0 | (ch >> 18));

2390 *p++ = (char)(0x80 | ((ch >> 12) & 0x3f));

2423 *p++ = (char)(0x80 | ((ch >> 12) & 0x3f));

2391 *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));

2424 *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));

2392 *p++ = (char)(0x80 | (ch & 0x3f));

2425 *p++ = (char)(0x80 | (ch & 0x3f));

2393 }

2426 }

2394 }

2427 }

2395

2428

2396 if (result == NULL) {

2429 if (result == NULL) {

2397 /* This was stack allocated. */

2430 /* This was stack allocated. */

2398 nneeded = p - stackbuf;

2431 nneeded = p - stackbuf;

2399 assert(nneeded <= nallocated);

2432 assert(nneeded <= nallocated);

2400 result = PyBytes_FromStringAndSize(stackbuf, nneeded);

2433 result = PyBytes_FromStringAndSize(stackbuf, nneeded);

2401 }

2434 }

2402 else {

2435 else {

2403 /* Cut back to size actually needed. */

2436 /* Cut back to size actually needed. */

2404 nneeded = p - PyBytes_AS_STRING(result);

2437 nneeded = p - PyBytes_AS_STRING(result);

2405 assert(nneeded <= nallocated);

2438 assert(nneeded <= nallocated);

2406 _PyBytes_Resize(&result, nneeded);

2439 _PyBytes_Resize(&result, nneeded);

2407 }

2440 }

2441 Py_XDECREF(errorHandler);

2408 return result;

2442 return result;

2443 error:

2444 Py_XDECREF(errorHandler);

2445 Py_XDECREF(result);

2446 return NULL;

2409

2447

2410 #undef MAX_SHORT_UNICHARS

2448 #undef MAX_SHORT_UNICHARS

2411 }

2449 }

2412

2450

2413 PyObject *PyUnicode_AsUTF8String(PyObject *unicode)

2451 PyObject *PyUnicode_AsUTF8String(PyObject *unicode)

2414 {

2452 {

2415 if (!PyUnicode_Check(unicode)) {

2453 if (!PyUnicode_Check(unicode)) {

2416 PyErr_BadArgument();

2454 PyErr_BadArgument();

2417 return NULL;

2455 return NULL;

2418 }

2456 }

(...skipping 1471 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...

3890 build arguments, call the callback and check the arguments,

3928 build arguments, call the callback and check the arguments,

3891 put the result into newpos and return the replacement string, which

3929 put the result into newpos and return the replacement string, which

3892 has to be freed by the caller */

3930 has to be freed by the caller */

3893 static PyObject *unicode_encode_call_errorhandler(const char *errors,

3931 static PyObject *unicode_encode_call_errorhandler(const char *errors,

3894 PyObject **errorHandler,

3932 PyObject **errorHandler,

3895 const char *encoding, const ch ar *reason,

3933 const char *encoding, const ch ar *reason,

3896 const Py_UNICODE *unicode, Py_ ssize_t size, PyObject **exceptionObject,

3934 const Py_UNICODE *unicode, Py_ ssize_t size, PyObject **exceptionObject,

3897 Py_ssize_t startpos, Py_ssize_ t endpos,

3935 Py_ssize_t startpos, Py_ssize_ t endpos,

3898 Py_ssize_t *newpos)

3936 Py_ssize_t *newpos)

3899 {

3937 {

3900 static char *argparse = "O!n;encoding error handler must return (str, int) t uple";

3938 static char *argparse = "On;encoding error handler must return (str/bytes, i nt) tuple";

3901

3939

3902 PyObject *restuple;

3940 PyObject *restuple;

3903 PyObject *resunicode;

3941 PyObject *resunicode;

3904

3942

3905 if (*errorHandler == NULL) {

3943 if (*errorHandler == NULL) {

3906 *errorHandler = PyCodec_LookupError(errors);

3944 *errorHandler = PyCodec_LookupError(errors);

3907 if (*errorHandler == NULL)

3945 if (*errorHandler == NULL)

3908 return NULL;

3946 return NULL;

3909 }

3947 }

3910

3948

3911 make_encode_exception(exceptionObject,

3949 make_encode_exception(exceptionObject,

3912 encoding, unicode, size, startpos, endpos, reason);

3950 encoding, unicode, size, startpos, endpos, reason);

3913 if (*exceptionObject == NULL)

3951 if (*exceptionObject == NULL)

3914 return NULL;

3952 return NULL;

3915

3953

3916 restuple = PyObject_CallFunctionObjArgs(

3954 restuple = PyObject_CallFunctionObjArgs(

3917 *errorHandler, *exceptionObject, NULL);

3955 *errorHandler, *exceptionObject, NULL);

3918 if (restuple == NULL)

3956 if (restuple == NULL)

3919 return NULL;

3957 return NULL;

3920 if (!PyTuple_Check(restuple)) {

3958 if (!PyTuple_Check(restuple)) {

3921 PyErr_SetString(PyExc_TypeError, &argparse[4]);

3959 PyErr_SetString(PyExc_TypeError, &argparse[3]);

3922 Py_DECREF(restuple);

3960 Py_DECREF(restuple);

3923 return NULL;

3961 return NULL;

3924 }

3962 }

3925 if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type,

3963 if (!PyArg_ParseTuple(restuple, argparse,

3926 &resunicode, newpos)) {

3964 &resunicode, newpos)) {

3927 Py_DECREF(restuple);

3965 Py_DECREF(restuple);

3928 return NULL;

3966 return NULL;

3929 }

3967 }

3968 if (!PyUnicode_Check(resunicode) && !PyBytes_Check(resunicode)) {

3969 PyErr_SetString(PyExc_TypeError, &argparse[3]);

3970 Py_DECREF(restuple);

3971 return NULL;

3972 }

3930 if (*newpos<0)

3973 if (*newpos<0)

3931 *newpos = size+*newpos;

3974 *newpos = size+*newpos;

3932 if (*newpos<0 || *newpos>size) {

3975 if (*newpos<0 || *newpos>size) {

3933 PyErr_Format(PyExc_IndexError, "position %zd from error handler out of b ounds", *newpos);

3976 PyErr_Format(PyExc_IndexError, "position %zd from error handler out of b ounds", *newpos);

3934 Py_DECREF(restuple);

3977 Py_DECREF(restuple);

3935 return NULL;

3978 return NULL;

3936 }

3979 }

3937 Py_INCREF(resunicode);

3980 Py_INCREF(resunicode);

3938 Py_DECREF(restuple);

3981 Py_DECREF(restuple);

3939 return resunicode;

3982 return resunicode;

(...skipping 117 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...

4057 str += sprintf(str, "&#%d;", (int)*p);

4100 str += sprintf(str, "&#%d;", (int)*p);

4058 }

4101 }

4059 p = collend;

4102 p = collend;

4060 break;

4103 break;

4061 default:

4104 default:

4062 repunicode = unicode_encode_call_errorhandler(errors, &errorHand ler,

4105 repunicode = unicode_encode_call_errorhandler(errors, &errorHand ler,

4063 encoding, reason, startp, size, &exc,

4106 encoding, reason, startp, size, &exc,

4064 collstart-startp, collend-startp, &newpos);

4107 collstart-startp, collend-startp, &newpos);

4065 if (repunicode == NULL)

4108 if (repunicode == NULL)

4066 goto onError;

4109 goto onError;

4110 if (!PyUnicode_Check(repunicode)) {

4111 /* Implementation limitation: byte results not supported yet . */

4112 PyErr_SetString(PyExc_TypeError, "error handler should retur n unicode");

4113 Py_DECREF(repunicode);

4114 goto onError;

4115 }

4067 /* need more space? (at least enough for what we

4116 /* need more space? (at least enough for what we

4068 have+the replacement+the rest of the string, so

4117 have+the replacement+the rest of the string, so

4069 we won't have to check space for encodable characters) */

4118 we won't have to check space for encodable characters) */

4070 respos = str - PyBytes_AS_STRING(res);

4119 respos = str - PyBytes_AS_STRING(res);

4071 repsize = PyUnicode_GET_SIZE(repunicode);

4120 repsize = PyUnicode_GET_SIZE(repunicode);

4072 requiredsize = respos+repsize+(endp-collend);

4121 requiredsize = respos+repsize+(endp-collend);

4073 if (requiredsize > ressize) {

4122 if (requiredsize > ressize) {

4074 if (requiredsize<2*ressize)

4123 if (requiredsize<2*ressize)

4075 requiredsize = 2*ressize;

4124 requiredsize = 2*ressize;

4076 if (_PyBytes_Resize(&res, requiredsize)) {

4125 if (_PyBytes_Resize(&res, requiredsize)) {

(...skipping 943 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...

5020 }

5069 }

5021 }

5070 }

5022 *inpos = collendpos;

5071 *inpos = collendpos;

5023 break;

5072 break;

5024 default:

5073 default:

5025 repunicode = unicode_encode_call_errorhandler(errors, errorHandler,

5074 repunicode = unicode_encode_call_errorhandler(errors, errorHandler,

5026 encoding, reason, p, size, exceptionObject,

5075 encoding, reason, p, size, exceptionObject,

5027 collstartpos, collendpos, &newpos);

5076 collstartpos, collendpos, &newpos);

5028 if (repunicode == NULL)

5077 if (repunicode == NULL)

5029 return -1;

5078 return -1;

5079 if (!PyUnicode_Check(repunicode)) {

5080 /* Implementation limitation: byte results not supported yet. */

5081 PyErr_SetString(PyExc_TypeError, "error handler should return unicod e");

5082 Py_DECREF(repunicode);

5083 return -1;

5084 }

5030 /* generate replacement */

5085 /* generate replacement */

5031 repsize = PyUnicode_GET_SIZE(repunicode);

5086 repsize = PyUnicode_GET_SIZE(repunicode);

5032 for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2) {

5087 for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2) {

5033 x = charmapencode_output(*uni2, mapping, res, respos);

5088 x = charmapencode_output(*uni2, mapping, res, respos);

5034 if (x==enc_EXCEPTION) {

5089 if (x==enc_EXCEPTION) {

5035 return -1;

5090 return -1;

5036 }

5091 }

5037 else if (x==enc_FAILED) {

5092 else if (x==enc_FAILED) {

5038 Py_DECREF(repunicode);

5093 Py_DECREF(repunicode);

5039 raise_encode_exception(exceptionObject, encoding, p, size, colls tartpos, collendpos, reason);

5094 raise_encode_exception(exceptionObject, encoding, p, size, colls tartpos, collendpos, reason);

(...skipping 541 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...

5581 for (p = collstart; p < collend; ++p)

5636 for (p = collstart; p < collend; ++p)

5582 output += sprintf(output, "&#%d;", (int)*p);

5637 output += sprintf(output, "&#%d;", (int)*p);

5583 p = collend;

5638 p = collend;

5584 break;

5639 break;

5585 default:

5640 default:

5586 repunicode = unicode_encode_call_errorhandler(errors, &errorHandler,

5641 repunicode = unicode_encode_call_errorhandler(errors, &errorHandler,

5587 encoding, reason, s, l ength, &exc,

5642 encoding, reason, s, l ength, &exc,

5588 collstart-s, collend-s , &newpos);

5643 collstart-s, collend-s , &newpos);

5589 if (repunicode == NULL)

5644 if (repunicode == NULL)

5590 goto onError;

5645 goto onError;

5646 if (!PyUnicode_Check(repunicode)) {

5647 /* Implementation limitation: byte results not supported yet. */

5648 PyErr_SetString(PyExc_TypeError, "error handler should return un icode");

5649 Py_DECREF(repunicode);

5650 goto onError;

5651 }

5591 /* generate replacement */

5652 /* generate replacement */

5592 repsize = PyUnicode_GET_SIZE(repunicode);

5653 repsize = PyUnicode_GET_SIZE(repunicode);

5593 for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2) {

5654 for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2) {

5594 Py_UNICODE ch = *uni2;

5655 Py_UNICODE ch = *uni2;

5595 if (Py_UNICODE_ISSPACE(ch))

5656 if (Py_UNICODE_ISSPACE(ch))

5596 *output++ = ' ';

5657 *output++ = ' ';

5597 else {

5658 else {

5598 decimal = Py_UNICODE_TODECIMAL(ch);

5659 decimal = Py_UNICODE_TODECIMAL(ch);

5599 if (decimal >= 0)

5660 if (decimal >= 0)

5600 *output++ = '0' + decimal;

5661 *output++ = '0' + decimal;

(...skipping 4265 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...

9866 }

9927 }

9867 #endif

9928 #endif

9868

9929

9869

9930

9870 /*

9931 /*

9871 Local variables:

9932 Local variables:

9872 c-basic-offset: 4

9933 c-basic-offset: 4

9873 indent-tabs-mode: nil

9934 indent-tabs-mode: nil

9874 End:

9935 End:

9875 */

9936 */

OLD

NEW