Auto merge of #127636 - nnethercote:fix-Parser-look_ahead, r=oli-obk · rust-lang/rust@62c068f (original) (raw)
`@@ -1376,6 +1376,127 @@ fn ttdelim_span() {
`
1376
1376
`});
`
1377
1377
`}
`
1378
1378
``
``
1379
`+
// Uses a macro rather than a function so that failure messages mention the
`
``
1380
`+
// correct line in the test function.
`
``
1381
`+
macro_rules! look {
`
``
1382
`+
($p:ident, dist:literal,dist:literal, dist:literal,kind:expr) => {
`
``
1383
`+
p.lookahead(p.look_ahead(p.lookahead(dist, |tok| assert_eq!($kind, tok.kind));
`
``
1384
`+
};
`
``
1385
`+
}
`
``
1386
+
``
1387
`+
#[test]
`
``
1388
`+
fn look_ahead() {
`
``
1389
`+
create_default_session_globals_then(|| {
`
``
1390
`+
let sym_f = Symbol::intern("f");
`
``
1391
`+
let sym_x = Symbol::intern("x");
`
``
1392
`+
#[allow(non_snake_case)]
`
``
1393
`+
let sym_S = Symbol::intern("S");
`
``
1394
`+
let raw_no = IdentIsRaw::No;
`
``
1395
+
``
1396
`+
let psess = psess();
`
``
1397
`+
let mut p = string_to_parser(&psess, "fn f(x: u32) { x } struct S;".to_string());
`
``
1398
+
``
1399
`` +
// Current position is the fn.
``
``
1400
`+
look!(p, 0, token::Ident(kw::Fn, raw_no));
`
``
1401
`+
look!(p, 1, token::Ident(sym_f, raw_no));
`
``
1402
`+
look!(p, 2, token::OpenDelim(Delimiter::Parenthesis));
`
``
1403
`+
look!(p, 3, token::Ident(sym_x, raw_no));
`
``
1404
`+
look!(p, 4, token::Colon);
`
``
1405
`+
look!(p, 5, token::Ident(sym::u32, raw_no));
`
``
1406
`+
look!(p, 6, token::CloseDelim(Delimiter::Parenthesis));
`
``
1407
`+
look!(p, 7, token::OpenDelim(Delimiter::Brace));
`
``
1408
`+
look!(p, 8, token::Ident(sym_x, raw_no));
`
``
1409
`+
look!(p, 9, token::CloseDelim(Delimiter::Brace));
`
``
1410
`+
look!(p, 10, token::Ident(kw::Struct, raw_no));
`
``
1411
`+
look!(p, 11, token::Ident(sym_S, raw_no));
`
``
1412
`+
look!(p, 12, token::Semi);
`
``
1413
`` +
// Any lookahead past the end of the token stream returns Eof.
``
``
1414
`+
look!(p, 13, token::Eof);
`
``
1415
`+
look!(p, 14, token::Eof);
`
``
1416
`+
look!(p, 15, token::Eof);
`
``
1417
`+
look!(p, 100, token::Eof);
`
``
1418
+
``
1419
`` +
// Move forward to the first x.
``
``
1420
`+
for _ in 0..3 {
`
``
1421
`+
p.bump();
`
``
1422
`+
}
`
``
1423
`+
look!(p, 0, token::Ident(sym_x, raw_no));
`
``
1424
`+
look!(p, 1, token::Colon);
`
``
1425
`+
look!(p, 2, token::Ident(sym::u32, raw_no));
`
``
1426
`+
look!(p, 3, token::CloseDelim(Delimiter::Parenthesis));
`
``
1427
`+
look!(p, 4, token::OpenDelim(Delimiter::Brace));
`
``
1428
`+
look!(p, 5, token::Ident(sym_x, raw_no));
`
``
1429
`+
look!(p, 6, token::CloseDelim(Delimiter::Brace));
`
``
1430
`+
look!(p, 7, token::Ident(kw::Struct, raw_no));
`
``
1431
`+
look!(p, 8, token::Ident(sym_S, raw_no));
`
``
1432
`+
look!(p, 9, token::Semi);
`
``
1433
`+
look!(p, 10, token::Eof);
`
``
1434
`+
look!(p, 11, token::Eof);
`
``
1435
`+
look!(p, 100, token::Eof);
`
``
1436
+
``
1437
`` +
// Move forward to the ;.
``
``
1438
`+
for _ in 0..9 {
`
``
1439
`+
p.bump();
`
``
1440
`+
}
`
``
1441
`+
look!(p, 0, token::Semi);
`
``
1442
`` +
// Any lookahead past the end of the token stream returns Eof.
``
``
1443
`+
look!(p, 1, token::Eof);
`
``
1444
`+
look!(p, 100, token::Eof);
`
``
1445
+
``
1446
`` +
// Move one past the ;, i.e. past the end of the token stream.
``
``
1447
`+
p.bump();
`
``
1448
`+
look!(p, 0, token::Eof);
`
``
1449
`+
look!(p, 1, token::Eof);
`
``
1450
`+
look!(p, 100, token::Eof);
`
``
1451
+
``
1452
`+
// Bumping after Eof is idempotent.
`
``
1453
`+
p.bump();
`
``
1454
`+
look!(p, 0, token::Eof);
`
``
1455
`+
look!(p, 1, token::Eof);
`
``
1456
`+
look!(p, 100, token::Eof);
`
``
1457
`+
});
`
``
1458
`+
}
`
``
1459
+
``
1460
`` +
/// There used to be some buggy behaviour when using look_ahead not within
``
``
1461
`+
/// the outermost token stream, which this test covers.
`
``
1462
`+
#[test]
`
``
1463
`+
fn look_ahead_non_outermost_stream() {
`
``
1464
`+
create_default_session_globals_then(|| {
`
``
1465
`+
let sym_f = Symbol::intern("f");
`
``
1466
`+
let sym_x = Symbol::intern("x");
`
``
1467
`+
#[allow(non_snake_case)]
`
``
1468
`+
let sym_S = Symbol::intern("S");
`
``
1469
`+
let raw_no = IdentIsRaw::No;
`
``
1470
+
``
1471
`+
let psess = psess();
`
``
1472
`+
let mut p = string_to_parser(&psess, "mod m { fn f(x: u32) { x } struct S; }".to_string());
`
``
1473
+
``
1474
`` +
// Move forward to the fn, which is not within the outermost token
``
``
1475
`` +
// stream (because it's inside the mod { ... }).
``
``
1476
`+
for _ in 0..3 {
`
``
1477
`+
p.bump();
`
``
1478
`+
}
`
``
1479
`+
look!(p, 0, token::Ident(kw::Fn, raw_no));
`
``
1480
`+
look!(p, 1, token::Ident(sym_f, raw_no));
`
``
1481
`+
look!(p, 2, token::OpenDelim(Delimiter::Parenthesis));
`
``
1482
`+
look!(p, 3, token::Ident(sym_x, raw_no));
`
``
1483
`+
look!(p, 4, token::Colon);
`
``
1484
`+
look!(p, 5, token::Ident(sym::u32, raw_no));
`
``
1485
`+
look!(p, 6, token::CloseDelim(Delimiter::Parenthesis));
`
``
1486
`+
look!(p, 7, token::OpenDelim(Delimiter::Brace));
`
``
1487
`+
look!(p, 8, token::Ident(sym_x, raw_no));
`
``
1488
`+
look!(p, 9, token::CloseDelim(Delimiter::Brace));
`
``
1489
`+
look!(p, 10, token::Ident(kw::Struct, raw_no));
`
``
1490
`+
look!(p, 11, token::Ident(sym_S, raw_no));
`
``
1491
`+
look!(p, 12, token::Semi);
`
``
1492
`+
look!(p, 13, token::CloseDelim(Delimiter::Brace));
`
``
1493
`` +
// Any lookahead past the end of the token stream returns Eof.
``
``
1494
`+
look!(p, 14, token::Eof);
`
``
1495
`+
look!(p, 15, token::Eof);
`
``
1496
`+
look!(p, 100, token::Eof);
`
``
1497
`+
});
`
``
1498
`+
}
`
``
1499
+
1379
1500
`// This tests that when parsing a string (rather than a file) we don't try
`
1380
1501
`// and read in a file for a module declaration and just parse a stub.
`
1381
1502
`` // See recurse_into_file_modules in the parser.
``