TokenLexer.cpp Source File (original) (raw)

27#include "llvm/ADT/ArrayRef.h"

28#include "llvm/ADT/STLExtras.h"

29#include "llvm/ADT/SmallVector.h"

30#include "llvm/ADT/iterator_range.h"

31#include

32#include

33#include

35using namespace clang;

43 destroy();

45 Macro = MI;

46 ActualArgs = Actuals;

47 CurTokenIdx = 0;

49 ExpandLocStart = Tok.getLocation();

50 ExpandLocEnd = ELEnd;

51 AtStartOfLine = Tok.isAtStartOfLine();

52 HasLeadingSpace = Tok.hasLeadingSpace();

53 NextTokGetsSpace = false;

54 Tokens = &*Macro->tokens_begin();

55 OwnsTokens = false;

56 DisableMacroExpansion = false;

57 IsReinject = false;

58 NumTokens = Macro->tokens_end()-Macro->tokens_begin();

62 MacroStartSLocOffset = SM.getNextLocalOffset();

64 if (NumTokens > 0) {

65 assert(Tokens[0].getLocation().isValid());

66 assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&

67 "Macro defined in macro?");

68 assert(ExpandLocStart.isValid());

74 MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation());

75 MacroDefLength = Macro->getDefinitionLength(SM);

76 MacroExpansionStart = SM.createExpansionLoc(MacroDefStart,

77 ExpandLocStart,

78 ExpandLocEnd,

79 MacroDefLength);

80 }

84 if (Macro->isFunctionLike() && Macro->getNumParams())

85 ExpandFunctionArguments();

90 Macro->DisableMacro();

91}

96 bool disableMacroExpansion, bool ownsTokens,

97 bool isReinject) {

98 assert(!isReinject || disableMacroExpansion);

100

101 destroy();

102

103 Macro = nullptr;

104 ActualArgs = nullptr;

105 Tokens = TokArray;

106 OwnsTokens = ownsTokens;

107 DisableMacroExpansion = disableMacroExpansion;

108 IsReinject = isReinject;

109 NumTokens = NumToks;

110 CurTokenIdx = 0;

112 AtStartOfLine = false;

113 HasLeadingSpace = false;

114 NextTokGetsSpace = false;

116

117

118

119 if (NumToks != 0) {

122 }

123}

124

125void TokenLexer::destroy() {

126

127

128 if (OwnsTokens) {

129 delete [] Tokens;

130 Tokens = nullptr;

131 OwnsTokens = false;

132 }

133

134

135 if (ActualArgs) ActualArgs->destroy(PP);

136}

137

138bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(

141

142 if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1)

143 return false;

144

145

146

147

148 if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat)

149 return false;

150

151

152

153

154

155

156 if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode

157 && Macro->getNumParams() < 2)

158 return false;

159

160

161 if (ResultToks.empty() || !ResultToks.back().is(tok::comma))

162 return false;

163

164

165 if (HasPasteOperator)

166 PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);

167

168

169 ResultToks.pop_back();

170

171 if (!ResultToks.empty()) {

172

173

174

175

176 if (ResultToks.back().is(tok::hashhash))

177 ResultToks.pop_back();

178

179

181 }

182

183

184 NextTokGetsSpace = false;

185 return true;

186}

187

188void TokenLexer::stringifyVAOPTContents(

192 const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;

193 Token *const VAOPTTokens =

194 NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] : nullptr;

195

196 SmallVector<Token, 64> ConcatenatedVAOPTResultToks;

197

198

199

200

201 for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;

202 ++CurTokenIdx) {

203 if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) {

204 assert(CurTokenIdx != 0 &&

205 "Can not have __VAOPT__ contents begin with a ##");

206 Token &LHS = VAOPTTokens[CurTokenIdx - 1];

207 pasteTokens(LHS, llvm::ArrayRef(VAOPTTokens, NumVAOptTokens),

208 CurTokenIdx);

209

210 ConcatenatedVAOPTResultToks.back() = LHS;

211 if (CurTokenIdx == NumVAOptTokens)

212 break;

213 }

214 ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]);

215 }

216

217 ConcatenatedVAOPTResultToks.push_back(VCtx.getEOFTok());

218

219

220

221

222

223 const SourceLocation ExpansionLocStartWithinMacro =

224 getExpansionLocForMacroDefLoc(VCtx.getVAOptLoc());

225 const SourceLocation ExpansionLocEndWithinMacro =

226 getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc);

227

229 &ConcatenatedVAOPTResultToks[0], PP, VCtx.hasCharifyBefore() ,

230 ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro);

231

234

236

237 ResultToks.resize(NumToksPriorToVAOpt + 1);

238 ResultToks.back() = StringifiedVAOPT;

239}

240

241

242

243void TokenLexer::ExpandFunctionArguments() {

244 SmallVector<Token, 128> ResultToks;

245

246

247

248

249 bool MadeChange = false;

250

251 std::optional CalledWithVariadicArguments;

252

253 VAOptExpansionContext VCtx(PP);

254

255 for (unsigned I = 0, E = NumTokens; I != E; ++I) {

256 const Token &CurTok = Tokens[I];

257

258

259

260

261

262

263 if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.hasLeadingSpace())

264 NextTokGetsSpace = true;

265

267 MadeChange = true;

268 assert(Tokens[I + 1].is(tok::l_paren) &&

269 "__VA_OPT__ must be followed by '('");

270

271 ++I;

273 ResultToks.size());

274

275 continue;

276 }

277

278

279

281

282

283

284

285

286

287

288

289

290

291 if (Tokens[I].is(tok::l_paren))

293

294

295

296

298

299 if (!CalledWithVariadicArguments) {

300 CalledWithVariadicArguments =

301 ActualArgs->invokedWithVariadicArgument(Macro, PP);

302 }

303 if (!*CalledWithVariadicArguments) {

304

305 continue;

306 }

307

308

309 } else {

310

311

312

313

314

315

316

318

319

320

321

322

323 stringifyVAOPTContents(ResultToks, VCtx,

324 Tokens[I].getLocation());

325

326 } else if (

328

329

330

331

332

333 if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) {

334 ResultToks.pop_back();

335 } else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) {

336 ++I;

337 }

338 } else {

339

340

341

342

347 tok::hashhash) &&

348 "no token paste before __VA_OPT__");

349 ResultToks.erase(ResultToks.begin() +

351 }

352

353

355 Tokens[I + 1].is(tok::hashhash)) {

356 ++I;

357 }

358 }

360

361

362 continue;

363 }

364 }

365

366

367

368

369

370 if (CurTok.isOneOf(tok::hash, tok::hashat)) {

371 int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo());

372 assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) &&

373 "Token following # is not an argument or __VA_OPT__!");

374

375 if (ArgNo == -1) {

376

378 CurTok.is(tok::hashat));

379 continue;

380 }

381

382 SourceLocation ExpansionLocStart =

383 getExpansionLocForMacroDefLoc(CurTok.getLocation());

384 SourceLocation ExpansionLocEnd =

385 getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation());

386

387 bool Charify = CurTok.is(tok::hashat);

388 const Token *UnexpArg = ActualArgs->getUnexpArgument(ArgNo);

390 UnexpArg, PP, Charify, ExpansionLocStart, ExpansionLocEnd);

392

393

394

395 if (NextTokGetsSpace)

397

398 ResultToks.push_back(Res);

399 MadeChange = true;

400 ++I;

401 NextTokGetsSpace = false;

402 continue;

403 }

404

405

406 bool NonEmptyPasteBefore =

407 !ResultToks.empty() && ResultToks.back().is(tok::hashhash);

408 bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash);

409 bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash);

410 bool RParenAfter = I+1 != E && Tokens[I+1].is(tok::r_paren);

411

412 assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) &&

413 "unexpected ## in ResultToks");

414

415

416

418 int ArgNo = II ? Macro->getParameterNum(II) : -1;

419 if (ArgNo == -1) {

420

421 ResultToks.push_back(CurTok);

422

423 if (NextTokGetsSpace) {

425 NextTokGetsSpace = false;

426 } else if (PasteBefore && !NonEmptyPasteBefore)

428

429 continue;

430 }

431

432

433

434 MadeChange = true;

435

436

437

438

439

440 if (!PasteBefore && ActualArgs->isVarargsElidedUse() &&

441 MaybeRemoveCommaBeforeVaArgs(ResultToks,

442 false,

443 Macro, ArgNo, PP))

444 continue;

445

446

447

448

449 if (!PasteBefore && !PasteAfter) {

450 const Token *ResultArgToks;

451

452

453

454 const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);

455 if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP))

456 ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0];

457 else

458 ResultArgToks = ArgTok;

459

460

461 if (ResultArgToks->isNot(tok::eof)) {

462 size_t FirstResult = ResultToks.size();

464 ResultToks.append(ResultArgToks, ResultArgToks+NumToks);

465

466

467

468

469

470 if (PP.getLangOpts().MSVCCompat && NumToks == 1 &&

471 ResultToks.back().is(tok::comma))

473

474

475

476 for (Token &Tok : llvm::drop_begin(ResultToks, FirstResult))

477 if (Tok.is(tok::hashhash))

479

480 if(ExpandLocStart.isValid()) {

481 updateLocForMacroArgTokens(CurTok.getLocation(),

482 ResultToks.begin()+FirstResult,

483 ResultToks.end());

484 }

485

486

487

488

490 NextTokGetsSpace);

492 NextTokGetsSpace = false;

493 } else {

494

495

496

497 if (NonEmptyPasteBefore) {

498

499

500 assert(VCtx.isInVAOpt() && "should only happen inside a __VA_OPT__");

502 } else if (RParenAfter)

504 }

505 continue;

506 }

507

508

509

510 const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);

512 if (NumToks) {

513 bool VaArgsPseudoPaste = false;

514

515

516

517

518 if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&

519 ResultToks[ResultToks.size()-2].is(tok::comma) &&

520 (unsigned)ArgNo == Macro->getNumParams()-1 &&

521 Macro->isVariadic()) {

522 VaArgsPseudoPaste = true;

523

524 PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);

525 }

526

527 ResultToks.append(ArgToks, ArgToks+NumToks);

528

529

530

531 for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks,

532 ResultToks.end())) {

533 if (Tok.is(tok::hashhash))

535 }

536

537 if (ExpandLocStart.isValid()) {

538 updateLocForMacroArgTokens(CurTok.getLocation(),

539 ResultToks.end()-NumToks, ResultToks.end());

540 }

541

542

543

544

545

546 if (!VaArgsPseudoPaste) {

547 ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine,

548 false);

549 ResultToks[ResultToks.size() - NumToks].setFlagValue(

551 }

552

553 NextTokGetsSpace = false;

554 continue;

555 }

556

557

558

559

560

561 if (PasteAfter) {

562

563

564 ++I;

565 continue;

566 }

567

568 if (RParenAfter && !NonEmptyPasteBefore)

570

571

572

573

574 assert(PasteBefore);

575 if (NonEmptyPasteBefore) {

576 assert(ResultToks.back().is(tok::hashhash));

577

578

579

580

583 ResultToks.pop_back();

584 else

586 }

587

588

589

590

591

592 if (ActualArgs->isVarargsElidedUse())

593 MaybeRemoveCommaBeforeVaArgs(ResultToks,

594 true,

595 Macro, ArgNo, PP);

596 }

597

598

599 if (MadeChange) {

600 assert(!OwnsTokens && "This would leak if we already own the token list");

601

602 NumTokens = ResultToks.size();

603

604

605 Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);

606

607

608 OwnsTokens = false;

609 }

610}

611

612

614 const Token &SecondTok) {

615 return FirstTok.is(tok::identifier) &&

618}

619

620

622

623 if (isAtEnd()) {

624

625

626 if (Macro) Macro->EnableMacro();

627

628 Tok.startToken();

631 if (CurTokenIdx == 0)

633 return PP.HandleEndOfTokenLexer(Tok);

634 }

635

637

638

639

640 bool isFirstToken = CurTokenIdx == 0;

641

642

643 Tok = Tokens[CurTokenIdx++];

644 if (IsReinject)

646

647 bool TokenIsFromPaste = false;

648

649

650

651 if (!isAtEnd() && Macro &&

652 (Tokens[CurTokenIdx].is(tok::hashhash) ||

653

654

655

656 (PP.getLangOpts().MSVCCompat &&

658

659

660 if (pasteTokens(Tok))

661 return true;

662

663 TokenIsFromPaste = true;

664 }

665

666

667

668

669

670

671 if (ExpandLocStart.isValid() &&

672

673 SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) {

675 if (Tok.is(tok::comment)) {

676 instLoc = SM.createExpansionLoc(Tok.getLocation(),

677 ExpandLocStart,

678 ExpandLocEnd,

679 Tok.getLength());

680 } else {

681 instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation());

682 }

683

684 Tok.setLocation(instLoc);

685 }

686

687

688

689 if (isFirstToken) {

692 } else {

693

694

697 }

698 AtStartOfLine = false;

699 HasLeadingSpace = false;

700

701

702 if ( Tok .isAnnotation() && Tok.getIdentifierInfo() != nullptr) {

703

704

707

708

709

710

711 if (II->isPoisoned() && TokenIsFromPaste) {

712 PP.HandlePoisonedIdentifier(Tok);

713 }

714

716 return PP.HandleIdentifier(Tok);

717 }

718

719

720 return true;

721}

722

723bool TokenLexer::pasteTokens(Token &Tok) {

724 return pasteTokens(Tok, llvm::ArrayRef(Tokens, NumTokens), CurTokenIdx);

725}

726

727

728

729

730

732 unsigned int &CurIdx) {

733 assert(CurIdx > 0 && "## can not be the first token within tokens");

734 assert((TokenStream[CurIdx].is(tok::hashhash) ||

737 "Token at this Index must be ## or part of the MSVC 'L "

738 "#macro-arg' pasting pair");

739

740

741

742

743 if (PP.getLangOpts().MicrosoftExt && (CurIdx >= 2) &&

744 TokenStream[CurIdx - 2].is(tok::hashhash))

746

748 const char *ResultTokStrPtr = nullptr;

751 bool HasUCNs = false;

752

753 auto IsAtEnd = [&TokenStream, &CurIdx] {

754 return TokenStream.size() == CurIdx;

755 };

756

757 do {

758

759 PasteOpLoc = TokenStream[CurIdx].getLocation();

760 if (TokenStream[CurIdx].is(tok::hashhash))

761 ++CurIdx;

762 assert(!IsAtEnd() && "No token on the RHS of a paste operator!");

763

764

765 const Token &RHS = TokenStream[CurIdx];

766

767

768

770

771

772 const char *BufPtr = &Buffer[0];

774 unsigned LHSLen = PP.getSpelling(LHSTok, BufPtr, &Invalid);

775 if (BufPtr != &Buffer[0])

776 memcpy(&Buffer[0], BufPtr, LHSLen);

778 return true;

779

780 BufPtr = Buffer.data() + LHSLen;

781 unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid);

783 return true;

784 if (RHSLen && BufPtr != &Buffer[LHSLen])

785

786 memcpy(&Buffer[LHSLen], BufPtr, RHSLen);

787

788

789 Buffer.resize(LHSLen+RHSLen);

790

791

792

793 Token ResultTokTmp;

795

796

797

798 ResultTokTmp.setKind(tok::string_literal);

799 PP.CreateString(Buffer, ResultTokTmp);

800 SourceLocation ResultTokLoc = ResultTokTmp.getLocation();

802

803

805

807

808

809 PP.IncrementPasteCounter(true);

811 Result.setKind(tok::raw_identifier);

812 Result.setRawIdentifierData(ResultTokStrPtr);

813 Result.setLocation(ResultTokLoc);

814 Result.setLength(LHSLen+RHSLen);

815 } else {

816 PP.IncrementPasteCounter(false);

817

818 assert(ResultTokLoc.isFileID() &&

819 "Should be a raw location into scratch buffer");

820 SourceManager &SourceMgr = PP.getSourceManager();

821 FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);

822

824 const char *ScratchBufStart

827 return false;

828

829

830

832 PP.getLangOpts(), ScratchBufStart,

833 ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);

834

835

836

837

838

840

841

842

844

845

846

847

849

850

851 SourceManager &SM = PP.getSourceManager();

852 SourceLocation Loc =

853 SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2);

854

855

856

857 if (PP.getLangOpts().MicrosoftExt && LHSTok.is(tok::slash) &&

858 RHS.is(tok::slash)) {

859 HandleMicrosoftCommentPaste(LHSTok, Loc);

860 return true;

861 }

862

863

864 if (!PP.getLangOpts().AsmPreprocessor) {

865

866

867

868 PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms

869 : diag::err_pp_bad_paste)

870 << Buffer;

871 }

872

873

874 break;

875 }

876

877

878

879 if (Result.is(tok::hashhash))

880 Result.setKind(tok::unknown);

881 }

882

883

886

887

888 ++CurIdx;

889

890

891 HasUCNs = LHSTok.hasUCN() || RHS.hasUCN() || HasUCNs;

893 } while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash));

894

895 SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation();

896

897

898

899

900

901

902 SourceManager &SM = PP.getSourceManager();

904 StartLoc = getExpansionLocForMacroDefLoc(StartLoc);

906 EndLoc = getExpansionLocForMacroDefLoc(EndLoc);

907 FileID MacroFID = SM.getFileID(MacroExpansionStart);

908 while (SM.getFileID(StartLoc) != MacroFID)

909 StartLoc = SM.getImmediateExpansionRange(StartLoc).getBegin();

910 while (SM.getFileID(EndLoc) != MacroFID)

911 EndLoc = SM.getImmediateExpansionRange(EndLoc).getEnd();

912

915

916

917

918

919 if (LHSTok.is(tok::raw_identifier)) {

920

921

922

923

924 if (HasUCNs)

926

927

928

929 PP.LookUpIdentifierInfo(LHSTok);

930 }

931 return false;

932}

933

934

935

936

938

939 if (isAtEnd())

940 return std::nullopt;

941 return Tokens[CurTokenIdx];

942}

943

944

945

947 return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd();

948}

949

950

951

952

953

954

956 PP.Diag(OpLoc, diag::ext_comment_paste_microsoft);

957

958

959

960

961

962

963 assert(Macro && "Token streams can't paste comments");

964 Macro->EnableMacro();

965

967}

968

969

970

971

972

974TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const {

975 assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() &&

976 "Not appropriate for token streams");

978

980 assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&

981 "Expected loc to come from the macro definition");

982

984 SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset);

986}

987

988

989

990

991

992

993

994

995

996

999 Token *&begin_tokens,

1000 Token * end_tokens) {

1001 assert(begin_tokens + 1 < end_tokens);

1005

1007

1008

1009

1013 return Distance <= MaxDistance;

1014 };

1015

1016

1017

1018

1020

1021

1022 Partition = All.take_while([&](const Token &T) {

1023 return T.getLocation().isFileID() && NearLast(T.getLocation());

1024 });

1025 } else {

1026

1027

1028 FileID BeginFID = SM.getFileID(BeginLoc);

1030 SM.getComposedLoc(BeginFID, SM.getFileIDSize(BeginFID));

1031 Partition = All.take_while([&](const Token &T) {

1032

1033

1034

1035

1036

1037

1038

1039

1040 return T.getLocation() >= BeginLoc && T.getLocation() <= Limit

1041 && NearLast(T.getLocation());

1042 });

1043 }

1044 assert(!Partition.empty());

1045

1046

1047

1049 Partition.back().getEndLoc().getRawEncoding() -

1050 Partition.front().getLocation().getRawEncoding();

1051

1053 SM.createMacroArgExpansionLoc(BeginLoc, ExpandLoc, FullLength);

1054

1055#ifdef EXPENSIVE_CHECKS

1056 assert(llvm::all_of(Partition.drop_front(),

1057 [&SM, ID = SM.getFileID(Partition.front().getLocation())](

1059 return ID == SM.getFileID(T.getLocation());

1060 }) &&

1061 "Must have the same FIleID!");

1062#endif

1063

1064

1065 for (Token& T : Partition) {

1067 T.getLocation().getRawEncoding() - BeginLoc.getRawEncoding();

1069 }

1070 begin_tokens = &Partition.back() + 1;

1071}

1072

1073

1074

1075

1076

1077

1078void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,

1079 Token *begin_tokens,

1080 Token *end_tokens) {

1081 SourceManager &SM = PP.getSourceManager();

1082

1083 SourceLocation ExpandLoc =

1084 getExpansionLocForMacroDefLoc(ArgIdSpellLoc);

1085

1086 while (begin_tokens < end_tokens) {

1087

1088 if (end_tokens - begin_tokens == 1) {

1089 Token &Tok = *begin_tokens;

1091 ExpandLoc,

1093 return;

1094 }

1095

1097 }

1098}

1099

1100void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {

1101 AtStartOfLine = Result.isAtStartOfLine();

1102 HasLeadingSpace = Result.hasLeadingSpace();

1103}

Defines the Diagnostic-related interfaces.

bool is(tok::TokenKind Kind) const

Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.

Defines the clang::LangOptions interface.

Defines the clang::MacroInfo and clang::MacroDirective classes.

Defines the clang::Preprocessor interface.

Defines the clang::SourceLocation class and associated facilities.

static bool isInvalid(LocType Loc, bool *Invalid)

Defines the SourceManager interface.

Defines the clang::TokenKind enum and support functions.

static bool isWideStringLiteralFromMacro(const Token &FirstTok, const Token &SecondTok)

Checks if two tokens form wide string literal.

Definition TokenLexer.cpp:613

static void updateConsecutiveMacroArgTokens(SourceManager &SM, SourceLocation ExpandLoc, Token *&begin_tokens, Token *end_tokens)

Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry,...

Definition TokenLexer.cpp:997

__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)

An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...

One of these records is kept for each identifier that is lexed.

bool isHandleIdentifierCase() const

Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier.

tok::TokenKind getTokenID() const

If this is a source-language token (e.g.

bool isPoisoned() const

Return true if this token has been poisoned.

bool isStr(const char(&Str)[StrLen]) const

Return true if this is the identifier for the specified string.

MacroArgs - An instance of this class captures information about the formal arguments specified to a ...

static unsigned getArgLength(const Token *ArgPtr)

getArgLength - Given a pointer to an expanded or unexpanded argument, return the number of tokens,...

static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP, bool Charify, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)

StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of tokens into the literal string...

void destroy(Preprocessor &PP)

destroy - Destroy and deallocate the memory for this object.

Encapsulates the data about a macro definition (e.g.

Engages in a tight little dance with the lexer to efficiently preprocess tokens.

SourceManager & getSourceManager() const

const LangOptions & getLangOpts() const

DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const

Forwarding function for diagnostics.

void HandleMicrosoftCommentPaste(Token &Tok)

When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...

Encodes a location in the source.

bool isValid() const

Return true if this is a valid SourceLocation object.

SourceLocation getLocWithOffset(IntTy Offset) const

Return a source location with the specified offset from this SourceLocation.

UIntTy getRawEncoding() const

When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it.

This class handles loading and caching of source files into memory.

FileID getFileID(SourceLocation SpellingLoc) const

Return the FileID for a SourceLocation.

StringRef getBufferData(FileID FID, bool *Invalid=nullptr) const

Return a StringRef to the source buffer data for the specified FileID.

SourceLocation getLocForStartOfFile(FileID FID) const

Return the source location corresponding to the first byte of the specified file.

bool isParsingPreprocessorDirective() const

isParsingPreprocessorDirective - Return true if we are in the middle of a preprocessor directive.

Definition TokenLexer.cpp:946

bool Lex(Token &Tok)

Lex and return a token from this macro stream.

Definition TokenLexer.cpp:621

std::optional< Token > peekNextPPToken() const

If TokenLexer::isAtEnd returns true(the next token lexed will pop thismacro off the expansion stack),...

Definition TokenLexer.cpp:937

void Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, MacroArgs *Actuals)

Initialize this TokenLexer to expand from the specified macro with the specified argument information...

Definition TokenLexer.cpp:39

Token - This structure provides full information about a lexed token.

IdentifierInfo * getIdentifierInfo() const

bool isAnyIdentifier() const

Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...

bool hasUCN() const

Returns true if this token contains a universal character name.

void clearFlag(TokenFlags Flag)

Unset the specified flag.

bool isLiteral() const

Return true if this is a "literal", like a numeric constant, string, etc.

SourceLocation getLocation() const

Return a source location identifier for the specified offset in the current file.

unsigned getLength() const

void setKind(tok::TokenKind K)

bool is(tok::TokenKind K) const

is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....

bool isAtStartOfLine() const

isAtStartOfLine - Return true if this token is at the start of a line.

bool isOneOf(Ts... Ks) const

bool hasLeadingSpace() const

Return true if this token has whitespace before it.

void setLocation(SourceLocation L)

bool isNot(tok::TokenKind K) const

bool stringifiedInMacro() const

Returns true if this token is formed by macro by stringizing or charizing operator.

void startToken()

Reset all flags to cleared.

const char * getLiteralData() const

getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...

void setFlag(TokenFlags Flag)

Set the specified flag.

A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a macro during...

void hasPlaceholderAfterHashhashAtStart()

bool isInVAOpt() const

Returns true if we have seen the VA_OPT and '(' but before having seen the matching ')'.

bool isVAOptToken(const Token &T) const

void sawHashOrHashAtBefore(const bool HasLeadingSpace, const bool IsHashAt)

SourceLocation getVAOptLoc() const

unsigned int getNumberOfTokensPriorToVAOpt() const

bool getLeadingSpaceForStringifiedToken() const

bool hasStringifyOrCharifyBefore() const

bool hasCharifyBefore() const

void sawOpeningParen(SourceLocation LParenLoc)

Call this function each time an lparen is seen.

void hasPlaceholderBeforeRParen()

const Token & getEOFTok() const

bool sawClosingParen()

Call this function each time an rparen is seen.

bool beginsWithPlaceholder() const

bool endsWithPlaceholder() const

void sawVAOptFollowedByOpeningParens(const SourceLocation VAOptLoc, const unsigned int NumPriorTokens)

The JSON file list parser is used to communicate input to InstallAPI.

@ Result

The result type of a method or function.

const FunctionProtoType * T