TokenLexer.cpp Source File (original) (raw)

27#include "llvm/ADT/ArrayRef.h"

28#include "llvm/ADT/STLExtras.h"

29#include "llvm/ADT/SmallVector.h"

30#include "llvm/ADT/iterator_range.h"

31#include

32#include

33#include

35using namespace clang;

43 destroy();

45 Macro = MI;

46 ActualArgs = Actuals;

47 CurTokenIdx = 0;

50 ExpandLocEnd = ELEnd;

53 NextTokGetsSpace = false;

55 OwnsTokens = false;

56 DisableMacroExpansion = false;

57 IsReinject = false;

62 MacroStartSLocOffset = SM.getNextLocalOffset();

64 if (NumTokens > 0) {

65 assert(Tokens[0].getLocation().isValid());

66 assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&

67 "Macro defined in macro?");

68 assert(ExpandLocStart.isValid());

74 MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation());

76 MacroExpansionStart = SM.createExpansionLoc(MacroDefStart,

77 ExpandLocStart,

78 ExpandLocEnd,

79 MacroDefLength);

80 }

85 ExpandFunctionArguments();

91}

96 bool disableMacroExpansion, bool ownsTokens,

97 bool isReinject) {

98 assert(!isReinject || disableMacroExpansion);

100

101 destroy();

102

103 Macro = nullptr;

104 ActualArgs = nullptr;

105 Tokens = TokArray;

106 OwnsTokens = ownsTokens;

107 DisableMacroExpansion = disableMacroExpansion;

108 IsReinject = isReinject;

109 NumTokens = NumToks;

110 CurTokenIdx = 0;

112 AtStartOfLine = false;

113 HasLeadingSpace = false;

114 NextTokGetsSpace = false;

116

117

118

119 if (NumToks != 0) {

122 }

123}

124

125void TokenLexer::destroy() {

126

127

128 if (OwnsTokens) {

129 delete [] Tokens;

130 Tokens = nullptr;

131 OwnsTokens = false;

132 }

133

134

135 if (ActualArgs) ActualArgs->destroy(PP);

136}

137

138bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(

141

142 if ( Macro ->isVariadic() || MacroArgNo != Macro->getNumParams()-1)

143 return false;

144

145

146

147

148 if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat)

149 return false;

150

151

152

153

154

155

157 && Macro->getNumParams() < 2)

158 return false;

159

160

161 if (ResultToks.empty() || !ResultToks.back().is(tok::comma))

162 return false;

163

164

165 if (HasPasteOperator)

166 PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);

167

168

169 ResultToks.pop_back();

170

171 if (!ResultToks.empty()) {

172

173

174

175

176 if (ResultToks.back().is(tok::hashhash))

177 ResultToks.pop_back();

178

179

181 }

182

183

184 NextTokGetsSpace = false;

185 return true;

186}

187

188void TokenLexer::stringifyVAOPTContents(

192 const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;

193 Token *const VAOPTTokens =

194 NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] : nullptr;

195

197

198

199

200

201 for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;

202 ++CurTokenIdx) {

203 if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) {

204 assert(CurTokenIdx != 0 &&

205 "Can not have __VAOPT__ contents begin with a ##");

206 Token &LHS = VAOPTTokens[CurTokenIdx - 1];

207 pasteTokens(LHS, llvm::ArrayRef(VAOPTTokens, NumVAOptTokens),

208 CurTokenIdx);

209

210 ConcatenatedVAOPTResultToks.back() = LHS;

211 if (CurTokenIdx == NumVAOptTokens)

212 break;

213 }

214 ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]);

215 }

216

217 ConcatenatedVAOPTResultToks.push_back(VCtx.getEOFTok());

218

219

220

221

222

224 getExpansionLocForMacroDefLoc(VCtx.getVAOptLoc());

226 getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc);

227

229 &ConcatenatedVAOPTResultToks[0], PP, VCtx.hasCharifyBefore() ,

230 ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro);

231

234

236

237 ResultToks.resize(NumToksPriorToVAOpt + 1);

238 ResultToks.back() = StringifiedVAOPT;

239}

240

241

242

243void TokenLexer::ExpandFunctionArguments() {

245

246

247

248

249 bool MadeChange = false;

250

251 std::optional CalledWithVariadicArguments;

252

254

255 for (unsigned I = 0, E = NumTokens; I != E; ++I) {

256 const Token &CurTok = Tokens[I];

257

258

259

260

261

262

263 if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.hasLeadingSpace())

264 NextTokGetsSpace = true;

265

267 MadeChange = true;

268 assert(Tokens[I + 1].is(tok::l_paren) &&

269 "__VA_OPT__ must be followed by '('");

270

271 ++I;

273 ResultToks.size());

274

275 continue;

276 }

277

278

279

281

282

283

284

285

286

287

288

289

290

291 if (Tokens[I].is(tok::l_paren))

293

294

295

296

297 if (!Tokens[I].is(tok::r_paren) || !VCtx.sawClosingParen()) {

298

299 if (!CalledWithVariadicArguments) {

300 CalledWithVariadicArguments =

302 }

303 if (!*CalledWithVariadicArguments) {

304

305 continue;

306 }

307

308

309 } else {

310

311

312

313

314

315

316

318

319

320

321

322

323 stringifyVAOPTContents(ResultToks, VCtx,

324 Tokens[I].getLocation());

325

326 } else if (

328

329

330

331

332

333 if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) {

334 ResultToks.pop_back();

335 } else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) {

336 ++I;

337 }

338 } else {

339

340

341

342

347 tok::hashhash) &&

348 "no token paste before __VA_OPT__");

349 ResultToks.erase(ResultToks.begin() +

351 }

352

353

355 Tokens[I + 1].is(tok::hashhash)) {

356 ++I;

357 }

358 }

360

361

362 continue;

363 }

364 }

365

366

367

368

369

370 if (CurTok.isOneOf(tok::hash, tok::hashat)) {

371 int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo());

372 assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) &&

373 "Token following # is not an argument or __VA_OPT__!");

374

375 if (ArgNo == -1) {

376

378 CurTok.is(tok::hashat));

379 continue;

380 }

381

383 getExpansionLocForMacroDefLoc(CurTok.getLocation());

385 getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation());

386

387 bool Charify = CurTok.is(tok::hashat);

390 UnexpArg, PP, Charify, ExpansionLocStart, ExpansionLocEnd);

392

393

394

395 if (NextTokGetsSpace)

397

398 ResultToks.push_back(Res);

399 MadeChange = true;

400 ++I;

401 NextTokGetsSpace = false;

402 continue;

403 }

404

405

406 bool NonEmptyPasteBefore =

407 !ResultToks.empty() && ResultToks.back().is(tok::hashhash);

408 bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash);

409 bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash);

410 bool RParenAfter = I+1 != E && Tokens[I+1].is(tok::r_paren);

411

412 assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) &&

413 "unexpected ## in ResultToks");

414

415

416

418 int ArgNo = II ? Macro->getParameterNum(II) : -1;

419 if (ArgNo == -1) {

420

421 ResultToks.push_back(CurTok);

422

423 if (NextTokGetsSpace) {

425 NextTokGetsSpace = false;

426 } else if (PasteBefore && !NonEmptyPasteBefore)

428

429 continue;

430 }

431

432

433

434 MadeChange = true;

435

436

437

438

439

441 MaybeRemoveCommaBeforeVaArgs(ResultToks,

442 false,

443 Macro, ArgNo, PP))

444 continue;

445

446

447

448

449 if (!PasteBefore && !PasteAfter) {

450 const Token *ResultArgToks;

451

452

453

457 else

458 ResultArgToks = ArgTok;

459

460

461 if (ResultArgToks->isNot(tok::eof)) {

462 size_t FirstResult = ResultToks.size();

464 ResultToks.append(ResultArgToks, ResultArgToks+NumToks);

465

466

467

468

469

470 if (PP.getLangOpts().MSVCCompat && NumToks == 1 &&

471 ResultToks.back().is(tok::comma))

473

474

475

476 for (Token &Tok : llvm::drop_begin(ResultToks, FirstResult))

477 if (Tok.is(tok::hashhash))

478 Tok.setKind(tok::unknown);

479

480 if(ExpandLocStart.isValid()) {

481 updateLocForMacroArgTokens(CurTok.getLocation(),

482 ResultToks.begin()+FirstResult,

483 ResultToks.end());

484 }

485

486

487

488

490 NextTokGetsSpace);

492 NextTokGetsSpace = false;

493 } else {

494

495

496

497 if (NonEmptyPasteBefore) {

498

499

500 assert(VCtx.isInVAOpt() && "should only happen inside a __VA_OPT__");

502 } else if (RParenAfter)

504 }

505 continue;

506 }

507

508

509

512 if (NumToks) {

513 bool VaArgsPseudoPaste = false;

514

515

516

517

518 if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&

519 ResultToks[ResultToks.size()-2].is(tok::comma) &&

520 (unsigned)ArgNo == Macro->getNumParams()-1 &&

521 Macro->isVariadic()) {

522 VaArgsPseudoPaste = true;

523

524 PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);

525 }

526

527 ResultToks.append(ArgToks, ArgToks+NumToks);

528

529

530

531 for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks,

532 ResultToks.end())) {

533 if (Tok.is(tok::hashhash))

534 Tok.setKind(tok::unknown);

535 }

536

537 if (ExpandLocStart.isValid()) {

538 updateLocForMacroArgTokens(CurTok.getLocation(),

539 ResultToks.end()-NumToks, ResultToks.end());

540 }

541

542

543

544

545

546 if (!VaArgsPseudoPaste) {

547 ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine,

548 false);

549 ResultToks[ResultToks.size() - NumToks].setFlagValue(

551 }

552

553 NextTokGetsSpace = false;

554 continue;

555 }

556

557

558

559

560

561 if (PasteAfter) {

562

563

564 ++I;

565 continue;

566 }

567

568 if (RParenAfter && !NonEmptyPasteBefore)

570

571

572

573

574 assert(PasteBefore);

575 if (NonEmptyPasteBefore) {

576 assert(ResultToks.back().is(tok::hashhash));

577

578

579

580

583 ResultToks.pop_back();

584 else

586 }

587

588

589

590

591

593 MaybeRemoveCommaBeforeVaArgs(ResultToks,

594 true,

595 Macro, ArgNo, PP);

596 }

597

598

599 if (MadeChange) {

600 assert(!OwnsTokens && "This would leak if we already own the token list");

601

602 NumTokens = ResultToks.size();

603

604

605 Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);

606

607

608 OwnsTokens = false;

609 }

610}

611

612

614 const Token &SecondTok) {

615 return FirstTok.is(tok::identifier) &&

618}

619

620

622

623 if (isAtEnd()) {

624

625

626 if (Macro) Macro->EnableMacro();

627

631 if (CurTokenIdx == 0)

634 }

635

637

638

639

640 bool isFirstToken = CurTokenIdx == 0;

641

642

643 Tok = Tokens[CurTokenIdx++];

644 if (IsReinject)

646

647 bool TokenIsFromPaste = false;

648

649

650

651 if (!isAtEnd() && Macro &&

652 (Tokens[CurTokenIdx].is(tok::hashhash) ||

653

654

655

658

659

660 if (pasteTokens(Tok))

661 return true;

662

663 TokenIsFromPaste = true;

664 }

665

666

667

668

669

670

671 if (ExpandLocStart.isValid() &&

672

673 SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) {

675 if (Tok.is(tok::comment)) {

676 instLoc = SM.createExpansionLoc(Tok.getLocation(),

677 ExpandLocStart,

678 ExpandLocEnd,

680 } else {

681 instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation());

682 }

683

685 }

686

687

688

689 if (isFirstToken) {

692 } else {

693

694

697 }

698 AtStartOfLine = false;

699 HasLeadingSpace = false;

700

701

703

704

707

708

709

710

711 if (II->isPoisoned() && TokenIsFromPaste) {

713 }

714

717 }

718

719

720 return true;

721}

722

723bool TokenLexer::pasteTokens(Token &Tok) {

724 return pasteTokens(Tok, llvm::ArrayRef(Tokens, NumTokens), CurTokenIdx);

725}

726

727

728

729

730

732 unsigned int &CurIdx) {

733 assert(CurIdx > 0 && "## can not be the first token within tokens");

734 assert((TokenStream[CurIdx].is(tok::hashhash) ||

737 "Token at this Index must be ## or part of the MSVC 'L "

738 "#macro-arg' pasting pair");

739

740

741

742

743 if (PP.getLangOpts().MicrosoftExt && (CurIdx >= 2) &&

744 TokenStream[CurIdx - 2].is(tok::hashhash))

746

748 const char *ResultTokStrPtr = nullptr;

751

752 auto IsAtEnd = [&TokenStream, &CurIdx] {

753 return TokenStream.size() == CurIdx;

754 };

755

756 do {

757

758 PasteOpLoc = TokenStream[CurIdx].getLocation();

759 if (TokenStream[CurIdx].is(tok::hashhash))

760 ++CurIdx;

761 assert(!IsAtEnd() && "No token on the RHS of a paste operator!");

762

763

764 const Token &RHS = TokenStream[CurIdx];

765

766

767

769

770

771 const char *BufPtr = &Buffer[0];

774 if (BufPtr != &Buffer[0])

775 memcpy(&Buffer[0], BufPtr, LHSLen);

777 return true;

778

779 BufPtr = Buffer.data() + LHSLen;

782 return true;

783 if (RHSLen && BufPtr != &Buffer[LHSLen])

784

785 memcpy(&Buffer[LHSLen], BufPtr, RHSLen);

786

787

788 Buffer.resize(LHSLen+RHSLen);

789

790

791

792 Token ResultTokTmp;

794

795

796

797 ResultTokTmp.setKind(tok::string_literal);

801

802

804

806

807

810 Result.setKind(tok::raw_identifier);

811 Result.setRawIdentifierData(ResultTokStrPtr);

812 Result.setLocation(ResultTokLoc);

813 Result.setLength(LHSLen+RHSLen);

814 } else {

816

817 assert(ResultTokLoc.isFileID() &&

818 "Should be a raw location into scratch buffer");

821

823 const char *ScratchBufStart

826 return false;

827

828

829

832 ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);

833

834

835

836

837

839

840

841

843

844

845

846

848

849

852 SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2);

853

854

855

856 if (PP.getLangOpts().MicrosoftExt && LHSTok.is(tok::slash) &&

857 RHS.is(tok::slash)) {

858 HandleMicrosoftCommentPaste(LHSTok, Loc);

859 return true;

860 }

861

862

864

865

866

867 PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms

868 : diag::err_pp_bad_paste)

869 << Buffer;

870 }

871

872

873 break;

874 }

875

876

877

878 if (Result.is(tok::hashhash))

879 Result.setKind(tok::unknown);

880 }

881

882

885

886

887 ++CurIdx;

889 } while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash));

890

891 SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation();

892

893

894

895

896

897

900 StartLoc = getExpansionLocForMacroDefLoc(StartLoc);

902 EndLoc = getExpansionLocForMacroDefLoc(EndLoc);

903 FileID MacroFID = SM.getFileID(MacroExpansionStart);

904 while (SM.getFileID(StartLoc) != MacroFID)

905 StartLoc = SM.getImmediateExpansionRange(StartLoc).getBegin();

906 while (SM.getFileID(EndLoc) != MacroFID)

907 EndLoc = SM.getImmediateExpansionRange(EndLoc).getEnd();

908

911

912

913

914

915 if (LHSTok.is(tok::raw_identifier)) {

916

917

919 }

920 return false;

921}

922

923

924

925

927

928 if (isAtEnd())

929 return 2;

930 return Tokens[CurTokenIdx].is(tok::l_paren);

931}

932

933

934

936 return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd();

937}

938

939

940

941

942

943

944void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) {

945 PP.Diag(OpLoc, diag::ext_comment_paste_microsoft);

946

947

948

949

950

951

952 assert(Macro && "Token streams can't paste comments");

953 Macro->EnableMacro();

954

956}

957

958

959

960

961

963TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const {

964 assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() &&

965 "Not appropriate for token streams");

967

969 assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&

970 "Expected loc to come from the macro definition");

971

973 SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset);

975}

976

977

978

979

980

981

982

983

984

985

988 Token *&begin_tokens,

989 Token * end_tokens) {

990 assert(begin_tokens + 1 < end_tokens);

994

996

997

998

1002 return Distance <= MaxDistance;

1003 };

1004

1005

1006

1007

1009

1010

1011 Partition = All.take_while([&](const Token &T) {

1012 return T.getLocation().isFileID() && NearLast(T.getLocation());

1013 });

1014 } else {

1015

1016

1017 FileID BeginFID = SM.getFileID(BeginLoc);

1019 SM.getComposedLoc(BeginFID, SM.getFileIDSize(BeginFID));

1020 Partition = All.take_while([&](const Token &T) {

1021

1022

1023

1024

1025

1026

1027

1028

1029 return T.getLocation() >= BeginLoc && T.getLocation() <= Limit

1030 && NearLast(T.getLocation());

1031 });

1032 }

1033 assert(!Partition.empty());

1034

1035

1036

1038 Partition.back().getEndLoc().getRawEncoding() -

1039 Partition.front().getLocation().getRawEncoding();

1040

1042 SM.createMacroArgExpansionLoc(BeginLoc, ExpandLoc, FullLength);

1043

1044#ifdef EXPENSIVE_CHECKS

1045 assert(llvm::all_of(Partition.drop_front(),

1046 [&SM, ID = SM.getFileID(Partition.front().getLocation())](

1048 return ID == SM.getFileID(T.getLocation());

1049 }) &&

1050 "Must have the same FIleID!");

1051#endif

1052

1053

1054 for (Token& T : Partition) {

1056 T.getLocation().getRawEncoding() - BeginLoc.getRawEncoding();

1058 }

1059 begin_tokens = &Partition.back() + 1;

1060}

1061

1062

1063

1064

1065

1066

1067void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,

1068 Token *begin_tokens,

1069 Token *end_tokens) {

1071

1073 getExpansionLocForMacroDefLoc(ArgIdSpellLoc);

1074

1075 while (begin_tokens < end_tokens) {

1076

1077 if (end_tokens - begin_tokens == 1) {

1078 Token &Tok = *begin_tokens;

1080 ExpandLoc,

1082 return;

1083 }

1084

1086 }

1087}

1088

1089void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {

1090 AtStartOfLine = Result.isAtStartOfLine();

1091 HasLeadingSpace = Result.hasLeadingSpace();

1092}

Defines the Diagnostic-related interfaces.

Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.

Defines the clang::LangOptions interface.

Defines the clang::MacroInfo and clang::MacroDirective classes.

Defines the clang::Preprocessor interface.

Defines the clang::SourceLocation class and associated facilities.

static bool isInvalid(LocType Loc, bool *Invalid)

Defines the SourceManager interface.

Defines the clang::TokenKind enum and support functions.

static bool isWideStringLiteralFromMacro(const Token &FirstTok, const Token &SecondTok)

Checks if two tokens form wide string literal.

static void updateConsecutiveMacroArgTokens(SourceManager &SM, SourceLocation ExpandLoc, Token *&begin_tokens, Token *end_tokens)

Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry,...

__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)

An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...

One of these records is kept for each identifier that is lexed.

bool isHandleIdentifierCase() const

Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier.

tok::TokenKind getTokenID() const

If this is a source-language token (e.g.

bool isPoisoned() const

Return true if this token has been poisoned.

bool isStr(const char(&Str)[StrLen]) const

Return true if this is the identifier for the specified string.

Lexer - This provides a simple interface that turns a text buffer into a stream of tokens.

MacroArgs - An instance of this class captures information about the formal arguments specified to a ...

const Token * getUnexpArgument(unsigned Arg) const

getUnexpArgument - Return a pointer to the first token of the unexpanded token list for the specified...

const std::vector< Token > & getPreExpArgument(unsigned Arg, Preprocessor &PP)

getPreExpArgument - Return the pre-expanded form of the specified argument.

static unsigned getArgLength(const Token *ArgPtr)

getArgLength - Given a pointer to an expanded or unexpanded argument, return the number of tokens,...

bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const

ArgNeedsPreexpansion - If we can prove that the argument won't be affected by pre-expansion,...

bool invokedWithVariadicArgument(const MacroInfo *const MI, Preprocessor &PP)

Returns true if the macro was defined with a variadic (ellipsis) parameter AND was invoked with at le...

bool isVarargsElidedUse() const

isVarargsElidedUse - Return true if this is a C99 style varargs macro invocation and there was no arg...

static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP, bool Charify, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)

StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of tokens into the literal string...

void destroy(Preprocessor &PP)

destroy - Destroy and deallocate the memory for this object.

Encapsulates the data about a macro definition (e.g.

bool isFunctionLike() const

const_tokens_iterator tokens_begin() const

const_tokens_iterator tokens_end() const

unsigned getNumParams() const

unsigned getDefinitionLength(const SourceManager &SM) const

Get length in characters of the macro definition.

Engages in a tight little dance with the lexer to efficiently preprocess tokens.

void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())

Plop the specified string into a scratch buffer and set the specified token's location and length to ...

IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const

Given a tok::raw_identifier token, look up the identifier information for the token and install it in...

bool HandleEndOfTokenLexer(Token &Result)

Callback invoked when the current TokenLexer hits the end of its token stream.

void HandlePoisonedIdentifier(Token &Identifier)

Display reason for poisoned identifier.

bool HandleIdentifier(Token &Identifier)

Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...

void IncrementPasteCounter(bool isFast)

Increment the counters for the number of token paste operations performed.

SourceManager & getSourceManager() const

StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const

Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...

const LangOptions & getLangOpts() const

DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const

Forwarding function for diagnostics.

void HandleMicrosoftCommentPaste(Token &Tok)

When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...

Encodes a location in the source.

bool isValid() const

Return true if this is a valid SourceLocation object.

SourceLocation getLocWithOffset(IntTy Offset) const

Return a source location with the specified offset from this SourceLocation.

UIntTy getRawEncoding() const

When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it.

This class handles loading and caching of source files into memory.

FileID getFileID(SourceLocation SpellingLoc) const

Return the FileID for a SourceLocation.

StringRef getBufferData(FileID FID, bool *Invalid=nullptr) const

Return a StringRef to the source buffer data for the specified FileID.

SourceLocation getLocForStartOfFile(FileID FID) const

Return the source location corresponding to the first byte of the specified file.

bool isParsingPreprocessorDirective() const

isParsingPreprocessorDirective - Return true if we are in the middle of a preprocessor directive.

unsigned isNextTokenLParen() const

If the next token lexed will pop this macro off the expansion stack, return 2.

bool Lex(Token &Tok)

Lex and return a token from this macro stream.

void Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, MacroArgs *Actuals)

Initialize this TokenLexer to expand from the specified macro with the specified argument information...

Token - This structure provides full information about a lexed token.

IdentifierInfo * getIdentifierInfo() const

bool isAnyIdentifier() const

Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...

void clearFlag(TokenFlags Flag)

Unset the specified flag.

bool isLiteral() const

Return true if this is a "literal", like a numeric constant, string, etc.

SourceLocation getLocation() const

Return a source location identifier for the specified offset in the current file.

unsigned getLength() const

void setKind(tok::TokenKind K)

bool is(tok::TokenKind K) const

is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....

bool isAtStartOfLine() const

isAtStartOfLine - Return true if this token is at the start of a line.

bool hasLeadingSpace() const

Return true if this token has whitespace before it.

void setLocation(SourceLocation L)

bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const

bool isNot(tok::TokenKind K) const

bool isAnnotation() const

Return true if this is any of tok::annot_* kind tokens.

bool stringifiedInMacro() const

Returns true if this token is formed by macro by stringizing or charizing operator.

void startToken()

Reset all flags to cleared.

void setFlagValue(TokenFlags Flag, bool Val)

Set a flag to either true or false.

const char * getLiteralData() const

getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...

void setFlag(TokenFlags Flag)

Set the specified flag.

A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a macro during...

void hasPlaceholderAfterHashhashAtStart()

bool isInVAOpt() const

Returns true if we have seen the VA_OPT and '(' but before having seen the matching ')'.

bool isVAOptToken(const Token &T) const

void sawHashOrHashAtBefore(const bool HasLeadingSpace, const bool IsHashAt)

SourceLocation getVAOptLoc() const

unsigned int getNumberOfTokensPriorToVAOpt() const

bool getLeadingSpaceForStringifiedToken() const

bool hasStringifyOrCharifyBefore() const

bool hasCharifyBefore() const

void sawOpeningParen(SourceLocation LParenLoc)

Call this function each time an lparen is seen.

void hasPlaceholderBeforeRParen()

const Token & getEOFTok() const

bool sawClosingParen()

Call this function each time an rparen is seen.

bool beginsWithPlaceholder() const

bool endsWithPlaceholder() const

void sawVAOptFollowedByOpeningParens(const SourceLocation VAOptLoc, const unsigned int NumPriorTokens)

The JSON file list parser is used to communicate input to InstallAPI.

@ Result

The result type of a method or function.

const FunctionProtoType * T