DependencyDirectivesScanner.cpp Source File (original) (raw)

23#include "llvm/ADT/ScopeExit.h"

24#include "llvm/ADT/SmallString.h"

25#include "llvm/ADT/StringMap.h"

26#include "llvm/ADT/StringSwitch.h"

27#include

29using namespace clang;

31using namespace llvm;

33namespace {

35struct DirectiveWithTokens {

37 unsigned NumTokens;

39 DirectiveWithTokens(DirectiveKind Kind, unsigned NumTokens)

40 : Kind(Kind), NumTokens(NumTokens) {}

41};

62struct Scanner {

63 Scanner(StringRef Input,

64 SmallVectorImpl<dependency_directives_scan::Token> &Tokens,

65 DiagnosticsEngine *Diags, SourceLocation InputSourceLoc)

66 : Input(Input), Tokens(Tokens), Diags(Diags),

67 InputSourceLoc(InputSourceLoc), LangOpts(getLangOptsForDepScanning()),

68 TheLexer(InputSourceLoc, LangOpts, Input.begin(), Input.begin(),

69 Input.end()) {}

71 static LangOptions getLangOptsForDepScanning() {

72 LangOptions LangOpts;

74 LangOpts.ObjC = true;

75 LangOpts.LineComment = true;

76 LangOpts.RawStringLiterals = true;

78 return LangOpts;

79 }

84 bool scan(SmallVectorImpl &Directives);

88private:

90 [[nodiscard]] dependency_directives_scan::Token &

91 lexToken(const char *&First, const char *const End);

93 [[nodiscard]] dependency_directives_scan::Token &

94 lexIncludeFilename(const char *&First, const char *const End);

96 void skipLine(const char *&First, const char *const End);

97 void skipDirective(StringRef Name, const char *&First, const char *const End);

100

101 StringRef cleanStringIfNeeded(const dependency_directives_scan::Token &Tok);

102

103

104

105

106

107

108 [[nodiscard]] std::optional

109 tryLexIdentifierOrSkipLine(const char *&First, const char *const End);

110

111

112 [[nodiscard]] StringRef lexIdentifier(const char *&First,

113 const char *const End);

114

115

116

117

118

119

120 [[nodiscard]] bool isNextIdentifierOrSkipLine(StringRef Id,

121 const char *&First,

122 const char *const End);

123

124

125

126

127

128

129 [[nodiscard]] bool isNextTokenOrSkipLine(tok::TokenKind K, const char *&First,

130 const char *const End);

131

132

133

134

135

136

137 [[nodiscard]] std::optional

138 tryLexStringLiteralOrSkipLine(const char *&First, const char *const End);

139

140 [[nodiscard]] bool scanImpl(const char *First, const char *const End);

141 [[nodiscard]] bool lexPPLine(const char *&First, const char *const End);

142 [[nodiscard]] bool lexAt(const char *&First, const char *const End);

143 [[nodiscard]] bool lexModule(const char *&First, const char *const End);

144 [[nodiscard]] bool lexDefine(const char *HashLoc, const char *&First,

145 const char *const End);

146 [[nodiscard]] bool lexPragma(const char *&First, const char *const End);

147 [[nodiscard]] bool lex_Pragma(const char *&First, const char *const End);

148 [[nodiscard]] bool lexEndif(const char *&First, const char *const End);

149 [[nodiscard]] bool lexDefault(DirectiveKind Kind, const char *&First,

150 const char *const End);

151 [[nodiscard]] bool lexModuleDirectiveBody(DirectiveKind Kind,

152 const char *&First,

153 const char *const End);

154 void lexPPDirectiveBody(const char *&First, const char *const End);

155

156 DirectiveWithTokens &pushDirective(DirectiveKind Kind) {

157 Tokens.append(CurDirToks);

158 DirsWithToks.emplace_back(Kind, CurDirToks.size());

159 CurDirToks.clear();

160 return DirsWithToks.back();

161 }

162 void popDirective() {

163 Tokens.pop_back_n(DirsWithToks.pop_back_val().NumTokens);

164 }

166 return DirsWithToks.empty() ? pp_none : DirsWithToks.back().Kind;

167 }

168

169 unsigned getOffsetAt(const char *CurPtr) const {

170 return CurPtr - Input.data();

171 }

172

173

174

175 bool reportError(const char *CurPtr, unsigned Err);

176

177 StringMap SplitIds;

178 StringRef Input;

179 SmallVectorImpl<dependency_directives_scan::Token> &Tokens;

180 DiagnosticsEngine *Diags;

181 SourceLocation InputSourceLoc;

182

183 const char *LastTokenPtr = nullptr;

184

185

186

187 SmallVector<dependency_directives_scan::Token, 32> CurDirToks;

188

189

190

191 SmallVector<DirectiveWithTokens, 64> DirsWithToks;

192 LangOptions LangOpts;

193 Lexer TheLexer;

194};

195

196}

197

198bool Scanner::reportError(const char *CurPtr, unsigned Err) {

199 if (!Diags)

200 return true;

201 assert(CurPtr >= Input.data() && "invalid buffer ptr");

203 return true;

204}

205

210

211

213 assert(Current > First);

214 --Current;

216

217 if (Current > First && *(Current - 1) == '\\') {

218

219

221 if (EscapeSize > 0) {

222

223

224 Current -= (1 + EscapeSize);

225 } else {

226 break;

227 }

228 } else {

229 break;

230 }

231 }

232 return *Current;

233}

234

236 const char *Current) {

237 assert(First <= Current);

238

239

240 if (*Current != '"' || First == Current)

241 return false;

242

243

245 return false;

246 if (First == Current ||

248 return true;

249

250

251 if (*Current == 'u' || *Current == 'U' || *Current == 'L')

252 return First == Current ||

254

255

256 if (*Current != '8' || First == Current ||

258 return false;

259 return First == Current ||

261}

262

264 assert(First[0] == '"');

265

267 while (Last != End && *Last != '(')

269 if (Last == End) {

270 First = Last;

271 return;

272 }

273

275 for (;;) {

276

278 while (First != End && *First != ')')

280 if (First == End)

281 return;

283

284

286 while (Last != End && size_t(Last - First) < Terminator.size() &&

289

290

291 if (Last == End) {

293 return;

294 }

295 if (size_t(Last - First) < Terminator.size())

296 continue;

297 if (*Last != '"')

298 continue;

300 return;

301 }

302}

303

304

305static unsigned isEOL(const char *First, const char *const End) {

306 if (First == End)

307 return 0;

310 return 2;

312}

313

315 assert(*First == '\'' || *First == '"' || *First == '<');

316 const char Terminator = *First == '<' ? '>' : *First;

318

320 return;

321 if (*First != '\\')

322 continue;

323

324

325

326 if (++First == End)

327 return;

329 continue;

330

331 const char *FirstAfterBackslashPastSpace = First;

333 if (unsigned NLSize = isEOL(FirstAfterBackslashPastSpace, End)) {

334

335

336 First = FirstAfterBackslashPastSpace + NLSize - 1;

337 }

338 }

339 if (First != End)

340 ++First;

341}

342

343

345 if (First == End)

346 return 0;

349 assert(Len && "expected newline");

351 return Len;

352}

353

355 for (;;) {

356 if (First == End)

357 return;

358

360 if (Len)

361 return;

362

363 char LastNonWhitespace = ' ';

364 do {

366 LastNonWhitespace = *First;

367 if (++First == End)

368 return;

370 } while (!Len);

371

372 if (LastNonWhitespace != '\\')

373 return;

374

376

377 }

378}

379

381 assert(First[0] == '/' && First[1] == '/');

384}

385

387 assert(First[0] == '/' && First[1] == '*');

388 if (End - First < 4) {

390 return;

391 }

393 if (First[-1] == '*' && First[0] == '/') {

395 return;

396 }

397}

398

399

400

402 const char *const Cur,

403 const char *const End) {

404 assert(*Cur == '\'' && "expected quotation character");

405

406

407 if (Start == Cur)

408 return false;

409

410

411

412 char Prev = *(Cur - 1);

413 if (Prev == 'L' || Prev == 'U' || Prev == 'u')

414 return false;

415 if (Prev == '8' && (Cur - 1 != Start) && *(Cur - 2) == 'u')

416 return false;

418 return false;

419

421}

422

423void Scanner::skipLine(const char *&First, const char *const End) {

424 for (;;) {

425 assert(First <= End);

426 if (First == End)

427 return;

428

431 return;

432 }

433 const char *Start = First;

434

435

436 char LastNonWhitespace = ' ';

438

439 if (*First == '"' ||

441 LastTokenPtr = First;

444 else

446 continue;

447 }

448

449

450 if (First + 1 < End && *First == '\\') {

451 if (unsigned Len = isEOL(First + 1, End)) {

452 First += 1 + Len;

453 continue;

454 }

455 }

456

457

458 if (*First != '/' || End - First < 2) {

459 LastTokenPtr = First;

461 LastNonWhitespace = *First;

463 continue;

464 }

465

466 if (First[1] == '/') {

467

469 continue;

470 }

471

472 if (First[1] != '*') {

473 LastTokenPtr = First;

475 LastNonWhitespace = *First;

477 continue;

478 }

479

480

482 }

483 if (First == End)

484 return;

485

486

488

489 if (LastNonWhitespace != '\\')

490 break;

491 }

492}

493

494void Scanner::skipDirective(StringRef Name, const char *&First,

495 const char *const End) {

496 if (llvm::StringSwitch(Name)

497 .Case("warning", true)

498 .Case("error", true)

500

502 else

503 skipLine(First, End);

504}

505

507 for (;;) {

508 assert(First <= End);

510

511 if (End - First < 2)

512 return;

513

514 if (*First == '\\') {

515 const char *Ptr = First + 1;

517 ++Ptr;

521 continue;

522 }

523 return;

524 }

525

526

527 if (First[0] != '/')

528 return;

529

530

531 if (First[1] == '/') {

533 return;

534 }

535

536

537 if (First[1] != '*')

538 return;

539

540

542 }

543}

544

545bool Scanner::lexModuleDirectiveBody(DirectiveKind Kind, const char *&First,

546 const char *const End) {

547 const char *DirectiveLoc = Input.data() + CurDirToks.front().Offset;

548 for (;;) {

549

551 const dependency_directives_scan::Token &Tok = lexToken(First, End);

552 if ((Tok.is(tok::hash) || Tok.is(tok::at)) &&

554 CurDirToks.pop_back();

556 return false;

557 }

558 if (Tok.is(tok::eof))

559 return reportError(

560 DirectiveLoc,

561 diag::err_dep_source_scanner_missing_semi_after_at_import);

562 if (Tok.is(tok::semi))

563 break;

564 }

565

566 const auto &Tok = lexToken(First, End);

567 pushDirective(Kind);

568 if (Tok.is(tok::eof) || Tok.is(tok::eod))

569 return false;

570 return reportError(DirectiveLoc,

571 diag::err_dep_source_scanner_unexpected_tokens_at_import);

572}

573

574dependency_directives_scan::Token &Scanner::lexToken(const char *&First,

575 const char *const End) {

576 clang::Token Tok;

579 assert(First <= End);

580

582 CurDirToks.emplace_back(Offset, Tok.getLength(), Tok.getKind(),

583 Tok.getFlags());

584 return CurDirToks.back();

585}

586

587dependency_directives_scan::Token &

588Scanner::lexIncludeFilename(const char *&First, const char *const End) {

589 clang::Token Tok;

592 assert(First <= End);

593

595 CurDirToks.emplace_back(Offset, Tok.getLength(), Tok.getKind(),

596 Tok.getFlags());

597 return CurDirToks.back();

598}

599

600void Scanner::lexPPDirectiveBody(const char *&First, const char *const End) {

601 while (true) {

602 const dependency_directives_scan::Token &Tok = lexToken(First, End);

603 if (Tok.is(tok::eod) || Tok.is(tok::eof))

604 break;

605 }

606}

607

608StringRef

609Scanner::cleanStringIfNeeded(const dependency_directives_scan::Token &Tok) {

611 if (LLVM_LIKELY(!NeedsCleaning))

613

614 SmallString<64> Spelling;

616

617

618

619

620 unsigned SpellingLength = 0;

621 const char *BufPtr = Input.begin() + Tok.Offset;

622 const char *AfterIdent = Input.begin() + Tok.getEnd();

623 while (BufPtr < AfterIdent) {

625 Spelling[SpellingLength++] = Char;

626 BufPtr += Size;

627 }

628

629 return SplitIds.try_emplace(StringRef(Spelling.begin(), SpellingLength), 0)

630 .first->first();

631}

632

633std::optional

634Scanner::tryLexIdentifierOrSkipLine(const char *&First, const char *const End) {

635 const dependency_directives_scan::Token &Tok = lexToken(First, End);

636 if (Tok.isNot(tok::raw_identifier)) {

637 if ( Tok .is(tok::eod))

638 skipLine(First, End);

639 return std::nullopt;

640 }

641

642 return cleanStringIfNeeded(Tok);

643}

644

645StringRef Scanner::lexIdentifier(const char *&First, const char *const End) {

646 std::optional Id = tryLexIdentifierOrSkipLine(First, End);

647 assert(Id && "expected identifier token");

648 return *Id;

649}

650

651bool Scanner::isNextIdentifierOrSkipLine(StringRef Id, const char *&First,

652 const char *const End) {

653 if (std::optional FoundId =

654 tryLexIdentifierOrSkipLine(First, End)) {

655 if (*FoundId == Id)

656 return true;

657 skipLine(First, End);

658 }

659 return false;

660}

661

663 const char *const End) {

664 const dependency_directives_scan::Token &Tok = lexToken(First, End);

666 return true;

667 skipLine(First, End);

668 return false;

669}

670

671std::optional

672Scanner::tryLexStringLiteralOrSkipLine(const char *&First,

673 const char *const End) {

674 const dependency_directives_scan::Token &Tok = lexToken(First, End);

676 if ( Tok .is(tok::eod))

677 skipLine(First, End);

678 return std::nullopt;

679 }

680

681 return cleanStringIfNeeded(Tok);

682}

683

684bool Scanner::lexAt(const char *&First, const char *const End) {

685

686

687

688 const dependency_directives_scan::Token &AtTok = lexToken(First, End);

689 assert(AtTok.is(tok::at));

690 (void)AtTok;

691

692 if (!isNextIdentifierOrSkipLine("import", First, End))

693 return false;

695}

696

697bool Scanner::lexModule(const char *&First, const char *const End) {

698 StringRef Id = lexIdentifier(First, End);

699 bool Export = false;

700 if (Id == "export") {

701 Export = true;

702 std::optional NextId = tryLexIdentifierOrSkipLine(First, End);

703 if (!NextId)

704 return false;

705 Id = *NextId;

706 }

707

708 if (Id != "module" && Id != "import") {

709 skipLine(First, End);

710 return false;

711 }

712

714

715

716

717

718 switch (*First) {

719 case ':': {

720

721 if (Id == "module") {

722 skipLine(First, End);

723 return false;

724 }

725

726

727

728 if (First + 1 != End && First[1] == ':') {

729 skipLine(First, End);

730 return false;

731 }

732

733 (void)lexToken(First, End);

734 if (!tryLexIdentifierOrSkipLine(First, End))

735 return false;

736 break;

737 }

738 case ';': {

739

740 if (Id == "module" && !Export)

741 break;

742 skipLine(First, End);

743 return false;

744 }

745 case '<':

746 case '"':

747 break;

748 default:

750 skipLine(First, End);

751 return false;

752 }

753 }

754

755 TheLexer.seek(getOffsetAt(First), false);

756

758 if (Id == "module")

760 else

762

763 return lexModuleDirectiveBody(Kind, First, End);

764}

765

766bool Scanner::lex_Pragma(const char *&First, const char *const End) {

767 if (!isNextTokenOrSkipLine(tok::l_paren, First, End))

768 return false;

769

770 std::optional Str = tryLexStringLiteralOrSkipLine(First, End);

771

772 if (!Str || !isNextTokenOrSkipLine(tok::r_paren, First, End))

773 return false;

774

775 SmallString<64> Buffer(*Str);

777

778

779

780

781 SmallVector<dependency_directives_scan::Token> DiscardTokens;

782 const char *Begin = Buffer.c_str();

783 Scanner PragmaScanner{StringRef(Begin, Buffer.size()), DiscardTokens, Diags,

784 InputSourceLoc};

785

787 if (PragmaScanner.lexPragma(Begin, Buffer.end()))

788 return true;

789

792 skipLine(First, End);

793 return false;

794 }

795

796 assert(Begin == Buffer.end());

797 pushDirective(K);

798 return false;

799}

800

801bool Scanner::lexPragma(const char *&First, const char *const End) {

802 std::optional FoundId = tryLexIdentifierOrSkipLine(First, End);

803 if (!FoundId)

804 return false;

805

806 StringRef Id = *FoundId;

807 auto Kind = llvm::StringSwitch(Id)

814 lexPPDirectiveBody(First, End);

815 pushDirective(Kind);

816 return false;

817 }

818

819 if (Id != "clang") {

820 skipLine(First, End);

821 return false;

822 }

823

824 FoundId = tryLexIdentifierOrSkipLine(First, End);

825 if (!FoundId)

826 return false;

827 Id = *FoundId;

828

829

830 if (Id == "system_header") {

831 lexPPDirectiveBody(First, End);

833 return false;

834 }

835

836 if (Id != "module") {

837 skipLine(First, End);

838 return false;

839 }

840

841

842 if (!isNextIdentifierOrSkipLine("import", First, End))

843 return false;

844

845

846 lexPPDirectiveBody(First, End);

848 return false;

849}

850

851bool Scanner::lexEndif(const char *&First, const char *const End) {

852

853 if (topDirective() == pp_else)

854 popDirective();

855

856

857

858

859

860

861

863 popDirective();

864 skipLine(First, End);

865 return false;

866 }

867

869}

870

872 const char *const End) {

873 lexPPDirectiveBody(First, End);

874 pushDirective(Kind);

875 return false;

876}

877

880 case '#':

881 case '@':

882 case 'i':

883 case 'e':

884 case 'm':

885 case '_':

886 return true;

887 }

888 return false;

889}

890

891bool Scanner::lexPPLine(const char *&First, const char *const End) {

892 assert(First != End);

893

895 assert(First <= End);

896 if (First == End)

897 return false;

898

900 skipLine(First, End);

901 assert(First <= End);

902 return false;

903 }

904

905 LastTokenPtr = First;

906

907 TheLexer.seek(getOffsetAt(First), true);

908

909 auto ScEx1 = make_scope_exit([&]() {

910

911

912 CurDirToks.clear();

913 });

914

915 if (*First == '_') {

916 if (isNextIdentifierOrSkipLine("_Pragma", First, End))

917 return lex_Pragma(First, End);

918 return false;

919 }

920

921

922

924 auto ScEx2 = make_scope_exit(

926

927

928 if (*First == '@')

929 return lexAt(First, End);

930

931

933 return lexModule(First, End);

934

935

936 const dependency_directives_scan::Token &HashTok = lexToken(First, End);

937 if (HashTok.is(tok::hashhash)) {

938

939

940

941 skipLine(First, End);

942 assert(First <= End);

943 return false;

944 }

945 assert(HashTok.is(tok::hash));

946 (void)HashTok;

947

948 std::optional FoundId = tryLexIdentifierOrSkipLine(First, End);

949 if (!FoundId)

950 return false;

951

952 StringRef Id = *FoundId;

953

954 if (Id == "pragma")

955 return lexPragma(First, End);

956

957 auto Kind = llvm::StringSwitch(Id)

964 .Case("if", pp_if)

974 skipDirective(Id, First, End);

975 return false;

976 }

977

979 return lexEndif(First, End);

980

981 switch (Kind) {

986

987 if (lexIncludeFilename(First, End).is(tok::eod)) {

988 return false;

989 }

990 break;

991 default:

992 break;

993 }

994

995

996 return lexDefault(Kind, First, End);

997}

998

1000 if ((End - First) >= 3 && First[0] == '\xef' && First[1] == '\xbb' &&

1001 First[2] == '\xbf')

1003}

1004

1005bool Scanner::scanImpl(const char *First, const char *const End) {

1007 while (First != End)

1008 if (lexPPLine(First, End))

1009 return true;

1010 return false;

1011}

1012

1013bool Scanner::scan(SmallVectorImpl &Directives) {

1014 bool Error = scanImpl(Input.begin(), Input.end());

1015

1017

1018 if (LastTokenPtr &&

1019 (Tokens.empty() || LastTokenPtr > Input.begin() + Tokens.back().Offset))

1021 pushDirective(pp_eof);

1022 }

1023

1024 ArrayRef<dependency_directives_scan::Token> RemainingTokens = Tokens;

1025 for (const DirectiveWithTokens &DirWithToks : DirsWithToks) {

1026 assert(RemainingTokens.size() >= DirWithToks.NumTokens);

1027 Directives.emplace_back(DirWithToks.Kind,

1028 RemainingTokens.take_front(DirWithToks.NumTokens));

1029 RemainingTokens = RemainingTokens.drop_front(DirWithToks.NumTokens);

1030 }

1031 assert(RemainingTokens.empty());

1032

1034}

1035

1040 return Scanner(Input, Tokens, Diags, InputSourceLoc).scan(Directives);

1041}

1042

1044 StringRef Source,

1046 llvm::raw_ostream &OS) {

1047

1048 auto needsSpaceSeparator =

1051 if (Prev == Tok.Kind)

1052 return Tok .isOneOf(tok::l_paren, tok::r_paren, tok::l_square,

1053 tok::r_square);

1054 if (Prev == tok::raw_identifier &&

1055 Tok.isOneOf(tok::hash, tok::numeric_constant, tok::string_literal,

1056 tok::char_constant, tok::header_name))

1057 return true;

1058 if (Prev == tok::r_paren &&

1059 Tok.isOneOf(tok::raw_identifier, tok::hash, tok::string_literal,

1060 tok::char_constant, tok::unknown))

1061 return true;

1062 if (Prev == tok::comma &&

1063 Tok.isOneOf(tok::l_paren, tok::string_literal, tok::less))

1064 return true;

1065 return false;

1066 };

1067

1070 OS << "";

1071 std::optionaltok::TokenKind PrevTokenKind;

1073 if (PrevTokenKind && needsSpaceSeparator(*PrevTokenKind, Tok))

1074 OS << ' ';

1075 PrevTokenKind = Tok.Kind;

1076 OS << Source.slice(Tok.Offset, Tok.getEnd());

1077 }

1078 }

1079}

1080

1082 const char *const End) {

1083 assert(First <= End);

1084 while (First != End) {

1085 if (*First == '#') {

1088 }

1090 if (const auto Len = isEOL(First, End)) {

1092 continue;

1093 }

1094 break;

1095 }

1096}

1097

1099 const char *First = Source.begin();

1100 const char *const End = Source.end();

1102 if (First == End)

1103 return false;

1104

1105

1106

1108 return false;

1109

1113 if (S.lexModule(First, End))

1114 return false;

1115 auto IsCXXNamedModuleDirective = [](const DirectiveWithTokens &D) {

1116 switch (D.Kind) {

1121 return true;

1122 default:

1123 return false;

1124 }

1125 };

1126 return llvm::any_of(S.DirsWithToks, IsCXXNamedModuleDirective);

1127}

Defines the Diagnostic-related interfaces.

static void skipBlockComment(const char *&First, const char *const End)

Definition DependencyDirectivesScanner.cpp:386

static void skipRawString(const char *&First, const char *const End)

Definition DependencyDirectivesScanner.cpp:263

static void skipString(const char *&First, const char *const End)

Definition DependencyDirectivesScanner.cpp:314

static bool isStartOfRelevantLine(char First)

Definition DependencyDirectivesScanner.cpp:878

static bool isRawStringLiteral(const char *First, const char *Current)

Definition DependencyDirectivesScanner.cpp:235

static void skipUntilMaybeCXX20ModuleDirective(const char *&First, const char *const End)

Definition DependencyDirectivesScanner.cpp:1081

static void skipOverSpaces(const char *&First, const char *const End)

Definition DependencyDirectivesScanner.cpp:206

static unsigned isEOL(const char *First, const char *const End)

Definition DependencyDirectivesScanner.cpp:305

static char previousChar(const char *First, const char *&Current)

Definition DependencyDirectivesScanner.cpp:212

static void skipToNewlineRaw(const char *&First, const char *const End)

Definition DependencyDirectivesScanner.cpp:354

static unsigned skipNewline(const char *&First, const char *End)

Definition DependencyDirectivesScanner.cpp:344

static void skipUTF8ByteOrderMark(const char *&First, const char *const End)

Definition DependencyDirectivesScanner.cpp:999

static void skipLineComment(const char *&First, const char *const End)

Definition DependencyDirectivesScanner.cpp:380

static bool isQuoteCppDigitSeparator(const char *const Start, const char *const Cur, const char *const End)

Definition DependencyDirectivesScanner.cpp:401

This is the interface for scanning header and source files to get the minimum necessary preprocessor ...

FormatToken * Previous

The previous token in the unwrapped line.

bool is(tok::TokenKind Kind) const

static unsigned skipWhitespace(unsigned Idx, StringRef Str, unsigned Length)

Skip over whitespace in the string, starting at the given index.

Concrete class used by the front-end to report problems and issues.

DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)

Issue the message to the client.

bool LexFromRawLexer(Token &Result)

LexFromRawLexer - Lex a token from a designated raw lexer (one with no associated preprocessor object...

static unsigned getEscapedNewLineSize(const char *P)

getEscapedNewLineSize - Return the size of the specified escaped newline, or 0 if it is not an escape...

void seek(unsigned Offset, bool IsAtStartOfLine)

Set the lexer's buffer pointer to Offset.

unsigned getCurrentBufferOffset()

Returns the current lexing offset.

static SizedChar getCharAndSizeNoWarn(const char *Ptr, const LangOptions &LangOpts)

getCharAndSizeNoWarn - Like the getCharAndSize method, but does not ever emit a warning.

void setParsingPreprocessorDirective(bool f)

Inform the lexer whether or not we are currently lexing a preprocessor directive.

void LexIncludeFilename(Token &FilenameTok)

Lex a token, producing a header-name token if possible.

Encodes a location in the source.

SourceLocation getLocWithOffset(IntTy Offset) const

Return a source location with the specified offset from this SourceLocation.

DirectiveKind

Represents the kind of preprocessor directive or a module declaration that is tracked by the scanner ...

@ tokens_present_before_eof

Indicates that there are tokens present between the last scanned directive and eof.

@ pp_pragma_system_header

@ pp_pragma_include_alias

bool isStringLiteral(TokenKind K)

Return true if this is a C or C++ string-literal (or C++11 user-defined-string-literal) token.

TokenKind

Provides a simple uniform namespace for tokens from all C languages.

The JSON file list parser is used to communicate input to InstallAPI.

LLVM_READONLY bool isVerticalWhitespace(unsigned char c)

Returns true if this character is vertical ASCII whitespace: '\n', '\r'.

LLVM_READONLY bool isAsciiIdentifierContinue(unsigned char c)

void printDependencyDirectivesAsSource(StringRef Source, ArrayRef< dependency_directives_scan::Directive > Directives, llvm::raw_ostream &OS)

Print the previously scanned dependency directives as minimized source text.

Definition DependencyDirectivesScanner.cpp:1043

bool scanInputForCXX20ModulesUsage(StringRef Source)

Scan an input source buffer for C++20 named module usage.

Definition DependencyDirectivesScanner.cpp:1098

LLVM_READONLY bool isHorizontalWhitespace(unsigned char c)

Returns true if this character is horizontal ASCII whitespace: ' ', '\t', '\f', '\v'.

bool scanSourceForDependencyDirectives(StringRef Input, SmallVectorImpl< dependency_directives_scan::Token > &Tokens, SmallVectorImpl< dependency_directives_scan::Directive > &Directives, DiagnosticsEngine *Diags=nullptr, SourceLocation InputSourceLoc=SourceLocation())

Scan the input for the preprocessor directives that might have an effect on the dependencies for a co...

Definition DependencyDirectivesScanner.cpp:1036

LLVM_READONLY bool isWhitespace(unsigned char c)

Return true if this character is horizontal or vertical ASCII whitespace: ' ', '\t',...

LLVM_READONLY bool isPreprocessingNumberBody(unsigned char c)

Return true if this is the body character of a C preprocessing number, which is [a-zA-Z0-9_.

void prepare_PragmaString(SmallVectorImpl< char > &StrVal)

Destringize a _Pragma("") string according to C11 6.10.9.1: "The string literal is destringized by de...

Diagnostic wrappers for TextAPI types for error reporting.

Represents a directive that's lexed as part of the dependency directives scanning.

DirectiveKind Kind

The kind of token.

Token lexed as part of dependency directive scanning.

bool isNot(tok::TokenKind K) const

unsigned Offset

Offset into the original source input.

bool is(tok::TokenKind K) const