PPCaching.cpp Source File (original) (raw)

15using namespace clang;

17std::pair<Preprocessor::CachedTokensTy::size_type, bool>

18Preprocessor::LastBacktrackPos() {

20 auto BacktrackPos = BacktrackPositions.back();

21 bool Unannotated =

22 static_castCachedTokensTy::difference\_type\(BacktrackPos) < 0;

23 return {Unannotated ? ~BacktrackPos : BacktrackPos, Unannotated};

24}

35 assert(LexLevel == 0 && "cannot use lookahead while lexing");

36 BacktrackPositions.push_back(Unannotated ? ~CachedLexPos : CachedLexPos);

37 if (Unannotated)

38 UnannotatedBacktrackTokens.emplace_back(CachedTokens, CachedTokens.size());

39 EnterCachingLexMode();

40}

42Preprocessor::CachedTokensTy Preprocessor::PopUnannotatedBacktrackTokens() {

44 auto [UnannotatedTokens, NumCachedToks] =

45 std::move(UnannotatedBacktrackTokens.back());

46 UnannotatedBacktrackTokens.pop_back();

50 UnannotatedBacktrackTokens.back().first.append(

51 UnannotatedTokens.begin() + NumCachedToks, UnannotatedTokens.end());

52 return std::move(UnannotatedTokens);

53}

57 assert(isBacktrackEnabled() && "EnableBacktrackAtThisPos was not called!");

58 auto [BacktrackPos, Unannotated] = LastBacktrackPos();

59 BacktrackPositions.pop_back();

60 if (Unannotated)

61 PopUnannotatedBacktrackTokens();

62}

67 assert(isBacktrackEnabled() && "EnableBacktrackAtThisPos was not called!");

68 auto [BacktrackPos, Unannotated] = LastBacktrackPos();

69 BacktrackPositions.pop_back();

70 CachedLexPos = BacktrackPos;

71 if (Unannotated)

72 CachedTokens = PopUnannotatedBacktrackTokens();

74}

76void Preprocessor::CachingLex(Token &Result) {

77 if (!InCachingLexMode())

78 return;

81 assert(LexLevel == 1 &&

82 "should not use token caching within the preprocessor");

87 return;

88 }

90 ExitCachingLexMode();

91 Lex(Result);

95 EnterCachingLexModeUnchecked();

97 ++CachedLexPos;

99 UnannotatedBacktrackTokens.back().first.push_back(Result);

100 return;

101 }

102

103 if (CachedLexPos < CachedTokens.size()) {

104 EnterCachingLexModeUnchecked();

105 } else {

106

107 CachedTokens.clear();

108 CachedLexPos = 0;

109 }

110}

111

112void Preprocessor::EnterCachingLexMode() {

113

114

115

116

117 assert(LexLevel == 0 &&

118 "entered caching lex mode while lexing something else");

119

120 if (InCachingLexMode()) {

121 assert(CurLexerCallback == CLK_CachingLexer && "Unexpected lexer kind");

122 return;

123 }

124

125 EnterCachingLexModeUnchecked();

126}

127

128void Preprocessor::EnterCachingLexModeUnchecked() {

129 assert(CurLexerCallback != CLK_CachingLexer && "already in caching lex mode");

130 PushIncludeMacroStack();

131 CurLexerCallback = CLK_CachingLexer;

132}

133

134

135const Token &Preprocessor::PeekAhead(unsigned N) {

136 assert(CachedLexPos + N > CachedTokens.size() && "Confused caching.");

137 ExitCachingLexMode();

138 for (size_t C = CachedLexPos + N - CachedTokens.size(); C > 0; --C) {

139 CachedTokens.push_back(Token());

140 Lex(CachedTokens.back());

142 UnannotatedBacktrackTokens.back().first.push_back(CachedTokens.back());

143 }

144 EnterCachingLexMode();

145 return CachedTokens.back();

146}

147

148void Preprocessor::AnnotatePreviousCachedTokens(const Token &Tok) {

149 assert(Tok.isAnnotation() && "Expected annotation token");

150 assert(CachedLexPos != 0 && "Expected to have some cached tokens");

152 && "The annotation should be until the most recent cached token");

153

154

155

156 for (CachedTokensTy::size_type i = CachedLexPos; i != 0; --i) {

157 CachedTokensTy::iterator AnnotBegin = CachedTokens.begin() + i-1;

158 if (AnnotBegin->getLocation() == Tok.getLocation()) {

160 "The backtrack pos points inside the annotated tokens!");

161

162 if (i < CachedLexPos)

163 CachedTokens.erase(AnnotBegin + 1, CachedTokens.begin() + CachedLexPos);

164 *AnnotBegin = Tok;

165 CachedLexPos = i;

166 return;

167 }

168 }

169}

170

172

173 if (!CachedLexPos)

174 return false;

175

176 const Token LastCachedTok = CachedTokens[CachedLexPos - 1];

177 if (LastCachedTok.getKind() != Tok.getKind())

178 return false;

179

183 RelOffset)

184 return false;

185

186 return true;

187}

188

190 assert(CachedLexPos != 0 && "Expected to have some cached tokens");

191 CachedTokens.insert(CachedTokens.begin() + CachedLexPos - 1, NewToks.begin(),

192 NewToks.end());

193 CachedTokens.erase(CachedTokens.begin() + CachedLexPos - 1 + NewToks.size());

194 CachedLexPos += NewToks.size() - 1;

195}

Defines the clang::Preprocessor interface.

bool IsPreviousCachedToken(const Token &Tok) const

Whether Tok is the most recent token (CachedLexPos - 1) in CachedTokens.

Definition PPCaching.cpp:171

void CommitBacktrackedTokens()

Disable the last EnableBacktrackAtThisPos call.

Definition PPCaching.cpp:56

void Lex(Token &Result)

Lex the next token for this preprocessor.

void Backtrack()

Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...

Definition PPCaching.cpp:66

SourceManager & getSourceManager() const

bool isBacktrackEnabled() const

True if EnableBacktrackAtThisPos() was called and caching of tokens is on.

bool isUnannotatedBacktrackEnabled() const

True if EnableBacktrackAtThisPos() was called and caching of unannotated tokens is on.

void recomputeCurLexerKind()

Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.

void ReplacePreviousCachedToken(ArrayRef< Token > NewToks)

Replace token in CachedLexPos - 1 in CachedTokens by the tokens in NewToks.

Definition PPCaching.cpp:189

void EnableBacktrackAtThisPos(bool Unannotated=false)

From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...

Definition PPCaching.cpp:34

SourceLocation getLastCachedTokenLocation() const

Get the location of the last cached token, suitable for setting the end location of an annotation tok...

Token - This structure provides full information about a lexed token.

SourceLocation getLocation() const

Return a source location identifier for the specified offset in the current file.

SourceLocation getAnnotationEndLoc() const

tok::TokenKind getKind() const

bool isAnnotation() const

Return true if this is any of tok::annot_* kind tokens.

The JSON file list parser is used to communicate input to InstallAPI.

@ Result

The result type of a method or function.

SmallVector< Token, 4 > CachedTokens

A set of tokens that has been cached for later parsing.