FormatTokenSource.h Source File (original) (raw)

15#ifndef LLVM_CLANG_LIB_FORMAT_FORMATTOKENSOURCE_H

16#define LLVM_CLANG_LIB_FORMAT_FORMATTOKENSOURCE_H

20#define DEBUG_TYPE "format-token-source"

23namespace format {

30public:

78 do {

80 assert(Tok);

81 } while (Tok->is(tok::comment));

82 return Tok;

83 }

84};

87public:

89 : Tokens(Tokens), Position(-1) {}

92 if (Position >= 0 && isEOF()) {

93 LLVM_DEBUG({

94 llvm::dbgs() << "Next ";

95 dbgToken(Position);

96 });

97 return Tokens[Position];

98 }

99 Position = successor(Position);

100 LLVM_DEBUG({

101 llvm::dbgs() << "Next ";

102 dbgToken(Position);

103 });

104 return Tokens[Position];

105 }

106

108 assert(Position <= 0 || Tokens[Position - 1]->isNot(tok::eof));

109 return Position > 0 ? Tokens[Position - 1] : nullptr;

110 }

111

114 return Tokens[Position];

115 int Next = successor(Position);

116 if (SkipComment)

117 while (Tokens[Next]->is(tok::comment))

118 Next = successor(Next);

119 LLVM_DEBUG({

120 llvm::dbgs() << "Peeking ";

121 dbgToken(Next);

122 });

123 return Tokens[Next];

124 }

125

127 return Position == -1 ? false : Tokens[Position]->is(tok::eof);

128 }

129

131 LLVM_DEBUG(llvm::dbgs() << "Getting Position: " << Position << "\n");

132 assert(Position >= 0);

133 return Position;

134 }

135

137 LLVM_DEBUG(llvm::dbgs() << "Setting Position: " << P << "\n");

138 Position = P;

139 return Tokens[Position];

140 }

141

143 assert(Position != -1);

144 assert((*New.rbegin())->Tok.is(tok::eof));

145 int Next = Tokens.size();

146 Tokens.append(New.begin(), New.end());

147 LLVM_DEBUG({

148 llvm::dbgs() << "Inserting:\n";

149 for (int I = Next, E = Tokens.size(); I != E; ++I)

150 dbgToken(I, " ");

151 llvm::dbgs() << " Jump from: " << (Tokens.size() - 1) << " -> "

152 << Position << "\n";

153 });

154 Jumps[Tokens.size() - 1] = Position;

155 Position = Next;

156 LLVM_DEBUG({

157 llvm::dbgs() << "At inserted token ";

158 dbgToken(Position);

159 });

160 return Tokens[Position];

161 }

162

163 void reset() { Position = -1; }

164

165private:

166 int successor(int Current) const {

167 int Next = Current + 1;

168 auto it = Jumps.find(Next);

169 if (it != Jumps.end()) {

170 Next = it->second;

171 assert(!Jumps.contains(Next));

172 }

173 return Next;

174 }

175

176 void dbgToken(int Position, StringRef Indent = "") {

177 FormatToken *Tok = Tokens[Position];

178 llvm::dbgs() << Indent << "[" << Position

179 << "] Token: " << Tok->Tok.getName() << " / " << Tok->TokenText

180 << ", Macro: " << !!Tok->MacroCtx << "\n";

181 }

182

183 SmallVector<FormatToken *> Tokens;

184 int Position;

185

186

187

188 llvm::DenseMap<int, int> Jumps;

189};

190

192public:

195 : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),

196 PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),

197 Token(nullptr), PreviousToken(nullptr) {

200 TokenSource = this;

201 Line.Level = 0;

202 Line.InPPDirective = true;

203

204 }

205

207 TokenSource = PreviousTokenSource;

208 ResetToken = Token;

211 Line.Level = PreviousLineLevel;

212 }

213

215

216

217 assert(!eof());

218 PreviousToken = Token;

220 if (eof())

221 return &FakeEOF;

223 }

224

227 }

228

230 if (eof())

231 return &FakeEOF;

232 return PreviousTokenSource->peekNextToken(SkipComment);

233 }

234

235 bool isEOF() override { return PreviousTokenSource->isEOF(); }

236

238

240 PreviousToken = nullptr;

243 }

244

246 llvm_unreachable("Cannot insert tokens while parsing a macro.");

247 return nullptr;

248 }

249

250private:

251 bool eof() {

252 return Token && Token->HasUnescapedNewline &&

254 PreviousToken);

255 }

256

257 FormatToken FakeEOF;

258 UnwrappedLine &Line;

259 FormatTokenSource *&TokenSource;

260 FormatToken *&ResetToken;

261 unsigned PreviousLineLevel;

262 FormatTokenSource *PreviousTokenSource;

263

264 FormatToken *Token;

265 FormatToken *PreviousToken;

266};

267

268}

269}

270

271#undef DEBUG_TYPE

272

273#endif

This file contains the declaration of the UnwrappedLineParser, which turns a stream of tokens into Un...

Token - This structure provides full information about a lexed token.

void setKind(tok::TokenKind K)

void startToken()

Reset all flags to cleared.

virtual FormatToken * peekNextToken(bool SkipComment=false)=0

virtual unsigned getPosition()=0

virtual FormatToken * insertTokens(ArrayRef< FormatToken * > Tokens)=0

FormatToken * getNextNonComment()

virtual FormatToken * getPreviousToken()=0

virtual FormatToken * setPosition(unsigned Position)=0

virtual ~FormatTokenSource()

virtual FormatToken * getNextToken()=0

FormatToken * getPreviousToken() override

IndexedTokenSource(ArrayRef< FormatToken * > Tokens)

FormatToken * getNextToken() override

unsigned getPosition() override

FormatToken * peekNextToken(bool SkipComment=false) override

FormatToken * setPosition(unsigned P) override

FormatToken * insertTokens(ArrayRef< FormatToken * > New) override

FormatToken * peekNextToken(bool SkipComment) override

FormatToken * getNextToken() override

ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource, FormatToken *&ResetToken)

FormatToken * setPosition(unsigned Position) override

FormatToken * getPreviousToken() override

FormatToken * insertTokens(ArrayRef< FormatToken * > Tokens) override

~ScopedMacroState() override

unsigned getPosition() override

bool continuesLineComment(const FormatToken &FormatTok, const FormatToken *Previous, const FormatToken *MinColumnToken)

The JSON file list parser is used to communicate input to InstallAPI.

A wrapper around a Token storing information about the whitespace characters preceding it.

bool is(tok::TokenKind Kind) const

An unwrapped line is a sequence of Token, that we would like to put on a single line if there was no ...

bool InMacroBody

Whether it is part of a macro body.

unsigned Level

The indent level of the UnwrappedLine.

bool InPPDirective

Whether this UnwrappedLine is part of a preprocessor directive.