TokenConcatenation.cpp Source File (original) (raw)

16#include "llvm/Support/ErrorHandling.h"

17using namespace clang;

24 if (Str[0] == 'L' ||

25 (CPlusPlus11 && (Str[0] == 'u' || Str[0] == 'U' || Str[0] == 'R'))) {

27 if (Str.size() == 1)

28 return true;

32 if (Str[1] == 'R' && Str[0] != 'R' && Str.size() == 2 && CPlusPlus11)

33 return true;

36 if (Str[0] == 'u' && Str[1] == '8') {

37 if (Str.size() == 2) return true;

38 if (Str.size() == 3 && Str[2] == 'R') return true;

39 }

40 }

42 return false;

43}

47bool TokenConcatenation::IsIdentifierStringPrefix(const Token &Tok) const {

52 return false;

54 const char *Ptr = SM.getCharacterData(SM.getSpellingLoc(Tok.getLocation()));

56 LangOpts.CPlusPlus11);

57 }

60 char Buffer[256];

61 const char *TokPtr = Buffer;

64 }

67}

70 memset(TokenInfo, 0, sizeof(TokenInfo));

73 TokenInfo[tok::identifier ] |= aci_custom;

74 TokenInfo[tok::numeric_constant] |= aci_custom_firstchar;

75 TokenInfo[tok::period ] |= aci_custom_firstchar;

76 TokenInfo[tok::amp ] |= aci_custom_firstchar;

77 TokenInfo[tok::plus ] |= aci_custom_firstchar;

78 TokenInfo[tok::minus ] |= aci_custom_firstchar;

79 TokenInfo[tok::slash ] |= aci_custom_firstchar;

80 TokenInfo[tok::less ] |= aci_custom_firstchar;

81 TokenInfo[tok::greater ] |= aci_custom_firstchar;

82 TokenInfo[tok::pipe ] |= aci_custom_firstchar;

83 TokenInfo[tok::percent ] |= aci_custom_firstchar;

84 TokenInfo[tok::colon ] |= aci_custom_firstchar;

85 TokenInfo[tok::hash ] |= aci_custom_firstchar;

86 TokenInfo[tok::arrow ] |= aci_custom_firstchar;

90 TokenInfo[tok::string_literal ] |= aci_custom;

91 TokenInfo[tok::wide_string_literal ] |= aci_custom;

92 TokenInfo[tok::utf8_string_literal ] |= aci_custom;

93 TokenInfo[tok::utf16_string_literal] |= aci_custom;

94 TokenInfo[tok::utf32_string_literal] |= aci_custom;

95 TokenInfo[tok::char_constant ] |= aci_custom;

96 TokenInfo[tok::wide_char_constant ] |= aci_custom;

97 TokenInfo[tok::utf16_char_constant ] |= aci_custom;

98 TokenInfo[tok::utf32_char_constant ] |= aci_custom;

99 }

100

101

103 TokenInfo[tok::utf8_char_constant] |= aci_custom;

104

105

107 TokenInfo[tok::lessequal ] |= aci_custom_firstchar;

108

109

110 TokenInfo[tok::amp ] |= aci_avoid_equal;

111 TokenInfo[tok::plus ] |= aci_avoid_equal;

112 TokenInfo[tok::minus ] |= aci_avoid_equal;

113 TokenInfo[tok::slash ] |= aci_avoid_equal;

114 TokenInfo[tok::less ] |= aci_avoid_equal;

115 TokenInfo[tok::greater ] |= aci_avoid_equal;

116 TokenInfo[tok::pipe ] |= aci_avoid_equal;

117 TokenInfo[tok::percent ] |= aci_avoid_equal;

118 TokenInfo[tok::star ] |= aci_avoid_equal;

119 TokenInfo[tok::exclaim ] |= aci_avoid_equal;

120 TokenInfo[tok::lessless ] |= aci_avoid_equal;

121 TokenInfo[tok::greatergreater] |= aci_avoid_equal;

122 TokenInfo[tok::caret ] |= aci_avoid_equal;

123 TokenInfo[tok::equal ] |= aci_avoid_equal;

124}

125

126

127

130

131 return II->getNameStart()[0];

135 } else {

137 return *SM.getCharacterData(SM.getSpellingLoc(Tok.getLocation()));

138 }

139 } else if (Tok.getLength() < 256) {

140 char Buffer[256];

141 const char *TokPtr = Buffer;

143 return TokPtr[0];

144 } else {

146 }

147}

148

149

150

151

152

153

154

155

156

157

158

159

161 const Token &PrevTok,

162 const Token &Tok) const {

163

164 if (PrevTok.is(tok::annot_header_unit) && Tok.is(tok::semi))

165 return false;

166

167

168

170 return true;

171

172

173

174

179 return false;

180

183 PrevKind = tok::identifier;

184

185

186 unsigned ConcatInfo = TokenInfo[PrevKind];

187

188

189 if (ConcatInfo == 0) return false;

190

191 if (ConcatInfo & aci_avoid_equal) {

192

193 if (Tok.isOneOf(tok::equal, tok::equalequal))

194 return true;

195 ConcatInfo &= ~aci_avoid_equal;

196 }

198

199 assert(Tok.isOneOf(tok::annot_module_include, tok::annot_module_begin,

200 tok::annot_module_end, tok::annot_embed) &&

201 "unexpected annotation in AvoidConcat");

202

203 ConcatInfo = 0;

204 if (Tok.is(tok::annot_embed))

205 return true;

206 }

207

208 if (ConcatInfo == 0)

209 return false;

210

211

212

213

214 char FirstChar = 0;

215 if (ConcatInfo & aci_custom) {

216

217 } else {

219 }

220

221 switch (PrevKind) {

222 default:

223 llvm_unreachable("InitAvoidConcatTokenInfo built wrong");

224

225 case tok::raw_identifier:

226 llvm_unreachable("tok::raw_identifier in non-raw lexing mode!");

227

228 case tok::string_literal:

229 case tok::wide_string_literal:

230 case tok::utf8_string_literal:

231 case tok::utf16_string_literal:

232 case tok::utf32_string_literal:

233 case tok::char_constant:

234 case tok::wide_char_constant:

235 case tok::utf8_char_constant:

236 case tok::utf16_char_constant:

237 case tok::utf32_char_constant:

239 return false;

240

241

242

244 return true;

245

246

247

249 return false;

250 [[fallthrough]];

251 case tok::identifier:

252

253 if (Tok.is(tok::numeric_constant))

255

257 Tok.isOneOf(tok::wide_string_literal, tok::utf8_string_literal,

258 tok::utf16_string_literal, tok::utf32_string_literal,

259 tok::wide_char_constant, tok::utf8_char_constant,

260 tok::utf16_char_constant, tok::utf32_char_constant))

261 return true;

262

263

264 if (Tok.isNot(tok::char_constant) && Tok.isNot(tok::string_literal))

265 return false;

266

267

268

269 return IsIdentifierStringPrefix(PrevTok);

270

271 case tok::numeric_constant:

273 FirstChar == '+' || FirstChar == '-';

274 case tok::period:

275 return (FirstChar == '.' && PrevPrevTok.is(tok::period)) ||

277 (PP.getLangOpts().CPlusPlus && FirstChar == '*');

278 case tok::amp:

279 return FirstChar == '&';

280 case tok:➕

281 return FirstChar == '+';

282 case tok:➖

283 return FirstChar == '-' || FirstChar == '>';

284 case tok::slash:

285 return FirstChar == '*' || FirstChar == '/';

286 case tok::less:

287 return FirstChar == '<' || FirstChar == ':' || FirstChar == '%';

288 case tok::greater:

289 return FirstChar == '>';

290 case tok::pipe:

291 return FirstChar == '|';

292 case tok::percent:

293 return FirstChar == '>' || FirstChar == ':';

294 case tok::colon:

295 return FirstChar == '>' ||

296 (PP.getLangOpts().CPlusPlus && FirstChar == ':');

297 case tok::hash:

298 return FirstChar == '#' || FirstChar == '@' || FirstChar == '%';

299 case tok::arrow:

300 return PP.getLangOpts().CPlusPlus && FirstChar == '*';

301 case tok::lessequal:

302 return PP.getLangOpts().CPlusPlus20 && FirstChar == '>';

303 }

304}

Defines the clang::Preprocessor interface.

static char GetFirstChar(const Preprocessor &PP, const Token &Tok)

GetFirstChar - Get the first character of the token.

static bool IsStringPrefix(StringRef Str, bool CPlusPlus11)

IsStringPrefix - Return true if Str is a string prefix.

__DEVICE__ void * memset(void *__a, int __b, size_t __c)

One of these records is kept for each identifier that is lexed.

Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...

Engages in a tight little dance with the lexer to efficiently preprocess tokens.

SourceManager & getSourceManager() const

StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const

Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...

const LangOptions & getLangOpts() const

Encodes a location in the source.

SourceLocation getLocWithOffset(IntTy Offset) const

Return a source location with the specified offset from this SourceLocation.

This class handles loading and caching of source files into memory.

TokenConcatenation(const Preprocessor &PP)

bool AvoidConcat(const Token &PrevPrevTok, const Token &PrevTok, const Token &Tok) const

AvoidConcat - If printing PrevTok immediately followed by Tok would cause the two individual tokens t...

Token - This structure provides full information about a lexed token.

IdentifierInfo * getIdentifierInfo() const

bool isLiteral() const

Return true if this is a "literal", like a numeric constant, string, etc.

SourceLocation getLocation() const

Return a source location identifier for the specified offset in the current file.

unsigned getLength() const

bool is(tok::TokenKind K) const

is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....

tok::TokenKind getKind() const

bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const

bool isNot(tok::TokenKind K) const

bool isAnnotation() const

Return true if this is any of tok::annot_* kind tokens.

bool hasUDSuffix() const

Return true if this token is a string or character literal which has a ud-suffix.

bool needsCleaning() const

Return true if this token has trigraphs or escaped newlines in it.

const char * getLiteralData() const

getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...

TokenKind

Provides a simple uniform namespace for tokens from all C languages.

The JSON file list parser is used to communicate input to InstallAPI.

LLVM_READONLY bool isDigit(unsigned char c)

Return true if this character is an ASCII digit: [0-9].

LLVM_READONLY bool isPreprocessingNumberBody(unsigned char c)

Return true if this is the body character of a C preprocessing number, which is [a-zA-Z0-9_.

float __ovld __cnfn length(float)

Return the length of vector p, i.e., sqrt(p.x2 + p.y 2 + ...)