Token.h Source File (original) (raw)

13#ifndef LLVM_CLANG_LEX_TOKEN_H

14#define LLVM_CLANG_LEX_TOKEN_H

18#include "llvm/ADT/ArrayRef.h"

19#include "llvm/ADT/StringRef.h"

20#include

24class IdentifierInfo;

25class LangOptions;

64 void *PtrData;

70 unsigned short Flags;

72public:

75 StartOfLine = 0x01,

77 LeadingSpace = 0x02,

79 DisableExpand = 0x04,

80 NeedsCleaning = 0x08,

82 HasUDSuffix = 0x20,

83 HasUCN = 0x40,

84 IgnoredComma = 0x80,

87 CommaAfterElided = 0x200,

89 IsReinjected = 0x800,

92 };

102 return is(K1) || is(K2);

103 }

106 }

107

108

109

112 }

113

114

115

118 }

119

120

122

123

124

125

128 }

129

130

131

134 }

136 assert( isAnnotation () && "Annotation tokens have no length field");

137 return UintData;

138 }

139

142 assert( isAnnotation () && "Annotation tokens have no length field");

143 UintData = Len;

144 }

145

147 assert(isAnnotation() && "Used AnnotEndLocID on non-annotation token");

149 }

151 assert(isAnnotation() && "Used AnnotEndLocID on non-annotation token");

153 }

154

157 }

158

162 }

163

164

165

168 }

172 }

173

175

176

178 Kind = tok::unknown;

179 Flags = 0;

180 PtrData = nullptr;

181 UintData = 0;

183 }

184

185 bool hasPtrData() const { return PtrData != nullptr; }

186

188 assert(isNot(tok::raw_identifier) &&

189 "getIdentifierInfo() on a tok::raw_identifier token!");

191 "getIdentifierInfo() on an annotation token!");

193 if (is(tok::eof)) return nullptr;

195 }

197 PtrData = (void*) II;

198 }

199

201 assert(is(tok::eof));

202 return reinterpret_cast<const void *>(PtrData);

203 }

205 assert(is(tok::eof));

206 assert(!PtrData);

207 PtrData = const_cast<void *>(D);

208 }

209

210

211

212

214 assert(is(tok::raw_identifier));

215 return StringRef(reinterpret_cast<const char *>(PtrData), getLength());

216 }

218 assert(is(tok::raw_identifier));

219 PtrData = const_cast<char*>(Ptr);

220 }

221

222

223

224

226 assert(isLiteral() && "Cannot get literal data of non-literal");

227 return reinterpret_cast<const char*>(PtrData);

228 }

230 assert(isLiteral() && "Cannot set literal data of non-literal");

231 PtrData = const_cast<char*>(Ptr);

232 }

233

235 assert(isAnnotation() && "Used AnnotVal on non-annotation token");

236 return PtrData;

237 }

239 assert(isAnnotation() && "Used AnnotVal on non-annotation token");

240 PtrData = val;

241 }

242

243

245 Flags |= Flag;

246 }

247

248

250 return (Flags & Flag) != 0;

251 }

252

253

255 Flags &= ~Flag;

256 }

257

258

259

260

261

263 return Flags;

264 }

265

266

268 if (Val)

270 else

272 }

273

274

275

277

278

279

281

282

283

285

286

288

289

291

293

294

296

297

298

300

301

302

304

305

307

308

309

311

312

314

315

316

317

318

319

321};

322

323

324

326

328

329

330

332

333

334

336

337

338

340};

341

342

347};

348}

349

350#endif

static constexpr bool isOneOf()

Defines the clang::SourceLocation class and associated facilities.

Defines the clang::TokenKind enum and support functions.

One of these records is kept for each identifier that is lexed.

Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...

Encodes a location in the source.

static SourceLocation getFromRawEncoding(UIntTy Encoding)

Turn a raw encoding of a SourceLocation object into a real SourceLocation.

SourceLocation getLocWithOffset(IntTy Offset) const

Return a source location with the specified offset from this SourceLocation.

UIntTy getRawEncoding() const

When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it.

A trivial tuple used to represent a source range.

SourceLocation getEnd() const

SourceLocation getBegin() const

Token - This structure provides full information about a lexed token.

IdentifierInfo * getIdentifierInfo() const

void setLiteralData(const char *Ptr)

bool isAnyIdentifier() const

Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...

SourceLocation getEndLoc() const

unsigned getFlags() const

Return the internal represtation of the flags.

void setAnnotationEndLoc(SourceLocation L)

bool hasUCN() const

Returns true if this token contains a universal character name.

void clearFlag(TokenFlags Flag)

Unset the specified flag.

bool isLiteral() const

Return true if this is a "literal", like a numeric constant, string, etc.

SourceLocation getLocation() const

Return a source location identifier for the specified offset in the current file.

const char * getName() const

unsigned getLength() const

void setLength(unsigned Len)

bool isEditorPlaceholder() const

Returns true if this token is an editor placeholder.

bool isExpandDisabled() const

Return true if this identifier token should never be expanded in the future, due to C99 6....

void setKind(tok::TokenKind K)

bool commaAfterElided() const

Returns true if the comma after this token was elided.

SourceLocation getAnnotationEndLoc() const

tok::ObjCKeywordKind getObjCKeywordID() const

Return the ObjC keyword kind.

bool is(tok::TokenKind K) const

is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {....

void * getAnnotationValue() const

bool isOneOf(tok::TokenKind K1, Ts... Ks) const

tok::TokenKind getKind() const

bool isRegularKeywordAttribute() const

Return true if the token is a keyword that is parsed in the same position as a standard attribute,...

bool isAtStartOfLine() const

isAtStartOfLine - Return true if this token is at the start of a line.

void setEofData(const void *D)

bool getFlag(TokenFlags Flag) const

Get the specified flag.

bool hasLeadingSpace() const

Return true if this token has whitespace before it.

SourceRange getAnnotationRange() const

SourceRange of the group of tokens that this annotation token represents.

void setLocation(SourceLocation L)

bool hasLeadingEmptyMacro() const

Return true if this token has an empty macro before it.

void setRawIdentifierData(const char *Ptr)

bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const

bool isNot(tok::TokenKind K) const

bool isAnnotation() const

Return true if this is any of tok::annot_* kind tokens.

void setAnnotationValue(void *val)

const void * getEofData() const

bool hasUDSuffix() const

Return true if this token is a string or character literal which has a ud-suffix.

bool stringifiedInMacro() const

Returns true if this token is formed by macro by stringizing or charizing operator.

bool isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const

Return true if we have an ObjC keyword identifier.

void setAnnotationRange(SourceRange R)

bool isSimpleTypeSpecifier(const LangOptions &LangOpts) const

Determine whether the token kind starts a simple-type-specifier.

void startToken()

Reset all flags to cleared.

bool needsCleaning() const

Return true if this token has trigraphs or escaped newlines in it.

void setIdentifierInfo(IdentifierInfo *II)

SourceLocation getLastLoc() const

void setFlagValue(TokenFlags Flag, bool Val)

Set a flag to either true or false.

StringRef getRawIdentifier() const

getRawIdentifier - For a raw identifier token (i.e., an identifier lexed in raw mode),...

const char * getLiteralData() const

getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...

void setFlag(TokenFlags Flag)

Set the specified flag.

const char * getTokenName(TokenKind Kind) LLVM_READNONE

Determines the name of a token as used within the front end.

bool isAnyIdentifier(TokenKind K)

Return true if this is a raw identifier or an identifier kind.

ObjCKeywordKind

Provides a namespace for Objective-C keywords which start with an '@'.

TokenKind

Provides a simple uniform namespace for tokens from all C languages.

constexpr bool isRegularKeywordAttribute(TokenKind K)

bool isLiteral(TokenKind K)

Return true if this is a "literal" kind, like a numeric constant, string, etc.

bool isAnnotation(TokenKind K)

Return true if this is any of tok::annot_* kinds.

The JSON file list parser is used to communicate input to InstallAPI.

Information about the conditional stack (#if directives) currently active.

bool FoundNonSkip

True if we have emitted tokens already, and now we're in an #else block or something.

SourceLocation IfLoc

Location where the conditional started.

bool WasSkipping

True if this was contained in a skipping directive, e.g., in a "\#if 0" block.

bool FoundElse

True if we've seen a #else in this block.