PGOCtxProfLowering.cpp Source File (original) (raw)

28#include

30using namespace llvm;

32#define DEBUG_TYPE "ctx-instr-lower"

37 "A function name, assumed to be global, which will be treated as the "

38 "root of an interesting graph, which will be profiled independently "

39 "from other similar graphs."));

48static auto StartCtx = "__llvm_ctx_profile_start_context";

49static auto ReleaseCtx = "__llvm_ctx_profile_release_context";

50static auto GetCtx = "__llvm_ctx_profile_get_context";

52static auto CallsiteTLS = "__llvm_ctx_profile_callsite";

53}

55namespace {

57class CtxInstrumentationLowerer final {

60 Type *ContextNodeTy = nullptr;

66 Function *ReleaseCtx = nullptr;

69 Constant *CannotBeRootInitializer = nullptr;

71public:

75};

81std::pair<uint32_t, uint32_t> getNumCountersAndCallsites(const Function &F) {

83 uint32_t NumCallsites = 0;

84 for (const auto &BB : F) {

85 for (const auto &I : BB) {

87 uint32_t V =

88 static_cast<uint32_t>(Incr->getNumCounters()->getZExtValue());

90 "expected all llvm.instrprof.increment[.step] intrinsics to "

91 "have the same total nr of counters parameter");

94 uint32_t V =

95 static_cast<uint32_t>(CSIntr->getNumCounters()->getZExtValue());

96 assert((!NumCallsites || V == NumCallsites) &&

97 "expected all llvm.instrprof.callsite intrinsics to have the "

98 "same total nr of callsites parameter");

99 NumCallsites = V;

100 }

101#ifdef NDEBUG

103 return std::make_pair(NumCounters, NumCallsites);

104#endif

105 }

106 }

108}

109

111 F.getContext().emitError("[ctxprof] The function " + F.getName() +

112 " was indicated as context root but " + Reason +

113 ", which is not supported.");

114}

115}

116

117

118

119

120CtxInstrumentationLowerer::CtxInstrumentationLowerer(Module &M,

123 auto *PointerTy = PointerType::get(M.getContext(), 0);

124 auto *SanitizerMutexType = Type::getInt8Ty(M.getContext());

125 auto *I32Ty = Type::getInt32Ty(M.getContext());

126 auto *I64Ty = Type::getInt64Ty(M.getContext());

127

128#define _PTRDECL(_, __) PointerTy,

129#define _VOLATILE_PTRDECL(_, __) PointerTy,

130#define _CONTEXT_ROOT PointerTy,

131#define _MUTEXDECL(_) SanitizerMutexType,

132

134 M.getContext(), {CTXPROF_FUNCTION_DATA(_PTRDECL, _CONTEXT_ROOT,

135 _VOLATILE_PTRDECL, _MUTEXDECL)});

136#undef _PTRDECL

137#undef _CONTEXT_ROOT

138#undef _VOLATILE_PTRDECL

139#undef _MUTEXDECL

140

141#define _PTRDECL(_, __) Constant::getNullValue(PointerTy),

142#define _VOLATILE_PTRDECL(_, __) _PTRDECL(_, __)

143#define _MUTEXDECL(_) Constant::getNullValue(SanitizerMutexType),

144#define _CONTEXT_ROOT \

145 Constant::getIntegerValue( \

146 PointerTy, \

147 APInt(M.getDataLayout().getPointerTypeSizeInBits(PointerTy), 1U)),

151#undef _PTRDECL

152#undef _CONTEXT_ROOT

153#undef _VOLATILE_PTRDECL

154#undef _MUTEXDECL

155

156

158 I64Ty,

159 PointerTy,

160 I32Ty,

161 I32Ty,

162 });

163

164

165

167 if (const auto *F = M.getFunction(Fname)) {

168 if (F->isDeclaration())

169 continue;

170 ContextRootSet.insert(F);

171 for (const auto &BB : *F)

172 for (const auto &I : BB)

174 if (CB->isMustTailCall())

175 emitUnsupportedRootError(*F, "it features musttail calls");

176 }

177 }

178

179

181 M.getOrInsertFunction(

184 {PointerTy,

185 I64Ty, I32Ty,

186 I32Ty },

187 false))

188 .getCallee());

192 {PointerTy,

193 PointerTy,

194 I64Ty,

195 I32Ty,

196 I32Ty},

197 false))

198 .getCallee());

201 FunctionType::get(Type::getVoidTy(M.getContext()),

202 {

203 PointerTy,

204 },

205 false))

206 .getCallee());

207

208

209 CallsiteInfoTLS =

212 CallsiteInfoTLS->setThreadLocal(true);

219}

220

223 CtxInstrumentationLowerer Lowerer(M, MAM);

225 for (auto &F : M)

226 Changed |= Lowerer.lowerFunction(F);

228}

229

230bool CtxInstrumentationLowerer::lowerFunction(Function &F) {

231 if (F.isDeclaration())

232 return false;

233

234

235

237 for (auto &BB : F)

240 I.eraseFromParent();

241 if (ContextRootSet.contains(&F))

242 emitUnsupportedRootError(F, "it does not return");

243 return true;

244 }

245

248

250 auto [NumCounters, NumCallsites] = getNumCountersAndCallsites(F);

251

252 Value *Context = nullptr;

253 Value *RealContext = nullptr;

254

255 StructType *ThisContextType = nullptr;

256 Value *TheRootFunctionData = nullptr;

257 Value *ExpectedCalleeTLSAddr = nullptr;

258 Value *CallsiteInfoTLSAddr = nullptr;

259 const bool HasMusttail = [&F]() {

260 for (auto &BB : F)

261 for (auto &I : BB)

263 if (CB->isMustTailCall())

264 return true;

265 return false;

266 }();

267

268 if (HasMusttail && ContextRootSet.contains(&F)) {

269 F.getContext().emitError(

270 "[ctx_prof] A function with musttail calls was explicitly requested as "

271 "root. That is not supported because we cannot instrument a return "

272 "instruction to release the context: " +

273 F.getName());

274 return false;

275 }

276 auto &Head = F.getEntryBlock();

277 for (auto &I : Head) {

278

280 assert(Mark->getIndex()->isZero());

281

283 Guid = Builder.getInt64(

285

286

287

289 F.getContext(),

290 {ContextNodeTy, ArrayType::get(Builder.getInt64Ty(), NumCounters),

291 ArrayType::get(Builder.getPtrTy(), NumCallsites)});

292

293

294

295

296

297

298

299

300

301

304 HasMusttail ? CannotBeRootInitializer

306

307 if (ContextRootSet.contains(&F)) {

308 Context = Builder.CreateCall(

309 StartCtx, {TheRootFunctionData, Guid, Builder.getInt32(NumCounters),

310 Builder.getInt32(NumCallsites)});

311 ORE.emit(

313 } else {

314 Context = Builder.CreateCall(GetCtx, {TheRootFunctionData, &F, Guid,

316 Builder.getInt32(NumCallsites)});

317 ORE.emit([&] {

319 });

320 }

321

322 auto *CtxAsInt = Builder.CreatePtrToInt(Context, Builder.getInt64Ty());

323 if (NumCallsites > 0) {

324

325

326 auto *Index = Builder.CreateAnd(CtxAsInt, Builder.getInt64(1));

327

328 ExpectedCalleeTLSAddr = Builder.CreateGEP(

330 Builder.CreateThreadLocalAddress(ExpectedCalleeTLS), {Index});

331 CallsiteInfoTLSAddr = Builder.CreateGEP(

332 Builder.getInt32Ty(),

333 Builder.CreateThreadLocalAddress(CallsiteInfoTLS), {Index});

334 }

335

336

337

338

339

340 RealContext = Builder.CreateIntToPtr(

341 Builder.CreateAnd(CtxAsInt, Builder.getInt64(-2)),

343 I.eraseFromParent();

344 break;

345 }

346 }

348 ORE.emit([&] {

350 << "Function doesn't have instrumentation, skipping";

351 });

352 return false;

353 }

354

355 bool ContextWasReleased = false;

356 for (auto &BB : F) {

360 switch (Instr->getIntrinsicID()) {

361 case llvm::Intrinsic::instrprof_increment:

362 case llvm::Intrinsic::instrprof_increment_step: {

363

364

366 auto *GEP = Builder.CreateGEP(

367 ThisContextType, RealContext,

368 {Builder.getInt32(0), Builder.getInt32(1), AsStep->getIndex()});

369 Builder.CreateStore(

370 Builder.CreateAdd(Builder.CreateLoad(Builder.getInt64Ty(), GEP),

371 AsStep->getStep()),

373 } break;

374 case llvm::Intrinsic::instrprof_callsite:

375

376

377

379 Builder.CreateStore(CSIntrinsic->getCallee(), ExpectedCalleeTLSAddr,

380 true);

381

382

383

384

385

386

387

388

389

390 Builder.CreateStore(

391 Builder.CreateGEP(ThisContextType, Context,

392 {Builder.getInt32(0), Builder.getInt32(2),

393 CSIntrinsic->getIndex()}),

394 CallsiteInfoTLSAddr, true);

395 break;

396 }

397 I.eraseFromParent();

399

401 Builder.CreateCall(ReleaseCtx, {TheRootFunctionData});

402 ContextWasReleased = true;

403 }

404 }

405 }

406 if (!HasMusttail && !ContextWasReleased)

408 "[ctx_prof] A function that doesn't have musttail calls was "

409 "instrumented but it has no `ret` "

410 "instructions above which to release the context: " +

411 F.getName());

412 return true;

413}

414

418 for (auto &F : M) {

419 if (F.isDeclaration())

420 continue;

421 if (F.hasFnAttribute(Attribute::NoInline))

422 continue;

423 if (.isWeakForLinker())

424 continue;

425

426 if (F.hasFnAttribute(Attribute::AlwaysInline))

427 F.removeFnAttr(Attribute::AlwaysInline);

428

429 F.addFnAttr(Attribute::NoInline);

431 }

435}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

This file contains the declarations for the subclasses of Constant, which represent the different fla...

#define CTXPROF_FUNCTION_DATA(PTRDECL, CONTEXT_PTR, VOLATILE_PTRDECL, MUTEXDECL)

The internal structure of FunctionData.

Module.h This file contains the declarations for the Module class.

This header defines various interfaces for pass management in LLVM.

#define _VOLATILE_PTRDECL(_, __)

static cl::list< std::string > ContextRoots("profile-context-root", cl::Hidden, cl::desc("A function name, assumed to be global, which will be treated as the " "root of an interesting graph, which will be profiled independently " "from other similar graphs."))

FunctionAnalysisManager FAM

ModuleAnalysisManager MAM

static LLVM_ABI uint64_t getGUID(const Function &F)

static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)

This is an important base class in LLVM.

static LLVM_ABI Constant * getNullValue(Type *Ty)

Constructor to create a '0' constant of arbitrary type.

Implements a dense probed hash-table based set.

@ HiddenVisibility

The GV is hidden.

@ InternalLinkage

Rename collisions when linking (static functions).

@ ExternalLinkage

Externally visible function.

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)

emitError - Emit an error message to the currently installed error handler with optional location inf...

A Module instance is used to store all the information related to an LLVM module.

PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)

Definition PGOCtxProfLowering.cpp:415

PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)

Definition PGOCtxProfLowering.cpp:221

static bool isCtxIRPGOInstrEnabled()

Definition PGOCtxProfLowering.cpp:41

static PointerType * getUnqual(Type *ElementType)

This constructs a pointer to an object of the specified type in the default address space (address sp...

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

StringRef - Represent a constant reference to a string, i.e.

Class to represent struct types.

static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)

This static method is the primary way to create a literal StructType.

The instances of the Type class are immutable: once they are created, they are never changed.

LLVM Value Representation.

LLVM_ABI LLVMContext & getContext() const

All values hold a context through their type.

Pass manager infrastructure for declaring and invalidating analyses.

static auto CallsiteTLS

Definition PGOCtxProfLowering.cpp:52

static auto ReleaseCtx

Definition PGOCtxProfLowering.cpp:49

static auto StartCtx

Definition PGOCtxProfLowering.cpp:48

static auto GetCtx

Definition PGOCtxProfLowering.cpp:50

static auto ExpectedCalleeTLS

Definition PGOCtxProfLowering.cpp:51

NodeAddr< InstrNode * > Instr

This is an optimization pass for GlobalISel generic memory operations.

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy

Provide the FunctionAnalysisManager to Module proxy.

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

AnalysisManager< Module > ModuleAnalysisManager

Convenience typedef for the Module analysis manager.

LLVM_ABI bool canReturn(const Function &F)

Return true if there is at least a path through which F can return, false if there is no such path.