InstrProfiling.cpp Source File (original) (raw)

59#include

60#include

61#include

62#include

64using namespace llvm;

66#define DEBUG_TYPE "instrprof"

68namespace llvm {

73 "profile-correlate",

74 cl::desc("Use debug info or binary file to correlate profiles."),

77 "No profile correlation"),

79 "Use debug info to correlate"),

81 "Use binary to correlate")));

82}

84namespace {

87 "hash-based-counter-split",

88 cl::desc("Rename counter variable of a comdat function based on cfg hash"),

92 RuntimeCounterRelocation("runtime-counter-relocation",

93 cl::desc("Enable relocating counters at runtime."),

97 "vp-static-alloc",

98 cl::desc("Do static counter allocation for value profiler"),

100

102 "vp-counters-per-site",

103 cl::desc("The average number of profile counters allocated "

104 "per value profiling site."),

105

106

107

108

110

112 "instrprof-atomic-counter-update-all",

113 cl::desc("Make all profile counter updates atomic (for testing only)"),

115

117 "atomic-counter-update-promoted",

118 cl::desc("Do counter update using atomic fetch add "

119 " for promoted counters only"),

121

123 "atomic-first-counter",

124 cl::desc("Use atomic fetch add for first counter in a function (usually "

125 "the entry counter)"),

127

129 "conditional-counter-update",

130 cl::desc("Do conditional counter updates in single byte counters mode)"),

132

133

134

135

136

137

138cl::opt DoCounterPromotion("do-counter-promotion",

139 cl::desc("Do counter register promotion"),

142 "max-counter-promotions-per-loop", cl::init(20),

143 cl::desc("Max number counter promotions per loop to avoid"

144 " increasing register pressure too much"));

145

146

148 MaxNumOfPromotions("max-counter-promotions", cl::init(-1),

149 cl::desc("Max number of allowed counter promotions"));

150

152 "speculative-counter-promotion-max-exiting", cl::init(3),

153 cl::desc("The max number of exiting blocks of a loop to allow "

154 " speculative counter promotion"));

155

157 "speculative-counter-promotion-to-loop",

158 cl::desc("When the option is false, if the target block is in a loop, "

159 "the promotion will be disallowed unless the promoted counter "

160 " update can be further/iteratively promoted into an acyclic "

161 " region."));

162

164 "iterative-counter-promotion", cl::init(true),

165 cl::desc("Allow counter promotion across the whole loop nest."));

166

168 "skip-ret-exit-block", cl::init(true),

169 cl::desc("Suppress counter promotion if exit blocks contain ret."));

170

173 cl::desc("Do PGO instrumentation sampling"));

174

176 "sampled-instr-period",

177 cl::desc("Set the profile instrumentation sample period. A sample period "

178 "of 0 is invalid. For each sample period, a fixed number of "

179 "consecutive samples will be recorded. The number is controlled "

180 "by 'sampled-instr-burst-duration' flag. The default sample "

181 "period of 65536 is optimized for generating efficient code that "

182 "leverages unsigned short integer wrapping in overflow, but this "

183 "is disabled under simple sampling (burst duration = 1)."),

185

187 "sampled-instr-burst-duration",

188 cl::desc("Set the profile instrumentation burst duration, which can range "

189 "from 1 to the value of 'sampled-instr-period' (0 is invalid). "

190 "This number of samples will be recorded for each "

191 "'sampled-instr-period' count update. Setting to 1 enables simple "

192 "sampling, in which case it is recommended to set "

193 "'sampled-instr-period' to a prime number."),

195

196struct SampledInstrumentationConfig {

197 unsigned BurstDuration;

198 unsigned Period;

199 bool UseShort;

200 bool IsSimpleSampling;

201 bool IsFastSampling;

202};

203

204static SampledInstrumentationConfig getSampledInstrumentationConfig() {

205 SampledInstrumentationConfig config;

206 config.BurstDuration = SampledInstrBurstDuration.getValue();

207 config.Period = SampledInstrPeriod.getValue();

208 if (config.BurstDuration > config.Period)

210 "SampledBurstDuration must be less than or equal to SampledPeriod");

211 if (config.Period == 0 || config.BurstDuration == 0)

213 "SampledPeriod and SampledBurstDuration must be greater than 0");

214 config.IsSimpleSampling = (config.BurstDuration == 1);

215

216

217 config.IsFastSampling =

218 (!config.IsSimpleSampling && config.Period == USHRT_MAX + 1);

219 config.UseShort = (config.Period <= USHRT_MAX) || config.IsFastSampling;

220 return config;

221}

222

223using LoadStorePair = std::pair<Instruction *, Instruction *>;

224

227 if (!MD)

228 return 0;

229

230

231

233}

234

235static bool enablesValueProfiling(const Module &M) {

237 getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0;

238}

239

240

241static bool profDataReferencedByCode(const Module &M) {

242 return enablesValueProfiling(M);

243}

244

245class InstrLowerer final {

246public:

247 InstrLowerer(Module &M, const InstrProfOptions &Options,

248 std::function<const TargetLibraryInfo &(Function &F)> GetTLI,

249 bool IsCS)

250 : M(M), Options(Options), TT(M.getTargetTriple()), IsCS(IsCS),

251 GetTLI(GetTLI), DataReferencedByCode(profDataReferencedByCode(M)) {}

252

253 bool lower();

254

255private:

257 const InstrProfOptions Options;

258 const Triple TT;

259

260 const bool IsCS;

261

262 std::function<const TargetLibraryInfo &(Function &F)> GetTLI;

263

264 const bool DataReferencedByCode;

265

266 struct PerFunctionProfileData {

267 uint32_t NumValueSites[IPVK_Last + 1] = {};

268 GlobalVariable *RegionCounters = nullptr;

269 GlobalVariable *DataVar = nullptr;

270 GlobalVariable *RegionBitmaps = nullptr;

271 uint32_t NumBitmapBytes = 0;

272

273 PerFunctionProfileData() = default;

274 };

275 DenseMap<GlobalVariable *, PerFunctionProfileData> ProfileDataMap;

276

277

278 DenseMap<GlobalVariable *, GlobalVariable *> VTableDataMap;

279

280

281 DenseMap<const Function *, LoadInst *> FunctionToProfileBiasMap;

282 std::vector<GlobalValue *> CompilerUsedVars;

283 std::vector<GlobalValue *> UsedVars;

284 std::vector<GlobalVariable *> ReferencedNames;

285

286

287 std::vector<GlobalVariable *> ReferencedVTables;

288 GlobalVariable *NamesVar = nullptr;

289 size_t NamesSize = 0;

290

291

292 std::vector PromotionCandidates;

293

294 int64_t TotalCountersPromoted = 0;

295

296

297

298 bool lowerIntrinsics(Function *F);

299

300

301 void promoteCounterLoadStores(Function *F);

302

303

304 bool isRuntimeCounterRelocationEnabled() const;

305

306

307 bool isCounterPromotionEnabled() const;

308

309

310 bool isSamplingEnabled() const;

311

312

313 void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins);

314

315

316 void lowerValueProfileInst(InstrProfValueProfileInst *Ins);

317

318

319 void lowerCover(InstrProfCoverInst *Inc);

320

321

322

323 void lowerTimestamp(InstrProfTimestampInst *TimestampInstruction);

324

325

326 void lowerIncrement(InstrProfIncrementInst *Inc);

327

328

329 void lowerCoverageData(GlobalVariable *CoverageNamesVar);

330

331

332

333 void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins);

334

335

336

337 GlobalVariable *getOrCreateBiasVar(StringRef VarName);

338

339

340

341 Value *getCounterAddress(InstrProfCntrInstBase *I);

342

343

344 void doSampling(Instruction *I);

345

346

347

348

349

350 GlobalVariable *getOrCreateRegionCounters(InstrProfCntrInstBase *Inc);

351

352

353 GlobalVariable *createRegionCounters(InstrProfCntrInstBase *Inc,

354 StringRef Name,

356

357

358

359 Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I);

360

361

362

363

364

365 GlobalVariable *getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc);

366

367

368

369

370

371

372 GlobalVariable *createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,

373 StringRef Name,

375

376

377 void maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, StringRef VarName);

378

379

380 GlobalVariable *setupProfileSection(InstrProfInstBase *Inc,

382

383

384 void createDataVariable(InstrProfCntrInstBase *Inc);

385

386

387 void getOrCreateVTableProfData(GlobalVariable *GV);

388

389

390 void emitNameData();

391

392

393 void emitVTableNames();

394

395

396 void emitVNodes();

397

398

399 void emitRegistration();

400

401

402

403 bool emitRuntimeHook();

404

405

406 void emitUses();

407

408

409

410 void emitInitialization();

411};

412

413

414

415

416

417

418

419

421public:

422 PGOCounterPromoterHelper(

423 Instruction *L, Instruction *S, SSAUpdater &SSA, Value *Init,

427 LoopInfo &LI)

428 : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),

429 InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {

432 SSA.AddAvailableValue(PH, Init);

433 }

434

435 void doExtraRewritesBeforeFinalDeletion() override {

436 for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {

437 BasicBlock *ExitBlock = ExitBlocks[i];

439

440

441

442 Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);

447

448

449

450

451

452

454 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);

455 Value *BiasInst = Builder.Insert(OrigBiasInst->clone());

456 Addr = Builder.CreateIntToPtr(BiasInst,

457 PointerType::getUnqual(Ty->getContext()));

458 }

459 if (AtomicCounterUpdatePromoted)

460

461

463 MaybeAlign(),

464 AtomicOrdering::SequentiallyConsistent);

465 else {

466 LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");

467 auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);

468 auto *NewStore = Builder.CreateStore(NewVal, Addr);

469

470

471 if (IterativeCounterPromotion) {

472 auto *TargetLoop = LI.getLoopFor(ExitBlock);

473 if (TargetLoop)

474 LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);

475 }

476 }

477 }

478 }

479

480private:

484 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;

485 LoopInfo &LI;

486};

487

488

489

490

491class PGOCounterPromoter {

492public:

493 PGOCounterPromoter(

495 Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)

496 : LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI) {

497

498

499

500 SmallVector<BasicBlock *, 8> LoopExitBlocks;

501 SmallPtrSet<BasicBlock *, 8> BlockSet;

502

503 L.getExitBlocks(LoopExitBlocks);

504 if (!isPromotionPossible(&L, LoopExitBlocks))

505 return;

506

507 for (BasicBlock *ExitBlock : LoopExitBlocks) {

508 if (BlockSet.insert(ExitBlock).second &&

511 })) {

512 ExitBlocks.push_back(ExitBlock);

514 }

515 }

516 }

517

518 bool run(int64_t *NumPromoted) {

519

520 if (ExitBlocks.size() == 0)

521 return false;

522

523

524

525

526

527

528 if (SkipRetExitBlock) {

529 for (auto *BB : ExitBlocks)

531 return false;

532 }

533

534 unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);

535 if (MaxProm == 0)

536 return false;

537

538 unsigned Promoted = 0;

539 for (auto &Cand : LoopToCandidates[&L]) {

540

542 SSAUpdater SSA(&NewPHIs);

543 Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);

544

545

546 if (BFI) {

547 auto *BB = Cand.first->getParent();

548 auto InstrCount = BFI->getBlockProfileCount(BB);

550 continue;

551 auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());

552

553

554 if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2))

555 continue;

556 }

557

558 PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,

559 L.getLoopPreheader(), ExitBlocks,

560 InsertPts, LoopToCandidates, LI);

561 Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));

562 Promoted++;

563 if (Promoted >= MaxProm)

564 break;

565

566 (*NumPromoted)++;

567 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)

568 break;

569 }

570

571 LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="

572 << L.getLoopDepth() << ")\n");

573 return Promoted != 0;

574 }

575

576private:

577 bool allowSpeculativeCounterPromotion(Loop *LP) {

578 SmallVector<BasicBlock *, 8> ExitingBlocks;

579 L.getExitingBlocks(ExitingBlocks);

580

581 if (ExitingBlocks.size() == 1)

582 return true;

583 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)

584 return false;

585 return true;

586 }

587

588

589

590 bool

591 isPromotionPossible(Loop *LP,

592 const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {

593

594 if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {

596 }))

597 return false;

598

600 return false;

601

603 if (!PH)

604 return false;

605

606 return true;

607 }

608

609

610 unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {

611 SmallVector<BasicBlock *, 8> LoopExitBlocks;

613 if (!isPromotionPossible(LP, LoopExitBlocks))

614 return 0;

615

616 SmallVector<BasicBlock *, 8> ExitingBlocks;

618

619

620 if (BFI)

621 return (unsigned)-1;

622

623

624 if (ExitingBlocks.size() == 1)

625 return MaxNumOfPromotionsPerLoop;

626

627 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)

628 return 0;

629

630

631 if (SpeculativeCounterPromotionToLoop)

632 return MaxNumOfPromotionsPerLoop;

633

634

635 unsigned MaxProm = MaxNumOfPromotionsPerLoop;

636 for (auto *TargetBlock : LoopExitBlocks) {

637 auto *TargetLoop = LI.getLoopFor(TargetBlock);

638 if (!TargetLoop)

639 continue;

640 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);

641 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();

642 MaxProm =

643 std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -

644 PendingCandsInTarget);

645 }

646 return MaxProm;

647 }

648

649 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;

650 SmallVector<BasicBlock *, 8> ExitBlocks;

651 SmallVector<Instruction *, 8> InsertPts;

652 Loop &L;

653 LoopInfo &LI;

654 BlockFrequencyInfo *BFI;

655};

656

657enum class ValueProfilingCallType {

658

659

661

662

664};

665

666}

667

674 };

675 InstrLowerer Lowerer(M, Options, GetTLI, IsCS);

676 if (!Lowerer.lower())

678

680}

681

682

683

684

685

686

687

688

689

690

691

692

693

694

695

696

697

698

699

700

701

702

703

704

705

706

707

708

709

710

711

712

713

714

715

716

717

718

719

720

721

722

723

724

725

726void InstrLowerer::doSampling(Instruction *I) {

727 if (!isSamplingEnabled())

728 return;

729

730 SampledInstrumentationConfig config = getSampledInstrumentationConfig();

732 if (config.UseShort)

733 return Builder.getInt16(C);

734 else

735 return Builder.getInt32(C);

736 };

737

739 if (config.UseShort)

741 else

743 auto *SamplingVar =

745 assert(SamplingVar && "SamplingVar not set properly");

746

747

749 Value *NewSamplingVarVal;

751 MDNode *BranchWeight;

753 auto *LoadSamplingVar = CondBuilder.CreateLoad(SamplingVarTy, SamplingVar);

754 if (config.IsSimpleSampling) {

755

757 NewSamplingVarVal =

758 IncBuilder.CreateAdd(LoadSamplingVar, GetConstant(IncBuilder, 1));

759 SamplingVarIncr = IncBuilder.CreateStore(NewSamplingVarVal, SamplingVar);

760 } else {

761

762 auto *DurationCond = CondBuilder.CreateICmpULE(

763 LoadSamplingVar, GetConstant(CondBuilder, config.BurstDuration - 1));

764 BranchWeight = MDB.createBranchWeights(

765 config.BurstDuration, config.Period - config.BurstDuration);

767 DurationCond, I, false, BranchWeight);

769 NewSamplingVarVal =

770 IncBuilder.CreateAdd(LoadSamplingVar, GetConstant(IncBuilder, 1));

771 SamplingVarIncr = IncBuilder.CreateStore(NewSamplingVarVal, SamplingVar);

773 }

774

775 if (config.IsFastSampling)

776 return;

777

778

780 IRBuilder<> PeriodCondBuilder(SamplingVarIncr);

781 auto *PeriodCond = PeriodCondBuilder.CreateICmpUGE(

782 NewSamplingVarVal, GetConstant(PeriodCondBuilder, config.Period));

783 BranchWeight = MDB.createBranchWeights(1, config.Period - 1);

785 &ElseTerm, BranchWeight);

786

787

788 if (config.IsSimpleSampling)

790

792 ResetBuilder.CreateStore(GetConstant(ResetBuilder, 0), SamplingVar);

794}

795

796bool InstrLowerer::lowerIntrinsics(Function *F) {

797 bool MadeChange = false;

798 PromotionCandidates.clear();

800

801

802

803

808 }

809 }

810

811 for (auto *Instr : InstrProfInsts) {

812 doSampling(Instr);

814 lowerIncrement(IPIS);

815 MadeChange = true;

817 lowerIncrement(IPI);

818 MadeChange = true;

820 lowerTimestamp(IPC);

821 MadeChange = true;

823 lowerCover(IPC);

824 MadeChange = true;

826 lowerValueProfileInst(IPVP);

827 MadeChange = true;

829 IPMP->eraseFromParent();

830 MadeChange = true;

832 lowerMCDCTestVectorBitmapUpdate(IPBU);

833 MadeChange = true;

834 }

835 }

836

837 if (!MadeChange)

838 return false;

839

840 promoteCounterLoadStores(F);

841 return true;

842}

843

844bool InstrLowerer::isRuntimeCounterRelocationEnabled() const {

845

846 if (TT.isOSBinFormatMachO())

847 return false;

848

849 if (RuntimeCounterRelocation.getNumOccurrences() > 0)

850 return RuntimeCounterRelocation;

851

852

853 return TT.isOSFuchsia();

854}

855

856bool InstrLowerer::isSamplingEnabled() const {

857 if (SampledInstr.getNumOccurrences() > 0)

858 return SampledInstr;

860}

861

862bool InstrLowerer::isCounterPromotionEnabled() const {

863 if (DoCounterPromotion.getNumOccurrences() > 0)

864 return DoCounterPromotion;

865

866 return Options.DoCounterPromotion;

867}

868

869void InstrLowerer::promoteCounterLoadStores(Function *F) {

870 if (!isCounterPromotionEnabled())

871 return;

872

876

877 std::unique_ptr BFI;

878 if (Options.UseBFIInPromotion) {

879 std::unique_ptr BPI;

882 }

883

884 for (const auto &LoadStore : PromotionCandidates) {

885 auto *CounterLoad = LoadStore.first;

886 auto *CounterStore = LoadStore.second;

889 if (!ParentLoop)

890 continue;

891 LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);

892 }

893

895

896

897

899 PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());

900 Promoter.run(&TotalCountersPromoted);

901 }

902}

903

905

906 if (TT.isOSFuchsia())

907 return false;

908

909 return true;

910}

911

912

914 auto containsIntrinsic = [&](int ID) {

916 return ->use_empty();

917 return false;

918 };

919 return containsIntrinsic(Intrinsic::instrprof_cover) ||

920 containsIntrinsic(Intrinsic::instrprof_increment) ||

921 containsIntrinsic(Intrinsic::instrprof_increment_step) ||

922 containsIntrinsic(Intrinsic::instrprof_timestamp) ||

923 containsIntrinsic(Intrinsic::instrprof_value_profile);

924}

925

926bool InstrLowerer::lower() {

927 bool MadeChange = false;

929 if (NeedsRuntimeHook)

930 MadeChange = emitRuntimeHook();

931

932 if (!IsCS && isSamplingEnabled())

934

938

939 if (!ContainsProfiling && !CoverageNamesVar)

940 return MadeChange;

941

942

943

944

948 for (auto I = BB.begin(), E = BB.end(); I != E; I++) {

950 computeNumValueSiteCounts(Ind);

951 else {

952 if (FirstProfInst == nullptr &&

955

957 static_cast<void>(getOrCreateRegionBitmaps(Params));

958 }

959 }

960 }

961

962

963

964 if (FirstProfInst != nullptr) {

965 static_cast<void>(getOrCreateRegionCounters(FirstProfInst));

966 }

967 }

968

971

972 if (GV.hasMetadata(LLVMContext::MD_type))

973 getOrCreateVTableProfData(&GV);

974

976 MadeChange |= lowerIntrinsics(&F);

977

978 if (CoverageNamesVar) {

979 lowerCoverageData(CoverageNamesVar);

980 MadeChange = true;

981 }

982

983 if (!MadeChange)

984 return false;

985

986 emitVNodes();

987 emitNameData();

988 emitVTableNames();

989

990

991

992

993

994 if (!NeedsRuntimeHook && ContainsProfiling)

995 emitRuntimeHook();

996

997 emitRegistration();

998 emitUses();

999 emitInitialization();

1000 return true;

1001}

1002

1005 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {

1008

1009 AttributeList AL;

1010 if (auto AK = TLI.getExtAttrForI32Param(false))

1011 AL = AL.addParamAttribute(M.getContext(), 2, AK);

1012

1013 assert((CallType == ValueProfilingCallType::Default ||

1014 CallType == ValueProfilingCallType::MemOp) &&

1015 "Must be Default or MemOp");

1016 Type *ParamTypes[] = {

1017#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType

1019 };

1020 auto *ValueProfilingCallTy =

1022 StringRef FuncName = CallType == ValueProfilingCallType::Default

1025 return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);

1026}

1027

1032 auto &PD = ProfileDataMap[Name];

1034 std::max(PD.NumValueSites[ValueKind], (uint32_t)(Index + 1));

1035}

1036

1038

1039

1040

1043 "Value profiling is not yet supported with lightweight instrumentation");

1045 auto It = ProfileDataMap.find(Name);

1046 assert(It != ProfileDataMap.end() && It->second.DataVar &&

1047 "value profiling detected in function with no counter increment");

1048

1053 Index += It->second.NumValueSites[Kind];

1054

1056 bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==

1057 llvm::InstrProfValueKind::IPVK_MemOPSize);

1059 auto *TLI = &GetTLI(*Ind->getFunction());

1062

1063

1064

1065

1066

1069 if (!IsMemOpSize) {

1073 OpBundles);

1074 } else {

1079 Args, OpBundles);

1080 }

1081 if (auto AK = TLI->getExtAttrForI32Param(false))

1085}

1086

1089 if (Bias)

1090 return Bias;

1091

1093

1094

1095

1096

1100

1101

1102

1103

1104 if (TT.supportsCOMDAT())

1105 Bias->setComdat(M.getOrInsertComdat(VarName));

1106

1107 return Bias;

1108}

1109

1111 auto *Counters = getOrCreateRegionCounters(I);

1113

1116

1118 Counters->getValueType(), Counters, 0, I->getIndex()->getZExtValue());

1119

1120 if (!isRuntimeCounterRelocationEnabled())

1121 return Addr;

1122

1125 LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];

1126 if (!BiasLI) {

1129 BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias, "profc_bias");

1130

1131 BiasLI->setMetadata(LLVMContext::MD_invariant_load,

1133 }

1136}

1137

1139 auto *Bitmaps = getOrCreateRegionBitmaps(I);

1140 if (!isRuntimeCounterRelocationEnabled())

1141 return Bitmaps;

1142

1143

1148 auto *BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias, "profbm_bias");

1149

1150 BiasLI->setMetadata(LLVMContext::MD_invariant_load,

1152

1153

1155 return Builder.CreatePtrAdd(Bitmaps, BiasLI, "profbm_addr");

1156}

1157

1158void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) {

1159 auto *Addr = getCounterAddress(CoverInstruction);

1161 if (ConditionalCounterUpdate) {

1163 auto &Ctx = CoverInstruction->getParent()->getContext();

1170 }

1171

1172

1175}

1176

1177void InstrLowerer::lowerTimestamp(

1180 "timestamp probes are always the first probe for a function");

1181 auto &Ctx = M.getContext();

1182 auto *TimestampAddr = getCounterAddress(TimestampInstruction);

1183 IRBuilder<> Builder(TimestampInstruction);

1184 auto *CalleeTy =

1186 auto Callee = M.getOrInsertFunction(

1188 Builder.CreateCall(Callee, {TimestampAddr});

1190}

1191

1193 auto *Addr = getCounterAddress(Inc);

1194

1196 if (Options.Atomic || AtomicCounterUpdateAll ||

1200 } else {

1205 if (isCounterPromotionEnabled())

1206 PromotionCandidates.emplace_back(cast(Load), Store);

1207 }

1209}

1210

1211void InstrLowerer::lowerCoverageData(GlobalVariable *CoverageNamesVar) {

1216 Value *V = NC->stripPointerCasts();

1219

1221 ReferencedNames.push_back(Name);

1223 NC->dropAllReferences();

1224 }

1226}

1227

1228void InstrLowerer::lowerMCDCTestVectorBitmapUpdate(

1230 auto &Ctx = M.getContext();

1235 auto *BitmapAddr = getBitmapAddress(Update);

1236

1237

1238

1242

1243

1244

1245 auto *BitmapByteOffset = Builder.CreateLShr(Temp, 0x3);

1246

1247

1248

1249 auto *BitmapByteAddr =

1251

1252

1253

1254

1256

1257

1258

1259 auto *ShiftedVal = Builder.CreateShl(Builder.getInt8(0x1), BitToSet);

1260

1261

1262

1263 auto *Bitmap = Builder.CreateLoad(Int8Ty, BitmapByteAddr, "mcdc.bits");

1264

1265 if (Options.Atomic || AtomicCounterUpdateAll) {

1266

1267

1268

1271

1272

1276

1277

1281 } else {

1282

1283

1284 auto *Result = Builder.CreateOr(Bitmap, ShiftedVal);

1285

1286

1287

1288 Builder.CreateStore(Result, BitmapByteAddr);

1289 }

1290

1292}

1293

1294

1296 bool &Renamed) {

1300 Module *M = F->getParent();

1301 if (!DoHashBasedCounterSplit || isIRPGOFlagSet (M) ||

1303 Renamed = false;

1304 return (Prefix + Name).str();

1305 }

1306 Renamed = true;

1310 return (Prefix + Name).str();

1311 return (Prefix + Name + "." + Twine(FuncHash)).str();

1312}

1313

1315

1316

1317

1318

1319 if (!profDataReferencedByCode(*F->getParent()))

1320 return false;

1321

1322

1323 bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();

1324 if (->hasLinkOnceLinkage() && ->hasLocalLinkage() &&

1325 !HasAvailableExternallyLinkage)

1326 return true;

1327

1328

1329

1330

1331 if (HasAvailableExternallyLinkage &&

1332 F->hasFnAttribute(Attribute::AlwaysInline))

1333 return false;

1334

1335

1336

1337

1338 if (F->hasLocalLinkage() && F->hasComdat())

1339 return false;

1340

1341

1342

1343

1344

1345

1346

1347

1348 return F->hasAddressTaken() || F->hasLinkOnceLinkage();

1349}

1350

1352

1354 return true;

1355

1356

1357

1359 return true;

1360

1361

1362

1363

1364

1365

1366

1367

1368 if (Fn->hasMetadata(LLVMContext::MD_type))

1369 return true;

1370

1371

1372

1373

1376 return true;

1377

1378

1379 return false;

1380}

1381

1384

1387

1388

1389

1391 return Fn;

1392

1393

1395 Fn->getName() + ".local", Fn);

1396

1397

1398

1399

1400

1401

1402

1403

1404

1405

1406

1410 }

1411

1412

1413

1414 return GA;

1415}

1416

1418

1419

1420 if (TT.isOSBinFormatELF() || TT.isOSBinFormatCOFF() ||

1421 TT.isOSBinFormatMachO() || TT.isOSBinFormatXCOFF() ||

1422 TT.isOSBinFormatWasm())

1423 return false;

1424

1425 return true;

1426}

1427

1430

1431

1432

1433

1435 bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());

1436

1437 if (!UseComdat)

1438 return;

1439

1440

1441

1442

1443

1444

1445

1446

1447

1448

1449

1450 StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode

1452 : CounterGroupName;

1453 Comdat *C = M.getOrInsertComdat(GroupName);

1454

1455 if (!NeedComdat) {

1456

1457

1458

1459

1460

1461

1463 }

1465

1466

1467

1470}

1471

1473 if (!profDataReferencedByCode(*GV->getParent()))

1474 return false;

1475

1478 return true;

1479

1480

1481

1483 return false;

1484

1485 return true;

1486}

1487

1488

1489

1491

1494

1495 return GV;

1496}

1497

1498void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) {

1500 "Value profiling is not supported with lightweight instrumentation");

1502 return;

1503

1504

1508 return;

1509

1510

1511 auto It = VTableDataMap.find(GV);

1512 if (It != VTableDataMap.end() && It->second)

1513 return;

1514

1517

1518

1519

1520 if (TT.isOSBinFormatXCOFF()) {

1523 }

1524

1526 Type *DataTypes[] = {

1527#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType,

1529#undef INSTR_PROF_VTABLE_DATA

1530 };

1531

1533

1534

1536 const std::string PGOVTableName = getPGOName(*GV);

1537

1538

1539 uint32_t VTableSizeVal =

1540 M.getDataLayout().getTypeAllocSize(GV->getValueType());

1541

1543#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init,

1545#undef INSTR_PROF_VTABLE_DATA

1546 };

1547

1548 auto *Data =

1552

1553 Data->setVisibility(Visibility);

1556

1557 maybeSetComdat(Data, GV, Data->getName());

1558

1559 VTableDataMap[GV] = Data;

1560

1561 ReferencedVTables.push_back(GV);

1562

1563

1564

1565 UsedVars.push_back(Data);

1566}

1567

1571

1572

1576

1577

1578

1582

1583

1584

1585

1586

1587

1588 if (TT.isOSBinFormatXCOFF()) {

1591 }

1592

1593 bool Renamed;

1597 if (IPSK == IPSK_cnts) {

1601 Ptr = createRegionCounters(CntrIncrement, VarName, Linkage);

1602 } else if (IPSK == IPSK_bitmap) {

1607 Ptr = createRegionBitmaps(BitmapUpdate, VarName, Linkage);

1608 } else {

1609 llvm_unreachable("Profile Section must be for Counters or Bitmaps");

1610 }

1611

1613

1614

1617 maybeSetComdat(Ptr, Fn, VarName);

1618 return Ptr;

1619}

1620

1630 return GV;

1631}

1632

1636 auto &PD = ProfileDataMap[NamePtr];

1637 if (PD.RegionBitmaps)

1638 return PD.RegionBitmaps;

1639

1640

1641

1642 auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap);

1643 PD.RegionBitmaps = BitmapPtr;

1645 return PD.RegionBitmaps;

1646}

1647

1652 auto &Ctx = M.getContext();

1657

1658 std::vector<Constant *> InitialValues(NumCounters,

1662 Name);

1664 } else {

1669 }

1670 return GV;

1671}

1672

1676 auto &PD = ProfileDataMap[NamePtr];

1677 if (PD.RegionCounters)

1678 return PD.RegionCounters;

1679

1680

1681

1682 auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts);

1683 PD.RegionCounters = CounterPtr;

1684

1690 Metadata *FunctionNameAnnotation[] = {

1693 };

1694 Metadata *CFGHashAnnotation[] = {

1697 };

1698 Metadata *NumCountersAnnotation[] = {

1701 };

1703 MDNode::get(Ctx, FunctionNameAnnotation),

1705 MDNode::get(Ctx, NumCountersAnnotation),

1706 });

1707 auto *DICounter = DB.createGlobalVariableExpression(

1708 SP, CounterPtr->getName(), StringRef(), SP->getFile(),

1709 0, DB.createUnspecifiedType("Profile Data Type"),

1710 CounterPtr->hasLocalLinkage(), true, nullptr,

1711 nullptr, nullptr, 0,

1713 CounterPtr->addDebugInfo(DICounter);

1714 DB.finalize();

1715 }

1716

1717

1718 CompilerUsedVars.push_back(PD.RegionCounters);

1719 }

1720

1721

1722 createDataVariable(Inc);

1723

1724 return PD.RegionCounters;

1725}

1726

1728

1729

1731 return;

1732

1734 auto &PD = ProfileDataMap[NamePtr];

1735

1736

1737 if (PD.DataVar)

1738 return;

1739

1741

1745

1746

1747

1748

1749

1750

1751 if (TT.isOSBinFormatXCOFF()) {

1754 }

1755

1757 bool Renamed;

1758

1759

1760 std::string CntsVarName =

1762 std::string DataVarName =

1764

1766

1767

1769 uint64_t NS = 0;

1770 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)

1771 NS += PD.NumValueSites[Kind];

1772 if (NS > 0 && ValueProfileStaticAlloc &&

1778 ValuesVar->setVisibility(Visibility);

1780 ValuesVar->setSection(

1782 ValuesVar->setAlignment(Align(8));

1783 maybeSetComdat(ValuesVar, Fn, CntsVarName);

1786 }

1787

1789 auto *CounterPtr = PD.RegionCounters;

1790

1792

1793

1794 auto *IntPtrTy = M.getDataLayout().getIntPtrType(M.getContext());

1797 Type *DataTypes[] = {

1798#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,

1800 };

1802

1804

1805 Constant *Int16ArrayVals[IPVK_Last + 1];

1806 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)

1807 Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);

1808

1812 }

1813

1814

1815

1816

1817

1818

1819

1820

1821

1822

1823

1824 else if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&

1825 (TT.isOSBinFormatELF() ||

1826 (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {

1829 }

1830 auto *Data =

1832 Constant *RelativeCounterPtr;

1834 Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0);

1836

1837

1839 DataSectionKind = IPSK_covdata;

1841 if (BitmapPtr != nullptr)

1843 } else {

1844

1845

1846 DataSectionKind = IPSK_data;

1847 RelativeCounterPtr =

1850 if (BitmapPtr != nullptr)

1851 RelativeBitmapPtr =

1854 }

1855

1857#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,

1859 };

1861

1862 Data->setVisibility(Visibility);

1863 Data->setSection(

1866 maybeSetComdat(Data, Fn, CntsVarName);

1867

1869

1870

1871 CompilerUsedVars.push_back(Data);

1872

1873

1874

1876

1877 ReferencedNames.push_back(NamePtr);

1878}

1879

1880void InstrLowerer::emitVNodes() {

1881 if (!ValueProfileStaticAlloc)

1882 return;

1883

1884

1885

1886

1888 return;

1889

1890 size_t TotalNS = 0;

1891 for (auto &PD : ProfileDataMap) {

1892 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)

1893 TotalNS += PD.second.NumValueSites[Kind];

1894 }

1895

1896 if (!TotalNS)

1897 return;

1898

1899 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;

1900

1901

1902

1903

1904

1905

1906

1907#define INSTR_PROF_MIN_VAL_COUNTS 10

1910

1911 auto &Ctx = M.getContext();

1912 Type *VNodeTypes[] = {

1913#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,

1915 };

1917

1923 VNodesVar->setSection(

1925 VNodesVar->setAlignment(M.getDataLayout().getABITypeAlign(VNodesTy));

1926

1927

1928 UsedVars.push_back(VNodesVar);

1929}

1930

1931void InstrLowerer::emitNameData() {

1932 if (ReferencedNames.empty())

1933 return;

1934

1935 std::string CompressedNameStr;

1939 }

1940

1941 auto &Ctx = M.getContext();

1942 auto *NamesVal =

1944 NamesVar = new GlobalVariable(M, NamesVal->getType(), true,

1950 }

1951

1952 NamesSize = CompressedNameStr.size();

1954 NamesVar->setSection(

1958

1959

1960

1961 NamesVar->setAlignment(Align(1));

1962

1963

1964 UsedVars.push_back(NamesVar);

1965

1966 for (auto *NamePtr : ReferencedNames)

1968}

1969

1970void InstrLowerer::emitVTableNames() {

1972 return;

1973

1974

1975 std::string CompressedVTableNames;

1979 }

1980

1981 auto &Ctx = M.getContext();

1983 Ctx, StringRef(CompressedVTableNames), false );

1985 new GlobalVariable(M, VTableNamesVal->getType(), true ,

1991

1992 UsedVars.push_back(VTableNamesVar);

1993}

1994

1995void InstrLowerer::emitRegistration() {

1997 return;

1998

1999

2008 RegisterF->addFnAttr(Attribute::NoRedZone);

2009

2010 auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);

2011 auto *RuntimeRegisterF =

2014

2016 for (Value *Data : CompilerUsedVars)

2018

2019 IRB.CreateCall(RuntimeRegisterF,

2020 IRB.CreatePointerBitCastOrAddrSpaceCast(Data, VoidPtrTy));

2023 IRB.CreateCall(RuntimeRegisterF,

2024 IRB.CreatePointerBitCastOrAddrSpaceCast(Data, VoidPtrTy));

2025

2026 if (NamesVar) {

2027 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};

2028 auto *NamesRegisterTy =

2030 auto *NamesRegisterF =

2033 IRB.CreateCall(NamesRegisterF, {IRB.CreatePointerBitCastOrAddrSpaceCast(

2034 NamesVar, VoidPtrTy),

2035 IRB.getInt64(NamesSize)});

2036 }

2037

2038 IRB.CreateRetVoid();

2039}

2040

2041bool InstrLowerer::emitRuntimeHook() {

2042

2043

2044 if (TT.isOSLinux() || TT.isOSAIX())

2045 return false;

2046

2047

2049 return false;

2050

2051

2053 auto *Var =

2058 else

2060

2061 if (TT.isOSBinFormatELF() && .isPS()) {

2062

2063 CompilerUsedVars.push_back(Var);

2064 } else {

2065

2069 User->addFnAttr(Attribute::NoInline);

2071 User->addFnAttr(Attribute::NoRedZone);

2073 if (TT.supportsCOMDAT())

2075

2077 auto *Load = IRB.CreateLoad(Int32Ty, Var);

2078 IRB.CreateRet(Load);

2079

2080

2081 CompilerUsedVars.push_back(User);

2082 }

2083 return true;

2084}

2085

2086void InstrLowerer::emitUses() {

2087

2088

2089

2090

2091

2092

2093

2094

2095

2096 if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||

2097 (TT.isOSBinFormatCOFF() && !DataReferencedByCode))

2099 else

2101

2102

2103

2104

2106}

2107

2108void InstrLowerer::emitInitialization() {

2109

2110

2111

2112

2113 if (!IsCS)

2116 if (!RegisterF)

2117 return;

2118

2119

2125 F->addFnAttr(Attribute::NoInline);

2127 F->addFnAttr(Attribute::NoRedZone);

2128

2129

2131 IRB.CreateCall(RegisterF, {});

2132 IRB.CreateRetVoid();

2133

2135}

2136

2137namespace llvm {

2138

2143 if (getSampledInstrumentationConfig().UseShort) {

2146 } else {

2149 }

2153 SamplingVar->setThreadLocal(true);

2154 Triple TT(M.getTargetTriple());

2155 if (TT.supportsCOMDAT()) {

2157 SamplingVar->setComdat(M.getOrInsertComdat(VarName));

2158 }

2160}

2161}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

This file contains the simple types necessary to represent the attributes associated with functions a...

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

#define clEnumValN(ENUMVAL, FLAGNAME, DESC)

This file contains the declarations for the subclasses of Constant, which represent the different fla...

static unsigned InstrCount

This file provides various utilities for inspecting and working with the control flow graph in LLVM I...

Module.h This file contains the declarations for the Module class.

#define INSTR_PROF_QUOTE(x)

#define INSTR_PROF_DATA_ALIGNMENT

#define INSTR_PROF_PROFILE_SET_TIMESTAMP

#define INSTR_PROF_PROFILE_SAMPLING_VAR

static bool shouldRecordVTableAddr(GlobalVariable *GV)

Definition InstrProfiling.cpp:1472

static bool shouldRecordFunctionAddr(Function *F)

Definition InstrProfiling.cpp:1314

static bool needsRuntimeHookUnconditionally(const Triple &TT)

Definition InstrProfiling.cpp:904

static bool containsProfilingIntrinsics(Module &M)

Check if the module contains uses of any profiling intrinsics.

Definition InstrProfiling.cpp:913

static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed)

Get the name of a profiling variable for a particular function.

Definition InstrProfiling.cpp:1295

#define INSTR_PROF_MIN_VAL_COUNTS

static Constant * getFuncAddrForProfData(Function *Fn)

Definition InstrProfiling.cpp:1382

static bool shouldUsePublicSymbol(Function *Fn)

Definition InstrProfiling.cpp:1351

static FunctionCallee getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, ValueProfilingCallType CallType=ValueProfilingCallType::Default)

Definition InstrProfiling.cpp:1003

static Constant * getVTableAddrForProfData(GlobalVariable *GV)

Definition InstrProfiling.cpp:1490

static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT)

Definition InstrProfiling.cpp:1417

This file provides the interface for LLVM's PGO Instrumentation lowering pass.

Machine Check Debug Module

This file provides the interface for IR based instrumentation passes ( (profile-gen,...

FunctionAnalysisManager FAM

std::unordered_set< BasicBlock * > BlockSet

This file defines the SmallVector class.

Class for arbitrary precision integers.

PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)

Get the result of an analysis pass for a given IR unit.

Annotations lets you mark points and ranges inside source code, for tests:

Class to represent array types.

static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)

This static method is the primary way to construct an ArrayType.

LLVM Basic Block Representation.

iterator begin()

Instruction iterator methods.

LLVM_ABI const_iterator getFirstInsertionPt() const

Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...

const Function * getParent() const

Return the enclosing method, or null if none.

static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)

Creates a new BasicBlock.

const Instruction & front() const

BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...

Analysis providing branch probability information.

LLVM_ABI void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const

Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.

void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)

Adds the attribute to the indicated argument.

This class represents a function call, abstracting a target machine's calling convention.

@ NoDeduplicate

No deduplication is performed.

ConstantArray - Constant Array Declarations.

static LLVM_ABI Constant * get(ArrayType *T, ArrayRef< Constant * > V)

static LLVM_ABI Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)

This method constructs a CDS and initializes it with a text string.

static LLVM_ABI Constant * getPointerBitCastOrAddrSpaceCast(Constant *C, Type *Ty)

Create a BitCast or AddrSpaceCast for a pointer type depending on the address space.

static LLVM_ABI Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)

static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)

uint64_t getZExtValue() const

Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...

static LLVM_ABI ConstantPointerNull * get(PointerType *T)

Static factory methods - Return objects of the specified value.

static LLVM_ABI Constant * get(StructType *T, ArrayRef< Constant * > V)

This is an important base class in LLVM.

static LLVM_ABI Constant * getIntegerValue(Type *Ty, const APInt &V)

Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...

static LLVM_ABI Constant * getAllOnesValue(Type *Ty)

static LLVM_ABI Constant * getNullValue(Type *Ty)

Constructor to create a '0' constant of arbitrary type.

LLVM_ABI bool isZeroValue() const

Return true if the value is negative zero or null value.

Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.

Lightweight error class with error context and mandatory checking.

A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...

static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)

This static method is the primary way of constructing a FunctionType.

static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)

const BasicBlock & getEntryBlock() const

DISubprogram * getSubprogram() const

Get the attached subprogram.

const Function & getFunction() const

LLVMContext & getContext() const

getContext - Return a reference to the LLVMContext associated with this function.

static LLVM_ABI GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)

If a parent module is specified, the alias is automatically inserted into the end of the specified mo...

bool hasMetadata() const

Return true if this value has any metadata attached to it.

LLVM_ABI void setComdat(Comdat *C)

LLVM_ABI void setSection(StringRef S)

Change the section for this global.

bool hasLinkOnceLinkage() const

VisibilityTypes getVisibility() const

LLVM_ABI bool isDeclaration() const

Return true if the primary definition of this global value is outside of the current translation unit...

LinkageTypes getLinkage() const

bool hasLocalLinkage() const

bool hasPrivateLinkage() const

void setLinkage(LinkageTypes LT)

bool isDeclarationForLinker() const

Module * getParent()

Get the module that this global value is contained inside of...

VisibilityTypes

An enumeration for the kinds of visibility of global values.

@ DefaultVisibility

The GV is visible.

@ HiddenVisibility

The GV is hidden.

@ ProtectedVisibility

The GV is protected.

void setVisibility(VisibilityTypes V)

bool hasAvailableExternallyLinkage() const

LinkageTypes

An enumeration for the kinds of linkage for global values.

@ PrivateLinkage

Like Internal, but omit from symbol table.

@ InternalLinkage

Rename collisions when linking (static functions).

@ ExternalLinkage

Externally visible function.

@ WeakAnyLinkage

Keep one copy of named function when linking (weak)

@ LinkOnceODRLinkage

Same, but only replaced by something equivalent.

Type * getValueType() const

const Constant * getInitializer() const

getInitializer - Return the initializer for this global variable.

LLVM_ABI void eraseFromParent()

eraseFromParent - This method unlinks 'this' from the containing module and deletes it.

void setAlignment(Align Align)

Sets the alignment attribute of the GlobalVariable.

Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")

Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)

ConstantInt * getInt8(uint8_t C)

Get a constant 8-bit value.

Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())

Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")

ConstantInt * getInt32(uint32_t C)

Get a constant 32-bit value.

LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)

Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...

Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateConstInBoundsGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0, unsigned Idx1, const Twine &Name="")

StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)

Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")

Value * CreateIsNotNull(Value *Arg, const Twine &Name="")

Return a boolean value testing if Arg != 0.

CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)

AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)

Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)

void SetInsertPoint(BasicBlock *TheBB)

This specifies that created instructions should be appended to the end of the specified block.

Value * CreateInBoundsPtrAdd(Value *Ptr, Value *Offset, const Twine &Name="")

Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="", bool IsDisjoint=false)

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

A base class for all instrprof counter intrinsics.

LLVM_ABI ConstantInt * getIndex() const

LLVM_ABI ConstantInt * getNumCounters() const

static LLVM_ABI const char * FunctionNameAttributeName

static LLVM_ABI const char * CFGHashAttributeName

static LLVM_ABI const char * NumCountersAttributeName

This represents the llvm.instrprof.cover intrinsic.

This represents the llvm.instrprof.increment intrinsic.

LLVM_ABI Value * getStep() const

A base class for all instrprof intrinsics.

GlobalVariable * getName() const

ConstantInt * getHash() const

A base class for instrprof mcdc intrinsics that require global bitmap bytes.

auto getNumBitmapBytes() const

This represents the llvm.instrprof.mcdc.tvbitmap.update intrinsic.

Value * getMCDCCondBitmapAddr() const

ConstantInt * getBitmapIndex() const

This represents the llvm.instrprof.timestamp intrinsic.

This represents the llvm.instrprof.value.profile intrinsic.

ConstantInt * getIndex() const

Value * getTargetValue() const

ConstantInt * getValueKind() const

LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)

Definition InstrProfiling.cpp:668

LLVM_ABI void moveBefore(InstListType::iterator InsertPos)

Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...

LLVM_ABI InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

LLVM_ABI const Function * getFunction() const

Return the function this instruction belongs to.

LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)

Set the metadata of the specified kind to the specified node.

Class to represent integer types.

This is an important class for using LLVM in a threaded context.

An instruction for reading from memory.

void getExitBlocks(SmallVectorImpl< BlockT * > &ExitBlocks) const

Return all of the successor blocks of this loop.

void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const

Return all blocks inside the loop that have successors outside of the loop.

BlockT * getLoopPreheader() const

If there is a preheader for this loop, return it.

bool hasDedicatedExits() const

Return true if no exit block for the loop has a predecessor that is outside the loop.

SmallVector< LoopT *, 4 > getLoopsInPreorder() const

Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...

LoopT * getLoopFor(const BlockT *BB) const

Return the inner most loop that BB lives in.

Represents a single loop in the control flow graph.

LLVM_ABI MDNode * createUnlikelyBranchWeights()

Return metadata containing two branch weights, with significant bias towards false destination.

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)

A Module instance is used to store all the information related to an LLVM module.

static PointerType * getUnqual(Type *ElementType)

This constructs a pointer to an object of the specified type in the default address space (address sp...

static LLVM_ABI PointerType * get(Type *ElementType, unsigned AddressSpace)

This constructs a pointer to an object of the specified type in a numbered address space.

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

constexpr StringRef substr(size_t Start, size_t N=npos) const

Return a reference to the substring from [Start, Start + N).

bool starts_with(StringRef Prefix) const

Check if this string starts with the given Prefix.

constexpr size_t size() const

size - Get the string size.

static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)

This static method is the primary way to create a literal StructType.

Analysis pass providing the TargetLibraryInfo.

Provides information about what library functions are available for the current target.

Triple - Helper class for working with autoconf configuration names.

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

The instances of the Type class are immutable: once they are created, they are never changed.

static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)

static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)

static LLVM_ABI Type * getVoidTy(LLVMContext &C)

static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)

static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)

LLVMContext & getContext() const

Return the LLVMContext in which this type was uniqued.

Value * getOperand(unsigned i) const

unsigned getNumOperands() const

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

LLVM_ABI void replaceAllUsesWith(Value *V)

Change all uses of this to point to a new Value.

LLVM_ABI LLVMContext & getContext() const

All values hold a context through their type.

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

const ParentTy * getParent() const

self_iterator getIterator()

NodeTy * getNextNode()

Get the next node, or nullptr for the list tail.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

constexpr char Args[]

Key for Kernel::Metadata::mArgs.

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

@ C

The default llvm calling convention, compatible with C.

@ BasicBlock

Various leaf nodes.

LLVM_ABI Function * getDeclarationIfExists(const Module *M, ID id)

Look up the Function declaration of the intrinsic id in the Module M and return it if it exists.

@ PD

PD - Prefix code for packed double precision vector floating point operations performed in the SSE re...

ValuesClass values(OptsTy... Options)

Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...

initializer< Ty > init(const Ty &Val)

PointerTypeMap run(const Module &M)

Compute the PointerTypeMap for the module M.

friend class Instruction

Iterator for Instructions in a `BasicBlock.

This is an optimization pass for GlobalISel generic memory operations.

FunctionAddr VTableAddr Value

FunctionAddr NumBitmapBytes

StringRef getInstrProfNameVarPrefix()

Return the name prefix of variables containing instrumented function names.

StringRef getInstrProfRuntimeHookVarName()

Return the name of the hook variable defined in profile runtime library.

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

FunctionAddr VTableAddr uintptr_t uintptr_t Int32Ty

LLVM_ABI void createProfileSamplingVar(Module &M)

Definition InstrProfiling.cpp:2139

StringRef getInstrProfBitmapVarPrefix()

Return the name prefix of profile bitmap variables.

LLVM_ABI cl::opt< bool > DoInstrProfNameCompression

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy

Provide the FunctionAnalysisManager to Module proxy.

FunctionAddr Int16ArrayTy

StringRef getInstrProfVTableNamesVarName()

StringRef getInstrProfDataVarPrefix()

Return the name prefix of variables containing per-function control data.

StringRef getCoverageUnusedNamesVarName()

Return the name of the internal variable recording the array of PGO name vars referenced by the cover...

LLVM_ABI std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)

Return the name of the profile section corresponding to IPSK.

LLVM_ABI bool needsComdatForCounter(const GlobalObject &GV, const Module &M)

Check if we can use Comdat for profile variables.

auto dyn_cast_or_null(const Y &Val)

LLVM_ABI std::string getPGOName(const GlobalVariable &V, bool InLTO=false)

StringRef getInstrProfInitFuncName()

Return the name of the runtime initialization method that is generated by the compiler.

StringRef getInstrProfValuesVarPrefix()

Return the name prefix of value profile variables.

bool any_of(R &&range, UnaryPredicate P)

Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.

StringRef getInstrProfCounterBiasVarName()

auto reverse(ContainerTy &&C)

StringRef getInstrProfRuntimeHookVarUseFuncName()

Return the name of the compiler generated function that references the runtime hook variable.

StringRef getInstrProfRegFuncsName()

Return the name of function that registers all the per-function control data at program startup time ...

LLVM_ABI Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)

Produce Result string with the same format described above.

LLVM_ABI void SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore, Instruction **ThenTerm, Instruction **ElseTerm, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr)

SplitBlockAndInsertIfThenElse is similar to SplitBlockAndInsertIfThen, but also creates the ElseBlock...

StringRef getInstrProfCountersVarPrefix()

Return the name prefix of profile counter variables.

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

bool none_of(R &&Range, UnaryPredicate P)

Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.

LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)

FunctionAddr VTableAddr Count

LLVM_ABI StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)

Return the initializer in string of the PGO name var NameVar.

StringRef getInstrProfBitmapBiasVarName()

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

StringRef getInstrProfValueProfMemOpFuncName()

Return the name profile runtime entry point to do memop size value profiling.

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

FunctionAddr VTableAddr uintptr_t uintptr_t Data

IRBuilder(LLVMContext &, FolderTy, InserterTy, MDNode *, ArrayRef< OperandBundleDef >) -> IRBuilder< FolderTy, InserterTy >

StringRef getInstrProfNamesRegFuncName()

Return the name of the runtime interface that registers the PGO name strings.

LLVM_ABI void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)

Adds global values to the llvm.compiler.used list.

LLVM_ABI Error collectVTableStrings(ArrayRef< GlobalVariable * > VTables, std::string &Result, bool doCompression)

LLVM_ABI void setGlobalVariableLargeSection(const Triple &TargetTriple, GlobalVariable &GV)

ArrayRef(const T &OneElt) -> ArrayRef< T >

std::string toString(const APInt &I, unsigned Radix, bool Signed, bool formatAsCLiteral=false, bool UpperCase=true, bool InsertSeparators=false)

LLVM_ABI bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)

Check if we can safely rename this Comdat function.

LLVM_ABI void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)

LLVM_ABI void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)

Append F to the list of global ctors of module M with the given Priority.

LLVM_ABI bool isPresplitCoroSuspendExitEdge(const BasicBlock &Src, const BasicBlock &Dest)

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

auto predecessors(const MachineBasicBlock *BB)

StringRef getInstrProfValueProfFuncName()

Return the name profile runtime entry point to do value profiling for a given site.

llvm:🆑:opt< llvm::InstrProfCorrelator::ProfCorrelatorKind > ProfileCorrelate

StringRef getInstrProfRegFuncName()

Return the name of the runtime interface that registers per-function control data for one instrumente...

LLVM_ABI Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)

Split the containing block at the specified instruction - everything before SplitBefore stays in the ...

AnalysisManager< Function > FunctionAnalysisManager

Convenience typedef for the Function analysis manager.

LLVM_ABI void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)

Adds global values to the llvm.used list.

StringRef getInstrProfNamesVarName()

Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...

LLVM_ABI bool isGPUProfTarget(const Module &M)

Determines whether module targets a GPU eligable for PGO instrumentation.

LLVM_ABI bool isIRPGOFlagSet(const Module *M)

Check if INSTR_PROF_RAW_VERSION_VAR is defined.

StringRef getInstrProfVNodesVarName()

Return the name of value profile node array variables:

StringRef toStringRef(bool B)

Construct a string ref from a boolean.

cl::opt< bool > EnableVTableValueProfiling("enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison."))

Definition InstrProfiling.cpp:71

StringRef getInstrProfVTableVarPrefix()

Return the name prefix of variables containing virtual table profile data.

AnalysisManager< Module > ModuleAnalysisManager

Convenience typedef for the Module analysis manager.

This struct is a compact representation of a valid (non-zero power of two) alignment.

This struct is a compact representation of a valid (power of two) or undefined (0) alignment.