PGOInstrumentation.cpp Source File (original) (raw)

116#include

117#include

118#include

119#include

120#include

121#include

122#include

123#include

124#include <unordered_map>

125#include

126#include

127

128using namespace llvm;

131

132#define DEBUG_TYPE "pgo-instrumentation"

133

134 STATISTIC(NumOfPGOInstrument, "Number of edges instrumented.");

135 STATISTIC(NumOfPGOSelectInsts, "Number of select instruction instrumented.");

136 STATISTIC(NumOfPGOMemIntrinsics, "Number of mem intrinsics instrumented.");

138 STATISTIC(NumOfPGOBB, "Number of basic-blocks.");

139 STATISTIC(NumOfPGOSplit, "Number of critical edge splits.");

140 STATISTIC(NumOfPGOFunc, "Number of functions having valid profile counts.");

141 STATISTIC(NumOfPGOMismatch, "Number of functions having mismatch profile.");

142 STATISTIC(NumOfPGOMissing, "Number of functions without profile.");

143 STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentations.");

144 STATISTIC(NumOfCSPGOInstrument, "Number of edges instrumented in CSPGO.");

146 "Number of select instruction instrumented in CSPGO.");

148 "Number of mem intrinsics instrumented in CSPGO.");

149 STATISTIC(NumOfCSPGOEdge, "Number of edges in CSPGO.");

150 STATISTIC(NumOfCSPGOBB, "Number of basic-blocks in CSPGO.");

151 STATISTIC(NumOfCSPGOSplit, "Number of critical edge splits in CSPGO.");

153 "Number of functions having valid profile counts in CSPGO.");

155 "Number of functions having mismatch profile in CSPGO.");

156 STATISTIC(NumOfCSPGOMissing, "Number of functions without profile in CSPGO.");

157 STATISTIC(NumCoveredBlocks, "Number of basic blocks that were executed");

158

159

160

164 cl::desc("Specify the path of profile data file. This is "

165 "mainly for test purpose."));

169 cl::desc("Specify the path of profile remapping file. This is mainly for "

170 "test purpose."));

171

172

173

176 cl::desc("Disable Value Profiling"));

177

178

179

182 cl::desc("Max number of annotations for a single indirect "

183 "call callsite"));

184

185

186

189 cl::desc("Max number of precise value annotations for a single memop"

190 "intrinsic"));

191

192

193

196 cl::desc("Append function hash to the name of COMDAT function to avoid "

197 "function hash mismatch due to the preinliner"));

198

199namespace llvm {

200

201

204 cl::desc("Use this option to turn on/off "

205 "warnings about missing profile data for "

206 "functions."));

207

208

209

212 cl::desc("Use this option to turn off/on "

213 "warnings about profile cfg mismatch."));

214

215

216

217

220 cl::desc("The option is used to turn on/off "

221 "warnings about hash mismatch for comdat "

222 "or weak functions."));

223}

224

225

228 cl::desc("Use this option to turn on/off SELECT "

229 "instruction instrumentation. "));

230

231

234 cl::desc("A boolean option to show CFG dag or text "

235 "with raw profile counts from "

236 "profile data. See also option "

237 "-pgo-view-counts. To limit graph "

238 "display to only one function, use "

239 "filtering option -view-bfi-func-name."),

243

244

247 cl::desc("Use this option to turn on/off "

248 "memory intrinsic size profiling."));

249

250

253 cl::desc("When this option is on, the annotated "

254 "branch probability will be emitted as "

255 "optimization remarks: -{Rpass|"

256 "pass-remarks}=pgo-instrumentation"));

257

260 cl::desc("Force to instrument function entry basicblock."));

261

265 cl::desc("Force to instrument loop entries."));

266

268 "pgo-function-entry-coverage", cl::Hidden,

270 "Use this option to enable function entry coverage instrumentation."));

271

273 "pgo-block-coverage",

274 cl::desc("Use this option to enable basic block coverage instrumentation"));

275

278 cl::desc("Create a dot file of CFGs with block "

279 "coverage inference information"));

280

282 "pgo-temporal-instrumentation",

283 cl::desc("Use this option to enable temporal instrumentation"));

284

287 cl::desc("Fix function entry count in profile use."));

288

291 cl::desc("Print out the non-match BFI count if a hot raw profile count "

292 "becomes non-hot, or a cold raw profile count becomes hot. "

293 "The print is enabled under -Rpass-analysis=pgo, or "

294 "internal option -pass-remarks-analysis=pgo."));

295

298 cl::desc("Print out mismatched BFI counts after setting profile metadata "

299 "The print is enabled under -Rpass-analysis=pgo, or "

300 "internal option -pass-remarks-analysis=pgo."));

301

304 cl::desc("Set the threshold for pgo-verify-bfi: only print out "

305 "mismatched BFI if the difference percentage is greater than "

306 "this value (in percentage)."));

307

310 cl::desc("Set the threshold for pgo-verify-bfi: skip the counts whose "

311 "profile count value is below."));

312

316 cl::desc("Trace the hash of the function with this name."));

317

319 "pgo-function-size-threshold", cl::Hidden,

320 cl::desc("Do not instrument functions smaller than this threshold."));

321

324 cl::desc("Do not instrument functions with the number of critical edges "

325 " greater than this threshold."));

326

329 cl::desc("For cold function instrumentation, skip instrumenting functions "

330 "whose entry count is above the given value."));

331

334 cl::desc("For cold function instrumentation, treat count unknown(e.g. "

335 "unprofiled) functions as cold."));

336

339 cl::desc("Enable cold function only instrumentation."));

340

342

343namespace llvm {

344

345

347

348

349

351

352

353

357}

358

359namespace {

360class FunctionInstrumenter final {

364 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;

368

370

371

372

373

374

375 bool isValueProfilingDisabled() const {

377 InstrumentationType == PGOInstrumentationType::CTXPROF;

378 }

379

380 bool shouldInstrumentEntryBB() const {

382 InstrumentationType == PGOInstrumentationType::CTXPROF;

383 }

384

386

387public:

388 FunctionInstrumenter(

390 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,

394 : M(M), F(F), TLI(TLI), ComdatMembers(ComdatMembers), BPI(BPI), BFI(BFI),

395 LI(LI), InstrumentationType(InstrumentationType) {}

396

398};

399}

400

401

402

404 BranchInst *BI = dyn_cast(TI);

406 return std::string();

407

410 if (!CI)

411 return std::string();

412

413 std::string result;

417

420 if (CV) {

422 OS << "_Zero";

423 else if (CV->isOne())

424 OS << "_One";

426 OS << "_MinusOne";

427 else

428 OS << "_Const";

429 }

430 return result;

431}

432

434#define VALUE_PROF_KIND(Enumerator, Value, Descr) Descr,

436};

437

438

439

443 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR));

445 uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);

446 if (InstrumentationType == PGOInstrumentationType::CSFDO)

447 ProfileVersion |= VARIANT_MASK_CSIR_PROF;

449 InstrumentationType == PGOInstrumentationType::CTXPROF)

450 ProfileVersion |= VARIANT_MASK_INSTR_ENTRY;

452 ProfileVersion |= VARIANT_MASK_INSTR_LOOP_ENTRIES;

454 ProfileVersion |= VARIANT_MASK_DBG_CORRELATE;

456 ProfileVersion |=

457 VARIANT_MASK_BYTE_COVERAGE | VARIANT_MASK_FUNCTION_ENTRY_ONLY;

459 ProfileVersion |= VARIANT_MASK_BYTE_COVERAGE;

461 ProfileVersion |= VARIANT_MASK_TEMPORAL_PROF;

466 Triple TT(M.getTargetTriple());

467 if (TT.supportsCOMDAT()) {

469 IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName));

470 }

471 return IRLevelVersionVariable;

472}

473

474namespace {

475

476

477

478

479

480

481enum VisitMode { VM_counting, VM_instrument, VM_annotate };

482class PGOUseFunc;

483

484

485struct SelectInstVisitor : public InstVisitor {

487 unsigned NSIs = 0;

488 VisitMode Mode = VM_counting;

489 unsigned *CurCtrIdx = nullptr;

490 unsigned TotalNumCtrs = 0;

493 PGOUseFunc *UseFunc = nullptr;

494 bool HasSingleByteCoverage;

495

496 SelectInstVisitor(Function &Func, bool HasSingleByteCoverage)

497 : F(Func), HasSingleByteCoverage(HasSingleByteCoverage) {}

498

499 void countSelects() {

500 NSIs = 0;

501 Mode = VM_counting;

503 }

504

505

506

507

508

509 void instrumentSelects(unsigned *Ind, unsigned TotalNC, GlobalValue *FNV,

511 Mode = VM_instrument;

512 CurCtrIdx = Ind;

513 TotalNumCtrs = TotalNC;

514 FuncHash = FHash;

515 FuncNameVar = FNV;

517 }

518

519

520 void annotateSelects(PGOUseFunc *UF, unsigned *Ind) {

521 Mode = VM_annotate;

522 UseFunc = UF;

523 CurCtrIdx = Ind;

525 }

526

527 void instrumentOneSelectInst(SelectInst &SI);

528 void annotateOneSelectInst(SelectInst &SI);

529

530

532

533

534

535 unsigned getNumOfSelectInsts() const { return NSIs; }

536};

537

538

539

540

541

542struct PGOEdge {

546 bool InMST = false;

547 bool Removed = false;

548 bool IsCritical = false;

549

551 : SrcBB(Src), DestBB(Dest), Weight(W) {}

552

553

554 std::string infoString() const {

555 return (Twine(Removed ? "-" : " ") + (InMST ? " " : "*") +

556 (IsCritical ? "c" : " ") + " W=" + Twine(Weight))

557 .str();

558 }

559};

560

561

562struct PGOBBInfo {

563 PGOBBInfo *Group;

566

567 PGOBBInfo(unsigned IX) : Group(this), Index(IX) {}

568

569

570 std::string infoString() const {

571 return (Twine("Index=") + Twine(Index)).str();

572 }

573};

574

575

576template <class Edge, class BBInfo> class FuncPGOInstrumentation {

577private:

579

580

581 bool IsCS;

582

583

584 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;

585

587

588 void computeCFGHash();

589 void renameComdatFunction();

590

591public:

593 std::vector<std::vector> ValueSites;

594 SelectInstVisitor SIVisitor;

595 std::string FuncName;

596 std::string DeprecatedFuncName;

598

599

601

602

604

605 const std::optional BCI;

606

607 static std::optional

608 constructBCI(Function &Func, bool HasSingleByteCoverage,

609 bool InstrumentFuncEntry) {

610 if (HasSingleByteCoverage)

612 return {};

613 }

614

615

616

617 void getInstrumentBBs(std::vector<BasicBlock *> &InstrumentBBs);

618

619

620

622

623

624 BBInfo &getBBInfo(const BasicBlock *BB) const { return MST.getBBInfo(BB); }

625

626

627 BBInfo *findBBInfo(const BasicBlock *BB) const { return MST.findBBInfo(BB); }

628

629

630 void dumpInfo(StringRef Str = "") const {

632 " Hash: " + Twine(FunctionHash) + "\t" + Str);

633 }

634

635 FuncPGOInstrumentation(

637 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,

640 bool IsCS = false, bool InstrumentFuncEntry = true,

641 bool InstrumentLoopEntries = false, bool HasSingleByteCoverage = false)

642 : F(Func), IsCS(IsCS), ComdatMembers(ComdatMembers), VPC(Func, TLI),

643 TLI(TLI), ValueSites(IPVK_Last + 1),

644 SIVisitor(Func, HasSingleByteCoverage),

645 MST(F, InstrumentFuncEntry, InstrumentLoopEntries, BPI, BFI, LI),

646 BCI(constructBCI(Func, HasSingleByteCoverage, InstrumentFuncEntry)) {

648 BCI->viewBlockCoverageGraph();

649

650 SIVisitor.countSelects();

651 ValueSites[IPVK_MemOPSize] = VPC.get(IPVK_MemOPSize);

652 if (!IsCS) {

653 NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts();

654 NumOfPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();

656 ValueSites[IPVK_IndirectCallTarget] = VPC.get(IPVK_IndirectCallTarget);

658 ValueSites[IPVK_VTableTarget] = VPC.get(IPVK_VTableTarget);

659 } else {

660 NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts();

661 NumOfCSPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();

663 }

664

667 computeCFGHash();

668 if (!ComdatMembers.empty())

669 renameComdatFunction();

670 LLVM_DEBUG(dumpInfo("after CFGMST"));

671

672 for (const auto &E : MST.allEdges()) {

673 if (E->Removed)

674 continue;

675 IsCS ? NumOfCSPGOEdge++ : NumOfPGOEdge++;

676 if (!E->InMST)

677 IsCS ? NumOfCSPGOInstrument++ : NumOfPGOInstrument++;

678 }

679

680 if (CreateGlobalVar)

682 }

683};

684

685}

686

687

688

689

690template <class Edge, class BBInfo>

691void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {

692 std::vector<uint8_t> Indexes;

694 for (auto &BB : F) {

696 auto BI = findBBInfo(Succ);

697 if (BI == nullptr)

698 continue;

700 for (int J = 0; J < 4; J++)

701 Indexes.push_back((uint8_t)(Index >> (J * 8)));

702 }

703 }

705

707

708 auto updateJCH = [&JCH](uint64_t Num) {

712 };

713 updateJCH((uint64_t)SIVisitor.getNumOfSelectInsts());

714 updateJCH((uint64_t)ValueSites[IPVK_IndirectCallTarget].size());

715 updateJCH((uint64_t)ValueSites[IPVK_MemOPSize].size());

716 if (BCI) {

717 updateJCH(BCI->getInstrumentedBlocksHash());

718 } else {

720 }

721

722

723

725

726

727 FunctionHash &= 0x0FFFFFFFFFFFFFFF;

728 if (IsCS)

730 LLVM_DEBUG(dbgs() << "Function Hash Computation for " << F.getName() << ":\n"

731 << " CRC = " << JC.getCRC()

732 << ", Selects = " << SIVisitor.getNumOfSelectInsts()

733 << ", Edges = " << MST.numEdges() << ", ICSites = "

734 << ValueSites[IPVK_IndirectCallTarget].size()

735 << ", Memops = " << ValueSites[IPVK_MemOPSize].size()

736 << ", High32 CRC = " << JCH.getCRC()

737 << ", Hash = " << FunctionHash << "\n";);

738

740 dbgs() << "Funcname=" << F.getName() << ", Hash=" << FunctionHash

741 << " in building " << F.getParent()->getSourceFileName() << "\n";

742}

743

744

747 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {

749 return false;

750

751

752

753

754

755

756

757

759 for (auto &&CM : make_range(ComdatMembers.equal_range(C))) {

760 assert(!isa(CM.second));

761 Function *FM = dyn_cast(CM.second);

762 if (FM != &F)

763 return false;

764 }

765 return true;

766}

767

768

769template <class Edge, class BBInfo>

770void FuncPGOInstrumentation<Edge, BBInfo>::renameComdatFunction() {

772 return;

773 std::string OrigName = F.getName().str();

774 std::string NewFuncName =

775 Twine(F.getName() + "." + Twine(FunctionHash)).str();

776 F.setName(Twine(NewFuncName));

778 FuncName = Twine(FuncName + "." + Twine(FunctionHash)).str();

781

782

783

784 if (.hasComdat()) {

786 NewComdat = M->getOrInsertComdat(StringRef(NewFuncName));

788 F.setComdat(NewComdat);

789 return;

790 }

791

792

793 Comdat *OrigComdat = F.getComdat();

794 std::string NewComdatName =

796 NewComdat = M->getOrInsertComdat(StringRef(NewComdatName));

798

799 for (auto &&CM : make_range(ComdatMembers.equal_range(OrigComdat))) {

800

801 cast(CM.second)->setComdat(NewComdat);

802 }

803}

804

805

806

807template <class Edge, class BBInfo>

808void FuncPGOInstrumentation<Edge, BBInfo>::getInstrumentBBs(

809 std::vector<BasicBlock *> &InstrumentBBs) {

810 if (BCI) {

811 for (auto &BB : F)

812 if (BCI->shouldInstrumentBlock(BB))

813 InstrumentBBs.push_back(&BB);

814 return;

815 }

816

817

818 std::vector<Edge *> EdgeList;

819 EdgeList.reserve(MST.numEdges());

820 for (const auto &E : MST.allEdges())

821 EdgeList.push_back(E.get());

822

823 for (auto &E : EdgeList) {

825 if (InstrBB)

826 InstrumentBBs.push_back(InstrBB);

827 }

828}

829

830

831

832template <class Edge, class BBInfo>

833BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) {

834 if (E->InMST || E->Removed)

835 return nullptr;

836

839

840 if (SrcBB == nullptr)

841 return DestBB;

842 if (DestBB == nullptr)

843 return SrcBB;

844

846

847

849 return nullptr;

850 return BB;

851 };

852

853

854

857 return canInstrument(SrcBB);

858 if (!E->IsCritical)

859 return canInstrument(DestBB);

860

861

862

865 isa(TI) ? nullptr : SplitCriticalEdge(TI, SuccNum);

866 if (!InstrBB) {

868 dbgs() << "Fail to split critical edge: not instrument this edge.\n");

869 return nullptr;

870 }

871

872

873 IsCS ? NumOfCSPGOSplit++ : NumOfPGOSplit++;

874 LLVM_DEBUG(dbgs() << "Split critical edge: " << getBBInfo(SrcBB).Index

875 << " --> " << getBBInfo(DestBB).Index << "\n");

876

877 MST.addEdge(SrcBB, InstrBB, 0);

878

879 Edge &NewEdge1 = MST.addEdge(InstrBB, DestBB, 0);

880 NewEdge1.InMST = true;

881 E->Removed = true;

882

883 return canInstrument(InstrBB);

884}

885

886

887

888

889

890

891static void

895 auto *OrigCall = dyn_cast(Cand.AnnotatedInst);

896 if (!OrigCall)

897 return;

898

899 if (!isa(OrigCall)) {

900

901

902 std::optional ParentFunclet =

904 if (ParentFunclet)

906 } else {

907

908

909

910 if (!BlockColors.empty()) {

911 const ColorVector &CV = BlockColors.find(OrigCall->getParent())->second;

912 assert(CV.size() == 1 && "non-unique color for block!");

916 }

917 }

918}

919

920

921

922void FunctionInstrumenter::instrument() {

924

925

927 }

928

929 const bool IsCtxProf = InstrumentationType == PGOInstrumentationType::CTXPROF;

930 FuncPGOInstrumentation<PGOEdge, PGOBBInfo> FuncInfo(

931 F, TLI, ComdatMembers, !IsCtxProf, BPI, BFI, LI,

932 InstrumentationType == PGOInstrumentationType::CSFDO,

933 shouldInstrumentEntryBB(), shouldInstrumentLoopEntries(),

935

936 auto *const Name = IsCtxProf ? cast(&F) : FuncInfo.FuncNameVar;

937 auto *const CFGHash =

938 ConstantInt::get(Type::getInt64Ty(M.getContext()), FuncInfo.FunctionHash);

939

940

942 Name, PointerType::get(M.getContext(), 0));

944 auto &EntryBB = F.getEntryBlock();

945 IRBuilder<> Builder(&EntryBB, EntryBB.getFirstInsertionPt());

946

947

948 Builder.CreateIntrinsic(

949 Intrinsic::instrprof_cover, {},

950 {NormalizedNamePtr, CFGHash, Builder.getInt32(1), Builder.getInt32(0)});

951 return;

952 }

953

954 std::vector<BasicBlock *> InstrumentBBs;

955 FuncInfo.getInstrumentBBs(InstrumentBBs);

956 unsigned NumCounters =

957 InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();

958

959 if (IsCtxProf) {

960 auto *CSIntrinsic =

962

963

964

965

966

967

968

969

971 for (auto &BB : F)

972 for (auto &Instr : BB)

973 if (auto *CS = dyn_cast(&Instr)) {

975 continue;

976 Visitor(CS);

977 }

978 };

979

980 uint32_t TotalNumCallsites = 0;

981 Visit([&TotalNumCallsites](auto *) { ++TotalNumCallsites; });

982

983

985 Visit([&](auto *CB) {

987 Builder.CreateCall(CSIntrinsic,

988 {Name, CFGHash, Builder.getInt32(TotalNumCallsites),

989 Builder.getInt32(CallsiteIndex++),

990 CB->getCalledOperand()});

991 });

992 }

993

997 auto &EntryBB = F.getEntryBlock();

998 IRBuilder<> Builder(&EntryBB, EntryBB.getFirstInsertionPt());

999

1000

1001 Builder.CreateIntrinsic(Intrinsic::instrprof_timestamp, {},

1002 {NormalizedNamePtr, CFGHash,

1003 Builder.getInt32(NumCounters),

1004 Builder.getInt32(I)});

1006 }

1007

1008 for (auto *InstrBB : InstrumentBBs) {

1010 assert(Builder.GetInsertPoint() != InstrBB->end() &&

1011 "Cannot get the Instrumentation point");

1012

1013

1014 Builder.CreateIntrinsic(PGOBlockCoverage ? Intrinsic::instrprof_cover

1015 : Intrinsic::instrprof_increment,

1016 {},

1017 {NormalizedNamePtr, CFGHash,

1018 Builder.getInt32(NumCounters),

1019 Builder.getInt32(I++)});

1020 }

1021

1022

1023 FuncInfo.SIVisitor.instrumentSelects(&I, NumCounters, Name,

1024 FuncInfo.FunctionHash);

1025 assert(I == NumCounters);

1026

1027 if (isValueProfilingDisabled())

1028 return;

1029

1030 NumOfPGOICall += FuncInfo.ValueSites[IPVK_IndirectCallTarget].size();

1031

1032

1033

1034

1035

1037 if (F.hasPersonalityFn() &&

1040

1041

1042 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {

1043 unsigned SiteIndex = 0;

1045 continue;

1046

1047 for (VPCandidateInfo Cand : FuncInfo.ValueSites[Kind]) {

1049 << " site: CallSite Index = " << SiteIndex << "\n");

1050

1052 assert(Builder.GetInsertPoint() != Cand.InsertPt->getParent()->end() &&

1053 "Cannot get the Instrumentation point");

1054

1055 Value *ToProfile = nullptr;

1056 if (Cand.V->getType()->isIntegerTy())

1057 ToProfile = Builder.CreateZExtOrTrunc(Cand.V, Builder.getInt64Ty());

1058 else if (Cand.V->getType()->isPointerTy())

1059 ToProfile = Builder.CreatePtrToInt(Cand.V, Builder.getInt64Ty());

1060 assert(ToProfile && "value profiling Value is of unexpected type");

1061

1063 Name, PointerType::get(M.getContext(), 0));

1064

1067 Builder.CreateCall(

1069 Intrinsic::instrprof_value_profile),

1070 {NormalizedNamePtr, Builder.getInt64(FuncInfo.FunctionHash),

1071 ToProfile, Builder.getInt32(Kind), Builder.getInt32(SiteIndex++)},

1072 OpBundles);

1073 }

1074 }

1075}

1076

1077namespace {

1078

1079

1080struct PGOUseEdge : public PGOEdge {

1081 using PGOEdge::PGOEdge;

1082

1083 std::optional<uint64_t> Count;

1084

1085

1087

1088

1089 std::string infoString() const {

1090 if (!Count)

1091 return PGOEdge::infoString();

1092 return (Twine(PGOEdge::infoString()) + " Count=" + Twine(*Count)).str();

1093 }

1094};

1095

1097

1098

1099struct PGOUseBBInfo : public PGOBBInfo {

1100 std::optional<uint64_t> Count;

1101 int32_t UnknownCountInEdge = 0;

1102 int32_t UnknownCountOutEdge = 0;

1103 DirectEdges InEdges;

1104 DirectEdges OutEdges;

1105

1106 PGOUseBBInfo(unsigned IX) : PGOBBInfo(IX) {}

1107

1108

1110

1111

1112 std::string infoString() const {

1113 if (!Count)

1114 return PGOBBInfo::infoString();

1115 return (Twine(PGOBBInfo::infoString()) + " Count=" + Twine(*Count)).str();

1116 }

1117

1118

1119 void addOutEdge(PGOUseEdge *E) {

1120 OutEdges.push_back(E);

1121 UnknownCountOutEdge++;

1122 }

1123

1124

1125 void addInEdge(PGOUseEdge *E) {

1126 InEdges.push_back(E);

1127 UnknownCountInEdge++;

1128 }

1129};

1130

1131}

1132

1133

1136 for (const auto &E : Edges) {

1137 if (E->Removed)

1138 continue;

1139 if (E->Count)

1140 Total += *E->Count;

1141 }

1143}

1144

1145namespace {

1146

1147class PGOUseFunc {

1148public:

1150 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,

1153 bool InstrumentFuncEntry, bool InstrumentLoopEntries,

1154 bool HasSingleByteCoverage)

1155 : F(Func), M(Modu), BFI(BFIin), PSI(PSI),

1156 FuncInfo(Func, TLI, ComdatMembers, false, BPI, BFIin, LI, IsCS,

1157 InstrumentFuncEntry, InstrumentLoopEntries,

1158 HasSingleByteCoverage),

1159 FreqAttr(FFA_Normal), IsCS(IsCS), VPC(Func, TLI) {}

1160

1161 void handleInstrProfError(Error Err, uint64_t MismatchedFuncSum);

1162

1163

1166

1167

1168 void populateCounters();

1169

1170

1172

1173

1175

1176

1177 void annotateValueSites();

1178

1179

1180 void annotateValueSites(uint32_t Kind);

1181

1182

1183 void annotateIrrLoopHeaderWeights();

1184

1185

1186 enum FuncFreqAttr { FFA_Normal, FFA_Cold, FFA_Hot };

1187

1188

1189 FuncFreqAttr getFuncFreqAttr() const { return FreqAttr; }

1190

1191

1193

1194

1195 InstrProfRecord &getProfileRecord() { return ProfileRecord; }

1196

1197

1198 PGOUseBBInfo &getBBInfo(const BasicBlock *BB) const {

1199 return FuncInfo.getBBInfo(BB);

1200 }

1201

1202

1203 PGOUseBBInfo *findBBInfo(const BasicBlock *BB) const {

1204 return FuncInfo.findBBInfo(BB);

1205 }

1206

1207 Function &getFunc() const { return F; }

1208

1209 void dumpInfo(StringRef Str = "") const { FuncInfo.dumpInfo(Str); }

1210

1211 uint64_t getProgramMaxCount() const { return ProgramMaxCount; }

1212

1213private:

1218

1219

1220 FuncPGOInstrumentation<PGOUseEdge, PGOUseBBInfo> FuncInfo;

1221

1222

1223

1225

1226

1228

1229

1230 uint32_t ProfileCountSize = 0;

1231

1232

1234

1235

1236 FuncFreqAttr FreqAttr;

1237

1238

1239 bool IsCS;

1240

1242

1243

1244 bool setInstrumentedCounts(const std::vector<uint64_t> &CountFromProfile);

1245

1246

1247

1248 void setEdgeCount(DirectEdges &Edges, uint64_t Value);

1249

1250

1251

1252

1253 void markFunctionAttributes(uint64_t EntryCount, uint64_t MaxCount) {

1255 FreqAttr = FFA_Hot;

1257 FreqAttr = FFA_Cold;

1258 }

1259};

1260

1261}

1262

1263

1265 const FuncPGOInstrumentation<PGOUseEdge, PGOUseBBInfo> &FuncInfo) {

1266

1267 if (FuncInfo.BCI)

1268 return;

1269 for (const auto &E : FuncInfo.MST.allEdges()) {

1270 if (E->Removed)

1271 continue;

1273 const BasicBlock *DestBB = E->DestBB;

1274 PGOUseBBInfo &SrcInfo = FuncInfo.getBBInfo(SrcBB);

1275 PGOUseBBInfo &DestInfo = FuncInfo.getBBInfo(DestBB);

1276 SrcInfo.addOutEdge(E.get());

1277 DestInfo.addInEdge(E.get());

1278 }

1279}

1280

1281

1282

1283bool PGOUseFunc::setInstrumentedCounts(

1284 const std::vector<uint64_t> &CountFromProfile) {

1285

1286 std::vector<BasicBlock *> InstrumentBBs;

1287 FuncInfo.getInstrumentBBs(InstrumentBBs);

1288

1290

1291 unsigned NumCounters =

1292 InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();

1293

1294

1295 if (NumCounters != CountFromProfile.size()) {

1296 return false;

1297 }

1298 auto *FuncEntry = &*F.begin();

1299

1300

1302 for (BasicBlock *InstrBB : InstrumentBBs) {

1303 uint64_t CountValue = CountFromProfile[I++];

1304 PGOUseBBInfo &Info = getBBInfo(InstrBB);

1305

1306

1307

1308 if (InstrBB == FuncEntry && CountValue == 0)

1309 CountValue = 1;

1310 Info.setBBInfoCount(CountValue);

1311 }

1312 ProfileCountSize = CountFromProfile.size();

1313 CountPosition = I;

1314

1315

1316 auto setEdgeCount = [this](PGOUseEdge *E, uint64_t Value) -> void {

1317 E->setEdgeCount(Value);

1318 this->getBBInfo(E->SrcBB).UnknownCountOutEdge--;

1319 this->getBBInfo(E->DestBB).UnknownCountInEdge--;

1320 };

1321

1322

1323

1324

1325 for (const auto &E : FuncInfo.MST.allEdges()) {

1326 if (E->Removed || E->InMST)

1327 continue;

1329 PGOUseBBInfo &SrcInfo = getBBInfo(SrcBB);

1330

1331

1332

1333 if (SrcInfo.Count && SrcInfo.OutEdges.size() == 1)

1334 setEdgeCount(E.get(), *SrcInfo.Count);

1335 else {

1336 const BasicBlock *DestBB = E->DestBB;

1337 PGOUseBBInfo &DestInfo = getBBInfo(DestBB);

1338

1339

1340 if (DestInfo.Count && DestInfo.InEdges.size() == 1)

1341 setEdgeCount(E.get(), *DestInfo.Count);

1342 }

1343 if (E->Count)

1344 continue;

1345

1346

1347 setEdgeCount(E.get(), 0);

1348 }

1349 return true;

1350}

1351

1352

1353

1354void PGOUseFunc::setEdgeCount(DirectEdges &Edges, uint64_t Value) {

1355 for (auto &E : Edges) {

1356 if (E->Count)

1357 continue;

1358 E->setEdgeCount(Value);

1359

1360 getBBInfo(E->SrcBB).UnknownCountOutEdge--;

1361 getBBInfo(E->DestBB).UnknownCountInEdge--;

1362 return;

1363 }

1365}

1366

1367

1369 const char MetadataName[] = "instr_prof_hash_mismatch";

1371

1372 auto *Existing = F.getMetadata(LLVMContext::MD_annotation);

1373 if (Existing) {

1374 MDTuple *Tuple = cast(Existing);

1375 for (const auto &N : Tuple->operands()) {

1376 if (N.equalsStr(MetadataName))

1377 return;

1379 }

1380 }

1381

1385 F.setMetadata(LLVMContext::MD_annotation, MD);

1386}

1387

1388void PGOUseFunc::handleInstrProfError(Error Err, uint64_t MismatchedFuncSum) {

1390 auto &Ctx = M->getContext();

1391 auto Err = IPE.get();

1392 bool SkipWarning = false;

1393 LLVM_DEBUG(dbgs() << "Error in reading profile for Func "

1394 << FuncInfo.FuncName << ": ");

1395 if (Err == instrprof_error::unknown_function) {

1396 IsCS ? NumOfCSPGOMissing++ : NumOfPGOMissing++;

1399 } else if (Err == instrprof_error::hash_mismatch ||

1400 Err == instrprof_error::malformed) {

1401 IsCS ? NumOfCSPGOMismatch++ : NumOfPGOMismatch++;

1402 SkipWarning =

1407 LLVM_DEBUG(dbgs() << "hash mismatch (hash= " << FuncInfo.FunctionHash

1408 << " skip=" << SkipWarning << ")");

1409

1411 }

1412

1414 if (SkipWarning)

1415 return;

1416

1417 std::string Msg =

1418 IPE.message() + std::string(" ") + F.getName().str() +

1419 std::string(" Hash = ") + std::to_string(FuncInfo.FunctionHash) +

1420 std::string(" up to ") + std::to_string(MismatchedFuncSum) +

1421 std::string(" count discarded");

1422

1423 Ctx.diagnose(

1425 });

1426}

1427

1428

1429

1430

1433 auto &Ctx = M->getContext();

1434 uint64_t MismatchedFuncSum = 0;

1436 FuncInfo.FuncName, FuncInfo.FunctionHash, FuncInfo.DeprecatedFuncName,

1437 &MismatchedFuncSum);

1439 handleInstrProfError(std::move(E), MismatchedFuncSum);

1440 return false;

1441 }

1442 ProfileRecord = std::move(Result.get());

1445 return true;

1446 }

1447 std::vector<uint64_t> &CountFromProfile = ProfileRecord.Counts;

1448

1449 IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;

1450 LLVM_DEBUG(dbgs() << CountFromProfile.size() << " counts\n");

1451

1453 for (unsigned I = 0, S = CountFromProfile.size(); I < S; I++) {

1454 LLVM_DEBUG(dbgs() << " " << I << ": " << CountFromProfile[I] << "\n");

1455 ValueSum += CountFromProfile[I];

1456 }

1457 AllZeros = (ValueSum == 0);

1458

1460

1461 getBBInfo(nullptr).UnknownCountOutEdge = 2;

1462 getBBInfo(nullptr).UnknownCountInEdge = 2;

1463

1464 if (!setInstrumentedCounts(CountFromProfile)) {

1466 dbgs() << "Inconsistent number of counts, skipping this function");

1468 M->getName().data(),

1469 Twine("Inconsistent number of counts in ") + F.getName().str() +

1470 Twine(": the profile may be stale or there is a function name "

1471 "collision."),

1473 return false;

1474 }

1476 return true;

1477}

1478

1480 uint64_t MismatchedFuncSum = 0;

1482 FuncInfo.FuncName, FuncInfo.FunctionHash, FuncInfo.DeprecatedFuncName,

1483 &MismatchedFuncSum);

1484 if (auto Err = Result.takeError()) {

1485 handleInstrProfError(std::move(Err), MismatchedFuncSum);

1486 return;

1487 }

1488 IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;

1489

1490 std::vector<uint64_t> &CountsFromProfile = Result.get().Counts;

1492 unsigned Index = 0;

1493 for (auto &BB : F)

1494 if (FuncInfo.BCI->shouldInstrumentBlock(BB))

1495 Coverage[&BB] = (CountsFromProfile[Index++] != 0);

1496 assert(Index == CountsFromProfile.size());

1497

1498

1500 InverseDependencies;

1501 for (auto &BB : F) {

1502 for (auto *Dep : FuncInfo.BCI->getDependencies(BB)) {

1503

1504 InverseDependencies[Dep].insert(&BB);

1505 }

1506 }

1507

1508

1509 std::stack<const BasicBlock *> CoveredBlocksToProcess;

1510 for (auto &[BB, IsCovered] : Coverage)

1511 if (IsCovered)

1512 CoveredBlocksToProcess.push(BB);

1513

1514 while (!CoveredBlocksToProcess.empty()) {

1515 auto *CoveredBlock = CoveredBlocksToProcess.top();

1516 assert(Coverage[CoveredBlock]);

1517 CoveredBlocksToProcess.pop();

1518 for (auto *BB : InverseDependencies[CoveredBlock]) {

1519

1520 if (Coverage[BB])

1521 continue;

1523 CoveredBlocksToProcess.push(BB);

1524 }

1525 }

1526

1527

1529

1530

1531 F.setEntryCount(Coverage[&F.getEntryBlock()] ? 10000 : 0);

1532 for (auto &BB : F) {

1533

1534

1535

1536

1537

1538

1541 Weights.push_back((Coverage[Succ] || !Coverage[&BB]) ? 1 : 0);

1542 if (Weights.size() >= 2)

1544 false);

1545 }

1546

1547 unsigned NumCorruptCoverage = 0;

1552 auto IsBlockDead = [&](const BasicBlock &BB) -> std::optional {

1553 if (auto C = BFI.getBlockProfileCount(&BB))

1554 return C == 0;

1555 return {};

1556 };

1557 LLVM_DEBUG(dbgs() << "Block Coverage: (Instrumented=*, Covered=X)\n");

1558 for (auto &BB : F) {

1559 LLVM_DEBUG(dbgs() << (FuncInfo.BCI->shouldInstrumentBlock(BB) ? "* " : " ")

1560 << (Coverage[&BB] ? "X " : " ") << " " << BB.getName()

1561 << "\n");

1562

1563

1564

1565

1566 if (Coverage[&BB] == IsBlockDead(BB).value_or(false)) {

1568 dbgs() << "Found inconsistent block covearge for " << BB.getName()

1569 << ": BCI=" << (Coverage[&BB] ? "Covered" : "Dead") << " BFI="

1570 << (IsBlockDead(BB).value() ? "Dead" : "Covered") << "\n");

1571 ++NumCorruptCoverage;

1572 }

1573 if (Coverage[&BB])

1574 ++NumCoveredBlocks;

1575 }

1577 auto &Ctx = M->getContext();

1579 M->getName().data(),

1580 Twine("Found inconsistent block coverage for function ") + F.getName() +

1581 " in " + Twine(NumCorruptCoverage) + " blocks.",

1583 }

1585 FuncInfo.BCI->viewBlockCoverageGraph(&Coverage);

1586}

1587

1588

1589

1590void PGOUseFunc::populateCounters() {

1591 bool Changes = true;

1592 unsigned NumPasses = 0;

1593 while (Changes) {

1594 NumPasses++;

1595 Changes = false;

1596

1597

1598

1599 for (auto &BB : reverse(F)) {

1600 PGOUseBBInfo *UseBBInfo = findBBInfo(&BB);

1601 if (UseBBInfo == nullptr)

1602 continue;

1603 if (!UseBBInfo->Count) {

1604 if (UseBBInfo->UnknownCountOutEdge == 0) {

1605 UseBBInfo->Count = sumEdgeCount(UseBBInfo->OutEdges);

1606 Changes = true;

1607 } else if (UseBBInfo->UnknownCountInEdge == 0) {

1608 UseBBInfo->Count = sumEdgeCount(UseBBInfo->InEdges);

1609 Changes = true;

1610 }

1611 }

1612 if (UseBBInfo->Count) {

1613 if (UseBBInfo->UnknownCountOutEdge == 1) {

1616

1617

1618

1619 if (*UseBBInfo->Count > OutSum)

1620 Total = *UseBBInfo->Count - OutSum;

1621 setEdgeCount(UseBBInfo->OutEdges, Total);

1622 Changes = true;

1623 }

1624 if (UseBBInfo->UnknownCountInEdge == 1) {

1627 if (*UseBBInfo->Count > InSum)

1628 Total = *UseBBInfo->Count - InSum;

1629 setEdgeCount(UseBBInfo->InEdges, Total);

1630 Changes = true;

1631 }

1632 }

1633 }

1634 }

1635

1636 LLVM_DEBUG(dbgs() << "Populate counts in " << NumPasses << " passes.\n");

1637 (void)NumPasses;

1638#ifndef NDEBUG

1639

1640 for (auto &BB : F) {

1641 auto BI = findBBInfo(&BB);

1642 if (BI == nullptr)

1643 continue;

1644 assert(BI->Count && "BB count is not valid");

1645 }

1646#endif

1647

1648 FuncInfo.SIVisitor.annotateSelects(this, &CountPosition);

1649 assert(CountPosition == ProfileCountSize);

1650

1653 for (auto &BB : F) {

1654 auto BI = findBBInfo(&BB);

1655 if (BI == nullptr)

1656 continue;

1657 FuncMaxCount = std::max(FuncMaxCount, *BI->Count);

1658 }

1659

1660

1664 markFunctionAttributes(FuncEntryCount, FuncMaxCount);

1665

1666 LLVM_DEBUG(FuncInfo.dumpInfo("after reading profile."));

1667}

1668

1669

1670void PGOUseFunc::setBranchWeights() {

1671

1672 LLVM_DEBUG(dbgs() << "\nSetting branch weights for func " << F.getName()

1673 << " IsCS=" << IsCS << "\n");

1674 for (auto &BB : F) {

1677 continue;

1678 if (!(isa(TI) || isa(TI) ||

1679 isa(TI) || isa(TI) ||

1680 isa(TI)))

1681 continue;

1682

1683 const PGOUseBBInfo &BBCountInfo = getBBInfo(&BB);

1684 if (!*BBCountInfo.Count)

1685 continue;

1686

1687

1688

1689

1690

1691 unsigned OutEdgesCount = BBCountInfo.OutEdges.size();

1692 unsigned SuccessorCount = BB.getTerminator()->getNumSuccessors();

1693 assert(OutEdgesCount <= SuccessorCount);

1694

1697 for (unsigned It = 0; It < OutEdgesCount; It++) {

1698 const PGOUseEdge *E = BBCountInfo.OutEdges[It];

1700 const BasicBlock *DestBB = E->DestBB;

1701 if (DestBB == nullptr)

1702 continue;

1704 uint64_t EdgeCount = *E->Count;

1705 if (EdgeCount > MaxCount)

1706 MaxCount = EdgeCount;

1707 EdgeCounts[SuccNum] = EdgeCount;

1708 }

1709

1710 if (MaxCount)

1712 else {

1713

1714

1715

1716 auto &Ctx = M->getContext();

1718 M->getName().data(),

1719 Twine("Profile in ") + F.getName().str() +

1720 Twine(" partially ignored") +

1721 Twine(", possibly due to the lack of a return path."),

1723 }

1724 }

1725}

1726

1729 if (isa(Pred->getTerminator()))

1730 return true;

1731 }

1732 return false;

1733}

1734

1735void PGOUseFunc::annotateIrrLoopHeaderWeights() {

1736 LLVM_DEBUG(dbgs() << "\nAnnotating irreducible loop header weights.\n");

1737

1738 for (auto &BB : F) {

1739

1740

1741

1744 const PGOUseBBInfo &BBCountInfo = getBBInfo(&BB);

1746 }

1747 }

1748}

1749

1750void SelectInstVisitor::instrumentOneSelectInst(SelectInst &SI) {

1754 auto *Step = Builder.CreateZExt(SI.getCondition(), Int64Ty);

1755 auto *NormalizedFuncNameVarPtr =

1757 FuncNameVar, PointerType::get(M->getContext(), 0));

1758 Builder.CreateIntrinsic(Intrinsic::instrprof_increment_step, {},

1759 {NormalizedFuncNameVarPtr, Builder.getInt64(FuncHash),

1760 Builder.getInt32(TotalNumCtrs),

1761 Builder.getInt32(*CurCtrIdx), Step});

1762 ++(*CurCtrIdx);

1763}

1764

1765void SelectInstVisitor::annotateOneSelectInst(SelectInst &SI) {

1766 std::vector<uint64_t> &CountFromProfile = UseFunc->getProfileRecord().Counts;

1767 assert(*CurCtrIdx < CountFromProfile.size() &&

1768 "Out of bound access of counters");

1770 SCounts[0] = CountFromProfile[*CurCtrIdx];

1771 ++(*CurCtrIdx);

1773 auto BI = UseFunc->findBBInfo(SI.getParent());

1774 if (BI != nullptr) {

1775 TotalCount = *BI->Count;

1776

1777

1778 if (TotalCount < SCounts[0])

1779 BI->Count = SCounts[0];

1780 }

1781

1782 SCounts[1] = (TotalCount > SCounts[0] ? TotalCount - SCounts[0] : 0);

1783 uint64_t MaxCount = std::max(SCounts[0], SCounts[1]);

1784 if (MaxCount)

1786}

1787

1788void SelectInstVisitor::visitSelectInst(SelectInst &SI) {

1790 return;

1791

1792 if (SI.getCondition()->getType()->isVectorTy())

1793 return;

1794

1795 switch (Mode) {

1796 case VM_counting:

1797 NSIs++;

1798 return;

1799 case VM_instrument:

1800 instrumentOneSelectInst(SI);

1801 return;

1802 case VM_annotate:

1803 annotateOneSelectInst(SI);

1804 return;

1805 }

1806

1808}

1809

1811 if (ValueProfKind == IPVK_MemOPSize)

1813 if (ValueProfKind == llvm::IPVK_VTableTarget)

1816}

1817

1818

1819void PGOUseFunc::annotateValueSites() {

1821 return;

1822

1823

1825

1827 annotateValueSites(Kind);

1828}

1829

1830

1831void PGOUseFunc::annotateValueSites(uint32_t Kind) {

1832 assert(Kind <= IPVK_Last);

1833 unsigned ValueSiteIndex = 0;

1834

1835 unsigned NumValueSites = ProfileRecord.getNumValueSites(Kind);

1836

1837

1838

1839

1840

1841

1842

1843

1844

1845

1846 if (NumValueSites > 0 && Kind == IPVK_VTableTarget &&

1847 NumValueSites != FuncInfo.ValueSites[IPVK_VTableTarget].size() &&

1849 FuncInfo.ValueSites[IPVK_VTableTarget] = VPC.get(IPVK_VTableTarget);

1850 auto &ValueSites = FuncInfo.ValueSites[Kind];

1851 if (NumValueSites != ValueSites.size()) {

1852 auto &Ctx = M->getContext();

1854 M->getName().data(),

1855 Twine("Inconsistent number of value sites for ") +

1857 F.getName().str() +

1858 Twine("\", possibly due to the use of a stale profile."),

1860 return;

1861 }

1862

1864 LLVM_DEBUG(dbgs() << "Read one value site profile (kind = " << Kind

1865 << "): Index = " << ValueSiteIndex << " out of "

1866 << NumValueSites << "\n");

1868 *M, *I.AnnotatedInst, ProfileRecord,

1871 ValueSiteIndex++;

1872 }

1873}

1874

1875

1876

1879 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {

1881 return;

1883 if (Comdat *C = F.getComdat())

1884 ComdatMembers.insert(std::make_pair(C, &F));

1886 if (Comdat *C = GV.getComdat())

1887 ComdatMembers.insert(std::make_pair(C, &GV));

1889 if (Comdat *C = GA.getComdat())

1890 ComdatMembers.insert(std::make_pair(C, &GA));

1891}

1892

1893

1895 if (F.isDeclaration())

1896 return true;

1897

1898

1899

1900 unsigned NumCriticalEdges = 0;

1901 for (auto &BB : F) {

1902 const Instruction *TI = BB.getTerminator();

1905 NumCriticalEdges++;

1906 }

1907 }

1910 << ", NumCriticalEdges=" << NumCriticalEdges

1911 << " exceed the threshold. Skip PGO.\n");

1912 return true;

1913 }

1914 return false;

1915}

1916

1917

1920 return true;

1921 if (F.hasFnAttribute(llvm::Attribute::Naked))

1922 return true;

1923 if (F.hasFnAttribute(llvm::Attribute::NoProfile))

1924 return true;

1925 if (F.hasFnAttribute(llvm::Attribute::SkipProfile))

1926 return true;

1928 return true;

1930 if (auto EntryCount = F.getEntryCount())

1933 }

1934 return false;

1935}

1936

1943

1944

1945 if (InstrumentationType == PGOInstrumentationType::FDO)

1947

1948 Triple TT(M.getTargetTriple());

1952 M.getName().data(),

1953 Twine("VTable value profiling is presently not "

1954 "supported for non-ELF object formats"),

1956 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;

1958

1959 for (auto &F : M) {

1961 continue;

1966 FunctionInstrumenter FI(M, F, TLI, ComdatMembers, BPI, BFI, LI,

1967 InstrumentationType);

1968 FI.instrument();

1969 }

1970 return true;

1971}

1972

1976

1977

1980 if (ProfileSampling)

1985 return PA;

1986}

1987

1993 };

1996 };

1999 };

2002 };

2003

2005 InstrumentationType))

2007

2009}

2010

2011

2012

2017#ifndef NDEBUG

2018 auto BFIEntryCount = F.getEntryCount();

2019 assert(BFIEntryCount && (BFIEntryCount->getCount() > 0) &&

2020 "Invalid BFI Entrycount");

2021#endif

2024 for (auto &BBI : F) {

2027 if (!Func.findBBInfo(&BBI))

2028 continue;

2030 CountValue = *Func.getBBInfo(&BBI).Count;

2031 BFICountValue = *BFICount;

2034 }

2035 if (SumCount.isZero())

2036 return;

2037

2039 "Incorrect sum of BFI counts");

2041 return;

2042 double Scale = (SumCount / SumBFICount).convertToDouble();

2043 if (Scale < 1.001 && Scale > 0.999)

2044 return;

2045

2048 if (NewEntryCount == 0)

2049 NewEntryCount = 1;

2052 LLVM_DEBUG(dbgs() << "FixFuncEntryCount: in " << F.getName()

2054 << NewEntryCount << "\n");

2055 }

2056}

2057

2058

2059

2066

2070

2071 unsigned BBNum = 0, BBMisMatchNum = 0, NonZeroBBNum = 0;

2072 for (auto &BBI : F) {

2075

2076 CountValue = Func.getBBInfo(&BBI).Count.value_or(CountValue);

2077

2078 BBNum++;

2079 if (CountValue)

2080 NonZeroBBNum++;

2082 if (BFICount)

2083 BFICountValue = *BFICount;

2084

2085 if (HotBBOnly) {

2086 bool rawIsHot = CountValue >= HotCountThreshold;

2087 bool BFIIsHot = BFICountValue >= HotCountThreshold;

2089 bool ShowCount = false;

2090 if (rawIsHot && !BFIIsHot) {

2091 Msg = "raw-Hot to BFI-nonHot";

2092 ShowCount = true;

2093 } else if (rawIsCold && BFIIsHot) {

2094 Msg = "raw-Cold to BFI-Hot";

2095 ShowCount = true;

2096 }

2097 if (!ShowCount)

2098 continue;

2099 } else {

2102 continue;

2103 uint64_t Diff = (BFICountValue >= CountValue)

2104 ? BFICountValue - CountValue

2105 : CountValue - BFICountValue;

2107 continue;

2108 }

2109 BBMisMatchNum++;

2110

2111 ORE.emit([&]() {

2113 F.getSubprogram(), &BBI);

2114 Remark << "BB " << ore::NV("Block", BBI.getName())

2115 << " Count=" << ore::NV("Count", CountValue)

2116 << " BFI_Count=" << ore::NV("Count", BFICountValue);

2117 if (!Msg.empty())

2118 Remark << " (" << Msg << ")";

2120 });

2121 }

2122 if (BBMisMatchNum)

2123 ORE.emit([&]() {

2125 F.getSubprogram(), &F.getEntryBlock())

2126 << "In Func " << ore::NV("Function", F.getName())

2127 << ": Num_of_BB=" << ore::NV("Count", BBNum)

2128 << ", Num_of_non_zerovalue_BB=" << ore::NV("Count", NonZeroBBNum)

2129 << ", Num_of_mis_matching_BB=" << ore::NV("Count", BBMisMatchNum);

2130 });

2131}

2132

2140 bool IsCS) {

2142 auto &Ctx = M.getContext();

2143

2145 ProfileRemappingFileName);

2146 if (Error E = ReaderOrErr.takeError()) {

2148 Ctx.diagnose(

2150 });

2151 return false;

2152 }

2153

2154 std::unique_ptr PGOReader =

2155 std::move(ReaderOrErr.get());

2156 if (!PGOReader) {

2158 StringRef("Cannot get PGOReader")));

2159 return false;

2160 }

2161 if (!PGOReader->hasCSIRLevelProfile() && IsCS)

2162 return false;

2163

2164

2165 if (!PGOReader->isIRLevelProfile()) {

2167 ProfileFileName.data(), "Not an IR level instrumentation profile"));

2168 return false;

2169 }

2170 if (PGOReader->functionEntryOnly()) {

2172 ProfileFileName.data(),

2173 "Function entry profiles are not yet supported for optimization"));

2174 return false;

2175 }

2176

2179 if (.hasName() || .hasMetadata(LLVMContext::MD_type))

2180 continue;

2181

2182

2184 }

2185 }

2186

2187

2188

2189

2190 M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()),

2194

2195 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;

2197 std::vector<Function *> HotFunctions;

2198 std::vector<Function *> ColdFunctions;

2199

2200

2201

2202 bool InstrumentFuncEntry = PGOReader->instrEntryBBEnabled();

2205 bool InstrumentLoopEntries = PGOReader->instrLoopEntriesEnabled();

2208

2209 bool HasSingleByteCoverage = PGOReader->hasSingleByteCoverage();

2210 for (auto &F : M) {

2212 continue;

2217 if (!HasSingleByteCoverage) {

2218

2219

2221 BFI);

2222 }

2223 PGOUseFunc Func(F, &M, TLI, ComdatMembers, BPI, BFI, LI, PSI, IsCS,

2224 InstrumentFuncEntry, InstrumentLoopEntries,

2225 HasSingleByteCoverage);

2226 if (HasSingleByteCoverage) {

2227 Func.populateCoverage(PGOReader.get());

2228 continue;

2229 }

2230

2231

2232

2233

2235 bool AllZeros = false;

2236 if (!Func.readCounters(PGOReader.get(), AllZeros, PseudoKind))

2237 continue;

2238 if (AllZeros) {

2240 if (Func.getProgramMaxCount() != 0)

2241 ColdFunctions.push_back(&F);

2242 continue;

2243 }

2245

2246 if (F.hasFnAttribute(Attribute::Cold))

2247 F.removeFnAttr(Attribute::Cold);

2248

2250 F.addFnAttr(Attribute::Hot);

2251 continue;

2252 }

2253 Func.populateCounters();

2254 Func.setBranchWeights();

2255 Func.annotateValueSites();

2256 Func.annotateIrrLoopHeaderWeights();

2257 PGOUseFunc::FuncFreqAttr FreqAttr = Func.getFuncFreqAttr();

2258 if (FreqAttr == PGOUseFunc::FFA_Cold)

2259 ColdFunctions.push_back(&F);

2260 else if (FreqAttr == PGOUseFunc::FFA_Hot)

2261 HotFunctions.push_back(&F);

2266 std::unique_ptr NewBPI =

2267 std::make_unique(F, LI);

2268 std::unique_ptr NewBFI =

2269 std::make_unique(F, *NewBPI, LI);

2271 NewBFI->view();

2273 dbgs() << "pgo-view-counts: " << Func.getFunc().getName() << "\n";

2274 NewBFI->print(dbgs());

2275 }

2276 }

2282 WriteGraph(&Func, Twine("PGORawCounts_") + Func.getFunc().getName());

2283 else

2284 ViewGraph(&Func, Twine("PGORawCounts_") + Func.getFunc().getName());

2286 dbgs() << "pgo-view-raw-counts: " << Func.getFunc().getName() << "\n";

2287 Func.dumpInfo();

2288 }

2289 }

2290

2294

2295

2298

2299

2304 }

2306 }

2307 }

2308

2309

2310

2311

2312

2313 for (auto &F : HotFunctions) {

2314 F->addFnAttr(Attribute::InlineHint);

2315 LLVM_DEBUG(dbgs() << "Set inline attribute to function: " << F->getName()

2316 << "\n");

2317 }

2318 for (auto &F : ColdFunctions) {

2319

2320

2321 if (F->hasFnAttribute(Attribute::Hot)) {

2322 auto &Ctx = M.getContext();

2323 std::string Msg = std::string("Function ") + F->getName().str() +

2324 std::string(" is annotated as a hot function but"

2325 " the profile is cold");

2326 Ctx.diagnose(

2328 continue;

2329 }

2330 F->addFnAttr(Attribute::Cold);

2331 LLVM_DEBUG(dbgs() << "Set cold attribute to function: " << F->getName()

2332 << "\n");

2333 }

2334 return true;

2335}

2336

2338 std::string Filename, std::string RemappingFilename, bool IsCS,

2340 : ProfileFileName(std::move(Filename)),

2341 ProfileRemappingFileName(std::move(RemappingFilename)), IsCS(IsCS),

2347 if (!FS)

2349}

2350

2353

2357 };

2360 };

2363 };

2366 };

2367

2369 if ( annotateAllFunctions (M, ProfileFileName, ProfileRemappingFileName, *FS,

2370 LookupTLI, LookupBPI, LookupBFI, LookupLI, PSI,

2371 IsCS))

2373

2375}

2376

2378 if ( Node ->getName().empty())

2379 return Node->getName().str();

2380

2381 std::string SimpleNodeName;

2384 return SimpleNodeName;

2385}

2386

2389 assert(MaxCount > 0 && "Bad max count");

2392 for (const auto &ECI : EdgeCounts)

2394

2395 LLVM_DEBUG(dbgs() << "Weight is: "; for (const auto &W

2396 : Weights) {

2397 dbgs() << W << " ";

2398 } dbgs() << "\n";);

2399

2401

2405 if (BrCondStr.empty())

2406 return;

2407

2409 std::accumulate(Weights.begin(), Weights.end(), (uint64_t)0,

2412 std::accumulate(EdgeCounts.begin(), EdgeCounts.end(), (uint64_t)0,

2417 std::string BranchProbStr;

2419 OS << BP;

2420 OS << " (total count : " << TotalCount << ")";

2423 ORE.emit([&]() {

2425 << BrCondStr << " is true with probability : " << BranchProbStr;

2426 });

2427 }

2428}

2429

2430namespace llvm {

2431

2434 TI->setMetadata(llvm::LLVMContext::MD_irr_loop,

2436}

2437

2442

2444 return &G->getFunc().front();

2445 }

2446

2449 }

2450

2452

2455 }

2456

2459 }

2460};

2461

2465

2467 return std::string(G->getFunc().getName());

2468 }

2469

2471 std::string Result;

2473

2475 PGOUseBBInfo *BI = Graph->findBBInfo(Node);

2476 OS << "Count : ";

2477 if (BI && BI->Count)

2478 OS << *BI->Count << "\\l";

2479 else

2480 OS << "Unknown\\l";

2481

2483 return Result;

2484

2486 if (!isa(&I))

2487 continue;

2488

2489 OS << "SELECT : { T = ";

2492 if (!HasProf)

2493 OS << "Unknown, F = Unknown }\\l";

2494 else

2495 OS << TC << ", F = " << FC << " }\\l";

2496 }

2497 return Result;

2498 }

2499};

2500

2501}

This file implements a class to represent arbitrary precision integral constant values and operations...

This file contains the simple types necessary to represent the attributes associated with functions a...

This file finds the minimum set of blocks on a CFG that must be instrumented to infer execution cover...

Analysis containing CSE Info

#define clEnumValN(ENUMVAL, FLAGNAME, DESC)

This file contains the declarations for the subclasses of Constant, which represent the different fla...

Given that RA is a live value

post inline ee instrument

static BasicBlock * getInstrBB(CFGMST< Edge, BBInfo > &MST, Edge &E, const DenseSet< const BasicBlock * > &ExecBlocks)

This file provides various utilities for inspecting and working with the control flow graph in LLVM I...

Module.h This file contains the declarations for the Module class.

This header defines various interfaces for pass management in LLVM.

static cl::opt< unsigned > ColdCountThreshold("mfs-count-threshold", cl::desc("Minimum number of times a block must be executed to be retained."), cl::init(1), cl::Hidden)

static cl::opt< bool > PGOInstrumentEntry("pgo-instrument-entry", cl::init(false), cl::Hidden, cl::desc("Force to instrument function entry basicblock."))

static GlobalVariable * createIRLevelProfileFlagVar(Module &M, PGOInstrumentationType InstrumentationType)

static cl::opt< std::string > PGOTestProfileRemappingFile("pgo-test-profile-remapping-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile remapping file. This is mainly for " "test purpose."))

static cl::opt< bool > PGOFixEntryCount("pgo-fix-entry-count", cl::init(true), cl::Hidden, cl::desc("Fix function entry count in profile use."))

static void fixFuncEntryCount(PGOUseFunc &Func, LoopInfo &LI, BranchProbabilityInfo &NBPI)

static void annotateFunctionWithHashMismatch(Function &F, LLVMContext &ctx)

cl::opt< unsigned > MaxNumVTableAnnotations

static cl::opt< unsigned > MaxNumMemOPAnnotations("memop-max-annotations", cl::init(4), cl::Hidden, cl::desc("Max number of precise value annotations for a single memop" "intrinsic"))

static cl::opt< bool > PGOTemporalInstrumentation("pgo-temporal-instrumentation", cl::desc("Use this option to enable temporal instrumentation"))

static cl::opt< unsigned > PGOFunctionSizeThreshold("pgo-function-size-threshold", cl::Hidden, cl::desc("Do not instrument functions smaller than this threshold."))

static cl::opt< unsigned > MaxNumAnnotations("icp-max-annotations", cl::init(3), cl::Hidden, cl::desc("Max number of annotations for a single indirect " "call callsite"))

static bool skipPGOGen(const Function &F)

static void collectComdatMembers(Module &M, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers)

static cl::opt< unsigned > PGOVerifyBFICutoff("pgo-verify-bfi-cutoff", cl::init(5), cl::Hidden, cl::desc("Set the threshold for pgo-verify-bfi: skip the counts whose " "profile count value is below."))

static cl::opt< std::string > PGOTraceFuncHash("pgo-trace-func-hash", cl::init("-"), cl::Hidden, cl::value_desc("function name"), cl::desc("Trace the hash of the function with this name."))

static void populateEHOperandBundle(VPCandidateInfo &Cand, DenseMap< BasicBlock *, ColorVector > &BlockColors, SmallVectorImpl< OperandBundleDef > &OpBundles)

static cl::opt< bool > PGOInstrSelect("pgo-instr-select", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off SELECT " "instruction instrumentation. "))

static cl::opt< bool > PGOFunctionEntryCoverage("pgo-function-entry-coverage", cl::Hidden, cl::desc("Use this option to enable function entry coverage instrumentation."))

static cl::opt< uint64_t > PGOColdInstrumentEntryThreshold("pgo-cold-instrument-entry-threshold", cl::init(0), cl::Hidden, cl::desc("For cold function instrumentation, skip instrumenting functions " "whose entry count is above the given value."))

static void verifyFuncBFI(PGOUseFunc &Func, LoopInfo &LI, BranchProbabilityInfo &NBPI, uint64_t HotCountThreshold, uint64_t ColdCountThreshold)

static cl::opt< unsigned > PGOVerifyBFIRatio("pgo-verify-bfi-ratio", cl::init(2), cl::Hidden, cl::desc("Set the threshold for pgo-verify-bfi: only print out " "mismatched BFI if the difference percentage is greater than " "this value (in percentage)."))

static cl::opt< bool > DoComdatRenaming("do-comdat-renaming", cl::init(false), cl::Hidden, cl::desc("Append function hash to the name of COMDAT function to avoid " "function hash mismatch due to the preinliner"))

static bool annotateAllFunctions(Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, vfs::FileSystem &FS, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, function_ref< LoopInfo *(Function &)> LookupLI, ProfileSummaryInfo *PSI, bool IsCS)

static cl::opt< unsigned > PGOFunctionCriticalEdgeThreshold("pgo-critical-edge-threshold", cl::init(20000), cl::Hidden, cl::desc("Do not instrument functions with the number of critical edges " " greater than this threshold."))

static void setupBBInfoEdges(const FuncPGOInstrumentation< PGOUseEdge, PGOUseBBInfo > &FuncInfo)

Set up InEdges/OutEdges for all BBs in the MST.

static bool skipPGOUse(const Function &F)

static bool canRenameComdat(Function &F, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers)

static cl::opt< bool > PGOVerifyHotBFI("pgo-verify-hot-bfi", cl::init(false), cl::Hidden, cl::desc("Print out the non-match BFI count if a hot raw profile count " "becomes non-hot, or a cold raw profile count becomes hot. " "The print is enabled under -Rpass-analysis=pgo, or " "internal option -pass-remarks-analysis=pgo."))

static cl::opt< bool > PGOBlockCoverage("pgo-block-coverage", cl::desc("Use this option to enable basic block coverage instrumentation"))

static cl::opt< bool > PGOInstrumentLoopEntries("pgo-instrument-loop-entries", cl::init(false), cl::Hidden, cl::desc("Force to instrument loop entries."))

static cl::opt< bool > PGOVerifyBFI("pgo-verify-bfi", cl::init(false), cl::Hidden, cl::desc("Print out mismatched BFI counts after setting profile metadata " "The print is enabled under -Rpass-analysis=pgo, or " "internal option -pass-remarks-analysis=pgo."))

static bool InstrumentAllFunctions(Module &M, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, function_ref< LoopInfo *(Function &)> LookupLI, PGOInstrumentationType InstrumentationType)

static uint64_t sumEdgeCount(const ArrayRef< PGOUseEdge * > Edges)

static cl::opt< bool > PGOInstrMemOP("pgo-instr-memop", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off " "memory intrinsic size profiling."))

static uint32_t getMaxNumAnnotations(InstrProfValueKind ValueProfKind)

Function::ProfileCount ProfileCount

static cl::opt< bool > EmitBranchProbability("pgo-emit-branch-prob", cl::init(false), cl::Hidden, cl::desc("When this option is on, the annotated " "branch probability will be emitted as " "optimization remarks: -{Rpass|" "pass-remarks}=pgo-instrumentation"))

static cl::opt< bool > DisableValueProfiling("disable-vp", cl::init(false), cl::Hidden, cl::desc("Disable Value Profiling"))

static std::string getSimpleNodeName(const BasicBlock *Node)

static cl::opt< bool > PGOViewBlockCoverageGraph("pgo-view-block-coverage-graph", cl::desc("Create a dot file of CFGs with block " "coverage inference information"))

static cl::opt< bool > PGOTreatUnknownAsCold("pgo-treat-unknown-as-cold", cl::init(false), cl::Hidden, cl::desc("For cold function instrumentation, treat count unknown(e.g. " "unprofiled) functions as cold."))

static bool isIndirectBrTarget(BasicBlock *BB)

static cl::opt< std::string > PGOTestProfileFile("pgo-test-profile-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is " "mainly for test purpose."))

static std::string getBranchCondString(Instruction *TI)

cl::opt< bool > PGOInstrumentColdFunctionOnly("pgo-instrument-cold-function-only", cl::init(false), cl::Hidden, cl::desc("Enable cold function only instrumentation."))

static cl::opt< PGOViewCountsType > PGOViewRawCounts("pgo-view-raw-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text " "with raw profile counts from " "profile data. See also option " "-pgo-view-counts. To limit graph " "display to only one function, use " "filtering option -view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))

static const char * ValueProfKindDescr[]

This file provides the interface for IR based instrumentation passes ( (profile-gen,...

FunctionAnalysisManager FAM

ModuleAnalysisManager MAM

This file contains the declarations for profiling metadata utility functions.

const SmallVectorImpl< MachineOperand > & Cond

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

static bool isSimple(Instruction *I)

This file defines the SmallVector class.

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

Defines the virtual file system interface vfs::FileSystem.

void printAsOperand(OutputBuffer &OB, Prec P=Prec::Default, bool StrictlyWorse=false) const

static APFloat getZero(const fltSemantics &Sem, bool Negative=false)

Factory for Positive and Negative Zero.

Class for arbitrary precision integers.

This templated class represents "all analyses that operate over " (e....

A container for analyses that lazily runs them and caches their results.

PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)

Get the result of an analysis pass for a given IR unit.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

LLVM Basic Block Representation.

const_iterator getFirstInsertionPt() const

Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...

const Instruction * getTerminator() const LLVM_READONLY

Returns the terminator instruction if the block is well formed or null if the block is not well forme...

Analysis pass which computes BlockFrequencyInfo.

BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...

std::optional< uint64_t > getBlockProfileCount(const BasicBlock *BB, bool AllowSynthetic=false) const

Returns the estimated profile count of BB.

Conditional or Unconditional Branch instruction.

bool isConditional() const

Value * getCondition() const

Analysis pass which computes BranchProbabilityInfo.

Analysis providing branch probability information.

An union-find based Minimum Spanning Tree for CFG.

Edge & addEdge(BasicBlock *Src, BasicBlock *Dest, uint64_t W)

const std::vector< std::unique_ptr< Edge > > & allEdges() const

size_t bbInfoSize() const

BBInfo * findBBInfo(const BasicBlock *BB) const

BBInfo & getBBInfo(const BasicBlock *BB) const

void dumpEdges(raw_ostream &OS, const Twine &Message) const

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

Predicate getPredicate() const

Return the predicate for this instruction.

StringRef getName() const

void setSelectionKind(SelectionKind Val)

SelectionKind getSelectionKind() const

static Constant * getPointerBitCastOrAddrSpaceCast(Constant *C, Type *Ty)

Create a BitCast or AddrSpaceCast for a pointer type depending on the address space.

This is the shared class of boolean and integer constants.

bool isMinusOne() const

This function will return true iff every bit in this constant is set to true.

bool isOne() const

This is just a convenience method to make client code smaller for a common case.

bool isZero() const

This is just a convenience method to make client code smaller for a common code.

static Constant * getIntegerValue(Type *Ty, const APInt &V)

Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...

iterator find(const_arg_type_t< KeyT > Val)

std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)

Diagnostic information for the PGO profiler.

Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.

Base class for error info classes.

virtual std::string message() const

Return the error message as a string.

Lightweight error class with error context and mandatory checking.

Tagged union holding either a T or a Error.

Class to represent profile counts.

static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)

If a parent module is specified, the alias is automatically inserted into the end of the specified mo...

@ HiddenVisibility

The GV is hidden.

@ ExternalLinkage

Externally visible function.

@ WeakAnyLinkage

Keep one copy of named function when linking (weak)

@ AvailableExternallyLinkage

Available for inspection, not emission.

@ LinkOnceODRLinkage

Same, but only replaced by something equivalent.

This instruction compares its operands according to the predicate given to the constructor.

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

Reader for the indexed binary instrprof format.

static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const Twine &RemappingPath="")

Factory method to create an indexed reader.

Expected< InstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName="", uint64_t *MismatchedFuncSum=nullptr)

Return the NamedInstrProfRecord associated with FuncName and FuncHash.

uint64_t getMaximumFunctionCount(bool UseCS)

Return the maximum of all known function counts.

An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...

Base class for instruction visitors.

void visit(Iterator Start, Iterator End)

RetTy visitSelectInst(SelectInst &I)

static bool canInstrumentCallsite(const CallBase &CB)

instrprof_error get() const

std::string message() const override

Return the error message as a string.

unsigned getNumSuccessors() const LLVM_READONLY

Return the number of successors that this instruction has.

bool isEHPad() const

Return true if the instruction is a variety of EH-block.

void setMetadata(unsigned KindID, MDNode *Node)

Set the metadata of the specified kind to the specified node.

A smart pointer to a reference-counted object that inherits from RefCountedBase or ThreadSafeRefCount...

void update(ArrayRef< uint8_t > Data)

This is an important class for using LLVM in a threaded context.

void diagnose(const DiagnosticInfo &DI)

Report a message to the currently installed diagnostic handler.

Analysis pass that exposes the LoopInfo for a function.

MDString * createString(StringRef Str)

Return the given string as metadata.

MDNode * createIrrLoopHeaderWeight(uint64_t Weight)

Return metadata containing an irreducible loop header weight.

ArrayRef< MDOperand > operands() const

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

A Module instance is used to store all the information related to an LLVM module.

PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)

PGOInstrumentationUse(std::string Filename="", std::string RemappingFilename="", bool IsCS=false, IntrusiveRefCntPtr< vfs::FileSystem > FS=nullptr)

PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

void preserveSet()

Mark an analysis set as preserved.

void preserve()

Mark an analysis as preserved.

An analysis pass based on the new PM to deliver ProfileSummaryInfo.

Analysis providing profile information.

uint64_t getOrCompColdCountThreshold() const

Returns ColdCountThreshold if set.

void refresh()

If no summary is present, attempt to refresh.

bool isColdCount(uint64_t C) const

Returns true if count C is considered cold.

bool isHotCount(uint64_t C) const

Returns true if count C is considered hot.

uint64_t getOrCompHotCountThreshold() const

Returns HotCountThreshold if set.

This class represents the LLVM 'select' instruction.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

reference emplace_back(ArgTypes &&... Args)

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

constexpr bool empty() const

empty - Check if the string is empty.

constexpr const char * data() const

data - Get a pointer to the start of the string (which may not be null terminated).

Analysis pass providing the TargetLibraryInfo.

Provides information about what library functions are available for the current target.

TinyPtrVector - This class is specialized for cases where there are normally 0 or 1 element in a vect...

Triple - Helper class for working with autoconf configuration names.

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

std::string str() const

Return the twine contents as a std::string.

The instances of the Type class are immutable: once they are created, they are never changed.

void print(raw_ostream &O, bool IsForDebug=false, bool NoDetails=false) const

Print the current type.

static IntegerType * getInt64Ty(LLVMContext &C)

Value * getOperand(unsigned i) const

Utility analysis that determines what values are worth profiling.

std::vector< CandidateInfo > get(InstrProfValueKind Kind) const

returns a list of value profiling candidates of the given kind

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

int getNumOccurrences() const

An efficient, type-erasing, non-owning reference to a callable.

const ParentTy * getParent() const

A raw_ostream that writes to an std::string.

The virtual file system interface.

This provides a very simple, boring adaptor for a begin and end iterator into a range type.

This file contains the declaration of the Comdat class, which represents a single COMDAT in LLVM.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

@ C

The default llvm calling convention, compatible with C.

Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})

Look up the Function declaration of the intrinsic id in the Module M.

ValuesClass values(OptsTy... Options)

Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...

initializer< Ty > init(const Ty &Val)

uint64_t getFuncHash(const FuncRecordTy *Record)

Return the structural hash associated with the function.

void checkExpectAnnotations(Instruction &I, const ArrayRef< uint32_t > ExistingWeights, bool IsFrontend)

checkExpectAnnotations - compares PGO counters to the thresholds used for llvm.expect and warns if th...

DiagnosticInfoOptimizationBase::Argument NV

NodeAddr< FuncNode * > Func

void write64le(void *P, uint64_t V)

IntrusiveRefCntPtr< FileSystem > getRealFileSystem()

Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.

This is an optimization pass for GlobalISel generic memory operations.

void setIrrLoopHeaderMetadata(Module *M, Instruction *TI, uint64_t Count)

auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)

Get the size of a range.

std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)

Please use getIRPGOFuncName for LLVM IR instrumentation.

void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)

Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.

unsigned GetSuccessorNumber(const BasicBlock *BB, const BasicBlock *Succ)

Search for the specified successor of basic block BB and return its position in the terminator instru...

std::string getIRPGOFuncName(const Function &F, bool InLTO=false)

Function::ProfileCount ProfileCount

auto successors(const MachineBasicBlock *BB)

void createProfileSamplingVar(Module &M)

void handleAllErrors(Error E, HandlerTs &&... Handlers)

Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...

iterator_range< T > make_range(T x, T y)

Convenience function for iterating over sub-ranges.

cl::opt< InstrProfCorrelator::ProfCorrelatorKind > ProfileCorrelate("profile-correlate", cl::desc("Use debug info or binary file to correlate profiles."), cl::init(InstrProfCorrelator::NONE), cl::values(clEnumValN(InstrProfCorrelator::NONE, "", "No profile correlation"), clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info", "Use debug info to correlate"), clEnumValN(InstrProfCorrelator::BINARY, "binary", "Use binary to correlate")))

DenseMap< BasicBlock *, ColorVector > colorEHFunclets(Function &F)

If an EH funclet personality is in use (see isFuncletEHPersonality), this will recompute which blocks...

void createPGONameMetadata(GlobalObject &GO, StringRef PGOName)

Create the PGOName metadata if a global object's PGO name is different from its mangled name.

cl::opt< bool > PGOWarnMissing

raw_ostream & WriteGraph(raw_ostream &O, const GraphType &G, bool ShortNames=false, const Twine &Title="")

bool SplitIndirectBrCriticalEdges(Function &F, bool IgnoreBlocksWithoutPHI, BranchProbabilityInfo *BPI=nullptr, BlockFrequencyInfo *BFI=nullptr)

cl::opt< bool > EnableVTableProfileUse("enable-vtable-profile-use", cl::init(false), cl::desc("If ThinLTO and WPD is enabled and this option is true, vtable " "profiles will be used by ICP pass for more efficient indirect " "call sequence. If false, type profiles won't be used."))

bool isScopedEHPersonality(EHPersonality Pers)

Returns true if this personality uses scope-style EH IR instructions: catchswitch,...

cl::opt< bool > DebugInfoCorrelate

OperandBundleDefT< Value * > OperandBundleDef

std::string getPGOName(const GlobalVariable &V, bool InLTO=false)

cl::opt< std::string > ViewBlockFreqFuncName("view-bfi-func-name", cl::Hidden, cl::desc("The option to specify " "the name of the function " "whose CFG will be displayed."))

GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)

Create and return the global variable for function name used in PGO instrumentation.

void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)

Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...

auto reverse(ContainerTy &&C)

void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected)

Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...

raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

EHPersonality classifyEHPersonality(const Value *Pers)

See if the given exception handling personality function is one that we understand.

cl::opt< bool > NoPGOWarnMismatch

RNSuccIterator< NodeRef, BlockT, RegionT > succ_begin(NodeRef Node)

cl::opt< PGOViewCountsType > PGOViewCounts("pgo-view-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text with " "block profile counts and branch probabilities " "right after PGO profile annotation step. The " "profile counts are computed using branch " "probabilities from the runtime profile data and " "block frequency propagation algorithm. To view " "the raw counts from the profile, use option " "-pgo-view-raw-counts instead. To limit graph " "display to only one function, use filtering option " "-view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))

RNSuccIterator< NodeRef, BlockT, RegionT > succ_end(NodeRef Node)

static uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)

Scale an individual branch count.

void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)

Adds global values to the llvm.compiler.used list.

BasicBlock * SplitCriticalEdge(Instruction *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")

If this edge is a critical edge, insert a new node to split the critical edge.

void ViewGraph(const GraphType &G, const Twine &Name, bool ShortNames=false, const Twine &Title="", GraphProgram::Name Program=GraphProgram::DOT)

ViewGraph - Emit a dot graph, run 'dot', run gv on the postscript file, then cleanup.

bool isCriticalEdge(const Instruction *TI, unsigned SuccNum, bool AllowIdenticalEdges=false)

Return true if the specified edge is a critical edge.

static uint64_t calculateCountScale(uint64_t MaxCount)

Calculate what to divide by to scale counts.

bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)

Check if we can safely rename this Comdat function.

void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)

OutputIt move(R &&Range, OutputIt Out)

Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.

bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)

Extract branch weights from MD_prof metadata.

auto predecessors(const MachineBasicBlock *BB)

void setProfMetadata(Module *M, Instruction *TI, ArrayRef< uint64_t > EdgeCounts, uint64_t MaxCount)

cl::opt< bool > EnableVTableValueProfiling("enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison."))

SuccIterator< const Instruction, const BasicBlock > const_succ_iterator

cl::opt< bool > NoPGOWarnMismatchComdatWeak

Implement std::hash so that hash_code can be used in STL containers.

static constexpr roundingMode rmNearestTiesToEven

static const fltSemantics & IEEEdouble() LLVM_READNONE

DOTGraphTraits(bool isSimple=false)

static std::string getGraphName(const PGOUseFunc *G)

std::string getNodeLabel(const BasicBlock *Node, const PGOUseFunc *Graph)

DOTGraphTraits - Template class that can be specialized to customize how graphs are converted to 'dot...

DefaultDOTGraphTraits - This class provides the default implementations of all of the DOTGraphTraits ...

static ChildIteratorType child_end(const NodeRef N)

static NodeRef getEntryNode(const PGOUseFunc *G)

static ChildIteratorType child_begin(const NodeRef N)

static nodes_iterator nodes_end(const PGOUseFunc *G)

static nodes_iterator nodes_begin(const PGOUseFunc *G)

Profiling information for a single function.

std::vector< uint64_t > Counts

CountPseudoKind getCountPseudoKind() const

uint32_t getNumValueSites(uint32_t ValueKind) const

Return the number of instrumented sites for ValueKind.

static void setCSFlagInHash(uint64_t &FuncHash)

Instruction * AnnotatedInst