PGOInstrumentation.cpp Source File (original) (raw)

118#include

119#include

120#include

121#include

122#include

123#include

124#include

125#include

126#include <unordered_map>

127#include

128#include

129

130using namespace llvm;

133

134#define DEBUG_TYPE "pgo-instrumentation"

135

136 STATISTIC(NumOfPGOInstrument, "Number of edges instrumented.");

137 STATISTIC(NumOfPGOSelectInsts, "Number of select instruction instrumented.");

138 STATISTIC(NumOfPGOMemIntrinsics, "Number of mem intrinsics instrumented.");

140 STATISTIC(NumOfPGOBB, "Number of basic-blocks.");

141 STATISTIC(NumOfPGOSplit, "Number of critical edge splits.");

142 STATISTIC(NumOfPGOFunc, "Number of functions having valid profile counts.");

143 STATISTIC(NumOfPGOMismatch, "Number of functions having mismatch profile.");

144 STATISTIC(NumOfPGOMissing, "Number of functions without profile.");

145 STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentations.");

146 STATISTIC(NumOfCSPGOInstrument, "Number of edges instrumented in CSPGO.");

148 "Number of select instruction instrumented in CSPGO.");

150 "Number of mem intrinsics instrumented in CSPGO.");

151 STATISTIC(NumOfCSPGOEdge, "Number of edges in CSPGO.");

152 STATISTIC(NumOfCSPGOBB, "Number of basic-blocks in CSPGO.");

153 STATISTIC(NumOfCSPGOSplit, "Number of critical edge splits in CSPGO.");

155 "Number of functions having valid profile counts in CSPGO.");

157 "Number of functions having mismatch profile in CSPGO.");

158 STATISTIC(NumOfCSPGOMissing, "Number of functions without profile in CSPGO.");

159 STATISTIC(NumCoveredBlocks, "Number of basic blocks that were executed");

160

161

162

166 cl::desc("Specify the path of profile data file. This is "

167 "mainly for test purpose."));

171 cl::desc("Specify the path of profile remapping file. This is mainly for "

172 "test purpose."));

173

174

175

178 cl::desc("Disable Value Profiling"));

179

180

181

184 cl::desc("Max number of annotations for a single indirect "

185 "call callsite"));

186

187

188

191 cl::desc("Max number of precise value annotations for a single memop"

192 "intrinsic"));

193

194

195

198 cl::desc("Append function hash to the name of COMDAT function to avoid "

199 "function hash mismatch due to the preinliner"));

200

201namespace llvm {

202

203

206 cl::desc("Use this option to turn on/off "

207 "warnings about missing profile data for "

208 "functions."));

209

210

211

214 cl::desc("Use this option to turn off/on "

215 "warnings about profile cfg mismatch."));

216

217

218

219

222 cl::desc("The option is used to turn on/off "

223 "warnings about hash mismatch for comdat "

224 "or weak functions."));

225

226

229 cl::desc("Use this option to turn on/off SELECT "

230 "instruction instrumentation. "));

231

232

235 cl::desc("A boolean option to show CFG dag or text "

236 "with raw profile counts from "

237 "profile data. See also option "

238 "-pgo-view-counts. To limit graph "

239 "display to only one function, use "

240 "filtering option -view-bfi-func-name."),

244

245

248 cl::desc("Use this option to turn on/off "

249 "memory intrinsic size profiling."));

250

251

254 cl::desc("When this option is on, the annotated "

255 "branch probability will be emitted as "

256 "optimization remarks: -{Rpass|"

257 "pass-remarks}=pgo-instrumentation"));

258

261 cl::desc("Force to instrument function entry basicblock."));

262

266 cl::desc("Force to instrument loop entries."));

267

269 "pgo-function-entry-coverage", cl::Hidden,

271 "Use this option to enable function entry coverage instrumentation."));

272

274 "pgo-block-coverage",

275 cl::desc("Use this option to enable basic block coverage instrumentation"));

276

279 cl::desc("Create a dot file of CFGs with block "

280 "coverage inference information"));

281

283 "pgo-temporal-instrumentation",

284 cl::desc("Use this option to enable temporal instrumentation"));

285

288 cl::desc("Fix function entry count in profile use."));

289

292 cl::desc("Print out the non-match BFI count if a hot raw profile count "

293 "becomes non-hot, or a cold raw profile count becomes hot. "

294 "The print is enabled under -Rpass-analysis=pgo, or "

295 "internal option -pass-remarks-analysis=pgo."));

296

299 cl::desc("Print out mismatched BFI counts after setting profile metadata "

300 "The print is enabled under -Rpass-analysis=pgo, or "

301 "internal option -pass-remarks-analysis=pgo."));

302

305 cl::desc("Set the threshold for pgo-verify-bfi: only print out "

306 "mismatched BFI if the difference percentage is greater than "

307 "this value (in percentage)."));

308

311 cl::desc("Set the threshold for pgo-verify-bfi: skip the counts whose "

312 "profile count value is below."));

313

317 cl::desc("Trace the hash of the function with this name."));

318

320 "pgo-function-size-threshold", cl::Hidden,

321 cl::desc("Do not instrument functions smaller than this threshold."));

322

325 cl::desc("Do not instrument functions with the number of critical edges "

326 " greater than this threshold."));

327

330 cl::desc("For cold function instrumentation, skip instrumenting functions "

331 "whose entry count is above the given value."));

332

335 cl::desc("For cold function instrumentation, treat count unknown(e.g. "

336 "unprofiled) functions as cold."));

337

340 cl::desc("Enable cold function only instrumentation."));

341

343 "ctx-prof-skip-callsite-instr", cl::Hidden,

344 cl::desc("Do not instrument callsites to functions in this list. Intended "

345 "for testing."));

346

348

349

350

352

353

354

356

357

358

363}

364

365namespace {

366class FunctionInstrumenter final {

370 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;

374

376

377

378

379

380

381 bool isValueProfilingDisabled() const {

384 }

385

386 bool shouldInstrumentEntryBB() const {

389 }

390

392

393public:

394 FunctionInstrumenter(

396 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,

400 : M(M), F(F), TLI(TLI), ComdatMembers(ComdatMembers), BPI(BPI), BFI(BFI),

401 LI(LI), InstrumentationType(InstrumentationType) {}

402

404};

405}

406

407

408

412 return std::string();

413

416 if (!CI)

417 return std::string();

418

419 std::string result;

423

426 if (CV) {

428 OS << "_Zero";

429 else if (CV->isOne())

430 OS << "_One";

432 OS << "_MinusOne";

433 else

434 OS << "_Const";

435 }

436 return result;

437}

438

440#define VALUE_PROF_KIND(Enumerator, Value, Descr) Descr,

442};

443

444

445

462 ProfileVersion |=

473 IRLevelVersionVariable->setVisibility(

475

476 Triple TT(M.getTargetTriple());

477 if (TT.supportsCOMDAT()) {

479 IRLevelVersionVariable->setComdat(M.getOrInsertComdat(VarName));

480 }

481 return IRLevelVersionVariable;

482}

483

484namespace {

485

486

487

488

489

490

491enum VisitMode { VM_counting, VM_instrument, VM_annotate };

492class PGOUseFunc;

493

494

495struct SelectInstVisitor : public InstVisitor {

497 unsigned NSIs = 0;

498 VisitMode Mode = VM_counting;

499 unsigned *CurCtrIdx = nullptr;

500 unsigned TotalNumCtrs = 0;

501 GlobalValue *FuncNameVar = nullptr;

502 uint64_t FuncHash = 0;

503 PGOUseFunc *UseFunc = nullptr;

504 bool HasSingleByteCoverage;

505

506 SelectInstVisitor(Function &Func, bool HasSingleByteCoverage)

507 : F(Func), HasSingleByteCoverage(HasSingleByteCoverage) {}

508

509 void countSelects() {

510 NSIs = 0;

511 Mode = VM_counting;

513 }

514

515

516

517

518

519 void instrumentSelects(unsigned *Ind, unsigned TotalNC, GlobalValue *FNV,

520 uint64_t FHash) {

521 Mode = VM_instrument;

522 CurCtrIdx = Ind;

523 TotalNumCtrs = TotalNC;

524 FuncHash = FHash;

525 FuncNameVar = FNV;

527 }

528

529

530 void annotateSelects(PGOUseFunc *UF, unsigned *Ind) {

531 Mode = VM_annotate;

532 UseFunc = UF;

533 CurCtrIdx = Ind;

535 }

536

537 void instrumentOneSelectInst(SelectInst &SI);

538 void annotateOneSelectInst(SelectInst &SI);

539

540

541 void visitSelectInst(SelectInst &SI);

542

543

544

545 unsigned getNumOfSelectInsts() const { return NSIs; }

546};

547

548

549

550

551

552struct PGOEdge {

555 uint64_t Weight;

556 bool InMST = false;

557 bool Removed = false;

558 bool IsCritical = false;

559

560 PGOEdge(BasicBlock *Src, BasicBlock *Dest, uint64_t W = 1)

561 : SrcBB(Src), DestBB(Dest), Weight(W) {}

562

563

564 std::string infoString() const {

565 return (Twine(Removed ? "-" : " ") + (InMST ? " " : "*") +

566 (IsCritical ? "c" : " ") + " W=" + Twine(Weight))

567 .str();

568 }

569};

570

571

572struct PGOBBInfo {

573 PGOBBInfo *Group;

574 uint32_t Index;

575 uint32_t Rank = 0;

576

577 PGOBBInfo(unsigned IX) : Group(this), Index(IX) {}

578

579

580 std::string infoString() const {

581 return (Twine("Index=") + Twine(Index)).str();

582 }

583};

584

585

586template <class Edge, class BBInfo> class FuncPGOInstrumentation {

587private:

589

590

591 bool IsCS;

592

593

594 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;

595

596 ValueProfileCollector VPC;

597

598 void computeCFGHash();

599 void renameComdatFunction();

600

601public:

602 const TargetLibraryInfo &TLI;

603 std::vector<std::vector> ValueSites;

604 SelectInstVisitor SIVisitor;

605 std::string FuncName;

606 std::string DeprecatedFuncName;

607 GlobalVariable *FuncNameVar;

608

609

610 uint64_t FunctionHash = 0;

611

612

613 CFGMST<Edge, BBInfo> MST;

614

615 const std::optional BCI;

616

617 static std::optional

618 constructBCI(Function &Func, bool HasSingleByteCoverage,

619 bool InstrumentFuncEntry) {

620 if (HasSingleByteCoverage)

621 return BlockCoverageInference(Func, InstrumentFuncEntry);

622 return {};

623 }

624

625

626

627 void getInstrumentBBs(std::vector<BasicBlock *> &InstrumentBBs);

628

629

630

632

633

634 BBInfo &getBBInfo(const BasicBlock *BB) const { return MST.getBBInfo(BB); }

635

636

637 BBInfo *findBBInfo(const BasicBlock *BB) const { return MST.findBBInfo(BB); }

638

639

640 void dumpInfo(StringRef Str = "") const {

641 MST.dumpEdges(dbgs(), Twine("Dump Function ") + FuncName +

642 " Hash: " + Twine(FunctionHash) + "\t" + Str);

643 }

644

645 FuncPGOInstrumentation(

646 Function &Func, TargetLibraryInfo &TLI,

647 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,

648 bool CreateGlobalVar = false, BranchProbabilityInfo *BPI = nullptr,

649 BlockFrequencyInfo *BFI = nullptr, LoopInfo *LI = nullptr,

650 bool IsCS = false, bool InstrumentFuncEntry = true,

651 bool InstrumentLoopEntries = false, bool HasSingleByteCoverage = false)

652 : F(Func), IsCS(IsCS), ComdatMembers(ComdatMembers), VPC(Func, TLI),

653 TLI(TLI), ValueSites(IPVK_Last + 1),

654 SIVisitor(Func, HasSingleByteCoverage),

655 MST(F, InstrumentFuncEntry, InstrumentLoopEntries, BPI, BFI, LI),

656 BCI(constructBCI(Func, HasSingleByteCoverage, InstrumentFuncEntry)) {

658 BCI->viewBlockCoverageGraph();

659

660 SIVisitor.countSelects();

661 ValueSites[IPVK_MemOPSize] = VPC.get(IPVK_MemOPSize);

662 if (!IsCS) {

663 NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts();

664 NumOfPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();

665 NumOfPGOBB += MST.bbInfoSize();

666 ValueSites[IPVK_IndirectCallTarget] = VPC.get(IPVK_IndirectCallTarget);

668 ValueSites[IPVK_VTableTarget] = VPC.get(IPVK_VTableTarget);

669 } else {

670 NumOfCSPGOSelectInsts += SIVisitor.getNumOfSelectInsts();

671 NumOfCSPGOMemIntrinsics += ValueSites[IPVK_MemOPSize].size();

672 NumOfCSPGOBB += MST.bbInfoSize();

673 }

674

677 computeCFGHash();

678 if (!ComdatMembers.empty())

679 renameComdatFunction();

680 LLVM_DEBUG(dumpInfo("after CFGMST"));

681

682 for (const auto &E : MST.allEdges()) {

683 if (E->Removed)

684 continue;

685 IsCS ? NumOfCSPGOEdge++ : NumOfPGOEdge++;

686 if (->InMST)

687 IsCS ? NumOfCSPGOInstrument++ : NumOfPGOInstrument++;

688 }

689

690 if (CreateGlobalVar)

692 }

693};

694

695}

696

697

698

699

700template <class Edge, class BBInfo>

701void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {

702 std::vector<uint8_t> Indexes;

704 for (auto &BB : F) {

706 auto BI = findBBInfo(Succ);

707 if (BI == nullptr)

708 continue;

710 for (int J = 0; J < 4; J++)

711 Indexes.push_back((uint8_t)(Index >> (J * 8)));

712 }

713 }

715

717

718 auto updateJCH = [&JCH](uint64_t Num) {

722 };

723 updateJCH((uint64_t)SIVisitor.getNumOfSelectInsts());

724 updateJCH((uint64_t)ValueSites[IPVK_IndirectCallTarget].size());

725 updateJCH((uint64_t)ValueSites[IPVK_MemOPSize].size());

726 if (BCI) {

727 updateJCH(BCI->getInstrumentedBlocksHash());

728 } else {

730 }

731

732

733

735

736

738 if (IsCS)

740 LLVM_DEBUG(dbgs() << "Function Hash Computation for " << F.getName() << ":\n"

741 << " CRC = " << JC.getCRC()

742 << ", Selects = " << SIVisitor.getNumOfSelectInsts()

743 << ", Edges = " << MST.numEdges() << ", ICSites = "

744 << ValueSites[IPVK_IndirectCallTarget].size()

745 << ", Memops = " << ValueSites[IPVK_MemOPSize].size()

746 << ", High32 CRC = " << JCH.getCRC()

747 << ", Hash = " << FunctionHash << "\n";);

748

750 dbgs() << "Funcname=" << F.getName() << ", Hash=" << FunctionHash

751 << " in building " << F.getParent()->getSourceFileName() << "\n";

752}

753

754

757 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {

759 return false;

760

761

762

763

764

765

766

767

769 for (auto &&CM : make_range(ComdatMembers.equal_range(C))) {

772 if (FM != &F)

773 return false;

774 }

775 return true;

776}

777

778

779template <class Edge, class BBInfo>

780void FuncPGOInstrumentation<Edge, BBInfo>::renameComdatFunction() {

782 return;

783 std::string OrigName = F.getName().str();

784 std::string NewFuncName =

785 Twine(F.getName() + "." + Twine(FunctionHash)).str();

786 F.setName(Twine(NewFuncName));

788 FuncName = Twine(FuncName + "." + Twine(FunctionHash)).str();

790 Module *M = F.getParent();

791

792

793

794 if (.hasComdat()) {

796 NewComdat = M->getOrInsertComdat(StringRef(NewFuncName));

798 F.setComdat(NewComdat);

799 return;

800 }

801

802

803 Comdat *OrigComdat = F.getComdat();

804 std::string NewComdatName =

806 NewComdat = M->getOrInsertComdat(StringRef(NewComdatName));

808

809 for (auto &&CM : make_range(ComdatMembers.equal_range(OrigComdat))) {

810

812 }

813}

814

815

816

817template <class Edge, class BBInfo>

818void FuncPGOInstrumentation<Edge, BBInfo>::getInstrumentBBs(

819 std::vector<BasicBlock *> &InstrumentBBs) {

820 if (BCI) {

821 for (auto &BB : F)

822 if (BCI->shouldInstrumentBlock(BB))

823 InstrumentBBs.push_back(&BB);

824 return;

825 }

826

827

828 std::vector<Edge *> EdgeList;

829 EdgeList.reserve(MST.numEdges());

830 for (const auto &E : MST.allEdges())

831 EdgeList.push_back(E.get());

832

833 for (auto &E : EdgeList) {

835 if (InstrBB)

836 InstrumentBBs.push_back(InstrBB);

837 }

838}

839

840

841

842template <class Edge, class BBInfo>

843BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) {

844 if (E->InMST || E->Removed)

845 return nullptr;

846

849

850 if (SrcBB == nullptr)

851 return DestBB;

852 if (DestBB == nullptr)

853 return SrcBB;

854

856

857

859 return nullptr;

860 return BB;

861 };

862

863

864

867 return canInstrument(SrcBB);

868 if (->IsCritical)

869 return canInstrument(DestBB);

870

871

872

876 if (!InstrBB) {

878 dbgs() << "Fail to split critical edge: not instrument this edge.\n");

879 return nullptr;

880 }

881

882

883 IsCS ? NumOfCSPGOSplit++ : NumOfPGOSplit++;

884 LLVM_DEBUG(dbgs() << "Split critical edge: " << getBBInfo(SrcBB).Index

885 << " --> " << getBBInfo(DestBB).Index << "\n");

886

887 MST.addEdge(SrcBB, InstrBB, 0);

888

889 Edge &NewEdge1 = MST.addEdge(InstrBB, DestBB, 0);

890 NewEdge1.InMST = true;

891 E->Removed = true;

892

893 return canInstrument(InstrBB);

894}

895

896

897

898

899

900

901static void

906 if (!OrigCall)

907 return;

908

910

911

912 std::optional ParentFunclet =

914 if (ParentFunclet)

916 } else {

917

918

919

920 if (!BlockColors.empty()) {

921 const ColorVector &CV = BlockColors.find(OrigCall->getParent())->second;

922 assert(CV.size() == 1 && "non-unique color for block!");

924 if (EHPadIt->isEHPad())

925 OpBundles.emplace_back("funclet", &*EHPadIt);

926 }

927 }

928}

929

930

931

932void FunctionInstrumenter::instrument() {

934

935

937 }

938

939 const bool IsCtxProf = InstrumentationType == PGOInstrumentationType::CTXPROF;

940 FuncPGOInstrumentation<PGOEdge, PGOBBInfo> FuncInfo(

941 F, TLI, ComdatMembers, !IsCtxProf, BPI, BFI, LI,

942 InstrumentationType == PGOInstrumentationType::CSFDO,

943 shouldInstrumentEntryBB(), shouldInstrumentLoopEntries(),

945

947 auto *const CFGHash =

948 ConstantInt::get(Type::getInt64Ty(M.getContext()), FuncInfo.FunctionHash);

949

950

952 Name, PointerType::get(M.getContext(), 0));

954 auto &EntryBB = F.getEntryBlock();

955 IRBuilder<> Builder(&EntryBB, EntryBB.getFirstNonPHIOrDbgOrAlloca());

956

957

958 Builder.CreateIntrinsic(

959 Intrinsic::instrprof_cover,

960 {NormalizedNamePtr, CFGHash, Builder.getInt32(1), Builder.getInt32(0)});

961 return;

962 }

963

964 std::vector<BasicBlock *> InstrumentBBs;

965 FuncInfo.getInstrumentBBs(InstrumentBBs);

967 InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();

968

969 if (IsCtxProf) {

971

972 auto *CSIntrinsic =

974

975

976

977

978

979

980

981

982 auto Visit = [&](llvm::function_ref<void(CallBase * CB)> Visitor) {

983 for (auto &BB : F)

984 for (auto &Instr : BB)

987 continue;

988 if (CS->getCalledFunction() &&

989 SkipCSInstr.contains(CS->getCalledFunction()->getName()))

990 continue;

991 Visitor(CS);

992 }

993 };

994

995 uint32_t TotalNumCallsites = 0;

996 Visit([&TotalNumCallsites](auto *) { ++TotalNumCallsites; });

997

998

1000 Visit([&](auto *CB) {

1002 Builder.CreateCall(CSIntrinsic,

1003 {Name, CFGHash, Builder.getInt32(TotalNumCallsites),

1005 CB->getCalledOperand()});

1006 });

1007 }

1008

1009 uint32_t I = 0;

1012 auto &EntryBB = F.getEntryBlock();

1013 IRBuilder<> Builder(&EntryBB, EntryBB.getFirstNonPHIOrDbgOrAlloca());

1014

1015

1016 Builder.CreateIntrinsic(Intrinsic::instrprof_timestamp,

1017 {NormalizedNamePtr, CFGHash,

1019 Builder.getInt32(I)});

1021 }

1022

1023 for (auto *InstrBB : InstrumentBBs) {

1025 assert(Builder.GetInsertPoint() != InstrBB->end() &&

1026 "Cannot get the Instrumentation point");

1027

1028

1029 Builder.CreateIntrinsic(PGOBlockCoverage ? Intrinsic::instrprof_cover

1030 : Intrinsic::instrprof_increment,

1031 {NormalizedNamePtr, CFGHash,

1033 Builder.getInt32(I++)});

1034 }

1035

1036

1037 FuncInfo.SIVisitor.instrumentSelects(&I, NumCounters, Name,

1038 FuncInfo.FunctionHash);

1040

1041 if (isValueProfilingDisabled())

1042 return;

1043

1044 NumOfPGOICall += FuncInfo.ValueSites[IPVK_IndirectCallTarget].size();

1045

1046

1047

1048

1049

1050 DenseMap<BasicBlock *, ColorVector> BlockColors;

1051 if (F.hasPersonalityFn() &&

1054

1055

1056 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {

1057 unsigned SiteIndex = 0;

1059 continue;

1060

1061 for (VPCandidateInfo Cand : FuncInfo.ValueSites[Kind]) {

1063 << " site: CallSite Index = " << SiteIndex << "\n");

1064

1066 assert(Builder.GetInsertPoint() != Cand.InsertPt->getParent()->end() &&

1067 "Cannot get the Instrumentation point");

1068

1069 Value *ToProfile = nullptr;

1070 if (Cand.V->getType()->isIntegerTy())

1071 ToProfile = Builder.CreateZExtOrTrunc(Cand.V, Builder.getInt64Ty());

1072 else if (Cand.V->getType()->isPointerTy())

1073 ToProfile = Builder.CreatePtrToInt(Cand.V, Builder.getInt64Ty());

1074 assert(ToProfile && "value profiling Value is of unexpected type");

1075

1077 Name, PointerType::get(M.getContext(), 0));

1078

1081 Builder.CreateCall(

1083 Intrinsic::instrprof_value_profile),

1084 {NormalizedNamePtr, Builder.getInt64(FuncInfo.FunctionHash),

1085 ToProfile, Builder.getInt32(Kind), Builder.getInt32(SiteIndex++)},

1086 OpBundles);

1087 }

1088 }

1089}

1090

1091namespace {

1092

1093

1094struct PGOUseEdge : public PGOEdge {

1095 using PGOEdge::PGOEdge;

1096

1097 std::optional<uint64_t> Count;

1098

1099

1101

1102

1103 std::string infoString() const {

1105 return PGOEdge::infoString();

1106 return (Twine(PGOEdge::infoString()) + " Count=" + Twine(*Count)).str();

1107 }

1108};

1109

1111

1112

1113struct PGOUseBBInfo : public PGOBBInfo {

1114 std::optional<uint64_t> Count;

1115 int32_t UnknownCountInEdge = 0;

1116 int32_t UnknownCountOutEdge = 0;

1117 DirectEdges InEdges;

1118 DirectEdges OutEdges;

1119

1120 PGOUseBBInfo(unsigned IX) : PGOBBInfo(IX) {}

1121

1122

1124

1125

1126 std::string infoString() const {

1128 return PGOBBInfo::infoString();

1129 return (Twine(PGOBBInfo::infoString()) + " Count=" + Twine(*Count)).str();

1130 }

1131

1132

1133 void addOutEdge(PGOUseEdge *E) {

1134 OutEdges.push_back(E);

1135 UnknownCountOutEdge++;

1136 }

1137

1138

1139 void addInEdge(PGOUseEdge *E) {

1140 InEdges.push_back(E);

1141 UnknownCountInEdge++;

1142 }

1143};

1144

1145}

1146

1147

1150 for (const auto &E : Edges) {

1151 if (E->Removed)

1152 continue;

1153 if (E->Count)

1155 }

1157}

1158

1159namespace {

1160

1161class PGOUseFunc {

1162public:

1163 PGOUseFunc(Function &Func, Module *Modu, TargetLibraryInfo &TLI,

1164 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,

1165 BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFIin,

1166 LoopInfo *LI, ProfileSummaryInfo *PSI, bool IsCS,

1167 bool InstrumentFuncEntry, bool InstrumentLoopEntries,

1168 bool HasSingleByteCoverage)

1169 : F(Func), M(Modu), BFI(BFIin), PSI(PSI),

1170 FuncInfo(Func, TLI, ComdatMembers, false, BPI, BFIin, LI, IsCS,

1171 InstrumentFuncEntry, InstrumentLoopEntries,

1172 HasSingleByteCoverage),

1173 FreqAttr(FFA_Normal), IsCS(IsCS), VPC(Func, TLI) {}

1174

1175 void handleInstrProfError(Error Err, uint64_t MismatchedFuncSum);

1176

1177

1178

1179

1180 bool getRecord(IndexedInstrProfReader *PGOReader);

1181

1182

1183 bool readCounters(bool &AllZeros,

1185

1186

1187 void populateCounters();

1188

1189

1190 void populateCoverage();

1191

1192

1194

1195

1196 void annotateValueSites();

1197

1198

1199 void annotateValueSites(uint32_t Kind);

1200

1201

1202 void annotateIrrLoopHeaderWeights();

1203

1204

1205 enum FuncFreqAttr { FFA_Normal, FFA_Cold, FFA_Hot };

1206

1207

1208 FuncFreqAttr getFuncFreqAttr() const { return FreqAttr; }

1209

1210

1211 uint64_t getFuncHash() const { return FuncInfo.FunctionHash; }

1212

1213

1214 NamedInstrProfRecord &getProfileRecord() { return ProfileRecord; }

1215

1216

1217 PGOUseBBInfo &getBBInfo(const BasicBlock *BB) const {

1218 return FuncInfo.getBBInfo(BB);

1219 }

1220

1221

1222 PGOUseBBInfo *findBBInfo(const BasicBlock *BB) const {

1223 return FuncInfo.findBBInfo(BB);

1224 }

1225

1226 Function &getFunc() const { return F; }

1227

1228 void dumpInfo(StringRef Str = "") const { FuncInfo.dumpInfo(Str); }

1229

1230 uint64_t getProgramMaxCount() const { return ProgramMaxCount; }

1231

1232private:

1235 BlockFrequencyInfo *BFI;

1236 ProfileSummaryInfo *PSI;

1237

1238

1239 FuncPGOInstrumentation<PGOUseEdge, PGOUseBBInfo> FuncInfo;

1240

1241

1242

1243 uint64_t ProgramMaxCount;

1244

1245

1246 uint32_t CountPosition = 0;

1247

1248

1249 uint32_t ProfileCountSize = 0;

1250

1251

1252 NamedInstrProfRecord ProfileRecord;

1253

1254

1255 FuncFreqAttr FreqAttr;

1256

1257

1258 bool IsCS;

1259

1260 ValueProfileCollector VPC;

1261

1262

1263 bool setInstrumentedCounts(const std::vector<uint64_t> &CountFromProfile);

1264

1265

1266

1267 void setEdgeCount(DirectEdges &Edges, uint64_t Value);

1268

1269

1270

1271

1272 void markFunctionAttributes(uint64_t EntryCount, uint64_t MaxCount) {

1274 FreqAttr = FFA_Hot;

1276 FreqAttr = FFA_Cold;

1277 }

1278};

1279

1280}

1281

1282

1284 const FuncPGOInstrumentation<PGOUseEdge, PGOUseBBInfo> &FuncInfo) {

1285

1286 if (FuncInfo.BCI)

1287 return;

1288 for (const auto &E : FuncInfo.MST.allEdges()) {

1289 if (E->Removed)

1290 continue;

1293 PGOUseBBInfo &SrcInfo = FuncInfo.getBBInfo(SrcBB);

1294 PGOUseBBInfo &DestInfo = FuncInfo.getBBInfo(DestBB);

1295 SrcInfo.addOutEdge(E.get());

1296 DestInfo.addInEdge(E.get());

1297 }

1298}

1299

1300

1301

1302bool PGOUseFunc::setInstrumentedCounts(

1303 const std::vector<uint64_t> &CountFromProfile) {

1304

1305 std::vector<BasicBlock *> InstrumentBBs;

1306 FuncInfo.getInstrumentBBs(InstrumentBBs);

1307

1309

1311 InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();

1312

1313

1314 if (NumCounters != CountFromProfile.size()) {

1315 return false;

1316 }

1317 auto *FuncEntry = &*F.begin();

1318

1319

1320 uint32_t I = 0;

1321 for (BasicBlock *InstrBB : InstrumentBBs) {

1322 uint64_t CountValue = CountFromProfile[I++];

1323 PGOUseBBInfo &Info = getBBInfo(InstrBB);

1324

1325

1326

1327 if (InstrBB == FuncEntry && CountValue == 0)

1328 CountValue = 1;

1329 Info.setBBInfoCount(CountValue);

1330 }

1331 ProfileCountSize = CountFromProfile.size();

1332 CountPosition = I;

1333

1334

1335 auto setEdgeCount = [this](PGOUseEdge *E, uint64_t Value) -> void {

1336 E->setEdgeCount(Value);

1337 this->getBBInfo(E->SrcBB).UnknownCountOutEdge--;

1338 this->getBBInfo(E->DestBB).UnknownCountInEdge--;

1339 };

1340

1341

1342

1343

1344 for (const auto &E : FuncInfo.MST.allEdges()) {

1345 if (E->Removed || E->InMST)

1346 continue;

1348 PGOUseBBInfo &SrcInfo = getBBInfo(SrcBB);

1349

1350

1351

1352 if (SrcInfo.Count && SrcInfo.OutEdges.size() == 1)

1353 setEdgeCount(E.get(), *SrcInfo.Count);

1354 else {

1356 PGOUseBBInfo &DestInfo = getBBInfo(DestBB);

1357

1358

1359 if (DestInfo.Count && DestInfo.InEdges.size() == 1)

1360 setEdgeCount(E.get(), *DestInfo.Count);

1361 }

1362 if (E->Count)

1363 continue;

1364

1365

1366 setEdgeCount(E.get(), 0);

1367 }

1368 return true;

1369}

1370

1371

1372

1373void PGOUseFunc::setEdgeCount(DirectEdges &Edges, uint64_t Value) {

1374 for (auto &E : Edges) {

1375 if (E->Count)

1376 continue;

1377 E->setEdgeCount(Value);

1378

1379 getBBInfo(E->SrcBB).UnknownCountOutEdge--;

1380 getBBInfo(E->DestBB).UnknownCountInEdge--;

1381 return;

1382 }

1384}

1385

1386

1388 const char MetadataName[] = "instr_prof_hash_mismatch";

1390

1391 auto *Existing = F.getMetadata(LLVMContext::MD_annotation);

1392 if (Existing) {

1394 for (const auto &N : Tuple->operands()) {

1395 if (N.equalsStr(MetadataName))

1396 return;

1398 }

1399 }

1400

1404 F.setMetadata(LLVMContext::MD_annotation, MD);

1405}

1406

1407void PGOUseFunc::handleInstrProfError(Error Err, uint64_t MismatchedFuncSum) {

1408 handleAllErrors(std::move(Err), [&](const InstrProfError &IPE) {

1409 auto &Ctx = M->getContext();

1410 auto Err = IPE.get();

1411 bool SkipWarning = false;

1412 LLVM_DEBUG(dbgs() << "Error in reading profile for Func "

1413 << FuncInfo.FuncName << ": ");

1414 if (Err == instrprof_error::unknown_function) {

1415 IsCS ? NumOfCSPGOMissing++ : NumOfPGOMissing++;

1418 } else if (Err == instrprof_error::hash_mismatch ||

1419 Err == instrprof_error::malformed) {

1420 IsCS ? NumOfCSPGOMismatch++ : NumOfPGOMismatch++;

1421 SkipWarning =

1426 LLVM_DEBUG(dbgs() << "hash mismatch (hash= " << FuncInfo.FunctionHash

1427 << " skip=" << SkipWarning << ")");

1428

1430 }

1431

1433 if (SkipWarning)

1434 return;

1435

1436 std::string Msg =

1437 IPE.message() + std::string(" ") + F.getName().str() +

1438 std::string(" Hash = ") + std::to_string(FuncInfo.FunctionHash) +

1439 std::string(" up to ") + std::to_string(MismatchedFuncSum) +

1440 std::string(" count discarded");

1441

1442 Ctx.diagnose(

1443 DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));

1444 });

1445}

1446

1447bool PGOUseFunc::getRecord(IndexedInstrProfReader *PGOReader) {

1448 uint64_t MismatchedFuncSum = 0;

1450 FuncInfo.FuncName, FuncInfo.FunctionHash, FuncInfo.DeprecatedFuncName,

1451 &MismatchedFuncSum);

1453 handleInstrProfError(std::move(E), MismatchedFuncSum);

1454 return false;

1455 }

1456 ProfileRecord = std::move(Result.get());

1458 return true;

1459}

1460

1461

1462

1463

1464bool PGOUseFunc::readCounters(bool &AllZeros,

1466 auto &Ctx = M->getContext();

1469 return true;

1470 }

1471 std::vector<uint64_t> &CountFromProfile = ProfileRecord.Counts;

1472

1473 IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;

1474 LLVM_DEBUG(dbgs() << CountFromProfile.size() << " counts\n");

1475

1476 uint64_t ValueSum = 0;

1477 for (unsigned I = 0, S = CountFromProfile.size(); I < S; I++) {

1478 LLVM_DEBUG(dbgs() << " " << I << ": " << CountFromProfile[I] << "\n");

1479 ValueSum += CountFromProfile[I];

1480 }

1481 AllZeros = (ValueSum == 0);

1482

1484

1485 getBBInfo(nullptr).UnknownCountOutEdge = 2;

1486 getBBInfo(nullptr).UnknownCountInEdge = 2;

1487

1488 if (!setInstrumentedCounts(CountFromProfile)) {

1490 dbgs() << "Inconsistent number of counts, skipping this function");

1491 Ctx.diagnose(DiagnosticInfoPGOProfile(

1492 M->getName().data(),

1493 Twine("Inconsistent number of counts in ") + F.getName().str() +

1494 Twine(": the profile may be stale or there is a function name "

1495 "collision."),

1497 return false;

1498 }

1499 return true;

1500}

1501

1502void PGOUseFunc::populateCoverage() {

1503 IsCS ? NumOfCSPGOFunc++ : NumOfPGOFunc++;

1504

1505 ArrayRef<uint64_t> CountsFromProfile = ProfileRecord.Counts;

1506 DenseMap<const BasicBlock *, bool> Coverage;

1507 unsigned Index = 0;

1508 for (auto &BB : F)

1509 if (FuncInfo.BCI->shouldInstrumentBlock(BB))

1510 Coverage[&BB] = (CountsFromProfile[Index++] != 0);

1511 assert(Index == CountsFromProfile.size());

1512

1513

1514 DenseMap<const BasicBlock *, DenseSet<const BasicBlock *>>

1515 InverseDependencies;

1516 for (auto &BB : F) {

1517 for (auto *Dep : FuncInfo.BCI->getDependencies(BB)) {

1518

1519 InverseDependencies[Dep].insert(&BB);

1520 }

1521 }

1522

1523

1524 std::stack<const BasicBlock *> CoveredBlocksToProcess;

1525 for (auto &[BB, IsCovered] : Coverage)

1526 if (IsCovered)

1527 CoveredBlocksToProcess.push(BB);

1528

1529 while (!CoveredBlocksToProcess.empty()) {

1530 auto *CoveredBlock = CoveredBlocksToProcess.top();

1531 assert(Coverage[CoveredBlock]);

1532 CoveredBlocksToProcess.pop();

1533 for (auto *BB : InverseDependencies[CoveredBlock]) {

1534

1536 if (Cov)

1537 continue;

1538 Cov = true;

1539 CoveredBlocksToProcess.push(BB);

1540 }

1541 }

1542

1543

1544 MDBuilder MDB(F.getContext());

1545

1546

1547 F.setEntryCount(Coverage[&F.getEntryBlock()] ? 10000 : 0);

1548 for (auto &BB : F) {

1549

1550

1551

1552

1553

1554

1555 SmallVector<uint32_t, 4> Weights;

1557 Weights.push_back((Coverage[Succ] || !Coverage[&BB]) ? 1 : 0);

1558 if (Weights.size() >= 2)

1560 false);

1561 }

1562

1563 unsigned NumCorruptCoverage = 0;

1564 DominatorTree DT(F);

1565 LoopInfo LI(DT);

1566 BranchProbabilityInfo BPI(F, LI);

1567 BlockFrequencyInfo BFI(F, BPI, LI);

1568 auto IsBlockDead = [&](const BasicBlock &BB) -> std::optional {

1570 return C == 0;

1571 return {};

1572 };

1573 LLVM_DEBUG(dbgs() << "Block Coverage: (Instrumented=*, Covered=X)\n");

1574 for (auto &BB : F) {

1575 LLVM_DEBUG(dbgs() << (FuncInfo.BCI->shouldInstrumentBlock(BB) ? "* " : " ")

1576 << (Coverage[&BB] ? "X " : " ") << " " << BB.getName()

1577 << "\n");

1578

1579

1580

1581

1582 const bool &Cov = Coverage[&BB];

1583 if (Cov == IsBlockDead(BB).value_or(false)) {

1585 dbgs() << "Found inconsistent block covearge for " << BB.getName()

1586 << ": BCI=" << (Cov ? "Covered" : "Dead") << " BFI="

1587 << (IsBlockDead(BB).value() ? "Dead" : "Covered") << "\n");

1588 ++NumCorruptCoverage;

1589 }

1590 if (Cov)

1591 ++NumCoveredBlocks;

1592 }

1594 auto &Ctx = M->getContext();

1595 Ctx.diagnose(DiagnosticInfoPGOProfile(

1596 M->getName().data(),

1597 Twine("Found inconsistent block coverage for function ") + F.getName() +

1598 " in " + Twine(NumCorruptCoverage) + " blocks.",

1600 }

1602 FuncInfo.BCI->viewBlockCoverageGraph(&Coverage);

1603}

1604

1605

1606

1607void PGOUseFunc::populateCounters() {

1608 bool Changes = true;

1609 unsigned NumPasses = 0;

1610 while (Changes) {

1611 NumPasses++;

1612 Changes = false;

1613

1614

1615

1616 for (auto &BB : reverse(F)) {

1617 PGOUseBBInfo *UseBBInfo = findBBInfo(&BB);

1618 if (UseBBInfo == nullptr)

1619 continue;

1620 if (!UseBBInfo->Count) {

1621 if (UseBBInfo->UnknownCountOutEdge == 0) {

1622 UseBBInfo->Count = sumEdgeCount(UseBBInfo->OutEdges);

1623 Changes = true;

1624 } else if (UseBBInfo->UnknownCountInEdge == 0) {

1625 UseBBInfo->Count = sumEdgeCount(UseBBInfo->InEdges);

1626 Changes = true;

1627 }

1628 }

1629 if (UseBBInfo->Count) {

1630 if (UseBBInfo->UnknownCountOutEdge == 1) {

1631 uint64_t Total = 0;

1632 uint64_t OutSum = sumEdgeCount(UseBBInfo->OutEdges);

1633

1634

1635

1636 if (*UseBBInfo->Count > OutSum)

1637 Total = *UseBBInfo->Count - OutSum;

1638 setEdgeCount(UseBBInfo->OutEdges, Total);

1639 Changes = true;

1640 }

1641 if (UseBBInfo->UnknownCountInEdge == 1) {

1642 uint64_t Total = 0;

1643 uint64_t InSum = sumEdgeCount(UseBBInfo->InEdges);

1644 if (*UseBBInfo->Count > InSum)

1645 Total = *UseBBInfo->Count - InSum;

1646 setEdgeCount(UseBBInfo->InEdges, Total);

1647 Changes = true;

1648 }

1649 }

1650 }

1651 }

1652

1653 LLVM_DEBUG(dbgs() << "Populate counts in " << NumPasses << " passes.\n");

1654 (void)NumPasses;

1655#ifndef NDEBUG

1656

1657 for (auto &BB : F) {

1658 auto BI = findBBInfo(&BB);

1659 if (BI == nullptr)

1660 continue;

1661 assert(BI->Count && "BB count is not valid");

1662 }

1663#endif

1664

1665 FuncInfo.SIVisitor.annotateSelects(this, &CountPosition);

1666 assert(CountPosition == ProfileCountSize);

1667

1668 uint64_t FuncEntryCount = *getBBInfo(&*F.begin()).Count;

1670 for (auto &BB : F) {

1671 auto BI = findBBInfo(&BB);

1672 if (BI == nullptr)

1673 continue;

1674 FuncMaxCount = std::max(FuncMaxCount, *BI->Count);

1675 }

1676

1677

1681 markFunctionAttributes(FuncEntryCount, FuncMaxCount);

1682

1683 LLVM_DEBUG(FuncInfo.dumpInfo("after reading profile."));

1684}

1685

1686

1687void PGOUseFunc::setBranchWeights() {

1688

1689 LLVM_DEBUG(dbgs() << "\nSetting branch weights for func " << F.getName()

1690 << " IsCS=" << IsCS << "\n");

1691 for (auto &BB : F) {

1694 continue;

1698 continue;

1699

1700 const PGOUseBBInfo &BBCountInfo = getBBInfo(&BB);

1701 if (!*BBCountInfo.Count)

1702 continue;

1703

1704

1705

1706

1707

1708 unsigned OutEdgesCount = BBCountInfo.OutEdges.size();

1709 unsigned SuccessorCount = BB.getTerminator()->getNumSuccessors();

1710 assert(OutEdgesCount <= SuccessorCount);

1711

1713 uint64_t MaxCount = 0;

1714 for (unsigned It = 0; It < OutEdgesCount; It++) {

1715 const PGOUseEdge *E = BBCountInfo.OutEdges[It];

1718 if (DestBB == nullptr)

1719 continue;

1721 uint64_t EdgeCount = *E->Count;

1722 if (EdgeCount > MaxCount)

1723 MaxCount = EdgeCount;

1724 EdgeCounts[SuccNum] = EdgeCount;

1725 }

1726

1727 if (MaxCount)

1729 else {

1730

1731

1732

1733 auto &Ctx = M->getContext();

1734 Ctx.diagnose(DiagnosticInfoPGOProfile(

1735 M->getName().data(),

1736 Twine("Profile in ") + F.getName().str() +

1737 Twine(" partially ignored") +

1738 Twine(", possibly due to the lack of a return path."),

1740 }

1741 }

1742}

1743

1747 return true;

1748 }

1749 return false;

1750}

1751

1752void PGOUseFunc::annotateIrrLoopHeaderWeights() {

1753 LLVM_DEBUG(dbgs() << "\nAnnotating irreducible loop header weights.\n");

1754

1755 for (auto &BB : F) {

1756

1757

1758

1761 const PGOUseBBInfo &BBCountInfo = getBBInfo(&BB);

1763 }

1764 }

1765}

1766

1767void SelectInstVisitor::instrumentOneSelectInst(SelectInst &SI) {

1770 Type *Int64Ty = Builder.getInt64Ty();

1771 auto *Step = Builder.CreateZExt(SI.getCondition(), Int64Ty);

1772 auto *NormalizedFuncNameVarPtr =

1774 FuncNameVar, PointerType::get(M->getContext(), 0));

1775 Builder.CreateIntrinsic(Intrinsic::instrprof_increment_step,

1776 {NormalizedFuncNameVarPtr, Builder.getInt64(FuncHash),

1777 Builder.getInt32(TotalNumCtrs),

1778 Builder.getInt32(*CurCtrIdx), Step});

1779 ++(*CurCtrIdx);

1780}

1781

1782void SelectInstVisitor::annotateOneSelectInst(SelectInst &SI) {

1783 std::vector<uint64_t> &CountFromProfile = UseFunc->getProfileRecord().Counts;

1784 assert(*CurCtrIdx < CountFromProfile.size() &&

1785 "Out of bound access of counters");

1786 uint64_t SCounts[2];

1787 SCounts[0] = CountFromProfile[*CurCtrIdx];

1788 ++(*CurCtrIdx);

1789 uint64_t TotalCount = 0;

1790 auto BI = UseFunc->findBBInfo(SI.getParent());

1791 if (BI != nullptr) {

1792 TotalCount = *BI->Count;

1793

1794

1795 if (TotalCount < SCounts[0])

1796 BI->Count = SCounts[0];

1797 }

1798

1799 SCounts[1] = (TotalCount > SCounts[0] ? TotalCount - SCounts[0] : 0);

1800 uint64_t MaxCount = std::max(SCounts[0], SCounts[1]);

1801 if (MaxCount)

1803}

1804

1805void SelectInstVisitor::visitSelectInst(SelectInst &SI) {

1807 return;

1808

1809 if (SI.getCondition()->getType()->isVectorTy())

1810 return;

1811

1812 switch (Mode) {

1813 case VM_counting:

1814 NSIs++;

1815 return;

1816 case VM_instrument:

1817 instrumentOneSelectInst(SI);

1818 return;

1819 case VM_annotate:

1820 annotateOneSelectInst(SI);

1821 return;

1822 }

1823

1825}

1826

1828 if (ValueProfKind == IPVK_MemOPSize)

1830 if (ValueProfKind == llvm::IPVK_VTableTarget)

1833}

1834

1835

1836void PGOUseFunc::annotateValueSites() {

1838 return;

1839

1840

1842

1843 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)

1844 annotateValueSites(Kind);

1845}

1846

1847

1848void PGOUseFunc::annotateValueSites(uint32_t Kind) {

1849 assert(Kind <= IPVK_Last);

1850 unsigned ValueSiteIndex = 0;

1851

1853

1854

1855

1856

1857

1858

1859

1860

1861

1862

1863 if (NumValueSites > 0 && Kind == IPVK_VTableTarget &&

1864 NumValueSites != FuncInfo.ValueSites[IPVK_VTableTarget].size() &&

1866 FuncInfo.ValueSites[IPVK_VTableTarget] = VPC.get(IPVK_VTableTarget);

1867 auto &ValueSites = FuncInfo.ValueSites[Kind];

1869 auto &Ctx = M->getContext();

1870 Ctx.diagnose(DiagnosticInfoPGOProfile(

1871 M->getName().data(),

1872 Twine("Inconsistent number of value sites for ") +

1874 F.getName().str() +

1875 Twine("\", possibly due to the use of a stale profile."),

1877 return;

1878 }

1879

1881 LLVM_DEBUG(dbgs() << "Read one value site profile (kind = " << Kind

1882 << "): Index = " << ValueSiteIndex << " out of "

1885 *M, *I.AnnotatedInst, ProfileRecord,

1888 ValueSiteIndex++;

1889 }

1890}

1891

1892

1893

1896 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {

1898 return;

1900 if (Comdat *C = F.getComdat())

1901 ComdatMembers.insert(std::make_pair(C, &F));

1903 if (Comdat *C = GV.getComdat())

1904 ComdatMembers.insert(std::make_pair(C, &GV));

1906 if (Comdat *C = GA.getComdat())

1907 ComdatMembers.insert(std::make_pair(C, &GA));

1908}

1909

1910

1912 if (F.isDeclaration())

1913 return true;

1914

1915

1916

1917 unsigned NumCriticalEdges = 0;

1918 for (auto &BB : F) {

1919 const Instruction *TI = BB.getTerminator();

1922 NumCriticalEdges++;

1923 }

1924 }

1927 << ", NumCriticalEdges=" << NumCriticalEdges

1928 << " exceed the threshold. Skip PGO.\n");

1929 return true;

1930 }

1931 return false;

1932}

1933

1934

1937 return true;

1938 if (F.hasFnAttribute(llvm::Attribute::Naked))

1939 return true;

1940 if (F.hasFnAttribute(llvm::Attribute::NoProfile))

1941 return true;

1942 if (F.hasFnAttribute(llvm::Attribute::SkipProfile))

1943 return true;

1945 return true;

1947 if (auto EntryCount = F.getEntryCount())

1950 }

1951 return false;

1952}

1953

1960

1961

1964

1965 Triple TT(M.getTargetTriple());

1969 M.getName().data(),

1970 Twine("VTable value profiling is presently not "

1971 "supported for non-ELF object formats"),

1973 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;

1975

1976 for (auto &F : M) {

1978 continue;

1983 FunctionInstrumenter FI(M, F, TLI, ComdatMembers, BPI, BFI, LI,

1984 InstrumentationType);

1985 FI.instrument();

1986 }

1987 return true;

1988}

1989

1990PreservedAnalyses

1993

1994

1997 if (ProfileSampling)

2002 return PA;

2003}

2004

2010 };

2013 };

2016 };

2019 };

2020

2022 InstrumentationType))

2024

2026}

2027

2028

2029

2034#ifndef NDEBUG

2035 auto BFIEntryCount = F.getEntryCount();

2036 assert(BFIEntryCount && (BFIEntryCount->getCount() > 0) &&

2037 "Invalid BFI Entrycount");

2038#endif

2041 for (auto &BBI : F) {

2044 if (!Func.findBBInfo(&BBI))

2045 continue;

2047 CountValue = *Func.getBBInfo(&BBI).Count;

2048 BFICountValue = *BFICount;

2051 }

2052 if (SumCount.isZero())

2053 return;

2054

2056 "Incorrect sum of BFI counts");

2058 return;

2059 double Scale = (SumCount / SumBFICount).convertToDouble();

2060 if (Scale < 1.001 && Scale > 0.999)

2061 return;

2062

2065 if (NewEntryCount == 0)

2066 NewEntryCount = 1;

2069 LLVM_DEBUG(dbgs() << "FixFuncEntryCount: in " << F.getName()

2071 << NewEntryCount << "\n");

2072 }

2073}

2074

2075

2076

2083

2087

2088 unsigned BBNum = 0, BBMisMatchNum = 0, NonZeroBBNum = 0;

2089 for (auto &BBI : F) {

2090 PGOUseBBInfo *BBInfo = Func.findBBInfo(&BBI);

2091 if (!BBInfo)

2092 continue;

2093

2094 uint64_t CountValue = BBInfo->Count.value_or(CountValue);

2096

2097 BBNum++;

2098 if (CountValue)

2099 NonZeroBBNum++;

2101 if (BFICount)

2102 BFICountValue = *BFICount;

2103

2104 if (HotBBOnly) {

2105 bool rawIsHot = CountValue >= HotCountThreshold;

2106 bool BFIIsHot = BFICountValue >= HotCountThreshold;

2108 bool ShowCount = false;

2109 if (rawIsHot && !BFIIsHot) {

2110 Msg = "raw-Hot to BFI-nonHot";

2111 ShowCount = true;

2112 } else if (rawIsCold && BFIIsHot) {

2113 Msg = "raw-Cold to BFI-Hot";

2114 ShowCount = true;

2115 }

2116 if (!ShowCount)

2117 continue;

2118 } else {

2121 continue;

2122 uint64_t Diff = (BFICountValue >= CountValue)

2123 ? BFICountValue - CountValue

2124 : CountValue - BFICountValue;

2126 continue;

2127 }

2128 BBMisMatchNum++;

2129

2130 ORE.emit([&]() {

2132 F.getSubprogram(), &BBI);

2133 Remark << "BB " << ore::NV("Block", BBI.getName())

2134 << " Count=" << ore::NV("Count", CountValue)

2135 << " BFI_Count=" << ore::NV("Count", BFICountValue);

2136 if (!Msg.empty())

2137 Remark << " (" << Msg << ")";

2139 });

2140 }

2141 if (BBMisMatchNum)

2142 ORE.emit([&]() {

2144 F.getSubprogram(), &F.getEntryBlock())

2145 << "In Func " << ore::NV("Function", F.getName())

2146 << ": Num_of_BB=" << ore::NV("Count", BBNum)

2147 << ", Num_of_non_zerovalue_BB=" << ore::NV("Count", NonZeroBBNum)

2148 << ", Num_of_mis_matching_BB=" << ore::NV("Count", BBMisMatchNum);

2149 });

2150}

2151

2159 bool IsCS) {

2161 auto &Ctx = M.getContext();

2162

2164 ProfileRemappingFileName);

2165 if (Error E = ReaderOrErr.takeError()) {

2167 Ctx.diagnose(

2169 });

2170 return false;

2171 }

2172

2173 std::unique_ptr PGOReader =

2174 std::move(ReaderOrErr.get());

2175 if (!PGOReader) {

2177 StringRef("Cannot get PGOReader")));

2178 return false;

2179 }

2180 if (!PGOReader->hasCSIRLevelProfile() && IsCS)

2181 return false;

2182

2183

2184 if (!PGOReader->isIRLevelProfile()) {

2186 ProfileFileName.data(), "Not an IR level instrumentation profile"));

2187 return false;

2188 }

2189 if (PGOReader->functionEntryOnly()) {

2191 ProfileFileName.data(),

2192 "Function entry profiles are not yet supported for optimization"));

2193 return false;

2194 }

2195

2198 if (.hasName() || .hasMetadata(LLVMContext::MD_type))

2199 continue;

2200

2201

2203 }

2204 }

2205

2206

2207

2208

2209 M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()),

2213

2214 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;

2216 std::vector<Function *> HotFunctions;

2217 std::vector<Function *> ColdFunctions;

2218

2219

2220

2221 bool InstrumentFuncEntry = PGOReader->instrEntryBBEnabled();

2224 bool InstrumentLoopEntries = PGOReader->instrLoopEntriesEnabled();

2227

2228 bool HasSingleByteCoverage = PGOReader->hasSingleByteCoverage();

2229 for (auto &F : M) {

2231 continue;

2236 if (!HasSingleByteCoverage) {

2237

2238

2240 BFI);

2241 }

2242 PGOUseFunc Func(F, &M, TLI, ComdatMembers, BPI, BFI, LI, PSI, IsCS,

2243 InstrumentFuncEntry, InstrumentLoopEntries,

2244 HasSingleByteCoverage);

2245 if (!Func.getRecord(PGOReader.get()))

2246 continue;

2247 if (HasSingleByteCoverage) {

2248 Func.populateCoverage();

2249 continue;

2250 }

2251

2252

2253

2254

2256 bool AllZeros = false;

2257 if (!Func.readCounters(AllZeros, PseudoKind))

2258 continue;

2259 if (AllZeros) {

2261 if (Func.getProgramMaxCount() != 0)

2262 ColdFunctions.push_back(&F);

2263 continue;

2264 }

2266

2267 if (F.hasFnAttribute(Attribute::Cold))

2268 F.removeFnAttr(Attribute::Cold);

2269

2271 F.addFnAttr(Attribute::Hot);

2272 continue;

2273 }

2274 Func.populateCounters();

2275 Func.setBranchWeights();

2276 Func.annotateValueSites();

2277 Func.annotateIrrLoopHeaderWeights();

2278 PGOUseFunc::FuncFreqAttr FreqAttr = Func.getFuncFreqAttr();

2279 if (FreqAttr == PGOUseFunc::FFA_Cold)

2280 ColdFunctions.push_back(&F);

2281 else if (FreqAttr == PGOUseFunc::FFA_Hot)

2282 HotFunctions.push_back(&F);

2287 std::unique_ptr NewBPI =

2288 std::make_unique(F, LI);

2289 std::unique_ptr NewBFI =

2290 std::make_unique(F, *NewBPI, LI);

2292 NewBFI->view();

2294 dbgs() << "pgo-view-counts: " << Func.getFunc().getName() << "\n";

2295 NewBFI->print(dbgs());

2296 }

2297 }

2303 WriteGraph(&Func, Twine("PGORawCounts_") + Func.getFunc().getName());

2304 else

2305 ViewGraph(&Func, Twine("PGORawCounts_") + Func.getFunc().getName());

2307 dbgs() << "pgo-view-raw-counts: " << Func.getFunc().getName() << "\n";

2308 Func.dumpInfo();

2309 }

2310 }

2311

2315

2316

2319

2320

2325 }

2327 }

2328 }

2329

2330

2331

2332

2333

2334 for (auto &F : HotFunctions) {

2335 F->addFnAttr(Attribute::InlineHint);

2336 LLVM_DEBUG(dbgs() << "Set inline attribute to function: " << F->getName()

2337 << "\n");

2338 }

2339 for (auto &F : ColdFunctions) {

2340

2341

2342 if (F->hasFnAttribute(Attribute::Hot)) {

2343 auto &Ctx = M.getContext();

2344 std::string Msg = std::string("Function ") + F->getName().str() +

2345 std::string(" is annotated as a hot function but"

2346 " the profile is cold");

2347 Ctx.diagnose(

2349 continue;

2350 }

2351 F->addFnAttr(Attribute::Cold);

2352 LLVM_DEBUG(dbgs() << "Set cold attribute to function: " << F->getName()

2353 << "\n");

2354 }

2355 return true;

2356}

2357

2359 std::string Filename, std::string RemappingFilename, bool IsCS,

2361 : ProfileFileName(std::move(Filename)),

2362 ProfileRemappingFileName(std::move(RemappingFilename)), IsCS(IsCS),

2368 if (!FS)

2370}

2371

2374

2378 };

2381 };

2384 };

2387 };

2388

2390 if ( annotateAllFunctions (M, ProfileFileName, ProfileRemappingFileName, *FS,

2391 LookupTLI, LookupBPI, LookupBFI, LookupLI, PSI,

2392 IsCS))

2394

2396}

2397

2399 if ( Node ->getName().empty())

2400 return Node->getName().str();

2401

2402 std::string SimpleNodeName;

2405 return SimpleNodeName;

2406}

2407

2411

2412 LLVM_DEBUG(dbgs() << "Weight is: "; for (const auto &W

2413 : Weights) {

2414 dbgs() << W << " ";

2415 } dbgs() << "\n";);

2416

2418

2422 if (BrCondStr.empty())

2423 return;

2424

2426 std::accumulate(Weights.begin(), Weights.end(), (uint64_t)0,

2429 std::accumulate(EdgeCounts.begin(), EdgeCounts.end(), (uint64_t)0,

2434 std::string BranchProbStr;

2436 OS << BP;

2437 OS << " (total count : " << TotalCount << ")";

2440 ORE.emit([&]() {

2442 << BrCondStr << " is true with probability : " << BranchProbStr;

2443 });

2444 }

2445}

2446

2447namespace llvm {

2448

2451 TI->setMetadata(llvm::LLVMContext::MD_irr_loop,

2453}

2454

2459

2461 return &G->getFunc().front();

2462 }

2463

2467

2469

2473

2477};

2478

2482

2484 return std::string(G->getFunc().getName());

2485 }

2486

2488 std::string Result;

2490

2492 PGOUseBBInfo *BI = Graph->findBBInfo(Node);

2493 OS << "Count : ";

2494 if (BI && BI->Count)

2495 OS << *BI->Count << "\\l";

2496 else

2497 OS << "Unknown\\l";

2498

2500 return Result;

2501

2504 continue;

2505

2506 OS << "SELECT : { T = ";

2509 if (!HasProf)

2510 OS << "Unknown, F = Unknown }\\l";

2511 else

2512 OS << TC << ", F = " << FC << " }\\l";

2513 }

2514 return Result;

2515 }

2516};

2517

2518}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

This file implements a class to represent arbitrary precision integral constant values and operations...

Function Alias Analysis false

This file contains the simple types necessary to represent the attributes associated with functions a...

This file finds the minimum set of blocks on a CFG that must be instrumented to infer execution cover...

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

Analysis containing CSE Info

#define clEnumValN(ENUMVAL, FLAGNAME, DESC)

This file contains the declarations for the subclasses of Constant, which represent the different fla...

post inline ee instrument

static BasicBlock * getInstrBB(CFGMST< Edge, BBInfo > &MST, Edge &E, const DenseSet< const BasicBlock * > &ExecBlocks)

This file provides various utilities for inspecting and working with the control flow graph in LLVM I...

Module.h This file contains the declarations for the Module class.

This header defines various interfaces for pass management in LLVM.

#define INSTR_PROF_QUOTE(x)

#define VARIANT_MASK_CSIR_PROF

#define VARIANT_MASK_DBG_CORRELATE

#define INSTR_PROF_RAW_VERSION

#define INSTR_PROF_RAW_VERSION_VAR

#define VARIANT_MASK_TEMPORAL_PROF

#define VARIANT_MASK_IR_PROF

#define VARIANT_MASK_BYTE_COVERAGE

#define VARIANT_MASK_INSTR_ENTRY

#define VARIANT_MASK_FUNCTION_ENTRY_ONLY

#define VARIANT_MASK_INSTR_LOOP_ENTRIES

Machine Check Debug Module

static cl::opt< unsigned > ColdCountThreshold("mfs-count-threshold", cl::desc("Minimum number of times a block must be executed to be retained."), cl::init(1), cl::Hidden)

static GlobalVariable * createIRLevelProfileFlagVar(Module &M, PGOInstrumentationType InstrumentationType)

Definition PGOInstrumentation.cpp:447

static cl::opt< std::string > PGOTestProfileRemappingFile("pgo-test-profile-remapping-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile remapping file. This is mainly for " "test purpose."))

static void fixFuncEntryCount(PGOUseFunc &Func, LoopInfo &LI, BranchProbabilityInfo &NBPI)

Definition PGOInstrumentation.cpp:2030

static void annotateFunctionWithHashMismatch(Function &F, LLVMContext &ctx)

Definition PGOInstrumentation.cpp:1387

static cl::opt< unsigned > MaxNumMemOPAnnotations("memop-max-annotations", cl::init(4), cl::Hidden, cl::desc("Max number of precise value annotations for a single memop" "intrinsic"))

static cl::opt< unsigned > MaxNumAnnotations("icp-max-annotations", cl::init(3), cl::Hidden, cl::desc("Max number of annotations for a single indirect " "call callsite"))

static bool skipPGOGen(const Function &F)

Definition PGOInstrumentation.cpp:1935

static void collectComdatMembers(Module &M, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers)

Definition PGOInstrumentation.cpp:1894

static void populateEHOperandBundle(VPCandidateInfo &Cand, DenseMap< BasicBlock *, ColorVector > &BlockColors, SmallVectorImpl< OperandBundleDef > &OpBundles)

Definition PGOInstrumentation.cpp:902

static void verifyFuncBFI(PGOUseFunc &Func, LoopInfo &LI, BranchProbabilityInfo &NBPI, uint64_t HotCountThreshold, uint64_t ColdCountThreshold)

Definition PGOInstrumentation.cpp:2077

static cl::opt< bool > DoComdatRenaming("do-comdat-renaming", cl::init(false), cl::Hidden, cl::desc("Append function hash to the name of COMDAT function to avoid " "function hash mismatch due to the preinliner"))

static bool annotateAllFunctions(Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, vfs::FileSystem &FS, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, function_ref< LoopInfo *(Function &)> LookupLI, ProfileSummaryInfo *PSI, bool IsCS)

Definition PGOInstrumentation.cpp:2152

static void setupBBInfoEdges(const FuncPGOInstrumentation< PGOUseEdge, PGOUseBBInfo > &FuncInfo)

Set up InEdges/OutEdges for all BBs in the MST.

Definition PGOInstrumentation.cpp:1283

static bool skipPGOUse(const Function &F)

Definition PGOInstrumentation.cpp:1911

static bool canRenameComdat(Function &F, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers)

Definition PGOInstrumentation.cpp:755

ValueProfileCollector::CandidateInfo VPCandidateInfo

Definition PGOInstrumentation.cpp:132

static bool InstrumentAllFunctions(Module &M, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, function_ref< LoopInfo *(Function &)> LookupLI, PGOInstrumentationType InstrumentationType)

Definition PGOInstrumentation.cpp:1954

static uint64_t sumEdgeCount(const ArrayRef< PGOUseEdge * > Edges)

Definition PGOInstrumentation.cpp:1148

static uint32_t getMaxNumAnnotations(InstrProfValueKind ValueProfKind)

Definition PGOInstrumentation.cpp:1827

static cl::opt< bool > DisableValueProfiling("disable-vp", cl::init(false), cl::Hidden, cl::desc("Disable Value Profiling"))

static std::string getSimpleNodeName(const BasicBlock *Node)

Definition PGOInstrumentation.cpp:2398

static bool isIndirectBrTarget(BasicBlock *BB)

Definition PGOInstrumentation.cpp:1744

static cl::opt< std::string > PGOTestProfileFile("pgo-test-profile-file", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path of profile data file. This is " "mainly for test purpose."))

static std::string getBranchCondString(Instruction *TI)

Definition PGOInstrumentation.cpp:409

static const char * ValueProfKindDescr[]

Definition PGOInstrumentation.cpp:439

This file provides the interface for IR based instrumentation passes ( (profile-gen,...

FunctionAnalysisManager FAM

ModuleAnalysisManager MAM

This file contains the declarations for profiling metadata utility functions.

const SmallVectorImpl< MachineOperand > & Cond

static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))

void visit(MachineFunction &MF, MachineBasicBlock &Start, std::function< void(MachineBasicBlock *)> op)

std::pair< BasicBlock *, BasicBlock * > Edge

This file defines the SmallVector class.

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

StringSet - A set-like wrapper for the StringMap.

Defines the virtual file system interface vfs::FileSystem.

void printAsOperand(OutputBuffer &OB, Prec P=Prec::Default, bool StrictlyWorse=false) const

static const fltSemantics & IEEEdouble()

static constexpr roundingMode rmNearestTiesToEven

static APFloat getZero(const fltSemantics &Sem, bool Negative=false)

Factory for Positive and Negative Zero.

Class for arbitrary precision integers.

This templated class represents "all analyses that operate over " (e....

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

LLVM Basic Block Representation.

LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const

Returns an iterator to the first instruction in this block that is not a PHINode instruction.

InstListType::iterator iterator

Instruction iterators...

LLVM_ABI const_iterator getFirstNonPHIOrDbgOrAlloca() const

Returns an iterator to the first instruction in this block that is not a PHINode, a debug intrinsic,...

const Instruction * getTerminator() const LLVM_READONLY

Returns the terminator instruction if the block is well formed or null if the block is not well forme...

Analysis pass which computes BlockFrequencyInfo.

BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...

LLVM_ABI bool isIrrLoopHeader(const BasicBlock *BB)

Returns true if BB is an irreducible loop header block.

LLVM_ABI std::optional< uint64_t > getBlockProfileCount(const BasicBlock *BB, bool AllowSynthetic=false) const

Returns the estimated profile count of BB.

Conditional or Unconditional Branch instruction.

bool isConditional() const

Value * getCondition() const

Analysis pass which computes BranchProbabilityInfo.

Analysis providing branch probability information.

Edge & addEdge(BasicBlock *Src, BasicBlock *Dest, uint64_t W)

const std::vector< std::unique_ptr< Edge > > & allEdges() const

Predicate getPredicate() const

Return the predicate for this instruction.

LLVM_ABI StringRef getName() const

void setSelectionKind(SelectionKind Val)

SelectionKind getSelectionKind() const

static LLVM_ABI Constant * getPointerBitCastOrAddrSpaceCast(Constant *C, Type *Ty)

Create a BitCast or AddrSpaceCast for a pointer type depending on the address space.

This is the shared class of boolean and integer constants.

bool isMinusOne() const

This function will return true iff every bit in this constant is set to true.

bool isOne() const

This is just a convenience method to make client code smaller for a common case.

bool isZero() const

This is just a convenience method to make client code smaller for a common code.

static LLVM_ABI Constant * getIntegerValue(Type *Ty, const APInt &V)

Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...

iterator find(const_arg_type_t< KeyT > Val)

std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)

Diagnostic information for the PGO profiler.

Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.

Base class for error info classes.

virtual std::string message() const

Return the error message as a string.

Lightweight error class with error context and mandatory checking.

Class to represent profile counts.

static LLVM_ABI GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)

If a parent module is specified, the alias is automatically inserted into the end of the specified mo...

@ HiddenVisibility

The GV is hidden.

@ ProtectedVisibility

The GV is protected.

@ ExternalLinkage

Externally visible function.

@ WeakAnyLinkage

Keep one copy of named function when linking (weak)

@ AvailableExternallyLinkage

Available for inspection, not emission.

@ LinkOnceODRLinkage

Same, but only replaced by something equivalent.

This instruction compares its operands according to the predicate given to the constructor.

static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const Twine &RemappingPath="")

Factory method to create an indexed reader.

uint64_t getMaximumFunctionCount(bool UseCS)

Return the maximum of all known function counts.

Expected< NamedInstrProfRecord > getInstrProfRecord(StringRef FuncName, uint64_t FuncHash, StringRef DeprecatedFuncName="", uint64_t *MismatchedFuncSum=nullptr)

Return the NamedInstrProfRecord associated with FuncName and FuncHash.

Base class for instruction visitors.

static bool canInstrumentCallsite(const CallBase &CB)

instrprof_error get() const

std::string message() const override

Return the error message as a string.

LLVM_ABI unsigned getNumSuccessors() const LLVM_READONLY

Return the number of successors that this instruction has.

LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)

Set the metadata of the specified kind to the specified node.

A smart pointer to a reference-counted object that inherits from RefCountedBase or ThreadSafeRefCount...

LLVM_ABI void update(ArrayRef< uint8_t > Data)

This is an important class for using LLVM in a threaded context.

LLVM_ABI void diagnose(const DiagnosticInfo &DI)

Report a message to the currently installed diagnostic handler.

Analysis pass that exposes the LoopInfo for a function.

LLVM_ABI MDString * createString(StringRef Str)

Return the given string as metadata.

LLVM_ABI MDNode * createIrrLoopHeaderWeight(uint64_t Weight)

Return metadata containing an irreducible loop header weight.

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

A Module instance is used to store all the information related to an LLVM module.

LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)

Definition PGOInstrumentation.cpp:1991

LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)

Definition PGOInstrumentation.cpp:2005

LLVM_ABI PGOInstrumentationUse(std::string Filename="", std::string RemappingFilename="", bool IsCS=false, IntrusiveRefCntPtr< vfs::FileSystem > FS=nullptr)

Definition PGOInstrumentation.cpp:2358

LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)

Definition PGOInstrumentation.cpp:2372

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

PreservedAnalyses & preserveSet()

Mark an analysis set as preserved.

PreservedAnalyses & preserve()

Mark an analysis as preserved.

An analysis pass based on the new PM to deliver ProfileSummaryInfo.

Analysis providing profile information.

LLVM_ABI uint64_t getOrCompColdCountThreshold() const

Returns ColdCountThreshold if set.

LLVM_ABI bool isColdCount(uint64_t C) const

Returns true if count C is considered cold.

LLVM_ABI void refresh(std::unique_ptr< ProfileSummary > &&Other=nullptr)

If a summary is provided as argument, use that.

LLVM_ABI bool isHotCount(uint64_t C) const

Returns true if count C is considered hot.

LLVM_ABI uint64_t getOrCompHotCountThreshold() const

Returns HotCountThreshold if set.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

reference emplace_back(ArgTypes &&... Args)

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

constexpr bool empty() const

empty - Check if the string is empty.

constexpr const char * data() const

data - Get a pointer to the start of the string (which may not be null terminated).

Analysis pass providing the TargetLibraryInfo.

Provides information about what library functions are available for the current target.

Triple - Helper class for working with autoconf configuration names.

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

LLVM_ABI std::string str() const

Return the twine contents as a std::string.

The instances of the Type class are immutable: once they are created, they are never changed.

static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)

LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false, bool NoDetails=false) const

Print the current type.

Value * getOperand(unsigned i) const

std::vector< CandidateInfo > get(InstrProfValueKind Kind) const

returns a list of value profiling candidates of the given kind

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

An efficient, type-erasing, non-owning reference to a callable.

const ParentTy * getParent() const

A raw_ostream that writes to an std::string.

The virtual file system interface.

This provides a very simple, boring adaptor for a begin and end iterator into a range type.

This file contains the declaration of the Comdat class, which represents a single COMDAT in LLVM.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

@ C

The default llvm calling convention, compatible with C.

@ BasicBlock

Various leaf nodes.

LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})

Look up the Function declaration of the intrinsic id in the Module M.

ValuesClass values(OptsTy... Options)

Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...

initializer< Ty > init(const Ty &Val)

uint64_t getFuncHash(const FuncRecordTy *Record)

Return the structural hash associated with the function.

void checkExpectAnnotations(const Instruction &I, ArrayRef< uint32_t > ExistingWeights, bool IsFrontend)

checkExpectAnnotations - compares PGO counters to the thresholds used for llvm.expect and warns if th...

DiagnosticInfoOptimizationBase::Argument NV

NodeAddr< FuncNode * > Func

friend class Instruction

Iterator for Instructions in a `BasicBlock.

void write64le(void *P, uint64_t V)

LLVM_ABI IntrusiveRefCntPtr< FileSystem > getRealFileSystem()

Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.

This is an optimization pass for GlobalISel generic memory operations.

static cl::opt< bool > PGOTreatUnknownAsCold("pgo-treat-unknown-as-cold", cl::init(false), cl::Hidden, cl::desc("For cold function instrumentation, treat count unknown(e.g. " "unprofiled) functions as cold."))

FunctionAddr VTableAddr Value

static cl::opt< bool > PGOInstrMemOP("pgo-instr-memop", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off " "memory intrinsic size profiling."))

LLVM_ABI void setIrrLoopHeaderMetadata(Module *M, Instruction *TI, uint64_t Count)

Definition PGOInstrumentation.cpp:2449

LLVM_ABI void setProfMetadata(Instruction *TI, ArrayRef< uint64_t > EdgeCounts, uint64_t MaxCount)

Definition PGOInstrumentation.cpp:2408

auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)

Get the size of a range.

LLVM_ABI std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)

Please use getIRPGOFuncName for LLVM IR instrumentation.

static cl::opt< bool > PGOViewBlockCoverageGraph("pgo-view-block-coverage-graph", cl::desc("Create a dot file of CFGs with block " "coverage inference information"))

LLVM_ABI void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)

Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.

LLVM_ABI unsigned GetSuccessorNumber(const BasicBlock *BB, const BasicBlock *Succ)

Search for the specified successor of basic block BB and return its position in the terminator instru...

LLVM_ABI std::string getIRPGOFuncName(const Function &F, bool InLTO=false)

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

auto successors(const MachineBasicBlock *BB)

LLVM_ABI void createProfileSamplingVar(Module &M)

void handleAllErrors(Error E, HandlerTs &&... Handlers)

Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...

constexpr from_range_t from_range

iterator_range< T > make_range(T x, T y)

Convenience function for iterating over sub-ranges.

LLVM_ABI DenseMap< BasicBlock *, ColorVector > colorEHFunclets(Function &F)

If an EH funclet personality is in use (see isFuncletEHPersonality), this will recompute which blocks...

LLVM_ABI void createPGONameMetadata(GlobalObject &GO, StringRef PGOName)

Create the PGOName metadata if a global object's PGO name is different from its mangled name.

InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy

Provide the FunctionAnalysisManager to Module proxy.

static cl::opt< bool > PGOBlockCoverage("pgo-block-coverage", cl::desc("Use this option to enable basic block coverage instrumentation"))

cl::opt< bool > PGOWarnMissing

raw_ostream & WriteGraph(raw_ostream &O, const GraphType &G, bool ShortNames=false, const Twine &Title="")

cl::opt< unsigned > MaxNumVTableAnnotations("icp-max-num-vtables", cl::init(6), cl::Hidden, cl::desc("Max number of vtables annotated for a vtable load instruction."))

static cl::opt< bool > PGOTemporalInstrumentation("pgo-temporal-instrumentation", cl::desc("Use this option to enable temporal instrumentation"))

LLVM_ABI bool SplitIndirectBrCriticalEdges(Function &F, bool IgnoreBlocksWithoutPHI, BranchProbabilityInfo *BPI=nullptr, BlockFrequencyInfo *BFI=nullptr)

cl::opt< bool > EnableVTableProfileUse("enable-vtable-profile-use", cl::init(false), cl::desc("If ThinLTO and WPD is enabled and this option is true, vtable " "profiles will be used by ICP pass for more efficient indirect " "call sequence. If false, type profiles won't be used."))

bool isScopedEHPersonality(EHPersonality Pers)

Returns true if this personality uses scope-style EH IR instructions: catchswitch,...

LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected, bool ElideAllZero=false)

Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...

LLVM_ABI std::string getPGOName(const GlobalVariable &V, bool InLTO=false)

cl::opt< std::string > ViewBlockFreqFuncName("view-bfi-func-name", cl::Hidden, cl::desc("The option to specify " "the name of the function " "whose CFG will be displayed."))

LLVM_ABI GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)

Create and return the global variable for function name used in PGO instrumentation.

LLVM_ABI void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)

Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...

auto reverse(ContainerTy &&C)

static cl::opt< bool > EmitBranchProbability("pgo-emit-branch-prob", cl::init(false), cl::Hidden, cl::desc("When this option is on, the annotated " "branch probability will be emitted as " "optimization remarks: -{Rpass|" "pass-remarks}=pgo-instrumentation"))

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

FunctionAddr NumValueSites[IPVK_Last+1]

FunctionAddr VTableAddr Count

Function::ProfileCount ProfileCount

static cl::opt< unsigned > PGOVerifyBFIRatio("pgo-verify-bfi-ratio", cl::init(2), cl::Hidden, cl::desc("Set the threshold for pgo-verify-bfi: only print out " "mismatched BFI if the difference percentage is greater than " "this value (in percentage)."))

static cl::opt< bool > PGOInstrumentLoopEntries("pgo-instrument-loop-entries", cl::init(false), cl::Hidden, cl::desc("Force to instrument loop entries."))

static cl::opt< unsigned > PGOFunctionSizeThreshold("pgo-function-size-threshold", cl::Hidden, cl::desc("Do not instrument functions smaller than this threshold."))

LLVM_ABI EHPersonality classifyEHPersonality(const Value *Pers)

See if the given exception handling personality function is one that we understand.

static cl::opt< bool > PGOFixEntryCount("pgo-fix-entry-count", cl::init(true), cl::Hidden, cl::desc("Fix function entry count in profile use."))

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

static cl::opt< PGOViewCountsType > PGOViewRawCounts("pgo-view-raw-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text " "with raw profile counts from " "profile data. See also option " "-pgo-view-counts. To limit graph " "display to only one function, use " "filtering option -view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))

static cl::opt< bool > PGOVerifyBFI("pgo-verify-bfi", cl::init(false), cl::Hidden, cl::desc("Print out mismatched BFI counts after setting profile metadata " "The print is enabled under -Rpass-analysis=pgo, or " "internal option -pass-remarks-analysis=pgo."))

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

cl::opt< bool > NoPGOWarnMismatch

RNSuccIterator< NodeRef, BlockT, RegionT > succ_begin(NodeRef Node)

static cl::opt< uint64_t > PGOColdInstrumentEntryThreshold("pgo-cold-instrument-entry-threshold", cl::init(0), cl::Hidden, cl::desc("For cold function instrumentation, skip instrumenting functions " "whose entry count is above the given value."))

FunctionAddr VTableAddr uintptr_t uintptr_t Data

IRBuilder(LLVMContext &, FolderTy, InserterTy, MDNode *, ArrayRef< OperandBundleDef >) -> IRBuilder< FolderTy, InserterTy >

cl::opt< PGOViewCountsType > PGOViewCounts("pgo-view-counts", cl::Hidden, cl::desc("A boolean option to show CFG dag or text with " "block profile counts and branch probabilities " "right after PGO profile annotation step. The " "profile counts are computed using branch " "probabilities from the runtime profile data and " "block frequency propagation algorithm. To view " "the raw counts from the profile, use option " "-pgo-view-raw-counts instead. To limit graph " "display to only one function, use filtering option " "-view-bfi-func-name."), cl::values(clEnumValN(PGOVCT_None, "none", "do not show."), clEnumValN(PGOVCT_Graph, "graph", "show a graph."), clEnumValN(PGOVCT_Text, "text", "show in text.")))

Definition PGOInstrumentation.cpp:351

RNSuccIterator< NodeRef, BlockT, RegionT > succ_end(NodeRef Node)

OperandBundleDefT< Value * > OperandBundleDef

LLVM_ABI void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)

Adds global values to the llvm.compiler.used list.

static cl::opt< unsigned > PGOVerifyBFICutoff("pgo-verify-bfi-cutoff", cl::init(5), cl::Hidden, cl::desc("Set the threshold for pgo-verify-bfi: skip the counts whose " "profile count value is below."))

LLVM_ABI BasicBlock * SplitCriticalEdge(Instruction *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")

If this edge is a critical edge, insert a new node to split the critical edge.

void ViewGraph(const GraphType &G, const Twine &Name, bool ShortNames=false, const Twine &Title="", GraphProgram::Name Program=GraphProgram::DOT)

ViewGraph - Emit a dot graph, run 'dot', run gv on the postscript file, then cleanup.

LLVM_ABI bool isCriticalEdge(const Instruction *TI, unsigned SuccNum, bool AllowIdenticalEdges=false)

Return true if the specified edge is a critical edge.

cl::opt< bool > PGOInstrumentColdFunctionOnly

cl::list< std::string > CtxPGOSkipCallsiteInstrument("ctx-prof-skip-callsite-instr", cl::Hidden, cl::desc("Do not instrument callsites to functions in this list. Intended " "for testing."))

LLVM_ABI bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)

Check if we can safely rename this Comdat function.

static cl::opt< bool > PGOInstrSelect("pgo-instr-select", cl::init(true), cl::Hidden, cl::desc("Use this option to turn on/off SELECT " "instruction instrumentation. "))

LLVM_ABI void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)

OutputIt move(R &&Range, OutputIt Out)

Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.

TinyPtrVector< BasicBlock * > ColorVector

LLVM_ABI bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)

Extract branch weights from MD_prof metadata.

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

auto predecessors(const MachineBasicBlock *BB)

llvm:🆑:opt< llvm::InstrProfCorrelator::ProfCorrelatorKind > ProfileCorrelate

Definition PGOInstrumentation.cpp:362

static cl::opt< bool > PGOFunctionEntryCoverage("pgo-function-entry-coverage", cl::Hidden, cl::desc("Use this option to enable function entry coverage instrumentation."))

static cl::opt< unsigned > PGOFunctionCriticalEdgeThreshold("pgo-critical-edge-threshold", cl::init(20000), cl::Hidden, cl::desc("Do not instrument functions with the number of critical edges " " greater than this threshold."))

uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)

Scale an individual branch count.

static cl::opt< bool > PGOVerifyHotBFI("pgo-verify-hot-bfi", cl::init(false), cl::Hidden, cl::desc("Print out the non-match BFI count if a hot raw profile count " "becomes non-hot, or a cold raw profile count becomes hot. " "The print is enabled under -Rpass-analysis=pgo, or " "internal option -pass-remarks-analysis=pgo."))

SuccIterator< const Instruction, const BasicBlock > const_succ_iterator

uint64_t calculateCountScale(uint64_t MaxCount)

Calculate what to divide by to scale counts.

LLVM_ABI SmallVector< uint32_t > downscaleWeights(ArrayRef< uint64_t > Weights, std::optional< uint64_t > KnownMaxCount=std::nullopt)

downscale the given weights preserving the ratio.

LLVM_ABI bool isGPUProfTarget(const Module &M)

Determines whether module targets a GPU eligable for PGO instrumentation.

cl::opt< bool > EnableVTableValueProfiling("enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison."))

static cl::opt< bool > PGOInstrumentEntry("pgo-instrument-entry", cl::init(false), cl::Hidden, cl::desc("Force to instrument function entry basicblock."))

AnalysisManager< Module > ModuleAnalysisManager

Convenience typedef for the Module analysis manager.

static cl::opt< std::string > PGOTraceFuncHash("pgo-trace-func-hash", cl::init("-"), cl::Hidden, cl::value_desc("function name"), cl::desc("Trace the hash of the function with this name."))

cl::opt< bool > NoPGOWarnMismatchComdatWeak

Implement std::hash so that hash_code can be used in STL containers.

DOTGraphTraits(bool isSimple=false)

Definition PGOInstrumentation.cpp:2480

static std::string getGraphName(const PGOUseFunc *G)

Definition PGOInstrumentation.cpp:2483

std::string getNodeLabel(const BasicBlock *Node, const PGOUseFunc *Graph)

Definition PGOInstrumentation.cpp:2487

DefaultDOTGraphTraits(bool simple=false)

static ChildIteratorType child_end(const NodeRef N)

Definition PGOInstrumentation.cpp:2468

static NodeRef getEntryNode(const PGOUseFunc *G)