IndirectCallPromotion.cpp Source File (original) (raw)

43#include

44#include

45#include

46#include

47#include <unordered_map>

48#include

49#include

51using namespace llvm;

53#define DEBUG_TYPE "pgo-icall-prom"

55 STATISTIC(NumOfPGOICallPromotion, "Number of indirect call promotions.");

56 STATISTIC(NumOfPGOICallsites, "Number of indirect call candidate sites.");

58namespace llvm {

62}

67 cl::desc("Disable indirect call promotion"));

75 cl::desc("Max number of promotions for this compilation"));

81 cl::desc("Skip Callsite up to this number for this compilation"));

86 cl::desc("Promote the target candidate even when the definition "

87 " is not available"));

93 cl::desc("Promote the target candidate only if it is a "

94 "hot function. Otherwise, warm functions can "

95 "also be promoted"));

101 cl::desc("Continue with the remaining targets instead of exiting "

102 "when failing in a candidate"));

103

104

105

106

108 cl::desc("Run indirect-call promotion in LTO "

109 "mode"));

110

111

112

115 cl::desc("Run indirect-call promotion in SamplePGO mode"));

116

117

118

121 cl::desc("Run indirect-call promotion for call instructions "

122 "only"));

123

124

125

128 cl::desc("Run indirect-call promotion for "

129 "invoke instruction only"));

130

131

132

135 cl::desc("Dump IR after transformation happens"));

136

137

138

141 cl::desc("The percentage threshold of vtable-count / function-count for "

142 "cost-benefit analysis."));

143

144

145

146

147

148

149

150

153 cl::desc("The maximum number of vtable for the last candidate."));

154

156 "icp-ignored-base-types", cl::Hidden,

158 "A list of mangled vtable type info names. Classes specified by the "

159 "type info names and their derived ones will not be vtable-ICP'ed. "

160 "Useful when the profiled types and actual types in the optimized "

161 "binary could be different due to profiling limitations. Type info "

162 "names are those string literals used in LLVM type metadata"));

163

164namespace {

165

166

167

168

169using VTableAddressPointOffsetValMap =

171

172

173struct VirtualCallSiteInfo {

174

176

178

180};

181

182

183using VirtualCallSiteTypeInfoMap =

185

186

188

189

190

191

192

193

194static std::optional<uint64_t>

195getAddressPointOffset(const GlobalVariable &VTableVar,

198 VTableVar.getMetadata(LLVMContext::MD_type, Types);

199

202 TypeId && TypeId->getString() == CompatibleType)

205 ->getZExtValue();

206

207 return std::nullopt;

208}

209

210

211

213 uint32_t AddressPointOffset) {

216 assert(AddressPointOffset <

217 M.getDataLayout().getTypeAllocSize(VTable->getValueType()) &&

218 "Out-of-bound access");

219

222 llvm::ConstantInt::get(Type::getInt32Ty(Context), AddressPointOffset));

223}

224

225

228 return PN->getIncomingBlock(U);

229

231}

232

233

234

235

236

238

240

243 "Guaranteed by ICP transformation");

244

248

249

251

253 continue;

254 UserBB = getUserBasicBlock(Use, UserInst);

255

256

257 if (UserBB != DestBB)

258 return false;

259 }

260 return UserBB != nullptr;

261}

262

263

264

265

266

268 if (!isDestBBSuitableForSink(I, DestBlock))

269 return false;

270

271

272

273 if (isa(I) || I->isEHPad() || I->mayThrow() || ->willReturn() ||

275 return false;

276

277

279 if (C->isInlineAsm() || C->cannotMerge() || C->isConvergent())

280 return false;

281

282

283 if (I->mayWriteToMemory())

284 return false;

285

286

287

288 if (I->mayReadFromMemory()) {

289

291 E = I->getParent()->end();

292 Scan != E; ++Scan) {

293

294

295

296 if (Scan->mayWriteToMemory())

297 return false;

298 }

299 }

300

302 I->moveBefore(*DestBlock, InsertPos);

303

304

305

306

307

308 return true;

309}

310

311

312

313static int tryToSinkInstructions(BasicBlock *OriginalBB,

315 int SinkCount = 0;

316

318 return SinkCount;

319

322 if (tryToSinkInstruction(&I, IndirectCallBB))

323 SinkCount++;

324

325 return SinkCount;

326}

327

328

329

330class IndirectCallPromoter {

331private:

334

335

336

337 InstrProfSymtab *const Symtab;

338

339 const bool SamplePGO;

340

341

342 const VirtualCallSiteTypeInfoMap &VirtualCSInfo;

343

344 VTableAddressPointOffsetValMap &VTableAddressPointOffsetVal;

345

346 OptimizationRemarkEmitter &ORE;

347

348 const DenseSet &IgnoredBaseTypes;

349

350

351 struct PromotionCandidate {

352 Function *const TargetFunction;

353 const uint64_t Count;

354 const uint32_t Index;

355

356

357

358

359

360

361

362

363 VTableGUIDCountsMap VTableGUIDAndCounts;

365

366 PromotionCandidate(Function *F, uint64_t C, uint32_t I)

367 : TargetFunction(F), Count(C), Index(I) {}

368 };

369

370

371

372

373

374

375 std::vector getPromotionCandidatesForCallSite(

377 uint64_t TotalCount, uint32_t NumCandidates);

378

379

380

381

382 bool tryToPromoteWithFuncCmp(

385 uint32_t NumCandidates, VTableGUIDCountsMap &VTableGUIDCounts);

386

387

388

389

390 bool tryToPromoteWithVTableCmp(

392 uint64_t TotalFuncCount, uint32_t NumCandidates,

394 VTableGUIDCountsMap &VTableGUIDCounts);

395

396

397 bool isProfitableToCompareVTables(const CallBase &CB,

399

400

401

402 bool shouldSkipVTable(uint64_t VTableGUID);

403

404

405

406

407

408

409

410

411 Instruction *computeVTableInfos(const CallBase *CB,

412 VTableGUIDCountsMap &VTableGUIDCounts,

413 std::vector &Candidates);

414

415 Constant *getOrCreateVTableAddressPointVar(GlobalVariable *GV,

416 uint64_t AddressPointOffset);

417

418 void updateFuncValueProfiles(CallBase &CB,

420 uint64_t Sum, uint32_t MaxMDCount);

421

422 void updateVPtrValueProfiles(Instruction *VPtr,

423 VTableGUIDCountsMap &VTableGUIDCounts);

424

425 bool isValidTarget(uint64_t, Function *, const CallBase &, uint64_t);

426

427public:

428 IndirectCallPromoter(

429 Function &Func, Module &M, InstrProfSymtab *Symtab, bool SamplePGO,

430 const VirtualCallSiteTypeInfoMap &VirtualCSInfo,

431 VTableAddressPointOffsetValMap &VTableAddressPointOffsetVal,

432 const DenseSet &IgnoredBaseTypes,

433 OptimizationRemarkEmitter &ORE)

434 : F(Func), M(M), Symtab(Symtab), SamplePGO(SamplePGO),

435 VirtualCSInfo(VirtualCSInfo),

436 VTableAddressPointOffsetVal(VTableAddressPointOffsetVal), ORE(ORE),

437 IgnoredBaseTypes(IgnoredBaseTypes) {}

438 IndirectCallPromoter(const IndirectCallPromoter &) = delete;

439 IndirectCallPromoter &operator=(const IndirectCallPromoter &) = delete;

440

442};

443

444}

445

446bool IndirectCallPromoter::isValidTarget(uint64_t Target,

449

450

451

452

453

454

455

456 using namespace ore;

457 if (TargetFunction == nullptr) {

458 LLVM_DEBUG(dbgs() << " Not promote: Cannot find the target\n");

459 ORE.emit([&]() {

460 return OptimizationRemarkMissed(DEBUG_TYPE, "UnableToFindTarget", &CB)

461 << "Cannot promote indirect call: target with md5sum "

462 << NV("target md5sum", Target)

463 << " not found (count=" << NV("Count", Count) << ")";

464 });

465 return false;

466 }

468 LLVM_DEBUG(dbgs() << " Not promote: target definition is not available\n");

469 ORE.emit([&]() {

470 return OptimizationRemarkMissed(DEBUG_TYPE, "NoTargetDef", &CB)

471 << "Do not promote indirect call: target with md5sum "

472 << NV("target md5sum", Target)

473 << " definition not available (count=" << ore::NV("Count", Count)

474 << ")";

475 });

476 return false;

477 }

478

479 const char *Reason = nullptr;

481

482 ORE.emit([&]() {

483 return OptimizationRemarkMissed(DEBUG_TYPE, "UnableToPromote", &CB)

484 << "Cannot promote indirect call to "

485 << NV("TargetFunction", TargetFunction)

486 << " (count=" << NV("Count", Count) << "): " << Reason;

487 });

488 return false;

489 }

490 return true;

491}

492

493

494

495std::vectorIndirectCallPromoter::PromotionCandidate

496IndirectCallPromoter::getPromotionCandidatesForCallSite(

498 uint64_t TotalCount, uint32_t NumCandidates) {

499 std::vector Ret;

500

501 LLVM_DEBUG(dbgs() << " \nWork on callsite #" << NumOfPGOICallsites << CB

502 << " Num_targets: " << ValueDataRef.size()

503 << " Num_candidates: " << NumCandidates << "\n");

504 NumOfPGOICallsites++;

507 return Ret;

508 }

509

510 for (uint32_t I = 0; I < NumCandidates; I++) {

511 uint64_t Count = ValueDataRef[I].Count;

513 (void)TotalCount;

514 uint64_t Target = ValueDataRef[I].Value;

516 << " Target_func: " << Target << "\n");

517

519 LLVM_DEBUG(dbgs() << " Not promote: User options.\n");

520 ORE.emit([&]() {

521 return OptimizationRemarkMissed(DEBUG_TYPE, "UserOptions", &CB)

522 << " Not promote: User options";

523 });

524 break;

525 }

527 LLVM_DEBUG(dbgs() << " Not promote: User option.\n");

528 ORE.emit([&]() {

529 return OptimizationRemarkMissed(DEBUG_TYPE, "UserOptions", &CB)

530 << " Not promote: User options";

531 });

532 break;

533 }

535 LLVM_DEBUG(dbgs() << " Not promote: Cutoff reached.\n");

536 ORE.emit([&]() {

537 return OptimizationRemarkMissed(DEBUG_TYPE, "CutOffReached", &CB)

538 << " Not promote: Cutoff reached";

539 });

540 break;

541 }

542

544 if (!isValidTarget(Target, TargetFunction, CB, Count)) {

546 continue;

547 else

548 break;

549 }

550

551 Ret.push_back(PromotionCandidate(TargetFunction, Count, I));

552 TotalCount -= Count;

553 }

554 return Ret;

555}

556

557Constant *IndirectCallPromoter::getOrCreateVTableAddressPointVar(

558 GlobalVariable *GV, uint64_t AddressPointOffset) {

560 VTableAddressPointOffsetVal[GV].try_emplace(AddressPointOffset, nullptr);

561 if (Inserted)

562 Iter->second = getVTableAddressPointOffset(GV, AddressPointOffset);

563 return Iter->second;

564}

565

566Instruction *IndirectCallPromoter::computeVTableInfos(

567 const CallBase *CB, VTableGUIDCountsMap &GUIDCountsMap,

568 std::vector &Candidates) {

570 return nullptr;

571

572

573

574

575

576

577

578

579

580

581

582

583

584

585

586

587

588

589

590

591

592

593

594

595

596 auto Iter = VirtualCSInfo.find(CB);

597 if (Iter == VirtualCSInfo.end())

598 return nullptr;

599

600 LLVM_DEBUG(dbgs() << "\nComputing vtable infos for callsite #"

601 << NumOfPGOICallsites << "\n");

602

603 const auto &VirtualCallInfo = Iter->second;

604 Instruction *VPtr = VirtualCallInfo.VPtr;

605

606 SmallDenseMap<Function *, int, 4> CalleeIndexMap;

607 for (size_t I = 0; I < Candidates.size(); I++)

608 CalleeIndexMap[Candidates[I].TargetFunction] = I;

609

610 uint64_t TotalVTableCount = 0;

611 auto VTableValueDataArray =

614 if (VTableValueDataArray.empty())

615 return VPtr;

616

617

618 for (const auto &V : VTableValueDataArray) {

619 uint64_t VTableVal = V.Value;

620 GUIDCountsMap[VTableVal] = V.Count;

621 GlobalVariable *VTableVar = Symtab->getGlobalVariable(VTableVal);

622 if (!VTableVar) {

623 LLVM_DEBUG(dbgs() << " Cannot find vtable definition for " << VTableVal

624 << "; maybe the vtable isn't imported\n");

625 continue;

626 }

627

628 std::optional<uint64_t> MaybeAddressPointOffset =

629 getAddressPointOffset(*VTableVar, VirtualCallInfo.CompatibleTypeStr);

630 if (!MaybeAddressPointOffset)

631 continue;

632

633 const uint64_t AddressPointOffset = *MaybeAddressPointOffset;

634

637 VTableVar, AddressPointOffset + VirtualCallInfo.FunctionOffset, M);

638 if (!Callee)

639 continue;

640 auto CalleeIndexIter = CalleeIndexMap.find(Callee);

641 if (CalleeIndexIter == CalleeIndexMap.end())

642 continue;

643

644 auto &Candidate = Candidates[CalleeIndexIter->second];

645

646

647

648 Candidate.VTableGUIDAndCounts[VTableVal] = V.Count;

649 Candidate.AddressPoints.push_back(

650 getOrCreateVTableAddressPointVar(VTableVar, AddressPointOffset));

651 }

652

653 return VPtr;

654}

655

656

657

665

668 bool AttachProfToDirectCall,

671 CB, DirectCallee,

673

674 if (AttachProfToDirectCall)

676 false);

677

678 using namespace ore;

679

680 if (ORE)

681 ORE->emit([&]() {

683 << "Promote indirect call to " << NV("DirectCallee", DirectCallee)

684 << " with count " << NV("Count", Count) << " out of "

685 << NV("TotalCount", TotalCount);

686 });

687 return NewInst;

688}

689

690

691bool IndirectCallPromoter::tryToPromoteWithFuncCmp(

694 uint32_t NumCandidates, VTableGUIDCountsMap &VTableGUIDCounts) {

696

697 for (const auto &C : Candidates) {

700 SamplePGO, &ORE);

701 assert(TotalCount >= FuncCount);

702 TotalCount -= FuncCount;

703 NumOfPGOICallPromotion++;

704 NumPromoted++;

705

706

707 ICallProfDataRef[C.Index].Count = 0;

709 continue;

710

711

712

713

714

716 for (const auto &[GUID, VTableCount] : C.VTableGUIDAndCounts)

717 SumVTableCount += VTableCount;

718

719 for (const auto &[GUID, VTableCount] : C.VTableGUIDAndCounts) {

720 APInt APFuncCount((unsigned)128, FuncCount, false );

721 APFuncCount *= VTableCount;

722 VTableGUIDCounts[GUID] -= APFuncCount.udiv(SumVTableCount).getZExtValue();

723 }

724 }

725 if (NumPromoted == 0)

726 return false;

727

728 assert(NumPromoted <= ICallProfDataRef.size() &&

729 "Number of promoted functions should not be greater than the number "

730 "of values in profile metadata");

731

732 updateFuncValueProfiles(CB, ICallProfDataRef, TotalCount, NumCandidates);

733 updateVPtrValueProfiles(VPtr, VTableGUIDCounts);

734 return true;

735}

736

737void IndirectCallPromoter::updateFuncValueProfiles(

739 uint64_t TotalCount, uint32_t MaxMDCount) {

740

741 CB.setMetadata(LLVMContext::MD_prof, nullptr);

742

743

745 const InstrProfValueData &RHS) {

746 return LHS.Count > RHS.Count;

747 });

748

752 [](uint64_t Count, const InstrProfValueData &ProfData) {

753 return ProfData.Count <= Count;

754 }));

755

756

757 if (TotalCount != 0)

758 annotateValueSite(M, CB, VDs, TotalCount, IPVK_IndirectCallTarget,

759 MaxMDCount);

760}

761

762void IndirectCallPromoter::updateVPtrValueProfiles(

763 Instruction *VPtr, VTableGUIDCountsMap &VTableGUIDCounts) {

765 !VPtr->getMetadata(LLVMContext::MD_prof))

766 return;

767 VPtr->setMetadata(LLVMContext::MD_prof, nullptr);

768 std::vector VTableValueProfiles;

769 uint64_t TotalVTableCount = 0;

770 for (auto [GUID, Count] : VTableGUIDCounts) {

772 continue;

773

774 VTableValueProfiles.push_back({GUID, Count});

775 TotalVTableCount += Count;

776 }

778 [](const InstrProfValueData &LHS, const InstrProfValueData &RHS) {

779 return LHS.Count > RHS.Count;

780 });

781

782 annotateValueSite(M, *VPtr, VTableValueProfiles, TotalVTableCount,

783 IPVK_VTableTarget, VTableValueProfiles.size());

784}

785

786bool IndirectCallPromoter::tryToPromoteWithVTableCmp(

788 uint64_t TotalFuncCount, uint32_t NumCandidates,

790 VTableGUIDCountsMap &VTableGUIDCounts) {

792

793 for (const auto &Candidate : Candidates) {

794 for (auto &[GUID, Count] : Candidate.VTableGUIDAndCounts)

796

797

798

799

802 CB, VPtr, Candidate.TargetFunction, Candidate.AddressPoints,

804 TotalFuncCount - Candidate.Count));

805

806 int SinkCount = tryToSinkInstructions(OriginalBB, CB.getParent());

807

808 ORE.emit([&]() {

810

811 const auto &VTableGUIDAndCounts = Candidate.VTableGUIDAndCounts;

812 Remark << "Promote indirect call to "

813 << ore::NV("DirectCallee", Candidate.TargetFunction)

814 << " with count " << ore::NV("Count", Candidate.Count)

815 << " out of " << ore::NV("TotalCount", TotalFuncCount) << ", sink "

816 << ore::NV("SinkCount", SinkCount)

817 << " instruction(s) and compare "

818 << ore::NV("VTable", VTableGUIDAndCounts.size())

819 << " vtable(s): {";

820

821

822 std::set<uint64_t> GUIDSet;

823 for (auto [GUID, Count] : VTableGUIDAndCounts)

824 GUIDSet.insert(GUID);

825 for (auto Iter = GUIDSet.begin(); Iter != GUIDSet.end(); Iter++) {

826 if (Iter != GUIDSet.begin())

829 }

830

832

834 });

835

836 PromotedFuncCount.push_back({Candidate.Index, Candidate.Count});

837

838 assert(TotalFuncCount >= Candidate.Count &&

839 "Within one prof metadata, total count is the sum of counts from "

840 "individual <target, count> pairs");

841

842

843

844 TotalFuncCount -= std::min(TotalFuncCount, Candidate.Count);

845 NumOfPGOICallPromotion++;

846 }

847

848 if (PromotedFuncCount.empty())

849 return false;

850

851

852

853

854

855

856

857 for (size_t I = 0; I < PromotedFuncCount.size(); I++) {

858 uint32_t Index = PromotedFuncCount[I].first;

859 ICallProfDataRef[Index].Count -=

860 std::max(PromotedFuncCount[I].second, ICallProfDataRef[Index].Count);

861 }

862 updateFuncValueProfiles(CB, ICallProfDataRef, TotalFuncCount, NumCandidates);

863 updateVPtrValueProfiles(VPtr, VTableGUIDCounts);

864 return true;

865}

866

867

868

869bool IndirectCallPromoter::processFunction(ProfileSummaryInfo *PSI) {

871 ICallPromotionAnalysis ICallAnalysis;

873 uint32_t NumCandidates;

874 uint64_t TotalCount;

876 CB, TotalCount, NumCandidates);

877 if (!NumCandidates)

878 continue;

880

882 LLVM_DEBUG(dbgs() << "Don't promote the cold candidate: TotalCount="

883 << TotalCount << "\n");

884 continue;

885 }

886

888 LLVM_DEBUG(dbgs() << "Don't promote the non-hot candidate: TotalCount="

889 << TotalCount << "\n");

890 continue;

891 }

892 }

893

894 auto PromotionCandidates = getPromotionCandidatesForCallSite(

895 *CB, ICallProfDataRef, TotalCount, NumCandidates);

896

897 VTableGUIDCountsMap VTableGUIDCounts;

899 computeVTableInfos(CB, VTableGUIDCounts, PromotionCandidates);

900

901 if (isProfitableToCompareVTables(*CB, PromotionCandidates))

902 Changed |= tryToPromoteWithVTableCmp(*CB, VPtr, PromotionCandidates,

903 TotalCount, NumCandidates,

904 ICallProfDataRef, VTableGUIDCounts);

905 else

906 Changed |= tryToPromoteWithFuncCmp(*CB, VPtr, PromotionCandidates,

907 TotalCount, ICallProfDataRef,

908 NumCandidates, VTableGUIDCounts);

909 }

911}

912

913

914

915bool IndirectCallPromoter::isProfitableToCompareVTables(

918 return false;

919 LLVM_DEBUG(dbgs() << "\nEvaluating vtable profitability for callsite #"

920 << NumOfPGOICallsites << CB << "\n");

921 const size_t CandidateSize = Candidates.size();

922 for (size_t I = 0; I < CandidateSize; I++) {

923 auto &Candidate = Candidates[I];

924 auto &VTableGUIDAndCounts = Candidate.VTableGUIDAndCounts;

925

927 dbgs() << " Candidate " << I << " FunctionCount: " << Candidate.Count

928 << ", VTableCounts:";

929 for (const auto &[GUID, Count] : VTableGUIDAndCounts)

932 dbgs() << "\n";

933 });

934

935 uint64_t CandidateVTableCount = 0;

936

937 for (auto &[GUID, Count] : VTableGUIDAndCounts) {

938 CandidateVTableCount += Count;

939

940 if (shouldSkipVTable(GUID))

941 return false;

942 }

943

946 dbgs() << " function count " << Candidate.Count

947 << " and its vtable sum count " << CandidateVTableCount

948 << " have discrepancies. Bail out vtable comparison.\n");

949 return false;

950 }

951

952

953

954

955

956

957

958 int MaxNumVTable = 1;

959 if (I == CandidateSize - 1)

961

962 if ((int)Candidate.AddressPoints.size() > MaxNumVTable) {

963 LLVM_DEBUG(dbgs() << " allow at most " << MaxNumVTable << " and got "

964 << Candidate.AddressPoints.size()

965 << " vtables. Bail out for vtable comparison.\n");

966 return false;

967 }

968 }

969

970 return true;

971}

972

973bool IndirectCallPromoter::shouldSkipVTable(uint64_t VTableGUID) {

974 if (IgnoredBaseTypes.empty())

975 return false;

976

978

979 assert(VTableVar && "VTableVar must exist for GUID in VTableGUIDAndCounts");

980

982 VTableVar->getMetadata(LLVMContext::MD_type, Types);

983

984 for (auto *Type : Types)

986 if (IgnoredBaseTypes.contains(TypeId->getString())) {

987 LLVM_DEBUG(dbgs() << " vtable profiles should be ignored. Bail "

988 "out of vtable comparison.");

989 return true;

990 }

991 return false;

992}

993

994

995

996

997

998

999

1000static void

1002 VirtualCallSiteTypeInfoMap &VirtualCSInfo) {

1003

1004

1005

1006

1007

1008

1009

1010

1013 if (!TypeTestFunc || TypeTestFunc->use_empty())

1014 return;

1015

1019 };

1020

1023 if (!CI)

1024 continue;

1026 if (!TypeMDVal)

1027 continue;

1028 auto *CompatibleTypeId = dyn_cast(TypeMDVal->getMetadata());

1029 if (!CompatibleTypeId)

1030 continue;

1031

1032

1033

1036 auto &DT = LookupDomTree(*CI->getFunction());

1038

1039 for (auto &DevirtCall : DevirtCalls) {

1040 CallBase &CB = DevirtCall.CB;

1041

1042

1045 if (!VTablePtr)

1046 continue;

1047 VirtualCSInfo[&CB] = {DevirtCall.Offset, VTablePtr,

1048 CompatibleTypeId->getString()};

1049 }

1050 }

1051}

1052

1053

1057 return false;

1060 std::string SymtabFailure = toString(std::move(E));

1061 M.getContext().emitError("Failed to create symtab: " + SymtabFailure);

1062 return false;

1063 }

1065 VirtualCallSiteTypeInfoMap VirtualCSInfo;

1066

1068

1071

1073 }

1074

1075

1076

1077

1078

1079

1080

1081

1082 VTableAddressPointOffsetValMap VTableAddressPointOffsetVal;

1083

1084 for (auto &F : M) {

1085 if (F.isDeclaration() || F.hasOptNone())

1086 continue;

1087

1088 auto &FAM =

1091

1092 IndirectCallPromoter CallPromoter(F, M, &Symtab, SamplePGO, VirtualCSInfo,

1093 VTableAddressPointOffsetVal,

1094 IgnoredBaseTypes, ORE);

1095 bool FuncChanged = CallPromoter.processFunction(PSI);

1099 }

1103 break;

1104 }

1105 }

1107}

1108

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

This file defines the DenseMap class.

This header defines various interfaces for pass management in LLVM.

Machine Check Debug Module

static bool processFunction(Function &F, NVPTXTargetMachine &TM)

This file provides the interface for IR based instrumentation passes ( (profile-gen,...

FunctionAnalysisManager FAM

ModuleAnalysisManager MAM

This file contains the declarations for profiling metadata utility functions.

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

Class for arbitrary precision integers.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

bool empty() const

empty - Check if the array is empty.

LLVM Basic Block Representation.

LLVM_ABI const_iterator getFirstInsertionPt() const

Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...

LLVM_ABI const BasicBlock * getUniquePredecessor() const

Return the predecessor of this block if it has a unique predecessor block.

InstListType::iterator iterator

Instruction iterators...

const Instruction * getTerminator() const LLVM_READONLY

Returns the terminator instruction if the block is well formed or null if the block is not well forme...

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

static Constant * getInBoundsGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList)

Create an "inbounds" getelementptr.

This is an important base class in LLVM.

iterator find(const_arg_type_t< KeyT > Val)

Implements a dense probed hash-table based set.

Analysis pass which computes a DominatorTree.

Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.

Lightweight error class with error context and mandatory checking.

MDNode * getMetadata(unsigned KindID) const

Get the current metadata attachments for the given kind, if any.

LLVM_ABI bool isDeclaration() const

Return true if the primary definition of this global value is outside of the current translation unit...

A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...

GlobalVariable * getGlobalVariable(uint64_t MD5Hash) const

Return the global variable corresponding to md5 hash.

LLVM_ABI Error create(object::SectionRef &Section)

Create InstrProfSymtab from an object file section which contains function PGO names.

Function * getFunction(uint64_t FuncMD5Hash) const

Return function from the name's md5 hash. Return nullptr if not found.

LLVM_ABI bool isDebugOrPseudoInst() const LLVM_READONLY

Return true if the instruction is a DbgInfoIntrinsic or PseudoProbeInst.

LLVM_ABI unsigned getNumSuccessors() const LLVM_READONLY

Return the number of successors that this instruction has.

MDNode * getMetadata(unsigned KindID) const

Get the metadata of given kind attached to this Instruction.

LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)

Set the metadata of the specified kind to the specified node.

This is an important class for using LLVM in a threaded context.

LLVM_ABI MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight, bool IsExpected=false)

Return metadata containing two branch weights.

A Module instance is used to store all the information related to an LLVM module.

MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

An analysis pass based on the new PM to deliver ProfileSummaryInfo.

Analysis providing profile information.

bool hasProfileSummary() const

Returns true if profile summary is available.

LLVM_ABI bool isColdCount(uint64_t C) const

Returns true if count C is considered cold.

LLVM_ABI bool isHotCount(uint64_t C) const

Returns true if count C is considered hot.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

Target - Wrapper for Target specific information.

The instances of the Type class are immutable: once they are created, they are never changed.

static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)

static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)

A Use represents the edge between a Value definition and its users.

User * getUser() const

Returns the User that contains this Use.

LLVM_ABI LLVMContext & getContext() const

All values hold a context through their type.

iterator_range< use_iterator > uses()

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

void insert_range(Range &&R)

const ParentTy * getParent() const

@ C

The default llvm calling convention, compatible with C.

@ BasicBlock

Various leaf nodes.

LLVM_ABI Function * getDeclarationIfExists(const Module *M, ID id)

Look up the Function declaration of the intrinsic id in the Module M and return it if it exists.

initializer< Ty > init(const Ty &Val)

Add a small namespace to avoid name clashes with the classes used in the streaming interface.

DiagnosticInfoOptimizationBase::Argument NV

LLVM_ABI CallBase & promoteIndirectCall(CallBase &CB, Function *F, uint64_t Count, uint64_t TotalCount, bool AttachProfToDirectCall, OptimizationRemarkEmitter *ORE)

Definition IndirectCallPromotion.cpp:666

NodeAddr< FuncNode * > Func

friend class Instruction

Iterator for Instructions in a `BasicBlock.

This is an optimization pass for GlobalISel generic memory operations.

auto drop_begin(T &&RangeOrContainer, size_t N=1)

Return a range covering RangeOrContainer with the first N elements excluded.

void stable_sort(R &&Range)

LLVM_ABI bool isLegalToPromote(const CallBase &CB, Function *Callee, const char **FailureReason=nullptr)

Return true if the given indirect call site can be made to call Callee.

std::vector< CallBase * > findIndirectCalls(Function &F)

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

LLVM_ABI CallBase & promoteCallWithIfThenElse(CallBase &CB, Function *Callee, MDNode *BranchWeights=nullptr)

Promote the given indirect call site to conditionally call Callee.

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy

Provide the FunctionAnalysisManager to Module proxy.

cl::opt< unsigned > MaxNumVTableAnnotations("icp-max-num-vtables", cl::init(6), cl::Hidden, cl::desc("Max number of vtables annotated for a vtable load instruction."))

auto upper_bound(R &&Range, T &&Value)

Provide wrappers to std::upper_bound which take ranges instead of having to pass begin/end explicitly...

cl::opt< bool > EnableVTableProfileUse("enable-vtable-profile-use", cl::init(false), cl::desc("If ThinLTO and WPD is enabled and this option is true, vtable " "profiles will be used by ICP pass for more efficient indirect " "call sequence. If false, type profiles won't be used."))

Definition IndirectCallPromotion.cpp:61

LLVM_ABI void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)

Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...

auto reverse(ContainerTy &&C)

void sort(IteratorTy Start, IteratorTy End)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

FunctionAddr VTableAddr Count

LLVM_ABI SmallVector< InstrProfValueData, 4 > getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)

Extract the value profile data from Inst and returns them if Inst is annotated with value profile dat...

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

MutableArrayRef(T &OneElt) -> MutableArrayRef< T >

ArrayRef(const T &OneElt) -> ArrayRef< T >

std::string toString(const APInt &I, unsigned Radix, bool Signed, bool formatAsCLiteral=false, bool UpperCase=true, bool InsertSeparators=false)

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)

Scale an individual branch count.

LLVM_ABI void setFittedBranchWeights(Instruction &I, ArrayRef< uint64_t > Weights, bool IsExpected, bool ElideAllZero=false)

Variant of setBranchWeights where the Weights will be fit first to uint32_t by shifting right.

uint64_t calculateCountScale(uint64_t MaxCount)

Calculate what to divide by to scale counts.

LLVM_ABI CallBase & promoteCallWithVTableCmp(CallBase &CB, Instruction *VPtr, Function *Callee, ArrayRef< Constant * > AddressPoints, MDNode *BranchWeights)

This is similar to promoteCallWithIfThenElse except that the condition to promote a virtual call is t...

void findDevirtualizableCallsForTypeTest(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< CallInst * > &Assumes, const CallInst *CI, DominatorTree &DT)

Given a call to the intrinsic @llvm.type.test, find all devirtualizable call sites based on the call ...

AnalysisManager< Module > ModuleAnalysisManager

Convenience typedef for the Module analysis manager.

std::pair< Function *, Constant * > getFunctionAtVTableOffset(GlobalVariable *GV, uint64_t Offset, Module &M)

Given a vtable and a specified offset, returns the function and the trivial pointer at the specified ...

static Instruction * tryGetVTableInstruction(CallBase *CB)