IndirectCallPromotion.cpp Source File (original) (raw)

43#include

44#include

45#include

46#include

47#include <unordered_map>

48#include

49#include

51using namespace llvm;

53#define DEBUG_TYPE "pgo-icall-prom"

55 STATISTIC(NumOfPGOICallPromotion, "Number of indirect call promotions.");

56 STATISTIC(NumOfPGOICallsites, "Number of indirect call candidate sites.");

60namespace llvm {

62}

67 cl::desc("Disable indirect call promotion"));

75 cl::desc("Max number of promotions for this compilation"));

81 cl::desc("Skip Callsite up to this number for this compilation"));

87 cl::desc("Run indirect-call promotion in LTO "

88 "mode"));

94 cl::desc("Run indirect-call promotion in SamplePGO mode"));

100 cl::desc("Run indirect-call promotion for call instructions "

101 "only"));

102

103

104

107 cl::desc("Run indirect-call promotion for "

108 "invoke instruction only"));

109

110

111

114 cl::desc("Dump IR after transformation happens"));

115

116

117

120 cl::desc("The percentage threshold of vtable-count / function-count for "

121 "cost-benefit analysis."));

122

123

124

125

126

127

128

129

132 cl::desc("The maximum number of vtable for the last candidate."));

133

135 "icp-ignored-base-types", cl::Hidden,

137 "A list of mangled vtable type info names. Classes specified by the "

138 "type info names and their derived ones will not be vtable-ICP'ed. "

139 "Useful when the profiled types and actual types in the optimized "

140 "binary could be different due to profiling limitations. Type info "

141 "names are those string literals used in LLVM type metadata"));

142

143namespace {

144

145

146

147

148using VTableAddressPointOffsetValMap =

150

151

152struct VirtualCallSiteInfo {

153

155

157

159};

160

161

162using VirtualCallSiteTypeInfoMap =

164

165

167

168

169

170

171

172

173static std::optional<uint64_t>

174getAddressPointOffset(const GlobalVariable &VTableVar,

177 VTableVar.getMetadata(LLVMContext::MD_type, Types);

178

180 if (auto *TypeId = dyn_cast(Type->getOperand(1).get());

181 TypeId && TypeId->getString() == CompatibleType)

182 return cast(

183 cast(Type->getOperand(0))->getValue())

184 ->getZExtValue();

185

186 return std::nullopt;

187}

188

189

190

192 uint32_t AddressPointOffset) {

195 assert(AddressPointOffset <

196 M.getDataLayout().getTypeAllocSize(VTable->getValueType()) &&

197 "Out-of-bound access");

198

201 llvm::ConstantInt::get(Type::getInt32Ty(Context), AddressPointOffset));

202}

203

204

206 if (PHINode *PN = dyn_cast(UserInst))

207 return PN->getIncomingBlock(U);

208

210}

211

212

213

214

215

217

219

222 "Guaranteed by ICP transformation");

223

227

228

230

232 continue;

233 UserBB = getUserBasicBlock(Use, UserInst);

234

235

236 if (UserBB != DestBB)

237 return false;

238 }

239 return UserBB != nullptr;

240}

241

242

243

244

245

247 if (!isDestBBSuitableForSink(I, DestBlock))

248 return false;

249

250

251

252 if (isa(I) || I->isEHPad() || I->mayThrow() || ->willReturn() ||

253 isa(I))

254 return false;

255

256

257 if (const auto *C = dyn_cast(I))

258 if (C->isInlineAsm() || C->cannotMerge() || C->isConvergent())

259 return false;

260

261

262 if (I->mayWriteToMemory())

263 return false;

264

265

266

267 if (I->mayReadFromMemory()) {

268

270 E = I->getParent()->end();

271 Scan != E; ++Scan) {

272

273

274

275 if (Scan->mayWriteToMemory())

276 return false;

277 }

278 }

279

281 I->moveBefore(*DestBlock, InsertPos);

282

283

284

285

286

287 return true;

288}

289

290

291

292static int tryToSinkInstructions(BasicBlock *OriginalBB,

294 int SinkCount = 0;

295

297 return SinkCount;

298

301 if (tryToSinkInstruction(&I, IndirectCallBB))

302 SinkCount++;

303

304 return SinkCount;

305}

306

307

308

309class IndirectCallPromoter {

310private:

313

314

315

317

318 const bool SamplePGO;

319

320

321 const VirtualCallSiteTypeInfoMap &VirtualCSInfo;

322

323 VTableAddressPointOffsetValMap &VTableAddressPointOffsetVal;

324

326

328

329

330 struct PromotionCandidate {

331 Function *const TargetFunction;

333

334

335

336

337

338

339

340

341 VTableGUIDCountsMap VTableGUIDAndCounts;

343

344 PromotionCandidate(Function *F, uint64_t C) : TargetFunction(F), Count(C) {}

345 };

346

347

348

349

350

351

352 std::vector getPromotionCandidatesForCallSite(

355

356

357

358

364 VTableGUIDCountsMap &VTableGUIDCounts);

365

366

367

368

369 bool tryToPromoteWithVTableCmp(

373 VTableGUIDCountsMap &VTableGUIDCounts);

374

375

376 bool isProfitableToCompareVTables(const CallBase &CB,

378

379

380

381 bool shouldSkipVTable(uint64_t VTableGUID);

382

383

384

385

386

387

388

389

391 VTableGUIDCountsMap &VTableGUIDCounts,

392 std::vector &Candidates);

393

395 uint64_t AddressPointOffset);

396

399

400 void updateVPtrValueProfiles(Instruction *VPtr,

401 VTableGUIDCountsMap &VTableGUIDCounts);

402

403public:

404 IndirectCallPromoter(

406 const VirtualCallSiteTypeInfoMap &VirtualCSInfo,

407 VTableAddressPointOffsetValMap &VTableAddressPointOffsetVal,

410 : F(Func), M(M), Symtab(Symtab), SamplePGO(SamplePGO),

411 VirtualCSInfo(VirtualCSInfo),

412 VTableAddressPointOffsetVal(VTableAddressPointOffsetVal), ORE(ORE),

413 IgnoredBaseTypes(IgnoredBaseTypes) {}

414 IndirectCallPromoter(const IndirectCallPromoter &) = delete;

415 IndirectCallPromoter &operator=(const IndirectCallPromoter &) = delete;

416

418};

419

420}

421

422

423

424std::vectorIndirectCallPromoter::PromotionCandidate

425IndirectCallPromoter::getPromotionCandidatesForCallSite(

428 std::vector Ret;

429

430 LLVM_DEBUG(dbgs() << " \nWork on callsite #" << NumOfPGOICallsites << CB

431 << " Num_targets: " << ValueDataRef.size()

432 << " Num_candidates: " << NumCandidates << "\n");

433 NumOfPGOICallsites++;

436 return Ret;

437 }

438

439 for (uint32_t I = 0; I < NumCandidates; I++) {

440 uint64_t Count = ValueDataRef[I].Count;

441 assert(Count <= TotalCount);

442 (void)TotalCount;

444 LLVM_DEBUG(dbgs() << " Candidate " << I << " Count=" << Count

445 << " Target_func: " << Target << "\n");

446

448 LLVM_DEBUG(dbgs() << " Not promote: User options.\n");

449 ORE.emit([&]() {

451 << " Not promote: User options";

452 });

453 break;

454 }

455 if (ICPCallOnly && isa(CB)) {

456 LLVM_DEBUG(dbgs() << " Not promote: User option.\n");

457 ORE.emit([&]() {

459 << " Not promote: User options";

460 });

461 break;

462 }

464 LLVM_DEBUG(dbgs() << " Not promote: Cutoff reached.\n");

465 ORE.emit([&]() {

467 << " Not promote: Cutoff reached";

468 });

469 break;

470 }

471

472

473

474

475

476

477

478

480 if (TargetFunction == nullptr || TargetFunction->isDeclaration()) {

481 LLVM_DEBUG(dbgs() << " Not promote: Cannot find the target\n");

482 ORE.emit([&]() {

484 << "Cannot promote indirect call: target with md5sum "

485 << ore::NV("target md5sum", Target) << " not found";

486 });

487 break;

488 }

489

490 const char *Reason = nullptr;

492 using namespace ore;

493

494 ORE.emit([&]() {

496 << "Cannot promote indirect call to "

497 << NV("TargetFunction", TargetFunction) << " with count of "

498 << NV("Count", Count) << ": " << Reason;

499 });

500 break;

501 }

502

503 Ret.push_back(PromotionCandidate(TargetFunction, Count));

504 TotalCount -= Count;

505 }

506 return Ret;

507}

508

509Constant *IndirectCallPromoter::getOrCreateVTableAddressPointVar(

512 VTableAddressPointOffsetVal[GV].try_emplace(AddressPointOffset, nullptr);

513 if (Inserted)

514 Iter->second = getVTableAddressPointOffset(GV, AddressPointOffset);

515 return Iter->second;

516}

517

518Instruction *IndirectCallPromoter::computeVTableInfos(

519 const CallBase *CB, VTableGUIDCountsMap &GUIDCountsMap,

520 std::vector &Candidates) {

522 return nullptr;

523

524

525

526

527

528

529

530

531

532

533

534

535

536

537

538

539

540

541

542

543

544

545

546

547

548 auto Iter = VirtualCSInfo.find(CB);

549 if (Iter == VirtualCSInfo.end())

550 return nullptr;

551

552 LLVM_DEBUG(dbgs() << "\nComputing vtable infos for callsite #"

553 << NumOfPGOICallsites << "\n");

554

555 const auto &VirtualCallInfo = Iter->second;

556 Instruction *VPtr = VirtualCallInfo.VPtr;

557

559 for (size_t I = 0; I < Candidates.size(); I++)

560 CalleeIndexMap[Candidates[I].TargetFunction] = I;

561

562 uint64_t TotalVTableCount = 0;

563 auto VTableValueDataArray =

566 if (VTableValueDataArray.empty())

567 return VPtr;

568

569

570 for (const auto &V : VTableValueDataArray) {

572 GUIDCountsMap[VTableVal] = V.Count;

573 GlobalVariable *VTableVar = Symtab->getGlobalVariable(VTableVal);

574 if (!VTableVar) {

575 LLVM_DEBUG(dbgs() << " Cannot find vtable definition for " << VTableVal

576 << "; maybe the vtable isn't imported\n");

577 continue;

578 }

579

580 std::optional<uint64_t> MaybeAddressPointOffset =

581 getAddressPointOffset(*VTableVar, VirtualCallInfo.CompatibleTypeStr);

582 if (!MaybeAddressPointOffset)

583 continue;

584

585 const uint64_t AddressPointOffset = *MaybeAddressPointOffset;

586

589 VTableVar, AddressPointOffset + VirtualCallInfo.FunctionOffset, M);

590 if (!Callee)

591 continue;

592 auto CalleeIndexIter = CalleeIndexMap.find(Callee);

593 if (CalleeIndexIter == CalleeIndexMap.end())

594 continue;

595

596 auto &Candidate = Candidates[CalleeIndexIter->second];

597

598

599

600 Candidate.VTableGUIDAndCounts[VTableVal] = V.Count;

601 Candidate.AddressPoints.push_back(

602 getOrCreateVTableAddressPointVar(VTableVar, AddressPointOffset));

603 }

604

605 return VPtr;

606}

607

608

609

616}

617

620 bool AttachProfToDirectCall,

623 CB, DirectCallee,

625

626 if (AttachProfToDirectCall)

628 false);

629

630 using namespace ore;

631

632 if (ORE)

633 ORE->emit([&]() {

635 << "Promote indirect call to " << NV("DirectCallee", DirectCallee)

636 << " with count " << NV("Count", Count) << " out of "

637 << NV("TotalCount", TotalCount);

638 });

639 return NewInst;

640}

641

642

643bool IndirectCallPromoter::tryToPromoteWithFuncCmp(

646 uint32_t NumCandidates, VTableGUIDCountsMap &VTableGUIDCounts) {

648

649 for (const auto &C : Candidates) {

652 SamplePGO, &ORE);

653 assert(TotalCount >= FuncCount);

654 TotalCount -= FuncCount;

655 NumOfPGOICallPromotion++;

656 NumPromoted++;

657

659 continue;

660

661

662

663

664

666 for (const auto &[GUID, VTableCount] : C.VTableGUIDAndCounts)

667 SumVTableCount += VTableCount;

668

669 for (const auto &[GUID, VTableCount] : C.VTableGUIDAndCounts) {

670 APInt APFuncCount((unsigned)128, FuncCount, false );

671 APFuncCount *= VTableCount;

672 VTableGUIDCounts[GUID] -= APFuncCount.udiv(SumVTableCount).getZExtValue();

673 }

674 }

675 if (NumPromoted == 0)

676 return false;

677

678 assert(NumPromoted <= ICallProfDataRef.size() &&

679 "Number of promoted functions should not be greater than the number "

680 "of values in profile metadata");

681

682

683 updateFuncValueProfiles(CB, ICallProfDataRef.slice(NumPromoted), TotalCount,

684 NumCandidates);

685 updateVPtrValueProfiles(VPtr, VTableGUIDCounts);

686 return true;

687}

688

689void IndirectCallPromoter::updateFuncValueProfiles(

692

693 CB.setMetadata(LLVMContext::MD_prof, nullptr);

694

695 if (TotalCount != 0)

696 annotateValueSite(M, CB, CallVDs, TotalCount, IPVK_IndirectCallTarget,

697 MaxMDCount);

698}

699

700void IndirectCallPromoter::updateVPtrValueProfiles(

701 Instruction *VPtr, VTableGUIDCountsMap &VTableGUIDCounts) {

703 !VPtr->getMetadata(LLVMContext::MD_prof))

704 return;

705 VPtr->setMetadata(LLVMContext::MD_prof, nullptr);

706 std::vector VTableValueProfiles;

707 uint64_t TotalVTableCount = 0;

708 for (auto [GUID, Count] : VTableGUIDCounts) {

709 if (Count == 0)

710 continue;

711

712 VTableValueProfiles.push_back({GUID, Count});

713 TotalVTableCount += Count;

714 }

716 [](const InstrProfValueData &LHS, const InstrProfValueData &RHS) {

717 return LHS.Count > RHS.Count;

718 });

719

720 annotateValueSite(M, *VPtr, VTableValueProfiles, TotalVTableCount,

721 IPVK_VTableTarget, VTableValueProfiles.size());

722}

723

724bool IndirectCallPromoter::tryToPromoteWithVTableCmp(

728 VTableGUIDCountsMap &VTableGUIDCounts) {

730

731 for (const auto &Candidate : Candidates) {

732 for (auto &[GUID, Count] : Candidate.VTableGUIDAndCounts)

733 VTableGUIDCounts[GUID] -= Count;

734

735

736

737

740 CB, VPtr, Candidate.TargetFunction, Candidate.AddressPoints,

742 TotalFuncCount - Candidate.Count));

743

744 int SinkCount = tryToSinkInstructions(OriginalBB, CB.getParent());

745

746 ORE.emit([&]() {

748

749 const auto &VTableGUIDAndCounts = Candidate.VTableGUIDAndCounts;

750 Remark << "Promote indirect call to "

751 << ore::NV("DirectCallee", Candidate.TargetFunction)

752 << " with count " << ore::NV("Count", Candidate.Count)

753 << " out of " << ore::NV("TotalCount", TotalFuncCount) << ", sink "

754 << ore::NV("SinkCount", SinkCount)

755 << " instruction(s) and compare "

756 << ore::NV("VTable", VTableGUIDAndCounts.size())

757 << " vtable(s): {";

758

759

760 std::set<uint64_t> GUIDSet;

761 for (auto [GUID, Count] : VTableGUIDAndCounts)

762 GUIDSet.insert(GUID);

763 for (auto Iter = GUIDSet.begin(); Iter != GUIDSet.end(); Iter++) {

764 if (Iter != GUIDSet.begin())

766 Remark << ore::NV("VTable", Symtab->getGlobalVariable(*Iter));

767 }

768

770

772 });

773

774 PromotedFuncCount.push_back(Candidate.Count);

775

776 assert(TotalFuncCount >= Candidate.Count &&

777 "Within one prof metadata, total count is the sum of counts from "

778 "individual <target, count> pairs");

779

780

781

782 TotalFuncCount -= std::min(TotalFuncCount, Candidate.Count);

783 NumOfPGOICallPromotion++;

784 }

785

786 if (PromotedFuncCount.empty())

787 return false;

788

789

790

791

792

793

794

795 for (size_t I = 0; I < PromotedFuncCount.size(); I++)

796 ICallProfDataRef[I].Count -=

797 std::max(PromotedFuncCount[I], ICallProfDataRef[I].Count);

798

799 llvm::stable_sort(ICallProfDataRef, [](const InstrProfValueData &LHS,

800 const InstrProfValueData &RHS) {

801 return LHS.Count > RHS.Count;

802 });

803

805 ICallProfDataRef.begin(),

807 [](uint64_t Count, const InstrProfValueData &ProfData) {

808 return ProfData.Count <= Count;

809 }));

810 updateFuncValueProfiles(CB, VDs, TotalFuncCount, NumCandidates);

811 updateVPtrValueProfiles(VPtr, VTableGUIDCounts);

812 return true;

813}

814

815

816

818 bool Changed = false;

824 CB, TotalCount, NumCandidates);

825 if (!NumCandidates ||

827 continue;

828

829 auto PromotionCandidates = getPromotionCandidatesForCallSite(

830 *CB, ICallProfDataRef, TotalCount, NumCandidates);

831

832 VTableGUIDCountsMap VTableGUIDCounts;

834 computeVTableInfos(CB, VTableGUIDCounts, PromotionCandidates);

835

836 if (isProfitableToCompareVTables(*CB, PromotionCandidates))

837 Changed |= tryToPromoteWithVTableCmp(*CB, VPtr, PromotionCandidates,

838 TotalCount, NumCandidates,

839 ICallProfDataRef, VTableGUIDCounts);

840 else

841 Changed |= tryToPromoteWithFuncCmp(*CB, VPtr, PromotionCandidates,

842 TotalCount, ICallProfDataRef,

843 NumCandidates, VTableGUIDCounts);

844 }

845 return Changed;

846}

847

848

849

850bool IndirectCallPromoter::isProfitableToCompareVTables(

853 return false;

854 LLVM_DEBUG(dbgs() << "\nEvaluating vtable profitability for callsite #"

855 << NumOfPGOICallsites << CB << "\n");

856 const size_t CandidateSize = Candidates.size();

857 for (size_t I = 0; I < CandidateSize; I++) {

858 auto &Candidate = Candidates[I];

859 auto &VTableGUIDAndCounts = Candidate.VTableGUIDAndCounts;

860

861 LLVM_DEBUG(dbgs() << " Candidate " << I << " FunctionCount: "

862 << Candidate.Count << ", VTableCounts:");

863

864 for ([[maybe_unused]] auto &[GUID, Count] : VTableGUIDAndCounts)

865 LLVM_DEBUG(dbgs() << " {" << Symtab->getGlobalVariable(GUID)->getName()

866 << ", " << Count << "}");

868

869 uint64_t CandidateVTableCount = 0;

870

871 for (auto &[GUID, Count] : VTableGUIDAndCounts) {

872 CandidateVTableCount += Count;

873

874 if (shouldSkipVTable(GUID))

875 return false;

876 }

877

880 dbgs() << " function count " << Candidate.Count

881 << " and its vtable sum count " << CandidateVTableCount

882 << " have discrepancies. Bail out vtable comparison.\n");

883 return false;

884 }

885

886

887

888

889

890

891

892 int MaxNumVTable = 1;

893 if (I == CandidateSize - 1)

895

896 if ((int)Candidate.AddressPoints.size() > MaxNumVTable) {

897 LLVM_DEBUG(dbgs() << " allow at most " << MaxNumVTable << " and got "

898 << Candidate.AddressPoints.size()

899 << " vtables. Bail out for vtable comparison.\n");

900 return false;

901 }

902 }

903

904 return true;

905}

906

907bool IndirectCallPromoter::shouldSkipVTable(uint64_t VTableGUID) {

908 if (IgnoredBaseTypes.empty())

909 return false;

910

911 auto *VTableVar = Symtab->getGlobalVariable(VTableGUID);

912

913 assert(VTableVar && "VTableVar must exist for GUID in VTableGUIDAndCounts");

914

916 VTableVar->getMetadata(LLVMContext::MD_type, Types);

917

918 for (auto *Type : Types)

919 if (auto *TypeId = dyn_cast(Type->getOperand(1).get()))

920 if (IgnoredBaseTypes.contains(TypeId->getString())) {

921 LLVM_DEBUG(dbgs() << " vtable profiles should be ignored. Bail "

922 "out of vtable comparison.");

923 return true;

924 }

925 return false;

926}

927

928

929

930

931

932

933

934static void

936 VirtualCallSiteTypeInfoMap &VirtualCSInfo) {

937

938

939

940

941

942

943

944

947 if (!TypeTestFunc || TypeTestFunc->use_empty())

948 return;

949

953 };

954

956 auto *CI = dyn_cast(U.getUser());

957 if (!CI)

958 continue;

959 auto *TypeMDVal = cast(CI->getArgOperand(1));

960 if (!TypeMDVal)

961 continue;

962 auto *CompatibleTypeId = dyn_cast(TypeMDVal->getMetadata());

963 if (!CompatibleTypeId)

964 continue;

965

966

967

970 auto &DT = LookupDomTree(*CI->getFunction());

972

973 for (auto &DevirtCall : DevirtCalls) {

974 CallBase &CB = DevirtCall.CB;

975

976

979 if (!VTablePtr)

980 continue;

981 VirtualCSInfo[&CB] = {DevirtCall.Offset, VTablePtr,

982 CompatibleTypeId->getString()};

983 }

984 }

985}

986

987

991 return false;

994 std::string SymtabFailure = toString(std::move(E));

995 M.getContext().emitError("Failed to create symtab: " + SymtabFailure);

996 return false;

997 }

998 bool Changed = false;

999 VirtualCallSiteTypeInfoMap VirtualCSInfo;

1000

1002

1005

1007 IgnoredBaseTypes.insert(Str);

1008 }

1009

1010

1011

1012

1013

1014

1015

1016

1017 VTableAddressPointOffsetValMap VTableAddressPointOffsetVal;

1018

1019 for (auto &F : M) {

1020 if (F.isDeclaration() || F.hasOptNone())

1021 continue;

1022

1023 auto &FAM =

1026

1027 IndirectCallPromoter CallPromoter(F, M, &Symtab, SamplePGO, VirtualCSInfo,

1028 VTableAddressPointOffsetVal,

1029 IgnoredBaseTypes, ORE);

1030 bool FuncChanged = CallPromoter.processFunction(PSI);

1034 }

1035 Changed |= FuncChanged;

1038 break;

1039 }

1040 }

1041 return Changed;

1042}

1043

1047

1051

1053}

This file defines the DenseMap class.

This header defines various interfaces for pass management in LLVM.

This file provides the interface for IR based instrumentation passes ( (profile-gen,...

FunctionAnalysisManager FAM

ModuleAnalysisManager MAM

This file contains the declarations for profiling metadata utility functions.

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

Class for arbitrary precision integers.

A container for analyses that lazily runs them and caches their results.

PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)

Get the result of an analysis pass for a given IR unit.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

bool empty() const

empty - Check if the array is empty.

ArrayRef< T > slice(size_t N, size_t M) const

slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.

LLVM Basic Block Representation.

const_iterator getFirstInsertionPt() const

Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...

const BasicBlock * getUniquePredecessor() const

Return the predecessor of this block if it has a unique predecessor block.

InstListType::iterator iterator

Instruction iterators...

const Instruction * getTerminator() const LLVM_READONLY

Returns the terminator instruction if the block is well formed or null if the block is not well forme...

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

static Constant * getInBoundsGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList)

Create an "inbounds" getelementptr.

This is an important base class in LLVM.

iterator find(const_arg_type_t< KeyT > Val)

Implements a dense probed hash-table based set.

Analysis pass which computes a DominatorTree.

Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.

Lightweight error class with error context and mandatory checking.

const Function & getFunction() const

MDNode * getMetadata(unsigned KindID) const

Get the current metadata attachments for the given kind, if any.

bool isDeclaration() const

Return true if the primary definition of this global value is outside of the current translation unit...

An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...

A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...

Error create(object::SectionRef &Section)

Create InstrProfSymtab from an object file section which contains function PGO names.

bool isDebugOrPseudoInst() const LLVM_READONLY

Return true if the instruction is a DbgInfoIntrinsic or PseudoProbeInst.

unsigned getNumSuccessors() const LLVM_READONLY

Return the number of successors that this instruction has.

MDNode * getMetadata(unsigned KindID) const

Get the metadata of given kind attached to this Instruction.

void setMetadata(unsigned KindID, MDNode *Node)

Set the metadata of the specified kind to the specified node.

This is an important class for using LLVM in a threaded context.

MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight, bool IsExpected=false)

Return metadata containing two branch weights.

A Module instance is used to store all the information related to an LLVM module.

MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

An analysis pass based on the new PM to deliver ProfileSummaryInfo.

Analysis providing profile information.

bool hasProfileSummary() const

Returns true if profile summary is available.

bool isHotCount(uint64_t C) const

Returns true if count C is considered hot.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

Target - Wrapper for Target specific information.

The instances of the Type class are immutable: once they are created, they are never changed.

static IntegerType * getInt8Ty(LLVMContext &C)

static IntegerType * getInt32Ty(LLVMContext &C)

A Use represents the edge between a Value definition and its users.

User * getUser() const

Returns the User that contains this Use.

LLVMContext & getContext() const

All values hold a context through their type.

iterator_range< use_iterator > uses()

std::pair< iterator, bool > insert(const ValueT &V)

const ParentTy * getParent() const

@ C

The default llvm calling convention, compatible with C.

Function * getDeclarationIfExists(Module *M, ID id, ArrayRef< Type * > Tys, FunctionType *FT=nullptr)

This version supports overloaded intrinsics.

initializer< Ty > init(const Ty &Val)

DiagnosticInfoOptimizationBase::Argument NV

CallBase & promoteIndirectCall(CallBase &CB, Function *F, uint64_t Count, uint64_t TotalCount, bool AttachProfToDirectCall, OptimizationRemarkEmitter *ORE)

NodeAddr< FuncNode * > Func

This is an optimization pass for GlobalISel generic memory operations.

auto drop_begin(T &&RangeOrContainer, size_t N=1)

Return a range covering RangeOrContainer with the first N elements excluded.

void stable_sort(R &&Range)

bool isLegalToPromote(const CallBase &CB, Function *Callee, const char **FailureReason=nullptr)

Return true if the given indirect call site can be made to call Callee.

std::vector< CallBase * > findIndirectCalls(Function &F)

CallBase & promoteCallWithIfThenElse(CallBase &CB, Function *Callee, MDNode *BranchWeights=nullptr)

Promote the given indirect call site to conditionally call Callee.

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

auto upper_bound(R &&Range, T &&Value)

Provide wrappers to std::upper_bound which take ranges instead of having to pass begin/end explicitly...

cl::opt< bool > EnableVTableProfileUse("enable-vtable-profile-use", cl::init(false), cl::desc("If ThinLTO and WPD is enabled and this option is true, vtable " "profiles will be used by ICP pass for more efficient indirect " "call sequence. If false, type profiles won't be used."))

void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)

Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...

auto reverse(ContainerTy &&C)

void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected)

Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...

void sort(IteratorTy Start, IteratorTy End)

raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

SmallVector< InstrProfValueData, 4 > getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)

Extract the value profile data from Inst and returns them if Inst is annotated with value profile dat...

static uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)

Scale an individual branch count.

static uint64_t calculateCountScale(uint64_t MaxCount)

Calculate what to divide by to scale counts.

const char * toString(DWARFSectionKind Kind)

CallBase & promoteCallWithVTableCmp(CallBase &CB, Instruction *VPtr, Function *Callee, ArrayRef< Constant * > AddressPoints, MDNode *BranchWeights)

This is similar to promoteCallWithIfThenElse except that the condition to promote a virtual call is t...

void findDevirtualizableCallsForTypeTest(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< CallInst * > &Assumes, const CallInst *CI, DominatorTree &DT)

Given a call to the intrinsic @llvm.type.test, find all devirtualizable call sites based on the call ...

std::pair< Function *, Constant * > getFunctionAtVTableOffset(GlobalVariable *GV, uint64_t Offset, Module &M)

Given a vtable and a specified offset, returns the function and the trivial pointer at the specified ...

static Instruction * tryGetVTableInstruction(CallBase *CB)