CallPromotionUtils.cpp Source File (original) (raw)

27using namespace llvm;

29#define DEBUG_TYPE "call-promotion-utils"

59 int Idx = Phi.getBasicBlockIndex(OrigBlock);

60 if (Idx == -1)

61 continue;

62 Phi.setIncomingBlock(Idx, MergeBlock);

63 }

64}

91 int Idx = Phi.getBasicBlockIndex(OrigBlock);

92 if (Idx == -1)

93 continue;

94 auto *V = Phi.getIncomingValue(Idx);

95 Phi.setIncomingBlock(Idx, ThenBlock);

96 Phi.addIncoming(V, ElseBlock);

97 }

98}

100

101

102

103

104

105

106

107

108

109

110

111

112

115

117 return;

118

119 Builder.SetInsertPoint(MergeBlock, MergeBlock->begin());

120 PHINode *Phi = Builder.CreatePHI(OrigInst->getType(), 0);

122 for (User *U : UsersToUpdate)

123 U->replaceUsesOfWith(OrigInst, Phi);

124 Phi->addIncoming(OrigInst, OrigInst->getParent());

125 Phi->addIncoming(NewInst, NewInst->getParent());

126}

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

169

170

171

173

174

175

178 InsertBefore =

179 SplitEdge(Invoke->getParent(), Invoke->getNormalDest())->begin();

180 else

181 InsertBefore = std::next(CB.getIterator());

182

183

185 if (RetBitCast)

186 *RetBitCast = Cast;

187

188

189 for (User *U : UsersToUpdate)

191}

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

288 MDNode *BranchWeights) {

289

293

295

296

300 ThenBlock->setName("if.true.direct_targ");

303

304

305 Value *NewRetVal = NewInst;

308 assert(BitCast->getOperand(0) == OrigInst &&

309 "bitcast following musttail call must use the call");

310 auto NewBitCast = BitCast->clone();

311 NewBitCast->replaceUsesOfWith(OrigInst, NewInst);

312 NewBitCast->insertBefore(ThenTerm->getIterator());

313 NewRetVal = NewBitCast;

314 Next = BitCast->getNextNode();

315 }

316

317

319 assert(Ret && "musttail call must precede a ret with an optional bitcast");

320 auto NewRet = Ret->clone();

323 NewRet->insertBefore(ThenTerm->getIterator());

324

325

326

328

329 return *NewInst;

330 }

331

332

333

334

341

342 ThenBlock->setName("if.true.direct_targ");

343 ElseBlock->setName("if.false.orig_indirect");

344 MergeBlock->setName("if.end.icp");

345

349

350

351

352

355

356

357

360

361

362 Builder.SetInsertPoint(MergeBlock);

363 Builder.CreateBr(OrigInvoke->getNormalDest());

364

365

368

369

370

371 OrigInvoke->setNormalDest(MergeBlock);

372 NewInvoke->setNormalDest(MergeBlock);

373 }

374

375

377

378 return *NewInst;

379}

380

381

382

384 MDNode *BranchWeights) {

385

387

388

389

393

395}

396

398 const char **FailureReason) {

400

401 auto &DL = Callee->getDataLayout();

402

403

404

406 Type *FuncRetTy = Callee->getReturnType();

407 if (CallRetTy != FuncRetTy)

409 if (FailureReason)

410 *FailureReason = "Return type mismatch";

411 return false;

412 }

413

414

415 unsigned NumParams = Callee->getFunctionType()->getNumParams();

416

417

418 unsigned NumArgs = CB.arg_size();

419

420

421

422 if (NumArgs != NumParams && !Callee->isVarArg()) {

423 if (FailureReason)

424 *FailureReason = "The number of arguments mismatch";

425 return false;

426 }

427

428

429

430

431 unsigned I = 0;

432 for (; I < NumParams; ++I) {

433

434

435 if (Callee->hasParamAttribute(I, Attribute::ByVal) !=

436 CB.getAttributes().hasParamAttr(I, Attribute::ByVal)) {

437 if (FailureReason)

438 *FailureReason = "byval mismatch";

439 return false;

440 }

441 if (Callee->hasParamAttribute(I, Attribute::InAlloca) !=

442 CB.getAttributes().hasParamAttr(I, Attribute::InAlloca)) {

443 if (FailureReason)

444 *FailureReason = "inalloca mismatch";

445 return false;

446 }

447

448 Type *FormalTy = Callee->getFunctionType()->getFunctionParamType(I);

450 if (FormalTy == ActualTy)

451 continue;

453 if (FailureReason)

454 *FailureReason = "Argument type mismatch";

455 return false;

456 }

457

458

459

464 if (FailureReason)

465 *FailureReason = "Musttail call Argument type mismatch";

466 return false;

467 }

468 }

469 }

470 for (; I < NumArgs; I++) {

471

472 assert(Callee->isVarArg());

474 if (FailureReason)

475 *FailureReason = "SRet arg to vararg function";

476 return false;

477 }

478 }

479

480 return true;

481}

482

486

487

488

490

491

492

493

494 CB.setMetadata(LLVMContext::MD_prof, nullptr);

495 CB.setMetadata(LLVMContext::MD_callees, nullptr);

496

497

498

500 return CB;

501

502

504 Type *CalleeRetTy = Callee->getReturnType();

505

506

508

509

510

511

512 auto CalleeType = Callee->getFunctionType();

513 auto CalleeParamNum = CalleeType->getNumParams();

514

516 const AttributeList &CallerPAL = CB.getAttributes();

517

519 bool AttributeChanged = false;

520

521 for (unsigned ArgNo = 0; ArgNo < CalleeParamNum; ++ArgNo) {

523 Type *FormalTy = CalleeType->getParamType(ArgNo);

524 Type *ActualTy = Arg->getType();

525 if (FormalTy != ActualTy) {

526 auto *Cast =

529

530

531 AttrBuilder ArgAttrs(Ctx, CallerPAL.getParamAttrs(ArgNo));

532 ArgAttrs.remove(AttributeFuncs::typeIncompatible(

533 FormalTy, CallerPAL.getParamAttrs(ArgNo)));

534

535

536 if (ArgAttrs.getByValType())

537 ArgAttrs.addByValAttr(Callee->getParamByValType(ArgNo));

538 if (ArgAttrs.getInAllocaType())

539 ArgAttrs.addInAllocaAttr(Callee->getParamInAllocaType(ArgNo));

540

542 AttributeChanged = true;

543 } else

544 NewArgAttrs.push_back(CallerPAL.getParamAttrs(ArgNo));

545 }

546

547

548

549

550 AttrBuilder RAttrs(Ctx, CallerPAL.getRetAttrs());

551 if (!CallSiteRetTy->isVoidTy() && CallSiteRetTy != CalleeRetTy) {

553 RAttrs.remove(

554 AttributeFuncs::typeIncompatible(CalleeRetTy, CallerPAL.getRetAttrs()));

555 AttributeChanged = true;

556 }

557

558

559 if (AttributeChanged)

560 CB.setAttributes(AttributeList::get(Ctx, CallerPAL.getFnAttrs(),

562 NewArgAttrs));

563

564 return CB;

565}

566

568 MDNode *BranchWeights) {

569

570

571

572

574

575

577}

578

583 return nullptr;

586 if (!CSInstr)

587 return nullptr;

588 const uint64_t CSIndex = CSInstr->getIndex()->getZExtValue();

589

591 versionCallSite(CB, &Callee, nullptr), &Callee);

595 NewCSInstr->setIndex(NewCSID);

596 NewCSInstr->setCallee(&Callee);

597 NewCSInstr->insertBefore(DirectCall.getIterator());

598 auto &DirectBB = *DirectCall.getParent();

599 auto &IndirectBB = *CB.getParent();

600

602 "The ICP direct BB is new, it shouldn't have instrumentation");

604 "The ICP indirect BB is new, it shouldn't have instrumentation");

605

606

609 auto *EntryBBIns =

612 DirectBBIns->setIndex(DirectID);

613 DirectBBIns->insertInto(&DirectBB, DirectBB.getFirstInsertionPt());

614

616 IndirectBBIns->setIndex(IndirectID);

617 IndirectBBIns->insertInto(&IndirectBB, IndirectBB.getFirstInsertionPt());

618

620 const uint32_t NewCountersSize = IndirectID + 1;

621

624 assert(NewCountersSize - 2 == Ctx.counters().size());

625

626 Ctx.resizeCounters(NewCountersSize);

627

628

629

630

631 if (!Ctx.hasCallsite(CSIndex))

632 return;

633 auto &CSData = Ctx.callsite(CSIndex);

634

636 for (const auto &[_, V] : CSData)

637 TotalCount += V.getEntrycount();

639

640

641

642 if (auto It = CSData.find(CalleeGUID); It != CSData.end()) {

643 assert(CalleeGUID == It->second.guid());

644 DirectCount = It->second.getEntrycount();

645

646

647 assert(Ctx.callsites().count(NewCSID) == 0);

648 Ctx.ingestContext(NewCSID, std::move(It->second));

649 CSData.erase(CalleeGUID);

650 }

651

652 assert(TotalCount >= DirectCount);

653 uint64_t IndirectCount = TotalCount - DirectCount;

654

655

656 Ctx.counters()[DirectID] = DirectCount;

657 Ctx.counters()[IndirectID] = IndirectCount;

658

659 };

660 CtxProf.update(ProfileUpdater, Caller);

662}

663

667 MDNode *BranchWeights) {

668 assert(!AddressPoints.empty() && "Caller should guarantee");

671 for (auto &AddressPoint : AddressPoints)

672 ICmps.push_back(Builder.CreateICmpEQ(VPtr, AddressPoint));

673

674

675 Value *Cond = Builder.CreateOr(ICmps);

676

677

678

680

681

683}

684

690

692 if (!VTableEntryLoad)

693 return false;

695 APInt VTableOffset(DL.getIndexTypeSizeInBits(VTableEntryPtr->getType()), 0);

697 DL, VTableOffset, true);

699 if (!VTablePtrLoad)

700 return false;

702 APInt ObjectOffset(DL.getIndexTypeSizeInBits(Object->getType()), 0);

703 Value *ObjectBase = Object->stripAndAccumulateConstantOffsets(

704 DL, ObjectOffset, true);

706

707 return false;

708

709

712 VTablePtrLoad, VTablePtrLoad->getParent(), BBI, 0, nullptr, nullptr);

714 return false;

715 APInt VTableOffsetGVBase(DL.getIndexTypeSizeInBits(VTablePtr->getType()), 0);

717 DL, VTableOffsetGVBase, true);

720

721 return false;

722

723 APInt VTableGVOffset = VTableOffsetGVBase + VTableOffset;

725 return false;

726

727 Function *DirectCallee = nullptr;

728 std::tie(DirectCallee, std::ignore) =

730 if (!DirectCallee)

731 return false;

732

734 return false;

735

736

738 return true;

739}

740

741#undef DEBUG_TYPE

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

Module.h This file contains the declarations for the Module class.

Reader for contextual iFDO profile, which comes in bitstream format.

const SmallVectorImpl< MachineOperand > & Cond

Class for arbitrary precision integers.

uint64_t getZExtValue() const

Get zero extended value.

unsigned getActiveBits() const

Compute the number of active bits in the value.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

bool empty() const

empty - Check if the array is empty.

static LLVM_ABI uint64_t getGUID(const Function &F)

static LLVM_ABI AttributeSet get(LLVMContext &C, const AttrBuilder &B)

LLVM Basic Block Representation.

iterator begin()

Instruction iterator methods.

iterator_range< const_phi_iterator > phis() const

Returns a range that iterates over the phis in the basic block.

InstListType::iterator iterator

Instruction iterators...

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

Function * getCalledFunction() const

Returns the function called, or null if this is an indirect function invocation or the function signa...

LLVM_ABI bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const

Determine whether the argument or parameter has the given attribute.

LLVM_ABI bool isMustTailCall() const

Tests if this call site must be tail call optimized.

LLVM_ABI bool isIndirectCall() const

Return true if the callsite is an indirect call.

Value * getCalledOperand() const

void setAttributes(AttributeList A)

Set the attributes for this call.

Value * getArgOperand(unsigned i) const

void mutateFunctionType(FunctionType *FTy)

void setArgOperand(unsigned i, Value *v)

FunctionType * getFunctionType() const

void setCalledOperand(Value *V)

unsigned arg_size() const

AttributeList getAttributes() const

Return the attributes for this call.

LLVM_ABI Function * getCaller()

Helper to get the caller (the parent function).

This is the base class for all instructions that perform data casts.

static LLVM_ABI bool isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy, const DataLayout &DL)

Check whether a bitcast, inttoptr, or ptrtoint cast between these types is valid and a no-op.

static LLVM_ABI CastInst * CreateBitOrPointerCast(Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)

Create a BitCast, a PtrToInt, or an IntToPTr cast instruction.

static LLVM_ABI InstrProfIncrementInst * getBBInstrumentation(BasicBlock &BB)

Get the instruction instrumenting a BB, or nullptr if not present.

static LLVM_ABI InstrProfCallsite * getCallsiteInstrumentation(CallBase &CB)

Get the instruction instrumenting a callsite, or nullptr if that cannot be found.

A parsed version of the target data layout string in and methods for querying it.

uint64_t GUID

Declare a type to represent a global unique identifier for a global value.

Module * getParent()

Get the module that this global value is contained inside of...

bool isConstant() const

If the value is a global constant, its value is immutable throughout the runtime execution of the pro...

bool hasDefinitiveInitializer() const

hasDefinitiveInitializer - Whether the global variable has an initializer, and any other instances of...

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

LLVM_ABI Instruction * clone() const

Create a copy of 'this' instruction that is identical in all ways except the following:

LLVM_ABI void moveBefore(InstListType::iterator InsertPos)

Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...

LLVM_ABI void insertBefore(InstListType::iterator InsertPos)

Insert an unlinked instruction into a basic block immediately before the specified position.

LLVM_ABI InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

LLVM_ABI const Function * getFunction() const

Return the function this instruction belongs to.

LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)

Set the metadata of the specified kind to the specified node.

BasicBlock * getUnwindDest() const

BasicBlock * getNormalDest() const

This is an important class for using LLVM in a threaded context.

An instruction for reading from memory.

Value * getPointerOperand()

A Module instance is used to store all the information related to an LLVM module.

The instrumented contextual profile, produced by the CtxProfAnalysis.

LLVM_ABI void update(Visitor, const Function &F)

uint32_t allocateNextCounterIndex(const Function &F)

uint32_t allocateNextCallsiteIndex(const Function &F)

bool isFunctionKnown(const Function &F) const

A node (context) in the loaded contextual profile, suitable for mutation during IPO passes.

Class to represent pointers.

unsigned getAddressSpace() const

Return the address space of the Pointer type.

Return a value (possibly void), from a function.

Value * getReturnValue() const

Convenience accessor. Returns null if there is no return value.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

The instances of the Type class are immutable: once they are created, they are never changed.

bool isPointerTy() const

True if this is an instance of PointerType.

bool isVoidTy() const

Return true if this is 'void'.

LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)

Replace uses of one Value with another.

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

LLVM_ABI void setName(const Twine &Name)

Change the name of the value.

iterator_range< user_iterator > users()

LLVM_ABI const Value * stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, bool AllowInvariantGroup=false, function_ref< bool(Value &Value, APInt &Offset)> ExternalAnalysis=nullptr, bool LookThroughIntToPtr=false) const

Accumulate the constant offset this value has compared to a base pointer.

const ParentTy * getParent() const

self_iterator getIterator()

NodeTy * getNextNode()

Get the next node, or nullptr for the list tail.

This is an optimization pass for GlobalISel generic memory operations.

LLVM_ABI bool isLegalToPromote(const CallBase &CB, Function *Callee, const char **FailureReason=nullptr)

Return true if the given indirect call site can be made to call Callee.

Definition CallPromotionUtils.cpp:397

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

LLVM_ABI CallBase & promoteCallWithIfThenElse(CallBase &CB, Function *Callee, MDNode *BranchWeights=nullptr)

Promote the given indirect call site to conditionally call Callee.

Definition CallPromotionUtils.cpp:567

LLVM_ABI Value * FindAvailableLoadedValue(LoadInst *Load, BasicBlock *ScanBB, BasicBlock::iterator &ScanFrom, unsigned MaxInstsToScan=DefMaxInstsToScan, BatchAAResults *AA=nullptr, bool *IsLoadCSE=nullptr, unsigned *NumScanedInst=nullptr)

Scan backwards to see if we have the value of the given load available locally within a small number ...

auto dyn_cast_or_null(const Y &Val)

LLVM_ABI void SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore, Instruction **ThenTerm, Instruction **ElseTerm, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr)

SplitBlockAndInsertIfThenElse is similar to SplitBlockAndInsertIfThen, but also creates the ElseBlock...

LLVM_ABI CallBase & versionCallSite(CallBase &CB, Value *Callee, MDNode *BranchWeights)

Predicate and clone the given call site.

Definition CallPromotionUtils.cpp:383

LLVM_ABI CallBase & promoteCall(CallBase &CB, Function *Callee, CastInst **RetBitCast=nullptr)

Promote the given indirect call site to unconditionally call Callee.

Definition CallPromotionUtils.cpp:483

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

FunctionAddr VTableAddr Next

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

LLVM_ABI bool tryPromoteCall(CallBase &CB)

Try to promote (devirtualize) a virtual call on an Alloca.

Definition CallPromotionUtils.cpp:685

LLVM_ABI Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)

Split the containing block at the specified instruction - everything before SplitBefore stays in the ...

LLVM_ABI BasicBlock * SplitEdge(BasicBlock *From, BasicBlock *To, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")

Split the edge connecting the specified blocks, and return the newly created basic block between From...

LLVM_ABI CallBase & promoteCallWithVTableCmp(CallBase &CB, Instruction *VPtr, Function *Callee, ArrayRef< Constant * > AddressPoints, MDNode *BranchWeights)

This is similar to promoteCallWithIfThenElse except that the condition to promote a virtual call is t...

Definition CallPromotionUtils.cpp:664

std::pair< Function *, Constant * > getFunctionAtVTableOffset(GlobalVariable *GV, uint64_t Offset, Module &M)

Given a vtable and a specified offset, returns the function and the trivial pointer at the specified ...