WholeProgramDevirt.cpp Source File (original) (raw)

97#include

98#include

99#include

100#include

101#include

102

103using namespace llvm;

104using namespace wholeprogramdevirt;

105

106#define DEBUG_TYPE "wholeprogramdevirt"

107

108 STATISTIC(NumDevirtTargets, "Number of whole program devirtualization targets");

109 STATISTIC(NumSingleImpl, "Number of single implementation devirtualizations");

110 STATISTIC(NumBranchFunnel, "Number of branch funnels");

111 STATISTIC(NumUniformRetVal, "Number of uniform return value optimizations");

112 STATISTIC(NumUniqueRetVal, "Number of unique return value optimizations");

114 "Number of 1 bit virtual constant propagations");

115 STATISTIC(NumVirtConstProp, "Number of virtual constant propagations");

116

118 "wholeprogramdevirt-summary-action",

119 cl::desc("What to do with the summary when running this pass"),

121 clEnumValN(PassSummaryAction::Import, "import",

122 "Import typeid resolutions from summary and globals"),

123 clEnumValN(PassSummaryAction::Export, "export",

124 "Export typeid resolutions to summary and globals")),

126

128 "wholeprogramdevirt-read-summary",

130 "Read summary from given bitcode or YAML file before running pass"),

132

134 "wholeprogramdevirt-write-summary",

135 cl::desc("Write summary to given bitcode or YAML file after running pass. "

136 "Output file format is deduced from extension: *.bc means writing "

137 "bitcode, otherwise YAML"),

139

143 cl::desc("Maximum number of call targets per "

144 "call site to enable branch funnels"));

145

148 cl::desc("Print index-based devirtualization messages"));

149

150

151

152

153

156 cl::desc("Enable whole program visibility"));

157

158

159

161 "disable-whole-program-visibility", cl::Hidden,

162 cl::desc("Disable whole program visibility (overrides enabling options)"));

163

164

167 cl::desc("Prevent function(s) from being devirtualized"),

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

188 "wholeprogramdevirt-keep-unreachable-function",

189 cl::desc("Regard unreachable functions as possible devirtualize targets."),

191

192

193

194

196 "wholeprogramdevirt-cutoff",

197 cl::desc("Max number of devirtualizations for devirt module pass"),

199

200

201

202

203

204

207 "wholeprogramdevirt-check", cl::Hidden,

208 cl::desc("Type of checking for incorrect devirtualizations"),

212 "Fallback to indirect when incorrect")));

213

214namespace {

215struct PatternList {

216 std::vector Patterns;

217 template void init(const T &StringList) {

218 for (const auto &S : StringList)

220 Patterns.push_back(std::move(*Pat));

221 }

224 if (P.match(S))

225 return true;

226 return false;

227 }

228};

229}

230

231

232

233

237

240 if (IsAfter)

241 MinByte = std::max(MinByte, Target.minAfterBytes());

242 else

243 MinByte = std::max(MinByte, Target.minBeforeBytes());

244 }

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266 std::vector<ArrayRef<uint8_t>> Used;

269 : Target.TM->Bits->Before.BytesUsed;

271 : MinByte - Target.minBeforeBytes();

272

273

274

277 }

278

279 if (Size == 1) {

280

281 for (unsigned I = 0;; ++I) {

283 for (auto &&B : Used)

284 if (I < B.size())

285 BitsUsed |= B[I];

286 if (BitsUsed != 0xff)

288 }

289 } else {

290

291

292 for (unsigned I = 0;; ++I) {

293 for (auto &&B : Used) {

294 unsigned Byte = 0;

295 while ((I + Byte) < B.size() && Byte < (Size / 8)) {

296 if (B[I + Byte])

297 goto NextI;

298 ++Byte;

299 }

300 }

301 return (MinByte + I) * 8;

302 NextI:;

303 }

304 }

305}

306

309 unsigned BitWidth, int64_t &OffsetByte, uint64_t &OffsetBit) {

311 OffsetByte = -(AllocBefore / 8 + 1);

312 else

313 OffsetByte = -((AllocBefore + 7) / 8 + (BitWidth + 7) / 8);

314 OffsetBit = AllocBefore % 8;

315

318 Target.setBeforeBit(AllocBefore);

319 else

320 Target.setBeforeBytes(AllocBefore, (BitWidth + 7) / 8);

321 }

322}

323

326 unsigned BitWidth, int64_t &OffsetByte, uint64_t &OffsetBit) {

328 OffsetByte = AllocAfter / 8;

329 else

330 OffsetByte = (AllocAfter + 7) / 8;

331 OffsetBit = AllocAfter % 8;

332

335 Target.setAfterBit(AllocAfter);

336 else

337 Target.setAfterBytes(AllocAfter, (BitWidth + 7) / 8);

338 }

339}

340

342 : Fn(Fn), TM(TM),

343 IsBigEndian(Fn->getDataLayout().isBigEndian()),

344 WasDevirt(false) {}

345

346namespace {

347

348

349static unsigned NumDevirtCalls = 0;

350

351

352

353

354struct VTableSlot {

357};

358

359}

360

361namespace llvm {

362

367 }

371 }

375 }

377 const VTableSlot &RHS) {

378 return LHS.TypeID == RHS.TypeID && LHS.ByteOffset == RHS.ByteOffset;

379 }

380};

381

386 }

390 }

394 }

397 return LHS.TypeID == RHS.TypeID && LHS.ByteOffset == RHS.ByteOffset;

398 }

399};

400

401}

402

403

404

405

406

407

408

409

412 return false;

413

414 if ((!TheFnVI) || TheFnVI.getSummaryList().empty()) {

415

416

417 return false;

418 }

419

420 for (const auto &Summary : TheFnVI.getSummaryList()) {

421

422

423 if (!Summary->isLive())

424 return false;

425 if (auto *FS = dyn_cast(Summary->getBaseObject())) {

426 if (!FS->fflags().MustBeUnreachable)

427 return false;

428 }

429

430 else

431 return false;

432 }

433

434

435 return true;

436}

437

438namespace {

439

440

441struct VirtualCallSite {

444

445

446

447

448 unsigned *NumUnsafeUses = nullptr;

449

450 void

456

457 using namespace ore;

459 << NV("Optimization", OptName)

460 << ": devirtualized a call to "

461 << NV("FunctionName", TargetName));

462 }

463

464 void replaceAndErase(

465 const StringRef OptName, const StringRef TargetName, bool RemarksEnabled,

468 if (RemarksEnabled)

469 emitRemark(OptName, TargetName, OREGetter);

471 if (auto *II = dyn_cast(&CB)) {

473 II->getUnwindDest()->removePredecessor(II->getParent());

474 }

476

477 if (NumUnsafeUses)

478 --*NumUnsafeUses;

479 }

480};

481

482

483

484

486

487

488

489

490 std::vector CallSites;

491

492

493

494

495 bool AllCallSitesDevirted = true;

496

497

498

499

500

501

502 bool SummaryHasTypeTestAssumeUsers = false;

503

504

505

506

507

508

509

510

511 std::vector<FunctionSummary *> SummaryTypeCheckedLoadUsers;

512 std::vector<FunctionSummary *> SummaryTypeTestAssumeUsers;

513

514 bool isExported() const {

515 return SummaryHasTypeTestAssumeUsers ||

516 !SummaryTypeCheckedLoadUsers.empty();

517 }

518

519 void addSummaryTypeCheckedLoadUser(FunctionSummary *FS) {

520 SummaryTypeCheckedLoadUsers.push_back(FS);

521 AllCallSitesDevirted = false;

522 }

523

525 SummaryTypeTestAssumeUsers.push_back(FS);

526 SummaryHasTypeTestAssumeUsers = true;

527 AllCallSitesDevirted = false;

528 }

529

530 void markDevirt() {

531 AllCallSitesDevirted = true;

532

533

534 SummaryTypeCheckedLoadUsers.clear();

535 }

536};

537

538

539struct VTableSlotInfo {

540

541

543

544

545

546 std::map<std::vector<uint64_t>, CallSiteInfo> ConstCSInfo;

547

548 void addCallSite(Value *VTable, CallBase &CB, unsigned *NumUnsafeUses);

549

550private:

552};

553

555 std::vector<uint64_t> Args;

556 auto *CBType = dyn_cast(CB.getType());

557 if (!CBType || CBType->getBitWidth() > 64 || CB.arg_empty())

558 return CSInfo;

560 auto *CI = dyn_cast(Arg);

561 if (!CI || CI->getBitWidth() > 64)

562 return CSInfo;

563 Args.push_back(CI->getZExtValue());

564 }

565 return ConstCSInfo[Args];

566}

567

568void VTableSlotInfo::addCallSite(Value *VTable, CallBase &CB,

569 unsigned *NumUnsafeUses) {

570 auto &CSI = findCallSiteInfo(CB);

571 CSI.AllCallSitesDevirted = false;

572 CSI.CallSites.push_back({VTable, CB, NumUnsafeUses});

573}

574

575struct DevirtModule {

579

582

588

589

590

592

593 bool RemarksEnabled;

595

597

598

599

600

602

603

604

606

607

608

609

610

611

612

613

614

615 std::map<CallInst *, unsigned> NumUnsafeUsesForTypeTest;

616 PatternList FunctionsToSkip;

617

623 : M(M), AARGetter(AARGetter), LookupDomTree(LookupDomTree),

624 ExportSummary(ExportSummary), ImportSummary(ImportSummary),

625 Int8Ty(Type::getInt8Ty(M.getContext())),

626 Int8PtrTy(PointerType::getUnqual(M.getContext())),

627 Int32Ty(Type::getInt32Ty(M.getContext())),

628 Int64Ty(Type::getInt64Ty(M.getContext())),

629 IntPtrTy(M.getDataLayout().getIntPtrType(M.getContext(), 0)),

631 RemarksEnabled(areRemarksEnabled()), OREGetter(OREGetter) {

632 assert(!(ExportSummary && ImportSummary));

634 }

635

636 bool areRemarksEnabled();

637

638 void

639 scanTypeTestUsers(Function *TypeTestFunc,

641 void scanTypeCheckedLoadUsers(Function *TypeCheckedLoadFunc);

642

643 void buildTypeIdentifierMap(

644 std::vector &Bits,

646

647 bool

648 tryFindVirtualCallTargets(std::vector &TargetsForSlot,

649 const std::set &TypeMemberInfos,

652

653 void applySingleImplDevirt(VTableSlotInfo &SlotInfo, Constant *TheFn,

654 bool &IsExported);

657 VTableSlotInfo &SlotInfo,

659

660 void applyICallBranchFunnel(VTableSlotInfo &SlotInfo, Constant *JT,

661 bool &IsExported);

663 VTableSlotInfo &SlotInfo,

665

666 bool tryEvaluateFunctionsWithArgs(

669

670 void applyUniformRetValOpt(CallSiteInfo &CSInfo, StringRef FnName,

673 CallSiteInfo &CSInfo,

675

676

677

680

681 bool shouldExportConstantsAsAbsoluteSymbols();

682

683

684

685

690

691

692

698

700

701 void applyUniqueRetValOpt(CallSiteInfo &CSInfo, StringRef FnName, bool IsOne,

703 bool tryUniqueRetValOpt(unsigned BitWidth,

705 CallSiteInfo &CSInfo,

708

709 void applyVirtualConstProp(CallSiteInfo &CSInfo, StringRef FnName,

712 VTableSlotInfo &SlotInfo,

714

716

717

718 void importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo);

719

720

721

722 void removeRedundantTypeTests();

723

724 bool run();

725

726

727

728

731

732

733

734

735

736

737

738

739

742

743

744

745 static bool

749};

750

751struct DevirtIndex {

753

754

755 std::setGlobalValue::GUID &ExportedGUIDs;

756

757

758

759 std::map<ValueInfo, std::vector> &LocalWPDTargetsMap;

760

762

763 PatternList FunctionsToSkip;

764

765 DevirtIndex(

767 std::setGlobalValue::GUID &ExportedGUIDs,

768 std::map<ValueInfo, std::vector> &LocalWPDTargetsMap)

769 : ExportSummary(ExportSummary), ExportedGUIDs(ExportedGUIDs),

770 LocalWPDTargetsMap(LocalWPDTargetsMap) {

772 }

773

774 bool tryFindVirtualCallTargets(std::vector &TargetsForSlot,

777

780 VTableSlotInfo &SlotInfo,

782 std::set &DevirtTargets);

783

784 void run();

785};

786}

787

793 };

796 };

799 };

800 if (UseCommandLine) {

801 if (!DevirtModule::runForTesting(M, AARGetter, OREGetter, LookupDomTree))

804 }

805 if (!DevirtModule(M, AARGetter, OREGetter, LookupDomTree, ExportSummary,

806 ImportSummary)

807 .run())

810}

811

812

813

817}

818

819static bool

822

823

824

825 if (TypeID.ends_with(".virtual"))

826 return false;

827

828

829

830

831 if ( TypeID .consume_front("_ZTS"))

832 return false;

833

834

835

836

837

838

839 std::string typeInfo = ("_ZTI" + TypeID).str();

840 return IsVisibleToRegularObj(typeInfo);

841}

842

843static bool

847 GV.getMetadata(LLVMContext::MD_type, Types);

848

849 for (auto Type : Types)

850 if (auto *TypeID = dyn_cast(Type->getOperand(1).get()))

852 IsVisibleToRegularObj);

853

854 return false;

855}

856

857

858

859

861 Module &M, bool WholeProgramVisibilityEnabledInLTO,

863 bool ValidateAllVtablesHaveTypeInfos,

866 return;

868

869

870

871 if (GV.hasMetadata(LLVMContext::MD_type) &&

873

874

876

877

878

879

880 !(ValidateAllVtablesHaveTypeInfos &&

883 }

884}

885

887 bool WholeProgramVisibilityEnabledInLTO) {

888 Function *PublicTypeTestFunc =

890 if (!PublicTypeTestFunc)

891 return;

896 auto *CI = cast(U.getUser());

898 TypeTestFunc, {CI->getArgOperand(0), CI->getArgOperand(1)}, {}, "",

900 CI->replaceAllUsesWith(NewCI);

901 CI->eraseFromParent();

902 }

903 } else {

906 auto *CI = cast(U.getUser());

907 CI->replaceAllUsesWith(True);

908 CI->eraseFromParent();

909 }

910 }

911}

912

913

914

919 for (const auto &typeID : Index.typeIdCompatibleVtableMap()) {

922 VisibleToRegularObjSymbols.insert(P.VTableVI.getGUID());

923 }

924}

925

926

927

928

934 return;

935 for (auto &P : Index) {

936

937

938 if (DynamicExportSymbols.count(P.first))

939 continue;

940 for (auto &S : P.second.SummaryList) {

941 auto *GVar = dyn_cast(S.get());

942 if (!GVar ||

944 continue;

945

946

947

948

949 if (VisibleToRegularObjSymbols.count(P.first))

950 continue;

952 }

953 }

954}

955

957 ModuleSummaryIndex &Summary, std::setGlobalValue::GUID &ExportedGUIDs,

958 std::map<ValueInfo, std::vector> &LocalWPDTargetsMap) {

959 DevirtIndex(Summary, ExportedGUIDs, LocalWPDTargetsMap).run();

960}

961

965 std::map<ValueInfo, std::vector> &LocalWPDTargetsMap) {

966 for (auto &T : LocalWPDTargetsMap) {

967 auto &VI = T.first;

968

969 assert(VI.getSummaryList().size() == 1 &&

970 "Devirt of local target has more than one copy");

971 auto &S = VI.getSummaryList()[0];

972 if (!isExported(S->modulePath(), VI))

973 continue;

974

975

976 for (auto &SlotSummary : T.second) {

977 auto *TIdSum = Summary.getTypeIdSummary(SlotSummary.TypeID);

979 auto WPDRes = TIdSum->WPDRes.find(SlotSummary.ByteOffset);

980 assert(WPDRes != TIdSum->WPDRes.end());

982 WPDRes->second.SingleImplName,

983 Summary.getModuleHash(S->modulePath()));

984 }

985 }

986}

987

989

990

991

992

993 const auto &ModPaths = Summary->modulePaths();

998 "combined summary should contain Regular LTO module");

1000}

1001

1002bool DevirtModule::runForTesting(

1006 std::unique_ptr Summary =

1007 std::make_unique(false);

1008

1009

1010

1013 ": ");

1014 auto ReadSummaryFile =

1016 if (Expected<std::unique_ptr> SummaryOrErr =

1018 Summary = std::move(*SummaryOrErr);

1020 } else {

1021

1023 yaml::Input In(ReadSummaryFile->getBuffer());

1026 }

1027 }

1028

1029 bool Changed =

1030 DevirtModule(M, AARGetter, OREGetter, LookupDomTree,

1032 : nullptr,

1034 : nullptr)

1035 .run();

1036

1039 "-wholeprogramdevirt-write-summary: " + ClWriteSummary + ": ");

1040 std::error_code EC;

1045 } else {

1048 yaml::Output Out(OS);

1050 }

1051 }

1052

1053 return Changed;

1054}

1055

1056void DevirtModule::buildTypeIdentifierMap(

1057 std::vector &Bits,

1058 DenseMap<Metadata *, std::set> &TypeIdMap) {

1060 Bits.reserve(M.global_size());

1064 GV.getMetadata(LLVMContext::MD_type, Types);

1066 continue;

1067

1068 VTableBits *&BitsPtr = GVToBits[&GV];

1069 if (!BitsPtr) {

1070 Bits.emplace_back();

1071 Bits.back().GV = &GV;

1072 Bits.back().ObjectSize =

1074 BitsPtr = &Bits.back();

1075 }

1076

1078 auto TypeID = Type->getOperand(1).get();

1079

1081 cast(

1082 cast(Type->getOperand(0))->getValue())

1083 ->getZExtValue();

1084

1085 TypeIdMap[TypeID].insert({BitsPtr, Offset});

1086 }

1087 }

1088}

1089

1090bool DevirtModule::tryFindVirtualCallTargets(

1091 std::vector &TargetsForSlot,

1092 const std::set &TypeMemberInfos, uint64_t ByteOffset,

1095 if (.Bits->GV->isConstant())

1096 return false;

1097

1098

1099

1100 if (TM.Bits->GV->getVCallVisibility() ==

1102 return false;

1103

1106 std::tie(Fn, C) =

1108

1109 if (!Fn)

1110 return false;

1111

1112 if (FunctionsToSkip.match(Fn->getName()))

1113 return false;

1114

1115

1116

1117 if (Fn->getName() == "__cxa_pure_virtual")

1118 continue;

1119

1120

1121

1123 continue;

1124

1125

1126

1127 auto GV = dyn_cast(C);

1129 TargetsForSlot.push_back({GV, &TM});

1130 }

1131

1132

1133 return !TargetsForSlot.empty();

1134}

1135

1136bool DevirtIndex::tryFindVirtualCallTargets(

1137 std::vector &TargetsForSlot,

1140

1141

1142

1143

1144

1145

1146

1147

1148

1149

1151 bool LocalFound = false;

1152 for (const auto &S : P.VTableVI.getSummaryList()) {

1154 if (LocalFound)

1155 return false;

1156 LocalFound = true;

1157 }

1158 auto *CurVS = cast(S->getBaseObject());

1159 if (!CurVS->vTableFuncs().empty() ||

1160

1161

1162

1163

1164

1165

1166

1168 VS = CurVS;

1169

1170

1172 return false;

1173 }

1174 }

1175

1176

1177 if (!VS)

1178 return false;

1179 if (->isLive())

1180 continue;

1181 for (auto VTP : VS->vTableFuncs()) {

1182 if (VTP.VTableOffset != P.AddressPointOffset + ByteOffset)

1183 continue;

1184

1186 continue;

1187

1188 TargetsForSlot.push_back(VTP.FuncVI);

1189 }

1190 }

1191

1192

1193 return !TargetsForSlot.empty();

1194}

1195

1196void DevirtModule::applySingleImplDevirt(VTableSlotInfo &SlotInfo,

1197 Constant *TheFn, bool &IsExported) {

1198

1199

1201 return;

1203 for (auto &&VCallSite : CSInfo.CallSites) {

1204 if (!OptimizedCalls.insert(&VCallSite.CB).second)

1205 continue;

1206

1207

1210 return;

1211

1212 if (RemarksEnabled)

1213 VCallSite.emitRemark("single-impl",

1215 NumSingleImpl++;

1216 NumDevirtCalls++;

1217 auto &CB = VCallSite.CB;

1222

1223

1224

1225

1229 Cond, &CB, false,

1231 Builder.SetInsertPoint(ThenTerm);

1234 auto *CallTrap = Builder.CreateCall(TrapFn);

1235 CallTrap->setDebugLoc(CB.getDebugLoc());

1236 }

1237

1238

1239

1240

1243

1244

1245

1248

1249

1250

1251 NewInst.setMetadata(LLVMContext::MD_prof, nullptr);

1252 NewInst.setMetadata(LLVMContext::MD_callees, nullptr);

1253

1254

1255 CB.setMetadata(LLVMContext::MD_prof, nullptr);

1256 CB.setMetadata(LLVMContext::MD_callees, nullptr);

1257 }

1258

1259

1260 else {

1261

1263

1264

1265

1266 CB.setMetadata(LLVMContext::MD_prof, nullptr);

1267 CB.setMetadata(LLVMContext::MD_callees, nullptr);

1273

1274 CallsWithPtrAuthBundleRemoved.push_back(&CB);

1275 }

1276 }

1277

1278

1279 if (VCallSite.NumUnsafeUses)

1280 --*VCallSite.NumUnsafeUses;

1281 }

1282 if (CSInfo.isExported())

1283 IsExported = true;

1284 CSInfo.markDevirt();

1285 };

1286 Apply(SlotInfo.CSInfo);

1287 for (auto &P : SlotInfo.ConstCSInfo)

1288 Apply(P.second);

1289}

1290

1292

1293 if (Callee.getSummaryList().empty())

1294 return false;

1295

1296

1297

1298

1299

1300 bool IsExported = false;

1301 auto &S = Callee.getSummaryList()[0];

1303 0);

1305 for (auto *FS : CSInfo.SummaryTypeCheckedLoadUsers) {

1306 FS->addCall({Callee, CI});

1307 IsExported |= S->modulePath() != FS->modulePath();

1308 }

1309 for (auto *FS : CSInfo.SummaryTypeTestAssumeUsers) {

1310 FS->addCall({Callee, CI});

1311 IsExported |= S->modulePath() != FS->modulePath();

1312 }

1313 };

1315 for (auto &P : SlotInfo.ConstCSInfo)

1317 return IsExported;

1318}

1319

1320bool DevirtModule::trySingleImplDevirt(

1324

1325

1326 auto *TheFn = TargetsForSlot[0].Fn;

1327 for (auto &&Target : TargetsForSlot)

1328 if (TheFn != Target.Fn)

1329 return false;

1330

1331

1333 TargetsForSlot[0].WasDevirt = true;

1334

1335 bool IsExported = false;

1336 applySingleImplDevirt(SlotInfo, TheFn, IsExported);

1337 if (!IsExported)

1338 return false;

1339

1340

1341

1342

1343 if (TheFn->hasLocalLinkage()) {

1344 std::string NewName = (TheFn->getName() + ".llvm.merged").str();

1345

1346

1347

1348

1349 if (Comdat *C = TheFn->getComdat()) {

1350 if (C->getName() == TheFn->getName()) {

1351 Comdat *NewC = M.getOrInsertComdat(NewName);

1354 if (GO.getComdat() == C)

1355 GO.setComdat(NewC);

1356 }

1357 }

1358

1361 TheFn->setName(NewName);

1362 }

1364

1365

1366 AddCalls(SlotInfo, TheFnVI);

1367

1370

1371 return true;

1372}

1373

1376 VTableSlotInfo &SlotInfo,

1378 std::set &DevirtTargets) {

1379

1380

1381 auto TheFn = TargetsForSlot[0];

1382 for (auto &&Target : TargetsForSlot)

1383 if (TheFn != Target)

1384 return false;

1385

1386

1387 auto Size = TheFn.getSummaryList().size();

1389 return false;

1390

1391

1392

1393 if (FunctionsToSkip.match(TheFn.name()))

1394 return false;

1395

1396

1397

1398 for (const auto &S : TheFn.getSummaryList())

1400 return false;

1401

1402

1404 DevirtTargets.insert(TheFn);

1405

1406 auto &S = TheFn.getSummaryList()[0];

1407 bool IsExported = AddCalls(SlotInfo, TheFn);

1408 if (IsExported)

1409 ExportedGUIDs.insert(TheFn.getGUID());

1410

1411

1412

1415 if (IsExported)

1416

1417

1418

1420 TheFn.name(), ExportSummary.getModuleHash(S->modulePath()));

1421 else {

1422 LocalWPDTargetsMap[TheFn].push_back(SlotSummary);

1424 }

1425 } else

1427

1428

1429

1430

1432

1433 return true;

1434}

1435

1436void DevirtModule::tryICallBranchFunnel(

1439 Triple T(M.getTargetTriple());

1441 return;

1442

1444 return;

1445

1446 bool HasNonDevirt = !SlotInfo.CSInfo.AllCallSitesDevirted;

1447 if (!HasNonDevirt)

1448 for (auto &P : SlotInfo.ConstCSInfo)

1449 if (.second.AllCallSitesDevirted) {

1450 HasNonDevirt = true;

1451 break;

1452 }

1453

1454 if (!HasNonDevirt)

1455 return;

1456

1460 if (isa(Slot.TypeID)) {

1462 M.getDataLayout().getProgramAddressSpace(),

1463 getGlobalName(Slot, {}, "branch_funnel"), &M);

1465 } else {

1467 M.getDataLayout().getProgramAddressSpace(),

1468 "branch_funnel", &M);

1469 }

1470 JT->addParamAttr(0, Attribute::Nest);

1471

1472 std::vector<Value *> JTArgs;

1473 JTArgs.push_back(JT->arg_begin());

1474 for (auto &T : TargetsForSlot) {

1475 JTArgs.push_back(getMemberAddr(T.TM));

1476 JTArgs.push_back(T.Fn);

1477 }

1478

1481 &M, llvm::Intrinsic::icall_branch_funnel, {});

1482

1486

1487 bool IsExported = false;

1488 applyICallBranchFunnel(SlotInfo, JT, IsExported);

1489 if (IsExported)

1491}

1492

1493void DevirtModule::applyICallBranchFunnel(VTableSlotInfo &SlotInfo,

1494 Constant *JT, bool &IsExported) {

1496 if (CSInfo.isExported())

1497 IsExported = true;

1498 if (CSInfo.AllCallSitesDevirted)

1499 return;

1500

1501 std::map<CallBase *, CallBase *> CallBases;

1502 for (auto &&VCallSite : CSInfo.CallSites) {

1503 CallBase &CB = VCallSite.CB;

1504

1505 if (CallBases.find(&CB) != CallBases.end()) {

1506

1507

1508

1509

1510

1511 continue;

1512 }

1513

1514

1516 if (!FSAttr.isValid() ||

1518 continue;

1519

1520 NumBranchFunnel++;

1521 if (RemarksEnabled)

1522 VCallSite.emitRemark("branch-funnel",

1523 JT->stripPointerCasts()->getName(), OREGetter);

1524

1525

1526

1527 std::vector<Type *> NewArgs;

1528 NewArgs.push_back(Int8PtrTy);

1534 std::vector<Value *> Args;

1535 Args.push_back(VCallSite.VTable);

1537

1539 if (isa(CB))

1540 NewCS = IRB.CreateCall(NewFT, JT, Args);

1541 else

1542 NewCS =

1543 IRB.CreateInvoke(NewFT, JT, cast(CB).getNormalDest(),

1544 cast(CB).getUnwindDest(), Args);

1546

1548 std::vector NewArgAttrs;

1551 M.getContext(), Attribute::Nest)}));

1552 for (unsigned I = 0; I + 2 < Attrs.getNumAttrSets(); ++I)

1553 NewArgAttrs.push_back(Attrs.getParamAttrs(I));

1556 Attrs.getRetAttrs(), NewArgAttrs));

1557

1558 CallBases[&CB] = NewCS;

1559

1560

1561 if (VCallSite.NumUnsafeUses)

1562 --*VCallSite.NumUnsafeUses;

1563 }

1564

1565

1566

1567

1568

1569 for (auto &[Old, New] : CallBases) {

1571 Old->eraseFromParent();

1572 }

1573 };

1574 Apply(SlotInfo.CSInfo);

1575 for (auto &P : SlotInfo.ConstCSInfo)

1576 Apply(P.second);

1577}

1578

1579bool DevirtModule::tryEvaluateFunctionsWithArgs(

1582

1583

1585

1586

1587

1588 auto Fn = dyn_cast(Target.Fn);

1589 if (!Fn)

1590 return false;

1591

1593 return false;

1594

1595 Evaluator Eval(M.getDataLayout(), nullptr);

1599 for (unsigned I = 0; I != Args.size(); ++I) {

1600 auto *ArgTy =

1602 if (!ArgTy)

1603 return false;

1604 EvalArgs.push_back(ConstantInt::get(ArgTy, Args[I]));

1605 }

1606

1608 if (!Eval.EvaluateFunction(Fn, RetVal, EvalArgs) ||

1609 !isa(RetVal))

1610 return false;

1611 Target.RetVal = cast(RetVal)->getZExtValue();

1612 }

1613 return true;

1614}

1615

1616void DevirtModule::applyUniformRetValOpt(CallSiteInfo &CSInfo, StringRef FnName,

1618 for (auto Call : CSInfo.CallSites) {

1619 if (!OptimizedCalls.insert(&Call.CB).second)

1620 continue;

1621 NumUniformRetVal++;

1622 Call.replaceAndErase(

1623 "uniform-ret-val", FnName, RemarksEnabled, OREGetter,

1624 ConstantInt::get(cast(Call.CB.getType()), TheRetVal));

1625 }

1626 CSInfo.markDevirt();

1627}

1628

1629bool DevirtModule::tryUniformRetValOpt(

1632

1633

1634 uint64_t TheRetVal = TargetsForSlot[0].RetVal;

1636 if (Target.RetVal != TheRetVal)

1637 return false;

1638

1639 if (CSInfo.isExported()) {

1641 Res->Info = TheRetVal;

1642 }

1643

1644 applyUniformRetValOpt(CSInfo, TargetsForSlot[0].Fn->getName(), TheRetVal);

1646 for (auto &&Target : TargetsForSlot)

1647 Target.WasDevirt = true;

1648 return true;

1649}

1650

1651std::string DevirtModule::getGlobalName(VTableSlot Slot,

1654 std::string FullName = "__typeid_";

1656 OS << cast(Slot.TypeID)->getString() << '_' << Slot.ByteOffset;

1658 OS << '_' << Arg;

1660 return FullName;

1661}

1662

1663bool DevirtModule::shouldExportConstantsAsAbsoluteSymbols() {

1664 Triple T(M.getTargetTriple());

1665 return T.isX86() && T.getObjectFormat() == Triple::ELF;

1666}

1667

1668void DevirtModule::exportGlobal(VTableSlot Slot, ArrayRef<uint64_t> Args,

1671 getGlobalName(Slot, Args, Name), C, &M);

1673}

1674

1675void DevirtModule::exportConstant(VTableSlot Slot, ArrayRef<uint64_t> Args,

1678 if (shouldExportConstantsAsAbsoluteSymbols()) {

1679 exportGlobal(

1680 Slot, Args, Name,

1682 return;

1683 }

1684

1685 Storage = Const;

1686}

1687

1691 M.getOrInsertGlobal(getGlobalName(Slot, Args, Name), Int8Arr0Ty);

1692 auto *GV = dyn_cast(C);

1693 if (GV)

1695 return C;

1696}

1697

1701 if (!shouldExportConstantsAsAbsoluteSymbols())

1702 return ConstantInt::get(IntTy, Storage);

1703

1705 auto *GV = cast(C->stripPointerCasts());

1707

1708

1709

1710 if (GV->hasMetadata(LLVMContext::MD_absolute_symbol))

1711 return C;

1712

1716 GV->setMetadata(LLVMContext::MD_absolute_symbol,

1718 };

1719 unsigned AbsWidth = IntTy->getBitWidth();

1720 if (AbsWidth == IntPtrTy->getBitWidth())

1721 SetAbsRange(~0ull, ~0ull);

1722 else

1723 SetAbsRange(0, 1ull << AbsWidth);

1724 return C;

1725}

1726

1727void DevirtModule::applyUniqueRetValOpt(CallSiteInfo &CSInfo, StringRef FnName,

1728 bool IsOne,

1729 Constant *UniqueMemberAddr) {

1730 for (auto &&Call : CSInfo.CallSites) {

1731 if (!OptimizedCalls.insert(&Call.CB).second)

1732 continue;

1736 B.CreateBitCast(UniqueMemberAddr, Call.VTable->getType()));

1737 Cmp = B.CreateZExt(Cmp, Call.CB.getType());

1738 NumUniqueRetVal++;

1739 Call.replaceAndErase("unique-ret-val", FnName, RemarksEnabled, OREGetter,

1740 Cmp);

1741 }

1742 CSInfo.markDevirt();

1743}

1744

1747 ConstantInt::get(Int64Ty, M->Offset));

1748}

1749

1750bool DevirtModule::tryUniqueRetValOpt(

1754

1755 auto tryUniqueRetValOptFor = [&](bool IsOne) {

1758 if (Target.RetVal == (IsOne ? 1 : 0)) {

1759 if (UniqueMember)

1760 return false;

1761 UniqueMember = Target.TM;

1762 }

1763 }

1764

1765

1766

1767 assert(UniqueMember);

1768

1769 Constant *UniqueMemberAddr = getMemberAddr(UniqueMember);

1770 if (CSInfo.isExported()) {

1772 Res->Info = IsOne;

1773

1774 exportGlobal(Slot, Args, "unique_member", UniqueMemberAddr);

1775 }

1776

1777

1778 applyUniqueRetValOpt(CSInfo, TargetsForSlot[0].Fn->getName(), IsOne,

1779 UniqueMemberAddr);

1780

1781

1783 for (auto &&Target : TargetsForSlot)

1784 Target.WasDevirt = true;

1785

1786 return true;

1787 };

1788

1790 if (tryUniqueRetValOptFor(true))

1791 return true;

1792 if (tryUniqueRetValOptFor(false))

1793 return true;

1794 }

1795 return false;

1796}

1797

1798void DevirtModule::applyVirtualConstProp(CallSiteInfo &CSInfo, StringRef FnName,

1800 for (auto Call : CSInfo.CallSites) {

1801 if (!OptimizedCalls.insert(&Call.CB).second)

1802 continue;

1803 auto *RetType = cast(Call.CB.getType());

1806 if (RetType->getBitWidth() == 1) {

1808 Value *BitsAndBit = B.CreateAnd(Bits, Bit);

1809 auto IsBitSet = B.CreateICmpNE(BitsAndBit, ConstantInt::get(Int8Ty, 0));

1810 NumVirtConstProp1Bit++;

1811 Call.replaceAndErase("virtual-const-prop-1-bit", FnName, RemarksEnabled,

1812 OREGetter, IsBitSet);

1813 } else {

1814 Value *Val = B.CreateLoad(RetType, Addr);

1815 NumVirtConstProp++;

1816 Call.replaceAndErase("virtual-const-prop", FnName, RemarksEnabled,

1817 OREGetter, Val);

1818 }

1819 }

1820 CSInfo.markDevirt();

1821}

1822

1823bool DevirtModule::tryVirtualConstProp(

1826

1827

1828

1829 auto Fn = dyn_cast(TargetsForSlot[0].Fn);

1830 if (!Fn)

1831 return false;

1832

1833 auto RetType = dyn_cast(Fn->getReturnType());

1834 if (!RetType)

1835 return false;

1836 unsigned BitWidth = RetType->getBitWidth();

1838 return false;

1839

1840

1841

1842

1843

1844

1845

1846

1847

1848

1849

1851

1852

1853

1854 auto Fn = dyn_cast(Target.Fn);

1855 if (!Fn)

1856 return false;

1857

1860 .doesNotAccessMemory() ||

1863 return false;

1864 }

1865

1866 for (auto &&CSByConstantArg : SlotInfo.ConstCSInfo) {

1867 if (!tryEvaluateFunctionsWithArgs(TargetsForSlot, CSByConstantArg.first))

1868 continue;

1869

1871 if (Res)

1872 ResByArg = &Res->ResByArg[CSByConstantArg.first];

1873

1874 if (tryUniformRetValOpt(TargetsForSlot, CSByConstantArg.second, ResByArg))

1875 continue;

1876

1877 if (tryUniqueRetValOpt(BitWidth, TargetsForSlot, CSByConstantArg.second,

1878 ResByArg, Slot, CSByConstantArg.first))

1879 continue;

1880

1881

1882

1887

1888

1889

1890 uint64_t TotalPaddingBefore = 0, TotalPaddingAfter = 0;

1891 for (auto &&Target : TargetsForSlot) {

1892 TotalPaddingBefore += std::max<int64_t>(

1893 (AllocBefore + 7) / 8 - Target.allocatedBeforeBytes() - 1, 0);

1894 TotalPaddingAfter += std::max<int64_t>(

1895 (AllocAfter + 7) / 8 - Target.allocatedAfterBytes() - 1, 0);

1896 }

1897

1898

1899

1900 if (std::min(TotalPaddingBefore, TotalPaddingAfter) > 128)

1901 continue;

1902

1903

1904

1905 int64_t OffsetByte;

1907 if (TotalPaddingBefore <= TotalPaddingAfter)

1909 OffsetBit);

1910 else

1912 OffsetBit);

1913

1915 for (auto &&Target : TargetsForSlot)

1916 Target.WasDevirt = true;

1917

1918

1919 if (CSByConstantArg.second.isExported()) {

1921 exportConstant(Slot, CSByConstantArg.first, "byte", OffsetByte,

1922 ResByArg->Byte);

1923 exportConstant(Slot, CSByConstantArg.first, "bit", 1ULL << OffsetBit,

1924 ResByArg->Bit);

1925 }

1926

1927

1928 Constant *ByteConst = ConstantInt::get(Int32Ty, OffsetByte);

1929 Constant *BitConst = ConstantInt::get(Int8Ty, 1ULL << OffsetBit);

1930 applyVirtualConstProp(CSByConstantArg.second,

1931 TargetsForSlot[0].Fn->getName(), ByteConst, BitConst);

1932 }

1933 return true;

1934}

1935

1936void DevirtModule::rebuildGlobal(VTableBits &B) {

1937 if (B.Before.Bytes.empty() && B.After.Bytes.empty())

1938 return;

1939

1940

1941

1942 Align Alignment = M.getDataLayout().getValueOrABITypeAlignment(

1943 B.GV->getAlign(), B.GV->getValueType());

1944 B.Before.Bytes.resize(alignTo(B.Before.Bytes.size(), Alignment));

1945

1946

1947 for (size_t I = 0, Size = B.Before.Bytes.size(); I != Size / 2; ++I)

1949

1950

1951

1954 B.GV->getInitializer(),

1956 auto NewGV =

1957 new GlobalVariable(M, NewInit->getType(), B.GV->isConstant(),

1959 NewGV->setSection(B.GV->getSection());

1960 NewGV->setComdat(B.GV->getComdat());

1961 NewGV->setAlignment(B.GV->getAlign());

1962

1963

1964

1965 NewGV->copyMetadata(B.GV, B.Before.Bytes.size());

1966

1967

1968

1970 B.GV->getInitializer()->getType(), 0, B.GV->getLinkage(), "",

1972 NewInit->getType(), NewGV,

1974 ConstantInt::get(Int32Ty, 1)}),

1975 &M);

1976 Alias->setVisibility(B.GV->getVisibility());

1977 Alias->takeName(B.GV);

1978

1979 B.GV->replaceAllUsesWith(Alias);

1980 B.GV->eraseFromParent();

1981}

1982

1983bool DevirtModule::areRemarksEnabled() {

1984 const auto &FL = M.getFunctionList();

1985 for (const Function &Fn : FL) {

1987 continue;

1989 return DI.isEnabled();

1990 }

1991 return false;

1992}

1993

1994void DevirtModule::scanTypeTestUsers(

1996 DenseMap<Metadata *, std::set> &TypeIdMap) {

1997

1998

1999

2000

2001

2003 auto *CI = dyn_cast(U.getUser());

2004 if (!CI)

2005 continue;

2006

2007

2010 auto &DT = LookupDomTree(*CI->getFunction());

2012

2014 cast(CI->getArgOperand(1))->getMetadata();

2015

2016 if (!Assumes.empty()) {

2017 Value *Ptr = CI->getArgOperand(0)->stripPointerCasts();

2019 CallSlots[{TypeId, Call.Offset}].addCallSite(Ptr, Call.CB, nullptr);

2020 }

2021

2022 auto RemoveTypeTestAssumes = [&]() {

2023

2024 for (auto *Assume : Assumes)

2025 Assume->eraseFromParent();

2026

2027

2028 if (CI->use_empty())

2029 CI->eraseFromParent();

2030 };

2031

2032

2033

2034

2035

2036

2037

2038

2039

2040

2041

2042

2043

2044 if (!TypeIdMap.count(TypeId))

2045 RemoveTypeTestAssumes();

2046

2047

2048

2049

2050

2051

2052

2053

2054

2055

2056 else if (ImportSummary && isa(TypeId)) {

2058 ImportSummary->getTypeIdSummary(cast(TypeId)->getString());

2059 if (!TidSummary)

2060 RemoveTypeTestAssumes();

2061 else

2062

2063

2065 }

2066 }

2067}

2068

2069void DevirtModule::scanTypeCheckedLoadUsers(Function *TypeCheckedLoadFunc) {

2072

2074 auto *CI = dyn_cast(U.getUser());

2075 if (!CI)

2076 continue;

2077

2078 Value *Ptr = CI->getArgOperand(0);

2080 Value *TypeIdValue = CI->getArgOperand(2);

2081 Metadata *TypeId = cast(TypeIdValue)->getMetadata();

2082

2086 bool HasNonCallUses = false;

2087 auto &DT = LookupDomTree(*CI->getFunction());

2089 HasNonCallUses, CI, DT);

2090

2091

2092

2093

2094

2095

2096

2098 (LoadedPtrs.size() == 1 && !HasNonCallUses) ? LoadedPtrs[0] : CI);

2099

2100 Value *LoadedValue = nullptr;

2102 Intrinsic::type_checked_load_relative) {

2104 LoadedValue = LoadB.CreateLoad(Int32Ty, GEP);

2105 LoadedValue = LoadB.CreateSExt(LoadedValue, IntPtrTy);

2106 GEP = LoadB.CreatePtrToInt(GEP, IntPtrTy);

2107 LoadedValue = LoadB.CreateAdd(GEP, LoadedValue);

2108 LoadedValue = LoadB.CreateIntToPtr(LoadedValue, Int8PtrTy);

2109 } else {

2111 LoadedValue = LoadB.CreateLoad(Int8PtrTy, GEP);

2112 }

2113

2114 for (Instruction *LoadedPtr : LoadedPtrs) {

2116 LoadedPtr->eraseFromParent();

2117 }

2118

2119

2120 IRBuilder<> CallB((Preds.size() == 1 && !HasNonCallUses) ? Preds[0] : CI);

2121 CallInst *TypeTestCall = CallB.CreateCall(TypeTestFunc, {Ptr, TypeIdValue});

2122

2125 Pred->eraseFromParent();

2126 }

2127

2128

2129

2130

2131

2132 if (!CI->use_empty()) {

2135 Pair = B.CreateInsertValue(Pair, LoadedValue, {0});

2136 Pair = B.CreateInsertValue(Pair, TypeTestCall, {1});

2137 CI->replaceAllUsesWith(Pair);

2138 }

2139

2140

2141 auto &NumUnsafeUses = NumUnsafeUsesForTypeTest[TypeTestCall];

2142 NumUnsafeUses = DevirtCalls.size();

2143

2144

2145

2146

2147 if (HasNonCallUses)

2148 ++NumUnsafeUses;

2150 CallSlots[{TypeId, Call.Offset}].addCallSite(Ptr, Call.CB,

2151 &NumUnsafeUses);

2152 }

2153

2154 CI->eraseFromParent();

2155 }

2156}

2157

2158void DevirtModule::importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo) {

2159 auto *TypeId = dyn_cast(Slot.TypeID);

2160 if (!TypeId)

2161 return;

2164 if (!TidSummary)

2165 return;

2166 auto ResI = TidSummary->WPDRes.find(Slot.ByteOffset);

2167 if (ResI == TidSummary->WPDRes.end())

2168 return;

2170

2173

2174

2176 cast(M.getOrInsertFunction(Res.SingleImplName,

2178 .getCallee());

2179

2180

2181 bool IsExported = false;

2182 applySingleImplDevirt(SlotInfo, SingleImpl, IsExported);

2183 assert(!IsExported);

2184 }

2185

2186 for (auto &CSByConstantArg : SlotInfo.ConstCSInfo) {

2187 auto I = Res.ResByArg.find(CSByConstantArg.first);

2189 continue;

2190 auto &ResByArg = I->second;

2191

2192

2193

2194

2195 switch (ResByArg.TheKind) {

2197 applyUniformRetValOpt(CSByConstantArg.second, "", ResByArg.Info);

2198 break;

2200 Constant *UniqueMemberAddr =

2201 importGlobal(Slot, CSByConstantArg.first, "unique_member");

2202 applyUniqueRetValOpt(CSByConstantArg.second, "", ResByArg.Info,

2203 UniqueMemberAddr);

2204 break;

2205 }

2207 Constant *Byte = importConstant(Slot, CSByConstantArg.first, "byte",

2208 Int32Ty, ResByArg.Byte);

2209 Constant *Bit = importConstant(Slot, CSByConstantArg.first, "bit", Int8Ty,

2210 ResByArg.Bit);

2211 applyVirtualConstProp(CSByConstantArg.second, "", Byte, Bit);

2212 break;

2213 }

2214 default:

2215 break;

2216 }

2217 }

2218

2220

2221

2223 M.getOrInsertFunction(getGlobalName(Slot, {}, "branch_funnel"),

2225 .getCallee());

2226 bool IsExported = false;

2227 applyICallBranchFunnel(SlotInfo, JT, IsExported);

2228 assert(!IsExported);

2229 }

2230}

2231

2232void DevirtModule::removeRedundantTypeTests() {

2234 for (auto &&U : NumUnsafeUsesForTypeTest) {

2235 if (U.second == 0) {

2236 U.first->replaceAllUsesWith(True);

2237 U.first->eraseFromParent();

2238 }

2239 }

2240}

2241

2243DevirtModule::lookUpFunctionValueInfo(Function *TheFn,

2245 assert((ExportSummary != nullptr) &&

2246 "Caller guarantees ExportSummary is not nullptr");

2247

2248 const auto TheFnGUID = TheFn->getGUID();

2250

2252

2253

2254

2255

2256

2257

2258

2259

2260 if ((!TheFnVI) && (TheFnGUID != TheFnGUIDWithExportedName)) {

2261 TheFnVI = ExportSummary->getValueInfo(TheFnGUIDWithExportedName);

2262 }

2263 return TheFnVI;

2264}

2265

2266bool DevirtModule::mustBeUnreachableFunction(

2269 return false;

2270

2271 if (->isDeclaration()) {

2272

2273

2274 return isa(F->getEntryBlock().getTerminator());

2275 }

2276

2277 return ExportSummary &&

2279 DevirtModule::lookUpFunctionValueInfo(F, ExportSummary));

2280}

2281

2282bool DevirtModule::run() {

2283

2284

2285

2286

2289 return false;

2290

2293 Function *TypeCheckedLoadFunc =

2296 &M, Intrinsic::type_checked_load_relative);

2299

2300

2301

2302

2303 if (!ExportSummary &&

2304 (!TypeTestFunc || TypeTestFunc->use_empty() || !AssumeFunc ||

2306 (!TypeCheckedLoadFunc || TypeCheckedLoadFunc->use_empty()) &&

2307 (!TypeCheckedLoadRelativeFunc ||

2308 TypeCheckedLoadRelativeFunc->use_empty()))

2309 return false;

2310

2311

2312 std::vector Bits;

2314 buildTypeIdentifierMap(Bits, TypeIdMap);

2315

2316 if (TypeTestFunc && AssumeFunc)

2317 scanTypeTestUsers(TypeTestFunc, TypeIdMap);

2318

2319 if (TypeCheckedLoadFunc)

2320 scanTypeCheckedLoadUsers(TypeCheckedLoadFunc);

2321

2322 if (TypeCheckedLoadRelativeFunc)

2323 scanTypeCheckedLoadUsers(TypeCheckedLoadRelativeFunc);

2324

2325 if (ImportSummary) {

2326 for (auto &S : CallSlots)

2327 importResolution(S.first, S.second);

2328

2329 removeRedundantTypeTests();

2330

2331

2332

2333

2335 GV.eraseMetadata(LLVMContext::MD_vcall_visibility);

2336

2337

2338

2339 return true;

2340 }

2341

2342 if (TypeIdMap.empty())

2343 return true;

2344

2345

2346 if (ExportSummary) {

2348 for (auto &P : TypeIdMap) {

2349 if (auto *TypeId = dyn_cast(P.first))

2351 TypeId);

2352 }

2353

2354 for (auto &P : *ExportSummary) {

2355 for (auto &S : P.second.SummaryList) {

2356 auto *FS = dyn_cast(S.get());

2357 if (!FS)

2358 continue;

2359

2361 for (Metadata *MD : MetadataByGUID[VF.GUID]) {

2362 CallSlots[{MD, VF.Offset}].CSInfo.addSummaryTypeTestAssumeUser(FS);

2363 }

2364 }

2366 for (Metadata *MD : MetadataByGUID[VF.GUID]) {

2367 CallSlots[{MD, VF.Offset}].CSInfo.addSummaryTypeCheckedLoadUser(FS);

2368 }

2369 }

2371 FS->type_test_assume_const_vcalls()) {

2372 for (Metadata *MD : MetadataByGUID[VC.VFunc.GUID]) {

2373 CallSlots[{MD, VC.VFunc.Offset}]

2374 .ConstCSInfo[VC.Args]

2375 .addSummaryTypeTestAssumeUser(FS);

2376 }

2377 }

2379 FS->type_checked_load_const_vcalls()) {

2380 for (Metadata *MD : MetadataByGUID[VC.VFunc.GUID]) {

2381 CallSlots[{MD, VC.VFunc.Offset}]

2382 .ConstCSInfo[VC.Args]

2383 .addSummaryTypeCheckedLoadUser(FS);

2384 }

2385 }

2386 }

2387 }

2388 }

2389

2390

2391 bool DidVirtualConstProp = false;

2392 std::map<std::string, GlobalValue *> DevirtTargets;

2393 for (auto &S : CallSlots) {

2394

2395

2396

2397 std::vector TargetsForSlot;

2399 const std::set &TypeMemberInfos = TypeIdMap[S.first.TypeID];

2400 if (ExportSummary && isa(S.first.TypeID) &&

2401 TypeMemberInfos.size())

2402

2403

2404

2405

2406

2407

2408 Res = &ExportSummary

2409 ->getOrInsertTypeIdSummary(

2410 cast(S.first.TypeID)->getString())

2411 .WPDRes[S.first.ByteOffset];

2412 if (tryFindVirtualCallTargets(TargetsForSlot, TypeMemberInfos,

2413 S.first.ByteOffset, ExportSummary)) {

2414

2415 if (!trySingleImplDevirt(ExportSummary, TargetsForSlot, S.second, Res)) {

2416 DidVirtualConstProp |=

2417 tryVirtualConstProp(TargetsForSlot, S.second, Res, S.first);

2418

2419 tryICallBranchFunnel(TargetsForSlot, S.second, Res, S.first);

2420 }

2421

2422

2424 for (const auto &T : TargetsForSlot)

2425 if (T.WasDevirt)

2426 DevirtTargets[std::string(T.Fn->getName())] = T.Fn;

2427 }

2428

2429

2430

2431

2432

2433 if (ExportSummary && isa(S.first.TypeID)) {

2436 for (auto *FS : S.second.CSInfo.SummaryTypeCheckedLoadUsers)

2437 FS->addTypeTest(GUID);

2438 for (auto &CCS : S.second.ConstCSInfo)

2439 for (auto *FS : CCS.second.SummaryTypeCheckedLoadUsers)

2440 FS->addTypeTest(GUID);

2441 }

2442 }

2443

2444 if (RemarksEnabled) {

2445

2446 for (const auto &DT : DevirtTargets) {

2448 auto F = dyn_cast(GV);

2449 if () {

2450 auto A = dyn_cast(GV);

2451 assert(A && isa(A->getAliasee()));

2452 F = dyn_cast(A->getAliasee());

2454 }

2455

2456 using namespace ore;

2458 << "devirtualized "

2459 << NV("FunctionName", DT.first));

2460 }

2461 }

2462

2463 NumDevirtTargets += DevirtTargets.size();

2464

2465 removeRedundantTypeTests();

2466

2467

2468

2469 if (DidVirtualConstProp)

2471 rebuildGlobal(B);

2472

2473

2474

2475

2477 GV.eraseMetadata(LLVMContext::MD_vcall_visibility);

2478

2479 for (auto *CI : CallsWithPtrAuthBundleRemoved)

2480 CI->eraseFromParent();

2481

2482 return true;

2483}

2484

2485void DevirtIndex::run() {

2486 if (ExportSummary.typeIdCompatibleVtableMap().empty())

2487 return;

2488

2490 for (const auto &P : ExportSummary.typeIdCompatibleVtableMap()) {

2492

2493

2494

2495

2496

2497

2498 ExportSummary.getOrInsertTypeIdSummary(P.first);

2499 }

2500

2501

2502 for (auto &P : ExportSummary) {

2503 for (auto &S : P.second.SummaryList) {

2504 auto *FS = dyn_cast(S.get());

2505 if (!FS)

2506 continue;

2507

2510 CallSlots[{Name, VF.Offset}].CSInfo.addSummaryTypeTestAssumeUser(FS);

2511 }

2512 }

2515 CallSlots[{Name, VF.Offset}].CSInfo.addSummaryTypeCheckedLoadUser(FS);

2516 }

2517 }

2519 FS->type_test_assume_const_vcalls()) {

2521 CallSlots[{Name, VC.VFunc.Offset}]

2522 .ConstCSInfo[VC.Args]

2523 .addSummaryTypeTestAssumeUser(FS);

2524 }

2525 }

2527 FS->type_checked_load_const_vcalls()) {

2529 CallSlots[{Name, VC.VFunc.Offset}]

2530 .ConstCSInfo[VC.Args]

2531 .addSummaryTypeCheckedLoadUser(FS);

2532 }

2533 }

2534 }

2535 }

2536

2537 std::set DevirtTargets;

2538

2539 for (auto &S : CallSlots) {

2540

2541

2542

2543 std::vector TargetsForSlot;

2544 auto TidSummary = ExportSummary.getTypeIdCompatibleVtableSummary(S.first.TypeID);

2546

2547

2549 &ExportSummary.getTypeIdSummary(S.first.TypeID)

2550 ->WPDRes[S.first.ByteOffset];

2551 if (tryFindVirtualCallTargets(TargetsForSlot, *TidSummary,

2552 S.first.ByteOffset)) {

2553

2554 if (!trySingleImplDevirt(TargetsForSlot, S.first, S.second, Res,

2555 DevirtTargets))

2556 continue;

2557 }

2558 }

2559

2560

2561

2563 for (const auto &DT : DevirtTargets)

2564 errs() << "Devirtualized call to " << DT << "\n";

2565

2566 NumDevirtTargets += DevirtTargets.size();

2567}

This is the interface for LLVM's primary stateless and local alias analysis.

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)

Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...

#define clEnumValN(ENUMVAL, FLAGNAME, DESC)

This file contains the declarations for the subclasses of Constant, which represent the different fla...

This file defines DenseMapInfo traits for DenseMap.

This file defines the DenseMap class.

This file defines the DenseSet and SmallDenseSet classes.

Provides passes for computing function attributes based on interprocedural analyses.

static void emitRemark(const Function &F, OptimizationRemarkEmitter &ORE, bool Skip)

Module.h This file contains the declarations for the Module class.

static cl::opt< std::string > ClReadSummary("lowertypetests-read-summary", cl::desc("Read summary from given YAML file before running pass"), cl::Hidden)

static cl::opt< PassSummaryAction > ClSummaryAction("lowertypetests-summary-action", cl::desc("What to do with the summary when running this pass"), cl::values(clEnumValN(PassSummaryAction::None, "none", "Do nothing"), clEnumValN(PassSummaryAction::Import, "import", "Import typeid resolutions from summary and globals"), clEnumValN(PassSummaryAction::Export, "export", "Export typeid resolutions to summary and globals")), cl::Hidden)

static cl::opt< std::string > ClWriteSummary("lowertypetests-write-summary", cl::desc("Write summary to given YAML file after running pass"), cl::Hidden)

This file implements a map that provides insertion order iteration.

static bool mustBeUnreachableFunction(const Function &F)

uint64_t IntrinsicInst * II

FunctionAnalysisManager FAM

const SmallVectorImpl< MachineOperand > & Cond

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

This file defines the SmallVector class.

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

static cl::opt< bool > DisableWholeProgramVisibility("disable-whole-program-visibility", cl::Hidden, cl::desc("Disable whole program visibility (overrides enabling options)"))

Provide a way to force disable whole program for debugging or workarounds, when enabled via the linke...

WPDCheckMode

Mechanism to add runtime checking of devirtualization decisions, optionally trapping or falling back ...

static cl::opt< PassSummaryAction > ClSummaryAction("wholeprogramdevirt-summary-action", cl::desc("What to do with the summary when running this pass"), cl::values(clEnumValN(PassSummaryAction::None, "none", "Do nothing"), clEnumValN(PassSummaryAction::Import, "import", "Import typeid resolutions from summary and globals"), clEnumValN(PassSummaryAction::Export, "export", "Export typeid resolutions to summary and globals")), cl::Hidden)

static cl::opt< bool > WholeProgramVisibility("whole-program-visibility", cl::Hidden, cl::desc("Enable whole program visibility"))

Provide a way to force enable whole program visibility in tests.

static bool typeIDVisibleToRegularObj(StringRef TypeID, function_ref< bool(StringRef)> IsVisibleToRegularObj)

static Error checkCombinedSummaryForTesting(ModuleSummaryIndex *Summary)

static cl::list< std::string > SkipFunctionNames("wholeprogramdevirt-skip", cl::desc("Prevent function(s) from being devirtualized"), cl::Hidden, cl::CommaSeparated)

Provide way to prevent certain function from being devirtualized.

static cl::opt< std::string > ClWriteSummary("wholeprogramdevirt-write-summary", cl::desc("Write summary to given bitcode or YAML file after running pass. " "Output file format is deduced from extension: *.bc means writing " "bitcode, otherwise YAML"), cl::Hidden)

static cl::opt< unsigned > ClThreshold("wholeprogramdevirt-branch-funnel-threshold", cl::Hidden, cl::init(10), cl::desc("Maximum number of call targets per " "call site to enable branch funnels"))

static cl::opt< WPDCheckMode > DevirtCheckMode("wholeprogramdevirt-check", cl::Hidden, cl::desc("Type of checking for incorrect devirtualizations"), cl::values(clEnumValN(WPDCheckMode::None, "none", "No checking"), clEnumValN(WPDCheckMode::Trap, "trap", "Trap when incorrect"), clEnumValN(WPDCheckMode::Fallback, "fallback", "Fallback to indirect when incorrect")))

static cl::opt< bool > WholeProgramDevirtKeepUnreachableFunction("wholeprogramdevirt-keep-unreachable-function", cl::desc("Regard unreachable functions as possible devirtualize targets."), cl::Hidden, cl::init(true))

With Clang, a pure virtual class's deleting destructor is emitted as a llvm.trap intrinsic followed b...

static cl::opt< std::string > ClReadSummary("wholeprogramdevirt-read-summary", cl::desc("Read summary from given bitcode or YAML file before running pass"), cl::Hidden)

static bool skipUpdateDueToValidation(GlobalVariable &GV, function_ref< bool(StringRef)> IsVisibleToRegularObj)

static bool AddCalls(VTableSlotInfo &SlotInfo, const ValueInfo &Callee)

static cl::opt< unsigned > WholeProgramDevirtCutoff("wholeprogramdevirt-cutoff", cl::desc("Max number of devirtualizations for devirt module pass"), cl::init(0))

If explicitly specified, the devirt module pass will stop transformation once the total number of dev...

static cl::opt< bool > PrintSummaryDevirt("wholeprogramdevirt-print-index-based", cl::Hidden, cl::desc("Print index-based devirtualization messages"))

A manager for alias analyses.

A container for analyses that lazily runs them and caches their results.

PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)

Get the result of an analysis pass for a given IR unit.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

ArrayRef< T > slice(size_t N, size_t M) const

slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.

static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)

Create an AttributeList with the specified parameters in it.

static AttributeSet get(LLVMContext &C, const AttrBuilder &B)

StringRef getValueAsString() const

Return the attribute's value as a string.

bool isValid() const

Return true if the attribute is any kind of attribute.

LLVM Basic Block Representation.

static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)

Creates a new BasicBlock.

static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

void setCallingConv(CallingConv::ID CC)

std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const

Return an operand bundle by name, if present.

Function * getCalledFunction() const

Returns the function called, or null if this is an indirect function invocation or the function signa...

CallingConv::ID getCallingConv() const

Value * getCalledOperand() const

void setAttributes(AttributeList A)

Set the attributes for this call.

FunctionType * getFunctionType() const

iterator_range< User::op_iterator > args()

Iteration adapter for range-for loops.

void setCalledOperand(Value *V)

static CallBase * removeOperandBundle(CallBase *CB, uint32_t ID, InsertPosition InsertPt=nullptr)

Create a clone of CB with operand bundle ID removed.

AttributeList getAttributes() const

Return the attributes for this call.

Function * getCaller()

Helper to get the caller (the parent function).

This class represents a function call, abstracting a target machine's calling convention.

static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)

void setSelectionKind(SelectionKind Val)

static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)

get() constructor - Return a constant with array type with an element count and element type matching...

static Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)

static Constant * getInBoundsGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList)

Create an "inbounds" getelementptr.

static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)

static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, GEPNoWrapFlags NW=GEPNoWrapFlags::none(), std::optional< ConstantRange > InRange=std::nullopt, Type *OnlyIfReducedTy=nullptr)

Getelementptr form.

static ConstantInt * getTrue(LLVMContext &Context)

static Constant * getAnon(ArrayRef< Constant * > V, bool Packed=false)

Return an anonymous struct that has the specified elements.

This is an important base class in LLVM.

const Constant * stripPointerCasts() const

static Constant * getNullValue(Type *Ty)

Constructor to create a '0' constant of arbitrary type.

Implements a dense probed hash-table based set.

Analysis pass which computes a DominatorTree.

Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.

Subclass of Error for the sole purpose of identifying the success path in the type system.

Lightweight error class with error context and mandatory checking.

This class evaluates LLVM IR, producing the Constant representing each SSA instruction.

Helper for check-and-exit error handling.

Tagged union holding either a T or a Error.

Function summary information to aid decisions and implementation of importing.

Type * getParamType(unsigned i) const

Parameter type accessors.

ArrayRef< Type * > params() const

Type * getReturnType() const

static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)

This static method is the primary way of constructing a FunctionType.

static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)

FunctionType * getFunctionType() const

Returns the FunctionType for me.

const BasicBlock & front() const

Attribute getFnAttribute(Attribute::AttrKind Kind) const

Return the attribute for the given attribute kind.

Intrinsic::ID getIntrinsicID() const LLVM_READONLY

getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...

Type * getReturnType() const

Returns the type of the ret val.

This class implements a glob pattern matcher similar to the one found in bash, but with some key diff...

static Expected< GlobPattern > create(StringRef Pat, std::optional< size_t > MaxSubPatterns={})

static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)

If a parent module is specified, the alias is automatically inserted into the end of the specified mo...

bool hasMetadata() const

Return true if this value has any metadata attached to it.

void setMetadata(unsigned KindID, MDNode *Node)

Set a particular kind of metadata attachment.

VCallVisibility getVCallVisibility() const

bool eraseMetadata(unsigned KindID)

Erase all metadata attachments with the given kind.

@ VCallVisibilityLinkageUnit

MDNode * getMetadata(unsigned KindID) const

Get the current metadata attachments for the given kind, if any.

void setVCallVisibilityMetadata(VCallVisibility Visibility)

static bool isLocalLinkage(LinkageTypes Linkage)

bool isDeclaration() const

Return true if the primary definition of this global value is outside of the current translation unit...

static bool isAvailableExternallyLinkage(LinkageTypes Linkage)

static GUID getGUID(StringRef GlobalName)

Return a 64-bit global unique ID constructed from global value name (i.e.

GUID getGUID() const

Return a 64-bit global unique ID constructed from global value name (i.e.

@ HiddenVisibility

The GV is hidden.

void setVisibility(VisibilityTypes V)

@ PrivateLinkage

Like Internal, but omit from symbol table.

@ InternalLinkage

Rename collisions when linking (static functions).

@ ExternalLinkage

Externally visible function.

Global variable summary information to aid decisions and implementation of importing.

const Constant * getInitializer() const

getInitializer - Return the initializer for this global variable.

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...

const DebugLoc & getDebugLoc() const

Return the debug location for this node as a DebugLoc.

InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

void setMetadata(unsigned KindID, MDNode *Node)

Set the metadata of the specified kind to the specified node.

Class to represent integer types.

unsigned getBitWidth() const

Get the number of bits in this IntegerType.

MDNode * createLikelyBranchWeights()

Return metadata containing two branch weights, with significant bias towards true destination.

MDNode * createUnlikelyBranchWeights()

Return metadata containing two branch weights, with significant bias towards false destination.

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

This class implements a map that also provides access to all stored values in a deterministic order.

static ErrorOr< std::unique_ptr< MemoryBuffer > > getFile(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, bool IsVolatile=false, std::optional< Align > Alignment=std::nullopt)

Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful,...

Class to hold module path string table and global value map, and encapsulate methods for operating on...

const TypeIdSummary * getTypeIdSummary(StringRef TypeId) const

This returns either a pointer to the type id summary (if present in the summary map) or null (if not ...

ValueInfo getValueInfo(const GlobalValueSummaryMapTy::value_type &R) const

Return a ValueInfo for the index value_type (convenient when iterating index).

const ModuleHash & getModuleHash(const StringRef ModPath) const

Get the module SHA1 hash recorded for the given module path.

static constexpr const char * getRegularLTOModuleName()

bool partiallySplitLTOUnits() const

static std::string getGlobalNameForLocal(StringRef Name, ModuleHash ModHash)

Convenience method for creating a promoted global name for the given value name of a local,...

A Module instance is used to store all the information related to an LLVM module.

MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...

static PoisonValue * get(Type *T)

Static factory methods - Return an 'poison' object of the specified type.

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, InsertPosition InsertBefore=nullptr)

std::pair< iterator, bool > insert(PtrType Ptr)

Inserts Ptr if and only if there is no element in the container equal to Ptr.

SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

bool contains(StringRef Other) const

Return true if the given string is a substring of *this, and false otherwise.

Target - Wrapper for Target specific information.

Triple - Helper class for working with autoconf configuration names.

The instances of the Type class are immutable: once they are created, they are never changed.

TypeID

Definitions of all of the base types for the Type system.

static Type * getVoidTy(LLVMContext &C)

A Use represents the edge between a Value definition and its users.

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

void setName(const Twine &Name)

Change the name of the value.

void replaceAllUsesWith(Value *V)

Change all uses of this to point to a new Value.

bool eraseMetadata(unsigned KindID)

Erase all metadata attachments with the given kind.

iterator_range< use_iterator > uses()

StringRef getName() const

Return a constant reference to the value's name.

std::pair< iterator, bool > insert(const ValueT &V)

size_type count(const_arg_type_t< ValueT > V) const

Return 1 if the specified key is in the set, 0 otherwise.

An efficient, type-erasing, non-owning reference to a callable.

const ParentTy * getParent() const

self_iterator getIterator()

A raw_ostream that writes to a file descriptor.

A raw_ostream that writes to an std::string.

constexpr char Args[]

Key for Kernel::Metadata::mArgs.

constexpr char Attrs[]

Key for Kernel::Metadata::mAttrs.

@ C

The default llvm calling convention, compatible with C.

Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})

Look up the Function declaration of the intrinsic id in the Module M.

Function * getDeclarationIfExists(Module *M, ID id, ArrayRef< Type * > Tys, FunctionType *FT=nullptr)

This version supports overloaded intrinsics.

bool match(Val *V, const Pattern &P)

ValuesClass values(OptsTy... Options)

Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...

initializer< Ty > init(const Ty &Val)

PointerTypeMap run(const Module &M)

Compute the PointerTypeMap for the module M.

@ Assume

Do not drop type tests (default).

DiagnosticInfoOptimizationBase::Argument NV

@ OF_TextWithCRLF

The file should be opened in text mode and use a carriage linefeed '\r '.

uint64_t findLowestOffset(ArrayRef< VirtualCallTarget > Targets, bool IsAfter, uint64_t Size)

void setAfterReturnValues(MutableArrayRef< VirtualCallTarget > Targets, uint64_t AllocAfter, unsigned BitWidth, int64_t &OffsetByte, uint64_t &OffsetBit)

void setBeforeReturnValues(MutableArrayRef< VirtualCallTarget > Targets, uint64_t AllocBefore, unsigned BitWidth, int64_t &OffsetByte, uint64_t &OffsetBit)

This is an optimization pass for GlobalISel generic memory operations.

auto drop_begin(T &&RangeOrContainer, size_t N=1)

Return a range covering RangeOrContainer with the first N elements excluded.

MemoryEffects computeFunctionBodyMemoryAccess(Function &F, AAResults &AAR)

Returns the memory access properties of this copy of the function.

@ Export

Export information to summary.

@ Import

Import information from summary.

void append_range(Container &C, Range &&R)

Wrapper function to append range R to container C.

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

Error createStringError(std::error_code EC, char const *Fmt, const Ts &... Vals)

Create formatted StringError object.

bool hasWholeProgramVisibility(bool WholeProgramVisibilityEnabledInLTO)

int countr_zero(T Val)

Count number of 0's from the least significant bit to the most stopping at the first 1.

void writeIndexToFile(const ModuleSummaryIndex &Index, raw_ostream &Out, const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex=nullptr, const GVSummaryPtrSet *DecSummaries=nullptr)

Write the specified module summary index to the given raw output stream, where it will be written in ...

Expected< std::unique_ptr< ModuleSummaryIndex > > getModuleSummaryIndex(MemoryBufferRef Buffer)

Parse the specified bitcode buffer, returning the module summary index.

void updatePublicTypeTestCalls(Module &M, bool WholeProgramVisibilityEnabledInLTO)

void getVisibleToRegularObjVtableGUIDs(ModuleSummaryIndex &Index, DenseSet< GlobalValue::GUID > &VisibleToRegularObjSymbols, function_ref< bool(StringRef)> IsVisibleToRegularObj)

Based on typeID string, get all associated vtable GUIDS that are visible to regular objects.

decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)

void findDevirtualizableCallsForTypeCheckedLoad(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< Instruction * > &LoadedPtrs, SmallVectorImpl< Instruction * > &Preds, bool &HasNonCallUses, const CallInst *CI, DominatorTree &DT)

Given a call to the intrinsic @llvm.type.checked.load, find all devirtualizable call sites based on t...

CallBase & versionCallSite(CallBase &CB, Value *Callee, MDNode *BranchWeights)

Predicate and clone the given call site.

bool AreStatisticsEnabled()

Check if statistics are enabled.

void updateIndexWPDForExports(ModuleSummaryIndex &Summary, function_ref< bool(StringRef, ValueInfo)> isExported, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)

Call after cross-module importing to update the recorded single impl devirt target names for any loca...

raw_fd_ostream & errs()

This returns a reference to a raw_ostream for standard error.

void runWholeProgramDevirtOnIndex(ModuleSummaryIndex &Summary, std::set< GlobalValue::GUID > &ExportedGUIDs, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)

Perform index-based whole program devirtualization on the Summary index.

uint64_t alignTo(uint64_t Size, Align A)

Returns a multiple of A needed to store Size bytes.

Expected< T > errorOrToExpected(ErrorOr< T > &&EO)

Convert an ErrorOr to an Expected.

constexpr unsigned BitWidth

Error errorCodeToError(std::error_code EC)

Helper for converting an std::error_code to a Error.

Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)

Split the containing block at the specified instruction - everything before SplitBefore stays in the ...

std::vector< TypeIdOffsetVtableInfo > TypeIdCompatibleVtableInfo

List of vtable definitions decorated by a particular type identifier, and their corresponding offsets...

void consumeError(Error Err)

Consume a Error without doing anything.

void findDevirtualizableCallsForTypeTest(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< CallInst * > &Assumes, const CallInst *CI, DominatorTree &DT)

Given a call to the intrinsic @llvm.type.test, find all devirtualizable call sites based on the call ...

void updateVCallVisibilityInModule(Module &M, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols, bool ValidateAllVtablesHaveTypeInfos, function_ref< bool(StringRef)> IsVisibleToRegularObj)

If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...

std::pair< Function *, Constant * > getFunctionAtVTableOffset(GlobalVariable *GV, uint64_t Offset, Module &M)

Given a vtable and a specified offset, returns the function and the trivial pointer at the specified ...

void updateVCallVisibilityInIndex(ModuleSummaryIndex &Index, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols, const DenseSet< GlobalValue::GUID > &VisibleToRegularObjSymbols)

If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...

void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)

Implement std::swap in terms of BitVector swap.

This struct is a compact representation of a valid (non-zero power of two) alignment.

Class to accumulate and hold information about a callee.

static unsigned getHashValue(const VTableSlotSummary &I)

static bool isEqual(const VTableSlotSummary &LHS, const VTableSlotSummary &RHS)

static VTableSlotSummary getTombstoneKey()

static VTableSlotSummary getEmptyKey()

static VTableSlot getTombstoneKey()

static VTableSlot getEmptyKey()

static bool isEqual(const VTableSlot &LHS, const VTableSlot &RHS)

static unsigned getHashValue(const VTableSlot &I)

An information struct used to provide DenseMap with the various necessary components for a given valu...

A call site that could be devirtualized.

A specification for a virtual function call with all constant integer arguments.

An "identifier" for a virtual function.

The following data structures summarize type metadata information.

std::map< uint64_t, WholeProgramDevirtResolution > WPDRes

Mapping from byte offset to whole-program devirt resolution for that (typeid, byte offset) pair.

@ Unsat

Unsatisfiable type (i.e. no global has this type metadata)

enum llvm::TypeTestResolution::Kind TheKind

Struct that holds a reference to a particular GUID in a global value summary.

ArrayRef< std::unique_ptr< GlobalValueSummary > > getSummaryList() const

PreservedAnalyses run(Module &M, ModuleAnalysisManager &)

@ UniformRetVal

Uniform return value optimization.

@ VirtualConstProp

Virtual constant propagation.

@ UniqueRetVal

Unique return value optimization.

uint64_t Info

Additional information for the resolution:

enum llvm::WholeProgramDevirtResolution::ByArg::Kind TheKind

enum llvm::WholeProgramDevirtResolution::Kind TheKind

std::map< std::vector< uint64_t >, ByArg > ResByArg

Resolutions for calls with all constant integer arguments (excluding the first argument,...

std::string SingleImplName

@ SingleImpl

Single implementation devirtualization.

@ BranchFunnel

When retpoline mitigation is enabled, use a branch funnel that is defined in the merged module.

VirtualCallTarget(GlobalValue *Fn, const TypeMemberInfo *TM)