LowerTypeTests.cpp Source File (original) (raw)

78#include

79#include

80#include

81#include

82#include

83#include <system_error>

84#include

85#include

87using namespace llvm;

90#define DEBUG_TYPE "lowertypetests"

92 STATISTIC(ByteArraySizeBits, "Byte array size in bits");

93 STATISTIC(ByteArraySizeBytes, "Byte array size in bytes");

94 STATISTIC(NumByteArraysCreated, "Number of byte arrays created");

95 STATISTIC(NumTypeTestCallsLowered, "Number of type test calls lowered");

96 STATISTIC(NumTypeIdDisjointSets, "Number of disjoint sets of type identifiers");

99 "lowertypetests-avoid-reuse",

100 cl::desc("Try to avoid reuse of byte array addresses using aliases"),

102

104 "lowertypetests-summary-action",

105 cl::desc("What to do with the summary when running this pass"),

108 "Import typeid resolutions from summary and globals"),

110 "Export typeid resolutions to summary and globals")),

112

114 "lowertypetests-read-summary",

115 cl::desc("Read summary from given YAML file before running pass"),

117

119 "lowertypetests-write-summary",

120 cl::desc("Write summary to given YAML file after running pass"),

122

125 cl::desc("Simply drop type test sequences"),

127 "Do not drop any type tests"),

128 clEnumValN(DropTestKind::Assume, "assume",

129 "Drop type test assume sequences"),

131 "Drop all type test sequences")),

133

136 return false;

137

139 return false;

140

142 if (BitOffset >= BitSize)

143 return false;

144

145 return Bits.count(BitSize - 1 - BitOffset);

146}

147

149 OS << "offset " << ByteOffset << " size " << BitSize << " align "

151

153 OS << " all-ones\n";

154 return;

155 }

156

157 OS << " { ";

159 OS << B << ' ';

160 OS << "}\n";

161}

162

166

167

168

169

170

171

176 }

177

180

182 if (Mask != 0)

184

185

186

190

191

192

193

195 }

196

197 return BSI;

198}

199

201

203 std::vector<uint64_t> &Fragment = Fragments.back();

205

206 for (auto ObjIndex : F) {

208 if (OldFragmentIndex == 0) {

209

210

211 Fragment.push_back(ObjIndex);

212 } else {

213

214

215

216

217

218 std::vector<uint64_t> &OldFragment = Fragments[OldFragmentIndex];

220 OldFragment.clear();

221 }

222 }

223

224

225 for (uint64_t ObjIndex : Fragment)

227}

228

232

233 unsigned Bit = 0;

236 Bit = I;

237

238 AllocByteOffset = BitAllocs[Bit];

239

240

241 unsigned ReqSize = AllocByteOffset + BitSize;

243 if (Bytes.size() < ReqSize)

244 Bytes.resize(ReqSize);

245

246

247 AllocMask = 1 << Bit;

249 Bytes[AllocByteOffset + B] |= AllocMask;

250}

251

253 if (F->isDeclarationForLinker())

254 return false;

256 F->getParent()->getModuleFlag("CFI Canonical Jump Tables"));

257 if (!CI || !CI->isZero())

258 return true;

259 return F->hasFnAttribute("cfi-canonical-jump-table");

260}

261

262namespace {

263

264struct ByteArrayInfo {

265 std::set<uint64_t> Bits;

269 uint8_t *MaskPtr = nullptr;

270};

271

272

273

274

275

276

277class GlobalTypeMember final : TrailingObjects<GlobalTypeMember, MDNode *> {

279

281 size_t NTypes;

282

283

284

285

286

287

288 bool IsJumpTableCanonical;

289

290

291

292 bool IsExported;

293

294public:

296 bool IsJumpTableCanonical, bool IsExported,

298 auto *GTM = static_cast<GlobalTypeMember *>(Alloc.Allocate(

299 totalSizeToAlloc<MDNode *>(Types.size()), alignof(GlobalTypeMember)));

300 GTM->GO = GO;

301 GTM->NTypes = Types.size();

302 GTM->IsJumpTableCanonical = IsJumpTableCanonical;

303 GTM->IsExported = IsExported;

304 llvm::copy(Types, GTM->getTrailingObjects());

305 return GTM;

306 }

307

308 GlobalObject *getGlobal() const {

309 return GO;

310 }

311

313 return IsJumpTableCanonical;

314 }

315

316 bool isExported() const {

317 return IsExported;

318 }

319

320 ArrayRef<MDNode *> types() const { return getTrailingObjects(NTypes); }

321};

322

323struct ICallBranchFunnel final

324 : TrailingObjects<ICallBranchFunnel, GlobalTypeMember *> {

327 unsigned UniqueId) {

328 auto *Call = static_cast<ICallBranchFunnel *>(

329 Alloc.Allocate(totalSizeToAlloc<GlobalTypeMember *>(Targets.size()),

330 alignof(ICallBranchFunnel)));

331 Call->CI = CI;

332 Call->UniqueId = UniqueId;

333 Call->NTargets = Targets.size();

336 }

337

338 CallInst *CI;

340 return getTrailingObjects(NTargets);

341 }

342

343 unsigned UniqueId;

344

345private:

346 size_t NTargets;

347};

348

349struct ScopedSaveAliaseesAndUsed {

352 std::vector<std::pair<GlobalAlias *, Function *>> FunctionAliases;

353 std::vector<std::pair<GlobalIFunc *, Function *>> ResolverIFuncs;

354

355

356

357

358 void collectAndEraseUsedFunctions(Module &M,

359 SmallVectorImpl<GlobalValue *> &Vec,

360 bool CompilerUsed) {

362 if (!GV)

363 return;

364

365

366

367 GV->eraseFromParent();

368 auto NonFuncBegin =

369 std::stable_partition(Vec.begin(), Vec.end(), [](GlobalValue *GV) {

370 return isa(GV);

371 });

372 if (CompilerUsed)

374 else

377 }

378

379 ScopedSaveAliaseesAndUsed(Module &M) : M(M) {

380

381

382

383

384

385

386

387

388

389

390

391

392 collectAndEraseUsedFunctions(M, Used, false);

393 collectAndEraseUsedFunctions(M, CompilerUsed, true);

394

395 for (auto &GA : M.aliases()) {

396

397

399 FunctionAliases.push_back({&GA, F});

400 }

401

402 for (auto &GI : M.ifuncs())

404 ResolverIFuncs.push_back({&GI, F});

405 }

406

407 ~ScopedSaveAliaseesAndUsed() {

410

411 for (auto P : FunctionAliases)

412 P.first->setAliasee(P.second);

413

414 for (auto P : ResolverIFuncs) {

415

416

417

418 P.first->setResolver(P.second);

419 }

420 }

421};

422

423class LowerTypeTestsModule {

425

426 ModuleSummaryIndex *ExportSummary;

427 const ModuleSummaryIndex *ImportSummary;

428

429

431

435

436

437

438

439 bool CanUseArmJumpTable = false, CanUseThumbBWJumpTable = false;

440

441

442 int HasBranchTargetEnforcement = -1;

443

444 IntegerType *Int1Ty = Type::getInt1Ty(M.getContext());

445 IntegerType *Int8Ty = Type::getInt8Ty(M.getContext());

446 PointerType *PtrTy = PointerType::getUnqual(M.getContext());

447 ArrayType *Int8Arr0Ty = ArrayType::get(Type::getInt8Ty(M.getContext()), 0);

448 IntegerType *Int32Ty = Type::getInt32Ty(M.getContext());

449 IntegerType *Int64Ty = Type::getInt64Ty(M.getContext());

450 IntegerType *IntPtrTy = M.getDataLayout().getIntPtrType(M.getContext(), 0);

451

452

453 uint64_t IndirectIndex = 1;

454

455

456

457

458 struct TypeIdUserInfo {

459 std::vector<CallInst *> CallSites;

460 bool IsExported = false;

461 };

462 DenseMap<Metadata *, TypeIdUserInfo> TypeIdUsers;

463

464

465

466

467

468 struct TypeIdLowering {

470

471

472

474

475

476

478

479

480

482

483

485

486

488

489

491 };

492

493 std::vector ByteArrayInfos;

494

495 Function *WeakInitializerFn = nullptr;

496

497 GlobalVariable *GlobalAnnotation;

498 DenseSet<Value *> FunctionAnnotations;

499

500 bool shouldExportConstantsAsAbsoluteSymbols();

501 uint8_t *exportTypeId(StringRef TypeId, const TypeIdLowering &TIL);

502 TypeIdLowering importTypeId(StringRef TypeId);

503 void importTypeTest(CallInst *CI);

505

506 ByteArrayInfo *createByteArray(const BitSetInfo &BSI);

507 void allocateByteArrays();

508 Value *createBitSetTest(IRBuilder<> &B, const TypeIdLowering &TIL,

509 Value *BitOffset);

510 void lowerTypeTestCalls(

512 const DenseMap<GlobalTypeMember *, uint64_t> &GlobalLayout);

513 Value *lowerTypeTestCall(Metadata *TypeId, CallInst *CI,

514 const TypeIdLowering &TIL);

515

520 bool hasBranchTargetEnforcement();

521 unsigned getJumpTableEntrySize(Triple::ArchType JumpTableArch);

522 InlineAsm *createJumpTableEntryAsm(Triple::ArchType JumpTableArch);

523 void verifyTypeMDNode(GlobalObject *GO, MDNode *Type);

530 void

534

535 void replaceWeakDeclarationWithJumpTablePtr(Function *F, Constant *JT,

536 bool IsJumpTableCanonical);

537 void moveInitializerToModuleConstructor(GlobalVariable *GV);

538 void findGlobalVariableUsersOf(Constant *C,

539 SmallSetVector<GlobalVariable *, 8> &Out);

540

543

544

545

546

547

548

549 void replaceCfiUses(Function *Old, Value *New, bool IsJumpTableCanonical);

550

551

552

553 void replaceDirectCalls(Value *Old, Value *New);

554

555 bool isFunctionAnnotation(Value *V) const {

556 return FunctionAnnotations.contains(V);

557 }

558

559 void maybeReplaceComdat(Function *F, StringRef OriginalName);

560

561public:

563 ModuleSummaryIndex *ExportSummary,

564 const ModuleSummaryIndex *ImportSummary,

566

567 bool lower();

568

569

570

572};

573}

574

575

581

582

583

585 Value *BitOffset) {

587 unsigned BitWidth = BitsType->getBitWidth();

588

589 BitOffset = B.CreateZExtOrTrunc(BitOffset, BitsType);

590 Value *BitIndex =

591 B.CreateAnd(BitOffset, ConstantInt::get(BitsType, BitWidth - 1));

592 Value *BitMask = B.CreateShl(ConstantInt::get(BitsType, 1), BitIndex);

593 Value *MaskedBits = B.CreateAnd(Bits, BitMask);

594 return B.CreateICmpNE(MaskedBits, ConstantInt::get(BitsType, 0));

595}

596

597ByteArrayInfo *LowerTypeTestsModule::createByteArray(const BitSetInfo &BSI) {

598

599

600

601 auto ByteArrayGlobal = new GlobalVariable(

603 auto MaskGlobal = new GlobalVariable(M, Int8Ty, true,

605

606 ByteArrayInfos.emplace_back();

607 ByteArrayInfo *BAI = &ByteArrayInfos.back();

608

609 BAI->Bits = BSI.Bits;

610 BAI->BitSize = BSI.BitSize;

611 BAI->ByteArray = ByteArrayGlobal;

612 BAI->MaskGlobal = MaskGlobal;

613 return BAI;

614}

615

616void LowerTypeTestsModule::allocateByteArrays() {

618 [](const ByteArrayInfo &BAI1, const ByteArrayInfo &BAI2) {

619 return BAI1.BitSize > BAI2.BitSize;

620 });

621

622 std::vector<uint64_t> ByteArrayOffsets(ByteArrayInfos.size());

623

625 for (unsigned I = 0; I != ByteArrayInfos.size(); ++I) {

626 ByteArrayInfo *BAI = &ByteArrayInfos[I];

627

628 uint8_t Mask;

629 BAB.allocate(BAI->Bits, BAI->BitSize, ByteArrayOffsets[I], Mask);

630

634 if (BAI->MaskPtr)

635 *BAI->MaskPtr = Mask;

636 }

637

640 new GlobalVariable(M, ByteArrayConst->getType(), true,

642

643 for (unsigned I = 0; I != ByteArrayInfos.size(); ++I) {

644 ByteArrayInfo *BAI = &ByteArrayInfos[I];

645

646 Constant *Idxs[] = {ConstantInt::get(IntPtrTy, 0),

647 ConstantInt::get(IntPtrTy, ByteArrayOffsets[I])};

649 ByteArrayConst->getType(), ByteArray, Idxs);

650

651

652

653

658 }

659

663 ByteArraySizeBytes = BAB.Bytes.size();

664}

665

666

667

669 const TypeIdLowering &TIL,

670 Value *BitOffset) {

672

673

675 } else {

678

679

680

681

683 "bits_use", ByteArray, &M);

684 }

685

686 Value *ByteAddr = B.CreateGEP(Int8Ty, ByteArray, BitOffset);

687 Value *Byte = B.CreateLoad(Int8Ty, ByteAddr);

688

689 Value *ByteAndMask =

691 return B.CreateICmpNE(ByteAndMask, ConstantInt::get(Int8Ty, 0));

692 }

693}

694

699 GV->getMetadata(LLVMContext::MD_type, Types);

701 if (Type->getOperand(1) != TypeId)

702 continue;

706 ->getZExtValue();

707 if (COffset == Offset)

708 return true;

709 }

710 return false;

711 }

712

714 APInt APOffset(DL.getIndexSizeInBits(0), 0);

715 bool Result = GEP->accumulateConstantOffset(DL, APOffset);

716 if (!Result)

717 return false;

720 }

721

723 if (Op->getOpcode() == Instruction::BitCast)

725

726 if (Op->getOpcode() == Instruction::Select)

729 }

730

731 return false;

732}

733

734

735

736Value *LowerTypeTestsModule::lowerTypeTestCall(Metadata *TypeId, CallInst *CI,

737 const TypeIdLowering &TIL) {

738

740 return nullptr;

743

745 const DataLayout &DL = M.getDataLayout();

748

750

752

753 Value *PtrAsInt = B.CreatePtrToInt(Ptr, IntPtrTy);

754

755 Constant *OffsetedGlobalAsInt =

758 return B.CreateICmpEQ(PtrAsInt, OffsetedGlobalAsInt);

759

760

761

762

763

764 Value *PtrOffset = B.CreateSub(OffsetedGlobalAsInt, PtrAsInt);

765

766

767

768

769

770

771

772

773

774 Value *BitOffset = B.CreateIntrinsic(IntPtrTy, Intrinsic::fshr,

775 {PtrOffset, PtrOffset, TIL.AlignLog2});

776

777 Value *OffsetInRange = B.CreateICmpULE(BitOffset, TIL.SizeM1);

778

779

781 return OffsetInRange;

782

783

784

785

786

794 Br->getMetadata(LLVMContext::MD_prof));

796

797

798 for (auto &Phi : Else->phis())

799 Phi.addIncoming(Phi.getIncomingValueForBlock(Then), InitialBB);

800

802 return createBitSetTest(ThenB, TIL, BitOffset);

803 }

804

806

807

808

809 Value *Bit = createBitSetTest(ThenB, TIL, BitOffset);

810

811

812

813

814 B.SetInsertPoint(CI);

815 PHINode *P = B.CreatePHI(Int1Ty, 2);

816 P->addIncoming(ConstantInt::get(Int1Ty, 0), InitialBB);

817 P->addIncoming(Bit, ThenB.GetInsertBlock());

818 return P;

819}

820

821

822

823void LowerTypeTestsModule::buildBitSetsFromGlobalVariables(

825

826

827

828

829

830 std::vector<Constant *> GlobalInits;

831 const DataLayout &DL = M.getDataLayout();

832 DenseMap<GlobalTypeMember *, uint64_t> GlobalLayout;

834 uint64_t CurOffset = 0;

835 uint64_t DesiredPadding = 0;

836 for (GlobalTypeMember *G : Globals) {

838 Align Alignment =

839 DL.getValueOrABITypeAlignment(GV->getAlign(), GV->getValueType());

840 MaxAlign = std::max(MaxAlign, Alignment);

841 uint64_t GVOffset = alignTo(CurOffset + DesiredPadding, Alignment);

842 GlobalLayout[G] = GVOffset;

843 if (GVOffset != 0) {

844 uint64_t Padding = GVOffset - CurOffset;

845 GlobalInits.push_back(

847 }

848

849 GlobalInits.push_back(GV->getInitializer());

850 uint64_t InitSize = DL.getTypeAllocSize(GV->getValueType());

851 CurOffset = GVOffset + InitSize;

852

853

854 DesiredPadding = NextPowerOf2(InitSize - 1) - InitSize;

855

856

857

858

859

860 if (DesiredPadding > 32)

861 DesiredPadding = alignTo(InitSize, 32) - InitSize;

862 }

863

865 auto *CombinedGlobal =

866 new GlobalVariable(M, NewInit->getType(), true,

868 CombinedGlobal->setAlignment(MaxAlign);

869

871 lowerTypeTestCalls(TypeIds, CombinedGlobal, GlobalLayout);

872

873

874

875

876 for (unsigned I = 0; I != Globals.size(); ++I) {

878

879

880 Constant *CombinedGlobalIdxs[] = {ConstantInt::get(Int32Ty, 0),

881 ConstantInt::get(Int32Ty, I * 2)};

883 NewInit->getType(), CombinedGlobal, CombinedGlobalIdxs);

885 GlobalAlias *GAlias =

887 "", CombinedGlobalElemPtr, &M);

892 }

893}

894

895bool LowerTypeTestsModule::shouldExportConstantsAsAbsoluteSymbols() {

898}

899

900

901

902

903

904

905

906

907

908uint8_t *LowerTypeTestsModule::exportTypeId(StringRef TypeId,

909 const TypeIdLowering &TIL) {

910 TypeTestResolution &TTRes =

912 TTRes.TheKind = TIL.TheKind;

913

914 auto ExportGlobal = [&](StringRef Name, Constant *C) {

915 GlobalAlias *GA =

917 "__typeid_" + TypeId + "_" + Name, C, &M);

919 };

920

921 auto ExportConstant = [&](StringRef Name, uint64_t &Storage, Constant *C) {

922 if (shouldExportConstantsAsAbsoluteSymbols())

924 else

926 };

927

929 ExportGlobal("global_addr", TIL.OffsetedGlobal);

930

934 ExportConstant("align", TTRes.AlignLog2, TIL.AlignLog2);

935 ExportConstant("size_m1", TTRes.SizeM1, TIL.SizeM1);

936

937 uint64_t BitSize = cast(TIL.SizeM1)->getZExtValue() + 1;

940 else

942 }

943

945 ExportGlobal("byte_array", TIL.TheByteArray);

946 if (shouldExportConstantsAsAbsoluteSymbols())

947 ExportGlobal("bit_mask", TIL.BitMask);

948 else

950 }

951

953 ExportConstant("inline_bits", TTRes.InlineBits, TIL.InlineBits);

954

955 return nullptr;

956}

957

958LowerTypeTestsModule::TypeIdLowering

959LowerTypeTestsModule::importTypeId(StringRef TypeId) {

960 const TypeIdSummary *TidSummary = ImportSummary->getTypeIdSummary(TypeId);

961 if (!TidSummary)

962 return {};

963 const TypeTestResolution &TTRes = TidSummary->TTRes;

964

965 TypeIdLowering TIL;

966 TIL.TheKind = TTRes.TheKind;

967

968 auto ImportGlobal = [&](StringRef Name) {

969

970

971 GlobalVariable *GV = M.getOrInsertGlobal(

972 ("__typeid_" + TypeId + "_" + Name).str(), Int8Arr0Ty);

974 return GV;

975 };

976

977 auto ImportConstant = [&](StringRef Name, uint64_t Const, unsigned AbsWidth,

979 if (!shouldExportConstantsAsAbsoluteSymbols()) {

981 ConstantInt::get(isa(Ty) ? Ty : Int64Ty, Const);

984 return C;

985 }

986

987 Constant *C = ImportGlobal(Name);

991 if (GV->getMetadata(LLVMContext::MD_absolute_symbol))

992 return C;

993

994 auto SetAbsRange = [&](uint64_t Min, uint64_t Max) {

997 GV->setMetadata(LLVMContext::MD_absolute_symbol,

999 };

1000 if (AbsWidth == IntPtrTy->getBitWidth())

1001 SetAbsRange(~0ull, ~0ull);

1002 else

1003 SetAbsRange(0, 1ull << AbsWidth);

1004 return C;

1005 };

1006

1008 auto *GV = ImportGlobal("global_addr");

1009

1010

1011

1012

1013

1014

1015

1016

1017

1018

1019

1021 TIL.OffsetedGlobal = GV;

1022 }

1023

1027 TIL.AlignLog2 = ImportConstant("align", TTRes.AlignLog2, 8, IntPtrTy);

1028 TIL.SizeM1 =

1030 }

1031

1033 TIL.TheByteArray = ImportGlobal("byte_array");

1034 TIL.BitMask = ImportConstant("bit_mask", TTRes.BitMask, 8, PtrTy);

1035 }

1036

1038 TIL.InlineBits = ImportConstant(

1041

1042 return TIL;

1043}

1044

1045void LowerTypeTestsModule::importTypeTest(CallInst *CI) {

1047 if (!TypeIdMDVal)

1048 report_fatal_error("Second argument of llvm.type.test must be metadata");

1049

1051

1052

1053

1054 if (!TypeIdStr)

1055 return;

1056

1057 TypeIdLowering TIL = importTypeId(TypeIdStr->getString());

1058 Value *Lowered = lowerTypeTestCall(TypeIdStr, CI, TIL);

1059 if (Lowered) {

1062 }

1063}

1064

1065void LowerTypeTestsModule::maybeReplaceComdat(Function *F,

1066 StringRef OriginalName) {

1067

1068

1069

1070

1071 if (F->hasComdat() && ObjectFormat == Triple::COFF &&

1072 F->getComdat()->getName() == OriginalName) {

1073 Comdat *OldComdat = F->getComdat();

1074 Comdat *NewComdat = M.getOrInsertComdat(F->getName());

1075 for (GlobalObject &GO : M.global_objects()) {

1076 if (GO.getComdat() == OldComdat)

1078 }

1079 }

1080}

1081

1082

1083

1084void LowerTypeTestsModule::importFunction(Function *F,

1086 assert(F->getType()->getAddressSpace() == 0);

1087

1089 std::string Name = std::string(F->getName());

1090

1092

1093

1094 if (F->isDSOLocal()) {

1097 F->getAddressSpace(),

1098 Name + ".cfi", &M);

1100 replaceDirectCalls(F, RealF);

1101 }

1102 return;

1103 }

1104

1107

1108

1110 F->getAddressSpace(), Name + ".cfi_jt", &M);

1112 } else {

1113 F->setName(Name + ".cfi");

1114 maybeReplaceComdat(F, Name);

1116 F->getAddressSpace(), Name, &M);

1119

1120

1121

1122

1123 for (auto &U : F->uses()) {

1125 std::string AliasName = A->getName().str() + ".cfi";

1128 F->getAddressSpace(), "", &M);

1130 A->replaceAllUsesWith(AliasDecl);

1131 A->setName(AliasName);

1132 }

1133 }

1134 }

1135

1136 if (F->hasExternalWeakLinkage())

1138 else

1140

1141

1142

1143 F->setVisibility(Visibility);

1144}

1145

1146static auto

1150

1151 for (Metadata *TypeId : TypeIds)

1152 OffsetsByTypeID[TypeId];

1153 for (const auto &[Mem, MemOff] : GlobalLayout) {

1155 auto It = OffsetsByTypeID.find(Type->getOperand(1));

1156 if (It == OffsetsByTypeID.end())

1157 continue;

1161 ->getZExtValue();

1162 It->second.push_back(MemOff + Offset);

1163 }

1164 }

1165

1168 for (Metadata *TypeId : TypeIds) {

1172 dbgs() << MDS->getString() << ": ";

1173 else

1174 dbgs() << ": ";

1175 BitSets.back().second.print(dbgs());

1176 });

1177 }

1178

1179 return BitSets;

1180}

1181

1182void LowerTypeTestsModule::lowerTypeTestCalls(

1184 const DenseMap<GlobalTypeMember *, uint64_t> &GlobalLayout) {

1185

1186 for (const auto &[TypeId, BSI] : buildBitSets(TypeIds, GlobalLayout)) {

1187 ByteArrayInfo *BAI = nullptr;

1188 TypeIdLowering TIL;

1189

1190 uint64_t GlobalOffset =

1193 Int8Ty, CombinedGlobalAddr, ConstantInt::get(IntPtrTy, GlobalOffset)),

1194 TIL.AlignLog2 = ConstantInt::get(IntPtrTy, BSI.AlignLog2);

1195 TIL.SizeM1 = ConstantInt::get(IntPtrTy, BSI.BitSize - 1);

1198 : TypeTestResolution::AllOnes;

1201 uint64_t InlineBits = 0;

1202 for (auto Bit : BSI.Bits)

1203 InlineBits |= uint64_t(1) << Bit;

1204 if (InlineBits == 0)

1206 else

1207 TIL.InlineBits = ConstantInt::get(

1208 (BSI.BitSize <= 32) ? Int32Ty : Int64Ty, InlineBits);

1209 } else {

1211 ++NumByteArraysCreated;

1212 BAI = createByteArray(BSI);

1213 TIL.TheByteArray = BAI->ByteArray;

1214 TIL.BitMask = BAI->MaskGlobal;

1215 }

1216

1217 TypeIdUserInfo &TIUI = TypeIdUsers[TypeId];

1218

1219 if (TIUI.IsExported) {

1220 uint8_t *MaskPtr = exportTypeId(cast(TypeId)->getString(), TIL);

1221 if (BAI)

1222 BAI->MaskPtr = MaskPtr;

1223 }

1224

1225

1226 for (CallInst *CI : TIUI.CallSites) {

1227 ++NumTypeTestCallsLowered;

1228 Value *Lowered = lowerTypeTestCall(TypeId, CI, TIL);

1229 if (Lowered) {

1232 }

1233 }

1234 }

1235}

1236

1237void LowerTypeTestsModule::verifyTypeMDNode(GlobalObject *GO, MDNode *Type) {

1238 if (Type->getNumOperands() != 2)

1239 report_fatal_error("All operands of type metadata must have 2 elements");

1240

1245 "A member of a type identifier may not have an explicit section");

1246

1247

1248

1249

1250

1252 if (!OffsetConstMD)

1255 if (!OffsetInt)

1257}

1258

1266

1267bool LowerTypeTestsModule::hasBranchTargetEnforcement() {

1268 if (HasBranchTargetEnforcement == -1) {

1269

1270

1272 M.getModuleFlag("branch-target-enforcement")))

1273 HasBranchTargetEnforcement = !BTE->isZero();

1274 else

1275 HasBranchTargetEnforcement = 0;

1276 }

1277 return HasBranchTargetEnforcement;

1278}

1279

1280unsigned

1281LowerTypeTestsModule::getJumpTableEntrySize(Triple::ArchType JumpTableArch) {

1282 switch (JumpTableArch) {

1286 M.getModuleFlag("cf-protection-branch")))

1287 if (MD->getZExtValue())

1293 if (CanUseThumbBWJumpTable) {

1294 if (hasBranchTargetEnforcement())

1297 } else {

1299 }

1301 if (hasBranchTargetEnforcement())

1309 default:

1311 }

1312}

1313

1314

1315

1316

1317InlineAsm *

1318LowerTypeTestsModule::createJumpTableEntryAsm(Triple::ArchType JumpTableArch) {

1319 std::string Asm;

1320 raw_string_ostream AsmOS(Asm);

1321

1323 bool Endbr = false;

1325 M.getModuleFlag("cf-protection-branch")))

1326 Endbr = !MD->isZero();

1327 if (Endbr)

1328 AsmOS << (JumpTableArch == Triple::x86 ? "endbr32\n" : "endbr64\n");

1329 AsmOS << "jmp ${0:c}@plt\n";

1330 if (Endbr)

1331 AsmOS << ".balign 16, 0xcc\n";

1332 else

1333 AsmOS << "int3\nint3\nint3\n";

1334 } else if (JumpTableArch == Triple::arm) {

1335 AsmOS << "b $0\n";

1337 if (hasBranchTargetEnforcement())

1338 AsmOS << "bti c\n";

1339 AsmOS << "b $0\n";

1341 if (!CanUseThumbBWJumpTable) {

1342

1343

1344

1345

1346

1347

1348

1349

1350

1351

1352

1353

1354

1355

1356

1357 AsmOS << "push {r0,r1}\n"

1358 << "ldr r0, 1f\n"

1359 << "0: add r0, r0, pc\n"

1360 << "str r0, [sp, #4]\n"

1361 << "pop {r0,pc}\n"

1362 << ".balign 4\n"

1363 << "1: .word $0 - (0b + 4)\n";

1364 } else {

1365 if (hasBranchTargetEnforcement())

1366 AsmOS << "bti\n";

1367 AsmOS << "b.w $0\n";

1368 }

1371 AsmOS << "tail $0@plt\n";

1373 AsmOS << "pcalau12i t0, %pc_hi20($0)\n"

1374 << "jirl r0, t0, %pc_lo12($0)\n";

1375 } else {

1377 }

1378

1381 AsmOS.str(), "s",

1382 true);

1383}

1384

1385

1386

1387void LowerTypeTestsModule::buildBitSetsFromFunctions(

1393 buildBitSetsFromFunctionsNative(TypeIds, Functions);

1395 buildBitSetsFromFunctionsWASM(TypeIds, Functions);

1396 else

1398}

1399

1400void LowerTypeTestsModule::moveInitializerToModuleConstructor(

1401 GlobalVariable *GV) {

1402 if (WeakInitializerFn == nullptr) {

1405 false),

1407 M.getDataLayout().getProgramAddressSpace(),

1408 "__cfi_global_var_init", &M);

1412 WeakInitializerFn->setSection(

1414 ? "__TEXT,__StaticInit,regular,pure_instructions"

1415 : ".text.startup");

1416

1417

1419 }

1420

1421 IRBuilder<> IRB(WeakInitializerFn->getEntryBlock().getTerminator());

1425}

1426

1427void LowerTypeTestsModule::findGlobalVariableUsersOf(

1428 Constant *C, SmallSetVector<GlobalVariable *, 8> &Out) {

1429 for (auto *U : C->users()){

1433 findGlobalVariableUsersOf(C2, Out);

1434 }

1435}

1436

1437

1438void LowerTypeTestsModule::replaceWeakDeclarationWithJumpTablePtr(

1439 Function *F, Constant *JT, bool IsJumpTableCanonical) {

1440

1441

1442 SmallSetVector<GlobalVariable *, 8> GlobalVarUsers;

1443 findGlobalVariableUsersOf(F, GlobalVarUsers);

1444 for (auto *GV : GlobalVarUsers) {

1445 if (GV == GlobalAnnotation)

1446 continue;

1447 moveInitializerToModuleConstructor(GV);

1448 }

1449

1450

1451

1455 F->getAddressSpace(), "", &M);

1456 replaceCfiUses(F, PlaceholderFn, IsJumpTableCanonical);

1457

1459

1460 while (!PlaceholderFn->use_empty()) {

1463 assert(InsertPt && "Non-instruction users should have been eliminated");

1465 if (PN)

1466 InsertPt = PN->getIncomingBlock(U)->getTerminator();

1470 Value *Select = Builder.CreateSelect(ICmp, JT,

1472

1475

1476

1477 if (PN)

1478 PN->setIncomingValueForBlock(InsertPt->getParent(), Select);

1479 else

1481 }

1483}

1484

1486 Attribute TFAttr = F->getFnAttribute("target-features");

1490 for (StringRef Feature : Features) {

1491 if (Feature == "-thumb-mode")

1492 return false;

1493 else if (Feature == "+thumb-mode")

1494 return true;

1495 }

1496 }

1497

1499}

1500

1501

1502

1503

1504Triple::ArchType LowerTypeTestsModule::selectJumpTableArmEncoding(

1507 return Arch;

1508

1509 if (!CanUseThumbBWJumpTable && CanUseArmJumpTable) {

1510

1511

1512

1514 }

1515

1516

1517 unsigned ArmCount = 0, ThumbCount = 0;

1518 for (const auto GTM : Functions) {

1519 if (!GTM->isJumpTableCanonical()) {

1520

1521

1522 ++ArmCount;

1523 continue;

1524 }

1525

1528 }

1529

1531}

1532

1533void LowerTypeTestsModule::createJumpTable(

1538

1539 InlineAsm *JumpTableAsm = createJumpTableEntryAsm(JumpTableArch);

1540

1541

1542

1543

1544

1545 bool areAllEntriesNounwind = true;

1546 for (GlobalTypeMember *GTM : Functions) {

1548 ->hasFnAttribute(llvm::Attribute::NoUnwind)) {

1549 areAllEntriesNounwind = false;

1550 }

1551 IRB.CreateCall(JumpTableAsm, GTM->getGlobal());

1552 }

1553 IRB.CreateUnreachable();

1554

1555

1556 F->setAlignment(Align(getJumpTableEntrySize(JumpTableArch)));

1557 F->addFnAttr(Attribute::Naked);

1559 F->addFnAttr("target-features", "-thumb-mode");

1561 if (hasBranchTargetEnforcement()) {

1562

1563

1564 F->addFnAttr("target-features", "+thumb-mode,+pacbti");

1565 } else {

1566 F->addFnAttr("target-features", "+thumb-mode");

1567 if (CanUseThumbBWJumpTable) {

1568

1569

1570 F->addFnAttr("target-cpu", "cortex-a8");

1571 }

1572 }

1573 }

1574

1575

1576

1578 if (F->hasFnAttribute("branch-target-enforcement"))

1579 F->removeFnAttr("branch-target-enforcement");

1580 if (F->hasFnAttribute("sign-return-address"))

1581 F->removeFnAttr("sign-return-address");

1582 }

1584

1585

1586 F->addFnAttr("target-features", "-c,-relax");

1587 }

1588

1589

1590

1592 F->addFnAttr(Attribute::NoCfCheck);

1593

1594

1595 if (areAllEntriesNounwind)

1596 F->addFnAttr(Attribute::NoUnwind);

1597

1598

1599 F->addFnAttr(Attribute::NoInline);

1600}

1601

1602

1603

1604void LowerTypeTestsModule::buildBitSetsFromFunctionsNative(

1606

1607

1608

1609

1610

1611

1612

1613

1614

1615

1616

1617

1618

1619

1620

1621

1622

1623

1624

1625

1626

1627

1628

1629

1630

1631

1632

1633

1634

1635

1636

1637

1638

1639

1640

1641

1642

1643

1644

1645

1646

1647

1648

1649

1650

1651

1652

1653

1654

1655

1656

1657

1658

1659

1660

1661

1662

1663

1664

1665

1666

1667

1668

1669

1670

1671

1672

1673

1674

1675

1676

1677

1678

1679

1680

1681

1683

1684

1685

1686 Triple::ArchType JumpTableArch = selectJumpTableArmEncoding(Functions);

1687

1688

1689 DenseMap<GlobalTypeMember *, uint64_t> GlobalLayout;

1690 unsigned EntrySize = getJumpTableEntrySize(JumpTableArch);

1691 for (unsigned I = 0; I != Functions.size(); ++I)

1692 GlobalLayout[Functions[I]] = I * EntrySize;

1693

1696 false),

1698 M.getDataLayout().getProgramAddressSpace(),

1699 ".cfi.jumptable", &M);

1705

1706 lowerTypeTestCalls(TypeIds, JumpTable, GlobalLayout);

1707

1708

1709

1710 for (unsigned I = 0; I != Functions.size(); ++I) {

1712 bool IsJumpTableCanonical = Functions[I]->isJumpTableCanonical();

1713

1715 JumpTableType, JumpTable,

1717 ConstantInt::get(IntPtrTy, I)});

1718

1719 const bool IsExported = Functions[I]->isExported();

1720 if (!IsJumpTableCanonical) {

1724 F->getName() + ".cfi_jt",

1725 CombinedGlobalElemPtr, &M);

1726 if (IsExported)

1728 else

1730 }

1731

1732 if (IsExported) {

1733 if (IsJumpTableCanonical)

1735 else

1737 }

1738

1739 if (!IsJumpTableCanonical) {

1740 if (F->hasExternalWeakLinkage())

1741 replaceWeakDeclarationWithJumpTablePtr(F, CombinedGlobalElemPtr,

1742 IsJumpTableCanonical);

1743 else

1744 replaceCfiUses(F, CombinedGlobalElemPtr, IsJumpTableCanonical);

1745 } else {

1746 assert(F->getType()->getAddressSpace() == 0);

1747

1748 GlobalAlias *FAlias =

1750 CombinedGlobalElemPtr, &M);

1753 if (FAlias->hasName()) {

1754 F->setName(FAlias->getName() + ".cfi");

1755 maybeReplaceComdat(F, FAlias->getName());

1756 }

1757 replaceCfiUses(F, FAlias, IsJumpTableCanonical);

1758 if (->hasLocalLinkage())

1760 }

1761 }

1762

1763 createJumpTable(JumpTableFn, Functions, JumpTableArch);

1764}

1765

1766

1767

1768

1769

1770

1771

1772void LowerTypeTestsModule::buildBitSetsFromFunctionsWASM(

1775

1776

1777 DenseMap<GlobalTypeMember *, uint64_t> GlobalLayout;

1778

1779 for (GlobalTypeMember *GTM : Functions) {

1781

1782

1783 if (->hasAddressTaken())

1784 continue;

1785

1786

1789 ConstantInt::get(Int64Ty, IndirectIndex))));

1790 F->setMetadata("wasm.index", MD);

1791

1792

1793 GlobalLayout[GTM] = IndirectIndex++;

1794 }

1795

1796

1797

1799 GlobalLayout);

1800}

1801

1802void LowerTypeTestsModule::buildBitSetsFromDisjointSet(

1805 DenseMap<Metadata *, uint64_t> TypeIdIndices;

1806 for (unsigned I = 0; I != TypeIds.size(); ++I)

1807 TypeIdIndices[TypeIds[I]] = I;

1808

1809

1810

1811 std::vector<std::set<uint64_t>> TypeMembers(TypeIds.size());

1812 unsigned GlobalIndex = 0;

1813 DenseMap<GlobalTypeMember *, uint64_t> GlobalIndices;

1814 for (GlobalTypeMember *GTM : Globals) {

1815 for (MDNode *Type : GTM->types()) {

1816

1817 auto I = TypeIdIndices.find(Type->getOperand(1));

1818 if (I != TypeIdIndices.end())

1819 TypeMembers[I->second].insert(GlobalIndex);

1820 }

1821 GlobalIndices[GTM] = GlobalIndex;

1822 GlobalIndex++;

1823 }

1824

1825 for (ICallBranchFunnel *JT : ICallBranchFunnels) {

1826 TypeMembers.emplace_back();

1827 std::set<uint64_t> &TMSet = TypeMembers.back();

1828 for (GlobalTypeMember *T : JT->targets())

1829 TMSet.insert(GlobalIndices[T]);

1830 }

1831

1832

1833

1835 const std::set<uint64_t> &O2) {

1836 return O1.size() < O2.size();

1837 });

1838

1839

1840

1841

1843 for (auto &&MemSet : TypeMembers)

1844 GLB.addFragment(MemSet);

1845

1846

1847 bool IsGlobalSet =

1849 std::vector<GlobalTypeMember *> OrderedGTMs(Globals.size());

1850 auto OGTMI = OrderedGTMs.begin();

1851 for (auto &&F : GLB.Fragments) {

1852 for (auto &&Offset : F) {

1855 "variables and functions");

1856 *OGTMI++ = Globals[Offset];

1857 }

1858 }

1859

1860

1861 if (IsGlobalSet)

1862 buildBitSetsFromGlobalVariables(TypeIds, OrderedGTMs);

1863 else

1864 buildBitSetsFromFunctions(TypeIds, OrderedGTMs);

1865}

1866

1867

1868LowerTypeTestsModule::LowerTypeTestsModule(

1870 const ModuleSummaryIndex *ImportSummary, DropTestKind DropTypeTests)

1871 : M(M), ExportSummary(ExportSummary), ImportSummary(ImportSummary),

1873 : DropTypeTests) {

1874 assert(!(ExportSummary && ImportSummary));

1875 Triple TargetTriple(M.getTargetTriple());

1876 Arch = TargetTriple.getArch();

1878 CanUseArmJumpTable = true;

1880 auto &FAM =

1883

1884 if (F.isDeclaration())

1885 continue;

1887 if (TTI.hasArmWideBranch(false))

1888 CanUseArmJumpTable = true;

1889 if (TTI.hasArmWideBranch(true))

1890 CanUseThumbBWJumpTable = true;

1891 }

1892 }

1893 OS = TargetTriple.getOS();

1894 ObjectFormat = TargetTriple.getObjectFormat();

1895

1896

1897

1898 GlobalAnnotation = M.getGlobalVariable("llvm.global.annotations");

1899 if (GlobalAnnotation && GlobalAnnotation->hasInitializer()) {

1902 FunctionAnnotations.insert_range(CA->operands());

1903 }

1904}

1905

1907 ModuleSummaryIndex Summary(false);

1908

1909

1910

1912 ExitOnError ExitOnErr("-lowertypetests-read-summary: " + ClReadSummary +

1913 ": ");

1916

1917 yaml::Input In(ReadSummaryFile->getBuffer());

1920 }

1921

1923 LowerTypeTestsModule(

1924 M, AM,

1925 ClSummaryAction == PassSummaryAction::Export ? &Summary : nullptr,

1926 ClSummaryAction == PassSummaryAction::Import ? &Summary : nullptr,

1927 DropTestKind::None)

1928 .lower();

1929

1931 ExitOnError ExitOnErr("-lowertypetests-write-summary: " + ClWriteSummary +

1932 ": ");

1933 std::error_code EC;

1936

1937 yaml::Output Out(OS);

1939 }

1940

1942}

1943

1946 if (Usr) {

1948 if (CB && CB->isCallee(&U))

1949 return true;

1950 }

1951 return false;

1952}

1953

1954void LowerTypeTestsModule::replaceCfiUses(Function *Old, Value *New,

1955 bool IsJumpTableCanonical) {

1956 SmallSetVector<Constant *, 4> Constants;

1958

1959

1961 continue;

1962

1963

1965 continue;

1966

1967

1968 if (isFunctionAnnotation(U.getUser()))

1969 continue;

1970

1971

1972

1975

1976

1978 continue;

1979 }

1980 }

1981

1982 U.set(New);

1983 }

1984

1985

1986 for (auto *C : Constants)

1987 C->handleOperandChange(Old, New);

1988}

1989

1990void LowerTypeTestsModule::replaceDirectCalls(Value *Old, Value *New) {

1992}

1993

1995 bool ShouldDropAll) {

1998

2001 Assume->eraseFromParent();

2002

2003

2004

2005

2006

2007

2010 return isa(U);

2011 }));

2013 }

2015 }

2016}

2017

2018bool LowerTypeTestsModule::lower() {

2021

2022 if (DropTypeTests != DropTestKind::None) {

2023 bool ShouldDropAll = DropTypeTests == DropTestKind::All;

2024 if (TypeTestFunc)

2026

2027

2028

2029 Function *PublicTypeTestFunc =

2031 if (PublicTypeTestFunc)

2032 dropTypeTests(M, *PublicTypeTestFunc, ShouldDropAll);

2033 if (TypeTestFunc || PublicTypeTestFunc) {

2034

2035

2036

2037 for (GlobalVariable &GV : M.globals())

2038 GV.eraseMetadata(LLVMContext::MD_vcall_visibility);

2039 return true;

2040 }

2041 return false;

2042 }

2043

2044

2045

2046

2047

2050 return false;

2051

2052 Function *ICallBranchFunnelFunc =

2054 if ((!TypeTestFunc || TypeTestFunc->use_empty()) &&

2055 (!ICallBranchFunnelFunc || ICallBranchFunnelFunc->use_empty()) &&

2056 !ExportSummary && !ImportSummary)

2057 return false;

2058

2059 if (ImportSummary) {

2060 if (TypeTestFunc)

2063

2064 if (ICallBranchFunnelFunc && !ICallBranchFunnelFunc->use_empty())

2066 "unexpected call to llvm.icall.branch.funnel during import phase");

2067

2070 for (auto &F : M) {

2071

2072

2073 if (F.hasLocalLinkage())

2074 continue;

2079 }

2080

2081 {

2082 ScopedSaveAliaseesAndUsed S(M);

2083 for (auto *F : Defs)

2084 importFunction(F, true);

2085 for (auto *F : Decls)

2086 importFunction(F, false);

2087 }

2088

2089 return true;

2090 }

2091

2092

2093

2094

2095 using GlobalClassesTy = EquivalenceClasses<

2096 PointerUnion<GlobalTypeMember *, Metadata *, ICallBranchFunnel *>>;

2097 GlobalClassesTy GlobalClasses;

2098

2099

2100

2101

2102

2103

2104

2105

2107 struct TIInfo {

2108 unsigned UniqueId;

2109 std::vector<GlobalTypeMember *> RefGlobals;

2110 };

2111 DenseMap<Metadata *, TIInfo> TypeIdInfo;

2112 unsigned CurUniqueId = 0;

2114

2115

2116

2117 const bool CrossDsoCfi = M.getModuleFlag("Cross-DSO CFI") != nullptr;

2118

2119 struct ExportedFunctionInfo {

2121 MDNode *FuncMD;

2122 };

2123 MapVector<StringRef, ExportedFunctionInfo> ExportedFunctions;

2124 if (ExportSummary) {

2125 NamedMDNode *CfiFunctionsMD = M.getNamedMetadata("cfi.functions");

2126 if (CfiFunctionsMD) {

2127

2128 DenseSetGlobalValue::GUID AddressTaken;

2129 for (auto &I : *ExportSummary)

2130 for (auto &GVS : I.second.getSummaryList())

2131 if (GVS->isLive())

2132 for (const auto &Ref : GVS->refs()) {

2133 AddressTaken.insert(Ref.getGUID());

2134 for (auto &RefGVS : Ref.getSummaryList())

2136 AddressTaken.insert(Alias->getAliaseeGUID());

2137 }

2139 if (AddressTaken.count(GUID))

2140 return true;

2141 auto VI = ExportSummary->getValueInfo(GUID);

2142 if (!VI)

2143 return false;

2144 for (auto &I : VI.getSummaryList())

2146 if (AddressTaken.count(Alias->getAliaseeGUID()))

2147 return true;

2148 return false;

2149 };

2150 for (auto *FuncMD : CfiFunctionsMD->operands()) {

2151 assert(FuncMD->getNumOperands() >= 2);

2152 StringRef FunctionName =

2156 ->getValue()

2157 ->getUniqueInteger()

2158 .getZExtValue());

2162

2163

2164 if (!ExportSummary->isGUIDLive(GUID))

2165 continue;

2168 continue;

2169

2171 if (auto VI = ExportSummary->getValueInfo(GUID))

2172 for (const auto &GVS : VI.getSummaryList())

2175

2176 if (!Exported)

2177 continue;

2178 }

2179 auto P = ExportedFunctions.insert({FunctionName, {Linkage, FuncMD}});

2181 P.first->second = {Linkage, FuncMD};

2182 }

2183

2184 for (const auto &P : ExportedFunctions) {

2185 StringRef FunctionName = P.first;

2187 MDNode *FuncMD = P.second.FuncMD;

2188 Function *F = M.getFunction(FunctionName);

2189 if (F && F->hasLocalLinkage()) {

2190

2191

2192

2193

2194

2195

2196 F->setName(F->getName() + ".1");

2197 F = nullptr;

2198 }

2199

2200 if ()

2202 FunctionType::get(Type::getVoidTy(M.getContext()), false),

2203 GlobalVariable::ExternalLinkage,

2204 M.getDataLayout().getProgramAddressSpace(), FunctionName, &M);

2205

2206

2207

2208

2209

2210

2211 if (F->hasAvailableExternallyLinkage()) {

2213 F->deleteBody();

2214 F->setComdat(nullptr);

2215 F->clearMetadata();

2216 }

2217

2218

2219

2222

2223

2224

2225

2226

2227 if (F->isDeclaration()) {

2230

2231 F->eraseMetadata(LLVMContext::MD_type);

2233 F->addMetadata(LLVMContext::MD_type,

2235 }

2236 }

2237 }

2238 }

2239

2240 struct AliasToCreate {

2242 std::string TargetName;

2243 };

2244 std::vector AliasesToCreate;

2245

2246

2247

2248 if (ExportSummary) {

2249 if (NamedMDNode *AliasesMD = M.getNamedMetadata("aliases")) {

2250 for (auto *AliasMD : AliasesMD->operands()) {

2252 for (Metadata *MD : AliasMD->operands()) {

2254 if (!MDS)

2255 continue;

2256 StringRef AliasName = MDS->getString();

2257 if (!ExportedFunctions.count(AliasName))

2258 continue;

2259 auto *AliasF = M.getFunction(AliasName);

2260 if (AliasF)

2262 }

2263

2264 if (Aliases.empty())

2265 continue;

2266

2267 for (unsigned I = 1; I != Aliases.size(); ++I) {

2268 auto *AliasF = Aliases[I];

2269 ExportedFunctions.erase(AliasF->getName());

2270 AliasesToCreate.push_back(

2271 {AliasF, std::string(Aliases[0]->getName())});

2272 }

2273 }

2274 }

2275 }

2276

2277 DenseMap<GlobalObject *, GlobalTypeMember *> GlobalTypeMembers;

2278 for (GlobalObject &GO : M.global_objects()) {

2280 continue;

2281

2283 GO.getMetadata(LLVMContext::MD_type, Types);

2284

2285 bool IsJumpTableCanonical = false;

2286 bool IsExported = false;

2289 if (auto It = ExportedFunctions.find(F->getName());

2290 It != ExportedFunctions.end()) {

2291 IsJumpTableCanonical |= It->second.Linkage == CFL_Definition;

2292 IsExported = true;

2293

2294

2295

2296

2297 } else if (->hasAddressTaken()) {

2298 if (!CrossDsoCfi || !IsJumpTableCanonical || F->hasLocalLinkage())

2299 continue;

2300 }

2301 }

2302

2303 auto *GTM = GlobalTypeMember::create(Alloc, &GO, IsJumpTableCanonical,

2304 IsExported, Types);

2305 GlobalTypeMembers[&GO] = GTM;

2306 for (MDNode *Type : Types) {

2307 verifyTypeMDNode(&GO, Type);

2308 auto &Info = TypeIdInfo[Type->getOperand(1)];

2309 Info.UniqueId = ++CurUniqueId;

2310 Info.RefGlobals.push_back(GTM);

2311 }

2312 }

2313

2314 auto AddTypeIdUse = [&](Metadata *TypeId) -> TypeIdUserInfo & {

2315

2316

2317

2318

2319 auto Ins = TypeIdUsers.insert({TypeId, {}});

2320 if (Ins.second) {

2321

2322 auto &GCI = GlobalClasses.insert(TypeId);

2323 GlobalClassesTy::member_iterator CurSet = GlobalClasses.findLeader(GCI);

2324

2325

2326 for (GlobalTypeMember *GTM : TypeIdInfo[TypeId].RefGlobals)

2327 CurSet = GlobalClasses.unionSets(

2328 CurSet, GlobalClasses.findLeader(GlobalClasses.insert(GTM)));

2329 }

2330

2331 return Ins.first->second;

2332 };

2333

2334 if (TypeTestFunc) {

2335 for (const Use &U : TypeTestFunc->uses()) {

2337

2338

2339

2340

2341

2342

2343 bool OnlyAssumeUses = !CI->use_empty();

2344 for (const Use &CIU : CI->uses()) {

2346 continue;

2347 OnlyAssumeUses = false;

2348 break;

2349 }

2350 if (OnlyAssumeUses)

2351 continue;

2352

2354 if (!TypeIdMDVal)

2355 report_fatal_error("Second argument of llvm.type.test must be metadata");

2356 auto TypeId = TypeIdMDVal->getMetadata();

2357 AddTypeIdUse(TypeId).CallSites.push_back(CI);

2358 }

2359 }

2360

2361 if (ICallBranchFunnelFunc) {

2362 for (const Use &U : ICallBranchFunnelFunc->uses()) {

2365 "llvm.icall.branch.funnel not supported on this target");

2366

2368

2369 std::vector<GlobalTypeMember *> Targets;

2370 if (CI->arg_size() % 2 != 1)

2372

2373 GlobalClassesTy::member_iterator CurSet;

2374 for (unsigned I = 1; I != CI->arg_size(); I += 2) {

2380 "Expected branch funnel operand to be global value");

2381

2382 GlobalTypeMember *GTM = GlobalTypeMembers[Base];

2383 Targets.push_back(GTM);

2384 GlobalClassesTy::member_iterator NewSet =

2385 GlobalClasses.findLeader(GlobalClasses.insert(GTM));

2386 if (I == 1)

2387 CurSet = NewSet;

2388 else

2389 CurSet = GlobalClasses.unionSets(CurSet, NewSet);

2390 }

2391

2392 GlobalClasses.unionSets(

2393 CurSet, GlobalClasses.findLeader(

2394 GlobalClasses.insert(ICallBranchFunnel::create(

2395 Alloc, CI, Targets, ++CurUniqueId))));

2396 }

2397 }

2398

2399 if (ExportSummary) {

2400 DenseMap<GlobalValue::GUID, TinyPtrVector<Metadata *>> MetadataByGUID;

2401 for (auto &P : TypeIdInfo) {

2404 TypeId->getString())]

2405 .push_back(TypeId);

2406 }

2407

2408 for (auto &P : *ExportSummary) {

2409 for (auto &S : P.second.getSummaryList()) {

2410 if (!ExportSummary->isGlobalValueLive(S.get()))

2411 continue;

2414 for (Metadata *MD : MetadataByGUID[G])

2415 AddTypeIdUse(MD).IsExported = true;

2416 }

2417 }

2418 }

2419

2420 if (GlobalClasses.empty())

2421 return false;

2422

2423 {

2424 ScopedSaveAliaseesAndUsed S(M);

2425

2426 for (const auto &C : GlobalClasses) {

2427 if (->isLeader())

2428 continue;

2429

2430 ++NumTypeIdDisjointSets;

2431

2432 std::vector<Metadata *> TypeIds;

2433 std::vector<GlobalTypeMember *> Globals;

2434 std::vector<ICallBranchFunnel *> ICallBranchFunnels;

2435 for (auto M : GlobalClasses.members(*C)) {

2440 else

2442 }

2443

2444

2445

2446

2448 return TypeIdInfo[M1].UniqueId < TypeIdInfo[M2].UniqueId;

2449 });

2450

2451

2453 [&](ICallBranchFunnel *F1, ICallBranchFunnel *F2) {

2454 return F1->UniqueId < F2->UniqueId;

2455 });

2456

2457

2458 buildBitSetsFromDisjointSet(TypeIds, Globals, ICallBranchFunnels);

2459 }

2460 }

2461

2462 allocateByteArrays();

2463

2464 for (auto A : AliasesToCreate) {

2465 auto *Target = M.getNamedValue(A.TargetName);

2467 continue;

2469 AliasGA->setVisibility(A.Alias->getVisibility());

2470 AliasGA->setLinkage(A.Alias->getLinkage());

2471 AliasGA->takeName(A.Alias);

2472 A.Alias->replaceAllUsesWith(AliasGA);

2473 A.Alias->eraseFromParent();

2474 }

2475

2476

2477 if (ExportSummary) {

2478 if (NamedMDNode *SymversMD = M.getNamedMetadata("symvers")) {

2479 for (auto *Symver : SymversMD->operands()) {

2480 assert(Symver->getNumOperands() >= 2);

2483 StringRef Alias = cast(Symver->getOperand(1))->getString();

2484

2485 if (!ExportedFunctions.count(SymbolName))

2486 continue;

2487

2488 M.appendModuleInlineAsm(

2489 (llvm::Twine(".symver ") + SymbolName + ", " + Alias).str());

2490 }

2491 }

2492 }

2493

2494 return true;

2495}

2496

2500 if (UseCommandLine)

2501 Changed = LowerTypeTestsModule::runForTesting(M, AM);

2502 else

2504 LowerTypeTestsModule(M, AM, ExportSummary, ImportSummary, DropTypeTests)

2505 .lower();

2509}

2510

2514

2515

2516

2517

2518

2519

2520

2521

2522

2523

2524

2525

2526

2527

2528

2529

2530

2531

2532 for (auto &GV : M.globals()) {

2535 continue;

2536

2539 auto MaySimplifyPtr = [&](Value *Ptr) {

2541 if (auto *CFIGV = M.getNamedValue((GV->getName() + ".cfi").str()))

2542 Ptr = CFIGV;

2544 };

2545 auto MaySimplifyInt = [&](Value *Op) {

2547 if (!PtrAsInt || PtrAsInt->getOpcode() != Instruction::PtrToInt)

2548 return false;

2549 return MaySimplifyPtr(PtrAsInt->getOperand(0));

2550 };

2554 MaySimplifyPtr(CI->getOperand(0))) {

2555

2556

2557

2561 continue;

2562 }

2563 }

2565 if (!CE || CE->getOpcode() != Instruction::PtrToInt)

2566 continue;

2569 if (U.getOperandNo() == 0 && CE &&

2570 CE->getOpcode() == Instruction::Sub &&

2571 MaySimplifyInt(CE->getOperand(1))) {

2572

2573

2574

2575

2576

2577 CE->replaceAllUsesWith(ConstantInt::get(CE->getType(), 0));

2579 }

2581 if (U.getOperandNo() == 1 && CI &&

2583 MaySimplifyInt(CI->getOperand(0))) {

2584

2585

2589 }

2590 }

2591 }

2592 }

2593

2600 return PA;

2601}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

AMDGPU Prepare AGPR Alloc

AMDGPU Register Bank Select

This file implements a class to represent arbitrary precision integral constant values and operations...

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

This file defines the BumpPtrAllocator interface.

This file contains the simple types necessary to represent the attributes associated with functions a...

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

Analysis containing CSE Info

#define clEnumValN(ENUMVAL, FLAGNAME, DESC)

This file contains the declarations for the subclasses of Constant, which represent the different fla...

This file defines the DenseMap class.

Generic implementation of equivalence classes through the use Tarjan's efficient union-find algorithm...

Module.h This file contains the declarations for the Module class.

This header defines various interfaces for pass management in LLVM.

This defines the Use class.

static const unsigned kARMJumpTableEntrySize

Definition LowerTypeTests.cpp:1261

static const unsigned kLOONGARCH64JumpTableEntrySize

Definition LowerTypeTests.cpp:1265

static bool isKnownTypeIdMember(Metadata *TypeId, const DataLayout &DL, Value *V, uint64_t COffset)

Definition LowerTypeTests.cpp:695

static const unsigned kX86IBTJumpTableEntrySize

Definition LowerTypeTests.cpp:1260

static cl::opt< std::string > ClReadSummary("lowertypetests-read-summary", cl::desc("Read summary from given YAML file before running pass"), cl::Hidden)

static const unsigned kRISCVJumpTableEntrySize

Definition LowerTypeTests.cpp:1264

static auto buildBitSets(ArrayRef< Metadata * > TypeIds, const DenseMap< GlobalTypeMember *, uint64_t > &GlobalLayout)

Definition LowerTypeTests.cpp:1147

static void dropTypeTests(Module &M, Function &TypeTestFunc, bool ShouldDropAll)

Definition LowerTypeTests.cpp:1994

static Value * createMaskedBitTest(IRBuilder<> &B, Value *Bits, Value *BitOffset)

Build a test that bit BitOffset mod sizeof(Bits)*8 is set in Bits.

Definition LowerTypeTests.cpp:584

static bool isThumbFunction(Function *F, Triple::ArchType ModuleArch)

Definition LowerTypeTests.cpp:1485

static const unsigned kX86JumpTableEntrySize

Definition LowerTypeTests.cpp:1259

static cl::opt< bool > AvoidReuse("lowertypetests-avoid-reuse", cl::desc("Try to avoid reuse of byte array addresses using aliases"), cl::Hidden, cl::init(true))

static cl::opt< PassSummaryAction > ClSummaryAction("lowertypetests-summary-action", cl::desc("What to do with the summary when running this pass"), cl::values(clEnumValN(PassSummaryAction::None, "none", "Do nothing"), clEnumValN(PassSummaryAction::Import, "import", "Import typeid resolutions from summary and globals"), clEnumValN(PassSummaryAction::Export, "export", "Export typeid resolutions to summary and globals")), cl::Hidden)

static const unsigned kARMBTIJumpTableEntrySize

Definition LowerTypeTests.cpp:1262

static cl::opt< std::string > ClWriteSummary("lowertypetests-write-summary", cl::desc("Write summary to given YAML file after running pass"), cl::Hidden)

static BitSetInfo buildBitSet(ArrayRef< uint64_t > Offsets)

Build a bit set for list of offsets.

Definition LowerTypeTests.cpp:576

static bool isDirectCall(Use &U)

Definition LowerTypeTests.cpp:1944

static const unsigned kARMv6MJumpTableEntrySize

Definition LowerTypeTests.cpp:1263

static cl::opt< DropTestKind > ClDropTypeTests("lowertypetests-drop-type-tests", cl::desc("Simply drop type test sequences"), cl::values(clEnumValN(DropTestKind::None, "none", "Do not drop any type tests"), clEnumValN(DropTestKind::Assume, "assume", "Drop type test assume sequences"), clEnumValN(DropTestKind::All, "all", "Drop all type test sequences")), cl::Hidden, cl::init(DropTestKind::None))

Machine Check Debug Module

ModuleSummaryIndex.h This file contains the declarations the classes that hold the module index and s...

FunctionAnalysisManager FAM

This file defines the PointerUnion class, which is a discriminated union of pointer types.

static StringRef getName(Value *V)

This file implements a set that has insertion order iteration characteristics.

This file defines the SmallVector class.

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

This pass exposes codegen information to IR-level passes.

This header defines support for implementing classes that have some trailing object (or arrays of obj...

Class for arbitrary precision integers.

uint64_t getZExtValue() const

Get zero extended value.

PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)

Get the result of an analysis pass for a given IR unit.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

bool empty() const

empty - Check if the array is empty.

static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)

This static method is the primary way to construct an ArrayType.

Functions, function parameters, and return types can have attributes to indicate how they should be t...

LLVM_ABI StringRef getValueAsString() const

Return the attribute's value as a string.

bool isValid() const

Return true if the attribute is any kind of attribute.

static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)

Creates a new BasicBlock.

LLVM_ABI BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)

Split the basic block into two basic blocks at the specified instruction.

const Instruction * getTerminator() const LLVM_READONLY

Returns the terminator instruction if the block is well formed or null if the block is not well forme...

static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)

Value * getArgOperand(unsigned i) const

unsigned arg_size() const

void emplace(Args &&...A)

size_t count(StringRef S) const

static LLVM_ABI ConstantAggregateZero * get(Type *Ty)

ConstantArray - Constant Array Declarations.

static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)

get() constructor - Return a constant with array type with an element count and element type matching...

static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)

static Constant * getInBoundsGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList)

Create an "inbounds" getelementptr.

static LLVM_ABI Constant * getPointerCast(Constant *C, Type *Ty)

Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.

static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)

static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, GEPNoWrapFlags NW=GEPNoWrapFlags::none(), std::optional< ConstantRange > InRange=std::nullopt, Type *OnlyIfReducedTy=nullptr)

Getelementptr form.

static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)

static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)

static LLVM_ABI ConstantPointerNull * get(PointerType *T)

Static factory methods - Return objects of the specified value.

static Constant * getAnon(ArrayRef< Constant * > V, bool Packed=false)

Return an anonymous struct that has the specified elements.

static LLVM_ABI Constant * getNullValue(Type *Ty)

Constructor to create a '0' constant of arbitrary type.

A parsed version of the target data layout string in and methods for querying it.

iterator find(const_arg_type_t< KeyT > Val)

Analysis pass which computes a DominatorTree.

static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)

This static method is the primary way of constructing a FunctionType.

static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)

void eraseFromParent()

eraseFromParent - This method unlinks 'this' from the containing module and deletes it.

static LLVM_ABI GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)

If a parent module is specified, the alias is automatically inserted into the end of the specified mo...

LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)

Set a particular kind of metadata attachment.

LLVM_ABI void setComdat(Comdat *C)

const Comdat * getComdat() const

LLVM_ABI bool eraseMetadata(unsigned KindID)

Erase all metadata attachments with the given kind.

bool hasSection() const

Check if this global has a custom object file section.

MDNode * getMetadata(unsigned KindID) const

Get the current metadata attachments for the given kind, if any.

static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)

Return a 64-bit global unique ID constructed from the name of a global symbol.

bool isThreadLocal() const

If the value is "Thread Local", its value isn't shared by the threads.

VisibilityTypes getVisibility() const

static bool isLocalLinkage(LinkageTypes Linkage)

LinkageTypes getLinkage() const

uint64_t GUID

Declare a type to represent a global unique identifier for a global value.

static StringRef dropLLVMManglingEscape(StringRef Name)

If the given string begins with the GlobalValue name mangling escape character '\1',...

bool isDeclarationForLinker() const

PointerType * getType() const

Global values are always pointers.

VisibilityTypes

An enumeration for the kinds of visibility of global values.

@ HiddenVisibility

The GV is hidden.

void setVisibility(VisibilityTypes V)

LinkageTypes

An enumeration for the kinds of linkage for global values.

@ PrivateLinkage

Like Internal, but omit from symbol table.

@ InternalLinkage

Rename collisions when linking (static functions).

@ ExternalLinkage

Externally visible function.

@ ExternalWeakLinkage

ExternalWeak linkage description.

Type * getValueType() const

const Constant * getInitializer() const

getInitializer - Return the initializer for this global variable.

LLVM_ABI void setInitializer(Constant *InitVal)

setInitializer - Sets the initializer for this global variable, removing any existing initializer if ...

MaybeAlign getAlign() const

Returns the alignment of the given variable.

void setConstant(bool Val)

LLVM_ABI void setCodeModel(CodeModel::Model CM)

Change the code model for this global.

LLVM_ABI void eraseFromParent()

eraseFromParent - This method unlinks 'this' from the containing module and deletes it.

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

static LLVM_ABI InlineAsm * get(FunctionType *Ty, StringRef AsmString, StringRef Constraints, bool hasSideEffects, bool isAlignStack=false, AsmDialect asmDialect=AD_ATT, bool canThrow=false)

InlineAsm::get - Return the specified uniqued inline asm string.

LLVM_ABI InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)

Set the metadata of the specified kind to the specified node.

unsigned getBitWidth() const

Get the number of bits in this IntegerType.

Analysis pass that exposes the LoopInfo for a function.

LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)

Definition LowerTypeTests.cpp:2497

const MDOperand & getOperand(unsigned I) const

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

unsigned getNumOperands() const

Return number of MDNode operands.

static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)

std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)

static ErrorOr< std::unique_ptr< MemoryBuffer > > getFile(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, bool IsVolatile=false, std::optional< Align > Alignment=std::nullopt)

Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful,...

TypeIdSummary & getOrInsertTypeIdSummary(StringRef TypeId)

Return an existing or new TypeIdSummary entry for TypeId.

const TypeIdSummary * getTypeIdSummary(StringRef TypeId) const

This returns either a pointer to the type id summary (if present in the summary map) or null (if not ...

CfiFunctionIndex & cfiFunctionDecls()

bool partiallySplitLTOUnits() const

CfiFunctionIndex & cfiFunctionDefs()

A Module instance is used to store all the information related to an LLVM module.

iterator_range< op_iterator > operands()

static PointerType * getUnqual(Type *ElementType)

This constructs a pointer to an object of the specified type in the default address space (address sp...

unsigned getAddressSpace() const

Return the address space of the Pointer type.

Analysis pass which computes a PostDominatorTree.

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

PreservedAnalyses & preserve()

Mark an analysis as preserved.

static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, InsertPosition InsertBefore=nullptr)

bool insert(const value_type &X)

Insert a new element into the SetVector.

LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)

Definition LowerTypeTests.cpp:2511

reference emplace_back(ArgTypes &&... Args)

void reserve(size_type N)

iterator erase(const_iterator CI)

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

std::pair< StringRef, StringRef > split(char Separator) const

Split into two substrings around the first occurrence of a separator character.

constexpr StringRef substr(size_t Start, size_t N=npos) const

Return a reference to the substring from [Start, Start + N).

bool starts_with(StringRef Prefix) const

Check if this string starts with the given Prefix.

constexpr size_t size() const

size - Get the string size.

bool ends_with(StringRef Suffix) const

Check if this string ends with the given Suffix.

Type * getElementType(unsigned N) const

Analysis pass providing the TargetTransformInfo.

See the file comment for details on the usage of the TrailingObjects type.

Triple - Helper class for working with autoconf configuration names.

The instances of the Type class are immutable: once they are created, they are never changed.

static LLVM_ABI Type * getVoidTy(LLVMContext &C)

A Use represents the edge between a Value definition and its users.

Value * getOperand(unsigned i) const

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

user_iterator user_begin()

bool hasOneUse() const

Return true if there is exactly one use of this value.

LLVM_ABI void replaceAllUsesWith(Value *V)

Change all uses of this to point to a new Value.

iterator_range< user_iterator > users()

LLVM_ABI void replaceUsesWithIf(Value *New, llvm::function_ref< bool(Use &U)> ShouldReplace)

Go through the uses list for this definition and make each use point to "V" if the callback ShouldRep...

iterator_range< use_iterator > uses()

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

LLVM_ABI void takeName(Value *V)

Transfer the name from V to this value.

std::pair< iterator, bool > insert(const ValueT &V)

bool contains(const_arg_type_t< ValueT > V) const

Check if the set contains the given element.

size_type count(const_arg_type_t< ValueT > V) const

Return 1 if the specified key is in the set, 0 otherwise.

const ParentTy * getParent() const

self_iterator getIterator()

NodeTy * getNextNode()

Get the next node, or nullptr for the list tail.

This class implements an extremely fast bulk output stream that can only output to a stream.

constexpr char Align[]

Key for Kernel::Arg::Metadata::mAlign.

constexpr char SymbolName[]

Key for Kernel::Metadata::mSymbolName.

constexpr std::underlying_type_t< E > Mask()

Get a bitmask with 1s in all places up to the high-order bit of E's largest value.

@ C

The default llvm calling convention, compatible with C.

@ BasicBlock

Various leaf nodes.

LLVM_ABI Function * getDeclarationIfExists(const Module *M, ID id)

Look up the Function declaration of the intrinsic id in the Module M and return it if it exists.

ValuesClass values(OptsTy... Options)

Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...

initializer< Ty > init(const Ty &Val)

DropTestKind

Specifies how to drop type tests.

@ Assume

Do not drop type tests (default).

LLVM_ABI bool isJumpTableCanonical(Function *F)

Definition LowerTypeTests.cpp:252

std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract_or_null(Y &&MD)

Extract a Value from Metadata, allowing null.

SmallVector< unsigned char, 0 > ByteArray

NodeAddr< PhiNode * > Phi

NodeAddr< UseNode * > Use

@ OF_TextWithCRLF

The file should be opened in text mode and use a carriage linefeed '\r '.

This is an optimization pass for GlobalISel generic memory operations.

LLVM_ABI void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)

Replace the instruction specified by BI with the instruction specified by I.

FunctionAddr VTableAddr Value

void stable_sort(R &&Range)

bool all_of(R &&range, UnaryPredicate P)

Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.

LLVM_ABI void setExplicitlyUnknownBranchWeightsIfProfiled(Instruction &I, StringRef PassName, const Function *F=nullptr)

Like setExplicitlyUnknownBranchWeights(...), but only sets unknown branch weights in the new instruct...

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

FunctionAddr VTableAddr uintptr_t uintptr_t Int32Ty

@ Export

Export information to summary.

@ Import

Import information from summary.

void append_range(Container &C, Range &&R)

Wrapper function to append range R to container C.

Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)

Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy

Provide the FunctionAnalysisManager to Module proxy.

int countr_zero(T Val)

Count number of 0's from the least significant bit to the most stopping at the first 1.

unsigned M1(unsigned Val)

LLVM_ABI bool convertUsersOfConstantsToInstructions(ArrayRef< Constant * > Consts, Function *RestrictToFunc=nullptr, bool RemoveDeadConstants=true, bool IncludeSelf=false)

Replace constant expressions users of the given constants with instructions.

void sort(IteratorTy Start, IteratorTy End)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

@ Ref

The access may reference the value stored in memory.

IRBuilder(LLVMContext &, FolderTy, InserterTy, MDNode *, ArrayRef< OperandBundleDef >) -> IRBuilder< FolderTy, InserterTy >

LLVM_ABI void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)

Adds global values to the llvm.compiler.used list.

uint64_t alignTo(uint64_t Size, Align A)

Returns a multiple of A needed to store Size bytes.

DWARFExpression::Operation Op

Expected< T > errorOrToExpected(ErrorOr< T > &&EO)

Convert an ErrorOr to an Expected.

ArrayRef(const T &OneElt) -> ArrayRef< T >

OutputIt copy(R &&Range, OutputIt Out)

constexpr unsigned BitWidth

LLVM_ABI void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)

Append F to the list of global ctors of module M with the given Priority.

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

LLVM_ABI Error errorCodeToError(std::error_code EC)

Helper for converting an std::error_code to a Error.

LLVM_ABI Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)

Split the containing block at the specified instruction - everything before SplitBefore stays in the ...

BumpPtrAllocatorImpl<> BumpPtrAllocator

The standard BumpPtrAllocator which just uses the default template parameters.

LLVM_ABI void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)

Adds global values to the llvm.used list.

CfiFunctionLinkage

The type of CFI jumptable needed for a function.

AnalysisManager< Module > ModuleAnalysisManager

Convenience typedef for the Module analysis manager.

constexpr uint64_t NextPowerOf2(uint64_t A)

Returns the next power of two (in 64-bits) that is strictly greater than A.

LLVM_ABI GlobalVariable * collectUsedGlobalVariables(const Module &M, SmallVectorImpl< GlobalValue * > &Vec, bool CompilerUsed)

Given "llvm.used" or "llvm.compiler.used" as a global name, collect the initializer elements of that ...

Kind

Specifies which kind of type check we should emit for this byte array.

@ Unknown

Unknown (analysis not performed, don't lower)

@ Single

Single element (last example in "Short Inline Bit Vectors")

@ Inline

Inlined bit vector ("Short Inline Bit Vectors")

@ Unsat

Unsatisfiable type (i.e. no global has this type metadata)

@ AllOnes

All-ones bit vector ("Eliminating Bit Vector Checks for All-Ones Bit Vectors")

@ ByteArray

Test a byte array (first example)

unsigned SizeM1BitWidth

Range of size-1 expressed as a bit width.

enum llvm::TypeTestResolution::Kind TheKind

LLVM_ABI BitSetInfo build()

Definition LowerTypeTests.cpp:163

SmallVector< uint64_t, 16 > Offsets

LLVM_ABI bool containsGlobalOffset(uint64_t Offset) const

Definition LowerTypeTests.cpp:134

LLVM_ABI void print(raw_ostream &OS) const

Definition LowerTypeTests.cpp:148

std::set< uint64_t > Bits

This class is used to build a byte array containing overlapping bit sets.

uint64_t BitAllocs[BitsPerByte]

The number of bytes allocated so far for each of the bits.

std::vector< uint8_t > Bytes

The byte array built so far.

LLVM_ABI void allocate(const std::set< uint64_t > &Bits, uint64_t BitSize, uint64_t &AllocByteOffset, uint8_t &AllocMask)

Allocate BitSize bits in the byte array where Bits contains the bits to set.

Definition LowerTypeTests.cpp:229

This class implements a layout algorithm for globals referenced by bit sets that tries to keep member...

std::vector< std::vector< uint64_t > > Fragments

The computed layout.

LLVM_ABI void addFragment(const std::set< uint64_t > &F)

Add F to the layout while trying to keep its indices contiguous.

Definition LowerTypeTests.cpp:200

std::vector< uint64_t > FragmentMap

Mapping from object index to fragment index.