LoopUtils.cpp Source File (original) (raw)

49using namespace llvm;

52#define DEBUG_TYPE "loop-utils"

59 bool PreserveLCSSA) {

60 bool Changed = false;

65 auto RewriteExit = [&](BasicBlock *BB) {

67 "Must start with an empty predecessors list!");

72 bool IsDedicatedExit = true;

74 if (L->contains(PredBB)) {

75 if (isa(PredBB->getTerminator()))

77 return false;

79 InLoopPredecessors.push_back(PredBB);

80 } else {

81 IsDedicatedExit = false;

82 }

84 assert(!InLoopPredecessors.empty() && "Must have *some* loop predecessor!");

87 if (IsDedicatedExit)

88 return false;

91 BB, InLoopPredecessors, ".loopexit", DT, LI, MSSAU, PreserveLCSSA);

93 if (!NewExitBB)

95 dbgs() << "WARNING: Can't create a dedicated exit block for loop: "

96 << *L << "\n");

97 else

98 LLVM_DEBUG(dbgs() << "LoopSimplify: Creating dedicated exit block "

99 << NewExitBB->getName() << "\n");

100 return true;

101 };

102

103

104

106 for (auto *BB : L->blocks())

107 for (auto *SuccBB : successors(BB)) {

108

109 if (L->contains(SuccBB))

110 continue;

111

112

113 if (!Visited.insert(SuccBB).second)

114 continue;

115

116 Changed |= RewriteExit(SuccBB);

117 }

118

119 return Changed;

120}

121

122

125

126 for (auto *Block : L->getBlocks())

127

128

129 for (auto &Inst : *Block) {

130 auto Users = Inst.users();

132 auto *Use = cast(U);

133 return !L->contains(Use->getParent());

134 }))

136 }

137

138 return UsedOutside;

139}

140

142

143

144

149

150

151

158

159

162

163

164

165

166

167

168

169

177

178

179}

180

181

182

183

184

185

186

187

188

200}

201

202

209}

210

211

212

213

215 unsigned V) {

217

219 if (LoopID) {

220 for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {

222

223 if (Node->getNumOperands() == 2) {

224 MDString *S = dyn_cast(Node->getOperand(0));

225 if (S && S->getString() == StringMD) {

227 mdconst::extract_or_null(Node->getOperand(1));

229

230 return;

231

232

233 continue;

234 }

235 }

237 }

238 }

239

241

244

247}

248

249std::optional

251 std::optional Width =

253

254 if (Width) {

256 TheLoop, "llvm.loop.vectorize.scalable.enable");

258 }

259

260 return std::nullopt;

261}

262

265 const char *InheritOptionsExceptPrefix, bool AlwaysNew) {

266 if (!OrigLoopID) {

267 if (AlwaysNew)

268 return nullptr;

269 return std::nullopt;

270 }

271

273

274 bool InheritAllAttrs = !InheritOptionsExceptPrefix;

275 bool InheritSomeAttrs =

276 InheritOptionsExceptPrefix && InheritOptionsExceptPrefix[0] != '\0';

279

280 bool Changed = false;

281 if (InheritAllAttrs || InheritSomeAttrs) {

283 MDNode *Op = cast(Existing.get());

284

285 auto InheritThisAttribute = [InheritSomeAttrs,

286 InheritOptionsExceptPrefix](MDNode *Op) {

287 if (!InheritSomeAttrs)

288 return false;

289

290

292 return true;

293 Metadata *NameMD = Op->getOperand(0).get();

294 if (!isa(NameMD))

295 return true;

296 StringRef AttrName = cast(NameMD)->getString();

297

298

299 return !AttrName.starts_with(InheritOptionsExceptPrefix);

300 };

301

302 if (InheritThisAttribute(Op))

304 else

305 Changed = true;

306 }

307 } else {

308

310 }

311

312 bool HasAnyFollowup = false;

313 for (StringRef OptionName : FollowupOptions) {

315 if (!FollowupNode)

316 continue;

317

318 HasAnyFollowup = true;

321 Changed = true;

322 }

323 }

324

325

326

327 if (!AlwaysNew && !HasAnyFollowup)

328 return std::nullopt;

329

330

331 if (!AlwaysNew && !Changed)

332 return OrigLoopID;

333

334

335 if (MDs.size() == 1)

336 return nullptr;

337

338

341 return FollowupLoopID;

342}

343

346}

347

350}

351

355

356 std::optional Count =

358 if (Count)

360

363

366

369

371}

372

376

377 std::optional Count =

379 if (Count)

381

384

387

389}

390

392 std::optional Enable =

394

397

398 std::optional VectorizeWidth =

400 std::optional InterleaveCount =

402

403

404

405 if (Enable == true && VectorizeWidth && VectorizeWidth->isScalar() &&

406 InterleaveCount == 1)

408

411

414

415 if ((VectorizeWidth && VectorizeWidth->isScalar()) && InterleaveCount == 1)

417

418 if ((VectorizeWidth && VectorizeWidth->isVector()) || InterleaveCount > 1)

420

423

425}

426

430

433

435}

436

440

443

445}

446

447

448

451 const Loop *CurLoop) {

453 auto AddRegionToWorklist = [&](DomTreeNode *DTN) {

454

457 Worklist.push_back(DTN->getBlock());

458 };

459

460 AddRegionToWorklist(N);

461

462 for (size_t I = 0; I < Worklist.size(); I++) {

464 AddRegionToWorklist(Child);

465 }

466

467 return Worklist;

468}

469

472 assert(LatchIdx != -1 && "LatchBlock is not a case in this PHINode");

474

476 if (U != Cond && U != IncV) return false;

477

479 if (U != Cond && U != PN) return false;

480 return true;

481}

482

483

486 assert((!DT || L->isLCSSAForm(*DT)) && "Expected LCSSA!");

487 auto *Preheader = L->getLoopPreheader();

488 assert(Preheader && "Preheader should exist!");

489

490 std::unique_ptr MSSAU;

491 if (MSSA)

492 MSSAU = std::make_unique(MSSA);

493

494

495

496

497

498

499

500

501

502

503 if (SE) {

506 }

507

508 Instruction *OldTerm = Preheader->getTerminator();

510 "Preheader must end with a side-effect-free terminator");

512 "Preheader must have a single successor");

513

514

515

516

517

518

519

520

521

522

523

524

525

526

527

528

529

530

531

532

533

534

535

536

537

539

540 auto *ExitBlock = L->getUniqueExitBlock();

541 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);

542 if (ExitBlock) {

543 assert(ExitBlock && "Should have a unique exit block!");

544 assert(L->hasDedicatedExits() && "Loop should have dedicated exits!");

545

547

549

550

551

552 for (PHINode &P : ExitBlock->phis()) {

553

554

555

556 int PredIndex = 0;

557 P.setIncomingBlock(PredIndex, Preheader);

558

559

560

561 P.removeIncomingValueIf([](unsigned Idx) { return Idx != 0; },

562 false);

563

564 assert((P.getNumIncomingValues() == 1 &&

565 P.getIncomingBlock(PredIndex) == Preheader) &&

566 "Should have exactly one value and that's from the preheader!");

567 }

568

569 if (DT) {

570 DTU.applyUpdates({{DominatorTree::Insert, Preheader, ExitBlock}});

571 if (MSSA) {

572 MSSAU->applyUpdates({{DominatorTree::Insert, Preheader, ExitBlock}},

573 *DT);

576 }

577 }

578

579

582

584 } else {

585 assert(L->hasNoExitBlocks() &&

586 "Loop should have either zero or one exit blocks.");

587

591 }

592

593 if (DT) {

594 DTU.applyUpdates({{DominatorTree::Delete, Preheader, L->getHeader()}});

595 if (MSSA) {

596 MSSAU->applyUpdates({{DominatorTree::Delete, Preheader, L->getHeader()}},

597 *DT);

599 L->block_end());

600 MSSAU->removeBlocks(DeadBlockSet);

603 }

604 }

605

606

610

611 if (ExitBlock) {

612

613

614

615

616

617

618

619

620 for (auto *Block : L->blocks())

624 if (auto *Usr = dyn_cast(U.getUser()))

625 if (L->contains(Usr->getParent()))

626 continue;

627

628

629 if (DT)

631 "Unexpected user in reachable block");

633 }

634

635

636 if (Block->IsNewDbgInfoFormat) {

639 DebugVariable Key(DVR.getVariable(), DVR.getExpression(),

640 DVR.getDebugLoc().get());

641 if (!DeadDebugSet.insert(Key).second)

642 continue;

643

644 DVR.removeFromParent();

645 DeadDbgVariableRecords.push_back(&DVR);

646 }

647 }

648

649

650

651

652 auto *DVI = dyn_cast(&I);

653 if (!DVI)

654 continue;

656 continue;

658 }

659

660

661

662

663

664

665

666 DIBuilder DIB(*ExitBlock->getModule());

668 ExitBlock->getFirstInsertionPt();

669 assert(InsertDbgValueBefore != ExitBlock->end() &&

670 "There should be a non-PHI instruction in exit block, else these "

671 "instructions will have no parent.");

672

673 for (auto *DVI : DeadDebugInst)

674 DVI->moveBefore(*ExitBlock, InsertDbgValueBefore);

675

676

677

678

679

681 ExitBlock->insertDbgRecordBefore(DVR, InsertDbgValueBefore);

682 }

683

684

685

686 for (auto *Block : L->blocks())

687 Block->dropAllReferences();

688

691

692 if (LI) {

693

694

695

696

698 BB->eraseFromParent();

699

700

701

702

704 blocks.insert(L->block_begin(), L->block_end());

707

708

709

710

711

712 if (Loop *ParentLoop = L->getParentLoop()) {

714 assert(I != ParentLoop->end() && "Couldn't find loop");

715 ParentLoop->removeChildLoop(I);

716 } else {

718 assert(I != LI->end() && "Couldn't find loop");

720 }

722 }

723}

724

727 auto *Latch = L->getLoopLatch();

728 assert(Latch && "multiple latches not yet supported");

729 auto *Header = L->getHeader();

730 Loop *OutermostLoop = L->getOutermostLoop();

731

734

735 std::unique_ptr MSSAU;

736 if (MSSA)

737 MSSAU = std::make_unique(MSSA);

738

739

740

741 [&]() -> void {

742 if (auto *BI = dyn_cast(Latch->getTerminator())) {

743 if (!BI->isConditional()) {

744 DomTreeUpdater DTU(&DT, DomTreeUpdater::UpdateStrategy::Eager);

746 MSSAU.get());

747 return;

748 }

749

750

751

752 if (L->isLoopExiting(Latch)) {

753

754

755

756

757 const unsigned ExitIdx = L->contains(BI->getSuccessor(0)) ? 1 : 0;

758 BasicBlock *ExitBB = BI->getSuccessor(ExitIdx);

759

760 DomTreeUpdater DTU(&DT, DomTreeUpdater::UpdateStrategy::Eager);

761 Header->removePredecessor(Latch, true);

762

764 auto *NewBI = Builder.CreateBr(ExitBB);

765

766

767 NewBI->copyMetadata(*BI, {LLVMContext::MD_dbg,

768 LLVMContext::MD_annotation});

769

770 BI->eraseFromParent();

771 DTU.applyUpdates({{DominatorTree::Delete, Latch, Header}});

772 if (MSSA)

773 MSSAU->applyUpdates({{DominatorTree::Delete, Latch, Header}}, DT);

774 return;

775 }

776 }

777

778

779

780

781 auto *BackedgeBB = SplitEdge(Latch, Header, &DT, &LI, MSSAU.get());

782

783 DomTreeUpdater DTU(&DT, DomTreeUpdater::UpdateStrategy::Eager);

785 true, &DTU, MSSAU.get());

786 }();

787

788

789

791

792

793

794

795

796

797 if (OutermostLoop != L)

799}

800

801

802

803

804

806 BasicBlock *Latch = L->getLoopLatch();

807 if (!Latch)

808 return nullptr;

809

811 if (!LatchBR || LatchBR->getNumSuccessors() != 2 || !L->isLoopExiting(Latch))

812 return nullptr;

813

815 LatchBR->getSuccessor(1) == L->getHeader()) &&

816 "At least one edge out of the latch must go to the header");

817

818 return LatchBR;

819}

820

821

822

826

827

828

829 uint64_t LoopWeight, ExitWeight;

831 return std::nullopt;

832

833 if (L->contains(ExitingBranch->getSuccessor(1)))

834 std::swap(LoopWeight, ExitWeight);

835

836 if (!ExitWeight)

837

838 return std::nullopt;

839

840 OrigExitWeight = ExitWeight;

841

842

843

845

846 return ExitCount + 1;

847}

848

849std::optional

851 unsigned *EstimatedLoopInvocationWeight) {

852

853

854

855

858 if (std::optional<uint64_t> EstTripCount =

860 if (EstimatedLoopInvocationWeight)

861 *EstimatedLoopInvocationWeight = ExitWeight;

862 return *EstTripCount;

863 }

864 }

865 return std::nullopt;

866}

867

869 unsigned EstimatedloopInvocationWeight) {

870

871

872

874 if (!LatchBranch)

875 return false;

876

877

878 unsigned LatchExitWeight = 0;

879 unsigned BackedgeTakenWeight = 0;

880

881 if (EstimatedTripCount > 0) {

882 LatchExitWeight = EstimatedloopInvocationWeight;

883 BackedgeTakenWeight = (EstimatedTripCount - 1) * LatchExitWeight;

884 }

885

886

887 if (LatchBranch->getSuccessor(0) != L->getHeader())

888 std::swap(BackedgeTakenWeight, LatchExitWeight);

889

891

892

894 LLVMContext::MD_prof,

896

897 return true;

898}

899

903 if (!OuterL)

904 return true;

905

906

908 const SCEV *InnerLoopBECountSC = SE.getExitCount(InnerLoop, InnerLoopLatch);

909 if (isa(InnerLoopBECountSC) ||

911 return false;

912

913

917 return false;

918

919 return true;

920}

921

923 switch (RK) {

924 default:

926 case RecurKind::Add:

927 return Intrinsic::vector_reduce_add;

928 case RecurKind::Mul:

929 return Intrinsic::vector_reduce_mul;

930 case RecurKind::And:

931 return Intrinsic::vector_reduce_and;

932 case RecurKind::Or:

933 return Intrinsic::vector_reduce_or;

934 case RecurKind::Xor:

935 return Intrinsic::vector_reduce_xor;

936 case RecurKind::FMulAdd:

937 case RecurKind::FAdd:

938 return Intrinsic::vector_reduce_fadd;

939 case RecurKind::FMul:

940 return Intrinsic::vector_reduce_fmul;

941 case RecurKind::SMax:

942 return Intrinsic::vector_reduce_smax;

943 case RecurKind::SMin:

944 return Intrinsic::vector_reduce_smin;

945 case RecurKind::UMax:

946 return Intrinsic::vector_reduce_umax;

947 case RecurKind::UMin:

948 return Intrinsic::vector_reduce_umin;

949 case RecurKind::FMax:

950 return Intrinsic::vector_reduce_fmax;

951 case RecurKind::FMin:

952 return Intrinsic::vector_reduce_fmin;

953 case RecurKind::FMaximum:

954 return Intrinsic::vector_reduce_fmaximum;

955 case RecurKind::FMinimum:

956 return Intrinsic::vector_reduce_fminimum;

957 }

958}

959

961 switch (RdxID) {

962 case Intrinsic::vector_reduce_fadd:

963 return Instruction::FAdd;

964 case Intrinsic::vector_reduce_fmul:

965 return Instruction::FMul;

966 case Intrinsic::vector_reduce_add:

967 return Instruction::Add;

968 case Intrinsic::vector_reduce_mul:

969 return Instruction::Mul;

970 case Intrinsic::vector_reduce_and:

971 return Instruction::And;

972 case Intrinsic::vector_reduce_or:

973 return Instruction::Or;

974 case Intrinsic::vector_reduce_xor:

975 return Instruction::Xor;

976 case Intrinsic::vector_reduce_smax:

977 case Intrinsic::vector_reduce_smin:

978 case Intrinsic::vector_reduce_umax:

979 case Intrinsic::vector_reduce_umin:

980 return Instruction::ICmp;

981 case Intrinsic::vector_reduce_fmax:

982 case Intrinsic::vector_reduce_fmin:

983 return Instruction::FCmp;

984 default:

986 }

987}

988

990 switch (RdxID) {

991 default:

993 case Intrinsic::vector_reduce_umin:

994 return Intrinsic::umin;

995 case Intrinsic::vector_reduce_umax:

996 return Intrinsic::umax;

997 case Intrinsic::vector_reduce_smin:

998 return Intrinsic::smin;

999 case Intrinsic::vector_reduce_smax:

1000 return Intrinsic::smax;

1001 case Intrinsic::vector_reduce_fmin:

1002 return Intrinsic::minnum;

1003 case Intrinsic::vector_reduce_fmax:

1004 return Intrinsic::maxnum;

1005 case Intrinsic::vector_reduce_fminimum:

1006 return Intrinsic::minimum;

1007 case Intrinsic::vector_reduce_fmaximum:

1008 return Intrinsic::maximum;

1009 }

1010}

1011

1013 switch (RK) {

1014 default:

1016 case RecurKind::UMin:

1017 return Intrinsic::umin;

1018 case RecurKind::UMax:

1019 return Intrinsic::umax;

1020 case RecurKind::SMin:

1021 return Intrinsic::smin;

1022 case RecurKind::SMax:

1023 return Intrinsic::smax;

1024 case RecurKind::FMin:

1025 return Intrinsic::minnum;

1026 case RecurKind::FMax:

1027 return Intrinsic::maxnum;

1028 case RecurKind::FMinimum:

1029 return Intrinsic::minimum;

1030 case RecurKind::FMaximum:

1031 return Intrinsic::maximum;

1032 }

1033}

1034

1036 switch (RdxID) {

1037 case Intrinsic::vector_reduce_smax:

1038 return RecurKind::SMax;

1039 case Intrinsic::vector_reduce_smin:

1040 return RecurKind::SMin;

1041 case Intrinsic::vector_reduce_umax:

1042 return RecurKind::UMax;

1043 case Intrinsic::vector_reduce_umin:

1044 return RecurKind::UMin;

1045 case Intrinsic::vector_reduce_fmax:

1046 return RecurKind::FMax;

1047 case Intrinsic::vector_reduce_fmin:

1048 return RecurKind::FMin;

1049 default:

1050 return RecurKind::None;

1051 }

1052}

1053

1055 switch (RK) {

1056 default:

1058 case RecurKind::UMin:

1060 case RecurKind::UMax:

1062 case RecurKind::SMin:

1064 case RecurKind::SMax:

1066 case RecurKind::FMin:

1068 case RecurKind::FMax:

1070

1071

1072

1073 }

1074}

1075

1080 (RK == RecurKind::FMinimum || RK == RecurKind::FMaximum)) {

1081

1084 "rdx.minmax");

1085 }

1090}

1091

1092

1095 unsigned VF = cast(Src->getType())->getNumElements();

1096

1097

1098

1099 Value *Result = Acc;

1100 for (unsigned ExtractIdx = 0; ExtractIdx != VF; ++ExtractIdx) {

1103

1104 if (Op != Instruction::ICmp && Op != Instruction::FCmp) {

1106 "bin.rdx");

1107 } else {

1109 "Invalid min/max");

1110 Result = createMinMaxOp(Builder, RdxKind, Result, Ext);

1111 }

1112 }

1113

1114 return Result;

1115}

1116

1117

1119 unsigned Op,

1122 unsigned VF = cast(Src->getType())->getNumElements();

1123

1124

1125

1127 "Reduction emission only supported for pow2 vectors!");

1128

1129

1130

1131

1132

1133

1134

1135 auto BuildShuffledOp = [&Builder, &Op,

1137 Value *&TmpVec) -> void {

1139 if (Op != Instruction::ICmp && Op != Instruction::FCmp) {

1141 "bin.rdx");

1142 } else {

1144 "Invalid min/max");

1145 TmpVec = createMinMaxOp(Builder, RdxKind, TmpVec, Shuf);

1146 }

1147 };

1148

1149 Value *TmpVec = Src;

1150 if (TargetTransformInfo::ReductionShuffle::Pairwise == RS) {

1152 for (unsigned stride = 1; stride < VF; stride <<= 1) {

1153

1154 std::fill(ShuffleMask.begin(), ShuffleMask.end(), -1);

1155 for (unsigned j = 0; j < VF; j += stride << 1) {

1156 ShuffleMask[j] = j + stride;

1157 }

1158 BuildShuffledOp(ShuffleMask, TmpVec);

1159 }

1160 } else {

1162 for (unsigned i = VF; i != 1; i >>= 1) {

1163

1164 for (unsigned j = 0; j != i / 2; ++j)

1165 ShuffleMask[j] = i / 2 + j;

1166

1167

1168 std::fill(&ShuffleMask[i / 2], ShuffleMask.end(), -1);

1169 BuildShuffledOp(ShuffleMask, TmpVec);

1170 }

1171 }

1172

1174}

1175

1181 "Unexpected reduction kind");

1182 Value *InitVal = Desc.getRecurrenceStartValue();

1183 Value *NewVal = nullptr;

1184

1185

1186

1188 for (auto *U : OrigPhi->users()) {

1189 if ((SI = dyn_cast(U)))

1190 break;

1191 }

1192 assert(SI && "One user of the original phi should be a select");

1193

1194 if (SI->getTrueValue() == OrigPhi)

1195 NewVal = SI->getFalseValue();

1196 else {

1197 assert(SI->getFalseValue() == OrigPhi &&

1198 "At least one input to the select should be the original Phi");

1199 NewVal = SI->getTrueValue();

1200 }

1201

1202

1204 Src->getType()->isVectorTy() ? Builder.CreateOrReduce(Src) : Src;

1205

1206

1208 return Builder.CreateSelect(AnyOf, NewVal, InitVal, "rdx.select");

1209}

1210

1214 Desc.getRecurrenceKind()) &&

1215 "Unexpected reduction kind");

1216 Value *StartVal = Desc.getRecurrenceStartValue();

1218 Value *MaxRdx = Src->getType()->isVectorTy()

1220 : Src;

1221

1222

1225 return Builder.CreateSelect(Cmp, MaxRdx, StartVal, "rdx.select");

1226}

1227

1230 bool Negative = false;

1231 switch (RdxID) {

1232 default:

1234 case Intrinsic::vector_reduce_add:

1235 case Intrinsic::vector_reduce_mul:

1236 case Intrinsic::vector_reduce_or:

1237 case Intrinsic::vector_reduce_xor:

1238 case Intrinsic::vector_reduce_and:

1239 case Intrinsic::vector_reduce_fadd:

1240 case Intrinsic::vector_reduce_fmul: {

1243 Flags.noSignedZeros());

1244 }

1245 case Intrinsic::vector_reduce_umax:

1246 case Intrinsic::vector_reduce_umin:

1247 case Intrinsic::vector_reduce_smin:

1248 case Intrinsic::vector_reduce_smax: {

1251 }

1252 case Intrinsic::vector_reduce_fmax:

1253 case Intrinsic::vector_reduce_fmaximum:

1254 Negative = true;

1255 [[fallthrough]];

1256 case Intrinsic::vector_reduce_fmin:

1257 case Intrinsic::vector_reduce_fminimum: {

1258 bool PropagatesNaN = RdxID == Intrinsic::vector_reduce_fminimum ||

1259 RdxID == Intrinsic::vector_reduce_fmaximum;

1261 return (!Flags.noNaNs() && !PropagatesNaN)

1263 : !Flags.noInfs()

1266 }

1267 }

1268}

1269

1271 assert((!(K == RecurKind::FMin || K == RecurKind::FMax) ||

1273 "nnan, nsz is expected to be set for FP min/max reduction.");

1276}

1277

1280 auto *SrcVecEltTy = cast(Src->getType())->getElementType();

1281 auto getIdentity = [&]() {

1284 };

1285 switch (RdxKind) {

1286 case RecurKind::Add:

1287 case RecurKind::Mul:

1288 case RecurKind::And:

1289 case RecurKind::Or:

1290 case RecurKind::Xor:

1291 case RecurKind::SMax:

1292 case RecurKind::SMin:

1293 case RecurKind::UMax:

1294 case RecurKind::UMin:

1295 case RecurKind::FMax:

1296 case RecurKind::FMin:

1297 case RecurKind::FMinimum:

1298 case RecurKind::FMaximum:

1300 case RecurKind::FMulAdd:

1301 case RecurKind::FAdd:

1303 case RecurKind::FMul:

1305 default:

1307 }

1308}

1309

1314 "AnyOf reduction is not supported.");

1316 auto *SrcTy = cast(Src->getType());

1317 Type *SrcEltTy = SrcTy->getElementType();

1319 Value *Ops[] = {Iden, Src};

1321}

1322

1326

1327

1328

1330 B.setFastMathFlags(Desc.getFastMathFlags());

1331

1337

1339}

1340

1344 assert((Desc.getRecurrenceKind() == RecurKind::FAdd ||

1345 Desc.getRecurrenceKind() == RecurKind::FMulAdd) &&

1346 "Unexpected reduction kind");

1347 assert(Src->getType()->isVectorTy() && "Expected a vector type");

1348 assert(!Start->getType()->isVectorTy() && "Expected a scalar type");

1349

1350 return B.CreateFAddReduce(Start, Src);

1351}

1352

1356 assert((Desc.getRecurrenceKind() == RecurKind::FAdd ||

1357 Desc.getRecurrenceKind() == RecurKind::FMulAdd) &&

1358 "Unexpected reduction kind");

1359 assert(Src->getType()->isVectorTy() && "Expected a vector type");

1360 assert(!Start->getType()->isVectorTy() && "Expected a scalar type");

1361

1363 auto *SrcTy = cast(Src->getType());

1364 Value *Ops[] = {Start, Src};

1366}

1367

1369 bool IncludeWrapFlags) {

1370 auto *VecOp = dyn_cast(I);

1371 if (!VecOp)

1372 return;

1373 auto *Intersection = (OpValue == nullptr) ? dyn_cast(VL[0])

1374 : dyn_cast(OpValue);

1375 if (!Intersection)

1376 return;

1377 const unsigned Opcode = Intersection->getOpcode();

1378 VecOp->copyIRFlags(Intersection, IncludeWrapFlags);

1379 for (auto *V : VL) {

1380 auto *Instr = dyn_cast(V);

1381 if (!Instr)

1382 continue;

1383 if (OpValue == nullptr || Opcode == Instr->getOpcode())

1384 VecOp->andIRFlags(V);

1385 }

1386}

1387

1393}

1394

1400}

1401

1407}

1408

1414}

1415

1418 unsigned BitWidth = cast(S->getType())->getBitWidth();

1421 auto Predicate = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;

1425}

1426

1429 unsigned BitWidth = cast(S->getType())->getBitWidth();

1432 auto Predicate = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;

1436}

1437

1438

1439

1440

1441

1442

1448 while (!WorkList.empty()) {

1450

1451 if (!L->contains(Curr))

1452 continue;

1453

1455 return true;

1456

1457 for (const auto *U : Curr->users()) {

1458 auto *UI = cast(U);

1459 if (Visited.insert(UI).second)

1461 }

1462 }

1463 return false;

1464}

1465

1466

1467

1469 PHINode *PN;

1470 unsigned Ith;

1473 bool HighCost;

1474

1476 bool H)

1479};

1480

1481

1482

1483

1485 BasicBlock *Preheader = L->getLoopPreheader();

1486

1487 if (!Preheader)

1488 return false;

1489

1490

1491

1492

1493

1495 L->getExitingBlocks(ExitingBlocks);

1497 L->getUniqueExitBlocks(ExitBlocks);

1498 if (ExitBlocks.size() != 1 || ExitingBlocks.size() != 1)

1499 return false;

1500

1501 BasicBlock *ExitBlock = ExitBlocks[0];

1503 while (PHINode *P = dyn_cast(BI)) {

1504 Value *Incoming = P->getIncomingValueForBlock(ExitingBlocks[0]);

1505

1506

1507

1508

1509 bool found = false;

1510 for (const RewritePhi &Phi : RewritePhiSet) {

1511 unsigned i = Phi.Ith;

1512 if (Phi.PN == P && (Phi.PN)->getIncomingValue(i) == Incoming) {

1513 found = true;

1514 break;

1515 }

1516 }

1517

1519 if (!found && (I = dyn_cast(Incoming)))

1520 if (!L->hasLoopInvariantOperands(I))

1521 return false;

1522

1523 ++BI;

1524 }

1525

1526 for (auto *BB : L->blocks())

1528 return I.mayHaveSideEffects();

1529 }))

1530 return false;

1531

1532 return true;

1533}

1534

1535

1536

1537

1540 if (!Phi)

1541 return false;

1542 if (!L->getLoopPreheader())

1543 return false;

1544 if (Phi->getParent() != L->getHeader())

1545 return false;

1547}

1548

1555

1556 assert(L->isRecursivelyLCSSAForm(*DT, *LI) &&

1557 "Indvars did not preserve LCSSA!");

1558

1560 L->getUniqueExitBlocks(ExitBlocks);

1561

1563

1564

1565

1566 for (BasicBlock *ExitBB : ExitBlocks) {

1567

1568

1569 PHINode *PN = dyn_cast(ExitBB->begin());

1570 if (!PN) continue;

1571

1573

1574

1576 while ((PN = dyn_cast(BBI++))) {

1578 continue;

1579

1581 continue;

1582

1583

1584 for (unsigned i = 0; i != NumPreds; ++i) {

1585

1586

1588 if (!isa(InVal))

1589 continue;

1590

1591

1593 continue;

1594

1595

1596 Instruction *Inst = cast(InVal);

1597 if (!L->contains(Inst))

1598 continue;

1599

1600

1601

1602

1603

1604

1607 PHINode *IndPhi = dyn_cast(Inst);

1608 if (IndPhi) {

1610 continue;

1611

1612

1614 if (!isa(U) && !isa(U))

1615 return true;

1616 BinaryOperator *B = dyn_cast(U);

1617 if (B && B != ID.getInductionBinOp())

1618 return true;

1619 return false;

1620 }))

1621 continue;

1622 } else {

1623

1624

1626 if ()

1627 continue;

1629 PHINode *Phi = dyn_cast(U);

1630 if (Phi != PN && !checkIsIndPhi(Phi, L, SE, ID))

1631 return true;

1632 return false;

1633 }))

1634 continue;

1635 if (B != ID.getInductionBinOp())

1636 continue;

1637 }

1638 }

1639

1640

1641

1642

1643

1644

1645

1646 const SCEV *ExitValue = SE->getSCEVAtScope(Inst, L->getParentLoop());

1647 if (isa(ExitValue) ||

1649 Rewriter .isSafeToExpand(ExitValue)) {

1650

1651

1652

1653

1655 if (isa(ExitCount))

1656 continue;

1657 if (auto *AddRec = dyn_cast(SE->getSCEV(Inst)))

1658 if (AddRec->getLoop() == L)

1659 ExitValue = AddRec->evaluateAtIteration(ExitCount, *SE);

1660 if (isa(ExitValue) ||

1662 Rewriter .isSafeToExpand(ExitValue))

1663 continue;

1664 }

1665

1666

1667

1668

1669

1670

1673 continue;

1674

1675

1676 bool HighCost = Rewriter.isHighCostExpansion(

1678

1679

1680

1681

1682

1683

1684

1685

1687 (isa(Inst) || isa(Inst)) ?

1688 &*Inst->getParent()->getFirstInsertionPt() : Inst;

1689 RewritePhiSet.emplace_back(PN, i, ExitValue, InsertPt, HighCost);

1690 }

1691 }

1692 }

1693

1694

1695

1696

1697

1698

1700 int NumReplaced = 0;

1701

1702

1703 for (const RewritePhi &Phi : RewritePhiSet) {

1705

1706

1707

1710 !LoopCanBeDel && Phi.HighCost)

1711 continue;

1712

1714 Phi.ExpansionSCEV, Phi.PN->getType(), Phi.ExpansionPoint);

1715

1716 LLVM_DEBUG(dbgs() << "rewriteLoopExitValues: AfterLoopVal = " << *ExitVal

1717 << '\n'

1718 << " LoopVal = " << *(Phi.ExpansionPoint) << "\n");

1719

1720#ifndef NDEBUG

1721

1722

1723

1724 if (auto *ExitInsn = dyn_cast(ExitVal))

1725 if (auto *EVL = LI->getLoopFor(ExitInsn->getParent()))

1726 if (EVL != L)

1727 assert(EVL->contains(L) && "LCSSA breach detected!");

1728#endif

1729

1730 NumReplaced++;

1733

1734

1735

1736

1737

1739

1740

1741

1744

1745

1750 }

1751 }

1752

1753

1754

1755 Rewriter.clearInsertPoint();

1756 return NumReplaced;

1757}

1758

1759

1760

1763 assert(UF > 0 && "Zero unrolled factor is not supported");

1764 assert(UnrolledLoop != RemainderLoop &&

1765 "Unrolled and Remainder loops are expected to distinct");

1766

1767

1768 unsigned OrigLoopInvocationWeight = 0;

1769 std::optional OrigAverageTripCount =

1771 if (!OrigAverageTripCount)

1772 return;

1773

1774

1775 unsigned UnrolledAverageTripCount = *OrigAverageTripCount / UF;

1776

1777 unsigned RemainderAverageTripCount = *OrigAverageTripCount % UF;

1778

1780 OrigLoopInvocationWeight);

1782 OrigLoopInvocationWeight);

1783}

1784

1785

1786

1787

1788template

1791

1792

1794

1795

1796

1798 assert(PreOrderLoops.empty() && "Must start with an empty preorder walk.");

1800 "Must start with an empty preorder walk worklist.");

1801 PreOrderWorklist.push_back(RootL);

1802 do {

1804 PreOrderWorklist.append(L->begin(), L->end());

1806 } while (!PreOrderWorklist.empty());

1807

1808 Worklist.insert(std::move(PreOrderLoops));

1809 PreOrderLoops.clear();

1810 }

1811}

1812

1813template

1817}

1818

1819template void llvm::appendLoopsToWorklist<ArrayRef<Loop *> &>(

1821

1822template void

1823llvm::appendLoopsToWorklist<Loop &>(Loop &L,

1825

1829}

1830

1834 if (PL)

1835 PL->addChildLoop(&New);

1836 else

1838

1839 if (LPM)

1841

1842

1845 New.addBasicBlockToLoop(cast(VM[BB]), *LI);

1846

1847

1848 for (Loop *I : *L)

1850

1851 return &New;

1852}

1853

1854

1855

1856

1857

1862};

1863

1864

1865

1871

1872 Value *Start = nullptr, *End = nullptr;

1873 LLVM_DEBUG(dbgs() << "LAA: Adding RT check for range:\n");

1875

1876

1877

1878

1879

1880

1881

1882

1883

1884

1885

1887 isa(High) && isa(Low)) {

1888 auto *HighAR = cast(High);

1889 auto *LowAR = cast(Low);

1892 const SCEV *Recur = LowAR->getStepRecurrence(SE);

1893 if (Recur == HighAR->getStepRecurrence(SE) &&

1894 HighAR->getLoop() == OuterLoop && LowAR->getLoop() == OuterLoop) {

1896 const SCEV *OuterExitCount = SE.getExitCount(OuterLoop, OuterLoopLatch);

1897 if (!isa(OuterExitCount) &&

1899 const SCEV *NewHigh =

1900 cast(High)->evaluateAtIteration(OuterExitCount, SE);

1901 if (!isa(NewHigh)) {

1902 LLVM_DEBUG(dbgs() << "LAA: Expanded RT check for range to include "

1903 "outer loop in order to permit hoisting\n");

1904 High = NewHigh;

1905 Low = cast(Low)->getStart();

1906

1907

1910 Stride = Recur;

1911 LLVM_DEBUG(dbgs() << "LAA: ... but need to check stride is "

1912 "positive: "

1913 << *Stride << '\n');

1914 }

1915 }

1916 }

1917 }

1918 }

1919

1920 Start = Exp.expandCodeFor(Low, PtrArithTy, Loc);

1921 End = Exp.expandCodeFor(High, PtrArithTy, Loc);

1924 Start = Builder.CreateFreeze(Start, Start->getName() + ".fr");

1926 }

1927 Value *StrideVal =

1928 Stride ? Exp.expandCodeFor(Stride, Stride->getType(), Loc) : nullptr;

1930 return {Start, End, StrideVal};

1931}

1932

1933

1934

1939

1940

1941

1942 transform(PointerChecks, std::back_inserter(ChecksWithBounds),

1948 return std::make_pair(First, Second);

1949 });

1950

1951 return ChecksWithBounds;

1952}

1953

1958

1959

1960 auto ExpandedChecks =

1962

1966

1967 Value *MemoryRuntimeCheck = nullptr;

1968

1969 for (const auto &[A, B] : ExpandedChecks) {

1970

1971

1972

1973 assert((A.Start->getType()->getPointerAddressSpace() ==

1974 B.End->getType()->getPointerAddressSpace()) &&

1975 (B.Start->getType()->getPointerAddressSpace() ==

1976 A.End->getType()->getPointerAddressSpace()) &&

1977 "Trying to bounds check pointers with different address spaces");

1978

1979

1980

1981

1982

1983

1984

1985

1986

1989 Value *IsConflict = ChkBuilder.CreateAnd(Cmp0, Cmp1, "found.conflict");

1990 if (A.StrideToCheck) {

1992 A.StrideToCheck, ConstantInt::get(A.StrideToCheck->getType(), 0),

1993 "stride.check");

1994 IsConflict = ChkBuilder.CreateOr(IsConflict, IsNegativeStride);

1995 }

1996 if (B.StrideToCheck) {

1998 B.StrideToCheck, ConstantInt::get(B.StrideToCheck->getType(), 0),

1999 "stride.check");

2000 IsConflict = ChkBuilder.CreateOr(IsConflict, IsNegativeStride);

2001 }

2002 if (MemoryRuntimeCheck) {

2003 IsConflict =

2004 ChkBuilder.CreateOr(MemoryRuntimeCheck, IsConflict, "conflict.rdx");

2005 }

2006 MemoryRuntimeCheck = IsConflict;

2007 }

2008

2009 return MemoryRuntimeCheck;

2010}

2011

2015

2019

2020 Value *MemoryRuntimeCheck = nullptr;

2021

2022 auto &SE = *Expander.getSE();

2023

2024

2026 for (const auto &[SrcStart, SinkStart, AccessSize, NeedsFreeze] : Checks) {

2027 Type *Ty = SinkStart->getType();

2028

2029 auto *VFTimesUFTimesSize =

2031 ConstantInt::get(Ty, IC * AccessSize));

2033 Expander.expandCodeFor(SE.getMinusSCEV(SinkStart, SrcStart), Ty, Loc);

2034

2035

2036

2037 Value *IsConflict = SeenCompares.lookup({Diff, VFTimesUFTimesSize});

2038 if (IsConflict)

2039 continue;

2040

2041 IsConflict =

2042 ChkBuilder.CreateICmpULT(Diff, VFTimesUFTimesSize, "diff.check");

2043 SeenCompares.insert({{Diff, VFTimesUFTimesSize}, IsConflict});

2044 if (NeedsFreeze)

2045 IsConflict =

2047 if (MemoryRuntimeCheck) {

2048 IsConflict =

2049 ChkBuilder.CreateOr(MemoryRuntimeCheck, IsConflict, "conflict.rdx");

2050 }

2051 MemoryRuntimeCheck = IsConflict;

2052 }

2053

2054 return MemoryRuntimeCheck;

2055}

2056

2057std::optional

2060 auto *TI = dyn_cast(L.getHeader()->getTerminator());

2061 if (!TI || !TI->isConditional())

2062 return {};

2063

2064 auto *CondI = dyn_cast(TI->getCondition());

2065

2066

2067

2068

2069 if (!CondI || !isa<CmpInst, TruncInst>(CondI) || !L.contains(CondI))

2070 return {};

2071

2073 InstToDuplicate.push_back(CondI);

2074

2076 WorkList.append(CondI->op_begin(), CondI->op_end());

2077

2080 while (!WorkList.empty()) {

2082 if ( || !L.contains(I))

2083 continue;

2084

2085

2086 if (!isa(I) && !isa(I))

2087 return {};

2088

2089

2090 if (auto *LI = dyn_cast(I))

2091 if (LI->isVolatile() || LI->isAtomic())

2092 return {};

2093

2096 if (auto *MemUse = dyn_cast_or_null(MA)) {

2097

2098 AccessesToCheck.push_back(MemUse->getDefiningAccess());

2100 } else {

2101

2102

2103 return {};

2104 }

2105 }

2106 WorkList.append(I->op_begin(), I->op_end());

2107 }

2108

2109 if (InstToDuplicate.empty())

2110 return {};

2111

2113 L.getExitingBlocks(ExitingBlocks);

2114 auto HasNoClobbersOnPath =

2115 [&L, &AA, &AccessedLocs, &ExitingBlocks, &InstToDuplicate,

2118 -> std::optional {

2120

2121

2126 Seen.insert(Header);

2127 Info.PathIsNoop &=

2128 all_of(*Header, [](Instruction &I) { return .mayHaveSideEffects(); });

2129

2130 while (!WorkList.empty()) {

2132 if (!L.contains(Current))

2133 continue;

2134 const auto &SeenIns = Seen.insert(Current);

2135 if (!SeenIns.second)

2136 continue;

2137

2139 *Current, [](Instruction &I) { return .mayHaveSideEffects(); });

2141 }

2142

2143

2144

2145 if (Seen.size() < 2)

2146 return {};

2147

2148

2149

2150

2151

2153 while (!AccessesToCheck.empty()) {

2155 auto SeenI = SeenAccesses.insert(Current);

2157 continue;

2158

2159

2161 return {};

2162

2163

2164 if (isa(Current))

2165 continue;

2166

2167

2168

2169 if (auto *CurrentDef = dyn_cast(Current)) {

2172 AA.getModRefInfo(CurrentDef->getMemoryInst(), Loc));

2173 }))

2174 return {};

2175 }

2176

2177 for (Use &U : Current->uses())

2178 AccessesToCheck.push_back(cast(U.getUser()));

2179 }

2180

2181

2182

2184

2185

2186

2187

2188 if (Info.PathIsNoop) {

2189 for (auto *Exiting : ExitingBlocks) {

2190 if (!Seen.contains(Exiting))

2191 continue;

2192 for (auto *Succ : successors(Exiting)) {

2193 if (L.contains(Succ))

2194 continue;

2195

2196 Info.PathIsNoop &= Succ->phis().empty() &&

2197 ( Info .ExitForPath || Info.ExitForPath == Succ);

2198 if ( Info .PathIsNoop)

2199 break;

2200 assert(( Info .ExitForPath || Info.ExitForPath == Succ) &&

2201 "cannot have multiple exit blocks");

2202 Info.ExitForPath = Succ;

2203 }

2204 }

2205 }

2206 if ( Info .ExitForPath)

2207 Info.PathIsNoop = false;

2208

2209 Info.InstToDuplicate = InstToDuplicate;

2210 return Info;

2211 };

2212

2213

2214

2215 if (TI->getSuccessor(0) == TI->getSuccessor(1))

2216 return {};

2217

2218 if (auto Info = HasNoClobbersOnPath(TI->getSuccessor(0), L.getHeader(),

2219 AccessesToCheck)) {

2221 return Info;

2222 }

2223 if (auto Info = HasNoClobbersOnPath(TI->getSuccessor(1), L.getHeader(),

2224 AccessesToCheck)) {

2226 return Info;

2227 }

2228

2229 return {};

2230}

AMDGPU Register Bank Select

This is the interface for LLVM's primary stateless and local alias analysis.

bbsections Prepares for basic block by splitting functions into clusters of basic blocks

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

Analysis containing CSE Info

Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx

This file defines the DenseSet and SmallDenseSet classes.

This is the interface for a simple mod/ref and alias analysis over globals.

static const HTTPClientCleanup Cleanup

Module.h This file contains the declarations for the Module class.

iv Induction Variable Users

static cl::opt< ReplaceExitVal > ReplaceExitValue("replexitval", cl::Hidden, cl::init(OnlyCheapRepl), cl::desc("Choose the strategy to replace exit value in IndVarSimplify"), cl::values(clEnumValN(NeverRepl, "never", "never replace exit value"), clEnumValN(OnlyCheapRepl, "cheap", "only replace exit value when the cost is cheap"), clEnumValN(UnusedIndVarInLoop, "unusedindvarinloop", "only replace exit value when it is an unused " "induction variable in the loop and has cheap replacement cost"), clEnumValN(NoHardUse, "noharduse", "only replace exit values when loop def likely dead"), clEnumValN(AlwaysRepl, "always", "always replace exit value whenever possible")))

static cl::opt< bool, true > HoistRuntimeChecks("hoist-runtime-checks", cl::Hidden, cl::desc("Hoist inner loop runtime memory checks to outer loop if possible"), cl::location(VectorizerParams::HoistRuntimeChecks), cl::init(true))

static std::optional< uint64_t > getEstimatedTripCount(BranchInst *ExitingBranch, Loop *L, uint64_t &OrigExitWeight)

Return the estimated trip count for any exiting branch which dominates the loop latch.

static bool hasHardUserWithinLoop(const Loop *L, const Instruction *I)

static const char * LLVMLoopDisableLICM

static PointerBounds expandBounds(const RuntimeCheckingPtrGroup *CG, Loop *TheLoop, Instruction *Loc, SCEVExpander &Exp, bool HoistRuntimeChecks)

Expand code for the lower and upper bound of the pointer group CG in TheLoop.

static bool canLoopBeDeleted(Loop *L, SmallVector< RewritePhi, 8 > &RewritePhiSet)

static const char * LLVMLoopDisableNonforced

static MDNode * createStringMetadata(Loop *TheLoop, StringRef Name, unsigned V)

Create MDNode for input string.

static BranchInst * getExpectedExitLoopLatchBranch(Loop *L)

Checks if L has an exiting latch branch.

static bool checkIsIndPhi(PHINode *Phi, Loop *L, ScalarEvolution *SE, InductionDescriptor &ID)

Checks if it is safe to call InductionDescriptor::isInductionPHI for Phi, and returns true if this Ph...

This file exposes an interface to building/using memory SSA to walk memory instructions using a use/d...

#define INITIALIZE_PASS_DEPENDENCY(depName)

This file provides a priority worklist.

This file contains the declarations for profiling metadata utility functions.

const SmallVectorImpl< MachineOperand > & Cond

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

This is the interface for a SCEV-based alias analysis.

This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...

This file implements a set that has insertion order iteration characteristics.

static cl::opt< unsigned > MSSAThreshold("simple-loop-unswitch-memoryssa-threshold", cl::desc("Max number of memory uses to explore during " "partial unswitching analysis"), cl::init(100), cl::Hidden)

This file defines the SmallPtrSet class.

This file defines the SmallVector class.

Virtual Register Rewriter

A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.

ModRefInfo getModRefInfo(const Instruction *I, const std::optional< MemoryLocation > &OptLoc)

Check whether or not an instruction may read or write the optionally specified memory location.

static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)

Returns the largest finite number in the given semantics.

Class for arbitrary precision integers.

static APInt getMaxValue(unsigned numBits)

Gets maximum unsigned value of APInt for specific bit width.

static APInt getSignedMaxValue(unsigned numBits)

Gets maximum signed value of APInt for a specific bit width.

static APInt getMinValue(unsigned numBits)

Gets minimum unsigned value of APInt for a specific bit width.

static APInt getSignedMinValue(unsigned numBits)

Gets minimum signed value of APInt for a specific bit width.

Represent the analysis usage information of a pass.

AnalysisUsage & addRequiredID(const void *ID)

AnalysisUsage & addPreservedID(const void *ID)

AnalysisUsage & addRequired()

AnalysisUsage & addPreserved()

Add the specified Pass class to the set of analyses preserved by this pass.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

Legacy wrapper pass to provide the BasicAAResult object.

LLVM Basic Block Representation.

iterator begin()

Instruction iterator methods.

InstListType::iterator iterator

Instruction iterators...

LLVMContext & getContext() const

Get the context in which this basic block lives.

const Instruction * getTerminator() const LLVM_READONLY

Returns the terminator instruction if the block is well formed or null if the block is not well forme...

Conditional or Unconditional Branch instruction.

unsigned getNumSuccessors() const

BasicBlock * getSuccessor(unsigned i) const

Predicate

This enumeration lists the possible predicates for CmpInst subclasses.

@ ICMP_SLT

signed less than

@ FCMP_OLT

0 1 0 0 True if ordered and less than

@ FCMP_OGT

0 0 1 0 True if ordered and greater than

@ ICMP_UGT

unsigned greater than

@ ICMP_SGT

signed greater than

@ ICMP_ULT

unsigned less than

static Constant * getIntrinsicIdentity(Intrinsic::ID, Type *Ty)

static Constant * getBinOpIdentity(unsigned Opcode, Type *Ty, bool AllowRHSConstant=false, bool NSZ=false)

Return the identity constant for a binary opcode.

static Constant * getInfinity(Type *Ty, bool Negative=false)

static Constant * getQNaN(Type *Ty, bool Negative=false, APInt *Payload=nullptr)

This is the shared class of boolean and integer constants.

static ConstantInt * getTrue(LLVMContext &Context)

static ConstantInt * getFalse(LLVMContext &Context)

int64_t getSExtValue() const

Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...

This class represents an Operation in the Expression.

uint64_t getNumOperands() const

Record of a variable value-assignment, aka a non instruction representation of the dbg....

Identifies a unique instance of a variable.

ValueT lookup(const_arg_type_t< KeyT > Val) const

lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...

std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)

iterator_range< iterator > children()

DomTreeNodeBase< NodeT > * getNode(const NodeT *BB) const

getNode - return the (Post)DominatorTree node for the specified basic block.

Legacy analysis pass which computes a DominatorTree.

Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.

bool isReachableFromEntry(const Use &U) const

Provide an overload for a Use.

static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)

Convenience struct for specifying and reasoning about fast-math flags.

bool noSignedZeros() const

void applyUpdates(ArrayRef< UpdateT > Updates)

Submit updates to all available trees.

Legacy wrapper pass to provide the GlobalsAAResult object.

Common base class shared among various IRBuilders.

CallInst * CreateFAddReduce(Value *Acc, Value *Src)

Create a sequential vector fadd reduction intrinsic of the source vector.

Value * CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")

UnreachableInst * CreateUnreachable()

Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)

Value * CreateFreeze(Value *V, const Twine &Name="")

CallInst * CreateOrReduce(Value *Src)

Create a vector int OR reduction intrinsic of the source vector.

CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")

Create a call to intrinsic ID with Args, mangled using Types.

ConstantInt * getInt32(uint32_t C)

Get a constant 32-bit value.

Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)

BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)

Create a conditional 'br Cond, TrueDest, FalseDest' instruction.

CallInst * CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, FMFSource FMFSource={}, const Twine &Name="")

Create a call to intrinsic ID with 1 operand which is mangled on its type.

FastMathFlags getFastMathFlags() const

Get the flags to be applied to created floating point ops.

Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")

Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")

CallInst * CreateIntMaxReduce(Value *Src, bool IsSigned=false)

Create a vector integer max reduction intrinsic of the source vector.

ConstantInt * getFalse()

Get the constant value for i1 false.

Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)

BranchInst * CreateBr(BasicBlock *Dest)

Create an unconditional 'br label X' instruction.

Value * CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name="")

void SetInsertPoint(BasicBlock *TheBB)

This specifies that created instructions should be appended to the end of the specified block.

CallInst * CreateFMulReduce(Value *Acc, Value *Src)

Create a sequential vector fmul reduction intrinsic of the source vector.

Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

A struct for saving information about induction variables.

static bool isInductionPHI(PHINode *Phi, const Loop *L, ScalarEvolution *SE, InductionDescriptor &D, const SCEV *Expr=nullptr, SmallVectorImpl< Instruction * > *CastsToIgnore=nullptr)

Returns true if Phi is an induction in the loop L.

InstSimplifyFolder - Use InstructionSimplify to fold operations to existing values.

unsigned getNumSuccessors() const LLVM_READONLY

Return the number of successors that this instruction has.

InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

bool mayHaveSideEffects() const LLVM_READONLY

Return true if the instruction may have side effects.

void setMetadata(unsigned KindID, MDNode *Node)

Set the metadata of the specified kind to the specified node.

void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())

Copy metadata from SrcInst to this instruction.

const DataLayout & getDataLayout() const

Get the data layout of the module this instruction belongs to.

This is an important class for using LLVM in a threaded context.

bool contains(const LoopT *L) const

Return true if the specified loop is contained within in this loop.

BlockT * getLoopLatch() const

If there is a single latch block for this loop, return it.

BlockT * getHeader() const

std::vector< Loop * >::const_iterator iterator

LoopT * getParentLoop() const

Return the parent loop if it exists or nullptr for top level loops.

void addTopLevelLoop(LoopT *New)

This adds the specified loop to the collection of top-level loops.

void removeBlock(BlockT *BB)

This method completely removes BB from all data structures, including all of the Loop objects it is n...

LoopT * AllocateLoop(ArgsTy &&...Args)

LoopT * removeLoop(iterator I)

This removes the specified top-level loop from this loop info object.

LoopT * getLoopFor(const BlockT *BB) const

Return the inner most loop that BB lives in.

void destroy(LoopT *L)

Destroy a loop that has been removed from the LoopInfo nest.

The legacy pass manager's analysis pass to compute loop information.

bool replacementPreservesLCSSAForm(Instruction *From, Value *To)

Returns true if replacing From with To everywhere is guaranteed to preserve LCSSA form.

void erase(Loop *L)

Update LoopInfo after removing the last backedge from a loop.

Represents a single loop in the control flow graph.

void setLoopID(MDNode *LoopID) const

Set the llvm.loop loop id metadata for this loop.

MDNode * getLoopID() const

Return the llvm.loop loop id metadata node for this loop if it is present.

MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight, bool IsExpected=false)

Return metadata containing two branch weights.

void replaceOperandWith(unsigned I, Metadata *New)

Replace a specific operand.

const MDOperand & getOperand(unsigned I) const

ArrayRef< MDOperand > operands() const

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

unsigned getNumOperands() const

Return number of MDNode operands.

LLVMContext & getContext() const

Tracking metadata reference owned by Metadata.

StringRef getString() const

static MDString * get(LLVMContext &Context, StringRef Str)

BasicBlock * getBlock() const

Representation for a specific memory location.

static MemoryLocation get(const LoadInst *LI)

Return a location with information about the memory reference by the given instruction.

Legacy analysis pass which computes MemorySSA.

Encapsulates MemorySSA, including all data associated with memory accesses.

void verifyMemorySSA(VerificationLevel=VerificationLevel::Fast) const

Verify that MemorySSA is self consistent (IE definitions dominate all uses, uses appear in the right ...

MemoryUseOrDef * getMemoryAccess(const Instruction *I) const

Given a memory Mod/Ref'ing instruction, get the MemorySSA access associated with it.

void setIncomingValue(unsigned i, Value *V)

BasicBlock * getIncomingBlock(unsigned i) const

Return incoming basic block number i.

Value * getIncomingValue(unsigned i) const

Return incoming value number x.

int getBasicBlockIndex(const BasicBlock *BB) const

Return the first index of the specified basic block in the value list for this PHI.

unsigned getNumIncomingValues() const

Return the number of incoming edges.

PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...

static PoisonValue * get(Type *T)

Static factory methods - Return an 'poison' object of the specified type.

bool insert(const T &X)

Insert a new element into the PriorityWorklist.

The RecurrenceDescriptor is used to identify recurrences variables in a loop.

static bool isAnyOfRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...

static bool isFindLastIVRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...

static bool isMinMaxRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is any min/max kind.

A global registry used in conjunction with static constructors to make pluggable components (like tar...

Legacy wrapper pass to provide the SCEVAAResult object.

This class uses information about analyze scalars to rewrite expressions in canonical form.

ScalarEvolution * getSE()

Value * expandCodeFor(const SCEV *SH, Type *Ty, BasicBlock::iterator I)

Insert code to directly compute the specified SCEV expression into the program.

This class represents an analyzed expression in the program.

Type * getType() const

Return the LLVM type of this SCEV expression.

The main scalar evolution driver.

bool isKnownNonNegative(const SCEV *S)

Test if the given expression is known to be non-negative.

bool isLoopEntryGuardedByCond(const Loop *L, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)

Test whether entry to the loop is protected by a conditional between LHS and RHS.

const SCEV * getSCEVAtScope(const SCEV *S, const Loop *L)

Return a SCEV expression for the specified value at the specified scope in the program.

const SCEV * getZero(Type *Ty)

Return a SCEV for the constant 0 of a specific type.

const SCEV * getConstant(ConstantInt *V)

const SCEV * getSCEV(Value *V)

Return a SCEV expression for the full generality of the specified expression.

void forgetLoop(const Loop *L)

This method should be called by the client when it has changed a loop in a way that may effect Scalar...

bool isLoopInvariant(const SCEV *S, const Loop *L)

Return true if the value of the given SCEV is unchanging in the specified loop.

LoopDisposition getLoopDisposition(const SCEV *S, const Loop *L)

Return the "disposition" of the given SCEV with respect to the given loop.

bool isSCEVable(Type *Ty) const

Test if values of the given type are analyzable within the SCEV framework.

void forgetValue(Value *V)

This method should be called by the client when it has changed a value in a way that may effect its v...

void forgetBlockAndLoopDispositions(Value *V=nullptr)

Called when the client has changed the disposition of values in a loop or block.

LoopDisposition

An enum describing the relationship between a SCEV and a loop.

@ LoopInvariant

The SCEV is loop-invariant.

bool isAvailableAtLoopEntry(const SCEV *S, const Loop *L)

Determine if the SCEV can be evaluated at loop's entry.

const SCEV * getExitCount(const Loop *L, const BasicBlock *ExitingBlock, ExitCountKind Kind=Exact)

Return the number of times the backedge executes before the given exit would be taken; if not exactly...

const SCEV * applyLoopGuards(const SCEV *Expr, const Loop *L)

Try to apply information from loop guards for L to Expr.

This class represents the LLVM 'select' instruction.

Implements a dense probed hash-table based set with some number of buckets stored inline.

A version of PriorityWorklist that selects small size optimized data structures for the vector and ma...

std::pair< iterator, bool > insert(PtrType Ptr)

Inserts Ptr if and only if there is no element in the container equal to Ptr.

bool contains(ConstPtrType Ptr) const

SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.

A SetVector that performs no allocations if smaller than a certain size.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

reference emplace_back(ArgTypes &&... Args)

void append(ItTy in_start, ItTy in_end)

Add the specified range to the end of the SmallVector.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

bool starts_with(StringRef Prefix) const

Check if this string starts with the given Prefix.

Provides information about what library functions are available for the current target.

This pass provides access to the codegen interfaces that are needed for IR-level transformations.

Value handle that tracks a Value across RAUW.

The instances of the Type class are immutable: once they are created, they are never changed.

const fltSemantics & getFltSemantics() const

bool isIntOrIntVectorTy() const

Return true if this is an integer type or a vector of integer types.

unsigned getScalarSizeInBits() const LLVM_READONLY

If this is a vector type, return the getPrimitiveSizeInBits value for the element type.

static IntegerType * getInt32Ty(LLVMContext &C)

bool isIntegerTy() const

True if this is an instance of IntegerType.

A Use represents the edge between a Value definition and its users.

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

void replaceAllUsesWith(Value *V)

Change all uses of this to point to a new Value.

iterator_range< user_iterator > users()

LLVMContext & getContext() const

All values hold a context through their type.

iterator_range< use_iterator > uses()

StringRef getName() const

Return a constant reference to the value's name.

Value * createSimpleReduction(Intrinsic::ID RdxID, Type *ValTy, ArrayRef< Value * > VecOpArray, const Twine &Name=Twine())

Emit a VP reduction intrinsic call for recurrence kind.

std::pair< iterator, bool > insert(const ValueT &V)

An efficient, type-erasing, non-owning reference to a callable.

const ParentTy * getParent() const

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

This is an optimization pass for GlobalISel generic memory operations.

auto drop_begin(T &&RangeOrContainer, size_t N=1)

Return a range covering RangeOrContainer with the first N elements excluded.

Value * createSimpleReduction(IRBuilderBase &B, Value *Src, RecurKind RdxKind)

Create a reduction of the given vector.

std::optional< ElementCount > getOptionalElementCountLoopAttribute(const Loop *TheLoop)

Find a combination of metadata ("llvm.loop.vectorize.width" and "llvm.loop.vectorize....

@ Low

Lower the current thread's priority such that it does not affect foreground tasks significantly.

Value * addRuntimeChecks(Instruction *Loc, Loop *TheLoop, const SmallVectorImpl< RuntimePointerCheck > &PointerChecks, SCEVExpander &Expander, bool HoistRuntimeChecks=false)

Add code that checks at runtime if the accessed arrays in PointerChecks overlap.

auto find(R &&Range, const T &Val)

Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.

std::optional< unsigned > getLoopEstimatedTripCount(Loop *L, unsigned *EstimatedLoopInvocationWeight=nullptr)

Returns a loop's estimated trip count based on branch weight metadata.

bool all_of(R &&range, UnaryPredicate P)

Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.

Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)

Returns the min/max intrinsic used when expanding a min/max reduction.

bool getBooleanLoopAttribute(const Loop *TheLoop, StringRef Name)

Returns true if Name is applied to TheLoop and enabled.

std::pair< const RuntimeCheckingPtrGroup *, const RuntimeCheckingPtrGroup * > RuntimePointerCheck

A memcheck which made up of a pair of grouped pointers.

detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)

bool isKnownNonPositiveInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE)

Returns true if we can prove that S is defined and always non-positive in loop L.

std::optional< bool > getOptionalBoolLoopAttribute(const Loop *TheLoop, StringRef Name)

void appendReversedLoopsToWorklist(RangeT &&, SmallPriorityWorklist< Loop *, 4 > &)

Utility that implements appending of loops onto a worklist given a range.

auto successors(const MachineBasicBlock *BB)

void initializeLoopPassPass(PassRegistry &)

Manually defined generic "LoopPass" dependency initialization.

bool formLCSSARecursively(Loop &L, const DominatorTree &DT, const LoopInfo *LI, ScalarEvolution *SE)

Put a loop nest into LCSSA form.

Value * getReductionIdentity(Intrinsic::ID RdxID, Type *Ty, FastMathFlags FMF)

Given information about an @llvm.vector.reduce.

std::optional< MDNode * > makeFollowupLoopID(MDNode *OrigLoopID, ArrayRef< StringRef > FollowupAttrs, const char *InheritOptionsAttrsPrefix="", bool AlwaysNew=false)

Create a new loop identifier for a loop created from a loop transformation.

unsigned getArithmeticReductionInstruction(Intrinsic::ID RdxID)

Returns the arithmetic instruction opcode used when expanding a reduction.

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

Value * createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left, Value *Right)

Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.

SmallVector< BasicBlock *, 16 > collectChildrenInLoop(DominatorTree *DT, DomTreeNode *N, const Loop *CurLoop)

Does a BFS from a given node to all of its children inside a given loop.

void addStringMetadataToLoop(Loop *TheLoop, const char *MDString, unsigned V=0)

Set input string into loop metadata by keeping other values intact.

bool cannotBeMaxInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE, bool Signed)

Returns true if S is defined and never is equal to signed/unsigned max.

Value * createAnyOfReduction(IRBuilderBase &B, Value *Src, const RecurrenceDescriptor &Desc, PHINode *OrigPhi)

Create a reduction of the given vector Src for a reduction of the kind RecurKind::IAnyOf or RecurKind...

constexpr T divideNearest(U Numerator, V Denominator)

Returns (Numerator / Denominator) rounded by round-half-up.

TransformationMode hasVectorizeTransformation(const Loop *L)

OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)

Wrapper function around std::transform to apply a function to a range and store the result elsewhere.

bool any_of(R &&range, UnaryPredicate P)

Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.

bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)

Return true if the result produced by the instruction is not used, and the instruction will return.

SmallVector< Instruction *, 8 > findDefsUsedOutsideOfLoop(Loop *L)

Returns the instructions that use values defined in the loop.

auto reverse(ContainerTy &&C)

constexpr Intrinsic::ID getReductionIntrinsicID(RecurKind RK)

Returns the llvm.vector.reduce intrinsic that corresponds to the recurrence kind.

bool isMustProgress(const Loop *L)

Return true if this loop can be assumed to make progress.

constexpr bool isPowerOf2_32(uint32_t Value)

Return true if the argument is a power of two > 0.

bool isModSet(const ModRefInfo MRI)

TransformationMode hasUnrollAndJamTransformation(const Loop *L)

void deleteDeadLoop(Loop *L, DominatorTree *DT, ScalarEvolution *SE, LoopInfo *LI, MemorySSA *MSSA=nullptr)

This function deletes dead loops.

raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

bool hasDisableAllTransformsHint(const Loop *L)

Look for the loop attribute that disables all transformation heuristic.

Value * createOrderedReduction(IRBuilderBase &B, const RecurrenceDescriptor &Desc, Value *Src, Value *Start)

Create an ordered reduction intrinsic using the given recurrence descriptor Desc.

cl::opt< unsigned > SCEVCheapExpansionBudget

Value * getShuffleReduction(IRBuilderBase &Builder, Value *Src, unsigned Op, TargetTransformInfo::ReductionShuffle RS, RecurKind MinMaxKind=RecurKind::None)

Generates a vector reduction using shufflevectors to reduce the value.

Value * createReduction(IRBuilderBase &B, const RecurrenceDescriptor &Desc, Value *Src, PHINode *OrigPhi=nullptr)

Create a generic reduction using a recurrence descriptor Desc Fast-math-flags are propagated using th...

TransformationMode hasUnrollTransformation(const Loop *L)

TransformationMode hasDistributeTransformation(const Loop *L)

void breakLoopBackedge(Loop *L, DominatorTree &DT, ScalarEvolution &SE, LoopInfo &LI, MemorySSA *MSSA)

Remove the backedge of the specified loop.

void getLoopAnalysisUsage(AnalysisUsage &AU)

Helper to consistently add the set of standard passes to a loop pass's AnalysisUsage.

void propagateIRFlags(Value *I, ArrayRef< Value * > VL, Value *OpValue=nullptr, bool IncludeWrapFlags=true)

Get the intersection (logical and) of all of the potential IR flags of each scalar operation (VL) tha...

bool isKnownPositiveInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE)

Returns true if we can prove that S is defined and always positive in loop L.

unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)

Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...

RNSuccIterator< NodeRef, BlockT, RegionT > succ_begin(NodeRef Node)

std::optional< int > getOptionalIntLoopAttribute(const Loop *TheLoop, StringRef Name)

Find named metadata for a loop with an integer value.

BasicBlock * SplitBlockPredecessors(BasicBlock *BB, ArrayRef< BasicBlock * > Preds, const char *Suffix, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, bool PreserveLCSSA=false)

This method introduces at least one new basic block into the function and moves some of the predecess...

@ First

Helpers to iterate all locations in the MemoryEffectsBase class.

CmpInst::Predicate getMinMaxReductionPredicate(RecurKind RK)

Returns the comparison predicate used when expanding a min/max reduction.

TransformationMode hasLICMVersioningTransformation(const Loop *L)

bool VerifyMemorySSA

Enables verification of MemorySSA.

Value * createFindLastIVReduction(IRBuilderBase &B, Value *Src, const RecurrenceDescriptor &Desc)

Create a reduction of the given vector Src for a reduction of the kind RecurKind::IFindLastIV or Recu...

TransformationMode

The mode sets how eager a transformation should be applied.

@ TM_Unspecified

The pass can use heuristics to determine whether a transformation should be applied.

@ TM_SuppressedByUser

The transformation must not be applied.

@ TM_ForcedByUser

The transformation was directed by the user, e.g.

@ TM_Disable

The transformation should not be applied.

@ TM_Enable

The transformation should be applied without considering a cost model.

RNSuccIterator< NodeRef, BlockT, RegionT > succ_end(NodeRef Node)

bool hasDisableLICMTransformsHint(const Loop *L)

Look for the loop attribute that disables the LICM transformation heuristics.

RecurKind

These are the kinds of recurrences that we support.

bool setLoopEstimatedTripCount(Loop *L, unsigned EstimatedTripCount, unsigned EstimatedLoopInvocationWeight)

Set a loop's branch weight metadata to reflect that loop has EstimatedTripCount iterations and Estima...

Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF)

Given information about an recurrence kind, return the identity for the @llvm.vector....

void setProfileInfoAfterUnrolling(Loop *OrigLoop, Loop *UnrolledLoop, Loop *RemainderLoop, uint64_t UF)

Set weights for UnrolledLoop and RemainderLoop based on weights for OrigLoop and the following distri...

bool formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI, MemorySSAUpdater *MSSAU, bool PreserveLCSSA)

Ensure that all exit blocks of the loop are dedicated exits.

DWARFExpression::Operation Op

void appendLoopsToWorklist(RangeT &&, SmallPriorityWorklist< Loop *, 4 > &)

Utility that implements appending of loops onto a worklist given a range.

bool isKnownNegativeInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE)

Returns true if we can prove that S is defined and always negative in loop L.

constexpr unsigned BitWidth

bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)

Extract branch weights from MD_prof metadata.

bool hasIterationCountInvariantInParent(Loop *L, ScalarEvolution &SE)

Check inner loop (L) backedge count is known to be invariant on all iterations of its outer loop.

bool isAlmostDeadIV(PHINode *IV, BasicBlock *LatchBlock, Value *Cond)

Return true if the induction variable IV in a Loop whose latch is LatchBlock would become dead if the...

auto predecessors(const MachineBasicBlock *BB)

int rewriteLoopExitValues(Loop *L, LoopInfo *LI, TargetLibraryInfo *TLI, ScalarEvolution *SE, const TargetTransformInfo *TTI, SCEVExpander &Rewriter, DominatorTree *DT, ReplaceExitVal ReplaceExitValue, SmallVector< WeakTrackingVH, 16 > &DeadInsts)

If the final value of any expressions that are recurrent in the loop can be computed,...

Value * addDiffRuntimeChecks(Instruction *Loc, ArrayRef< PointerDiffInfo > Checks, SCEVExpander &Expander, function_ref< Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC)

RecurKind getMinMaxReductionRecurKind(Intrinsic::ID RdxID)

Returns the recurence kind used when expanding a min/max reduction.

BasicBlock * SplitEdge(BasicBlock *From, BasicBlock *To, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")

Split the edge connecting the specified blocks, and return the newly created basic block between From...

std::optional< IVConditionInfo > hasPartialIVCondition(const Loop &L, unsigned MSSAThreshold, const MemorySSA &MSSA, AAResults &AA)

Check if the loop header has a conditional branch that is not loop-invariant, because it involves loa...

static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)

Filter the DbgRecord range to DbgVariableRecord types only and downcast.

bool cannotBeMinInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE, bool Signed)

Returns true if S is defined and never is equal to signed/unsigned min.

bool isKnownNonNegativeInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE)

Returns true if we can prove that S is defined and always non-negative in loop L.

Value * getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src, unsigned Op, RecurKind MinMaxKind=RecurKind::None)

Generates an ordered vector reduction using extracts to reduce the value.

MDNode * findOptionMDForLoopID(MDNode *LoopID, StringRef Name)

Find and return the loop attribute node for the attribute Name in LoopID.

Loop * cloneLoop(Loop *L, Loop *PL, ValueToValueMapTy &VM, LoopInfo *LI, LPPassManager *LPM)

Recursively clone the specified loop and all of its children, mapping the blocks with the specified m...

void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)

Implement std::swap in terms of BitVector swap.

IR Values for the lower and upper bounds of a pointer evolution.

TrackingVH< Value > Start

RewritePhi(PHINode *P, unsigned I, const SCEV *Val, Instruction *ExpansionPt, bool H)

const SCEV * ExpansionSCEV

Instruction * ExpansionPoint

Description of the encoding of one expression Op.

Struct to hold information about a partially invariant condition.

Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...

unsigned AddressSpace

Address space of the involved pointers.

bool NeedsFreeze

Whether the pointer needs to be frozen after expansion, e.g.

const SCEV * High

The SCEV expression which represents the upper bound of all the pointers in this group.

const SCEV * Low

The SCEV expression which represents the lower bound of all the pointers in this group.