LoopUtils.cpp Source File (original) (raw)

50using namespace llvm;

53#define DEBUG_TYPE "loop-utils"

60 bool PreserveLCSSA) {

66 auto RewriteExit = [&](BasicBlock *BB) {

68 "Must start with an empty predecessors list!");

73 bool IsDedicatedExit = true;

75 if (L->contains(PredBB)) {

78 return false;

80 InLoopPredecessors.push_back(PredBB);

81 } else {

82 IsDedicatedExit = false;

83 }

85 assert(!InLoopPredecessors.empty() && "Must have *some* loop predecessor!");

88 if (IsDedicatedExit)

89 return false;

92 BB, InLoopPredecessors, ".loopexit", DT, LI, MSSAU, PreserveLCSSA);

94 if (!NewExitBB)

96 dbgs() << "WARNING: Can't create a dedicated exit block for loop: "

97 << *L << "\n");

98 else

99 LLVM_DEBUG(dbgs() << "LoopSimplify: Creating dedicated exit block "

100 << NewExitBB->getName() << "\n");

101 return true;

102 };

103

104

105

107 for (auto *BB : L->blocks())

108 for (auto *SuccBB : successors(BB)) {

109

110 if (L->contains(SuccBB))

111 continue;

112

113

114 if (!Visited.insert(SuccBB).second)

115 continue;

116

117 Changed |= RewriteExit(SuccBB);

118 }

119

121}

122

123

126

127 for (auto *Block : L->getBlocks())

128

129

130 for (auto &Inst : *Block) {

131 auto Users = Inst.users();

134 return !L->contains(Use->getParent());

135 }))

137 }

138

139 return UsedOutside;

140}

141

143

144

145

150

151

152

159

160

163

164

165

166

167

168

169

170

178

179

180}

181

182

183

184

185

186

187

188

189

202

203

211

212

213

214

216 unsigned V) {

218

220 if (LoopID) {

221 for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {

223

224 if (Node->getNumOperands() == 2) {

226 if (S && S->getString() == StringMD) {

230

231 return;

232

233

234 continue;

235 }

236 }

238 }

239 }

240

242

245

248}

249

250std::optional

252 std::optional Width =

254

255 if (Width) {

257 TheLoop, "llvm.loop.vectorize.scalable.enable");

259 }

260

261 return std::nullopt;

262}

263

266 const char *InheritOptionsExceptPrefix, bool AlwaysNew) {

267 if (!OrigLoopID) {

268 if (AlwaysNew)

269 return nullptr;

270 return std::nullopt;

271 }

272

274

275 bool InheritAllAttrs = !InheritOptionsExceptPrefix;

276 bool InheritSomeAttrs =

277 InheritOptionsExceptPrefix && InheritOptionsExceptPrefix[0] != '\0';

280

282 if (InheritAllAttrs || InheritSomeAttrs) {

285

286 auto InheritThisAttribute = [InheritSomeAttrs,

287 InheritOptionsExceptPrefix](MDNode *Op) {

288 if (!InheritSomeAttrs)

289 return false;

290

291

292 if (Op->getNumOperands() == 0)

293 return true;

294 Metadata *NameMD = Op->getOperand(0).get();

296 return true;

298

299

300 return !AttrName.starts_with(InheritOptionsExceptPrefix);

301 };

302

303 if (InheritThisAttribute(Op))

305 else

307 }

308 } else {

309

311 }

312

313 bool HasAnyFollowup = false;

314 for (StringRef OptionName : FollowupOptions) {

316 if (!FollowupNode)

317 continue;

318

319 HasAnyFollowup = true;

323 }

324 }

325

326

327

328 if (!AlwaysNew && !HasAnyFollowup)

329 return std::nullopt;

330

331

332 if (!AlwaysNew && Changed )

333 return OrigLoopID;

334

335

336 if (MDs.size() == 1)

337 return nullptr;

338

339

342 return FollowupLoopID;

343}

344

348

352

356

357 std::optional Count =

361

364

367

370

372}

373

377

378 std::optional Count =

382

385

388

390}

391

393 std::optional Enable =

395

398

399 std::optional VectorizeWidth =

401 std::optional InterleaveCount =

403

404

405

406 if (Enable == true && VectorizeWidth && VectorizeWidth->isScalar() &&

407 InterleaveCount == 1)

409

412

415

416 if ((VectorizeWidth && VectorizeWidth->isScalar()) && InterleaveCount == 1)

418

419 if ((VectorizeWidth && VectorizeWidth->isVector()) || InterleaveCount > 1)

421

424

426}

427

437

447

448

449

452 const Loop *CurLoop) {

454 auto AddRegionToWorklist = [&](DomTreeNode *DTN) {

455

458 Worklist.push_back(DTN->getBlock());

459 };

460

461 AddRegionToWorklist(N);

462

463 for (size_t I = 0; I < Worklist.size(); I++) {

465 AddRegionToWorklist(Child);

466 }

467

468 return Worklist;

469}

470

473 assert(LatchIdx != -1 && "LatchBlock is not a case in this PHINode");

475

477 if (U != Cond && U != IncV) return false;

478

480 if (U != Cond && U != PN) return false;

481 return true;

482}

483

484

487 assert((!DT || L->isLCSSAForm(*DT)) && "Expected LCSSA!");

488 auto *Preheader = L->getLoopPreheader();

489 assert(Preheader && "Preheader should exist!");

490

491 std::unique_ptr MSSAU;

492 if (MSSA)

493 MSSAU = std::make_unique(MSSA);

494

495

496

497

498

499

500

501

502

503

504 if (SE) {

507 }

508

509 Instruction *OldTerm = Preheader->getTerminator();

511 "Preheader must end with a side-effect-free terminator");

513 "Preheader must have a single successor");

514

515

516

517

518

519

520

521

522

523

524

525

526

527

528

529

530

531

532

533

534

535

536

537

538

540

541 auto *ExitBlock = L->getUniqueExitBlock();

542 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);

543 if (ExitBlock) {

544 assert(ExitBlock && "Should have a unique exit block!");

545 assert(L->hasDedicatedExits() && "Loop should have dedicated exits!");

546

547 Builder.CreateCondBr(Builder.getFalse(), L->getHeader(), ExitBlock);

548

550

551

552

553 for (PHINode &P : ExitBlock->phis()) {

554

555

556

557 int PredIndex = 0;

558 P.setIncomingBlock(PredIndex, Preheader);

559

560

561

562 P.removeIncomingValueIf([](unsigned Idx) { return Idx != 0; },

563 false);

564

565 assert((P.getNumIncomingValues() == 1 &&

566 P.getIncomingBlock(PredIndex) == Preheader) &&

567 "Should have exactly one value and that's from the preheader!");

568 }

569

570 if (DT) {

572 if (MSSA) {

574 *DT);

577 }

578 }

579

580

581 Builder.SetInsertPoint(Preheader->getTerminator());

582 Builder.CreateBr(ExitBlock);

583

584 Preheader->getTerminator()->eraseFromParent();

585 } else {

586 assert(L->hasNoExitBlocks() &&

587 "Loop should have either zero or one exit blocks.");

588

589 Builder.SetInsertPoint(OldTerm);

590 Builder.CreateUnreachable();

591 Preheader->getTerminator()->eraseFromParent();

592 }

593

594 if (DT) {

596 if (MSSA) {

598 *DT);

600 L->block_end());

601 MSSAU->removeBlocks(DeadBlockSet);

604 }

605 }

606

607

610

611 if (ExitBlock) {

612

613

614

615

616

617

618

619

620 for (auto *Block : L->blocks())

625 if (L->contains(Usr->getParent()))

626 continue;

627

628

629 if (DT)

631 "Unexpected user in reachable block");

633 }

634

635

636

637

641 DVR.getDebugLoc().get());

642 if (!DeadDebugSet.insert(Key).second)

643 continue;

644

645 DVR.removeFromParent();

646 DeadDbgVariableRecords.push_back(&DVR);

647 }

648 }

649

650

651

652

653

654

655

656 DIBuilder DIB(*ExitBlock->getModule());

658 ExitBlock->getFirstInsertionPt();

659 assert(InsertDbgValueBefore != ExitBlock->end() &&

660 "There should be a non-PHI instruction in exit block, else these "

661 "instructions will have no parent.");

662

663

664

665

666

668 ExitBlock->insertDbgRecordBefore(DVR, InsertDbgValueBefore);

669 }

670

671

672

673 for (auto *Block : L->blocks())

674 Block->dropAllReferences();

675

678

679 if (LI) {

680

681

682

683

685 BB->eraseFromParent();

686

687

688

689

693

694

695

696

697

698 if (Loop *ParentLoop = L->getParentLoop()) {

700 assert(I != ParentLoop->end() && "Couldn't find loop");

701 ParentLoop->removeChildLoop(I);

702 } else {

704 assert(I != LI->end() && "Couldn't find loop");

706 }

708 }

709}

710

713 auto *Latch = L->getLoopLatch();

714 assert(Latch && "multiple latches not yet supported");

715 auto *Header = L->getHeader();

716 Loop *OutermostLoop = L->getOutermostLoop();

717

720

721 std::unique_ptr MSSAU;

722 if (MSSA)

723 MSSAU = std::make_unique(MSSA);

724

725

726

727 [&]() -> void {

729 if (!BI->isConditional()) {

730 DomTreeUpdater DTU(&DT, DomTreeUpdater::UpdateStrategy::Eager);

732 MSSAU.get());

733 return;

734 }

735

736

737

738 if (L->isLoopExiting(Latch)) {

739

740

741

742

743 const unsigned ExitIdx = L->contains(BI->getSuccessor(0)) ? 1 : 0;

744 BasicBlock *ExitBB = BI->getSuccessor(ExitIdx);

745

746 DomTreeUpdater DTU(&DT, DomTreeUpdater::UpdateStrategy::Eager);

747 Header->removePredecessor(Latch, true);

748

750 auto *NewBI = Builder.CreateBr(ExitBB);

751

752

753 NewBI->copyMetadata(*BI, {LLVMContext::MD_dbg,

754 LLVMContext::MD_annotation});

755

756 BI->eraseFromParent();

758 if (MSSA)

760 return;

761 }

762 }

763

764

765

766

767 auto *BackedgeBB = SplitEdge(Latch, Header, &DT, &LI, MSSAU.get());

768

769 DomTreeUpdater DTU(&DT, DomTreeUpdater::UpdateStrategy::Eager);

771 true, &DTU, MSSAU.get());

772 }();

773

774

775

777

778

779

780

781

782

783 if (OutermostLoop != L)

785}

786

787

788

789

790

792 BasicBlock *Latch = L->getLoopLatch();

793 if (!Latch)

794 return nullptr;

795

797 if (!LatchBR || LatchBR->getNumSuccessors() != 2 || !L->isLoopExiting(Latch))

798 return nullptr;

799

801 LatchBR->getSuccessor(1) == L->getHeader()) &&

802 "At least one edge out of the latch must go to the header");

803

804 return LatchBR;

805}

806

811

812#ifndef NDEBUG

814 OS << "function ";

815 D.L->getHeader()->getParent()->printAsOperand(OS, false);

816 return OS << " " << *D.L;

817}

818#endif

819

821

822

823

824

826 if (!ExitingBranch) {

827 LLVM_DEBUG(dbgs() << "estimateLoopTripCount: Failed to find exiting "

828 << "latch branch of required form in " << DbgLoop(L)

829 << "\n");

830 return std::nullopt;

831 }

832

833

834

835

836 uint64_t LoopWeight, ExitWeight;

838 LLVM_DEBUG(dbgs() << "estimateLoopTripCount: Failed to extract branch "

839 << "weights for " << DbgLoop(L) << "\n");

840 return std::nullopt;

841 }

842

843 if (L->contains(ExitingBranch->getSuccessor(1)))

844 std::swap(LoopWeight, ExitWeight);

845

846 if (!ExitWeight) {

847

848 LLVM_DEBUG(dbgs() << "estimateLoopTripCount: Failed because of zero exit "

849 << "probability for " << DbgLoop(L) << "\n");

850 return std::nullopt;

851 }

852

853

854

856

857

858 if (ExitCount >= std::numeric_limits::max())

859 return std::numeric_limits::max();

860

861

863 LLVM_DEBUG(dbgs() << "estimateLoopTripCount: Estimated trip count of " << TC

864 << " for " << DbgLoop(L) << "\n");

865 return TC;

866}

867

868std::optional

870 unsigned *EstimatedLoopInvocationWeight) {

871

872

873

874

875

876

877

878

879

880

881

882

883

884

885

886

887

888

889

890

891

892

894 if (!ExitingBranch)

895 return std::nullopt;

896

897

898

899

900

901

902

903 if (EstimatedLoopInvocationWeight) {

904 uint64_t LoopWeight = 0, ExitWeight = 0;

906 return std::nullopt;

907 if (L->contains(ExitingBranch->getSuccessor(1)))

908 std::swap(LoopWeight, ExitWeight);

909 if (!ExitWeight)

910 return std::nullopt;

911 *EstimatedLoopInvocationWeight = ExitWeight;

912 }

913

914

915

916

917

918

919

920

921

922

923

924

925

926

927

928

932 << "count of " << *TC

933 << (*TC == 0 ? " (returning std::nullopt)" : "")

934 << " for " << DbgLoop(L) << "\n");

935 return *TC == 0 ? std::nullopt : std::optional(*TC);

936 }

937

938

940}

941

943 Loop *L, unsigned EstimatedTripCount,

944 std::optional EstimatedloopInvocationWeight) {

945

946

947

948

949

951 if (!LatchBranch)

952 return false;

953

954

956

957

958

959

960

961

962

963

964 if (!EstimatedloopInvocationWeight)

965 return true;

966

967

968 unsigned LatchExitWeight = 0;

969 unsigned BackedgeTakenWeight = 0;

970

971 if (EstimatedTripCount != 0) {

972 LatchExitWeight = *EstimatedloopInvocationWeight;

973 BackedgeTakenWeight = (EstimatedTripCount - 1) * LatchExitWeight;

974 }

975

976

977 if (LatchBranch->getSuccessor(0) != L->getHeader())

978 std::swap(BackedgeTakenWeight, LatchExitWeight);

979

980

981 setBranchWeights(*LatchBranch, {BackedgeTakenWeight, LatchExitWeight},

982 false);

983

984 return true;

985}

986

989 if (!LatchBranch)

991 bool FirstTargetIsLoop = LatchBranch->getSuccessor(0) == L->getHeader();

993}

994

997 if (!LatchBranch)

998 return false;

999 bool FirstTargetIsLoop = LatchBranch->getSuccessor(0) == L->getHeader();

1001}

1002

1004 bool ForFirstTarget) {

1005 if (B->getNumSuccessors() != 2)

1010 uint64_t Denominator = Weight0 + Weight1;

1011 if (Denominator == 0)

1013 if (!ForFirstTarget)

1016}

1017

1019 bool ForFirstTarget) {

1020 if (B->getNumSuccessors() != 2)

1021 return false;

1024 if (!ForFirstTarget)

1027 false);

1028 return true;

1029}

1030

1034 if (!OuterL)

1035 return true;

1036

1037

1039 const SCEV *InnerLoopBECountSC = SE.getExitCount(InnerLoop, InnerLoopLatch);

1042 return false;

1043

1044

1048 return false;

1049

1050 return true;

1051}

1052

1054 switch (RK) {

1055 default:

1060 return Intrinsic::vector_reduce_add;

1062 return Intrinsic::vector_reduce_mul;

1064 return Intrinsic::vector_reduce_and;

1066 return Intrinsic::vector_reduce_or;

1068 return Intrinsic::vector_reduce_xor;

1071 return Intrinsic::vector_reduce_fadd;

1073 return Intrinsic::vector_reduce_fmul;

1075 return Intrinsic::vector_reduce_smax;

1077 return Intrinsic::vector_reduce_smin;

1079 return Intrinsic::vector_reduce_umax;

1081 return Intrinsic::vector_reduce_umin;

1084 return Intrinsic::vector_reduce_fmax;

1087 return Intrinsic::vector_reduce_fmin;

1089 return Intrinsic::vector_reduce_fmaximum;

1091 return Intrinsic::vector_reduce_fminimum;

1093 return Intrinsic::vector_reduce_fmax;

1095 return Intrinsic::vector_reduce_fmin;

1096 }

1097}

1098

1100 switch (IID) {

1101 default:

1103 case Intrinsic::umin:

1104 return Intrinsic::vector_reduce_umin;

1105 case Intrinsic::umax:

1106 return Intrinsic::vector_reduce_umax;

1107 case Intrinsic::smin:

1108 return Intrinsic::vector_reduce_smin;

1109 case Intrinsic::smax:

1110 return Intrinsic::vector_reduce_smax;

1111 }

1112}

1113

1114

1116 switch (RdxID) {

1117 case Intrinsic::vector_reduce_fadd:

1118 return Instruction::FAdd;

1119 case Intrinsic::vector_reduce_fmul:

1120 return Instruction::FMul;

1121 case Intrinsic::vector_reduce_add:

1122 return Instruction::Add;

1123 case Intrinsic::vector_reduce_mul:

1124 return Instruction::Mul;

1125 case Intrinsic::vector_reduce_and:

1126 return Instruction::And;

1127 case Intrinsic::vector_reduce_or:

1128 return Instruction::Or;

1129 case Intrinsic::vector_reduce_xor:

1130 return Instruction::Xor;

1131 case Intrinsic::vector_reduce_smax:

1132 case Intrinsic::vector_reduce_smin:

1133 case Intrinsic::vector_reduce_umax:

1134 case Intrinsic::vector_reduce_umin:

1135 return Instruction::ICmp;

1136 case Intrinsic::vector_reduce_fmax:

1137 case Intrinsic::vector_reduce_fmin:

1138 return Instruction::FCmp;

1139 default:

1141 }

1142}

1143

1144

1146 switch (Opc) {

1147 default:

1148 break;

1149 case Instruction::Add:

1150 return Intrinsic::vector_reduce_add;

1151 case Instruction::Mul:

1152 return Intrinsic::vector_reduce_mul;

1153 case Instruction::And:

1154 return Intrinsic::vector_reduce_and;

1155 case Instruction::Or:

1156 return Intrinsic::vector_reduce_or;

1157 case Instruction::Xor:

1158 return Intrinsic::vector_reduce_xor;

1159 }

1161}

1162

1164 switch (RdxID) {

1165 default:

1167 case Intrinsic::vector_reduce_umin:

1168 return Intrinsic::umin;

1169 case Intrinsic::vector_reduce_umax:

1170 return Intrinsic::umax;

1171 case Intrinsic::vector_reduce_smin:

1172 return Intrinsic::smin;

1173 case Intrinsic::vector_reduce_smax:

1174 return Intrinsic::smax;

1175 case Intrinsic::vector_reduce_fmin:

1176 return Intrinsic::minnum;

1177 case Intrinsic::vector_reduce_fmax:

1178 return Intrinsic::maxnum;

1179 case Intrinsic::vector_reduce_fminimum:

1180 return Intrinsic::minimum;

1181 case Intrinsic::vector_reduce_fmaximum:

1182 return Intrinsic::maximum;

1183 }

1184}

1185

1187 switch (RK) {

1188 default:

1191 return Intrinsic::umin;

1193 return Intrinsic::umax;

1195 return Intrinsic::smin;

1197 return Intrinsic::smax;

1200 return Intrinsic::minnum;

1203 return Intrinsic::maxnum;

1205 return Intrinsic::minimum;

1207 return Intrinsic::maximum;

1209 return Intrinsic::minimumnum;

1211 return Intrinsic::maximumnum;

1212 }

1213}

1214

1216 switch (RdxID) {

1217 case Intrinsic::vector_reduce_smax:

1219 case Intrinsic::vector_reduce_smin:

1221 case Intrinsic::vector_reduce_umax:

1223 case Intrinsic::vector_reduce_umin:

1225 case Intrinsic::vector_reduce_fmax:

1227 case Intrinsic::vector_reduce_fmin:

1229 default:

1231 }

1232}

1233

1235 switch (RK) {

1236 default:

1250

1251

1252

1253 }

1254}

1255

1259 if (Ty->isIntOrIntVectorTy() ||

1264 return Builder.CreateIntrinsic(Ty, Id, {Left, Right}, nullptr,

1265 "rdx.minmax");

1266 }

1268 Value *Cmp = Builder.CreateCmp(Pred, Left, Right, "rdx.minmax.cmp");

1269 Value *Select = Builder.CreateSelect(Cmp, Left, Right, "rdx.minmax.select");

1271}

1272

1273

1277

1278

1279

1280 Value *Result = Acc;

1281 for (unsigned ExtractIdx = 0; ExtractIdx != VF; ++ExtractIdx) {

1283 Builder.CreateExtractElement(Src, Builder.getInt32(ExtractIdx));

1284

1285 if (Op != Instruction::ICmp && Op != Instruction::FCmp) {

1287 "bin.rdx");

1288 } else {

1290 "Invalid min/max");

1291 Result = createMinMaxOp(Builder, RdxKind, Result, Ext);

1292 }

1293 }

1294

1295 return Result;

1296}

1297

1298

1300 unsigned Op,

1304

1305

1306

1308 "Reduction emission only supported for pow2 vectors!");

1309

1310

1311

1312

1313

1314

1315

1316 auto BuildShuffledOp = [&Builder, &Op,

1318 Value *&TmpVec) -> void {

1319 Value *Shuf = Builder.CreateShuffleVector(TmpVec, ShuffleMask, "rdx.shuf");

1320 if (Op != Instruction::ICmp && Op != Instruction::FCmp) {

1322 "bin.rdx");

1323 } else {

1325 "Invalid min/max");

1326 TmpVec = createMinMaxOp(Builder, RdxKind, TmpVec, Shuf);

1327 }

1328 };

1329

1330 Value *TmpVec = Src;

1333 for (unsigned stride = 1; stride < VF; stride <<= 1) {

1334

1336 for (unsigned j = 0; j < VF; j += stride << 1) {

1337 ShuffleMask[j] = j + stride;

1338 }

1339 BuildShuffledOp(ShuffleMask, TmpVec);

1340 }

1341 } else {

1343 for (unsigned i = VF; i != 1; i >>= 1) {

1344

1345 for (unsigned j = 0; j != i / 2; ++j)

1346 ShuffleMask[j] = i / 2 + j;

1347

1348

1349 std::fill(&ShuffleMask[i / 2], ShuffleMask.end(), -1);

1350 BuildShuffledOp(ShuffleMask, TmpVec);

1351 }

1352 }

1353

1354 return Builder.CreateExtractElement(TmpVec, Builder.getInt32(0));

1355}

1356

1359 Value *NewVal = nullptr;

1360

1361

1362

1364 for (auto *U : OrigPhi->users()) {

1366 break;

1367 }

1368 assert(SI && "One user of the original phi should be a select");

1369

1370 if (SI->getTrueValue() == OrigPhi)

1371 NewVal = SI->getFalseValue();

1372 else {

1373 assert(SI->getFalseValue() == OrigPhi &&

1374 "At least one input to the select should be the original Phi");

1375 NewVal = SI->getTrueValue();

1376 }

1377

1378

1380 Src->getType()->isVectorTy() ? Builder.CreateOrReduce(Src) : Src;

1381

1382

1383 AnyOf = Builder.CreateFreeze(AnyOf);

1384 return Builder.CreateSelect(AnyOf, NewVal, InitVal, "rdx.select");

1385}

1386

1393 ? (IsMaxRdx ? Builder.CreateIntMaxReduce(Src, IsSigned)

1394 : Builder.CreateIntMinReduce(Src, IsSigned))

1395 : Src;

1396

1397

1400 return Builder.CreateSelect(Cmp, MaxRdx, Start, "rdx.select");

1401}

1402

1405 bool Negative = false;

1406 switch (RdxID) {

1407 default:

1409 case Intrinsic::vector_reduce_add:

1410 case Intrinsic::vector_reduce_mul:

1411 case Intrinsic::vector_reduce_or:

1412 case Intrinsic::vector_reduce_xor:

1413 case Intrinsic::vector_reduce_and:

1414 case Intrinsic::vector_reduce_fadd:

1415 case Intrinsic::vector_reduce_fmul: {

1418 Flags.noSignedZeros());

1419 }

1420 case Intrinsic::vector_reduce_umax:

1421 case Intrinsic::vector_reduce_umin:

1422 case Intrinsic::vector_reduce_smin:

1423 case Intrinsic::vector_reduce_smax: {

1426 }

1427 case Intrinsic::vector_reduce_fmax:

1428 case Intrinsic::vector_reduce_fmaximum:

1429 Negative = true;

1430 [[fallthrough]];

1431 case Intrinsic::vector_reduce_fmin:

1432 case Intrinsic::vector_reduce_fminimum: {

1433 bool PropagatesNaN = RdxID == Intrinsic::vector_reduce_fminimum ||

1434 RdxID == Intrinsic::vector_reduce_fmaximum;

1435 const fltSemantics &Semantics = Ty->getFltSemantics();

1436 return (!Flags.noNaNs() && !PropagatesNaN)

1438 : !Flags.noInfs()

1441 }

1442 }

1443}

1444

1448 "nnan, nsz is expected to be set for FP min/max reduction.");

1451}

1452

1455 auto *SrcVecEltTy = cast(Src->getType())->getElementType();

1456 auto getIdentity = [&]() {

1458 Builder.getFastMathFlags());

1459 };

1460 switch (RdxKind) {

1483 return Builder.CreateFAddReduce(getIdentity(), Src);

1485 return Builder.CreateFMulReduce(getIdentity(), Src);

1486 default:

1488 }

1489}

1490

1495 "AnyOf and FindIV reductions are not supported.");

1499 "No VPIntrinsic for this reduction");

1500 auto *EltTy = cast(Src->getType())->getElementType();

1502 Value *Ops[] = {Iden, Src, Mask, EVL};

1503 return Builder.CreateIntrinsic(EltTy, VPID, Ops);

1504}

1505

1509 "Unexpected reduction kind");

1510 assert(Src->getType()->isVectorTy() && "Expected a vector type");

1511 assert(!Start->getType()->isVectorTy() && "Expected a scalar type");

1512

1513 return B.CreateFAddReduce(Start, Src);

1514}

1515

1520 "Unexpected reduction kind");

1521 assert(Src->getType()->isVectorTy() && "Expected a vector type");

1522 assert(!Start->getType()->isVectorTy() && "Expected a scalar type");

1523

1527 "No VPIntrinsic for this reduction");

1528 auto *EltTy = cast(Src->getType())->getElementType();

1529 Value *Ops[] = {Start, Src, Mask, EVL};

1530 return Builder.CreateIntrinsic(EltTy, VPID, Ops);

1531}

1532

1534 bool IncludeWrapFlags) {

1536 if (!VecOp)

1537 return;

1540 if (!Intersection)

1541 return;

1542 const unsigned Opcode = Intersection->getOpcode();

1543 VecOp->copyIRFlags(Intersection, IncludeWrapFlags);

1544 for (auto *V : VL) {

1546 if (!Instr)

1547 continue;

1548 if (OpValue == nullptr || Opcode == Instr->getOpcode())

1549 VecOp->andIRFlags(V);

1550 }

1551}

1552

1559

1566

1573

1580

1591

1602

1603

1604

1605

1606

1607

1613 while (!WorkList.empty()) {

1615

1616 if (!L->contains(Curr))

1617 continue;

1618

1620 return true;

1621

1622 for (const auto *U : Curr->users()) {

1624 if (Visited.insert(UI).second)

1626 }

1627 }

1628 return false;

1629}

1630

1631

1632

1645

1646

1647

1648

1650 BasicBlock *Preheader = L->getLoopPreheader();

1651

1652 if (!Preheader)

1653 return false;

1654

1655

1656

1657

1658

1660 L->getExitingBlocks(ExitingBlocks);

1662 L->getUniqueExitBlocks(ExitBlocks);

1663 if (ExitBlocks.size() != 1 || ExitingBlocks.size() != 1)

1664 return false;

1665

1666 BasicBlock *ExitBlock = ExitBlocks[0];

1669 Value *Incoming = P->getIncomingValueForBlock(ExitingBlocks[0]);

1670

1671

1672

1673

1674 bool found = false;

1675 for (const RewritePhi &Phi : RewritePhiSet) {

1676 unsigned i = Phi.Ith;

1677 if (Phi.PN == P && (Phi.PN)->getIncomingValue(i) == Incoming) {

1678 found = true;

1679 break;

1680 }

1681 }

1682

1685 if (!L->hasLoopInvariantOperands(I))

1686 return false;

1687

1688 ++BI;

1689 }

1690

1691 for (auto *BB : L->blocks())

1693 return I.mayHaveSideEffects();

1694 }))

1695 return false;

1696

1697 return true;

1698}

1699

1700

1701

1702

1705 if (!Phi)

1706 return false;

1707 if (!L->getLoopPreheader())

1708 return false;

1709 if (Phi->getParent() != L->getHeader())

1710 return false;

1712}

1713

1720

1721 assert(L->isRecursivelyLCSSAForm(*DT, *LI) &&

1722 "Indvars did not preserve LCSSA!");

1723

1725 L->getUniqueExitBlocks(ExitBlocks);

1726

1728

1729

1730

1731 for (BasicBlock *ExitBB : ExitBlocks) {

1732

1733

1735 if (!PN) continue;

1736

1738

1739

1743 continue;

1744

1746 continue;

1747

1748

1749 for (unsigned i = 0; i != NumPreds; ++i) {

1750

1751

1754 continue;

1755

1756

1758 continue;

1759

1760

1762 if (!L->contains(Inst))

1763 continue;

1764

1765

1766

1767

1768

1769

1773 if (IndPhi) {

1775 continue;

1776

1777

1779 if (!isa(U) && !isa(U))

1780 return true;

1781 BinaryOperator *B = dyn_cast(U);

1782 if (B && B != ID.getInductionBinOp())

1783 return true;

1784 return false;

1785 }))

1786 continue;

1787 } else {

1788

1789

1791 if ()

1792 continue;

1794 PHINode *Phi = dyn_cast(U);

1795 if (Phi != PN && !checkIsIndPhi(Phi, L, SE, ID))

1796 return true;

1797 return false;

1798 }))

1799 continue;

1800 if (B != ID.getInductionBinOp())

1801 continue;

1802 }

1803 }

1804

1805

1806

1807

1808

1809

1810

1811 const SCEV *ExitValue = SE->getSCEVAtScope(Inst, L->getParentLoop());

1814 !Rewriter.isSafeToExpand(ExitValue)) {

1815

1816

1817

1818

1821 continue;

1823 if (AddRec->getLoop() == L)

1824 ExitValue = AddRec->evaluateAtIteration(ExitCount, *SE);

1827 !Rewriter.isSafeToExpand(ExitValue))

1828 continue;

1829 }

1830

1831

1832

1833

1834

1835

1838 continue;

1839

1840

1841 bool HighCost = Rewriter.isHighCostExpansion(

1843

1844

1845

1846

1847

1848

1849

1850

1853 &*Inst->getParent()->getFirstInsertionPt() : Inst;

1854 RewritePhiSet.emplace_back(PN, i, ExitValue, InsertPt, HighCost);

1855 }

1856 }

1857 }

1858

1859

1860

1861

1862

1863

1865 int NumReplaced = 0;

1866

1867

1868 for (const RewritePhi &Phi : RewritePhiSet) {

1870

1871

1872

1875 !LoopCanBeDel && Phi.HighCost)

1876 continue;

1877

1878 Value *ExitVal = Rewriter.expandCodeFor(

1879 Phi.ExpansionSCEV, Phi.PN->getType(), Phi.ExpansionPoint);

1880

1881 LLVM_DEBUG(dbgs() << "rewriteLoopExitValues: AfterLoopVal = " << *ExitVal

1882 << '\n'

1883 << " LoopVal = " << *(Phi.ExpansionPoint) << "\n");

1884

1885#ifndef NDEBUG

1886

1887

1888

1890 if (auto *EVL = LI->getLoopFor(ExitInsn->getParent()))

1891 if (EVL != L)

1892 assert(EVL->contains(L) && "LCSSA breach detected!");

1893#endif

1894

1895 NumReplaced++;

1898

1899

1900

1901

1902

1904

1905

1906

1909

1910

1915 }

1916 }

1917

1918

1919

1920 Rewriter.clearInsertPoint();

1921 return NumReplaced;

1922}

1923

1924

1925

1926

1927template

1930

1931

1933

1934

1935

1937 assert(PreOrderLoops.empty() && "Must start with an empty preorder walk.");

1939 "Must start with an empty preorder walk worklist.");

1940 PreOrderWorklist.push_back(RootL);

1941 do {

1943 PreOrderWorklist.append(L->begin(), L->end());

1945 } while (!PreOrderWorklist.empty());

1946

1947 Worklist.insert(std::move(PreOrderLoops));

1948 PreOrderLoops.clear();

1949 }

1950}

1951

1952template

1956}

1957

1961

1965

1970

1974 if (PL)

1975 PL->addChildLoop(&New);

1976 else

1978

1979 if (LPM)

1981

1982

1986

1987

1988 for (Loop *I : *L)

1990

1991 return &New;

1992}

1993

1994

1995

1996

1997

2003

2004

2005

2011

2012 Value *Start = nullptr, *End = nullptr;

2013 LLVM_DEBUG(dbgs() << "LAA: Adding RT check for range:\n");

2015

2016

2017

2018

2019

2020

2021

2022

2023

2024

2025

2032 const SCEV *Recur = LowAR->getStepRecurrence(SE);

2033 if (Recur == HighAR->getStepRecurrence(SE) &&

2034 HighAR->getLoop() == OuterLoop && LowAR->getLoop() == OuterLoop) {

2036 const SCEV *OuterExitCount = SE.getExitCount(OuterLoop, OuterLoopLatch);

2039 const SCEV *NewHigh =

2042 LLVM_DEBUG(dbgs() << "LAA: Expanded RT check for range to include "

2043 "outer loop in order to permit hoisting\n");

2044 High = NewHigh;

2046

2047

2050 Stride = Recur;

2051 LLVM_DEBUG(dbgs() << "LAA: ... but need to check stride is "

2052 "positive: "

2053 << *Stride << '\n');

2054 }

2055 }

2056 }

2057 }

2058 }

2059

2060 Start = Exp.expandCodeFor(Low, PtrArithTy, Loc);

2061 End = Exp.expandCodeFor(High, PtrArithTy, Loc);

2064 Start = Builder.CreateFreeze(Start, Start->getName() + ".fr");

2065 End = Builder.CreateFreeze(End, End->getName() + ".fr");

2066 }

2067 Value *StrideVal =

2068 Stride ? Exp.expandCodeFor(Stride, Stride->getType(), Loc) : nullptr;

2070 return {Start, End, StrideVal};

2071}

2072

2073

2074

2079

2080

2081

2082 transform(PointerChecks, std::back_inserter(ChecksWithBounds),

2088 return std::make_pair(First, Second);

2089 });

2090

2091 return ChecksWithBounds;

2092}

2093

2098

2099

2100 auto ExpandedChecks =

2102

2106

2107 Value *MemoryRuntimeCheck = nullptr;

2108

2109 for (const auto &[A, B] : ExpandedChecks) {

2110

2111

2112

2113 assert((A.Start->getType()->getPointerAddressSpace() ==

2114 B.End->getType()->getPointerAddressSpace()) &&

2115 (B.Start->getType()->getPointerAddressSpace() ==

2116 A.End->getType()->getPointerAddressSpace()) &&

2117 "Trying to bounds check pointers with different address spaces");

2118

2119

2120

2121

2122

2123

2124

2125

2126

2129 Value *IsConflict = ChkBuilder.CreateAnd(Cmp0, Cmp1, "found.conflict");

2130 if (A.StrideToCheck) {

2132 A.StrideToCheck, ConstantInt::get(A.StrideToCheck->getType(), 0),

2133 "stride.check");

2134 IsConflict = ChkBuilder.CreateOr(IsConflict, IsNegativeStride);

2135 }

2136 if (B.StrideToCheck) {

2138 B.StrideToCheck, ConstantInt::get(B.StrideToCheck->getType(), 0),

2139 "stride.check");

2140 IsConflict = ChkBuilder.CreateOr(IsConflict, IsNegativeStride);

2141 }

2142 if (MemoryRuntimeCheck) {

2143 IsConflict =

2144 ChkBuilder.CreateOr(MemoryRuntimeCheck, IsConflict, "conflict.rdx");

2145 }

2146 MemoryRuntimeCheck = IsConflict;

2147 }

2148

2149 Exp.eraseDeadInstructions(MemoryRuntimeCheck);

2150 return MemoryRuntimeCheck;

2151}

2152

2156

2160

2161 Value *MemoryRuntimeCheck = nullptr;

2162

2163 auto &SE = *Expander.getSE();

2164

2165

2167 for (const auto &[SrcStart, SinkStart, AccessSize, NeedsFreeze] : Checks) {

2168 Type *Ty = SinkStart->getType();

2169

2170 auto *VFTimesICTimesSize =

2171 ChkBuilder.CreateMul(GetVF(ChkBuilder, Ty->getScalarSizeInBits()),

2172 ConstantInt::get(Ty, IC * AccessSize));

2174 Expander.expandCodeFor(SE.getMinusSCEV(SinkStart, SrcStart), Ty, Loc);

2175

2176

2177

2178 Value *IsConflict = SeenCompares.lookup({Diff, VFTimesICTimesSize});

2179 if (IsConflict)

2180 continue;

2181

2182 IsConflict =

2183 ChkBuilder.CreateICmpULT(Diff, VFTimesICTimesSize, "diff.check");

2184 SeenCompares.insert({{Diff, VFTimesICTimesSize}, IsConflict});

2185 if (NeedsFreeze)

2186 IsConflict =

2188 if (MemoryRuntimeCheck) {

2189 IsConflict =

2190 ChkBuilder.CreateOr(MemoryRuntimeCheck, IsConflict, "conflict.rdx");

2191 }

2192 MemoryRuntimeCheck = IsConflict;

2193 }

2194

2196 return MemoryRuntimeCheck;

2197}

2198

2199std::optional

2203 if (!TI || !TI->isConditional())

2204 return {};

2205

2207

2208

2209

2210

2212 return {};

2213

2215 InstToDuplicate.push_back(CondI);

2216

2218 WorkList.append(CondI->op_begin(), CondI->op_end());

2219

2222 while (!WorkList.empty()) {

2224 if ( || !L.contains(I))

2225 continue;

2226

2227

2229 return {};

2230

2231

2233 if (LI->isVolatile() || LI->isAtomic())

2234 return {};

2235

2239

2240 AccessesToCheck.push_back(MemUse->getDefiningAccess());

2242 } else {

2243

2244

2245 return {};

2246 }

2247 }

2248 WorkList.append(I->op_begin(), I->op_end());

2249 }

2250

2251 if (InstToDuplicate.empty())

2252 return {};

2253

2255 L.getExitingBlocks(ExitingBlocks);

2256 auto HasNoClobbersOnPath =

2257 [&L, &AA, &AccessedLocs, &ExitingBlocks, &InstToDuplicate,

2260 -> std::optional {

2262

2263

2268 Seen.insert(Header);

2269 Info.PathIsNoop &=

2270 all_of(*Header, [](Instruction &I) { return .mayHaveSideEffects(); });

2271

2272 while (!WorkList.empty()) {

2274 if (!L.contains(Current))

2275 continue;

2276 const auto &SeenIns = Seen.insert(Current);

2277 if (!SeenIns.second)

2278 continue;

2279

2280 Info.PathIsNoop &= all_of(

2281 *Current, [](Instruction &I) { return .mayHaveSideEffects(); });

2283 }

2284

2285

2286

2287 if (Seen.size() < 2)

2288 return {};

2289

2290

2291

2292

2293

2295 while (!AccessesToCheck.empty()) {

2297 auto SeenI = SeenAccesses.insert(Current);

2299 continue;

2300

2301

2303 return {};

2304

2305

2307 continue;

2308

2309

2310

2314 AA.getModRefInfo(CurrentDef->getMemoryInst(), Loc));

2315 }))

2316 return {};

2317 }

2318

2319 for (Use &U : Current->uses())

2321 }

2322

2323

2324

2326

2327

2328

2329

2330 if (Info.PathIsNoop) {

2331 for (auto *Exiting : ExitingBlocks) {

2332 if (!Seen.contains(Exiting))

2333 continue;

2334 for (auto *Succ : successors(Exiting)) {

2335 if (L.contains(Succ))

2336 continue;

2337

2338 Info.PathIsNoop &= Succ->phis().empty() &&

2339 (!Info.ExitForPath || Info.ExitForPath == Succ);

2340 if (!Info.PathIsNoop)

2341 break;

2342 assert((!Info.ExitForPath || Info.ExitForPath == Succ) &&

2343 "cannot have multiple exit blocks");

2344 Info.ExitForPath = Succ;

2345 }

2346 }

2347 }

2348 if (!Info.ExitForPath)

2349 Info.PathIsNoop = false;

2350

2351 Info.InstToDuplicate = InstToDuplicate;

2352 return Info;

2353 };

2354

2355

2356

2357 if (TI->getSuccessor(0) == TI->getSuccessor(1))

2358 return {};

2359

2360 if (auto Info = HasNoClobbersOnPath(TI->getSuccessor(0), L.getHeader(),

2361 AccessesToCheck)) {

2363 return Info;

2364 }

2365 if (auto Info = HasNoClobbersOnPath(TI->getSuccessor(1), L.getHeader(),

2366 AccessesToCheck)) {

2368 return Info;

2369 }

2370

2371 return {};

2372}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

AMDGPU Register Bank Select

This is the interface for LLVM's primary stateless and local alias analysis.

bbsections Prepares for basic block by splitting functions into clusters of basic blocks

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

#define LLVM_EXPORT_TEMPLATE

This file defines the DenseSet and SmallDenseSet classes.

This is the interface for a simple mod/ref and alias analysis over globals.

static const HTTPClientCleanup Cleanup

Module.h This file contains the declarations for the Module class.

iv Induction Variable Users

static cl::opt< ReplaceExitVal > ReplaceExitValue("replexitval", cl::Hidden, cl::init(OnlyCheapRepl), cl::desc("Choose the strategy to replace exit value in IndVarSimplify"), cl::values(clEnumValN(NeverRepl, "never", "never replace exit value"), clEnumValN(OnlyCheapRepl, "cheap", "only replace exit value when the cost is cheap"), clEnumValN(UnusedIndVarInLoop, "unusedindvarinloop", "only replace exit value when it is an unused " "induction variable in the loop and has cheap replacement cost"), clEnumValN(NoHardUse, "noharduse", "only replace exit values when loop def likely dead"), clEnumValN(AlwaysRepl, "always", "always replace exit value whenever possible")))

const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]

static cl::opt< bool, true > HoistRuntimeChecks("hoist-runtime-checks", cl::Hidden, cl::desc("Hoist inner loop runtime memory checks to outer loop if possible"), cl::location(VectorizerParams::HoistRuntimeChecks), cl::init(true))

static bool hasHardUserWithinLoop(const Loop *L, const Instruction *I)

Definition LoopUtils.cpp:1608

static const char * LLVMLoopDisableLICM

Definition LoopUtils.cpp:56

static PointerBounds expandBounds(const RuntimeCheckingPtrGroup *CG, Loop *TheLoop, Instruction *Loc, SCEVExpander &Exp, bool HoistRuntimeChecks)

Expand code for the lower and upper bound of the pointer group CG in TheLoop.

Definition LoopUtils.cpp:2006

static bool canLoopBeDeleted(Loop *L, SmallVector< RewritePhi, 8 > &RewritePhiSet)

Definition LoopUtils.cpp:1649

static const char * LLVMLoopDisableNonforced

Definition LoopUtils.cpp:55

static MDNode * createStringMetadata(Loop *TheLoop, StringRef Name, unsigned V)

Create MDNode for input string.

Definition LoopUtils.cpp:204

static BranchInst * getExpectedExitLoopLatchBranch(Loop *L)

Checks if L has an exiting latch branch.

Definition LoopUtils.cpp:791

static std::optional< unsigned > estimateLoopTripCount(Loop *L)

Definition LoopUtils.cpp:820

static bool checkIsIndPhi(PHINode *Phi, Loop *L, ScalarEvolution *SE, InductionDescriptor &ID)

Checks if it is safe to call InductionDescriptor::isInductionPHI for Phi, and returns true if this Ph...

Definition LoopUtils.cpp:1703

This file exposes an interface to building/using memory SSA to walk memory instructions using a use/d...

#define INITIALIZE_PASS_DEPENDENCY(depName)

This file provides a priority worklist.

This file contains the declarations for profiling metadata utility functions.

const SmallVectorImpl< MachineOperand > & Cond

This is the interface for a SCEV-based alias analysis.

This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...

This file implements a set that has insertion order iteration characteristics.

This file defines the SmallPtrSet class.

This file defines the SmallVector class.

A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.

static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)

Returns the largest finite number in the given semantics.

Class for arbitrary precision integers.

static APInt getMaxValue(unsigned numBits)

Gets maximum unsigned value of APInt for specific bit width.

static APInt getSignedMaxValue(unsigned numBits)

Gets maximum signed value of APInt for a specific bit width.

static APInt getMinValue(unsigned numBits)

Gets minimum unsigned value of APInt for a specific bit width.

static APInt getSignedMinValue(unsigned numBits)

Gets minimum signed value of APInt for a specific bit width.

Represent the analysis usage information of a pass.

LLVM_ABI AnalysisUsage & addRequiredID(const void *ID)

AnalysisUsage & addPreservedID(const void *ID)

AnalysisUsage & addRequired()

AnalysisUsage & addPreserved()

Add the specified Pass class to the set of analyses preserved by this pass.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

Legacy wrapper pass to provide the BasicAAResult object.

LLVM Basic Block Representation.

iterator begin()

Instruction iterator methods.

InstListType::iterator iterator

Instruction iterators...

LLVM_ABI LLVMContext & getContext() const

Get the context in which this basic block lives.

const Instruction * getTerminator() const LLVM_READONLY

Returns the terminator instruction if the block is well formed or null if the block is not well forme...

Conditional or Unconditional Branch instruction.

unsigned getNumSuccessors() const

BasicBlock * getSuccessor(unsigned i) const

static LLVM_ABI BranchProbability getBranchProbability(uint64_t Numerator, uint64_t Denominator)

static BranchProbability getUnknown()

uint32_t getNumerator() const

Predicate

This enumeration lists the possible predicates for CmpInst subclasses.

@ ICMP_SLT

signed less than

@ ICMP_SLE

signed less or equal

@ FCMP_OLT

0 1 0 0 True if ordered and less than

@ FCMP_OGT

0 0 1 0 True if ordered and greater than

@ ICMP_UGT

unsigned greater than

@ ICMP_SGT

signed greater than

@ ICMP_ULT

unsigned less than

@ ICMP_SGE

signed greater or equal

static LLVM_ABI Constant * getIntrinsicIdentity(Intrinsic::ID, Type *Ty)

static LLVM_ABI Constant * getBinOpIdentity(unsigned Opcode, Type *Ty, bool AllowRHSConstant=false, bool NSZ=false)

Return the identity constant for a binary opcode.

static LLVM_ABI Constant * getInfinity(Type *Ty, bool Negative=false)

static LLVM_ABI Constant * getQNaN(Type *Ty, bool Negative=false, APInt *Payload=nullptr)

This is the shared class of boolean and integer constants.

static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)

static LLVM_ABI ConstantInt * getFalse(LLVMContext &Context)

int64_t getSExtValue() const

Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...

Record of a variable value-assignment, aka a non instruction representation of the dbg....

Identifies a unique instance of a variable.

ValueT lookup(const_arg_type_t< KeyT > Val) const

lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...

std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)

iterator_range< iterator > children()

static constexpr UpdateKind Delete

static constexpr UpdateKind Insert

DomTreeNodeBase< NodeT > * getNode(const NodeT *BB) const

getNode - return the (Post)DominatorTree node for the specified basic block.

Legacy analysis pass which computes a DominatorTree.

Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.

LLVM_ABI bool isReachableFromEntry(const Use &U) const

Provide an overload for a Use.

static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)

Convenience struct for specifying and reasoning about fast-math flags.

bool noSignedZeros() const

void applyUpdates(ArrayRef< UpdateT > Updates)

Submit updates to all available trees.

Legacy wrapper pass to provide the GlobalsAAResult object.

Common base class shared among various IRBuilders.

Value * CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateFreeze(Value *V, const Twine &Name="")

Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name="")

void SetInsertPoint(BasicBlock *TheBB)

This specifies that created instructions should be appended to the end of the specified block.

Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="", bool IsDisjoint=false)

Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

A struct for saving information about induction variables.

static LLVM_ABI bool isInductionPHI(PHINode *Phi, const Loop *L, ScalarEvolution *SE, InductionDescriptor &D, const SCEV *Expr=nullptr, SmallVectorImpl< Instruction * > *CastsToIgnore=nullptr)

Returns true if Phi is an induction in the loop L.

InstSimplifyFolder - Use InstructionSimplify to fold operations to existing values.

LLVM_ABI unsigned getNumSuccessors() const LLVM_READONLY

Return the number of successors that this instruction has.

LLVM_ABI InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

LLVM_ABI bool mayHaveSideEffects() const LLVM_READONLY

Return true if the instruction may have side effects.

This is an important class for using LLVM in a threaded context.

bool contains(const LoopT *L) const

Return true if the specified loop is contained within in this loop.

typename std::vector< Loop * >::const_iterator iterator

BlockT * getLoopLatch() const

If there is a single latch block for this loop, return it.

BlockT * getHeader() const

LoopT * getParentLoop() const

Return the parent loop if it exists or nullptr for top level loops.

void addTopLevelLoop(LoopT *New)

This adds the specified loop to the collection of top-level loops.

void removeBlock(BlockT *BB)

This method completely removes BB from all data structures, including all of the Loop objects it is n...

LoopT * AllocateLoop(ArgsTy &&...Args)

LoopT * removeLoop(iterator I)

This removes the specified top-level loop from this loop info object.

LoopT * getLoopFor(const BlockT *BB) const

Return the inner most loop that BB lives in.

void destroy(LoopT *L)

Destroy a loop that has been removed from the LoopInfo nest.

The legacy pass manager's analysis pass to compute loop information.

bool replacementPreservesLCSSAForm(Instruction *From, Value *To)

Returns true if replacing From with To everywhere is guaranteed to preserve LCSSA form.

LLVM_ABI void erase(Loop *L)

Update LoopInfo after removing the last backedge from a loop.

Represents a single loop in the control flow graph.

void setLoopID(MDNode *LoopID) const

Set the llvm.loop loop id metadata for this loop.

MDNode * getLoopID() const

Return the llvm.loop loop id metadata node for this loop if it is present.

LLVM_ABI void replaceOperandWith(unsigned I, Metadata *New)

Replace a specific operand.

const MDOperand & getOperand(unsigned I) const

ArrayRef< MDOperand > operands() const

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

unsigned getNumOperands() const

Return number of MDNode operands.

LLVMContext & getContext() const

Tracking metadata reference owned by Metadata.

LLVM_ABI StringRef getString() const

static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)

BasicBlock * getBlock() const

Representation for a specific memory location.

static LLVM_ABI MemoryLocation get(const LoadInst *LI)

Return a location with information about the memory reference by the given instruction.

Legacy analysis pass which computes MemorySSA.

Encapsulates MemorySSA, including all data associated with memory accesses.

LLVM_ABI void verifyMemorySSA(VerificationLevel=VerificationLevel::Fast) const

Verify that MemorySSA is self consistent (IE definitions dominate all uses, uses appear in the right ...

MemoryUseOrDef * getMemoryAccess(const Instruction *I) const

Given a memory Mod/Ref'ing instruction, get the MemorySSA access associated with it.

void setIncomingValue(unsigned i, Value *V)

BasicBlock * getIncomingBlock(unsigned i) const

Return incoming basic block number i.

Value * getIncomingValue(unsigned i) const

Return incoming value number x.

int getBasicBlockIndex(const BasicBlock *BB) const

Return the first index of the specified basic block in the value list for this PHI.

unsigned getNumIncomingValues() const

Return the number of incoming edges.

PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...

static LLVM_ABI PointerType * get(Type *ElementType, unsigned AddressSpace)

This constructs a pointer to an object of the specified type in a numbered address space.

static LLVM_ABI PoisonValue * get(Type *T)

Static factory methods - Return an 'poison' object of the specified type.

bool insert(const T &X)

Insert a new element into the PriorityWorklist.

static bool isSignedRecurrenceKind(RecurKind Kind)

Returns true if recurrece kind is a signed redux kind.

static bool isAnyOfRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...

static bool isFindLastIVRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...

static bool isFindIVRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...

static bool isMinMaxRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is any min/max kind.

A global registry used in conjunction with static constructors to make pluggable components (like tar...

Legacy wrapper pass to provide the SCEVAAResult object.

This class uses information about analyze scalars to rewrite expressions in canonical form.

ScalarEvolution * getSE()

LLVM_ABI Value * expandCodeFor(const SCEV *SH, Type *Ty, BasicBlock::iterator I)

Insert code to directly compute the specified SCEV expression into the program.

void eraseDeadInstructions(Value *Root)

Remove inserted instructions that are dead, e.g.

This class represents an analyzed expression in the program.

LLVM_ABI Type * getType() const

Return the LLVM type of this SCEV expression.

The main scalar evolution driver.

LLVM_ABI bool isKnownNonNegative(const SCEV *S)

Test if the given expression is known to be non-negative.

LLVM_ABI bool isLoopEntryGuardedByCond(const Loop *L, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)

Test whether entry to the loop is protected by a conditional between LHS and RHS.

LLVM_ABI const SCEV * getSCEVAtScope(const SCEV *S, const Loop *L)

Return a SCEV expression for the specified value at the specified scope in the program.

const SCEV * getZero(Type *Ty)

Return a SCEV for the constant 0 of a specific type.

LLVM_ABI const SCEV * getConstant(ConstantInt *V)

LLVM_ABI const SCEV * getSCEV(Value *V)

Return a SCEV expression for the full generality of the specified expression.

LLVM_ABI void forgetLoop(const Loop *L)

This method should be called by the client when it has changed a loop in a way that may effect Scalar...

LLVM_ABI bool isLoopInvariant(const SCEV *S, const Loop *L)

Return true if the value of the given SCEV is unchanging in the specified loop.

LLVM_ABI LoopDisposition getLoopDisposition(const SCEV *S, const Loop *L)

Return the "disposition" of the given SCEV with respect to the given loop.

LLVM_ABI bool isSCEVable(Type *Ty) const

Test if values of the given type are analyzable within the SCEV framework.

LLVM_ABI void forgetValue(Value *V)

This method should be called by the client when it has changed a value in a way that may effect its v...

LLVM_ABI void forgetBlockAndLoopDispositions(Value *V=nullptr)

Called when the client has changed the disposition of values in a loop or block.

LoopDisposition

An enum describing the relationship between a SCEV and a loop.

@ LoopInvariant

The SCEV is loop-invariant.

LLVM_ABI bool isAvailableAtLoopEntry(const SCEV *S, const Loop *L)

Determine if the SCEV can be evaluated at loop's entry.

LLVM_ABI const SCEV * getExitCount(const Loop *L, const BasicBlock *ExitingBlock, ExitCountKind Kind=Exact)

Return the number of times the backedge executes before the given exit would be taken; if not exactly...

LLVM_ABI const SCEV * applyLoopGuards(const SCEV *Expr, const Loop *L)

Try to apply information from loop guards for L to Expr.

This class represents the LLVM 'select' instruction.

Implements a dense probed hash-table based set with some number of buckets stored inline.

A version of PriorityWorklist that selects small size optimized data structures for the vector and ma...

std::pair< iterator, bool > insert(PtrType Ptr)

Inserts Ptr if and only if there is no element in the container equal to Ptr.

bool contains(ConstPtrType Ptr) const

SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.

A SetVector that performs no allocations if smaller than a certain size.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

reference emplace_back(ArgTypes &&... Args)

void append(ItTy in_start, ItTy in_end)

Add the specified range to the end of the SmallVector.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

bool starts_with(StringRef Prefix) const

Check if this string starts with the given Prefix.

Provides information about what library functions are available for the current target.

This pass provides access to the codegen interfaces that are needed for IR-level transformations.

Value handle that tracks a Value across RAUW.

The instances of the Type class are immutable: once they are created, they are never changed.

bool isVectorTy() const

True if this is an instance of VectorType.

static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)

bool isIntegerTy() const

True if this is an instance of IntegerType.

A Use represents the edge between a Value definition and its users.

static LLVM_ABI Intrinsic::ID getForIntrinsic(Intrinsic::ID Id)

The llvm.vp.

static LLVM_ABI bool isVPReduction(Intrinsic::ID ID)

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

LLVM_ABI void replaceAllUsesWith(Value *V)

Change all uses of this to point to a new Value.

iterator_range< user_iterator > users()

iterator_range< use_iterator > uses()

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

std::pair< iterator, bool > insert(const ValueT &V)

An efficient, type-erasing, non-owning reference to a callable.

const ParentTy * getParent() const

This class implements an extremely fast bulk output stream that can only output to a stream.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

Abstract Attribute helper functions.

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract_or_null(Y &&MD)

Extract a Value from Metadata, allowing null.

This is an optimization pass for GlobalISel generic memory operations.

auto drop_begin(T &&RangeOrContainer, size_t N=1)

Return a range covering RangeOrContainer with the first N elements excluded.

LLVM_ABI Value * createSimpleReduction(IRBuilderBase &B, Value *Src, RecurKind RdxKind)

Create a reduction of the given vector.

Definition LoopUtils.cpp:1453

LLVM_ABI std::optional< ElementCount > getOptionalElementCountLoopAttribute(const Loop *TheLoop)

Find a combination of metadata ("llvm.loop.vectorize.width" and "llvm.loop.vectorize....

Definition LoopUtils.cpp:251

@ Low

Lower the current thread's priority such that it does not affect foreground tasks significantly.

LLVM_ABI Value * addRuntimeChecks(Instruction *Loc, Loop *TheLoop, const SmallVectorImpl< RuntimePointerCheck > &PointerChecks, SCEVExpander &Expander, bool HoistRuntimeChecks=false)

Add code that checks at runtime if the accessed arrays in PointerChecks overlap.

Definition LoopUtils.cpp:2094

LLVM_ABI Value * createFindLastIVReduction(IRBuilderBase &B, Value *Src, RecurKind RdxKind, Value *Start, Value *Sentinel)

Create a reduction of the given vector Src for a reduction of the kind RecurKind::FindLastIV.

Definition LoopUtils.cpp:1387

auto find(R &&Range, const T &Val)

Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.

void fill(R &&Range, T &&Value)

Provide wrappers to std::fill which take ranges instead of having to pass begin/end explicitly.

LLVM_ABI std::optional< unsigned > getLoopEstimatedTripCount(Loop *L, unsigned *EstimatedLoopInvocationWeight=nullptr)

Return either:

Definition LoopUtils.cpp:869

bool all_of(R &&range, UnaryPredicate P)

Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.

LLVM_ABI Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)

Returns the min/max intrinsic used when expanding a min/max reduction.

Definition LoopUtils.cpp:1163

LLVM_ABI bool getBooleanLoopAttribute(const Loop *TheLoop, StringRef Name)

Returns true if Name is applied to TheLoop and enabled.

std::pair< const RuntimeCheckingPtrGroup *, const RuntimeCheckingPtrGroup * > RuntimePointerCheck

A memcheck which made up of a pair of grouped pointers.

detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)

LLVM_ABI bool isKnownNonPositiveInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE)

Returns true if we can prove that S is defined and always non-positive in loop L.

Definition LoopUtils.cpp:1574

LLVM_ABI std::optional< bool > getOptionalBoolLoopAttribute(const Loop *TheLoop, StringRef Name)

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

void appendReversedLoopsToWorklist(RangeT &&, SmallPriorityWorklist< Loop *, 4 > &)

Utility that implements appending of loops onto a worklist given a range.

Definition LoopUtils.cpp:1928

auto successors(const MachineBasicBlock *BB)

BranchProbability getBranchProbability(BranchInst *B, bool ForFirstTarget)

Based on branch weight metadata, return either:

Definition LoopUtils.cpp:1003

LLVM_ABI void initializeLoopPassPass(PassRegistry &)

Manually defined generic "LoopPass" dependency initialization.

Definition LoopUtils.cpp:190

constexpr from_range_t from_range

LLVM_ABI bool formLCSSARecursively(Loop &L, const DominatorTree &DT, const LoopInfo *LI, ScalarEvolution *SE)

Put a loop nest into LCSSA form.

LLVM_ABI Value * getReductionIdentity(Intrinsic::ID RdxID, Type *Ty, FastMathFlags FMF)

Given information about an @llvm.vector.reduce.

Definition LoopUtils.cpp:1403

LLVM_ABI std::optional< MDNode * > makeFollowupLoopID(MDNode *OrigLoopID, ArrayRef< StringRef > FollowupAttrs, const char *InheritOptionsAttrsPrefix="", bool AlwaysNew=false)

Create a new loop identifier for a loop created from a loop transformation.

Definition LoopUtils.cpp:264

LLVM_ABI unsigned getArithmeticReductionInstruction(Intrinsic::ID RdxID)

Returns the arithmetic instruction opcode used when expanding a reduction.

Definition LoopUtils.cpp:1115

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

LLVM_ABI char & LoopSimplifyID

LLVM_ABI Value * createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left, Value *Right)

Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.

Definition LoopUtils.cpp:1256

LLVM_ABI SmallVector< BasicBlock *, 16 > collectChildrenInLoop(DominatorTree *DT, DomTreeNode *N, const Loop *CurLoop)

Does a BFS from a given node to all of its children inside a given loop.

Definition LoopUtils.cpp:450

LLVM_ABI void addStringMetadataToLoop(Loop *TheLoop, const char *MDString, unsigned V=0)

Set input string into loop metadata by keeping other values intact.

Definition LoopUtils.cpp:215

LLVM_ABI bool cannotBeMaxInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE, bool Signed)

Returns true if S is defined and never is equal to signed/unsigned max.

Definition LoopUtils.cpp:1592

LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected, bool ElideAllZero=false)

Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...

DomTreeNodeBase< BasicBlock > DomTreeNode

constexpr T divideNearest(U Numerator, V Denominator)

Returns (Numerator / Denominator) rounded by round-half-up.

LLVM_ABI TransformationMode hasVectorizeTransformation(const Loop *L)

Definition LoopUtils.cpp:392

bool setBranchProbability(BranchInst *B, BranchProbability P, bool ForFirstTarget)

Set branch weight metadata for B to indicate that P and 1 - P are the probabilities of control flowin...

Definition LoopUtils.cpp:1018

auto dyn_cast_or_null(const Y &Val)

OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)

Wrapper function around std::transform to apply a function to a range and store the result elsewhere.

bool any_of(R &&range, UnaryPredicate P)

Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.

LLVM_ABI bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)

Return true if the result produced by the instruction is not used, and the instruction will return.

LLVM_ABI SmallVector< Instruction *, 8 > findDefsUsedOutsideOfLoop(Loop *L)

Returns the instructions that use values defined in the loop.

Definition LoopUtils.cpp:124

auto reverse(ContainerTy &&C)

LLVM_ABI constexpr Intrinsic::ID getReductionIntrinsicID(RecurKind RK)

Returns the llvm.vector.reduce intrinsic that corresponds to the recurrence kind.

Definition LoopUtils.cpp:1053

LLVM_ABI bool isMustProgress(const Loop *L)

Return true if this loop can be assumed to make progress.

constexpr bool isPowerOf2_32(uint32_t Value)

Return true if the argument is a power of two > 0.

bool isModSet(const ModRefInfo MRI)

LLVM_ABI TransformationMode hasUnrollAndJamTransformation(const Loop *L)

Definition LoopUtils.cpp:374

LLVM_ABI void deleteDeadLoop(Loop *L, DominatorTree *DT, ScalarEvolution *SE, LoopInfo *LI, MemorySSA *MSSA=nullptr)

This function deletes dead loops.

Definition LoopUtils.cpp:485

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

LLVM_ABI bool hasDisableAllTransformsHint(const Loop *L)

Look for the loop attribute that disables all transformation heuristic.

Definition LoopUtils.cpp:345

LLVM_TEMPLATE_ABI void appendLoopsToWorklist(RangeT &&, SmallPriorityWorklist< Loop *, 4 > &)

Utility that implements appending of loops onto a worklist given a range.

LLVM_ABI cl::opt< unsigned > SCEVCheapExpansionBudget

FunctionAddr VTableAddr Count

LLVM_ABI Value * getShuffleReduction(IRBuilderBase &Builder, Value *Src, unsigned Op, TargetTransformInfo::ReductionShuffle RS, RecurKind MinMaxKind=RecurKind::None)

Generates a vector reduction using shufflevectors to reduce the value.

Definition LoopUtils.cpp:1299

LLVM_ABI TransformationMode hasUnrollTransformation(const Loop *L)

Definition LoopUtils.cpp:353

BranchProbability getLoopProbability(Loop *L)

Based on branch weight metadata, return either:

Definition LoopUtils.cpp:987

LLVM_ABI TransformationMode hasDistributeTransformation(const Loop *L)

Definition LoopUtils.cpp:428

LLVM_ABI void breakLoopBackedge(Loop *L, DominatorTree &DT, ScalarEvolution &SE, LoopInfo &LI, MemorySSA *MSSA)

Remove the backedge of the specified loop.

Definition LoopUtils.cpp:711

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

LLVM_ABI void getLoopAnalysisUsage(AnalysisUsage &AU)

Helper to consistently add the set of standard passes to a loop pass's AnalysisUsage.

Definition LoopUtils.cpp:142

LLVM_ABI void propagateIRFlags(Value *I, ArrayRef< Value * > VL, Value *OpValue=nullptr, bool IncludeWrapFlags=true)

Get the intersection (logical and) of all of the potential IR flags of each scalar operation (VL) tha...

Definition LoopUtils.cpp:1533

LLVM_ABI bool isKnownPositiveInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE)

Returns true if we can prove that S is defined and always positive in loop L.

Definition LoopUtils.cpp:1567

LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key

LLVM_ABI unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)

Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...

RNSuccIterator< NodeRef, BlockT, RegionT > succ_begin(NodeRef Node)

LLVM_ABI std::optional< int > getOptionalIntLoopAttribute(const Loop *TheLoop, StringRef Name)

Find named metadata for a loop with an integer value.

bool setLoopProbability(Loop *L, BranchProbability P)

Set branch weight metadata for the latch of L to indicate that, at the end of any iteration,...

Definition LoopUtils.cpp:995

LLVM_ABI BasicBlock * SplitBlockPredecessors(BasicBlock *BB, ArrayRef< BasicBlock * > Preds, const char *Suffix, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, bool PreserveLCSSA=false)

This method introduces at least one new basic block into the function and moves some of the predecess...

@ First

Helpers to iterate all locations in the MemoryEffectsBase class.

LLVM_ABI CmpInst::Predicate getMinMaxReductionPredicate(RecurKind RK)

Returns the comparison predicate used when expanding a min/max reduction.

Definition LoopUtils.cpp:1234

LLVM_ABI TransformationMode hasLICMVersioningTransformation(const Loop *L)

Definition LoopUtils.cpp:438

LLVM_ABI bool VerifyMemorySSA

Enables verification of MemorySSA.

TransformationMode

The mode sets how eager a transformation should be applied.

@ TM_Unspecified

The pass can use heuristics to determine whether a transformation should be applied.

@ TM_SuppressedByUser

The transformation must not be applied.

@ TM_ForcedByUser

The transformation was directed by the user, e.g.

@ TM_Disable

The transformation should not be applied.

@ TM_Enable

The transformation should be applied without considering a cost model.

RNSuccIterator< NodeRef, BlockT, RegionT > succ_end(NodeRef Node)

LLVM_ABI bool hasDisableLICMTransformsHint(const Loop *L)

Look for the loop attribute that disables the LICM transformation heuristics.

Definition LoopUtils.cpp:349

template LLVM_TEMPLATE_ABI void appendLoopsToWorklist< Loop & >(Loop &L, SmallPriorityWorklist< Loop *, 4 > &Worklist)

LLVM_ABI Intrinsic::ID getReductionForBinop(Instruction::BinaryOps Opc)

Returns the reduction intrinsic id corresponding to the binary operation.

Definition LoopUtils.cpp:1145

RecurKind

These are the kinds of recurrences that we support.

@ UMin

Unsigned integer min implemented in terms of select(cmp()).

@ FMinimumNum

FP min with llvm.minimumnum semantics.

@ Or

Bitwise or logical OR of integers.

@ FMinimum

FP min with llvm.minimum semantics.

@ FMaxNum

FP max with llvm.maxnum semantics including NaNs.

@ Mul

Product of integers.

@ AnyOf

AnyOf reduction with select(cmp(),x,y) where one of (x,y) is loop invariant, and both x and y are int...

@ Xor

Bitwise or logical XOR of integers.

@ FMax

FP max implemented in terms of select(cmp()).

@ FMaximum

FP max with llvm.maximum semantics.

@ FMulAdd

Sum of float products with llvm.fmuladd(a * b + sum).

@ SMax

Signed integer max implemented in terms of select(cmp()).

@ And

Bitwise or logical AND of integers.

@ SMin

Signed integer min implemented in terms of select(cmp()).

@ FMin

FP min implemented in terms of select(cmp()).

@ FMinNum

FP min with llvm.minnum semantics including NaNs.

@ Sub

Subtraction of integers.

@ AddChainWithSubs

A chain of adds and subs.

@ FMaximumNum

FP max with llvm.maximumnum semantics.

@ UMax

Unsigned integer max implemented in terms of select(cmp()).

LLVM_ABI Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF)

Given information about an recurrence kind, return the identity for the @llvm.vector....

Definition LoopUtils.cpp:1445

LLVM_ABI bool formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI, MemorySSAUpdater *MSSAU, bool PreserveLCSSA)

Ensure that all exit blocks of the loop are dedicated exits.

Definition LoopUtils.cpp:58

DWARFExpression::Operation Op

raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)

LLVM_ABI bool isKnownNegativeInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE)

Returns true if we can prove that S is defined and always negative in loop L.

Definition LoopUtils.cpp:1553

constexpr unsigned BitWidth

ValueMap< const Value *, WeakTrackingVH > ValueToValueMapTy

LLVM_ABI bool setLoopEstimatedTripCount(Loop *L, unsigned EstimatedTripCount, std::optional< unsigned > EstimatedLoopInvocationWeight=std::nullopt)

Set llvm.loop.estimated_trip_count with the value EstimatedTripCount in the loop metadata of L.

Definition LoopUtils.cpp:942

LLVM_ABI bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)

Extract branch weights from MD_prof metadata.

LLVM_ABI const char * LLVMLoopEstimatedTripCount

Profile-based loop metadata that should be accessed only by using llvm::getLoopEstimatedTripCount and...

LLVM_ABI bool hasIterationCountInvariantInParent(Loop *L, ScalarEvolution &SE)

Check inner loop (L) backedge count is known to be invariant on all iterations of its outer loop.

Definition LoopUtils.cpp:1031

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

static cl::opt< unsigned > MSSAThreshold("simple-loop-unswitch-memoryssa-threshold", cl::desc("Max number of memory uses to explore during " "partial unswitching analysis"), cl::init(100), cl::Hidden)

LLVM_ABI bool isAlmostDeadIV(PHINode *IV, BasicBlock *LatchBlock, Value *Cond)

Return true if the induction variable IV in a Loop whose latch is LatchBlock would become dead if the...

Definition LoopUtils.cpp:471

auto predecessors(const MachineBasicBlock *BB)

LLVM_ABI int rewriteLoopExitValues(Loop *L, LoopInfo *LI, TargetLibraryInfo *TLI, ScalarEvolution *SE, const TargetTransformInfo *TTI, SCEVExpander &Rewriter, DominatorTree *DT, ReplaceExitVal ReplaceExitValue, SmallVector< WeakTrackingVH, 16 > &DeadInsts)

If the final value of any expressions that are recurrent in the loop can be computed,...

Definition LoopUtils.cpp:1714

LLVM_ABI Value * createOrderedReduction(IRBuilderBase &B, RecurKind RdxKind, Value *Src, Value *Start)

Create an ordered reduction intrinsic using the given recurrence kind RdxKind.

Definition LoopUtils.cpp:1506

LLVM_ABI Value * addDiffRuntimeChecks(Instruction *Loc, ArrayRef< PointerDiffInfo > Checks, SCEVExpander &Expander, function_ref< Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC)

Definition LoopUtils.cpp:2153

LLVM_ABI RecurKind getMinMaxReductionRecurKind(Intrinsic::ID RdxID)

Returns the recurence kind used when expanding a min/max reduction.

Definition LoopUtils.cpp:1215

LLVM_ABI BasicBlock * SplitEdge(BasicBlock *From, BasicBlock *To, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")

Split the edge connecting the specified blocks, and return the newly created basic block between From...

LLVM_ABI std::optional< IVConditionInfo > hasPartialIVCondition(const Loop &L, unsigned MSSAThreshold, const MemorySSA &MSSA, AAResults &AA)

Check if the loop header has a conditional branch that is not loop-invariant, because it involves loa...

Definition LoopUtils.cpp:2200

static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)

Filter the DbgRecord range to DbgVariableRecord types only and downcast.

LLVM_ABI Value * createAnyOfReduction(IRBuilderBase &B, Value *Src, Value *InitVal, PHINode *OrigPhi)

Create a reduction of the given vector Src for a reduction of kind RecurKind::AnyOf.

Definition LoopUtils.cpp:1357

LLVM_ABI bool cannotBeMinInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE, bool Signed)

Returns true if S is defined and never is equal to signed/unsigned min.

Definition LoopUtils.cpp:1581

LLVM_ABI bool isKnownNonNegativeInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE)

Returns true if we can prove that S is defined and always non-negative in loop L.

Definition LoopUtils.cpp:1560

LLVM_ABI Value * getOrderedReduction(IRBuilderBase &Builder, Value *Acc, Value *Src, unsigned Op, RecurKind MinMaxKind=RecurKind::None)

Generates an ordered vector reduction using extracts to reduce the value.

Definition LoopUtils.cpp:1274

LLVM_ABI MDNode * findOptionMDForLoopID(MDNode *LoopID, StringRef Name)

Find and return the loop attribute node for the attribute Name in LoopID.

LLVM_ABI Intrinsic::ID getMinMaxReductionIntrinsicID(Intrinsic::ID IID)

Returns the llvm.vector.reduce min/max intrinsic that corresponds to the intrinsic op.

Definition LoopUtils.cpp:1099

LLVM_ABI Loop * cloneLoop(Loop *L, Loop *PL, ValueToValueMapTy &VM, LoopInfo *LI, LPPassManager *LPM)

Recursively clone the specified loop and all of its children, mapping the blocks with the specified m...

Definition LoopUtils.cpp:1971

void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)

Implement std::swap in terms of BitVector swap.

Definition LoopUtils.cpp:807

DbgLoop(const Loop *L)

Definition LoopUtils.cpp:809

const Loop * L

Definition LoopUtils.cpp:808

IR Values for the lower and upper bounds of a pointer evolution.

Definition LoopUtils.cpp:1998

TrackingVH< Value > Start

Definition LoopUtils.cpp:1999

TrackingVH< Value > End

Definition LoopUtils.cpp:2000

Value * StrideToCheck

Definition LoopUtils.cpp:2001

Definition LoopUtils.cpp:1633

bool HighCost

Definition LoopUtils.cpp:1638

unsigned Ith

Definition LoopUtils.cpp:1635

RewritePhi(PHINode *P, unsigned I, const SCEV *Val, Instruction *ExpansionPt, bool H)

Definition LoopUtils.cpp:1640

const SCEV * ExpansionSCEV

Definition LoopUtils.cpp:1636

PHINode * PN

Definition LoopUtils.cpp:1634

Instruction * ExpansionPoint

Definition LoopUtils.cpp:1637

Struct to hold information about a partially invariant condition.

Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...

unsigned AddressSpace

Address space of the involved pointers.

bool NeedsFreeze

Whether the pointer needs to be frozen after expansion, e.g.

const SCEV * High

The SCEV expression which represents the upper bound of all the pointers in this group.

const SCEV * Low

The SCEV expression which represents the lower bound of all the pointers in this group.