LLVM: lib/Transforms/Vectorize/VPlanTransforms.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
13
30
31using namespace llvm;
32
36 GetIntOrFpInductionDescriptor,
38
40 Plan->getVectorLoopRegion());
41 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(RPOT)) {
42
43 if (!VPBB->getParent())
44 break;
46 auto EndIter = Term ? Term->getIterator() : VPBB->end();
47
50
51 VPValue *VPV = Ingredient.getVPSingleValue();
53
55 if (auto *VPPhi = dyn_cast(&Ingredient)) {
56 auto *Phi = cast(VPPhi->getUnderlyingValue());
57 const auto *II = GetIntOrFpInductionDescriptor(Phi);
58 if ()
59 continue;
60
61 VPValue *Start = Plan->getOrAddLiveIn(II->getStartValue());
65 Phi, Start, Step, &Plan->getVF(), *II, Ingredient.getDebugLoc());
66 } else {
67 assert(isa(&Ingredient) &&
68 "only VPInstructions expected here");
69 assert(!isa(Inst) && "phis should be handled above");
70
71 if (LoadInst *Load = dyn_cast(Inst)) {
73 *Load, Ingredient.getOperand(0), nullptr ,
74 false , false ,
75 Ingredient.getDebugLoc());
76 } else if (StoreInst *Store = dyn_cast(Inst)) {
78 *Store, Ingredient.getOperand(1), Ingredient.getOperand(0),
79 nullptr , false , false ,
80 Ingredient.getDebugLoc());
83 } else if (CallInst *CI = dyn_cast(Inst)) {
86 {Ingredient.op_begin(), Ingredient.op_end() - 1}, CI->getType(),
87 CI->getDebugLoc());
88 } else if (SelectInst *SI = dyn_cast(Inst)) {
90 } else if (auto *CI = dyn_cast(Inst)) {
92 CI->getOpcode(), Ingredient.getOperand(0), CI->getType(), *CI);
93 } else {
95 }
96 }
97
101 else
103 "Only recpies with zero or one defined values expected");
104 Ingredient.eraseFromParent();
105 }
106 }
107}
108
111 bool Changed = false;
112
113
115 for (VPRegionBlock *VPR : VPBlockUtils::blocksOnly(Iter)) {
117 if (!VPR->isReplicator() || EntryVPBB->getSuccessors().size() != 2)
118 continue;
121 continue;
122 for (auto &Recipe : *VPBB) {
123 for (VPValue *Op : Recipe.operands())
124 if (auto *Def =
125 dyn_cast_or_null(Op->getDefiningRecipe()))
126 WorkList.insert(std::make_pair(VPBB, Def));
127 }
128 }
129
131
132 for (unsigned I = 0; I != WorkList.size(); ++I) {
135 std::tie(SinkTo, SinkCandidate) = WorkList[I];
136 if (SinkCandidate->getParent() == SinkTo ||
139 continue;
140 if (auto *RepR = dyn_cast(SinkCandidate)) {
141 if (!ScalarVFOnly && RepR->isUniform())
142 continue;
143 } else if (!isa(SinkCandidate))
144 continue;
145
146 bool NeedsDuplicating = false;
147
148
149
150
151 auto CanSinkWithUser = [SinkTo, &NeedsDuplicating,
152 SinkCandidate](VPUser *U) {
153 auto *UI = cast(U);
154 if (UI->getParent() == SinkTo)
155 return true;
156 NeedsDuplicating = UI->onlyFirstLaneUsed(SinkCandidate);
157
158 return NeedsDuplicating && isa(SinkCandidate);
159 };
160 if ((SinkCandidate->users(), CanSinkWithUser))
161 continue;
162
163 if (NeedsDuplicating) {
164 if (ScalarVFOnly)
165 continue;
168
169
170 Clone->insertBefore(SinkCandidate);
172 return cast(&U)->getParent() != SinkTo;
173 });
174 }
177 if (auto *Def =
178 dyn_cast_or_null(Op->getDefiningRecipe()))
179 WorkList.insert(std::make_pair(SinkTo, Def));
180 Changed = true;
181 }
182 return Changed;
183}
184
185
186
188 auto *EntryBB = dyn_cast(R->getEntry());
189 if (!EntryBB || EntryBB->size() != 1 ||
190 !isa(EntryBB->begin()))
191 return nullptr;
192
193 return cast(&*EntryBB->begin())->getOperand(0);
194}
195
196
198 auto *EntryBB = cast(R->getEntry());
199 if (EntryBB->getNumSuccessors() != 2)
200 return nullptr;
201
202 auto *Succ0 = dyn_cast(EntryBB->getSuccessors()[0]);
203 auto *Succ1 = dyn_cast(EntryBB->getSuccessors()[1]);
204 if (!Succ0 || !Succ1)
205 return nullptr;
206
207 if (Succ0->getNumSuccessors() + Succ1->getNumSuccessors() != 1)
208 return nullptr;
209 if (Succ0->getSingleSuccessor() == Succ1)
210 return Succ0;
211 if (Succ1->getSingleSuccessor() == Succ0)
212 return Succ1;
213 return nullptr;
214}
215
216
217
218
221
222
223
224
226 for (VPRegionBlock *Region1 : VPBlockUtils::blocksOnly(
228 if (!Region1->isReplicator())
229 continue;
230 auto *MiddleBasicBlock =
231 dyn_cast_or_null(Region1->getSingleSuccessor());
232 if (!MiddleBasicBlock || !MiddleBasicBlock->empty())
233 continue;
234
235 auto *Region2 =
236 dyn_cast_or_null(MiddleBasicBlock->getSingleSuccessor());
237 if (!Region2 || !Region2->isReplicator())
238 continue;
239
242 if (!Mask1 || Mask1 != Mask2)
243 continue;
244
245 assert(Mask1 && Mask2 && "both region must have conditions");
247 }
248
249
251 if (TransformedRegions.contains(Region1))
252 continue;
253 auto *MiddleBasicBlock = cast(Region1->getSingleSuccessor());
254 auto *Region2 = cast(MiddleBasicBlock->getSingleSuccessor());
255
258 if (!Then1 || !Then2)
259 continue;
260
261
262
263
264
265
268
271
272
273
274
277 cast(&Phi1ToMove)->getOperand(0);
278 VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
280 return cast(&U)->getParent() == Then2;
281 });
282
283
284 if (Phi1ToMove.getVPSingleValue()->getNumUsers() == 0) {
285 Phi1ToMove.eraseFromParent();
286 continue;
287 }
288 Phi1ToMove.moveBefore(*Merge2, Merge2->begin());
289 }
290
291
295 }
297 TransformedRegions.insert(Region1);
298 }
299
300 return !TransformedRegions.empty();
301}
302
306
307 std::string RegionName = (Twine("pred.") + Instr->getOpcodeName()).str();
308 assert(Instr->getParent() && "Predicated instruction not in any basic block");
309 auto *BlockInMask = PredRecipe->getMask();
311 auto *Entry =
313
314
315
320 auto *Pred =
322
326 RecipeWithoutMask->getDebugLoc());
328 PHIRecipe->setOperand(0, RecipeWithoutMask);
329 }
331 auto *Exiting =
335
336
337
340
342}
343
346 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(
349 if (auto *RepR = dyn_cast(&R)) {
350 if (RepR->isPredicated())
352 }
353 }
354
355 unsigned BBNum = 0;
359
363
367 }
368}
369
370
371
374 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(
376
377
378
379 if (!VPBB->getParent())
380 continue;
381 auto *PredVPBB =
382 dyn_cast_or_null(VPBB->getSinglePredecessor());
383 if (!PredVPBB || PredVPBB->getNumSuccessors() != 1 ||
384 isa(PredVPBB))
385 continue;
387 }
388
390 VPBasicBlock *PredVPBB = cast(VPBB->getSinglePredecessor());
392 R.moveBefore(*PredVPBB, PredVPBB->end());
394 auto *ParentRegion = cast_or_null(VPBB->getParent());
395 if (ParentRegion && ParentRegion->getExiting() == VPBB)
396 ParentRegion->setExiting(PredVPBB);
397 for (auto *Succ : to_vector(VPBB->successors())) {
400 }
401
402 }
403 return !WorkList.empty();
404}
405
407
409
410 bool ShouldSimplify = true;
411 while (ShouldSimplify) {
415 }
416}
417
418
419
420
421
422
423
426 auto *IV = dyn_cast(&Phi);
427 if ( || IV->getTruncInst())
428 continue;
429
430
431
432
433
434
435
436
437
438 auto &Casts = IV->getInductionDescriptor().getCastInsts();
442 for (auto *U : FindMyCast->users()) {
443 auto *UserCast = dyn_cast(U);
444 if (UserCast && UserCast->getUnderlyingValue() == IRCast) {
445 FoundUserCast = UserCast;
446 break;
447 }
448 }
449 FindMyCast = FoundUserCast;
450 }
452 }
453}
454
455
456
461 WidenNewIV = dyn_cast(U);
462 if (WidenNewIV)
463 break;
464 }
465
466 if (!WidenNewIV)
467 return;
468
471 auto *WidenOriginalIV = dyn_cast(&Phi);
472
473 if (!WidenOriginalIV || !WidenOriginalIV->isCanonical())
474 continue;
475
476
477
478
479
480 if (any_of(WidenOriginalIV->users(),
481 [WidenOriginalIV](VPUser *U) {
482 return !U->usesScalars(WidenOriginalIV);
483 }) ||
487 return;
488 }
489 }
490}
491
492
495
496
497 auto *RepR = dyn_cast(&R);
498 bool IsConditionalAssume =
499 RepR && RepR->isPredicated() &&
500 match(RepR->getUnderlyingInstr(), m_IntrinsicIntrinsic::assume());
501 if (IsConditionalAssume)
502 return true;
503
504 if (R.mayHaveSideEffects())
505 return false;
506
507
508 return all_of(R.definedValues(),
509 [](VPValue *V) { return V->getNumUsers() == 0; });
510}
511
515
516 for (VPBasicBlock *VPBB : reverse(VPBlockUtils::blocksOnly(RPOT))) {
517
518
521 R.eraseFromParent();
522 }
523 }
524}
525
535 Kind, FPBinOp, StartV, CanonicalIV, Step, "offset.idx");
536
537
541 if (TruncI) {
544 "Not truncating.");
545 assert(ResultTy->isIntegerTy() && "Truncation requires an integer type");
546 BaseIV = Builder.createScalarCast(Instruction::Trunc, BaseIV, TruncTy, DL);
547 ResultTy = TruncTy;
548 }
549
550
552 if (ResultTy != StepTy) {
554 "Not truncating.");
555 assert(StepTy->isIntegerTy() && "Truncation requires an integer type");
556 auto *VecPreheader =
560 Step = Builder.createScalarCast(Instruction::Trunc, Step, ResultTy, DL);
561 }
562 return Builder.createScalarIVSteps(InductionOpcode, FPBinOp, BaseIV, Step);
563}
564
567 for (unsigned I = 0; I != Users.size(); ++I) {
569 if (isa(Cur))
570 continue;
572 Users.insert(V->user_begin(), V->user_end());
573 }
574 return Users.takeVector();
575}
576
577
578
579
580
581
582
583
584
585
586
587
588
595 auto *PhiR = dyn_cast(&Phi);
596 if (!PhiR)
597 continue;
598
599
600
601
602
605 auto *Def = dyn_cast(U);
606 auto *RepR = dyn_cast(U);
607
608 if (!Def || !isa<VPReplicateRecipe, VPWidenRecipe>(Def) ||
609 Def->getNumUsers() == 0 || !Def->getUnderlyingValue() ||
610 (RepR && (RepR->isUniform() || RepR->isPredicated())))
611 continue;
612
613
616 continue;
617
619 Def->operands(), true);
620 Clone->insertAfter(Def);
621 Def->replaceAllUsesWith(Clone);
622 }
623
624
625
626 if (auto *PtrIV = dyn_cast(&Phi)) {
627 if (!PtrIV->onlyScalarsGenerated(Plan.hasScalableVF()))
628 continue;
629
632 Plan.getOrAddLiveIn(ConstantInt::get(ID.getStep()->getType(), 0));
633 VPValue *StepV = PtrIV->getOperand(1);
636 nullptr, StartV, StepV, PtrIV->getDebugLoc(), Builder);
637
640
642 continue;
643 }
644
645
646
647 auto *WideIV = cast(&Phi);
648 if (HasOnlyVectorVFs && none_of(WideIV->users(), [WideIV](VPUser *U) {
649 return U->usesScalars(WideIV);
650 }))
651 continue;
652
655 Plan, ID.getKind(), ID.getInductionOpcode(),
656 dyn_cast_or_null(ID.getInductionBinOp()),
657 WideIV->getTruncInst(), WideIV->getStartValue(), WideIV->getStepValue(),
658 WideIV->getDebugLoc(), Builder);
659
660
661 if (!HasOnlyVectorVFs)
662 WideIV->replaceAllUsesWith(Steps);
663 else
664 WideIV->replaceUsesWithIf(Steps, [WideIV](VPUser &U, unsigned) {
665 return U.usesScalars(WideIV);
666 });
667 }
668}
669
670
671
672
674 auto *WideIV = dyn_cast(VPV);
675 if (WideIV) {
676
677
678 auto *IntOrFpIV = dyn_cast(WideIV);
679 return (IntOrFpIV && IntOrFpIV->getTruncInst()) ? nullptr : WideIV;
680 }
681
682
684 if (!Def || Def->getNumOperands() != 2)
685 return nullptr;
686 WideIV = dyn_cast(Def->getOperand(0));
687 if (!WideIV)
688 WideIV = dyn_cast(Def->getOperand(1));
689 if (!WideIV)
690 return nullptr;
691
692 auto IsWideIVInc = [&]() {
693 using namespace VPlanPatternMatch;
694 auto &ID = WideIV->getInductionDescriptor();
695
696
697 VPValue *IVStep = WideIV->getStepValue();
698 switch (ID.getInductionOpcode()) {
699 case Instruction::Add:
700 return match(VPV, m_c_BinaryInstruction::Add(m_Specific(WideIV),
702 case Instruction::FAdd:
703 return match(VPV, m_c_BinaryInstruction::FAdd(m_Specific(WideIV),
705 case Instruction::FSub:
706 return match(VPV, m_BinaryInstruction::FSub(m_Specific(WideIV),
708 case Instruction::Sub: {
709
710
713 m_BinaryInstruction::Sub(m_VPValue(), m_VPValue(Step))) ||
715 return false;
716 auto *StepCI = dyn_cast(Step->getLiveInIRValue());
717 auto *IVStepCI = dyn_cast(IVStep->getLiveInIRValue());
718 return StepCI && IVStepCI &&
719 StepCI->getValue() == (-1 * IVStepCI->getValue());
720 }
721 default:
724 m_Specific(WideIV->getStepValue())));
725 }
726 llvm_unreachable("should have been covered by switch above");
727 };
728 return IsWideIVInc() ? WideIV : nullptr;
729}
730
733 using namespace VPlanPatternMatch;
735 if (ExitVPBBs.size() != 1)
736 return;
737
740 if (!PredVPBB)
741 return;
743 "predecessor must be the middle block");
744
748 auto *ExitIRI = cast(&R);
749 if (!isa(ExitIRI->getInstruction()))
750 break;
751
753 if ((ExitIRI->getOperand(0),
754 m_VPInstructionVPInstruction::ExtractFromEnd(
756 continue;
757
759 if (!WideIV)
760 continue;
762 assert(EndValue && "end value must have been pre-computed");
763
765 ExitIRI->setOperand(0, EndValue);
766 continue;
767 }
768
769 VPValue *Escape = nullptr;
770 VPValue *Step = WideIV->getStepValue();
773 Escape =
774 B.createNaryOp(Instruction::Sub, {EndValue, Step}, {}, "ind.escape");
778 Escape = B.createPtrAdd(EndValue,
779 B.createNaryOp(Instruction::Sub, {Zero, Step}),
780 {}, "ind.escape");
782 const auto &ID = WideIV->getInductionDescriptor();
783 Escape = B.createNaryOp(
784 ID.getInductionBinOp()->getOpcode() == Instruction::FAdd
785 ? Instruction::FSub
786 : Instruction::FAdd,
787 {EndValue, Step}, {ID.getInductionBinOp()->getFastMathFlags()});
788 } else {
789 llvm_unreachable("all possible induction types must be handled");
790 }
791 ExitIRI->setOperand(0, Escape);
792 }
793}
794
795
796
799
802 auto *ExpR = dyn_cast(&R);
803 if (!ExpR)
804 continue;
805
806 auto I = SCEV2VPV.insert({ExpR->getSCEV(), ExpR});
807 if (I.second)
808 continue;
809 ExpR->replaceAllUsesWith(I.first->second);
810 ExpR->eraseFromParent();
811 }
812}
813
818
819 while (!WorkList.empty()) {
821 if (!Seen.insert(Cur).second)
822 continue;
824 if (!R)
825 continue;
827 continue;
828 WorkList.append(R->op_begin(), R->op_end());
829 R->eraseFromParent();
830 }
831}
832
833
836
837 if (auto *Blend = dyn_cast(&R)) {
838
840 if (Blend->isNormalized() || (Blend->getMask(0), m_False()))
841 UniqueValues.insert(Blend->getIncomingValue(0));
842 for (unsigned I = 1; I != Blend->getNumIncomingValues(); ++I)
843 if ((Blend->getMask(I), m_False()))
844 UniqueValues.insert(Blend->getIncomingValue(I));
845
846 if (UniqueValues.size() == 1) {
847 Blend->replaceAllUsesWith(*UniqueValues.begin());
848 Blend->eraseFromParent();
849 return;
850 }
851
852 if (Blend->isNormalized())
853 return;
854
855
856
857
858 unsigned StartIndex = 0;
859 for (unsigned I = 0; I != Blend->getNumIncomingValues(); ++I) {
860
861
862
863 VPValue *Mask = Blend->getMask(I);
864 if (Mask->getNumUsers() == 1 && (Mask, m_False())) {
865 StartIndex = I;
866 break;
867 }
868 }
869
871 OperandsWithMask.push_back(Blend->getIncomingValue(StartIndex));
872
873 for (unsigned I = 0; I != Blend->getNumIncomingValues(); ++I) {
874 if (I == StartIndex)
875 continue;
876 OperandsWithMask.push_back(Blend->getIncomingValue(I));
877 OperandsWithMask.push_back(Blend->getMask(I));
878 }
879
881 cast(Blend->getUnderlyingValue()), OperandsWithMask);
882 NewBlend->insertBefore(&R);
883
884 VPValue *DeadMask = Blend->getMask(StartIndex);
886 Blend->eraseFromParent();
888 return;
889 }
890
893 VPValue *Trunc = R.getVPSingleValue();
896 if (TruncTy == ATy) {
898 } else {
899
900 if (isa(&R))
901 return;
903
904 unsigned ExtOpcode = match(R.getOperand(0), m_SExt(m_VPValue()))
905 ? Instruction::SExt
906 : Instruction::ZExt;
907 auto *VPC =
909 if (auto *UnderlyingExt = R.getOperand(0)->getUnderlyingValue()) {
910
911 VPC->setUnderlyingValue(UnderlyingExt);
912 }
913 VPC->insertBefore(&R);
917 VPC->insertBefore(&R);
919 }
920 }
921#ifndef NDEBUG
922
923
925 R.getParent()->getPlan()->getCanonicalIV()->getScalarType());
927 for (VPUser *U : A->users()) {
928 auto *R = cast(U);
929 for (VPValue *VPV : R->definedValues())
931 }
932#endif
933 }
934
935
936
937
938
941 m_c_BinaryOr(m_LogicalAnd(m_VPValue(X), m_VPValue(Y)),
944 R.getVPSingleValue()->replaceAllUsesWith(X);
945 R.eraseFromParent();
946 return;
947 }
948
950 return R.getVPSingleValue()->replaceAllUsesWith(A);
951
953 return R.getVPSingleValue()->replaceAllUsesWith(A);
954
955
962 return R.getVPSingleValue()->replaceAllUsesWith(R.getOperand(1));
963}
964
965
966
971 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(RPOT)) {
974 }
975 }
976}
977
979 unsigned BestUF,
981 assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
982 assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
985 auto *Term = &ExitingVPBB->back();
986
987
988
989
990
992 if ((Term, m_BranchOnCount(m_VPValue(), m_VPValue())) &&
994 m_BranchOnCond(m_Not(m_ActiveLaneMask(m_VPValue(), m_VPValue())))))
995 return;
996
998 const SCEV *TripCount =
1000 assert(!isa(TripCount) &&
1001 "Trip count SCEV must be computable");
1004 if (TripCount->isZero() ||
1006 return;
1007
1008
1009
1010
1011 auto *Header = cast(VectorRegion->getEntry());
1014 Header->phis(),
1015 IsaPred<VPCanonicalIVPHIRecipe, VPFirstOrderRecurrencePHIRecipe>)) {
1017 auto *HeaderPhiR = cast(&HeaderR);
1018 HeaderPhiR->replaceAllUsesWith(HeaderPhiR->getStartValue());
1019 HeaderPhiR->eraseFromParent();
1020 }
1021
1026
1028 B->setParent(nullptr);
1029
1033 } else {
1034
1035
1041 }
1042
1043 Term->eraseFromParent();
1045
1046 Plan.setVF(BestVF);
1047 Plan.setUF(BestUF);
1048
1049
1050
1051}
1052
1053
1054
1055
1056static bool
1060
1063 Seen.insert(Previous);
1064 auto TryToPushSinkCandidate = [&](VPRecipeBase *SinkCandidate) {
1065
1066
1067 if (SinkCandidate == Previous)
1068 return false;
1069
1070 if (isa(SinkCandidate) ||
1071 !Seen.insert(SinkCandidate).second ||
1073 return true;
1074
1075 if (SinkCandidate->mayHaveSideEffects())
1076 return false;
1077
1078 WorkList.push_back(SinkCandidate);
1079 return true;
1080 };
1081
1082
1084 for (unsigned I = 0; I != WorkList.size(); ++I) {
1087 "only recipes with a single defined value expected");
1088
1090 if (!TryToPushSinkCandidate(cast(User)))
1091 return false;
1092 }
1093 }
1094
1095
1096
1099 });
1100
1101 for (VPRecipeBase *SinkCandidate : WorkList) {
1102 if (SinkCandidate == FOR)
1103 continue;
1104
1105 SinkCandidate->moveAfter(Previous);
1106 Previous = SinkCandidate;
1107 }
1108 return true;
1109}
1110
1111
1116 return false;
1117
1118
1122
1123
1124 for (VPUser *U : FOR->users()) {
1125 auto *R = cast(U);
1127 HoistPoint = R;
1128 }
1130 [&VPDT, HoistPoint](VPUser *U) {
1131 auto *R = cast(U);
1132 return HoistPoint == R ||
1133 VPDT.properlyDominates(HoistPoint, R);
1134 }) &&
1135 "HoistPoint must dominate all users of FOR");
1136
1137 auto NeedsHoisting = [HoistPoint, &VPDT,
1139 VPRecipeBase *HoistCandidate = HoistCandidateV->getDefiningRecipe();
1140 if (!HoistCandidate)
1141 return nullptr;
1146 "CFG in VPlan should still be flat, without replicate regions");
1147
1148 if (!Visited.insert(HoistCandidate).second)
1149 return nullptr;
1150
1151
1152
1153 if (!EnclosingLoopRegion || isa(HoistCandidate))
1154 return nullptr;
1155
1156
1157
1159 return nullptr;
1160 return HoistCandidate;
1161 };
1162 auto CanHoist = [&](VPRecipeBase *HoistCandidate) {
1163
1164
1166 };
1167
1169 return true;
1170
1171
1172 HoistCandidates.push_back(Previous);
1173
1174 for (unsigned I = 0; I != HoistCandidates.size(); ++I) {
1177 "only recipes with a single defined value expected");
1178 if (!CanHoist(Current))
1179 return false;
1180
1182
1183
1184
1185
1186 if (Op == FOR)
1187 return false;
1188
1189 if (auto *R = NeedsHoisting(Op))
1191 }
1192 }
1193
1194
1195
1198 });
1199
1200 for (VPRecipeBase *HoistCandidate : HoistCandidates) {
1201 HoistCandidate->moveBefore(*HoistPoint->getParent(),
1203 }
1204
1205 return true;
1206}
1207
1212
1216 if (auto *FOR = dyn_cast(&R))
1218
1221 VPRecipeBase *Previous = FOR->getBackedgeValue()->getDefiningRecipe();
1222
1223
1224 while (auto *PrevPhi =
1225 dyn_cast_or_null(Previous)) {
1226 assert(PrevPhi->getParent() == FOR->getParent());
1228 Previous = PrevPhi->getBackedgeValue()->getDefiningRecipe();
1229 }
1230
1233 return false;
1234
1235
1236
1238 if (isa(Previous))
1240 else
1243
1244 auto *RecurSplice = cast(
1246 {FOR, FOR->getBackedgeValue()}));
1247
1248 FOR->replaceAllUsesWith(RecurSplice);
1249
1250
1251 RecurSplice->setOperand(0, FOR);
1252 }
1253 return true;
1254}
1255
1259 auto *PhiR = dyn_cast(&R);
1260 if (!PhiR)
1261 continue;
1265 continue;
1266
1268 if (auto *RecWithFlags = dyn_cast(U)) {
1269 RecWithFlags->dropPoisonGeneratingFlags();
1270 }
1271 }
1272}
1273
1274
1277
1278
1279
1280
1281 auto CannotHoistRecipe = [](VPRecipeBase &R) {
1282
1283 auto *RepR = dyn_cast(&R);
1284 return RepR && RepR->getOpcode() == Instruction::Alloca;
1285 };
1286
1287
1288
1289
1291 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(
1294 if (CannotHoistRecipe(R))
1295 continue;
1296
1297
1298 if (R.mayHaveSideEffects() || R.mayReadFromMemory() || R.isPhi() ||
1300 return !Op->isDefinedOutsideLoopRegions();
1301 }))
1302 continue;
1303 R.moveBefore(*Preheader, Preheader->end());
1304 }
1305 }
1306}
1307
1310#ifndef NDEBUG
1311
1312
1313 unsigned NumProcessedRecipes = 0;
1314#endif
1315
1316
1317
1318
1323 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(
1328 continue;
1329
1330 VPValue *ResultVPV = R.getVPSingleValue();
1331 auto *UI = cast_or_null(ResultVPV->getUnderlyingValue());
1332 unsigned NewResSizeInBits = MinBWs.lookup(UI);
1333 if (!NewResSizeInBits)
1334 continue;
1335
1336#ifndef NDEBUG
1337 NumProcessedRecipes++;
1338#endif
1339
1340
1341
1342
1343 if (isa<VPReplicateRecipe, VPWidenCastRecipe>(&R)) {
1344#ifndef NDEBUG
1345
1346
1347
1348
1349
1350
1351
1352 for (VPValue *Op : R.operands()) {
1353 if (->isLiveIn())
1354 continue;
1355 auto *UV = dyn_cast_or_null(Op->getUnderlyingValue());
1358 IsaPred<VPWidenRecipe, VPWidenSelectRecipe>)) {
1359
1360
1361 ProcessedTruncs[Op] = nullptr;
1362 NumProcessedRecipes += 1;
1363 }
1364 }
1365#endif
1366 continue;
1367 }
1368
1371 assert(OldResTy->isIntegerTy() && "only integer types supported");
1372 (void)OldResSizeInBits;
1373
1376
1377
1378
1379
1380 if (auto *VPW = dyn_cast(&R))
1381 VPW->dropPoisonGeneratingFlags();
1382
1384 if (OldResSizeInBits != NewResSizeInBits &&
1385 (&R, m_BinaryInstruction::ICmp(m_VPValue(), m_VPValue()))) {
1386
1387 auto *Ext =
1389 Ext->insertAfter(&R);
1391 Ext->setOperand(0, ResultVPV);
1392 assert(OldResSizeInBits > NewResSizeInBits && "Nothing to shrink?");
1393 } else {
1395 match(&R, m_BinaryInstruction::ICmp(m_VPValue(), m_VPValue())) &&
1396 "Only ICmps should not need extending the result.");
1397 }
1398
1399 assert(!isa(&R) && "stores cannot be narrowed");
1400 if (isa(&R))
1401 continue;
1402
1403
1404 unsigned StartIdx = isa(&R) ? 1 : 0;
1405 for (unsigned Idx = StartIdx; Idx != R.getNumOperands(); ++Idx) {
1406 auto *Op = R.getOperand(Idx);
1407 unsigned OpSizeInBits =
1409 if (OpSizeInBits == NewResSizeInBits)
1410 continue;
1411 assert(OpSizeInBits > NewResSizeInBits && "nothing to truncate");
1412 auto [ProcessedIter, IterIsEmpty] =
1413 ProcessedTruncs.insert({Op, nullptr});
1415 IterIsEmpty
1417 : ProcessedIter->second;
1418 R.setOperand(Idx, NewOp);
1419 if (!IterIsEmpty)
1420 continue;
1421 ProcessedIter->second = NewOp;
1422 if (->isLiveIn()) {
1424 } else {
1426#ifndef NDEBUG
1427 auto *OpInst = dyn_cast(Op->getLiveInIRValue());
1428 bool IsContained = MinBWs.contains(OpInst);
1429 NumProcessedRecipes += IsContained;
1430#endif
1431 }
1432 }
1433
1434 }
1435 }
1436
1437 assert(MinBWs.size() == NumProcessedRecipes &&
1438 "some entries in MinBWs haven't been processed");
1439}
1440
1444
1451
1455}
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1495 VPValue *StartV = CanonicalIVPHI->getStartValue();
1496
1497 auto *CanonicalIVIncrement =
1498 cast(CanonicalIVPHI->getBackedgeValue());
1499
1500
1501 CanonicalIVIncrement->dropPoisonGeneratingFlags();
1502 DebugLoc DL = CanonicalIVIncrement->getDebugLoc();
1503
1504
1505
1507 VPBuilder Builder(VecPreheader);
1508
1509
1511
1512 VPValue *TripCount, *IncrementValue;
1514
1515
1516
1517 IncrementValue = CanonicalIVIncrement;
1518 TripCount = TC;
1519 } else {
1520
1521
1522
1523 IncrementValue = CanonicalIVPHI;
1525 {TC}, DL);
1526 }
1529 "index.part.next");
1530
1531
1532 auto *EntryALM =
1534 DL, "active.lane.mask.entry");
1535
1536
1537
1539 LaneMaskPhi->insertAfter(CanonicalIVPHI);
1540
1541
1542
1545 auto *InLoopIncrement =
1547 {IncrementValue}, {false, false}, DL);
1549 {InLoopIncrement, TripCount}, DL,
1550 "active.lane.mask.next");
1552
1553
1554
1555 auto *NotMask = Builder.createNot(ALM, DL);
1558 return LaneMaskPhi;
1559}
1560
1561
1562
1563
1564
1567 auto *FoundWidenCanonicalIVUser =
1569 [](VPUser *U) { return isa(U); });
1571 [](VPUser *U) { return isa(U); }) <=
1572 1 &&
1573 "Must have at most one VPWideCanonicalIVRecipe");
1575 auto *WideCanonicalIV =
1576 cast(*FoundWidenCanonicalIVUser);
1577 WideCanonicalIVs.push_back(WideCanonicalIV);
1578 }
1579
1580
1581
1584 auto *WidenOriginalIV = dyn_cast(&Phi);
1585 if (WidenOriginalIV && WidenOriginalIV->isCanonical())
1586 WideCanonicalIVs.push_back(WidenOriginalIV);
1587 }
1588
1589
1590
1592 for (auto *Wide : WideCanonicalIVs) {
1594 auto *HeaderMask = dyn_cast(U);
1596 continue;
1597
1598 assert(HeaderMask->getOperand(0) == Wide &&
1599 "WidenCanonicalIV must be the first operand of the compare");
1600 HeaderMasks.push_back(HeaderMask);
1601 }
1602 }
1603 return HeaderMasks;
1604}
1605
1607 VPlan &Plan, bool UseActiveLaneMaskForControlFlow,
1610 UseActiveLaneMaskForControlFlow) &&
1611 "DataAndControlFlowWithoutRuntimeCheck implies "
1612 "UseActiveLaneMaskForControlFlow");
1613
1614 auto *FoundWidenCanonicalIVUser =
1616 [](VPUser *U) { return isa(U); });
1617 assert(FoundWidenCanonicalIVUser &&
1618 "Must have widened canonical IV when tail folding!");
1619 auto *WideCanonicalIV =
1620 cast(*FoundWidenCanonicalIVUser);
1622 if (UseActiveLaneMaskForControlFlow) {
1625 } else {
1628 {WideCanonicalIV, Plan.getTripCount()}, nullptr,
1629 "active.lane.mask");
1630 }
1631
1632
1633
1634
1636 HeaderMask->replaceAllUsesWith(LaneMask);
1637}
1638
1639
1640
1641
1642
1643
1644
1645
1646
1652 auto GetNewMask = [&](VPValue *OrigMask) -> VPValue * {
1653 assert(OrigMask && "Unmasked recipe when folding tail");
1654 return HeaderMask == OrigMask ? nullptr : OrigMask;
1655 };
1656
1659 VPValue *NewMask = GetNewMask(L->getMask());
1661 })
1665 })
1667 unsigned Opcode = W->getOpcode();
1669 return nullptr;
1671 })
1673 VPValue *NewMask = GetNewMask(Red->getCondOp());
1675 })
1676 .Case<VPWidenIntrinsicRecipe, VPWidenCastRecipe>(
1679 if (auto *CallR = dyn_cast(CR)) {
1680 VPID =
1682 } else {
1683 auto *CastR = cast(CR);
1685 }
1686
1687
1689 return nullptr;
1692 "Expected VP intrinsic to have mask and EVL");
1693
1698 VPID, Ops, TypeInfo.inferScalarType(CR), CR->getDebugLoc());
1699 })
1706 })
1709
1710
1711
1712
1714 m_VPValue(RHS))))
1715 return nullptr;
1716
1717
1719 Intrinsic::vp_merge, {&AllOneMask, LHS, RHS, &EVL},
1721 })
1723}
1724
1725
1731
1733 if (auto *R = dyn_cast(U))
1734 R->setOperand(1, &EVL);
1735 }
1736
1738
1741 auto *CurRecipe = cast(U);
1743 createEVLRecipe(HeaderMask, *CurRecipe, TypeInfo, *AllOneMask, EVL);
1744 if (!EVLRecipe)
1745 continue;
1746
1748 assert(NumDefVal == CurRecipe->getNumDefinedValues() &&
1749 "New recipe must define the same number of values as the "
1750 "original.");
1752 NumDefVal <= 1 &&
1753 "Only supports recipes with a single definition or without users.");
1755 if (isa<VPSingleDefRecipe, VPWidenLoadEVLRecipe>(EVLRecipe)) {
1756 VPValue *CurVPV = CurRecipe->getVPSingleValue();
1758 }
1759
1760
1762 }
1763 }
1764
1767 R->eraseFromParent();
1768 for (VPValue *Op : PossiblyDead)
1770 }
1771}
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1813 VPlan &Plan, const std::optional &MaxSafeElements) {
1815
1816
1817
1818 bool ContainsWidenInductions = any_of(
1819 Header->phis(),
1820 IsaPred<VPWidenIntOrFpInductionRecipe, VPWidenPointerInductionRecipe>);
1821 if (ContainsWidenInductions)
1822 return false;
1823
1825 VPValue *StartV = CanonicalIVPHI->getStartValue();
1826
1827
1829 EVLPhi->insertAfter(CanonicalIVPHI);
1830 VPBuilder Builder(Header, Header->getFirstNonPhi());
1831
1834 if (MaxSafeElements) {
1835
1837 ConstantInt::get(CanonicalIVPHI->getScalarType(), *MaxSafeElements));
1840 }
1843
1844 auto *CanonicalIVIncrement =
1845 cast(CanonicalIVPHI->getBackedgeValue());
1847 if (unsigned IVSize = CanonicalIVPHI->getScalarType()->getScalarSizeInBits();
1848 IVSize != 32) {
1850 IVSize < 32 ? Instruction::Trunc : Instruction::ZExt, OpVPEVL,
1851 CanonicalIVPHI->getScalarType(), CanonicalIVIncrement->getDebugLoc());
1852 OpVPEVL->insertBefore(CanonicalIVIncrement);
1853 }
1854 auto *NextEVLIV =
1855 new VPInstruction(Instruction::Add, {OpVPEVL, EVLPhi},
1856 {CanonicalIVIncrement->hasNoUnsignedWrap(),
1857 CanonicalIVIncrement->hasNoSignedWrap()},
1858 CanonicalIVIncrement->getDebugLoc(), "index.evl.next");
1859 NextEVLIV->insertBefore(CanonicalIVIncrement);
1860 EVLPhi->addOperand(NextEVLIV);
1861
1863
1864
1865
1866 CanonicalIVPHI->replaceAllUsesWith(EVLPhi);
1867 CanonicalIVIncrement->setOperand(0, CanonicalIVPHI);
1868
1870 return true;
1871}
1872
1875
1876
1878 auto CollectPoisonGeneratingInstrsInBackwardSlice([&](VPRecipeBase *Root) {
1881
1882
1883 while (!Worklist.empty()) {
1884 VPRecipeBase *CurRec = Worklist.pop_back_val();
1885
1886 if (!Visited.insert(CurRec).second)
1887 continue;
1888
1889
1890
1891
1892
1893 if (isa<VPWidenMemoryRecipe, VPInterleaveRecipe, VPScalarIVStepsRecipe,
1894 VPHeaderPHIRecipe>(CurRec))
1895 continue;
1896
1897
1898
1899
1900 if (auto *RecWithFlags = dyn_cast(CurRec)) {
1901 VPValue *A, *B;
1902 using namespace llvm::VPlanPatternMatch;
1903
1904
1905
1906
1907
1908 if (match(RecWithFlags, m_BinaryOr(m_VPValue(A), m_VPValue(B))) &&
1909 RecWithFlags->isDisjoint()) {
1910 VPBuilder Builder(RecWithFlags);
1911 VPInstruction *New = Builder.createOverflowingOp(
1912 Instruction::Add, {A, B}, {false, false},
1913 RecWithFlags->getDebugLoc());
1914 New->setUnderlyingValue(RecWithFlags->getUnderlyingValue());
1915 RecWithFlags->replaceAllUsesWith(New);
1916 RecWithFlags->eraseFromParent();
1917 CurRec = New;
1918 } else
1919 RecWithFlags->dropPoisonGeneratingFlags();
1920 } else {
1921 Instruction *Instr = dyn_cast_or_null(
1922 CurRec->getVPSingleValue()->getUnderlyingValue());
1923 (void)Instr;
1924 assert((!Instr || !Instr->hasPoisonGeneratingFlags()) &&
1925 "found instruction with poison generating flags not covered by "
1926 "VPRecipeWithIRFlags");
1927 }
1928
1929
1930 for (VPValue *Operand : CurRec->operands())
1931 if (VPRecipeBase *OpDef = Operand->getDefiningRecipe())
1933 }
1934 });
1935
1936
1937
1938
1940 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(Iter)) {
1942 if (auto *WidenRec = dyn_cast(&Recipe)) {
1943 Instruction &UnderlyingInstr = WidenRec->getIngredient();
1944 VPRecipeBase *AddrDef = WidenRec->getAddr()->getDefiningRecipe();
1945 if (AddrDef && WidenRec->isConsecutive() &&
1946 BlockNeedsPredication(UnderlyingInstr.getParent()))
1947 CollectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
1948 } else if (auto *InterleaveRec = dyn_cast(&Recipe)) {
1949 VPRecipeBase *AddrDef = InterleaveRec->getAddr()->getDefiningRecipe();
1950 if (AddrDef) {
1951
1953 InterleaveRec->getInterleaveGroup();
1954 bool NeedPredication = false;
1955 for (int I = 0, NumMembers = InterGroup->getNumMembers();
1958 if (Member)
1959 NeedPredication |= BlockNeedsPredication(Member->getParent());
1960 }
1961
1962 if (NeedPredication)
1963 CollectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
1964 }
1965 }
1966 }
1967 }
1968}
1969
1973 &InterleaveGroups,
1974 VPRecipeBuilder &RecipeBuilder, bool ScalarEpilogueAllowed) {
1975 if (InterleaveGroups.empty())
1976 return;
1977
1978
1979
1980
1983 for (const auto *IG : InterleaveGroups) {
1985 for (unsigned i = 0; i < IG->getFactor(); ++i)
1986 if (auto *SI = dyn_cast_or_null(IG->getMember(i))) {
1987 auto *StoreR = cast(RecipeBuilder.getRecipe(SI));
1988 StoredValues.push_back(StoreR->getStoredValue());
1989 }
1990
1991 bool NeedsMaskForGaps =
1992 IG->requiresScalarEpilogue() && !ScalarEpilogueAllowed;
1993
1994 Instruction *IRInsertPos = IG->getInsertPos();
1995 auto *InsertPos =
1996 cast(RecipeBuilder.getRecipe(IRInsertPos));
1997
1998
1999 auto *Start =
2000 cast(RecipeBuilder.getRecipe(IG->getMember(0)));
2004
2005
2006 bool InBounds = false;
2007 if (auto *Gep = dyn_cast(
2009 InBounds = Gep->isInBounds();
2010
2011
2012
2013
2014
2015 assert(IG->getIndex(IRInsertPos) != 0 &&
2016 "index of insert position shouldn't be zero");
2020 IG->getIndex(IRInsertPos),
2021 true);
2023 ConstantInt::get(IRInsertPos->getParent()->getContext(), -Offset));
2025 Addr = InBounds ? B.createInBoundsPtrAdd(InsertPos->getAddr(), OffsetVPV)
2026 : B.createPtrAdd(InsertPos->getAddr(), OffsetVPV);
2027 }
2029 InsertPos->getMask(), NeedsMaskForGaps);
2030 VPIG->insertBefore(InsertPos);
2031
2032 unsigned J = 0;
2033 for (unsigned i = 0; i < IG->getFactor(); ++i)
2034 if (Instruction *Member = IG->getMember(i)) {
2036 if (!Member->getType()->isVoidTy()) {
2039 J++;
2040 }
2042 }
2043 }
2044}
2045
2047 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(
2050 if (!isa<VPCanonicalIVPHIRecipe, VPEVLBasedIVPHIRecipe>(&R))
2051 continue;
2052 auto *PhiR = cast(&R);
2054 isa(PhiR) ? "index" : "evl.based.iv";
2055 auto *ScalarR =
2056 new VPScalarPHIRecipe(PhiR->getStartValue(), PhiR->getBackedgeValue(),
2057 PhiR->getDebugLoc(), Name);
2058 ScalarR->insertBefore(PhiR);
2059 PhiR->replaceAllUsesWith(ScalarR);
2060 PhiR->eraseFromParent();
2061 }
2062 }
2063}
2064
2069 auto *LatchVPBB = cast(LoopRegion->getExiting());
2070 VPBuilder Builder(LatchVPBB->getTerminator());
2072 VPValue *IsEarlyExitTaken = nullptr;
2073
2074
2075
2076
2077
2078 auto *EarlyExitingBranch =
2079 cast(UncountableExitingBlock->getTerminator());
2080 BasicBlock *TrueSucc = EarlyExitingBranch->getSuccessor(0);
2081 BasicBlock *FalseSucc = EarlyExitingBranch->getSuccessor(1);
2082
2083
2084
2085
2086
2089 VPEarlyExitBlock = cast(MiddleVPBB->getSuccessors()[0]);
2090 } else {
2092 !OrigLoop->contains(TrueSucc) ? TrueSucc : FalseSucc);
2093 }
2094
2096 OrigLoop->contains(TrueSucc) ? TrueSucc : FalseSucc);
2097 auto *EarlyExitTakenCond = Builder.createNot(EarlyExitNotTakenCond);
2098 IsEarlyExitTaken =
2100
2105
2106 VPBuilder MiddleBuilder(NewMiddle);
2108
2109
2110
2111
2112 auto *LatchExitingBranch = cast(LatchVPBB->getTerminator());
2114 "Unexpected terminator");
2115 auto *IsLatchExitTaken =
2117 LatchExitingBranch->getOperand(1));
2119 Instruction::Or, {IsEarlyExitTaken, IsLatchExitTaken});
2122}
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
iv Induction Variable Users
static bool mergeBlocksIntoPredecessors(Loop &L, DominatorTree &DT, LoopInfo &LI, MemorySSAUpdater *MSSAU, ScalarEvolution &SE)
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file implements the TypeSwitch template, which mimics a switch() statement whose cases are type ...
This file implements dominator tree analysis for a single level of a VPlan's H-CFG.
static bool sinkScalarOperands(VPlan &Plan)
static void removeRedundantInductionCasts(VPlan &Plan)
Remove redundant casts of inductions.
static VPScalarIVStepsRecipe * createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind, Instruction::BinaryOps InductionOpcode, FPMathOperator *FPBinOp, Instruction *TruncI, VPValue *StartV, VPValue *Step, DebugLoc DL, VPBuilder &Builder)
static bool sinkRecurrenceUsersAfterPrevious(VPFirstOrderRecurrencePHIRecipe *FOR, VPRecipeBase *Previous, VPDominatorTree &VPDT)
Sink users of FOR after the recipe defining the previous value Previous of the recurrence.
static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan)
static VPActiveLaneMaskPHIRecipe * addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan, bool DataAndControlFlowWithoutRuntimeCheck)
static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL)
Replace recipes with their EVL variants.
static bool isDeadRecipe(VPRecipeBase &R)
Returns true if R is dead and can be removed.
static void legalizeAndOptimizeInductions(VPlan &Plan)
Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd (IndStart, ScalarIVSteps (0,...
static void addReplicateRegions(VPlan &Plan)
static void simplifyRecipes(VPlan &Plan, Type *CanonicalIVTy)
Try to simplify the recipes in Plan.
static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo)
Try to simplify recipe R.
static void removeRedundantExpandSCEVRecipes(VPlan &Plan)
Remove redundant EpxandSCEVRecipes in Plan's entry block by replacing them with already existing reci...
static SmallVector< VPValue * > collectAllHeaderMasks(VPlan &Plan)
Collect all VPValues representing a header mask through the (ICMP_ULE, WideCanonicalIV,...
static bool hoistPreviousBeforeFORUsers(VPFirstOrderRecurrencePHIRecipe *FOR, VPRecipeBase *Previous, VPDominatorTree &VPDT)
Try to hoist Previous and its operands before all users of FOR.
static VPWidenInductionRecipe * getOptimizableIVOf(VPValue *VPV)
Check if VPV is an untruncated wide induction, either before or after the increment.
static SmallVector< VPUser * > collectUsersRecursively(VPValue *V)
static void recursivelyDeleteDeadRecipes(VPValue *V)
static VPRegionBlock * createReplicateRegion(VPReplicateRecipe *PredRecipe, VPlan &Plan)
static VPBasicBlock * getPredicatedThenBlock(VPRegionBlock *R)
If R is a triangle region, return the 'then' block of the triangle.
static VPRecipeBase * createEVLRecipe(VPValue *HeaderMask, VPRecipeBase &CurRecipe, VPTypeAnalysis &TypeInfo, VPValue &AllOneMask, VPValue &EVL)
Try to convert CurRecipe to a corresponding EVL-based recipe.
VPValue * getPredicatedMask(VPRegionBlock *R)
If R is a region with a VPBranchOnMaskRecipe in the entry block, return the mask.
static void removeRedundantCanonicalIVs(VPlan &Plan)
Try to replace VPWidenCanonicalIVRecipes with a widened canonical IV recipe, if it exists.
This file provides utility VPlan to VPlan transformations.
This file contains the declarations of the Vectorization Plan base classes:
static const uint32_t IV[8]
Class for arbitrary precision integers.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
This class represents a function call, abstracting a target machine's calling convention.
@ ICMP_ULT
unsigned less than
@ ICMP_ULE
unsigned less or equal
static ConstantInt * getTrue(LLVMContext &Context)
This class represents an Operation in the Expression.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
void recalculate(ParentType &Func)
recalculate - compute a dominator tree for the given function
static constexpr ElementCount getFixed(ScalarTy MinVal)
Utility class for floating point operations which can have information about relaxed accuracy require...
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
A struct for saving information about induction variables.
InductionKind
This enum represents the kinds of inductions that we support.
@ IK_PtrInduction
Pointer induction var. Step = C.
@ IK_IntInduction
Integer induction variable. Step = C.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
The group of interleaved loads/stores sharing the same stride and close to each other.
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
uint32_t getNumMembers() const
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
BlockT * getUniqueExitBlock() const
If getUniqueExitBlocks would return exactly one block, return that block.
Represents a single loop in the control flow graph.
This class implements a map that also provides access to all stored values in a deterministic order.
bool contains(const KeyT &Key) const
ValueT lookup(const KeyT &Key) const
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
RecurKind getRecurrenceKind() const
This class represents an analyzed expression in the program.
bool isZero() const
Return true if the expression is a constant zero.
Type * getType() const
Return the LLVM type of this SCEV expression.
The main scalar evolution driver.
const SCEV * getElementCount(Type *Ty, ElementCount EC)
bool isKnownPredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
LLVMContext & getContext() const
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
size_type size() const
Determine the number of elements in the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Provides information about what library functions are available for the current target.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
This class implements a switch-like dispatch statement for a value of 'T' using dyn_cast functionalit...
TypeSwitch< T, ResultT > & Case(CallableT &&caseFn)
Add a case on the given type.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isPointerTy() const
True if this is an instance of PointerType.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
A recipe for generating the active lane mask for the vector loop that is used to predicate the vector...
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
VPRegionBlock * getEnclosingLoopRegion()
VPBasicBlock * splitAt(iterator SplitAt)
Split current block at SplitAt by inserting a new block between the current block and its successors ...
VPRecipeBase * getTerminator()
If the block has multiple successors, return the branch recipe terminating the block.
const VPRecipeBase & back() const
A recipe for vectorizing a phi-node as a sequence of mask-based select instructions.
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
const VPBasicBlock * getExitingBasicBlock() const
void swapSuccessors()
Swap successors of the block. The block must have exactly 2 successors.
VPBlockBase * getSinglePredecessor() const
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleHierarchicalPredecessor()
VPBlockBase * getSingleSuccessor() const
const VPBlocksTy & getSuccessors() const
static void insertOnEdge(VPBlockBase *From, VPBlockBase *To, VPBlockBase *BlockPtr)
Inserts BlockPtr on the edge between From and To.
static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBases IfTrue and IfFalse after BlockPtr.
static void connectBlocks(VPBlockBase *From, VPBlockBase *To, unsigned PredIdx=-1u, unsigned SuccIdx=-1u)
Connect VPBlockBases From and To bi-directionally.
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
A recipe for generating conditional branches on the bits of a mask.
RAII object that stores the current insertion point and restores it when the object is destroyed.
VPlan-based builder utility analogous to IRBuilder.
VPValue * createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B, DebugLoc DL={}, const Twine &Name="")
Create a new ICmp VPInstruction with predicate Pred and operands A and B.
VPDerivedIVRecipe * createDerivedIV(InductionDescriptor::InductionKind Kind, FPMathOperator *FPBinOp, VPValue *Start, VPValue *Current, VPValue *Step, const Twine &Name="")
Convert the input value Current to the corresponding value of an induction with Start and Step values...
VPInstruction * createPtrAdd(VPValue *Ptr, VPValue *Offset, DebugLoc DL={}, const Twine &Name="")
VPScalarCastRecipe * createScalarCast(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, DebugLoc DL)
static VPBuilder getToInsertAfter(VPRecipeBase *R)
Create a VPBuilder to insert after R.
VPScalarIVStepsRecipe * createScalarIVSteps(Instruction::BinaryOps InductionOpcode, FPMathOperator *FPBinOp, VPValue *IV, VPValue *Step)
VPInstruction * createOverflowingOp(unsigned Opcode, std::initializer_list< VPValue * > Operands, VPRecipeWithIRFlags::WrapFlagsTy WrapFlags, DebugLoc DL={}, const Twine &Name="")
VPInstruction * createNaryOp(unsigned Opcode, ArrayRef< VPValue * > Operands, Instruction *Inst=nullptr, const Twine &Name="")
Create an N-ary operation with Opcode, Operands and set Inst as its underlying Instruction.
VPValue * createNot(VPValue *Operand, DebugLoc DL={}, const Twine &Name="")
VPValue * createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal, DebugLoc DL={}, const Twine &Name="", std::optional< FastMathFlags > FMFs=std::nullopt)
void setInsertPoint(VPBasicBlock *TheBB)
This specifies that created VPInstructions should be appended to the end of the specified block.
Canonical scalar induction phi of the vector loop.
Type * getScalarType() const
Returns the scalar type of the induction.
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
ArrayRef< VPValue * > definedValues()
Returns an ArrayRef of the values defined by the VPDef.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
Template specialization of the standard LLVM dominator tree utility for VPBlockBases.
bool properlyDominates(const VPRecipeBase *A, const VPRecipeBase *B)
Returns true if A properly dominates B.
A recipe for generating the phi node for the current index of elements, adjusted in accordance with E...
A special type of VPBasicBlock that wraps an existing IR basic block.
This is a concrete Recipe that models a single VPlan-level instruction.
@ FirstOrderRecurrenceSplice
@ CanonicalIVIncrementForPart
@ CalculateTripCountMinusVF
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
static std::optional< unsigned > getMaskParamPos(Intrinsic::ID IntrinsicID)
static std::optional< unsigned > getVectorLengthParamPos(Intrinsic::ID IntrinsicID)
static Intrinsic::ID getForOpcode(unsigned OC)
The llvm.vp.* intrinsics for this instruction Opcode.
static Intrinsic::ID getForIntrinsic(Intrinsic::ID Id)
The llvm.vp.
VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when control converges back from ...
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
bool mayReadOrWriteMemory() const
Returns true if the recipe may read from or write to memory.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
VPBasicBlock * getParent()
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Helper class to create VPRecipies from IR instructions.
VPValue * getBlockInMask(BasicBlock *BB) const
Returns the entry mask for the block BB.
VPRecipeBase * getRecipe(Instruction *I)
Return the recipe created for given ingredient.
A recipe to represent inloop reduction operations with vector-predication intrinsics,...
A recipe to represent inloop reduction operations, performing a reduction on a vector operand into a ...
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
const VPBlockBase * getExiting() const
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
VPValue * getMask()
Return the mask of a predicated VPReplicateRecipe.
VPScalarCastRecipe is a recipe to create scalar cast instructions.
A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...
Recipe to generate a scalar PHI.
VPSingleDef is a base class for recipes for modeling a sequence of one or more output IR that define ...
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
An analysis for type-inference for VPValues.
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void setOperand(unsigned I, VPValue *New)
operand_iterator op_end()
operand_iterator op_begin()
void addOperand(VPValue *Operand)
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
void replaceAllUsesWith(VPValue *New)
unsigned getNumUsers() const
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
bool isLiveIn() const
Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
void replaceUsesWithIf(VPValue *New, llvm::function_ref< bool(VPUser &U, unsigned Idx)> ShouldReplace)
Go through the uses list for this VPValue and make each use point to New if the callback ShouldReplac...
A Recipe for widening the canonical induction variable of the vector loop.
VPWidenCastRecipe is a recipe to create vector cast instructions.
A recipe for widening operations with vector-predication intrinsics with explicit vector length (EVL)...
A recipe for handling GEP instructions.
Base class for widened induction (VPWidenIntOrFpInductionRecipe and VPWidenPointerInductionRecipe),...
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
A recipe for widening vector intrinsics.
VPValue * getMask() const
Return the mask used by this recipe.
VPWidenRecipe is a recipe for producing a widened instruction using the opcode and operands of the re...
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
VPBasicBlock * getEntry()
VPRegionBlock * createVPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exiting, const std::string &Name="", bool IsReplicator=false)
Create a new VPRegionBlock with Entry, Exiting and Name.
VPValue & getVF()
Returns the VF of the vector loop region.
VPValue * getTripCount() const
The trip count of the original loop.
bool hasVF(ElementCount VF)
bool hasUF(unsigned UF) const
void setVF(ElementCount VF)
auto getExitBlocks()
Return an iterator range over the VPIRBasicBlock wrapping the exit blocks of the VPlan,...
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
const VPBasicBlock * getMiddleBlock() const
Returns the 'middle' block of the plan, that is the block that selects whether to execute the scalar ...
VPBasicBlock * createVPBasicBlock(const Twine &Name, VPRecipeBase *Recipe=nullptr)
Create a new VPBasicBlock with Name and containing Recipe if present.
VPIRBasicBlock * createVPIRBasicBlock(BasicBlock *IRBB)
Create a VPIRBasicBlock from IRBB containing VPIRInstructions for all instructions in IRBB,...
VPValue * getOrAddLiveIn(Value *V)
Gets the live-in VPValue for V or adds a new live-in (if none exists yet) for V.
bool hasScalarVFOnly() const
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
VPBasicBlock * getVectorPreheader()
Returns the preheader of the vector loop region, if one exists, or null otherwise.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
StringRef getName() const
Return a constant reference to the value's name.
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
CastInst_match< OpTy, TruncInst > m_Trunc(const OpTy &Op)
Matches Trunc.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
BinaryOp_match< LHS, RHS, Instruction::Mul, true > m_c_Mul(const LHS &L, const RHS &R)
Matches a Mul with LHS and RHS in either order.
bool isUniformAfterVectorization(const VPValue *VPV)
Returns true if VPV is uniform after vectorization.
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
const SCEV * getSCEVExprForVPValue(VPValue *V, ScalarEvolution &SE)
Return the SCEV expression for V.
bool onlyFirstLaneUsed(const VPValue *Def)
Returns true if only the first lane of Def is used.
bool isHeaderMask(const VPValue *V, VPlan &Plan)
Return true if V is a header mask in Plan.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
void sort(IteratorTy Start, IteratorTy End)
std::unique_ptr< VPlan > VPlanPtr
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
bool isa(const From &Val)
isa - Return true if the parameter to the template is an instance of one of the template type argu...
RecurKind
These are the kinds of recurrences that we support.
@ Mul
Product of integers.
DWARFExpression::Operation Op
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Type * getLoadStoreType(const Value *I)
A helper function that returns the type of a load or store instruction.
@ DataAndControlFlowWithoutRuntimeCheck
Use predicate to control both data and control flow, but modify the trip count so that a runtime over...
@ Default
The result values are uniform if and only if all operands are uniform.
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
A recipe for handling first-order recurrence phis.
A recipe for widening load operations with vector-predication intrinsics, using the address to load f...
A recipe for widening load operations, using the address to load from and an optional mask.
A recipe for widening select instructions.
A recipe for widening store operations with vector-predication intrinsics, using the value to store,...
A recipe for widening store operations, using the stored value, the address to store to and an option...
static void handleUncountableEarlyExit(VPlan &Plan, ScalarEvolution &SE, Loop *OrigLoop, BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder)
Update Plan to account for the uncountable early exit block in UncountableExitingBlock by.
static void optimizeInductionExitUsers(VPlan &Plan, DenseMap< VPValue *, VPValue * > &EndValues)
If there's a single exit block, optimize its phi recipes that use exiting IV values by feeding them p...
static void createAndOptimizeReplicateRegions(VPlan &Plan)
Wrap predicated VPReplicateRecipes with a mask operand in an if-then region block and remove the mask...
static void convertToConcreteRecipes(VPlan &Plan)
Lower abstract recipes to concrete ones, that can be codegen'd.
static void dropPoisonGeneratingRecipes(VPlan &Plan, function_ref< bool(BasicBlock *)> BlockNeedsPredication)
Drop poison flags from recipes that may generate a poison value that is used after vectorization,...
static void createInterleaveGroups(VPlan &Plan, const SmallPtrSetImpl< const InterleaveGroup< Instruction > * > &InterleaveGroups, VPRecipeBuilder &RecipeBuilder, bool ScalarEpilogueAllowed)
static void removeDeadRecipes(VPlan &Plan)
Remove dead recipes from Plan.
static void clearReductionWrapFlags(VPlan &Plan)
Clear NSW/NUW flags from reduction instructions if necessary.
static bool tryAddExplicitVectorLength(VPlan &Plan, const std::optional< unsigned > &MaxEVLSafeElements)
Add a VPEVLBasedIVPHIRecipe and related recipes to Plan and replaces all uses except the canonical IV...
static void VPInstructionsToVPRecipes(VPlanPtr &Plan, function_ref< const InductionDescriptor *(PHINode *)> GetIntOrFpInductionDescriptor, ScalarEvolution &SE, const TargetLibraryInfo &TLI)
Replaces the VPInstructions in Plan with corresponding widen recipes.
static void addActiveLaneMask(VPlan &Plan, bool UseActiveLaneMaskForControlFlow, bool DataAndControlFlowWithoutRuntimeCheck)
Replace (ICMP_ULE, wide canonical IV, backedge-taken-count) checks with an (active-lane-mask recipe,...
static void optimize(VPlan &Plan)
Apply VPlan-to-VPlan optimizations to Plan, including induction recipe optimizations,...
static void truncateToMinimalBitwidths(VPlan &Plan, const MapVector< Instruction *, uint64_t > &MinBWs)
Insert truncates and extends for any truncated recipe.
static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder)
Try to have all users of fixed-order recurrences appear after the recipe defining their previous valu...
static void optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
Optimize Plan based on BestVF and BestUF.