LLVM: lib/Transforms/Vectorize/VPlan.h Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
25#define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
26
42#include
43#include
44#include
45#include
46#include
47#include
48
49namespace llvm {
50
70
72
74typedef unsigned ID;
75}
76
77using VPlanPtr = std::unique_ptr;
78
79
80
83
84 const unsigned char SubclassID;
85
86
87 std::string Name;
88
89
90
92
93
95
96
98
99
100
101 VPlan *Plan = nullptr;
102
103
107 }
108
109
110 void appendPredecessor(VPBlockBase *Predecessor) {
111 assert(Predecessor && "Cannot add nullptr predecessor!");
112 Predecessors.push_back(Predecessor);
113 }
114
115
116 void removePredecessor(VPBlockBase *Predecessor) {
117 auto Pos = find(Predecessors, Predecessor);
118 assert(Pos && "Predecessor does not exist");
119 Predecessors.erase(Pos);
120 }
121
122
123 void removeSuccessor(VPBlockBase *Successor) {
124 auto Pos = find(Successors, Successor);
125 assert(Pos && "Successor does not exist");
126 Successors.erase(Pos);
127 }
128
129
130
131 void replacePredecessor(VPBlockBase *Old, VPBlockBase *New) {
132 auto I = find(Predecessors, Old);
134 assert(Old->getParent() == New->getParent() &&
135 "replaced predecessor must have the same parent");
137 }
138
139
140
141 void replaceSuccessor(VPBlockBase *Old, VPBlockBase *New) {
142 auto I = find(Successors, Old);
144 assert(Old->getParent() == New->getParent() &&
145 "replaced successor must have the same parent");
147 }
148
149protected:
150 VPBlockBase(const unsigned char SC, const std::string &N)
151 : SubclassID(SC), Name(N) {}
152
153public:
154
155
156
157
158 using VPBlockTy = enum { VPRegionBlockSC, VPBasicBlockSC, VPIRBasicBlockSC };
159
161
163
164 const std::string &getName() const { return Name; }
165
167
168
169
170
172
175
176
177 VPlan *getPlan();
178 const VPlan *getPlan() const;
179
180
181
182 void setPlan(VPlan *ParentPlan);
183
185
186
187
188
189 const VPBasicBlock *getEntryBasicBlock() const;
191
192
193
194
195 const VPBasicBlock *getExitingBasicBlock() const;
197
200
203
206
207
208
210 return (Successors.size() == 1 ? *Successors.begin() : nullptr);
211 }
212
213
214
216 return (Predecessors.size() == 1 ? *Predecessors.begin() : nullptr);
217 }
218
221
222
224
225
226
227
228
229 VPBlockBase *getEnclosingBlockWithSuccessors();
230
231
232
233
234 VPBlockBase *getEnclosingBlockWithPredecessors();
235
236
237
238
239
240
241
245
246
247
251
252
253
254
255
256
257
261
262
263
267
268
269
270
272 assert(Successors.empty() && "Setting one successor when others exist.");
274 "connected blocks must have the same parent");
276 }
277
278
279
280
281
283 assert(Successors.empty() && "Setting two successors when others exist.");
284 appendSuccessor(IfTrue);
285 appendSuccessor(IfFalse);
286 }
287
288
289
290
292 assert(Predecessors.empty() && "Block predecessors already set.");
293 for (auto *Pred : NewPreds)
294 appendPredecessor(Pred);
295 }
296
297
298
299
301 assert(Successors.empty() && "Block successors already set.");
302 for (auto *Succ : NewSuccs)
303 appendSuccessor(Succ);
304 }
305
306
308
309
311
312
313
315 assert(Predecessors.size() == 2 && "must have 2 predecessors to swap");
316 std::swap(Predecessors[0], Predecessors[1]);
317 }
318
319
320
321
323 assert(Successors.size() == 2 && "must have 2 successors to swap");
324 std::swap(Successors[0], Successors[1]);
325 }
326
327
330 "must have Pred exactly once in Predecessors");
331 return std::distance(Predecessors.begin(), find(Predecessors, Pred));
332 }
333
334
337 "must have Succ exactly once in Successors");
338 return std::distance(Successors.begin(), find(Successors, Succ));
339 }
340
341
342
344
345
347
348#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
352
353
354
355
356
357
358
361
362
364
365
366
368
369
371#endif
372
373
374
375
377};
378
379
380
381
382
383
388 friend VPBasicBlock;
390
391
392 VPBasicBlock *Parent = nullptr;
393
394
396
397public:
401
403
404
406
407
409 const VPBasicBlock *getParent() const { return Parent; }
410
411
414
415
416
418
419
420
421
423
424
425
427
428
430
431
432
434
435
436
438
439
440
441
443
444
445
447
448
449
450
452
453
455
456 return true;
457 }
458
460
461
463
464
465 bool isPhi() const;
466
467
468 bool mayReadFromMemory() const;
469
470
471 bool mayWriteToMemory() const;
472
473
477
478
480
481
482 bool isScalarCast() const;
483
484
486
487#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
488
491#endif
492
493protected:
494
495
496
499
500#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
501
502
505#endif
506};
507
508
509#define VP_CLASSOF_IMPL(VPDefID) \
510 static inline bool classof(const VPDef *D) { \
511 return D->getVPDefID() == VPDefID; \
512 } \
513 static inline bool classof(const VPValue *V) { \
514 auto *R = V->getDefiningRecipe(); \
515 return R && R->getVPDefID() == VPDefID; \
516 } \
517 static inline bool classof(const VPUser *U) { \
518 auto *R = dyn_cast(U); \
519 return R && R->getVPDefID() == VPDefID; \
520 } \
521 static inline bool classof(const VPRecipeBase *R) { \
522 return R->getVPDefID() == VPDefID; \
523 } \
524 static inline bool classof(const VPSingleDefRecipe *R) { \
525 return R->getVPDefID() == VPDefID; \
526 }
527
528
529
530
532public:
536
540
542 switch (R->getVPDefID()) {
543 case VPRecipeBase::VPDerivedIVSC:
544 case VPRecipeBase::VPEVLBasedIVPHISC:
545 case VPRecipeBase::VPExpandSCEVSC:
546 case VPRecipeBase::VPExpressionSC:
547 case VPRecipeBase::VPInstructionSC:
548 case VPRecipeBase::VPReductionEVLSC:
549 case VPRecipeBase::VPReductionSC:
550 case VPRecipeBase::VPReplicateSC:
551 case VPRecipeBase::VPScalarIVStepsSC:
552 case VPRecipeBase::VPVectorPointerSC:
553 case VPRecipeBase::VPVectorEndPointerSC:
554 case VPRecipeBase::VPWidenCallSC:
555 case VPRecipeBase::VPWidenCanonicalIVSC:
556 case VPRecipeBase::VPWidenCastSC:
557 case VPRecipeBase::VPWidenGEPSC:
558 case VPRecipeBase::VPWidenIntrinsicSC:
559 case VPRecipeBase::VPWidenSC:
560 case VPRecipeBase::VPWidenSelectSC:
561 case VPRecipeBase::VPBlendSC:
562 case VPRecipeBase::VPPredInstPHISC:
563 case VPRecipeBase::VPCanonicalIVPHISC:
564 case VPRecipeBase::VPActiveLaneMaskPHISC:
565 case VPRecipeBase::VPFirstOrderRecurrencePHISC:
566 case VPRecipeBase::VPWidenPHISC:
567 case VPRecipeBase::VPWidenIntOrFpInductionSC:
568 case VPRecipeBase::VPWidenPointerInductionSC:
569 case VPRecipeBase::VPReductionPHISC:
570 return true;
571 case VPRecipeBase::VPBranchOnMaskSC:
572 case VPRecipeBase::VPInterleaveEVLSC:
573 case VPRecipeBase::VPInterleaveSC:
574 case VPRecipeBase::VPIRInstructionSC:
575 case VPRecipeBase::VPWidenLoadEVLSC:
576 case VPRecipeBase::VPWidenLoadSC:
577 case VPRecipeBase::VPWidenStoreEVLSC:
578 case VPRecipeBase::VPWidenStoreSC:
579 case VPRecipeBase::VPHistogramSC:
580
581
582 return false;
583 }
585 }
586
591
593
594
601
602#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
603
605#endif
606};
607
608
610 enum class OperationType : unsigned char {
611 Cmp,
612 FCmp,
613 OverflowingBinOp,
614 Trunc,
615 DisjointOp,
616 PossiblyExactOp,
617 GEPOp,
618 FPMathOp,
619 NonNegOp,
620 Other
621 };
622
623public:
630
637
642
647
648private:
649 struct ExactFlagsTy {
650 char IsExact : 1;
651 };
652 struct FastMathFlagsTy {
653 char AllowReassoc : 1;
654 char NoNaNs : 1;
655 char NoInfs : 1;
656 char NoSignedZeros : 1;
657 char AllowReciprocal : 1;
658 char AllowContract : 1;
659 char ApproxFunc : 1;
660
662 };
663
664
665 struct FCmpFlagsTy {
667 FastMathFlagsTy FMFs;
668 };
669
670 OperationType OpType;
671
672 union {
683 };
684
685public:
687
690 OpType = OperationType::FCmp;
691 FCmpFlags.Pred = FCmp->getPredicate();
692 FCmpFlags.FMFs = FCmp->getFastMathFlags();
694 OpType = OperationType::Cmp;
697 OpType = OperationType::DisjointOp;
700 OpType = OperationType::OverflowingBinOp;
701 WrapFlags = {Op->hasNoUnsignedWrap(), Op->hasNoSignedWrap()};
703 OpType = OperationType::Trunc;
704 TruncFlags = {Op->hasNoUnsignedWrap(), Op->hasNoSignedWrap()};
706 OpType = OperationType::PossiblyExactOp;
709 OpType = OperationType::GEPOp;
712 OpType = OperationType::NonNegOp;
715 OpType = OperationType::FPMathOp;
716 FMFs = Op->getFastMathFlags();
717 } else {
718 OpType = OperationType::Other;
720 }
721 }
722
725
727 : OpType(OperationType::FCmp) {
730 }
731
734
737
739
742
745
748
750 OpType = Other.OpType;
752 }
753
754
755
757
758
760
761
762 switch (OpType) {
763 case OperationType::OverflowingBinOp:
766 break;
767 case OperationType::Trunc:
770 break;
771 case OperationType::DisjointOp:
773 break;
774 case OperationType::PossiblyExactOp:
776 break;
777 case OperationType::GEPOp:
779 break;
780 case OperationType::FPMathOp:
781 case OperationType::FCmp:
782 getFMFsRef().NoNaNs = false;
783 getFMFsRef().NoInfs = false;
784 break;
785 case OperationType::NonNegOp:
787 break;
788 case OperationType::Cmp:
789 case OperationType::Other:
790 break;
791 }
792 }
793
794
796 switch (OpType) {
797 case OperationType::OverflowingBinOp:
798 I.setHasNoUnsignedWrap(WrapFlags.HasNUW);
799 I.setHasNoSignedWrap(WrapFlags.HasNSW);
800 break;
801 case OperationType::Trunc:
802 I.setHasNoUnsignedWrap(TruncFlags.HasNUW);
804 break;
805 case OperationType::DisjointOp:
807 break;
808 case OperationType::PossiblyExactOp:
810 break;
811 case OperationType::GEPOp:
813 break;
814 case OperationType::FPMathOp:
815 case OperationType::FCmp: {
816 const FastMathFlagsTy &F = getFMFsRef();
817 I.setHasAllowReassoc(F.AllowReassoc);
820 I.setHasNoSignedZeros(F.NoSignedZeros);
821 I.setHasAllowReciprocal(F.AllowReciprocal);
822 I.setHasAllowContract(F.AllowContract);
823 I.setHasApproxFunc(F.ApproxFunc);
824 break;
825 }
826 case OperationType::NonNegOp:
828 break;
829 case OperationType::Cmp:
830 case OperationType::Other:
831 break;
832 }
833 }
834
836 assert((OpType == OperationType::Cmp || OpType == OperationType::FCmp) &&
837 "recipe doesn't have a compare predicate");
839 }
840
842 assert((OpType == OperationType::Cmp || OpType == OperationType::FCmp) &&
843 "recipe doesn't have a compare predicate");
844 if (OpType == OperationType::FCmp)
846 else
848 }
849
851
852
854 return OpType == OperationType::Cmp || OpType == OperationType::FCmp;
855 }
856
857
859 return OpType == OperationType::FPMathOp || OpType == OperationType::FCmp;
860 }
861
863
864
865 bool hasNonNegFlag() const { return OpType == OperationType::NonNegOp; }
866
868 assert(OpType == OperationType::NonNegOp &&
869 "recipe doesn't have a NNEG flag");
871 }
872
874 switch (OpType) {
875 case OperationType::OverflowingBinOp:
877 case OperationType::Trunc:
879 default:
881 }
882 }
883
885 switch (OpType) {
886 case OperationType::OverflowingBinOp:
888 case OperationType::Trunc:
890 default:
892 }
893 }
894
896 assert(OpType == OperationType::DisjointOp &&
897 "recipe cannot have a disjoing flag");
899 }
900
901private:
902
903 FastMathFlagsTy &getFMFsRef() {
904 return OpType == OperationType::FCmp ? FCmpFlags.FMFs : FMFs;
905 }
906 const FastMathFlagsTy &getFMFsRef() const {
907 return OpType == OperationType::FCmp ? FCmpFlags.FMFs : FMFs;
908 }
909
910public:
911#if !defined(NDEBUG)
912
914#endif
915
916#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
917 void printFlags(raw_ostream &O) const;
918#endif
919};
920
921
922
928
930 return R->getVPDefID() == VPRecipeBase::VPInstructionSC ||
931 R->getVPDefID() == VPRecipeBase::VPWidenSC ||
932 R->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
933 R->getVPDefID() == VPRecipeBase::VPWidenCallSC ||
934 R->getVPDefID() == VPRecipeBase::VPWidenCastSC ||
935 R->getVPDefID() == VPRecipeBase::VPWidenIntrinsicSC ||
936 R->getVPDefID() == VPRecipeBase::VPWidenSelectSC ||
937 R->getVPDefID() == VPRecipeBase::VPReductionSC ||
938 R->getVPDefID() == VPRecipeBase::VPReductionEVLSC ||
939 R->getVPDefID() == VPRecipeBase::VPReplicateSC ||
940 R->getVPDefID() == VPRecipeBase::VPVectorEndPointerSC ||
941 R->getVPDefID() == VPRecipeBase::VPVectorPointerSC;
942 }
943
948
953
955
960
962
963
966};
967
968
969
971protected:
972
973
975
976
978};
979
980
981
984
985public:
987
988
989
991
992
994
996
997
999
1000
1001
1003 auto It =
1004 llvm::find_if(Metadata, [Kind](const std::pair<unsigned, MDNode *> &P) {
1005 return P.first == Kind;
1006 });
1007 if (It != Metadata.end())
1008 It->second = Node;
1009 else
1010 Metadata.emplace_back(Kind, Node);
1011 }
1012
1013
1014
1016
1017
1019 auto It =
1020 find_if(Metadata, [Kind](const auto &P) { return P.first == Kind; });
1021 return It != Metadata.end() ? It->second : nullptr;
1022 }
1023
1024#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1025
1027#endif
1028};
1029
1030
1031
1032
1033
1038
1039public:
1040
1041 enum {
1043 Instruction::OtherOpsEnd + 1,
1044
1048
1049
1050
1051
1055
1060
1061
1062
1064
1065
1067
1068
1069
1070
1072
1073
1077
1079
1081
1082
1083
1086
1087
1088
1090
1091
1093
1094
1095
1096
1097
1099
1100
1101
1102
1103
1105
1106
1107
1108
1109
1110
1112
1113
1114
1115
1116
1118
1119
1120
1122
1124
1125
1126
1128
1129
1131
1134 };
1135
1136
1137
1138
1139
1140
1141
1142 bool doesGeneratePerAllLanes() const;
1143
1144private:
1145 typedef unsigned char OpcodeTy;
1146 OpcodeTy Opcode;
1147
1148
1149 std::string Name;
1150
1151
1152
1153 bool canGenerateScalarForFirstLane() const;
1154
1155
1156
1157
1159
1160#if !defined(NDEBUG)
1161
1162
1163
1164 static unsigned getNumOperandsForOpcode(unsigned Opcode);
1165#endif
1166
1167public:
1169 const VPIRFlags &Flags = {}, const VPIRMetadata &MD = {},
1170 DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "");
1171
1173
1181
1183
1184
1185
1186
1188
1189
1192
1193#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1194
1196#endif
1197
1199
1200
1202 case Instruction::Ret:
1203 case Instruction::Br:
1204 case Instruction::Store:
1205 case Instruction::Switch:
1206 case Instruction::IndirectBr:
1207 case Instruction::Resume:
1208 case Instruction::CatchRet:
1209 case Instruction::Unreachable:
1210 case Instruction::Fence:
1211 case Instruction::AtomicRMW:
1214 return false;
1215 default:
1216 return true;
1217 }
1218 }
1219
1220
1221 bool opcodeMayReadOrWriteFromMemory() const;
1222
1223
1224 bool usesFirstLaneOnly(const VPValue *Op) const override;
1225
1226
1227 bool usesFirstPartOnly(const VPValue *Op) const override;
1228
1229
1230
1231 bool isVectorToScalar() const;
1232
1233
1234
1235 bool isSingleScalar() const;
1236
1237
1239
1240
1242
1243protected:
1244#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1245
1248#endif
1249};
1250
1251
1252
1253
1254
1255
1257
1258 Type *ResultTy;
1259
1260public:
1265 const Twine &Name = "")
1267 ResultTy(ResultTy) {}
1268
1270
1271
1272 if (R->isScalarCast())
1273 return true;
1275 if (!VPI)
1276 return false;
1277 switch (VPI->getOpcode()) {
1281 return true;
1282 default:
1283 return false;
1284 }
1285 }
1286
1290
1292 auto *New =
1296 return New;
1297 }
1298
1300
1301
1304
1305 return 0;
1306 }
1307
1309
1310protected:
1311#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1312
1315#endif
1316};
1317
1318
1319
1321protected:
1322
1324
1325public:
1327
1328
1332
1333
1335
1336
1340
1341
1346
1349
1350
1352 std::function<const VPBasicBlock *(size_t)> GetBlock = [this](size_t Idx) {
1354 };
1356 }
1357
1358
1359
1365
1366
1367
1369
1370#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1371
1373#endif
1374};
1375
1379
1382 return VPI && VPI->getOpcode() == Instruction::PHI;
1383 }
1384
1387 return VPI && VPI->getOpcode() == Instruction::PHI;
1388 }
1389
1392 return VPI && VPI->getOpcode() == Instruction::PHI;
1393 }
1394
1400
1402
1403protected:
1404#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1405
1408#endif
1409
1411};
1412
1413
1414
1415
1418
1419protected:
1420
1421
1424
1425public:
1427
1428
1429
1431
1433
1435 auto *R = create(I);
1437 R->addOperand(Op);
1438 return R;
1439 }
1440
1442
1443
1446
1448
1451 "Op must be an operand of the recipe");
1452 return true;
1453 }
1454
1457 "Op must be an operand of the recipe");
1458 return true;
1459 }
1460
1463 "Op must be an operand of the recipe");
1464 return true;
1465 }
1466
1467
1468
1469
1471
1472protected:
1473#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1474
1477#endif
1478};
1479
1480
1481
1482
1483
1487
1492
1494
1496
1497protected:
1498#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1499
1502#endif
1503
1505};
1506
1507
1508
1509
1510
1513 unsigned Opcode;
1514
1515public:
1518 DebugLoc DL = {})
1519 : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL),
1521 setUnderlyingValue(&I);
1522 }
1523
1525
1530
1532
1533
1534
1536
1537
1540
1542
1543protected:
1544#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1545
1548#endif
1549};
1550
1551
1553
1555
1556
1557 Type *ResultTy;
1558
1559public:
1567 "Set flags not supported for the provided opcode");
1569 }
1570
1572
1578
1580
1581
1583
1584
1587
1589
1590
1592
1593protected:
1594#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1595
1598#endif
1599};
1600
1601
1603
1605
1606
1607 Type *ResultTy;
1608
1609
1610 bool MayReadFromMemory;
1611
1612
1613 bool MayWriteToMemory;
1614
1615
1616 bool MayHaveSideEffects;
1617
1618public:
1625 DL),
1626 VPIRMetadata(MD), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
1631 }
1632
1639 DL),
1641 ResultTy(Ty) {
1642 LLVMContext &Ctx = Ty->getContext();
1645 MayReadFromMemory = !ME.onlyWritesMemory();
1646 MayWriteToMemory = !ME.onlyReadsMemory();
1647 MayHaveSideEffects = MayWriteToMemory ||
1648 !Attrs.hasAttribute(Attribute::NoUnwind) ||
1649 !Attrs.hasAttribute(Attribute::WillReturn);
1650 }
1651
1653
1657 operands(), ResultTy, *this, *this,
1661 }
1662
1664
1665
1667
1668
1671
1672
1674
1675
1677
1678
1680
1681
1683
1684
1686
1687
1689
1691
1692protected:
1693#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1694
1697#endif
1698};
1699
1700
1703
1704
1705
1707
1708public:
1713 : VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments, Flags, DL),
1715 setUnderlyingValue(UV);
1717 isa(getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
1718 "last operand must be the called function");
1719 }
1720
1722
1727
1729
1730
1732
1733
1736
1740
1743
1744protected:
1745#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1746
1749#endif
1750};
1751
1752
1753
1754
1755
1756
1758
1759 unsigned Opcode;
1760
1761public:
1765
1767
1771
1773
1774
1776
1777
1780
1782
1783
1784
1788
1789protected:
1790#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1791
1794#endif
1795};
1796
1797
1798
1799
1804 DebugLoc DL = {})
1805 : VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, Flags, DL),
1806 VPIRMetadata(MD) {
1807 setUnderlyingValue(SI);
1808 }
1809
1811
1816
1818
1819
1821
1822
1825
1826 unsigned getOpcode() const { return Instruction::Select; }
1827
1831
1832
1835 "Op must be an operand of the recipe");
1836 return Op == getCond() && Op->isDefinedOutsideLoopRegions();
1837 }
1838
1839protected:
1840#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1841
1844#endif
1845};
1846
1847
1849 Type *SourceElementTy;
1850
1851 bool isPointerLoopInvariant() const {
1852 return getOperand(0)->isDefinedOutsideLoopRegions();
1853 }
1854
1855 bool isIndexLoopInvariant(unsigned I) const {
1856 return getOperand(I + 1)->isDefinedOutsideLoopRegions();
1857 }
1858
1859public:
1864 SourceElementTy(GEP->getSourceElementType()) {
1865 setUnderlyingValue(GEP);
1869 assert(Metadata.empty() && "unexpected metadata on GEP");
1870 }
1871
1873
1878
1880
1881
1883
1884
1886
1888
1889
1892
1893 return 0;
1894 }
1895
1896
1897 bool usesFirstLaneOnly(const VPValue *Op) const override;
1898
1899protected:
1900#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1901
1904#endif
1905};
1906
1907
1908
1909
1912 Type *IndexedTy;
1913
1914
1915
1916 int64_t Stride;
1917
1918public:
1923 IndexedTy(IndexedTy), Stride(Stride) {
1924 assert(Stride < 0 && "Stride must be negative");
1925 }
1926
1928
1931
1933
1936 "Op must be an operand of the recipe");
1937 return true;
1938 }
1939
1940
1943
1944 return 0;
1945 }
1946
1947
1950 "Op must be an operand of the recipe");
1952 return true;
1953 }
1954
1960
1961protected:
1962#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1963
1966#endif
1967};
1968
1969
1972 Type *SourceElementTy;
1973
1974public:
1979 SourceElementTy(SourceElementTy) {}
1980
1982
1984
1986
1989 "Op must be an operand of the recipe");
1990 return true;
1991 }
1992
1993
1996 "Op must be an operand of the recipe");
1998 return true;
1999 }
2000
2005
2006
2007
2009
2010
2013
2014 return 0;
2015 }
2016
2017protected:
2018#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2019
2022#endif
2023};
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2050protected:
2054 UnderlyingInstr, DL) {}
2055
2057
2058public:
2060
2061
2063 return R->getVPDefID() >= VPDef::VPFirstHeaderPHISC &&
2064 R->getVPDefID() <= VPDef::VPLastHeaderPHISC;
2065 }
2072
2073
2075
2076
2079
2080
2087
2088
2090
2091
2095
2096
2098
2099
2100
2104
2105protected:
2106#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2107
2110#endif
2111};
2112
2113
2114
2115
2118
2119public:
2126
2128 return R->getVPDefID() == VPDef::VPWidenIntOrFpInductionSC ||
2129 R->getVPDefID() == VPDef::VPWidenPointerInductionSC;
2130 }
2131
2133 auto *R = V->getDefiningRecipe();
2135 }
2136
2140
2142
2143
2146
2147
2149
2152
2153
2154
2155
2157
2159
2160
2162
2164
2165
2167 "VPWidenIntOrFpInductionRecipe generates its own backedge value");
2168 }
2169
2171
2172
2174 "VPWidenIntOrFpInductionRecipe generates its own backedge value");
2175 }
2176
2177
2180 "Op must be an operand of the recipe");
2181
2182
2183
2185 }
2186};
2187
2188
2189
2190
2194
2195
2196 bool isUnrolled() const { return getNumOperands() == 5; }
2197
2198public:
2203 Step, IndDesc, DL),
2204 VPIRFlags(Flags), Trunc(nullptr) {
2206 }
2207
2213 Step, IndDesc, DL),
2214 VPIRFlags(Flags), Trunc(Trunc) {
2218 if (Trunc)
2220 assert(Metadata.empty() && "unexpected metadata on Trunc");
2221 }
2222
2224
2230
2232
2234 llvm_unreachable("cannot execute this recipe, should be expanded via "
2235 "expandVPWidenIntOrFpInductionRecipe");
2236 }
2237
2243
2244
2245
2246
2248
2249
2250
2253
2254
2255
2256
2258
2259
2261 return Trunc ? Trunc->getType()
2263 }
2264
2265
2266
2267
2271
2272protected:
2273#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2274
2277#endif
2278};
2279
2281public:
2282
2283
2284
2286 VPValue *NumUnrolledElems,
2289 Step, IndDesc, DL) {
2291 }
2292
2294
2300
2302
2303
2305 llvm_unreachable("cannot execute this recipe, should be expanded via "
2306 "expandVPWidenPointerInduction");
2307 };
2308
2309
2311
2312protected:
2313#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2314
2317#endif
2318};
2319
2320
2321
2322
2323
2324
2327
2328 std::string Name;
2329
2330public:
2331
2332
2339
2345 return C;
2346 }
2347
2349
2351
2352
2354
2355protected:
2356#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2357
2360#endif
2361
2363};
2364
2365
2366
2367
2371
2373
2378
2380
2381
2384
2385
2388 "Op must be an operand of the recipe");
2390 }
2391
2392protected:
2393#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2394
2397#endif
2398};
2399
2400
2401
2402
2404
2406
2407
2411using ReductionStyle = std::variant<RdxOrdered, RdxInLoop, RdxUnordered>;
2412
2414 unsigned ScaleFactor) {
2415 assert((!Ordered || InLoop) && "Ordered implies in-loop");
2416 if (Ordered)
2418 if (InLoop)
2420 return RdxUnordered{ScaleFactor};
2421}
2422
2423
2424
2425
2428
2430
2432
2433
2434
2435
2436
2437 bool HasUsesOutsideReductionChain;
2438
2439public:
2440
2443 bool HasUsesOutsideReductionChain = false)
2445 Style(Style),
2446 HasUsesOutsideReductionChain(HasUsesOutsideReductionChain) {}
2447
2449
2453 *getOperand(0), Style, HasUsesOutsideReductionChain);
2455 return R;
2456 }
2457
2459
2460
2462
2463
2464
2466 auto *Partial = std::get_if(&Style);
2467 return Partial ? Partial->VFScaleFactor : 1;
2468 }
2469
2470
2471
2472
2474
2475
2477
2478
2479 bool isOrdered() const { return std::holds_alternative(Style); }
2480
2481
2483 return std::holds_alternative(Style) ||
2484 std::holds_alternative(Style);
2485 }
2486
2487
2489
2490
2492 return HasUsesOutsideReductionChain;
2493 }
2494
2495
2498 "Op must be an operand of the recipe");
2500 }
2501
2502protected:
2503#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2504
2507#endif
2508};
2509
2510
2511
2513public:
2514
2515
2516
2517
2520 assert(Operands.size() > 0 && "Expected at least one operand!");
2521 }
2522
2527
2529
2530
2531
2533
2534
2535
2539
2540
2544
2545
2550
2551
2556
2558 llvm_unreachable("VPBlendRecipe should be expanded by simplifyBlends");
2559 }
2560
2561
2564
2565
2568 "Op must be an operand of the recipe");
2569
2570
2572 [this](VPUser *U) { return U->usesFirstLaneOnly(this); });
2573 }
2574
2575protected:
2576#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2577
2580#endif
2581};
2582
2583
2584
2585
2586
2587
2591
2592
2593
2594 bool HasMask = false;
2595
2596
2597
2598 bool NeedsMaskForGaps = false;
2599
2600protected:
2607 NeedsMaskForGaps(NeedsMaskForGaps) {
2608
2609 assert((!Mask || !IG->isReverse()) &&
2610 "Reversed masked interleave-group not supported.");
2611 for (unsigned I = 0; I < IG->getFactor(); ++I)
2613 if (Inst->getType()->isVoidTy())
2614 continue;
2615 new VPValue(Inst, this);
2616 }
2617
2618 for (auto *SV : StoredValues)
2620 if (Mask) {
2621 HasMask = true;
2623 }
2624 }
2625
2626public:
2628
2630 return R->getVPDefID() == VPRecipeBase::VPInterleaveSC ||
2631 R->getVPDefID() == VPRecipeBase::VPInterleaveEVLSC;
2632 }
2633
2638
2639
2643
2644
2645
2650
2651
2653
2655
2657
2661
2662
2665
2666
2668
2669
2670
2672
2673
2674
2680};
2681
2682
2683
2684
2685
2687public:
2692 NeedsMaskForGaps, MD, DL) {}
2693
2695
2701
2703
2704
2706
2709 "Op must be an operand of the recipe");
2711 }
2712
2716
2717protected:
2718#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2719
2722#endif
2723};
2724
2725
2726
2727
2729public:
2733 R.getStoredValues(), Mask, R.needsMaskForGaps(), R,
2734 R.getDebugLoc()) {
2735 assert(!getInterleaveGroup()->isReverse() &&
2736 "Reversed interleave-group with tail folding is not supported.");
2737 assert(!needsMaskForGaps() && "Interleaved access with gap mask is not "
2738 "supported for scalable vector.");
2739 }
2740
2742
2746
2748
2749
2751
2752
2754
2755
2758 "Op must be an operand of the recipe");
2761 }
2762
2766
2767protected:
2768#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2769
2772#endif
2773};
2774
2775
2776
2777
2778
2780
2781
2783
2784 bool IsConditional = false;
2786
2787protected:
2793 Style(Style) {
2794 if (CondOp) {
2795 IsConditional = true;
2797 }
2799 }
2800
2801public:
2808
2815
2817
2823
2825 return R->getVPDefID() == VPRecipeBase::VPReductionSC ||
2826 R->getVPDefID() == VPRecipeBase::VPReductionEVLSC;
2827 }
2828
2833
2838
2842
2843
2845
2846
2849
2850
2852
2853 bool isOrdered() const { return std::holds_alternative(Style); };
2854
2856
2858
2860 return std::holds_alternative(Style) ||
2861 std::holds_alternative(Style);
2862 }
2863
2865
2867
2871
2872
2874 auto *Partial = std::get_if(&Style);
2875 return Partial ? Partial->VFScaleFactor : 1;
2876 }
2877
2878protected:
2879#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2880
2883#endif
2884};
2885
2886
2887
2888
2889
2891public:
2900
2902
2906
2908
2909
2911
2912
2914
2915
2918 "Op must be an operand of the recipe");
2920 }
2921
2922protected:
2923#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2924
2927#endif
2928};
2929
2930
2931
2932
2933
2936
2937 bool IsSingleScalar;
2938
2939
2940 bool IsPredicated;
2941
2942public:
2944 bool IsSingleScalar, VPValue *Mask = nullptr,
2946 DebugLoc DL = DebugLoc::getUnknown())
2947 : VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, Flags, DL),
2948 VPIRMetadata(Metadata), IsSingleScalar(IsSingleScalar),
2949 IsPredicated(Mask) {
2950 setUnderlyingValue(I);
2951 if (Mask)
2953 }
2954
2956
2961 Copy->transferFlags(*this);
2962 return Copy;
2963 }
2964
2966
2967
2968
2969
2971
2972
2975
2977
2979
2980
2983 "Op must be an operand of the recipe");
2985 }
2986
2987
2990 "Op must be an operand of the recipe");
2991 return true;
2992 }
2993
2994
2995
2996
2997 bool shouldPack() const;
2998
2999
3004
3006
3007protected:
3008#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3009
3012#endif
3013};
3014
3015
3017public:
3020
3024
3026
3027
3028
3030
3031
3034
3035#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3036
3039 O << Indent << "BRANCH-ON-MASK ";
3041 }
3042#endif
3043
3044
3047 "Op must be an operand of the recipe");
3048 return true;
3049 }
3050};
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3062
3063
3065
3066
3067
3069
3070 enum class ExpressionTypes {
3071
3072
3073
3074 ExtendedReduction,
3075
3076
3077
3078 ExtMulAccReduction,
3079
3080
3081
3082 MulAccReduction,
3083
3084
3085
3086 ExtNegatedMulAccReduction,
3087 };
3088
3089
3090 ExpressionTypes ExpressionType;
3091
3092
3093
3094
3095
3096
3097
3100
3101public:
3103 : VPExpressionRecipe(ExpressionTypes::ExtendedReduction, {Ext, Red}) {}
3105 : VPExpressionRecipe(ExpressionTypes::MulAccReduction, {Mul, Red}) {}
3108 : VPExpressionRecipe(ExpressionTypes::ExtMulAccReduction,
3109 {Ext0, Ext1, Mul, Red}) {}
3113 : VPExpressionRecipe(ExpressionTypes::ExtNegatedMulAccReduction,
3114 {Ext0, Ext1, Mul, Sub, Red}) {
3115 assert(Mul->getOpcode() == Instruction::Mul && "Expected a mul");
3117 "Expected an add reduction");
3120 assert(SubConst && SubConst->getValue() == 0 &&
3121 Sub->getOpcode() == Instruction::Sub && "Expected a negating sub");
3122 }
3123
3126 for (auto *R : reverse(ExpressionRecipes)) {
3127 if (ExpressionRecipesSeen.insert(R).second)
3128 delete R;
3129 }
3130 for (VPValue *T : LiveInPlaceholders)
3131 delete T;
3132 }
3133
3135
3136 VPExpressionRecipe *clone() override {
3137 assert(!ExpressionRecipes.empty() && "empty expressions should be removed");
3139 for (auto *R : ExpressionRecipes)
3140 NewExpressiondRecipes.push_back(R->clone());
3141 for (auto *New : NewExpressiondRecipes) {
3142 for (const auto &[Idx, Old] : enumerate(ExpressionRecipes))
3143 New->replaceUsesOfWith(Old, NewExpressiondRecipes[Idx]);
3144
3145
3146 for (const auto &[Placeholder, OutsideOp] :
3148 New->replaceUsesOfWith(Placeholder, OutsideOp);
3149 }
3150 return new VPExpressionRecipe(ExpressionType, NewExpressiondRecipes);
3151 }
3152
3153
3155 unsigned OpIdx =
3157 : 1;
3159 }
3160
3161
3162
3163
3165
3168 return PR ? PR->getVFScaleFactor() : 1;
3169 }
3170
3171
3175
3178
3179
3180
3182
3183
3184
3186
3187
3189
3190protected:
3191#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3192
3195#endif
3196};
3197
3198
3199
3200
3201
3202
3204public:
3205
3206
3210
3214
3216
3217
3218
3220
3221
3224
3225 return 0;
3226 }
3227
3228
3231 "Op must be an operand of the recipe");
3232 return true;
3233 }
3234
3235protected:
3236#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3237
3240#endif
3241};
3242
3243
3244
3247protected:
3249
3250
3252
3253
3255
3256
3258
3259
3261
3264 if (!Mask)
3265 return;
3268 }
3269
3271 std::initializer_list<VPValue *> Operands,
3279 "Reversed acccess without VPVectorEndPointerRecipe address?");
3280 }
3281
3282public:
3286
3288 return R->getVPDefID() == VPRecipeBase::VPWidenLoadSC ||
3289 R->getVPDefID() == VPRecipeBase::VPWidenStoreSC ||
3290 R->getVPDefID() == VPRecipeBase::VPWidenLoadEVLSC ||
3291 R->getVPDefID() == VPRecipeBase::VPWidenStoreEVLSC;
3292 }
3293
3298
3299
3301
3302
3303
3305
3306
3308
3309
3311
3312
3313
3318
3319
3321
3322
3324 llvm_unreachable("VPWidenMemoryRecipe should not be instantiated.");
3325 }
3326
3327
3330
3332};
3333
3334
3335
3344 setMask(Mask);
3345 }
3346
3352
3354
3355
3357
3358
3361 "Op must be an operand of the recipe");
3362
3363
3365 }
3366
3367protected:
3368#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3369
3372#endif
3373};
3374
3375
3376
3377
3382 {Addr, &EVL}, L.isConsecutive(), L.isReverse(), L,
3383 L.getDebugLoc()),
3386 }
3387
3389
3390
3392
3393
3395
3396
3399
3400
3403 "Op must be an operand of the recipe");
3404
3405
3407 }
3408
3409protected:
3410#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3411
3414#endif
3415};
3416
3417
3418
3425 setMask(Mask);
3426 }
3427
3433
3435
3436
3438
3439
3441
3442
3445 "Op must be an operand of the recipe");
3446
3447
3449 }
3450
3451protected:
3452#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3453
3456#endif
3457};
3458
3459
3460
3461
3466 {Addr, S.getStoredValue(), &EVL}, S.isConsecutive(),
3467 S.isReverse(), S, S.getDebugLoc()) {
3469 }
3470
3472
3473
3475
3476
3478
3479
3481
3482
3485
3486
3489 "Op must be an operand of the recipe");
3492 return true;
3493 }
3494
3495
3496
3498 }
3499
3500protected:
3501#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3502
3505#endif
3506};
3507
3508
3510 const SCEV *Expr;
3511
3512public:
3515
3517
3519
3521
3523 llvm_unreachable("SCEV expressions must be expanded before final execute");
3524 }
3525
3526
3529
3530 return 0;
3531 }
3532
3534
3535protected:
3536#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3537
3540#endif
3541};
3542
3543
3544
3545
3546
3548public:
3551
3553
3559
3561
3563 llvm_unreachable("cannot execute this recipe, should be replaced by a "
3564 "scalar phi recipe");
3565 }
3566
3567
3571
3572
3575 "Op must be an operand of the recipe");
3576 return true;
3577 }
3578
3579
3582 "Op must be an operand of the recipe");
3583 return true;
3584 }
3585
3586
3589
3590 return 0;
3591 }
3592
3593protected:
3594#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3595
3598#endif
3599};
3600
3601
3602
3603
3604
3606public:
3610
3612
3619
3621
3622
3624
3625protected:
3626#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3627
3630#endif
3631};
3632
3633
3634
3635
3636
3638public:
3641
3643
3647
3649
3651 llvm_unreachable("cannot execute this recipe, should be replaced by a "
3652 "scalar phi recipe");
3653 }
3654
3655
3658
3659 return 0;
3660 }
3661
3662
3665 "Op must be an operand of the recipe");
3666 return true;
3667 }
3668
3669protected:
3670#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3671
3674#endif
3675};
3676
3677
3680public:
3683
3685
3690
3692
3693
3694
3695
3697
3698
3701
3702 return 0;
3703 }
3704
3705protected:
3706#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3707
3710#endif
3711};
3712
3713
3714
3715
3717
3719
3720
3722
3723
3724 std::string Name;
3725
3726public:
3729 const Twine &Name = "")
3731 IndDesc.getKind(),
3733 Start, CanonicalIV, Step, Name) {}
3734
3739 FPBinOp(FPBinOp), Name(Name.str()) {}
3740
3742
3747
3749
3750
3751
3753
3754
3757
3758 return 0;
3759 }
3760
3764
3767
3768
3771 "Op must be an operand of the recipe");
3772 return true;
3773 }
3774
3775protected:
3776#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3777
3780#endif
3781};
3782
3783
3784
3788
3789public:
3795 InductionOpcode(Opcode) {}
3796
3801 IV, Step, VF, IndDesc.getInductionOpcode(),
3805 DL) {}
3806
3808
3815
3816
3817
3819
3821
3822
3824
3825
3828
3829 return 0;
3830 }
3831
3833
3834
3837 "Op must be an operand of the recipe");
3838 return true;
3839 }
3840
3841protected:
3842#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3843
3846#endif
3847};
3848
3849
3850
3857
3858
3859template
3861
3863
3864
3867 switch (R->getVPDefID()) {
3868 case VPDef::VPInstructionSC:
3870 case VPDef::VPIRInstructionSC:
3872 case VPDef::VPWidenPHISC:
3874 default:
3876 }
3877 }());
3878 }
3879
3880
3883 return nullptr;
3885 }
3886};
3887template <>
3890template <>
3893
3894
3895
3897template <typename DstTy, typename RecipeBasePtrTy>
3899 switch (R->getVPDefID()) {
3900 case VPDef::VPInstructionSC:
3902 case VPDef::VPWidenSC:
3904 case VPDef::VPWidenCastSC:
3906 case VPDef::VPWidenIntrinsicSC:
3908 case VPDef::VPWidenCallSC:
3910 case VPDef::VPWidenSelectSC:
3912 case VPDef::VPReplicateSC:
3914 case VPDef::VPInterleaveSC:
3915 case VPDef::VPInterleaveEVLSC:
3917 case VPDef::VPWidenLoadSC:
3918 case VPDef::VPWidenLoadEVLSC:
3919 case VPDef::VPWidenStoreSC:
3920 case VPDef::VPWidenStoreEVLSC:
3922 default:
3924 }
3925}
3926}
3927
3928
3929
3930template <typename DstTy, typename SrcTy>
3941
3943
3944
3948
3949
3952 return nullptr;
3954 }
3955};
3956template <>
3959template <>
3962
3963
3964
3965
3968
3969
3970 VPBasicBlock(const Twine &Name = "", VPRecipeBase *Recipe = nullptr)
3971 : VPBlockBase(VPBasicBlockSC, Name.str()) {
3972 if (Recipe)
3974 }
3975
3976public:
3978
3979protected:
3980
3982
3985
3986public:
3988 while (.empty())
3990 }
3991
3992
3997
3998
3999
4000
4005
4010
4017
4018
4020
4021
4025
4026
4028 return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC ||
4029 V->getVPBlockID() == VPBlockBase::VPIRBasicBlockSC;
4030 }
4031
4033 assert(Recipe && "No recipe to append.");
4034 assert(!Recipe->Parent && "Recipe already in VPlan");
4035 Recipe->Parent = this;
4036 Recipes.insert(InsertPt, Recipe);
4037 }
4038
4039
4040
4042
4043
4044
4046
4047
4049
4050
4052
4053
4057
4058
4059
4060
4062
4064 const VPRegionBlock *getEnclosingLoopRegion() const;
4065
4066#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4067
4068
4069
4070
4071
4075#endif
4076
4077
4078
4081
4082
4083 bool isExiting() const;
4084
4085
4086
4088
4089
4090
4091
4092
4093 const VPBasicBlock *getCFGPredecessor(unsigned Idx) const;
4094
4095protected:
4096
4098
4099
4100
4102
4103private:
4104
4105
4107};
4108
4109inline const VPBasicBlock *
4113
4114
4115
4116
4117
4118
4119class VPIRBasicBlock : public VPBasicBlock {
4121
4123
4124
4126 : VPBasicBlock(VPIRBasicBlockSC,
4128 IRBB(IRBB) {}
4129
4130public:
4132
4134 return V->getVPBlockID() == VPBlockBase::VPIRBasicBlockSC;
4135 }
4136
4137
4138
4140
4142
4144};
4145
4146
4147
4148
4149
4150
4151
4152
4153
4156
4157
4159
4160
4161
4163
4164
4165
4166 bool IsReplicator;
4167
4168
4170 const std::string &Name = "", bool IsReplicator = false)
4171 : VPBlockBase(VPRegionBlockSC, Name), Entry(Entry), Exiting(Exiting),
4172 IsReplicator(IsReplicator) {
4173 assert(Entry->getPredecessors().empty() && "Entry block has predecessors.");
4174 assert(Exiting->getSuccessors().empty() && "Exit block has successors.");
4175 Entry->setParent(this);
4176 Exiting->setParent(this);
4177 }
4178 VPRegionBlock(const std::string &Name = "", bool IsReplicator = false)
4179 : VPBlockBase(VPRegionBlockSC, Name), Entry(nullptr), Exiting(nullptr),
4180 IsReplicator(IsReplicator) {}
4181
4182public:
4184
4185
4187 return V->getVPBlockID() == VPBlockBase::VPRegionBlockSC;
4188 }
4189
4192
4193
4194
4197 "Entry block cannot have predecessors.");
4198 Entry = EntryBlock;
4200 }
4201
4204
4205
4206
4209 "Exit block cannot have successors.");
4210 Exiting = ExitingBlock;
4212 }
4213
4214
4219
4220
4221
4223
4224
4225
4227
4228
4230
4231#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4232
4233
4234
4235
4236
4237
4241#endif
4242
4243
4244
4246
4247
4248
4249 void dissolveToCFGLoop();
4250
4251
4254 if (EntryVPBB->empty()) {
4255
4257 }
4259 }
4261 return const_cast<VPRegionBlock *>(this)->getCanonicalIV();
4262 }
4263
4264
4269};
4270
4274
4278
4279
4280
4281
4282
4283
4284class VPlan {
4287
4288
4289
4290
4291
4292
4294
4295
4297
4298
4299
4300
4302
4303
4305
4306
4307
4309
4310
4311 std::string Name;
4312
4313
4314
4315 VPValue *TripCount = nullptr;
4316
4317
4318
4319 VPValue *BackedgeTakenCount = nullptr;
4320
4321
4322 VPValue VectorTripCount;
4323
4324
4326
4327
4329
4330
4331
4333
4334
4335
4337
4338
4339
4341
4342
4343
4345 : Entry(Entry), ScalarHeader(ScalarHeader) {
4346 Entry->setPlan(this);
4347 assert(ScalarHeader->getNumSuccessors() == 0 &&
4348 "scalar header must be a leaf node");
4349 }
4350
4351public:
4352
4353
4354
4356
4357
4358
4363
4365
4367 Entry = VPBB;
4369 }
4370
4371
4373
4374
4376
4379
4380
4381
4384 return VectorRegion
4386 : nullptr;
4387 }
4388
4389
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4405 LoopRegion &&
4406 "cannot call the function after vector loop region has been removed");
4408 if (RegionSucc->getSingleSuccessor() ||
4410 return RegionSucc;
4411
4413 }
4414
4418
4419
4423
4424
4426
4427
4428
4430
4431
4432
4434
4435
4437
4438
4440 assert(TripCount && "trip count needs to be set before accessing it");
4441 return TripCount;
4442 }
4443
4444
4445
4447 assert(!TripCount && NewTripCount && "TripCount should not be set yet.");
4448 TripCount = NewTripCount;
4449 }
4450
4451
4452
4454 assert(TripCount && NewTripCount && TripCount->getNumUsers() == 0 &&
4455 "TripCount must be set when resetting");
4456 TripCount = NewTripCount;
4457 }
4458
4459
4461 if (!BackedgeTakenCount)
4462 BackedgeTakenCount = new VPValue();
4463 return BackedgeTakenCount;
4464 }
4466
4467
4469
4470
4473
4474
4476
4480
4482
4484 assert(hasVF(VF) && "Cannot set VF not already in plan");
4485 VFs.clear();
4486 VFs.insert(VF);
4487 }
4488
4493
4494
4499
4501 bool HasScalarVFOnly = VFs.size() == 1 && VFs[0].isScalar();
4503 "Plan with scalar VF should only have a single VF");
4504 return HasScalarVFOnly;
4505 }
4506
4507 bool hasUF(unsigned UF) const { return UFs.empty() || UFs.contains(UF); }
4508
4510 assert(UFs.size() == 1 && "Expected a single UF");
4511 return UFs[0];
4512 }
4513
4515 assert(hasUF(UF) && "Cannot set the UF not already in plan");
4516 UFs.clear();
4517 UFs.insert(UF);
4518 }
4519
4520
4521
4523
4524
4525 std::string getName() const;
4526
4528
4529
4530
4532 assert(V && "Trying to get or add the VPValue of a null Value");
4533 auto [It, Inserted] = Value2VPValue.try_emplace(V);
4534 if (Inserted) {
4536 VPLiveIns.push_back(VPV);
4537 assert(VPV->isLiveIn() && "VPV must be a live-in.");
4538 It->second = VPV;
4539 }
4540
4541 assert(It->second->isLiveIn() && "Only live-ins should be in mapping");
4542 return It->second;
4543 }
4544
4545
4547
4548
4550
4551
4553 return getOrAddLiveIn(ConstantInt::get(Ty, Val, IsSigned));
4554 }
4555
4556
4558 bool IsSigned = false) {
4560 }
4561
4562
4566
4567
4569
4570
4573 [this](const auto &P) {
4575 }) &&
4576 "all VPValues in Value2VPValue must also be in VPLiveIns");
4577 return VPLiveIns;
4578 }
4579
4580#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4581
4583
4584
4586
4587
4589
4590
4592#endif
4593
4594
4595
4597
4598
4599
4600
4604 CreatedBlocks.push_back(VPB);
4605 return VPB;
4606 }
4607
4608
4609
4610
4614 auto *VPB = Entry ? new VPRegionBlock(Entry, Exiting, Name)
4616 CreatedBlocks.push_back(VPB);
4617 return VPB;
4618 }
4619
4620
4621
4622
4624 const std::string &Name = "") {
4625 auto *VPB = new VPRegionBlock(Entry, Exiting, Name, true);
4626 CreatedBlocks.push_back(VPB);
4627 return VPB;
4628 }
4629
4630
4631
4632
4634
4635
4636
4637
4638
4640
4641
4642
4643
4644
4648 1 ||
4649 (ExitBlocks.size() == 1 && ExitBlocks[0]->getNumPredecessors() > 1);
4650 }
4651
4652
4653
4654
4659};
4660
4661#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4666#endif
4667
4668}
4669
4670#endif
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static MCDisassembler::DecodeStatus addOperand(MCInst &Inst, const MCOperand &Opnd)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
#define LLVM_ABI_FOR_TEST
This file defines the DenseMap class.
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
static std::pair< Value *, APInt > getMask(Value *WideMask, unsigned Factor, ElementCount LeafValueEC)
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first DebugLoc that has line number information, given a range of instructions.
This file provides utility analysis objects describing memory locations.
MachineInstr unsigned OpIdx
StandardInstrumentations SI(Mod->getContext(), Debug, VerifyEach)
static StringRef getName(Value *V)
static bool mayHaveSideEffects(MachineInstr &MI)
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static const BasicSubtargetSubTypeKV * find(StringRef S, ArrayRef< BasicSubtargetSubTypeKV > A)
Find KV in array using binary search.
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
This file contains the declarations of the entities induced by Vectorization Plans,...
#define VP_CLASSOF_IMPL(VPDefID)
Definition VPlan.h:509
static const uint32_t IV[8]
Class for arbitrary precision integers.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
LLVM_ABI LLVMContext & getContext() const
Get the context in which this basic block lives.
This class represents a function call, abstracting a target machine's calling convention.
This is the base class for all instructions that perform data casts.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
static DebugLoc getUnknown()
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
static constexpr ElementCount getFixed(ScalarTy MinVal)
Utility class for floating point operations which can have information about relaxed accuracy require...
Convenience struct for specifying and reasoning about fast-math flags.
Represents flags for the getelementptr instruction/expression.
static GEPNoWrapFlags none()
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Common base class shared among various IRBuilders.
A struct for saving information about induction variables.
InductionKind
This enum represents the kinds of inductions that we support.
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
The group of interleaved loads/stores sharing the same stride and close to each other.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
LoopVectorizationCostModel - estimates the expected speedups due to vectorization.
Represents a single loop in the control flow graph.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
This class represents an analyzed expression in the program.
This class represents the LLVM 'select' instruction.
This class provides computation of slot numbers for LLVM Assembly writing.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
iterator erase(const_iterator CI)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
std::string str() const
str - Get the contents as an std::string.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
LLVM_ABI std::string str() const
Return the twine contents as a std::string.
The instances of the Type class are immutable: once they are created, they are never changed.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
VPActiveLaneMaskPHIRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3613
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPActiveLaneMaskPHIRecipe(VPValue *StartMask, DebugLoc DL)
Definition VPlan.h:3607
~VPActiveLaneMaskPHIRecipe() override=default
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition VPlan.h:3966
RecipeListTy::const_iterator const_iterator
Definition VPlan.h:3994
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
Definition VPlan.h:4041
RecipeListTy::const_reverse_iterator const_reverse_iterator
Definition VPlan.h:3996
RecipeListTy::iterator iterator
Instruction iterators...
Definition VPlan.h:3993
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
Definition VPlan.h:4019
iplist< VPRecipeBase > RecipeListTy
Definition VPlan.h:3977
VPBasicBlock(const unsigned char BlockSC, const Twine &Name="")
Definition VPlan.h:3983
iterator end()
Definition VPlan.h:4003
iterator begin()
Recipe iterator methods.
Definition VPlan.h:4001
RecipeListTy::reverse_iterator reverse_iterator
Definition VPlan.h:3995
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
Definition VPlan.h:4054
const VPBasicBlock * getCFGPredecessor(unsigned Idx) const
Returns the predecessor block at index Idx with the predecessors as per the corresponding plain CFG.
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
~VPBasicBlock() override
Definition VPlan.h:3987
const_reverse_iterator rbegin() const
Definition VPlan.h:4007
reverse_iterator rend()
Definition VPlan.h:4008
RecipeListTy Recipes
The VPRecipes held in the order of output instructions to generate.
Definition VPlan.h:3981
VPRecipeBase & back()
Definition VPlan.h:4016
const VPRecipeBase & front() const
Definition VPlan.h:4013
const_iterator begin() const
Definition VPlan.h:4002
VPRecipeBase & front()
Definition VPlan.h:4014
const VPRecipeBase & back() const
Definition VPlan.h:4015
void insert(VPRecipeBase *Recipe, iterator InsertPt)
Definition VPlan.h:4032
bool empty() const
Definition VPlan.h:4012
const_iterator end() const
Definition VPlan.h:4004
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition VPlan.h:4027
static RecipeListTy VPBasicBlock::* getSublistAccess(VPRecipeBase *)
Returns a pointer to a member of the recipe list.
Definition VPlan.h:4022
reverse_iterator rbegin()
Definition VPlan.h:4006
friend class VPlan
Definition VPlan.h:3967
size_t size() const
Definition VPlan.h:4011
const_reverse_iterator rend() const
Definition VPlan.h:4009
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
Definition VPlan.h:2541
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
Definition VPlan.h:2546
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account when normalized the first incoming value wi...
Definition VPlan.h:2536
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
Definition VPlan.h:2557
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:2566
VPBlendRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2523
VPBlendRecipe(PHINode *Phi, ArrayRef< VPValue * > Operands, DebugLoc DL)
The blend operation is a User of the incoming values and of their respective masks,...
Definition VPlan.h:2518
void setMask(unsigned Idx, VPValue *V)
Set mask number Idx to V.
Definition VPlan.h:2552
bool isNormalized() const
A normalized blend is one that has an odd number of operands, whereby the first operand does not have...
Definition VPlan.h:2532
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
Definition VPlan.h:81
void setSuccessors(ArrayRef< VPBlockBase * > NewSuccs)
Set each VPBasicBlock in NewSuccss as successor of this VPBlockBase.
Definition VPlan.h:300
VPRegionBlock * getParent()
Definition VPlan.h:173
VPBlocksTy & getPredecessors()
Definition VPlan.h:205
iterator_range< VPBlockBase ** > predecessors()
Definition VPlan.h:202
LLVM_DUMP_METHOD void dump() const
Dump this VPBlockBase to dbgs().
Definition VPlan.h:370
void setName(const Twine &newName)
Definition VPlan.h:166
size_t getNumSuccessors() const
Definition VPlan.h:219
iterator_range< VPBlockBase ** > successors()
Definition VPlan.h:201
virtual void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const =0
Print plain-text dump of this VPBlockBase to O, prefixing all lines with Indent.
bool hasPredecessors() const
Returns true if this block has any predecessors.
Definition VPlan.h:223
void swapSuccessors()
Swap successors of the block. The block must have exactly 2 successors.
Definition VPlan.h:322
void printSuccessors(raw_ostream &O, const Twine &Indent) const
Print the successors of this block to O, prefixing all lines with Indent.
SmallVectorImpl< VPBlockBase * > VPBlocksTy
Definition VPlan.h:160
virtual ~VPBlockBase()=default
const VPBlocksTy & getHierarchicalPredecessors()
Definition VPlan.h:258
unsigned getIndexForSuccessor(const VPBlockBase *Succ) const
Returns the index for Succ in the blocks successor list.
Definition VPlan.h:335
size_t getNumPredecessors() const
Definition VPlan.h:220
void setPredecessors(ArrayRef< VPBlockBase * > NewPreds)
Set each VPBasicBlock in NewPreds as predecessor of this VPBlockBase.
Definition VPlan.h:291
VPBlockBase * getEnclosingBlockWithPredecessors()
unsigned getIndexForPredecessor(const VPBlockBase *Pred) const
Returns the index for Pred in the blocks predecessors list.
Definition VPlan.h:328
const VPBlocksTy & getPredecessors() const
Definition VPlan.h:204
virtual VPBlockBase * clone()=0
Clone the current block and it's recipes without updating the operands of the cloned recipes,...
enum { VPRegionBlockSC, VPBasicBlockSC, VPIRBasicBlockSC } VPBlockTy
An enumeration for keeping track of the concrete subclass of VPBlockBase that are actually instantiat...
Definition VPlan.h:158
virtual InstructionCost cost(ElementCount VF, VPCostContext &Ctx)=0
Return the cost of the block.
void setPlan(VPlan *ParentPlan)
Sets the pointer of the plan containing the block.
const VPRegionBlock * getParent() const
Definition VPlan.h:174
const std::string & getName() const
Definition VPlan.h:164
void clearSuccessors()
Remove all the successors of this block.
Definition VPlan.h:310
VPBlockBase * getSingleHierarchicalSuccessor()
Definition VPlan.h:248
void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse)
Set two given VPBlockBases IfTrue and IfFalse to be the two successors of this VPBlockBase.
Definition VPlan.h:282
VPBlockBase * getSinglePredecessor() const
Definition VPlan.h:215
virtual void execute(VPTransformState *State)=0
The method which generates the output IR that correspond to this VPBlockBase, thereby "executing" the...
const VPBlocksTy & getHierarchicalSuccessors()
Definition VPlan.h:242
void clearPredecessors()
Remove all the predecessor of this block.
Definition VPlan.h:307
friend class VPBlockUtils
Definition VPlan.h:82
unsigned getVPBlockID() const
Definition VPlan.h:171
void printAsOperand(raw_ostream &OS, bool PrintType=false) const
Definition VPlan.h:349
void swapPredecessors()
Swap predecessors of the block.
Definition VPlan.h:314
VPBlockBase(const unsigned char SC, const std::string &N)
Definition VPlan.h:150
VPBlocksTy & getSuccessors()
Definition VPlan.h:199
VPBlockBase * getEnclosingBlockWithSuccessors()
An Enclosing Block of a block B is any block containing B, including B itself.
const VPBasicBlock * getEntryBasicBlock() const
void setOneSuccessor(VPBlockBase *Successor)
Set a given VPBlockBase Successor as the single successor of this VPBlockBase.
Definition VPlan.h:271
void setParent(VPRegionBlock *P)
Definition VPlan.h:184
VPBlockBase * getSingleHierarchicalPredecessor()
Definition VPlan.h:264
VPBlockBase * getSingleSuccessor() const
Definition VPlan.h:209
const VPBlocksTy & getSuccessors() const
Definition VPlan.h:198
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition VPlan.h:3037
VPBranchOnMaskRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3021
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
Definition VPlan.h:3045
VPBranchOnMaskRecipe(VPValue *BlockInMask, DebugLoc DL)
Definition VPlan.h:3018
VPlan-based builder utility analogous to IRBuilder.
Canonical scalar induction phi of the vector loop.
Definition VPlan.h:3547
~VPCanonicalIVPHIRecipe() override=default
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:3573
VPCanonicalIVPHIRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3554
LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool usesFirstPartOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first part of operand Op.
Definition VPlan.h:3580
VPCanonicalIVPHIRecipe(VPValue *StartV, DebugLoc DL)
Definition VPlan.h:3549
Type * getScalarType() const
Returns the scalar type of the induction.
Definition VPlan.h:3568
void execute(VPTransformState &State) override
Generate the phi nodes.
Definition VPlan.h:3562
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPCanonicalIVPHIRecipe.
Definition VPlan.h:3587
This class augments a recipe with a set of VPValues defined by the recipe.
VPDef(const unsigned char SC)
void execute(VPTransformState &State) override
Generate the transformed value of the induction at offset StartValue (1.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPDerivedIVRecipe.
Definition VPlan.h:3755
VPValue * getStepValue() const
Definition VPlan.h:3766
Type * getScalarType() const
Definition VPlan.h:3761
VPDerivedIVRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3743
VPDerivedIVRecipe(InductionDescriptor::InductionKind Kind, const FPMathOperator *FPBinOp, VPValue *Start, VPValue *IV, VPValue *Step, const Twine &Name="")
Definition VPlan.h:3735
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
~VPDerivedIVRecipe() override=default
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:3769
VPValue * getStartValue() const
Definition VPlan.h:3765
VPDerivedIVRecipe(const InductionDescriptor &IndDesc, VPValue *Start, VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step, const Twine &Name="")
Definition VPlan.h:3727
Template specialization of the standard LLVM dominator tree utility for VPBlockBases.
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:3663
LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPEVLBasedIVPHIRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3644
~VPEVLBasedIVPHIRecipe() override=default
void execute(VPTransformState &State) override
Generate the phi nodes.
Definition VPlan.h:3650
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPEVLBasedIVPHIRecipe.
Definition VPlan.h:3656
VPEVLBasedIVPHIRecipe(VPValue *StartIV, DebugLoc DL)
Definition VPlan.h:3639
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
Definition VPlan.h:3522
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPExpandSCEVRecipe.
Definition VPlan.h:3527
VPExpandSCEVRecipe(const SCEV *Expr)
Definition VPlan.h:3513
const SCEV * getSCEV() const
Definition VPlan.h:3533
VPExpandSCEVRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3518
~VPExpandSCEVRecipe() override=default
void execute(VPTransformState &State) override
Method for generating code, must not be called as this recipe is abstract.
Definition VPlan.h:3172
VPValue * getOperandOfResultType() const
Return the VPValue to use to infer the result type of the recipe.
Definition VPlan.h:3154
VPExpressionRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3136
void decompose()
Insert the recipes of the expression back into the VPlan, directly before the current recipe.
~VPExpressionRecipe() override
Definition VPlan.h:3124
bool isSingleScalar() const
Returns true if the result of this VPExpressionRecipe is a single-scalar.
VPExpressionRecipe(VPWidenCastRecipe *Ext0, VPWidenCastRecipe *Ext1, VPWidenRecipe *Mul, VPWidenRecipe *Sub, VPReductionRecipe *Red)
Definition VPlan.h:3110
VPExpressionRecipe(VPWidenCastRecipe *Ext, VPReductionRecipe *Red)
Definition VPlan.h:3102
bool mayHaveSideEffects() const
Returns true if this expression contains recipes that may have side effects.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Compute the cost of this recipe either using a recipe's specialized implementation or using the legac...
bool mayReadOrWriteMemory() const
Returns true if this expression contains recipes that may read from or write to memory.
VPExpressionRecipe(VPWidenCastRecipe *Ext0, VPWidenCastRecipe *Ext1, VPWidenRecipe *Mul, VPReductionRecipe *Red)
Definition VPlan.h:3106
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
unsigned getVFScaleFactor() const
Definition VPlan.h:3166
VPExpressionRecipe(VPWidenRecipe *Mul, VPReductionRecipe *Red)
Definition VPlan.h:3104
void execute(VPTransformState &State) override
Produce a vectorized histogram operation.
VP_CLASSOF_IMPL(VPDef::VPHistogramSC)
VPHistogramRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:1768
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPHistogramRecipe.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getMask() const
Return the mask operand if one was provided, or a null pointer if all lanes should be executed uncond...
Definition VPlan.h:1785
unsigned getOpcode() const
Definition VPlan.h:1781
VPHistogramRecipe(unsigned Opcode, ArrayRef< VPValue * > Operands, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:1762
~VPHistogramRecipe() override=default
A special type of VPBasicBlock that wraps an existing IR basic block.
Definition VPlan.h:4119
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPBasicBlock,...
BasicBlock * getIRBasicBlock() const
Definition VPlan.h:4143
static bool classof(const VPBlockBase *V)
Definition VPlan.h:4133
~VPIRBasicBlock() override=default
friend class VPlan
Definition VPlan.h:4120
VPIRBasicBlock * clone() override
Clone the current block and it's recipes, without updating the operands of the cloned recipes.
Class to record and manage LLVM IR flags.
Definition VPlan.h:609
FastMathFlagsTy FMFs
Definition VPlan.h:680
bool flagsValidForOpcode(unsigned Opcode) const
Returns true if the set flags are valid for Opcode.
VPIRFlags(DisjointFlagsTy DisjointFlags)
Definition VPlan.h:740
VPIRFlags(WrapFlagsTy WrapFlags)
Definition VPlan.h:732
WrapFlagsTy WrapFlags
Definition VPlan.h:674
CmpInst::Predicate CmpPredicate
Definition VPlan.h:673
void printFlags(raw_ostream &O) const
VPIRFlags(CmpInst::Predicate Pred, FastMathFlags FMFs)
Definition VPlan.h:726
GEPNoWrapFlags GEPFlags
Definition VPlan.h:678
bool hasFastMathFlags() const
Returns true if the recipe has fast-math flags.
Definition VPlan.h:858
LLVM_ABI_FOR_TEST FastMathFlags getFastMathFlags() const
TruncFlagsTy TruncFlags
Definition VPlan.h:675
CmpInst::Predicate getPredicate() const
Definition VPlan.h:835
bool hasNonNegFlag() const
Returns true if the recipe has non-negative flag.
Definition VPlan.h:865
void transferFlags(VPIRFlags &Other)
Definition VPlan.h:749
ExactFlagsTy ExactFlags
Definition VPlan.h:677
bool hasNoSignedWrap() const
Definition VPlan.h:884
void intersectFlags(const VPIRFlags &Other)
Only keep flags also present in Other.
VPIRFlags()
Definition VPlan.h:686
bool isDisjoint() const
Definition VPlan.h:895
VPIRFlags(TruncFlagsTy TruncFlags)
Definition VPlan.h:735
VPIRFlags(FastMathFlags FMFs)
Definition VPlan.h:738
VPIRFlags(NonNegFlagsTy NonNegFlags)
Definition VPlan.h:743
VPIRFlags(CmpInst::Predicate Pred)
Definition VPlan.h:723
bool isNonNeg() const
Definition VPlan.h:867
GEPNoWrapFlags getGEPNoWrapFlags() const
Definition VPlan.h:850
bool hasPredicate() const
Returns true if the recipe has a comparison predicate.
Definition VPlan.h:853
DisjointFlagsTy DisjointFlags
Definition VPlan.h:676
unsigned AllFlags
Definition VPlan.h:682
void setPredicate(CmpInst::Predicate Pred)
Definition VPlan.h:841
bool hasNoUnsignedWrap() const
Definition VPlan.h:873
FCmpFlagsTy FCmpFlags
Definition VPlan.h:681
NonNegFlagsTy NonNegFlags
Definition VPlan.h:679
void dropPoisonGeneratingFlags()
Drop all poison-generating flags.
Definition VPlan.h:759
void applyFlags(Instruction &I) const
Apply the IR flags to I.
Definition VPlan.h:795
VPIRFlags(GEPNoWrapFlags GEPFlags)
Definition VPlan.h:746
VPIRFlags(Instruction &I)
Definition VPlan.h:688
Instruction & getInstruction() const
Definition VPlan.h:1447
bool usesFirstPartOnly(const VPValue *Op) const override
Returns true if the VPUser only uses the first part of operand Op.
Definition VPlan.h:1455
void extractLastLaneOfLastPartOfFirstOperand(VPBuilder &Builder)
Update the recipe's first operand to the last lane of the last part of the operand using Builder.
~VPIRInstruction() override=default
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
VPIRInstruction * clone() override
Clone the current recipe.
Definition VPlan.h:1434
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the VPUser only uses the first lane of operand Op.
Definition VPlan.h:1461
static LLVM_ABI_FOR_TEST VPIRInstruction * create(Instruction &I)
Create a new VPIRPhi for \I , if it is a PHINode, otherwise create a VPIRInstruction.
LLVM_ABI_FOR_TEST InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPIRInstruction.
bool usesScalars(const VPValue *Op) const override
Returns true if the VPUser uses scalars of operand Op.
Definition VPlan.h:1449
VPIRInstruction(Instruction &I)
VPIRInstruction::create() should be used to create VPIRInstructions, as subclasses may need to be cre...
Definition VPlan.h:1422
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPInstructionWithType(unsigned Opcode, ArrayRef< VPValue * > Operands, Type *ResultTy, const VPIRFlags &Flags={}, const VPIRMetadata &Metadata={}, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="")
Definition VPlan.h:1261
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPInstruction.
Definition VPlan.h:1302
static bool classof(const VPUser *R)
Definition VPlan.h:1287
static bool classof(const VPRecipeBase *R)
Definition VPlan.h:1269
Type * getResultType() const
Definition VPlan.h:1308
VPInstruction * clone() override
Clone the current recipe.
Definition VPlan.h:1291
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the instruction.
This is a concrete Recipe that models a single VPlan-level instruction.
Definition VPlan.h:1036
VPInstruction * clone() override
Clone the current recipe.
Definition VPlan.h:1174
@ BranchOnCond
Definition VPlan.h:1058
@ PtrAdd
Definition VPlan.h:1089
@ ExtractLane
Extracts a single lane (first operand) from a set of vector operands.
Definition VPlan.h:1127
@ Broadcast
Definition VPlan.h:1059
@ LastActiveLane
Definition VPlan.h:1111
@ ComputeAnyOfResult
Compute the final result of a AnyOf reduction with select(cmp(),x,y), where one of (x,...
Definition VPlan.h:1074
@ BranchOnCount
Definition VPlan.h:1057
@ WideIVStep
Scale the first operand (vector step) by the second operand (scalar-step).
Definition VPlan.h:1117
@ ExtractLastPart
Definition VPlan.h:1078
@ ExtractPenultimateElement
Definition VPlan.h:1084
@ ResumeForEpilogue
Explicit user for the resume phi of the canonical induction in the main VPlan, used by the epilogue v...
Definition VPlan.h:1130
@ Unpack
Extracts all lanes from its (non-scalable) vector operand.
Definition VPlan.h:1071
@ ActiveLaneMask
Definition VPlan.h:1052
@ FirstActiveLane
Definition VPlan.h:1104
@ FirstOrderRecurrenceSplice
Definition VPlan.h:1042
@ ExplicitVectorLength
Definition VPlan.h:1053
@ ReductionStartVector
Start vector for reductions with 3 operands: the original start value, the identity value for the red...
Definition VPlan.h:1121
@ SLPStore
Definition VPlan.h:1047
@ BuildVector
Creates a fixed-width vector containing all operands.
Definition VPlan.h:1066
@ OpsEnd
Definition VPlan.h:1133
@ WidePtrAdd
Definition VPlan.h:1092
@ LogicalAnd
Definition VPlan.h:1085
@ BuildStructVector
Given operands of (the same) struct type, creates a struct of fixed- width vectors each containing a ...
Definition VPlan.h:1063
@ VScale
Returns the value for vscale.
Definition VPlan.h:1132
@ CanonicalIVIncrementForPart
Definition VPlan.h:1056
@ SLPLoad
Definition VPlan.h:1046
@ ComputeReductionResult
Definition VPlan.h:1076
@ Not
Definition VPlan.h:1045
@ CalculateTripCountMinusVF
Definition VPlan.h:1054
@ StepVector
Definition VPlan.h:1123
@ ExtractLastLane
Definition VPlan.h:1080
@ AnyOf
Definition VPlan.h:1098
@ ComputeFindIVResult
Definition VPlan.h:1075
bool hasResult() const
Definition VPlan.h:1198
StringRef getName() const
Returns the symbolic name assigned to the VPInstruction.
Definition VPlan.h:1238
unsigned getOpcode() const
Definition VPlan.h:1182
VPInstruction(unsigned Opcode, ArrayRef< VPValue * > Operands, const VPIRFlags &Flags={}, const VPIRMetadata &MD={}, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="")
void setName(StringRef NewName)
Set the symbolic name for the VPInstruction.
Definition VPlan.h:1241
friend class VPlanSlp
Definition VPlan.h:1037
virtual unsigned getNumStoreOperands() const =0
Returns the number of stored operands of this interleave group.
bool usesFirstLaneOnly(const VPValue *Op) const override=0
Returns true if the recipe only uses the first lane of operand Op.
bool needsMaskForGaps() const
Return true if the access needs a mask because of the gaps.
Definition VPlan.h:2652
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
Definition VPlan.h:2658
static bool classof(const VPUser *U)
Definition VPlan.h:2634
VPInterleaveBase(const unsigned char SC, const InterleaveGroup< Instruction > *IG, ArrayRef< VPValue * > Operands, ArrayRef< VPValue * > StoredValues, VPValue *Mask, bool NeedsMaskForGaps, const VPIRMetadata &MD, DebugLoc DL)
Definition VPlan.h:2601
Instruction * getInsertPos() const
Definition VPlan.h:2656
static bool classof(const VPRecipeBase *R)
Definition VPlan.h:2629
const InterleaveGroup< Instruction > * getInterleaveGroup() const
Definition VPlan.h:2654
VPValue * getMask() const
Return the mask used by this recipe.
Definition VPlan.h:2646
ArrayRef< VPValue * > getStoredValues() const
Return the VPValues stored by this interleave group.
Definition VPlan.h:2675
VPInterleaveBase * clone() override=0
Clone the current recipe.
VPValue * getAddr() const
Return the address accessed by this recipe.
Definition VPlan.h:2640
A recipe for interleaved memory operations with vector-predication intrinsics.
Definition VPlan.h:2728
bool usesFirstLaneOnly(const VPValue *Op) const override
The recipe only uses the first lane of the address, and EVL operand.
Definition VPlan.h:2756
VPValue * getEVL() const
The VPValue of the explicit vector length.
Definition VPlan.h:2750
~VPInterleaveEVLRecipe() override=default
unsigned getNumStoreOperands() const override
Returns the number of stored operands of this interleave group.
Definition VPlan.h:2763
VPInterleaveEVLRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2743
VPInterleaveEVLRecipe(VPInterleaveRecipe &R, VPValue &EVL, VPValue *Mask)
Definition VPlan.h:2730
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
Definition VPlan.h:2686
unsigned getNumStoreOperands() const override
Returns the number of stored operands of this interleave group.
Definition VPlan.h:2713
~VPInterleaveRecipe() override=default
VPInterleaveRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2696
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:2707
VPInterleaveRecipe(const InterleaveGroup< Instruction > *IG, VPValue *Addr, ArrayRef< VPValue * > StoredValues, VPValue *Mask, bool NeedsMaskForGaps, const VPIRMetadata &MD, DebugLoc DL)
Definition VPlan.h:2688
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
Helper type to provide functions to access incoming values and blocks for phi-like recipes.
Definition VPlan.h:1320
virtual const VPRecipeBase * getAsRecipe() const =0
Return a VPRecipeBase* to the current object.
VPUser::const_operand_range incoming_values() const
Returns an interator range over the incoming values.
Definition VPlan.h:1342
virtual unsigned getNumIncoming() const
Returns the number of incoming values, also number of incoming blocks.
Definition VPlan.h:1337
void removeIncomingValueFor(VPBlockBase *IncomingBlock) const
Removes the incoming value for IncomingBlock, which must be a predecessor.
const VPBasicBlock * getIncomingBlock(unsigned Idx) const
Returns the incoming block with index Idx.
Definition VPlan.h:4110
detail::zippy< llvm::detail::zip_first, VPUser::const_operand_range, const_incoming_blocks_range > incoming_values_and_blocks() const
Returns an iterator range over pairs of incoming values and corresponding incoming blocks.
Definition VPlan.h:1362
VPValue * getIncomingValue(unsigned Idx) const
Returns the incoming VPValue with index Idx.
Definition VPlan.h:1329
virtual ~VPPhiAccessors()=default
void printPhiOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the recipe.
iterator_range< mapped_iterator< detail::index_iterator, std::function< const VPBasicBlock *(size_t)> > > const_incoming_blocks_range
Definition VPlan.h:1347
const_incoming_blocks_range incoming_blocks() const
Returns an iterator range over the incoming blocks.
Definition VPlan.h:1351
~VPPredInstPHIRecipe() override=default
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
Definition VPlan.h:3229
VPPredInstPHIRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3211
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPPredInstPHIRecipe.
Definition VPlan.h:3222
VPPredInstPHIRecipe(VPValue *PredV, DebugLoc DL)
Construct a VPPredInstPHIRecipe given PredInst whose value needs a phi nodes after merging back from ...
Definition VPlan.h:3207
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition VPlan.h:387
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
bool mayReadOrWriteMemory() const
Returns true if the recipe may read from or write to memory.
Definition VPlan.h:474
virtual void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const =0
Each concrete VPRecipe prints itself, without printing common information, like debug info or metadat...
VPRegionBlock * getRegion()
Definition VPlan.h:4271
void setDebugLoc(DebugLoc NewDL)
Set the recipe's debug location to NewDL.
Definition VPlan.h:485
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
~VPRecipeBase() override=default
VPBasicBlock * getParent()
Definition VPlan.h:408
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
Definition VPlan.h:479
virtual void execute(VPTransformState &State)=0
The method which generates the output IR instructions that correspond to this VPRecipe,...
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition VPlan.h:454
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
virtual VPRecipeBase * clone()=0
Clone the current recipe.
friend class VPBlockUtils
Definition VPlan.h:389
const VPBasicBlock * getParent() const
Definition VPlan.h:409
InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this recipe, taking into account if the cost computation should be skipped and the...
static bool classof(const VPUser *U)
Definition VPlan.h:459
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
VPRecipeBase(const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:398
VPValue * getEVL() const
The VPValue of the explicit vector length.
Definition VPlan.h:2913
VPReductionEVLRecipe(VPReductionRecipe &R, VPValue &EVL, VPValue *CondOp, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:2892
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:2916
VPReductionEVLRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2903
~VPReductionEVLRecipe() override=default
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
Definition VPlan.h:2479
VPReductionPHIRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2450
unsigned getVFScaleFactor() const
Get the factor that the VF of this recipe's output should be scaled by, or 1 if it isn't scaled.
Definition VPlan.h:2465
~VPReductionPHIRecipe() override=default
bool hasUsesOutsideReductionChain() const
Returns true, if the phi is part of a multi-use reduction.
Definition VPlan.h:2491
unsigned getNumIncoming() const override
Returns the number of incoming values, also number of incoming blocks.
Definition VPlan.h:2473
bool isInLoop() const
Returns true if the phi is part of an in-loop reduction.
Definition VPlan.h:2482
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:2496
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool isPartialReduction() const
Returns true if the reduction outputs a vector with a scaled down VF.
Definition VPlan.h:2488
VPReductionPHIRecipe(PHINode *Phi, RecurKind Kind, VPValue &Start, ReductionStyle Style, bool HasUsesOutsideReductionChain=false)
Create a new VPReductionPHIRecipe for the reduction Phi.
Definition VPlan.h:2441
RecurKind getRecurrenceKind() const
Returns the recurrence kind of the reduction.
Definition VPlan.h:2476
A recipe to represent inloop, ordered or partial reduction operations.
Definition VPlan.h:2779
VPReductionRecipe(const unsigned char SC, RecurKind RdxKind, FastMathFlags FMFs, Instruction *I, ArrayRef< VPValue * > Operands, VPValue *CondOp, ReductionStyle Style, DebugLoc DL)
Definition VPlan.h:2788
bool isConditional() const
Return true if the in-loop reduction is conditional.
Definition VPlan.h:2855
static bool classof(const VPRecipeBase *R)
Definition VPlan.h:2824
static bool classof(const VPSingleDefRecipe *R)
Definition VPlan.h:2839
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
Definition VPlan.h:2866
VPValue * getCondOp() const
The VPValue of the condition for the block.
Definition VPlan.h:2868
RecurKind getRecurrenceKind() const
Return the recurrence kind for the in-loop reduction.
Definition VPlan.h:2851
VPReductionRecipe(RecurKind RdxKind, FastMathFlags FMFs, Instruction *I, VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp, ReductionStyle Style, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:2802
bool isOrdered() const
Return true if the in-loop reduction is ordered.
Definition VPlan.h:2853
VPReductionRecipe(const RecurKind RdxKind, FastMathFlags FMFs, VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp, ReductionStyle Style, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:2809
bool isPartialReduction() const
Returns true if the reduction outputs a vector with a scaled down VF.
Definition VPlan.h:2857
~VPReductionRecipe() override=default
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
Definition VPlan.h:2864
bool isInLoop() const
Returns true if the reduction is in-loop.
Definition VPlan.h:2859
VPReductionRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2818
static bool classof(const VPUser *U)
Definition VPlan.h:2829
static bool classof(const VPValue *VPV)
Definition VPlan.h:2834
unsigned getVFScaleFactor() const
Get the factor that the VF of this recipe's output should be scaled by, or 1 if it isn't scaled.
Definition VPlan.h:2873
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition VPlan.h:4154
const VPBlockBase * getEntry() const
Definition VPlan.h:4190
Type * getCanonicalIVType()
Return the type of the canonical IV for loop regions.
Definition VPlan.h:4265
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
Definition VPlan.h:4222
~VPRegionBlock() override=default
void setExiting(VPBlockBase *ExitingBlock)
Set ExitingBlock as the exiting VPBlockBase of this VPRegionBlock.
Definition VPlan.h:4207
VPBlockBase * getExiting()
Definition VPlan.h:4203
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the region.
Definition VPlan.h:4252
void setEntry(VPBlockBase *EntryBlock)
Set EntryBlock as the entry VPBlockBase of this VPRegionBlock.
Definition VPlan.h:4195
const Type * getCanonicalIVType() const
Definition VPlan.h:4266
const VPBlockBase * getExiting() const
Definition VPlan.h:4202
VPBlockBase * getEntry()
Definition VPlan.h:4191
const VPCanonicalIVPHIRecipe * getCanonicalIV() const
Definition VPlan.h:4260
VPBasicBlock * getPreheaderVPBB()
Returns the pre-header VPBasicBlock of the loop region.
Definition VPlan.h:4215
friend class VPlan
Definition VPlan.h:4155
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition VPlan.h:4186
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
Definition VPlan.h:2935
bool isSingleScalar() const
Definition VPlan.h:2976
VPReplicateRecipe(Instruction *I, ArrayRef< VPValue * > Operands, bool IsSingleScalar, VPValue *Mask=nullptr, const VPIRFlags &Flags={}, VPIRMetadata Metadata={}, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:2943
~VPReplicateRecipe() override=default
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
Definition VPlan.h:2988
bool isPredicated() const
Definition VPlan.h:2978
VPReplicateRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2957
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:2981
unsigned getOpcode() const
Definition VPlan.h:3005
VPValue * getMask()
Return the mask of a predicated VPReplicateRecipe.
Definition VPlan.h:3000
VPValue * getStepValue() const
Definition VPlan.h:3832
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPScalarIVStepsRecipe.
Definition VPlan.h:3826
VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, VPValue *IV, VPValue *Step, VPValue *VF, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:3797
bool isPart0() const
Return true if this VPScalarIVStepsRecipe corresponds to part 0.
Definition VPlan.h:3818
VPScalarIVStepsRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3809
VPScalarIVStepsRecipe(VPValue *IV, VPValue *Step, VPValue *VF, Instruction::BinaryOps Opcode, FastMathFlags FMFs, DebugLoc DL)
Definition VPlan.h:3790
~VPScalarIVStepsRecipe() override=default
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:3835
VPSingleDef is a base class for recipes for modeling a sequence of one or more output IR that define ...
Definition VPlan.h:531
VPSingleDefRecipe(const unsigned char SC, ArrayRef< VPValue * > Operands, Value *UV, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:537
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
Definition VPlan.h:595
static bool classof(const VPRecipeBase *R)
Definition VPlan.h:541
const Instruction * getUnderlyingInstr() const
Definition VPlan.h:598
static bool classof(const VPUser *U)
Definition VPlan.h:587
LLVM_ABI_FOR_TEST LLVM_DUMP_METHOD void dump() const
Print this VPSingleDefRecipe to dbgs() (for debugging).
VPSingleDefRecipe * clone() override=0
Clone the current recipe.
VPSingleDefRecipe(const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:533
This class can be used to assign names to VPValues.
Helper to access the operand that contains the unroll part for this recipe after unrolling.
Definition VPlan.h:970
VPValue * getUnrollPartOperand(const VPUser &U) const
Return the VPValue operand containing the unroll part or null if there is no such operand.
unsigned getUnrollPart(const VPUser &U) const
Return the unroll part.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
void setOperand(unsigned I, VPValue *New)
unsigned getNumOperands() const
operand_iterator op_end()
VPValue * getOperand(unsigned N) const
VPUser(ArrayRef< VPValue * > Operands)
iterator_range< const_operand_iterator > const_operand_range
iterator_range< operand_iterator > operand_range
This is the base class of the VPlan Def/Use graph, used for modeling the data flow into,...
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
friend class VPExpressionRecipe
Value * getLiveInIRValue() const
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
VPValue(const unsigned char SC, Value *UV=nullptr, VPDef *Def=nullptr)
void setUnderlyingValue(Value *Val)
unsigned getNumUsers() const
bool isLiveIn() const
Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the VPUser only uses the first lane of operand Op.
Definition VPlan.h:1934
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getVFValue()
Definition VPlan.h:1929
VPVectorEndPointerRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:1955
const VPValue * getVFValue() const
Definition VPlan.h:1930
bool usesFirstPartOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first part of operand Op.
Definition VPlan.h:1948
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPVectorPointerRecipe.
Definition VPlan.h:1941
VPVectorEndPointerRecipe(VPValue *Ptr, VPValue *VF, Type *IndexedTy, int64_t Stride, GEPNoWrapFlags GEPFlags, DebugLoc DL)
Definition VPlan.h:1919
bool isFirstPart() const
Return true if this VPVectorPointerRecipe corresponds to part 0.
Definition VPlan.h:2008
Type * getSourceElementType() const
Definition VPlan.h:1985
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the VPUser only uses the first lane of operand Op.
Definition VPlan.h:1987
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
bool usesFirstPartOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first part of operand Op.
Definition VPlan.h:1994
VPVectorPointerRecipe(VPValue *Ptr, Type *SourceElementTy, GEPNoWrapFlags GEPFlags, DebugLoc DL)
Definition VPlan.h:1975
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPHeaderPHIRecipe.
Definition VPlan.h:2011
VPVectorPointerRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2001
A recipe for widening Call instructions using library calls.
Definition VPlan.h:1702
VPWidenCallRecipe(Value *UV, Function *Variant, ArrayRef< VPValue * > CallArguments, const VPIRFlags &Flags={}, const VPIRMetadata &Metadata={}, DebugLoc DL={})
Definition VPlan.h:1709
const_operand_range args() const
Definition VPlan.h:1742
VPWidenCallRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:1723
operand_range args()
Definition VPlan.h:1741
Function * getCalledScalarFunction() const
Definition VPlan.h:1737
~VPWidenCallRecipe() override=default
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
~VPWidenCanonicalIVRecipe() override=default
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenCanonicalIVPHIRecipe.
Definition VPlan.h:3699
VPWidenCanonicalIVRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3686
VPWidenCanonicalIVRecipe(VPCanonicalIVPHIRecipe *CanonicalIV)
Definition VPlan.h:3681
VPWidenCastRecipe is a recipe to create vector cast instructions.
Definition VPlan.h:1552
Instruction::CastOps getOpcode() const
Definition VPlan.h:1588
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getResultType() const
Returns the result type of the cast.
Definition VPlan.h:1591
void execute(VPTransformState &State) override
Produce widened copies of the cast.
~VPWidenCastRecipe() override=default
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenCastRecipe.
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, CastInst *CI=nullptr, const VPIRFlags &Flags={}, const VPIRMetadata &Metadata={}, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:1560
VPWidenCastRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:1573
unsigned getOpcode() const
This recipe generates a GEP instruction.
Definition VPlan.h:1882
Type * getSourceElementType() const
Definition VPlan.h:1887
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenGEPRecipe.
Definition VPlan.h:1890
VPWidenGEPRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:1874
~VPWidenGEPRecipe() override=default
VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef< VPValue * > Operands, const VPIRFlags &Flags={}, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:1860
void execute(VPTransformState &State) override=0
Generate the phi nodes.
VPValue * getVFValue()
Definition VPlan.h:2150
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:2178
static bool classof(const VPValue *V)
Definition VPlan.h:2132
void setStepValue(VPValue *V)
Update the step value of the recipe.
Definition VPlan.h:2148
VPValue * getBackedgeValue() override
Returns the incoming value from the loop backedge.
Definition VPlan.h:2163
unsigned getNumIncoming() const override
Returns the number of incoming values, also number of incoming blocks.
Definition VPlan.h:2156
PHINode * getPHINode() const
Definition VPlan.h:2158
VPWidenInductionRecipe(unsigned char Kind, PHINode *IV, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, DebugLoc DL)
Definition VPlan.h:2120
VPValue * getStepValue()
Returns the step value of the induction.
Definition VPlan.h:2144
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
Definition VPlan.h:2161
VPRecipeBase & getBackedgeRecipe() override
Returns the backedge value as a recipe.
Definition VPlan.h:2170
static bool classof(const VPRecipeBase *R)
Definition VPlan.h:2127
const VPValue * getVFValue() const
Definition VPlan.h:2151
static bool classof(const VPSingleDefRecipe *R)
Definition VPlan.h:2137
const VPValue * getStepValue() const
Definition VPlan.h:2145
const TruncInst * getTruncInst() const
Definition VPlan.h:2252
void execute(VPTransformState &State) override
Generate the phi nodes.
Definition VPlan.h:2233
~VPWidenIntOrFpInductionRecipe() override=default
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, VPValue *VF, const InductionDescriptor &IndDesc, TruncInst *Trunc, const VPIRFlags &Flags, DebugLoc DL)
Definition VPlan.h:2208
VPWidenIntOrFpInductionRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2225
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
Definition VPlan.h:2251
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, VPValue *VF, const InductionDescriptor &IndDesc, const VPIRFlags &Flags, DebugLoc DL)
Definition VPlan.h:2199
VPValue * getLastUnrolledPartOperand()
Returns the VPValue representing the value of this induction at the last unrolled part,...
Definition VPlan.h:2268
unsigned getNumIncoming() const override
Returns the number of incoming values, also number of incoming blocks.
Definition VPlan.h:2247
Type * getScalarType() const
Returns the scalar type of the induction.
Definition VPlan.h:2260
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getSplatVFValue()
Definition VPlan.h:2238
A recipe for widening vector intrinsics.
Definition VPlan.h:1602
VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID, ArrayRef< VPValue * > CallArguments, Type *Ty, const VPIRFlags &Flags={}, const VPIRMetadata &Metadata={}, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:1633
Intrinsic::ID getVectorIntrinsicID() const
Return the ID of the intrinsic.
Definition VPlan.h:1673
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool mayReadFromMemory() const
Returns true if the intrinsic may read from memory.
Definition VPlan.h:1682
StringRef getIntrinsicName() const
Return to name of the intrinsic as string.
VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID, ArrayRef< VPValue * > CallArguments, Type *Ty, const VPIRFlags &Flags={}, const VPIRMetadata &MD={}, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:1619
bool mayHaveSideEffects() const
Returns true if the intrinsic may have side-effects.
Definition VPlan.h:1688
VPWidenIntrinsicRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:1654
bool mayWriteToMemory() const
Returns true if the intrinsic may write to memory.
Definition VPlan.h:1685
~VPWidenIntrinsicRecipe() override=default
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the VPUser only uses the first lane of operand Op.
Type * getResultType() const
Return the scalar return type of the intrinsic.
Definition VPlan.h:1676
void execute(VPTransformState &State) override
Produce a widened version of the vector intrinsic.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this vector intrinsic.
bool IsMasked
Whether the memory access is masked.
Definition VPlan.h:3260
bool Reverse
Whether the consecutive accessed addresses are in reverse order.
Definition VPlan.h:3257
bool isConsecutive() const
Return whether the loaded-from / stored-to addresses are consecutive.
Definition VPlan.h:3300
static bool classof(const VPUser *U)
Definition VPlan.h:3294
void execute(VPTransformState &State) override
Generate the wide load/store.
Definition VPlan.h:3323
Instruction & Ingredient
Definition VPlan.h:3248
VPWidenMemoryRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3283
Instruction & getIngredient() const
Definition VPlan.h:3331
bool Consecutive
Whether the accessed addresses are consecutive.
Definition VPlan.h:3254
static bool classof(const VPRecipeBase *R)
Definition VPlan.h:3287
VPValue * getMask() const
Return the mask used by this recipe.
Definition VPlan.h:3314
Align Alignment
Alignment information for this memory access.
Definition VPlan.h:3251
bool isMasked() const
Returns true if the recipe is masked.
Definition VPlan.h:3310
VPWidenMemoryRecipe(const char unsigned SC, Instruction &I, std::initializer_list< VPValue * > Operands, bool Consecutive, bool Reverse, const VPIRMetadata &Metadata, DebugLoc DL)
Definition VPlan.h:3270
void setMask(VPValue *Mask)
Definition VPlan.h:3262
Align getAlign() const
Returns the alignment of the memory access.
Definition VPlan.h:3320
VPValue * getAddr() const
Return the address accessed by this recipe.
Definition VPlan.h:3307
bool isReverse() const
Return whether the consecutive loaded/stored addresses are in reverse order.
Definition VPlan.h:3304
const VPRecipeBase * getAsRecipe() const override
Return a VPRecipeBase* to the current object.
Definition VPlan.h:2362
VPWidenPHIRecipe(PHINode *Phi, VPValue *Start=nullptr, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="")
Create a new VPWidenPHIRecipe for Phi with start value Start and debug location DL.
Definition VPlan.h:2333
VPWidenPHIRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2340
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
~VPWidenPHIRecipe() override=default
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPWidenPointerInductionRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2295
~VPWidenPointerInductionRecipe() override=default
bool onlyScalarsGenerated(bool IsScalable)
Returns true if only scalar values will be generated.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate vector values for the pointer induction.
Definition VPlan.h:2304
VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, VPValue *Step, VPValue *NumUnrolledElems, const InductionDescriptor &IndDesc, DebugLoc DL)
Create a new VPWidenPointerInductionRecipe for Phi with start value Start and the number of elements ...
Definition VPlan.h:2285
VPWidenRecipe is a recipe for producing a widened instruction using the opcode and operands of the re...
Definition VPlan.h:1512
VPWidenRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:1526
VPWidenRecipe(Instruction &I, ArrayRef< VPValue * > Operands, const VPIRFlags &Flags={}, const VPIRMetadata &Metadata={}, DebugLoc DL={})
Definition VPlan.h:1516
~VPWidenRecipe() override=default
unsigned getOpcode() const
Definition VPlan.h:1541
Class that maps (parts of) an existing VPlan to trees of combined VPInstructions.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition VPlan.h:4284
LLVM_ABI_FOR_TEST void printDOT(raw_ostream &O) const
Print this VPlan in DOT format to O.
friend class VPSlotTracker
Definition VPlan.h:4286
std::string getName() const
Return a string with the name of the plan and the applicable VFs and UFs.
bool hasVF(ElementCount VF) const
Definition VPlan.h:4489
LLVMContext & getContext() const
Definition VPlan.h:4477
VPBasicBlock * getEntry()
Definition VPlan.h:4377
VPValue & getVectorTripCount()
The vector trip count.
Definition VPlan.h:4468
void setName(const Twine &newName)
Definition VPlan.h:4527
bool hasScalableVF() const
Definition VPlan.h:4490
VPValue & getVFxUF()
Returns VF * UF of the vector loop region.
Definition VPlan.h:4475
VPValue & getVF()
Returns the VF of the vector loop region.
Definition VPlan.h:4471
VPValue * getTripCount() const
The trip count of the original loop.
Definition VPlan.h:4439
VPValue * getTrue()
Return a VPValue wrapping i1 true.
Definition VPlan.h:4546
VPValue * getOrCreateBackedgeTakenCount()
The backedge taken count of the original loop.
Definition VPlan.h:4460
iterator_range< SmallSetVector< ElementCount, 2 >::iterator > vectorFactors() const
Returns an iterator range over all VFs of the plan.
Definition VPlan.h:4496
VPIRBasicBlock * getExitBlock(BasicBlock *IRBB) const
Return the VPIRBasicBlock corresponding to IRBB.
LLVM_ABI_FOR_TEST ~VPlan()
const VPValue & getVF() const
Definition VPlan.h:4472
bool isExitBlock(VPBlockBase *VPBB)
Returns true if VPBB is an exit block.
const VPBasicBlock * getEntry() const
Definition VPlan.h:4378
friend class VPlanPrinter
Definition VPlan.h:4285
VPValue * getConstantInt(const APInt &Val)
Return a VPValue wrapping a ConstantInt with the given APInt value.
Definition VPlan.h:4563
unsigned getUF() const
Definition VPlan.h:4509
VPRegionBlock * createReplicateRegion(VPBlockBase *Entry, VPBlockBase *Exiting, const std::string &Name="")
Create a new replicate region with Entry, Exiting and Name.
Definition VPlan.h:4623
VPIRBasicBlock * createEmptyVPIRBasicBlock(BasicBlock *IRBB)
Create a VPIRBasicBlock wrapping IRBB, but do not create VPIRInstructions wrapping the instructions i...
bool hasUF(unsigned UF) const
Definition VPlan.h:4507
ArrayRef< VPIRBasicBlock * > getExitBlocks() const
Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of the original scalar loop.
Definition VPlan.h:4429
VPValue * getConstantInt(Type *Ty, uint64_t Val, bool IsSigned=false)
Return a VPValue wrapping a ConstantInt with the given type and value.
Definition VPlan.h:4552
VPValue * getBackedgeTakenCount() const
Definition VPlan.h:4465
void setVF(ElementCount VF)
Definition VPlan.h:4483
bool isUnrolled() const
Returns true if the VPlan already has been unrolled, i.e.
Definition VPlan.h:4522
LLVM_ABI_FOR_TEST VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
bool hasEarlyExit() const
Returns true if the VPlan is based on a loop with an early exit.
Definition VPlan.h:4645
InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this plan.
const VPBasicBlock * getMiddleBlock() const
Definition VPlan.h:4415
void setTripCount(VPValue *NewTripCount)
Set the trip count assuming it is currently null; if it is not - use resetTripCount().
Definition VPlan.h:4446
void resetTripCount(VPValue *NewTripCount)
Resets the trip count for the VPlan.
Definition VPlan.h:4453
VPBasicBlock * getMiddleBlock()
Returns the 'middle' block of the plan, that is the block that selects whether to execute the scalar ...
Definition VPlan.h:4402
void setEntry(VPBasicBlock *VPBB)
Definition VPlan.h:4366
VPBasicBlock * createVPBasicBlock(const Twine &Name, VPRecipeBase *Recipe=nullptr)
Create a new VPBasicBlock with Name and containing Recipe if present.
Definition VPlan.h:4601
LLVM_ABI_FOR_TEST VPIRBasicBlock * createVPIRBasicBlock(BasicBlock *IRBB)
Create a VPIRBasicBlock from IRBB containing VPIRInstructions for all instructions in IRBB,...
VPValue * getFalse()
Return a VPValue wrapping i1 false.
Definition VPlan.h:4549
VPValue * getOrAddLiveIn(Value *V)
Gets the live-in VPValue for V or adds a new live-in (if none exists yet) for V.
Definition VPlan.h:4531
VPRegionBlock * createLoopRegion(const std::string &Name="", VPBlockBase *Entry=nullptr, VPBlockBase *Exiting=nullptr)
Create a new loop region with Name and entry and exiting blocks set to Entry and Exiting respectively...
Definition VPlan.h:4611
LLVM_DUMP_METHOD void dump() const
Dump the plan to stderr (for debugging).
bool hasScalarVFOnly() const
Definition VPlan.h:4500
VPBasicBlock * getScalarPreheader() const
Return the VPBasicBlock for the preheader of the scalar loop.
Definition VPlan.h:4420
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
ArrayRef< VPValue * > getLiveIns() const
Return the list of live-in VPValues available in the VPlan.
Definition VPlan.h:4571
LLVM_ABI_FOR_TEST void print(raw_ostream &O) const
Print this VPlan to O.
void addVF(ElementCount VF)
Definition VPlan.h:4481
VPIRBasicBlock * getScalarHeader() const
Return the VPIRBasicBlock wrapping the header of the scalar loop.
Definition VPlan.h:4425
VPValue * getLiveIn(Value *V) const
Return the live-in VPValue for V, if there is one or nullptr otherwise.
Definition VPlan.h:4568
VPValue * getConstantInt(unsigned BitWidth, uint64_t Val, bool IsSigned=false)
Return a VPValue wrapping a ConstantInt with the given bitwidth and value.
Definition VPlan.h:4557
void printLiveIns(raw_ostream &O) const
Print the live-ins of this VPlan to O.
VPBasicBlock * getVectorPreheader()
Returns the preheader of the vector loop region, if one exists, or null otherwise.
Definition VPlan.h:4382
void setUF(unsigned UF)
Definition VPlan.h:4514
bool hasScalarTail() const
Returns true if the scalar tail may execute after the vector loop.
Definition VPlan.h:4655
LLVM_ABI_FOR_TEST VPlan * duplicate()
Clone the current VPlan, update all VPValues of the new VPlan and cloned recipes to refer to the clon...
VPlan(BasicBlock *ScalarHeaderBB)
Construct a VPlan with a new VPBasicBlock as entry, a VPIRBasicBlock wrapping ScalarHeaderBB and a tr...
Definition VPlan.h:4359
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
ilist_node_with_parent()=default
Increasing range of size_t indices.
typename base_list_type::const_reverse_iterator const_reverse_iterator
typename base_list_type::reverse_iterator reverse_iterator
typename base_list_type::iterator iterator
typename base_list_type::const_iterator const_iterator
An intrusive list with ownership and callbacks specified/controlled by ilist_traits,...
A range adaptor for a pair of iterators.
This class implements an extremely fast bulk output stream that can only output to a stream.
This file defines classes to implement an intrusive doubly linked list class (i.e.
This file defines the ilist_node class template, which is a convenient base class for creating classe...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI AttributeSet getFnAttributes(LLVMContext &C, ID id)
Return the function attributes for an intrinsic.
std::variant< std::monostate, Loc::Single, Loc::Multi, Loc::MMI, Loc::EntryValue > Variant
Alias for the std::variant specialization base class of DbgVariable.
static auto castToVPIRMetadata(RecipeBasePtrTy R) -> DstTy
Definition VPlan.h:3898
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
detail::zippy< detail::zip_first, T, U, Args... > zip_equal(T &&t, U &&u, Args &&...args)
zip iterator that assumes that all iteratees have the same length.
ReductionStyle getReductionStyle(bool InLoop, bool Ordered, unsigned ScaleFactor)
Definition VPlan.h:2413
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast - Return the argument parameter cast to the specified type.
LLVM_ABI void getMetadataToPropagate(Instruction *Inst, SmallVectorImpl< std::pair< unsigned, MDNode * > > &Metadata)
Add metadata from Inst to Metadata, if it can be preserved after vectorization.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
auto cast_or_null(const Y &Val)
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
MemoryEffectsBase< IRMemLocation > MemoryEffects
Summary of how a function affects memory in the program.
auto map_range(ContainerTy &&C, FuncTy F)
auto dyn_cast_or_null(const Y &Val)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isa(const From &Val)
isa - Return true if the parameter to the template is an instance of one of the template type argu...
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
RecurKind
These are the kinds of recurrences that we support.
@ Mul
Product of integers.
@ Sub
Subtraction of integers.
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
DWARFExpression::Operation Op
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
decltype(auto) cast(const From &Val)
cast - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
DenseMap< Value *, VPValue * > Value2VPValueTy
std::variant< RdxOrdered, RdxInLoop, RdxUnordered > ReductionStyle
Definition VPlan.h:2411
std::unique_ptr< VPlan > VPlanPtr
Definition VPlan.h:77
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Support casting from VPRecipeBase -> VPPhiAccessors, by down-casting to the recipe types implementing...
Definition VPlan.h:3860
static VPPhiAccessors * doCastIfPossible(SrcTy f)
doCastIfPossible is used by dyn_cast<>.
Definition VPlan.h:3881
CastInfo< VPPhiAccessors, SrcTy > Self
Definition VPlan.h:3862
static VPPhiAccessors * doCast(SrcTy R)
doCast is used by cast<>.
Definition VPlan.h:3865
This struct provides a method for customizing the way a cast is performed.
static bool isPossible(const VPRecipeBase *f)
Definition VPlan.h:3852
This struct provides a way to check if a given cast is possible.
static bool isPossible(const SrcTy &f)
This reduction is in-loop.
Definition VPlan.h:2405
Possible variants of a reduction.
Definition VPlan.h:2403
This reduction is unordered with the partial result scaled down by some factor.
Definition VPlan.h:2408
unsigned VFScaleFactor
Definition VPlan.h:2409
Struct to hold various analysis needed for cost computations.
void execute(VPTransformState &State) override
Generate the phi nodes.
VPFirstOrderRecurrencePHIRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:2374
VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start)
Definition VPlan.h:2369
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this first-order recurrence phi recipe.
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:2386
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
DisjointFlagsTy(bool IsDisjoint)
Definition VPlan.h:640
char IsDisjoint
Definition VPlan.h:639
NonNegFlagsTy(bool IsNonNeg)
Definition VPlan.h:645
char NonNeg
Definition VPlan.h:644
TruncFlagsTy(bool HasNUW, bool HasNSW)
Definition VPlan.h:635
char HasNSW
Definition VPlan.h:633
char HasNUW
Definition VPlan.h:632
WrapFlagsTy(bool HasNUW, bool HasNSW)
Definition VPlan.h:628
char HasNSW
Definition VPlan.h:626
char HasNUW
Definition VPlan.h:625
PHINode & getIRPhi()
Definition VPlan.h:1493
VPIRPhi(PHINode &PN)
Definition VPlan.h:1486
static bool classof(const VPRecipeBase *U)
Definition VPlan.h:1488
const VPRecipeBase * getAsRecipe() const override
Return a VPRecipeBase* to the current object.
Definition VPlan.h:1504
static bool classof(const VPUser *U)
Definition VPlan.h:1380
VPPhi * clone() override
Clone the current recipe.
Definition VPlan.h:1395
const VPRecipeBase * getAsRecipe() const override
Return a VPRecipeBase* to the current object.
Definition VPlan.h:1410
VPPhi(ArrayRef< VPValue * > Operands, DebugLoc DL, const Twine &Name="")
Definition VPlan.h:1377
static bool classof(const VPSingleDefRecipe *SDR)
Definition VPlan.h:1390
static bool classof(const VPValue *V)
Definition VPlan.h:1385
A pure-virtual common base class for recipes defining a single VPValue and using IR flags.
Definition VPlan.h:923
static bool classof(const VPRecipeBase *R)
Definition VPlan.h:929
InstructionCost getCostForRecipeWithOpcode(unsigned Opcode, ElementCount VF, VPCostContext &Ctx) const
Compute the cost for this recipe for VF, using Opcode and Ctx.
VPRecipeWithIRFlags(const unsigned char SC, ArrayRef< VPValue * > Operands, const VPIRFlags &Flags, DebugLoc DL=DebugLoc::getUnknown())
Definition VPlan.h:924
static bool classof(const VPValue *V)
Definition VPlan.h:949
static bool classof(const VPSingleDefRecipe *U)
Definition VPlan.h:956
void execute(VPTransformState &State) override=0
The method which generates the output IR instructions that correspond to this VPRecipe,...
VPRecipeWithIRFlags * clone() override=0
Clone the current recipe.
static bool classof(const VPUser *U)
Definition VPlan.h:944
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
A recipe for widening load operations with vector-predication intrinsics, using the address to load f...
Definition VPlan.h:3378
void execute(VPTransformState &State) override
Generate the wide load or gather.
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenLoadEVLRecipe.
VPValue * getEVL() const
Return the EVL operand.
Definition VPlan.h:3391
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue *Addr, VPValue &EVL, VPValue *Mask)
Definition VPlan.h:3379
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:3401
A recipe for widening load operations, using the address to load from and an optional mask.
Definition VPlan.h:3337
VP_CLASSOF_IMPL(VPDef::VPWidenLoadSC)
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:3359
void execute(VPTransformState &State) override
Generate a wide load or gather.
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask, bool Consecutive, bool Reverse, const VPIRMetadata &Metadata, DebugLoc DL)
Definition VPlan.h:3338
VPWidenLoadRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3347
A recipe for widening select instructions.
Definition VPlan.h:1801
VPWidenSelectRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:1812
VPWidenSelectRecipe(SelectInst *SI, ArrayRef< VPValue * > Operands, const VPIRFlags &Flags={}, const VPIRMetadata &MD={}, DebugLoc DL={})
Definition VPlan.h:1802
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:1833
VPValue * getCond() const
Definition VPlan.h:1828
unsigned getOpcode() const
Definition VPlan.h:1826
~VPWidenSelectRecipe() override=default
A recipe for widening store operations with vector-predication intrinsics, using the value to store,...
Definition VPlan.h:3462
VPValue * getStoredValue() const
Return the address accessed by this recipe.
Definition VPlan.h:3474
void execute(VPTransformState &State) override
Generate the wide store or scatter.
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:3487
VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue *Addr, VPValue &EVL, VPValue *Mask)
Definition VPlan.h:3463
void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenStoreEVLRecipe.
VPValue * getEVL() const
Return the EVL operand.
Definition VPlan.h:3477
A recipe for widening store operations, using the stored value, the address to store to and an option...
Definition VPlan.h:3419
VP_CLASSOF_IMPL(VPDef::VPWidenStoreSC)
VPValue * getStoredValue() const
Return the value stored by this recipe.
Definition VPlan.h:3437
VPWidenStoreRecipe * clone() override
Clone the current recipe.
Definition VPlan.h:3428
bool usesFirstLaneOnly(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Definition VPlan.h:3443
VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal, VPValue *Mask, bool Consecutive, bool Reverse, const VPIRMetadata &Metadata, DebugLoc DL)
Definition VPlan.h:3420