LLVM: lib/CodeGen/SelectionDAG/LegalizeDAG.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
51#include
52#include
53#include
54#include
55
56using namespace llvm;
57
58#define DEBUG_TYPE "legalizedag"
59
60namespace {
61
62
63
64struct FloatSignAsInt {
65 EVT FloatVT;
74};
75
76
77
78
79
80
81
82
83
84
85
86
87class SelectionDAGLegalize {
91
92
93
95
96
98
99 EVT getSetCCResultType(EVT VT) const {
101 }
102
103
104
105public:
110 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
111
112
114
115private:
117
120
122
123
124
125
126
130
131 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
133 bool IsSigned, EVT RetVT);
134 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
135
136 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
138 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
139 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
140 RTLIB::Libcall Call_F128,
141 RTLIB::Libcall Call_PPCF128,
143
144 void
145 ExpandFastFPLibCall(SDNode *Node, bool IsFast,
146 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
147 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
148 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
149 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
150 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
152
154 RTLIB::Libcall Call_I16, RTLIB::Libcall Call_I32,
155 RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128);
156 void ExpandArgFPLibCall(SDNode *Node,
157 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
158 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
159 RTLIB::Libcall Call_PPCF128,
161 SDValue ExpandBitCountingLibCall(SDNode *Node, RTLIB::Libcall CallI32,
162 RTLIB::Libcall CallI64,
163 RTLIB::Libcall CallI128);
165
167
169 const SDLoc &dl);
175 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
177 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
179 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
180 SDValue NewIntValue) const;
186
188 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
190 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
193
194
195
196
197
198
200
202
207
210
211
213 void ConvertNodeToLibcall(SDNode *Node);
215
216public:
217
218
219 void ReplacedNode(SDNode *N) {
221 if (UpdatedNodes)
223 }
224
227 dbgs() << " with: "; New->dump(&DAG));
228
230 "Replacing one node with another that produces a different number "
231 "of values!");
233 if (UpdatedNodes)
234 UpdatedNodes->insert(New);
235 ReplacedNode(Old);
236 }
237
240 dbgs() << " with: "; New->dump(&DAG));
241
243 if (UpdatedNodes)
244 UpdatedNodes->insert(New.getNode());
245 ReplacedNode(Old.getNode());
246 }
247
248 void ReplaceNode(SDNode *Old, const SDValue *New) {
250
252 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
253 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
254 New[i]->dump(&DAG));
255 if (UpdatedNodes)
257 }
258 ReplacedNode(Old);
259 }
260
261 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
263 dbgs() << " with: "; New->dump(&DAG));
264
266 if (UpdatedNodes)
267 UpdatedNodes->insert(New.getNode());
268 ReplacedNode(Old.getNode());
269 }
270};
271
272}
273
274
275
278 bool isObjectScalable) {
286 ObjectSize, MFI.getObjectAlign(FI));
287}
288
289
290
291
292
293SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
298 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
299
300 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
301
302 if (NumEltsGrowth == 1)
304
305 SmallVector<int, 8> NewMask;
306 for (unsigned i = 0; i != NumMaskElts; ++i) {
307 int Idx = Mask[i];
308 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
309 if (Idx < 0)
311 else
312 NewMask.push_back(Idx * NumEltsGrowth + j);
313 }
314 }
315 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
318}
319
320
321
323SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
324 bool Extend = false;
325 SDLoc dl(CFP);
326
327
328
329
330
331
332
335 if (!UseCP) {
336 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
338 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
339 }
340
342 EVT OrigVT = VT;
343 EVT SVT = VT;
344
345
346
348 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
351
352
357 Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout()));
358 VT = SVT;
359 Extend = true;
360 }
361 }
362 }
363
367 if (Extend) {
371 Alignment);
373 }
378}
379
380
381SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
382 SDLoc dl(CP);
383 EVT VT = CP->getValueType(0);
391}
392
393SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Op) {
397 SDLoc dl(Op);
398
400
401
402
408
410
411
412
413 SmallVector<int, 8> ShufOps;
414 for (unsigned i = 0; i != NumElts; ++i)
415 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
416
418 }
419 }
420 return ExpandInsertToVectorThroughStack(Op);
421}
422
423SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
426
427 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
428
429
430
431
432
433
438 AAMDNodes AAInfo = ST->getAAInfo();
439 SDLoc dl(ST);
440
441
444
449 bitcastToAPInt().zextOrTrunc(32),
450 SDLoc(CFP), MVT::i32);
451 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
452 ST->getBaseAlign(), MMOFlags, AAInfo);
453 }
454
457
460 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
461 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
462 ST->getBaseAlign(), MMOFlags, AAInfo);
463 }
464
465 if (TLI.isTypeLegal(MVT::i32) && ->isVolatile()) {
466
467
468
474
475 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(),
476 ST->getBaseAlign(), MMOFlags, AAInfo);
479 ST->getPointerInfo().getWithOffset(4),
480 ST->getBaseAlign(), MMOFlags, AAInfo);
481
483 }
484 }
485 }
487}
488
489void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
493 SDLoc dl(Node);
494
496 AAMDNodes AAInfo = ST->getAAInfo();
497
498 if (->isTruncatingStore()) {
499 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
500 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
501 ReplaceNode(ST, OptStore);
502 return;
503 }
504
506 MVT VT = Value.getSimpleValueType();
508 default: llvm_unreachable("This action is not supported yet!");
509 case TargetLowering::Legal: {
510
511
512 EVT MemVT = ST->getMemoryVT();
515 *ST->getMemOperand())) {
516 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
518 ReplaceNode(SDValue(ST, 0), Result);
519 } else
521 break;
522 }
523 case TargetLowering::Custom: {
526 if (Res && Res != SDValue(Node, 0))
527 ReplaceNode(SDValue(Node, 0), Res);
528 return;
529 }
530 case TargetLowering::Promote: {
533 "Can only promote stores to same size type");
536 ST->getBaseAlign(), MMOFlags, AAInfo);
537 ReplaceNode(SDValue(Node, 0), Result);
538 break;
539 }
540 }
541 return;
542 }
543
544 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
546 EVT StVT = ST->getMemoryVT();
550
551 if (StWidth != StSize) {
552
553
554
559 ST->getBaseAlign(), MMOFlags, AAInfo);
560 ReplaceNode(SDValue(Node, 0), Result);
562
563 assert(!StVT.isVector() && "Unsupported truncstore!");
565 unsigned LogStWidth = Log2_32(StWidthBits);
566 assert(LogStWidth < 32);
567 unsigned RoundWidth = 1 << LogStWidth;
568 assert(RoundWidth < StWidthBits);
569 unsigned ExtraWidth = StWidthBits - RoundWidth;
570 assert(ExtraWidth < RoundWidth);
571 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
572 "Store size not an integral number of bytes!");
576 unsigned IncrementSize;
577
578 if (DL.isLittleEndian()) {
579
580
582 RoundVT, ST->getBaseAlign(), MMOFlags, AAInfo);
583
584
585 IncrementSize = RoundWidth / 8;
586 Ptr =
592 ST->getPointerInfo().getWithOffset(IncrementSize),
593 ExtraVT, ST->getBaseAlign(), MMOFlags, AAInfo);
594 } else {
595
596
597
601 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT,
602 ST->getBaseAlign(), MMOFlags, AAInfo);
603
604
605 IncrementSize = RoundWidth / 8;
610 ST->getPointerInfo().getWithOffset(IncrementSize),
611 ExtraVT, ST->getBaseAlign(), MMOFlags, AAInfo);
612 }
613
614
616 ReplaceNode(SDValue(Node, 0), Result);
617 } else {
619 default: llvm_unreachable("This action is not supported yet!");
620 case TargetLowering::Legal: {
621 EVT MemVT = ST->getMemoryVT();
622
623
625 *ST->getMemOperand())) {
627 ReplaceNode(SDValue(ST, 0), Result);
628 }
629 break;
630 }
631 case TargetLowering::Custom: {
633 if (Res && Res != SDValue(Node, 0))
634 ReplaceNode(SDValue(Node, 0), Res);
635 return;
636 }
637 case TargetLowering::Expand:
639 "Vector Stores are handled in LegalizeVectorOps");
640
642
643
647 ST->getBaseAlign(), MMOFlags, AAInfo);
648 } else {
649
650
655 StVT, ST->getBaseAlign(), MMOFlags, AAInfo);
656 }
657
658 ReplaceNode(SDValue(Node, 0), Result);
659 break;
660 }
661 }
662}
663
664void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
666 SDValue Chain = LD->getChain();
667 SDValue Ptr = LD->getBasePtr();
669 SDLoc dl(Node);
670
673 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
674 MVT VT = Node->getSimpleValueType(0);
677
679 default: llvm_unreachable("This action is not supported yet!");
680 case TargetLowering::Legal: {
681 EVT MemVT = LD->getMemoryVT();
683
684
686 *LD->getMemOperand())) {
688 }
689 break;
690 }
691 case TargetLowering::Custom:
693 RVal = Res;
695 }
696 break;
697
698 case TargetLowering::Promote: {
701 "Can only promote loads to same size type");
702
703
704
705 if (const MDNode *MD = LD->getRanges()) {
709 LD->getMemOperand()->clearRanges();
710 }
711 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
712 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
714 break;
715 }
716 }
717 if (RChain.getNode() != Node) {
718 assert(RVal.getNode() != Node && "Load must be completely replaced");
721 if (UpdatedNodes) {
722 UpdatedNodes->insert(RVal.getNode());
723 UpdatedNodes->insert(RChain.getNode());
724 }
725 ReplacedNode(Node);
726 }
727 return;
728 }
729
730 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
731 EVT SrcVT = LD->getMemoryVT();
734 AAMDNodes AAInfo = LD->getAAInfo();
735
737
738
739
740
741
742
743
744 (SrcVT != MVT::i1 ||
746 TargetLowering::Promote)) {
747
748
752
753
754
755
758
760 Chain, Ptr, LD->getPointerInfo(), NVT,
761 LD->getBaseAlign(), MMOFlags, AAInfo);
762
763 Ch = Result.getValue(1);
764
766
768 Result.getValueType(),
771
773 Result.getValueType(), Result,
775
777 Chain = Ch;
779
781 unsigned SrcWidthBits = SrcWidth.getFixedValue();
782 unsigned LogSrcWidth = Log2_32(SrcWidthBits);
783 assert(LogSrcWidth < 32);
784 unsigned RoundWidth = 1 << LogSrcWidth;
785 assert(RoundWidth < SrcWidthBits);
786 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
787 assert(ExtraWidth < RoundWidth);
788 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
789 "Load size not an integral number of bytes!");
793 unsigned IncrementSize;
795
796 if (DL.isLittleEndian()) {
797
798
800 LD->getPointerInfo(), RoundVT, LD->getBaseAlign(),
801 MMOFlags, AAInfo);
802
803
804 IncrementSize = RoundWidth / 8;
805 Ptr =
807 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
808 LD->getPointerInfo().getWithOffset(IncrementSize),
809 ExtraVT, LD->getBaseAlign(), MMOFlags, AAInfo);
810
811
812
814 Hi.getValue(1));
815
816
820
821
823 } else {
824
825
826
827 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
828 LD->getPointerInfo(), RoundVT, LD->getBaseAlign(),
829 MMOFlags, AAInfo);
830
831
832 IncrementSize = RoundWidth / 8;
833 Ptr =
836 LD->getPointerInfo().getWithOffset(IncrementSize),
837 ExtraVT, LD->getBaseAlign(), MMOFlags, AAInfo);
838
839
840
842 Hi.getValue(1));
843
844
848
849
851 }
852
853 Chain = Ch;
854 } else {
855 bool isCustom = false;
858 default: llvm_unreachable("This action is not supported yet!");
859 case TargetLowering::Custom:
860 isCustom = true;
861 [[fallthrough]];
862 case TargetLowering::Legal:
864 Chain = SDValue(Node, 1);
865
866 if (isCustom) {
870 }
871 } else {
872
873
874 EVT MemVT = LD->getMemoryVT();
877 *LD->getMemOperand())) {
879 }
880 }
881 break;
882
883 case TargetLowering::Expand: {
884 EVT DestVT = Node->getValueType(0);
886
887
890 (TLI.isTypeLegal(SrcVT) ||
892
893
896
898 SrcVT, LD->getMemOperand());
899 unsigned ExtendOp =
901 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
902 Chain = Load.getValue(1);
903 break;
904 }
905
906
907
908
909
911 if (SVT == MVT::f16 || SVT == MVT::bf16) {
915
917 Ptr, ISrcVT, LD->getMemOperand());
919 DAG.getNode(SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
920 dl, DestVT, Result);
921 Chain = Result.getValue(1);
922 break;
923 }
924 }
925
927 "Vector Loads are handled in LegalizeVectorOps");
928
929
930
931
932
934 "EXTLOAD should always be supported!");
935
936
938 Node->getValueType(0),
939 Chain, Ptr, SrcVT,
940 LD->getMemOperand());
944 Result.getValueType(),
946 else
949 Chain = Result.getValue(1);
950 break;
951 }
952 }
953 }
954
955
956
957 if (Chain.getNode() != Node) {
958 assert(Value.getNode() != Node && "Load must be completely replaced");
961 if (UpdatedNodes) {
962 UpdatedNodes->insert(Value.getNode());
963 UpdatedNodes->insert(Chain.getNode());
964 }
965 ReplacedNode(Node);
966 }
967}
968
969
970void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
972
973
976 return;
977
978#ifndef NDEBUG
979 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
981 TargetLowering::TypeLegal &&
982 "Unexpected illegal type!");
983
986 TargetLowering::TypeLegal ||
989 "Unexpected illegal type!");
990#endif
991
992
993 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
994 bool SimpleFinishLegalizing = true;
995 switch (Node->getOpcode()) {
996
997
998
999
1000
1001
1002
1003
1006 ReplaceNode(Node, UndefNode.getNode());
1007 break;
1008 }
1012 case ISD::STACKSAVE:
1014 break;
1015 case ISD::GET_DYNAMIC_AREA_OFFSET:
1017 Node->getValueType(0));
1018 break;
1019 case ISD::VAARG:
1021 Node->getValueType(0));
1022 if (Action != TargetLowering::Promote)
1024 break;
1025 case ISD::SET_FPENV:
1026 case ISD::SET_FPMODE:
1028 Node->getOperand(1).getValueType());
1029 break;
1030 case ISD::FP_TO_FP16:
1031 case ISD::FP_TO_BF16:
1035 case ISD::LROUND:
1036 case ISD::LLROUND:
1037 case ISD::LRINT:
1038 case ISD::LLRINT:
1040 Node->getOperand(0).getValueType());
1041 break;
1042 case ISD::STRICT_FP_TO_FP16:
1043 case ISD::STRICT_FP_TO_BF16:
1050
1051
1052
1054 Node->getOperand(1).getValueType());
1055 break;
1059 break;
1060 }
1061 case ISD::ATOMIC_STORE:
1063 Node->getOperand(1).getValueType());
1064 break;
1070 case ISD::VP_SETCC:
1071 case ISD::BR_CC: {
1072 unsigned Opc = Node->getOpcode();
1078 : 1;
1079 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1082 : 0;
1083 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1087 if (Action == TargetLowering::Legal) {
1090 Node->getValueType(0));
1091 else
1093 }
1094 break;
1095 }
1096 case ISD::LOAD:
1097 case ISD::STORE:
1098
1099
1100 SimpleFinishLegalizing = false;
1101 break;
1102 case ISD::CALLSEQ_START:
1103 case ISD::CALLSEQ_END:
1104
1105
1106
1107 SimpleFinishLegalizing = false;
1108 break;
1118
1119
1121 if (Action == TargetLowering::Legal)
1122 Action = TargetLowering::Expand;
1123 break;
1124 case ISD::INIT_TRAMPOLINE:
1125 case ISD::ADJUST_TRAMPOLINE:
1130
1131
1133 if (Action == TargetLowering::Legal)
1134 Action = TargetLowering::Custom;
1135 break;
1137
1138
1140 break;
1141 case ISD::READCYCLECOUNTER:
1142 case ISD::READSTEADYCOUNTER:
1143
1144
1146 break;
1149
1150
1151
1152 Action = TargetLowering::Legal;
1153 break;
1154 case ISD::UBSANTRAP:
1156 if (Action == TargetLowering::Expand) {
1157
1159 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1160 Node->getOperand(0));
1161 ReplaceNode(Node, NewVal.getNode());
1162 LegalizeOp(NewVal.getNode());
1163 return;
1164 }
1165 break;
1166 case ISD::DEBUGTRAP:
1168 if (Action == TargetLowering::Expand) {
1169
1171 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1172 Node->getOperand(0));
1173 ReplaceNode(Node, NewVal.getNode());
1174 LegalizeOp(NewVal.getNode());
1175 return;
1176 }
1177 break;
1189 break;
1198 unsigned Scale = Node->getConstantOperandVal(2);
1200 Node->getValueType(0), Scale);
1201 break;
1202 }
1203 case ISD::MSCATTER:
1206 break;
1207 case ISD::MSTORE:
1210 break;
1211 case ISD::VP_SCATTER:
1213 Node->getOpcode(),
1215 break;
1216 case ISD::VP_STORE:
1218 Node->getOpcode(),
1220 break;
1221 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1223 Node->getOpcode(),
1225 break;
1226 case ISD::VECREDUCE_FADD:
1227 case ISD::VECREDUCE_FMUL:
1228 case ISD::VECREDUCE_ADD:
1229 case ISD::VECREDUCE_MUL:
1230 case ISD::VECREDUCE_AND:
1231 case ISD::VECREDUCE_OR:
1232 case ISD::VECREDUCE_XOR:
1233 case ISD::VECREDUCE_SMAX:
1234 case ISD::VECREDUCE_SMIN:
1235 case ISD::VECREDUCE_UMAX:
1236 case ISD::VECREDUCE_UMIN:
1237 case ISD::VECREDUCE_FMAX:
1238 case ISD::VECREDUCE_FMIN:
1239 case ISD::VECREDUCE_FMAXIMUM:
1240 case ISD::VECREDUCE_FMINIMUM:
1243 Node->getOpcode(), Node->getOperand(0).getValueType());
1244 break;
1245 case ISD::VECREDUCE_SEQ_FADD:
1246 case ISD::VECREDUCE_SEQ_FMUL:
1247 case ISD::VP_REDUCE_FADD:
1248 case ISD::VP_REDUCE_FMUL:
1249 case ISD::VP_REDUCE_ADD:
1250 case ISD::VP_REDUCE_MUL:
1251 case ISD::VP_REDUCE_AND:
1252 case ISD::VP_REDUCE_OR:
1253 case ISD::VP_REDUCE_XOR:
1254 case ISD::VP_REDUCE_SMAX:
1255 case ISD::VP_REDUCE_SMIN:
1256 case ISD::VP_REDUCE_UMAX:
1257 case ISD::VP_REDUCE_UMIN:
1258 case ISD::VP_REDUCE_FMAX:
1259 case ISD::VP_REDUCE_FMIN:
1260 case ISD::VP_REDUCE_FMAXIMUM:
1261 case ISD::VP_REDUCE_FMINIMUM:
1262 case ISD::VP_REDUCE_SEQ_FADD:
1263 case ISD::VP_REDUCE_SEQ_FMUL:
1265 Node->getOpcode(), Node->getOperand(1).getValueType());
1266 break;
1267 case ISD::VP_CTTZ_ELTS:
1268 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
1270 Node->getOperand(0).getValueType());
1271 break;
1272 case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM:
1274 Node->getOpcode(),
1276 break;
1277 default:
1280 } else {
1282 }
1283 break;
1284 }
1285
1286 if (SimpleFinishLegalizing) {
1287 SDNode *NewNode = Node;
1288 switch (Node->getOpcode()) {
1289 default: break;
1295
1296
1301
1302
1303
1304
1305
1306 if (SAO != Op1)
1308 }
1309 }
1310 break;
1316
1317
1323
1324
1325
1326 if (SAO != Op2)
1328 }
1329 break;
1330 }
1331 }
1332
1333 if (NewNode != Node) {
1334 ReplaceNode(Node, NewNode);
1335 Node = NewNode;
1336 }
1337 switch (Action) {
1338 case TargetLowering::Legal:
1339 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1340 return;
1341 case TargetLowering::Custom:
1342 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1343
1344
1347 return;
1348
1349 if (Node->getNumValues() == 1) {
1350
1351
1353 Node->getValueType(0) == MVT::Glue) &&
1354 "Type mismatch for custom legalized operation");
1355 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1356
1357 ReplaceNode(SDValue(Node, 0), Res);
1358 return;
1359 }
1360
1362 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1363
1364
1366 Node->getValueType(i) == MVT::Glue) &&
1367 "Type mismatch for custom legalized operation");
1369 }
1370 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1371 ReplaceNode(Node, ResultVals.data());
1372 return;
1373 }
1374 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1375 [[fallthrough]];
1376 case TargetLowering::Expand:
1377 if (ExpandNode(Node))
1378 return;
1379 [[fallthrough]];
1380 case TargetLowering::LibCall:
1381 ConvertNodeToLibcall(Node);
1382 return;
1383 case TargetLowering::Promote:
1384 PromoteNode(Node);
1385 return;
1386 }
1387 }
1388
1389 switch (Node->getOpcode()) {
1390 default:
1391#ifndef NDEBUG
1392 dbgs() << "NODE: ";
1393 Node->dump( &DAG);
1394 dbgs() << "\n";
1395#endif
1396 llvm_unreachable("Do not know how to legalize this operator!");
1397
1398 case ISD::CALLSEQ_START:
1399 case ISD::CALLSEQ_END:
1400 break;
1401 case ISD::LOAD:
1402 return LegalizeLoadOps(Node);
1403 case ISD::STORE:
1404 return LegalizeStoreOps(Node);
1405 }
1406}
1407
1408SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1411 SDLoc dl(Op);
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421 SmallPtrSet<const SDNode *, 32> Visited;
1423 Visited.insert(Op.getNode());
1426 for (SDNode *User : Vec.getNode()->users()) {
1428 if (ST->isIndexed() || ST->isTruncatingStore() ||
1429 ST->getValue() != Vec)
1430 continue;
1431
1432
1433
1434 if (->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1435 continue;
1436
1437
1438
1439
1440
1441
1443 ST->hasPredecessor(Op.getNode()))
1444 continue;
1445
1448 break;
1449 }
1450 }
1451
1453
1455
1460 }
1461
1463 Align ElementAlignment =
1466 Op.getValueType().getTypeForEVT(*DAG.getContext())));
1467
1468 if (Op.getValueType().isVector()) {
1470 Op.getValueType(), Idx);
1471 NewLoad = DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,
1472 MachinePointerInfo(), ElementAlignment);
1473 } else {
1477 ElementAlignment);
1478 }
1479
1480
1482
1483
1484
1486 NewLoadOperands[0] = Ch;
1487 NewLoad =
1489 return NewLoad;
1490}
1491
1492SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1493 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1494
1496 SDValue Part = Op.getOperand(1);
1498 SDLoc dl(Op);
1499
1500
1505 MachinePointerInfo PtrInfo =
1507
1508
1509 Align BaseVecAlignment =
1512 BaseVecAlignment);
1513
1514
1516
1519
1520
1524
1525
1527 Ch, dl, Part, SubStackPtr,
1529 PartAlignment);
1530 } else {
1533
1534
1536 Ch, dl, Part, SubStackPtr,
1539 }
1540
1542 "ElementAlignment does not match!");
1543
1544
1545 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo,
1546 BaseVecAlignment);
1547}
1548
1549SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
1551 SDLoc DL(Node);
1553 unsigned NumOperands = Node->getNumOperands();
1555 EVT VectorValueType = Node->getOperand(0).getValueType();
1559 for (unsigned I = 0; I < NumOperands; ++I) {
1561 for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
1563 SubOp,
1565 }
1566 }
1568}
1569
1570SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1573 "Unexpected opcode!");
1574
1575
1576
1577
1578
1579 EVT VT = Node->getValueType(0);
1581 : Node->getOperand(0).getValueType();
1582 SDLoc dl(Node);
1585 MachinePointerInfo PtrInfo =
1587
1588
1590 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1591 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1592
1593
1594
1596 MemVT.bitsLT(Node->getOperand(0).getValueType());
1597
1598
1599 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1600
1601 if (Node->getOperand(i).isUndef()) continue;
1602
1603 unsigned Offset = TypeByteSize*i;
1604
1607
1608 if (Truncate)
1610 Node->getOperand(i), Idx,
1612 else
1615 }
1616
1618 if (!Stores.empty())
1620 else
1622
1623
1624 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1625}
1626
1627
1628
1629
1630void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1631 const SDLoc &DL,
1633 EVT FloatVT = Value.getValueType();
1635 State.FloatVT = FloatVT;
1637
1639 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1641 State.SignBit = NumBits - 1;
1642 return;
1643 }
1644
1646
1648
1651
1656 State.FloatPointerInfo);
1657
1659 if (DataLayout.isBigEndian()) {
1660 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1661
1663 State.IntPointerInfo = State.FloatPointerInfo;
1664 } else {
1665
1666 unsigned ByteOffset = (NumBits / 8) - 1;
1667 IntPtr =
1670 ByteOffset);
1671 }
1672
1673 State.IntPtr = IntPtr;
1675 State.IntPointerInfo, MVT::i8);
1677 State.SignBit = 7;
1678}
1679
1680
1681
1682SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1683 const SDLoc &DL,
1684 SDValue NewIntValue) const {
1685 if (!State.Chain)
1686 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1687
1688
1690 State.IntPointerInfo, MVT::i8);
1691 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1692 State.FloatPointerInfo);
1693}
1694
1695SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1696 SDLoc DL(Node);
1699
1700
1701 FloatSignAsInt SignAsInt;
1702 getSignAsIntValue(SignAsInt, DL, Sign);
1703
1704 EVT IntVT = SignAsInt.IntValue.getValueType();
1707 SignMask);
1708
1709
1710
1714 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1715 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1718 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1719 }
1720
1721
1722 FloatSignAsInt MagAsInt;
1723 getSignAsIntValue(MagAsInt, DL, Mag);
1724 EVT MagVT = MagAsInt.IntValue.getValueType();
1727 ClearSignMask);
1728
1729
1730 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1731 EVT ShiftVT = IntVT;
1735 ShiftVT = MagVT;
1736 }
1737 if (ShiftAmount > 0) {
1739 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1740 } else if (ShiftAmount < 0) {
1742 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1743 }
1747 }
1748
1749
1752
1753 return modifySignAsInt(MagAsInt, DL, CopiedSign);
1754}
1755
1756SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1757
1758 SDLoc DL(Node);
1759 FloatSignAsInt SignAsInt;
1760 getSignAsIntValue(SignAsInt, DL, Node->getOperand(0));
1761 EVT IntVT = SignAsInt.IntValue.getValueType();
1762
1763
1767
1768
1769 return modifySignAsInt(SignAsInt, DL, SignFlip);
1770}
1771
1772SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1773 SDLoc DL(Node);
1775
1776
1777 EVT FloatVT = Value.getValueType();
1781 }
1782
1783
1784 FloatSignAsInt ValueAsInt;
1785 getSignAsIntValue(ValueAsInt, DL, Value);
1786 EVT IntVT = ValueAsInt.IntValue.getValueType();
1789 ClearSignMask);
1790 return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1791}
1792
1793void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1794 SmallVectorImpl &Results) {
1796 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1797 " not tell us which reg is the stack pointer!");
1798 SDLoc dl(Node);
1799 EVT VT = Node->getValueType(0);
1804
1805
1806
1808
1811 Chain = SP.getValue(1);
1814 unsigned Opc =
1817
1820 if (Alignment > StackAlign)
1823 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);
1824
1826
1827 Results.push_back(Tmp1);
1828 Results.push_back(Tmp2);
1829}
1830
1831
1832
1833
1834
1835SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1836 EVT DestVT, const SDLoc &dl) {
1837 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1838}
1839
1840SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1841 EVT DestVT, const SDLoc &dl,
1846
1847
1848 if ((SrcVT.bitsGT(SlotVT) &&
1850 (SlotVT.bitsLT(DestVT) &&
1853
1854
1858
1860 int SPFI = StackPtrFI->getIndex();
1861 MachinePointerInfo PtrInfo =
1863
1864
1865
1867
1868 if (SrcVT.bitsGT(SlotVT))
1870 SlotVT, SrcAlign);
1871 else {
1872 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1873 Store = DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1874 }
1875
1876
1877 if (SlotVT.bitsEq(DestVT))
1878 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1879
1880 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1882 DestAlign);
1883}
1884
1885SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1886 SDLoc dl(Node);
1887
1888
1890
1892 int SPFI = StackPtrFI->getIndex();
1893
1897 Node->getValueType(0).getVectorElementType());
1899 Node->getValueType(0), dl, Ch, StackPtr,
1901}
1902
1903static bool
1906 unsigned NumElems = Node->getNumOperands();
1908 EVT VT = Node->getValueType(0);
1909
1910
1911
1912
1913
1914
1915
1916
1919 NewIntermedVals;
1920 for (unsigned i = 0; i < NumElems; ++i) {
1922 if (V.isUndef())
1923 continue;
1924
1929 }
1930
1931 while (IntermedVals.size() > 2) {
1932 NewIntermedVals.clear();
1933 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1934
1935
1937
1939 FinalIndices.reserve(IntermedVals[i].second.size() +
1940 IntermedVals[i+1].second.size());
1941
1942 int k = 0;
1943 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1944 ++j, ++k) {
1945 ShuffleVec[k] = j;
1946 FinalIndices.push_back(IntermedVals[i].second[j]);
1947 }
1948 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1949 ++j, ++k) {
1950 ShuffleVec[k] = NumElems + j;
1951 FinalIndices.push_back(IntermedVals[i+1].second[j]);
1952 }
1953
1956 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1957 IntermedVals[i+1].first,
1958 ShuffleVec);
1960 return false;
1962 std::make_pair(Shuffle, std::move(FinalIndices)));
1963 }
1964
1965
1966
1967 if ((IntermedVals.size() & 1) != 0)
1969
1970 IntermedVals.swap(NewIntermedVals);
1971 }
1972
1973 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1974 "Invalid number of intermediate vectors");
1975 SDValue Vec1 = IntermedVals[0].first;
1977 if (IntermedVals.size() > 1)
1978 Vec2 = IntermedVals[1].first;
1981
1983 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1984 ShuffleVec[IntermedVals[0].second[i]] = i;
1985 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1986 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1987
1991 return false;
1992 }
1993
1994 return true;
1995}
1996
1997
1998
1999SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
2000 unsigned NumElems = Node->getNumOperands();
2002 SDLoc dl(Node);
2003 EVT VT = Node->getValueType(0);
2004 EVT OpVT = Node->getOperand(0).getValueType();
2006
2007
2008
2009 bool isOnlyLowElement = true;
2010 bool MoreThanTwoValues = false;
2012 for (unsigned i = 0; i < NumElems; ++i) {
2014 if (V.isUndef())
2015 continue;
2016 if (i > 0)
2017 isOnlyLowElement = false;
2020
2021 if (!Value1.getNode()) {
2022 Value1 = V;
2023 } else if (!Value2.getNode()) {
2024 if (V != Value1)
2025 Value2 = V;
2026 } else if (V != Value1 && V != Value2) {
2027 MoreThanTwoValues = true;
2028 }
2029 }
2030
2033
2034 if (isOnlyLowElement)
2036
2037
2040 for (unsigned i = 0, e = NumElems; i != e; ++i) {
2041 if (ConstantFPSDNode *V =
2044 } else if (ConstantSDNode *V =
2046 if (OpVT==EltVT)
2047 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
2048 else {
2049
2050
2051
2052 const ConstantInt *CI = V->getConstantIntValue();
2055 }
2056 } else {
2057 assert(Node->getOperand(i).isUndef());
2060 }
2061 }
2069 Alignment);
2070 }
2071
2072 SmallSet<SDValue, 16> DefinedValues;
2073 for (unsigned i = 0; i < NumElems; ++i) {
2074 if (Node->getOperand(i).isUndef())
2075 continue;
2076 DefinedValues.insert(Node->getOperand(i));
2077 }
2078
2080 if (!MoreThanTwoValues) {
2081 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2082 for (unsigned i = 0; i < NumElems; ++i) {
2084 if (V.isUndef())
2085 continue;
2086 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2087 }
2089
2094 else
2096
2097
2098 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2099 }
2100 } else {
2103 return Res;
2104 }
2105 }
2106
2107
2108 return ExpandVectorBuildThroughStack(Node);
2109}
2110
2111SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2112 SDLoc DL(Node);
2113 EVT VT = Node->getValueType(0);
2115
2117}
2118
2119
2120
2121
2122
2123
2124std::pair<SDValue, SDValue>
2125SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2126 TargetLowering::ArgListTy &&Args,
2127 bool IsSigned, EVT RetVT) {
2131 if (LCImpl != RTLIB::Unsupported)
2133 else {
2136 Node->getOperationName(&DAG));
2137 }
2138
2140
2141
2142
2143
2144
2146
2147
2148
2149 SDValue TCChain = InChain;
2151 bool isTailCall =
2153 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2154 if (isTailCall)
2155 InChain = TCChain;
2156
2157 TargetLowering::CallLoweringInfo CLI(DAG);
2159 CLI.setDebugLoc(SDLoc(Node))
2160 .setChain(InChain)
2162 std::move(Args))
2163 .setTailCall(isTailCall)
2164 .setSExtResult(signExtend)
2165 .setZExtResult(!signExtend)
2166 .setIsPostTypeLegalization(true);
2167
2168 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2169
2170 if (!CallInfo.second.getNode()) {
2172
2174 }
2175
2176 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2177 return CallInfo;
2178}
2179
2180std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2182 TargetLowering::ArgListTy Args;
2184 EVT ArgVT = Op.getValueType();
2186 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2189 Args.push_back(Entry);
2190 }
2191
2192 return ExpandLibCall(LC, Node, std::move(Args), isSigned,
2193 Node->getValueType(0));
2194}
2195
2196void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2197 RTLIB::Libcall LC,
2198 SmallVectorImpl &Results) {
2199 if (LC == RTLIB::UNKNOWN_LIBCALL)
2201
2202 if (Node->isStrictFPOpcode()) {
2203 EVT RetVT = Node->getValueType(0);
2205 TargetLowering::MakeLibCallOptions CallOptions;
2207
2208 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2209 Ops, CallOptions,
2210 SDLoc(Node),
2211 Node->getOperand(0));
2212 Results.push_back(Tmp.first);
2213 Results.push_back(Tmp.second);
2214 } else {
2215 bool IsSignedArgument = Node->getOpcode() == ISD::FLDEXP;
2216 SDValue Tmp = ExpandLibCall(LC, Node, IsSignedArgument).first;
2218 }
2219}
2220
2221
2222void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2223 RTLIB::Libcall Call_F32,
2224 RTLIB::Libcall Call_F64,
2225 RTLIB::Libcall Call_F80,
2226 RTLIB::Libcall Call_F128,
2227 RTLIB::Libcall Call_PPCF128,
2228 SmallVectorImpl &Results) {
2230 Call_F32, Call_F64, Call_F80,
2231 Call_F128, Call_PPCF128);
2232 ExpandFPLibCall(Node, LC, Results);
2233}
2234
2235void SelectionDAGLegalize::ExpandFastFPLibCall(
2236 SDNode *Node, bool IsFast,
2237 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F32,
2238 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F64,
2239 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F80,
2240 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_F128,
2241 std::pair<RTLIB::Libcall, RTLIB::Libcall> Call_PPCF128,
2242 SmallVectorImpl &Results) {
2243
2244 EVT VT = Node->getSimpleValueType(0);
2245
2246 RTLIB::Libcall LC;
2247
2248
2249
2250
2251 if (IsFast) {
2252 LC = RTLIB::getFPLibCall(VT, Call_F32.first, Call_F64.first, Call_F80.first,
2253 Call_F128.first, Call_PPCF128.first);
2254 }
2255
2256 if (!IsFast || TLI.getLibcallImpl(LC) == RTLIB::Unsupported) {
2257
2259 Call_F80.second, Call_F128.second,
2260 Call_PPCF128.second);
2261 }
2262
2263 ExpandFPLibCall(Node, LC, Results);
2264}
2265
2266SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2267 RTLIB::Libcall Call_I8,
2268 RTLIB::Libcall Call_I16,
2269 RTLIB::Libcall Call_I32,
2270 RTLIB::Libcall Call_I64,
2271 RTLIB::Libcall Call_I128) {
2272 RTLIB::Libcall LC;
2273 switch (Node->getSimpleValueType(0).SimpleTy) {
2274 default: llvm_unreachable("Unexpected request for libcall!");
2275 case MVT::i8: LC = Call_I8; break;
2276 case MVT::i16: LC = Call_I16; break;
2277 case MVT::i32: LC = Call_I32; break;
2278 case MVT::i64: LC = Call_I64; break;
2279 case MVT::i128: LC = Call_I128; break;
2280 }
2281 return ExpandLibCall(LC, Node, isSigned).first;
2282}
2283
2284
2285
2286void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2287 RTLIB::Libcall Call_F32,
2288 RTLIB::Libcall Call_F64,
2289 RTLIB::Libcall Call_F80,
2290 RTLIB::Libcall Call_F128,
2291 RTLIB::Libcall Call_PPCF128,
2292 SmallVectorImpl &Results) {
2293 EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2295 Call_F32, Call_F64, Call_F80,
2296 Call_F128, Call_PPCF128);
2297 ExpandFPLibCall(Node, LC, Results);
2298}
2299
2300SDValue SelectionDAGLegalize::ExpandBitCountingLibCall(
2301 SDNode *Node, RTLIB::Libcall CallI32, RTLIB::Libcall CallI64,
2302 RTLIB::Libcall CallI128) {
2303 RTLIB::Libcall LC;
2304 switch (Node->getSimpleValueType(0).SimpleTy) {
2305 default:
2307 case MVT::i32:
2308 LC = CallI32;
2309 break;
2310 case MVT::i64:
2311 LC = CallI64;
2312 break;
2313 case MVT::i128:
2314 LC = CallI128;
2315 break;
2316 }
2317
2318
2319
2320
2322 EVT IntVT =
2324
2325 EVT ArgVT = Op.getValueType();
2327 TargetLowering::ArgListEntry Arg(Op, ArgTy);
2329 Arg.IsZExt = !Arg.IsSExt;
2330
2331 SDValue Res = ExpandLibCall(LC, Node, TargetLowering::ArgListTy{Arg},
2332 true, IntVT)
2333 .first;
2334
2335
2336
2339 return Res;
2340}
2341
2342
2343void
2344SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2345 SmallVectorImpl &Results) {
2346 unsigned Opcode = Node->getOpcode();
2348
2349 RTLIB::Libcall LC;
2350 switch (Node->getSimpleValueType(0).SimpleTy) {
2351 default: llvm_unreachable("Unexpected request for libcall!");
2352 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2353 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2354 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2355 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2356 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2357 }
2358
2359
2360
2361
2363
2364 EVT RetVT = Node->getValueType(0);
2366
2367 TargetLowering::ArgListTy Args;
2369 EVT ArgVT = Op.getValueType();
2371 TargetLowering::ArgListEntry Entry(Op, ArgTy);
2374 Args.push_back(Entry);
2375 }
2376
2377
2379 TargetLowering::ArgListEntry Entry(
2380 FIPtr, PointerType::getUnqual(RetTy->getContext()));
2383 Args.push_back(Entry);
2384
2385 RTLIB::LibcallImpl LibcallImpl = TLI.getLibcallImpl(LC);
2386 if (LibcallImpl == RTLIB::Unsupported) {
2388 Node->getOperationName(&DAG));
2392 return;
2393 }
2394
2397
2398 SDLoc dl(Node);
2399 TargetLowering::CallLoweringInfo CLI(DAG);
2400 CLI.setDebugLoc(dl)
2401 .setChain(InChain)
2403 std::move(Args))
2406
2407 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2408
2409
2411 MachinePointerInfo PtrInfo =
2413
2414 SDValue Rem = DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, PtrInfo);
2415 Results.push_back(CallInfo.first);
2417}
2418
2419
2425
2426
2428 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2429 ? ISD::FCOS : ISD::FSIN;
2430
2434 continue;
2435
2436 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2437 return true;
2438 }
2439 return false;
2440}
2441
2442SDValue SelectionDAGLegalize::ExpandSincosStretLibCall(SDNode *Node) const {
2443
2444
2445 SDLoc dl(Node);
2449 RTLIB::LibcallImpl SincosStret = TLI.getLibcallImpl(LC);
2450 if (SincosStret == RTLIB::Unsupported)
2452
2453
2454
2455
2456
2457
2459
2461
2462 auto [FuncTy, FuncAttrs] = CallsInfo.getFunctionTy(
2464
2465 Type *SincosStretRetTy = FuncTy->getReturnType();
2467
2470
2471 TargetLowering::ArgListTy Args;
2473
2474 int FrameIdx;
2475 if (FuncTy->getParamType(0)->isPointerTy()) {
2476
2478
2479 AttributeSet PtrAttrs = FuncAttrs.getParamAttrs(0);
2481 const uint64_t ByteSize = DL.getTypeAllocSize(StructTy);
2482 const Align StackAlign = DL.getPrefTypeAlign(StructTy);
2483
2486
2487 TargetLowering::ArgListEntry Entry(SRet, FuncTy->getParamType(0));
2488 Entry.IsSRet = true;
2489 Entry.IndirectType = StructTy;
2490 Entry.Alignment = StackAlign;
2491
2492 Args.push_back(Entry);
2493 Args.emplace_back(Arg, FuncTy->getParamType(1));
2494 } else {
2495 Args.emplace_back(Arg, FuncTy->getParamType(0));
2496 }
2497
2498 TargetLowering::CallLoweringInfo CLI(DAG);
2499 CLI.setDebugLoc(dl)
2501 .setLibCallee(CallConv, SincosStretRetTy, Callee, std::move(Args))
2502 .setIsPostTypeLegalization();
2503
2504 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
2505
2506 if (SRet) {
2507 MachinePointerInfo PtrInfo =
2509 SDValue LoadSin = DAG.getLoad(ArgVT, dl, CallResult.second, SRet, PtrInfo);
2510
2512
2513
2517
2518 SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);
2521 }
2522
2523 if (!CallResult.first.getValueType().isVector())
2524 return CallResult.first;
2525
2532 SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);
2534}
2535
2536SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2537 SDLoc dl(Node);
2538 EVT VT = Node->getValueType(0);
2541 EVT ExpVT = N.getValueType();
2543 if (AsIntVT == EVT())
2545
2548
2549 SDNodeFlags NSW;
2551 SDNodeFlags NUW_NSW;
2554
2555 EVT SetCCVT =
2558
2559 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2560 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2561 const int Precision = APFloat::semanticsPrecision(FltSem);
2562
2565
2567
2568 const APFloat One(FltSem, "1.0");
2569 APFloat ScaleUpK = scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);
2570
2571
2573 scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);
2574
2575
2576
2577
2578
2580
2585 DAG.getNode(ISD::SUB, dl, ExpVT, ClampN_Big, DoubleMaxExp, NSW);
2586
2589
2593
2598
2599
2601
2602 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT);
2603 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT);
2604
2606
2608 DAG.getSignedConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT);
2611 DAG.getNode(ISD::ADD, dl, ExpVT, ClampN_Small, Increment1, NSW);
2612
2616
2618 dl, SetCCVT, N,
2620
2624 DAG.getNode(ISD::SELECT, dl, VT, ScaleDownTwice, ScaleDown1, ScaleDown0);
2625
2626
2627
2629 ISD::SELECT, dl, VT, NGtMaxExp, SelectX_Big,
2631
2633 ISD::SELECT, dl, ExpVT, NGtMaxExp, SelectN_Big,
2635
2637
2638 SDValue ExponentShiftAmt =
2641
2643 ExponentShiftAmt, NUW_NSW);
2644 SDValue AsFP = DAG.getNode(ISD::BITCAST, dl, VT, AsInt);
2646}
2647
2648SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2649 SDLoc dl(Node);
2652 EVT ExpVT = Node->getValueType(1);
2654 if (AsIntVT == EVT())
2656
2658 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2659 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2678 AsIntVT);
2679
2682 AsIntVT);
2683
2684
2687
2688
2689
2690
2691 APInt FractSignMaskVal = APInt::getBitsSet(BitSize, 0, Precision - 1);
2692 FractSignMaskVal.setBit(BitSize - 1);
2693
2696
2697 SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT);
2698
2699 const APFloat One(FltSem, "1.0");
2700
2701
2703 scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);
2704
2707
2708 EVT SetCCVT =
2710
2711 SDValue AsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, Val);
2712
2714
2715 SDValue AddNegSmallestNormal =
2716 DAG.getNode(ISD::ADD, dl, AsIntVT, Abs, NegSmallestNormalizedInt);
2717 SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal,
2719
2722
2725
2726 SDValue ScaledAsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, ScaleUp);
2728 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ScaledAsInt, AsInt);
2729
2731 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledAsInt, ExpMask);
2732
2734 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ExpMaskScaled, Abs);
2735
2736
2737 SDValue ExponentShiftAmt =
2740 DAG.getNode(ISD::SRL, dl, AsIntVT, ScaledValue, ExponentShiftAmt);
2742
2744 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT);
2745 SDValue DenormalExpBias =
2746 DAG.getNode(ISD::SELECT, dl, ExpVT, IsDenormal, DenormalOffset, Zero);
2747
2748 SDValue MaskedFractAsInt =
2749 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledSelect, FractSignMask);
2750 const APFloat Half(FltSem, "0.5");
2751 SDValue FPHalf = DAG.getConstant(Half.bitcastToAPInt(), dl, AsIntVT);
2753 SDValue MaskedFract = DAG.getNode(ISD::BITCAST, dl, VT, Or);
2754
2756 DAG.getNode(ISD::ADD, dl, ExpVT, NormalBiasedExp, DenormalExpBias);
2757
2760
2762 DAG.getNode(ISD::SELECT, dl, ExpVT, DenormOrZero, Zero, ComputedExp);
2763
2765}
2766
2767
2768
2769
2770
2771SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2775 EVT DestVT = Node->getValueType(0);
2776 SDLoc dl(Node);
2777 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2780
2781
2783 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2784 (DestVT.bitsLE(MVT::f64) ||
2786 : ISD::FP_EXTEND,
2787 DestVT))) {
2788 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2789 "expansion\n");
2790
2791
2793
2795
2797
2799 DAG.getConstant(0x80000000u, dl, MVT::i32));
2800 }
2801
2803
2804
2807
2809
2810
2812 MachinePointerInfo());
2813
2817 DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo());
2819
2820
2822 DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo());
2823
2827 dl, MVT::f64);
2828
2831 if (Node->isStrictFPOpcode()) {
2833 {Node->getOperand(0), Load, Bias});
2835 if (DestVT != Sub.getValueType()) {
2836 std::pair<SDValue, SDValue> ResultPair;
2837 ResultPair =
2839 Result = ResultPair.first;
2840 Chain = ResultPair.second;
2841 }
2842 else
2844 } else {
2847 }
2849 }
2850
2853
2854
2855 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2856 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2857 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872 EVT SetCCVT = getSetCCResultType(SrcVT);
2873
2876
2882
2884 if (Node->isStrictFPOpcode()) {
2885
2886
2888
2889
2890
2891 SDNodeFlags Flags;
2892 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2894 {Node->getOperand(0), InCvt}, Flags);
2895 Flags.setNoFPExcept(true);
2899 } else {
2901 Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt);
2903 }
2904
2905 return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast);
2906 }
2907
2908
2912
2913
2914
2915
2918 "Cannot perform lossless SINT_TO_FP!");
2919
2921 if (Node->isStrictFPOpcode()) {
2923 { Node->getOperand(0), Op0 });
2924 } else
2926
2927 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2932 SignSet, Four, Zero);
2933
2934
2935
2936
2937 uint64_t FF;
2939 default:
2941 case MVT::i8 : FF = 0x43800000ULL; break;
2942 case MVT::i16: FF = 0x47800000ULL; break;
2943 case MVT::i32: FF = 0x4F800000ULL; break;
2944 case MVT::i64: FF = 0x5F800000ULL; break;
2945 }
2947 FF <<= 32;
2948 Constant *FudgeFactor = ConstantInt::get(
2949 Type::getInt64Ty(*DAG.getContext()), FF);
2950
2957 if (DestVT == MVT::f32)
2958 FudgeInReg = DAG.getLoad(
2961 Alignment);
2962 else {
2966 Alignment);
2967 HandleSDNode Handle(Load);
2968 LegalizeOp(Load.getNode());
2969 FudgeInReg = Handle.getValue();
2970 }
2971
2972 if (Node->isStrictFPOpcode()) {
2974 { Tmp1.getValue(1), Tmp1, FudgeInReg });
2975 Chain = Result.getValue(1);
2977 }
2978
2979 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2980}
2981
2982
2983
2984
2985
2986
2987void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2988 SDNode *N, const SDLoc &dl, SmallVectorImpl &Results) {
2992 EVT DestVT = N->getValueType(0);
2993 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2996
2997
2999
3000 unsigned OpToUse = 0;
3001
3002
3003 while (true) {
3005 assert(NewInTy.isInteger() && "Ran out of possibilities!");
3006
3007
3009 OpToUse = SIntOp;
3010 break;
3011 }
3012 if (IsSigned)
3013 continue;
3014
3015
3017 OpToUse = UIntOp;
3018 break;
3019 }
3020
3021
3022 }
3023
3024
3025
3026 if (IsStrict) {
3028 DAG.getNode(OpToUse, dl, {DestVT, MVT::Other},
3029 {N->getOperand(0),
3031 dl, NewInTy, LegalOp)});
3034 return;
3035 }
3036
3038 DAG.getNode(OpToUse, dl, DestVT,
3040 dl, NewInTy, LegalOp)));
3041}
3042
3043
3044
3045
3046
3047
3048void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
3049 SmallVectorImpl &Results) {
3050 bool IsStrict = N->isStrictFPOpcode();
3053 EVT DestVT = N->getValueType(0);
3055
3056 EVT NewOutTy = DestVT;
3057
3058 unsigned OpToUse = 0;
3059
3060
3061 while (true) {
3063 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3064
3065
3066
3069 break;
3070
3071
3074 break;
3075
3076
3077 }
3078
3079
3081 if (IsStrict) {
3082 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
3083 Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp);
3084 } else
3086
3087
3088
3090 Results.push_back(Trunc);
3091 if (IsStrict)
3093}
3094
3095
3096
3097
3098SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
3099 const SDLoc &dl) {
3100 unsigned Opcode = Node->getOpcode();
3101
3102
3103 EVT NewOutTy = Node->getValueType(0);
3104 while (true) {
3106 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
3107
3109 break;
3110 }
3111
3112
3113
3115 Node->getOperand(1));
3117}
3118
3119
3120SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
3121 EVT VT = Op.getValueType();
3124
3125
3129 } else {
3131 for (unsigned i = Log2_32_Ceil(Sz); i != 0;) {
3133 DAG.getConstant(1ULL << (--i), dl, ShVT));
3135 }
3136 }
3137
3139}
3140
3141SDValue SelectionDAGLegalize::PromoteReduction(SDNode *Node) {
3143 MVT VecVT = IsVPOpcode ? Node->getOperand(1).getSimpleValueType()
3144 : Node->getOperand(0).getSimpleValueType();
3146 MVT ScalarVT = Node->getSimpleValueType(0);
3148
3149 SDLoc DL(Node);
3151
3152
3153 assert(Node->getOperand(0).getValueType().isFloatingPoint() &&
3154 "Only FP promotion is supported");
3155
3156 for (unsigned j = 0; j != Node->getNumOperands(); ++j)
3157 if (Node->getOperand(j).getValueType().isVector() &&
3158 !(IsVPOpcode &&
3160
3161
3162 assert(Node->getOperand(j).getValueType().isFloatingPoint() &&
3163 "Only FP promotion is supported");
3164 Operands[j] =
3165 DAG.getNode(ISD::FP_EXTEND, DL, NewVecVT, Node->getOperand(j));
3166 } else if (Node->getOperand(j).getValueType().isFloatingPoint()) {
3167
3168 Operands[j] =
3169 DAG.getNode(ISD::FP_EXTEND, DL, NewScalarVT, Node->getOperand(j));
3170 } else {
3171 Operands[j] = Node->getOperand(j);
3172 }
3173
3175 Node->getFlags());
3176
3180}
3181
3182bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
3185 SDLoc dl(Node);
3186 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3187 bool NeedInvert;
3188 switch (Node->getOpcode()) {
3190 if ((Tmp1 = TLI.expandABS(Node, DAG)))
3191 Results.push_back(Tmp1);
3192 break;
3195 if ((Tmp1 = TLI.expandABD(Node, DAG)))
3196 Results.push_back(Tmp1);
3197 break;
3202 if ((Tmp1 = TLI.expandAVG(Node, DAG)))
3203 Results.push_back(Tmp1);
3204 break;
3206 if ((Tmp1 = TLI.expandCTPOP(Node, DAG)))
3207 Results.push_back(Tmp1);
3208 break;
3211 if ((Tmp1 = TLI.expandCTLZ(Node, DAG)))
3212 Results.push_back(Tmp1);
3213 break;
3216 if ((Tmp1 = TLI.expandCTTZ(Node, DAG)))
3217 Results.push_back(Tmp1);
3218 break;
3221 Results.push_back(Tmp1);
3222 break;
3224 if ((Tmp1 = TLI.expandBSWAP(Node, DAG)))
3225 Results.push_back(Tmp1);
3226 break;
3228 Results.push_back(ExpandPARITY(Node->getOperand(0), dl));
3229 break;
3234 break;
3242 CfaArg);
3248 break;
3249 }
3253 break;
3255 case ISD::EH_LABEL:
3256 case ISD::PREFETCH:
3257 case ISD::VAEND:
3259
3260
3262 break;
3263 case ISD::READCYCLECOUNTER:
3264 case ISD::READSTEADYCOUNTER:
3265
3266
3267 Results.append(Node->getNumValues() - 1,
3270 break;
3272
3273
3276 break;
3277 case ISD::ATOMIC_LOAD: {
3278
3280 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3282 ISD::ATOMIC_CMP_SWAP, dl, cast(Node)->getMemoryVT(), VTs,
3283 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
3287 break;
3288 }
3289 case ISD::ATOMIC_STORE: {
3290
3293 Node->getOperand(0), Node->getOperand(2), Node->getOperand(1),
3296 break;
3297 }
3298 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
3299
3300
3301
3302 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3304 ISD::ATOMIC_CMP_SWAP, dl, cast(Node)->getMemoryVT(), VTs,
3305 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
3307
3311
3313 EVT OuterType = Node->getValueType(0);
3320 ExtRes = LHS;
3321 break;
3326 ExtRes = LHS;
3327 break;
3331 break;
3332 default:
3334 }
3335
3338
3342 break;
3343 }
3344 case ISD::ATOMIC_LOAD_SUB: {
3345 SDLoc DL(Node);
3346 EVT VT = Node->getValueType(0);
3351 RHS = RHS->getOperand(0);
3355 Node->getOperand(0), Node->getOperand(1),
3359 break;
3360 }
3361 case ISD::DYNAMIC_STACKALLOC:
3362 ExpandDYNAMIC_STACKALLOC(Node, Results);
3363 break;
3365 for (unsigned i = 0; i < Node->getNumValues(); i++)
3367 break;
3370 EVT VT = Node->getValueType(0);
3373 else {
3376 }
3377 break;
3378 }
3380
3381
3383 break;
3384
3385
3387 Node->getValueType(0))
3388 == TargetLowering::Legal)
3389 break;
3390
3391
3392 if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0),
3393 Node->getValueType(0), dl,
3394 Node->getOperand(0)))) {
3395 ReplaceNode(Node, Tmp1.getNode());
3396 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3397 return true;
3398 }
3399 break;
3402 Results.push_back(Tmp1);
3403 break;
3404 }
3405
3406 [[fallthrough]];
3407 }
3408 case ISD::BITCAST:
3409 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
3410 Node->getValueType(0), dl)))
3411 Results.push_back(Tmp1);
3412 break;
3414
3415
3417 break;
3418
3419
3421 Node->getValueType(0))
3422 == TargetLowering::Legal)
3423 break;
3424
3425
3426 if ((Tmp1 = EmitStackConvert(
3427 Node->getOperand(1), Node->getOperand(1).getValueType(),
3428 Node->getValueType(0), dl, Node->getOperand(0)))) {
3429 ReplaceNode(Node, Tmp1.getNode());
3430 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3431 return true;
3432 }
3433 break;
3434 case ISD::FP_EXTEND: {
3436 EVT SrcVT = Op.getValueType();
3437 EVT DstVT = Node->getValueType(0);
3439 Results.push_back(DAG.getNode(ISD::BF16_TO_FP, SDLoc(Node), DstVT, Op));
3440 break;
3441 }
3442
3443 if ((Tmp1 = EmitStackConvert(Op, SrcVT, DstVT, dl)))
3444 Results.push_back(Tmp1);
3445 break;
3446 }
3447 case ISD::BF16_TO_FP: {
3448
3449
3450
3451
3453 if (Op.getValueType() == MVT::bf16) {
3455 DAG.getNode(ISD::BITCAST, dl, MVT::i16, Op));
3456 } else {
3458 }
3461 Op = DAG.getNode(ISD::BITCAST, dl, MVT::f32, Op);
3462
3463 if (Node->getValueType(0) != MVT::f32)
3464 Op = DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Op);
3466 break;
3467 }
3468 case ISD::FP_TO_BF16: {
3470 if (Op.getValueType() != MVT::f32)
3473
3476 }
3478 DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op),
3480
3481
3482 if (Node->getValueType(0) == MVT::bf16) {
3483 Op = DAG.getNode(ISD::BITCAST, dl, MVT::bf16,
3485 } else {
3487 }
3489 break;
3490 }
3492
3493
3494
3495
3496
3497
3498
3499
3500
3505
3506
3507 SDNodeFlags CanonicalizeFlags = Node->getFlags();
3510 {Chain, Operand, One}, CanonicalizeFlags);
3511
3513 break;
3514 }
3517 EVT VT = Node->getValueType(0);
3518
3519
3520
3521
3522
3523
3524
3525
3532 break;
3533 }
3534
3535
3536
3542 Results.push_back(Tmp1);
3543 break;
3544 }
3548 Results.push_back(Tmp1);
3549 if (Node->isStrictFPOpcode())
3550 Results.push_back(Tmp2);
3551 break;
3552 }
3553 [[fallthrough]];
3556 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) {
3557 Results.push_back(Tmp1);
3558 if (Node->isStrictFPOpcode())
3559 Results.push_back(Tmp2);
3560 }
3561 break;
3564 Results.push_back(Tmp1);
3565 break;
3568 ReplaceNode(Node, Tmp1.getNode());
3569 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3570 return true;
3571 }
3572 break;
3575 Results.push_back(Tmp1);
3576 break;
3579
3581
3582 ReplaceNodeWithValue(SDValue(Node, 0), Tmp1);
3583 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3584 return true;
3585 }
3586 break;
3590 break;
3591 case ISD::LROUND:
3592 case ISD::LLROUND: {
3595 EVT ResVT = Node->getValueType(0);
3596 SDLoc dl(Node);
3597 SDValue RoundNode = DAG.getNode(ISD::FROUND, dl, ArgVT, Arg);
3599 break;
3600 }
3601 case ISD::VAARG:
3604 break;
3605 case ISD::VACOPY:
3607 break;
3609 if (Node->getOperand(0).getValueType().getVectorElementCount().isScalar())
3610
3611 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3612 Node->getOperand(0));
3613 else
3614 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3615 Results.push_back(Tmp1);
3616 break;
3618 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3619 break;
3621 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3622 break;
3624 if (EVT VectorValueType = Node->getOperand(0).getValueType();
3627 Results.push_back(ExpandVectorBuildThroughStack(Node));
3628 else
3629 Results.push_back(ExpandConcatVectors(Node));
3630 break;
3632 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3633 break;
3635 Results.push_back(ExpandINSERT_VECTOR_ELT(SDValue(Node, 0)));
3636 break;
3640
3641 EVT VT = Node->getValueType(0);
3647
3648
3649
3650
3651 if (NewEltVT.bitsLT(EltVT)) {
3652
3653
3654
3655
3656
3657 EVT NewVT =
3661
3662
3663 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3664 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3665
3666
3667 unsigned int factor =
3669
3670
3672
3674 if (Mask[i] < 0) {
3675 for (unsigned fi = 0; fi < factor; ++fi)
3677 }
3678 else {
3679 for (unsigned fi = 0; fi < factor; ++fi)
3680 NewMask.push_back(Mask[i]*factor+fi);
3681 }
3682 }
3683 Mask = NewMask;
3684 VT = NewVT;
3685 }
3686 EltVT = NewEltVT;
3687 }
3690 for (unsigned i = 0; i != NumElems; ++i) {
3691 if (Mask[i] < 0) {
3693 continue;
3694 }
3695 unsigned Idx = Mask[i];
3696 if (Idx < NumElems)
3699 else
3700 Ops.push_back(
3703 }
3704
3706
3707 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3708 Results.push_back(Tmp1);
3709 break;
3710 }
3713 break;
3714 }
3716 unsigned Factor = Node->getNumOperands();
3718 break;
3720 EVT VecVT = Node->getValueType(0);
3722
3723
3729
3730
3731 for (unsigned I = 0; I < Factor / 2; I++) {
3734 {L.getValue(I), R.getValue(I)});
3737 }
3738 break;
3739 }
3741 unsigned Factor = Node->getNumOperands();
3743 break;
3744 EVT VecVT = Node->getValueType(0);
3747
3748
3749 for (unsigned I = 0; I < Factor / 2; I++) {
3752 {Node->getOperand(I), Node->getOperand(I + Factor / 2)});
3755 }
3756
3757
3760 for (unsigned I = 0; I < Factor / 2; I++)
3761 Results.push_back(L.getValue(I));
3762 for (unsigned I = 0; I < Factor / 2; I++)
3763 Results.push_back(R.getValue(I));
3764 break;
3765 }
3767 EVT OpTy = Node->getOperand(0).getValueType();
3768 if (Node->getConstantOperandVal(1)) {
3769
3774 } else {
3775
3777 Node->getOperand(0));
3778 }
3779 Results.push_back(Tmp1);
3780 break;
3781 }
3782 case ISD::STACKSAVE:
3783
3784
3787 Node->getValueType(0)));
3789 } else {
3792 }
3793 break;
3794 case ISD::STACKRESTORE:
3795
3796
3799 Node->getOperand(1)));
3800 } else {
3802 }
3803 break;
3804 case ISD::GET_DYNAMIC_AREA_OFFSET:
3807 break;
3809 Results.push_back(ExpandFCOPYSIGN(Node));
3810 break;
3811 case ISD::FNEG:
3812 Results.push_back(ExpandFNEG(Node));
3813 break;
3814 case ISD::FABS:
3815 Results.push_back(ExpandFABS(Node));
3816 break;
3821 Test, Node->getFlags(), SDLoc(Node), DAG))
3822 Results.push_back(Expanded);
3823 break;
3824 }
3829
3831 switch (Node->getOpcode()) {
3837 }
3838 Tmp1 = Node->getOperand(0);
3839 Tmp2 = Node->getOperand(1);
3840 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3841 Results.push_back(Tmp1);
3842 break;
3843 }
3844 case ISD::FMINNUM:
3845 case ISD::FMAXNUM: {
3847 Results.push_back(Expanded);
3848 break;
3849 }
3850 case ISD::FMINIMUM:
3851 case ISD::FMAXIMUM: {
3853 Results.push_back(Expanded);
3854 break;
3855 }
3856 case ISD::FMINIMUMNUM:
3857 case ISD::FMAXIMUMNUM: {
3859 break;
3860 }
3861 case ISD::FSIN:
3862 case ISD::FCOS: {
3863 EVT VT = Node->getValueType(0);
3864
3865
3869 SDVTList VTs = DAG.getVTList(VT, VT);
3870 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3871 if (Node->getOpcode() == ISD::FCOS)
3873 Results.push_back(Tmp1);
3874 }
3875 break;
3876 }
3877 case ISD::FLDEXP:
3879 EVT VT = Node->getValueType(0);
3881
3882
3884 break;
3885
3886 if (SDValue Expanded = expandLdexp(Node)) {
3887 Results.push_back(Expanded);
3889 Results.push_back(Expanded.getValue(1));
3890 }
3891
3892 break;
3893 }
3894 case ISD::FFREXP: {
3896
3897
3899 break;
3900
3901 if (SDValue Expanded = expandFrexp(Node)) {
3902 Results.push_back(Expanded);
3903 Results.push_back(Expanded.getValue(1));
3904 }
3905 break;
3906 }
3907 case ISD::FSINCOS: {
3909 break;
3910 EVT VT = Node->getValueType(0);
3912 SDNodeFlags Flags = Node->getFlags();
3913 Tmp1 = DAG.getNode(ISD::FSIN, dl, VT, Op, Flags);
3914 Tmp2 = DAG.getNode(ISD::FCOS, dl, VT, Op, Flags);
3915 Results.append({Tmp1, Tmp2});
3916 break;
3917 }
3920
3921 case ISD::FP16_TO_FP:
3922 if (Node->getValueType(0) != MVT::f32) {
3923
3924
3925
3927 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3929 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3930 }
3931 break;
3932 case ISD::STRICT_BF16_TO_FP:
3933 case ISD::STRICT_FP16_TO_FP:
3934 if (Node->getValueType(0) != MVT::f32) {
3935
3936
3937
3938 SDValue Res = DAG.getNode(Node->getOpcode(), dl, {MVT::f32, MVT::Other},
3939 {Node->getOperand(0), Node->getOperand(1)});
3941 {Node->getValueType(0), MVT::Other},
3945 }
3946 break;
3947 case ISD::FP_TO_FP16:
3949 if (Node->getFlags().hasApproximateFuncs() && !TLI.useSoftFloat()) {
3951 MVT SVT = Op.getSimpleValueType();
3952 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3954
3955
3960 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3961 }
3962 }
3963 break;
3966
3967
3970 Results.push_back(ExpandConstantFP(CFP, true));
3971 break;
3972 }
3975 Results.push_back(ExpandConstant(CP));
3976 break;
3977 }
3979 EVT VT = Node->getValueType(0);
3982 const SDNodeFlags Flags = Node->getFlags();
3983 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3985 Results.push_back(Tmp1);
3986 }
3987 break;
3988 }
3990 EVT VT = Node->getValueType(0);
3993 "Don't know how to expand this subtraction!");
3994 Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT);
3997 break;
3998 }
4001 if (TLI.expandREM(Node, Tmp1, DAG))
4002 Results.push_back(Tmp1);
4003 break;
4008 EVT VT = Node->getValueType(0);
4010 SDVTList VTs = DAG.getVTList(VT, VT);
4011 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
4012 Node->getOperand(1));
4013 Results.push_back(Tmp1);
4014 }
4015 break;
4016 }
4019 unsigned ExpandOpcode =
4021 EVT VT = Node->getValueType(0);
4022 SDVTList VTs = DAG.getVTList(VT, VT);
4023
4024 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
4025 Node->getOperand(1));
4027 break;
4028 }
4033 MVT VT = LHS.getSimpleValueType();
4034 unsigned MULHOpcode =
4036
4040 break;
4041 }
4042
4044 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
4047 HalfType, DAG,
4048 TargetLowering::MulExpansionKind::Always)) {
4049 for (unsigned i = 0; i < 2; ++i) {
4056 }
4057 break;
4058 }
4059 break;
4060 }
4062 EVT VT = Node->getValueType(0);
4063 SDVTList VTs = DAG.getVTList(VT, VT);
4064
4065
4066
4067
4068
4073 unsigned OpToUse = 0;
4074 if (HasSMUL_LOHI && !HasMULHS) {
4076 } else if (HasUMUL_LOHI && !HasMULHU) {
4078 } else if (HasSMUL_LOHI) {
4080 } else if (HasUMUL_LOHI) {
4082 }
4083 if (OpToUse) {
4084 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
4085 Node->getOperand(1)));
4086 break;
4087 }
4088
4096 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
4103 }
4104 break;
4105 }
4109 Results.push_back(Expanded);
4110 break;
4113 if (SDValue Expanded = TLI.expandROT(Node, true , DAG))
4114 Results.push_back(Expanded);
4115 break;
4121 break;
4125 break;
4129 break;
4135 break;
4141 Node->getOperand(0),
4142 Node->getOperand(1),
4143 Node->getConstantOperandVal(2),
4144 DAG)) {
4146 break;
4147 }
4148
4149
4150
4151
4152
4153
4154
4161
4163
4164
4166 EVT VT = LHS.getValueType();
4168
4169
4170 EVT CarryType = Node->getValueType(1);
4171 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
4174
4175
4180
4181
4182
4183
4184
4189 Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2,
4191
4193 DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2);
4194
4195 Results.push_back(Sum2);
4197 break;
4198 }
4203 Results.push_back(Result);
4204 Results.push_back(Overflow);
4205 break;
4206 }
4211 Results.push_back(Result);
4212 Results.push_back(Overflow);
4213 break;
4214 }
4218 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
4219 Results.push_back(Result);
4220 Results.push_back(Overflow);
4221 }
4222 break;
4223 }
4232 break;
4233 }
4235 Tmp1 = Node->getOperand(0);
4236 Tmp2 = Node->getOperand(1);
4237 Tmp3 = Node->getOperand(2);
4242 } else {
4243 Tmp1 =
4246 }
4247 Results.push_back(Tmp1);
4248 break;
4249 case ISD::BR_JT: {
4254
4257
4258 unsigned EntrySize =
4260
4261
4262
4263
4264
4269 else
4273
4278 Addr = LD;
4280
4281
4282
4284 Addr, dl);
4285 }
4286
4288 Results.push_back(Tmp1);
4289 break;
4290 }
4291 case ISD::BRCOND:
4292
4293
4294 Tmp1 = Node->getOperand(0);
4295 Tmp2 = Node->getOperand(1);
4299 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2),
4301 Node->getOperand(2));
4302 } else {
4303
4306 Tmp3 = Tmp2;
4307 else
4310 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
4313 Node->getOperand(2));
4314 }
4315 Results.push_back(Tmp1);
4316 break;
4318 case ISD::VP_SETCC:
4321 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
4326 unsigned Offset = IsStrict ? 1 : 0;
4327 Tmp1 = Node->getOperand(0 + Offset);
4328 Tmp2 = Node->getOperand(1 + Offset);
4329 Tmp3 = Node->getOperand(2 + Offset);
4331 if (IsVP) {
4334 }
4336 DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
4337 Chain, IsSignaling);
4338
4339 if (Legalized) {
4340
4341
4343 if (IsStrict) {
4344 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(),
4345 {Chain, Tmp1, Tmp2, Tmp3}, Node->getFlags());
4347 } else if (IsVP) {
4348 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0),
4349 {Tmp1, Tmp2, Tmp3, Mask, EVL}, Node->getFlags());
4350 } else {
4351 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1,
4352 Tmp2, Tmp3, Node->getFlags());
4353 }
4354 }
4355
4356
4357
4358 if (NeedInvert) {
4359 if (!IsVP)
4361 else
4362 Tmp1 =
4364 }
4365
4366 Results.push_back(Tmp1);
4367 if (IsStrict)
4368 Results.push_back(Chain);
4369
4370 break;
4371 }
4372
4373
4374
4375 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4376
4377
4378
4379
4380 EVT VT = Node->getValueType(0);
4385 Node->getFlags());
4386 Results.push_back(Tmp1);
4387 break;
4388 }
4390
4391 Tmp1 = Node->getOperand(0);
4392 Tmp2 = Node->getOperand(1);
4393 Tmp3 = Node->getOperand(2);
4394 Tmp4 = Node->getOperand(3);
4395 EVT VT = Node->getValueType(0);
4399
4401
4402
4405 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4406 "expanded.");
4407 EVT CCVT = getSetCCResultType(CmpVT);
4411 break;
4412 }
4413
4414
4415 bool Legalized = false;
4416
4417
4418
4421
4422 Legalized = true;
4423 Tmp1 =
4424 DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC, Node->getFlags());
4425 } else {
4426
4427
4430
4431
4432 Legalized = true;
4433 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC,
4434 Node->getFlags());
4435 }
4436 }
4437
4438 if (!Legalized) {
4440 DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC,
4441 SDValue(), SDValue(), NeedInvert, dl, Chain);
4442
4443 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4444
4445
4446
4447 if (NeedInvert)
4449
4450
4451
4454 Tmp2, Tmp3, Tmp4, CC, Node->getFlags());
4455 } else {
4459 Tmp2, Tmp3, Tmp4, CC, Node->getFlags());
4460 }
4461 }
4462 Results.push_back(Tmp1);
4463 break;
4464 }
4465 case ISD::BR_CC: {
4466
4468 Tmp1 = Node->getOperand(0);
4469 Tmp2 = Node->getOperand(2);
4470 Tmp3 = Node->getOperand(3);
4471 Tmp4 = Node->getOperand(1);
4472
4474 DAG, getSetCCResultType(Tmp2.getValueType()), Tmp2, Tmp3, Tmp4,
4475 SDValue(), SDValue(), NeedInvert, dl, Chain);
4476 (void)Legalized;
4477 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4478
4479
4480
4482 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4483
4484 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
4485 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
4486 } else {
4489 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
4490 Tmp2, Tmp3, Node->getOperand(4));
4491 }
4492 Results.push_back(Tmp1);
4493 break;
4494 }
4496 Results.push_back(ExpandBUILD_VECTOR(Node));
4497 break;
4499 Results.push_back(ExpandSPLAT_VECTOR(Node));
4500 break;
4504
4505 EVT VT = Node->getValueType(0);
4506 assert(VT.isVector() && "Unable to legalize non-vector shift");
4509
4511 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4520 }
4521
4523 Results.push_back(Result);
4524 break;
4525 }
4526 case ISD::VECREDUCE_FADD:
4527 case ISD::VECREDUCE_FMUL:
4528 case ISD::VECREDUCE_ADD:
4529 case ISD::VECREDUCE_MUL:
4530 case ISD::VECREDUCE_AND:
4531 case ISD::VECREDUCE_OR:
4532 case ISD::VECREDUCE_XOR:
4533 case ISD::VECREDUCE_SMAX:
4534 case ISD::VECREDUCE_SMIN:
4535 case ISD::VECREDUCE_UMAX:
4536 case ISD::VECREDUCE_UMIN:
4537 case ISD::VECREDUCE_FMAX:
4538 case ISD::VECREDUCE_FMIN:
4539 case ISD::VECREDUCE_FMAXIMUM:
4540 case ISD::VECREDUCE_FMINIMUM:
4542 break;
4543 case ISD::VP_CTTZ_ELTS:
4544 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
4546 break;
4548
4549
4551 break;
4552 case ISD::LRINT:
4553 case ISD::LLRINT: {
4556 EVT ResVT = Node->getValueType(0);
4557 SDLoc dl(Node);
4558 SDValue RoundNode = DAG.getNode(ISD::FRINT, dl, ArgVT, Arg);
4560 break;
4561 }
4562 case ISD::ADDRSPACECAST:
4564 break;
4574
4575
4576
4577 return true;
4578 }
4579
4581
4582
4583
4584
4585
4586 switch (Node->getOpcode()) {
4587 default:
4589 Node->getValueType(0))
4590 == TargetLowering::Legal)
4591 return true;
4592 break;
4596 return true;
4599 break;
4600
4601 EVT VT = Node->getValueType(0);
4602 const SDNodeFlags Flags = Node->getFlags();
4603 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags);
4605 {Node->getOperand(0), Node->getOperand(1), Neg},
4606 Flags);
4607
4608 Results.push_back(Fadd);
4610 break;
4611 }
4618
4619
4621 Node->getOperand(1).getValueType())
4622 == TargetLowering::Legal)
4623 return true;
4624 break;
4625 }
4626 }
4627
4628
4631 return false;
4632 }
4633
4634 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4635 ReplaceNode(Node, Results.data());
4636 return true;
4637}
4638
4639
4640
4641
4643
4644
4645
4647 return Flags.hasApproximateFuncs() && Flags.hasNoNaNs() &&
4648 Flags.hasNoInfs() && Flags.hasNoSignedZeros();
4649}
4650
4651void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4652 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4654 SDLoc dl(Node);
4655 TargetLowering::MakeLibCallOptions CallOptions;
4657
4658 unsigned Opc = Node->getOpcode();
4659 switch (Opc) {
4660 case ISD::ATOMIC_FENCE: {
4661
4662
4663 TargetLowering::ArgListTy Args;
4664
4665 TargetLowering::CallLoweringInfo CLI(DAG);
4666 CLI.setDebugLoc(dl)
4667 .setChain(Node->getOperand(0))
4668 .setLibCallee(
4669 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4672 std::move(Args));
4673
4674 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4675
4676 Results.push_back(CallResult.second);
4677 break;
4678 }
4679
4680
4681
4682 case ISD::ATOMIC_SWAP:
4683 case ISD::ATOMIC_LOAD_ADD:
4684 case ISD::ATOMIC_LOAD_SUB:
4685 case ISD::ATOMIC_LOAD_AND:
4686 case ISD::ATOMIC_LOAD_CLR:
4687 case ISD::ATOMIC_LOAD_OR:
4688 case ISD::ATOMIC_LOAD_XOR:
4689 case ISD::ATOMIC_LOAD_NAND:
4690 case ISD::ATOMIC_LOAD_MIN:
4691 case ISD::ATOMIC_LOAD_MAX:
4692 case ISD::ATOMIC_LOAD_UMIN:
4693 case ISD::ATOMIC_LOAD_UMAX:
4694 case ISD::ATOMIC_CMP_SWAP: {
4698 EVT RetVT = Node->getValueType(0);
4701
4702 Ops.append(Node->op_begin() + 2, Node->op_end());
4703 Ops.push_back(Node->getOperand(1));
4704
4705 } else {
4707 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4708 "Unexpected atomic op or value type!");
4709
4710 Ops.append(Node->op_begin() + 1, Node->op_end());
4711 }
4712 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4713 Ops, CallOptions,
4714 SDLoc(Node),
4715 Node->getOperand(0));
4716 Results.push_back(Tmp.first);
4717 Results.push_back(Tmp.second);
4718 break;
4719 }
4720 case ISD::TRAP: {
4721
4722 TargetLowering::ArgListTy Args;
4723 TargetLowering::CallLoweringInfo CLI(DAG);
4724 CLI.setDebugLoc(dl)
4725 .setChain(Node->getOperand(0))
4726 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4729 std::move(Args));
4730 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4731
4732 Results.push_back(CallResult.second);
4733 break;
4734 }
4736 SDValue InputChain = Node->getOperand(0);
4739 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4740 DAG, RTLIB::CLEAR_CACHE, MVT::isVoid, {StartVal, EndVal}, CallOptions,
4741 SDLoc(Node), InputChain);
4742 Results.push_back(Tmp.second);
4743 break;
4744 }
4745 case ISD::FMINNUM:
4747 ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4748 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4749 RTLIB::FMIN_PPCF128, Results);
4750 break;
4751
4752
4753
4754 case ISD::FMAXNUM:
4756 ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4757 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4758 RTLIB::FMAX_PPCF128, Results);
4759 break;
4760 case ISD::FMINIMUMNUM:
4761 ExpandFPLibCall(Node, RTLIB::FMINIMUM_NUM_F32, RTLIB::FMINIMUM_NUM_F64,
4762 RTLIB::FMINIMUM_NUM_F80, RTLIB::FMINIMUM_NUM_F128,
4763 RTLIB::FMINIMUM_NUM_PPCF128, Results);
4764 break;
4765 case ISD::FMAXIMUMNUM:
4766 ExpandFPLibCall(Node, RTLIB::FMAXIMUM_NUM_F32, RTLIB::FMAXIMUM_NUM_F64,
4767 RTLIB::FMAXIMUM_NUM_F80, RTLIB::FMAXIMUM_NUM_F128,
4768 RTLIB::FMAXIMUM_NUM_PPCF128, Results);
4769 break;
4770 case ISD::FSQRT:
4772
4773
4775 {RTLIB::FAST_SQRT_F32, RTLIB::SQRT_F32},
4776 {RTLIB::FAST_SQRT_F64, RTLIB::SQRT_F64},
4777 {RTLIB::FAST_SQRT_F80, RTLIB::SQRT_F80},
4778 {RTLIB::FAST_SQRT_F128, RTLIB::SQRT_F128},
4779 {RTLIB::FAST_SQRT_PPCF128, RTLIB::SQRT_PPCF128},
4781 break;
4782 }
4783 case ISD::FCBRT:
4784 ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4785 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4786 RTLIB::CBRT_PPCF128, Results);
4787 break;
4788 case ISD::FSIN:
4790 ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4791 RTLIB::SIN_F80, RTLIB::SIN_F128,
4792 RTLIB::SIN_PPCF128, Results);
4793 break;
4794 case ISD::FCOS:
4796 ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
4797 RTLIB::COS_F80, RTLIB::COS_F128,
4798 RTLIB::COS_PPCF128, Results);
4799 break;
4800 case ISD::FTAN:
4802 ExpandFPLibCall(Node, RTLIB::TAN_F32, RTLIB::TAN_F64, RTLIB::TAN_F80,
4803 RTLIB::TAN_F128, RTLIB::TAN_PPCF128, Results);
4804 break;
4805 case ISD::FASIN:
4807 ExpandFPLibCall(Node, RTLIB::ASIN_F32, RTLIB::ASIN_F64, RTLIB::ASIN_F80,
4808 RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128, Results);
4809 break;
4810 case ISD::FACOS:
4812 ExpandFPLibCall(Node, RTLIB::ACOS_F32, RTLIB::ACOS_F64, RTLIB::ACOS_F80,
4813 RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128, Results);
4814 break;
4815 case ISD::FATAN:
4817 ExpandFPLibCall(Node, RTLIB::ATAN_F32, RTLIB::ATAN_F64, RTLIB::ATAN_F80,
4818 RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128, Results);
4819 break;
4820 case ISD::FATAN2:
4822 ExpandFPLibCall(Node, RTLIB::ATAN2_F32, RTLIB::ATAN2_F64, RTLIB::ATAN2_F80,
4823 RTLIB::ATAN2_F128, RTLIB::ATAN2_PPCF128, Results);
4824 break;
4825 case ISD::FSINH:
4827 ExpandFPLibCall(Node, RTLIB::SINH_F32, RTLIB::SINH_F64, RTLIB::SINH_F80,
4828 RTLIB::SINH_F128, RTLIB::SINH_PPCF128, Results);
4829 break;
4830 case ISD::FCOSH:
4832 ExpandFPLibCall(Node, RTLIB::COSH_F32, RTLIB::COSH_F64, RTLIB::COSH_F80,
4833 RTLIB::COSH_F128, RTLIB::COSH_PPCF128, Results);
4834 break;
4835 case ISD::FTANH:
4837 ExpandFPLibCall(Node, RTLIB::TANH_F32, RTLIB::TANH_F64, RTLIB::TANH_F80,
4838 RTLIB::TANH_F128, RTLIB::TANH_PPCF128, Results);
4839 break;
4840 case ISD::FSINCOS:
4841 case ISD::FSINCOSPI: {
4842 EVT VT = Node->getValueType(0);
4843
4844 if (Node->getOpcode() == ISD::FSINCOS) {
4846 if (SincosStret != RTLIB::UNKNOWN_LIBCALL) {
4847 if (SDValue Expanded = ExpandSincosStretLibCall(Node)) {
4848 Results.push_back(Expanded);
4849 Results.push_back(Expanded.getValue(1));
4850 break;
4851 }
4852 }
4853 }
4854
4855 RTLIB::Libcall LC = Node->getOpcode() == ISD::FSINCOS
4859 if (!Expanded) {
4861 Node->getOperationName(&DAG));
4865 }
4866
4867 break;
4868 }
4869 case ISD::FLOG:
4871 ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4872 RTLIB::LOG_F128, RTLIB::LOG_PPCF128, Results);
4873 break;
4874 case ISD::FLOG2:
4876 ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4877 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128, Results);
4878 break;
4879 case ISD::FLOG10:
4881 ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4882 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128, Results);
4883 break;
4884 case ISD::FEXP:
4886 ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4887 RTLIB::EXP_F128, RTLIB::EXP_PPCF128, Results);
4888 break;
4889 case ISD::FEXP2:
4891 ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4892 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128, Results);
4893 break;
4894 case ISD::FEXP10:
4895 ExpandFPLibCall(Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80,
4896 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128, Results);
4897 break;
4898 case ISD::FTRUNC:
4900 ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4901 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4902 RTLIB::TRUNC_PPCF128, Results);
4903 break;
4904 case ISD::FFLOOR:
4906 ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4907 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4908 RTLIB::FLOOR_PPCF128, Results);
4909 break;
4910 case ISD::FCEIL:
4912 ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4913 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4914 RTLIB::CEIL_PPCF128, Results);
4915 break;
4916 case ISD::FRINT:
4918 ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4919 RTLIB::RINT_F80, RTLIB::RINT_F128,
4920 RTLIB::RINT_PPCF128, Results);
4921 break;
4922 case ISD::FNEARBYINT:
4924 ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4925 RTLIB::NEARBYINT_F64,
4926 RTLIB::NEARBYINT_F80,
4927 RTLIB::NEARBYINT_F128,
4928 RTLIB::NEARBYINT_PPCF128, Results);
4929 break;
4930 case ISD::FROUND:
4932 ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4933 RTLIB::ROUND_F64,
4934 RTLIB::ROUND_F80,
4935 RTLIB::ROUND_F128,
4936 RTLIB::ROUND_PPCF128, Results);
4937 break;
4938 case ISD::FROUNDEVEN:
4940 ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32,
4941 RTLIB::ROUNDEVEN_F64,
4942 RTLIB::ROUNDEVEN_F80,
4943 RTLIB::ROUNDEVEN_F128,
4944 RTLIB::ROUNDEVEN_PPCF128, Results);
4945 break;
4946 case ISD::FLDEXP:
4948 ExpandFPLibCall(Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,
4949 RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128, Results);
4950 break;
4951 case ISD::FMODF:
4952 case ISD::FFREXP: {
4953 EVT VT = Node->getValueType(0);
4954 RTLIB::Libcall LC = Node->getOpcode() == ISD::FMODF ? RTLIB::getMODF(VT)
4957 0);
4958 if (!Expanded)
4959 llvm_unreachable("Expected scalar FFREXP/FMODF to expand to libcall!");
4960 break;
4961 }
4962 case ISD::FPOWI:
4965 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
4967
4968 if (Node->isStrictFPOpcode()) {
4971 {Node->getValueType(0), Node->getValueType(1)},
4972 {Node->getOperand(0), Node->getOperand(2)});
4975 {Node->getValueType(0), Node->getValueType(1)},
4977 Results.push_back(FPOW);
4979 } else {
4982 Node->getOperand(1));
4983 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node),
4984 Node->getValueType(0),
4986 }
4987 break;
4988 }
4989 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
4990 bool ExponentHasSizeOfInt =
4992 Node->getOperand(1 + Offset).getValueType().getSizeInBits();
4993 if (!ExponentHasSizeOfInt) {
4994
4995
4996 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
4998 break;
4999 }
5000 ExpandFPLibCall(Node, LC, Results);
5001 break;
5002 }
5003 case ISD::FPOW:
5005 ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
5006 RTLIB::POW_F128, RTLIB::POW_PPCF128, Results);
5007 break;
5008 case ISD::LROUND:
5010 ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,
5011 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
5012 RTLIB::LROUND_F128,
5013 RTLIB::LROUND_PPCF128, Results);
5014 break;
5015 case ISD::LLROUND:
5017 ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,
5018 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
5019 RTLIB::LLROUND_F128,
5020 RTLIB::LLROUND_PPCF128, Results);
5021 break;
5022 case ISD::LRINT:
5024 ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,
5025 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
5026 RTLIB::LRINT_F128,
5027 RTLIB::LRINT_PPCF128, Results);
5028 break;
5029 case ISD::LLRINT:
5031 ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,
5032 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
5033 RTLIB::LLRINT_F128,
5034 RTLIB::LLRINT_PPCF128, Results);
5035 break;
5039 {RTLIB::FAST_DIV_F32, RTLIB::DIV_F32},
5040 {RTLIB::FAST_DIV_F64, RTLIB::DIV_F64},
5041 {RTLIB::FAST_DIV_F80, RTLIB::DIV_F80},
5042 {RTLIB::FAST_DIV_F128, RTLIB::DIV_F128},
5043 {RTLIB::FAST_DIV_PPCF128, RTLIB::DIV_PPCF128}, Results);
5044 break;
5045 }
5048 ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
5049 RTLIB::REM_F80, RTLIB::REM_F128,
5050 RTLIB::REM_PPCF128, Results);
5051 break;
5054 ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
5055 RTLIB::FMA_F80, RTLIB::FMA_F128,
5056 RTLIB::FMA_PPCF128, Results);
5057 break;
5061 {RTLIB::FAST_ADD_F32, RTLIB::ADD_F32},
5062 {RTLIB::FAST_ADD_F64, RTLIB::ADD_F64},
5063 {RTLIB::FAST_ADD_F80, RTLIB::ADD_F80},
5064 {RTLIB::FAST_ADD_F128, RTLIB::ADD_F128},
5065 {RTLIB::FAST_ADD_PPCF128, RTLIB::ADD_PPCF128}, Results);
5066 break;
5067 }
5071 {RTLIB::FAST_MUL_F32, RTLIB::MUL_F32},
5072 {RTLIB::FAST_MUL_F64, RTLIB::MUL_F64},
5073 {RTLIB::FAST_MUL_F80, RTLIB::MUL_F80},
5074 {RTLIB::FAST_MUL_F128, RTLIB::MUL_F128},
5075 {RTLIB::FAST_MUL_PPCF128, RTLIB::MUL_PPCF128}, Results);
5076 break;
5077 }
5078 case ISD::FP16_TO_FP:
5079 if (Node->getValueType(0) == MVT::f32) {
5080 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false).first);
5081 }
5082 break;
5083 case ISD::STRICT_BF16_TO_FP:
5084 if (Node->getValueType(0) == MVT::f32) {
5085 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5086 DAG, RTLIB::FPEXT_BF16_F32, MVT::f32, Node->getOperand(1),
5087 CallOptions, SDLoc(Node), Node->getOperand(0));
5088 Results.push_back(Tmp.first);
5089 Results.push_back(Tmp.second);
5090 }
5091 break;
5092 case ISD::STRICT_FP16_TO_FP: {
5093 if (Node->getValueType(0) == MVT::f32) {
5094 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
5095 DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions,
5096 SDLoc(Node), Node->getOperand(0));
5097 Results.push_back(Tmp.first);
5098 Results.push_back(Tmp.second);
5099 }
5100 break;
5101 }
5102 case ISD::FP_TO_FP16: {
5103 RTLIB::Libcall LC =
5105 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
5106 Results.push_back(ExpandLibCall(LC, Node, false).first);
5107 break;
5108 }
5109 case ISD::FP_TO_BF16: {
5110 RTLIB::Libcall LC =
5112 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
5113 Results.push_back(ExpandLibCall(LC, Node, false).first);
5114 break;
5115 }
5120
5121 bool IsStrict = Node->isStrictFPOpcode();
5124 EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType();
5125 EVT RVT = Node->getValueType(0);
5126 EVT NVT = EVT();
5127 SDLoc dl(Node);
5128
5129
5130
5131
5132 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5133 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
5134 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5135 ++t) {
5137
5138 if (NVT.bitsGE(SVT))
5141 }
5142 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5143
5145
5147 NVT, Node->getOperand(IsStrict ? 1 : 0));
5149 std::pair<SDValue, SDValue> Tmp =
5150 TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, dl, Chain);
5151 Results.push_back(Tmp.first);
5152 if (IsStrict)
5153 Results.push_back(Tmp.second);
5154 break;
5155 }
5160
5161 bool IsStrict = Node->isStrictFPOpcode();
5164
5165 SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
5166 EVT SVT = Op.getValueType();
5167 EVT RVT = Node->getValueType(0);
5168 EVT NVT = EVT();
5169 SDLoc dl(Node);
5170
5171
5172
5173
5174 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5175 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
5176 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
5177 ++IntVT) {
5179
5180 if (NVT.bitsGE(RVT))
5183 }
5184 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5185
5187 std::pair<SDValue, SDValue> Tmp =
5188 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
5189
5190
5192 if (IsStrict)
5193 Results.push_back(Tmp.second);
5194 break;
5195 }
5196
5199
5200
5201
5202
5203
5204 bool IsStrict = Node->isStrictFPOpcode();
5205 SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
5207 EVT VT = Node->getValueType(0);
5209 "Unable to expand as libcall if it is not normal rounding");
5210
5212 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5213
5214 std::pair<SDValue, SDValue> Tmp =
5215 TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain);
5216 Results.push_back(Tmp.first);
5217 if (IsStrict)
5218 Results.push_back(Tmp.second);
5219 break;
5220 }
5221 case ISD::FP_EXTEND: {
5224 Node->getValueType(0)),
5225 Node, false).first);
5226 break;
5227 }
5229 case ISD::STRICT_FP_TO_FP16:
5230 case ISD::STRICT_FP_TO_BF16: {
5231 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5232 if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16)
5234 else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16)
5236 else
5238 Node->getValueType(0));
5239
5240 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
5241
5242 std::pair<SDValue, SDValue> Tmp =
5244 CallOptions, SDLoc(Node), Node->getOperand(0));
5245 Results.push_back(Tmp.first);
5246 Results.push_back(Tmp.second);
5247 break;
5248 }
5252 {RTLIB::FAST_SUB_F32, RTLIB::SUB_F32},
5253 {RTLIB::FAST_SUB_F64, RTLIB::SUB_F64},
5254 {RTLIB::FAST_SUB_F80, RTLIB::SUB_F80},
5255 {RTLIB::FAST_SUB_F128, RTLIB::SUB_F128},
5256 {RTLIB::FAST_SUB_PPCF128, RTLIB::SUB_PPCF128}, Results);
5257 break;
5258 }
5260 Results.push_back(ExpandIntLibCall(Node, true,
5261 RTLIB::SREM_I8,
5262 RTLIB::SREM_I16, RTLIB::SREM_I32,
5263 RTLIB::SREM_I64, RTLIB::SREM_I128));
5264 break;
5266 Results.push_back(ExpandIntLibCall(Node, false,
5267 RTLIB::UREM_I8,
5268 RTLIB::UREM_I16, RTLIB::UREM_I32,
5269 RTLIB::UREM_I64, RTLIB::UREM_I128));
5270 break;
5272 Results.push_back(ExpandIntLibCall(Node, true,
5273 RTLIB::SDIV_I8,
5274 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
5275 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
5276 break;
5278 Results.push_back(ExpandIntLibCall(Node, false,
5279 RTLIB::UDIV_I8,
5280 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
5281 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
5282 break;
5285
5286 ExpandDivRemLibCall(Node, Results);
5287 break;
5289 Results.push_back(ExpandIntLibCall(Node, false,
5290 RTLIB::MUL_I8,
5291 RTLIB::MUL_I16, RTLIB::MUL_I32,
5292 RTLIB::MUL_I64, RTLIB::MUL_I128));
5293 break;
5295 Results.push_back(ExpandBitCountingLibCall(
5296 Node, RTLIB::CTLZ_I32, RTLIB::CTLZ_I64, RTLIB::CTLZ_I128));
5297 break;
5299 Results.push_back(ExpandBitCountingLibCall(
5300 Node, RTLIB::CTPOP_I32, RTLIB::CTPOP_I64, RTLIB::CTPOP_I128));
5301 break;
5302 case ISD::RESET_FPENV: {
5303
5304
5310 break;
5311 }
5312 case ISD::GET_FPENV_MEM: {
5317 break;
5318 }
5319 case ISD::SET_FPENV_MEM: {
5324 break;
5325 }
5326 case ISD::GET_FPMODE: {
5327
5328
5329 EVT ModeVT = Node->getValueType(0);
5333 Node->getOperand(0), dl);
5335 ModeVT, dl, Chain, StackPtr,
5337 Results.push_back(LdInst);
5339 break;
5340 }
5341 case ISD::SET_FPMODE: {
5342
5343
5345 EVT ModeVT = Mode.getValueType();
5349 Node->getOperand(0), dl, Mode, StackPtr,
5353 break;
5354 }
5355 case ISD::RESET_FPMODE: {
5356
5357
5358
5363 Node->getOperand(0), dl));
5364 break;
5365 }
5366 }
5367
5368
5370 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
5371 ReplaceNode(Node, Results.data());
5372 } else
5373 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
5374}
5375
5376
5377
5379 MVT EltVT, MVT NewEltVT) {
5381 MVT MidVT = OldEltsPerNewElt == 1
5382 ? NewEltVT
5385 return MidVT;
5386}
5387
5388void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
5391 MVT OVT = Node->getSimpleValueType(0);
5397 Node->getOpcode() == ISD::VECREDUCE_FMAX ||
5398 Node->getOpcode() == ISD::VECREDUCE_FMIN ||
5399 Node->getOpcode() == ISD::VECREDUCE_FMAXIMUM ||
5400 Node->getOpcode() == ISD::VECREDUCE_FMINIMUM) {
5401 OVT = Node->getOperand(0).getSimpleValueType();
5402 }
5403 if (Node->getOpcode() == ISD::ATOMIC_STORE ||
5408 Node->getOpcode() == ISD::VP_REDUCE_FADD ||
5409 Node->getOpcode() == ISD::VP_REDUCE_FMUL ||
5410 Node->getOpcode() == ISD::VP_REDUCE_FMAX ||
5411 Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
5412 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||
5413 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||
5414 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
5415 OVT = Node->getOperand(1).getSimpleValueType();
5416 if (Node->getOpcode() == ISD::BR_CC ||
5418 OVT = Node->getOperand(2).getSimpleValueType();
5419
5421 SelectionDAG::FlagInserter FlagsInserter(DAG, FastMathFlags);
5423 SDLoc dl(Node);
5424 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
5425 switch (Node->getOpcode()) {
5430
5434 else
5436
5437 unsigned NewOpc = Node->getOpcode();
5439
5440
5441
5447 }
5448
5449
5450 Tmp1 = DAG.getNode(NewOpc, dl, NVT, Tmp1);
5452
5456 }
5459 break;
5460 }
5462
5463
5464
5465
5466 auto AnyExtendedNode =
5468
5469
5472 auto LeftShiftResult =
5473 DAG.getNode(ISD::SHL, dl, NVT, AnyExtendedNode, ShiftConstant);
5474
5475
5476 auto CTLZResult = DAG.getNode(Node->getOpcode(), dl, NVT, LeftShiftResult);
5478 break;
5479 }
5484 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5487
5489 break;
5490 }
5495 PromoteLegalFP_TO_INT(Node, dl, Results);
5496 break;
5499 Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl));
5500 break;
5505 PromoteLegalINT_TO_FP(Node, dl, Results);
5506 break;
5507 case ISD::VAARG: {
5508 SDValue Chain = Node->getOperand(0);
5509 SDValue Ptr = Node->getOperand(1);
5510
5511 unsigned TruncOp;
5513 TruncOp = ISD::BITCAST;
5514 } else {
5516 && "VAARG promotion is supported only for vectors or integer types");
5518 }
5519
5520
5521 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
5522 Node->getConstantOperandVal(3));
5524
5525 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
5526
5527
5528
5531 if (UpdatedNodes) {
5532 UpdatedNodes->insert(Tmp2.getNode());
5533 UpdatedNodes->insert(Chain.getNode());
5534 }
5535 ReplacedNode(Node);
5536 break;
5537 }
5550 unsigned ExtOp, TruncOp;
5552 ExtOp = ISD::BITCAST;
5553 TruncOp = ISD::BITCAST;
5554 } else {
5555 assert(OVT.isInteger() && "Cannot promote logic operation");
5556
5557 switch (Node->getOpcode()) {
5558 default:
5560 break;
5566 break;
5570 break;
5575 else
5577 break;
5578 }
5580 }
5581
5582 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5583 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5584
5585 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5586 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
5587 break;
5588 }
5591
5594 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5595 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5597
5603 break;
5604 }
5606 unsigned ExtOp, TruncOp;
5607 if (Node->getValueType(0).isVector() ||
5609 ExtOp = ISD::BITCAST;
5610 TruncOp = ISD::BITCAST;
5611 } else if (Node->getValueType(0).isInteger()) {
5614 } else {
5615 ExtOp = ISD::FP_EXTEND;
5617 }
5618 Tmp1 = Node->getOperand(0);
5619
5620 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5621 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
5622
5623 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
5625 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
5626 else
5627 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
5629 Results.push_back(Tmp1);
5630 break;
5631 }
5634
5635
5636 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
5637 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
5638
5639
5640 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
5641 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
5642 Results.push_back(Tmp1);
5643 break;
5644 }
5649 Node->getOperand(2));
5651 break;
5652 }
5656
5657 MVT CVT = Node->getSimpleValueType(0);
5658 assert(CVT == OVT && "not handled");
5659
5660 unsigned ExtOp = ISD::FP_EXTEND;
5663 }
5664
5665
5667 Tmp1 = Node->getOperand(0);
5668 Tmp2 = Node->getOperand(1);
5669 } else {
5670 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5671 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5672 }
5673
5674 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
5675 Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
5676
5678 Node->getFlags());
5679
5680
5681 if (ExtOp != ISD::FP_EXTEND)
5683 else
5686
5687 Results.push_back(Tmp1);
5688 break;
5689 }
5693 unsigned ExtOp = ISD::FP_EXTEND;
5699 else
5701 }
5702 if (Node->isStrictFPOpcode()) {
5704 std::tie(Tmp1, std::ignore) =
5706 std::tie(Tmp2, std::ignore) =
5710 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
5712 {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
5713 Node->getFlags()));
5715 break;
5716 }
5717 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5718 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5720 Tmp2, Node->getOperand(2), Node->getFlags()));
5721 break;
5722 }
5723 case ISD::BR_CC: {
5724 unsigned ExtOp = ISD::FP_EXTEND;
5729 }
5730 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
5731 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
5732 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
5733 Node->getOperand(0), Node->getOperand(1),
5734 Tmp1, Tmp2, Node->getOperand(4)));
5735 break;
5736 }
5742 case ISD::FMINNUM:
5743 case ISD::FMAXNUM:
5744 case ISD::FMINIMUM:
5745 case ISD::FMAXIMUM:
5746 case ISD::FMINIMUMNUM:
5747 case ISD::FMAXIMUMNUM:
5748 case ISD::FPOW:
5749 case ISD::FATAN2:
5750 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5751 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
5752 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5756 break;
5757
5761 SDVTList VTs = DAG.getVTList(NVT, MVT::Other);
5763 Node->getOperand(1));
5765 Node->getOperand(2));
5767 Tmp3 = DAG.getNode(Node->getOpcode(), dl, VTs, Ops, Node->getFlags());
5769 InChain, Tmp3,
5771 Results.push_back(Tmp4);
5773 break;
5774 }
5775
5786 {Node->getOperand(0), Node->getOperand(1)});
5788 {Node->getOperand(0), Node->getOperand(2)});
5791 Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5792 {Tmp3, Tmp1, Tmp2});
5796 Results.push_back(Tmp1);
5798 break;
5800 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5801 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
5802 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
5805 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
5807 break;
5810 {Node->getOperand(0), Node->getOperand(1)});
5812 {Node->getOperand(0), Node->getOperand(2)});
5814 {Node->getOperand(0), Node->getOperand(3)});
5817 Tmp4 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5818 {Tmp4, Tmp1, Tmp2, Tmp3});
5822 Results.push_back(Tmp4);
5824 break;
5826 case ISD::FLDEXP:
5827 case ISD::FPOWI: {
5828 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5829 Tmp2 = Node->getOperand(1);
5830 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5831
5832
5833
5834
5835
5836
5841 break;
5842 }
5845 {Node->getOperand(0), Node->getOperand(1)});
5846 Tmp2 = Node->getOperand(2);
5848 {Tmp1.getValue(1), Tmp1, Tmp2});
5852 Results.push_back(Tmp4);
5854 break;
5855 }
5858 {Node->getOperand(0), Node->getOperand(1)});
5859 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5860 {Tmp1.getValue(1), Tmp1, Node->getOperand(2)});
5864 Results.push_back(Tmp3);
5866 break;
5867 case ISD::FFREXP: {
5868 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5869 Tmp2 = DAG.getNode(ISD::FFREXP, dl, {NVT, Node->getValueType(1)}, Tmp1);
5870
5874
5876 break;
5877 }
5878 case ISD::FMODF:
5879 case ISD::FSINCOS:
5880 case ISD::FSINCOSPI: {
5881 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5884 for (unsigned ResNum = 0; ResNum < Node->getNumValues(); ResNum++)
5887 break;
5888 }
5889 case ISD::FFLOOR:
5890 case ISD::FCEIL:
5891 case ISD::FRINT:
5892 case ISD::FNEARBYINT:
5893 case ISD::FROUND:
5894 case ISD::FROUNDEVEN:
5895 case ISD::FTRUNC:
5896 case ISD::FNEG:
5897 case ISD::FSQRT:
5898 case ISD::FSIN:
5899 case ISD::FCOS:
5900 case ISD::FTAN:
5901 case ISD::FASIN:
5902 case ISD::FACOS:
5903 case ISD::FATAN:
5904 case ISD::FSINH:
5905 case ISD::FCOSH:
5906 case ISD::FTANH:
5907 case ISD::FLOG:
5908 case ISD::FLOG2:
5909 case ISD::FLOG10:
5910 case ISD::FABS:
5911 case ISD::FEXP:
5912 case ISD::FEXP2:
5913 case ISD::FEXP10:
5915 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5916 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5920 break;
5944 {Node->getOperand(0), Node->getOperand(1)});
5945 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5946 {Tmp1.getValue(1), Tmp1});
5950 Results.push_back(Tmp3);
5952 break;
5956
5957
5958
5959
5960
5961
5962
5964 "Invalid promote type for build_vector");
5965 assert(NewEltVT.bitsLE(EltVT) && "not handled");
5966
5968
5972
5973 SDLoc SL(Node);
5976 SL, NVT, NewOps);
5978 Results.push_back(CvtVec);
5979 break;
5980 }
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5997 "Invalid promote type for extract_vector_elt");
5998 assert(NewEltVT.bitsLT(EltVT) && "not handled");
5999
6002
6005 SDLoc SL(Node);
6008
6009 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
6010
6012 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6015
6017 CastVec, TmpIdx);
6019 }
6020
6022 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
6023 break;
6024 }
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6043 "Invalid promote type for insert_vector_elt");
6044 assert(NewEltVT.bitsLT(EltVT) && "not handled");
6045
6048
6052 SDLoc SL(Node);
6053
6056
6057 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
6058 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
6059
6060 SDValue NewVec = CastVec;
6061 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
6064
6066 CastVal, IdxOffset);
6067
6069 NewVec, Elt, InEltIdx);
6070 }
6071
6072 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
6073 break;
6074 }
6078
6079
6080
6081
6082
6083
6084
6085
6088 SDLoc SL(Node);
6089
6090 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
6092
6097
6100 Results.push_back(CvtVec);
6101 break;
6102 }
6103 case ISD::ATOMIC_SWAP:
6104 case ISD::ATOMIC_STORE: {
6106 SDLoc SL(Node);
6109 "unexpected promotion type");
6111 "unexpected atomic_swap with illegal type");
6112
6115
6116
6117
6118 if (AM->getOpcode() == ISD::ATOMIC_STORE)
6120
6123
6124 if (AM->getOpcode() != ISD::ATOMIC_STORE) {
6125 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
6127 } else
6128 Results.push_back(NewAtomic);
6129 break;
6130 }
6131 case ISD::ATOMIC_LOAD: {
6133 SDLoc SL(Node);
6135 "unexpected promotion type");
6137 "unexpected atomic_load with illegal type");
6138
6140 DAG.getAtomic(ISD::ATOMIC_LOAD, SL, NVT, DAG.getVTList(NVT, MVT::Other),
6141 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand());
6142 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
6144 break;
6145 }
6148 MVT ScalarType = Scalar.getSimpleValueType();
6152 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
6154 break;
6155 }
6156 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NewScalarType, Scalar);
6157 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
6161 break;
6162 }
6163 case ISD::VECREDUCE_FMAX:
6164 case ISD::VECREDUCE_FMIN:
6165 case ISD::VECREDUCE_FMAXIMUM:
6166 case ISD::VECREDUCE_FMINIMUM:
6167 case ISD::VP_REDUCE_FMAX:
6168 case ISD::VP_REDUCE_FMIN:
6169 case ISD::VP_REDUCE_FMAXIMUM:
6170 case ISD::VP_REDUCE_FMINIMUM:
6171 Results.push_back(PromoteReduction(Node));
6172 break;
6173 }
6174
6175
6177 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
6178 ReplaceNode(Node, Results.data());
6179 } else
6181}
6182
6183
6186
6188
6189
6190
6191
6193 *this,
6195
6196 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
6197
6198
6199
6200
6201
6202 while (true) {
6203 bool AnyLegalized = false;
6205 --NI;
6206
6209 ++NI;
6211 continue;
6212 }
6213
6214 if (LegalizedNodes.insert(N).second) {
6215 AnyLegalized = true;
6217
6219 ++NI;
6221 }
6222 }
6223 }
6224 if (!AnyLegalized)
6225 break;
6226
6227 }
6228
6229
6231}
6232
6236 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
6237
6238
6239
6240 LegalizedNodes.insert(N);
6242
6243 return LegalizedNodes.count(N);
6244}
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static bool isSigned(unsigned int Opcode)
Utilities for dealing with flags related to floating point properties and mode controls.
static MaybeAlign getAlign(Value *Ptr)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, const TargetLowering &TLI, SDValue &Res)
Definition LegalizeDAG.cpp:1904
static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI)
Return true if sincos or __sincos_stret libcall is available.
Definition LegalizeDAG.cpp:2420
static bool useSinCos(SDNode *Node)
Only issue sincos libcall if both sin and cos are needed.
Definition LegalizeDAG.cpp:2427
static bool canUseFastMathLibcall(const SDNode *Node)
Return if we can use the FAST_* variant of a math libcall for the node.
Definition LegalizeDAG.cpp:4642
static MachineMemOperand * getStackAlignedMMO(SDValue StackPtr, MachineFunction &MF, bool isObjectScalable)
Definition LegalizeDAG.cpp:276
static MVT getPromotedVectorElementType(const TargetLowering &TLI, MVT EltVT, MVT NewEltVT)
Definition LegalizeDAG.cpp:5378
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
Promote Memory to Register
PowerPC Reduce CR logical Operation
static constexpr MCPhysReg SPReg
const SmallVectorImpl< MachineOperand > & Cond
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
This file describes how to lower LLVM code to machine code.
static constexpr int Concat[]
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
APInt bitcastToAPInt() const
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const SDValue & getBasePtr() const
const SDValue & getVal() const
LLVM_ABI Type * getStructRetType() const
static LLVM_ABI bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
const ConstantFP * getConstantFPValue() const
const APFloat & getValueAPF() const
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static LLVM_ABI Constant * get(ArrayRef< Constant * > V)
bool isLittleEndian() const
Layout endianness...
LLVM_ABI Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
const BasicBlock & back() const
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
uint64_t getScalarSizeInBits() const
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
LLVM_ABI unsigned getEntrySize(const DataLayout &TD) const
getEntrySize - Return the size of each entry in the jump table.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOStore
The memory access writes data.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isStrictFPOpcode()
Test if this node is a strict floating point pseudo-op.
ArrayRef< SDUse > ops() const
LLVM_ABI void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
iterator_range< user_iterator > users()
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op)
Return the specified value casted to the target's desired shift amount type.
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
bool isKnownNeverSNaN(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
const TargetSubtargetInfo & getSubtarget() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
LLVM_ABI SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
LLVM_ABI SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
LLVM_ABI SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
LLVM_ABI bool shouldOptForSize() const
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
LLVM_ABI SDValue expandVACopy(SDNode *Node)
Expand the specified ISD::VACOPY node as the Legalize pass would.
allnodes_const_iterator allnodes_begin() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
allnodes_const_iterator allnodes_end() const
LLVM_ABI void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue expandVAArg(SDNode *Node)
Expand the specified ISD::VAARG node as the Legalize pass would.
LLVM_ABI void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
Definition LegalizeDAG.cpp:6184
LLVM_ABI SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
LLVM_ABI bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
Definition LegalizeDAG.cpp:6233
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
LLVM_ABI SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
LLVM_ABI SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
LLVM_ABI void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
LLVM_ABI SDValue makeStateFunctionCall(unsigned LibFunc, SDValue Ptr, SDValue InChain, const SDLoc &DLoc)
Helper used to make a call to a library function that has one argument of pointer type.
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
LLVM_ABI void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
LLVM_ABI std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
LLVM_ABI SDValue getVPLogicalNOT(const SDLoc &DL, SDValue Val, SDValue Mask, SDValue EVL, EVT VT)
Create a vector-predicated logical NOT operation as (VP_XOR Val, BooleanOne, Mask,...
LLVM_ABI SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
LLVM_ABI unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
const TargetLibraryInfo & getLibInfo() const
LLVM_ABI SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getPOISON(EVT VT)
Return a POISON node. POISON does not have a useful SDLoc.
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
LLVM_ABI SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
LLVM_ABI SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
LLVM_ABI SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
bool insert(const value_type &X)
Insert a new element into the SetVector.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
void swap(SmallVectorImpl &RHS)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
virtual bool shouldExpandBuildVectorWithShuffles(EVT, unsigned DefinedValues) const
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
bool isOperationLegalOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal using promotion.
LegalizeAction getCondCodeAction(ISD::CondCode CC, MVT VT) const
Return how the condition code should be treated: either it is legal, needs to be expanded to some oth...
CallingConv::ID getLibcallImplCallingConv(RTLIB::LibcallImpl Call) const
Get the CallingConv that should be used for the specified libcall implementation.
virtual bool isFPImmLegal(const APFloat &, EVT, bool ForCodeSize=false) const
Returns true if the target can instruction select the specified FP immediate natively.
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
bool isStrictFPEnabled() const
Return true if the target support strict float operation.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal for a comparison of the specified types on this ...
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
MVT getProgramPointerTy(const DataLayout &DL) const
Return the type for code pointers, which is determined by the program address space specified through...
virtual bool isJumpTableRelative() const
virtual bool ShouldShrinkFPConstant(EVT) const
If true, then instruction selection should seek to shrink the FP constant of the specified type to a ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
LegalizeAction getTruncStoreAction(EVT ValVT, EVT MemVT) const
Return how this store with truncation should be treated: either it is legal, needs to be promoted to ...
LegalizeAction getLoadExtAction(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return how this load with extension should be treated: either it is legal, needs to be promoted to a ...
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
virtual LegalizeAction getCustomOperationAction(SDNode &Op) const
How to legalize this custom operation?
bool isLoadExtLegalOrCustom(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal or custom on this target.
LegalizeAction getStrictFPOperationAction(unsigned Op, EVT VT) const
bool isLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal on this target.
RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const
Get the libcall impl routine name for the specified libcall.
virtual bool useSoftFloat() const
bool isTruncStoreLegalOrCustom(EVT ValVT, EVT MemVT) const
Return true if the specified store with truncation has solution on this target.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
virtual bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const
Returns true if arguments should be sign-extended in lib calls.
std::vector< ArgListEntry > ArgListTy
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
bool isCondCodeLegalOrCustom(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal or custom for a comparison of the specified type...
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
MVT getTypeToPromoteTo(unsigned Op, MVT VT) const
If the action for this operation is to promote, this method returns the ValueType to promote to.
const RTLIB::RuntimeLibcallsInfo & getRuntimeLibcallsInfo() const
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
bool expandMultipleResultFPLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, SDNode *Node, SmallVectorImpl< SDValue > &Results, std::optional< unsigned > CallRetResNo={}) const
Expands a node with multiple results to an FP or vector libcall.
bool expandMULO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]MULO.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand CTLZ/CTLZ_ZERO_UNDEF nodes.
SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand BITREVERSE nodes.
SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand CTTZ/CTTZ_ZERO_UNDEF nodes.
virtual SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const
Expands target specific indirect branch for the case of JumpTable expansion.
SDValue expandABD(SDNode *N, SelectionDAG &DAG) const
Expand ABDS/ABDU nodes.
SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]SHLSAT.
SDValue expandIS_FPCLASS(EVT ResultVT, SDValue Op, FPClassTest Test, SDNodeFlags Flags, const SDLoc &DL, SelectionDAG &DAG) const
Expand check for floating point class.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
void expandSADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::S(ADD|SUB)O.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue expandVPCTTZElements(SDNode *N, SelectionDAG &DAG) const
Expand VP_CTTZ_ELTS/VP_CTTZ_ELTS_ZERO_UNDEF nodes.
bool expandFP_TO_UINT(SDNode *N, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const
Expand float to UINT conversion.
bool expandREM(SDNode *Node, SDValue &Result, SelectionDAG &DAG) const
Expand an SREM or UREM using SDIV/UDIV or SDIVREM/UDIVREM, if legal.
std::pair< SDValue, SDValue > expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Expands an unaligned load to 2 half-size loads for an integer, and possibly more for vectors.
SDValue expandFMINIMUMNUM_FMAXIMUMNUM(SDNode *N, SelectionDAG &DAG) const
Expand fminimumnum/fmaximumnum into multiple comparison with selects.
SDValue expandVectorSplice(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::VECTOR_SPLICE.
SDValue getVectorSubVecPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, EVT SubVecVT, SDValue Index, const SDNodeFlags PtrArithFlags=SDNodeFlags()) const
Get a pointer to a sub-vector of type SubVecVT at index Idx located in memory for a vector of type Ve...
SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand CTPOP nodes.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand BSWAP nodes.
SDValue expandFMINIMUM_FMAXIMUM(SDNode *N, SelectionDAG &DAG) const
Expand fminimum/fmaximum into multiple comparison with selects.
bool expandFP_TO_SINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const
Expand float(f32) to SINT(i64) conversion.
virtual SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) const
Returns relocation base for the given PIC jumptable.
bool isInTailCallPosition(SelectionDAG &DAG, SDNode *Node, SDValue &Chain) const
Check whether a given call node is in tail position within its function.
SDValue expandFunnelShift(SDNode *N, SelectionDAG &DAG) const
Expand funnel shift.
bool LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, SDValue Mask, SDValue EVL, bool &NeedInvert, const SDLoc &dl, SDValue &Chain, bool IsSignaling=false) const
Legalize a SETCC or VP_SETCC with given LHS and RHS and condition code CC on the current target.
virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const
This callback is invoked for operations that are unsupported by the target, which are registered to u...
SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, SDValue LHS, SDValue RHS, unsigned Scale, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]DIVFIX[SAT].
SDValue expandFP_ROUND(SDNode *Node, SelectionDAG &DAG) const
Expand round(fp) to fp conversion.
SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const
Expand rotations.
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index, const SDNodeFlags PtrArithFlags=SDNodeFlags()) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
SDValue expandFMINNUM_FMAXNUM(SDNode *N, SelectionDAG &DAG) const
Expand fminnum/fmaxnum into fminnum_ieee/fmaxnum_ieee with quieted inputs.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandCMP(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]CMP.
SDValue expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[U|S]MULFIX[SAT].
void expandUADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::U(ADD|SUB)O.
bool expandUINT_TO_FP(SDNode *N, SDValue &Result, SDValue &Chain, SelectionDAG &DAG) const
Expand UINT(i64) to double(f64) conversion.
bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively,...
SDValue expandAVG(SDNode *N, SelectionDAG &DAG) const
Expand vector/scalar AVGCEILS/AVGCEILU/AVGFLOORS/AVGFLOORU nodes.
Primary interface to the complete machine description for the target machine.
const Triple & getTargetTriple() const
virtual const TargetFrameLowering * getFrameLowering() const
static constexpr TypeSize getFixed(ScalarTy ExactSize)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isVoidTy() const
Return true if this is 'void'.
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
LLVM Value Representation.
constexpr ScalarTy getFixedValue() const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ POISON
POISON - A poison node.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ FRAME_TO_ARGS_OFFSET
FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to first (possible) on-stack ar...
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ SSUBO
Same for subtraction.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor to...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ GLOBAL_OFFSET_TABLE
The address of the GOT.
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2, ...) - Returns N vectors from N input vectors, where N is the factor ...
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
LLVM_ABI NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
LLVM_ABI std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
LLVM_ABI CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
LLVM_ABI bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
LLVM_ABI Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSINCOSPI(EVT RetVT)
getSINCOSPI - Return the SINCOSPI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
LLVM_ABI Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getMODF(EVT VT)
getMODF - Return the MODF_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
LLVM_ABI Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSINCOS_STRET(EVT RetVT)
Return the SINCOS_STRET_ value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getSINCOS(EVT RetVT)
getSINCOS - Return the SINCOS_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Undef
Value of the register doesn't matter.
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
NodeAddr< NodeBase * > Node
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
FunctionAddr VTableAddr Value
decltype(auto) dyn_cast(const From &Val)
dyn_cast - Return the argument parameter cast to the specified type.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM)
Returns: X * 2^Exp for integral exponents.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa - Return true if the parameter to the template is an instance of one of the template type argu...
AtomicOrdering
Atomic ordering for LLVM's memory model.
To bit_cast(const From &from) noexcept
@ Or
Bitwise or logical OR of integers.
@ And
Bitwise or logical AND of integers.
@ Sub
Subtraction of integers.
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
decltype(auto) cast(const From &Val)
cast - Return the argument parameter cast to the specified type.
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
LLVM_ABI const fltSemantics & getFltSemantics() const
Returns an APFloat semantics tag appropriate for the value type.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static LLVM_ABI MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
CallingConv::ID getLibcallImplCallingConv(RTLIB::LibcallImpl Call) const
Get the CallingConv that should be used for the specified libcall.
std::pair< FunctionType *, AttributeList > getFunctionTy(LLVMContext &Ctx, const Triple &TT, const DataLayout &DL, RTLIB::LibcallImpl LibcallImpl) const
These are IR-level optimization flags that may be propagated to SDNodes.
void setNoFPExcept(bool b)
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
bool IsPostTypeLegalization
MakeLibCallOptions & setIsSigned(bool Value=true)