LLVM: lib/Target/AMDGPU/R600ISelLowering.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
13
23#include "llvm/IR/IntrinsicsAMDGPU.h"
24#include "llvm/IR/IntrinsicsR600.h"
26
27using namespace llvm;
28
29#include "R600GenCallingConv.inc"
30
34 Gen(STI.getGeneration()) {
41
44
46
47
49
50
51
57 }
58
59
62
65
66 setOperationAction(ISD::STORE, {MVT::i8, MVT::i32, MVT::v2i32, MVT::v4i32},
68
71
72
83
84
87
88
93
96
98
100
103
105
107 {MVT::f32, MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32,
108 MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v16f32},
110
111 setOperationAction({ISD::FCEIL, ISD::FTRUNC, ISD::FROUNDEVEN, ISD::FFLOOR},
113
115
119
122
123
124
125 if (Subtarget->hasCARRY())
127
128 if (Subtarget->hasBORROW())
130
131
132 if (!Subtarget->hasBFE())
134
136
137 if (!Subtarget->hasBFE())
140
141 if (!Subtarget->hasBFE())
144
147
149
151
153 {MVT::v2i32, MVT::v2f32, MVT::v4i32, MVT::v4f32}, Custom);
154
156 {MVT::v2i32, MVT::v2f32, MVT::v4i32, MVT::v4f32}, Custom);
157
158
159
162
163 if (!Subtarget->hasFMA())
165
166
168
169 if (!Subtarget->hasBFI())
170
172
173 if (!Subtarget->hasBCNT(32))
175
176 if (!Subtarget->hasBCNT(64))
178
179 if (Subtarget->hasFFBH())
181
182 if (Subtarget->hasFFBL())
184
185
186
187 if (Subtarget->hasBFE())
189
192
193 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
194 for (MVT VT : ScalarIntVTs)
197
198
199
201
202
205
207
210}
211
213 if (std::next(I) == I->getParent()->end())
214 return false;
215 return std::next(I)->getOpcode() == R600::RETURN;
216}
217
225
226 switch (MI.getOpcode()) {
227 default:
228
229
230 if (TII->isLDSRetInstr(MI.getOpcode())) {
231 int DstIdx = TII->getOperandIdx(MI.getOpcode(), R600::OpName::dst);
232 assert(DstIdx != -1);
234
235
236 if (.use_empty(MI.getOperand(DstIdx).getReg()) ||
237 MI.getOpcode() == R600::LDS_CMPST_RET)
238 return BB;
239
243 NewMI.add(MO);
244 } else {
246 }
247 break;
248
249 case R600::FABS_R600: {
251 *BB, I, R600::MOV, MI.getOperand(0).getReg(),
252 MI.getOperand(1).getReg());
254 break;
255 }
256
257 case R600::FNEG_R600: {
259 *BB, I, R600::MOV, MI.getOperand(0).getReg(),
260 MI.getOperand(1).getReg());
262 break;
263 }
264
265 case R600::MASK_WRITE: {
266 Register maskedRegister = MI.getOperand(0).getReg();
268 MachineInstr * defInstr = MRI.getVRegDef(maskedRegister);
270 break;
271 }
272
273 case R600::MOV_IMM_F32:
274 TII->buildMovImm(*BB, I, MI.getOperand(0).getReg(), MI.getOperand(1)
275 .getFPImm()
276 ->getValueAPF()
277 .bitcastToAPInt()
278 .getZExtValue());
279 break;
280
281 case R600::MOV_IMM_I32:
282 TII->buildMovImm(*BB, I, MI.getOperand(0).getReg(),
283 MI.getOperand(1).getImm());
284 break;
285
286 case R600::MOV_IMM_GLOBAL_ADDR: {
287
288 auto MIB = TII->buildDefaultInstruction(
289 *BB, MI, R600::MOV, MI.getOperand(0).getReg(), R600::ALU_LITERAL_X);
290 int Idx = TII->getOperandIdx(*MIB, R600::OpName::literal);
291
295 break;
296 }
297
298 case R600::CONST_COPY: {
300 *BB, MI, R600::MOV, MI.getOperand(0).getReg(), R600::ALU_CONST);
301 TII->setImmOperand(*NewMI, R600::OpName::src0_sel,
302 MI.getOperand(1).getImm());
303 break;
304 }
305
306 case R600::RAT_WRITE_CACHELESS_32_eg:
307 case R600::RAT_WRITE_CACHELESS_64_eg:
308 case R600::RAT_WRITE_CACHELESS_128_eg:
313 break;
314
315 case R600::RAT_STORE_TYPED_eg:
321 break;
322
323 case R600::BRANCH:
326 break;
327
328 case R600::BRANCH_COND_f32: {
331 R600::PREDICATE_BIT)
333 .addImm(R600::PRED_SETNE)
339 break;
340 }
341
342 case R600::BRANCH_COND_i32: {
345 R600::PREDICATE_BIT)
347 .addImm(R600::PRED_SETNE_INT)
353 break;
354 }
355
356 case R600::EG_ExportSwz:
357 case R600::R600_ExportSwz: {
358
359 bool isLastInstructionOfItsType = true;
360 unsigned InstExportType = MI.getOperand(1).getImm();
362 EndBlock = BB->end(); NextExportInst != EndBlock;
363 NextExportInst = std::next(NextExportInst)) {
364 if (NextExportInst->getOpcode() == R600::EG_ExportSwz ||
365 NextExportInst->getOpcode() == R600::R600_ExportSwz) {
366 unsigned CurrentInstExportType = NextExportInst->getOperand(1)
367 .getImm();
368 if (CurrentInstExportType == InstExportType) {
369 isLastInstructionOfItsType = false;
370 break;
371 }
372 }
373 }
375 if (!EOP && !isLastInstructionOfItsType)
376 return BB;
377 unsigned CfInst = (MI.getOpcode() == R600::EG_ExportSwz) ? 84 : 40;
388 break;
389 }
390 case R600::RETURN: {
391 return BB;
392 }
393 }
394
395 MI.eraseFromParent();
396 return BB;
397}
398
399
400
401
402
406 switch (Op.getOpcode()) {
415 case ISD::FCOS:
416 case ISD::FSIN: return LowerTrig(Op, DAG);
418 case ISD::STORE: return LowerSTORE(Op, DAG);
419 case ISD::LOAD: {
420 SDValue Result = LowerLOAD(Op, DAG);
421 assert((!Result.getNode() ||
422 Result.getNode()->getNumValues() == 2) &&
423 "Load should return a value and a chain");
424 return Result;
425 }
426
427 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
430 case ISD::ADDRSPACECAST:
431 return lowerADDRSPACECAST(Op, DAG);
433 SDValue Chain = Op.getOperand(0);
434 unsigned IntrinsicID = Op.getConstantOperandVal(1);
435 switch (IntrinsicID) {
436 case Intrinsic::r600_store_swizzle: {
438 const SDValue Args[8] = {
439 Chain,
440 Op.getOperand(2),
441 Op.getOperand(3),
442 Op.getOperand(4),
447 };
449 }
450
451
452 default: break;
453 }
454
455 break;
456 }
458 unsigned IntrinsicID = Op.getConstantOperandVal(0);
459 EVT VT = Op.getValueType();
461 switch (IntrinsicID) {
462 case Intrinsic::r600_tex:
463 case Intrinsic::r600_texc: {
464 unsigned TextureOp;
465 switch (IntrinsicID) {
466 case Intrinsic::r600_tex:
467 TextureOp = 0;
468 break;
469 case Intrinsic::r600_texc:
470 TextureOp = 1;
471 break;
472 default:
474 }
475
478 Op.getOperand(1),
483 Op.getOperand(2),
484 Op.getOperand(3),
485 Op.getOperand(4),
490 Op.getOperand(5),
491 Op.getOperand(6),
492 Op.getOperand(7),
493 Op.getOperand(8),
494 Op.getOperand(9),
495 Op.getOperand(10)
496 };
498 }
499 case Intrinsic::r600_dot4: {
517 };
519 }
520
521 case Intrinsic::r600_implicitarg_ptr: {
525 }
526 case Intrinsic::r600_read_ngroups_x:
527 return LowerImplicitParameter(DAG, VT, DL, 0);
528 case Intrinsic::r600_read_ngroups_y:
529 return LowerImplicitParameter(DAG, VT, DL, 1);
530 case Intrinsic::r600_read_ngroups_z:
531 return LowerImplicitParameter(DAG, VT, DL, 2);
532 case Intrinsic::r600_read_global_size_x:
533 return LowerImplicitParameter(DAG, VT, DL, 3);
534 case Intrinsic::r600_read_global_size_y:
535 return LowerImplicitParameter(DAG, VT, DL, 4);
536 case Intrinsic::r600_read_global_size_z:
537 return LowerImplicitParameter(DAG, VT, DL, 5);
538 case Intrinsic::r600_read_local_size_x:
539 return LowerImplicitParameter(DAG, VT, DL, 6);
540 case Intrinsic::r600_read_local_size_y:
541 return LowerImplicitParameter(DAG, VT, DL, 7);
542 case Intrinsic::r600_read_local_size_z:
543 return LowerImplicitParameter(DAG, VT, DL, 8);
544
545 case Intrinsic::r600_read_tgid_x:
546 case Intrinsic::amdgcn_workgroup_id_x:
548 R600::T1_X, VT);
549 case Intrinsic::r600_read_tgid_y:
550 case Intrinsic::amdgcn_workgroup_id_y:
552 R600::T1_Y, VT);
553 case Intrinsic::r600_read_tgid_z:
554 case Intrinsic::amdgcn_workgroup_id_z:
556 R600::T1_Z, VT);
557 case Intrinsic::r600_read_tidig_x:
558 case Intrinsic::amdgcn_workitem_id_x:
560 R600::T0_X, VT);
561 case Intrinsic::r600_read_tidig_y:
562 case Intrinsic::amdgcn_workitem_id_y:
564 R600::T0_Y, VT);
565 case Intrinsic::r600_read_tidig_z:
566 case Intrinsic::amdgcn_workitem_id_z:
568 R600::T0_Z, VT);
569
570 case Intrinsic::r600_recipsqrt_ieee:
571 return DAG.getNode(AMDGPUISD::RSQ, DL, VT, Op.getOperand(1));
572
573 case Intrinsic::r600_recipsqrt_clamped:
574 return DAG.getNode(AMDGPUISD::RSQ_CLAMP, DL, VT, Op.getOperand(1));
575 default:
576 return Op;
577 }
578
579
580 break;
581 }
582 }
584}
585
589 switch (N->getOpcode()) {
590 default:
592 return;
594 if (N->getValueType(0) == MVT::i1) {
595 Results.push_back(lowerFP_TO_UINT(N->getOperand(0), DAG));
596 return;
597 }
598
599
600
601 [[fallthrough]];
603 if (N->getValueType(0) == MVT::i1) {
604 Results.push_back(lowerFP_TO_SINT(N->getOperand(0), DAG));
605 return;
606 }
607
610 Results.push_back(Result);
611 return;
612 }
618 break;
619 }
623 break;
624 }
625 }
626}
627
631 EVT VecVT = Vector.getValueType();
634
638 }
639
641}
642
643SDValue R600TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
648
651 return Op;
652
653 Vector = vectorToVerticalVector(DAG, Vector);
656}
657
658SDValue R600TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
664
667 return Op;
668
669 Vector = vectorToVerticalVector(DAG, Vector);
672 return vectorToVerticalVector(DAG, Insert);
673}
674
681
683 const GlobalValue *GV = GSD->getGlobal();
685
687 return DAG.getNode(AMDGPUISD::CONST_DATA_PTR, SDLoc(GSD), ConstPtrVT, GA);
688}
689
691
692
693 EVT VT = Op.getValueType();
696
697
703 unsigned TrigNode;
704 switch (Op.getOpcode()) {
705 case ISD::FCOS:
706 TrigNode = AMDGPUISD::COS_HW;
707 break;
708 case ISD::FSIN:
709 TrigNode = AMDGPUISD::SIN_HW;
710 break;
711 default:
713 }
718 return TrigVal;
719
722}
723
729}
730
732 unsigned mainop, unsigned ovf) const {
734 EVT VT = Op.getValueType();
735
738
740
743
745
747}
748
754 MVT::i1,
757}
758
764 MVT::i1,
767}
768
771 unsigned DwordOffset) const {
772 unsigned ByteOffset = DwordOffset * 4;
775
776
778
780 DAG.getConstant(ByteOffset, DL, MVT::i32),
782}
783
784bool R600TargetLowering::isZero(SDValue Op) const {
786 return Cst->isZero();
788 return CstFP->isZero();
789 return false;
790}
791
792bool R600TargetLowering::isHWTrueValue(SDValue Op) const {
794 return CFP->isExactlyValue(1.0);
795 }
797}
798
799bool R600TargetLowering::isHWFalseValue(SDValue Op) const {
801 return CFP->getValueAPF().isZero();
802 }
804}
805
808 EVT VT = Op.getValueType();
809
813 SDValue False = Op.getOperand(3);
816
817 if (VT == MVT::f32) {
820 if (MinMax)
821 return MinMax;
822 }
823
824
825 EVT CompareVT = LHS.getValueType();
826
827
828
829
830
831
832
833
834
835
836
837
838
839 if (isHWTrueValue(False) && isHWFalseValue(True)) {
845 } else {
851 }
852 }
853 }
854
855 if (isHWTrueValue(True) && isHWFalseValue(False) &&
856 (CompareVT == VT || VT == MVT::i32)) {
857
859 }
860
861
862
863
864
865
866
867
868
869
870
871
872 if (isZero(LHS)) {
874
879 } else {
880
887 }
888 }
889 }
890 if (isZero(RHS)) {
894 if (CompareVT != VT) {
895
896
897
898
899 True = DAG.getNode(ISD::BITCAST, DL, CompareVT, True);
900 False = DAG.getNode(ISD::BITCAST, DL, CompareVT, False);
901 }
902
903 switch (CCOpcode) {
908 Temp = True;
909 True = False;
910 False = Temp;
911 break;
912 default:
913 break;
914 }
917 True, False,
919 return DAG.getNode(ISD::BITCAST, DL, VT, SelectNode);
920 }
921
922
923
925
926 if (CompareVT == MVT::f32) {
929 } else if (CompareVT == MVT::i32) {
932 }
933 else {
935 }
936
937
938
940
942 Cond, HWFalse,
943 True, False,
945}
946
949 SDLoc SL(Op);
950 EVT VT = Op.getValueType();
951
952 const R600TargetMachine &TM =
954
958
960 return DAG.getSignedConstant(TM.getNullPointerValue(DestAS), SL, VT);
961
962 return Op;
963}
964
965
966
967
968
969
970SDValue R600TargetLowering::stackPtrToRegIndex(SDValue Ptr,
971 unsigned StackWidth,
973 unsigned SRLPad;
974 switch(StackWidth) {
975 case 1:
976 SRLPad = 2;
977 break;
978 case 2:
979 SRLPad = 3;
980 break;
981 case 4:
982 SRLPad = 4;
983 break;
985 }
986
987 SDLoc DL(Ptr);
990}
991
992void R600TargetLowering::getStackAddress(unsigned StackWidth,
993 unsigned ElemIdx,
994 unsigned &Channel,
995 unsigned &PtrIncr) const {
996 switch (StackWidth) {
997 default:
998 case 1:
999 Channel = 0;
1000 if (ElemIdx > 0) {
1001 PtrIncr = 1;
1002 } else {
1003 PtrIncr = 0;
1004 }
1005 break;
1006 case 2:
1007 Channel = ElemIdx % 2;
1008 if (ElemIdx == 2) {
1009 PtrIncr = 1;
1010 } else {
1011 PtrIncr = 0;
1012 }
1013 break;
1014 case 4:
1015 Channel = ElemIdx;
1016 PtrIncr = 0;
1017 break;
1018 }
1019}
1020
1021SDValue R600TargetLowering::lowerPrivateTruncStore(StoreSDNode *Store,
1023 SDLoc DL(Store);
1024
1026 || Store->getValue().getValueType() == MVT::i8);
1028
1030 if (Store->getMemoryVT() == MVT::i8) {
1033 } else if (Store->getMemoryVT() == MVT::i16) {
1036 } else {
1038 }
1039
1042
1043 SDValue Chain = VectorTrunc ? OldChain->getOperand(0) : OldChain;
1046 EVT MemVT = Store->getMemoryVT();
1047
1049 if (.isUndef()) {
1051 }
1052
1053
1054
1057
1058
1059
1061 SDValue Dst = DAG.getLoad(MVT::i32, DL, Chain, Ptr, PtrInfo);
1062
1063 Chain = Dst.getValue(1);
1064
1065
1068
1069
1072
1073
1074
1076 Store->getValue());
1077
1078
1080
1081
1083 MaskedValue, ShiftAmt);
1084
1085
1087
1088
1089
1090 DstMask = DAG.getNOT(DL, DstMask, MVT::i32);
1091
1092
1094
1095
1097
1098
1099
1101
1102
1103 if (VectorTrunc) {
1104
1107 }
1108 return NewStore;
1109}
1110
1114
1118
1119 EVT VT = Value.getValueType();
1122
1124
1126
1127
1129 TruncatingStore) &&
1132
1135 NewOps[0] = NewChain;
1137 }
1138
1140 }
1141
1146 nullptr)) {
1148 }
1149
1152
1154
1155
1156 if (TruncatingStore) {
1159 if (MemVT == MVT::i8) {
1160 MaskConstant = DAG.getConstant(0xFF, DL, MVT::i32);
1161 } else {
1162 assert(MemVT == MVT::i16);
1164 MaskConstant = DAG.getConstant(0xFFFF, DL, MVT::i32);
1165 }
1166
1171
1172
1174
1175
1178
1179
1180
1182 ShiftedValue,
1186 };
1188 SDValue Args[3] = { Chain, Input, DWordAddr };
1190 Op->getVTList(), Args, MemVT,
1192 }
1193 if (Ptr->getOpcode() != AMDGPUISD::DWORDADDR && VT.bitsGE(MVT::i32)) {
1194
1195 Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, PtrVT, DWordAddr);
1196
1199 } else {
1201 }
1202 return Chain;
1203 }
1204 }
1205
1206
1209
1210 if (MemVT.bitsLT(MVT::i32))
1211 return lowerPrivateTruncStore(StoreNode, DAG);
1212
1213
1214
1215 if (Ptr.getOpcode() != AMDGPUISD::DWORDADDR) {
1216 Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, PtrVT, DWordAddr);
1218 }
1219
1220
1222}
1223
1224
1225static int
1229 return 512;
1231 return 512 + 4096;
1233 return 512 + 4096 * 2;
1235 return 512 + 4096 * 3;
1237 return 512 + 4096 * 4;
1239 return 512 + 4096 * 5;
1241 return 512 + 4096 * 6;
1243 return 512 + 4096 * 7;
1245 return 512 + 4096 * 8;
1247 return 512 + 4096 * 9;
1249 return 512 + 4096 * 10;
1251 return 512 + 4096 * 11;
1253 return 512 + 4096 * 12;
1255 return 512 + 4096 * 13;
1257 return 512 + 4096 * 14;
1259 return 512 + 4096 * 15;
1260 default:
1261 return -1;
1262 }
1263}
1264
1270 EVT MemVT = Load->getMemoryVT();
1272
1276
1278 if (.isUndef()) {
1280 }
1281
1282
1283
1286
1287
1288
1291
1292
1295
1296
1299
1300
1302
1303
1305
1309 } else {
1311 }
1312
1314 Ret,
1316 };
1317
1319}
1320
1326
1329 return lowerPrivateExtLoad(Op, DAG);
1330 }
1331
1333 EVT VT = Op.getValueType();
1336
1343 }
1344
1345
1347 if (ConstantBlock > -1 &&
1353 return constBufferLoad(LoadNode, LoadNode->getAddressSpace(), DAG);
1354 }
1355
1356
1362 DL, MVT::i32));
1363
1367 }
1368
1369 SDValue MergedValues[2] = {
1371 Chain
1372 };
1374 }
1375
1376
1377
1378
1379
1380
1381
1382
1384 assert(!MemVT.isVector() && (MemVT == MVT::i16 || MemVT == MVT::i8));
1390
1391 SDValue MergedValues[2] = { Res, Chain };
1393 }
1394
1397 }
1398
1399
1400 if (Ptr.getOpcode() != AMDGPUISD::DWORDADDR) {
1401 assert(VT == MVT::i32);
1403 Ptr = DAG.getNode(AMDGPUISD::DWORDADDR, DL, MVT::i32, Ptr);
1405 }
1407}
1408
1410 SDValue Chain = Op.getOperand(0);
1412 SDValue Jump = Op.getOperand(2);
1413
1414 return DAG.getNode(AMDGPUISD::BRANCH_COND, SDLoc(Op), Op.getValueType(),
1415 Chain, Jump, Cond);
1416}
1417
1421 const R600FrameLowering *TFL = Subtarget->getFrameLowering();
1422
1424
1427 StackOffset Offset =
1430 SDLoc(Op), Op.getValueType());
1431}
1432
1434 bool IsVarArg) const {
1435 switch (CC) {
1449 return CC_R600;
1450 default:
1452 }
1453}
1454
1455
1456
1457
1466
1469 } else {
1471 }
1472
1473 for (unsigned i = 0, e = Ins.size(); i < e; ++i) {
1476 EVT VT = In.VT;
1479
1481 }
1482
1487 continue;
1488 }
1489
1490
1491
1492
1493
1494
1495
1496
1499
1500
1501
1502
1504 }
1505
1506
1507
1508
1509
1512
1517 PtrInfo,
1521
1523 }
1524 return Chain;
1525}
1526
1528 EVT VT) const {
1530 return MVT::i32;
1532}
1533
1536
1539 }
1540 return true;
1541}
1542
1545 unsigned *IsFast) const {
1546 if (IsFast)
1547 *IsFast = 0;
1548
1549 if (!VT.isSimple() || VT == MVT::Other)
1550 return false;
1551
1552 if (VT.bitsLT(MVT::i32))
1553 return false;
1554
1555
1556 if (IsFast)
1557 *IsFast = 1;
1558
1559 return VT.bitsGT(MVT::i32) && Alignment >= Align(4);
1560}
1561
1566
1569
1571 for (unsigned i = 0; i < 4; i++)
1574
1575 for (unsigned i = 0; i < 4; i++) {
1576 if (NewBldVec[i].isUndef())
1577
1578
1579
1580 RemapSwizzle[i] = 7;
1582 if (C->isZero()) {
1583 RemapSwizzle[i] = 4;
1584 NewBldVec[i] = DAG.getUNDEF(MVT::f32);
1585 } else if (C->isExactlyValue(1.0)) {
1586 RemapSwizzle[i] = 5;
1587 NewBldVec[i] = DAG.getUNDEF(MVT::f32);
1588 }
1589 }
1590
1591 if (NewBldVec[i].isUndef())
1592 continue;
1593
1594 for (unsigned j = 0; j < i; j++) {
1595 if (NewBldVec[i] == NewBldVec[j]) {
1597 RemapSwizzle[i] = j;
1598 break;
1599 }
1600 }
1601 }
1602
1604 NewBldVec);
1605}
1606
1610
1613
1615 bool isUnmovable[4] = {false, false, false, false};
1616 for (unsigned i = 0; i < 4; i++)
1619
1620 for (unsigned i = 0; i < 4; i++) {
1621 RemapSwizzle[i] = i;
1624 if (i == Idx)
1625 isUnmovable[Idx] = true;
1626 }
1627 }
1628
1629 for (unsigned i = 0; i < 4; i++) {
1632 if (isUnmovable[Idx])
1633 continue;
1634
1635 std::swap(NewBldVec[Idx], NewBldVec[i]);
1636 std::swap(RemapSwizzle[i], RemapSwizzle[Idx]);
1637 break;
1638 }
1639 }
1640
1642 NewBldVec);
1643}
1644
1648
1649 DenseMap<unsigned, unsigned> SwizzleRemap;
1650
1652 for (unsigned i = 0; i < 4; i++) {
1654 auto It = SwizzleRemap.find(Idx);
1655 if (It != SwizzleRemap.end())
1656 Swz[i] = DAG.getConstant(It->second, DL, MVT::i32);
1657 }
1658
1659 SwizzleRemap.clear();
1660 BuildVector = ReorganizeVector(DAG, BuildVector, SwizzleRemap);
1661 for (unsigned i = 0; i < 4; i++) {
1663 auto It = SwizzleRemap.find(Idx);
1664 if (It != SwizzleRemap.end())
1665 Swz[i] = DAG.getConstant(It->second, DL, MVT::i32);
1666 }
1667
1668 return BuildVector;
1669}
1670
1673 SDLoc DL(LoadNode);
1678
1679
1682
1685
1687
1689 for (unsigned i = 0; i < 4; i++) {
1690
1691
1692
1693
1694
1696 DAG.getConstant(4 * i + ConstantBlock * 16, DL, MVT::i32));
1698 }
1699 EVT NewVT = MVT::v4i32;
1700 unsigned NumElements = 4;
1702 NewVT = VT;
1704 }
1709 }
1710 SDValue MergedValues[2] = {
1712 Chain
1713 };
1715}
1716
1717
1718
1719
1720
1725
1726 switch (N->getOpcode()) {
1727
1729 SDValue Arg = N->getOperand(0);
1733 }
1734 break;
1735 }
1736
1737
1738
1739
1740
1741
1743 SDValue FNeg = N->getOperand(0);
1744 if (FNeg.getOpcode() != ISD::FNEG) {
1746 }
1751 !isHWTrueValue(SelectCC.getOperand(2)) ||
1752 !isHWFalseValue(SelectCC.getOperand(3))) {
1754 }
1755
1762 }
1763
1764
1765
1767 SDValue InVec = N->getOperand(0);
1768 SDValue InVal = N->getOperand(1);
1769 SDValue EltNo = N->getOperand(2);
1770
1771
1773 return InVec;
1774
1776
1777
1780
1781
1785
1786
1787
1788
1793 } else if (InVec.isUndef()) {
1796 } else {
1798 }
1799
1800
1801 if (Elt < Ops.size()) {
1802
1803
1804 EVT OpVT = Ops[0].getValueType();
1809 Ops[Elt] = InVal;
1810 }
1811
1812
1814 }
1815
1816
1817
1819 SDValue Arg = N->getOperand(0);
1822 unsigned Element = Const->getZExtValue();
1824 }
1825 }
1826 if (Arg.getOpcode() == ISD::BITCAST &&
1831 unsigned Element = Const->getZExtValue();
1832 return DAG.getNode(ISD::BITCAST, DL, N->getVTList(),
1834 }
1835 }
1836 break;
1837 }
1838
1840
1842 return Ret;
1843
1844
1845
1846
1847
1848
1849 SDValue LHS = N->getOperand(0);
1852 }
1853
1854 SDValue RHS = N->getOperand(1);
1855 SDValue True = N->getOperand(2);
1856 SDValue False = N->getOperand(3);
1858
1859 if (LHS.getOperand(2).getNode() != True.getNode() ||
1861 RHS.getNode() != False.getNode()) {
1863 }
1864
1865 switch (NCC) {
1866 default: return SDValue();
1872 isCondCodeLegal(LHSCC, LHS.getOperand(0).getSimpleValueType()))
1874 LHS.getOperand(0),
1875 LHS.getOperand(1),
1876 LHS.getOperand(2),
1877 LHS.getOperand(3),
1878 LHSCC);
1879 break;
1880 }
1881 }
1883 }
1884
1886 SDValue Arg = N->getOperand(1);
1888 break;
1889
1891 N->getOperand(0),
1893 N->getOperand(2),
1894 N->getOperand(3),
1895 N->getOperand(4),
1896 N->getOperand(5),
1897 N->getOperand(6),
1898 N->getOperand(7)
1899 };
1900 NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[4], DAG, DL);
1902 }
1904 SDValue Arg = N->getOperand(1);
1906 break;
1907
1909 N->getOperand(0),
1910 N->getOperand(1),
1911 N->getOperand(2),
1912 N->getOperand(3),
1913 N->getOperand(4),
1914 N->getOperand(5),
1915 N->getOperand(6),
1916 N->getOperand(7),
1917 N->getOperand(8),
1918 N->getOperand(9),
1919 N->getOperand(10),
1920 N->getOperand(11),
1921 N->getOperand(12),
1922 N->getOperand(13),
1923 N->getOperand(14),
1924 N->getOperand(15),
1925 N->getOperand(16),
1926 N->getOperand(17),
1927 N->getOperand(18),
1928 };
1929 NewArgs[1] = OptimizeSwizzle(N->getOperand(1), &NewArgs[2], DAG, DL);
1931 }
1932
1933 case ISD::LOAD: {
1939 break;
1940 }
1941
1942 default: break;
1943 }
1944
1946}
1947
1948bool R600TargetLowering::FoldOperand(SDNode *ParentNode, unsigned SrcIdx,
1953 if (!Src.isMachineOpcode())
1954 return false;
1955
1956 switch (Src.getMachineOpcode()) {
1957 case R600::FNEG_R600:
1959 return false;
1960 Src = Src.getOperand(0);
1962 return true;
1963 case R600::FABS_R600:
1965 return false;
1968 return true;
1969 case R600::CONST_COPY: {
1971 bool HasDst = TII->getOperandIdx(Opcode, R600::OpName::dst) > -1;
1972
1974 return false;
1975
1978 return false;
1979
1980
1981 int SrcIndices[] = {
1982 TII->getOperandIdx(Opcode, R600::OpName::src0),
1983 TII->getOperandIdx(Opcode, R600::OpName::src1),
1984 TII->getOperandIdx(Opcode, R600::OpName::src2),
1985 TII->getOperandIdx(Opcode, R600::OpName::src0_X),
1986 TII->getOperandIdx(Opcode, R600::OpName::src0_Y),
1987 TII->getOperandIdx(Opcode, R600::OpName::src0_Z),
1988 TII->getOperandIdx(Opcode, R600::OpName::src0_W),
1989 TII->getOperandIdx(Opcode, R600::OpName::src1_X),
1990 TII->getOperandIdx(Opcode, R600::OpName::src1_Y),
1991 TII->getOperandIdx(Opcode, R600::OpName::src1_Z),
1992 TII->getOperandIdx(Opcode, R600::OpName::src1_W)
1993 };
1994 std::vector Consts;
1995 for (int OtherSrcIdx : SrcIndices) {
1996 int OtherSelIdx = TII->getSelIdx(Opcode, OtherSrcIdx);
1997 if (OtherSrcIdx < 0 || OtherSelIdx < 0)
1998 continue;
1999 if (HasDst) {
2000 OtherSrcIdx--;
2001 OtherSelIdx--;
2002 }
2003 if (RegisterSDNode *Reg =
2005 if (Reg->getReg() == R600::ALU_CONST) {
2007 }
2008 }
2009 }
2010
2013 if (->fitsConstReadLimitations(Consts)) {
2014 return false;
2015 }
2016
2017 Sel = CstOffset;
2018 Src = DAG.getRegister(R600::ALU_CONST, MVT::f32);
2019 return true;
2020 }
2021 case R600::MOV_IMM_GLOBAL_ADDR:
2022
2023 if (Imm->getAsZExtVal())
2024 return false;
2025 Imm = Src.getOperand(0);
2026 Src = DAG.getRegister(R600::ALU_LITERAL_X, MVT::i32);
2027 return true;
2028 case R600::MOV_IMM_I32:
2029 case R600::MOV_IMM_F32: {
2030 unsigned ImmReg = R600::ALU_LITERAL_X;
2031 uint64_t ImmValue = 0;
2032
2033 if (Src.getMachineOpcode() == R600::MOV_IMM_F32) {
2036 if (FloatValue == 0.0) {
2037 ImmReg = R600::ZERO;
2038 } else if (FloatValue == 0.5) {
2039 ImmReg = R600::HALF;
2040 } else if (FloatValue == 1.0) {
2041 ImmReg = R600::ONE;
2042 } else {
2044 }
2045 } else {
2046 uint64_t Value = Src.getConstantOperandVal(0);
2047 if (Value == 0) {
2048 ImmReg = R600::ZERO;
2049 } else if (Value == 1) {
2050 ImmReg = R600::ONE_INT;
2051 } else {
2052 ImmValue = Value;
2053 }
2054 }
2055
2056
2057
2058
2059 if (ImmReg == R600::ALU_LITERAL_X) {
2060 if (.getNode())
2061 return false;
2063 if (C->getZExtValue())
2064 return false;
2066 }
2067 Src = DAG.getRegister(ImmReg, MVT::i32);
2068 return true;
2069 }
2070 default:
2071 return false;
2072 }
2073}
2074
2075
2078 const R600InstrInfo *TII = Subtarget->getInstrInfo();
2079 if (->isMachineOpcode())
2080 return Node;
2081
2082 unsigned Opcode = Node->getMachineOpcode();
2084
2085 std::vector Ops(Node->op_begin(), Node->op_end());
2086
2087 if (Opcode == R600::DOT_4) {
2088 int OperandIdx[] = {
2089 TII->getOperandIdx(Opcode, R600::OpName::src0_X),
2090 TII->getOperandIdx(Opcode, R600::OpName::src0_Y),
2091 TII->getOperandIdx(Opcode, R600::OpName::src0_Z),
2092 TII->getOperandIdx(Opcode, R600::OpName::src0_W),
2093 TII->getOperandIdx(Opcode, R600::OpName::src1_X),
2094 TII->getOperandIdx(Opcode, R600::OpName::src1_Y),
2095 TII->getOperandIdx(Opcode, R600::OpName::src1_Z),
2096 TII->getOperandIdx(Opcode, R600::OpName::src1_W)
2097 };
2098 int NegIdx[] = {
2099 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_X),
2100 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_Y),
2101 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_Z),
2102 TII->getOperandIdx(Opcode, R600::OpName::src0_neg_W),
2103 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_X),
2104 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_Y),
2105 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_Z),
2106 TII->getOperandIdx(Opcode, R600::OpName::src1_neg_W)
2107 };
2108 int AbsIdx[] = {
2109 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_X),
2110 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_Y),
2111 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_Z),
2112 TII->getOperandIdx(Opcode, R600::OpName::src0_abs_W),
2113 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_X),
2114 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_Y),
2115 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_Z),
2116 TII->getOperandIdx(Opcode, R600::OpName::src1_abs_W)
2117 };
2118 for (unsigned i = 0; i < 8; i++) {
2119 if (OperandIdx[i] < 0)
2120 return Node;
2121 SDValue &Src = Ops[OperandIdx[i] - 1];
2124 bool HasDst = TII->getOperandIdx(Opcode, R600::OpName::dst) > -1;
2125 int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]);
2126 if (HasDst)
2127 SelIdx--;
2128 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp;
2129 if (FoldOperand(Node, i, Src, Neg, Abs, Sel, FakeOp, DAG))
2131 }
2132 } else if (Opcode == R600::REG_SEQUENCE) {
2133 for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2) {
2135 if (FoldOperand(Node, i, Src, FakeOp, FakeOp, FakeOp, FakeOp, DAG))
2137 }
2138 } else {
2139 if (->hasInstrModifiers(Opcode))
2140 return Node;
2141 int OperandIdx[] = {
2142 TII->getOperandIdx(Opcode, R600::OpName::src0),
2143 TII->getOperandIdx(Opcode, R600::OpName::src1),
2144 TII->getOperandIdx(Opcode, R600::OpName::src2)
2145 };
2146 int NegIdx[] = {
2147 TII->getOperandIdx(Opcode, R600::OpName::src0_neg),
2148 TII->getOperandIdx(Opcode, R600::OpName::src1_neg),
2149 TII->getOperandIdx(Opcode, R600::OpName::src2_neg)
2150 };
2151 int AbsIdx[] = {
2152 TII->getOperandIdx(Opcode, R600::OpName::src0_abs),
2153 TII->getOperandIdx(Opcode, R600::OpName::src1_abs),
2154 -1
2155 };
2156 for (unsigned i = 0; i < 3; i++) {
2157 if (OperandIdx[i] < 0)
2158 return Node;
2159 SDValue &Src = Ops[OperandIdx[i] - 1];
2162 SDValue &Abs = (AbsIdx[i] > -1) ? Ops[AbsIdx[i] - 1] : FakeAbs;
2163 bool HasDst = TII->getOperandIdx(Opcode, R600::OpName::dst) > -1;
2164 int SelIdx = TII->getSelIdx(Opcode, OperandIdx[i]);
2165 int ImmIdx = TII->getOperandIdx(Opcode, R600::OpName::literal);
2166 if (HasDst) {
2167 SelIdx--;
2168 ImmIdx--;
2169 }
2170 SDValue &Sel = (SelIdx > -1) ? Ops[SelIdx] : FakeOp;
2172 if (FoldOperand(Node, i, Src, Neg, Abs, Sel, Imm, DAG))
2174 }
2175 }
2176
2177 return Node;
2178}
2179
2181R600TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
2193
2194
2198 unsigned ValSize = DL.getTypeSizeInBits(RMW->getType());
2199 if (ValSize == 32 || ValSize == 64)
2202 }
2203 default:
2205 unsigned Size = IntTy->getBitWidth();
2206 if (Size == 32 || Size == 64)
2208 }
2209
2211 }
2212
2214}
unsigned const MachineRegisterInfo * MRI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const TargetInstrInfo & TII
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
Interfaces for producing common pass manager configurations.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool isUndef(const MachineInstr &MI)
Promote Memory to Register
static bool isEOP(MachineBasicBlock::iterator I)
Definition R600ISelLowering.cpp:212
static SDValue ReorganizeVector(SelectionDAG &DAG, SDValue VectorEntry, DenseMap< unsigned, unsigned > &RemapSwizzle)
Definition R600ISelLowering.cpp:1607
static int ConstantAddressBlock(unsigned AddressSpace)
Definition R600ISelLowering.cpp:1226
static SDValue CompactSwizzlableVector(SelectionDAG &DAG, SDValue VectorEntry, DenseMap< unsigned, unsigned > &RemapSwizzle)
Definition R600ISelLowering.cpp:1562
R600 DAG Lowering interface definition.
Provides R600 specific target descriptions.
AMDGPU R600 specific subclass of TargetSubtarget.
The AMDGPU TargetMachine interface definition for hw codegen targets.
const SmallVectorImpl< MachineOperand > & Cond
static Type * getValueType(Value *V)
Returns the type of the given value/instruction V.
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
unsigned getStackWidth(const MachineFunction &MF) const
SDValue combineFMinMaxLegacy(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, SDValue False, SDValue CC, DAGCombinerInfo &DCI) const
Generate Min/Max node.
void analyzeFormalArgumentsCompute(CCState &State, const SmallVectorImpl< ISD::InputArg > &Ins) const
The SelectionDAGBuilder will automatically promote function arguments with illegal types.
SDValue LowerSDIVREM(SDValue Op, SelectionDAG &DAG) const
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
uint32_t getImplicitParameterOffset(const MachineFunction &MF, const ImplicitParameter Param) const
Helper function that returns the byte offset of the given type of implicit parameter.
virtual SDValue LowerGlobalAddress(AMDGPUMachineFunction *MFI, SDValue Op, SelectionDAG &DAG) const
SDValue CreateLiveInRegisterRaw(SelectionDAG &DAG, const TargetRegisterClass *RC, Register Reg, EVT VT) const
AMDGPUTargetLowering(const TargetMachine &TM, const TargetSubtargetInfo &STI, const AMDGPUSubtarget &AMDGPUSTI)
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
void LowerUDIVREM64(SDValue Op, SelectionDAG &DAG, SmallVectorImpl< SDValue > &Results) const
LLVM_ABI float convertToFloat() const
Converts this APFloat to host float value.
APInt bitcastToAPInt() const
uint64_t getZExtValue() const
Get zero extended value.
unsigned getSrcAddressSpace() const
unsigned getDestAddressSpace() const
an instruction that atomically reads a memory location, combines it with another value,...
@ USubCond
Subtract only if no unsigned overflow.
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ UIncWrap
Increment one up to a maximum value.
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
BinOp getOperation() const
CCState - This class holds information needed while lowering arguments and return values.
LLVM_ABI void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
int64_t getLocMemOffset() const
const APFloat & getValueAPF() const
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
uint64_t getZExtValue() const
A parsed version of the target data layout string in and methods for querying it.
iterator find(const_arg_type_t< KeyT > Val)
const DataLayout & getDataLayout() const
Get the data layout of the module this function belongs to.
LLVM_ABI unsigned getAddressSpace() const
const GlobalValue * getGlobal() const
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
This is an important class for using LLVM in a threaded context.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
static auto integer_valuetypes()
LLVM_ABI DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
Flags
Flags values. These may be or'd together.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
Flags getFlags() const
Return the raw flags of the source value,.
const Value * getValue() const
Return the base address of the memory access.
MachineOperand class - Representation of each machine instruction operand.
const GlobalValue * getGlobal() const
unsigned getTargetFlags() const
int64_t getOffset() const
Return the offset from the symbol in this operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
An SDNode that represents everything that will be needed to construct a MachineInstr.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
static LLVM_ABI PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
StackOffset getFrameIndexReference(const MachineFunction &MF, int FI, Register &FrameReg) const override
const R600InstrInfo * getInstrInfo() const override
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
Definition R600ISelLowering.cpp:1721
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Definition R600ISelLowering.cpp:219
bool canMergeStoresTo(unsigned AS, EVT MemVT, const MachineFunction &MF) const override
Returns if it's reasonable to merge stores to MemVT size.
Definition R600ISelLowering.cpp:1534
R600TargetLowering(const TargetMachine &TM, const R600Subtarget &STI)
Definition R600ISelLowering.cpp:31
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &, EVT VT) const override
Return the ValueType of the result of SETCC operations.
Definition R600ISelLowering.cpp:1527
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
Definition R600ISelLowering.cpp:586
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *IsFast=nullptr) const override
Determine if the target supports unaligned memory accesses.
Definition R600ISelLowering.cpp:1543
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const
Definition R600ISelLowering.cpp:1433
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
XXX Only kernel functions are supported, so we can assume for now that every function is a kernel fun...
Definition R600ISelLowering.cpp:1458
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
Definition R600ISelLowering.cpp:403
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
op_iterator op_end() const
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=LocationSize::precise(0), const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
LLVMContext * getContext() const
LLVM_ABI SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
const TargetMachine & getTargetMachine() const
void setHasExtractBitsInsn(bool hasExtractInsn=true)
Tells the code generator that the target has BitExtract instructions.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal for a comparison of the specified types on this ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrNegativeOneBooleanContent
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
bool expandFP_TO_SINT(SDNode *N, SDValue &Result, SelectionDAG &DAG) const
Expand float(f32) to SINT(i64) conversion.
std::pair< SDValue, SDValue > scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Turn load of vector type into a load of the individual elements.
virtual MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
void expandShiftParts(SDNode *N, SDValue &Lo, SDValue &Hi, SelectionDAG &DAG) const
Expand shift-by-parts.
Primary interface to the complete machine description for the target machine.
Type * getType() const
All values are typed, get the type of this value.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ LOCAL_ADDRESS
Address space for local memory.
@ PARAM_I_ADDRESS
Address space for indirect addressable parameter memory (VTX1).
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ FLAT_ADDRESS
Address space for flat memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ PRIVATE_ADDRESS
Address space for private memory.
@ BUILD_VERTICAL_VECTOR
This node is for VLIW targets and it is used to represent a vector that is stored in consecutive regi...
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
LLVM_READNONE constexpr bool isShader(CallingConv::ID CC)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_VS
Used for Mesa vertex shaders, or AMDPAL last shader stage before rasterization (vertex shader if tess...
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ AMDGPU_HS
Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
@ AMDGPU_GS
Used for Mesa/AMDPAL geometry shaders.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ Cold
Attempts to make code in the caller as efficient as possible under the assumption that the call is no...
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ AMDGPU_ES
Used for AMDPAL shader stage before geometry shader if geometry is in use.
@ AMDGPU_LS
Used for AMDPAL vertex shader if tessellation is in use.
@ C
The default llvm calling convention, compatible with C.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ FADD
Simple binary floating point operators.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ SIGN_EXTEND
Conversion operators.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ SHL
Shift and rotation operations.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
LLVM_ABI CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
int getLDSNoRetOp(uint16_t Opcode)
@ Kill
The last use of a register.
NodeAddr< NodeBase * > Node
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
FunctionAddr VTableAddr Value
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
decltype(auto) dyn_cast(const From &Val)
dyn_cast - Return the argument parameter cast to the specified type.
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa - Return true if the parameter to the template is an instance of one of the template type argu...
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
decltype(auto) cast(const From &Val)
cast - Return the argument parameter cast to the specified type.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
This class contains a discriminated union of information about pointers in memory operands,...
bool isBeforeLegalizeOps() const