LLVM: lib/Target/Hexagon/HexagonISelLoweringHVX.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
21#include "llvm/IR/IntrinsicsHexagon.h"
23
24#include
25#include
26#include
27
28using namespace llvm;
29
32 cl::desc("Lower threshold (in bytes) for widening to HVX vectors"));
33
36 cl::desc("Enable FP fast conversion routine."));
37
38static const MVT LegalV64[] = { MVT::v64i8, MVT::v32i16, MVT::v16i32 };
39static const MVT LegalW64[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
40static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
41static const MVT LegalW128[] = { MVT::v256i8, MVT::v128i16, MVT::v64i32 };
42
44
46 switch (ElemTy.SimpleTy) {
47 case MVT::f16:
48 return std::make_tuple(5, 15, 10);
49 case MVT::f32:
50 return std::make_tuple(8, 127, 23);
51 case MVT::f64:
52 return std::make_tuple(11, 1023, 52);
53 default:
54 break;
55 }
57}
58
59void
60HexagonTargetLowering::initializeHVXLowering() {
61 if (Subtarget.useHVX64BOps()) {
68
69
70
71
72
73
74
75
79 } else if (Subtarget.useHVX128BOps()) {
89 if (Subtarget.useHVXV68Ops() && Subtarget.useHVXFloatingPoint()) {
94 }
95 if (Subtarget.useHVXV81Ops()) {
98 }
99 }
100
101
102
103 bool Use64b = Subtarget.useHVX64BOps();
106 MVT ByteV = Use64b ? MVT::v64i8 : MVT::v128i8;
107 MVT WordV = Use64b ? MVT::v16i32 : MVT::v32i32;
108 MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8;
109
110 auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) {
113 };
114
115
116
117
127
128 if (Subtarget.useHVX128BOps()) {
131 }
132 if (Subtarget.useHVX128BOps() && Subtarget.useHVXV68Ops() &&
133 Subtarget.useHVXFloatingPoint()) {
134
135 static const MVT FloatV[] = { MVT::v64f16, MVT::v32f32 };
136 static const MVT FloatW[] = { MVT::v128f16, MVT::v64f32 };
137
138 for (MVT T : FloatV) {
144
147
150
153
154
155
157 }
158
159
160
161
165
166
167
172
173 if (Subtarget.useHVXV81Ops()) {
176 setPromoteTo(ISD::SETCC, MVT::v64bf16, MVT::v64f32);
177 setPromoteTo(ISD::FADD, MVT::v64bf16, MVT::v64f32);
178 setPromoteTo(ISD::FSUB, MVT::v64bf16, MVT::v64f32);
179 setPromoteTo(ISD::FMUL, MVT::v64bf16, MVT::v64f32);
180 setPromoteTo(ISD::FMINNUM, MVT::v64bf16, MVT::v64f32);
181 setPromoteTo(ISD::FMAXNUM, MVT::v64bf16, MVT::v64f32);
182
186
191
195 }
196
197 for (MVT P : FloatW) {
207
208
209
210
212
214
217 }
218
219 if (Subtarget.useHVXQFloatOps()) {
222 } else if (Subtarget.useHVXIEEEFPOps()) {
225 }
226 }
227
228 for (MVT T : LegalV) {
231
247 if (T != ByteV) {
251 }
252
255 if (T.getScalarType() != MVT::i32) {
258 }
259
264 if (T.getScalarType() != MVT::i32) {
267 }
268
270
281 if (T != ByteV) {
283
287
288
290 }
291
292 if (Subtarget.useHVXFloatingPoint()) {
293
298 }
299
307 }
308
309 for (MVT T : LegalW) {
310
311
312
314
316
317
318
327
336
351 if (T != ByteW) {
355
356
358 }
361
364 if (T.getScalarType() != MVT::i32) {
367 }
368
369 if (Subtarget.useHVXFloatingPoint()) {
370
375 }
376 }
377
378
383
398
413
414
415
416 for (MVT T : LegalW) {
417
418
419
420
421
422
428
431 }
432
433 for (MVT T : LegalV) {
445 }
446
447 if (Use64b) {
448 for (MVT T: {MVT::v32i8, MVT::v32i16, MVT::v16i8, MVT::v16i16, MVT::v16i32})
450 } else {
451 for (MVT T: {MVT::v64i8, MVT::v64i16, MVT::v32i8, MVT::v32i16, MVT::v32i32})
453 }
454
455
456 unsigned HwLen = Subtarget.getVectorLength();
457 for (MVT ElemTy : Subtarget.getHVXElementTypes()) {
458 if (ElemTy == MVT::i1)
459 continue;
460 int ElemWidth = ElemTy.getFixedSizeInBits();
461 int MaxElems = (8*HwLen) / ElemWidth;
462 for (int N = 2; N < MaxElems; N *= 2) {
474 if (Subtarget.useHVXFloatingPoint()) {
479 }
480
484 }
485 }
486 }
487
488
492
494}
495
496unsigned
497HexagonTargetLowering::getPreferredHvxVectorAction(MVT VecTy) const {
498
500 return ~0u;
501
504 unsigned HwLen = Subtarget.getVectorLength();
505
506
507 if (ElemTy == MVT::i1 && VecLen > HwLen)
509
510 ArrayRef Tys = Subtarget.getHVXElementTypes();
511
512
513 if (ElemTy == MVT::i1) {
514 for (MVT T : Tys) {
517 if (A != ~0u)
518 return A;
519 }
520 return ~0u;
521 }
522
523
524
525
528 unsigned HwWidth = 8*HwLen;
529 if (VecWidth > 2*HwWidth)
531
535 if (VecWidth >= HwWidth/2 && VecWidth < HwWidth)
537 }
538
539
540 return ~0u;
541}
542
543unsigned
544HexagonTargetLowering::getCustomHvxOperationAction(SDNode &Op) const {
545 unsigned Opc = Op.getOpcode();
546 switch (Opc) {
551 }
553}
554
562}
563
565HexagonTargetLowering::typeJoin(const TypePair &Tys) const {
566 assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType());
567
569 return MVT::getVectorVT(ElemTy, Tys.first.getVectorNumElements() +
570 Tys.second.getVectorNumElements());
571}
572
573HexagonTargetLowering::TypePair
574HexagonTargetLowering::typeSplit(MVT VecTy) const {
577 assert((NumElem % 2) == 0 && "Expecting even-sized vector type");
579 return { HalfTy, HalfTy };
580}
581
583HexagonTargetLowering::typeExtElem(MVT VecTy, unsigned Factor) const {
587}
588
590HexagonTargetLowering::typeTruncElem(MVT VecTy, unsigned Factor) const {
594}
595
597HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy,
600 return Vec;
603}
604
606HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl,
610}
611
612HexagonTargetLowering::VectorPair
613HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl,
615 TypePair Tys = typeSplit(ty(Vec));
618 return DAG.SplitVector(Vec, dl, Tys.first, Tys.second);
619}
620
621bool
622HexagonTargetLowering::isHvxSingleTy(MVT Ty) const {
623 return Subtarget.isHVXVectorType(Ty) &&
624 Ty.getSizeInBits() == 8 * Subtarget.getVectorLength();
625}
626
627bool
628HexagonTargetLowering::isHvxPairTy(MVT Ty) const {
629 return Subtarget.isHVXVectorType(Ty) &&
630 Ty.getSizeInBits() == 16 * Subtarget.getVectorLength();
631}
632
633bool
634HexagonTargetLowering::isHvxBoolTy(MVT Ty) const {
635 return Subtarget.isHVXVectorType(Ty, true) &&
637}
638
639bool HexagonTargetLowering::allowsHvxMemoryAccess(
641
642
643
644
645 if (VecTy.getSizeInBits() > 8*Subtarget.getVectorLength())
646 return false;
647 if (!Subtarget.isHVXVectorType(VecTy, false))
648 return false;
651 return true;
652}
653
654bool HexagonTargetLowering::allowsHvxMisalignedMemoryAccesses(
656 if (!Subtarget.isHVXVectorType(VecTy))
657 return false;
658
661 return true;
662}
663
664void HexagonTargetLowering::AdjustHvxInstrPostInstrSelection(
666 unsigned Opc = MI.getOpcode();
667 const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
668 MachineBasicBlock &MB = *MI.getParent();
669 MachineFunction &MF = *MB.getParent();
672 auto At = MI.getIterator();
673
674 switch (Opc) {
675 case Hexagon::PS_vsplatib:
676 if (Subtarget.useHVXV62Ops()) {
677
678
679 Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
682 Register OutV = MI.getOperand(0).getReg();
685 } else {
686
687
688 Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
689 const MachineOperand &InpOp = MI.getOperand(1);
691 uint32_t V = InpOp.getImm() & 0xFF;
693 .addImm(V << 24 | V << 16 | V << 8 | V);
694 Register OutV = MI.getOperand(0).getReg();
696 }
698 break;
699 case Hexagon::PS_vsplatrb:
700 if (Subtarget.useHVXV62Ops()) {
701
702 Register OutV = MI.getOperand(0).getReg();
705 } else {
706 Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
707 const MachineOperand &InpOp = MI.getOperand(1);
708 BuildMI(MB, At, DL, TII.get(Hexagon::S2_vsplatrb), SplatV)
710 Register OutV = MI.getOperand(0).getReg();
713 }
715 break;
716 case Hexagon::PS_vsplatih:
717 if (Subtarget.useHVXV62Ops()) {
718
719
720 Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
723 Register OutV = MI.getOperand(0).getReg();
726 } else {
727
728
729 Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
730 const MachineOperand &InpOp = MI.getOperand(1);
732 uint32_t V = InpOp.getImm() & 0xFFFF;
734 .addImm(V << 16 | V);
735 Register OutV = MI.getOperand(0).getReg();
737 }
739 break;
740 case Hexagon::PS_vsplatrh:
741 if (Subtarget.useHVXV62Ops()) {
742
743 Register OutV = MI.getOperand(0).getReg();
746 } else {
747
748
749 Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
750 const MachineOperand &InpOp = MI.getOperand(1);
751 BuildMI(MB, At, DL, TII.get(Hexagon::A2_combine_ll), SplatV)
754 Register OutV = MI.getOperand(0).getReg();
756 }
758 break;
759 case Hexagon::PS_vsplatiw:
760 case Hexagon::PS_vsplatrw:
761 if (Opc == Hexagon::PS_vsplatiw) {
762
763 Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
766 MI.getOperand(1).ChangeToRegister(SplatV, false);
767 }
768
769 MI.setDesc(TII.get(Hexagon::V6_lvsplatw));
770 break;
771 }
772}
773
775HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy,
778 ElemIdx = DAG.getBitcast(MVT::i32, ElemIdx);
779
781 if (ElemWidth == 8)
782 return ElemIdx;
783
784 unsigned L = Log2_32(ElemWidth/8);
785 const SDLoc &dl(ElemIdx);
787 {ElemIdx, DAG.getConstant(L, dl, MVT::i32)});
788}
789
791HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy,
794 assert(ElemWidth >= 8 && ElemWidth <= 32);
795 if (ElemWidth == 32)
796 return Idx;
797
798 if (ty(Idx) != MVT::i32)
800 const SDLoc &dl(Idx);
803 return SubIdx;
804}
805
807HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0,
810 MVT OpTy = ty(Op0);
811 assert(OpTy == ty(Op1));
812
814 if (ElemTy == MVT::i8)
817
818 MVT ResTy = tyVector(OpTy, MVT::i8);
820
821 SmallVector<int,128> ByteMask;
822 for (int M : Mask) {
823 if (M < 0) {
824 for (unsigned I = 0; I != ElemSize; ++I)
826 } else {
827 int NewM = M*ElemSize;
828 for (unsigned I = 0; I != ElemSize; ++I)
830 }
831 }
833 return DAG.getVectorShuffle(ResTy, dl, opCastElem(Op0, MVT::i8, DAG),
834 opCastElem(Op1, MVT::i8, DAG), ByteMask);
835}
836
838HexagonTargetLowering::buildHvxVectorReg(ArrayRef Values,
841 unsigned VecLen = Values.size();
845 unsigned HwLen = Subtarget.getVectorLength();
846
847 unsigned ElemSize = ElemWidth / 8;
848 assert(ElemSize*VecLen == HwLen);
850
852 !(Subtarget.useHVXFloatingPoint() &&
854 assert((ElemSize == 1 || ElemSize == 2) && "Invalid element size");
855 unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2;
857 for (unsigned i = 0; i != VecLen; i += OpsPerWord) {
858 SDValue W = buildVector32(Values.slice(i, OpsPerWord), dl, PartVT, DAG);
860 }
861 } else {
864 }
866 unsigned NumValues = Values.size();
867 assert(NumValues > 0);
868 bool IsUndef = true;
869 for (unsigned i = 0; i != NumValues; ++i) {
870 if (Values[i].isUndef())
871 continue;
872 IsUndef = false;
873 if (!SplatV.getNode())
874 SplatV = Values[i];
875 else if (SplatV != Values[i])
876 return false;
877 }
878 if (IsUndef)
879 SplatV = Values[0];
880 return true;
881 };
882
883 unsigned NumWords = Words.size();
885 bool IsSplat = isSplat(Words, SplatV);
886 if (IsSplat && isUndef(SplatV))
888 if (IsSplat) {
891 return getZero(dl, VecTy, DAG);
895 }
896
897
898
900 bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts);
901 if (AllConst) {
903 (Constant**)Consts.end());
905 Align Alignment(HwLen);
910 }
911
912
913
914
915
916
917
918
919 auto IsBuildFromExtracts = [this,&Values] (SDValue &SrcVec,
920 SmallVectorImpl &SrcIdx) {
922 for (SDValue V : Values) {
923 if (isUndef(V)) {
924 SrcIdx.push_back(-1);
925 continue;
926 }
928 return false;
929
931 if (Vec.getNode() != nullptr && T.getNode() != Vec.getNode())
932 return false;
933 Vec = T;
935 if (C == nullptr)
936 return false;
937 int I = C->getSExtValue();
938 assert(I >= 0 && "Negative element index");
939 SrcIdx.push_back(I);
940 }
941 SrcVec = Vec;
942 return true;
943 };
944
945 SmallVector<int,128> ExtIdx;
947 if (IsBuildFromExtracts(ExtVec, ExtIdx)) {
948 MVT ExtTy = ty(ExtVec);
950 if (ExtLen == VecLen || ExtLen == 2*VecLen) {
951
952
953
954 SmallVector<int,128> Mask;
955 BitVector Used(ExtLen);
956
957 for (int M : ExtIdx) {
958 Mask.push_back(M);
959 if (M >= 0)
961 }
962
963
964
965
966
967 for (unsigned I = 0; I != ExtLen; ++I) {
968 if (Mask.size() == ExtLen)
969 break;
970 if (.test(I))
972 }
973
976 return ExtLen == VecLen ? S : LoHalf(S, DAG);
977 }
978 }
979
980
981
982
983
984 assert(4*Words.size() == Subtarget.getVectorLength());
985 int VecHist[32];
986 int n = 0;
987 for (unsigned i = 0; i != NumWords; ++i) {
988 VecHist[i] = 0;
989 if (Words[i].isUndef())
990 continue;
991 for (unsigned j = i; j != NumWords; ++j)
992 if (Words[i] == Words[j])
993 VecHist[i]++;
994
995 if (VecHist[i] > VecHist[n])
996 n = i;
997 }
998
999 SDValue HalfV = getZero(dl, VecTy, DAG);
1000 if (VecHist[n] > 1) {
1003 {HalfV, SplatV, DAG.getConstant(HwLen/2, dl, MVT::i32)});
1004 }
1007
1008
1009
1010
1011 int Rn = 0, Rm = 0;
1015 for (unsigned i = 0; i != NumWords/2; ++i) {
1016
1017 if (Words[i] != Words[n] || VecHist[n] <= 1) {
1021 {HalfV0, Words[i]});
1022 Rn = 0;
1023 }
1024 if (Words[i+NumWords/2] != Words[n] || VecHist[n] <= 1) {
1028 {HalfV1, Words[i+NumWords/2]});
1029 Rm = 0;
1030 }
1031 Rn += 4;
1032 Rm += 4;
1033 }
1034
1035 Sn = DAG.getConstant(Rn+HwLen/2, dl, MVT::i32);
1039
1042
1044
1047 return OutV;
1048}
1049
1051HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl,
1052 unsigned BitBytes, bool ZeroFill, SelectionDAG &DAG) const {
1053 MVT PredTy = ty(PredV);
1054 unsigned HwLen = Subtarget.getVectorLength();
1056
1057 if (Subtarget.isHVXVectorType(PredTy, true)) {
1058
1059
1060
1061
1062
1063
1064
1065
1067 SmallVector<int,128> Mask(HwLen);
1068
1071
1072 for (unsigned i = 0; i != HwLen; ++i) {
1073 unsigned Num = i % Scale;
1074 unsigned Off = i / Scale;
1075 Mask[BlockLen*Num + Off] = i;
1076 }
1078 if (!ZeroFill)
1079 return S;
1080
1081
1082
1083 assert(BlockLen < HwLen && "vsetq(v1) prerequisite");
1085 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
1086 {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
1089 }
1090
1091
1092 assert(PredTy == MVT::v2i1 || PredTy == MVT::v4i1 || PredTy == MVT::v8i1);
1093
1096 unsigned IdxW = 0;
1097
1098 SDValue W0 = isUndef(PredV)
1101 if (Bytes < BitBytes) {
1102 Words[IdxW].push_back(HiHalf(W0, DAG));
1103 Words[IdxW].push_back(LoHalf(W0, DAG));
1104 } else
1106
1107 while (Bytes < BitBytes) {
1108 IdxW ^= 1;
1109 Words[IdxW].clear();
1110
1111 if (Bytes < 4) {
1112 for (const SDValue &W : Words[IdxW ^ 1]) {
1113 SDValue T = expandPredicate(W, dl, DAG);
1114 Words[IdxW].push_back(HiHalf(T, DAG));
1115 Words[IdxW].push_back(LoHalf(T, DAG));
1116 }
1117 } else {
1118 for (const SDValue &W : Words[IdxW ^ 1]) {
1121 }
1122 }
1123 Bytes *= 2;
1124 }
1125
1126 while (Bytes > BitBytes) {
1127 IdxW ^= 1;
1128 Words[IdxW].clear();
1129
1130 if (Bytes <= 4) {
1131 for (const SDValue &W : Words[IdxW ^ 1]) {
1132 SDValue T = contractPredicate(W, dl, DAG);
1134 }
1135 } else {
1136 for (const SDValue &W : Words[IdxW ^ 1]) {
1138 }
1139 }
1140 Bytes /= 2;
1141 }
1142
1143 assert(Bytes == BitBytes);
1144 if (BitBytes == 1 && PredTy == MVT::v2i1)
1146
1147 SDValue Vec = ZeroFill ? getZero(dl, ByteTy, DAG) : DAG.getUNDEF(ByteTy);
1149 for (const SDValue &W : Words[IdxW]) {
1152 }
1153
1154 return Vec;
1155}
1156
1158HexagonTargetLowering::buildHvxVectorPred(ArrayRef Values,
1161
1162
1163 unsigned VecLen = Values.size();
1164 unsigned HwLen = Subtarget.getVectorLength();
1165 assert(VecLen <= HwLen || VecLen == 8*HwLen);
1167 bool AllT = true, AllF = true;
1168
1169 auto IsTrue = [] (SDValue V) {
1171 return ->isZero();
1172 return false;
1173 };
1174 auto IsFalse = [] (SDValue V) {
1176 return N->isZero();
1177 return false;
1178 };
1179
1180 if (VecLen <= HwLen) {
1181
1182
1183
1184 assert(HwLen % VecLen == 0);
1185 unsigned BitBytes = HwLen / VecLen;
1186 for (SDValue V : Values) {
1187 AllT &= IsTrue(V);
1188 AllF &= IsFalse(V);
1189
1192 for (unsigned B = 0; B != BitBytes; ++B)
1194 }
1195 } else {
1196
1197
1198
1199 for (unsigned I = 0; I != VecLen; I += 8) {
1200 unsigned B = 0;
1201
1203 if (!Values[I+B].isUndef())
1204 break;
1205 }
1207 AllT &= IsTrue(F);
1208 AllF &= IsFalse(F);
1209
1213
1214
1216 assert(Values[I+B].isUndef() || Values[I+B] == F);
1217 }
1218 }
1219
1220 if (AllT)
1222 if (AllF)
1224
1226 SDValue ByteVec = buildHvxVectorReg(Bytes, dl, ByteTy, DAG);
1228}
1229
1231HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV,
1234
1236 assert(ElemWidth >= 8 && ElemWidth <= 32);
1237 (void)ElemWidth;
1238
1239 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
1241 {VecV, ByteIdx});
1242 if (ElemTy == MVT::i32)
1243 return ExWord;
1244
1245
1246
1247
1248 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
1249
1250 SDValue ExVec = DAG.getBitcast(tyVector(ty(ExWord), ElemTy), ExWord);
1251 return extractVector(ExVec, SubIdx, dl, ElemTy, MVT::i32, DAG);
1252}
1253
1255HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV,
1257
1258 assert(ResTy == MVT::i1);
1259
1260 unsigned HwLen = Subtarget.getVectorLength();
1263
1264 unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
1267
1268 SDValue ExtB = extractHvxElementReg(ByteVec, IdxV, dl, MVT::i32, DAG);
1270 return getInstr(Hexagon::C2_cmpgtui, dl, MVT::i1, {ExtB, Zero}, DAG);
1271}
1272
1274HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV,
1277
1279 assert(ElemWidth >= 8 && ElemWidth <= 32);
1280 (void)ElemWidth;
1281
1282 auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV,
1284 MVT VecTy = ty(VecV);
1285 unsigned HwLen = Subtarget.getVectorLength();
1292 {DAG.getConstant(HwLen, dl, MVT::i32), MaskV});
1294 return TorV;
1295 };
1296
1297 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
1298 if (ElemTy == MVT::i32)
1299 return InsertWord(VecV, ValV, ByteIdx);
1300
1301
1302
1304 {ByteIdx, DAG.getConstant(2, dl, MVT::i32)});
1305 SDValue Ext = extractHvxElementReg(opCastElem(VecV, MVT::i32, DAG), WordIdx,
1306 dl, MVT::i32, DAG);
1307
1308
1309
1310 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
1311 MVT SubVecTy = tyVector(ty(Ext), ElemTy);
1313 ValV, SubIdx, dl, ElemTy, DAG);
1314
1315
1316 return InsertWord(VecV, Ins, ByteIdx);
1317}
1318
1320HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV,
1322 unsigned HwLen = Subtarget.getVectorLength();
1325
1326 unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
1330
1331 SDValue InsV = insertHvxElementReg(ByteVec, IdxV, ValV, dl, DAG);
1333}
1334
1336HexagonTargetLowering::extractHvxSubvectorReg(SDValue OrigOp, SDValue VecV,
1338 MVT VecTy = ty(VecV);
1339 unsigned HwLen = Subtarget.getVectorLength();
1343
1344
1345
1346
1347 if (isHvxPairTy(VecTy)) {
1348 unsigned SubIdx = Hexagon::vsub_lo;
1349 if (Idx * ElemWidth >= 8 * HwLen) {
1350 SubIdx = Hexagon::vsub_hi;
1352 }
1353
1354 VecTy = typeSplit(VecTy).first;
1356 if (VecTy == ResTy)
1357 return VecV;
1358 }
1359
1360
1361
1363
1364 MVT WordTy = tyVector(VecTy, MVT::i32);
1366 unsigned WordIdx = (Idx*ElemWidth) / 32;
1367
1369 SDValue W0 = extractHvxElementReg(WordVec, W0Idx, dl, MVT::i32, DAG);
1372
1374 SDValue W1 = extractHvxElementReg(WordVec, W1Idx, dl, MVT::i32, DAG);
1375 SDValue WW = getCombine(W1, W0, dl, MVT::i64, DAG);
1377}
1378
1380HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV,
1382 MVT VecTy = ty(VecV);
1383 unsigned HwLen = Subtarget.getVectorLength();
1386
1388
1391 unsigned Offset = Idx * BitBytes;
1393 SmallVector<int,128> Mask;
1394
1395 if (Subtarget.isHVXVectorType(ResTy, true)) {
1396
1397
1398
1399
1402 for (unsigned i = 0; i != HwLen/Rep; ++i) {
1403 for (unsigned j = 0; j != Rep; ++j)
1405 }
1408 }
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420 unsigned Rep = 8 / ResLen;
1421
1422
1423 for (unsigned r = 0; r != HwLen/ResLen; ++r) {
1424
1425 for (unsigned i = 0; i != ResLen; ++i) {
1426 for (unsigned j = 0; j != Rep; ++j)
1427 Mask.push_back(Offset + i*BitBytes);
1428 }
1429 }
1430
1431 SDValue Zero = getZero(dl, MVT::i32, DAG);
1433
1434
1437 {ShuffV, DAG.getConstant(4, dl, MVT::i32)});
1438 SDValue Vec64 = getCombine(W1, W0, dl, MVT::v8i8, DAG);
1439 return getInstr(Hexagon::A4_vcmpbgtui, dl, ResTy,
1441}
1442
1444HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV,
1446 MVT VecTy = ty(VecV);
1447 MVT SubTy = ty(SubV);
1448 unsigned HwLen = Subtarget.getVectorLength();
1451
1452 bool IsPair = isHvxPairTy(VecTy);
1453 MVT SingleTy = MVT::getVectorVT(ElemTy, (8*HwLen)/ElemWidth);
1454
1458
1459 if (IsPair) {
1460 V0 = LoHalf(VecV, DAG);
1461 V1 = HiHalf(VecV, DAG);
1462
1464 dl, MVT::i32);
1466 if (isHvxSingleTy(SubTy)) {
1468 unsigned Idx = CN->getZExtValue();
1470 unsigned SubIdx = (Idx == 0) ? Hexagon::vsub_lo : Hexagon::vsub_hi;
1472 }
1473
1474
1475
1479 }
1480
1481
1482
1486 }
1487
1488
1489
1491
1493 if (!IdxN || !IdxN->isZero()) {
1495 DAG.getConstant(ElemWidth/8, dl, MVT::i32));
1497 }
1498
1499
1500
1501 unsigned RolBase = HwLen;
1505 } else {
1507 SDValue R0 = LoHalf(V, DAG);
1508 SDValue R1 = HiHalf(V, DAG);
1513 RolBase = HwLen-4;
1514 }
1515
1516 if (RolBase != 4 || !IdxN || !IdxN->isZero()) {
1518 DAG.getConstant(RolBase, dl, MVT::i32), IdxV);
1520 }
1521
1522 if (IsPair) {
1526 }
1527 return SingleV;
1528}
1529
1531HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV,
1533 MVT VecTy = ty(VecV);
1534 MVT SubTy = ty(SubV);
1535 assert(Subtarget.isHVXVectorType(VecTy, true));
1536
1537
1538
1540 unsigned HwLen = Subtarget.getVectorLength();
1541 assert(HwLen % VecLen == 0 && "Unexpected vector type");
1542
1544 unsigned BitBytes = HwLen / VecLen;
1545 unsigned BlockLen = HwLen / Scale;
1546
1549 SDValue ByteSub = createHvxPrefixPred(SubV, dl, BitBytes, false, DAG);
1551
1553 if (!IdxN || !IdxN->isZero()) {
1555 DAG.getConstant(BitBytes, dl, MVT::i32));
1557 }
1558
1559
1560
1561
1562 assert(BlockLen < HwLen && "vsetq(v1) prerequisite");
1564 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
1565 {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
1566 ByteVec = getInstr(Hexagon::V6_vmux, dl, ByteTy, {Q, ByteSub, ByteVec}, DAG);
1567
1568 if (!IdxN || !IdxN->isZero()) {
1572 }
1574}
1575
1577HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl,
1579
1580
1581
1582 assert(Subtarget.isHVXVectorType(ResTy));
1583 if (!ZeroExt)
1585
1589 SDValue False = getZero(dl, ResTy, DAG);
1590 return DAG.getSelect(dl, ResTy, VecV, True, False);
1591}
1592
1594HexagonTargetLowering::compressHvxPred(SDValue VecQ, const SDLoc &dl,
1596
1597
1598
1599
1600
1602 unsigned HwLen = Subtarget.getVectorLength();
1604 MVT PredTy = ty(VecQ);
1606 assert(HwLen % PredLen == 0);
1608
1611
1612
1613 for (unsigned i = 0; i != HwLen/8; ++i) {
1614 for (unsigned j = 0; j != 8; ++j)
1615 Tmp.push_back(ConstantInt::get(Int8Ty, 1ull << j));
1616 }
1618 Align Alignment(HwLen);
1624
1625
1627 getZero(dl, VecTy, DAG));
1628
1629
1630
1633 SDValue Vrmpy = getInstr(Hexagon::V6_vrmpyub, dl, ByteTy, {Sel, All1}, DAG);
1634
1635 SDValue Rot = getInstr(Hexagon::V6_valignbi, dl, ByteTy,
1638
1639
1640
1641
1642 SmallVector<int,128> Mask;
1643 for (unsigned i = 0; i != HwLen; ++i)
1644 Mask.push_back((8*i) % HwLen + i/(HwLen/8));
1647 return DAG.getBitcast(ResTy, Collect);
1648}
1649
1651HexagonTargetLowering::resizeToWidth(SDValue VecV, MVT ResTy, bool Signed,
1653
1654 MVT InpTy = ty(VecV);
1655 if (InpTy == ResTy)
1656 return VecV;
1657
1660
1662 return InpWidth < ResWidth
1663 ? DAG.getNode(ISD::FP_EXTEND, dl, ResTy, VecV)
1666 }
1667
1669
1670 if (InpWidth < ResWidth) {
1672 return DAG.getNode(ExtOpc, dl, ResTy, VecV);
1673 } else {
1676 }
1677}
1678
1680HexagonTargetLowering::extractSubvector(SDValue Vec, MVT SubTy, unsigned SubIdx,
1683
1684 const SDLoc &dl(Vec);
1687 {Vec, DAG.getConstant(ElemIdx, dl, MVT::i32)});
1688}
1689
1692 const {
1693 const SDLoc &dl(Op);
1694 MVT VecTy = ty(Op);
1695
1698 for (unsigned i = 0; i != Size; ++i)
1699 Ops.push_back(Op.getOperand(i));
1700
1702 return buildHvxVectorPred(Ops, dl, VecTy, DAG);
1703
1704
1705
1706
1710 for (unsigned i = 0; i != Size; i++)
1712
1716 }
1717
1718
1719
1720
1721 if (VecTy.getSizeInBits() == 16 * Subtarget.getVectorLength()) {
1723 MVT SingleTy = typeSplit(VecTy).first;
1724 SDValue V0 = buildHvxVectorReg(A.take_front(Size / 2), dl, SingleTy, DAG);
1725 SDValue V1 = buildHvxVectorReg(A.drop_front(Size / 2), dl, SingleTy, DAG);
1727 }
1728
1729 return buildHvxVectorReg(Ops, dl, VecTy, DAG);
1730}
1731
1734 const {
1735 const SDLoc &dl(Op);
1736 MVT VecTy = ty(Op);
1737 MVT ArgTy = ty(Op.getOperand(0));
1738
1739 if (ArgTy == MVT::f16 || ArgTy == MVT::bf16) {
1745 }
1746
1748}
1749
1752 const {
1753
1754
1755
1756 MVT VecTy = ty(Op);
1757 const SDLoc &dl(Op);
1760 if (NumOp == 2)
1761 return Op;
1762
1764 for (SDValue V : Op.getNode()->ops())
1766
1767
1768
1769
1770
1771 for (SDValue &V : Elems) {
1772 MVT Ty = ty(V);
1774 MVT NTy = typeLegalize(Ty, DAG);
1778 V.getOperand(0), V.getOperand(1)),
1780 continue;
1781 }
1782
1783 switch (V.getOpcode()) {
1786 break;
1789 break;
1792 break;
1793 default:
1795 }
1796 }
1797 }
1799 }
1800
1802 unsigned HwLen = Subtarget.getVectorLength();
1804
1806
1807
1808
1809 if (Subtarget.isHVXVectorType(ty(Op0), true)) {
1810 if (NumOp == 2)
1812
1816
1817 MVT HalfTy = typeSplit(VecTy).first;
1819 Ops.take_front(NumOp/2));
1821 Ops.take_back(NumOp/2));
1823 }
1824
1825
1826
1828
1830 for (SDValue V : Op.getNode()->op_values()) {
1831 SDValue P = createHvxPrefixPred(V, dl, BitBytes, true, DAG);
1833 }
1834
1835 unsigned InpLen = ty(Op.getOperand(0)).getVectorNumElements();
1838 SDValue Res = getZero(dl, ByteTy, DAG);
1839 for (unsigned i = 0, e = Prefixes.size(); i != e; ++i) {
1841 Res = DAG.getNode(ISD::OR, dl, ByteTy, Res, Prefixes[e-i-1]);
1842 }
1844}
1845
1848 const {
1849
1850 SDValue VecV = Op.getOperand(0);
1852 const SDLoc &dl(Op);
1853 SDValue IdxV = Op.getOperand(1);
1854 if (ElemTy == MVT::i1)
1855 return extractHvxElementPred(VecV, IdxV, dl, ty(Op), DAG);
1856
1857 return extractHvxElementReg(VecV, IdxV, dl, ty(Op), DAG);
1858}
1859
1862 const {
1863 const SDLoc &dl(Op);
1864 MVT VecTy = ty(Op);
1865 SDValue VecV = Op.getOperand(0);
1866 SDValue ValV = Op.getOperand(1);
1867 SDValue IdxV = Op.getOperand(2);
1869 if (ElemTy == MVT::i1)
1870 return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG);
1871
1872 if (ElemTy == MVT::f16 || ElemTy == MVT::bf16) {
1874 tyVector(VecTy, MVT::i16),
1875 DAG.getBitcast(tyVector(VecTy, MVT::i16), VecV),
1876 DAG.getBitcast(MVT::i16, ValV), IdxV);
1877 return DAG.getBitcast(tyVector(VecTy, ElemTy), T0);
1878 }
1879
1880 return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG);
1881}
1882
1885 const {
1886 SDValue SrcV = Op.getOperand(0);
1887 MVT SrcTy = ty(SrcV);
1888 MVT DstTy = ty(Op);
1889 SDValue IdxV = Op.getOperand(1);
1892 (void)Idx;
1893 const SDLoc &dl(Op);
1894
1896 if (ElemTy == MVT::i1)
1897 return extractHvxSubvectorPred(SrcV, IdxV, dl, DstTy, DAG);
1898
1899 return extractHvxSubvectorReg(Op, SrcV, IdxV, dl, DstTy, DAG);
1900}
1901
1904 const {
1905
1906 SDValue VecV = Op.getOperand(0);
1907 SDValue ValV = Op.getOperand(1);
1908 SDValue IdxV = Op.getOperand(2);
1909
1910 const SDLoc &dl(Op);
1911 MVT VecTy = ty(VecV);
1913 if (ElemTy == MVT::i1)
1914 return insertHvxSubvectorPred(VecV, ValV, IdxV, dl, DAG);
1915
1916 return insertHvxSubvectorReg(VecV, ValV, IdxV, dl, DAG);
1917}
1918
1921
1922
1923
1924
1925 MVT ResTy = ty(Op);
1926 SDValue InpV = Op.getOperand(0);
1928 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1929 return LowerHvxSignExt(Op, DAG);
1931}
1932
1935 MVT ResTy = ty(Op);
1936 SDValue InpV = Op.getOperand(0);
1938 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1939 return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), false, DAG);
1940 return Op;
1941}
1942
1945 MVT ResTy = ty(Op);
1946 SDValue InpV = Op.getOperand(0);
1948 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1949 return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), true, DAG);
1950 return Op;
1951}
1952
1955
1956
1957 const SDLoc &dl(Op);
1958 MVT ResTy = ty(Op);
1959 SDValue InpV = Op.getOperand(0);
1960 assert(ResTy == ty(InpV));
1961
1962
1965
1969 DAG.getConstant(ElemWidth, dl, MVT::i32));
1972
1973
1974
1975
1981}
1982
1985 const SDLoc &dl(Op);
1986 MVT ResTy = ty(Op);
1988
1991
1992 SDVTList ResTys = DAG.getVTList(ResTy, ResTy);
1993 unsigned Opc = Op.getOpcode();
1994
1995
2000
2001#ifndef NDEBUG
2002 Op.dump(&DAG);
2003#endif
2005}
2006
2009 const SDLoc &dl(Op);
2010 unsigned Opc = Op.getOpcode();
2013
2014
2015 if (auto HiVal = Op.getValue(1); HiVal.use_empty()) {
2016
2020 }
2021
2024
2025
2026
2027 if (Subtarget.useHVXV62Ops())
2028 return emitHvxMulLoHiV62(Vu, SignedVu, Vv, SignedVv, dl, DAG);
2029
2031
2032
2033 if (auto LoVal = Op.getValue(0); LoVal.use_empty()) {
2034 SDValue Hi = emitHvxMulHsV60(Vu, Vv, dl, DAG);
2037 }
2038 }
2039
2040 return emitHvxMulLoHiV60(Vu, SignedVu, Vv, SignedVv, dl, DAG);
2041}
2042
2046 MVT ResTy = ty(Op);
2047 MVT ValTy = ty(Val);
2048 const SDLoc &dl(Op);
2049
2051 unsigned HwLen = Subtarget.getVectorLength();
2053 SDValue VQ = compressHvxPred(Val, dl, WordTy, DAG);
2055
2057 SDValue W0 = extractHvxElementReg(VQ, DAG.getConstant(0, dl, MVT::i32),
2058 dl, MVT::i32, DAG);
2060 return W0;
2063 }
2064
2065
2068 for (unsigned i = 0; i != BitWidth/32; ++i) {
2069 SDValue W = extractHvxElementReg(
2070 VQ, DAG.getConstant(i, dl, MVT::i32), dl, MVT::i32, DAG);
2072 }
2075 for (unsigned i = 0, e = Words.size(); i < e; i += 2) {
2076 SDValue C = getCombine(Words[i+1], Words[i], dl, MVT::i64, DAG);
2078 }
2079
2081 return Combines[0];
2082
2084 }
2085
2086
2087
2088
2089 auto bitcastI32ToV32I1 = [&](SDValue Val32) {
2090 assert(Val32.getValueType().getSizeInBits() == 32 &&
2091 "Input must be 32 bits");
2095 for (unsigned i = 0; i < 32; ++i)
2096 Mask.push_back(DAG.getConstant(1ull << i, dl, MVT::i32));
2097
2101 };
2102
2103 if (ResTy == MVT::v32i1 &&
2104 (ValTy == MVT::i32 || ValTy == MVT::v2i16 || ValTy == MVT::v4i8) &&
2105 Subtarget.useHVX128BOps()) {
2107 if (ValTy == MVT::v2i16 || ValTy == MVT::v4i8)
2108 Val32 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Val);
2109 return bitcastI32ToV32I1(Val32);
2110 }
2111
2112 if (ResTy == MVT::v64i1 && ValTy == MVT::i64 && Subtarget.useHVX128BOps()) {
2113
2118
2119
2120 SDValue LoRes = bitcastI32ToV32I1(Lo);
2121 SDValue HiRes = bitcastI32ToV32I1(Hi);
2122
2123
2125 }
2126
2128
2130 unsigned HwLen = Subtarget.getVectorLength();
2132
2135
2136
2137
2139
2140
2142 for (unsigned I = 0; I != HwLen / 8; ++I) {
2146 for (unsigned J = 0; J != 8; ++J) {
2149 }
2150 }
2151
2154 SDValue I2V = buildHvxVectorReg(Bytes, dl, ConstantVecTy, DAG);
2155
2156
2157 I2V = DAG.getNode(ISD::AND, dl, ConstantVecTy, {I2V, ConstantVec});
2159 }
2160
2161 return Op;
2162}
2163
2166
2169 Op.getOperand(0));
2170}
2171
2174 MVT ResTy = ty(Op);
2176 return Op;
2177
2178 const SDLoc &dl(Op);
2179 unsigned HwLen = Subtarget.getVectorLength();
2181 assert(HwLen % VecLen == 0);
2182 unsigned ElemSize = HwLen / VecLen;
2183
2190}
2191
2194 if (SDValue S = getVectorShiftByInt(Op, DAG))
2195 return S;
2196 return Op;
2197}
2198
2200HexagonTargetLowering::LowerHvxFunnelShift(SDValue Op,
2202 unsigned Opc = Op.getOpcode();
2204
2205
2206
2210
2211 MVT InpTy = ty(A);
2213
2214 const SDLoc &dl(Op);
2217
2218
2219
2220 bool UseShifts = ElemTy != MVT::i8;
2221 if (Subtarget.useHVXV65Ops() && ElemTy == MVT::i32)
2222 UseShifts = false;
2223
2224 if (SDValue SplatV = getSplatValue(S, DAG); SplatV && UseShifts) {
2225
2232 {DAG.getConstant(ElemWidth, dl, MVT::i32), ModS});
2234 DAG.getSetCC(dl, MVT::i1, ModS, getZero(dl, MVT::i32, DAG), ISD::SETEQ);
2235
2236
2242
2243
2245 }
2246
2248 InpTy, dl, DAG.getConstant(ElemWidth - 1, dl, ElemTy));
2249
2251 return DAG.getNode(MOpc, dl, ty(Op),
2253}
2254
2257 const SDLoc &dl(Op);
2258 unsigned IntNo = Op.getConstantOperandVal(0);
2260
2261 auto Swap = [&](SDValue P) {
2262 return DAG.getMergeValues({P.getValue(1), P.getValue(0)}, dl);
2263 };
2264
2265 switch (IntNo) {
2266 case Intrinsic::hexagon_V6_pred_typecast:
2267 case Intrinsic::hexagon_V6_pred_typecast_128B: {
2268 MVT ResTy = ty(Op), InpTy = ty(Ops[1]);
2269 if (isHvxBoolTy(ResTy) && isHvxBoolTy(InpTy)) {
2270 if (ResTy == InpTy)
2271 return Ops[1];
2273 }
2274 break;
2275 }
2276 case Intrinsic::hexagon_V6_vmpyss_parts:
2277 case Intrinsic::hexagon_V6_vmpyss_parts_128B:
2279 {Ops[1], Ops[2]}));
2280 case Intrinsic::hexagon_V6_vmpyuu_parts:
2281 case Intrinsic::hexagon_V6_vmpyuu_parts_128B:
2283 {Ops[1], Ops[2]}));
2284 case Intrinsic::hexagon_V6_vmpyus_parts:
2285 case Intrinsic::hexagon_V6_vmpyus_parts_128B: {
2287 {Ops[1], Ops[2]}));
2288 }
2289 }
2290
2291 return Op;
2292}
2293
2296 const SDLoc &dl(Op);
2297 unsigned HwLen = Subtarget.getVectorLength();
2301 SDValue Chain = MaskN->getChain();
2304
2305 unsigned Opc = Op->getOpcode();
2306 assert(Opc == ISD::MLOAD || Opc == ISD::MSTORE);
2307
2308 if (Opc == ISD::MLOAD) {
2309 MVT ValTy = ty(Op);
2312 if (isUndef(Thru))
2313 return Load;
2316 }
2317
2318
2319
2320
2321
2322 unsigned StoreOpc = Hexagon::V6_vS32b_qpred_ai;
2325
2326 if (MaskN->getAlign().value() % HwLen == 0) {
2327 SDValue Store = getInstr(StoreOpc, dl, MVT::Other,
2331 }
2332
2333
2335 SDValue Z = getZero(dl, ty(V), DAG);
2336
2337
2338
2339 SDValue LoV = getInstr(Hexagon::V6_vlalignb, dl, ty(V), {V, Z, A}, DAG);
2340 SDValue HiV = getInstr(Hexagon::V6_vlalignb, dl, ty(V), {Z, V, A}, DAG);
2341 return std::make_pair(LoV, HiV);
2342 };
2343
2347 VectorPair Tmp = StoreAlign(MaskV, Base);
2350 VectorPair ValueU = StoreAlign(Value, Base);
2351
2354 getInstr(StoreOpc, dl, MVT::Other,
2355 {MaskU.first, Base, Offset0, ValueU.first, Chain}, DAG);
2357 getInstr(StoreOpc, dl, MVT::Other,
2358 {MaskU.second, Base, Offset1, ValueU.second, Chain}, DAG);
2362}
2363
2366
2367
2368 assert(Subtarget.useHVXQFloatOps());
2369
2370 assert(Op->getOpcode() == ISD::FP_EXTEND);
2371
2372 MVT VecTy = ty(Op);
2373 MVT ArgTy = ty(Op.getOperand(0));
2374 const SDLoc &dl(Op);
2375
2376 if (ArgTy == MVT::v64bf16) {
2377 MVT HalfTy = typeSplit(VecTy).first;
2378 SDValue BF16Vec = Op.getOperand(0);
2380 getInstr(Hexagon::V6_vxor, dl, HalfTy, {BF16Vec, BF16Vec}, DAG);
2381
2382
2383
2385 getInstr(Hexagon::V6_vshufoeh, dl, VecTy, {BF16Vec, Zeroes}, DAG);
2386 VectorPair VecPair = opSplit(ShuffVec, dl, DAG);
2387 SDValue Result = getInstr(Hexagon::V6_vshuffvdd, dl, VecTy,
2388 {VecPair.second, VecPair.first,
2390 DAG);
2392 }
2393
2394 assert(VecTy == MVT::v64f32 && ArgTy == MVT::v64f16);
2395
2396 SDValue F16Vec = Op.getOperand(0);
2397
2399 bool Ignored;
2403 getInstr(Hexagon::V6_vmpy_qf32_hf, dl, VecTy, {F16Vec, Fp16Ones}, DAG);
2404
2405 MVT HalfTy = typeSplit(VecTy).first;
2406 VectorPair Pair = opSplit(VmpyVec, dl, DAG);
2408 getInstr(Hexagon::V6_vconv_sf_qf32, dl, HalfTy, {Pair.first}, DAG);
2410 getInstr(Hexagon::V6_vconv_sf_qf32, dl, HalfTy, {Pair.second}, DAG);
2411
2413 getInstr(Hexagon::V6_vshuffvdd, dl, VecTy,
2415
2416 return ShuffVec;
2417}
2418
2421
2424
2425 MVT ResTy = ty(Op);
2426 MVT FpTy = ty(Op.getOperand(0)).getVectorElementType();
2428
2429 if (Subtarget.useHVXIEEEFPOps()) {
2430
2431 if (FpTy == MVT::f16) {
2432
2433 assert(IntTy == MVT::i8 || IntTy == MVT::i16 || IntTy == MVT::i32);
2434
2435 if (IntTy == MVT::i8 || IntTy == MVT::i16)
2436 return Op;
2437 }
2438 }
2439
2441 return EqualizeFpIntConversion(Op, DAG);
2442
2443 return ExpandHvxFpToInt(Op, DAG);
2444}
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456SDValue HexagonTargetLowering::LowerHvxPred32ToFp(SDValue PredOp,
2458
2459 MVT ResTy = ty(PredOp);
2460 const SDLoc &dl(PredOp);
2461
2463 SDNode *RegConst = DAG.getMachineNode(Hexagon::A2_tfrsi, dl, MVT::i32, Const);
2464 SDNode *SplatConst = DAG.getMachineNode(Hexagon::V6_lvsplatw, dl, MVT::v32i32,
2466 SDNode *PredTransfer =
2467 DAG.getMachineNode(Hexagon::V6_vandvrt, dl, MVT::v32i1,
2469 SDNode *PrefixSum = DAG.getMachineNode(Hexagon::V6_vprefixqw, dl, MVT::v32i32,
2470 SDValue(PredTransfer, 0));
2472 Hexagon::V6_lvsplatw, dl, MVT::v32i32,
2474 SDNode *Vsub =
2475 DAG.getMachineNode(Hexagon::V6_vsubw, dl, MVT::v32i32,
2477 SDNode *IndexShift =
2478 DAG.getMachineNode(Hexagon::V6_vlsrwv, dl, MVT::v32i32,
2480 SDNode *MaskOff =
2481 DAG.getMachineNode(Hexagon::V6_vand, dl, MVT::v32i32,
2483 SDNode *Convert = DAG.getMachineNode(Hexagon::V6_vconv_sf_w, dl, ResTy,
2485 return SDValue(Convert, 0);
2486}
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505SDValue HexagonTargetLowering::LowerHvxPred64ToFp(SDValue PredOp,
2507
2508 MVT ResTy = ty(PredOp);
2509 const SDLoc &dl(PredOp);
2510
2512
2517
2519 SDNode *RegConst = DAG.getMachineNode(Hexagon::A2_tfrsi, dl, MVT::i32, Const);
2520 SDNode *SplatConst = DAG.getMachineNode(Hexagon::V6_lvsplatw, dl, MVT::v32i32,
2522
2523 SDNode *SplatHi =
2524 DAG.getMachineNode(Hexagon::V6_lvsplatw, dl, MVT::v32i32,
2525 DAG.getNode(ISD::BITCAST, dl, MVT::i32, HiReg));
2526 SDNode *SplatLo =
2527 DAG.getMachineNode(Hexagon::V6_lvsplatw, dl, MVT::v32i32,
2528 DAG.getNode(ISD::BITCAST, dl, MVT::i32, LoReg));
2529
2530 SDNode *PredTransfer =
2531 DAG.getMachineNode(Hexagon::V6_vandvrt, dl, MVT::v32i1,
2533
2534 SDNode *PrefixSum = DAG.getMachineNode(Hexagon::V6_vprefixqw, dl, MVT::v32i32,
2535 SDValue(PredTransfer, 0));
2536
2537 SDNode *Vsub =
2538 DAG.getMachineNode(Hexagon::V6_vsubw, dl, MVT::v32i32,
2540
2541 SDNode *IndexShift_hi =
2542 DAG.getMachineNode(Hexagon::V6_vlsrwv, dl, MVT::v32i32,
2544 SDNode *IndexShift_lo =
2545 DAG.getMachineNode(Hexagon::V6_vlsrwv, dl, MVT::v32i32,
2547
2548 SDNode *MaskOff_hi =
2549 DAG.getMachineNode(Hexagon::V6_vand, dl, MVT::v32i32,
2551 SDNode *MaskOff_lo =
2552 DAG.getMachineNode(Hexagon::V6_vand, dl, MVT::v32i32,
2554
2555 SDNode *Pack =
2556 DAG.getMachineNode(Hexagon::V6_vpackeh, dl, MVT::v64i16,
2558 SDNode *Convert =
2560 return SDValue(Convert, 0);
2561}
2562
2565
2568
2569 MVT ResTy = ty(Op);
2572
2574 if (ResTy == MVT::v32f32 && ty(Op.getOperand(0)) == MVT::v32i1)
2575 return LowerHvxPred32ToFp(Op, DAG);
2576 if (ResTy == MVT::v64f16 && ty(Op.getOperand(0)) == MVT::v64i1)
2577 return LowerHvxPred64ToFp(Op, DAG);
2578 }
2579
2580 if (Subtarget.useHVXIEEEFPOps()) {
2581
2582 if (FpTy == MVT::f16) {
2583
2584 assert(IntTy == MVT::i8 || IntTy == MVT::i16 || IntTy == MVT::i32);
2585
2586 if (IntTy == MVT::i8 || IntTy == MVT::i16)
2587 return Op;
2588 }
2589 }
2590
2592 return EqualizeFpIntConversion(Op, DAG);
2593
2594 return ExpandHvxIntToFp(Op, DAG);
2595}
2596
2597HexagonTargetLowering::TypePair
2598HexagonTargetLowering::typeExtendToWider(MVT Ty0, MVT Ty1) const {
2599
2600
2601
2603
2606
2609 unsigned MaxWidth = std::max(Width0, Width1);
2610
2611 auto getScalarWithWidth = [](MVT ScalarTy, unsigned Width) {
2616 };
2617
2618 MVT WideETy0 = getScalarWithWidth(ElemTy0, MaxWidth);
2619 MVT WideETy1 = getScalarWithWidth(ElemTy1, MaxWidth);
2620
2622
2623 return {WideETy0, WideETy1};
2624 }
2625
2626
2629
2632}
2633
2634HexagonTargetLowering::TypePair
2635HexagonTargetLowering::typeWidenToWider(MVT Ty0, MVT Ty1) const {
2636
2637
2639
2642 if (Len0 == Len1)
2643 return {Ty0, Ty1};
2644
2645 unsigned MaxLen = std::max(Len0, Len1);
2648}
2649
2651HexagonTargetLowering::typeLegalize(MVT Ty, SelectionDAG &DAG) const {
2655}
2656
2658HexagonTargetLowering::typeWidenToHvx(MVT Ty) const {
2659 unsigned HwWidth = 8 * Subtarget.getVectorLength();
2662 return Ty;
2663
2666}
2667
2668HexagonTargetLowering::VectorPair
2669HexagonTargetLowering::emitHvxAddWithOverflow(SDValue A, SDValue B,
2671
2672
2673 MVT ResTy = ty(A);
2676
2678
2679
2682 return {Add, Ovf};
2683 }
2684
2685
2686
2687
2696 return {Add, MSB};
2697}
2698
2699HexagonTargetLowering::VectorPair
2700HexagonTargetLowering::emitHvxShiftRightRnd(SDValue Val, unsigned Amt,
2702
2703
2704
2705 const SDLoc &dl(Val);
2706 MVT ValTy = ty(Val);
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2720 MVT IntTy = tyVector(ValTy, ElemTy);
2723
2725 SDValue LowBits = DAG.getConstant((1ull << (Amt - 1)) - 1, dl, IntTy);
2726
2732 auto [Tmp0, Ovf] = emitHvxAddWithOverflow(Inp, LowBits, dl, Signed, DAG);
2733
2735 SDValue Tmp1 = DAG.getNode(ShRight, dl, IntTy, Inp, AmtM1);
2736 SDValue Tmp2 = DAG.getNode(ShRight, dl, IntTy, Tmp0, AmtM1);
2738
2741 SDValue Tmp4 = DAG.getNode(ShRight, dl, IntTy, {Tmp2, One});
2742 SDValue Tmp5 = DAG.getNode(ShRight, dl, IntTy, {Tmp3, One});
2744 return {Mux, Ovf};
2745}
2746
2750 MVT VecTy = ty(A);
2751 MVT PairTy = typeJoin({VecTy, VecTy});
2753
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777 SDValue T0 = getInstr(Hexagon::V6_vmpyewuh, dl, VecTy, {B, A}, DAG);
2778
2779 SDValue T1 = getInstr(Hexagon::V6_vasrw, dl, VecTy, {A, S16}, DAG);
2780
2781 SDValue P0 = getInstr(Hexagon::V6_vmpyhus, dl, PairTy, {T1, B}, DAG);
2782
2783 SDValue T2 = LoHalf(P0, DAG);
2784
2785
2786
2787 SDValue P1 = getInstr(Hexagon::V6_vadduhw, dl, PairTy, {T0, T2}, DAG);
2788
2789 SDValue P2 = getInstr(Hexagon::V6_vaddhw, dl, PairTy, {T0, T2}, DAG);
2790
2791
2792 SDValue T3 = getInstr(Hexagon::V6_vasrw_acc, dl, VecTy,
2793 {HiHalf(P2, DAG), LoHalf(P1, DAG), S16}, DAG);
2794 SDValue T4 = getInstr(Hexagon::V6_vasrw, dl, VecTy, {B, S16}, DAG);
2795
2796
2797 SDValue P3 = getInstr(Hexagon::V6_vmpyhv, dl, PairTy, {T1, T4}, DAG);
2798 SDValue T5 = LoHalf(P3, DAG);
2799
2801 return T6;
2802}
2803
2805HexagonTargetLowering::emitHvxMulLoHiV60(SDValue A, bool SignedA, SDValue B,
2806 bool SignedB, const SDLoc &dl,
2808 MVT VecTy = ty(A);
2809 MVT PairTy = typeJoin({VecTy, VecTy});
2811
2813
2814 if (SignedA && !SignedB) {
2815
2818 }
2819
2820
2821
2822
2824
2825
2826
2827 SDValue P0 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {A, B}, DAG);
2828
2829
2830 SDValue T0 = getInstr(Hexagon::V6_lvsplatw, dl, VecTy,
2831 {DAG.getConstant(0x02020202, dl, MVT::i32)}, DAG);
2832 SDValue T1 = getInstr(Hexagon::V6_vdelta, dl, VecTy, {B, T0}, DAG);
2833
2834
2835
2836 SDValue P1 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {A, T1}, DAG);
2837
2838
2839
2840 SDValue P2 = getInstr(Hexagon::V6_vadduhw, dl, PairTy,
2841 {HiHalf(P1, DAG), LoHalf(P1, DAG)}, DAG);
2842
2844 getInstr(Hexagon::V6_vlsrw, dl, VecTy, {LoHalf(P0, DAG), S16}, DAG);
2846
2847
2848 SDValue T4 = getInstr(Hexagon::V6_vasrw_acc, dl, VecTy,
2849 {HiHalf(P2, DAG), T3, S16}, DAG);
2850
2851
2852 Lo = getInstr(Hexagon::V6_vaslw_acc, dl, VecTy,
2853 {LoHalf(P0, DAG), LoHalf(P2, DAG), S16}, DAG);
2855
2856 if (SignedA) {
2857 assert(SignedB && "Signed A and unsigned B should have been inverted");
2858
2864 SDValue X1 = getInstr(Hexagon::V6_vaddwq, dl, VecTy, {Q1, X0, A}, DAG);
2865 Hi = getInstr(Hexagon::V6_vsubw, dl, VecTy, {Hi, X1}, DAG);
2866 } else if (SignedB) {
2867
2868
2872 Hi = getInstr(Hexagon::V6_vsubwq, dl, VecTy, {Q1, Hi, A}, DAG);
2873 } else {
2874 assert(!SignedA && !SignedB);
2875 }
2876
2878}
2879
2881HexagonTargetLowering::emitHvxMulLoHiV62(SDValue A, bool SignedA,
2883 const SDLoc &dl,
2885 MVT VecTy = ty(A);
2886 MVT PairTy = typeJoin({VecTy, VecTy});
2888
2889 if (SignedA && !SignedB) {
2890
2893 }
2894
2895
2896 SDValue P0 = getInstr(Hexagon::V6_vmpyewuh_64, dl, PairTy, {A, B}, DAG);
2898 getInstr(Hexagon::V6_vmpyowh_64_acc, dl, PairTy, {P0, A, B}, DAG);
2901
2902 if (!SignedB) {
2903 assert(!SignedA && "Signed A and unsigned B should have been inverted");
2906
2907
2908
2909
2910
2911
2912
2915 SDValue T0 = getInstr(Hexagon::V6_vandvqv, dl, VecTy, {Q0, B}, DAG);
2916 SDValue T1 = getInstr(Hexagon::V6_vaddwq, dl, VecTy, {Q1, T0, A}, DAG);
2917 Hi = getInstr(Hexagon::V6_vaddw, dl, VecTy, {Hi, T1}, DAG);
2918 } else if (!SignedA) {
2921
2922
2923
2924
2925
2926
2928 Hi = getInstr(Hexagon::V6_vaddwq, dl, VecTy, {Q0, Hi, B}, DAG);
2929 }
2930
2932}
2933
2936 const {
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946 unsigned Opc = Op.getOpcode();
2949
2951 MVT InpTy = ty(Inp);
2952 MVT ResTy = ty(Op);
2953
2954 if (InpTy == ResTy)
2955 return Op;
2956
2957 const SDLoc &dl(Op);
2959
2960 auto [WInpTy, WResTy] = typeExtendToWider(InpTy, ResTy);
2961 SDValue WInp = resizeToWidth(Inp, WInpTy, Signed, dl, DAG);
2963 SDValue Res = resizeToWidth(Conv, ResTy, Signed, dl, DAG);
2964 return Res;
2965}
2966
2969 unsigned Opc = Op.getOpcode();
2971
2972 const SDLoc &dl(Op);
2974 MVT InpTy = ty(Op0);
2975 MVT ResTy = ty(Op);
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987 if (InpTy == MVT::v64f16) {
2988 if (Subtarget.useHVXV81Ops()) {
2989
2991 getInstr(Hexagon::V6_vconv_h_hf_rnd, dl, ResTy, {Op0}, DAG);
2992 return ConvVec;
2994
2995 SDValue ConvVec = getInstr(Hexagon::V6_vconv_h_hf, dl, ResTy, {Op0}, DAG);
2996 return ConvVec;
2997 }
2999
3000 SDValue ConvVec = getInstr(Hexagon::V6_vconv_w_sf, dl, ResTy, {Op0}, DAG);
3001 return ConvVec;
3002 }
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3080 unsigned ElemWidth = 1 + ExpWidth + FracWidth;
3081 assert((1ull << (ExpWidth - 1)) == (1 + ExpBias));
3082
3087 SDValue M7F = DAG.getConstant((1ull << (ElemWidth - 1)) - 1, dl, ResTy);
3093
3097
3103
3105
3112 } else {
3117 }
3118
3119 return Int;
3120}
3121
3124 unsigned Opc = Op.getOpcode();
3126
3127 const SDLoc &dl(Op);
3129 MVT InpTy = ty(Op0);
3130 MVT ResTy = ty(Op);
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3160
3162 unsigned ElemWidth = 1 + ExpWidth + FracWidth;
3163
3171
3172 auto [Frac, Ovf] = emitHvxShiftRightRnd(Frac0, ExpWidth + 1, false, DAG);
3177 Frac = DAG.getNode(ISD::OR, dl, InpTy, {Sign, Frac});
3178 }
3179
3185 {Exp2, DAG.getConstant(FracWidth, dl, InpTy)});
3189
3190 return Flt;
3191}
3192
3195 unsigned Opc = Op.getOpcode();
3196 unsigned TLOpc;
3197 switch (Opc) {
3202 break;
3205 break;
3206#ifndef NDEBUG
3207 Op.dump(&DAG);
3208#endif
3210 }
3211
3212 const SDLoc &dl(Op);
3213 return DAG.getNode(TLOpc, dl, ty(Op), Op.getOperand(0),
3214 DAG.getUNDEF(MVT::i128),
3216}
3217
3222 unsigned Opc = Op.getConstantOperandVal(2);
3223 return DAG.getNode(Opc, SDLoc(Op), ty(Op), Op.getOperand(0));
3224}
3225
3226HexagonTargetLowering::VectorPair
3228 assert(.isMachineOpcode());
3230 const SDLoc &dl(Op);
3231
3232 auto SplitVTNode = [&DAG, this](const VTSDNode *N) {
3233 MVT Ty = typeSplit(N->getVT().getSimpleVT()).first;
3235 return std::make_pair(TV, TV);
3236 };
3237
3238 for (SDValue A : Op.getNode()->ops()) {
3240 ty(A).isVector() ? opSplit(A, dl, DAG) : std::make_pair(A, A);
3241
3242 switch (Op.getOpcode()) {
3243 case ISD::SIGN_EXTEND_INREG:
3244 case HexagonISD::SSAT:
3245 case HexagonISD::USAT:
3246 if (const auto *N = dyn_cast(A.getNode()))
3247 std::tie(Lo, Hi) = SplitVTNode(N);
3248 break;
3249 }
3252 }
3253
3254 MVT ResTy = ty(Op);
3255 MVT HalfTy = typeSplit(ResTy).first;
3259}
3260
3264
3265 if (!MemN->getMemoryVT().isSimple())
3266 return Op;
3267
3268 MVT MemTy = MemN->getMemoryVT().getSimpleVT();
3269 if (!isHvxPairTy(MemTy))
3270 return Op;
3271
3272 const SDLoc &dl(Op);
3273 unsigned HwLen = Subtarget.getVectorLength();
3274 MVT SingleTy = typeSplit(MemTy).first;
3275 SDValue Chain = MemN->getChain();
3276 SDValue Base0 = MemN->getBasePtr();
3279 unsigned MemOpc = MemN->getOpcode();
3280
3281 MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr;
3282 if (MachineMemOperand *MMO = MemN->getMemOperand()) {
3284 uint64_t MemSize = (MemOpc == ISD::MLOAD || MemOpc == ISD::MSTORE)
3286 : HwLen;
3289 }
3290
3291 if (MemOpc == ISD::LOAD) {
3293 SDValue Load0 = DAG.getLoad(SingleTy, dl, Chain, Base0, MOp0);
3294 SDValue Load1 = DAG.getLoad(SingleTy, dl, Chain, Base1, MOp1);
3299 }
3300 if (MemOpc == ISD::STORE) {
3303 SDValue Store0 = DAG.getStore(Chain, dl, Vals.first, Base0, MOp0);
3304 SDValue Store1 = DAG.getStore(Chain, dl, Vals.second, Base1, MOp1);
3306 }
3307
3308 assert(MemOpc == ISD::MLOAD || MemOpc == ISD::MSTORE);
3309
3311 assert(MaskN->isUnindexed());
3312 VectorPair Masks = opSplit(MaskN->getMask(), dl, DAG);
3314
3315 if (MemOpc == ISD::MLOAD) {
3316 VectorPair Thru =
3330 }
3331 if (MemOpc == ISD::MSTORE) {
3334 Masks.first, SingleTy, MOp0,
3337 Masks.second, SingleTy, MOp1,
3340 }
3341
3342 std::string Name = "Unexpected operation: " + Op->getOperationName(&DAG);
3344}
3345
3348 const SDLoc &dl(Op);
3350 assert(LoadN->isUnindexed() && "Not widening indexed loads yet");
3351 assert(LoadN->getMemoryVT().getVectorElementType() != MVT::i1 &&
3352 "Not widening loads of i1 yet");
3353
3354 SDValue Chain = LoadN->getChain();
3357
3358 MVT ResTy = ty(Op);
3359 unsigned HwLen = Subtarget.getVectorLength();
3361 assert(ResLen < HwLen && "vsetq(v1) prerequisite");
3362
3364 SDValue Mask = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
3365 {DAG.getConstant(ResLen, dl, MVT::i32)}, DAG);
3366
3370
3372 DAG.getUNDEF(LoadTy), LoadTy, MemOp,
3376}
3377
3380 const SDLoc &dl(Op);
3382 assert(StoreN->isUnindexed() && "Not widening indexed stores yet");
3383 assert(StoreN->getMemoryVT().getVectorElementType() != MVT::i1 &&
3384 "Not widening stores of i1 yet");
3385
3386 SDValue Chain = StoreN->getChain();
3389
3390 SDValue Value = opCastElem(StoreN->getValue(), MVT::i8, DAG);
3391 MVT ValueTy = ty(Value);
3393 unsigned HwLen = Subtarget.getVectorLength();
3395
3396 for (unsigned Len = ValueLen; Len < HwLen; ) {
3398 Len = ty(Value).getVectorNumElements();
3399 }
3400 assert(ty(Value).getVectorNumElements() == HwLen);
3401
3402 assert(ValueLen < HwLen && "vsetq(v1) prerequisite");
3404 SDValue Mask = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
3405 {DAG.getConstant(ValueLen, dl, MVT::i32)}, DAG);
3407 auto *MemOp = MF.getMachineMemOperand(StoreN->getMemOperand(), 0, HwLen);
3410}
3411
3414 const SDLoc &dl(Op);
3415 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
3417 unsigned HwLen = Subtarget.getVectorLength();
3418
3419 unsigned WideOpLen = (8 * HwLen) / ElemTy.getSizeInBits();
3422 if (!Subtarget.isHVXVectorType(WideOpTy, true))
3424
3425 SDValue WideOp0 = appendUndef(Op0, WideOpTy, DAG);
3426 SDValue WideOp1 = appendUndef(Op1, WideOpTy, DAG);
3427 EVT ResTy =
3430 {WideOp0, WideOp1, Op.getOperand(2)});
3431
3432 EVT RetTy = typeLegalize(ty(Op), DAG);
3434 {SetCC, getZero(dl, MVT::i32, DAG)});
3435}
3436
3439 const SDLoc &dl(Op);
3440 unsigned HwWidth = 8 * Subtarget.getVectorLength();
3441 bool IsResInterleaved = false;
3442
3445 MVT ResTy = ty(Op);
3446 MVT OpTy = ty(Op1);
3447 if (!Subtarget.isHVXElementType(OpTy) || !Subtarget.isHVXElementType(ResTy))
3449
3450 auto getFactor = [HwWidth](MVT Ty) {
3452 assert(HwWidth % Width == 0);
3453 return HwWidth / Width;
3454 };
3455
3456 auto getWideTy = [getFactor](MVT Ty) {
3459 };
3460
3463 SDValue WideOp1 = appendUndef(Op1, getWideTy(OpTy), DAG);
3466 WideOp2 = Op2;
3467 } else {
3468 WideOp2 = appendUndef(Op2, getWideTy(OpTy), DAG);
3469 }
3470 unsigned WidenFactor = getFactor(OpTy);
3473
3474 switch (IID) {
3475 default:
3477 case Intrinsic::hexagon_vasrsat_su:
3478 case Intrinsic::hexagon_vasrsat_uu:
3479 case Intrinsic::hexagon_vasrsat_ss:
3481 DAG.getConstant(IID, dl, MVT::i32), WideOp1, WideOp2,
3482 Op.getOperand(3));
3483 break;
3484 case Intrinsic::hexagon_vadd_su:
3485 case Intrinsic::hexagon_vadd_uu:
3486 case Intrinsic::hexagon_vadd_ss:
3487 case Intrinsic::hexagon_vadd_us:
3488
3489 case Intrinsic::hexagon_vsub_su:
3490 case Intrinsic::hexagon_vsub_uu:
3491 case Intrinsic::hexagon_vsub_ss:
3492 case Intrinsic::hexagon_vsub_us:
3493
3494 case Intrinsic::hexagon_vmpy_su:
3495 case Intrinsic::hexagon_vmpy_uu:
3496 case Intrinsic::hexagon_vmpy_ss:
3497 case Intrinsic::hexagon_vmpy_us:
3498 case Intrinsic::hexagon_vmpy_ub_ub:
3499 case Intrinsic::hexagon_vmpy_ub_b:
3500 case Intrinsic::hexagon_vmpy_uh_uh:
3501 case Intrinsic::hexagon_vmpy_h_h:
3502 IsResInterleaved = true;
3504 DAG.getConstant(IID, dl, MVT::i32), WideOp1, WideOp2);
3505 break;
3506 case Intrinsic::hexagon_vavgu:
3507 case Intrinsic::hexagon_vavgs:
3509 DAG.getConstant(IID, dl, MVT::i32), WideOp1, WideOp2);
3510 break;
3511 }
3513 assert(OrigLen % 2 == 0);
3514 unsigned HalfOrigLen = OrigLen / 2;
3515 unsigned SplitLen = WideLen / 2;
3516 if (IsResInterleaved) {
3517
3518
3519
3520
3521 SmallVector<int, 128> ShuffV;
3522 for (unsigned j = 0; j < WidenFactor; j++) {
3523 for (unsigned i = 0; i < HalfOrigLen; i++)
3524 ShuffV.push_back(j * HalfOrigLen + i);
3525 for (unsigned i = 0; i < HalfOrigLen; i++)
3526 ShuffV.push_back(SplitLen + j * HalfOrigLen + i);
3527 }
3529 DAG.getUNDEF(WideResTy), ShuffV);
3530 }
3532 {WideRes, getZero(dl, MVT::i32, DAG)});
3533}
3534
3537 unsigned Opc = Op.getOpcode();
3538 bool IsPairOp = isHvxPairTy(ty(Op)) ||
3540 return isHvxPairTy(ty(V));
3541 });
3542
3543 if (IsPairOp) {
3544 switch (Opc) {
3545 default:
3546 break;
3547 case ISD::LOAD:
3548 case ISD::STORE:
3549 case ISD::MLOAD:
3550 case ISD::MSTORE:
3551 return SplitHvxMemOp(Op, DAG);
3556 if (ty(Op).getSizeInBits() == ty(Op.getOperand(0)).getSizeInBits())
3557 return opJoin(SplitVectorOp(Op, DAG), SDLoc(Op), DAG);
3558 break;
3567 case ISD::FMINIMUMNUM:
3568 case ISD::FMAXIMUMNUM:
3587 return opJoin(SplitVectorOp(Op, DAG), SDLoc(Op), DAG);
3590
3591
3592 if (ty(Op.getOperand(0)).getVectorElementType() == MVT::i1)
3593 return opJoin(SplitVectorOp(Op, DAG), SDLoc(Op), DAG);
3594 break;
3595 }
3596 }
3597
3598 switch (Opc) {
3599 default:
3600 break;
3608 case ISD::BITCAST: return LowerHvxBitcast(Op, DAG);
3612 case ISD::CTTZ: return LowerHvxCttz(Op, DAG);
3613 case ISD::SELECT: return LowerHvxSelect(Op, DAG);
3616 case ISD::SRL: return LowerHvxShift(Op, DAG);
3618 case ISD::FSHR: return LowerHvxFunnelShift(Op, DAG);
3620 case ISD::MULHU: return LowerHvxMulh(Op, DAG);
3627 case ISD::MLOAD:
3628 case ISD::MSTORE: return LowerHvxMaskedOp(Op, DAG);
3629
3630 case ISD::LOAD: return SDValue();
3631 case ISD::FP_EXTEND: return LowerHvxFpExtend(Op, DAG);
3636
3637
3641 }
3642#ifndef NDEBUG
3643 Op.dumpr(&DAG);
3644#endif
3646}
3647
3650 const {
3651
3652
3653
3654
3655
3656
3657 unsigned Opc = Op.getOpcode();
3658 switch (Opc) {
3663 break;
3669 break;
3670#ifndef NDEBUG
3671 Op.dump(&DAG);
3672#endif
3674 }
3675
3677 MVT InpTy = ty(Inp);
3678 MVT ResTy = ty(Op);
3679
3682 assert(InpWidth != ResWidth);
3683
3684 if (InpWidth == 2 * ResWidth || ResWidth == 2 * InpWidth)
3685 return Op;
3686
3687 const SDLoc &dl(Op);
3690
3691 auto repeatOp = [&](unsigned NewWidth, SDValue Arg) {
3693 switch (Opc) {
3699 return DAG.getNode(Opc, dl, Ty, {Arg, Op.getOperand(1), Op.getOperand(2)});
3700 default:
3702 }
3703 };
3704
3706 if (InpWidth < ResWidth) {
3708 while (InpWidth * 2 <= ResWidth)
3709 S = repeatOp(InpWidth *= 2, S);
3710 } else {
3711
3713 while (InpWidth / 2 >= ResWidth)
3714 S = repeatOp(InpWidth /= 2, S);
3715 }
3716 return S;
3717}
3718
3721 SDValue Inp0 = Op.getOperand(0);
3722 MVT InpTy = ty(Inp0);
3723 MVT ResTy = ty(Op);
3726 unsigned Opc = Op.getOpcode();
3727
3728 if (shouldWidenToHvx(InpTy, DAG) || shouldWidenToHvx(ResTy, DAG)) {
3729
3730
3731
3732
3733 auto [WInpTy, WResTy] =
3734 InpWidth < ResWidth ? typeWidenToWider(typeWidenToHvx(InpTy), ResTy)
3735 : typeWidenToWider(InpTy, typeWidenToHvx(ResTy));
3736 SDValue W = appendUndef(Inp0, WInpTy, DAG);
3739 S = DAG.getNode(Opc, SDLoc(Op), WResTy, W, Op.getOperand(1),
3740 Op.getOperand(2));
3741 } else {
3743 }
3744 SDValue T = ExpandHvxResizeIntoSteps(S, DAG);
3745 return extractSubvector(T, typeLegalize(ResTy, DAG), 0, DAG);
3746 } else if (shouldSplitToHvx(InpWidth < ResWidth ? ResTy : InpTy, DAG)) {
3747 return opJoin(SplitVectorOp(Op, DAG), SDLoc(Op), DAG);
3748 } else {
3750 return RemoveTLWrapper(Op, DAG);
3751 }
3753}
3754
3755void
3756HexagonTargetLowering::LowerHvxOperationWrapper(SDNode *N,
3758 unsigned Opc = N->getOpcode();
3760 SDValue Inp0;
3761 if (N->getNumOperands() > 0)
3762 Inp0 = Op.getOperand(0);
3763
3764 switch (Opc) {
3769 if (Subtarget.isHVXElementType(ty(Op)) &&
3770 Subtarget.isHVXElementType(ty(Inp0))) {
3771 Results.push_back(CreateTLWrapper(Op, DAG));
3772 }
3773 break;
3775 if (shouldWidenToHvx(ty(Inp0), DAG)) {
3776 if (SDValue T = WidenHvxSetCC(Op, DAG))
3778 }
3779 break;
3780 case ISD::STORE: {
3783 Results.push_back(Store);
3784 }
3785 break;
3786 }
3787 case ISD::MLOAD:
3788 if (isHvxPairTy(ty(Op))) {
3789 SDValue S = SplitHvxMemOp(Op, DAG);
3793 }
3794 break;
3795 case ISD::MSTORE:
3796 if (isHvxPairTy(ty(Op->getOperand(1)))) {
3797 SDValue S = SplitHvxMemOp(Op, DAG);
3799 }
3800 break;
3802 if (shouldWidenToHvx(ty(Op.getOperand(1)), DAG)) {
3803 if (SDValue T = WidenHvxIntrinsic(Op, DAG))
3805 }
3806 break;
3811 if (ty(Op).getSizeInBits() != ty(Inp0).getSizeInBits()) {
3812 SDValue T = EqualizeFpIntConversion(Op, DAG);
3814 }
3815 break;
3820 Results.push_back(LegalizeHvxResize(Op, DAG));
3821 break;
3822 default:
3823 break;
3824 }
3825}
3826
3827void
3828HexagonTargetLowering::ReplaceHvxNodeResults(SDNode *N,
3830 unsigned Opc = N->getOpcode();
3832 SDValue Inp0;
3833 if (N->getNumOperands() > 0)
3834 Inp0 = Op.getOperand(0);
3835
3836 switch (Opc) {
3841 if (Subtarget.isHVXElementType(ty(Op)) &&
3842 Subtarget.isHVXElementType(ty(Inp0))) {
3843 Results.push_back(CreateTLWrapper(Op, DAG));
3844 }
3845 break;
3847 if (shouldWidenToHvx(ty(Op), DAG)) {
3848 if (SDValue T = WidenHvxSetCC(Op, DAG))
3850 }
3851 break;
3852 case ISD::LOAD: {
3853 if (shouldWidenToHvx(ty(Op), DAG)) {
3858 }
3859 break;
3860 }
3861 case ISD::BITCAST:
3862 if (isHvxBoolTy(ty(Inp0))) {
3863 SDValue C = LowerHvxBitcast(Op, DAG);
3865 }
3866 break;
3868 assert(shouldWidenToHvx(ty(N->getOperand(1)), DAG) && "Not widening?");
3869 if (SDValue T = WidenHvxIntrinsic(Op, DAG))
3871 break;
3874 if (ty(Op).getSizeInBits() != ty(Inp0).getSizeInBits()) {
3875 SDValue T = EqualizeFpIntConversion(Op, DAG);
3877 }
3878 break;
3883 Results.push_back(LegalizeHvxResize(Op, DAG));
3884 break;
3885 default:
3886 break;
3887 }
3888}
3889
3891HexagonTargetLowering::combineTruncateBeforeLegal(SDValue Op,
3892 DAGCombinerInfo &DCI) const {
3893
3894
3895
3897 SelectionDAG &DAG = DCI.DAG;
3898 const SDLoc &dl(Op);
3899
3900 if (Op.getOperand(0).getOpcode() == ISD::BITCAST)
3902 SDValue Cast = Op.getOperand(0);
3904
3905 EVT TruncTy = Op.getValueType();
3907 EVT SrcTy = Src.getValueType();
3914 if (2 * CastLen != SrcLen)
3916
3917 SmallVector<int, 128> Mask(SrcLen);
3918 for (int i = 0; i != static_cast<int>(CastLen); ++i) {
3919 Mask[i] = 2 * i;
3920 Mask[i + CastLen] = 2 * i + 1;
3921 }
3924 return opSplit(Deal, dl, DAG).first;
3925}
3926
3928HexagonTargetLowering::combineConcatVectorsBeforeLegal(
3929 SDValue Op, DAGCombinerInfo &DCI) const {
3930
3931
3932
3933
3936
3937 SelectionDAG &DAG = DCI.DAG;
3938 const SDLoc &dl(Op);
3941
3946
3947 SetVector Order;
3952
3953 if (Order.size() > 2)
3955
3956
3957
3961
3962 SmallVector<int, 128> LongMask;
3963 auto AppendToMask = [&](SDValue Shuffle) {
3965 ArrayRef Mask = SV->getMask();
3966 SDValue X = Shuffle.getOperand(0);
3967 SDValue Y = Shuffle.getOperand(1);
3968 for (int M : Mask) {
3969 if (M == -1) {
3971 continue;
3972 }
3973 SDValue Src = static_cast<unsigned>(M) < InpLen ? X : Y;
3974 if (static_cast<unsigned>(M) >= InpLen)
3975 M -= InpLen;
3976
3977 int OutOffset = Order[0] == Src ? 0 : InpLen;
3978 LongMask.push_back(M + OutOffset);
3979 }
3980 };
3981
3982 AppendToMask(V0);
3983 AppendToMask(V1);
3984
3986 SDValue C1 = Order.back();
3988
3991}
3992
3994HexagonTargetLowering::PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
3995 const {
3996 const SDLoc &dl(N);
3997 SelectionDAG &DAG = DCI.DAG;
3999 unsigned Opc = Op.getOpcode();
4000
4002
4004 return combineTruncateBeforeLegal(Op, DCI);
4006 return combineConcatVectorsBeforeLegal(Op, DCI);
4007
4008 if (DCI.isBeforeLegalizeOps())
4010
4011 switch (Opc) {
4013
4019 }
4020 break;
4021 }
4027 }
4028 break;
4034 return getZero(dl, ty(Op), DAG);
4035 break;
4037 if (isUndef(Ops[1]))
4038 return Ops[0];
4039 break;
4046 }
4047 break;
4048 }
4049 }
4050
4052}
4053
4054bool
4055HexagonTargetLowering::shouldSplitToHvx(MVT Ty, SelectionDAG &DAG) const {
4056 if (Subtarget.isHVXVectorType(Ty, true))
4057 return false;
4058 auto Action = getPreferredHvxVectorAction(Ty);
4060 return Subtarget.isHVXVectorType(typeLegalize(Ty, DAG), true);
4061 return false;
4062}
4063
4064bool
4065HexagonTargetLowering::shouldWidenToHvx(MVT Ty, SelectionDAG &DAG) const {
4066 if (Subtarget.isHVXVectorType(Ty, true))
4067 return false;
4068 auto Action = getPreferredHvxVectorAction(Ty);
4070 return Subtarget.isHVXVectorType(typeLegalize(Ty, DAG), true);
4071 return false;
4072}
4073
4074bool
4075HexagonTargetLowering::isHvxOperation(SDNode *N, SelectionDAG &DAG) const {
4076 if (!Subtarget.useHVXOps())
4077 return false;
4078
4079
4080 auto IsHvxTy = [this](EVT Ty) {
4081 return Ty.isSimple() && Subtarget.isHVXVectorType(Ty.getSimpleVT(), true);
4082 };
4083 auto IsHvxOp = [this](SDValue Op) {
4084 return Op.getValueType().isSimple() &&
4085 Subtarget.isHVXVectorType(ty(Op), true);
4086 };
4088 return true;
4089
4090
4091 auto IsWidenedToHvx = [this, &DAG](SDValue Op) {
4092 if (.getValueType().isSimple())
4093 return false;
4094 MVT ValTy = ty(Op);
4095 return ValTy.isVector() && shouldWidenToHvx(ValTy, DAG);
4096 };
4097
4098 for (int i = 0, e = N->getNumValues(); i != e; ++i) {
4099 if (IsWidenedToHvx(SDValue(N, i)))
4100 return true;
4101 }
4103}
unsigned const MachineRegisterInfo * MRI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const TargetInstrInfo & TII
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static std::tuple< unsigned, unsigned, unsigned > getIEEEProperties(MVT Ty)
Definition HexagonISelLoweringHVX.cpp:43
static const MVT LegalV128[]
Definition HexagonISelLoweringHVX.cpp:40
static const MVT LegalW128[]
Definition HexagonISelLoweringHVX.cpp:41
static const MVT LegalW64[]
Definition HexagonISelLoweringHVX.cpp:39
static const MVT LegalV64[]
Definition HexagonISelLoweringHVX.cpp:38
static cl::opt< unsigned > HvxWidenThreshold("hexagon-hvx-widen", cl::Hidden, cl::init(16), cl::desc("Lower threshold (in bytes) for widening to HVX vectors"))
static cl::opt< bool > EnableFpFastConvert("hexagon-fp-fast-convert", cl::Hidden, cl::init(false), cl::desc("Enable FP fast conversion routine."))
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
Promote Memory to Register
This file provides utility analysis objects describing memory locations.
const SmallVectorImpl< MachineOperand > & Cond
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallVector class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static llvm::Type * getVectorElementType(llvm::Type *Ty)
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEhalf()
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
static LLVM_ABI Constant * get(ArrayRef< Constant * > V)
uint64_t getNumOperands() const
SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const
EVT getSetCCResultType(const DataLayout &, LLVMContext &C, EVT VT) const override
Return the ValueType of the result of SETCC operations.
LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Return the preferred vector type legalization action.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
static MVT getFloatingPointVT(unsigned BitWidth)
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
MVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
LLVM_ABI instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
Flags
Flags values. These may be or'd together.
unsigned getSubReg() const
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
Register getReg() const
getReg - Returns the register number.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
LLVM_ABI SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
LLVM_ABI std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
LLVM_ABI SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
MachineFunction & getMachineFunction() const
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVMContext * getContext() const
LLVM_ABI SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
LLVM_ABI SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
size_type size() const
Determine the number of elements in the SetVector.
const value_type & front() const
Return the first element of the SetVector.
const value_type & back() const
Return the last element of the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ ADD
Simple integer binary arithmetic operators.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ SIGN_EXTEND
Conversion operators.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
@ Undef
Value of the register doesn't matter.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
FunctionAddr VTableAddr Value
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
decltype(auto) dyn_cast(const From &Val)
dyn_cast - Return the argument parameter cast to the specified type.
SmallVectorImpl< T >::const_pointer c_str(SmallVectorImpl< T > &str)
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
@ Or
Bitwise or logical OR of integers.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast - Return the argument parameter cast to the specified type.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool isVector() const
Return true if this is a vector value type.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
static LLVM_ABI MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.