LLVM: lib/Target/Hexagon/HexagonISelLoweringHVX.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
21#include "llvm/IR/IntrinsicsHexagon.h"
23
24#include
25#include
26#include
27
28using namespace llvm;
29
32 cl::desc("Lower threshold (in bytes) for widening to HVX vectors"));
33
36 cl::desc("Enable FP fast conversion routine."));
37
38static const MVT LegalV64[] = { MVT::v64i8, MVT::v32i16, MVT::v16i32 };
39static const MVT LegalW64[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
40static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
41static const MVT LegalW128[] = { MVT::v256i8, MVT::v128i16, MVT::v64i32 };
42
44
46 switch (ElemTy.SimpleTy) {
47 case MVT::f16:
48 return std::make_tuple(5, 15, 10);
49 case MVT::f32:
50 return std::make_tuple(8, 127, 23);
51 case MVT::f64:
52 return std::make_tuple(11, 1023, 52);
53 default:
54 break;
55 }
57}
58
59void
60HexagonTargetLowering::initializeHVXLowering() {
61 if (Subtarget.useHVX64BOps()) {
68
69
70
71
72
73
74
75
79 } else if (Subtarget.useHVX128BOps()) {
89 if (Subtarget.useHVXV68Ops() && Subtarget.useHVXFloatingPoint()) {
94 }
95 if (Subtarget.useHVXV81Ops()) {
98 }
99 }
100
101
102
103 bool Use64b = Subtarget.useHVX64BOps();
106 MVT ByteV = Use64b ? MVT::v64i8 : MVT::v128i8;
107 MVT WordV = Use64b ? MVT::v16i32 : MVT::v32i32;
108 MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8;
109
110 auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) {
113 };
114
115
116
117
127
128 if (Subtarget.useHVX128BOps()) {
131 }
132 if (Subtarget.useHVX128BOps() && Subtarget.useHVXV68Ops() &&
133 Subtarget.useHVXFloatingPoint()) {
134
135 static const MVT FloatV[] = { MVT::v64f16, MVT::v32f32 };
136 static const MVT FloatW[] = { MVT::v128f16, MVT::v64f32 };
137
138 for (MVT T : FloatV) {
144
147
150
153
154
155
157 }
158
159
160
161
165
166
167
172
173 if (Subtarget.useHVXV81Ops()) {
176 setPromoteTo(ISD::SETCC, MVT::v64bf16, MVT::v64f32);
177 setPromoteTo(ISD::FADD, MVT::v64bf16, MVT::v64f32);
178 setPromoteTo(ISD::FSUB, MVT::v64bf16, MVT::v64f32);
179 setPromoteTo(ISD::FMUL, MVT::v64bf16, MVT::v64f32);
180 setPromoteTo(ISD::FMINNUM, MVT::v64bf16, MVT::v64f32);
181 setPromoteTo(ISD::FMAXNUM, MVT::v64bf16, MVT::v64f32);
182
186
189
194
199
203 }
204
205 for (MVT P : FloatW) {
215
216
217
218
220
222
225 }
226
227 if (Subtarget.useHVXQFloatOps()) {
230 } else if (Subtarget.useHVXIEEEFPOps()) {
233 }
234 }
235
236 for (MVT T : LegalV) {
239
255 if (T != ByteV) {
259 }
260
263 if (T.getScalarType() != MVT::i32) {
266 }
267
272 if (T.getScalarType() != MVT::i32) {
275 }
276
278
289 if (T != ByteV) {
291
295
296
298 }
299
300 if (Subtarget.useHVXFloatingPoint()) {
301
306 }
307
315 }
316
317 for (MVT T : LegalW) {
318
319
320
322
324
325
326
335
344
359 if (T != ByteW) {
363
364
366 }
369
372 if (T.getScalarType() != MVT::i32) {
375 }
376
377 if (Subtarget.useHVXFloatingPoint()) {
378
383 }
384 }
385
386
391
406
421
422
423
424 for (MVT T : LegalW) {
425
426
427
428
429
430
436
439 }
440
441 for (MVT T : LegalV) {
453 }
454
455 if (Use64b) {
456 for (MVT T: {MVT::v32i8, MVT::v32i16, MVT::v16i8, MVT::v16i16, MVT::v16i32})
458 } else {
459 for (MVT T: {MVT::v64i8, MVT::v64i16, MVT::v32i8, MVT::v32i16, MVT::v32i32})
461 }
462
463
464 unsigned HwLen = Subtarget.getVectorLength();
465 for (MVT ElemTy : Subtarget.getHVXElementTypes()) {
466 if (ElemTy == MVT::i1)
467 continue;
468 int ElemWidth = ElemTy.getFixedSizeInBits();
469 int MaxElems = (8*HwLen) / ElemWidth;
470 for (int N = 2; N < MaxElems; N *= 2) {
481 if (Subtarget.useHVXFloatingPoint()) {
486 }
487
491 }
492 }
493 }
494
495
499
501}
502
503unsigned
504HexagonTargetLowering::getPreferredHvxVectorAction(MVT VecTy) const {
505
507 return ~0u;
508
511 unsigned HwLen = Subtarget.getVectorLength();
512
513
514 if (ElemTy == MVT::i1 && VecLen > HwLen)
516
517 ArrayRef Tys = Subtarget.getHVXElementTypes();
518
519
520 if (ElemTy == MVT::i1) {
521 for (MVT T : Tys) {
524 if (A != ~0u)
525 return A;
526 }
527 return ~0u;
528 }
529
530
531
532
535 unsigned HwWidth = 8*HwLen;
536 if (VecWidth > 2*HwWidth)
538
542 if (VecWidth >= HwWidth/2 && VecWidth < HwWidth)
544 }
545
546
547 return ~0u;
548}
549
550unsigned
551HexagonTargetLowering::getCustomHvxOperationAction(SDNode &Op) const {
552 unsigned Opc = Op.getOpcode();
553 switch (Opc) {
558 }
560}
561
569}
570
572HexagonTargetLowering::typeJoin(const TypePair &Tys) const {
573 assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType());
574
576 return MVT::getVectorVT(ElemTy, Tys.first.getVectorNumElements() +
577 Tys.second.getVectorNumElements());
578}
579
580HexagonTargetLowering::TypePair
581HexagonTargetLowering::typeSplit(MVT VecTy) const {
584 assert((NumElem % 2) == 0 && "Expecting even-sized vector type");
586 return { HalfTy, HalfTy };
587}
588
590HexagonTargetLowering::typeExtElem(MVT VecTy, unsigned Factor) const {
594}
595
597HexagonTargetLowering::typeTruncElem(MVT VecTy, unsigned Factor) const {
601}
602
604HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy,
607 return Vec;
610}
611
613HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl,
617}
618
619HexagonTargetLowering::VectorPair
620HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl,
622 TypePair Tys = typeSplit(ty(Vec));
625 return DAG.SplitVector(Vec, dl, Tys.first, Tys.second);
626}
627
628bool
629HexagonTargetLowering::isHvxSingleTy(MVT Ty) const {
630 return Subtarget.isHVXVectorType(Ty) &&
631 Ty.getSizeInBits() == 8 * Subtarget.getVectorLength();
632}
633
634bool
635HexagonTargetLowering::isHvxPairTy(MVT Ty) const {
636 return Subtarget.isHVXVectorType(Ty) &&
637 Ty.getSizeInBits() == 16 * Subtarget.getVectorLength();
638}
639
640bool
641HexagonTargetLowering::isHvxBoolTy(MVT Ty) const {
642 return Subtarget.isHVXVectorType(Ty, true) &&
644}
645
646bool HexagonTargetLowering::allowsHvxMemoryAccess(
648
649
650
651
652 if (VecTy.getSizeInBits() > 8*Subtarget.getVectorLength())
653 return false;
654 if (!Subtarget.isHVXVectorType(VecTy, false))
655 return false;
658 return true;
659}
660
661bool HexagonTargetLowering::allowsHvxMisalignedMemoryAccesses(
663 if (!Subtarget.isHVXVectorType(VecTy))
664 return false;
665
668 return true;
669}
670
671void HexagonTargetLowering::AdjustHvxInstrPostInstrSelection(
673 unsigned Opc = MI.getOpcode();
674 const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
675 MachineBasicBlock &MB = *MI.getParent();
676 MachineFunction &MF = *MB.getParent();
679 auto At = MI.getIterator();
680
681 switch (Opc) {
682 case Hexagon::PS_vsplatib:
683 if (Subtarget.useHVXV62Ops()) {
684
685
686 Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
687 BuildMI(MB, At, DL, TII.get(Hexagon::A2_tfrsi), SplatV)
689 Register OutV = MI.getOperand(0).getReg();
690 BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatb), OutV)
692 } else {
693
694
695 Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
696 const MachineOperand &InpOp = MI.getOperand(1);
698 uint32_t V = InpOp.getImm() & 0xFF;
699 BuildMI(MB, At, DL, TII.get(Hexagon::A2_tfrsi), SplatV)
700 .addImm(V << 24 | V << 16 | V << 8 | V);
701 Register OutV = MI.getOperand(0).getReg();
702 BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatw), OutV).addReg(SplatV);
703 }
705 break;
706 case Hexagon::PS_vsplatrb:
707 if (Subtarget.useHVXV62Ops()) {
708
709 Register OutV = MI.getOperand(0).getReg();
710 BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatb), OutV)
712 } else {
713 Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
714 const MachineOperand &InpOp = MI.getOperand(1);
715 BuildMI(MB, At, DL, TII.get(Hexagon::S2_vsplatrb), SplatV)
717 Register OutV = MI.getOperand(0).getReg();
718 BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatw), OutV)
720 }
722 break;
723 case Hexagon::PS_vsplatih:
724 if (Subtarget.useHVXV62Ops()) {
725
726
727 Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
728 BuildMI(MB, At, DL, TII.get(Hexagon::A2_tfrsi), SplatV)
730 Register OutV = MI.getOperand(0).getReg();
731 BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplath), OutV)
733 } else {
734
735
736 Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
737 const MachineOperand &InpOp = MI.getOperand(1);
739 uint32_t V = InpOp.getImm() & 0xFFFF;
740 BuildMI(MB, At, DL, TII.get(Hexagon::A2_tfrsi), SplatV)
741 .addImm(V << 16 | V);
742 Register OutV = MI.getOperand(0).getReg();
743 BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatw), OutV).addReg(SplatV);
744 }
746 break;
747 case Hexagon::PS_vsplatrh:
748 if (Subtarget.useHVXV62Ops()) {
749
750 Register OutV = MI.getOperand(0).getReg();
751 BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplath), OutV)
753 } else {
754
755
756 Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
757 const MachineOperand &InpOp = MI.getOperand(1);
758 BuildMI(MB, At, DL, TII.get(Hexagon::A2_combine_ll), SplatV)
761 Register OutV = MI.getOperand(0).getReg();
762 BuildMI(MB, At, DL, TII.get(Hexagon::V6_lvsplatw), OutV).addReg(SplatV);
763 }
765 break;
766 case Hexagon::PS_vsplatiw:
767 case Hexagon::PS_vsplatrw:
768 if (Opc == Hexagon::PS_vsplatiw) {
769
770 Register SplatV = MRI.createVirtualRegister(&Hexagon::IntRegsRegClass);
771 BuildMI(MB, At, DL, TII.get(Hexagon::A2_tfrsi), SplatV)
773 MI.getOperand(1).ChangeToRegister(SplatV, false);
774 }
775
776 MI.setDesc(TII.get(Hexagon::V6_lvsplatw));
777 break;
778 }
779}
780
782HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy,
785 ElemIdx = DAG.getBitcast(MVT::i32, ElemIdx);
786
788 if (ElemWidth == 8)
789 return ElemIdx;
790
791 unsigned L = Log2_32(ElemWidth/8);
792 const SDLoc &dl(ElemIdx);
794 {ElemIdx, DAG.getConstant(L, dl, MVT::i32)});
795}
796
798HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy,
801 assert(ElemWidth >= 8 && ElemWidth <= 32);
802 if (ElemWidth == 32)
803 return Idx;
804
805 if (ty(Idx) != MVT::i32)
807 const SDLoc &dl(Idx);
810 return SubIdx;
811}
812
814HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0,
817 MVT OpTy = ty(Op0);
818 assert(OpTy == ty(Op1));
819
821 if (ElemTy == MVT::i8)
824
825 MVT ResTy = tyVector(OpTy, MVT::i8);
827
828 SmallVector<int,128> ByteMask;
829 for (int M : Mask) {
830 if (M < 0) {
831 for (unsigned I = 0; I != ElemSize; ++I)
833 } else {
834 int NewM = M*ElemSize;
835 for (unsigned I = 0; I != ElemSize; ++I)
837 }
838 }
840 return DAG.getVectorShuffle(ResTy, dl, opCastElem(Op0, MVT::i8, DAG),
841 opCastElem(Op1, MVT::i8, DAG), ByteMask);
842}
843
845HexagonTargetLowering::buildHvxVectorReg(ArrayRef Values,
848 unsigned VecLen = Values.size();
852 unsigned HwLen = Subtarget.getVectorLength();
853
854 unsigned ElemSize = ElemWidth / 8;
855 assert(ElemSize*VecLen == HwLen);
857
859 !(Subtarget.useHVXFloatingPoint() &&
861 assert((ElemSize == 1 || ElemSize == 2) && "Invalid element size");
862 unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2;
864 for (unsigned i = 0; i != VecLen; i += OpsPerWord) {
865 SDValue W = buildVector32(Values.slice(i, OpsPerWord), dl, PartVT, DAG);
867 }
868 } else {
871 }
873 unsigned NumValues = Values.size();
874 assert(NumValues > 0);
875 bool IsUndef = true;
876 for (unsigned i = 0; i != NumValues; ++i) {
877 if (Values[i].isUndef())
878 continue;
879 IsUndef = false;
880 if (!SplatV.getNode())
881 SplatV = Values[i];
882 else if (SplatV != Values[i])
883 return false;
884 }
885 if (IsUndef)
886 SplatV = Values[0];
887 return true;
888 };
889
890 unsigned NumWords = Words.size();
892 bool IsSplat = isSplat(Words, SplatV);
893 if (IsSplat && isUndef(SplatV))
895 if (IsSplat) {
898 return getZero(dl, VecTy, DAG);
902 }
903
904
905
907 bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts);
908 if (AllConst) {
910 (Constant**)Consts.end());
912 Align Alignment(HwLen);
917 }
918
919
920
921
922
923
924
925
926 auto IsBuildFromExtracts = [this,&Values] (SDValue &SrcVec,
927 SmallVectorImpl &SrcIdx) {
929 for (SDValue V : Values) {
930 if (isUndef(V)) {
931 SrcIdx.push_back(-1);
932 continue;
933 }
935 return false;
936
938 if (Vec.getNode() != nullptr && T.getNode() != Vec.getNode())
939 return false;
940 Vec = T;
942 if (C == nullptr)
943 return false;
944 int I = C->getSExtValue();
945 assert(I >= 0 && "Negative element index");
946 SrcIdx.push_back(I);
947 }
948 SrcVec = Vec;
949 return true;
950 };
951
952 SmallVector<int,128> ExtIdx;
954 if (IsBuildFromExtracts(ExtVec, ExtIdx)) {
955 MVT ExtTy = ty(ExtVec);
957 if (ExtLen == VecLen || ExtLen == 2*VecLen) {
958
959
960
961 SmallVector<int,128> Mask;
962 BitVector Used(ExtLen);
963
964 for (int M : ExtIdx) {
965 Mask.push_back(M);
966 if (M >= 0)
968 }
969
970
971
972
973
974 for (unsigned I = 0; I != ExtLen; ++I) {
975 if (Mask.size() == ExtLen)
976 break;
977 if (.test(I))
979 }
980
983 return ExtLen == VecLen ? S : LoHalf(S, DAG);
984 }
985 }
986
987
988
989
990
991 assert(4*Words.size() == Subtarget.getVectorLength());
992 int VecHist[32];
993 int n = 0;
994 for (unsigned i = 0; i != NumWords; ++i) {
995 VecHist[i] = 0;
996 if (Words[i].isUndef())
997 continue;
998 for (unsigned j = i; j != NumWords; ++j)
999 if (Words[i] == Words[j])
1000 VecHist[i]++;
1001
1002 if (VecHist[i] > VecHist[n])
1003 n = i;
1004 }
1005
1006 SDValue HalfV = getZero(dl, VecTy, DAG);
1007 if (VecHist[n] > 1) {
1010 {HalfV, SplatV, DAG.getConstant(HwLen/2, dl, MVT::i32)});
1011 }
1014
1015
1016
1017
1018 int Rn = 0, Rm = 0;
1022 for (unsigned i = 0; i != NumWords/2; ++i) {
1023
1024 if (Words[i] != Words[n] || VecHist[n] <= 1) {
1028 {HalfV0, Words[i]});
1029 Rn = 0;
1030 }
1031 if (Words[i+NumWords/2] != Words[n] || VecHist[n] <= 1) {
1035 {HalfV1, Words[i+NumWords/2]});
1036 Rm = 0;
1037 }
1038 Rn += 4;
1039 Rm += 4;
1040 }
1041
1042 Sn = DAG.getConstant(Rn+HwLen/2, dl, MVT::i32);
1046
1049
1051
1054 return OutV;
1055}
1056
1058HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl,
1059 unsigned BitBytes, bool ZeroFill, SelectionDAG &DAG) const {
1060 MVT PredTy = ty(PredV);
1061 unsigned HwLen = Subtarget.getVectorLength();
1063
1064 if (Subtarget.isHVXVectorType(PredTy, true)) {
1065
1066
1067
1068
1069
1070
1071
1072
1074 SmallVector<int,128> Mask(HwLen);
1075
1078
1079 for (unsigned i = 0; i != HwLen; ++i) {
1080 unsigned Num = i % Scale;
1081 unsigned Off = i / Scale;
1082 Mask[BlockLen*Num + Off] = i;
1083 }
1085 if (!ZeroFill)
1086 return S;
1087
1088
1089
1090 assert(BlockLen < HwLen && "vsetq(v1) prerequisite");
1092 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
1093 {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
1096 }
1097
1098
1099 assert(PredTy == MVT::v2i1 || PredTy == MVT::v4i1 || PredTy == MVT::v8i1);
1100
1103 unsigned IdxW = 0;
1104
1105 SDValue W0 = isUndef(PredV)
1108 if (Bytes < BitBytes) {
1109 Words[IdxW].push_back(HiHalf(W0, DAG));
1110 Words[IdxW].push_back(LoHalf(W0, DAG));
1111 } else
1113
1114 while (Bytes < BitBytes) {
1115 IdxW ^= 1;
1116 Words[IdxW].clear();
1117
1118 if (Bytes < 4) {
1119 for (const SDValue &W : Words[IdxW ^ 1]) {
1120 SDValue T = expandPredicate(W, dl, DAG);
1121 Words[IdxW].push_back(HiHalf(T, DAG));
1122 Words[IdxW].push_back(LoHalf(T, DAG));
1123 }
1124 } else {
1125 for (const SDValue &W : Words[IdxW ^ 1]) {
1128 }
1129 }
1130 Bytes *= 2;
1131 }
1132
1133 while (Bytes > BitBytes) {
1134 IdxW ^= 1;
1135 Words[IdxW].clear();
1136
1137 if (Bytes <= 4) {
1138 for (const SDValue &W : Words[IdxW ^ 1]) {
1139 SDValue T = contractPredicate(W, dl, DAG);
1141 }
1142 } else {
1143 for (const SDValue &W : Words[IdxW ^ 1]) {
1145 }
1146 }
1147 Bytes /= 2;
1148 }
1149
1150 assert(Bytes == BitBytes);
1151 if (BitBytes == 1 && PredTy == MVT::v2i1)
1153
1154 SDValue Vec = ZeroFill ? getZero(dl, ByteTy, DAG) : DAG.getUNDEF(ByteTy);
1156 for (const SDValue &W : Words[IdxW]) {
1159 }
1160
1161 return Vec;
1162}
1163
1165HexagonTargetLowering::buildHvxVectorPred(ArrayRef Values,
1168
1169
1170 unsigned VecLen = Values.size();
1171 unsigned HwLen = Subtarget.getVectorLength();
1172 assert(VecLen <= HwLen || VecLen == 8*HwLen);
1174 bool AllT = true, AllF = true;
1175
1176 auto IsTrue = [] (SDValue V) {
1178 return ->isZero();
1179 return false;
1180 };
1181 auto IsFalse = [] (SDValue V) {
1183 return N->isZero();
1184 return false;
1185 };
1186
1187 if (VecLen <= HwLen) {
1188
1189
1190
1191 assert(HwLen % VecLen == 0);
1192 unsigned BitBytes = HwLen / VecLen;
1193 for (SDValue V : Values) {
1194 AllT &= IsTrue(V);
1195 AllF &= IsFalse(V);
1196
1199 for (unsigned B = 0; B != BitBytes; ++B)
1201 }
1202 } else {
1203
1204
1205
1206 for (unsigned I = 0; I != VecLen; I += 8) {
1207 unsigned B = 0;
1208
1210 if (!Values[I+B].isUndef())
1211 break;
1212 }
1214 AllT &= IsTrue(F);
1215 AllF &= IsFalse(F);
1216
1220
1221
1223 assert(Values[I+B].isUndef() || Values[I+B] == F);
1224 }
1225 }
1226
1227 if (AllT)
1229 if (AllF)
1231
1233 SDValue ByteVec = buildHvxVectorReg(Bytes, dl, ByteTy, DAG);
1235}
1236
1238HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV,
1241
1243 assert(ElemWidth >= 8 && ElemWidth <= 32);
1244 (void)ElemWidth;
1245
1246 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
1248 {VecV, ByteIdx});
1249 if (ElemTy == MVT::i32)
1250 return ExWord;
1251
1252
1253
1254
1255 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
1256
1257 SDValue ExVec = DAG.getBitcast(tyVector(ty(ExWord), ElemTy), ExWord);
1258 return extractVector(ExVec, SubIdx, dl, ElemTy, MVT::i32, DAG);
1259}
1260
1262HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV,
1264
1265 assert(ResTy == MVT::i1);
1266
1267 unsigned HwLen = Subtarget.getVectorLength();
1270
1271 unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
1274
1275 SDValue ExtB = extractHvxElementReg(ByteVec, IdxV, dl, MVT::i32, DAG);
1277 return getInstr(Hexagon::C2_cmpgtui, dl, MVT::i1, {ExtB, Zero}, DAG);
1278}
1279
1281HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV,
1284
1286 assert(ElemWidth >= 8 && ElemWidth <= 32);
1287 (void)ElemWidth;
1288
1289 auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV,
1291 MVT VecTy = ty(VecV);
1292 unsigned HwLen = Subtarget.getVectorLength();
1299 {DAG.getConstant(HwLen, dl, MVT::i32), MaskV});
1301 return TorV;
1302 };
1303
1304 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
1305 if (ElemTy == MVT::i32)
1306 return InsertWord(VecV, ValV, ByteIdx);
1307
1308
1309
1311 {ByteIdx, DAG.getConstant(2, dl, MVT::i32)});
1312 SDValue Ext = extractHvxElementReg(opCastElem(VecV, MVT::i32, DAG), WordIdx,
1313 dl, MVT::i32, DAG);
1314
1315
1316
1317 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
1318 MVT SubVecTy = tyVector(ty(Ext), ElemTy);
1320 ValV, SubIdx, dl, ElemTy, DAG);
1321
1322
1323 return InsertWord(VecV, Ins, ByteIdx);
1324}
1325
1327HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV,
1329 unsigned HwLen = Subtarget.getVectorLength();
1332
1333 unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
1337
1338 SDValue InsV = insertHvxElementReg(ByteVec, IdxV, ValV, dl, DAG);
1340}
1341
1343HexagonTargetLowering::extractHvxSubvectorReg(SDValue OrigOp, SDValue VecV,
1345 MVT VecTy = ty(VecV);
1346 unsigned HwLen = Subtarget.getVectorLength();
1350
1351
1352
1353
1354 if (isHvxPairTy(VecTy)) {
1355 unsigned SubIdx = Hexagon::vsub_lo;
1356 if (Idx * ElemWidth >= 8 * HwLen) {
1357 SubIdx = Hexagon::vsub_hi;
1359 }
1360
1361 VecTy = typeSplit(VecTy).first;
1363 if (VecTy == ResTy)
1364 return VecV;
1365 }
1366
1367
1368
1370
1371 MVT WordTy = tyVector(VecTy, MVT::i32);
1373 unsigned WordIdx = (Idx*ElemWidth) / 32;
1374
1376 SDValue W0 = extractHvxElementReg(WordVec, W0Idx, dl, MVT::i32, DAG);
1379
1381 SDValue W1 = extractHvxElementReg(WordVec, W1Idx, dl, MVT::i32, DAG);
1382 SDValue WW = getCombine(W1, W0, dl, MVT::i64, DAG);
1384}
1385
1387HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV,
1389 MVT VecTy = ty(VecV);
1390 unsigned HwLen = Subtarget.getVectorLength();
1393
1395
1398 unsigned Offset = Idx * BitBytes;
1400 SmallVector<int,128> Mask;
1401
1402 if (Subtarget.isHVXVectorType(ResTy, true)) {
1403
1404
1405
1406
1409 for (unsigned i = 0; i != HwLen/Rep; ++i) {
1410 for (unsigned j = 0; j != Rep; ++j)
1412 }
1415 }
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427 unsigned Rep = 8 / ResLen;
1428
1429
1430 for (unsigned r = 0; r != HwLen/ResLen; ++r) {
1431
1432 for (unsigned i = 0; i != ResLen; ++i) {
1433 for (unsigned j = 0; j != Rep; ++j)
1434 Mask.push_back(Offset + i*BitBytes);
1435 }
1436 }
1437
1438 SDValue Zero = getZero(dl, MVT::i32, DAG);
1440
1441
1444 {ShuffV, DAG.getConstant(4, dl, MVT::i32)});
1445 SDValue Vec64 = getCombine(W1, W0, dl, MVT::v8i8, DAG);
1446 return getInstr(Hexagon::A4_vcmpbgtui, dl, ResTy,
1448}
1449
1451HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV,
1453 MVT VecTy = ty(VecV);
1454 MVT SubTy = ty(SubV);
1455 unsigned HwLen = Subtarget.getVectorLength();
1458
1459 bool IsPair = isHvxPairTy(VecTy);
1460 MVT SingleTy = MVT::getVectorVT(ElemTy, (8*HwLen)/ElemWidth);
1461
1465
1466 if (IsPair) {
1467 V0 = LoHalf(VecV, DAG);
1468 V1 = HiHalf(VecV, DAG);
1469
1471 dl, MVT::i32);
1473 if (isHvxSingleTy(SubTy)) {
1475 unsigned Idx = CN->getZExtValue();
1477 unsigned SubIdx = (Idx == 0) ? Hexagon::vsub_lo : Hexagon::vsub_hi;
1479 }
1480
1481
1482
1486 }
1487
1488
1489
1493 }
1494
1495
1496
1498
1500 if (!IdxN || !IdxN->isZero()) {
1502 DAG.getConstant(ElemWidth/8, dl, MVT::i32));
1504 }
1505
1506
1507
1508 unsigned RolBase = HwLen;
1512 } else {
1514 SDValue R0 = LoHalf(V, DAG);
1515 SDValue R1 = HiHalf(V, DAG);
1520 RolBase = HwLen-4;
1521 }
1522
1523 if (RolBase != 4 || !IdxN || !IdxN->isZero()) {
1525 DAG.getConstant(RolBase, dl, MVT::i32), IdxV);
1527 }
1528
1529 if (IsPair) {
1533 }
1534 return SingleV;
1535}
1536
1538HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV,
1540 MVT VecTy = ty(VecV);
1541 MVT SubTy = ty(SubV);
1542 assert(Subtarget.isHVXVectorType(VecTy, true));
1543
1544
1545
1547 unsigned HwLen = Subtarget.getVectorLength();
1548 assert(HwLen % VecLen == 0 && "Unexpected vector type");
1549
1551 unsigned BitBytes = HwLen / VecLen;
1552 unsigned BlockLen = HwLen / Scale;
1553
1556 SDValue ByteSub = createHvxPrefixPred(SubV, dl, BitBytes, false, DAG);
1558
1560 if (!IdxN || !IdxN->isZero()) {
1562 DAG.getConstant(BitBytes, dl, MVT::i32));
1564 }
1565
1566
1567
1568
1569 assert(BlockLen < HwLen && "vsetq(v1) prerequisite");
1571 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
1572 {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
1573 ByteVec = getInstr(Hexagon::V6_vmux, dl, ByteTy, {Q, ByteSub, ByteVec}, DAG);
1574
1575 if (!IdxN || !IdxN->isZero()) {
1579 }
1581}
1582
1584HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl,
1586
1587
1588
1589 assert(Subtarget.isHVXVectorType(ResTy));
1590 if (!ZeroExt)
1592
1596 SDValue False = getZero(dl, ResTy, DAG);
1597 return DAG.getSelect(dl, ResTy, VecV, True, False);
1598}
1599
1601HexagonTargetLowering::compressHvxPred(SDValue VecQ, const SDLoc &dl,
1603
1604
1605
1606
1607
1609 unsigned HwLen = Subtarget.getVectorLength();
1611 MVT PredTy = ty(VecQ);
1613 assert(HwLen % PredLen == 0);
1615
1618
1619
1620 for (unsigned i = 0; i != HwLen/8; ++i) {
1621 for (unsigned j = 0; j != 8; ++j)
1622 Tmp.push_back(ConstantInt::get(Int8Ty, 1ull << j));
1623 }
1625 Align Alignment(HwLen);
1631
1632
1634 getZero(dl, VecTy, DAG));
1635
1636
1637
1640 SDValue Vrmpy = getInstr(Hexagon::V6_vrmpyub, dl, ByteTy, {Sel, All1}, DAG);
1641
1642 SDValue Rot = getInstr(Hexagon::V6_valignbi, dl, ByteTy,
1645
1646
1647
1648
1649 SmallVector<int,128> Mask;
1650 for (unsigned i = 0; i != HwLen; ++i)
1651 Mask.push_back((8*i) % HwLen + i/(HwLen/8));
1654 return DAG.getBitcast(ResTy, Collect);
1655}
1656
1658HexagonTargetLowering::resizeToWidth(SDValue VecV, MVT ResTy, bool Signed,
1660
1661 MVT InpTy = ty(VecV);
1662 if (InpTy == ResTy)
1663 return VecV;
1664
1667
1669 return InpWidth < ResWidth
1670 ? DAG.getNode(ISD::FP_EXTEND, dl, ResTy, VecV)
1673 }
1674
1676
1677 if (InpWidth < ResWidth) {
1679 return DAG.getNode(ExtOpc, dl, ResTy, VecV);
1680 } else {
1683 }
1684}
1685
1687HexagonTargetLowering::extractSubvector(SDValue Vec, MVT SubTy, unsigned SubIdx,
1690
1691 const SDLoc &dl(Vec);
1694 {Vec, DAG.getConstant(ElemIdx, dl, MVT::i32)});
1695}
1696
1699 const {
1700 const SDLoc &dl(Op);
1701 MVT VecTy = ty(Op);
1702
1705 for (unsigned i = 0; i != Size; ++i)
1706 Ops.push_back(Op.getOperand(i));
1707
1709 return buildHvxVectorPred(Ops, dl, VecTy, DAG);
1710
1711
1712
1713
1717 for (unsigned i = 0; i != Size; i++)
1719
1723 }
1724
1725
1726
1727
1728 if (VecTy.getSizeInBits() == 16 * Subtarget.getVectorLength()) {
1730 MVT SingleTy = typeSplit(VecTy).first;
1731 SDValue V0 = buildHvxVectorReg(A.take_front(Size / 2), dl, SingleTy, DAG);
1732 SDValue V1 = buildHvxVectorReg(A.drop_front(Size / 2), dl, SingleTy, DAG);
1734 }
1735
1736 return buildHvxVectorReg(Ops, dl, VecTy, DAG);
1737}
1738
1741 const {
1742 const SDLoc &dl(Op);
1743 MVT VecTy = ty(Op);
1744 MVT ArgTy = ty(Op.getOperand(0));
1745
1746 if (ArgTy == MVT::f16 || ArgTy == MVT::bf16) {
1752 }
1753
1755}
1756
1759 const {
1760
1761
1762
1763 MVT VecTy = ty(Op);
1764 const SDLoc &dl(Op);
1767 if (NumOp == 2)
1768 return Op;
1769
1771 for (SDValue V : Op.getNode()->ops())
1773
1774
1775
1776
1777
1778 for (SDValue &V : Elems) {
1779 MVT Ty = ty(V);
1781 MVT NTy = typeLegalize(Ty, DAG);
1785 V.getOperand(0), V.getOperand(1)),
1787 continue;
1788 }
1789
1790 switch (V.getOpcode()) {
1793 break;
1796 break;
1799 break;
1800 default:
1802 }
1803 }
1804 }
1806 }
1807
1809 unsigned HwLen = Subtarget.getVectorLength();
1811
1813
1814
1815
1816 if (Subtarget.isHVXVectorType(ty(Op0), true)) {
1817 if (NumOp == 2)
1819
1823
1824 MVT HalfTy = typeSplit(VecTy).first;
1826 Ops.take_front(NumOp/2));
1828 Ops.take_back(NumOp/2));
1830 }
1831
1832
1833
1835
1837 for (SDValue V : Op.getNode()->op_values()) {
1838 SDValue P = createHvxPrefixPred(V, dl, BitBytes, true, DAG);
1840 }
1841
1842 unsigned InpLen = ty(Op.getOperand(0)).getVectorNumElements();
1845 SDValue Res = getZero(dl, ByteTy, DAG);
1846 for (unsigned i = 0, e = Prefixes.size(); i != e; ++i) {
1848 Res = DAG.getNode(ISD::OR, dl, ByteTy, Res, Prefixes[e-i-1]);
1849 }
1851}
1852
1855 const {
1856
1857 SDValue VecV = Op.getOperand(0);
1859 const SDLoc &dl(Op);
1860 SDValue IdxV = Op.getOperand(1);
1861 if (ElemTy == MVT::i1)
1862 return extractHvxElementPred(VecV, IdxV, dl, ty(Op), DAG);
1863
1864 return extractHvxElementReg(VecV, IdxV, dl, ty(Op), DAG);
1865}
1866
1869 const {
1870 const SDLoc &dl(Op);
1871 MVT VecTy = ty(Op);
1872 SDValue VecV = Op.getOperand(0);
1873 SDValue ValV = Op.getOperand(1);
1874 SDValue IdxV = Op.getOperand(2);
1876 if (ElemTy == MVT::i1)
1877 return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG);
1878
1879 if (ElemTy == MVT::f16 || ElemTy == MVT::bf16) {
1881 tyVector(VecTy, MVT::i16),
1882 DAG.getBitcast(tyVector(VecTy, MVT::i16), VecV),
1883 DAG.getBitcast(MVT::i16, ValV), IdxV);
1884 return DAG.getBitcast(tyVector(VecTy, ElemTy), T0);
1885 }
1886
1887 return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG);
1888}
1889
1892 const {
1893 SDValue SrcV = Op.getOperand(0);
1894 MVT SrcTy = ty(SrcV);
1895 MVT DstTy = ty(Op);
1896 SDValue IdxV = Op.getOperand(1);
1899 (void)Idx;
1900 const SDLoc &dl(Op);
1901
1903 if (ElemTy == MVT::i1)
1904 return extractHvxSubvectorPred(SrcV, IdxV, dl, DstTy, DAG);
1905
1906 return extractHvxSubvectorReg(Op, SrcV, IdxV, dl, DstTy, DAG);
1907}
1908
1911 const {
1912
1913 SDValue VecV = Op.getOperand(0);
1914 SDValue ValV = Op.getOperand(1);
1915 SDValue IdxV = Op.getOperand(2);
1916
1917 const SDLoc &dl(Op);
1918 MVT VecTy = ty(VecV);
1920 if (ElemTy == MVT::i1)
1921 return insertHvxSubvectorPred(VecV, ValV, IdxV, dl, DAG);
1922
1923 return insertHvxSubvectorReg(VecV, ValV, IdxV, dl, DAG);
1924}
1925
1928
1929
1930
1931
1932 MVT ResTy = ty(Op);
1933 SDValue InpV = Op.getOperand(0);
1935 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1936 return LowerHvxSignExt(Op, DAG);
1938}
1939
1942 MVT ResTy = ty(Op);
1943 SDValue InpV = Op.getOperand(0);
1945 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1946 return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), false, DAG);
1947 return Op;
1948}
1949
1952 MVT ResTy = ty(Op);
1953 SDValue InpV = Op.getOperand(0);
1955 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1956 return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), true, DAG);
1957 return Op;
1958}
1959
1962
1963
1964 const SDLoc &dl(Op);
1965 MVT ResTy = ty(Op);
1966 SDValue InpV = Op.getOperand(0);
1967 assert(ResTy == ty(InpV));
1968
1969
1972
1976 DAG.getConstant(ElemWidth, dl, MVT::i32));
1979
1980
1981
1982
1988}
1989
1992 const SDLoc &dl(Op);
1993 MVT ResTy = ty(Op);
1995
1998
1999 SDVTList ResTys = DAG.getVTList(ResTy, ResTy);
2000 unsigned Opc = Op.getOpcode();
2001
2002
2007
2008#ifndef NDEBUG
2009 Op.dump(&DAG);
2010#endif
2012}
2013
2016 const SDLoc &dl(Op);
2017 unsigned Opc = Op.getOpcode();
2020
2021
2022 if (auto HiVal = Op.getValue(1); HiVal.use_empty()) {
2023
2027 }
2028
2031
2032
2033
2034 if (Subtarget.useHVXV62Ops())
2035 return emitHvxMulLoHiV62(Vu, SignedVu, Vv, SignedVv, dl, DAG);
2036
2038
2039
2040 if (auto LoVal = Op.getValue(0); LoVal.use_empty()) {
2041 SDValue Hi = emitHvxMulHsV60(Vu, Vv, dl, DAG);
2044 }
2045 }
2046
2047 return emitHvxMulLoHiV60(Vu, SignedVu, Vv, SignedVv, dl, DAG);
2048}
2049
2053 MVT ResTy = ty(Op);
2054 MVT ValTy = ty(Val);
2055 const SDLoc &dl(Op);
2056
2058 unsigned HwLen = Subtarget.getVectorLength();
2060 SDValue VQ = compressHvxPred(Val, dl, WordTy, DAG);
2062
2064 SDValue W0 = extractHvxElementReg(VQ, DAG.getConstant(0, dl, MVT::i32),
2065 dl, MVT::i32, DAG);
2067 return W0;
2070 }
2071
2072
2075 for (unsigned i = 0; i != BitWidth/32; ++i) {
2076 SDValue W = extractHvxElementReg(
2077 VQ, DAG.getConstant(i, dl, MVT::i32), dl, MVT::i32, DAG);
2079 }
2082 for (unsigned i = 0, e = Words.size(); i < e; i += 2) {
2083 SDValue C = getCombine(Words[i+1], Words[i], dl, MVT::i64, DAG);
2085 }
2086
2088 return Combines[0];
2089
2091 }
2092
2093
2094
2095
2096 auto bitcastI32ToV32I1 = [&](SDValue Val32) {
2097 assert(Val32.getValueType().getSizeInBits() == 32 &&
2098 "Input must be 32 bits");
2102 for (unsigned i = 0; i < 32; ++i)
2103 Mask.push_back(DAG.getConstant(1ull << i, dl, MVT::i32));
2104
2108 };
2109
2110 if (ResTy == MVT::v32i1 &&
2111 (ValTy == MVT::i32 || ValTy == MVT::v2i16 || ValTy == MVT::v4i8) &&
2112 Subtarget.useHVX128BOps()) {
2114 if (ValTy == MVT::v2i16 || ValTy == MVT::v4i8)
2115 Val32 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Val);
2116 return bitcastI32ToV32I1(Val32);
2117 }
2118
2119 if (ResTy == MVT::v64i1 && ValTy == MVT::i64 && Subtarget.useHVX128BOps()) {
2120
2125
2126
2127 SDValue LoRes = bitcastI32ToV32I1(Lo);
2128 SDValue HiRes = bitcastI32ToV32I1(Hi);
2129
2130
2132 }
2133
2135
2137 unsigned HwLen = Subtarget.getVectorLength();
2139
2142
2143
2144
2146
2147
2149 for (unsigned I = 0; I != HwLen / 8; ++I) {
2153 for (unsigned J = 0; J != 8; ++J) {
2156 }
2157 }
2158
2161 SDValue I2V = buildHvxVectorReg(Bytes, dl, ConstantVecTy, DAG);
2162
2163
2164 I2V = DAG.getNode(ISD::AND, dl, ConstantVecTy, {I2V, ConstantVec});
2166 }
2167
2168 return Op;
2169}
2170
2173
2176 Op.getOperand(0));
2177}
2178
2181 MVT ResTy = ty(Op);
2183 return Op;
2184
2185 const SDLoc &dl(Op);
2186 unsigned HwLen = Subtarget.getVectorLength();
2188 assert(HwLen % VecLen == 0);
2189 unsigned ElemSize = HwLen / VecLen;
2190
2197}
2198
2201 if (SDValue S = getVectorShiftByInt(Op, DAG))
2202 return S;
2203 return Op;
2204}
2205
2207HexagonTargetLowering::LowerHvxFunnelShift(SDValue Op,
2209 unsigned Opc = Op.getOpcode();
2211
2212
2213
2217
2218 MVT InpTy = ty(A);
2220
2221 const SDLoc &dl(Op);
2224
2225
2226
2227 bool UseShifts = ElemTy != MVT::i8;
2228 if (Subtarget.useHVXV65Ops() && ElemTy == MVT::i32)
2229 UseShifts = false;
2230
2231 if (SDValue SplatV = getSplatValue(S, DAG); SplatV && UseShifts) {
2232
2239 {DAG.getConstant(ElemWidth, dl, MVT::i32), ModS});
2241 DAG.getSetCC(dl, MVT::i1, ModS, getZero(dl, MVT::i32, DAG), ISD::SETEQ);
2242
2243
2249
2250
2252 }
2253
2255 InpTy, dl, DAG.getConstant(ElemWidth - 1, dl, ElemTy));
2256
2258 return DAG.getNode(MOpc, dl, ty(Op),
2260}
2261
2264 const SDLoc &dl(Op);
2265 unsigned IntNo = Op.getConstantOperandVal(0);
2267
2268 auto Swap = [&](SDValue P) {
2269 return DAG.getMergeValues({P.getValue(1), P.getValue(0)}, dl);
2270 };
2271
2272 switch (IntNo) {
2273 case Intrinsic::hexagon_V6_pred_typecast:
2274 case Intrinsic::hexagon_V6_pred_typecast_128B: {
2275 MVT ResTy = ty(Op), InpTy = ty(Ops[1]);
2276 if (isHvxBoolTy(ResTy) && isHvxBoolTy(InpTy)) {
2277 if (ResTy == InpTy)
2278 return Ops[1];
2280 }
2281 break;
2282 }
2283 case Intrinsic::hexagon_V6_vmpyss_parts:
2284 case Intrinsic::hexagon_V6_vmpyss_parts_128B:
2286 {Ops[1], Ops[2]}));
2287 case Intrinsic::hexagon_V6_vmpyuu_parts:
2288 case Intrinsic::hexagon_V6_vmpyuu_parts_128B:
2290 {Ops[1], Ops[2]}));
2291 case Intrinsic::hexagon_V6_vmpyus_parts:
2292 case Intrinsic::hexagon_V6_vmpyus_parts_128B: {
2294 {Ops[1], Ops[2]}));
2295 }
2296 }
2297
2298 return Op;
2299}
2300
2303 const SDLoc &dl(Op);
2304 unsigned HwLen = Subtarget.getVectorLength();
2308 SDValue Chain = MaskN->getChain();
2311
2312 unsigned Opc = Op->getOpcode();
2313 assert(Opc == ISD::MLOAD || Opc == ISD::MSTORE);
2314
2315 if (Opc == ISD::MLOAD) {
2316 MVT ValTy = ty(Op);
2319 if (isUndef(Thru))
2320 return Load;
2323 }
2324
2325
2326
2327
2328
2329 unsigned StoreOpc = Hexagon::V6_vS32b_qpred_ai;
2332
2333 if (MaskN->getAlign().value() % HwLen == 0) {
2334 SDValue Store = getInstr(StoreOpc, dl, MVT::Other,
2338 }
2339
2340
2342 SDValue Z = getZero(dl, ty(V), DAG);
2343
2344
2345
2346 SDValue LoV = getInstr(Hexagon::V6_vlalignb, dl, ty(V), {V, Z, A}, DAG);
2347 SDValue HiV = getInstr(Hexagon::V6_vlalignb, dl, ty(V), {Z, V, A}, DAG);
2348 return std::make_pair(LoV, HiV);
2349 };
2350
2354 VectorPair Tmp = StoreAlign(MaskV, Base);
2357 VectorPair ValueU = StoreAlign(Value, Base);
2358
2361 getInstr(StoreOpc, dl, MVT::Other,
2362 {MaskU.first, Base, Offset0, ValueU.first, Chain}, DAG);
2364 getInstr(StoreOpc, dl, MVT::Other,
2365 {MaskU.second, Base, Offset1, ValueU.second, Chain}, DAG);
2369}
2370
2373
2374
2375 assert(Subtarget.useHVXQFloatOps());
2376
2377 assert(Op->getOpcode() == ISD::FP_EXTEND);
2378
2379 MVT VecTy = ty(Op);
2380 MVT ArgTy = ty(Op.getOperand(0));
2381 const SDLoc &dl(Op);
2382
2383 if (ArgTy == MVT::v64bf16) {
2384 MVT HalfTy = typeSplit(VecTy).first;
2385 SDValue BF16Vec = Op.getOperand(0);
2387 getInstr(Hexagon::V6_vxor, dl, HalfTy, {BF16Vec, BF16Vec}, DAG);
2388
2389
2390
2392 getInstr(Hexagon::V6_vshufoeh, dl, VecTy, {BF16Vec, Zeroes}, DAG);
2393 VectorPair VecPair = opSplit(ShuffVec, dl, DAG);
2394 SDValue Result = getInstr(Hexagon::V6_vshuffvdd, dl, VecTy,
2395 {VecPair.second, VecPair.first,
2397 DAG);
2399 }
2400
2401 assert(VecTy == MVT::v64f32 && ArgTy == MVT::v64f16);
2402
2403 SDValue F16Vec = Op.getOperand(0);
2404
2406 bool Ignored;
2410 getInstr(Hexagon::V6_vmpy_qf32_hf, dl, VecTy, {F16Vec, Fp16Ones}, DAG);
2411
2412 MVT HalfTy = typeSplit(VecTy).first;
2413 VectorPair Pair = opSplit(VmpyVec, dl, DAG);
2415 getInstr(Hexagon::V6_vconv_sf_qf32, dl, HalfTy, {Pair.first}, DAG);
2417 getInstr(Hexagon::V6_vconv_sf_qf32, dl, HalfTy, {Pair.second}, DAG);
2418
2420 getInstr(Hexagon::V6_vshuffvdd, dl, VecTy,
2422
2423 return ShuffVec;
2424}
2425
2428
2431
2432 MVT ResTy = ty(Op);
2433 MVT FpTy = ty(Op.getOperand(0)).getVectorElementType();
2435
2436 if (Subtarget.useHVXIEEEFPOps()) {
2437
2438 if (FpTy == MVT::f16) {
2439
2440 assert(IntTy == MVT::i8 || IntTy == MVT::i16 || IntTy == MVT::i32);
2441
2442 if (IntTy == MVT::i8 || IntTy == MVT::i16)
2443 return Op;
2444 }
2445 }
2446
2448 return EqualizeFpIntConversion(Op, DAG);
2449
2450 return ExpandHvxFpToInt(Op, DAG);
2451}
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463SDValue HexagonTargetLowering::LowerHvxPred32ToFp(SDValue PredOp,
2465
2466 MVT ResTy = ty(PredOp);
2467 const SDLoc &dl(PredOp);
2468
2470 SDNode *RegConst = DAG.getMachineNode(Hexagon::A2_tfrsi, dl, MVT::i32, Const);
2471 SDNode *SplatConst = DAG.getMachineNode(Hexagon::V6_lvsplatw, dl, MVT::v32i32,
2473 SDNode *PredTransfer =
2474 DAG.getMachineNode(Hexagon::V6_vandvrt, dl, MVT::v32i1,
2476 SDNode *PrefixSum = DAG.getMachineNode(Hexagon::V6_vprefixqw, dl, MVT::v32i32,
2477 SDValue(PredTransfer, 0));
2479 Hexagon::V6_lvsplatw, dl, MVT::v32i32,
2481 SDNode *Vsub =
2482 DAG.getMachineNode(Hexagon::V6_vsubw, dl, MVT::v32i32,
2484 SDNode *IndexShift =
2485 DAG.getMachineNode(Hexagon::V6_vlsrwv, dl, MVT::v32i32,
2487 SDNode *MaskOff =
2488 DAG.getMachineNode(Hexagon::V6_vand, dl, MVT::v32i32,
2490 SDNode *Convert = DAG.getMachineNode(Hexagon::V6_vconv_sf_w, dl, ResTy,
2492 return SDValue(Convert, 0);
2493}
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512SDValue HexagonTargetLowering::LowerHvxPred64ToFp(SDValue PredOp,
2514
2515 MVT ResTy = ty(PredOp);
2516 const SDLoc &dl(PredOp);
2517
2519
2524
2526 SDNode *RegConst = DAG.getMachineNode(Hexagon::A2_tfrsi, dl, MVT::i32, Const);
2527 SDNode *SplatConst = DAG.getMachineNode(Hexagon::V6_lvsplatw, dl, MVT::v32i32,
2529
2530 SDNode *SplatHi =
2531 DAG.getMachineNode(Hexagon::V6_lvsplatw, dl, MVT::v32i32,
2532 DAG.getNode(ISD::BITCAST, dl, MVT::i32, HiReg));
2533 SDNode *SplatLo =
2534 DAG.getMachineNode(Hexagon::V6_lvsplatw, dl, MVT::v32i32,
2535 DAG.getNode(ISD::BITCAST, dl, MVT::i32, LoReg));
2536
2537 SDNode *PredTransfer =
2538 DAG.getMachineNode(Hexagon::V6_vandvrt, dl, MVT::v32i1,
2540
2541 SDNode *PrefixSum = DAG.getMachineNode(Hexagon::V6_vprefixqw, dl, MVT::v32i32,
2542 SDValue(PredTransfer, 0));
2543
2544 SDNode *Vsub =
2545 DAG.getMachineNode(Hexagon::V6_vsubw, dl, MVT::v32i32,
2547
2548 SDNode *IndexShift_hi =
2549 DAG.getMachineNode(Hexagon::V6_vlsrwv, dl, MVT::v32i32,
2551 SDNode *IndexShift_lo =
2552 DAG.getMachineNode(Hexagon::V6_vlsrwv, dl, MVT::v32i32,
2554
2555 SDNode *MaskOff_hi =
2556 DAG.getMachineNode(Hexagon::V6_vand, dl, MVT::v32i32,
2558 SDNode *MaskOff_lo =
2559 DAG.getMachineNode(Hexagon::V6_vand, dl, MVT::v32i32,
2561
2562 SDNode *Pack =
2563 DAG.getMachineNode(Hexagon::V6_vpackeh, dl, MVT::v64i16,
2565 SDNode *Convert =
2567 return SDValue(Convert, 0);
2568}
2569
2572
2575
2576 MVT ResTy = ty(Op);
2579
2581 if (ResTy == MVT::v32f32 && ty(Op.getOperand(0)) == MVT::v32i1)
2582 return LowerHvxPred32ToFp(Op, DAG);
2583 if (ResTy == MVT::v64f16 && ty(Op.getOperand(0)) == MVT::v64i1)
2584 return LowerHvxPred64ToFp(Op, DAG);
2585 }
2586
2587 if (Subtarget.useHVXIEEEFPOps()) {
2588
2589 if (FpTy == MVT::f16) {
2590
2591 assert(IntTy == MVT::i8 || IntTy == MVT::i16 || IntTy == MVT::i32);
2592
2593 if (IntTy == MVT::i8 || IntTy == MVT::i16)
2594 return Op;
2595 }
2596 }
2597
2599 return EqualizeFpIntConversion(Op, DAG);
2600
2601 return ExpandHvxIntToFp(Op, DAG);
2602}
2603
2604HexagonTargetLowering::TypePair
2605HexagonTargetLowering::typeExtendToWider(MVT Ty0, MVT Ty1) const {
2606
2607
2608
2610
2613
2616 unsigned MaxWidth = std::max(Width0, Width1);
2617
2618 auto getScalarWithWidth = [](MVT ScalarTy, unsigned Width) {
2623 };
2624
2625 MVT WideETy0 = getScalarWithWidth(ElemTy0, MaxWidth);
2626 MVT WideETy1 = getScalarWithWidth(ElemTy1, MaxWidth);
2627
2629
2630 return {WideETy0, WideETy1};
2631 }
2632
2633
2636
2639}
2640
2641HexagonTargetLowering::TypePair
2642HexagonTargetLowering::typeWidenToWider(MVT Ty0, MVT Ty1) const {
2643
2644
2646
2649 if (Len0 == Len1)
2650 return {Ty0, Ty1};
2651
2652 unsigned MaxLen = std::max(Len0, Len1);
2655}
2656
2658HexagonTargetLowering::typeLegalize(MVT Ty, SelectionDAG &DAG) const {
2662}
2663
2665HexagonTargetLowering::typeWidenToHvx(MVT Ty) const {
2666 unsigned HwWidth = 8 * Subtarget.getVectorLength();
2669 return Ty;
2670
2673}
2674
2675HexagonTargetLowering::VectorPair
2676HexagonTargetLowering::emitHvxAddWithOverflow(SDValue A, SDValue B,
2678
2679
2680 MVT ResTy = ty(A);
2683
2685
2686
2689 return {Add, Ovf};
2690 }
2691
2692
2693
2694
2703 return {Add, MSB};
2704}
2705
2706HexagonTargetLowering::VectorPair
2707HexagonTargetLowering::emitHvxShiftRightRnd(SDValue Val, unsigned Amt,
2709
2710
2711
2712 const SDLoc &dl(Val);
2713 MVT ValTy = ty(Val);
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2727 MVT IntTy = tyVector(ValTy, ElemTy);
2730
2732 SDValue LowBits = DAG.getConstant((1ull << (Amt - 1)) - 1, dl, IntTy);
2733
2739 auto [Tmp0, Ovf] = emitHvxAddWithOverflow(Inp, LowBits, dl, Signed, DAG);
2740
2742 SDValue Tmp1 = DAG.getNode(ShRight, dl, IntTy, Inp, AmtM1);
2743 SDValue Tmp2 = DAG.getNode(ShRight, dl, IntTy, Tmp0, AmtM1);
2745
2748 SDValue Tmp4 = DAG.getNode(ShRight, dl, IntTy, {Tmp2, One});
2749 SDValue Tmp5 = DAG.getNode(ShRight, dl, IntTy, {Tmp3, One});
2751 return {Mux, Ovf};
2752}
2753
2757 MVT VecTy = ty(A);
2758 MVT PairTy = typeJoin({VecTy, VecTy});
2760
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784 SDValue T0 = getInstr(Hexagon::V6_vmpyewuh, dl, VecTy, {B, A}, DAG);
2785
2786 SDValue T1 = getInstr(Hexagon::V6_vasrw, dl, VecTy, {A, S16}, DAG);
2787
2788 SDValue P0 = getInstr(Hexagon::V6_vmpyhus, dl, PairTy, {T1, B}, DAG);
2789
2790 SDValue T2 = LoHalf(P0, DAG);
2791
2792
2793
2794 SDValue P1 = getInstr(Hexagon::V6_vadduhw, dl, PairTy, {T0, T2}, DAG);
2795
2796 SDValue P2 = getInstr(Hexagon::V6_vaddhw, dl, PairTy, {T0, T2}, DAG);
2797
2798
2799 SDValue T3 = getInstr(Hexagon::V6_vasrw_acc, dl, VecTy,
2800 {HiHalf(P2, DAG), LoHalf(P1, DAG), S16}, DAG);
2801 SDValue T4 = getInstr(Hexagon::V6_vasrw, dl, VecTy, {B, S16}, DAG);
2802
2803
2804 SDValue P3 = getInstr(Hexagon::V6_vmpyhv, dl, PairTy, {T1, T4}, DAG);
2805 SDValue T5 = LoHalf(P3, DAG);
2806
2808 return T6;
2809}
2810
2812HexagonTargetLowering::emitHvxMulLoHiV60(SDValue A, bool SignedA, SDValue B,
2813 bool SignedB, const SDLoc &dl,
2815 MVT VecTy = ty(A);
2816 MVT PairTy = typeJoin({VecTy, VecTy});
2818
2820
2821 if (SignedA && !SignedB) {
2822
2825 }
2826
2827
2828
2829
2831
2832
2833
2834 SDValue P0 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {A, B}, DAG);
2835
2836
2837 SDValue T0 = getInstr(Hexagon::V6_lvsplatw, dl, VecTy,
2838 {DAG.getConstant(0x02020202, dl, MVT::i32)}, DAG);
2839 SDValue T1 = getInstr(Hexagon::V6_vdelta, dl, VecTy, {B, T0}, DAG);
2840
2841
2842
2843 SDValue P1 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {A, T1}, DAG);
2844
2845
2846
2847 SDValue P2 = getInstr(Hexagon::V6_vadduhw, dl, PairTy,
2848 {HiHalf(P1, DAG), LoHalf(P1, DAG)}, DAG);
2849
2851 getInstr(Hexagon::V6_vlsrw, dl, VecTy, {LoHalf(P0, DAG), S16}, DAG);
2853
2854
2855 SDValue T4 = getInstr(Hexagon::V6_vasrw_acc, dl, VecTy,
2856 {HiHalf(P2, DAG), T3, S16}, DAG);
2857
2858
2859 Lo = getInstr(Hexagon::V6_vaslw_acc, dl, VecTy,
2860 {LoHalf(P0, DAG), LoHalf(P2, DAG), S16}, DAG);
2862
2863 if (SignedA) {
2864 assert(SignedB && "Signed A and unsigned B should have been inverted");
2865
2871 SDValue X1 = getInstr(Hexagon::V6_vaddwq, dl, VecTy, {Q1, X0, A}, DAG);
2872 Hi = getInstr(Hexagon::V6_vsubw, dl, VecTy, {Hi, X1}, DAG);
2873 } else if (SignedB) {
2874
2875
2879 Hi = getInstr(Hexagon::V6_vsubwq, dl, VecTy, {Q1, Hi, A}, DAG);
2880 } else {
2881 assert(!SignedA && !SignedB);
2882 }
2883
2885}
2886
2888HexagonTargetLowering::emitHvxMulLoHiV62(SDValue A, bool SignedA,
2890 const SDLoc &dl,
2892 MVT VecTy = ty(A);
2893 MVT PairTy = typeJoin({VecTy, VecTy});
2895
2896 if (SignedA && !SignedB) {
2897
2900 }
2901
2902
2903 SDValue P0 = getInstr(Hexagon::V6_vmpyewuh_64, dl, PairTy, {A, B}, DAG);
2905 getInstr(Hexagon::V6_vmpyowh_64_acc, dl, PairTy, {P0, A, B}, DAG);
2908
2909 if (!SignedB) {
2910 assert(!SignedA && "Signed A and unsigned B should have been inverted");
2913
2914
2915
2916
2917
2918
2919
2922 SDValue T0 = getInstr(Hexagon::V6_vandvqv, dl, VecTy, {Q0, B}, DAG);
2923 SDValue T1 = getInstr(Hexagon::V6_vaddwq, dl, VecTy, {Q1, T0, A}, DAG);
2924 Hi = getInstr(Hexagon::V6_vaddw, dl, VecTy, {Hi, T1}, DAG);
2925 } else if (!SignedA) {
2928
2929
2930
2931
2932
2933
2935 Hi = getInstr(Hexagon::V6_vaddwq, dl, VecTy, {Q0, Hi, B}, DAG);
2936 }
2937
2939}
2940
2943 const {
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953 unsigned Opc = Op.getOpcode();
2956
2958 MVT InpTy = ty(Inp);
2959 MVT ResTy = ty(Op);
2960
2961 if (InpTy == ResTy)
2962 return Op;
2963
2964 const SDLoc &dl(Op);
2966
2967 auto [WInpTy, WResTy] = typeExtendToWider(InpTy, ResTy);
2968 SDValue WInp = resizeToWidth(Inp, WInpTy, Signed, dl, DAG);
2970 SDValue Res = resizeToWidth(Conv, ResTy, Signed, dl, DAG);
2971 return Res;
2972}
2973
2976 unsigned Opc = Op.getOpcode();
2978
2979 const SDLoc &dl(Op);
2981 MVT InpTy = ty(Op0);
2982 MVT ResTy = ty(Op);
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994 if (InpTy == MVT::v64f16) {
2995 if (Subtarget.useHVXV81Ops()) {
2996
2998 getInstr(Hexagon::V6_vconv_h_hf_rnd, dl, ResTy, {Op0}, DAG);
2999 return ConvVec;
3001
3002 SDValue ConvVec = getInstr(Hexagon::V6_vconv_h_hf, dl, ResTy, {Op0}, DAG);
3003 return ConvVec;
3004 }
3006
3007 SDValue ConvVec = getInstr(Hexagon::V6_vconv_w_sf, dl, ResTy, {Op0}, DAG);
3008 return ConvVec;
3009 }
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3087 unsigned ElemWidth = 1 + ExpWidth + FracWidth;
3088 assert((1ull << (ExpWidth - 1)) == (1 + ExpBias));
3089
3094 SDValue M7F = DAG.getConstant((1ull << (ElemWidth - 1)) - 1, dl, ResTy);
3100
3104
3110
3112
3119 } else {
3124 }
3125
3126 return Int;
3127}
3128
3131 unsigned Opc = Op.getOpcode();
3133
3134 const SDLoc &dl(Op);
3136 MVT InpTy = ty(Op0);
3137 MVT ResTy = ty(Op);
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3167
3169 unsigned ElemWidth = 1 + ExpWidth + FracWidth;
3170
3178
3179 auto [Frac, Ovf] = emitHvxShiftRightRnd(Frac0, ExpWidth + 1, false, DAG);
3184 Frac = DAG.getNode(ISD::OR, dl, InpTy, {Sign, Frac});
3185 }
3186
3192 {Exp2, DAG.getConstant(FracWidth, dl, InpTy)});
3196
3197 return Flt;
3198}
3199
3202 unsigned Opc = Op.getOpcode();
3203 unsigned TLOpc;
3204 switch (Opc) {
3209 break;
3212 break;
3213#ifndef NDEBUG
3214 Op.dump(&DAG);
3215#endif
3217 }
3218
3219 const SDLoc &dl(Op);
3220 return DAG.getNode(TLOpc, dl, ty(Op), Op.getOperand(0),
3221 DAG.getUNDEF(MVT::i128),
3223}
3224
3229 unsigned Opc = Op.getConstantOperandVal(2);
3230 return DAG.getNode(Opc, SDLoc(Op), ty(Op), Op.getOperand(0));
3231}
3232
3233HexagonTargetLowering::VectorPair
3235 assert(.isMachineOpcode());
3237 const SDLoc &dl(Op);
3238
3239 auto SplitVTNode = [&DAG, this](const VTSDNode *N) {
3240 MVT Ty = typeSplit(N->getVT().getSimpleVT()).first;
3242 return std::make_pair(TV, TV);
3243 };
3244
3245 for (SDValue A : Op.getNode()->ops()) {
3247 ty(A).isVector() ? opSplit(A, dl, DAG) : std::make_pair(A, A);
3248
3249 switch (Op.getOpcode()) {
3250 case ISD::SIGN_EXTEND_INREG:
3251 case HexagonISD::SSAT:
3252 case HexagonISD::USAT:
3253 if (const auto *N = dyn_cast(A.getNode()))
3254 std::tie(Lo, Hi) = SplitVTNode(N);
3255 break;
3256 }
3259 }
3260
3261 MVT ResTy = ty(Op);
3262 MVT HalfTy = typeSplit(ResTy).first;
3266}
3267
3271
3272 if (!MemN->getMemoryVT().isSimple())
3273 return Op;
3274
3275 MVT MemTy = MemN->getMemoryVT().getSimpleVT();
3276 if (!isHvxPairTy(MemTy))
3277 return Op;
3278
3279 const SDLoc &dl(Op);
3280 unsigned HwLen = Subtarget.getVectorLength();
3281 MVT SingleTy = typeSplit(MemTy).first;
3282 SDValue Chain = MemN->getChain();
3283 SDValue Base0 = MemN->getBasePtr();
3286 unsigned MemOpc = MemN->getOpcode();
3287
3288 MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr;
3289 if (MachineMemOperand *MMO = MemN->getMemOperand()) {
3291 uint64_t MemSize = (MemOpc == ISD::MLOAD || MemOpc == ISD::MSTORE)
3293 : HwLen;
3296 }
3297
3298 if (MemOpc == ISD::LOAD) {
3300 SDValue Load0 = DAG.getLoad(SingleTy, dl, Chain, Base0, MOp0);
3301 SDValue Load1 = DAG.getLoad(SingleTy, dl, Chain, Base1, MOp1);
3306 }
3307 if (MemOpc == ISD::STORE) {
3310 SDValue Store0 = DAG.getStore(Chain, dl, Vals.first, Base0, MOp0);
3311 SDValue Store1 = DAG.getStore(Chain, dl, Vals.second, Base1, MOp1);
3313 }
3314
3315 assert(MemOpc == ISD::MLOAD || MemOpc == ISD::MSTORE);
3316
3318 assert(MaskN->isUnindexed());
3319 VectorPair Masks = opSplit(MaskN->getMask(), dl, DAG);
3321
3322 if (MemOpc == ISD::MLOAD) {
3323 VectorPair Thru =
3337 }
3338 if (MemOpc == ISD::MSTORE) {
3341 Masks.first, SingleTy, MOp0,
3344 Masks.second, SingleTy, MOp1,
3347 }
3348
3349 std::string Name = "Unexpected operation: " + Op->getOperationName(&DAG);
3351}
3352
3355 const SDLoc &dl(Op);
3357 assert(LoadN->isUnindexed() && "Not widening indexed loads yet");
3358 assert(LoadN->getMemoryVT().getVectorElementType() != MVT::i1 &&
3359 "Not widening loads of i1 yet");
3360
3361 SDValue Chain = LoadN->getChain();
3364
3365 MVT ResTy = ty(Op);
3366 unsigned HwLen = Subtarget.getVectorLength();
3368 assert(ResLen < HwLen && "vsetq(v1) prerequisite");
3369
3371 SDValue Mask = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
3372 {DAG.getConstant(ResLen, dl, MVT::i32)}, DAG);
3373
3377
3379 DAG.getUNDEF(LoadTy), LoadTy, MemOp,
3383}
3384
3387 const SDLoc &dl(Op);
3389 assert(StoreN->isUnindexed() && "Not widening indexed stores yet");
3390 assert(StoreN->getMemoryVT().getVectorElementType() != MVT::i1 &&
3391 "Not widening stores of i1 yet");
3392
3393 SDValue Chain = StoreN->getChain();
3396
3397 SDValue Value = opCastElem(StoreN->getValue(), MVT::i8, DAG);
3398 MVT ValueTy = ty(Value);
3400 unsigned HwLen = Subtarget.getVectorLength();
3402
3403 for (unsigned Len = ValueLen; Len < HwLen; ) {
3405 Len = ty(Value).getVectorNumElements();
3406 }
3407 assert(ty(Value).getVectorNumElements() == HwLen);
3408
3409 assert(ValueLen < HwLen && "vsetq(v1) prerequisite");
3411 SDValue Mask = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
3412 {DAG.getConstant(ValueLen, dl, MVT::i32)}, DAG);
3414 auto *MemOp = MF.getMachineMemOperand(StoreN->getMemOperand(), 0, HwLen);
3417}
3418
3421 const SDLoc &dl(Op);
3422 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
3424 unsigned HwLen = Subtarget.getVectorLength();
3425
3426 unsigned WideOpLen = (8 * HwLen) / ElemTy.getSizeInBits();
3429 if (!Subtarget.isHVXVectorType(WideOpTy, true))
3431
3432 SDValue WideOp0 = appendUndef(Op0, WideOpTy, DAG);
3433 SDValue WideOp1 = appendUndef(Op1, WideOpTy, DAG);
3434 EVT ResTy =
3437 {WideOp0, WideOp1, Op.getOperand(2)});
3438
3439 EVT RetTy = typeLegalize(ty(Op), DAG);
3441 {SetCC, getZero(dl, MVT::i32, DAG)});
3442}
3443
3446 unsigned Opc = Op.getOpcode();
3447 bool IsPairOp = isHvxPairTy(ty(Op)) ||
3449 return isHvxPairTy(ty(V));
3450 });
3451
3452 if (IsPairOp) {
3453 switch (Opc) {
3454 default:
3455 break;
3456 case ISD::LOAD:
3457 case ISD::STORE:
3458 case ISD::MLOAD:
3459 case ISD::MSTORE:
3460 return SplitHvxMemOp(Op, DAG);
3465 if (ty(Op).getSizeInBits() == ty(Op.getOperand(0)).getSizeInBits())
3466 return opJoin(SplitVectorOp(Op, DAG), SDLoc(Op), DAG);
3467 break;
3476 case ISD::FMINIMUMNUM:
3477 case ISD::FMAXIMUMNUM:
3496 return opJoin(SplitVectorOp(Op, DAG), SDLoc(Op), DAG);
3499
3500
3501 if (ty(Op.getOperand(0)).getVectorElementType() == MVT::i1)
3502 return opJoin(SplitVectorOp(Op, DAG), SDLoc(Op), DAG);
3503 break;
3504 }
3505 }
3506
3507 switch (Opc) {
3508 default:
3509 break;
3517 case ISD::BITCAST: return LowerHvxBitcast(Op, DAG);
3521 case ISD::CTTZ: return LowerHvxCttz(Op, DAG);
3522 case ISD::SELECT: return LowerHvxSelect(Op, DAG);
3525 case ISD::SRL: return LowerHvxShift(Op, DAG);
3527 case ISD::FSHR: return LowerHvxFunnelShift(Op, DAG);
3529 case ISD::MULHU: return LowerHvxMulh(Op, DAG);
3536 case ISD::MLOAD:
3537 case ISD::MSTORE: return LowerHvxMaskedOp(Op, DAG);
3538
3539 case ISD::LOAD: return SDValue();
3540 case ISD::FP_EXTEND: return LowerHvxFpExtend(Op, DAG);
3545
3546
3550 }
3551#ifndef NDEBUG
3552 Op.dumpr(&DAG);
3553#endif
3555}
3556
3559 const {
3560
3561
3562
3563
3564
3565
3566 unsigned Opc = Op.getOpcode();
3567 switch (Opc) {
3572 break;
3578 break;
3579#ifndef NDEBUG
3580 Op.dump(&DAG);
3581#endif
3583 }
3584
3586 MVT InpTy = ty(Inp);
3587 MVT ResTy = ty(Op);
3588
3591 assert(InpWidth != ResWidth);
3592
3593 if (InpWidth == 2 * ResWidth || ResWidth == 2 * InpWidth)
3594 return Op;
3595
3596 const SDLoc &dl(Op);
3599
3600 auto repeatOp = [&](unsigned NewWidth, SDValue Arg) {
3602 switch (Opc) {
3608 return DAG.getNode(Opc, dl, Ty, {Arg, Op.getOperand(1), Op.getOperand(2)});
3609 default:
3611 }
3612 };
3613
3615 if (InpWidth < ResWidth) {
3617 while (InpWidth * 2 <= ResWidth)
3618 S = repeatOp(InpWidth *= 2, S);
3619 } else {
3620
3622 while (InpWidth / 2 >= ResWidth)
3623 S = repeatOp(InpWidth /= 2, S);
3624 }
3625 return S;
3626}
3627
3630 SDValue Inp0 = Op.getOperand(0);
3631 MVT InpTy = ty(Inp0);
3632 MVT ResTy = ty(Op);
3635 unsigned Opc = Op.getOpcode();
3636
3637 if (shouldWidenToHvx(InpTy, DAG) || shouldWidenToHvx(ResTy, DAG)) {
3638
3639
3640
3641
3642 auto [WInpTy, WResTy] =
3643 InpWidth < ResWidth ? typeWidenToWider(typeWidenToHvx(InpTy), ResTy)
3644 : typeWidenToWider(InpTy, typeWidenToHvx(ResTy));
3645 SDValue W = appendUndef(Inp0, WInpTy, DAG);
3648 S = DAG.getNode(Opc, SDLoc(Op), WResTy, W, Op.getOperand(1),
3649 Op.getOperand(2));
3650 } else {
3652 }
3653 SDValue T = ExpandHvxResizeIntoSteps(S, DAG);
3654 return extractSubvector(T, typeLegalize(ResTy, DAG), 0, DAG);
3655 } else if (shouldSplitToHvx(InpWidth < ResWidth ? ResTy : InpTy, DAG)) {
3656 return opJoin(SplitVectorOp(Op, DAG), SDLoc(Op), DAG);
3657 } else {
3659 return RemoveTLWrapper(Op, DAG);
3660 }
3662}
3663
3664void
3665HexagonTargetLowering::LowerHvxOperationWrapper(SDNode *N,
3667 unsigned Opc = N->getOpcode();
3669 SDValue Inp0;
3670 if (N->getNumOperands() > 0)
3671 Inp0 = Op.getOperand(0);
3672
3673 switch (Opc) {
3678 if (Subtarget.isHVXElementType(ty(Op)) &&
3679 Subtarget.isHVXElementType(ty(Inp0))) {
3680 Results.push_back(CreateTLWrapper(Op, DAG));
3681 }
3682 break;
3684 if (shouldWidenToHvx(ty(Inp0), DAG)) {
3685 if (SDValue T = WidenHvxSetCC(Op, DAG))
3687 }
3688 break;
3689 case ISD::STORE: {
3692 Results.push_back(Store);
3693 }
3694 break;
3695 }
3696 case ISD::MLOAD:
3697 if (isHvxPairTy(ty(Op))) {
3698 SDValue S = SplitHvxMemOp(Op, DAG);
3702 }
3703 break;
3704 case ISD::MSTORE:
3705 if (isHvxPairTy(ty(Op->getOperand(1)))) {
3706 SDValue S = SplitHvxMemOp(Op, DAG);
3708 }
3709 break;
3714 if (ty(Op).getSizeInBits() != ty(Inp0).getSizeInBits()) {
3715 SDValue T = EqualizeFpIntConversion(Op, DAG);
3717 }
3718 break;
3723 Results.push_back(LegalizeHvxResize(Op, DAG));
3724 break;
3725 default:
3726 break;
3727 }
3728}
3729
3730void
3731HexagonTargetLowering::ReplaceHvxNodeResults(SDNode *N,
3733 unsigned Opc = N->getOpcode();
3735 SDValue Inp0;
3736 if (N->getNumOperands() > 0)
3737 Inp0 = Op.getOperand(0);
3738
3739 switch (Opc) {
3744 if (Subtarget.isHVXElementType(ty(Op)) &&
3745 Subtarget.isHVXElementType(ty(Inp0))) {
3746 Results.push_back(CreateTLWrapper(Op, DAG));
3747 }
3748 break;
3750 if (shouldWidenToHvx(ty(Op), DAG)) {
3751 if (SDValue T = WidenHvxSetCC(Op, DAG))
3753 }
3754 break;
3755 case ISD::LOAD: {
3756 if (shouldWidenToHvx(ty(Op), DAG)) {
3761 }
3762 break;
3763 }
3764 case ISD::BITCAST:
3765 if (isHvxBoolTy(ty(Inp0))) {
3766 SDValue C = LowerHvxBitcast(Op, DAG);
3768 }
3769 break;
3772 if (ty(Op).getSizeInBits() != ty(Inp0).getSizeInBits()) {
3773 SDValue T = EqualizeFpIntConversion(Op, DAG);
3775 }
3776 break;
3781 Results.push_back(LegalizeHvxResize(Op, DAG));
3782 break;
3783 default:
3784 break;
3785 }
3786}
3787
3789HexagonTargetLowering::combineTruncateBeforeLegal(SDValue Op,
3790 DAGCombinerInfo &DCI) const {
3791
3792
3793
3795 SelectionDAG &DAG = DCI.DAG;
3796 const SDLoc &dl(Op);
3797
3798 if (Op.getOperand(0).getOpcode() == ISD::BITCAST)
3800 SDValue Cast = Op.getOperand(0);
3802
3803 EVT TruncTy = Op.getValueType();
3805 EVT SrcTy = Src.getValueType();
3812 if (2 * CastLen != SrcLen)
3814
3815 SmallVector<int, 128> Mask(SrcLen);
3816 for (int i = 0; i != static_cast<int>(CastLen); ++i) {
3817 Mask[i] = 2 * i;
3818 Mask[i + CastLen] = 2 * i + 1;
3819 }
3822 return opSplit(Deal, dl, DAG).first;
3823}
3824
3826HexagonTargetLowering::combineConcatVectorsBeforeLegal(
3827 SDValue Op, DAGCombinerInfo &DCI) const {
3828
3829
3830
3831
3834
3835 SelectionDAG &DAG = DCI.DAG;
3836 const SDLoc &dl(Op);
3839
3844
3845 SetVector Order;
3850
3851 if (Order.size() > 2)
3853
3854
3855
3859
3860 SmallVector<int, 128> LongMask;
3861 auto AppendToMask = [&](SDValue Shuffle) {
3863 ArrayRef Mask = SV->getMask();
3864 SDValue X = Shuffle.getOperand(0);
3865 SDValue Y = Shuffle.getOperand(1);
3866 for (int M : Mask) {
3867 if (M == -1) {
3869 continue;
3870 }
3871 SDValue Src = static_cast<unsigned>(M) < InpLen ? X : Y;
3872 if (static_cast<unsigned>(M) >= InpLen)
3873 M -= InpLen;
3874
3875 int OutOffset = Order[0] == Src ? 0 : InpLen;
3876 LongMask.push_back(M + OutOffset);
3877 }
3878 };
3879
3880 AppendToMask(V0);
3881 AppendToMask(V1);
3882
3884 SDValue C1 = Order.back();
3886
3889}
3890
3892HexagonTargetLowering::PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
3893 const {
3894 const SDLoc &dl(N);
3895 SelectionDAG &DAG = DCI.DAG;
3897 unsigned Opc = Op.getOpcode();
3898
3900
3902 return combineTruncateBeforeLegal(Op, DCI);
3904 return combineConcatVectorsBeforeLegal(Op, DCI);
3905
3906 if (DCI.isBeforeLegalizeOps())
3908
3909 switch (Opc) {
3911
3917 }
3918 break;
3919 }
3925 }
3926 break;
3932 return getZero(dl, ty(Op), DAG);
3933 break;
3935 if (isUndef(Ops[1]))
3936 return Ops[0];
3937 break;
3944 }
3945 break;
3946 }
3947 }
3948
3950}
3951
3952bool
3953HexagonTargetLowering::shouldSplitToHvx(MVT Ty, SelectionDAG &DAG) const {
3954 if (Subtarget.isHVXVectorType(Ty, true))
3955 return false;
3956 auto Action = getPreferredHvxVectorAction(Ty);
3958 return Subtarget.isHVXVectorType(typeLegalize(Ty, DAG), true);
3959 return false;
3960}
3961
3962bool
3963HexagonTargetLowering::shouldWidenToHvx(MVT Ty, SelectionDAG &DAG) const {
3964 if (Subtarget.isHVXVectorType(Ty, true))
3965 return false;
3966 auto Action = getPreferredHvxVectorAction(Ty);
3968 return Subtarget.isHVXVectorType(typeLegalize(Ty, DAG), true);
3969 return false;
3970}
3971
3972bool
3973HexagonTargetLowering::isHvxOperation(SDNode *N, SelectionDAG &DAG) const {
3974 if (!Subtarget.useHVXOps())
3975 return false;
3976
3977
3978 auto IsHvxTy = [this](EVT Ty) {
3979 return Ty.isSimple() && Subtarget.isHVXVectorType(Ty.getSimpleVT(), true);
3980 };
3981 auto IsHvxOp = [this](SDValue Op) {
3982 return Op.getValueType().isSimple() &&
3983 Subtarget.isHVXVectorType(ty(Op), true);
3984 };
3986 return true;
3987
3988
3989 auto IsWidenedToHvx = [this, &DAG](SDValue Op) {
3990 if (.getValueType().isSimple())
3991 return false;
3992 MVT ValTy = ty(Op);
3993 return ValTy.isVector() && shouldWidenToHvx(ValTy, DAG);
3994 };
3995
3996 for (int i = 0, e = N->getNumValues(); i != e; ++i) {
3997 if (IsWidenedToHvx(SDValue(N, i)))
3998 return true;
3999 }
4001}
unsigned const MachineRegisterInfo * MRI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
const HexagonInstrInfo * TII
static std::tuple< unsigned, unsigned, unsigned > getIEEEProperties(MVT Ty)
Definition HexagonISelLoweringHVX.cpp:43
static const MVT LegalV128[]
Definition HexagonISelLoweringHVX.cpp:40
static const MVT LegalW128[]
Definition HexagonISelLoweringHVX.cpp:41
static const MVT LegalW64[]
Definition HexagonISelLoweringHVX.cpp:39
static const MVT LegalV64[]
Definition HexagonISelLoweringHVX.cpp:38
static cl::opt< unsigned > HvxWidenThreshold("hexagon-hvx-widen", cl::Hidden, cl::init(16), cl::desc("Lower threshold (in bytes) for widening to HVX vectors"))
static cl::opt< bool > EnableFpFastConvert("hexagon-fp-fast-convert", cl::Hidden, cl::init(false), cl::desc("Enable FP fast conversion routine."))
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
Promote Memory to Register
This file provides utility analysis objects describing memory locations.
const SmallVectorImpl< MachineOperand > & Cond
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallVector class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static llvm::Type * getVectorElementType(llvm::Type *Ty)
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEhalf()
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
static LLVM_ABI Constant * get(ArrayRef< Constant * > V)
uint64_t getNumOperands() const
SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const
EVT getSetCCResultType(const DataLayout &, LLVMContext &C, EVT VT) const override
Return the ValueType of the result of SETCC operations.
LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Return the preferred vector type legalization action.
static MVT getFloatingPointVT(unsigned BitWidth)
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
MVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
LLVM_ABI instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
Flags
Flags values. These may be or'd together.
unsigned getSubReg() const
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
Register getReg() const
getReg - Returns the register number.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
LLVM_ABI SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
LLVM_ABI std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
LLVM_ABI SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
LLVM_ABI SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
LLVM_ABI SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
MachineFunction & getMachineFunction() const
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVMContext * getContext() const
LLVM_ABI SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
LLVM_ABI SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
size_type size() const
Determine the number of elements in the SetVector.
const value_type & front() const
Return the first element of the SetVector.
const value_type & back() const
Return the last element of the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ ADD
Simple integer binary arithmetic operators.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ SIGN_EXTEND
Conversion operators.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
@ Undef
Value of the register doesn't matter.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
FunctionAddr VTableAddr Value
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
decltype(auto) dyn_cast(const From &Val)
dyn_cast - Return the argument parameter cast to the specified type.
SmallVectorImpl< T >::const_pointer c_str(SmallVectorImpl< T > &str)
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
@ Or
Bitwise or logical OR of integers.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast - Return the argument parameter cast to the specified type.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool isVector() const
Return true if this is a vector value type.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
static LLVM_ABI MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.