LLVM: lib/Target/ARM/ARMISelDAGToDAG.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
31#include "llvm/IR/IntrinsicsARM.h"
36#include
37
38using namespace llvm;
39
40#define DEBUG_TYPE "arm-isel"
41#define PASS_NAME "ARM Instruction Selection"
42
45 cl::desc("Disable isel of shifter-op"),
47
48
49
50
51
52namespace {
53
55
56
58
59public:
60 ARMDAGToDAGISel() = delete;
61
64
65 bool runOnMachineFunction(MachineFunction &MF) override {
66
69 return true;
70 }
71
72 void PreprocessISelDAG() override;
73
74
75
76 inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
77 return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
78 }
79
80 void Select(SDNode *N) override;
81
82
83
84 bool ComplexPatternFuncMutatesDAG() const override { return true; }
85
86 bool hasNoVMLxHazardUse(SDNode *N) const;
87 bool isShifterOpProfitable(const SDValue &Shift,
91 bool CheckProfitability = true);
93 SDValue &B, bool CheckProfitability = true);
96
97 return SelectRegShifterOperand(N, A, B, C, false);
98 }
100
101 return SelectImmShifterOperand(N, A, B, false);
102 }
104 if (.hasOneUse())
105 return false;
106 return SelectImmShifterOperand(N, A, B, false);
107 }
108
109 bool SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out);
110
113
114 bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
116 bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
118 bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
123 bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
130
132
133
145 template
147
148
150 template
154 bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
156 template
157 bool SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N, SDValue &OffImm);
158 bool SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N, SDValue &OffImm,
159 unsigned Shift);
160 template
165
166 template<int Min, int Max>
168
169 inline bool is_so_imm(unsigned Imm) const {
171 }
172
173 inline bool is_so_imm_not(unsigned Imm) const {
175 }
176
177 inline bool is_t2_so_imm(unsigned Imm) const {
179 }
180
181 inline bool is_t2_so_imm_not(unsigned Imm) const {
183 }
184
185
186#include "ARMGenDAGISel.inc"
187
188private:
189 void transferMemOperands(SDNode *Src, SDNode *Dst);
190
191
192 bool tryARMIndexedLoad(SDNode *N);
193 bool tryT1IndexedLoad(SDNode *N);
194 bool tryT2IndexedLoad(SDNode *N);
195 bool tryMVEIndexedLoad(SDNode *N);
196 bool tryFMULFixed(SDNode *N, SDLoc dl);
197 bool tryFP_TO_INT(SDNode *N, SDLoc dl);
198 bool transformFixedFloatingPointConversion(SDNode *N, SDNode *FMul,
199 bool IsUnsigned,
200 bool FixedToFloat);
201
202
203
204
205
206 void SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
207 const uint16_t *DOpcodes, const uint16_t *QOpcodes0,
208 const uint16_t *QOpcodes1);
209
210
211
212
213
214 void SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
215 const uint16_t *DOpcodes, const uint16_t *QOpcodes0,
216 const uint16_t *QOpcodes1);
217
218
219
220
221 void SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
222 unsigned NumVecs, const uint16_t *DOpcodes,
223 const uint16_t *QOpcodes);
224
225
226 template
227 void AddMVEPredicateToOps(SDValueVector &Ops, SDLoc Loc,
229 template
230 void AddMVEPredicateToOps(SDValueVector &Ops, SDLoc Loc,
232
233 template
234 void AddEmptyMVEPredicateToOps(SDValueVector &Ops, SDLoc Loc);
235 template
236 void AddEmptyMVEPredicateToOps(SDValueVector &Ops, SDLoc Loc, EVT InactiveTy);
237
238
239 void SelectMVE_WB(SDNode *N, const uint16_t *Opcodes, bool Predicated);
240
241
242 void SelectMVE_LongShift(SDNode *N, uint16_t Opcode, bool Immediate,
243 bool HasSaturationOperand);
244
245
246 void SelectMVE_VADCSBC(SDNode *N, uint16_t OpcodeWithCarry,
247 uint16_t OpcodeWithNoCarry, bool Add, bool Predicated);
248
249
250
251 void SelectMVE_VSHLC(SDNode *N, bool Predicated);
252
253
254
255
256
257
258
259
260
261 void SelectBaseMVE_VMLLDAV(SDNode *N, bool Predicated,
262 const uint16_t *OpcodesS, const uint16_t *OpcodesU,
263 size_t Stride, size_t TySize);
264
265
266
267 void SelectMVE_VMLLDAV(SDNode *N, bool Predicated, const uint16_t *OpcodesS,
268 const uint16_t *OpcodesU);
269
270
271 void SelectMVE_VRMLLDAVH(SDNode *N, bool Predicated, const uint16_t *OpcodesS,
272 const uint16_t *OpcodesU);
273
274
275
276
277
278
279 void SelectMVE_VLD(SDNode *N, unsigned NumVecs,
280 const uint16_t *const *Opcodes, bool HasWriteback);
281
282
283
284 void SelectMVE_VxDUP(SDNode *N, const uint16_t *Opcodes,
286
287
288
289
290
291
292
293 void SelectCDE_CXxD(SDNode *N, uint16_t Opcode, size_t NumExtraOps,
294 bool HasAccum);
295
296
297
298
299 void SelectVLDDup(SDNode *N, bool IsIntrinsic, bool isUpdating,
300 unsigned NumVecs, const uint16_t *DOpcodes,
301 const uint16_t *QOpcodes0 = nullptr,
302 const uint16_t *QOpcodes1 = nullptr);
303
304
305 bool tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
306
307 bool tryInsertVectorElt(SDNode *N);
308
309 bool tryReadRegister(SDNode *N);
310 bool tryWriteRegister(SDNode *N);
311
312 bool tryInlineAsm(SDNode *N);
313
314 void SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI);
315
316 void SelectCMP_SWAP(SDNode *N);
317
318
319
320 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
322 std::vector &OutOps) override;
323
324
326 SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1);
327 SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1);
328 SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1);
329
330
334
335
336 SDValue GetVLDSTAlign(SDValue Align, const SDLoc &dl, unsigned NumVecs,
337 bool is64BitVector);
338
339
340
341
342
343
344 bool canExtractShiftFromMul(const SDValue &N, unsigned MaxShift,
345 unsigned &PowerOfTwo, SDValue &NewMulConst) const;
346
347
348
350};
351
353public:
354 static char ID;
355 ARMDAGToDAGISelLegacy(ARMBaseTargetMachine &tm, CodeGenOptLevel OptLevel)
356 : SelectionDAGISelLegacy(
357 ID, std::make_unique(tm, OptLevel)) {}
358};
359}
360
361char ARMDAGToDAGISelLegacy::ID = 0;
362
364
365
366
368 if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
369 Imm = N->getAsZExtVal();
370 return true;
371 }
372 return false;
373}
374
375
376
380
381
382
383
385 return N->getOpcode() == Opc &&
387}
388
389
390
391
392
394 int RangeMin, int RangeMax,
395 int &ScaledConstant) {
396 assert(Scale > 0 && "Invalid scale!");
397
398
400 if ()
401 return false;
402
403 ScaledConstant = (int) C->getZExtValue();
404 if ((ScaledConstant % Scale) != 0)
405 return false;
406
407 ScaledConstant /= Scale;
408 return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
409}
410
411void ARMDAGToDAGISel::PreprocessISelDAG() {
412 if (!Subtarget->hasV6T2Ops())
413 return;
414
415 bool isThumb2 = Subtarget->isThumb();
416
419 continue;
420
421
422
423
424
425
426
427
428
429
430 SDValue N0 = N.getOperand(0);
431 SDValue N1 = N.getOperand(1);
432 unsigned And_imm = 0;
436 }
437 if (!And_imm)
438 continue;
439
440
442 if (TZ != 1 && TZ != 2)
443
444
445
446
447
448
449
450
451
452 continue;
453 And_imm >>= TZ;
454 if (And_imm & (And_imm + 1))
455 continue;
456
457
459 unsigned Srl_imm = 0;
461 (Srl_imm <= 2))
462 continue;
463
464
465
469 if (isThumb2) {
470 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1))
471 continue;
472 } else {
473 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) ||
474 SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2))
475 continue;
476 }
477
478
479 Srl = CurDAG->getNode(ISD::SRL, SDLoc(Srl), MVT::i32,
481 CurDAG->getConstant(Srl_imm + TZ, SDLoc(Srl),
482 MVT::i32));
483 N1 = CurDAG->getNode(ISD::AND, SDLoc(N1), MVT::i32,
484 Srl,
485 CurDAG->getConstant(And_imm, SDLoc(Srl), MVT::i32));
486 N1 = CurDAG->getNode(ISD::SHL, SDLoc(N1), MVT::i32,
487 N1, CurDAG->getConstant(TZ, SDLoc(Srl), MVT::i32));
488 CurDAG->UpdateNodeOperands(&N, N0, N1);
489 }
490}
491
492
493
494
495bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
496 if (OptLevel == CodeGenOptLevel::None)
497 return true;
498
499 if (!Subtarget->hasVMLxHazards())
500 return true;
501
502 if (->hasOneUse())
503 return false;
504
505 SDNode *User = *N->user_begin();
507 return true;
508 if (User->isMachineOpcode()) {
509 const ARMBaseInstrInfo *TII = static_cast<const ARMBaseInstrInfo *>(
510 CurDAG->getSubtarget().getInstrInfo());
511
512 const MCInstrDesc &MCID = TII->get(User->getMachineOpcode());
514 return true;
515 unsigned Opcode = MCID.getOpcode();
516 if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
517 return true;
518
519
520
521
522
523
524
525
526
527
528
529
530
531 return TII->isFpMLxInstruction(Opcode);
532 }
533
534 return false;
535}
536
537bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
539 unsigned ShAmt) {
541 return true;
543 return true;
544
546 (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1));
547}
548
549bool ARMDAGToDAGISel::canExtractShiftFromMul(const SDValue &N,
550 unsigned MaxShift,
551 unsigned &PowerOfTwo,
552 SDValue &NewMulConst) const {
554 assert(MaxShift > 0);
555
556
557
558 if (.hasOneUse()) return false;
559
561 if (!MulConst) return false;
562
563
564 if (!MulConst->hasOneUse()) return false;
565 unsigned MulConstVal = MulConst->getZExtValue();
566 if (MulConstVal == 0) return false;
567
568
569 PowerOfTwo = MaxShift;
570 while ((MulConstVal % (1 << PowerOfTwo)) != 0) {
571 --PowerOfTwo;
572 if (PowerOfTwo == 0) return false;
573 }
574
575
576 unsigned NewMulConstVal = MulConstVal / (1 << PowerOfTwo);
577 NewMulConst = CurDAG->getConstant(NewMulConstVal, SDLoc(N), MVT::i32);
580 return NewCost < OldCost;
581}
582
583void ARMDAGToDAGISel::replaceDAGValue(const SDValue &N, SDValue M) {
584 CurDAG->RepositionNode(N.getNode()->getIterator(), M.getNode());
585 ReplaceUses(N, M);
586}
587
588bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
591 bool CheckProfitability) {
593 return false;
594
595
596
598 unsigned PowerOfTwo = 0;
600 if (canExtractShiftFromMul(N, 31, PowerOfTwo, NewMulConst)) {
601 HandleSDNode Handle(N);
602 SDLoc Loc(N);
603 replaceDAGValue(N.getOperand(1), NewMulConst);
604 BaseReg = Handle.getValue();
605 Opc = CurDAG->getTargetConstant(
607 return true;
608 }
609 }
610
612
613
614
616
618 unsigned ShImmVal = 0;
620 if () return false;
621 ShImmVal = RHS->getZExtValue() & 31;
623 SDLoc(N), MVT::i32);
624 return true;
625}
626
627bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
631 bool CheckProfitability) {
633 return false;
634
636
637
638
640
642 unsigned ShImmVal = 0;
644 if (RHS) return false;
645
647 if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
648 return false;
650 SDLoc(N), MVT::i32);
651 return true;
652}
653
654
655
656bool ARMDAGToDAGISel::SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out) {
658 Out = N;
659 return CurDAG->haveNoCommonBitsSet(N, Parent->getOperand(1));
660}
661
662
663bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
666
667
668
670 !CurDAG->isBaseWithConstantOffset(N)) {
672
674 Base = CurDAG->getTargetFrameIndex(
675 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
676 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
677 return true;
678 }
679
680 if (N.getOpcode() == ARMISD::Wrapper &&
685 } else
687 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
688 return true;
689 }
690
692 int RHSC = (int)RHS->getSExtValue();
694 RHSC = -RHSC;
695
696 if (RHSC > -0x1000 && RHSC < 0x1000) {
700 Base = CurDAG->getTargetFrameIndex(
701 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
702 }
703 OffImm = CurDAG->getSignedTargetConstant(RHSC, SDLoc(N), MVT::i32);
704 return true;
705 }
706 }
707
708
710 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
711 return true;
712}
713
714
715
719 ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) {
721
722 int RHSC = (int)RHS->getZExtValue();
723 if (RHSC & 1) {
724 RHSC = RHSC & ~1;
726 if (RHSC < 0) {
728 RHSC = - RHSC;
729 }
731 unsigned ShAmt = Log2_32(RHSC);
735 SDLoc(N), MVT::i32);
736 return true;
737 }
738 }
739 }
740 }
741
743
744 !CurDAG->isBaseWithConstantOffset(N))
745 return false;
746
747
749 int RHSC;
751 -0x1000+1, 0x1000, RHSC))
752 return false;
753 }
754
755
759 unsigned ShAmt = 0;
760
763
765
766
767 if (ConstantSDNode *Sh =
769 ShAmt = Sh->getZExtValue();
770 if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
771 Offset = N.getOperand(1).getOperand(0);
772 else {
773 ShAmt = 0;
775 }
776 } else {
778 }
779 }
780
781
784 N.getOperand(0).hasOneUse())) {
787
788
789 if (ConstantSDNode *Sh =
791 ShAmt = Sh->getZExtValue();
792 if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
793 Offset = N.getOperand(0).getOperand(0);
795 } else {
796 ShAmt = 0;
798 }
799 } else {
801 }
802 }
803 }
804
805
806
808 unsigned PowerOfTwo = 0;
810 if (canExtractShiftFromMul(Offset, 31, PowerOfTwo, NewMulConst)) {
811 HandleSDNode Handle(Offset);
812 replaceDAGValue(Offset.getOperand(1), NewMulConst);
813 Offset = Handle.getValue();
814 ShAmt = PowerOfTwo;
816 }
817 }
818
820 SDLoc(N), MVT::i32);
821 return true;
822}
823
824bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
826 unsigned Opcode = Op->getOpcode();
832 int Val;
834 return false;
835
838 unsigned ShAmt = 0;
840
841
843 ShAmt = Sh->getZExtValue();
844 if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
846 else {
847 ShAmt = 0;
849 }
850 } else {
852 }
853 }
854
856 SDLoc(N), MVT::i32);
857 return true;
858}
859
860bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
862 unsigned Opcode = Op->getOpcode();
868 int Val;
871 Offset = CurDAG->getRegister(0, MVT::i32);
872 Opc = CurDAG->getSignedTargetConstant(Val, SDLoc(Op), MVT::i32);
873 return true;
874 }
875
876 return false;
877}
878
879
880bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
882 unsigned Opcode = Op->getOpcode();
888 int Val;
890 Offset = CurDAG->getRegister(0, MVT::i32);
893 SDLoc(Op), MVT::i32);
894 return true;
895 }
896
897 return false;
898}
899
902 return true;
903}
904
905bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
909
913 MVT::i32);
914 return true;
915 }
916
917 if (!CurDAG->isBaseWithConstantOffset(N)) {
921 Base = CurDAG->getTargetFrameIndex(
922 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
923 }
924 Offset = CurDAG->getRegister(0, MVT::i32);
926 MVT::i32);
927 return true;
928 }
929
930
931 int RHSC;
933 -256 + 1, 256, RHSC)) {
937 Base = CurDAG->getTargetFrameIndex(
938 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
939 }
940 Offset = CurDAG->getRegister(0, MVT::i32);
941
943 if (RHSC < 0) {
945 RHSC = -RHSC;
946 }
948 MVT::i32);
949 return true;
950 }
951
955 MVT::i32);
956 return true;
957}
958
959bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
961 unsigned Opcode = Op->getOpcode();
967 int Val;
969 Offset = CurDAG->getRegister(0, MVT::i32);
971 MVT::i32);
972 return true;
973 }
974
977 MVT::i32);
978 return true;
979}
980
982 bool FP16) {
983 if (!CurDAG->isBaseWithConstantOffset(N)) {
987 Base = CurDAG->getTargetFrameIndex(
988 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
989 } else if (N.getOpcode() == ARMISD::Wrapper &&
994 }
996 SDLoc(N), MVT::i32);
997 return true;
998 }
999
1000
1001 int RHSC;
1002 const int Scale = FP16 ? 2 : 4;
1003
1008 Base = CurDAG->getTargetFrameIndex(
1009 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1010 }
1011
1013 if (RHSC < 0) {
1015 RHSC = -RHSC;
1016 }
1017
1018 if (FP16)
1020 SDLoc(N), MVT::i32);
1021 else
1023 SDLoc(N), MVT::i32);
1024
1025 return true;
1026 }
1027
1029
1030 if (FP16)
1032 SDLoc(N), MVT::i32);
1033 else
1035 SDLoc(N), MVT::i32);
1036
1037 return true;
1038}
1039
1040bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
1042 return IsAddressingMode5(N, Base, Offset, false);
1043}
1044
1045bool ARMDAGToDAGISel::SelectAddrMode5FP16(SDValue N,
1047 return IsAddressingMode5(N, Base, Offset, true);
1048}
1049
1050bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
1052 Addr = N;
1053
1054 unsigned Alignment = 0;
1055
1057
1062
1063
1064 llvm::Align MMOAlign = MemN->getAlign();
1066 if (MMOAlign.value() >= MemSize && MemSize > 1)
1067 Alignment = MemSize;
1068 } else {
1069
1070
1071
1073 }
1074
1075 Align = CurDAG->getTargetConstant(Alignment, SDLoc(N), MVT::i32);
1076 return true;
1077}
1078
1079bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N,
1084 return false;
1088 Offset = CurDAG->getRegister(0, MVT::i32);
1089 }
1090 return true;
1091}
1092
1093bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
1095 if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
1096 Offset = N.getOperand(0);
1098 Label = CurDAG->getTargetConstant(N1->getAsZExtVal(), SDLoc(N), MVT::i32);
1099 return true;
1100 }
1101
1102 return false;
1103}
1104
1105
1106
1107
1108
1109
1111
1112
1113
1115 return false;
1116
1117
1119 return C->getSExtValue() < 0 && C->getSExtValue() >= -255;
1120
1121 return false;
1122}
1123
1124bool ARMDAGToDAGISel::SelectThumbAddrModeRRSext(SDValue N, SDValue &Base,
1126 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) {
1128 return false;
1129
1131 return true;
1132 }
1133
1135 Offset = N.getOperand(1);
1136 return true;
1137}
1138
1142 return false;
1143 return SelectThumbAddrModeRRSext(N, Base, Offset);
1144}
1145
1146bool
1147ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
1151 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1152 return true;
1153 }
1154
1155 if (!CurDAG->isBaseWithConstantOffset(N)) {
1156 if (N.getOpcode() == ISD::ADD) {
1157 return false;
1158 } else if (N.getOpcode() == ARMISD::Wrapper &&
1164 } else {
1166 }
1167
1168 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1169 return true;
1170 }
1171
1172
1173 int RHSC;
1176 OffImm = CurDAG->getSignedTargetConstant(RHSC, SDLoc(N), MVT::i32);
1177 return true;
1178 }
1179
1180
1181 return false;
1182}
1183
1184bool
1187 return SelectThumbAddrModeImm5S(N, 4, Base, OffImm);
1188}
1189
1190bool
1193 return SelectThumbAddrModeImm5S(N, 2, Base, OffImm);
1194}
1195
1196bool
1199 return SelectThumbAddrModeImm5S(N, 1, Base, OffImm);
1200}
1201
1202bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
1206
1207
1208 MachineFrameInfo &MFI = MF->getFrameInfo();
1211 Base = CurDAG->getTargetFrameIndex(
1212 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1213 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1214 return true;
1215 }
1216
1217 if (!CurDAG->isBaseWithConstantOffset(N))
1218 return false;
1219
1221
1222 int RHSC;
1226
1227
1228
1229 MachineFrameInfo &MFI = MF->getFrameInfo();
1231
1232
1236 Base = CurDAG->getTargetFrameIndex(
1237 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1238 OffImm = CurDAG->getSignedTargetConstant(RHSC, SDLoc(N), MVT::i32);
1239 return true;
1240 }
1241 }
1242 }
1243 }
1244
1245 return false;
1246}
1247
1248template
1251 if (N.getOpcode() == ISD::SUB || CurDAG->isBaseWithConstantOffset(N)) {
1252 int RHSC;
1254 RHSC)) {
1257 RHSC = -RHSC;
1258 OffImm = CurDAG->getSignedTargetConstant(RHSC * (1 << Shift), SDLoc(N),
1259 MVT::i32);
1260 return true;
1261 }
1262 }
1263
1264
1266 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1267 return true;
1268}
1269
1270
1271
1272
1273
1274
1275
1276bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
1278
1279
1280
1282 !CurDAG->isBaseWithConstantOffset(N)) {
1284
1286 Base = CurDAG->getTargetFrameIndex(
1287 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1288 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1289 return true;
1290 }
1291
1292 if (N.getOpcode() == ARMISD::Wrapper &&
1298 return false;
1299 } else
1301 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1302 return true;
1303 }
1304
1306 if (SelectT2AddrModeImm8(N, Base, OffImm))
1307
1308 return false;
1309
1310 int RHSC = (int)RHS->getZExtValue();
1312 RHSC = -RHSC;
1313
1314 if (RHSC >= 0 && RHSC < 0x1000) {
1318 Base = CurDAG->getTargetFrameIndex(
1319 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1320 }
1321 OffImm = CurDAG->getSignedTargetConstant(RHSC, SDLoc(N), MVT::i32);
1322 return true;
1323 }
1324 }
1325
1326
1328 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1329 return true;
1330}
1331
1332template
1335 if (N.getOpcode() == ISD::SUB || CurDAG->isBaseWithConstantOffset(N)) {
1336 int RHSC;
1341 Base = CurDAG->getTargetFrameIndex(
1342 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1343 }
1344
1346 RHSC = -RHSC;
1347 OffImm = CurDAG->getSignedTargetConstant(RHSC * (1 << Shift), SDLoc(N),
1348 MVT::i32);
1349 return true;
1350 }
1351 }
1352
1353
1355 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1356 return true;
1357}
1358
1359bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
1361
1363 !CurDAG->isBaseWithConstantOffset(N))
1364 return false;
1365
1367 int RHSC = (int)RHS->getSExtValue();
1369 RHSC = -RHSC;
1370
1371 if ((RHSC >= -255) && (RHSC < 0)) {
1375 Base = CurDAG->getTargetFrameIndex(
1376 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1377 }
1378 OffImm = CurDAG->getSignedTargetConstant(RHSC, SDLoc(N), MVT::i32);
1379 return true;
1380 }
1381 }
1382
1383 return false;
1384}
1385
1386bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
1388 unsigned Opcode = Op->getOpcode();
1392 int RHSC;
1395 ? CurDAG->getSignedTargetConstant(RHSC, SDLoc(N), MVT::i32)
1396 : CurDAG->getSignedTargetConstant(-RHSC, SDLoc(N), MVT::i32);
1397 return true;
1398 }
1399
1400 return false;
1401}
1402
1403template
1406 if (N.getOpcode() == ISD::SUB || CurDAG->isBaseWithConstantOffset(N)) {
1407 int RHSC;
1409 RHSC)) {
1413 Base = CurDAG->getTargetFrameIndex(
1414 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1415 }
1416
1418 RHSC = -RHSC;
1419 OffImm = CurDAG->getSignedTargetConstant(RHSC * (1 << Shift), SDLoc(N),
1420 MVT::i32);
1421 return true;
1422 }
1423 }
1424
1425
1427 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1428 return true;
1429}
1430
1431template
1432bool ARMDAGToDAGISel::SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N,
1434 return SelectT2AddrModeImm7Offset(Op, N, OffImm, Shift);
1435}
1436
1437bool ARMDAGToDAGISel::SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N,
1439 unsigned Shift) {
1440 unsigned Opcode = Op->getOpcode();
1442 switch (Opcode) {
1443 case ISD::LOAD:
1445 break;
1446 case ISD::STORE:
1448 break;
1449 case ISD::MLOAD:
1451 break;
1452 case ISD::MSTORE:
1454 break;
1455 default:
1457 }
1458
1459 int RHSC;
1460
1463 ? CurDAG->getSignedTargetConstant(RHSC * (1 << Shift),
1464 SDLoc(N), MVT::i32)
1465 : CurDAG->getSignedTargetConstant(-RHSC * (1 << Shift),
1466 SDLoc(N), MVT::i32);
1467 return true;
1468 }
1469 return false;
1470}
1471
1472template <int Min, int Max>
1473bool ARMDAGToDAGISel::SelectImmediateInRange(SDValue N, SDValue &OffImm) {
1474 int Val;
1476 OffImm = CurDAG->getSignedTargetConstant(Val, SDLoc(N), MVT::i32);
1477 return true;
1478 }
1479 return false;
1480}
1481
1482bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
1485
1486 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N))
1487 return false;
1488
1489
1491 int RHSC = (int)RHS->getZExtValue();
1492 if (RHSC >= 0 && RHSC < 0x1000)
1493 return false;
1494 else if (RHSC < 0 && RHSC >= -255)
1495 return false;
1496 }
1497
1498
1499 unsigned ShAmt = 0;
1502
1503
1509 }
1510
1512
1513
1515 ShAmt = Sh->getZExtValue();
1516 if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1518 else {
1519 ShAmt = 0;
1520 }
1521 }
1522 }
1523
1524
1525
1527 unsigned PowerOfTwo = 0;
1529 if (canExtractShiftFromMul(OffReg, 3, PowerOfTwo, NewMulConst)) {
1530 HandleSDNode Handle(OffReg);
1531 replaceDAGValue(OffReg.getOperand(1), NewMulConst);
1533 ShAmt = PowerOfTwo;
1534 }
1535 }
1536
1537 ShImm = CurDAG->getTargetConstant(ShAmt, SDLoc(N), MVT::i32);
1538
1539 return true;
1540}
1541
1542bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base,
1544
1545
1547 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1548
1549 if (N.getOpcode() != ISD::ADD || !CurDAG->isBaseWithConstantOffset(N))
1550 return true;
1551
1553 if ()
1554 return true;
1555
1556 uint32_t RHSC = (int)RHS->getZExtValue();
1557 if (RHSC > 1020 || RHSC % 4 != 0)
1558 return true;
1559
1563 Base = CurDAG->getTargetFrameIndex(
1564 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1565 }
1566
1567 OffImm = CurDAG->getTargetConstant(RHSC/4, SDLoc(N), MVT::i32);
1568 return true;
1569}
1570
1571
1572
1573
1577
1578void ARMDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) {
1579 MachineMemOperand *MemOp = cast(N)->getMemOperand();
1581}
1582
1583bool ARMDAGToDAGISel::tryARMIndexedLoad(SDNode *N) {
1587 return false;
1588
1589 EVT LoadedVT = LD->getMemoryVT();
1592 unsigned Opcode = 0;
1593 bool Match = false;
1594 if (LoadedVT == MVT::i32 && isPre &&
1595 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1596 Opcode = ARM::LDR_PRE_IMM;
1597 Match = true;
1598 } else if (LoadedVT == MVT::i32 && !isPre &&
1599 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1600 Opcode = ARM::LDR_POST_IMM;
1601 Match = true;
1602 } else if (LoadedVT == MVT::i32 &&
1603 SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1604 Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1605 Match = true;
1606
1607 } else if (LoadedVT == MVT::i16 &&
1608 SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1609 Match = true;
1611 ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1612 : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1613 } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1615 if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1616 Match = true;
1617 Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1618 }
1619 } else {
1620 if (isPre &&
1621 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1622 Match = true;
1623 Opcode = ARM::LDRB_PRE_IMM;
1624 } else if (!isPre &&
1625 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1626 Match = true;
1627 Opcode = ARM::LDRB_POST_IMM;
1628 } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1629 Match = true;
1630 Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1631 }
1632 }
1633 }
1634
1635 if (Match) {
1636 if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1640 CurDAG->getRegister(0, MVT::i32), Chain };
1641 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1642 MVT::Other, Ops);
1643 transferMemOperands(N, New);
1644 ReplaceNode(N, New);
1645 return true;
1646 } else {
1650 CurDAG->getRegister(0, MVT::i32), Chain };
1651 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1652 MVT::Other, Ops);
1653 transferMemOperands(N, New);
1654 ReplaceNode(N, New);
1655 return true;
1656 }
1657 }
1658
1659 return false;
1660}
1661
1662bool ARMDAGToDAGISel::tryT1IndexedLoad(SDNode *N) {
1664 EVT LoadedVT = LD->getMemoryVT();
1668 return false;
1669
1671 if (!COffs || COffs->getZExtValue() != 4)
1672 return false;
1673
1674
1675
1676
1677
1681 CurDAG->getRegister(0, MVT::i32), Chain };
1682 SDNode *New = CurDAG->getMachineNode(ARM::tLDR_postidx, SDLoc(N), MVT::i32,
1683 MVT::i32, MVT::Other, Ops);
1684 transferMemOperands(N, New);
1685 ReplaceNode(N, New);
1686 return true;
1687}
1688
1689bool ARMDAGToDAGISel::tryT2IndexedLoad(SDNode *N) {
1693 return false;
1694
1695 EVT LoadedVT = LD->getMemoryVT();
1696 bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1699 unsigned Opcode = 0;
1700 bool Match = false;
1701 if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
1703 case MVT::i32:
1704 Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1705 break;
1706 case MVT::i16:
1707 if (isSExtLd)
1708 Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1709 else
1710 Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1711 break;
1712 case MVT::i8:
1713 case MVT::i1:
1714 if (isSExtLd)
1715 Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1716 else
1717 Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1718 break;
1719 default:
1720 return false;
1721 }
1722 Match = true;
1723 }
1724
1725 if (Match) {
1729 CurDAG->getRegister(0, MVT::i32), Chain };
1730 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1731 MVT::Other, Ops);
1732 transferMemOperands(N, New);
1733 ReplaceNode(N, New);
1734 return true;
1735 }
1736
1737 return false;
1738}
1739
1740bool ARMDAGToDAGISel::tryMVEIndexedLoad(SDNode *N) {
1741 EVT LoadedVT;
1742 unsigned Opcode = 0;
1743 bool isSExtLd, isPre;
1744 Align Alignment;
1748
1752 return false;
1753 LoadedVT = LD->getMemoryVT();
1755 return false;
1756
1757 Chain = LD->getChain();
1760 Alignment = LD->getAlign();
1764 PredReg = CurDAG->getRegister(0, MVT::i32);
1768 return false;
1769 LoadedVT = LD->getMemoryVT();
1771 return false;
1772
1773 Chain = LD->getChain();
1776 Alignment = LD->getAlign();
1780 PredReg = LD->getMask();
1781 } else
1783
1784
1785
1786
1788
1790 if (Alignment >= Align(2) && LoadedVT == MVT::v4i16 &&
1791 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 1)) {
1792 if (isSExtLd)
1793 Opcode = isPre ? ARM::MVE_VLDRHS32_pre : ARM::MVE_VLDRHS32_post;
1794 else
1795 Opcode = isPre ? ARM::MVE_VLDRHU32_pre : ARM::MVE_VLDRHU32_post;
1796 } else if (LoadedVT == MVT::v8i8 &&
1797 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 0)) {
1798 if (isSExtLd)
1799 Opcode = isPre ? ARM::MVE_VLDRBS16_pre : ARM::MVE_VLDRBS16_post;
1800 else
1801 Opcode = isPre ? ARM::MVE_VLDRBU16_pre : ARM::MVE_VLDRBU16_post;
1802 } else if (LoadedVT == MVT::v4i8 &&
1803 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 0)) {
1804 if (isSExtLd)
1805 Opcode = isPre ? ARM::MVE_VLDRBS32_pre : ARM::MVE_VLDRBS32_post;
1806 else
1807 Opcode = isPre ? ARM::MVE_VLDRBU32_pre : ARM::MVE_VLDRBU32_post;
1808 } else if (Alignment >= Align(4) &&
1809 (CanChangeType || LoadedVT == MVT::v4i32 ||
1810 LoadedVT == MVT::v4f32) &&
1811 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 2))
1812 Opcode = isPre ? ARM::MVE_VLDRWU32_pre : ARM::MVE_VLDRWU32_post;
1813 else if (Alignment >= Align(2) &&
1814 (CanChangeType || LoadedVT == MVT::v8i16 ||
1815 LoadedVT == MVT::v8f16) &&
1816 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 1))
1817 Opcode = isPre ? ARM::MVE_VLDRHU16_pre : ARM::MVE_VLDRHU16_post;
1818 else if ((CanChangeType || LoadedVT == MVT::v16i8) &&
1819 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 0))
1820 Opcode = isPre ? ARM::MVE_VLDRBU8_pre : ARM::MVE_VLDRBU8_post;
1821 else
1822 return false;
1823
1825 NewOffset,
1826 CurDAG->getTargetConstant(Pred, SDLoc(N), MVT::i32),
1827 PredReg,
1828 CurDAG->getRegister(0, MVT::i32),
1829 Chain};
1830 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32,
1831 N->getValueType(0), MVT::Other, Ops);
1832 transferMemOperands(N, New);
1836 CurDAG->RemoveDeadNode(N);
1837 return true;
1838}
1839
1840
1841SDNode *ARMDAGToDAGISel::createGPRPairNode(EVT VT, SDValue V0, SDValue V1) {
1844 CurDAG->getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
1845 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
1846 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
1847 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1848 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1849}
1850
1851
1852SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1855 CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, dl, MVT::i32);
1856 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1857 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1858 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1859 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1860}
1861
1862
1863SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1865 SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl,
1866 MVT::i32);
1867 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1868 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1869 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1870 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1871}
1872
1873
1874SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1876 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1877 MVT::i32);
1878 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1879 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1880 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1881 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1882}
1883
1884
1885SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1,
1889 CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, dl, MVT::i32);
1890 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1891 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1892 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl, MVT::i32);
1893 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl, MVT::i32);
1894 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1895 V2, SubReg2, V3, SubReg3 };
1896 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1897}
1898
1899
1900SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1,
1903 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1904 MVT::i32);
1905 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1906 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1907 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl, MVT::i32);
1908 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl, MVT::i32);
1909 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1910 V2, SubReg2, V3, SubReg3 };
1911 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1912}
1913
1914
1915SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1,
1918 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl,
1919 MVT::i32);
1920 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1921 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1922 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl, MVT::i32);
1923 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl, MVT::i32);
1924 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1925 V2, SubReg2, V3, SubReg3 };
1926 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1927}
1928
1929
1930
1931
1932SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, const SDLoc &dl,
1933 unsigned NumVecs, bool is64BitVector) {
1934 unsigned NumRegs = NumVecs;
1935 if (!is64BitVector && NumVecs < 3)
1936 NumRegs *= 2;
1937
1938 unsigned Alignment = Align->getAsZExtVal();
1939 if (Alignment >= 32 && NumRegs == 4)
1940 Alignment = 32;
1941 else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1942 Alignment = 16;
1943 else if (Alignment >= 8)
1944 Alignment = 8;
1945 else
1946 Alignment = 0;
1947
1948 return CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
1949}
1950
1952{
1953 switch (Opc) {
1954 default: return false;
1955 case ARM::VLD1d8wb_fixed : return true;
1956 case ARM::VLD1d16wb_fixed : return true;
1957 case ARM::VLD1d64Qwb_fixed : return true;
1958 case ARM::VLD1d32wb_fixed : return true;
1959 case ARM::VLD1d64wb_fixed : return true;
1960 case ARM::VLD1d8TPseudoWB_fixed : return true;
1961 case ARM::VLD1d16TPseudoWB_fixed : return true;
1962 case ARM::VLD1d32TPseudoWB_fixed : return true;
1963 case ARM::VLD1d64TPseudoWB_fixed : return true;
1964 case ARM::VLD1d8QPseudoWB_fixed : return true;
1965 case ARM::VLD1d16QPseudoWB_fixed : return true;
1966 case ARM::VLD1d32QPseudoWB_fixed : return true;
1967 case ARM::VLD1d64QPseudoWB_fixed : return true;
1968 case ARM::VLD1q8wb_fixed : return true;
1969 case ARM::VLD1q16wb_fixed : return true;
1970 case ARM::VLD1q32wb_fixed : return true;
1971 case ARM::VLD1q64wb_fixed : return true;
1972 case ARM::VLD1DUPd8wb_fixed : return true;
1973 case ARM::VLD1DUPd16wb_fixed : return true;
1974 case ARM::VLD1DUPd32wb_fixed : return true;
1975 case ARM::VLD1DUPq8wb_fixed : return true;
1976 case ARM::VLD1DUPq16wb_fixed : return true;
1977 case ARM::VLD1DUPq32wb_fixed : return true;
1978 case ARM::VLD2d8wb_fixed : return true;
1979 case ARM::VLD2d16wb_fixed : return true;
1980 case ARM::VLD2d32wb_fixed : return true;
1981 case ARM::VLD2q8PseudoWB_fixed : return true;
1982 case ARM::VLD2q16PseudoWB_fixed : return true;
1983 case ARM::VLD2q32PseudoWB_fixed : return true;
1984 case ARM::VLD2DUPd8wb_fixed : return true;
1985 case ARM::VLD2DUPd16wb_fixed : return true;
1986 case ARM::VLD2DUPd32wb_fixed : return true;
1987 case ARM::VLD2DUPq8OddPseudoWB_fixed: return true;
1988 case ARM::VLD2DUPq16OddPseudoWB_fixed: return true;
1989 case ARM::VLD2DUPq32OddPseudoWB_fixed: return true;
1990 }
1991}
1992
1994{
1995 switch (Opc) {
1996 default: return false;
1997 case ARM::VST1d8wb_fixed : return true;
1998 case ARM::VST1d16wb_fixed : return true;
1999 case ARM::VST1d32wb_fixed : return true;
2000 case ARM::VST1d64wb_fixed : return true;
2001 case ARM::VST1q8wb_fixed : return true;
2002 case ARM::VST1q16wb_fixed : return true;
2003 case ARM::VST1q32wb_fixed : return true;
2004 case ARM::VST1q64wb_fixed : return true;
2005 case ARM::VST1d8TPseudoWB_fixed : return true;
2006 case ARM::VST1d16TPseudoWB_fixed : return true;
2007 case ARM::VST1d32TPseudoWB_fixed : return true;
2008 case ARM::VST1d64TPseudoWB_fixed : return true;
2009 case ARM::VST1d8QPseudoWB_fixed : return true;
2010 case ARM::VST1d16QPseudoWB_fixed : return true;
2011 case ARM::VST1d32QPseudoWB_fixed : return true;
2012 case ARM::VST1d64QPseudoWB_fixed : return true;
2013 case ARM::VST2d8wb_fixed : return true;
2014 case ARM::VST2d16wb_fixed : return true;
2015 case ARM::VST2d32wb_fixed : return true;
2016 case ARM::VST2q8PseudoWB_fixed : return true;
2017 case ARM::VST2q16PseudoWB_fixed : return true;
2018 case ARM::VST2q32PseudoWB_fixed : return true;
2019 }
2020}
2021
2022
2023
2026 && "Incorrect fixed stride updating instruction.");
2027 switch (Opc) {
2028 default: break;
2029 case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register;
2030 case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register;
2031 case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register;
2032 case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register;
2033 case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register;
2034 case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register;
2035 case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register;
2036 case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register;
2037 case ARM::VLD1d64Twb_fixed: return ARM::VLD1d64Twb_register;
2038 case ARM::VLD1d64Qwb_fixed: return ARM::VLD1d64Qwb_register;
2039 case ARM::VLD1d8TPseudoWB_fixed: return ARM::VLD1d8TPseudoWB_register;
2040 case ARM::VLD1d16TPseudoWB_fixed: return ARM::VLD1d16TPseudoWB_register;
2041 case ARM::VLD1d32TPseudoWB_fixed: return ARM::VLD1d32TPseudoWB_register;
2042 case ARM::VLD1d64TPseudoWB_fixed: return ARM::VLD1d64TPseudoWB_register;
2043 case ARM::VLD1d8QPseudoWB_fixed: return ARM::VLD1d8QPseudoWB_register;
2044 case ARM::VLD1d16QPseudoWB_fixed: return ARM::VLD1d16QPseudoWB_register;
2045 case ARM::VLD1d32QPseudoWB_fixed: return ARM::VLD1d32QPseudoWB_register;
2046 case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register;
2047 case ARM::VLD1DUPd8wb_fixed : return ARM::VLD1DUPd8wb_register;
2048 case ARM::VLD1DUPd16wb_fixed : return ARM::VLD1DUPd16wb_register;
2049 case ARM::VLD1DUPd32wb_fixed : return ARM::VLD1DUPd32wb_register;
2050 case ARM::VLD1DUPq8wb_fixed : return ARM::VLD1DUPq8wb_register;
2051 case ARM::VLD1DUPq16wb_fixed : return ARM::VLD1DUPq16wb_register;
2052 case ARM::VLD1DUPq32wb_fixed : return ARM::VLD1DUPq32wb_register;
2053 case ARM::VLD2DUPq8OddPseudoWB_fixed: return ARM::VLD2DUPq8OddPseudoWB_register;
2054 case ARM::VLD2DUPq16OddPseudoWB_fixed: return ARM::VLD2DUPq16OddPseudoWB_register;
2055 case ARM::VLD2DUPq32OddPseudoWB_fixed: return ARM::VLD2DUPq32OddPseudoWB_register;
2056
2057 case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register;
2058 case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register;
2059 case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register;
2060 case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register;
2061 case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register;
2062 case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register;
2063 case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register;
2064 case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register;
2065 case ARM::VST1d8TPseudoWB_fixed: return ARM::VST1d8TPseudoWB_register;
2066 case ARM::VST1d16TPseudoWB_fixed: return ARM::VST1d16TPseudoWB_register;
2067 case ARM::VST1d32TPseudoWB_fixed: return ARM::VST1d32TPseudoWB_register;
2068 case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register;
2069 case ARM::VST1d8QPseudoWB_fixed: return ARM::VST1d8QPseudoWB_register;
2070 case ARM::VST1d16QPseudoWB_fixed: return ARM::VST1d16QPseudoWB_register;
2071 case ARM::VST1d32QPseudoWB_fixed: return ARM::VST1d32QPseudoWB_register;
2072 case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register;
2073
2074 case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register;
2075 case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register;
2076 case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register;
2077 case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register;
2078 case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register;
2079 case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register;
2080
2081 case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register;
2082 case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register;
2083 case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register;
2084 case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register;
2085 case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register;
2086 case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register;
2087
2088 case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register;
2089 case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register;
2090 case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register;
2091 }
2092 return Opc;
2093}
2094
2095
2096
2097
2100 return C && C->getZExtValue() == VecTy.getSizeInBits() / 8 * NumVecs;
2101}
2102
2103void ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
2104 const uint16_t *DOpcodes,
2105 const uint16_t *QOpcodes0,
2106 const uint16_t *QOpcodes1) {
2107 assert(Subtarget->hasNEON());
2108 assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
2109 SDLoc dl(N);
2110
2112 bool IsIntrinsic = !isUpdating;
2113
2114 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2115 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2116 return;
2117
2119 EVT VT = N->getValueType(0);
2121 Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
2122
2126
2128 case MVT::v4f16:
2129 case MVT::v4bf16:
2130 case MVT::v4i16: OpcodeIndex = 1; break;
2131 case MVT::v2f32:
2132 case MVT::v2i32: OpcodeIndex = 2; break;
2133 case MVT::v1i64: OpcodeIndex = 3; break;
2134
2135 case MVT::v16i8: OpcodeIndex = 0; break;
2136 case MVT::v8f16:
2137 case MVT::v8bf16:
2138 case MVT::v8i16: OpcodeIndex = 1; break;
2139 case MVT::v4f32:
2140 case MVT::v4i32: OpcodeIndex = 2; break;
2141 case MVT::v2f64:
2142 case MVT::v2i64: OpcodeIndex = 3; break;
2143 }
2144
2145 EVT ResTy;
2146 if (NumVecs == 1)
2147 ResTy = VT;
2148 else {
2149 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2150 if (!is64BitVector)
2151 ResTyElts *= 2;
2152 ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
2153 }
2154 std::vector ResTys;
2155 ResTys.push_back(ResTy);
2156 if (isUpdating)
2157 ResTys.push_back(MVT::i32);
2158 ResTys.push_back(MVT::Other);
2159
2161 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2162 SDNode *VLd;
2164
2165
2166 if (is64BitVector || NumVecs <= 2) {
2167 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
2169 Ops.push_back(MemAddr);
2170 Ops.push_back(Align);
2171 if (isUpdating) {
2172 SDValue Inc = N->getOperand(AddrOpIdx + 1);
2174 if (!IsImmUpdate) {
2175
2176
2179 Ops.push_back(Inc);
2180
2181
2183 Ops.push_back(Reg0);
2184 }
2185 Ops.push_back(Pred);
2186 Ops.push_back(Reg0);
2187 Ops.push_back(Chain);
2188 VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2189
2190 } else {
2191
2192
2194
2195
2196
2198 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
2199 const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
2200 SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
2201 ResTy, AddrTy, MVT::Other, OpsA);
2202 Chain = SDValue(VLdA, 2);
2203
2204
2206 Ops.push_back(Align);
2207 if (isUpdating) {
2210 "only constant post-increment update allowed for VLD3/4");
2211 (void)Inc;
2212 Ops.push_back(Reg0);
2213 }
2215 Ops.push_back(Pred);
2216 Ops.push_back(Reg0);
2217 Ops.push_back(Chain);
2218 VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops);
2219 }
2220
2221
2224
2225 if (NumVecs == 1) {
2226 ReplaceNode(N, VLd);
2227 return;
2228 }
2229
2230
2232 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2233 ARM::qsub_3 == ARM::qsub_0 + 3,
2234 "Unexpected subreg numbering");
2235 unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
2236 for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2237 ReplaceUses(SDValue(N, Vec),
2238 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2240 if (isUpdating)
2242 CurDAG->RemoveDeadNode(N);
2243}
2244
2245void ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
2246 const uint16_t *DOpcodes,
2247 const uint16_t *QOpcodes0,
2248 const uint16_t *QOpcodes1) {
2249 assert(Subtarget->hasNEON());
2250 assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
2251 SDLoc dl(N);
2252
2254 bool IsIntrinsic = !isUpdating;
2255
2256 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2257 unsigned Vec0Idx = 3;
2258 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2259 return;
2260
2262
2264 EVT VT = N->getOperand(Vec0Idx).getValueType();
2266 Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
2267
2271
2273 case MVT::v4f16:
2274 case MVT::v4bf16:
2275 case MVT::v4i16: OpcodeIndex = 1; break;
2276 case MVT::v2f32:
2277 case MVT::v2i32: OpcodeIndex = 2; break;
2278 case MVT::v1i64: OpcodeIndex = 3; break;
2279
2280 case MVT::v16i8: OpcodeIndex = 0; break;
2281 case MVT::v8f16:
2282 case MVT::v8bf16:
2283 case MVT::v8i16: OpcodeIndex = 1; break;
2284 case MVT::v4f32:
2285 case MVT::v4i32: OpcodeIndex = 2; break;
2286 case MVT::v2f64:
2287 case MVT::v2i64: OpcodeIndex = 3; break;
2288 }
2289
2290 std::vector ResTys;
2291 if (isUpdating)
2292 ResTys.push_back(MVT::i32);
2293 ResTys.push_back(MVT::Other);
2294
2296 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2298
2299
2300 if (is64BitVector || NumVecs <= 2) {
2302 if (NumVecs == 1) {
2304 } else if (is64BitVector) {
2305
2308 if (NumVecs == 2)
2309 SrcReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2310 else {
2312
2313
2314 SDValue V3 = (NumVecs == 3)
2315 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
2317 SrcReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2318 }
2319 } else {
2320
2321 SDValue Q0 = N->getOperand(Vec0Idx);
2322 SDValue Q1 = N->getOperand(Vec0Idx + 1);
2323 SrcReg = SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0);
2324 }
2325
2326 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
2328 Ops.push_back(MemAddr);
2329 Ops.push_back(Align);
2330 if (isUpdating) {
2333 if (!IsImmUpdate) {
2334
2335
2338 Ops.push_back(Inc);
2339 }
2340
2341
2343 Ops.push_back(Reg0);
2344 }
2345 Ops.push_back(SrcReg);
2346 Ops.push_back(Pred);
2347 Ops.push_back(Reg0);
2348 Ops.push_back(Chain);
2349 SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2350
2351
2353
2354 ReplaceNode(N, VSt);
2355 return;
2356 }
2357
2358
2359
2360
2361
2365 SDValue V3 = (NumVecs == 3)
2366 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2368 SDValue RegSeq = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2369
2370
2371
2372 const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain };
2373 SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
2375 MVT::Other, OpsA);
2377 Chain = SDValue(VStA, 1);
2378
2379
2381 Ops.push_back(Align);
2382 if (isUpdating) {
2385 "only constant post-increment update allowed for VST3/4");
2386 (void)Inc;
2387 Ops.push_back(Reg0);
2388 }
2389 Ops.push_back(RegSeq);
2390 Ops.push_back(Pred);
2391 Ops.push_back(Reg0);
2392 Ops.push_back(Chain);
2393 SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
2396 ReplaceNode(N, VStB);
2397}
2398
2399void ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
2400 unsigned NumVecs,
2401 const uint16_t *DOpcodes,
2402 const uint16_t *QOpcodes) {
2403 assert(Subtarget->hasNEON());
2404 assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
2405 SDLoc dl(N);
2406
2408 bool IsIntrinsic = !isUpdating;
2409
2410 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2411 unsigned Vec0Idx = 3;
2412 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2413 return;
2414
2416
2419 EVT VT = N->getOperand(Vec0Idx).getValueType();
2421
2422 unsigned Alignment = 0;
2423 if (NumVecs != 3) {
2424 Alignment = Align->getAsZExtVal();
2426 if (Alignment > NumBytes)
2427 Alignment = NumBytes;
2428 if (Alignment < 8 && Alignment < NumBytes)
2429 Alignment = 0;
2430
2431 Alignment = (Alignment & -Alignment);
2432 if (Alignment == 1)
2433 Alignment = 0;
2434 }
2435 Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2436
2440
2442 case MVT::v4f16:
2443 case MVT::v4bf16:
2444 case MVT::v4i16: OpcodeIndex = 1; break;
2445 case MVT::v2f32:
2446 case MVT::v2i32: OpcodeIndex = 2; break;
2447
2448 case MVT::v8f16:
2449 case MVT::v8bf16:
2450 case MVT::v8i16: OpcodeIndex = 0; break;
2451 case MVT::v4f32:
2452 case MVT::v4i32: OpcodeIndex = 1; break;
2453 }
2454
2455 std::vector ResTys;
2456 if (IsLoad) {
2457 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2458 if (!is64BitVector)
2459 ResTyElts *= 2;
2461 MVT::i64, ResTyElts));
2462 }
2463 if (isUpdating)
2464 ResTys.push_back(MVT::i32);
2465 ResTys.push_back(MVT::Other);
2466
2468 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2469
2471 Ops.push_back(MemAddr);
2472 Ops.push_back(Align);
2473 if (isUpdating) {
2475 bool IsImmUpdate =
2477 Ops.push_back(IsImmUpdate ? Reg0 : Inc);
2478 }
2479
2483 if (NumVecs == 2) {
2484 if (is64BitVector)
2485 SuperReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2486 else
2487 SuperReg = SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0);
2488 } else {
2490 SDValue V3 = (NumVecs == 3)
2491 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2493 if (is64BitVector)
2494 SuperReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2495 else
2496 SuperReg = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2497 }
2498 Ops.push_back(SuperReg);
2499 Ops.push_back(getI32Imm(Lane, dl));
2500 Ops.push_back(Pred);
2501 Ops.push_back(Reg0);
2502 Ops.push_back(Chain);
2503
2504 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
2506 SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2508 if (!IsLoad) {
2509 ReplaceNode(N, VLdLn);
2510 return;
2511 }
2512
2513
2514 SuperReg = SDValue(VLdLn, 0);
2515 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2516 ARM::qsub_3 == ARM::qsub_0 + 3,
2517 "Unexpected subreg numbering");
2518 unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2519 for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2520 ReplaceUses(SDValue(N, Vec),
2521 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2523 if (isUpdating)
2524 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2));
2525 CurDAG->RemoveDeadNode(N);
2526}
2527
2528template
2529void ARMDAGToDAGISel::AddMVEPredicateToOps(SDValueVector &Ops, SDLoc Loc,
2530 SDValue PredicateMask) {
2531 Ops.push_back(CurDAG->getTargetConstant(ARMVCC::Then, Loc, MVT::i32));
2532 Ops.push_back(PredicateMask);
2533 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2534}
2535
2536template
2537void ARMDAGToDAGISel::AddMVEPredicateToOps(SDValueVector &Ops, SDLoc Loc,
2540 Ops.push_back(CurDAG->getTargetConstant(ARMVCC::Then, Loc, MVT::i32));
2541 Ops.push_back(PredicateMask);
2542 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2543 Ops.push_back(Inactive);
2544}
2545
2546template
2547void ARMDAGToDAGISel::AddEmptyMVEPredicateToOps(SDValueVector &Ops, SDLoc Loc) {
2548 Ops.push_back(CurDAG->getTargetConstant(ARMVCC::None, Loc, MVT::i32));
2549 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2550 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2551}
2552
2553template
2554void ARMDAGToDAGISel::AddEmptyMVEPredicateToOps(SDValueVector &Ops, SDLoc Loc,
2555 EVT InactiveTy) {
2556 Ops.push_back(CurDAG->getTargetConstant(ARMVCC::None, Loc, MVT::i32));
2557 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2558 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2560 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, InactiveTy), 0));
2561}
2562
2563void ARMDAGToDAGISel::SelectMVE_WB(SDNode *N, const uint16_t *Opcodes,
2565 SDLoc Loc(N);
2567
2568 uint16_t Opcode;
2569 switch (N->getValueType(1).getVectorElementType().getSizeInBits()) {
2570 case 32:
2571 Opcode = Opcodes[0];
2572 break;
2573 case 64:
2574 Opcode = Opcodes[1];
2575 break;
2576 default:
2577 llvm_unreachable("bad vector element size in SelectMVE_WB");
2578 }
2579
2580 Ops.push_back(N->getOperand(2));
2581
2582 int32_t ImmValue = N->getConstantOperandVal(3);
2583 Ops.push_back(getI32Imm(ImmValue, Loc));
2584
2586 AddMVEPredicateToOps(Ops, Loc, N->getOperand(4));
2587 else
2588 AddEmptyMVEPredicateToOps(Ops, Loc);
2589
2590 Ops.push_back(N->getOperand(0));
2591
2596
2597 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), VTs, Ops);
2601 transferMemOperands(N, New);
2602 CurDAG->RemoveDeadNode(N);
2603}
2604
2605void ARMDAGToDAGISel::SelectMVE_LongShift(SDNode *N, uint16_t Opcode,
2606 bool Immediate,
2607 bool HasSaturationOperand) {
2608 SDLoc Loc(N);
2610
2611
2612 Ops.push_back(N->getOperand(1));
2613 Ops.push_back(N->getOperand(2));
2614
2615
2616 if (Immediate) {
2617 int32_t ImmValue = N->getConstantOperandVal(3);
2618 Ops.push_back(getI32Imm(ImmValue, Loc));
2619 } else {
2620 Ops.push_back(N->getOperand(3));
2621 }
2622
2623
2624 if (HasSaturationOperand) {
2625 int32_t SatOp = N->getConstantOperandVal(4);
2626 int SatBit = (SatOp == 64 ? 0 : 1);
2627 Ops.push_back(getI32Imm(SatBit, Loc));
2628 }
2629
2630
2631
2632 Ops.push_back(getAL(CurDAG, Loc));
2633 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2634
2635 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), ArrayRef(Ops));
2636}
2637
2638void ARMDAGToDAGISel::SelectMVE_VADCSBC(SDNode *N, uint16_t OpcodeWithCarry,
2639 uint16_t OpcodeWithNoCarry,
2641 SDLoc Loc(N);
2643 uint16_t Opcode;
2644
2645 unsigned FirstInputOp = Predicated ? 2 : 1;
2646
2647
2648 Ops.push_back(N->getOperand(FirstInputOp));
2649 Ops.push_back(N->getOperand(FirstInputOp + 1));
2650 SDValue CarryIn = N->getOperand(FirstInputOp + 2);
2652 uint32_t CarryMask = 1 << 29;
2653 uint32_t CarryExpected = Add ? 0 : CarryMask;
2654 if (CarryInConstant &&
2655 (CarryInConstant->getZExtValue() & CarryMask) == CarryExpected) {
2656 Opcode = OpcodeWithNoCarry;
2657 } else {
2658 Ops.push_back(CarryIn);
2659 Opcode = OpcodeWithCarry;
2660 }
2661
2663 AddMVEPredicateToOps(Ops, Loc,
2664 N->getOperand(FirstInputOp + 3),
2665 N->getOperand(FirstInputOp - 1));
2666 else
2667 AddEmptyMVEPredicateToOps(Ops, Loc, N->getValueType(0));
2668
2669 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), ArrayRef(Ops));
2670}
2671
2672void ARMDAGToDAGISel::SelectMVE_VSHLC(SDNode *N, bool Predicated) {
2673 SDLoc Loc(N);
2675
2676
2677
2678 Ops.push_back(N->getOperand(1));
2679 Ops.push_back(N->getOperand(2));
2680 int32_t ImmValue = N->getConstantOperandVal(3);
2681 Ops.push_back(getI32Imm(ImmValue, Loc));
2682
2684 AddMVEPredicateToOps(Ops, Loc, N->getOperand(4));
2685 else
2686 AddEmptyMVEPredicateToOps(Ops, Loc);
2687
2688 CurDAG->SelectNodeTo(N, ARM::MVE_VSHLC, N->getVTList(), ArrayRef(Ops));
2689}
2690
2698
2699void ARMDAGToDAGISel::SelectBaseMVE_VMLLDAV(SDNode *N, bool Predicated,
2700 const uint16_t *OpcodesS,
2701 const uint16_t *OpcodesU,
2702 size_t Stride, size_t TySize) {
2703 assert(TySize < Stride && "Invalid TySize");
2707 if (IsUnsigned) {
2709 "Unsigned versions of vmlsldav[a]/vrmlsldavh[a] do not exist");
2710 assert(!IsExchange &&
2711 "Unsigned versions of vmlaldav[a]x/vrmlaldavh[a]x do not exist");
2712 }
2713
2714 auto OpIsZero = [N](size_t OpNo) {
2716 };
2717
2718
2719
2720 bool IsAccum = !(OpIsZero(4) && OpIsZero(5));
2721
2722 const uint16_t *Opcodes = IsUnsigned ? OpcodesU : OpcodesS;
2723 if (IsSub)
2724 Opcodes += 4 * Stride;
2725 if (IsExchange)
2726 Opcodes += 2 * Stride;
2727 if (IsAccum)
2728 Opcodes += Stride;
2729 uint16_t Opcode = Opcodes[TySize];
2730
2731 SDLoc Loc(N);
2733
2734 if (IsAccum) {
2735 Ops.push_back(N->getOperand(4));
2736 Ops.push_back(N->getOperand(5));
2737 }
2738
2739 Ops.push_back(N->getOperand(6));
2740 Ops.push_back(N->getOperand(7));
2741
2743 AddMVEPredicateToOps(Ops, Loc, N->getOperand(8));
2744 else
2745 AddEmptyMVEPredicateToOps(Ops, Loc);
2746
2747 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), ArrayRef(Ops));
2748}
2749
2750void ARMDAGToDAGISel::SelectMVE_VMLLDAV(SDNode *N, bool Predicated,
2751 const uint16_t *OpcodesS,
2752 const uint16_t *OpcodesU) {
2753 EVT VecTy = N->getOperand(6).getValueType();
2754 size_t SizeIndex;
2756 case 16:
2757 SizeIndex = 0;
2758 break;
2759 case 32:
2760 SizeIndex = 1;
2761 break;
2762 default:
2764 }
2765
2766 SelectBaseMVE_VMLLDAV(N, Predicated, OpcodesS, OpcodesU, 2, SizeIndex);
2767}
2768
2769void ARMDAGToDAGISel::SelectMVE_VRMLLDAVH(SDNode *N, bool Predicated,
2770 const uint16_t *OpcodesS,
2771 const uint16_t *OpcodesU) {
2773 N->getOperand(6).getValueType().getVectorElementType().getSizeInBits() ==
2774 32 &&
2775 "bad vector element size");
2776 SelectBaseMVE_VMLLDAV(N, Predicated, OpcodesS, OpcodesU, 1, 0);
2777}
2778
2779void ARMDAGToDAGISel::SelectMVE_VLD(SDNode *N, unsigned NumVecs,
2780 const uint16_t *const *Opcodes,
2781 bool HasWriteback) {
2782 EVT VT = N->getValueType(0);
2783 SDLoc Loc(N);
2784
2785 const uint16_t *OurOpcodes;
2787 case 8:
2788 OurOpcodes = Opcodes[0];
2789 break;
2790 case 16:
2791 OurOpcodes = Opcodes[1];
2792 break;
2793 case 32:
2794 OurOpcodes = Opcodes[2];
2795 break;
2796 default:
2797 llvm_unreachable("bad vector element size in SelectMVE_VLD");
2798 }
2799
2800 EVT DataTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, NumVecs * 2);
2802 unsigned PtrOperand = HasWriteback ? 1 : 2;
2803
2805 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, DataTy), 0);
2807
2808 for (unsigned Stage = 0; Stage < NumVecs - 1; ++Stage) {
2809 SDValue Ops[] = {Data, N->getOperand(PtrOperand), Chain};
2810 auto LoadInst =
2811 CurDAG->getMachineNode(OurOpcodes[Stage], Loc, ResultTys, Ops);
2813 Chain = SDValue(LoadInst, 1);
2814 transferMemOperands(N, LoadInst);
2815 }
2816
2817 if (HasWriteback)
2818 ResultTys = {DataTy, MVT::i32, MVT::Other};
2819 SDValue Ops[] = {Data, N->getOperand(PtrOperand), Chain};
2820 auto LoadInst =
2821 CurDAG->getMachineNode(OurOpcodes[NumVecs - 1], Loc, ResultTys, Ops);
2822 transferMemOperands(N, LoadInst);
2823
2824 unsigned i;
2825 for (i = 0; i < NumVecs; i++)
2827 CurDAG->getTargetExtractSubreg(ARM::qsub_0 + i, Loc, VT,
2829 if (HasWriteback)
2831 ReplaceUses(SDValue(N, i), SDValue(LoadInst, HasWriteback ? 2 : 1));
2832 CurDAG->RemoveDeadNode(N);
2833}
2834
2835void ARMDAGToDAGISel::SelectMVE_VxDUP(SDNode *N, const uint16_t *Opcodes,
2837 EVT VT = N->getValueType(0);
2838 SDLoc Loc(N);
2839
2840 uint16_t Opcode;
2842 case 8:
2843 Opcode = Opcodes[0];
2844 break;
2845 case 16:
2846 Opcode = Opcodes[1];
2847 break;
2848 case 32:
2849 Opcode = Opcodes[2];
2850 break;
2851 default:
2852 llvm_unreachable("bad vector element size in SelectMVE_VxDUP");
2853 }
2854
2856 unsigned OpIdx = 1;
2857
2861
2862 Ops.push_back(N->getOperand(OpIdx++));
2863 if (Wrapping)
2864 Ops.push_back(N->getOperand(OpIdx++));
2865
2868 Ops.push_back(getI32Imm(ImmValue, Loc));
2869
2871 AddMVEPredicateToOps(Ops, Loc, N->getOperand(OpIdx), Inactive);
2872 else
2873 AddEmptyMVEPredicateToOps(Ops, Loc, N->getValueType(0));
2874
2875 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), ArrayRef(Ops));
2876}
2877
2878void ARMDAGToDAGISel::SelectCDE_CXxD(SDNode *N, uint16_t Opcode,
2879 size_t NumExtraOps, bool HasAccum) {
2880 bool IsBigEndian = CurDAG->getDataLayout().isBigEndian();
2881 SDLoc Loc(N);
2883
2884 unsigned OpIdx = 1;
2885
2886
2888 uint32_t ImmCoprocVal = ImmCorpoc->getAsZExtVal();
2889 Ops.push_back(getI32Imm(ImmCoprocVal, Loc));
2890
2891
2892
2893 if (HasAccum) {
2896 if (IsBigEndian)
2899 }
2900
2901
2902 for (size_t I = 0; I < NumExtraOps; I++)
2903 Ops.push_back(N->getOperand(OpIdx++));
2904
2905
2907 uint32_t ImmVal = Imm->getAsZExtVal();
2908 Ops.push_back(getI32Imm(ImmVal, Loc));
2909
2910
2911 if (HasAccum) {
2913 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2914 Ops.push_back(Pred);
2915 Ops.push_back(PredReg);
2916 }
2917
2918
2919 SDNode *InstrNode = CurDAG->getMachineNode(Opcode, Loc, MVT::Untyped, Ops);
2921
2922
2923
2924
2925
2926 uint16_t SubRegs[2] = {ARM::gsub_0, ARM::gsub_1};
2927 if (IsBigEndian)
2928 std::swap(SubRegs[0], SubRegs[1]);
2929
2930 for (size_t ResIdx = 0; ResIdx < 2; ResIdx++) {
2931 if (SDValue(N, ResIdx).use_empty())
2932 continue;
2933 SDValue SubReg = CurDAG->getTargetExtractSubreg(SubRegs[ResIdx], Loc,
2934 MVT::i32, ResultPair);
2936 }
2937
2938 CurDAG->RemoveDeadNode(N);
2939}
2940
2941void ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool IsIntrinsic,
2942 bool isUpdating, unsigned NumVecs,
2943 const uint16_t *DOpcodes,
2944 const uint16_t *QOpcodes0,
2945 const uint16_t *QOpcodes1) {
2946 assert(Subtarget->hasNEON());
2947 assert(NumVecs >= 1 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
2948 SDLoc dl(N);
2949
2951 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2952 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2953 return;
2954
2956 EVT VT = N->getValueType(0);
2958
2959 unsigned Alignment = 0;
2960 if (NumVecs != 3) {
2961 Alignment = Align->getAsZExtVal();
2963 if (Alignment > NumBytes)
2964 Alignment = NumBytes;
2965 if (Alignment < 8 && Alignment < NumBytes)
2966 Alignment = 0;
2967
2968 Alignment = (Alignment & -Alignment);
2969 if (Alignment == 1)
2970 Alignment = 0;
2971 }
2972 Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2973
2977 case MVT::v8i8:
2978 case MVT::v16i8: OpcodeIndex = 0; break;
2979 case MVT::v4i16:
2980 case MVT::v8i16:
2981 case MVT::v4f16:
2982 case MVT::v8f16:
2983 case MVT::v4bf16:
2984 case MVT::v8bf16:
2986 case MVT::v2f32:
2987 case MVT::v2i32:
2988 case MVT::v4f32:
2989 case MVT::v4i32: OpcodeIndex = 2; break;
2990 case MVT::v1f64:
2991 case MVT::v1i64: OpcodeIndex = 3; break;
2992 }
2993
2994 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2995 if (!is64BitVector)
2996 ResTyElts *= 2;
2997 EVT ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
2998
2999 std::vector ResTys;
3000 ResTys.push_back(ResTy);
3001 if (isUpdating)
3002 ResTys.push_back(MVT::i32);
3003 ResTys.push_back(MVT::Other);
3004
3006 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3007
3009 Ops.push_back(MemAddr);
3010 Ops.push_back(Align);
3011 unsigned Opc = is64BitVector ? DOpcodes[OpcodeIndex]
3012 : (NumVecs == 1) ? QOpcodes0[OpcodeIndex]
3014 if (isUpdating) {
3016 bool IsImmUpdate =
3018 if (IsImmUpdate) {
3020 Ops.push_back(Reg0);
3021 } else {
3024 Ops.push_back(Inc);
3025 }
3026 }
3027 if (is64BitVector || NumVecs == 1) {
3028
3029 } else {
3031 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
3032 const SDValue OpsA[] = {MemAddr, Align, ImplDef, Pred, Reg0, Chain};
3033 SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, ResTy,
3034 MVT::Other, OpsA);
3036 Chain = SDValue(VLdA, 1);
3037 }
3038
3039 Ops.push_back(Pred);
3040 Ops.push_back(Reg0);
3041 Ops.push_back(Chain);
3042
3043 SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
3044
3045
3048
3049
3050 if (NumVecs == 1) {
3052 } else {
3054 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7, "Unexpected subreg numbering");
3055 unsigned SubIdx = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
3056 for (unsigned Vec = 0; Vec != NumVecs; ++Vec) {
3057 ReplaceUses(SDValue(N, Vec),
3058 CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
3059 }
3060 }
3062 if (isUpdating)
3063 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2));
3064 CurDAG->RemoveDeadNode(N);
3065}
3066
3067bool ARMDAGToDAGISel::tryInsertVectorElt(SDNode *N) {
3068 if (!Subtarget->hasMVEIntegerOps())
3069 return false;
3070
3071 SDLoc dl(N);
3072
3073
3074
3075
3077 SDValue Ins2 = N->getOperand(0);
3082 (VT != MVT::v8f16 && VT != MVT::v8i16) || (Ins2.getValueType() != VT))
3083 return false;
3084
3087 if (Lane2 % 2 != 0 || Lane1 != Lane2 + 1)
3088 return false;
3089
3090
3091
3095 return false;
3096
3097
3099 Val1.getOpcode() == ARMISD::VGETLANEu) &&
3101 Val2.getOpcode() == ARMISD::VGETLANEu) &&
3110
3111
3112
3114 ExtractLane1 == ExtractLane2 + 1) {
3115 SDValue NewExt = CurDAG->getTargetExtractSubreg(
3116 ARM::ssub_0 + ExtractLane2 / 2, dl, MVT::f32, Val1.getOperand(0));
3117 SDValue NewIns = CurDAG->getTargetInsertSubreg(
3118 ARM::ssub_0 + Lane2 / 2, dl, VT, Ins2.getOperand(0),
3119 NewExt);
3120 ReplaceUses(Ins1, NewIns);
3121 return true;
3122 }
3123
3124
3125
3126 if (VT == MVT::v8i16 && Subtarget->hasFullFP16()) {
3127 SDValue Inp1 = CurDAG->getTargetExtractSubreg(
3128 ARM::ssub_0 + ExtractLane1 / 2, dl, MVT::f32, Val1.getOperand(0));
3129 SDValue Inp2 = CurDAG->getTargetExtractSubreg(
3130 ARM::ssub_0 + ExtractLane2 / 2, dl, MVT::f32, Val2.getOperand(0));
3131 if (ExtractLane1 % 2 != 0)
3132 Inp1 = SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl, MVT::f32, Inp1), 0);
3133 if (ExtractLane2 % 2 != 0)
3134 Inp2 = SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl, MVT::f32, Inp2), 0);
3135 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl, MVT::f32, Inp2, Inp1);
3137 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl, MVT::v4f32,
3139 ReplaceUses(Ins1, NewIns);
3140 return true;
3141 }
3142 }
3143
3144
3145
3146 if (VT == MVT::v8f16 && Subtarget->hasFullFP16()) {
3147 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl, MVT::f32, Val2, Val1);
3149 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl, MVT::v4f32,
3151 ReplaceUses(Ins1, NewIns);
3152 return true;
3153 }
3154
3155 return false;
3156}
3157
3158bool ARMDAGToDAGISel::transformFixedFloatingPointConversion(SDNode *N,
3159 SDNode *FMul,
3160 bool IsUnsigned,
3161 bool FixedToFloat) {
3164 if (ScalarBits > 32)
3165 return false;
3166
3167 SDNodeFlags FMulFlags = FMul->getFlags();
3168
3169
3170 if (ScalarBits == 16 && !FMulFlags.hasNoInfs() && IsUnsigned)
3171 return false;
3172
3178
3180 return false;
3181
3182 if (ImmNode.getOpcode() == ISD::BITCAST) {
3184 return false;
3186 }
3187
3189 return false;
3190
3193 case ARMISD::VMOVIMM:
3194 case ARMISD::VDUP: {
3196 return false;
3198 if (ImmNode.getOpcode() == ARMISD::VMOVIMM)
3200 ImmAPF =
3201 APFloat(ScalarBits == 32 ? APFloat::IEEEsingle() : APFloat::IEEEhalf(),
3202 APInt(ScalarBits, Imm));
3203 break;
3204 }
3205 case ARMISD::VMOVFPIMM: {
3207 break;
3208 }
3209 default:
3210 return false;
3211 }
3212
3213
3214
3215
3216
3217 APFloat ToConvert = ImmAPF;
3218 if (FixedToFloat) {
3219 if (!ImmAPF.getExactInverse(&ToConvert))
3220 return false;
3221 }
3222 APSInt Converted(64, false);
3223 bool IsExact;
3225 &IsExact);
3226 if (!IsExact || !Converted.isPowerOf2())
3227 return false;
3228
3229 unsigned FracBits = Converted.logBase2();
3230 if (FracBits > ScalarBits)
3231 return false;
3232
3234 VecVal, CurDAG->getConstant(FracBits, SDLoc(N), MVT::i32)};
3235 AddEmptyMVEPredicateToOps(Ops, SDLoc(N), Type);
3236
3237 unsigned int Opcode;
3238 switch (ScalarBits) {
3239 case 16:
3240 if (FixedToFloat)
3241 Opcode = IsUnsigned ? ARM::MVE_VCVTf16u16_fix : ARM::MVE_VCVTf16s16_fix;
3242 else
3243 Opcode = IsUnsigned ? ARM::MVE_VCVTu16f16_fix : ARM::MVE_VCVTs16f16_fix;
3244 break;
3245 case 32:
3246 if (FixedToFloat)
3247 Opcode = IsUnsigned ? ARM::MVE_VCVTf32u32_fix : ARM::MVE_VCVTf32s32_fix;
3248 else
3249 Opcode = IsUnsigned ? ARM::MVE_VCVTu32f32_fix : ARM::MVE_VCVTs32f32_fix;
3250 break;
3251 default:
3253 break;
3254 }
3255
3256 ReplaceNode(N, CurDAG->getMachineNode(Opcode, SDLoc(N), Type, Ops));
3257 return true;
3258}
3259
3260bool ARMDAGToDAGISel::tryFP_TO_INT(SDNode *N, SDLoc dl) {
3261
3262 if (!Subtarget->hasMVEFloatOps())
3263 return false;
3264 EVT Type = N->getValueType(0);
3265 if (.isVector())
3266 return false;
3267 unsigned int ScalarBits = Type.getScalarSizeInBits();
3268
3271 SDNode *Node = N->getOperand(0).getNode();
3272
3273
3274
3276 if (Node->getOperand(0) != Node->getOperand(1))
3277 return false;
3278 SDNodeFlags Flags = Node->getFlags();
3279
3280
3281 if (ScalarBits == 16 && .hasNoInfs() && IsUnsigned)
3282 return false;
3283
3284 unsigned Opcode;
3285 switch (ScalarBits) {
3286 case 16:
3287 Opcode = IsUnsigned ? ARM::MVE_VCVTu16f16_fix : ARM::MVE_VCVTs16f16_fix;
3288 break;
3289 case 32:
3290 Opcode = IsUnsigned ? ARM::MVE_VCVTu32f32_fix : ARM::MVE_VCVTs32f32_fix;
3291 break;
3292 }
3294 CurDAG->getConstant(1, dl, MVT::i32)};
3295 AddEmptyMVEPredicateToOps(Ops, dl, Type);
3296
3297 ReplaceNode(N, CurDAG->getMachineNode(Opcode, dl, Type, Ops));
3298 return true;
3299 }
3300
3302 return false;
3303
3304 return transformFixedFloatingPointConversion(N, Node, IsUnsigned, false);
3305}
3306
3307bool ARMDAGToDAGISel::tryFMULFixed(SDNode *N, SDLoc dl) {
3308
3309 if (!Subtarget->hasMVEFloatOps())
3310 return false;
3311 auto Type = N->getValueType(0);
3312 if (.isVector())
3313 return false;
3314
3315 auto LHS = N->getOperand(0);
3317 return false;
3318
3319 return transformFixedFloatingPointConversion(
3321}
3322
3323bool ARMDAGToDAGISel::tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned) {
3324 if (!Subtarget->hasV6T2Ops())
3325 return false;
3326
3328 ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
3329 : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
3330 SDLoc dl(N);
3331
3332
3333 unsigned And_imm = 0;
3334 if (N->getOpcode() == ISD::AND) {
3336
3337
3338 if (And_imm & (And_imm + 1))
3339 return false;
3340
3341 unsigned Srl_imm = 0;
3343 Srl_imm)) {
3344 assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
3345
3346
3347
3348
3349 And_imm &= -1U >> Srl_imm;
3350
3351
3353 unsigned LSB = Srl_imm;
3354
3355 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3356
3357 if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) {
3358
3359 if (Subtarget->isThumb()) {
3360 Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri;
3361 SDValue Ops[] = { N->getOperand(0).getOperand(0),
3362 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3363 getAL(CurDAG, dl), Reg0, Reg0 };
3364 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
3365 return true;
3366 }
3367
3368
3372 MVT::i32);
3373 SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc,
3374 getAL(CurDAG, dl), Reg0, Reg0 };
3375 CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops);
3376 return true;
3377 }
3378
3379 assert(LSB + Width + 1 <= 32 && "Shouldn't create an invalid ubfx");
3380 SDValue Ops[] = { N->getOperand(0).getOperand(0),
3381 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3382 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3383 getAL(CurDAG, dl), Reg0 };
3384 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
3385 return true;
3386 }
3387 }
3388 return false;
3389 }
3390
3391
3392 unsigned Shl_imm = 0;
3394 assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
3395 unsigned Srl_imm = 0;
3397 assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
3398
3399 unsigned Width = 32 - Srl_imm - 1;
3400 int LSB = Srl_imm - Shl_imm;
3401 if (LSB < 0)
3402 return false;
3403 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3404 assert(LSB + Width + 1 <= 32 && "Shouldn't create an invalid ubfx");
3405 SDValue Ops[] = { N->getOperand(0).getOperand(0),
3406 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3407 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3408 getAL(CurDAG, dl), Reg0 };
3409 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
3410 return true;
3411 }
3412 }
3413
3414
3417 unsigned Srl_imm = 0;
3419
3420 if (isInt32Immediate(N->getOperand(1), Srl_imm) && Srl_imm == LSB) {
3421 assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
3423
3424 unsigned Width = MSB - LSB;
3425 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3426 assert(Srl_imm + Width + 1 <= 32 && "Shouldn't create an invalid ubfx");
3427 SDValue Ops[] = { N->getOperand(0).getOperand(0),
3428 CurDAG->getTargetConstant(Srl_imm, dl, MVT::i32),
3429 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3430 getAL(CurDAG, dl), Reg0 };
3431 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
3432 return true;
3433 }
3434 }
3435
3437 unsigned Width = cast(N->getOperand(1))->getVT().getSizeInBits();
3438 unsigned LSB = 0;
3441 return false;
3442
3443 if (LSB + Width > 32)
3444 return false;
3445
3446 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3447 assert(LSB + Width <= 32 && "Shouldn't create an invalid ubfx");
3448 SDValue Ops[] = { N->getOperand(0).getOperand(0),
3449 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3450 CurDAG->getTargetConstant(Width - 1, dl, MVT::i32),
3451 getAL(CurDAG, dl), Reg0 };
3452 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
3453 return true;
3454 }
3455
3456 return false;
3457}
3458
3459
3460void ARMDAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
3461 unsigned Opcode;
3463 if (MemTy == MVT::i8)
3464 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_8 : ARM::CMP_SWAP_8;
3465 else if (MemTy == MVT::i16)
3466 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_16 : ARM::CMP_SWAP_16;
3467 else if (MemTy == MVT::i32)
3468 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_32 : ARM::CMP_SWAP_32;
3469 else
3471
3472 SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3),
3473 N->getOperand(0)};
3474 SDNode *CmpSwap = CurDAG->getMachineNode(
3475 Opcode, SDLoc(N),
3476 CurDAG->getVTList(MVT::i32, MVT::i32, MVT::Other), Ops);
3477
3478 MachineMemOperand *MemOp = cast(N)->getMemOperand();
3480
3483 CurDAG->RemoveDeadNode(N);
3484}
3485
3486static std::optional<std::pair<unsigned, unsigned>>
3488 unsigned FirstOne = A.getBitWidth() - A.countl_zero() - 1;
3489 unsigned LastOne = A.countr_zero();
3490 if (A.popcount() != (FirstOne - LastOne + 1))
3491 return std::nullopt;
3492 return std::make_pair(FirstOne, LastOne);
3493}
3494
3495void ARMDAGToDAGISel::SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI) {
3496 assert(N->getOpcode() == ARMISD::CMPZ);
3497 SwitchEQNEToPLMI = false;
3498
3499 if (!Subtarget->isThumb())
3500
3501
3502 return;
3503
3504
3506 if (->hasOneUse())
3507 return;
3508
3511 return;
3514
3515 if ()
3516 return;
3519 return;
3520
3521
3522 SDNode *NewN;
3523 SDLoc dl(N);
3524
3525 auto EmitShift = [&](unsigned Opc, SDValue Src, unsigned Imm) -> SDNode* {
3526 if (Subtarget->isThumb2()) {
3527 Opc = (Opc == ARM::tLSLri) ? ARM::t2LSLri : ARM::t2LSRri;
3528 SDValue Ops[] = { Src, CurDAG->getTargetConstant(Imm, dl, MVT::i32),
3529 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
3530 CurDAG->getRegister(0, MVT::i32) };
3531 return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
3532 } else {
3533 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), Src,
3534 CurDAG->getTargetConstant(Imm, dl, MVT::i32),
3535 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
3536 return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
3537 }
3538 };
3539
3540 if (Range->second == 0) {
3541
3542 NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
3543 ReplaceNode(And.getNode(), NewN);
3544 } else if (Range->first == 31) {
3545
3546 NewN = EmitShift(ARM::tLSRri, X, Range->second);
3547 ReplaceNode(And.getNode(), NewN);
3548 } else if (Range->first == Range->second) {
3549
3550
3551
3552
3553 if (->hasOneUse())
3554 return;
3555 NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
3556 ReplaceNode(And.getNode(), NewN);
3557
3558 SwitchEQNEToPLMI = true;
3559 } else if (!Subtarget->hasV6T2Ops()) {
3560
3561
3562 NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
3563 NewN = EmitShift(ARM::tLSRri, SDValue(NewN, 0),
3564 Range->second + (31 - Range->first));
3565 ReplaceNode(And.getNode(), NewN);
3566 }
3567}
3568
3570 unsigned Opc128[3]) {
3572 "Unexpected vector shuffle length");
3574 default:
3576 case 8:
3577 return VT.is64BitVector() ? Opc64[0] : Opc128[0];
3578 case 16:
3579 return VT.is64BitVector() ? Opc64[1] : Opc128[1];
3580 case 32:
3581 return VT.is64BitVector() ? Opc64[2] : Opc128[2];
3582 }
3583}
3584
3585void ARMDAGToDAGISel::Select(SDNode *N) {
3586 SDLoc dl(N);
3587
3588 if (N->isMachineOpcode()) {
3589 N->setNodeId(-1);
3590 return;
3591 }
3592
3593 switch (N->getOpcode()) {
3594 default: break;
3595 case ISD::STORE: {
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3608 if (Subtarget->isThumb1Only() && ST->isUnindexed()) {
3609 int RHSC = 0;
3613
3618 CurDAG->getRegister(ARM::SP, MVT::i32),
3619 CurDAG->getTargetConstant(RHSC, dl, MVT::i32),
3620 getAL(CurDAG, dl),
3621 CurDAG->getRegister(0, MVT::i32),
3622 ST->getChain()};
3623 MachineSDNode *ResNode =
3624 CurDAG->getMachineNode(ARM::tSTRspi, dl, MVT::Other, Ops);
3625 MachineMemOperand *MemOp = ST->getMemOperand();
3627 ReplaceNode(N, ResNode);
3628 return;
3629 }
3630 }
3631 break;
3632 }
3634 if (tryWriteRegister(N))
3635 return;
3636 break;
3638 if (tryReadRegister(N))
3639 return;
3640 break;
3641 case ISD::INLINEASM:
3642 case ISD::INLINEASM_BR:
3643 if (tryInlineAsm(N))
3644 return;
3645 break;
3647 unsigned Val = N->getAsZExtVal();
3648
3650 !Subtarget->genExecuteOnly()) {
3651 SDValue CPIdx = CurDAG->getTargetConstantPool(
3652 ConstantInt::get(Type::getInt32Ty(*CurDAG->getContext()), Val),
3653 TLI->getPointerTy(CurDAG->getDataLayout()));
3654
3655 SDNode *ResNode;
3656 if (Subtarget->isThumb()) {
3658 CPIdx,
3659 getAL(CurDAG, dl),
3660 CurDAG->getRegister(0, MVT::i32),
3661 CurDAG->getEntryNode()
3662 };
3663 ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
3665 } else {
3667 CPIdx,
3668 CurDAG->getTargetConstant(0, dl, MVT::i32),
3669 getAL(CurDAG, dl),
3670 CurDAG->getRegister(0, MVT::i32),
3671 CurDAG->getEntryNode()
3672 };
3673 ResNode = CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
3675 }
3676
3677
3678
3679 MachineFunction& MF = CurDAG->getMachineFunction();
3680 MachineMemOperand *MemOp =
3683
3685
3686 ReplaceNode(N, ResNode);
3687 return;
3688 }
3689
3690
3691 break;
3692 }
3694
3696 SDValue TFI = CurDAG->getTargetFrameIndex(
3697 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
3699
3700
3704 CurDAG->SelectNodeTo(N, ARM::tADDframe, MVT::i32, TFI,
3705 CurDAG->getTargetConstant(0, dl, MVT::i32));
3706 return;
3707 } else {
3708 unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
3709 ARM::t2ADDri : ARM::ADDri);
3710 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, dl, MVT::i32),
3711 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
3712 CurDAG->getRegister(0, MVT::i32) };
3713 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
3714 return;
3715 }
3716 }
3718 if (tryInsertVectorElt(N))
3719 return;
3720 break;
3721 }
3723 if (tryV6T2BitfieldExtractOp(N, false))
3724 return;
3725 break;
3728 if (tryV6T2BitfieldExtractOp(N, true))
3729 return;
3730 break;
3735 if (tryFP_TO_INT(N, dl))
3736 return;
3737 break;
3739 if (tryFMULFixed(N, dl))
3740 return;
3741 break;
3744 break;
3746 unsigned RHSV = C->getZExtValue();
3747 if (!RHSV) break;
3749 unsigned ShImm = Log2_32(RHSV-1);
3750 if (ShImm >= 32)
3751 break;
3754 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
3755 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3756 if (Subtarget->isThumb()) {
3757 SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
3758 CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops);
3759 return;
3760 } else {
3762 Reg0 };
3763 CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops);
3764 return;
3765 }
3766 }
3768 unsigned ShImm = Log2_32(RHSV+1);
3769 if (ShImm >= 32)
3770 break;
3773 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
3774 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3775 if (Subtarget->isThumb()) {
3776 SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
3777 CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops);
3778 return;
3779 } else {
3781 Reg0 };
3782 CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops);
3783 return;
3784 }
3785 }
3786 }
3787 break;
3789
3790 if (tryV6T2BitfieldExtractOp(N, false))
3791 return;
3792
3793
3794
3795
3797 if (N1C && N1C->hasOneUse() && Subtarget->isThumb()) {
3798 uint32_t Imm = (uint32_t) N1C->getZExtValue();
3799
3800
3801
3802
3803
3804 bool PreferImmediateEncoding =
3805 Subtarget->hasThumb2() && (is_t2_so_imm(Imm) || is_t2_so_imm_not(Imm));
3806 if (!PreferImmediateEncoding &&
3809
3810
3811 SDValue NewImm = CurDAG->getConstant(~Imm, dl, MVT::i32);
3812
3813
3815 CurDAG->RepositionNode(N->getIterator(), NewImm.getNode());
3816
3817 if (!Subtarget->hasThumb2()) {
3818 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32),
3819 N->getOperand(0), NewImm, getAL(CurDAG, dl),
3820 CurDAG->getRegister(0, MVT::i32)};
3821 ReplaceNode(N, CurDAG->getMachineNode(ARM::tBIC, dl, MVT::i32, Ops));
3822 return;
3823 } else {
3824 SDValue Ops[] = {N->getOperand(0), NewImm, getAL(CurDAG, dl),
3825 CurDAG->getRegister(0, MVT::i32),
3826 CurDAG->getRegister(0, MVT::i32)};
3827 ReplaceNode(N,
3828 CurDAG->getMachineNode(ARM::t2BICrr, dl, MVT::i32, Ops));
3829 return;
3830 }
3831 }
3832 }
3833
3834
3835
3836
3837
3838
3839 EVT VT = N->getValueType(0);
3840 if (VT != MVT::i32)
3841 break;
3842 unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
3843 ? ARM::t2MOVTi16
3844 : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
3845 if ()
3846 break;
3849 if (!N1C)
3850 break;
3854 if (!N2C)
3855 break;
3856 unsigned N1CVal = N1C->getZExtValue();
3858 if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
3859 (N1CVal & 0xffffU) == 0xffffU &&
3860 (N2CVal & 0xffffU) == 0x0U) {
3861 SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
3862 dl, MVT::i32);
3864 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) };
3865 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
3866 return;
3867 }
3868 }
3869
3870 break;
3871 }
3872 case ARMISD::UMAAL: {
3873 unsigned Opc = Subtarget->isThumb() ? ARM::t2UMAAL : ARM::UMAAL;
3874 SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
3875 N->getOperand(2), N->getOperand(3),
3876 getAL(CurDAG, dl),
3877 CurDAG->getRegister(0, MVT::i32) };
3878 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::i32, Ops));
3879 return;
3880 }
3881 case ARMISD::UMLAL:{
3882 if (Subtarget->isThumb()) {
3883 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
3884 N->getOperand(3), getAL(CurDAG, dl),
3885 CurDAG->getRegister(0, MVT::i32)};
3886 ReplaceNode(
3887 N, CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops));
3888 return;
3889 }else{
3890 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
3891 N->getOperand(3), getAL(CurDAG, dl),
3892 CurDAG->getRegister(0, MVT::i32),
3893 CurDAG->getRegister(0, MVT::i32) };
3894 ReplaceNode(N, CurDAG->getMachineNode(
3895 Subtarget->hasV6Ops() ? ARM::UMLAL : ARM::UMLALv5, dl,
3896 MVT::i32, MVT::i32, Ops));
3897 return;
3898 }
3899 }
3900 case ARMISD::SMLAL:{
3901 if (Subtarget->isThumb()) {
3902 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
3903 N->getOperand(3), getAL(CurDAG, dl),
3904 CurDAG->getRegister(0, MVT::i32)};
3905 ReplaceNode(
3906 N, CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops));
3907 return;
3908 }else{
3909 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
3910 N->getOperand(3), getAL(CurDAG, dl),
3911 CurDAG->getRegister(0, MVT::i32),
3912 CurDAG->getRegister(0, MVT::i32) };
3913 ReplaceNode(N, CurDAG->getMachineNode(
3914 Subtarget->hasV6Ops() ? ARM::SMLAL : ARM::SMLALv5, dl,
3915 MVT::i32, MVT::i32, Ops));
3916 return;
3917 }
3918 }
3919 case ARMISD::SUBE: {
3920 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
3921 break;
3922
3923
3925 N->getOperand(2).getOpcode() != ARMISD::SUBC ||
3927 break;
3928
3929 if (Subtarget->isThumb())
3930 assert(Subtarget->hasThumb2() &&
3931 "This pattern should not be generated for Thumb");
3932
3933 SDValue SmulLoHi = N->getOperand(1);
3934 SDValue Subc = N->getOperand(2);
3936
3940 break;
3941
3942 unsigned Opc = Subtarget->isThumb2() ? ARM::t2SMMLS : ARM::SMMLS;
3945 CurDAG->getRegister(0, MVT::i32) };
3946 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops));
3947 return;
3948 }
3949 case ISD::LOAD: {
3950 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(N))
3951 return;
3952 if (Subtarget->isThumb() && Subtarget->hasThumb2()) {
3953 if (tryT2IndexedLoad(N))
3954 return;
3955 } else if (Subtarget->isThumb()) {
3956 if (tryT1IndexedLoad(N))
3957 return;
3958 } else if (tryARMIndexedLoad(N))
3959 return;
3960
3961 break;
3962 }
3963 case ISD::MLOAD:
3964 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(N))
3965 return;
3966
3967 break;
3968 case ARMISD::LDRD: {
3970 break;
3974 SelectAddrMode3(Addr, Base, RegOffset, ImmOffset);
3975 if (RegOffset != CurDAG->getRegister(0, MVT::i32)) {
3976
3977
3978
3979
3980 Base = Addr;
3981 RegOffset = CurDAG->getRegister(0, MVT::i32);
3982 }
3983 SDValue Ops[] = {Base, RegOffset, ImmOffset, Chain};
3984 SDNode *New = CurDAG->getMachineNode(ARM::LOADDUAL, dl,
3985 {MVT::Untyped, MVT::Other}, Ops);
3986 SDValue Lo = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
3988 SDValue Hi = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
3990 transferMemOperands(N, New);
3994 CurDAG->RemoveDeadNode(N);
3995 return;
3996 }
3997 case ARMISD::STRD: {
3999 break;
4003 SelectAddrMode3(Addr, Base, RegOffset, ImmOffset);
4004 if (RegOffset != CurDAG->getRegister(0, MVT::i32)) {
4005
4006
4007
4008
4009 Base = Addr;
4010 RegOffset = CurDAG->getRegister(0, MVT::i32);
4011 }
4012 SDNode *RegPair =
4015 SDNode *New = CurDAG->getMachineNode(ARM::STOREDUAL, dl, MVT::Other, Ops);
4016 transferMemOperands(N, New);
4018 CurDAG->RemoveDeadNode(N);
4019 return;
4020 }
4021 case ARMISD::BRCOND: {
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034 unsigned Opc = Subtarget->isThumb() ?
4035 ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
4042
4044
4045 if (Flags.getOpcode() == ARMISD::CMPZ) {
4048 uint64_t ID = Int->getConstantOperandVal(1);
4049
4050
4051 if (ID == Intrinsic::loop_decrement_reg) {
4053 SDValue Size = CurDAG->getTargetConstant(Int.getConstantOperandVal(3),
4054 dl, MVT::i32);
4055
4057 SDNode *LoopDec =
4058 CurDAG->getMachineNode(ARM::t2LoopDec, dl,
4059 CurDAG->getVTList(MVT::i32, MVT::Other),
4060 Args);
4061 ReplaceUses(Int.getNode(), LoopDec);
4062
4063 SDValue EndArgs[] = { SDValue(LoopDec, 0), N1, Chain };
4064 SDNode *LoopEnd =
4065 CurDAG->getMachineNode(ARM::t2LoopEnd, dl, MVT::Other, EndArgs);
4066
4067 ReplaceUses(N, LoopEnd);
4068 CurDAG->RemoveDeadNode(N);
4069 CurDAG->RemoveDeadNode(Flags.getNode());
4070 CurDAG->RemoveDeadNode(Int.getNode());
4071 return;
4072 }
4073 }
4074
4075 bool SwitchEQNEToPLMI;
4076 SelectCMPZ(Flags.getNode(), SwitchEQNEToPLMI);
4077 Flags = N->getOperand(3);
4078
4079 if (SwitchEQNEToPLMI) {
4084 break;
4087 break;
4088 }
4089 }
4090 }
4091
4092 SDValue Tmp2 = CurDAG->getTargetConstant(CC, dl, MVT::i32);
4093 Chain = CurDAG->getCopyToReg(Chain, dl, ARM::CPSR, Flags, SDValue());
4094 SDValue Ops[] = {N1, Tmp2, CurDAG->getRegister(ARM::CPSR, MVT::i32), Chain,
4096 CurDAG->SelectNodeTo(N, Opc, MVT::Other, Ops);
4097 return;
4098 }
4099
4100 case ARMISD::CMPZ: {
4101
4102
4103
4104
4107 if (C && C->getSExtValue() < 0 && Subtarget->isThumb()) {
4108 int64_t Addend = -C->getSExtValue();
4109
4110 SDNode *Add = nullptr;
4111
4112
4113
4114
4115 if (Addend < 1<<8) {
4116 if (Subtarget->isThumb2()) {
4117 SDValue Ops[] = { X, CurDAG->getTargetConstant(Addend, dl, MVT::i32),
4118 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
4119 CurDAG->getRegister(0, MVT::i32) };
4120 Add = CurDAG->getMachineNode(ARM::t2ADDri, dl, MVT::i32, Ops);
4121 } else {
4122 unsigned Opc = (Addend < 1<<3) ? ARM::tADDi3 : ARM::tADDi8;
4123 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), X,
4124 CurDAG->getTargetConstant(Addend, dl, MVT::i32),
4125 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
4126 Add = CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
4127 }
4128 }
4129 if (Add) {
4130 SDValue Ops2[] = {SDValue(Add, 0), CurDAG->getConstant(0, dl, MVT::i32)};
4131 CurDAG->MorphNodeTo(N, ARMISD::CMPZ, N->getVTList(), Ops2);
4132 }
4133 }
4134
4135 break;
4136 }
4137
4138 case ARMISD::CMOV: {
4140
4141 if (Flags.getOpcode() == ARMISD::CMPZ) {
4142 bool SwitchEQNEToPLMI;
4143 SelectCMPZ(Flags.getNode(), SwitchEQNEToPLMI);
4144
4145 if (SwitchEQNEToPLMI) {
4146 SDValue ARMcc = N->getOperand(2);
4148
4149 switch (CC) {
4153 break;
4156 break;
4157 }
4158 SDValue NewARMcc = CurDAG->getConstant((unsigned)CC, dl, MVT::i32);
4159 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), NewARMcc,
4161 CurDAG->MorphNodeTo(N, ARMISD::CMOV, N->getVTList(), Ops);
4162 }
4163 }
4164
4165 break;
4166 }
4167 case ARMISD::VZIP: {
4168 EVT VT = N->getValueType(0);
4169
4170 unsigned Opc64[] = {ARM::VZIPd8, ARM::VZIPd16, ARM::VTRNd32};
4171 unsigned Opc128[] = {ARM::VZIPq8, ARM::VZIPq16, ARM::VZIPq32};
4174 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4175 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), Pred, PredReg};
4176 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4177 return;
4178 }
4179 case ARMISD::VUZP: {
4180 EVT VT = N->getValueType(0);
4181
4182 unsigned Opc64[] = {ARM::VUZPd8, ARM::VUZPd16, ARM::VTRNd32};
4183 unsigned Opc128[] = {ARM::VUZPq8, ARM::VUZPq16, ARM::VUZPq32};
4186 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4187 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), Pred, PredReg};
4188 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4189 return;
4190 }
4191 case ARMISD::VTRN: {
4192 EVT VT = N->getValueType(0);
4193 unsigned Opc64[] = {ARM::VTRNd8, ARM::VTRNd16, ARM::VTRNd32};
4194 unsigned Opc128[] = {ARM::VTRNq8, ARM::VTRNq16, ARM::VTRNq32};
4197 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4198 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), Pred, PredReg};
4199 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4200 return;
4201 }
4203 EVT VecVT = N->getValueType(0);
4206 if (EltVT == MVT::f64) {
4207 assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
4208 ReplaceNode(
4209 N, createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)));
4210 return;
4211 }
4212 assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR");
4213 if (NumElts == 2) {
4214 ReplaceNode(
4215 N, createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)));
4216 return;
4217 }
4218 assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
4219 ReplaceNode(N,
4220 createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1),
4221 N->getOperand(2), N->getOperand(3)));
4222 return;
4223 }
4224
4226 static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8, ARM::VLD1DUPd16,
4227 ARM::VLD1DUPd32 };
4228 static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8, ARM::VLD1DUPq16,
4229 ARM::VLD1DUPq32 };
4230 SelectVLDDup(N, false, false, 1, DOpcodes, QOpcodes);
4231 return;
4232 }
4233
4235 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
4236 ARM::VLD2DUPd32 };
4237 SelectVLDDup(N, false, false, 2, Opcodes);
4238 return;
4239 }
4240
4242 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
4243 ARM::VLD3DUPd16Pseudo,
4244 ARM::VLD3DUPd32Pseudo };
4245 SelectVLDDup(N, false, false, 3, Opcodes);
4246 return;
4247 }
4248
4250 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
4251 ARM::VLD4DUPd16Pseudo,
4252 ARM::VLD4DUPd32Pseudo };
4253 SelectVLDDup(N, false, false, 4, Opcodes);
4254 return;
4255 }
4256
4258 static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8wb_fixed,
4259 ARM::VLD1DUPd16wb_fixed,
4260 ARM::VLD1DUPd32wb_fixed };
4261 static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8wb_fixed,
4262 ARM::VLD1DUPq16wb_fixed,
4263 ARM::VLD1DUPq32wb_fixed };
4264 SelectVLDDup(N, false, true, 1, DOpcodes, QOpcodes);
4265 return;
4266 }
4267
4269 static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8wb_fixed,
4270 ARM::VLD2DUPd16wb_fixed,
4271 ARM::VLD2DUPd32wb_fixed,
4272 ARM::VLD1q64wb_fixed };
4273 static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
4274 ARM::VLD2DUPq16EvenPseudo,
4275 ARM::VLD2DUPq32EvenPseudo };
4276 static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudoWB_fixed,
4277 ARM::VLD2DUPq16OddPseudoWB_fixed,
4278 ARM::VLD2DUPq32OddPseudoWB_fixed };
4279 SelectVLDDup(N, false, true, 2, DOpcodes, QOpcodes0, QOpcodes1);
4280 return;
4281 }
4282
4284 static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
4285 ARM::VLD3DUPd16Pseudo_UPD,
4286 ARM::VLD3DUPd32Pseudo_UPD,
4287 ARM::VLD1d64TPseudoWB_fixed };
4288 static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
4289 ARM::VLD3DUPq16EvenPseudo,
4290 ARM::VLD3DUPq32EvenPseudo };
4291 static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo_UPD,
4292 ARM::VLD3DUPq16OddPseudo_UPD,
4293 ARM::VLD3DUPq32OddPseudo_UPD };
4294 SelectVLDDup(N, false, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4295 return;
4296 }
4297
4299 static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
4300 ARM::VLD4DUPd16Pseudo_UPD,
4301 ARM::VLD4DUPd32Pseudo_UPD,
4302 ARM::VLD1d64QPseudoWB_fixed };
4303 static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
4304 ARM::VLD4DUPq16EvenPseudo,
4305 ARM::VLD4DUPq32EvenPseudo };
4306 static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo_UPD,
4307 ARM::VLD4DUPq16OddPseudo_UPD,
4308 ARM::VLD4DUPq32OddPseudo_UPD };
4309 SelectVLDDup(N, false, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4310 return;
4311 }
4312
4314 static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
4315 ARM::VLD1d16wb_fixed,
4316 ARM::VLD1d32wb_fixed,
4317 ARM::VLD1d64wb_fixed };
4318 static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
4319 ARM::VLD1q16wb_fixed,
4320 ARM::VLD1q32wb_fixed,
4321 ARM::VLD1q64wb_fixed };
4322 SelectVLD(N, true, 1, DOpcodes, QOpcodes, nullptr);
4323 return;
4324 }
4325
4327 if (Subtarget->hasNEON()) {
4328 static const uint16_t DOpcodes[] = {
4329 ARM::VLD2d8wb_fixed, ARM::VLD2d16wb_fixed, ARM::VLD2d32wb_fixed,
4330 ARM::VLD1q64wb_fixed};
4331 static const uint16_t QOpcodes[] = {ARM::VLD2q8PseudoWB_fixed,
4332 ARM::VLD2q16PseudoWB_fixed,
4333 ARM::VLD2q32PseudoWB_fixed};
4334 SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr);
4335 } else {
4336 static const uint16_t Opcodes8[] = {ARM::MVE_VLD20_8,
4337 ARM::MVE_VLD21_8_wb};
4338 static const uint16_t Opcodes16[] = {ARM::MVE_VLD20_16,
4339 ARM::MVE_VLD21_16_wb};
4340 static const uint16_t Opcodes32[] = {ARM::MVE_VLD20_32,
4341 ARM::MVE_VLD21_32_wb};
4342 static const uint16_t *const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
4343 SelectMVE_VLD(N, 2, Opcodes, true);
4344 }
4345 return;
4346 }
4347
4349 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
4350 ARM::VLD3d16Pseudo_UPD,
4351 ARM::VLD3d32Pseudo_UPD,
4352 ARM::VLD1d64TPseudoWB_fixed};
4353 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
4354 ARM::VLD3q16Pseudo_UPD,
4355 ARM::VLD3q32Pseudo_UPD };
4356 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
4357 ARM::VLD3q16oddPseudo_UPD,
4358 ARM::VLD3q32oddPseudo_UPD };
4359 SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4360 return;
4361 }
4362
4364 if (Subtarget->hasNEON()) {
4365 static const uint16_t DOpcodes[] = {
4366 ARM::VLD4d8Pseudo_UPD, ARM::VLD4d16Pseudo_UPD, ARM::VLD4d32Pseudo_UPD,
4367 ARM::VLD1d64QPseudoWB_fixed};
4368 static const uint16_t QOpcodes0[] = {ARM::VLD4q8Pseudo_UPD,
4369 ARM::VLD4q16Pseudo_UPD,
4370 ARM::VLD4q32Pseudo_UPD};
4371 static const uint16_t QOpcodes1[] = {ARM::VLD4q8oddPseudo_UPD,
4372 ARM::VLD4q16oddPseudo_UPD,
4373 ARM::VLD4q32oddPseudo_UPD};
4374 SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4375 } else {
4376 static const uint16_t Opcodes8[] = {ARM::MVE_VLD40_8, ARM::MVE_VLD41_8,
4377 ARM::MVE_VLD42_8,
4378 ARM::MVE_VLD43_8_wb};
4379 static const uint16_t Opcodes16[] = {ARM::MVE_VLD40_16, ARM::MVE_VLD41_16,
4380 ARM::MVE_VLD42_16,
4381 ARM::MVE_VLD43_16_wb};
4382 static const uint16_t Opcodes32[] = {ARM::MVE_VLD40_32, ARM::MVE_VLD41_32,
4383 ARM::MVE_VLD42_32,
4384 ARM::MVE_VLD43_32_wb};
4385 static const uint16_t *const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
4386 SelectMVE_VLD(N, 4, Opcodes, true);
4387 }
4388 return;
4389 }
4390
4392 if (Subtarget->hasNEON()) {
4393 static const uint16_t DOpcodes[] = {
4394 ARM::VLD1q8wb_fixed, ARM::VLD1q16wb_fixed, ARM::VLD1q32wb_fixed,
4395 ARM::VLD1q64wb_fixed};
4396 static const uint16_t QOpcodes[] = {
4397 ARM::VLD1d8QPseudoWB_fixed, ARM::VLD1d16QPseudoWB_fixed,
4398 ARM::VLD1d32QPseudoWB_fixed, ARM::VLD1d64QPseudoWB_fixed};
4399 SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr);
4400 return;
4401 }
4402 break;
4403 }
4404
4406 if (Subtarget->hasNEON()) {
4407 static const uint16_t DOpcodes[] = {
4408 ARM::VLD1d8TPseudoWB_fixed, ARM::VLD1d16TPseudoWB_fixed,
4409 ARM::VLD1d32TPseudoWB_fixed, ARM::VLD1d64TPseudoWB_fixed};
4410 static const uint16_t QOpcodes0[] = {
4411 ARM::VLD1q8LowTPseudo_UPD, ARM::VLD1q16LowTPseudo_UPD,
4412 ARM::VLD1q32LowTPseudo_UPD, ARM::VLD1q64LowTPseudo_UPD};
4413 static const uint16_t QOpcodes1[] = {
4414 ARM::VLD1q8HighTPseudo_UPD, ARM::VLD1q16HighTPseudo_UPD,
4415 ARM::VLD1q32HighTPseudo_UPD, ARM::VLD1q64HighTPseudo_UPD};
4416 SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4417 return;
4418 }
4419 break;
4420 }
4421
4423 if (Subtarget->hasNEON()) {
4424 static const uint16_t DOpcodes[] = {
4425 ARM::VLD1d8QPseudoWB_fixed, ARM::VLD1d16QPseudoWB_fixed,
4426 ARM::VLD1d32QPseudoWB_fixed, ARM::VLD1d64QPseudoWB_fixed};
4427 static const uint16_t QOpcodes0[] = {
4428 ARM::VLD1q8LowQPseudo_UPD, ARM::VLD1q16LowQPseudo_UPD,
4429 ARM::VLD1q32LowQPseudo_UPD, ARM::VLD1q64LowQPseudo_UPD};
4430 static const uint16_t QOpcodes1[] = {
4431 ARM::VLD1q8HighQPseudo_UPD, ARM::VLD1q16HighQPseudo_UPD,
4432 ARM::VLD1q32HighQPseudo_UPD, ARM::VLD1q64HighQPseudo_UPD};
4433 SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4434 return;
4435 }
4436 break;
4437 }
4438
4440 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
4441 ARM::VLD2LNd16Pseudo_UPD,
4442 ARM::VLD2LNd32Pseudo_UPD };
4443 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
4444 ARM::VLD2LNq32Pseudo_UPD };
4445 SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes);
4446 return;
4447 }
4448
4450 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
4451 ARM::VLD3LNd16Pseudo_UPD,
4452 ARM::VLD3LNd32Pseudo_UPD };
4453 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
4454 ARM::VLD3LNq32Pseudo_UPD };
4455 SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes);
4456 return;
4457 }
4458
4460 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
4461 ARM::VLD4LNd16Pseudo_UPD,
4462 ARM::VLD4LNd32Pseudo_UPD };
4463 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
4464 ARM::VLD4LNq32Pseudo_UPD };
4465 SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes);
4466 return;
4467 }
4468
4470 static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
4471 ARM::VST1d16wb_fixed,
4472 ARM::VST1d32wb_fixed,
4473 ARM::VST1d64wb_fixed };
4474 static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
4475 ARM::VST1q16wb_fixed,
4476 ARM::VST1q32wb_fixed,
4477 ARM::VST1q64wb_fixed };
4478 SelectVST(N, true, 1, DOpcodes, QOpcodes, nullptr);
4479 return;
4480 }
4481
4482 case ARMISD::VST2_UPD: {
4483 if (Subtarget->hasNEON()) {
4484 static const uint16_t DOpcodes[] = {
4485 ARM::VST2d8wb_fixed, ARM::VST2d16wb_fixed, ARM::VST2d32wb_fixed,
4486 ARM::VST1q64wb_fixed};
4487 static const uint16_t QOpcodes[] = {ARM::VST2q8PseudoWB_fixed,
4488 ARM::VST2q16PseudoWB_fixed,
4489 ARM::VST2q32PseudoWB_fixed};
4490 SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr);
4491 return;
4492 }
4493 break;
4494 }
4495
4497 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
4498 ARM::VST3d16Pseudo_UPD,
4499 ARM::VST3d32Pseudo_UPD,
4500 ARM::VST1d64TPseudoWB_fixed};
4501 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
4502 ARM::VST3q16Pseudo_UPD,
4503 ARM::VST3q32Pseudo_UPD };
4504 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
4505 ARM::VST3q16oddPseudo_UPD,
4506 ARM::VST3q32oddPseudo_UPD };
4507 SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4508 return;
4509 }
4510
4511 case ARMISD::VST4_UPD: {
4512 if (Subtarget->hasNEON()) {
4513 static const uint16_t DOpcodes[] = {
4514 ARM::VST4d8Pseudo_UPD, ARM::VST4d16Pseudo_UPD, ARM::VST4d32Pseudo_UPD,
4515 ARM::VST1d64QPseudoWB_fixed};
4516 static const uint16_t QOpcodes0[] = {ARM::VST4q8Pseudo_UPD,
4517 ARM::VST4q16Pseudo_UPD,
4518 ARM::VST4q32Pseudo_UPD};
4519 static const uint16_t QOpcodes1[] = {ARM::VST4q8oddPseudo_UPD,
4520 ARM::VST4q16oddPseudo_UPD,
4521 ARM::VST4q32oddPseudo_UPD};
4522 SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4523 return;
4524 }
4525 break;
4526 }
4527
4529 if (Subtarget->hasNEON()) {
4530 static const uint16_t DOpcodes[] = { ARM::VST1q8wb_fixed,
4531 ARM::VST1q16wb_fixed,
4532 ARM::VST1q32wb_fixed,
4533 ARM::VST1q64wb_fixed};
4534 static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudoWB_fixed,
4535 ARM::VST1d16QPseudoWB_fixed,
4536 ARM::VST1d32QPseudoWB_fixed,
4537 ARM::VST1d64QPseudoWB_fixed };
4538 SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr);
4539 return;
4540 }
4541 break;
4542 }
4543
4545 if (Subtarget->hasNEON()) {
4546 static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudoWB_fixed,
4547 ARM::VST1d16TPseudoWB_fixed,
4548 ARM::VST1d32TPseudoWB_fixed,
4549 ARM::VST1d64TPseudoWB_fixed };
4550 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD,
4551 ARM::VST1q16LowTPseudo_UPD,
4552 ARM::VST1q32LowTPseudo_UPD,
4553 ARM::VST1q64LowTPseudo_UPD };
4554 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo_UPD,
4555 ARM::VST1q16HighTPseudo_UPD,
4556 ARM::VST1q32HighTPseudo_UPD,
4557 ARM::VST1q64HighTPseudo_UPD };
4558 SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4559 return;
4560 }
4561 break;
4562 }
4563
4565 if (Subtarget->hasNEON()) {
4566 static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudoWB_fixed,
4567 ARM::VST1d16QPseudoWB_fixed,
4568 ARM::VST1d32QPseudoWB_fixed,
4569 ARM::VST1d64QPseudoWB_fixed };
4570 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD,
4571 ARM::VST1q16LowQPseudo_UPD,
4572 ARM::VST1q32LowQPseudo_UPD,
4573 ARM::VST1q64LowQPseudo_UPD };
4574 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo_UPD,
4575 ARM::VST1q16HighQPseudo_UPD,
4576 ARM::VST1q32HighQPseudo_UPD,
4577 ARM::VST1q64HighQPseudo_UPD };
4578 SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4579 return;
4580 }
4581 break;
4582 }
4584 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
4585 ARM::VST2LNd16Pseudo_UPD,
4586 ARM::VST2LNd32Pseudo_UPD };
4587 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
4588 ARM::VST2LNq32Pseudo_UPD };
4589 SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes);
4590 return;
4591 }
4592
4594 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
4595 ARM::VST3LNd16Pseudo_UPD,
4596 ARM::VST3LNd32Pseudo_UPD };
4597 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
4598 ARM::VST3LNq32Pseudo_UPD };
4599 SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes);
4600 return;
4601 }
4602
4604 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
4605 ARM::VST4LNd16Pseudo_UPD,
4606 ARM::VST4LNd32Pseudo_UPD };
4607 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
4608 ARM::VST4LNq32Pseudo_UPD };
4609 SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes);
4610 return;
4611 }
4612
4615 unsigned IntNo = N->getConstantOperandVal(1);
4616 switch (IntNo) {
4617 default:
4618 break;
4619
4620 case Intrinsic::arm_mrrc:
4621 case Intrinsic::arm_mrrc2: {
4622 SDLoc dl(N);
4624 unsigned Opc;
4625
4626 if (Subtarget->isThumb())
4627 Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::t2MRRC : ARM::t2MRRC2);
4628 else
4629 Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::MRRC : ARM::MRRC2);
4630
4632 Ops.push_back(getI32Imm(N->getConstantOperandVal(2), dl));
4633 Ops.push_back(getI32Imm(N->getConstantOperandVal(3), dl));
4634 Ops.push_back(getI32Imm(N->getConstantOperandVal(4), dl));
4635
4636
4637
4638
4639 if (Opc != ARM::MRRC2) {
4640 Ops.push_back(getAL(CurDAG, dl));
4641 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
4642 }
4643
4644 Ops.push_back(Chain);
4645
4646
4647 const EVT RetType[] = {MVT::i32, MVT::i32, MVT::Other};
4648
4649 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, RetType, Ops));
4650 return;
4651 }
4652 case Intrinsic::arm_ldaexd:
4653 case Intrinsic::arm_ldrexd: {
4654 SDLoc dl(N);
4657 bool isThumb = Subtarget->isThumb() && Subtarget->hasV8MBaselineOps();
4658
4659 bool IsAcquire = IntNo == Intrinsic::arm_ldaexd;
4660 unsigned NewOpc = isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD)
4661 : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD);
4662
4663
4664 std::vector ResTys;
4666 ResTys.push_back(MVT::i32);
4667 ResTys.push_back(MVT::i32);
4668 } else
4669 ResTys.push_back(MVT::Untyped);
4670 ResTys.push_back(MVT::Other);
4671
4672
4674 CurDAG->getRegister(0, MVT::i32), Chain};
4675 SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
4676
4679
4680
4682 if ((N, 0).use_empty()) {
4686 else {
4688 CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
4689 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
4690 dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
4692 }
4693 ReplaceUses(SDValue(N, 0), Result);
4694 }
4695 if ((N, 1).use_empty()) {
4699 else {
4701 CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
4702 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
4703 dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
4705 }
4706 ReplaceUses(SDValue(N, 1), Result);
4707 }
4708 ReplaceUses(SDValue(N, 2), OutChain);
4709 CurDAG->RemoveDeadNode(N);
4710 return;
4711 }
4712 case Intrinsic::arm_stlexd:
4713 case Intrinsic::arm_strexd: {
4714 SDLoc dl(N);
4719
4720
4721
4722 const EVT ResTys[] = {MVT::i32, MVT::Other};
4723
4724 bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2();
4725
4728 Ops.push_back(Val0);
4729 Ops.push_back(Val1);
4730 } else
4731
4733 Ops.push_back(MemAddr);
4734 Ops.push_back(getAL(CurDAG, dl));
4735 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
4736 Ops.push_back(Chain);
4737
4738 bool IsRelease = IntNo == Intrinsic::arm_stlexd;
4739 unsigned NewOpc = isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD)
4740 : (IsRelease ? ARM::STLEXD : ARM::STREXD);
4741
4742 SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
4743
4746
4747 ReplaceNode(N, St);
4748 return;
4749 }
4750
4751 case Intrinsic::arm_neon_vld1: {
4752 static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
4753 ARM::VLD1d32, ARM::VLD1d64 };
4754 static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
4755 ARM::VLD1q32, ARM::VLD1q64};
4756 SelectVLD(N, false, 1, DOpcodes, QOpcodes, nullptr);
4757 return;
4758 }
4759
4760 case Intrinsic::arm_neon_vld1x2: {
4761 static const uint16_t DOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
4762 ARM::VLD1q32, ARM::VLD1q64 };
4763 static const uint16_t QOpcodes[] = { ARM::VLD1d8QPseudo,
4764 ARM::VLD1d16QPseudo,
4765 ARM::VLD1d32QPseudo,
4766 ARM::VLD1d64QPseudo };
4767 SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr);
4768 return;
4769 }
4770
4771 case Intrinsic::arm_neon_vld1x3: {
4772 static const uint16_t DOpcodes[] = { ARM::VLD1d8TPseudo,
4773 ARM::VLD1d16TPseudo,
4774 ARM::VLD1d32TPseudo,
4775 ARM::VLD1d64TPseudo };
4776 static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowTPseudo_UPD,
4777 ARM::VLD1q16LowTPseudo_UPD,
4778 ARM::VLD1q32LowTPseudo_UPD,
4779 ARM::VLD1q64LowTPseudo_UPD };
4780 static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighTPseudo,
4781 ARM::VLD1q16HighTPseudo,
4782 ARM::VLD1q32HighTPseudo,
4783 ARM::VLD1q64HighTPseudo };
4784 SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
4785 return;
4786 }
4787
4788 case Intrinsic::arm_neon_vld1x4: {
4789 static const uint16_t DOpcodes[] = { ARM::VLD1d8QPseudo,
4790 ARM::VLD1d16QPseudo,
4791 ARM::VLD1d32QPseudo,
4792 ARM::VLD1d64QPseudo };
4793 static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowQPseudo_UPD,
4794 ARM::VLD1q16LowQPseudo_UPD,
4795 ARM::VLD1q32LowQPseudo_UPD,
4796 ARM::VLD1q64LowQPseudo_UPD };
4797 static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighQPseudo,
4798 ARM::VLD1q16HighQPseudo,
4799 ARM::VLD1q32HighQPseudo,
4800 ARM::VLD1q64HighQPseudo };
4801 SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
4802 return;
4803 }
4804
4805 case Intrinsic::arm_neon_vld2: {
4806 static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
4807 ARM::VLD2d32, ARM::VLD1q64 };
4808 static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
4809 ARM::VLD2q32Pseudo };
4810 SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr);
4811 return;
4812 }
4813
4814 case Intrinsic::arm_neon_vld3: {
4815 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo,
4816 ARM::VLD3d16Pseudo,
4817 ARM::VLD3d32Pseudo,
4818 ARM::VLD1d64TPseudo };
4819 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
4820 ARM::VLD3q16Pseudo_UPD,
4821 ARM::VLD3q32Pseudo_UPD };
4822 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo,
4823 ARM::VLD3q16oddPseudo,
4824 ARM::VLD3q32oddPseudo };
4825 SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
4826 return;
4827 }
4828
4829 case Intrinsic::arm_neon_vld4: {
4830 static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo,
4831 ARM::VLD4d16Pseudo,
4832 ARM::VLD4d32Pseudo,
4833 ARM::VLD1d64QPseudo };
4834 static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
4835 ARM::VLD4q16Pseudo_UPD,
4836 ARM::VLD4q32Pseudo_UPD };
4837 static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo,
4838 ARM::VLD4q16oddPseudo,
4839 ARM::VLD4q32oddPseudo };
4840 SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
4841 return;
4842 }
4843
4844 case Intrinsic::arm_neon_vld2dup: {
4845 static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
4846 ARM::VLD2DUPd32, ARM::VLD1q64 };
4847 static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
4848 ARM::VLD2DUPq16EvenPseudo,
4849 ARM::VLD2DUPq32EvenPseudo };
4850 static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudo,
4851 ARM::VLD2DUPq16OddPseudo,
4852 ARM::VLD2DUPq32OddPseudo };
4853 SelectVLDDup(N, true, false, 2,
4854 DOpcodes, QOpcodes0, QOpcodes1);
4855 return;
4856 }
4857
4858 case Intrinsic::arm_neon_vld3dup: {
4859 static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo,
4860 ARM::VLD3DUPd16Pseudo,
4861 ARM::VLD3DUPd32Pseudo,
4862 ARM::VLD1d64TPseudo };
4863 static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
4864 ARM::VLD3DUPq16EvenPseudo,
4865 ARM::VLD3DUPq32EvenPseudo };
4866 static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo,
4867 ARM::VLD3DUPq16OddPseudo,
4868 ARM::VLD3DUPq32OddPseudo };
4869 SelectVLDDup(N, true, false, 3,
4870 DOpcodes, QOpcodes0, QOpcodes1);
4871 return;
4872 }
4873
4874 case Intrinsic::arm_neon_vld4dup: {
4875 static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo,
4876 ARM::VLD4DUPd16Pseudo,
4877 ARM::VLD4DUPd32Pseudo,
4878 ARM::VLD1d64QPseudo };
4879 static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
4880 ARM::VLD4DUPq16EvenPseudo,
4881 ARM::VLD4DUPq32EvenPseudo };
4882 static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo,
4883 ARM::VLD4DUPq16OddPseudo,
4884 ARM::VLD4DUPq32OddPseudo };
4885 SelectVLDDup(N, true, false, 4,
4886 DOpcodes, QOpcodes0, QOpcodes1);
4887 return;
4888 }
4889
4890 case Intrinsic::arm_neon_vld2lane: {
4891 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo,
4892 ARM::VLD2LNd16Pseudo,
4893 ARM::VLD2LNd32Pseudo };
4894 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo,
4895 ARM::VLD2LNq32Pseudo };
4896 SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes);
4897 return;
4898 }
4899
4900 case Intrinsic::arm_neon_vld3lane: {
4901 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo,
4902 ARM::VLD3LNd16Pseudo,
4903 ARM::VLD3LNd32Pseudo };
4904 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo,
4905 ARM::VLD3LNq32Pseudo };
4906 SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes);
4907 return;
4908 }
4909
4910 case Intrinsic::arm_neon_vld4lane: {
4911 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo,
4912 ARM::VLD4LNd16Pseudo,
4913 ARM::VLD4LNd32Pseudo };
4914 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo,
4915 ARM::VLD4LNq32Pseudo };
4916 SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes);
4917 return;
4918 }
4919
4920 case Intrinsic::arm_neon_vst1: {
4921 static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
4922 ARM::VST1d32, ARM::VST1d64 };
4923 static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
4924 ARM::VST1q32, ARM::VST1q64 };
4925 SelectVST(N, false, 1, DOpcodes, QOpcodes, nullptr);
4926 return;
4927 }
4928
4929 case Intrinsic::arm_neon_vst1x2: {
4930 static const uint16_t DOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
4931 ARM::VST1q32, ARM::VST1q64 };
4932 static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudo,
4933 ARM::VST1d16QPseudo,
4934 ARM::VST1d32QPseudo,
4935 ARM::VST1d64QPseudo };
4936 SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr);
4937 return;
4938 }
4939
4940 case Intrinsic::arm_neon_vst1x3: {
4941 static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudo,
4942 ARM::VST1d16TPseudo,
4943 ARM::VST1d32TPseudo,
4944 ARM::VST1d64TPseudo };
4945 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD,
4946 ARM::VST1q16LowTPseudo_UPD,
4947 ARM::VST1q32LowTPseudo_UPD,
4948 ARM::VST1q64LowTPseudo_UPD };
4949 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo,
4950 ARM::VST1q16HighTPseudo,
4951 ARM::VST1q32HighTPseudo,
4952 ARM::VST1q64HighTPseudo };
4953 SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
4954 return;
4955 }
4956
4957 case Intrinsic::arm_neon_vst1x4: {
4958 static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudo,
4959 ARM::VST1d16QPseudo,
4960 ARM::VST1d32QPseudo,
4961 ARM::VST1d64QPseudo };
4962 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD,
4963 ARM::VST1q16LowQPseudo_UPD,
4964 ARM::VST1q32LowQPseudo_UPD,
4965 ARM::VST1q64LowQPseudo_UPD };
4966 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo,
4967 ARM::VST1q16HighQPseudo,
4968 ARM::VST1q32HighQPseudo,
4969 ARM::VST1q64HighQPseudo };
4970 SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
4971 return;
4972 }
4973
4974 case Intrinsic::arm_neon_vst2: {
4975 static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
4976 ARM::VST2d32, ARM::VST1q64 };
4977 static const uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
4978 ARM::VST2q32Pseudo };
4979 SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr);
4980 return;
4981 }
4982
4983 case Intrinsic::arm_neon_vst3: {
4984 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo,
4985 ARM::VST3d16Pseudo,
4986 ARM::VST3d32Pseudo,
4987 ARM::VST1d64TPseudo };
4988 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
4989 ARM::VST3q16Pseudo_UPD,
4990 ARM::VST3q32Pseudo_UPD };
4991 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo,
4992 ARM::VST3q16oddPseudo,
4993 ARM::VST3q32oddPseudo };
4994 SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
4995 return;
4996 }
4997
4998 case Intrinsic::arm_neon_vst4: {
4999 static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo,
5000 ARM::VST4d16Pseudo,
5001 ARM::VST4d32Pseudo,
5002 ARM::VST1d64QPseudo };
5003 static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
5004 ARM::VST4q16Pseudo_UPD,
5005 ARM::VST4q32Pseudo_UPD };
5006 static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo,
5007 ARM::VST4q16oddPseudo,
5008 ARM::VST4q32oddPseudo };
5009 SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
5010 return;
5011 }
5012
5013 case Intrinsic::arm_neon_vst2lane: {
5014 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo,
5015 ARM::VST2LNd16Pseudo,
5016 ARM::VST2LNd32Pseudo };
5017 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo,
5018 ARM::VST2LNq32Pseudo };
5019 SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes);
5020 return;
5021 }
5022
5023 case Intrinsic::arm_neon_vst3lane: {
5024 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo,
5025 ARM::VST3LNd16Pseudo,
5026 ARM::VST3LNd32Pseudo };
5027 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo,
5028 ARM::VST3LNq32Pseudo };
5029 SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes);
5030 return;
5031 }
5032
5033 case Intrinsic::arm_neon_vst4lane: {
5034 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo,
5035 ARM::VST4LNd16Pseudo,
5036 ARM::VST4LNd32Pseudo };
5037 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo,
5038 ARM::VST4LNq32Pseudo };
5039 SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes);
5040 return;
5041 }
5042
5043 case Intrinsic::arm_mve_vldr_gather_base_wb:
5044 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
5045 static const uint16_t Opcodes[] = {ARM::MVE_VLDRWU32_qi_pre,
5046 ARM::MVE_VLDRDU64_qi_pre};
5047 SelectMVE_WB(N, Opcodes,
5048 IntNo == Intrinsic::arm_mve_vldr_gather_base_wb_predicated);
5049 return;
5050 }
5051
5052 case Intrinsic::arm_mve_vld2q: {
5053 static const uint16_t Opcodes8[] = {ARM::MVE_VLD20_8, ARM::MVE_VLD21_8};
5054 static const uint16_t Opcodes16[] = {ARM::MVE_VLD20_16,
5055 ARM::MVE_VLD21_16};
5056 static const uint16_t Opcodes32[] = {ARM::MVE_VLD20_32,
5057 ARM::MVE_VLD21_32};
5058 static const uint16_t *const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
5059 SelectMVE_VLD(N, 2, Opcodes, false);
5060 return;
5061 }
5062
5063 case Intrinsic::arm_mve_vld4q: {
5064 static const uint16_t Opcodes8[] = {ARM::MVE_VLD40_8, ARM::MVE_VLD41_8,
5065 ARM::MVE_VLD42_8, ARM::MVE_VLD43_8};
5066 static const uint16_t Opcodes16[] = {ARM::MVE_VLD40_16, ARM::MVE_VLD41_16,
5067 ARM::MVE_VLD42_16,
5068 ARM::MVE_VLD43_16};
5069 static const uint16_t Opcodes32[] = {ARM::MVE_VLD40_32, ARM::MVE_VLD41_32,
5070 ARM::MVE_VLD42_32,
5071 ARM::MVE_VLD43_32};
5072 static const uint16_t *const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
5073 SelectMVE_VLD(N, 4, Opcodes, false);
5074 return;
5075 }
5076 }
5077 break;
5078 }
5079
5081 unsigned IntNo = N->getConstantOperandVal(0);
5082 switch (IntNo) {
5083 default:
5084 break;
5085
5086
5087 case Intrinsic::arm_neon_vcvtbfp2bf: {
5088 SDLoc dl(N);
5089 const SDValue &Src = N->getOperand(1);
5090 llvm::EVT DestTy = N->getValueType(0);
5092 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
5093 SDValue Ops[] = { Src, Src, Pred, Reg0 };
5094 CurDAG->SelectNodeTo(N, ARM::BF16_VCVTB, DestTy, Ops);
5095 return;
5096 }
5097
5098
5099 case Intrinsic::arm_neon_vcvtfp2bf: {
5100 SDLoc dl(N);
5101 const SDValue &Src = N->getOperand(1);
5103 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
5104 SDValue Ops[] = { Src, Pred, Reg0 };
5105 CurDAG->SelectNodeTo(N, ARM::BF16_VCVT, MVT::v4bf16, Ops);
5106 return;
5107 }
5108
5109 case Intrinsic::arm_mve_urshrl:
5110 SelectMVE_LongShift(N, ARM::MVE_URSHRL, true, false);
5111 return;
5112 case Intrinsic::arm_mve_uqshll:
5113 SelectMVE_LongShift(N, ARM::MVE_UQSHLL, true, false);
5114 return;
5115 case Intrinsic::arm_mve_srshrl:
5116 SelectMVE_LongShift(N, ARM::MVE_SRSHRL, true, false);
5117 return;
5118 case Intrinsic::arm_mve_sqshll:
5119 SelectMVE_LongShift(N, ARM::MVE_SQSHLL, true, false);
5120 return;
5121 case Intrinsic::arm_mve_uqrshll:
5122 SelectMVE_LongShift(N, ARM::MVE_UQRSHLL, false, true);
5123 return;
5124 case Intrinsic::arm_mve_sqrshrl:
5125 SelectMVE_LongShift(N, ARM::MVE_SQRSHRL, false, true);
5126 return;
5127
5128 case Intrinsic::arm_mve_vadc:
5129 case Intrinsic::arm_mve_vadc_predicated:
5130 SelectMVE_VADCSBC(N, ARM::MVE_VADC, ARM::MVE_VADCI, true,
5131 IntNo == Intrinsic::arm_mve_vadc_predicated);
5132 return;
5133 case Intrinsic::arm_mve_vsbc:
5134 case Intrinsic::arm_mve_vsbc_predicated:
5135 SelectMVE_VADCSBC(N, ARM::MVE_VSBC, ARM::MVE_VSBCI, false,
5136 IntNo == Intrinsic::arm_mve_vsbc_predicated);
5137 return;
5138 case Intrinsic::arm_mve_vshlc:
5139 case Intrinsic::arm_mve_vshlc_predicated:
5140 SelectMVE_VSHLC(N, IntNo == Intrinsic::arm_mve_vshlc_predicated);
5141 return;
5142
5143 case Intrinsic::arm_mve_vmlldava:
5144 case Intrinsic::arm_mve_vmlldava_predicated: {
5145 static const uint16_t OpcodesU[] = {
5146 ARM::MVE_VMLALDAVu16, ARM::MVE_VMLALDAVu32,
5147 ARM::MVE_VMLALDAVau16, ARM::MVE_VMLALDAVau32,
5148 };
5149 static const uint16_t OpcodesS[] = {
5150 ARM::MVE_VMLALDAVs16, ARM::MVE_VMLALDAVs32,
5151 ARM::MVE_VMLALDAVas16, ARM::MVE_VMLALDAVas32,
5152 ARM::MVE_VMLALDAVxs16, ARM::MVE_VMLALDAVxs32,
5153 ARM::MVE_VMLALDAVaxs16, ARM::MVE_VMLALDAVaxs32,
5154 ARM::MVE_VMLSLDAVs16, ARM::MVE_VMLSLDAVs32,
5155 ARM::MVE_VMLSLDAVas16, ARM::MVE_VMLSLDAVas32,
5156 ARM::MVE_VMLSLDAVxs16, ARM::MVE_VMLSLDAVxs32,
5157 ARM::MVE_VMLSLDAVaxs16, ARM::MVE_VMLSLDAVaxs32,
5158 };
5159 SelectMVE_VMLLDAV(N, IntNo == Intrinsic::arm_mve_vmlldava_predicated,
5160 OpcodesS, OpcodesU);
5161 return;
5162 }
5163
5164 case Intrinsic::arm_mve_vrmlldavha:
5165 case Intrinsic::arm_mve_vrmlldavha_predicated: {
5166 static const uint16_t OpcodesU[] = {
5167 ARM::MVE_VRMLALDAVHu32, ARM::MVE_VRMLALDAVHau32,
5168 };
5169 static const uint16_t OpcodesS[] = {
5170 ARM::MVE_VRMLALDAVHs32, ARM::MVE_VRMLALDAVHas32,
5171 ARM::MVE_VRMLALDAVHxs32, ARM::MVE_VRMLALDAVHaxs32,
5172 ARM::MVE_VRMLSLDAVHs32, ARM::MVE_VRMLSLDAVHas32,
5173 ARM::MVE_VRMLSLDAVHxs32, ARM::MVE_VRMLSLDAVHaxs32,
5174 };
5175 SelectMVE_VRMLLDAVH(N, IntNo == Intrinsic::arm_mve_vrmlldavha_predicated,
5176 OpcodesS, OpcodesU);
5177 return;
5178 }
5179
5180 case Intrinsic::arm_mve_vidup:
5181 case Intrinsic::arm_mve_vidup_predicated: {
5182 static const uint16_t Opcodes[] = {
5183 ARM::MVE_VIDUPu8, ARM::MVE_VIDUPu16, ARM::MVE_VIDUPu32,
5184 };
5185 SelectMVE_VxDUP(N, Opcodes, false,
5186 IntNo == Intrinsic::arm_mve_vidup_predicated);
5187 return;
5188 }
5189
5190 case Intrinsic::arm_mve_vddup:
5191 case Intrinsic::arm_mve_vddup_predicated: {
5192 static const uint16_t Opcodes[] = {
5193 ARM::MVE_VDDUPu8, ARM::MVE_VDDUPu16, ARM::MVE_VDDUPu32,
5194 };
5195 SelectMVE_VxDUP(N, Opcodes, false,
5196 IntNo == Intrinsic::arm_mve_vddup_predicated);
5197 return;
5198 }
5199
5200 case Intrinsic::arm_mve_viwdup:
5201 case Intrinsic::arm_mve_viwdup_predicated: {
5202 static const uint16_t Opcodes[] = {
5203 ARM::MVE_VIWDUPu8, ARM::MVE_VIWDUPu16, ARM::MVE_VIWDUPu32,
5204 };
5205 SelectMVE_VxDUP(N, Opcodes, true,
5206 IntNo == Intrinsic::arm_mve_viwdup_predicated);
5207 return;
5208 }
5209
5210 case Intrinsic::arm_mve_vdwdup:
5211 case Intrinsic::arm_mve_vdwdup_predicated: {
5212 static const uint16_t Opcodes[] = {
5213 ARM::MVE_VDWDUPu8, ARM::MVE_VDWDUPu16, ARM::MVE_VDWDUPu32,
5214 };
5215 SelectMVE_VxDUP(N, Opcodes, true,
5216 IntNo == Intrinsic::arm_mve_vdwdup_predicated);
5217 return;
5218 }
5219
5220 case Intrinsic::arm_cde_cx1d:
5221 case Intrinsic::arm_cde_cx1da:
5222 case Intrinsic::arm_cde_cx2d:
5223 case Intrinsic::arm_cde_cx2da:
5224 case Intrinsic::arm_cde_cx3d:
5225 case Intrinsic::arm_cde_cx3da: {
5226 bool HasAccum = IntNo == Intrinsic::arm_cde_cx1da ||
5227 IntNo == Intrinsic::arm_cde_cx2da ||
5228 IntNo == Intrinsic::arm_cde_cx3da;
5229 size_t NumExtraOps;
5230 uint16_t Opcode;
5231 switch (IntNo) {
5232 case Intrinsic::arm_cde_cx1d:
5233 case Intrinsic::arm_cde_cx1da:
5234 NumExtraOps = 0;
5235 Opcode = HasAccum ? ARM::CDE_CX1DA : ARM::CDE_CX1D;
5236 break;
5237 case Intrinsic::arm_cde_cx2d:
5238 case Intrinsic::arm_cde_cx2da:
5239 NumExtraOps = 1;
5240 Opcode = HasAccum ? ARM::CDE_CX2DA : ARM::CDE_CX2D;
5241 break;
5242 case Intrinsic::arm_cde_cx3d:
5243 case Intrinsic::arm_cde_cx3da:
5244 NumExtraOps = 2;
5245 Opcode = HasAccum ? ARM::CDE_CX3DA : ARM::CDE_CX3D;
5246 break;
5247 default:
5249 }
5250 SelectCDE_CXxD(N, Opcode, NumExtraOps, HasAccum);
5251 return;
5252 }
5253 }
5254 break;
5255 }
5256
5257 case ISD::ATOMIC_CMP_SWAP:
5258 SelectCMP_SWAP(N);
5259 return;
5260 }
5261
5262 SelectCode(N);
5263}
5264
5265
5266
5267
5268
5269
5273 std::vector &Ops) {
5275 RegString.split(Fields, ':');
5276
5277 if (Fields.size() > 1) {
5278 bool AllIntFields = true;
5279
5281
5282 unsigned IntField;
5283 AllIntFields &= .trim("CPcp").getAsInteger(10, IntField);
5285 }
5286
5287 assert(AllIntFields &&
5288 "Unexpected non-integer value in special register string.");
5289 (void)AllIntFields;
5290 }
5291}
5292
5293
5294
5295
5296
5297
5299 auto TheReg = ARMBankedReg::lookupBankedRegByName(RegString.lower());
5300 if (!TheReg)
5301 return -1;
5302 return TheReg->Encoding;
5303}
5304
5305
5306
5307
5310 .Case("", 0x2)
5311
5312 .Case("g", 0x1)
5313 .Case("nzcvq", 0x2)
5314 .Case("nzcvqg", 0x3)
5316}
5317
5318
5319
5320
5322 auto TheReg = ARMSysReg::lookupMClassSysRegByName(Reg);
5323 const FeatureBitset &FeatureBits = Subtarget->getFeatureBits();
5324 if (!TheReg || !TheReg->hasRequiredFeatures(FeatureBits))
5325 return -1;
5326 return (int)(TheReg->Encoding & 0xFFF);
5327}
5328
5330
5331
5332
5333
5334 int Mask = 0;
5335 if (Reg == "apsr") {
5336
5337
5338
5340 if (Mask == -1)
5341 return -1;
5342 return Mask << 2;
5343 }
5344
5345 if (Reg != "cpsr" && Reg != "spsr") {
5346 return -1;
5347 }
5348
5349
5350 if (Flags.empty() || Flags == "all")
5351 return Mask | 0x9;
5352
5353
5354
5355 for (char Flag : Flags) {
5356 int FlagVal;
5357 switch (Flag) {
5358 case 'c':
5359 FlagVal = 0x1;
5360 break;
5361 case 'x':
5362 FlagVal = 0x2;
5363 break;
5364 case 's':
5365 FlagVal = 0x4;
5366 break;
5367 case 'f':
5368 FlagVal = 0x8;
5369 break;
5370 default:
5371 FlagVal = 0;
5372 }
5373
5374
5375 if (!FlagVal || (Mask & FlagVal))
5376 return -1;
5377 Mask |= FlagVal;
5378 }
5379
5380
5381 if (Reg == "spsr")
5382 Mask |= 0x10;
5383
5384 return Mask;
5385}
5386
5387
5388
5389
5390bool ARMDAGToDAGISel::tryReadRegister(SDNode *N){
5392 const auto *RegString = cast(MD->getMD()->getOperand(0));
5393 bool IsThumb2 = Subtarget->isThumb2();
5395
5396 std::vector Ops;
5398
5399 if (.empty()) {
5400
5401
5402
5403
5404 unsigned Opcode;
5406 if (Ops.size() == 5){
5407 Opcode = IsThumb2 ? ARM::t2MRC : ARM::MRC;
5408 ResTypes.append({ MVT::i32, MVT::Other });
5409 } else {
5411 "Invalid number of fields in special register string.");
5412 Opcode = IsThumb2 ? ARM::t2MRRC : ARM::MRRC;
5413 ResTypes.append({ MVT::i32, MVT::i32, MVT::Other });
5414 }
5415
5417 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
5418 Ops.push_back(N->getOperand(0));
5419 ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, ResTypes, Ops));
5420 return true;
5421 }
5422
5423 std::string SpecialReg = RegString->getString().lower();
5424
5426 if (BankedReg != -1) {
5427 Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32),
5428 getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
5430 ReplaceNode(
5431 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSbanked : ARM::MRSbanked,
5432 DL, MVT::i32, MVT::Other, Ops));
5433 return true;
5434 }
5435
5436
5437
5438
5439 unsigned Opcode = StringSwitch(SpecialReg)
5440 .Case("fpscr", ARM::VMRS)
5441 .Case("fpexc", ARM::VMRS_FPEXC)
5442 .Case("fpsid", ARM::VMRS_FPSID)
5443 .Case("mvfr0", ARM::VMRS_MVFR0)
5444 .Case("mvfr1", ARM::VMRS_MVFR1)
5445 .Case("mvfr2", ARM::VMRS_MVFR2)
5446 .Case("fpinst", ARM::VMRS_FPINST)
5447 .Case("fpinst2", ARM::VMRS_FPINST2)
5448 .Default(0);
5449
5450
5451 if (Opcode) {
5453 return false;
5454 if (Opcode == ARM::VMRS_MVFR2 && !Subtarget->hasFPARMv8Base())
5455 return false;
5456
5457 Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
5459 ReplaceNode(N,
5460 CurDAG->getMachineNode(Opcode, DL, MVT::i32, MVT::Other, Ops));
5461 return true;
5462 }
5463
5464
5465
5466
5467 if (Subtarget->isMClass()) {
5469 if (SYSmValue == -1)
5470 return false;
5471
5472 SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
5473 getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
5475 ReplaceNode(
5476 N, CurDAG->getMachineNode(ARM::t2MRS_M, DL, MVT::i32, MVT::Other, Ops));
5477 return true;
5478 }
5479
5480
5481
5482 if (SpecialReg == "apsr" || SpecialReg == "cpsr") {
5483 Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
5485 ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRS_AR : ARM::MRS,
5486 DL, MVT::i32, MVT::Other, Ops));
5487 return true;
5488 }
5489
5490 if (SpecialReg == "spsr") {
5491 Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
5493 ReplaceNode(
5494 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSsys_AR : ARM::MRSsys, DL,
5495 MVT::i32, MVT::Other, Ops));
5496 return true;
5497 }
5498
5499 return false;
5500}
5501
5502
5503
5504
5505bool ARMDAGToDAGISel::tryWriteRegister(SDNode *N){
5507 const auto *RegString = cast(MD->getMD()->getOperand(0));
5508 bool IsThumb2 = Subtarget->isThumb2();
5510
5511 std::vector Ops;
5513
5514 if (.empty()) {
5515
5516
5517
5518
5519 unsigned Opcode;
5520 if (Ops.size() == 5) {
5521 Opcode = IsThumb2 ? ARM::t2MCR : ARM::MCR;
5522 Ops.insert(Ops.begin()+2, N->getOperand(2));
5523 } else {
5525 "Invalid number of fields in special register string.");
5526 Opcode = IsThumb2 ? ARM::t2MCRR : ARM::MCRR;
5527 SDValue WriteValue[] = { N->getOperand(2), N->getOperand(3) };
5528 Ops.insert(Ops.begin()+2, WriteValue, WriteValue+2);
5529 }
5530
5532 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
5533 Ops.push_back(N->getOperand(0));
5534
5535 ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
5536 return true;
5537 }
5538
5539 std::string SpecialReg = RegString->getString().lower();
5541 if (BankedReg != -1) {
5542 Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32), N->getOperand(2),
5543 getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
5545 ReplaceNode(
5546 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSRbanked : ARM::MSRbanked,
5548 return true;
5549 }
5550
5551
5552
5553
5554 unsigned Opcode = StringSwitch(SpecialReg)
5555 .Case("fpscr", ARM::VMSR)
5556 .Case("fpexc", ARM::VMSR_FPEXC)
5557 .Case("fpsid", ARM::VMSR_FPSID)
5558 .Case("fpinst", ARM::VMSR_FPINST)
5559 .Case("fpinst2", ARM::VMSR_FPINST2)
5560 .Default(0);
5561
5562 if (Opcode) {
5564 return false;
5565 Ops = { N->getOperand(2), getAL(CurDAG, DL),
5566 CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
5567 ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
5568 return true;
5569 }
5570
5571 std::pair<StringRef, StringRef> Fields;
5572 Fields = StringRef(SpecialReg).rsplit('_');
5573 std::string Reg = Fields.first.str();
5574 StringRef Flags = Fields.second;
5575
5576
5577
5578 if (Subtarget->isMClass()) {
5580 if (SYSmValue == -1)
5581 return false;
5582
5583 SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
5584 N->getOperand(2), getAL(CurDAG, DL),
5585 CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
5586 ReplaceNode(N, CurDAG->getMachineNode(ARM::t2MSR_M, DL, MVT::Other, Ops));
5587 return true;
5588 }
5589
5590
5591
5592
5594 if (Mask != -1) {
5595 Ops = { CurDAG->getTargetConstant(Mask, DL, MVT::i32), N->getOperand(2),
5596 getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
5598 ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSR_AR : ARM::MSR,
5600 return true;
5601 }
5602
5603 return false;
5604}
5605
5606bool ARMDAGToDAGISel::tryInlineAsm(SDNode *N){
5607 std::vector AsmNodeOperands;
5608 InlineAsm::Flag Flag;
5610 unsigned NumOps = N->getNumOperands();
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620 SDLoc dl(N);
5622
5624
5625 for(unsigned i = 0, e = N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) {
5627 AsmNodeOperands.push_back(op);
5628
5630 continue;
5631
5633 Flag = InlineAsm::Flag(C->getZExtValue());
5634 else
5635 continue;
5636
5637
5638
5639
5640
5641 if (Flag.isImmKind()) {
5643 AsmNodeOperands.push_back(op);
5644 continue;
5645 }
5646
5647 const unsigned NumRegs = Flag.getNumOperandRegisters();
5648 if (NumRegs)
5650
5651 unsigned DefIdx = 0;
5652 bool IsTiedToChangedOp = false;
5653
5654
5655 if (Changed && Flag.isUseOperandTiedToDef(DefIdx))
5656 IsTiedToChangedOp = OpChanged[DefIdx];
5657
5658
5659
5660
5661
5662
5663 if (Flag.isMemKind()) {
5665 AsmNodeOperands.push_back(op);
5666 continue;
5667 }
5668
5669 if (.isRegUseKind() &&
.isRegDefKind() &&
5670 .isRegDefEarlyClobberKind())
5671 continue;
5672
5673 unsigned RC;
5674 const bool HasRC = Flag.hasRegClassConstraint(RC);
5675 if ((!IsTiedToChangedOp && (!HasRC || RC != ARM::GPRRegClassID))
5676 || NumRegs != 2)
5677 continue;
5678
5679 assert((i+2 < NumOps) && "Invalid number of operands in inline asm");
5686
5687 if (Flag.isRegDefKind() || Flag.isRegDefEarlyClobberKind()) {
5688
5689
5690
5691 Register GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
5692 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
5694
5695 SDNode *GU = N->getGluedUser();
5696 SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped,
5698
5699
5700 SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
5701 RegCopy);
5702 SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
5703 RegCopy);
5704 SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0,
5706 SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1));
5707
5708
5710 Ops.push_back(T1.getValue(1));
5711 CurDAG->UpdateNodeOperands(GU, Ops);
5712 } else {
5713
5714
5716
5717
5718 SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32,
5720 SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32,
5723
5724
5725
5726 Register GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
5727 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
5728 Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair, T1.getValue(1));
5729
5732 }
5733
5735
5736 if(PairedReg.getNode()) {
5737 OpChanged[OpChanged.size() -1 ] = true;
5738 Flag = InlineAsm::Flag(Flag.getKind(), 1 );
5739 if (IsTiedToChangedOp)
5740 Flag.setMatchingOp(DefIdx);
5741 else
5742 Flag.setRegClass(ARM::GPRPairRegClassID);
5743
5744 AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant(
5745 Flag, dl, MVT::i32);
5746
5747 AsmNodeOperands.push_back(PairedReg);
5748
5749 i += 2;
5750 }
5751 }
5752
5754 AsmNodeOperands.push_back(Glue);
5756 return false;
5757
5758 SDValue New = CurDAG->getNode(N->getOpcode(), SDLoc(N),
5759 CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
5760 New->setNodeId(-1);
5761 ReplaceNode(N, New.getNode());
5762 return true;
5763}
5764
5765bool ARMDAGToDAGISel::SelectInlineAsmMemoryOperand(
5767 std::vector &OutOps) {
5768 switch(ConstraintID) {
5769 default:
5771 case InlineAsm::ConstraintCode::m:
5772 case InlineAsm::ConstraintCode::o:
5773 case InlineAsm::ConstraintCode::Q:
5774 case InlineAsm::ConstraintCode::Um:
5775 case InlineAsm::ConstraintCode::Un:
5776 case InlineAsm::ConstraintCode::Uq:
5777 case InlineAsm::ConstraintCode::Us:
5778 case InlineAsm::ConstraintCode::Ut:
5779 case InlineAsm::ConstraintCode::Uv:
5780 case InlineAsm::ConstraintCode::Uy:
5781
5782
5783
5784 OutOps.push_back(Op);
5785 return false;
5786 }
5787 return true;
5788}
5789
5790
5791
5792
5795 return new ARMDAGToDAGISelLegacy(TM, OptLevel);
5796}
unsigned const MachineRegisterInfo * MRI
static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc, uint64_t &Imm)
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const TargetInstrInfo & TII
AMDGPU Register Bank Select
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
static bool isThumb(const MCSubtargetInfo &STI)
static unsigned getVectorShuffleOpcode(EVT VT, unsigned Opc64[3], unsigned Opc128[3])
Definition ARMISelDAGToDAG.cpp:3569
static int getBankedRegisterMask(StringRef RegString)
Definition ARMISelDAGToDAG.cpp:5298
static bool isPerfectIncrement(SDValue Inc, EVT VecTy, unsigned NumVecs)
Returns true if the given increment is a Constant known to be equal to the access size performed by a...
Definition ARMISelDAGToDAG.cpp:2098
static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc)
Definition ARMISelDAGToDAG.cpp:2024
static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned &Imm)
Definition ARMISelDAGToDAG.cpp:384
static bool isVSTfixed(unsigned Opc)
Definition ARMISelDAGToDAG.cpp:1993
static bool isVLDfixed(unsigned Opc)
Definition ARMISelDAGToDAG.cpp:1951
static bool isInt32Immediate(SDNode *N, unsigned &Imm)
isInt32Immediate - This method tests to see if the node is a 32-bit constant operand.
Definition ARMISelDAGToDAG.cpp:367
static std::optional< std::pair< unsigned, unsigned > > getContiguousRangeOfSetBits(const APInt &A)
Definition ARMISelDAGToDAG.cpp:3487
static void getIntOperandsFromRegisterString(StringRef RegString, SelectionDAG *CurDAG, const SDLoc &DL, std::vector< SDValue > &Ops)
Definition ARMISelDAGToDAG.cpp:5270
static int getARClassRegisterMask(StringRef Reg, StringRef Flags)
Definition ARMISelDAGToDAG.cpp:5329
static int getMClassRegisterMask(StringRef Reg, const ARMSubtarget *Subtarget)
Definition ARMISelDAGToDAG.cpp:5321
static cl::opt< bool > DisableShifterOp("disable-shifter-op", cl::Hidden, cl::desc("Disable isel of shifter-op"), cl::init(false))
static SDValue getAL(SelectionDAG *CurDAG, const SDLoc &dl)
getAL - Returns a ARMCC::AL immediate node.
Definition ARMISelDAGToDAG.cpp:1574
static bool shouldUseZeroOffsetLdSt(SDValue N)
Definition ARMISelDAGToDAG.cpp:1110
static int getMClassFlagsMask(StringRef Flags)
Definition ARMISelDAGToDAG.cpp:5308
static bool SDValueToConstBool(SDValue SDVal)
Definition ARMISelDAGToDAG.cpp:2691
static bool isScaledConstantInRange(SDValue Node, int Scale, int RangeMin, int RangeMax, int &ScaledConstant)
Check whether a particular node is a constant value representable as (N * Scale) where (N in [RangeMi...
Definition ARMISelDAGToDAG.cpp:393
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static bool isSigned(unsigned int Opcode)
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Promote Memory to Register
MachineInstr unsigned OpIdx
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
OptimizedStructLayoutField Field
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
This file implements the StringSwitch template, which mimics a switch() statement whose cases are str...
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
This file describes how to lower LLVM code to machine code.
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
bool isThumb1Only() const
bool hasFPARMv8Base() const
uint64_t getZExtValue() const
Container class for subtarget features.
FunctionPass class - This class is used to implement most global optimizations.
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc,...
bool mayStore() const
Return true if this instruction could possibly modify memory.
unsigned getOpcode() const
Return the opcode number for this descriptor.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const MachineOperand & getOperand(unsigned i) const
@ MOLoad
The memory access reads data.
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
int getNodeId() const
Return the unique node id.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
op_iterator op_end() const
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual bool runOnMachineFunction(MachineFunction &mf)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
LLVM_ABI std::string lower() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
LLVM Value Representation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
uint64_t decodeVMOVModImm(unsigned ModImm, unsigned &EltBits)
decodeVMOVModImm - Decode a NEON/MVE modified immediate value into the element value and the element ...
float getFPImmFloat(unsigned Imm)
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
unsigned getAM2Opc(AddrOpc Opc, unsigned Imm12, ShiftOpc SO, unsigned IdxMode=0)
unsigned getAM5Opc(AddrOpc Opc, unsigned char Offset)
getAM5Opc - This function encodes the addrmode5 opc field.
unsigned getAM5FP16Opc(AddrOpc Opc, unsigned char Offset)
getAM5FP16Opc - This function encodes the addrmode5fp16 opc field.
unsigned getAM3Opc(AddrOpc Opc, unsigned char Offset, unsigned IdxMode=0)
getAM3Opc - This function encodes the addrmode3 opc field.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ ADD
Simple integer binary arithmetic operators.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ FADD
Simple binary floating point operators.
@ BasicBlock
Various leaf nodes.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ SHL
Shift and rotation operations.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
initializer< Ty > init(const Ty &Val)
@ User
could "use" a pointer
NodeAddr< NodeBase * > Node
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
decltype(auto) dyn_cast(const From &Val)
dyn_cast - Return the argument parameter cast to the specified type.
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
FunctionPass * createARMISelDag(ARMBaseTargetMachine &TM, CodeGenOptLevel OptLevel)
createARMISelDag - This pass converts a legalized DAG into a ARM-specific DAG, ready for instruction ...
Definition ARMISelDAGToDAG.cpp:5793
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
CodeGenOptLevel
Code generation optimization level.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa - Return true if the parameter to the template is an instance of one of the template type argu...
FunctionAddr VTableAddr uintptr_t uintptr_t Data
unsigned ConstantMaterializationCost(unsigned Val, const ARMSubtarget *Subtarget, bool ForCodesize=false)
Returns the number of instructions required to materialize the given constant in a register,...
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
@ NearestTiesToEven
roundTiesToEven.
ArrayRef(const T &OneElt) -> ArrayRef< T >
decltype(auto) cast(const From &Val)
cast - Return the argument parameter cast to the specified type.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
bool isVector() const
Return true if this is a vector value type.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool is64BitVector() const
Return true if this is a 64-bit vector type.
static LLVM_ABI MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.