LLVM: lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
37#include
38#include
39#include
40
41using namespace llvm;
42
43#define AARCH64_EXPAND_PSEUDO_NAME "AArch64 pseudo instruction expansion pass"
44
45namespace {
46
48public:
50
51 static char ID;
52
54
56
58
59private:
67 unsigned ContiguousOpc, unsigned StridedOpc);
71 unsigned Size);
73 unsigned BitSize);
74
78 unsigned LdarOp, unsigned StlrOp, unsigned CmpOp,
79 unsigned ExtendImm, unsigned ZeroReg,
89 unsigned N);
95 struct ConditionalBlocks {
98 };
109};
110
111}
112
113char AArch64ExpandPseudo::ID = 0;
114
117
118
119
125 assert(MO.isReg() && MO.getReg());
126 if (MO.isUse())
127 UseMI.add(MO);
128 else
129 DefMI.add(MO);
130 }
131}
132
133
134
137 unsigned BitSize) {
139 Register DstReg = MI.getOperand(0).getReg();
140 uint64_t RenamableState =
142 uint64_t Imm = MI.getOperand(1).getImm();
143
144 if (DstReg == AArch64::XZR || DstReg == AArch64::WZR) {
145
146
147 MI.eraseFromParent();
148 return true;
149 }
150
154
155 SmallVector<MachineInstrBuilder, 4> MIBS;
156 for (auto I = Insn.begin(), E = Insn.end(); I != E; ++I) {
157 bool LastItem = std::next(I) == E;
158 switch (I->Opcode)
159 {
161
162 case AArch64::ORRWri:
163 case AArch64::ORRXri:
164 if (I->Op1 == 0) {
167 .addReg(BitSize == 32 ? AArch64::WZR : AArch64::XZR)
169 } else {
170 Register DstReg = MI.getOperand(0).getReg();
171 bool DstIsDead = MI.getOperand(0).isDead();
176 RenamableState)
177 .addReg(DstReg)
178 .addImm(I->Op2));
179 }
180 break;
181 case AArch64::ORRWrs:
182 case AArch64::ORRXrs: {
183 Register DstReg = MI.getOperand(0).getReg();
184 bool DstIsDead = MI.getOperand(0).isDead();
189 RenamableState)
190 .addReg(DstReg)
191 .addReg(DstReg)
192 .addImm(I->Op2));
193 } break;
194 case AArch64::ANDXri:
195 case AArch64::EORXri:
196 if (I->Op1 == 0) {
199 .addReg(BitSize == 32 ? AArch64::WZR : AArch64::XZR)
201 } else {
202 Register DstReg = MI.getOperand(0).getReg();
203 bool DstIsDead = MI.getOperand(0).isDead();
208 RenamableState)
209 .addReg(DstReg)
210 .addImm(I->Op2));
211 }
212 break;
213 case AArch64::MOVNWi:
214 case AArch64::MOVNXi:
215 case AArch64::MOVZWi:
216 case AArch64::MOVZXi: {
217 bool DstIsDead = MI.getOperand(0).isDead();
221 RenamableState)
222 .addImm(I->Op1)
224 } break;
225 case AArch64::MOVKWi:
226 case AArch64::MOVKXi: {
227 Register DstReg = MI.getOperand(0).getReg();
228 bool DstIsDead = MI.getOperand(0).isDead();
233 RenamableState)
234 .addReg(DstReg)
235 .addImm(I->Op1)
237 } break;
238 }
239 }
240 transferImpOps(MI, MIBS.front(), MIBS.back());
241 MI.eraseFromParent();
242 return true;
243}
244
245bool AArch64ExpandPseudo::expandCMP_SWAP(
247 unsigned StlrOp, unsigned CmpOp, unsigned ExtendImm, unsigned ZeroReg,
250 MIMetadata MIMD(MI);
251 const MachineOperand &Dest = MI.getOperand(0);
252 Register StatusReg = MI.getOperand(1).getReg();
253 bool StatusDead = MI.getOperand(1).isDead();
254
255
256 assert(.getOperand(2).isUndef() && "cannot handle undef");
257 Register AddrReg = MI.getOperand(2).getReg();
258 Register DesiredReg = MI.getOperand(3).getReg();
259 Register NewReg = MI.getOperand(4).getReg();
260
265
267 MF->insert(++LoadCmpBB->getIterator(), StoreBB);
268 MF->insert(++StoreBB->getIterator(), DoneBB);
269
270
271
272
273
274
275 if (!StatusDead)
276 BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::MOVZWi), StatusReg)
280 BuildMI(LoadCmpBB, MIMD, TII->get(CmpOp), ZeroReg)
288 LoadCmpBB->addSuccessor(DoneBB);
289 LoadCmpBB->addSuccessor(StoreBB);
290
291
292
293
294 BuildMI(StoreBB, MIMD, TII->get(StlrOp), StatusReg)
300 StoreBB->addSuccessor(LoadCmpBB);
301 StoreBB->addSuccessor(DoneBB);
302
303 DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
304 DoneBB->transferSuccessors(&MBB);
305
307
309 MI.eraseFromParent();
310
311
312 LivePhysRegs LiveRegs;
316
317 StoreBB->clearLiveIns();
319 LoadCmpBB->clearLiveIns();
321
322 return true;
323}
324
325bool AArch64ExpandPseudo::expandCMP_SWAP_128(
329 MIMetadata MIMD(MI);
330 MachineOperand &DestLo = MI.getOperand(0);
331 MachineOperand &DestHi = MI.getOperand(1);
332 Register StatusReg = MI.getOperand(2).getReg();
333 bool StatusDead = MI.getOperand(2).isDead();
334
335
336 assert(.getOperand(3).isUndef() && "cannot handle undef");
337 Register AddrReg = MI.getOperand(3).getReg();
338 Register DesiredLoReg = MI.getOperand(4).getReg();
339 Register DesiredHiReg = MI.getOperand(5).getReg();
340 Register NewLoReg = MI.getOperand(6).getReg();
341 Register NewHiReg = MI.getOperand(7).getReg();
342
343 unsigned LdxpOp, StxpOp;
344
345 switch (MI.getOpcode()) {
346 case AArch64::CMP_SWAP_128_MONOTONIC:
347 LdxpOp = AArch64::LDXPX;
348 StxpOp = AArch64::STXPX;
349 break;
350 case AArch64::CMP_SWAP_128_RELEASE:
351 LdxpOp = AArch64::LDXPX;
352 StxpOp = AArch64::STLXPX;
353 break;
354 case AArch64::CMP_SWAP_128_ACQUIRE:
355 LdxpOp = AArch64::LDAXPX;
356 StxpOp = AArch64::STXPX;
357 break;
358 case AArch64::CMP_SWAP_128:
359 LdxpOp = AArch64::LDAXPX;
360 StxpOp = AArch64::STLXPX;
361 break;
362 default:
364 }
365
371
373 MF->insert(++LoadCmpBB->getIterator(), StoreBB);
374 MF->insert(++StoreBB->getIterator(), FailBB);
375 MF->insert(++FailBB->getIterator(), DoneBB);
376
377
378
379
380
381
386 BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::SUBSXrs), AArch64::XZR)
388 .addReg(DesiredLoReg)
390 BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::CSINCWr), StatusReg)
391 .addUse(AArch64::WZR)
392 .addUse(AArch64::WZR)
394 BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::SUBSXrs), AArch64::XZR)
396 .addReg(DesiredHiReg)
398 BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::CSINCWr), StatusReg)
402 BuildMI(LoadCmpBB, MIMD, TII->get(AArch64::CBNZW))
405 LoadCmpBB->addSuccessor(FailBB);
406 LoadCmpBB->addSuccessor(StoreBB);
407
408
409
410
411 BuildMI(StoreBB, MIMD, TII->get(StxpOp), StatusReg)
419 StoreBB->addSuccessor(LoadCmpBB);
420 StoreBB->addSuccessor(DoneBB);
421
422
423
424
425 BuildMI(FailBB, MIMD, TII->get(StxpOp), StatusReg)
432 FailBB->addSuccessor(LoadCmpBB);
433 FailBB->addSuccessor(DoneBB);
434
435 DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
436 DoneBB->transferSuccessors(&MBB);
437
439
441 MI.eraseFromParent();
442
443
444 LivePhysRegs LiveRegs;
449
450
451 FailBB->clearLiveIns();
453 StoreBB->clearLiveIns();
455 LoadCmpBB->clearLiveIns();
457
458 return true;
459}
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499bool AArch64ExpandPseudo::expand_DestructiveOp(
500 MachineInstr &MI,
501 MachineBasicBlock &MBB,
507 Register DstReg = MI.getOperand(0).getReg();
508 bool DstIsDead = MI.getOperand(0).isDead();
509 bool UseRev = false;
510 unsigned PredIdx, DOPIdx, SrcIdx, Src2Idx;
511
512 switch (DType) {
515 if (DstReg == MI.getOperand(3).getReg()) {
516
517 std::tie(PredIdx, DOPIdx, SrcIdx) = std::make_tuple(1, 3, 2);
518 UseRev = true;
519 break;
520 }
521 [[fallthrough]];
524 std::tie(PredIdx, DOPIdx, SrcIdx) = std::make_tuple(1, 2, 3);
525 break;
527 std::tie(PredIdx, DOPIdx, SrcIdx) = std::make_tuple(2, 3, 3);
528 break;
530 std::tie(PredIdx, DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 2, 3, 4);
531 if (DstReg == MI.getOperand(3).getReg()) {
532
533 std::tie(PredIdx, DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 3, 4, 2);
534 UseRev = true;
535 } else if (DstReg == MI.getOperand(4).getReg()) {
536
537 std::tie(PredIdx, DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 4, 3, 2);
538 UseRev = true;
539 }
540 break;
542
543
544 std::tie(DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 1, 2);
545 break;
546 default:
548 }
549
550
551
552
553 bool DOPRegIsUnique = false;
554 switch (DType) {
556 DOPRegIsUnique = DstReg != MI.getOperand(SrcIdx).getReg();
557 break;
560 DOPRegIsUnique =
561 DstReg != MI.getOperand(DOPIdx).getReg() ||
562 MI.getOperand(DOPIdx).getReg() != MI.getOperand(SrcIdx).getReg();
563 break;
567 DOPRegIsUnique = true;
568 break;
570 DOPRegIsUnique =
571 DstReg != MI.getOperand(DOPIdx).getReg() ||
572 (MI.getOperand(DOPIdx).getReg() != MI.getOperand(SrcIdx).getReg() &&
573 MI.getOperand(DOPIdx).getReg() != MI.getOperand(Src2Idx).getReg());
574 break;
575 }
576
577
578 if (UseRev) {
579 int NewOpcode;
580
582 Opcode = NewOpcode;
583
585 Opcode = NewOpcode;
586 }
587
588
589 uint64_t ElementSize = TII->getElementSizeForOpcode(Opcode);
590 unsigned MovPrfx, LSLZero, MovPrfxZero;
591 switch (ElementSize) {
594 MovPrfx = AArch64::MOVPRFX_ZZ;
595 LSLZero = AArch64::LSL_ZPmI_B;
596 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_B;
597 break;
599 MovPrfx = AArch64::MOVPRFX_ZZ;
600 LSLZero = AArch64::LSL_ZPmI_H;
601 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_H;
602 break;
604 MovPrfx = AArch64::MOVPRFX_ZZ;
605 LSLZero = AArch64::LSL_ZPmI_S;
606 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_S;
607 break;
609 MovPrfx = AArch64::MOVPRFX_ZZ;
610 LSLZero = AArch64::LSL_ZPmI_D;
611 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_D;
612 break;
613 default:
615 }
616
617
618 unsigned DOPRegState = MI.getOperand(DOPIdx).isUndef() ? RegState::Undef : 0;
619
620
621
622
623 MachineInstrBuilder PRFX, DOP;
624 if (FalseZero) {
625
626
630 "The destructive operand should be unique");
632 "This instruction is unpredicated");
633
634
637 .addReg(MI.getOperand(PredIdx).getReg())
638 .addReg(MI.getOperand(DOPIdx).getReg(), DOPRegState);
639
640
641 DOPIdx = 0;
642 DOPRegState = 0;
643
644
645
646
650 !DOPRegIsUnique) {
653 .add(MI.getOperand(PredIdx))
656 }
657 } else if (DstReg != MI.getOperand(DOPIdx).getReg()) {
658 assert(DOPRegIsUnique && "The destructive operand should be unique");
661 .addReg(MI.getOperand(DOPIdx).getReg(), DOPRegState);
662 DOPIdx = 0;
663 DOPRegState = 0;
664 }
665
666
667
668
672
673 switch (DType) {
675 DOP.addReg(MI.getOperand(DOPIdx).getReg(), DOPRegState)
676 .add(MI.getOperand(PredIdx))
677 .add(MI.getOperand(SrcIdx));
678 break;
683 DOP.add(MI.getOperand(PredIdx))
684 .addReg(MI.getOperand(DOPIdx).getReg(), DOPRegState)
685 .add(MI.getOperand(SrcIdx));
686 break;
688 DOP.add(MI.getOperand(PredIdx))
689 .addReg(MI.getOperand(DOPIdx).getReg(), DOPRegState)
690 .add(MI.getOperand(SrcIdx))
691 .add(MI.getOperand(Src2Idx));
692 break;
694 DOP.addReg(MI.getOperand(DOPIdx).getReg(), DOPRegState)
695 .add(MI.getOperand(SrcIdx))
696 .add(MI.getOperand(Src2Idx));
697 break;
698 }
699
700 if (PRFX) {
701 transferImpOps(MI, PRFX, DOP);
703 } else
704 transferImpOps(MI, DOP, DOP);
705
706 MI.eraseFromParent();
707 return true;
708}
709
710bool AArch64ExpandPseudo::expandSetTagLoop(
715 Register SizeReg = MI.getOperand(0).getReg();
716 Register AddressReg = MI.getOperand(1).getReg();
717
719
720 bool ZeroData = MI.getOpcode() == AArch64::STZGloop_wback;
721 const unsigned OpCode1 =
722 ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex;
723 const unsigned OpCode2 =
724 ZeroData ? AArch64::STZ2GPostIndex : AArch64::ST2GPostIndex;
725
726 unsigned Size = MI.getOperand(2).getImm();
728 if (Size % (16 * 2) != 0) {
734 }
739
742
744 MF->insert(++LoopBB->getIterator(), DoneBB);
745
762
763 LoopBB->addSuccessor(LoopBB);
764 LoopBB->addSuccessor(DoneBB);
765
766 DoneBB->splice(DoneBB->end(), &MBB, MI, MBB.end());
767 DoneBB->transferSuccessors(&MBB);
768
770
772 MI.eraseFromParent();
773
774 LivePhysRegs LiveRegs;
777
778
779 LoopBB->clearLiveIns();
781 DoneBB->clearLiveIns();
783
784 return true;
785}
786
787bool AArch64ExpandPseudo::expandSVESpillFill(MachineBasicBlock &MBB,
789 unsigned Opc, unsigned N) {
790 assert((Opc == AArch64::LDR_ZXI || Opc == AArch64::STR_ZXI ||
791 Opc == AArch64::LDR_PXI || Opc == AArch64::STR_PXI) &&
792 "Unexpected opcode");
793 unsigned RState = (Opc == AArch64::LDR_ZXI || Opc == AArch64::LDR_PXI)
795 : 0;
796 unsigned sub0 = (Opc == AArch64::LDR_ZXI || Opc == AArch64::STR_ZXI)
797 ? AArch64::zsub0
798 : AArch64::psub0;
799 const TargetRegisterInfo *TRI =
803 int ImmOffset = MI.getOperand(2).getImm() + Offset;
804 bool Kill = (Offset + 1 == N) ? MI.getOperand(1).isKill() : false;
805 assert(ImmOffset >= -256 && ImmOffset < 256 &&
806 "Immediate spill offset out of range");
808 .addReg(TRI->getSubReg(MI.getOperand(0).getReg(), sub0 + Offset),
809 RState)
812 }
813 MI.eraseFromParent();
814 return true;
815}
816
817
818
822 unsigned Opcode,
824 unsigned RegMaskStartIdx) {
825
827 .add(ExplicitOps)
829
830
831
832
833 while (->getOperand(RegMaskStartIdx).isRegMask()) {
835 assert(MOP.isReg() && "can only add register operands");
837 MOP.getReg(), false, true, false,
838 false, MOP.isUndef()));
839 RegMaskStartIdx++;
840 }
843 Call->addOperand(MO);
844
846}
847
848
849
854 unsigned RegMaskStartIdx) {
855 unsigned Opc = CallTarget.isGlobal() ? AArch64::BL : AArch64::BLR;
856
858 "invalid operand for regular call");
860}
861
862bool AArch64ExpandPseudo::expandCALL_RVMARKER(
864
865
866
867
868
870 MachineOperand &RVTarget = MI.getOperand(0);
871 bool DoEmitMarker = MI.getOperand(1).getImm();
872 assert(RVTarget.isGlobal() && "invalid operand for attached call");
873
874 MachineInstr *OriginalCall = nullptr;
875
876 if (MI.getOpcode() == AArch64::BLRA_RVMARKER) {
877
878 const MachineOperand &CallTarget = MI.getOperand(2);
879 const MachineOperand &Key = MI.getOperand(3);
880 const MachineOperand &IntDisc = MI.getOperand(4);
881 const MachineOperand &AddrDisc = MI.getOperand(5);
882
885 "Invalid auth call key");
886
887 MachineOperand Ops[] = {CallTarget, Key, IntDisc, AddrDisc};
888
890 6);
891 } else {
892 assert(MI.getOpcode() == AArch64::BLR_RVMARKER && "unknown rvmarker MI");
894
895 3);
896 }
897
898 if (DoEmitMarker)
901 .addReg(AArch64::XZR)
904
906 .add(RVTarget)
908
909 if (MI.shouldUpdateAdditionalCallInfo())
911
912 MI.eraseFromParent();
914 std::next(RVCall->getIterator()));
915 return true;
916}
917
918bool AArch64ExpandPseudo::expandCALL_BTI(MachineBasicBlock &MBB,
920
921
922
923
924
927
928 1);
929
931
932 MachineInstr *BTI =
934
937
938 if (MI.shouldUpdateAdditionalCallInfo())
940
941 MI.eraseFromParent();
943 return true;
944}
945
946bool AArch64ExpandPseudo::expandStoreSwiftAsyncContext(
948 Register CtxReg = MBBI->getOperand(0).getReg();
950 int Offset = MBBI->getOperand(2).getImm();
953
954 if (STI.getTargetTriple().getArchName() != "arm64e") {
961 return true;
962 }
963
964
965
966
967
968
969
970
971 unsigned Opc = Offset >= 0 ? AArch64::ADDXri : AArch64::SUBXri;
978 .addUse(AArch64::X16)
982
983
985 .addUse(AArch64::XZR)
990 .addUse(AArch64::X17)
991 .addUse(AArch64::X16)
994 .addUse(AArch64::X17)
998
1000 return true;
1001}
1002
1003AArch64ExpandPseudo::ConditionalBlocks
1004AArch64ExpandPseudo::expandConditionalPseudo(MachineBasicBlock &MBB,
1007 MachineInstrBuilder &Branch) {
1010 "Unexpected unreachable in block");
1011
1012
1013
1014
1015
1016 MachineInstr &PrevMI = *std::prev(MBBI);
1017 MachineBasicBlock *CondBB = MBB.splitAt(PrevMI, true);
1018 MachineBasicBlock *EndBB =
1019 std::next(MBBI) == CondBB->end()
1021 : CondBB->splitAt(*MBBI, true);
1022
1023
1024 Branch.addMBB(CondBB);
1028
1029
1030
1032 return {*CondBB, *EndBB};
1033}
1034
1035MachineBasicBlock *
1036AArch64ExpandPseudo::expandRestoreZASave(MachineBasicBlock &MBB,
1038 MachineInstr &MI = *MBBI;
1040
1041
1042 MachineInstrBuilder Branch =
1044
1045 auto [CondBB, EndBB] = expandConditionalPseudo(MBB, MBBI, DL, Branch);
1046
1047 MachineInstrBuilder MIB =
1049
1050 for (unsigned I = 2; I < MI.getNumOperands(); ++I)
1051 MIB.add(MI.getOperand(I));
1052
1054
1055 MI.eraseFromParent();
1056 return &EndBB;
1057}
1058
1060
1066 [[maybe_unused]] auto *RI = MBB.getParent()->getSubtarget().getRegisterInfo();
1067
1068
1071
1072 auto [CondBB, EndBB] = expandConditionalPseudo(MBB, MBBI, DL, Branch);
1073
1076
1077 for (unsigned I = 3; I < MI.getNumOperands(); ++I)
1078 MIB.add(MI.getOperand(I));
1079
1081 .addImm(AArch64SysReg::TPIDR2_EL0)
1082 .addReg(AArch64::XZR);
1083 bool ZeroZA = MI.getOperand(1).getImm() != 0;
1084 bool ZeroZT0 = MI.getOperand(2).getImm() != 0;
1085 if (ZeroZA) {
1086 assert(MI.definesRegister(AArch64::ZAB0, RI) && "should define ZA!");
1087 BuildMI(CondBB, CondBB.back(), DL, TII->get(AArch64::ZERO_M))
1090 }
1091 if (ZeroZT0) {
1092 assert(MI.definesRegister(AArch64::ZT0, RI) && "should define ZT0!");
1093 BuildMI(CondBB, CondBB.back(), DL, TII->get(AArch64::ZERO_T))
1094 .addDef(AArch64::ZT0);
1095 }
1096
1097 MI.eraseFromParent();
1098 return &EndBB;
1099}
1100
1101MachineBasicBlock *
1102AArch64ExpandPseudo::expandCondSMToggle(MachineBasicBlock &MBB,
1104 MachineInstr &MI = *MBBI;
1105
1106
1107
1108
1109
1111 MI.getParent()->successors().begin() ==
1112 MI.getParent()->successors().end()) {
1113 MI.eraseFromParent();
1114 return &MBB;
1115 }
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1148
1149
1150
1151
1152
1153
1154
1155 unsigned Opc;
1156 switch (MI.getOperand(2).getImm()) {
1158 llvm_unreachable("Should have matched to instruction directly");
1160 Opc = AArch64::TBNZW;
1161 break;
1163 Opc = AArch64::TBZW;
1164 break;
1165 }
1166 auto PStateSM = MI.getOperand(3).getReg();
1168 unsigned SMReg32 = TRI->getSubReg(PStateSM, AArch64::sub_32);
1169 MachineInstrBuilder Tbx =
1171
1172 auto [CondBB, EndBB] = expandConditionalPseudo(MBB, MBBI, DL, Tbx);
1173
1174 MachineInstrBuilder MIB = BuildMI(CondBB, CondBB.back(), MI.getDebugLoc(),
1175 TII->get(AArch64::MSRpstatesvcrImm1));
1176
1177
1178
1179 MIB.add(MI.getOperand(0));
1180 MIB.add(MI.getOperand(1));
1181 for (unsigned i = 4; i < MI.getNumOperands(); ++i)
1182 MIB.add(MI.getOperand(i));
1183
1184 MI.eraseFromParent();
1185 return &EndBB;
1186}
1187
1188bool AArch64ExpandPseudo::expandMultiVecPseudo(
1190 TargetRegisterClass ContiguousClass, TargetRegisterClass StridedClass,
1191 unsigned ContiguousOp, unsigned StridedOpc) {
1192 MachineInstr &MI = *MBBI;
1193 Register Tuple = MI.getOperand(0).getReg();
1194
1195 auto ContiguousRange = ContiguousClass.getRegisters();
1196 auto StridedRange = StridedClass.getRegisters();
1197 unsigned Opc;
1199 Opc = ContiguousOp;
1201 Opc = StridedOpc;
1202 } else
1204
1210 transferImpOps(MI, MIB, MIB);
1211 MI.eraseFromParent();
1212 return true;
1213}
1214
1215bool AArch64ExpandPseudo::expandFormTuplePseudo(
1218 assert((Size == 2 || Size == 4) && "Invalid Tuple Size");
1219 MachineInstr &MI = *MBBI;
1220 Register ReturnTuple = MI.getOperand(0).getReg();
1221
1222 const TargetRegisterInfo *TRI =
1224 for (unsigned I = 0; I < Size; ++I) {
1225 Register FormTupleOpReg = MI.getOperand(I + 1).getReg();
1226 Register ReturnTupleSubReg =
1227 TRI->getSubReg(ReturnTuple, AArch64::zsub0 + I);
1228
1229
1230 if (FormTupleOpReg != ReturnTupleSubReg)
1233 .addReg(FormTupleOpReg)
1234 .addReg(FormTupleOpReg);
1235 }
1236
1237 MI.eraseFromParent();
1238 return true;
1239}
1240
1241
1242
1243bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
1246 MachineInstr &MI = *MBBI;
1247 unsigned Opcode = MI.getOpcode();
1248
1249
1251 if (OrigInstr != -1) {
1252 auto &Orig = TII->get(OrigInstr);
1255 return expand_DestructiveOp(MI, MBB, MBBI);
1256 }
1257 }
1258
1259 switch (Opcode) {
1260 default:
1261 break;
1262
1263 case AArch64::BSPv8i8:
1264 case AArch64::BSPv16i8: {
1265 Register DstReg = MI.getOperand(0).getReg();
1266 if (DstReg == MI.getOperand(3).getReg()) {
1267
1269 TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::BITv8i8
1270 : AArch64::BITv16i8))
1275 transferImpOps(MI, I, I);
1276 } else if (DstReg == MI.getOperand(2).getReg()) {
1277
1279 TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::BIFv8i8
1280 : AArch64::BIFv16i8))
1285 transferImpOps(MI, I, I);
1286 } else {
1287
1288 if (DstReg == MI.getOperand(1).getReg()) {
1289 auto I =
1291 TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::BSLv8i8
1292 : AArch64::BSLv16i8))
1297 transferImpOps(MI, I, I);
1298 } else {
1299 unsigned RegState =
1302 MI.getOperand(1).isKill() &&
1303 MI.getOperand(1).getReg() != MI.getOperand(2).getReg() &&
1304 MI.getOperand(1).getReg() != MI.getOperand(3).getReg());
1306 TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::ORRv8i8
1307 : AArch64::ORRv16i8))
1311 .addReg(MI.getOperand(1).getReg(), RegState)
1312 .addReg(MI.getOperand(1).getReg(), RegState);
1313 auto I2 =
1315 TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::BSLv8i8
1316 : AArch64::BSLv16i8))
1320 MI.getOperand(0).isRenamable()))
1323 transferImpOps(MI, I2, I2);
1324 }
1325 }
1326 MI.eraseFromParent();
1327 return true;
1328 }
1329
1330 case AArch64::ADDWrr:
1331 case AArch64::SUBWrr:
1332 case AArch64::ADDXrr:
1333 case AArch64::SUBXrr:
1334 case AArch64::ADDSWrr:
1335 case AArch64::SUBSWrr:
1336 case AArch64::ADDSXrr:
1337 case AArch64::SUBSXrr:
1338 case AArch64::ANDWrr:
1339 case AArch64::ANDXrr:
1340 case AArch64::BICWrr:
1341 case AArch64::BICXrr:
1342 case AArch64::ANDSWrr:
1343 case AArch64::ANDSXrr:
1344 case AArch64::BICSWrr:
1345 case AArch64::BICSXrr:
1346 case AArch64::EONWrr:
1347 case AArch64::EONXrr:
1348 case AArch64::EORWrr:
1349 case AArch64::EORXrr:
1350 case AArch64::ORNWrr:
1351 case AArch64::ORNXrr:
1352 case AArch64::ORRWrr:
1353 case AArch64::ORRXrr: {
1354 unsigned Opcode;
1355 switch (MI.getOpcode()) {
1356 default:
1357 return false;
1358 case AArch64::ADDWrr: Opcode = AArch64::ADDWrs; break;
1359 case AArch64::SUBWrr: Opcode = AArch64::SUBWrs; break;
1360 case AArch64::ADDXrr: Opcode = AArch64::ADDXrs; break;
1361 case AArch64::SUBXrr: Opcode = AArch64::SUBXrs; break;
1362 case AArch64::ADDSWrr: Opcode = AArch64::ADDSWrs; break;
1363 case AArch64::SUBSWrr: Opcode = AArch64::SUBSWrs; break;
1364 case AArch64::ADDSXrr: Opcode = AArch64::ADDSXrs; break;
1365 case AArch64::SUBSXrr: Opcode = AArch64::SUBSXrs; break;
1366 case AArch64::ANDWrr: Opcode = AArch64::ANDWrs; break;
1367 case AArch64::ANDXrr: Opcode = AArch64::ANDXrs; break;
1368 case AArch64::BICWrr: Opcode = AArch64::BICWrs; break;
1369 case AArch64::BICXrr: Opcode = AArch64::BICXrs; break;
1370 case AArch64::ANDSWrr: Opcode = AArch64::ANDSWrs; break;
1371 case AArch64::ANDSXrr: Opcode = AArch64::ANDSXrs; break;
1372 case AArch64::BICSWrr: Opcode = AArch64::BICSWrs; break;
1373 case AArch64::BICSXrr: Opcode = AArch64::BICSXrs; break;
1374 case AArch64::EONWrr: Opcode = AArch64::EONWrs; break;
1375 case AArch64::EONXrr: Opcode = AArch64::EONXrs; break;
1376 case AArch64::EORWrr: Opcode = AArch64::EORWrs; break;
1377 case AArch64::EORXrr: Opcode = AArch64::EORXrs; break;
1378 case AArch64::ORNWrr: Opcode = AArch64::ORNWrs; break;
1379 case AArch64::ORNXrr: Opcode = AArch64::ORNXrs; break;
1380 case AArch64::ORRWrr: Opcode = AArch64::ORRWrs; break;
1381 case AArch64::ORRXrr: Opcode = AArch64::ORRXrs; break;
1382 }
1384
1385 MachineInstr *NewMI = MF.CreateMachineInstr(
1386 TII->get(Opcode), MI.getDebugLoc(), true);
1388 MachineInstrBuilder MIB1(MF, NewMI);
1389 MIB1->setPCSections(MF, MI.getPCSections());
1391 .add(MI.getOperand(1))
1392 .add(MI.getOperand(2))
1394 transferImpOps(MI, MIB1, MIB1);
1395 if (auto DebugNumber = MI.peekDebugInstrNum())
1397 MI.eraseFromParent();
1398 return true;
1399 }
1400
1401 case AArch64::LOADgot: {
1403 Register DstReg = MI.getOperand(0).getReg();
1404 const MachineOperand &MO1 = MI.getOperand(1);
1406
1408
1409 MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
1410 TII->get(AArch64::LDRXl), DstReg);
1411
1414 } else if (MO1.isSymbol()) {
1416 } else {
1418 "Only expect globals, externalsymbols, or constant pools");
1420 }
1421 } else {
1422
1423 MachineFunction &MF = *MI.getParent()->getParent();
1425 MachineInstrBuilder MIB1 =
1427
1428 MachineInstrBuilder MIB2;
1429 if (MF.getSubtarget().isTargetILP32()) {
1431 unsigned Reg32 = TRI->getSubReg(DstReg, AArch64::sub_32);
1432 unsigned DstFlags = MI.getOperand(0).getTargetFlags();
1437 } else {
1438 Register DstReg = MI.getOperand(0).getReg();
1442 }
1443
1448 } else if (MO1.isSymbol()) {
1453 } else {
1455 "Only expect globals, externalsymbols, or constant pools");
1461 }
1462
1463
1464
1465
1466 if (MI.peekDebugInstrNum() != 0)
1468 transferImpOps(MI, MIB1, MIB2);
1469 }
1470 MI.eraseFromParent();
1471 return true;
1472 }
1473 case AArch64::MOVaddrBA: {
1474 MachineFunction &MF = *MI.getParent()->getParent();
1475 if (MF.getSubtarget().isTargetMachO()) {
1476
1477
1478
1479 const BlockAddress *BA = MI.getOperand(1).getBlockAddress();
1480 assert(MI.getOperand(1).getOffset() == 0 && "unexpected offset");
1481
1484
1485 Register DstReg = MI.getOperand(0).getReg();
1486 auto MIB1 =
1490 TII->get(AArch64::LDRXui), DstReg)
1494 transferImpOps(MI, MIB1, MIB2);
1495 MI.eraseFromParent();
1496 return true;
1497 }
1498 }
1499 [[fallthrough]];
1500 case AArch64::MOVaddr:
1501 case AArch64::MOVaddrJT:
1502 case AArch64::MOVaddrCP:
1503 case AArch64::MOVaddrTLS:
1504 case AArch64::MOVaddrEXT: {
1505
1506 Register DstReg = MI.getOperand(0).getReg();
1507 assert(DstReg != AArch64::XZR);
1508 MachineInstrBuilder MIB1 =
1511
1513
1514
1515
1516
1517
1518
1519
1520 auto Tag = MI.getOperand(1);
1522 Tag.setOffset(0x100000000);
1527 }
1528
1529 MachineInstrBuilder MIB2 =
1535
1536 transferImpOps(MI, MIB1, MIB2);
1537 MI.eraseFromParent();
1538 return true;
1539 }
1540 case AArch64::ADDlowTLS:
1541
1547 MI.eraseFromParent();
1548 return true;
1549
1550 case AArch64::MOVbaseTLS: {
1551 Register DstReg = MI.getOperand(0).getReg();
1552 auto SysReg = AArch64SysReg::TPIDR_EL0;
1554 if (MF->getSubtarget().useEL3ForTP())
1555 SysReg = AArch64SysReg::TPIDR_EL3;
1556 else if (MF->getSubtarget().useEL2ForTP())
1557 SysReg = AArch64SysReg::TPIDR_EL2;
1558 else if (MF->getSubtarget().useEL1ForTP())
1559 SysReg = AArch64SysReg::TPIDR_EL1;
1560 else if (MF->getSubtarget().useROEL0ForTP())
1561 SysReg = AArch64SysReg::TPIDRRO_EL0;
1564 MI.eraseFromParent();
1565 return true;
1566 }
1567
1568 case AArch64::MOVi32imm:
1570 case AArch64::MOVi64imm:
1572 case AArch64::RET_ReallyLR: {
1573
1574
1575
1576
1577
1578 MachineInstrBuilder MIB =
1581 transferImpOps(MI, MIB, MIB);
1582 MI.eraseFromParent();
1583 return true;
1584 }
1585 case AArch64::CMP_SWAP_8:
1586 return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRB, AArch64::STLXRB,
1587 AArch64::SUBSWrx,
1589 AArch64::WZR, NextMBBI);
1590 case AArch64::CMP_SWAP_16:
1591 return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRH, AArch64::STLXRH,
1592 AArch64::SUBSWrx,
1594 AArch64::WZR, NextMBBI);
1595 case AArch64::CMP_SWAP_32:
1596 return expandCMP_SWAP(MBB, MBBI, AArch64::LDAXRW, AArch64::STLXRW,
1597 AArch64::SUBSWrs,
1599 AArch64::WZR, NextMBBI);
1600 case AArch64::CMP_SWAP_64:
1601 return expandCMP_SWAP(MBB, MBBI,
1602 AArch64::LDAXRX, AArch64::STLXRX, AArch64::SUBSXrs,
1604 AArch64::XZR, NextMBBI);
1605 case AArch64::CMP_SWAP_128:
1606 case AArch64::CMP_SWAP_128_RELEASE:
1607 case AArch64::CMP_SWAP_128_ACQUIRE:
1608 case AArch64::CMP_SWAP_128_MONOTONIC:
1609 return expandCMP_SWAP_128(MBB, MBBI, NextMBBI);
1610
1611 case AArch64::AESMCrrTied:
1612 case AArch64::AESIMCrrTied: {
1613 MachineInstrBuilder MIB =
1615 TII->get(Opcode == AArch64::AESMCrrTied ? AArch64::AESMCrr :
1616 AArch64::AESIMCrr))
1619 transferImpOps(MI, MIB, MIB);
1620 MI.eraseFromParent();
1621 return true;
1622 }
1623 case AArch64::IRGstack: {
1625 const AArch64FunctionInfo *AFI = MF.getInfo();
1626 const AArch64FrameLowering *TFI =
1627 MF.getSubtarget().getFrameLowering();
1628
1629
1630
1631
1634 StackOffset FrameRegOffset = TFI->resolveFrameOffsetReference(
1636 FrameReg,
1637 false,
1638 true);
1640 if (FrameRegOffset) {
1641
1642 SrcReg = MI.getOperand(0).getReg();
1644 FrameRegOffset, TII);
1645 }
1650 MI.eraseFromParent();
1651 return true;
1652 }
1653 case AArch64::TAGPstack: {
1654 int64_t Offset = MI.getOperand(2).getImm();
1656 TII->get(Offset >= 0 ? AArch64::ADDG : AArch64::SUBG))
1661 MI.eraseFromParent();
1662 return true;
1663 }
1664 case AArch64::STGloop_wback:
1665 case AArch64::STZGloop_wback:
1666 return expandSetTagLoop(MBB, MBBI, NextMBBI);
1667 case AArch64::STGloop:
1668 case AArch64::STZGloop:
1670 "Non-writeback variants of STGloop / STZGloop should not "
1671 "survive past PrologEpilogInserter.");
1672 case AArch64::STR_ZZZZXI:
1673 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
1674 return expandSVESpillFill(MBB, MBBI, AArch64::STR_ZXI, 4);
1675 case AArch64::STR_ZZZXI:
1676 return expandSVESpillFill(MBB, MBBI, AArch64::STR_ZXI, 3);
1677 case AArch64::STR_ZZXI:
1678 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
1679 return expandSVESpillFill(MBB, MBBI, AArch64::STR_ZXI, 2);
1680 case AArch64::STR_PPXI:
1681 return expandSVESpillFill(MBB, MBBI, AArch64::STR_PXI, 2);
1682 case AArch64::LDR_ZZZZXI:
1683 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
1684 return expandSVESpillFill(MBB, MBBI, AArch64::LDR_ZXI, 4);
1685 case AArch64::LDR_ZZZXI:
1686 return expandSVESpillFill(MBB, MBBI, AArch64::LDR_ZXI, 3);
1687 case AArch64::LDR_ZZXI:
1688 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
1689 return expandSVESpillFill(MBB, MBBI, AArch64::LDR_ZXI, 2);
1690 case AArch64::LDR_PPXI:
1691 return expandSVESpillFill(MBB, MBBI, AArch64::LDR_PXI, 2);
1692 case AArch64::BLR_RVMARKER:
1693 case AArch64::BLRA_RVMARKER:
1694 return expandCALL_RVMARKER(MBB, MBBI);
1695 case AArch64::BLR_BTI:
1696 return expandCALL_BTI(MBB, MBBI);
1697 case AArch64::StoreSwiftAsyncContext:
1698 return expandStoreSwiftAsyncContext(MBB, MBBI);
1699 case AArch64::RestoreZAPseudo:
1700 case AArch64::CommitZASavePseudo:
1701 case AArch64::MSRpstatePseudo: {
1702 auto *NewMBB = [&] {
1703 switch (Opcode) {
1704 case AArch64::RestoreZAPseudo:
1705 return expandRestoreZASave(MBB, MBBI);
1706 case AArch64::CommitZASavePseudo:
1707 return expandCommitZASave(MBB, MBBI);
1708 case AArch64::MSRpstatePseudo:
1709 return expandCondSMToggle(MBB, MBBI);
1710 default:
1712 }
1713 }();
1714 if (NewMBB != &MBB)
1716 return true;
1717 }
1718 case AArch64::InOutZAUsePseudo:
1719 case AArch64::RequiresZASavePseudo:
1720 case AArch64::RequiresZT0SavePseudo:
1721 case AArch64::SMEStateAllocPseudo:
1722 case AArch64::COALESCER_BARRIER_FPR16:
1723 case AArch64::COALESCER_BARRIER_FPR32:
1724 case AArch64::COALESCER_BARRIER_FPR64:
1725 case AArch64::COALESCER_BARRIER_FPR128:
1726 MI.eraseFromParent();
1727 return true;
1728 case AArch64::LD1B_2Z_IMM_PSEUDO:
1729 return expandMultiVecPseudo(
1730 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1731 AArch64::LD1B_2Z_IMM, AArch64::LD1B_2Z_STRIDED_IMM);
1732 case AArch64::LD1H_2Z_IMM_PSEUDO:
1733 return expandMultiVecPseudo(
1734 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1735 AArch64::LD1H_2Z_IMM, AArch64::LD1H_2Z_STRIDED_IMM);
1736 case AArch64::LD1W_2Z_IMM_PSEUDO:
1737 return expandMultiVecPseudo(
1738 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1739 AArch64::LD1W_2Z_IMM, AArch64::LD1W_2Z_STRIDED_IMM);
1740 case AArch64::LD1D_2Z_IMM_PSEUDO:
1741 return expandMultiVecPseudo(
1742 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1743 AArch64::LD1D_2Z_IMM, AArch64::LD1D_2Z_STRIDED_IMM);
1744 case AArch64::LDNT1B_2Z_IMM_PSEUDO:
1745 return expandMultiVecPseudo(
1746 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1747 AArch64::LDNT1B_2Z_IMM, AArch64::LDNT1B_2Z_STRIDED_IMM);
1748 case AArch64::LDNT1H_2Z_IMM_PSEUDO:
1749 return expandMultiVecPseudo(
1750 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1751 AArch64::LDNT1H_2Z_IMM, AArch64::LDNT1H_2Z_STRIDED_IMM);
1752 case AArch64::LDNT1W_2Z_IMM_PSEUDO:
1753 return expandMultiVecPseudo(
1754 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1755 AArch64::LDNT1W_2Z_IMM, AArch64::LDNT1W_2Z_STRIDED_IMM);
1756 case AArch64::LDNT1D_2Z_IMM_PSEUDO:
1757 return expandMultiVecPseudo(
1758 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1759 AArch64::LDNT1D_2Z_IMM, AArch64::LDNT1D_2Z_STRIDED_IMM);
1760 case AArch64::LD1B_2Z_PSEUDO:
1761 return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR2RegClass,
1762 AArch64::ZPR2StridedRegClass, AArch64::LD1B_2Z,
1763 AArch64::LD1B_2Z_STRIDED);
1764 case AArch64::LD1H_2Z_PSEUDO:
1765 return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR2RegClass,
1766 AArch64::ZPR2StridedRegClass, AArch64::LD1H_2Z,
1767 AArch64::LD1H_2Z_STRIDED);
1768 case AArch64::LD1W_2Z_PSEUDO:
1769 return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR2RegClass,
1770 AArch64::ZPR2StridedRegClass, AArch64::LD1W_2Z,
1771 AArch64::LD1W_2Z_STRIDED);
1772 case AArch64::LD1D_2Z_PSEUDO:
1773 return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR2RegClass,
1774 AArch64::ZPR2StridedRegClass, AArch64::LD1D_2Z,
1775 AArch64::LD1D_2Z_STRIDED);
1776 case AArch64::LDNT1B_2Z_PSEUDO:
1777 return expandMultiVecPseudo(
1778 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1779 AArch64::LDNT1B_2Z, AArch64::LDNT1B_2Z_STRIDED);
1780 case AArch64::LDNT1H_2Z_PSEUDO:
1781 return expandMultiVecPseudo(
1782 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1783 AArch64::LDNT1H_2Z, AArch64::LDNT1H_2Z_STRIDED);
1784 case AArch64::LDNT1W_2Z_PSEUDO:
1785 return expandMultiVecPseudo(
1786 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1787 AArch64::LDNT1W_2Z, AArch64::LDNT1W_2Z_STRIDED);
1788 case AArch64::LDNT1D_2Z_PSEUDO:
1789 return expandMultiVecPseudo(
1790 MBB, MBBI, AArch64::ZPR2RegClass, AArch64::ZPR2StridedRegClass,
1791 AArch64::LDNT1D_2Z, AArch64::LDNT1D_2Z_STRIDED);
1792 case AArch64::LD1B_4Z_IMM_PSEUDO:
1793 return expandMultiVecPseudo(
1794 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1795 AArch64::LD1B_4Z_IMM, AArch64::LD1B_4Z_STRIDED_IMM);
1796 case AArch64::LD1H_4Z_IMM_PSEUDO:
1797 return expandMultiVecPseudo(
1798 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1799 AArch64::LD1H_4Z_IMM, AArch64::LD1H_4Z_STRIDED_IMM);
1800 case AArch64::LD1W_4Z_IMM_PSEUDO:
1801 return expandMultiVecPseudo(
1802 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1803 AArch64::LD1W_4Z_IMM, AArch64::LD1W_4Z_STRIDED_IMM);
1804 case AArch64::LD1D_4Z_IMM_PSEUDO:
1805 return expandMultiVecPseudo(
1806 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1807 AArch64::LD1D_4Z_IMM, AArch64::LD1D_4Z_STRIDED_IMM);
1808 case AArch64::LDNT1B_4Z_IMM_PSEUDO:
1809 return expandMultiVecPseudo(
1810 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1811 AArch64::LDNT1B_4Z_IMM, AArch64::LDNT1B_4Z_STRIDED_IMM);
1812 case AArch64::LDNT1H_4Z_IMM_PSEUDO:
1813 return expandMultiVecPseudo(
1814 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1815 AArch64::LDNT1H_4Z_IMM, AArch64::LDNT1H_4Z_STRIDED_IMM);
1816 case AArch64::LDNT1W_4Z_IMM_PSEUDO:
1817 return expandMultiVecPseudo(
1818 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1819 AArch64::LDNT1W_4Z_IMM, AArch64::LDNT1W_4Z_STRIDED_IMM);
1820 case AArch64::LDNT1D_4Z_IMM_PSEUDO:
1821 return expandMultiVecPseudo(
1822 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1823 AArch64::LDNT1D_4Z_IMM, AArch64::LDNT1D_4Z_STRIDED_IMM);
1824 case AArch64::LD1B_4Z_PSEUDO:
1825 return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR4RegClass,
1826 AArch64::ZPR4StridedRegClass, AArch64::LD1B_4Z,
1827 AArch64::LD1B_4Z_STRIDED);
1828 case AArch64::LD1H_4Z_PSEUDO:
1829 return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR4RegClass,
1830 AArch64::ZPR4StridedRegClass, AArch64::LD1H_4Z,
1831 AArch64::LD1H_4Z_STRIDED);
1832 case AArch64::LD1W_4Z_PSEUDO:
1833 return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR4RegClass,
1834 AArch64::ZPR4StridedRegClass, AArch64::LD1W_4Z,
1835 AArch64::LD1W_4Z_STRIDED);
1836 case AArch64::LD1D_4Z_PSEUDO:
1837 return expandMultiVecPseudo(MBB, MBBI, AArch64::ZPR4RegClass,
1838 AArch64::ZPR4StridedRegClass, AArch64::LD1D_4Z,
1839 AArch64::LD1D_4Z_STRIDED);
1840 case AArch64::LDNT1B_4Z_PSEUDO:
1841 return expandMultiVecPseudo(
1842 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1843 AArch64::LDNT1B_4Z, AArch64::LDNT1B_4Z_STRIDED);
1844 case AArch64::LDNT1H_4Z_PSEUDO:
1845 return expandMultiVecPseudo(
1846 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1847 AArch64::LDNT1H_4Z, AArch64::LDNT1H_4Z_STRIDED);
1848 case AArch64::LDNT1W_4Z_PSEUDO:
1849 return expandMultiVecPseudo(
1850 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1851 AArch64::LDNT1W_4Z, AArch64::LDNT1W_4Z_STRIDED);
1852 case AArch64::LDNT1D_4Z_PSEUDO:
1853 return expandMultiVecPseudo(
1854 MBB, MBBI, AArch64::ZPR4RegClass, AArch64::ZPR4StridedRegClass,
1855 AArch64::LDNT1D_4Z, AArch64::LDNT1D_4Z_STRIDED);
1856 case AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO:
1857 return expandFormTuplePseudo(MBB, MBBI, NextMBBI, 2);
1858 case AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO:
1859 return expandFormTuplePseudo(MBB, MBBI, NextMBBI, 4);
1860 }
1861 return false;
1862}
1863
1864
1865
1866bool AArch64ExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
1868
1873 MBBI = NMBBI;
1874 }
1875
1877}
1878
1879bool AArch64ExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
1880 TII = MF.getSubtarget().getInstrInfo();
1881
1883 for (auto &MBB : MF)
1886}
1887
1888
1890 return new AArch64ExpandPseudo();
1891}
#define AARCH64_EXPAND_PSEUDO_NAME
Definition AArch64ExpandPseudoInsts.cpp:43
MachineInstrBuilder & UseMI
Definition AArch64ExpandPseudoInsts.cpp:120
static MachineInstr * createCallWithOps(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const AArch64InstrInfo *TII, unsigned Opcode, ArrayRef< MachineOperand > ExplicitOps, unsigned RegMaskStartIdx)
Definition AArch64ExpandPseudoInsts.cpp:819
static constexpr unsigned ZERO_ALL_ZA_MASK
Definition AArch64ExpandPseudoInsts.cpp:1059
static MachineInstr * createCall(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const AArch64InstrInfo *TII, MachineOperand &CallTarget, unsigned RegMaskStartIdx)
Definition AArch64ExpandPseudoInsts.cpp:850
MachineInstrBuilder MachineInstrBuilder & DefMI
Definition AArch64ExpandPseudoInsts.cpp:121
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const TargetInstrInfo & TII
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
Register const TargetRegisterInfo * TRI
Promote Memory to Register
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
unsigned getTaggedBasePointerOffset() const
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
FunctionPass class - This class is used to implement most global optimizations.
Describe properties that are true of each instruction in the target description file.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
LLVM_ABI instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
LLVM_ABI MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
LLVM_ABI void eraseFromParent()
This method unlinks 'this' from the containing function and deletes it.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator_range< succ_iterator > successors()
MachineInstrBundleIterator< MachineInstr > iterator
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void moveAdditionalCallInfo(const MachineInstr *Old, const MachineInstr *New)
Move the call site info from Old to \New call site info.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
void setDebugInstrNum(unsigned Num)
Set instruction number of this MachineInstr.
MachineOperand class - Representation of each machine instruction operand.
const GlobalValue * getGlobal() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isCPI() const
isCPI - Tests if this is a MO_ConstantPoolIndex operand.
bool isSymbol() const
isSymbol - Tests if this is a MO_ExternalSymbol operand.
unsigned getTargetFlags() const
bool isGlobal() const
isGlobal - Tests if this is a MO_GlobalAddress operand.
const char * getSymbolName() const
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
int64_t getOffset() const
Return the offset from the symbol in this operand.
MCRegister asMCReg() const
Utility to check-convert this value to a MCRegister.
void push_back(const T &Elt)
StringRef - Represent a constant reference to a string, i.e.
CodeModel::Model getCodeModel() const
Returns the code model.
ArrayRef< MCPhysReg > getRegisters() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_PREL
MO_PREL - Indicates that the bits of the symbol operand represented by MO_G0 etc are PC relative.
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
int getSVERevInstr(uint16_t Opcode)
@ Destructive2xRegImmUnpred
@ DestructiveInstTypeMask
@ DestructiveUnaryPassthru
@ DestructiveTernaryCommWithRev
@ DestructiveBinaryCommWithRev
int getSVEPseudoMap(uint16_t Opcode)
int getSVENonRevInstr(uint16_t Opcode)
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Renamable
Register that may be renamed.
@ Define
Register definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
LLVM_ABI void finalizeBundle(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
finalizeBundle - Finalize a machine instruction bundle which includes a sequence of instructions star...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
APFloat abs(APFloat X)
Returns the absolute value of the argument.
unsigned getDeadRegState(bool B)
FunctionPass * createAArch64ExpandPseudoPass()
Returns an instance of the pseudo instruction expansion pass.
Definition AArch64ExpandPseudoInsts.cpp:1889
void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, StackOffset Offset, const TargetInstrInfo *TII, MachineInstr::MIFlag=MachineInstr::NoFlags, bool SetNZCV=false, bool NeedsWinCFI=false, bool *HasWinCFI=nullptr, bool EmitCFAOffset=false, StackOffset InitialOffset={}, unsigned FrameReg=AArch64::SP)
emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg plus Offset.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
unsigned getKillRegState(bool B)
unsigned getRenamableRegState(bool B)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
void computeAndAddLiveIns(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB)
Convenience function combining computeLiveIns() and addLiveIns().