LLVM: lib/Target/RISCV/RISCVInstrInfo.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
36
37using namespace llvm;
38
39#define GEN_CHECK_COMPRESS_INSTR
40#include "RISCVGenCompressInstEmitter.inc"
41
42#define GET_INSTRINFO_CTOR_DTOR
43#define GET_INSTRINFO_NAMED_OPS
44#include "RISCVGenInstrInfo.inc"
45
48 cl::desc("Prefer whole register move for vector registers."));
49
51 "riscv-force-machine-combiner-strategy", cl::Hidden,
52 cl::desc("Force machine combiner to use a specific strategy for machine "
53 "trace metrics evaluation."),
54 cl::init(MachineTraceStrategy::TS_NumStrategies),
56 "Local strategy."),
57 clEnumValN(MachineTraceStrategy::TS_MinInstrCount, "min-instr",
58 "MinInstrCount strategy.")));
59
61
62using namespace RISCV;
63
64#define GET_RISCVVPseudosTable_IMPL
65#include "RISCVGenSearchableTables.inc"
66
67}
68
70
71#define GET_RISCVMaskedPseudosTable_IMPL
72#include "RISCVGenSearchableTables.inc"
73
74}
75
77 : RISCVGenInstrInfo(RISCV::ADJCALLSTACKDOWN, RISCV::ADJCALLSTACKUP),
78 STI(STI) {}
79
87}
88
90 int &FrameIndex) const {
91 unsigned Dummy;
93}
94
96 int &FrameIndex,
97 unsigned &MemBytes) const {
98 switch (MI.getOpcode()) {
99 default:
100 return 0;
101 case RISCV::LB:
102 case RISCV::LBU:
103 MemBytes = 1;
104 break;
105 case RISCV::LH:
106 case RISCV::LH_INX:
107 case RISCV::LHU:
108 case RISCV::FLH:
109 MemBytes = 2;
110 break;
111 case RISCV::LW:
112 case RISCV::LW_INX:
113 case RISCV::FLW:
114 case RISCV::LWU:
115 MemBytes = 4;
116 break;
117 case RISCV::LD:
118 case RISCV::FLD:
119 MemBytes = 8;
120 break;
121 }
122
123 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
124 MI.getOperand(2).getImm() == 0) {
125 FrameIndex = MI.getOperand(1).getIndex();
126 return MI.getOperand(0).getReg();
127 }
128
129 return 0;
130}
131
133 int &FrameIndex) const {
134 unsigned Dummy;
136}
137
139 int &FrameIndex,
140 unsigned &MemBytes) const {
141 switch (MI.getOpcode()) {
142 default:
143 return 0;
144 case RISCV::SB:
145 MemBytes = 1;
146 break;
147 case RISCV::SH:
148 case RISCV::SH_INX:
149 case RISCV::FSH:
150 MemBytes = 2;
151 break;
152 case RISCV::SW:
153 case RISCV::SW_INX:
154 case RISCV::FSW:
155 MemBytes = 4;
156 break;
157 case RISCV::SD:
158 case RISCV::FSD:
159 MemBytes = 8;
160 break;
161 }
162
163 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
164 MI.getOperand(2).getImm() == 0) {
165 FrameIndex = MI.getOperand(1).getIndex();
166 return MI.getOperand(0).getReg();
167 }
168
169 return 0;
170}
171
175 case RISCV::VMV_V_X:
176 case RISCV::VFMV_V_F:
177 case RISCV::VMV_V_I:
178 case RISCV::VMV_S_X:
179 case RISCV::VFMV_S_F:
180 case RISCV::VID_V:
181 return MI.getOperand(1).isUndef();
182 default:
184 }
185}
186
188 unsigned NumRegs) {
189 return DstReg > SrcReg && (DstReg - SrcReg) < NumRegs;
190}
191
198 return false;
199
200 assert(MBBI->getOpcode() == TargetOpcode::COPY &&
201 "Unexpected COPY instruction.");
202 Register SrcReg = MBBI->getOperand(1).getReg();
204
205 bool FoundDef = false;
206 bool FirstVSetVLI = false;
207 unsigned FirstSEW = 0;
210 if (MBBI->isMetaInstruction())
211 continue;
212
213 if (MBBI->getOpcode() == RISCV::PseudoVSETVLI ||
214 MBBI->getOpcode() == RISCV::PseudoVSETVLIX0 ||
215 MBBI->getOpcode() == RISCV::PseudoVSETIVLI) {
216
217
218
219
220
221
222 if (!FoundDef) {
223 if (!FirstVSetVLI) {
224 FirstVSetVLI = true;
225 unsigned FirstVType = MBBI->getOperand(2).getImm();
228
229
230 if (FirstLMul != LMul)
231 return false;
232 }
233
234
235 if (MBBI->getOperand(0).getReg() != RISCV::X0)
236 return false;
237 if (MBBI->getOperand(1).isImm())
238 return false;
239 if (MBBI->getOperand(1).getReg() != RISCV::X0)
240 return false;
241 continue;
242 }
243
244
245 unsigned VType = MBBI->getOperand(2).getImm();
246
247 if (FirstVSetVLI) {
248
250 return false;
251 }
252
253
255 return false;
256
257
258
259
260
261
263 } else if (MBBI->isInlineAsm() || MBBI->isCall()) {
264 return false;
265 } else if (MBBI->getNumDefs()) {
266
267
268 if (MBBI->modifiesRegister(RISCV::VL, nullptr))
269 return false;
270
271
272
274 if (!MO.isReg() || !MO.isDef())
275 continue;
276 if (!FoundDef && TRI->regsOverlap(MO.getReg(), SrcReg)) {
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291 if (MO.getReg() != SrcReg)
292 return false;
293
294
295
296
297
298
299
300
301
302
305 return false;
306
307
308
310 return false;
311
312
313 FoundDef = true;
314 DefMBBI = MBBI;
315 break;
316 }
317 }
318 }
319 }
320
321 return false;
322}
323
331
332 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
333 uint16_t DstEncoding = TRI->getEncodingValue(DstReg);
335 assert(!Fractional && "It is impossible be fractional lmul here.");
336 unsigned NumRegs = NF * LMulVal;
337 bool ReversedCopy =
339 if (ReversedCopy) {
340
341
342 SrcEncoding += NumRegs - 1;
343 DstEncoding += NumRegs - 1;
344 }
345
346 unsigned I = 0;
347 auto GetCopyInfo = [&](uint16_t SrcEncoding, uint16_t DstEncoding)
349 unsigned, unsigned> {
350 if (ReversedCopy) {
351
352
353
354
355
356
357 uint16_t Diff = DstEncoding - SrcEncoding;
358 if (I + 8 <= NumRegs && Diff >= 8 && SrcEncoding % 8 == 7 &&
359 DstEncoding % 8 == 7)
360 return {RISCVII::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
361 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
362 if (I + 4 <= NumRegs && Diff >= 4 && SrcEncoding % 4 == 3 &&
363 DstEncoding % 4 == 3)
364 return {RISCVII::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
365 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
366 if (I + 2 <= NumRegs && Diff >= 2 && SrcEncoding % 2 == 1 &&
367 DstEncoding % 2 == 1)
368 return {RISCVII::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
369 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
370
371 return {RISCVII::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
372 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
373 }
374
375
376
377 if (I + 8 <= NumRegs && SrcEncoding % 8 == 0 && DstEncoding % 8 == 0)
378 return {RISCVII::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
379 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
380 if (I + 4 <= NumRegs && SrcEncoding % 4 == 0 && DstEncoding % 4 == 0)
381 return {RISCVII::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
382 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
383 if (I + 2 <= NumRegs && SrcEncoding % 2 == 0 && DstEncoding % 2 == 0)
384 return {RISCVII::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
385 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
386
387 return {RISCVII::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
388 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
389 };
392 MCRegister Reg = RISCV::V0 + Encoding;
394 return Reg;
395 return TRI->getMatchingSuperReg(Reg, RISCV::sub_vrm1_0, &RegClass);
396 };
397 while (I != NumRegs) {
398
399
400
401
402 auto [LMulCopied, RegClass, Opc, VVOpc, VIOpc] =
403 GetCopyInfo(SrcEncoding, DstEncoding);
405
407 if (LMul == LMulCopied &&
409 Opc = VVOpc;
410 if (DefMBBI->getOpcode() == VIOpc)
411 Opc = VIOpc;
412 }
413
414
415
416 MCRegister ActualSrcReg = FindRegWithEncoding(
417 RegClass, ReversedCopy ? (SrcEncoding - NumCopied + 1) : SrcEncoding);
418 MCRegister ActualDstReg = FindRegWithEncoding(
419 RegClass, ReversedCopy ? (DstEncoding - NumCopied + 1) : DstEncoding);
420
424 if (UseVMV)
426 if (UseVMV_V_I)
427 MIB = MIB.add(DefMBBI->getOperand(2));
428 else
429 MIB = MIB.addReg(ActualSrcReg, getKillRegState(KillSrc));
430 if (UseVMV) {
433 unsigned Log2SEW =
435 MIB.addImm(Log2SEW ? Log2SEW : 3);
436 MIB.addImm(0);
439 }
440
441
442 SrcEncoding += (ReversedCopy ? -NumCopied : NumCopied);
443 DstEncoding += (ReversedCopy ? -NumCopied : NumCopied);
444 I += NumCopied;
445 }
446}
447
452 bool RenamableDest, bool RenamableSrc) const {
454
455 if (RISCV::GPRRegClass.contains(DstReg, SrcReg)) {
460 return;
461 }
462
463 if (RISCV::GPRF16RegClass.contains(DstReg, SrcReg)) {
467 return;
468 }
469
470 if (RISCV::GPRF32RegClass.contains(DstReg, SrcReg)) {
474 return;
475 }
476
477 if (RISCV::GPRPairRegClass.contains(DstReg, SrcReg)) {
478
480 TRI->getSubReg(DstReg, RISCV::sub_gpr_even))
481 .addReg(TRI->getSubReg(SrcReg, RISCV::sub_gpr_even),
485 TRI->getSubReg(DstReg, RISCV::sub_gpr_odd))
486 .addReg(TRI->getSubReg(SrcReg, RISCV::sub_gpr_odd),
489 return;
490 }
491
492
493 if (RISCV::VCSRRegClass.contains(SrcReg) &&
494 RISCV::GPRRegClass.contains(DstReg)) {
496 .addImm(RISCVSysReg::lookupSysRegByName(TRI->getName(SrcReg))->Encoding)
498 return;
499 }
500
501 if (RISCV::FPR16RegClass.contains(DstReg, SrcReg)) {
502 unsigned Opc;
503 if (STI.hasStdExtZfh()) {
504 Opc = RISCV::FSGNJ_H;
505 } else {
507 (STI.hasStdExtZfhmin() || STI.hasStdExtZfbfmin()) &&
508 "Unexpected extensions");
509
510 DstReg = TRI->getMatchingSuperReg(DstReg, RISCV::sub_16,
511 &RISCV::FPR32RegClass);
512 SrcReg = TRI->getMatchingSuperReg(SrcReg, RISCV::sub_16,
513 &RISCV::FPR32RegClass);
514 Opc = RISCV::FSGNJ_S;
515 }
519 return;
520 }
521
522 if (RISCV::FPR32RegClass.contains(DstReg, SrcReg)) {
526 return;
527 }
528
529 if (RISCV::FPR64RegClass.contains(DstReg, SrcReg)) {
533 return;
534 }
535
536 if (RISCV::FPR32RegClass.contains(DstReg) &&
537 RISCV::GPRRegClass.contains(SrcReg)) {
540 return;
541 }
542
543 if (RISCV::GPRRegClass.contains(DstReg) &&
544 RISCV::FPR32RegClass.contains(SrcReg)) {
547 return;
548 }
549
550 if (RISCV::FPR64RegClass.contains(DstReg) &&
551 RISCV::GPRRegClass.contains(SrcReg)) {
555 return;
556 }
557
558 if (RISCV::GPRRegClass.contains(DstReg) &&
559 RISCV::FPR64RegClass.contains(SrcReg)) {
563 return;
564 }
565
566
568 TRI->getCommonMinimalPhysRegClass(SrcReg, DstReg);
571 return;
572 }
573
575}
576
579 Register SrcReg, bool IsKill, int FI,
586
587 unsigned Opcode;
588 bool IsScalableVector = true;
589 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
590 Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
591 RISCV::SW : RISCV::SD;
592 IsScalableVector = false;
593 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
594 Opcode = RISCV::SH_INX;
595 IsScalableVector = false;
596 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
597 Opcode = RISCV::SW_INX;
598 IsScalableVector = false;
599 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
600 Opcode = RISCV::PseudoRV32ZdinxSD;
601 IsScalableVector = false;
602 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
603 Opcode = RISCV::FSH;
604 IsScalableVector = false;
605 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
606 Opcode = RISCV::FSW;
607 IsScalableVector = false;
608 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
609 Opcode = RISCV::FSD;
610 IsScalableVector = false;
611 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
612 Opcode = RISCV::VS1R_V;
613 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
614 Opcode = RISCV::VS2R_V;
615 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
616 Opcode = RISCV::VS4R_V;
617 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
618 Opcode = RISCV::VS8R_V;
619 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
620 Opcode = RISCV::PseudoVSPILL2_M1;
621 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
622 Opcode = RISCV::PseudoVSPILL2_M2;
623 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
624 Opcode = RISCV::PseudoVSPILL2_M4;
625 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
626 Opcode = RISCV::PseudoVSPILL3_M1;
627 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
628 Opcode = RISCV::PseudoVSPILL3_M2;
629 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
630 Opcode = RISCV::PseudoVSPILL4_M1;
631 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
632 Opcode = RISCV::PseudoVSPILL4_M2;
633 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
634 Opcode = RISCV::PseudoVSPILL5_M1;
635 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
636 Opcode = RISCV::PseudoVSPILL6_M1;
637 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
638 Opcode = RISCV::PseudoVSPILL7_M1;
639 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
640 Opcode = RISCV::PseudoVSPILL8_M1;
641 else
643
644 if (IsScalableVector) {
648
654 } else {
658
664 }
665}
666
673
674 unsigned Opcode;
675 bool IsScalableVector = true;
676 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
677 Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
678 RISCV::LW : RISCV::LD;
679 IsScalableVector = false;
680 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
681 Opcode = RISCV::LH_INX;
682 IsScalableVector = false;
683 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
684 Opcode = RISCV::LW_INX;
685 IsScalableVector = false;
686 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
687 Opcode = RISCV::PseudoRV32ZdinxLD;
688 IsScalableVector = false;
689 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
690 Opcode = RISCV::FLH;
691 IsScalableVector = false;
692 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
693 Opcode = RISCV::FLW;
694 IsScalableVector = false;
695 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
696 Opcode = RISCV::FLD;
697 IsScalableVector = false;
698 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
699 Opcode = RISCV::VL1RE8_V;
700 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
701 Opcode = RISCV::VL2RE8_V;
702 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
703 Opcode = RISCV::VL4RE8_V;
704 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
705 Opcode = RISCV::VL8RE8_V;
706 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
707 Opcode = RISCV::PseudoVRELOAD2_M1;
708 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
709 Opcode = RISCV::PseudoVRELOAD2_M2;
710 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
711 Opcode = RISCV::PseudoVRELOAD2_M4;
712 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
713 Opcode = RISCV::PseudoVRELOAD3_M1;
714 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
715 Opcode = RISCV::PseudoVRELOAD3_M2;
716 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
717 Opcode = RISCV::PseudoVRELOAD4_M1;
718 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
719 Opcode = RISCV::PseudoVRELOAD4_M2;
720 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
721 Opcode = RISCV::PseudoVRELOAD5_M1;
722 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
723 Opcode = RISCV::PseudoVRELOAD6_M1;
724 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
725 Opcode = RISCV::PseudoVRELOAD7_M1;
726 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
727 Opcode = RISCV::PseudoVRELOAD8_M1;
728 else
729 llvm_unreachable("Can't load this register from stack slot");
730
731 if (IsScalableVector) {
735
740 } else {
744
749 }
750}
751
756
757
758
760 return nullptr;
761
762
763 if (Ops.size() != 1 || Ops[0] != 1)
764 return nullptr;
765
766 unsigned LoadOpc;
767 switch (MI.getOpcode()) {
768 default:
770 LoadOpc = RISCV::LW;
771 break;
772 }
774 LoadOpc = RISCV::LWU;
775 break;
776 }
778 LoadOpc = RISCV::LBU;
779 break;
780 }
782 unsigned Log2SEW =
785 return nullptr;
786 switch (Log2SEW) {
787 case 3:
788 LoadOpc = RISCV::LB;
789 break;
790 case 4:
791 LoadOpc = RISCV::LH;
792 break;
793 case 5:
794 LoadOpc = RISCV::LW;
795 break;
796 case 6:
797 LoadOpc = RISCV::LD;
798 break;
799 default:
801 }
802 break;
803 }
805 unsigned Log2SEW =
807 switch (Log2SEW) {
808 case 4:
809 LoadOpc = RISCV::FLH;
810 break;
811 case 5:
812 LoadOpc = RISCV::FLW;
813 break;
814 case 6:
815 LoadOpc = RISCV::FLD;
816 break;
817 default:
819 }
820 break;
821 }
822 return nullptr;
823 case RISCV::SEXT_H:
824 LoadOpc = RISCV::LH;
825 break;
826 case RISCV::SEXT_B:
827 LoadOpc = RISCV::LB;
828 break;
829 case RISCV::ZEXT_H_RV32:
830 case RISCV::ZEXT_H_RV64:
831 LoadOpc = RISCV::LHU;
832 break;
833 }
834
835 Register DstReg = MI.getOperand(0).getReg();
836 return BuildMI(*MI.getParent(), InsertPt, MI.getDebugLoc(), get(LoadOpc),
837 DstReg)
840}
841
846 bool DstIsDead) const {
848
849
850 if (.is64Bit() && !isInt<32>(Val)) {
851
852 if (!isUInt<32>(Val))
853 report_fatal_error("Should only materialize 32-bit constants for RV32");
854
855
856 Val = SignExtend64<32>(Val);
857 }
858
861
862 bool SrcRenamable = false;
863 unsigned Num = 0;
864
866 bool LastItem = ++Num == Seq.size();
867 unsigned DstRegState = getDeadRegState(DstIsDead && LastItem) |
869 unsigned SrcRegState = getKillRegState(SrcReg != RISCV::X0) |
871 switch (Inst.getOpndKind()) {
875 .addImm(Inst.getImm())
877 break;
881 .addReg(SrcReg, SrcRegState)
884 break;
888 .addReg(SrcReg, SrcRegState)
889 .addReg(SrcReg, SrcRegState)
891 break;
895 .addReg(SrcReg, SrcRegState)
896 .addImm(Inst.getImm())
898 break;
899 }
900
901
902 SrcReg = DstReg;
903 SrcRenamable = DstRenamable;
904 }
905}
906
908 switch (Opc) {
909 default:
911 case RISCV::CV_BEQIMM:
913 case RISCV::CV_BNEIMM:
915 case RISCV::BEQ:
917 case RISCV::BNE:
919 case RISCV::BLT:
921 case RISCV::BGE:
923 case RISCV::BLTU:
925 case RISCV::BGEU:
927 }
928}
929
930
931
932
935
937 "Unknown conditional branch");
943}
944
946 switch (CC) {
947 default:
950 return Imm ? RISCV::CV_BEQIMM : RISCV::BEQ;
952 return Imm ? RISCV::CV_BNEIMM : RISCV::BNE;
954 return RISCV::BLT;
956 return RISCV::BGE;
958 return RISCV::BLTU;
960 return RISCV::BGEU;
961 }
962}
963
965 bool Imm) const {
967}
968
970 switch (CC) {
971 default:
985 }
986}
987
992 bool AllowModify) const {
993 TBB = FBB = nullptr;
994 Cond.clear();
995
996
998 if (I == MBB.end() || !isUnpredicatedTerminator(*I))
999 return false;
1000
1001
1002
1004 int NumTerminators = 0;
1005 for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(*J);
1006 J++) {
1007 NumTerminators++;
1008 if (J->getDesc().isUnconditionalBranch() ||
1009 J->getDesc().isIndirectBranch()) {
1010 FirstUncondOrIndirectBr = J.getReverse();
1011 }
1012 }
1013
1014
1015
1016 if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) {
1017 while (std::next(FirstUncondOrIndirectBr) != MBB.end()) {
1018 std::next(FirstUncondOrIndirectBr)->eraseFromParent();
1019 NumTerminators--;
1020 }
1021 I = FirstUncondOrIndirectBr;
1022 }
1023
1024
1025 if (I->getDesc().isIndirectBranch())
1026 return true;
1027
1028
1029 if (I->isPreISelOpcode())
1030 return true;
1031
1032
1033 if (NumTerminators > 2)
1034 return true;
1035
1036
1037 if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) {
1039 return false;
1040 }
1041
1042
1043 if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) {
1045 return false;
1046 }
1047
1048
1049 if (NumTerminators == 2 && std::prev(I)->getDesc().isConditionalBranch() &&
1050 I->getDesc().isUnconditionalBranch()) {
1053 return false;
1054 }
1055
1056
1057 return true;
1058}
1059
1061 int *BytesRemoved) const {
1062 if (BytesRemoved)
1063 *BytesRemoved = 0;
1066 return 0;
1067
1068 if (->getDesc().isUnconditionalBranch() &&
1069 ->getDesc().isConditionalBranch())
1070 return 0;
1071
1072
1073 if (BytesRemoved)
1075 I->eraseFromParent();
1076
1078
1080 return 1;
1081 --I;
1082 if (->getDesc().isConditionalBranch())
1083 return 1;
1084
1085
1086 if (BytesRemoved)
1088 I->eraseFromParent();
1089 return 2;
1090}
1091
1092
1093
1097 if (BytesAdded)
1098 *BytesAdded = 0;
1099
1100
1101 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1103 "RISC-V branch conditions have two components!");
1104
1105
1106 if (Cond.empty()) {
1108 if (BytesAdded)
1110 return 1;
1111 }
1112
1113
1119 if (BytesAdded)
1121
1122
1123 if (!FBB)
1124 return 1;
1125
1126
1128 if (BytesAdded)
1130 return 2;
1131}
1132
1136 const DebugLoc &DL, int64_t BrOffset,
1138 assert(RS && "RegScavenger required for long branching");
1140 "new block should be inserted for expanding unconditional branch");
1143 "restore block should be inserted for restoring clobbered registers");
1144
1149
1150 if (!isInt<32>(BrOffset))
1152 "Branch offsets outside of the signed 32-bit range not supported");
1153
1154
1155
1156
1157 Register ScratchReg = MRI.createVirtualRegister(&RISCV::GPRJALRRegClass);
1159
1163
1167 false, 0,
1168 false);
1169 if (TmpGPR != RISCV::NoRegister)
1171 else {
1172
1173
1174
1175 TmpGPR = RISCV::X27;
1176
1178 if (FrameIndex == -1)
1180
1183 TRI->eliminateFrameIndex(std::prev(MI.getIterator()),
1184 0, 1);
1185
1186 MI.getOperand(1).setMBB(&RestoreBB);
1187
1190 TRI->eliminateFrameIndex(RestoreBB.back(),
1191 0, 1);
1192 }
1193
1194 MRI.replaceRegWith(ScratchReg, TmpGPR);
1195 MRI.clearVirtRegs();
1196}
1197
1200 assert((Cond.size() == 3) && "Invalid branch condition!");
1203 return false;
1204}
1205
1209
1213 return false;
1214
1217
1219 return false;
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240 auto isLoadImm = [](const MachineInstr *MI, int64_t &Imm) -> bool {
1241 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(1).isReg() &&
1242 MI->getOperand(1).getReg() == RISCV::X0) {
1243 Imm = MI->getOperand(2).getImm();
1244 return true;
1245 }
1246 return false;
1247 };
1248
1249 auto isFromLoadImm = [&](const MachineOperand &Op, int64_t &Imm) -> bool {
1250 if (.isReg())
1251 return false;
1253 return Reg.isVirtual() && isLoadImm(MRI.getVRegDef(Reg), Imm);
1254 };
1255
1258
1259
1260 auto searchConst = [&](int64_t C1) -> Register {
1262 auto DefC1 = std::find_if(++II, E, [&](const MachineInstr &I) -> bool {
1263 int64_t Imm;
1264 return isLoadImm(&I, Imm) && Imm == C1 &&
1265 I.getOperand(0).getReg().isVirtual();
1266 });
1267 if (DefC1 != E)
1268 return DefC1->getOperand(0).getReg();
1269
1271 };
1272
1273 bool Modify = false;
1274 int64_t C0;
1275 if (isFromLoadImm(LHS, C0) && MRI.hasOneUse(LHS.getReg())) {
1276
1277
1278
1280 if (Register RegZ = searchConst(C0 + 1)) {
1284
1285
1286 MRI.clearKillFlags(RegZ);
1287 Modify = true;
1288 }
1289 } else if (isFromLoadImm(RHS, C0) && MRI.hasOneUse(RHS.getReg())) {
1290
1291
1292
1294 if (Register RegZ = searchConst(C0 - 1)) {
1298
1299
1300 MRI.clearKillFlags(RegZ);
1301 Modify = true;
1302 }
1303 }
1304
1305 if (!Modify)
1306 return false;
1307
1308
1314 MI.eraseFromParent();
1315
1316 return true;
1317}
1318
1321 assert(MI.getDesc().isBranch() && "Unexpected opcode!");
1322
1323 int NumOp = MI.getNumExplicitOperands();
1324 return MI.getOperand(NumOp - 1).getMBB();
1325}
1326
1328 int64_t BrOffset) const {
1330
1331
1332
1333 switch (BranchOp) {
1334 default:
1336 case RISCV::BEQ:
1337 case RISCV::BNE:
1338 case RISCV::BLT:
1339 case RISCV::BGE:
1340 case RISCV::BLTU:
1341 case RISCV::BGEU:
1342 case RISCV::CV_BEQIMM:
1343 case RISCV::CV_BNEIMM:
1344 return isIntN(13, BrOffset);
1345 case RISCV::JAL:
1346 case RISCV::PseudoBR:
1347 return isIntN(21, BrOffset);
1348 case RISCV::PseudoJump:
1350 }
1351}
1352
1353
1354
1355
1357 switch (Opcode) {
1358 case RISCV::ADD: return RISCV::PseudoCCADD; break;
1359 case RISCV::SUB: return RISCV::PseudoCCSUB; break;
1360 case RISCV::SLL: return RISCV::PseudoCCSLL; break;
1361 case RISCV::SRL: return RISCV::PseudoCCSRL; break;
1362 case RISCV::SRA: return RISCV::PseudoCCSRA; break;
1363 case RISCV::AND: return RISCV::PseudoCCAND; break;
1364 case RISCV::OR: return RISCV::PseudoCCOR; break;
1365 case RISCV::XOR: return RISCV::PseudoCCXOR; break;
1366
1367 case RISCV::ADDI: return RISCV::PseudoCCADDI; break;
1368 case RISCV::SLLI: return RISCV::PseudoCCSLLI; break;
1369 case RISCV::SRLI: return RISCV::PseudoCCSRLI; break;
1370 case RISCV::SRAI: return RISCV::PseudoCCSRAI; break;
1371 case RISCV::ANDI: return RISCV::PseudoCCANDI; break;
1372 case RISCV::ORI: return RISCV::PseudoCCORI; break;
1373 case RISCV::XORI: return RISCV::PseudoCCXORI; break;
1374
1375 case RISCV::ADDW: return RISCV::PseudoCCADDW; break;
1376 case RISCV::SUBW: return RISCV::PseudoCCSUBW; break;
1377 case RISCV::SLLW: return RISCV::PseudoCCSLLW; break;
1378 case RISCV::SRLW: return RISCV::PseudoCCSRLW; break;
1379 case RISCV::SRAW: return RISCV::PseudoCCSRAW; break;
1380
1381 case RISCV::ADDIW: return RISCV::PseudoCCADDIW; break;
1382 case RISCV::SLLIW: return RISCV::PseudoCCSLLIW; break;
1383 case RISCV::SRLIW: return RISCV::PseudoCCSRLIW; break;
1384 case RISCV::SRAIW: return RISCV::PseudoCCSRAIW; break;
1385
1386 case RISCV::ANDN: return RISCV::PseudoCCANDN; break;
1387 case RISCV::ORN: return RISCV::PseudoCCORN; break;
1388 case RISCV::XNOR: return RISCV::PseudoCCXNOR; break;
1389 }
1390
1391 return RISCV::INSTRUCTION_LIST_END;
1392}
1393
1394
1395
1399 if (!Reg.isVirtual())
1400 return nullptr;
1401 if (.hasOneNonDBGUse(Reg))
1402 return nullptr;
1404 if ()
1405 return nullptr;
1406
1408 return nullptr;
1409
1410 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(1).isReg() &&
1411 MI->getOperand(1).getReg() == RISCV::X0)
1412 return nullptr;
1413
1415
1416 if (MO.isFI() || MO.isCPI() || MO.isJTI())
1417 return nullptr;
1418 if (!MO.isReg())
1419 continue;
1420
1421 if (MO.isTied())
1422 return nullptr;
1423 if (MO.isDef())
1424 return nullptr;
1425
1426 if (MO.getReg().isPhysical() && .isConstantPhysReg(MO.getReg()))
1427 return nullptr;
1428 }
1429 bool DontMoveAcrossStores = true;
1430 if (->isSafeToMove(DontMoveAcrossStores))
1431 return nullptr;
1432 return MI;
1433}
1434
1437 unsigned &TrueOp, unsigned &FalseOp,
1438 bool &Optimizable) const {
1439 assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1440 "Unknown select instruction");
1441
1442
1443
1444
1445
1446
1447
1448 TrueOp = 5;
1449 FalseOp = 4;
1450 Cond.push_back(MI.getOperand(1));
1451 Cond.push_back(MI.getOperand(2));
1452 Cond.push_back(MI.getOperand(3));
1453
1454 Optimizable = STI.hasShortForwardBranchOpt();
1455 return false;
1456}
1457
1461 bool PreferFalse) const {
1462 assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1463 "Unknown select instruction");
1464 if (.hasShortForwardBranchOpt())
1465 return nullptr;
1466
1470 bool Invert = ;
1474 return nullptr;
1475
1476
1478 Register DestReg = MI.getOperand(0).getReg();
1480 if (.constrainRegClass(DestReg, PreviousClass))
1481 return nullptr;
1482
1484 assert(PredOpc != RISCV::INSTRUCTION_LIST_END && "Unexpected opcode!");
1485
1486
1488 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(PredOpc), DestReg);
1489
1490
1491 NewMI.add(MI.getOperand(1));
1492 NewMI.add(MI.getOperand(2));
1493
1494
1496 if (Invert)
1499
1500
1501 NewMI.add(FalseReg);
1502
1503
1505 for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
1507
1508
1509 SeenMIs.insert(NewMI);
1511
1512
1513
1514
1515
1518
1519
1521 return NewMI;
1522}
1523
1525 if (MI.isMetaInstruction())
1526 return 0;
1527
1528 unsigned Opcode = MI.getOpcode();
1529
1530 if (Opcode == TargetOpcode::INLINEASM ||
1531 Opcode == TargetOpcode::INLINEASM_BR) {
1533 return getInlineAsmLength(MI.getOperand(0).getSymbolName(),
1535 }
1536
1537 if (.memoperands_empty()) {
1541 if (isCompressibleInst(MI, STI))
1542 return 4;
1543 return 6;
1544 }
1545 return 8;
1546 }
1547 }
1548
1549 if (Opcode == TargetOpcode::BUNDLE)
1550 return getInstBundleLength(MI);
1551
1552 if (MI.getParent() && MI.getParent()->getParent()) {
1553 if (isCompressibleInst(MI, STI))
1554 return 2;
1555 }
1556
1557 switch (Opcode) {
1558 case RISCV::PseudoMV_FPR16INX:
1559 case RISCV::PseudoMV_FPR32INX:
1560
1562 case TargetOpcode::STACKMAP:
1563
1565 case TargetOpcode::PATCHPOINT:
1566
1568 case TargetOpcode::STATEPOINT: {
1569
1571
1572 return std::max(NumBytes, 8U);
1573 }
1574 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1575 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1576 case TargetOpcode::PATCHABLE_TAIL_CALL: {
1579 if (Opcode == TargetOpcode::PATCHABLE_FUNCTION_ENTER &&
1580 F.hasFnAttribute("patchable-function-entry")) {
1581 unsigned Num;
1582 if (F.getFnAttribute("patchable-function-entry")
1583 .getValueAsString()
1584 .getAsInteger(10, Num))
1585 return get(Opcode).getSize();
1586
1587
1589 }
1590
1591
1593 }
1594 default:
1595 return get(Opcode).getSize();
1596 }
1597}
1598
1599unsigned RISCVInstrInfo::getInstBundleLength(const MachineInstr &MI) const {
1600 unsigned Size = 0;
1603 while (++I != E && I->isInsideBundle()) {
1604 assert(->isBundle() && "No nested bundle!");
1606 }
1607 return Size;
1608}
1609
1611 const unsigned Opcode = MI.getOpcode();
1612 switch (Opcode) {
1613 default:
1614 break;
1615 case RISCV::FSGNJ_D:
1616 case RISCV::FSGNJ_S:
1617 case RISCV::FSGNJ_H:
1618 case RISCV::FSGNJ_D_INX:
1619 case RISCV::FSGNJ_D_IN32X:
1620 case RISCV::FSGNJ_S_INX:
1621 case RISCV::FSGNJ_H_INX:
1622
1623 return MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
1624 MI.getOperand(1).getReg() == MI.getOperand(2).getReg();
1625 case RISCV::ADDI:
1626 case RISCV::ORI:
1627 case RISCV::XORI:
1628 return (MI.getOperand(1).isReg() &&
1629 MI.getOperand(1).getReg() == RISCV::X0) ||
1630 (MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0);
1631 }
1632 return MI.isAsCheapAsAMove();
1633}
1634
1635std::optional
1637 if (MI.isMoveReg())
1639 switch (MI.getOpcode()) {
1640 default:
1641 break;
1642 case RISCV::ADDI:
1643
1644 if (MI.getOperand(1).isReg() && MI.getOperand(2).isImm() &&
1645 MI.getOperand(2).getImm() == 0)
1647 break;
1648 case RISCV::FSGNJ_D:
1649 case RISCV::FSGNJ_S:
1650 case RISCV::FSGNJ_H:
1651 case RISCV::FSGNJ_D_INX:
1652 case RISCV::FSGNJ_D_IN32X:
1653 case RISCV::FSGNJ_S_INX:
1654 case RISCV::FSGNJ_H_INX:
1655
1656 if (MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
1657 MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
1659 break;
1660 }
1661 return std::nullopt;
1662}
1663
1666
1667
1668
1669 const auto &SchedModel = STI.getSchedModel();
1670 return (!SchedModel.hasInstrSchedModel() || SchedModel.isOutOfOrder())
1673 }
1674
1676}
1677
1681 int16_t FrmOpIdx =
1683 if (FrmOpIdx < 0) {
1687 RISCV::OpName::frm) < 0;
1688 }) &&
1689 "New instructions require FRM whereas the old one does not have it");
1690 return;
1691 }
1692
1695
1696 for (auto *NewMI : InsInstrs) {
1697
1699 NewMI->getOpcode(), RISCV::OpName::frm)) != NewMI->getNumOperands())
1700 continue;
1702 MIB.add(FRM);
1705 }
1706}
1707
1709 switch (Opc) {
1710 default:
1711 return false;
1712 case RISCV::FADD_H:
1713 case RISCV::FADD_S:
1714 case RISCV::FADD_D:
1715 return true;
1716 }
1717}
1718
1720 switch (Opc) {
1721 default:
1722 return false;
1723 case RISCV::FSUB_H:
1724 case RISCV::FSUB_S:
1725 case RISCV::FSUB_D:
1726 return true;
1727 }
1728}
1729
1731 switch (Opc) {
1732 default:
1733 return false;
1734 case RISCV::FMUL_H:
1735 case RISCV::FMUL_S:
1736 case RISCV::FMUL_D:
1737 return true;
1738 }
1739}
1740
1741bool RISCVInstrInfo::isVectorAssociativeAndCommutative(const MachineInstr &Inst,
1742 bool Invert) const {
1743#define OPCODE_LMUL_CASE(OPC) \
1744 case RISCV::OPC##_M1: \
1745 case RISCV::OPC##_M2: \
1746 case RISCV::OPC##_M4: \
1747 case RISCV::OPC##_M8: \
1748 case RISCV::OPC##_MF2: \
1749 case RISCV::OPC##_MF4: \
1750 case RISCV::OPC##_MF8
1751
1752#define OPCODE_LMUL_MASK_CASE(OPC) \
1753 case RISCV::OPC##_M1_MASK: \
1754 case RISCV::OPC##_M2_MASK: \
1755 case RISCV::OPC##_M4_MASK: \
1756 case RISCV::OPC##_M8_MASK: \
1757 case RISCV::OPC##_MF2_MASK: \
1758 case RISCV::OPC##_MF4_MASK: \
1759 case RISCV::OPC##_MF8_MASK
1760
1761 unsigned Opcode = Inst.getOpcode();
1762 if (Invert) {
1764 Opcode = *InvOpcode;
1765 else
1766 return false;
1767 }
1768
1769
1770 switch (Opcode) {
1771 default:
1772 return false;
1777 return true;
1778 }
1779
1780
1781#undef OPCODE_LMUL_MASK_CASE
1782#undef OPCODE_LMUL_CASE
1783}
1784
1785bool RISCVInstrInfo::areRVVInstsReassociable(const MachineInstr &Root,
1788 return false;
1789
1793
1794
1797
1798 auto checkImmOperand = [&](unsigned OpIdx) {
1800 };
1801
1802 auto checkRegOperand = [&](unsigned OpIdx) {
1804 };
1805
1806
1807
1808
1809
1810 if (!checkRegOperand(1))
1811 return false;
1812
1813
1816 return false;
1817
1818
1824
1825 bool SeenMI2 = false;
1826 for (auto End = MBB->rend(), It = It1; It != End; ++It) {
1827 if (It == It2) {
1828 SeenMI2 = true;
1830
1831
1832 break;
1833 }
1834
1835 if (It->modifiesRegister(RISCV::V0, TRI)) {
1836 Register SrcReg = It->getOperand(1).getReg();
1837
1838
1840 return false;
1841
1842 if (!MI1VReg.isValid()) {
1843
1844 MI1VReg = SrcReg;
1845 continue;
1846 }
1847
1848
1849 if (!SeenMI2)
1850 continue;
1851
1852
1853
1854 if (MI1VReg != SrcReg)
1855 return false;
1856 else
1857 break;
1858 }
1859 }
1860
1861
1862
1863 assert(SeenMI2 && "Prev is expected to appear before Root");
1864 }
1865
1866
1869 return false;
1870
1871
1877 return false;
1878 switch (Op1.getType()) {
1881 return false;
1882 break;
1885 return false;
1886 break;
1887 default:
1889 }
1890 }
1891
1892
1895 return false;
1896
1897 return true;
1898}
1899
1900
1901
1902bool RISCVInstrInfo::hasReassociableVectorSibling(const MachineInstr &Inst,
1903 bool &Commuted) const {
1907 "Expect the present of passthrough operand.");
1910
1911
1912
1913 Commuted = !areRVVInstsReassociable(Inst, *MI1) &&
1914 areRVVInstsReassociable(Inst, *MI2);
1915 if (Commuted)
1917
1918 return areRVVInstsReassociable(Inst, *MI1) &&
1919 (isVectorAssociativeAndCommutative(*MI1) ||
1920 isVectorAssociativeAndCommutative(*MI1, true)) &&
1923}
1924
1927 if (!isVectorAssociativeAndCommutative(Inst) &&
1928 !isVectorAssociativeAndCommutative(Inst, true))
1930
1934
1935
1936
1940 MI1 = MRI.getUniqueVRegDef(Op1.getReg());
1942 MI2 = MRI.getUniqueVRegDef(Op2.getReg());
1943
1944
1946}
1947
1953
1954 for (unsigned I = 0; I < 5; ++I)
1956 }
1957}
1958
1960 bool &Commuted) const {
1961 if (isVectorAssociativeAndCommutative(Inst) ||
1962 isVectorAssociativeAndCommutative(Inst, true))
1963 return hasReassociableVectorSibling(Inst, Commuted);
1964
1966 return false;
1967
1969 unsigned OperandIdx = Commuted ? 2 : 1;
1972
1973 int16_t InstFrmOpIdx =
1975 int16_t SiblingFrmOpIdx =
1977
1978 return (InstFrmOpIdx < 0 && SiblingFrmOpIdx < 0) ||
1980}
1981
1983 bool Invert) const {
1984 if (isVectorAssociativeAndCommutative(Inst, Invert))
1985 return true;
1986
1987 unsigned Opc = Inst.getOpcode();
1988 if (Invert) {
1990 if (!InverseOpcode)
1991 return false;
1992 Opc = *InverseOpcode;
1993 }
1994
1998
1999 switch (Opc) {
2000 default:
2001 return false;
2002 case RISCV::ADD:
2003 case RISCV::ADDW:
2004 case RISCV::AND:
2005 case RISCV::OR:
2006 case RISCV::XOR:
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019 case RISCV::MUL:
2020 case RISCV::MULW:
2021 case RISCV::MIN:
2022 case RISCV::MINU:
2023 case RISCV::MAX:
2024 case RISCV::MAXU:
2025 case RISCV::FMIN_H:
2026 case RISCV::FMIN_S:
2027 case RISCV::FMIN_D:
2028 case RISCV::FMAX_H:
2029 case RISCV::FMAX_S:
2030 case RISCV::FMAX_D:
2031 return true;
2032 }
2033
2034 return false;
2035}
2036
2037std::optional
2039#define RVV_OPC_LMUL_CASE(OPC, INV) \
2040 case RISCV::OPC##_M1: \
2041 return RISCV::INV##_M1; \
2042 case RISCV::OPC##_M2: \
2043 return RISCV::INV##_M2; \
2044 case RISCV::OPC##_M4: \
2045 return RISCV::INV##_M4; \
2046 case RISCV::OPC##_M8: \
2047 return RISCV::INV##_M8; \
2048 case RISCV::OPC##_MF2: \
2049 return RISCV::INV##_MF2; \
2050 case RISCV::OPC##_MF4: \
2051 return RISCV::INV##_MF4; \
2052 case RISCV::OPC##_MF8: \
2053 return RISCV::INV##_MF8
2054
2055#define RVV_OPC_LMUL_MASK_CASE(OPC, INV) \
2056 case RISCV::OPC##_M1_MASK: \
2057 return RISCV::INV##_M1_MASK; \
2058 case RISCV::OPC##_M2_MASK: \
2059 return RISCV::INV##_M2_MASK; \
2060 case RISCV::OPC##_M4_MASK: \
2061 return RISCV::INV##_M4_MASK; \
2062 case RISCV::OPC##_M8_MASK: \
2063 return RISCV::INV##_M8_MASK; \
2064 case RISCV::OPC##_MF2_MASK: \
2065 return RISCV::INV##_MF2_MASK; \
2066 case RISCV::OPC##_MF4_MASK: \
2067 return RISCV::INV##_MF4_MASK; \
2068 case RISCV::OPC##_MF8_MASK: \
2069 return RISCV::INV##_MF8_MASK
2070
2071 switch (Opcode) {
2072 default:
2073 return std::nullopt;
2074 case RISCV::FADD_H:
2075 return RISCV::FSUB_H;
2076 case RISCV::FADD_S:
2077 return RISCV::FSUB_S;
2078 case RISCV::FADD_D:
2079 return RISCV::FSUB_D;
2080 case RISCV::FSUB_H:
2081 return RISCV::FADD_H;
2082 case RISCV::FSUB_S:
2083 return RISCV::FADD_S;
2084 case RISCV::FSUB_D:
2085 return RISCV::FADD_D;
2086 case RISCV::ADD:
2087 return RISCV::SUB;
2088 case RISCV::SUB:
2089 return RISCV::ADD;
2090 case RISCV::ADDW:
2091 return RISCV::SUBW;
2092 case RISCV::SUBW:
2093 return RISCV::ADDW;
2094
2099
2100 }
2101
2102#undef RVV_OPC_LMUL_MASK_CASE
2103#undef RVV_OPC_LMUL_CASE
2104}
2105
2108 bool DoRegPressureReduce) {
2110 return false;
2113 if ( ||
(MI->getOpcode()))
2114 return false;
2115
2118 return false;
2119
2120
2121
2122
2123
2124 if (DoRegPressureReduce && .hasOneNonDBGUse(MI->getOperand(0).getReg()))
2125 return false;
2126
2127
2128 if (Root.getParent() != MI->getParent())
2129 return false;
2131}
2132
2135 bool DoRegPressureReduce) {
2136 unsigned Opc = Root.getOpcode();
2137 bool IsFAdd = isFADD(Opc);
2138 if (!IsFAdd && (Opc))
2139 return false;
2140 bool Added = false;
2142 DoRegPressureReduce)) {
2145 Added = true;
2146 }
2148 DoRegPressureReduce)) {
2151 Added = true;
2152 }
2153 return Added;
2154}
2155
2158 bool DoRegPressureReduce) {
2160}
2161
2162
2163
2166 unsigned CombineOpc) {
2169
2172
2173 if ( || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
2174 return nullptr;
2175
2176 if (.hasOneNonDBGUse(MI->getOperand(0).getReg()))
2177 return nullptr;
2178
2179 return MI;
2180}
2181
2182
2183
2184
2187 unsigned OuterShiftAmt) {
2189 if (!ShiftMI)
2190 return false;
2191
2193 if (InnerShiftAmt < OuterShiftAmt || (InnerShiftAmt - OuterShiftAmt) > 3)
2194 return false;
2195
2196 return true;
2197}
2198
2199
2200
2202 switch (Opc) {
2203 default:
2204 return 0;
2205 case RISCV::SH1ADD:
2206 return 1;
2207 case RISCV::SH2ADD:
2208 return 2;
2209 case RISCV::SH3ADD:
2210 return 3;
2211 }
2212}
2213
2214
2215
2219 if (!ShiftAmt)
2220 return false;
2221
2223
2225 if (!AddMI)
2226 return false;
2227
2228 bool Found = false;
2231 Found = true;
2232 }
2235 Found = true;
2236 }
2237
2238 return Found;
2239}
2240
2248 default:
2250 }
2251}
2252
2255 bool DoRegPressureReduce) const {
2256
2257 if (getFPPatterns(Root, Patterns, DoRegPressureReduce))
2258 return true;
2259
2261 return true;
2262
2264 DoRegPressureReduce);
2265}
2266
2268 switch (RootOpc) {
2269 default:
2271 case RISCV::FADD_H:
2272 return RISCV::FMADD_H;
2273 case RISCV::FADD_S:
2274 return RISCV::FMADD_S;
2275 case RISCV::FADD_D:
2276 return RISCV::FMADD_D;
2277 case RISCV::FSUB_H:
2279 : RISCV::FNMSUB_H;
2280 case RISCV::FSUB_S:
2282 : RISCV::FNMSUB_S;
2283 case RISCV::FSUB_D:
2285 : RISCV::FNMSUB_D;
2286 }
2287}
2288
2291 default:
2295 return 2;
2298 return 1;
2299 }
2300}
2301
2309
2314
2315 Register DstReg = Dst.getReg();
2320
2321 bool Mul1IsKill = Mul1.isKill();
2322 bool Mul2IsKill = Mul2.isKill();
2323 bool AddendIsKill = Addend.isKill();
2324
2325
2326
2327
2328 MRI.clearKillFlags(Mul1.getReg());
2329 MRI.clearKillFlags(Mul2.getReg());
2330
2332 BuildMI(*MF, MergedLoc, TII->get(FusedOpc), DstReg)
2337
2342}
2343
2344
2345
2346
2347static void
2355
2357 assert(OuterShiftAmt != 0 && "Unexpected opcode");
2358
2362
2364 assert(InnerShiftAmt >= OuterShiftAmt && "Unexpected shift amount");
2365
2366 unsigned InnerOpc;
2367 switch (InnerShiftAmt - OuterShiftAmt) {
2368 default:
2370 case 0:
2371 InnerOpc = RISCV::ADD;
2372 break;
2373 case 1:
2374 InnerOpc = RISCV::SH1ADD;
2375 break;
2376 case 2:
2377 InnerOpc = RISCV::SH2ADD;
2378 break;
2379 case 3:
2380 InnerOpc = RISCV::SH3ADD;
2381 break;
2382 }
2383
2387
2388 Register NewVR = MRI.createVirtualRegister(&RISCV::GPRRegClass);
2389
2397
2398 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
2404}
2405
2413 default:
2415 DelInstrs, InstrIdxForVirtReg);
2416 return;
2421 return;
2422 }
2427 return;
2428 }
2430 genShXAddAddShift(Root, 1, InsInstrs, DelInstrs, InstrIdxForVirtReg);
2431 return;
2433 genShXAddAddShift(Root, 2, InsInstrs, DelInstrs, InstrIdxForVirtReg);
2434 return;
2435 }
2436}
2437
2441
2442 for (const auto &[Index, Operand] : enumerate(Desc.operands())) {
2443 unsigned OpType = Operand.OperandType;
2447 if (MO.isReg()) {
2448 ErrInfo = "Expected a non-register operand.";
2449 return false;
2450 }
2451 if (MO.isImm()) {
2452 int64_t Imm = MO.getImm();
2453 bool Ok;
2454 switch (OpType) {
2455 default:
2457
2458
2459#define CASE_OPERAND_UIMM(NUM) \
2460 case RISCVOp::OPERAND_UIMM##NUM: \
2461 Ok = isUInt(Imm); \
2462 break;
2463#define CASE_OPERAND_SIMM(NUM) \
2464 case RISCVOp::OPERAND_SIMM##NUM: \
2465 Ok = isInt(Imm); \
2466 break;
2478
2480 Ok = isShiftedUInt<1, 1>(Imm);
2481 break;
2483 Ok = isShiftedUInt<4, 1>(Imm);
2484 break;
2486 Ok = isShiftedUInt<5, 1>(Imm);
2487 break;
2489 Ok = isShiftedUInt<5, 2>(Imm);
2490 break;
2492 Ok = isShiftedUInt<6, 2>(Imm);
2493 break;
2495 Ok = isShiftedUInt<5, 3>(Imm);
2496 break;
2498 Ok = isUInt<8>(Imm) && Imm >= 32;
2499 break;
2501 Ok = isShiftedUInt<6, 3>(Imm);
2502 break;
2504 Ok = isShiftedInt<6, 4>(Imm) && (Imm != 0);
2505 break;
2507 Ok = isShiftedUInt<8, 2>(Imm) && (Imm != 0);
2508 break;
2510 Ok = Imm == 0;
2511 break;
2512
2517
2519 Ok = (isInt<5>(Imm) && Imm != -16) || Imm == 16;
2520 break;
2522 Ok = Imm != 0 && isInt<6>(Imm);
2523 break;
2525 Ok = isUInt<10>(Imm);
2526 break;
2528 Ok = isUInt<11>(Imm);
2529 break;
2531 Ok = isShiftedInt<7, 5>(Imm);
2532 break;
2534 Ok = STI.is64Bit() ? isUInt<6>(Imm) : isUInt<5>(Imm);
2535 break;
2537 Ok = STI.is64Bit() ? isUInt<6>(Imm) : isUInt<5>(Imm);
2538 Ok = Ok && Imm != 0;
2539 break;
2541 Ok = (isUInt<5>(Imm) && Imm != 0) ||
2542 (Imm >= 0xfffe0 && Imm <= 0xfffff);
2543 break;
2545 Ok = Imm >= 0 && Imm <= 10;
2546 break;
2548 Ok = Imm >= 0 && Imm <= 7;
2549 break;
2551 Ok = Imm >= 1 && Imm <= 10;
2552 break;
2554 Ok = Imm >= 2 && Imm <= 14;
2555 break;
2557 Ok = (Imm & 0xf) == 0;
2558 break;
2561 break;
2564 break;
2567 break;
2570 break;
2573 break;
2575 Ok = Imm == 0;
2576 break;
2580 Ok = isUInt<2>(Imm);
2581 else
2583 break;
2584 }
2585 if (!Ok) {
2586 ErrInfo = "Invalid immediate";
2587 return false;
2588 }
2589 }
2590 }
2591 }
2592
2596 if (.isImm() &&
.isReg()) {
2597 ErrInfo = "Invalid operand type for VL operand";
2598 return false;
2599 }
2600 if (Op.isReg() && Op.getReg() != RISCV::NoRegister) {
2602 auto *RC = MRI.getRegClass(Op.getReg());
2603 if (!RISCV::GPRRegClass.hasSubClassEq(RC)) {
2604 ErrInfo = "Invalid register class for VL operand";
2605 return false;
2606 }
2607 }
2609 ErrInfo = "VL operand w/o SEW operand?";
2610 return false;
2611 }
2612 }
2615 if (.getOperand(OpIdx).isImm()) {
2616 ErrInfo = "SEW value expected to be an immediate";
2617 return false;
2618 }
2619 uint64_t Log2SEW = MI.getOperand(OpIdx).getImm();
2620 if (Log2SEW > 31) {
2621 ErrInfo = "Unexpected SEW value";
2622 return false;
2623 }
2624 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
2626 ErrInfo = "Unexpected SEW value";
2627 return false;
2628 }
2629 }
2632 if (.getOperand(OpIdx).isImm()) {
2633 ErrInfo = "Policy operand expected to be an immediate";
2634 return false;
2635 }
2636 uint64_t Policy = MI.getOperand(OpIdx).getImm();
2638 ErrInfo = "Invalid Policy Value";
2639 return false;
2640 }
2642 ErrInfo = "policy operand w/o VL operand?";
2643 return false;
2644 }
2645
2646
2647
2648
2649 unsigned UseOpIdx;
2650 if (.isRegTiedToUseOperand(0, &UseOpIdx)) {
2651 ErrInfo = "policy operand w/o tied operand?";
2652 return false;
2653 }
2654 }
2655
2658 .readsRegister(RISCV::FRM, nullptr)) {
2659 ErrInfo = "dynamic rounding mode should read FRM";
2660 return false;
2661 }
2662
2663 return true;
2664}
2665
2670 default:
2671 return false;
2672 case RISCV::LB:
2673 case RISCV::LBU:
2674 case RISCV::LH:
2675 case RISCV::LH_INX:
2676 case RISCV::LHU:
2677 case RISCV::LW:
2678 case RISCV::LW_INX:
2679 case RISCV::LWU:
2680 case RISCV::LD:
2681 case RISCV::FLH:
2682 case RISCV::FLW:
2683 case RISCV::FLD:
2684 case RISCV::SB:
2685 case RISCV::SH:
2686 case RISCV::SH_INX:
2687 case RISCV::SW:
2688 case RISCV::SW_INX:
2689 case RISCV::SD:
2690 case RISCV::FSH:
2691 case RISCV::FSW:
2692 case RISCV::FSD:
2693 break;
2694 }
2695
2697 return false;
2698
2701 return false;
2702
2705 int64_t NewOffset = OldOffset + Disp;
2707 NewOffset = SignExtend64<32>(NewOffset);
2708
2709 if (!isInt<12>(NewOffset))
2710 return false;
2711
2717 return true;
2718}
2719
2722
2725
2727 "Addressing mode not supported for folding");
2728
2736}
2737
2743 return false;
2744
2745
2747 case RISCV::LB:
2748 case RISCV::LBU:
2749 case RISCV::SB:
2750 case RISCV::LH:
2751 case RISCV::LH_INX:
2752 case RISCV::LHU:
2753 case RISCV::FLH:
2754 case RISCV::SH:
2755 case RISCV::SH_INX:
2756 case RISCV::FSH:
2757 case RISCV::LW:
2758 case RISCV::LW_INX:
2759 case RISCV::LWU:
2760 case RISCV::FLW:
2761 case RISCV::SW:
2762 case RISCV::SW_INX:
2763 case RISCV::FSW:
2764 case RISCV::LD:
2765 case RISCV::FLD:
2766 case RISCV::SD:
2767 case RISCV::FSD:
2768 break;
2769 default:
2770 return false;
2771 }
2773 OffsetIsScalable = false;
2775 return false;
2777 return true;
2778}
2779
2780
2781
2786
2787
2788
2789 if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front()))
2790 return true;
2791
2793 return false;
2794
2797 if (MO1->getAddrSpace() != MO2->getAddrSpace())
2798 return false;
2799
2800 auto Base1 = MO1->getValue();
2801 auto Base2 = MO2->getValue();
2802 if (!Base1 || !Base2)
2803 return false;
2806
2807 if (isa(Base1) || isa(Base2))
2808 return false;
2809
2810 return Base1 == Base2;
2811}
2812
2816 int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize,
2817 unsigned NumBytes) const {
2818
2819
2820 if (!BaseOps1.empty() && !BaseOps2.empty()) {
2822 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
2824 return false;
2825 } else if (!BaseOps1.empty() || !BaseOps2.empty()) {
2826
2827 return false;
2828 }
2829
2831 BaseOps1.front()->getParent()->getMF()->getSubtarget().getCacheLineSize();
2832
2834
2835
2836
2837 return ClusterSize <= 4 && std::abs(Offset1 - Offset2) < CacheLineSize;
2838}
2839
2840
2841
2842
2843
2844
2845
2846
2851 return false;
2852
2853
2854
2855
2857 return false;
2860 return false;
2861
2863 return false;
2864
2868 return true;
2869}
2870
2875
2878 return false;
2879
2880
2881
2882
2883
2884
2886 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
2887 int64_t OffsetA = 0, OffsetB = 0;
2892 int LowOffset = std::min(OffsetA, OffsetB);
2893 int HighOffset = std::max(OffsetA, OffsetB);
2894 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
2896 LowOffset + (int)LowWidth.getValue() <= HighOffset)
2897 return true;
2898 }
2899 }
2900 return false;
2901}
2902
2903std::pair<unsigned, unsigned>
2906 return std::make_pair(TF & Mask, TF & ~Mask);
2907}
2908
2911 using namespace RISCVII;
2912 static const std::pair<unsigned, const char *> TargetFlags[] = {
2913 {MO_CALL, "riscv-call"},
2914 {MO_LO, "riscv-lo"},
2915 {MO_HI, "riscv-hi"},
2916 {MO_PCREL_LO, "riscv-pcrel-lo"},
2917 {MO_PCREL_HI, "riscv-pcrel-hi"},
2918 {MO_GOT_HI, "riscv-got-hi"},
2919 {MO_TPREL_LO, "riscv-tprel-lo"},
2920 {MO_TPREL_HI, "riscv-tprel-hi"},
2921 {MO_TPREL_ADD, "riscv-tprel-add"},
2922 {MO_TLS_GOT_HI, "riscv-tls-got-hi"},
2923 {MO_TLS_GD_HI, "riscv-tls-gd-hi"},
2924 {MO_TLSDESC_HI, "riscv-tlsdesc-hi"},
2925 {MO_TLSDESC_LOAD_LO, "riscv-tlsdesc-load-lo"},
2926 {MO_TLSDESC_ADD_LO, "riscv-tlsdesc-add-lo"},
2927 {MO_TLSDESC_CALL, "riscv-tlsdesc-call"}};
2928 return ArrayRef(TargetFlags);
2929}
2931 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
2933
2934
2935 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
2936 return false;
2937
2938
2939
2940 if (F.hasSection())
2941 return false;
2942
2943
2944 return true;
2945}
2946
2948 unsigned &Flags) const {
2949
2951}
2952
2953
2958
2962}
2963
2967 return F.getFnAttribute("fentry-call").getValueAsBool() ||
2968 F.hasFnAttribute("patchable-function-entry");
2969}
2970
2972 unsigned RegNo) {
2973 return MI.readsRegister(RegNo, TRI) ||
2974 MI.getDesc().hasImplicitUseOfPhysReg(RegNo);
2975}
2976
2979 return MI.modifiesRegister(RegNo, TRI) ||
2980 MI.getDesc().hasImplicitDefOfPhysReg(RegNo);
2981}
2982
2985 return true;
2987 return true;
2988
2989
2990
2991
2993 unsigned TailExpandUseRegNo =
2997 return true;
2999 break;
3000 }
3001 return false;
3002}
3003
3004static std::optional
3006
3007
3008 if (C.back().isReturn()) {
3010 "The candidate who uses return instruction must be outlined "
3011 "using tail call");
3013 }
3014
3018 return isMIModifiesReg(MI, TRI, RISCV::X5);
3019 }))
3020 return true;
3021 return .isAvailableAcrossAndOutOfSeq(RISCV::X5, *TRI);
3022 };
3023
3024 if (!CandidateUsesX5(C))
3026
3027 return std::nullopt;
3028}
3029
3030std::optional<std::unique_ptroutliner::OutlinedFunction>
3033 std::vectoroutliner::Candidate &RepeatedSequenceLocs,
3034 unsigned MinRepeats) const {
3035
3036
3039 if (!CandidateInfo)
3040 RepeatedSequenceLocs.clear();
3041
3042
3043 if (RepeatedSequenceLocs.size() < MinRepeats)
3044 return std::nullopt;
3045
3046 unsigned InstrSizeCExt =
3048 : 4;
3049 unsigned CallOverhead = 0, FrameOverhead = 0;
3050
3052 switch (MOCI) {
3054
3055 CallOverhead = 8;
3056
3057 FrameOverhead = InstrSizeCExt;
3058 break;
3060
3061 CallOverhead = 4 + InstrSizeCExt;
3062
3063 FrameOverhead = 0;
3064 break;
3065 }
3066
3067 for (auto &C : RepeatedSequenceLocs)
3068 C.setCallInfo(MOCI, CallOverhead);
3069
3070 unsigned SequenceSize = 0;
3071 for (auto &MI : Candidate)
3073
3074 return std::make_uniqueoutliner::OutlinedFunction(
3075 RepeatedSequenceLocs, SequenceSize, FrameOverhead, MOCI);
3076}
3077
3081 unsigned Flags) const {
3086 const auto &F = MI.getMF()->getFunction();
3087
3088
3089 if (MI.isCFIInstruction())
3090
3091
3092
3095
3099
3100
3101 for (const auto &MO : MI.operands()) {
3102
3103
3104
3106 (MI.getMF()->getTarget().getFunctionSections() || F.hasComdat() ||
3107 F.hasSection() || F.getSectionPrefix()))
3109 }
3110
3112}
3113
3117
3118
3119 bool Changed = true;
3120 while (Changed) {
3121 Changed = false;
3125 if (I->isCFIInstruction()) {
3126 I->removeFromParent();
3127 Changed = true;
3128 break;
3129 }
3130 }
3131 }
3132
3134 return;
3135
3137
3138
3141 .addReg(RISCV::X5)
3142 .addImm(0));
3143}
3144
3148
3151 .addGlobalAddress(M.getNamedValue(MF.getName()),
3153 return It;
3154 }
3155
3156
3159 .addGlobalAddress(M.getNamedValue(MF.getName()), 0,
3161 return It;
3162}
3163
3166
3167
3169 if (!Op0.isReg() || Reg != Op0.getReg())
3170 return std::nullopt;
3171
3172
3173
3174 if (MI.getOpcode() == RISCV::ADDI && MI.getOperand(1).isReg() &&
3175 MI.getOperand(2).isImm())
3176 return RegImmPair{MI.getOperand(1).getReg(), MI.getOperand(2).getImm()};
3177
3178 return std::nullopt;
3179}
3180
3181
3185
3186 std::string GenericComment =
3188 if (!GenericComment.empty())
3189 return GenericComment;
3190
3191
3192 if (.isImm())
3193 return std::string();
3194
3196 if (OpIdx >= Desc.getNumOperands())
3197 return std::string();
3198
3199 std::string Comment;
3201
3203
3204
3205
3209 unsigned Imm = Op.getImm();
3211 break;
3212 }
3215 unsigned Log2SEW = Op.getImm();
3216 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
3218 OS << "e" << SEW;
3219 break;
3220 }
3222 unsigned Policy = Op.getImm();
3224 "Invalid Policy Value");
3227 break;
3228 }
3229
3230 return Comment;
3231}
3232
3233
3234#define CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL) \
3235 RISCV::Pseudo##OP##_##LMUL
3236
3237#define CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL) \
3238 RISCV::Pseudo##OP##_##LMUL##_MASK
3239
3240#define CASE_RVV_OPCODE_LMUL(OP, LMUL) \
3241 CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL): \
3242 case CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL)
3243
3244#define CASE_RVV_OPCODE_UNMASK_WIDEN(OP) \
3245 CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF8): \
3246 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF4): \
3247 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF2): \
3248 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M1): \
3249 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M2): \
3250 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M4)
3251
3252#define CASE_RVV_OPCODE_UNMASK(OP) \
3253 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
3254 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M8)
3255
3256#define CASE_RVV_OPCODE_MASK_WIDEN(OP) \
3257 CASE_RVV_OPCODE_MASK_LMUL(OP, MF8): \
3258 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF4): \
3259 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF2): \
3260 case CASE_RVV_OPCODE_MASK_LMUL(OP, M1): \
3261 case CASE_RVV_OPCODE_MASK_LMUL(OP, M2): \
3262 case CASE_RVV_OPCODE_MASK_LMUL(OP, M4)
3263
3264#define CASE_RVV_OPCODE_MASK(OP) \
3265 CASE_RVV_OPCODE_MASK_WIDEN(OP): \
3266 case CASE_RVV_OPCODE_MASK_LMUL(OP, M8)
3267
3268#define CASE_RVV_OPCODE_WIDEN(OP) \
3269 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
3270 case CASE_RVV_OPCODE_MASK_WIDEN(OP)
3271
3272#define CASE_RVV_OPCODE(OP) \
3273 CASE_RVV_OPCODE_UNMASK(OP): \
3274 case CASE_RVV_OPCODE_MASK(OP)
3275
3276
3277
3278#define CASE_VMA_OPCODE_COMMON(OP, TYPE, LMUL) \
3279 RISCV::PseudoV##OP##_##TYPE##_##LMUL
3280
3281#define CASE_VMA_OPCODE_LMULS_M1(OP, TYPE) \
3282 CASE_VMA_OPCODE_COMMON(OP, TYPE, M1): \
3283 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M2): \
3284 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M4): \
3285 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M8)
3286
3287#define CASE_VMA_OPCODE_LMULS_MF2(OP, TYPE) \
3288 CASE_VMA_OPCODE_COMMON(OP, TYPE, MF2): \
3289 case CASE_VMA_OPCODE_LMULS_M1(OP, TYPE)
3290
3291#define CASE_VMA_OPCODE_LMULS_MF4(OP, TYPE) \
3292 CASE_VMA_OPCODE_COMMON(OP, TYPE, MF4): \
3293 case CASE_VMA_OPCODE_LMULS_MF2(OP, TYPE)
3294
3295#define CASE_VMA_OPCODE_LMULS(OP, TYPE) \
3296 CASE_VMA_OPCODE_COMMON(OP, TYPE, MF8): \
3297 case CASE_VMA_OPCODE_LMULS_MF4(OP, TYPE)
3298
3299
3300#define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL, SEW) \
3301 RISCV::PseudoV##OP##_##TYPE##_##LMUL##_##SEW
3302
3303#define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW) \
3304 CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1, SEW): \
3305 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2, SEW): \
3306 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4, SEW): \
3307 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8, SEW)
3308
3309#define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW) \
3310 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2, SEW): \
3311 case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW)
3312
3313#define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE, SEW) \
3314 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4, SEW): \
3315 case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW)
3316
3317#define CASE_VFMA_OPCODE_VV(OP) \
3318 CASE_VFMA_OPCODE_LMULS_MF4(OP, VV, E16): \
3319 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VV, E32): \
3320 case CASE_VFMA_OPCODE_LMULS_M1(OP, VV, E64)
3321
3322#define CASE_VFMA_SPLATS(OP) \
3323 CASE_VFMA_OPCODE_LMULS_MF4(OP, VFPR16, E16): \
3324 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VFPR32, E32): \
3325 case CASE_VFMA_OPCODE_LMULS_M1(OP, VFPR64, E64)
3326
3327
3329 unsigned &SrcOpIdx1,
3330 unsigned &SrcOpIdx2) const {
3332 if (.isCommutable())
3333 return false;
3334
3335 switch (MI.getOpcode()) {
3336 case RISCV::TH_MVEQZ:
3337 case RISCV::TH_MVNEZ:
3338
3339
3340
3341 if (MI.getOperand(2).getReg() == RISCV::X0)
3342 return false;
3343
3344 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1, 2);
3345 case RISCV::TH_MULA:
3346 case RISCV::TH_MULAW:
3347 case RISCV::TH_MULAH:
3348 case RISCV::TH_MULS:
3349 case RISCV::TH_MULSW:
3350 case RISCV::TH_MULSH:
3351
3352 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
3353 case RISCV::PseudoCCMOVGPRNoX0:
3354 case RISCV::PseudoCCMOVGPR:
3355
3356 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 4, 5);
3382
3383 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
3402
3404 if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0)
3405 return false;
3406
3407
3408
3409 unsigned CommutableOpIdx1 = 1;
3410 unsigned CommutableOpIdx2 = 3;
3411 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
3412 CommutableOpIdx2))
3413 return false;
3414 return true;
3415 }
3422
3424 if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0)
3425 return false;
3426
3427
3428
3429
3430
3431 if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3)
3432 return false;
3433 if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3)
3434 return false;
3435
3436
3437 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
3438 SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1)
3439 return false;
3440
3441
3442
3443
3444 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
3445 SrcOpIdx2 == CommuteAnyOperandIndex) {
3446
3447
3448 unsigned CommutableOpIdx1 = SrcOpIdx1;
3449 if (SrcOpIdx1 == SrcOpIdx2) {
3450
3451
3452 CommutableOpIdx1 = 1;
3453 } else if (SrcOpIdx1 == CommuteAnyOperandIndex) {
3454
3455 CommutableOpIdx1 = SrcOpIdx2;
3456 }
3457
3458
3459
3460 unsigned CommutableOpIdx2;
3461 if (CommutableOpIdx1 != 1) {
3462
3463 CommutableOpIdx2 = 1;
3464 } else {
3465 Register Op1Reg = MI.getOperand(CommutableOpIdx1).getReg();
3466
3467
3468
3469
3470 if (Op1Reg != MI.getOperand(2).getReg())
3471 CommutableOpIdx2 = 2;
3472 else
3473 CommutableOpIdx2 = 3;
3474 }
3475
3476
3477
3478 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
3479 CommutableOpIdx2))
3480 return false;
3481 }
3482
3483 return true;
3484 }
3485 }
3486
3488}
3489
3490
3491#define CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL) \
3492 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL: \
3493 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL; \
3494 break;
3495
3496#define CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE) \
3497 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1) \
3498 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2) \
3499 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4) \
3500 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)
3501
3502#define CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE) \
3503 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2) \
3504 CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE)
3505
3506#define CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE) \
3507 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4) \
3508 CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE)
3509
3510#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE) \
3511 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8) \
3512 CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE)
3513
3514#define CASE_VMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \
3515 CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16) \
3516 CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32) \
3517 CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64)
3518
3519
3520#define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL, SEW) \
3521 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL##_##SEW: \
3522 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL##_##SEW; \
3523 break;
3524
3525#define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW) \
3526 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1, SEW) \
3527 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2, SEW) \
3528 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4, SEW) \
3529 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8, SEW)
3530
3531#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW) \
3532 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2, SEW) \
3533 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)
3534
3535#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP) \
3536 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VV, E16) \
3537 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32) \
3538 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VV, E64)
3539
3540#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW) \
3541 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4, SEW) \
3542 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW)
3543
3544#define CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE, SEW) \
3545 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8, SEW) \
3546 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW)
3547
3548#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \
3549 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16, E16) \
3550 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32, E32) \
3551 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64, E64)
3552
3554 bool NewMI,
3555 unsigned OpIdx1,
3556 unsigned OpIdx2) const {
3558 if (NewMI)
3559 return *MI.getParent()->getParent()->CloneMachineInstr(&MI);
3560 return MI;
3561 };
3562
3563 switch (MI.getOpcode()) {
3564 case RISCV::TH_MVEQZ:
3565 case RISCV::TH_MVNEZ: {
3566 auto &WorkingMI = cloneIfNew(MI);
3567 WorkingMI.setDesc(get(MI.getOpcode() == RISCV::TH_MVEQZ ? RISCV::TH_MVNEZ
3568 : RISCV::TH_MVEQZ));
3570 OpIdx2);
3571 }
3572 case RISCV::PseudoCCMOVGPRNoX0:
3573 case RISCV::PseudoCCMOVGPR: {
3574
3577 auto &WorkingMI = cloneIfNew(MI);
3578 WorkingMI.getOperand(3).setImm(CC);
3580 OpIdx1, OpIdx2);
3581 }
3600
3601
3602 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
3603 assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index");
3604 unsigned Opc;
3605 switch (MI.getOpcode()) {
3606 default:
3626 }
3627
3628 auto &WorkingMI = cloneIfNew(MI);
3629 WorkingMI.setDesc(get(Opc));
3631 OpIdx1, OpIdx2);
3632 }
3639 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
3640
3641
3642 if (OpIdx1 == 3 || OpIdx2 == 3) {
3643 unsigned Opc;
3644 switch (MI.getOpcode()) {
3645 default:
3653 }
3654
3655 auto &WorkingMI = cloneIfNew(MI);
3656 WorkingMI.setDesc(get(Opc));
3658 OpIdx1, OpIdx2);
3659 }
3660
3661 break;
3662 }
3663 }
3664
3666}
3667
3668#undef CASE_RVV_OPCODE_UNMASK_LMUL
3669#undef CASE_RVV_OPCODE_MASK_LMUL
3670#undef CASE_RVV_OPCODE_LMUL
3671#undef CASE_RVV_OPCODE_UNMASK_WIDEN
3672#undef CASE_RVV_OPCODE_UNMASK
3673#undef CASE_RVV_OPCODE_MASK_WIDEN
3674#undef CASE_RVV_OPCODE_MASK
3675#undef CASE_RVV_OPCODE_WIDEN
3676#undef CASE_RVV_OPCODE
3677
3678#undef CASE_VMA_OPCODE_COMMON
3679#undef CASE_VMA_OPCODE_LMULS_M1
3680#undef CASE_VMA_OPCODE_LMULS_MF2
3681#undef CASE_VMA_OPCODE_LMULS_MF4
3682#undef CASE_VMA_OPCODE_LMULS
3683#undef CASE_VFMA_OPCODE_COMMON
3684#undef CASE_VFMA_OPCODE_LMULS_M1
3685#undef CASE_VFMA_OPCODE_LMULS_MF2
3686#undef CASE_VFMA_OPCODE_LMULS_MF4
3687#undef CASE_VFMA_OPCODE_VV
3688#undef CASE_VFMA_SPLATS
3689
3690
3691#define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL) \
3692 RISCV::PseudoV##OP##_##LMUL##_TIED
3693
3694#define CASE_WIDEOP_OPCODE_LMULS_MF4(OP) \
3695 CASE_WIDEOP_OPCODE_COMMON(OP, MF4): \
3696 case CASE_WIDEOP_OPCODE_COMMON(OP, MF2): \
3697 case CASE_WIDEOP_OPCODE_COMMON(OP, M1): \
3698 case CASE_WIDEOP_OPCODE_COMMON(OP, M2): \
3699 case CASE_WIDEOP_OPCODE_COMMON(OP, M4)
3700
3701#define CASE_WIDEOP_OPCODE_LMULS(OP) \
3702 CASE_WIDEOP_OPCODE_COMMON(OP, MF8): \
3703 case CASE_WIDEOP_OPCODE_LMULS_MF4(OP)
3704
3705#define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL) \
3706 case RISCV::PseudoV##OP##_##LMUL##_TIED: \
3707 NewOpc = RISCV::PseudoV##OP##_##LMUL; \
3708 break;
3709
3710#define CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP) \
3711 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4) \
3712 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2) \
3713 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1) \
3714 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2) \
3715 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4)
3716
3717#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
3718 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8) \
3719 CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
3720
3721
3722#define CASE_FP_WIDEOP_OPCODE_COMMON(OP, LMUL, SEW) \
3723 RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED
3724
3725#define CASE_FP_WIDEOP_OPCODE_LMULS_MF4(OP) \
3726 CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16): \
3727 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16): \
3728 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E32): \
3729 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16): \
3730 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E32): \
3731 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16): \
3732 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E32): \
3733 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16): \
3734 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E32) \
3735
3736#define CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL, SEW) \
3737 case RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED: \
3738 NewOpc = RISCV::PseudoV##OP##_##LMUL##_##SEW; \
3739 break;
3740
3741#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP) \
3742 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16) \
3743 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16) \
3744 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E32) \
3745 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16) \
3746 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E32) \
3747 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16) \
3748 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E32) \
3749 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16) \
3750 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E32) \
3751
3752#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
3753 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
3754
3755
3760 switch (MI.getOpcode()) {
3761 default:
3762 return nullptr;
3766 MI.getNumExplicitOperands() == 7 &&
3767 "Expect 7 explicit operands rd, rs2, rs1, rm, vl, sew, policy");
3768
3770 1) == 0)
3771 return nullptr;
3772
3773 unsigned NewOpc;
3774 switch (MI.getOpcode()) {
3775 default:
3779 }
3780
3781
3792 break;
3793 }
3798
3800 MI.getNumExplicitOperands() == 6);
3801 if ((MI.getOperand(5).getImm() & 1) == 0)
3802 return nullptr;
3803
3804
3805 unsigned NewOpc;
3806 switch (MI.getOpcode()) {
3807 default:
3813 }
3814
3815
3825 break;
3826 }
3827 }
3829
3830 if (LV) {
3831 unsigned NumOps = MI.getNumOperands();
3832 for (unsigned I = 1; I < NumOps; ++I) {
3834 if (Op.isReg() && Op.isKill())
3836 }
3837 }
3838
3839 if (LIS) {
3841
3842 if (MI.getOperand(0).isEarlyClobber()) {
3843
3844
3845
3848 if (S->end == Idx.getRegSlot(true))
3849 S->end = Idx.getRegSlot();
3850 }
3851 }
3852
3853 return MIB;
3854}
3855
3856#undef CASE_WIDEOP_OPCODE_COMMON
3857#undef CASE_WIDEOP_OPCODE_LMULS_MF4
3858#undef CASE_WIDEOP_OPCODE_LMULS
3859#undef CASE_WIDEOP_CHANGE_OPCODE_COMMON
3860#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4
3861#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS
3862#undef CASE_FP_WIDEOP_OPCODE_COMMON
3863#undef CASE_FP_WIDEOP_OPCODE_LMULS_MF4
3864#undef CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON
3865#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4
3866#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS
3867
3873 if (llvm::has_single_bit<uint32_t>(Amount)) {
3875 if (ShiftAmount == 0)
3876 return;
3879 .addImm(ShiftAmount)
3881 } else if (STI.hasStdExtZba() &&
3882 ((Amount % 3 == 0 && isPowerOf2_64(Amount / 3)) ||
3883 (Amount % 5 == 0 && isPowerOf2_64(Amount / 5)) ||
3884 (Amount % 9 == 0 && isPowerOf2_64(Amount / 9)))) {
3885
3886 unsigned Opc;
3888 if (Amount % 9 == 0) {
3889 Opc = RISCV::SH3ADD;
3890 ShiftAmount = Log2_64(Amount / 9);
3891 } else if (Amount % 5 == 0) {
3892 Opc = RISCV::SH2ADD;
3893 ShiftAmount = Log2_64(Amount / 5);
3894 } else if (Amount % 3 == 0) {
3895 Opc = RISCV::SH1ADD;
3896 ShiftAmount = Log2_64(Amount / 3);
3897 } else {
3899 }
3900 if (ShiftAmount)
3903 .addImm(ShiftAmount)
3909 } else if (llvm::has_single_bit<uint32_t>(Amount - 1)) {
3910 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);
3914 .addImm(ShiftAmount)
3920 } else if (llvm::has_single_bit<uint32_t>(Amount + 1)) {
3921 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);
3925 .addImm(ShiftAmount)
3931 } else if (STI.hasStdExtZmmul()) {
3932 Register N = MRI.createVirtualRegister(&RISCV::GPRRegClass);
3938 } else {
3940 uint32_t PrevShiftAmount = 0;
3941 for (uint32_t ShiftAmount = 0; Amount >> ShiftAmount; ShiftAmount++) {
3942 if (Amount & (1U << ShiftAmount)) {
3943 if (ShiftAmount)
3946 .addImm(ShiftAmount - PrevShiftAmount)
3948 if (Amount >> (ShiftAmount + 1)) {
3949
3950 if (!Acc) {
3951 Acc = MRI.createVirtualRegister(&RISCV::GPRRegClass);
3955 } else {
3960 }
3961 }
3962 PrevShiftAmount = ShiftAmount;
3963 }
3964 }
3965 assert(Acc && "Expected valid accumulator");
3970 }
3971}
3972
3975 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
3978 return ArrayRef(TargetFlags);
3979}
3980
3984 : 2;
3985}
3986
3987
3989 return MI.getOpcode() == RISCV::ADDIW && MI.getOperand(1).isReg() &&
3990 MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0;
3991}
3992
3993
3995 return MI.getOpcode() == RISCV::ADD_UW && MI.getOperand(1).isReg() &&
3996 MI.getOperand(2).isReg() && MI.getOperand(2).getReg() == RISCV::X0;
3997}
3998
3999
4001 return MI.getOpcode() == RISCV::ANDI && MI.getOperand(1).isReg() &&
4002 MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 255;
4003}
4004
4006 switch (Opcode) {
4007 default:
4008 return false;
4009 case RISCV::VS1R_V:
4010 case RISCV::VS2R_V:
4011 case RISCV::VS4R_V:
4012 case RISCV::VS8R_V:
4013 case RISCV::VL1RE8_V:
4014 case RISCV::VL2RE8_V:
4015 case RISCV::VL4RE8_V:
4016 case RISCV::VL8RE8_V:
4017 case RISCV::VL1RE16_V:
4018 case RISCV::VL2RE16_V:
4019 case RISCV::VL4RE16_V:
4020 case RISCV::VL8RE16_V:
4021 case RISCV::VL1RE32_V:
4022 case RISCV::VL2RE32_V:
4023 case RISCV::VL4RE32_V:
4024 case RISCV::VL8RE32_V:
4025 case RISCV::VL1RE64_V:
4026 case RISCV::VL2RE64_V:
4027 case RISCV::VL4RE64_V:
4028 case RISCV::VL8RE64_V:
4029 return true;
4030 }
4031}
4032
4034
4035
4036 unsigned Opcode = MI.getOpcode();
4037 if (!RISCVVPseudosTable::getPseudoInfo(Opcode) &&
4039 return false;
4040 return true;
4041}
4042
4043std::optional<std::pair<unsigned, unsigned>>
4045 switch (Opcode) {
4046 default:
4047 return std::nullopt;
4048 case RISCV::PseudoVSPILL2_M1:
4049 case RISCV::PseudoVRELOAD2_M1:
4050 return std::make_pair(2u, 1u);
4051 case RISCV::PseudoVSPILL2_M2:
4052 case RISCV::PseudoVRELOAD2_M2:
4053 return std::make_pair(2u, 2u);
4054 case RISCV::PseudoVSPILL2_M4:
4055 case RISCV::PseudoVRELOAD2_M4:
4056 return std::make_pair(2u, 4u);
4057 case RISCV::PseudoVSPILL3_M1:
4058 case RISCV::PseudoVRELOAD3_M1:
4059 return std::make_pair(3u, 1u);
4060 case RISCV::PseudoVSPILL3_M2:
4061 case RISCV::PseudoVRELOAD3_M2:
4062 return std::make_pair(3u, 2u);
4063 case RISCV::PseudoVSPILL4_M1:
4064 case RISCV::PseudoVRELOAD4_M1:
4065 return std::make_pair(4u, 1u);
4066 case RISCV::PseudoVSPILL4_M2:
4067 case RISCV::PseudoVRELOAD4_M2:
4068 return std::make_pair(4u, 2u);
4069 case RISCV::PseudoVSPILL5_M1:
4070 case RISCV::PseudoVRELOAD5_M1:
4071 return std::make_pair(5u, 1u);
4072 case RISCV::PseudoVSPILL6_M1:
4073 case RISCV::PseudoVRELOAD6_M1:
4074 return std::make_pair(6u, 1u);
4075 case RISCV::PseudoVSPILL7_M1:
4076 case RISCV::PseudoVRELOAD7_M1:
4077 return std::make_pair(7u, 1u);
4078 case RISCV::PseudoVSPILL8_M1:
4079 case RISCV::PseudoVRELOAD8_M1:
4080 return std::make_pair(8u, 1u);
4081 }
4082}
4083
4085 return MI.getNumExplicitDefs() == 2 &&
4086 MI.modifiesRegister(RISCV::VL, nullptr) && .isInlineAsm();
4087}
4088
4090 int16_t MI1FrmOpIdx =
4092 int16_t MI2FrmOpIdx =
4094 if (MI1FrmOpIdx < 0 || MI2FrmOpIdx < 0)
4095 return false;
4099}
4100
4101std::optional
4103
4104 switch (Opcode) {
4105 default:
4106 return std::nullopt;
4107
4108
4109 case RISCV::VSLL_VX:
4110 case RISCV::VSRL_VX:
4111 case RISCV::VSRA_VX:
4112
4113 case RISCV::VSSRL_VX:
4114 case RISCV::VSSRA_VX:
4115
4116 return Log2SEW;
4117
4118
4119 case RISCV::VNSRL_WX:
4120 case RISCV::VNSRA_WX:
4121
4122 case RISCV::VNCLIPU_WX:
4123 case RISCV::VNCLIP_WX:
4124
4125 return Log2SEW + 1;
4126
4127
4128 case RISCV::VADD_VX:
4129 case RISCV::VSUB_VX:
4130 case RISCV::VRSUB_VX:
4131
4132 case RISCV::VWADDU_VX:
4133 case RISCV::VWSUBU_VX:
4134 case RISCV::VWADD_VX:
4135 case RISCV::VWSUB_VX:
4136 case RISCV::VWADDU_WX:
4137 case RISCV::VWSUBU_WX:
4138 case RISCV::VWADD_WX:
4139 case RISCV::VWSUB_WX:
4140
4141 case RISCV::VADC_VXM:
4142 case RISCV::VADC_VIM:
4143 case RISCV::VMADC_VXM:
4144 case RISCV::VMADC_VIM:
4145 case RISCV::VMADC_VX:
4146 case RISCV::VSBC_VXM:
4147 case RISCV::VMSBC_VXM:
4148 case RISCV::VMSBC_VX:
4149
4150 case RISCV::VAND_VX:
4151 case RISCV::VOR_VX:
4152 case RISCV::VXOR_VX:
4153
4154 case RISCV::VMSEQ_VX:
4155 case RISCV::VMSNE_VX:
4156 case RISCV::VMSLTU_VX:
4157 case RISCV::VMSLT_VX:
4158 case RISCV::VMSLEU_VX:
4159 case RISCV::VMSLE_VX:
4160 case RISCV::VMSGTU_VX:
4161 case RISCV::VMSGT_VX:
4162
4163 case RISCV::VMINU_VX:
4164 case RISCV::VMIN_VX:
4165 case RISCV::VMAXU_VX:
4166 case RISCV::VMAX_VX:
4167
4168 case RISCV::VMUL_VX:
4169 case RISCV::VMULH_VX:
4170 case RISCV::VMULHU_VX:
4171 case RISCV::VMULHSU_VX:
4172
4173 case RISCV::VDIVU_VX:
4174 case RISCV::VDIV_VX:
4175 case RISCV::VREMU_VX:
4176 case RISCV::VREM_VX:
4177
4178 case RISCV::VWMUL_VX:
4179 case RISCV::VWMULU_VX:
4180 case RISCV::VWMULSU_VX:
4181
4182 case RISCV::VMACC_VX:
4183 case RISCV::VNMSAC_VX:
4184 case RISCV::VMADD_VX:
4185 case RISCV::VNMSUB_VX:
4186
4187 case RISCV::VWMACCU_VX:
4188 case RISCV::VWMACC_VX:
4189 case RISCV::VWMACCSU_VX:
4190 case RISCV::VWMACCUS_VX:
4191
4192 case RISCV::VMERGE_VXM:
4193
4194 case RISCV::VMV_V_X:
4195
4196 case RISCV::VSADDU_VX:
4197 case RISCV::VSADD_VX:
4198 case RISCV::VSSUBU_VX:
4199 case RISCV::VSSUB_VX:
4200
4201 case RISCV::VAADDU_VX:
4202 case RISCV::VAADD_VX:
4203 case RISCV::VASUBU_VX:
4204 case RISCV::VASUB_VX:
4205
4206 case RISCV::VSMUL_VX:
4207
4208 case RISCV::VMV_S_X:
4209 return 1U << Log2SEW;
4210 }
4211}
4212
4215 RISCVVPseudosTable::getPseudoInfo(RVVPseudoOpcode);
4216 if ()
4217 return 0;
4218 return RVV->BaseInstr;
4219}
4220
4222 unsigned DestEEW =
4224
4225 if (DestEEW == 0)
4226 return 0;
4227
4228 unsigned Scaled = Log2SEW + (DestEEW - 1);
4231}
4232
4233
4235 if (LHS.isReg() && RHS.isReg() && LHS.getReg().isVirtual() &&
4236 LHS.getReg() == RHS.getReg())
4237 return true;
4239 return true;
4241 return false;
4242 if (.isImm() ||
.isImm())
4243 return false;
4244 return LHS.getImm() <= RHS.getImm();
4245}
4246
4247namespace {
4252
4253public:
4257
4258 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
4259
4260
4262 return true;
4264 return true;
4265 return false;
4266 }
4267
4268 std::optional createTripCountGreaterCondition(
4271
4272
4273
4274 CondParam = Cond;
4275 return {};
4276 }
4277
4279
4280 void adjustTripCount(int TripCountAdjust) override {}
4281};
4282}
4283
4284std::unique_ptrTargetInstrInfo::PipelinerLoopInfo
4289 return nullptr;
4290
4291
4292 if (TBB == LoopBB && FBB == LoopBB)
4293 return nullptr;
4294
4295
4296 if (FBB == nullptr)
4297 return nullptr;
4298
4299 assert((TBB == LoopBB || FBB == LoopBB) &&
4300 "The Loop must be a single-basic-block loop");
4301
4302
4303 if (TBB == LoopBB)
4305
4308 if (.isReg())
4309 return nullptr;
4311 if (!Reg.isVirtual())
4312 return nullptr;
4313 return MRI.getVRegDef(Reg);
4314 };
4315
4319 return nullptr;
4321 return nullptr;
4322
4323 return std::make_unique(LHS, RHS, Cond);
4324}
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder MachineInstrBuilder & DefMI
static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg, unsigned NumRegs)
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc, unsigned ZeroReg=0, bool CheckZeroReg=false)
static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, SmallVectorImpl< MachineOperand > &Cond)
@ MachineOutlinerTailCall
Emit a save, restore, call, and return.
static ARCCC::CondCode getOppositeBranchCondition(ARCCC::CondCode CC)
Return the inverse of passed condition, i.e. turning COND_E to COND_NE.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
Module.h This file contains the declarations for the Module class.
static M68k::CondCode getCondFromBranchOpc(unsigned BrOpc)
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static bool cannotInsertTailCall(const MachineBasicBlock &MBB)
#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP)
#define CASE_OPERAND_SIMM(NUM)
static bool isRVVWholeLoadStore(unsigned Opcode)
#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP)
static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc, unsigned Pattern)
#define RVV_OPC_LMUL_CASE(OPC, INV)
static void combineFPFusedMultiply(MachineInstr &Root, MachineInstr &Prev, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs)
static unsigned getAddendOperandIdx(unsigned Pattern)
#define CASE_RVV_OPCODE_UNMASK(OP)
#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP)
static std::optional< MachineOutlinerConstructionID > analyzeCandidate(outliner::Candidate &C)
static cl::opt< bool > PreferWholeRegisterMove("riscv-prefer-whole-register-move", cl::init(false), cl::Hidden, cl::desc("Prefer whole register move for vector registers."))
#define CASE_VFMA_SPLATS(OP)
unsigned getPredicatedOpcode(unsigned Opcode)
static void genShXAddAddShift(MachineInstr &Root, unsigned AddOpIdx, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg)
#define CASE_WIDEOP_OPCODE_LMULS(OP)
#define OPCODE_LMUL_MASK_CASE(OPC)
static bool isFSUB(unsigned Opc)
#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE)
#define CASE_RVV_OPCODE(OP)
#define CASE_VFMA_OPCODE_VV(OP)
MachineOutlinerConstructionID
#define CASE_RVV_OPCODE_WIDEN(OP)
#define CASE_VMA_OPCODE_LMULS(OP, TYPE)
static bool isFMUL(unsigned Opc)
static bool getFPPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce)
#define OPCODE_LMUL_CASE(OPC)
#define CASE_OPERAND_UIMM(NUM)
static bool canCombineShiftIntoShXAdd(const MachineBasicBlock &MBB, const MachineOperand &MO, unsigned OuterShiftAmt)
Utility routine that checks if.
static bool isCandidatePatchable(const MachineBasicBlock &MBB)
static bool isMIReadsReg(const MachineInstr &MI, const TargetRegisterInfo *TRI, unsigned RegNo)
static bool isFADD(unsigned Opc)
#define CASE_FP_WIDEOP_OPCODE_LMULS_MF4(OP)
static bool isConvertibleToVMV_V_V(const RISCVSubtarget &STI, const MachineBasicBlock &MBB, MachineBasicBlock::const_iterator MBBI, MachineBasicBlock::const_iterator &DefMBBI, RISCVII::VLMUL LMul)
static bool isMIModifiesReg(const MachineInstr &MI, const TargetRegisterInfo *TRI, unsigned RegNo)
static MachineInstr * canFoldAsPredicatedOp(Register Reg, const MachineRegisterInfo &MRI, const TargetInstrInfo *TII)
Identify instructions that can be folded into a CCMOV instruction, and return the defining instructio...
static bool canCombineFPFusedMultiply(const MachineInstr &Root, const MachineOperand &MO, bool DoRegPressureReduce)
static bool getSHXADDPatterns(const MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static bool getFPFusedMultiplyPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce)
static cl::opt< MachineTraceStrategy > ForceMachineCombinerStrategy("riscv-force-machine-combiner-strategy", cl::Hidden, cl::desc("Force machine combiner to use a specific strategy for machine " "trace metrics evaluation."), cl::init(MachineTraceStrategy::TS_NumStrategies), cl::values(clEnumValN(MachineTraceStrategy::TS_Local, "local", "Local strategy."), clEnumValN(MachineTraceStrategy::TS_MinInstrCount, "min-instr", "MinInstrCount strategy.")))
static unsigned getSHXADDShiftAmount(unsigned Opc)
#define CASE_RVV_OPCODE_MASK(OP)
#define RVV_OPC_LMUL_MASK_CASE(OPC, INV)
#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
This file declares the machine register scavenger class.
static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, ArrayRef< const MachineOperand * > BaseOps1, const MachineInstr &MI2, ArrayRef< const MachineOperand * > BaseOps2)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI)
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file defines the SmallVector class.
static cl::opt< unsigned > CacheLineSize("cache-line-size", cl::init(0), cl::Hidden, cl::desc("Use this to override the target cache line size when " "specified by the user."))
static unsigned getSize(unsigned Kind)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
static DILocation * getMergedLocation(DILocation *LocA, DILocation *LocB)
When two instructions are combined into a single instruction we also need to combine the original loc...
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
LiveInterval - This class represents the liveness of a register, or stack slot.
LiveInterval & getInterval(Register Reg)
SlotIndex ReplaceMachineInstrInMaps(MachineInstr &MI, MachineInstr &NewMI)
const Segment * getSegmentContaining(SlotIndex Idx) const
Return the segment that contains the specified index, or null if there is none.
void replaceKillInstruction(Register Reg, MachineInstr &OldMI, MachineInstr &NewMI)
replaceKillInstruction - Update register kill info by replacing a kill instruction with a new one.
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
TypeSize getValue() const
MCInstBuilder & addReg(MCRegister Reg)
Add a new register operand.
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Instances of this class represent a single low-level machine instruction.
Describe properties that are true of each instruction in the target description file.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
bool isConditionalBranch() const
Return true if this is a branch which may fall through to the next instruction or may transfer contro...
This holds information about one operand of a machine instruction, indicating the register class for ...
uint8_t OperandType
Information about the type of the operand.
Wrapper class representing physical registers. Should be passed by value.
const FeatureBitset & getFeatureBits() const
unsigned pred_size() const
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
iterator getLastNonDebugInstr(bool SkipPseudoOp=true)
Returns an iterator to the last non-debug instruction in the basic block, or end().
Instructions::const_iterator const_instr_iterator
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void setStackID(int ObjectIdx, uint8_t ID)
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & copyImplicitOps(const MachineInstr &OtherMI) const
Copy all the implicit operands from OtherMI onto this one.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool isReturn(QueryType Type=AnyInBundle) const
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
bool hasOneMemOperand() const
Return true if this instruction has exactly one MachineMemOperand.
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
void clearKillInfo()
Clears kill flags on all operands.
A description of a memory reference used in the backend.
bool isNonTemporal() const
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
static MachineOperand CreateImm(int64_t Val)
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
@ MO_Immediate
Immediate operand.
@ MO_Register
Register operand.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
A Module instance is used to store all the information related to an LLVM module.
MI-level patchpoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given patchpoint should emit.
MachineInstr * convertToThreeAddress(MachineInstr &MI, LiveVariables *LV, LiveIntervals *LIS) const override
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
std::optional< std::unique_ptr< outliner::OutlinedFunction > > getOutliningCandidateInfo(const MachineModuleInfo &MMI, std::vector< outliner::Candidate > &RepeatedSequenceLocs, unsigned MinRepeats) const override
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
void movImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, Register DstReg, uint64_t Val, MachineInstr::MIFlag Flag=MachineInstr::NoFlags, bool DstRenamable=false, bool DstIsDead=false) const
MachineInstr * emitLdStWithAddr(MachineInstr &MemI, const ExtAddrMode &AM) const override
void mulImm(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator II, const DebugLoc &DL, Register DestReg, uint32_t Amt, MachineInstr::MIFlag Flag) const
Generate code to multiply the value in DestReg by Amt - handles all the common optimizations for this...
bool isReallyTriviallyReMaterializable(const MachineInstr &MI) const override
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool IsKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
bool isFunctionSafeToOutlineFrom(MachineFunction &MF, bool OutlineFromLinkOnceODRs) const override
std::unique_ptr< TargetInstrInfo::PipelinerLoopInfo > analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &dl, int *BytesAdded=nullptr) const override
bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) const override
RISCVInstrInfo(RISCVSubtarget &STI)
void copyPhysRegVector(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, const TargetRegisterClass *RegClass) const
void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg) const override
const MCInstrDesc & getBrCond(RISCVCC::CondCode CC, bool Imm=false) const
MachineInstr * optimizeSelect(MachineInstr &MI, SmallPtrSetImpl< MachineInstr * > &SeenMIs, bool) const override
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, bool RenamableDest=false, bool RenamableSrc=false) const override
bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg, const MachineInstr &AddrI, ExtAddrMode &AM) const override
void insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &NewDestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const override
bool isAsCheapAsAMove(const MachineInstr &MI) const override
bool verifyInstruction(const MachineInstr &MI, StringRef &ErrInfo) const override
bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset, LocationSize &Width, const TargetRegisterInfo *TRI) const
unsigned getTailDuplicateSize(CodeGenOptLevel OptLevel) const override
void getReassociateOperandIndices(const MachineInstr &Root, unsigned Pattern, std::array< unsigned, 5 > &OperandIndices) const override
const RISCVSubtarget & STI
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
std::optional< unsigned > getInverseOpcode(unsigned Opcode) const override
ArrayRef< std::pair< unsigned, const char * > > getSerializableDirectMachineOperandTargetFlags() const override
virtual outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI, MachineBasicBlock::iterator &MBBI, unsigned Flags) const override
MachineTraceStrategy getMachineCombinerTraceStrategy() const override
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
std::optional< RegImmPair > isAddImmediate(const MachineInstr &MI, Register Reg) const override
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
ArrayRef< std::pair< MachineMemOperand::Flags, const char * > > getSerializableMachineMemOperandTargetFlags() const override
MCInst getNop() const override
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, unsigned &Flags) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &MI, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override
void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF, const outliner::OutlinedFunction &OF) const override
void finalizeInsInstrs(MachineInstr &Root, unsigned &Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs) const override
std::pair< unsigned, unsigned > decomposeMachineOperandsTargetFlags(unsigned TF) const override
MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const override
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DstReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
bool hasReassociableOperands(const MachineInstr &Inst, const MachineBasicBlock *MBB) const override
MachineBasicBlock * getBranchDestBlock(const MachineInstr &MI) const override
std::string createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx, const TargetRegisterInfo *TRI) const override
bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override
bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const override
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, outliner::Candidate &C) const override
bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override
bool isAssociativeAndCommutative(const MachineInstr &Inst, bool Invert) const override
CombinerObjective getCombinerObjective(unsigned Pattern) const override
bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const override
bool optimizeCondBranch(MachineInstr &MI) const override
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
bool analyzeSelect(const MachineInstr &MI, SmallVectorImpl< MachineOperand > &Cond, unsigned &TrueOp, unsigned &FalseOp, bool &Optimizable) const override
bool shouldClusterMemOps(ArrayRef< const MachineOperand * > BaseOps1, int64_t Offset1, bool OffsetIsScalable1, ArrayRef< const MachineOperand * > BaseOps2, int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize, unsigned NumBytes) const override
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
int getBranchRelaxationScratchFrameIndex() const
bool hasStdExtCOrZca() const
unsigned getTailDupAggressiveThreshold() const
const RISCVRegisterInfo * getRegisterInfo() const override
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
void setRegUsed(Register Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Tell the scavenger a register is used.
Register scavengeRegisterBackwards(const TargetRegisterClass &RC, MachineBasicBlock::iterator To, bool RestoreAfter, int SPAdj, bool AllowSpill=true)
Make a register of the specific register class available from the current position backwards to the p...
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
SlotIndex - An opaque wrapper around machine indexes.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MI-level stackmap operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
MI-level Statepoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given statepoint should emit.
StringRef - Represent a constant reference to a string, i.e.
Object returned by analyzeLoopForPipelining.
TargetInstrInfo - Interface to description of machine instruction set.
virtual bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const
Returns true iff the routine could find two commutable operands in the given machine instruction.
virtual bool hasReassociableOperands(const MachineInstr &Inst, const MachineBasicBlock *MBB) const
Return true when \P Inst has reassociable operands in the same \P MBB.
virtual void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
virtual bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, unsigned &Flags) const
Optional target hook that returns true if MBB is safe to outline from, and returns any target-specifi...
virtual void getReassociateOperandIndices(const MachineInstr &Root, unsigned Pattern, std::array< unsigned, 5 > &OperandIndices) const
The returned array encodes the operand index for each parameter because the operands may be commuted;...
virtual bool isReallyTriviallyReMaterializable(const MachineInstr &MI) const
For instructions with opcodes for which the M_REMATERIALIZABLE flag is set, this hook lets the target...
virtual CombinerObjective getCombinerObjective(unsigned Pattern) const
Return the objective of a combiner pattern.
virtual MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const
This method commutes the operands of the given machine instruction MI.
virtual bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) const
Return true when \P Inst has reassociable sibling.
virtual std::string createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx, const TargetRegisterInfo *TRI) const
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
const uint8_t TSFlags
Configurable target specific flags.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
CondCode getOppositeBranchCondition(CondCode)
unsigned getBrCond(CondCode CC, bool Imm=false)
static bool isValidRoundingMode(unsigned Mode)
static unsigned getVecPolicyOpNum(const MCInstrDesc &Desc)
static bool usesMaskPolicy(uint64_t TSFlags)
static bool hasRoundModeOp(uint64_t TSFlags)
static unsigned getVLOpNum(const MCInstrDesc &Desc)
static bool hasVLOp(uint64_t TSFlags)
static int getFRMOpNum(const MCInstrDesc &Desc)
static bool hasVecPolicyOp(uint64_t TSFlags)
static bool usesVXRM(uint64_t TSFlags)
static unsigned getTailExpandUseRegNo(const FeatureBitset &FeatureBits)
static bool isRVVWideningReduction(uint64_t TSFlags)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
static bool hasSEWOp(uint64_t TSFlags)
static bool isFirstDefTiedToFirstUse(const MCInstrDesc &Desc)
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
@ OPERAND_UIMMLOG2XLEN_NONZERO
@ OPERAND_SIMM12_LSB00000
@ OPERAND_FIRST_RISCV_IMM
@ OPERAND_UIMM10_LSB00_NONZERO
@ OPERAND_SIMM10_LSB0000_NONZERO
static RISCVII::VLMUL getLMul(uint64_t TSFlags)
static unsigned getNF(uint64_t TSFlags)
static bool isTailAgnostic(unsigned VType)
static RISCVII::VLMUL getVLMUL(unsigned VType)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
static bool isValidSEW(unsigned SEW)
void printVType(unsigned VType, raw_ostream &OS)
static unsigned getSEW(unsigned VType)
bool hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2)
bool isVLKnownLE(const MachineOperand &LHS, const MachineOperand &RHS)
Given two VL operands, do we know that LHS <= RHS?
std::optional< unsigned > getVectorLowDemandedScalarBits(uint16_t Opcode, unsigned Log2SEW)
int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex)
unsigned getRVVMCOpcode(unsigned RVVPseudoOpcode)
unsigned getDestLog2EEW(const MCInstrDesc &Desc, unsigned Log2SEW)
bool isSEXT_W(const MachineInstr &MI)
bool isFaultFirstLoad(const MachineInstr &MI)
std::optional< std::pair< unsigned, unsigned > > isRVVSpillForZvlsseg(unsigned Opcode)
bool isZEXT_B(const MachineInstr &MI)
bool isRVVSpill(const MachineInstr &MI)
static constexpr int64_t VLMaxSentinel
bool isZEXT_W(const MachineInstr &MI)
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
InstrType
Represents how an instruction should be mapped by the outliner.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
MachineTraceStrategy
Strategies for selecting traces.
@ TS_MinInstrCount
Select the trace through a block that has the fewest instructions.
@ TS_Local
Select the trace that contains only the current basic block.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static const MachineMemOperand::Flags MONontemporalBit1
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
static const MachineMemOperand::Flags MONontemporalBit0
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
unsigned getDeadRegState(bool B)
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
CombinerObjective
The combiner's goal may differ based on which pattern it is attempting to optimize.
CodeGenOptLevel
Code generation optimization level.
unsigned getKillRegState(bool B)
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
unsigned getRenamableRegState(bool B)
DWARFExpression::Operation Op
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Description of the encoding of one expression Op.
Used to describe addressing mode similar to ExtAddrMode in CodeGenPrepare.
This represents a simple continuous liveness interval for a value.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
static bool isRVVRegClass(const TargetRegisterClass *RC)
Used to describe a register and immediate addition.
An individual sequence of instructions to be replaced with a call to an outlined function.
MachineFunction * getMF() const
The information necessary to create an outlined function for some class of candidate.
unsigned FrameConstructionID
Target-defined identifier for constructing a frame for this function.