LLVM: lib/Target/RISCV/RISCVInstrInfo.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
36
37using namespace llvm;
38
39#define GEN_CHECK_COMPRESS_INSTR
40#include "RISCVGenCompressInstEmitter.inc"
41
42#define GET_INSTRINFO_CTOR_DTOR
43#define GET_INSTRINFO_NAMED_OPS
44#include "RISCVGenInstrInfo.inc"
45
48 cl::desc("Prefer whole register move for vector registers."));
49
51 "riscv-force-machine-combiner-strategy", cl::Hidden,
52 cl::desc("Force machine combiner to use a specific strategy for machine "
53 "trace metrics evaluation."),
54 cl::init(MachineTraceStrategy::TS_NumStrategies),
56 "Local strategy."),
57 clEnumValN(MachineTraceStrategy::TS_MinInstrCount, "min-instr",
58 "MinInstrCount strategy.")));
59
61
62using namespace RISCV;
63
64#define GET_RISCVVPseudosTable_IMPL
65#include "RISCVGenSearchableTables.inc"
66
67}
68
70
71#define GET_RISCVMaskedPseudosTable_IMPL
72#include "RISCVGenSearchableTables.inc"
73
74}
75
77 : RISCVGenInstrInfo(RISCV::ADJCALLSTACKDOWN, RISCV::ADJCALLSTACKUP),
78 STI(STI) {}
79
87}
88
90 int &FrameIndex) const {
91 unsigned Dummy;
93}
94
96 int &FrameIndex,
97 unsigned &MemBytes) const {
98 switch (MI.getOpcode()) {
99 default:
100 return 0;
101 case RISCV::LB:
102 case RISCV::LBU:
103 MemBytes = 1;
104 break;
105 case RISCV::LH:
106 case RISCV::LH_INX:
107 case RISCV::LHU:
108 case RISCV::FLH:
109 MemBytes = 2;
110 break;
111 case RISCV::LW:
112 case RISCV::LW_INX:
113 case RISCV::FLW:
114 case RISCV::LWU:
115 MemBytes = 4;
116 break;
117 case RISCV::LD:
118 case RISCV::FLD:
119 MemBytes = 8;
120 break;
121 }
122
123 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
124 MI.getOperand(2).getImm() == 0) {
125 FrameIndex = MI.getOperand(1).getIndex();
126 return MI.getOperand(0).getReg();
127 }
128
129 return 0;
130}
131
133 int &FrameIndex) const {
134 unsigned Dummy;
136}
137
139 int &FrameIndex,
140 unsigned &MemBytes) const {
141 switch (MI.getOpcode()) {
142 default:
143 return 0;
144 case RISCV::SB:
145 MemBytes = 1;
146 break;
147 case RISCV::SH:
148 case RISCV::SH_INX:
149 case RISCV::FSH:
150 MemBytes = 2;
151 break;
152 case RISCV::SW:
153 case RISCV::SW_INX:
154 case RISCV::FSW:
155 MemBytes = 4;
156 break;
157 case RISCV::SD:
158 case RISCV::FSD:
159 MemBytes = 8;
160 break;
161 }
162
163 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
164 MI.getOperand(2).getImm() == 0) {
165 FrameIndex = MI.getOperand(1).getIndex();
166 return MI.getOperand(0).getReg();
167 }
168
169 return 0;
170}
171
175 case RISCV::VMV_V_X:
176 case RISCV::VFMV_V_F:
177 case RISCV::VMV_V_I:
178 case RISCV::VMV_S_X:
179 case RISCV::VFMV_S_F:
180 case RISCV::VID_V:
181 return MI.getOperand(1).isUndef();
182 default:
184 }
185}
186
188 unsigned NumRegs) {
189 return DstReg > SrcReg && (DstReg - SrcReg) < NumRegs;
190}
191
198 return false;
199
200 assert(MBBI->getOpcode() == TargetOpcode::COPY &&
201 "Unexpected COPY instruction.");
202 Register SrcReg = MBBI->getOperand(1).getReg();
204
205 bool FoundDef = false;
206 bool FirstVSetVLI = false;
207 unsigned FirstSEW = 0;
210 if (MBBI->isMetaInstruction())
211 continue;
212
213 if (MBBI->getOpcode() == RISCV::PseudoVSETVLI ||
214 MBBI->getOpcode() == RISCV::PseudoVSETVLIX0 ||
215 MBBI->getOpcode() == RISCV::PseudoVSETIVLI) {
216
217
218
219
220
221
222 if (!FoundDef) {
223 if (!FirstVSetVLI) {
224 FirstVSetVLI = true;
225 unsigned FirstVType = MBBI->getOperand(2).getImm();
228
229
230 if (FirstLMul != LMul)
231 return false;
232 }
233
234
235 if (MBBI->getOperand(0).getReg() != RISCV::X0)
236 return false;
237 if (MBBI->getOperand(1).isImm())
238 return false;
239 if (MBBI->getOperand(1).getReg() != RISCV::X0)
240 return false;
241 continue;
242 }
243
244
245 unsigned VType = MBBI->getOperand(2).getImm();
246
247 if (FirstVSetVLI) {
248
250 return false;
251 }
252
253
255 return false;
256
257
258
259
260
261
263 } else if (MBBI->isInlineAsm() || MBBI->isCall()) {
264 return false;
265 } else if (MBBI->getNumDefs()) {
266
267
268 if (MBBI->modifiesRegister(RISCV::VL, nullptr))
269 return false;
270
271
272
274 if (!MO.isReg() || !MO.isDef())
275 continue;
276 if (!FoundDef && TRI->regsOverlap(MO.getReg(), SrcReg)) {
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291 if (MO.getReg() != SrcReg)
292 return false;
293
294
295
296
297
298
299
300
301
302
305 return false;
306
307
308
310 return false;
311
312
313 FoundDef = true;
314 DefMBBI = MBBI;
315 break;
316 }
317 }
318 }
319 }
320
321 return false;
322}
323
331
332 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
333 uint16_t DstEncoding = TRI->getEncodingValue(DstReg);
335 assert(!Fractional && "It is impossible be fractional lmul here.");
336 unsigned NumRegs = NF * LMulVal;
337 bool ReversedCopy =
339 if (ReversedCopy) {
340
341
342 SrcEncoding += NumRegs - 1;
343 DstEncoding += NumRegs - 1;
344 }
345
346 unsigned I = 0;
347 auto GetCopyInfo = [&](uint16_t SrcEncoding, uint16_t DstEncoding)
349 unsigned, unsigned> {
350 if (ReversedCopy) {
351
352
353
354
355
356
357 uint16_t Diff = DstEncoding - SrcEncoding;
358 if (I + 8 <= NumRegs && Diff >= 8 && SrcEncoding % 8 == 7 &&
359 DstEncoding % 8 == 7)
360 return {RISCVII::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
361 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
362 if (I + 4 <= NumRegs && Diff >= 4 && SrcEncoding % 4 == 3 &&
363 DstEncoding % 4 == 3)
364 return {RISCVII::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
365 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
366 if (I + 2 <= NumRegs && Diff >= 2 && SrcEncoding % 2 == 1 &&
367 DstEncoding % 2 == 1)
368 return {RISCVII::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
369 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
370
371 return {RISCVII::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
372 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
373 }
374
375
376
377 if (I + 8 <= NumRegs && SrcEncoding % 8 == 0 && DstEncoding % 8 == 0)
378 return {RISCVII::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
379 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
380 if (I + 4 <= NumRegs && SrcEncoding % 4 == 0 && DstEncoding % 4 == 0)
381 return {RISCVII::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
382 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
383 if (I + 2 <= NumRegs && SrcEncoding % 2 == 0 && DstEncoding % 2 == 0)
384 return {RISCVII::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
385 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
386
387 return {RISCVII::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
388 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
389 };
392 MCRegister Reg = RISCV::V0 + Encoding;
394 return Reg;
395 return TRI->getMatchingSuperReg(Reg, RISCV::sub_vrm1_0, &RegClass);
396 };
397 while (I != NumRegs) {
398
399
400
401
402 auto [LMulCopied, RegClass, Opc, VVOpc, VIOpc] =
403 GetCopyInfo(SrcEncoding, DstEncoding);
405
407 if (LMul == LMulCopied &&
409 Opc = VVOpc;
410 if (DefMBBI->getOpcode() == VIOpc)
411 Opc = VIOpc;
412 }
413
414
415
416 MCRegister ActualSrcReg = FindRegWithEncoding(
417 RegClass, ReversedCopy ? (SrcEncoding - NumCopied + 1) : SrcEncoding);
418 MCRegister ActualDstReg = FindRegWithEncoding(
419 RegClass, ReversedCopy ? (DstEncoding - NumCopied + 1) : DstEncoding);
420
424 if (UseVMV)
426 if (UseVMV_V_I)
427 MIB = MIB.add(DefMBBI->getOperand(2));
428 else
429 MIB = MIB.addReg(ActualSrcReg, getKillRegState(KillSrc));
430 if (UseVMV) {
433 unsigned Log2SEW =
435 MIB.addImm(Log2SEW ? Log2SEW : 3);
436 MIB.addImm(0);
439 }
440
441
442 SrcEncoding += (ReversedCopy ? -NumCopied : NumCopied);
443 DstEncoding += (ReversedCopy ? -NumCopied : NumCopied);
444 I += NumCopied;
445 }
446}
447
452 bool RenamableDest, bool RenamableSrc) const {
454
455 if (RISCV::GPRRegClass.contains(DstReg, SrcReg)) {
460 return;
461 }
462
463 if (RISCV::GPRF16RegClass.contains(DstReg, SrcReg)) {
467 return;
468 }
469
470 if (RISCV::GPRF32RegClass.contains(DstReg, SrcReg)) {
474 return;
475 }
476
477 if (RISCV::GPRPairRegClass.contains(DstReg, SrcReg)) {
478
480 TRI->getSubReg(DstReg, RISCV::sub_gpr_even))
481 .addReg(TRI->getSubReg(SrcReg, RISCV::sub_gpr_even),
485 TRI->getSubReg(DstReg, RISCV::sub_gpr_odd))
486 .addReg(TRI->getSubReg(SrcReg, RISCV::sub_gpr_odd),
489 return;
490 }
491
492
493 if (RISCV::VCSRRegClass.contains(SrcReg) &&
494 RISCV::GPRRegClass.contains(DstReg)) {
496 .addImm(RISCVSysReg::lookupSysRegByName(TRI->getName(SrcReg))->Encoding)
498 return;
499 }
500
501 if (RISCV::FPR16RegClass.contains(DstReg, SrcReg)) {
502 unsigned Opc;
503 if (STI.hasStdExtZfh()) {
504 Opc = RISCV::FSGNJ_H;
505 } else {
507 (STI.hasStdExtZfhmin() || STI.hasStdExtZfbfmin()) &&
508 "Unexpected extensions");
509
510 DstReg = TRI->getMatchingSuperReg(DstReg, RISCV::sub_16,
511 &RISCV::FPR32RegClass);
512 SrcReg = TRI->getMatchingSuperReg(SrcReg, RISCV::sub_16,
513 &RISCV::FPR32RegClass);
514 Opc = RISCV::FSGNJ_S;
515 }
519 return;
520 }
521
522 if (RISCV::FPR32RegClass.contains(DstReg, SrcReg)) {
526 return;
527 }
528
529 if (RISCV::FPR64RegClass.contains(DstReg, SrcReg)) {
533 return;
534 }
535
536 if (RISCV::FPR32RegClass.contains(DstReg) &&
537 RISCV::GPRRegClass.contains(SrcReg)) {
540 return;
541 }
542
543 if (RISCV::GPRRegClass.contains(DstReg) &&
544 RISCV::FPR32RegClass.contains(SrcReg)) {
547 return;
548 }
549
550 if (RISCV::FPR64RegClass.contains(DstReg) &&
551 RISCV::GPRRegClass.contains(SrcReg)) {
555 return;
556 }
557
558 if (RISCV::GPRRegClass.contains(DstReg) &&
559 RISCV::FPR64RegClass.contains(SrcReg)) {
563 return;
564 }
565
566
568 TRI->getCommonMinimalPhysRegClass(SrcReg, DstReg);
571 return;
572 }
573
575}
576
579 Register SrcReg, bool IsKill, int FI,
585
586 unsigned Opcode;
587 bool IsScalableVector = true;
588 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
589 Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
590 RISCV::SW : RISCV::SD;
591 IsScalableVector = false;
592 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
593 Opcode = RISCV::SH_INX;
594 IsScalableVector = false;
595 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
596 Opcode = RISCV::SW_INX;
597 IsScalableVector = false;
598 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
599 Opcode = RISCV::PseudoRV32ZdinxSD;
600 IsScalableVector = false;
601 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
602 Opcode = RISCV::FSH;
603 IsScalableVector = false;
604 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
605 Opcode = RISCV::FSW;
606 IsScalableVector = false;
607 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
608 Opcode = RISCV::FSD;
609 IsScalableVector = false;
610 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
611 Opcode = RISCV::VS1R_V;
612 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
613 Opcode = RISCV::VS2R_V;
614 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
615 Opcode = RISCV::VS4R_V;
616 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
617 Opcode = RISCV::VS8R_V;
618 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
619 Opcode = RISCV::PseudoVSPILL2_M1;
620 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
621 Opcode = RISCV::PseudoVSPILL2_M2;
622 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
623 Opcode = RISCV::PseudoVSPILL2_M4;
624 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
625 Opcode = RISCV::PseudoVSPILL3_M1;
626 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
627 Opcode = RISCV::PseudoVSPILL3_M2;
628 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
629 Opcode = RISCV::PseudoVSPILL4_M1;
630 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
631 Opcode = RISCV::PseudoVSPILL4_M2;
632 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
633 Opcode = RISCV::PseudoVSPILL5_M1;
634 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
635 Opcode = RISCV::PseudoVSPILL6_M1;
636 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
637 Opcode = RISCV::PseudoVSPILL7_M1;
638 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
639 Opcode = RISCV::PseudoVSPILL8_M1;
640 else
642
643 if (IsScalableVector) {
647
653 } else {
657
663 }
664}
665
674
675 unsigned Opcode;
676 bool IsScalableVector = true;
677 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
678 Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
679 RISCV::LW : RISCV::LD;
680 IsScalableVector = false;
681 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
682 Opcode = RISCV::LH_INX;
683 IsScalableVector = false;
684 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
685 Opcode = RISCV::LW_INX;
686 IsScalableVector = false;
687 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
688 Opcode = RISCV::PseudoRV32ZdinxLD;
689 IsScalableVector = false;
690 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
691 Opcode = RISCV::FLH;
692 IsScalableVector = false;
693 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
694 Opcode = RISCV::FLW;
695 IsScalableVector = false;
696 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
697 Opcode = RISCV::FLD;
698 IsScalableVector = false;
699 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
700 Opcode = RISCV::VL1RE8_V;
701 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
702 Opcode = RISCV::VL2RE8_V;
703 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
704 Opcode = RISCV::VL4RE8_V;
705 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
706 Opcode = RISCV::VL8RE8_V;
707 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
708 Opcode = RISCV::PseudoVRELOAD2_M1;
709 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
710 Opcode = RISCV::PseudoVRELOAD2_M2;
711 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
712 Opcode = RISCV::PseudoVRELOAD2_M4;
713 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
714 Opcode = RISCV::PseudoVRELOAD3_M1;
715 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
716 Opcode = RISCV::PseudoVRELOAD3_M2;
717 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
718 Opcode = RISCV::PseudoVRELOAD4_M1;
719 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
720 Opcode = RISCV::PseudoVRELOAD4_M2;
721 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
722 Opcode = RISCV::PseudoVRELOAD5_M1;
723 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
724 Opcode = RISCV::PseudoVRELOAD6_M1;
725 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
726 Opcode = RISCV::PseudoVRELOAD7_M1;
727 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
728 Opcode = RISCV::PseudoVRELOAD8_M1;
729 else
730 llvm_unreachable("Can't load this register from stack slot");
731
732 if (IsScalableVector) {
736
741 } else {
745
750 }
751}
752
757
758
759
761 return nullptr;
762
763
764 if (Ops.size() != 1 || Ops[0] != 1)
765 return nullptr;
766
767 unsigned LoadOpc;
768 switch (MI.getOpcode()) {
769 default:
771 LoadOpc = RISCV::LW;
772 break;
773 }
775 LoadOpc = RISCV::LWU;
776 break;
777 }
779 LoadOpc = RISCV::LBU;
780 break;
781 }
783 unsigned Log2SEW =
786 return nullptr;
787 switch (Log2SEW) {
788 case 3:
789 LoadOpc = RISCV::LB;
790 break;
791 case 4:
792 LoadOpc = RISCV::LH;
793 break;
794 case 5:
795 LoadOpc = RISCV::LW;
796 break;
797 case 6:
798 LoadOpc = RISCV::LD;
799 break;
800 default:
802 }
803 break;
804 }
806 unsigned Log2SEW =
808 switch (Log2SEW) {
809 case 4:
810 LoadOpc = RISCV::FLH;
811 break;
812 case 5:
813 LoadOpc = RISCV::FLW;
814 break;
815 case 6:
816 LoadOpc = RISCV::FLD;
817 break;
818 default:
820 }
821 break;
822 }
823 return nullptr;
824 case RISCV::SEXT_H:
825 LoadOpc = RISCV::LH;
826 break;
827 case RISCV::SEXT_B:
828 LoadOpc = RISCV::LB;
829 break;
830 case RISCV::ZEXT_H_RV32:
831 case RISCV::ZEXT_H_RV64:
832 LoadOpc = RISCV::LHU;
833 break;
834 }
835
836 Register DstReg = MI.getOperand(0).getReg();
837 return BuildMI(*MI.getParent(), InsertPt, MI.getDebugLoc(), get(LoadOpc),
838 DstReg)
841}
842
847 bool DstIsDead) const {
849
850
851 if (.is64Bit() && !isInt<32>(Val)) {
852
853 if (!isUInt<32>(Val))
854 report_fatal_error("Should only materialize 32-bit constants for RV32");
855
856
857 Val = SignExtend64<32>(Val);
858 }
859
862
863 bool SrcRenamable = false;
864 unsigned Num = 0;
865
867 bool LastItem = ++Num == Seq.size();
868 unsigned DstRegState = getDeadRegState(DstIsDead && LastItem) |
870 unsigned SrcRegState = getKillRegState(SrcReg != RISCV::X0) |
872 switch (Inst.getOpndKind()) {
876 .addImm(Inst.getImm())
878 break;
882 .addReg(SrcReg, SrcRegState)
885 break;
889 .addReg(SrcReg, SrcRegState)
890 .addReg(SrcReg, SrcRegState)
892 break;
896 .addReg(SrcReg, SrcRegState)
897 .addImm(Inst.getImm())
899 break;
900 }
901
902
903 SrcReg = DstReg;
904 SrcRenamable = DstRenamable;
905 }
906}
907
909 switch (Opc) {
910 default:
912 case RISCV::CV_BEQIMM:
914 case RISCV::CV_BNEIMM:
916 case RISCV::BEQ:
918 case RISCV::BNE:
920 case RISCV::BLT:
922 case RISCV::BGE:
924 case RISCV::BLTU:
926 case RISCV::BGEU:
928 }
929}
930
931
932
933
936
938 "Unknown conditional branch");
944}
945
947 switch (CC) {
948 default:
951 return Imm ? RISCV::CV_BEQIMM : RISCV::BEQ;
953 return Imm ? RISCV::CV_BNEIMM : RISCV::BNE;
955 return RISCV::BLT;
957 return RISCV::BGE;
959 return RISCV::BLTU;
961 return RISCV::BGEU;
962 }
963}
964
966 bool Imm) const {
968}
969
971 switch (CC) {
972 default:
986 }
987}
988
993 bool AllowModify) const {
994 TBB = FBB = nullptr;
995 Cond.clear();
996
997
999 if (I == MBB.end() || !isUnpredicatedTerminator(*I))
1000 return false;
1001
1002
1003
1005 int NumTerminators = 0;
1006 for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(*J);
1007 J++) {
1008 NumTerminators++;
1009 if (J->getDesc().isUnconditionalBranch() ||
1010 J->getDesc().isIndirectBranch()) {
1011 FirstUncondOrIndirectBr = J.getReverse();
1012 }
1013 }
1014
1015
1016
1017 if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) {
1018 while (std::next(FirstUncondOrIndirectBr) != MBB.end()) {
1019 std::next(FirstUncondOrIndirectBr)->eraseFromParent();
1020 NumTerminators--;
1021 }
1022 I = FirstUncondOrIndirectBr;
1023 }
1024
1025
1026 if (I->getDesc().isIndirectBranch())
1027 return true;
1028
1029
1030 if (I->isPreISelOpcode())
1031 return true;
1032
1033
1034 if (NumTerminators > 2)
1035 return true;
1036
1037
1038 if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) {
1040 return false;
1041 }
1042
1043
1044 if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) {
1046 return false;
1047 }
1048
1049
1050 if (NumTerminators == 2 && std::prev(I)->getDesc().isConditionalBranch() &&
1051 I->getDesc().isUnconditionalBranch()) {
1054 return false;
1055 }
1056
1057
1058 return true;
1059}
1060
1062 int *BytesRemoved) const {
1063 if (BytesRemoved)
1064 *BytesRemoved = 0;
1067 return 0;
1068
1069 if (->getDesc().isUnconditionalBranch() &&
1070 ->getDesc().isConditionalBranch())
1071 return 0;
1072
1073
1074 if (BytesRemoved)
1076 I->eraseFromParent();
1077
1079
1081 return 1;
1082 --I;
1083 if (->getDesc().isConditionalBranch())
1084 return 1;
1085
1086
1087 if (BytesRemoved)
1089 I->eraseFromParent();
1090 return 2;
1091}
1092
1093
1094
1098 if (BytesAdded)
1099 *BytesAdded = 0;
1100
1101
1102 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1104 "RISC-V branch conditions have two components!");
1105
1106
1107 if (Cond.empty()) {
1109 if (BytesAdded)
1111 return 1;
1112 }
1113
1114
1120 if (BytesAdded)
1122
1123
1124 if (!FBB)
1125 return 1;
1126
1127
1129 if (BytesAdded)
1131 return 2;
1132}
1133
1137 const DebugLoc &DL, int64_t BrOffset,
1139 assert(RS && "RegScavenger required for long branching");
1141 "new block should be inserted for expanding unconditional branch");
1144 "restore block should be inserted for restoring clobbered registers");
1145
1150
1151 if (!isInt<32>(BrOffset))
1153 "Branch offsets outside of the signed 32-bit range not supported");
1154
1155
1156
1157
1158 Register ScratchReg = MRI.createVirtualRegister(&RISCV::GPRJALRRegClass);
1160
1164
1168 false, 0,
1169 false);
1170 if (TmpGPR != RISCV::NoRegister)
1172 else {
1173
1174
1175
1176 TmpGPR = RISCV::X27;
1177
1179 if (FrameIndex == -1)
1181
1184 TRI->eliminateFrameIndex(std::prev(MI.getIterator()),
1185 0, 1);
1186
1187 MI.getOperand(1).setMBB(&RestoreBB);
1188
1191 TRI->eliminateFrameIndex(RestoreBB.back(),
1192 0, 1);
1193 }
1194
1195 MRI.replaceRegWith(ScratchReg, TmpGPR);
1196 MRI.clearVirtRegs();
1197}
1198
1201 assert((Cond.size() == 3) && "Invalid branch condition!");
1204 return false;
1205}
1206
1210
1214 return false;
1215
1218
1220 return false;
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241 auto isLoadImm = [](const MachineInstr *MI, int64_t &Imm) -> bool {
1242 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(1).isReg() &&
1243 MI->getOperand(1).getReg() == RISCV::X0) {
1244 Imm = MI->getOperand(2).getImm();
1245 return true;
1246 }
1247 return false;
1248 };
1249
1250 auto isFromLoadImm = [&](const MachineOperand &Op, int64_t &Imm) -> bool {
1251 if (.isReg())
1252 return false;
1254 return Reg.isVirtual() && isLoadImm(MRI.getVRegDef(Reg), Imm);
1255 };
1256
1259
1260
1261 auto searchConst = [&](int64_t C1) -> Register {
1263 auto DefC1 = std::find_if(++II, E, [&](const MachineInstr &I) -> bool {
1264 int64_t Imm;
1265 return isLoadImm(&I, Imm) && Imm == C1 &&
1266 I.getOperand(0).getReg().isVirtual();
1267 });
1268 if (DefC1 != E)
1269 return DefC1->getOperand(0).getReg();
1270
1272 };
1273
1274 bool Modify = false;
1275 int64_t C0;
1276 if (isFromLoadImm(LHS, C0) && MRI.hasOneUse(LHS.getReg())) {
1277
1278
1279
1281 if (Register RegZ = searchConst(C0 + 1)) {
1285
1286
1287 MRI.clearKillFlags(RegZ);
1288 Modify = true;
1289 }
1290 } else if (isFromLoadImm(RHS, C0) && MRI.hasOneUse(RHS.getReg())) {
1291
1292
1293
1295 if (Register RegZ = searchConst(C0 - 1)) {
1299
1300
1301 MRI.clearKillFlags(RegZ);
1302 Modify = true;
1303 }
1304 }
1305
1306 if (!Modify)
1307 return false;
1308
1309
1315 MI.eraseFromParent();
1316
1317 return true;
1318}
1319
1322 assert(MI.getDesc().isBranch() && "Unexpected opcode!");
1323
1324 int NumOp = MI.getNumExplicitOperands();
1325 return MI.getOperand(NumOp - 1).getMBB();
1326}
1327
1329 int64_t BrOffset) const {
1331
1332
1333
1334 switch (BranchOp) {
1335 default:
1337 case RISCV::BEQ:
1338 case RISCV::BNE:
1339 case RISCV::BLT:
1340 case RISCV::BGE:
1341 case RISCV::BLTU:
1342 case RISCV::BGEU:
1343 case RISCV::CV_BEQIMM:
1344 case RISCV::CV_BNEIMM:
1345 return isIntN(13, BrOffset);
1346 case RISCV::JAL:
1347 case RISCV::PseudoBR:
1348 return isIntN(21, BrOffset);
1349 case RISCV::PseudoJump:
1351 }
1352}
1353
1354
1355
1356
1358 switch (Opcode) {
1359 case RISCV::ADD: return RISCV::PseudoCCADD; break;
1360 case RISCV::SUB: return RISCV::PseudoCCSUB; break;
1361 case RISCV::SLL: return RISCV::PseudoCCSLL; break;
1362 case RISCV::SRL: return RISCV::PseudoCCSRL; break;
1363 case RISCV::SRA: return RISCV::PseudoCCSRA; break;
1364 case RISCV::AND: return RISCV::PseudoCCAND; break;
1365 case RISCV::OR: return RISCV::PseudoCCOR; break;
1366 case RISCV::XOR: return RISCV::PseudoCCXOR; break;
1367
1368 case RISCV::ADDI: return RISCV::PseudoCCADDI; break;
1369 case RISCV::SLLI: return RISCV::PseudoCCSLLI; break;
1370 case RISCV::SRLI: return RISCV::PseudoCCSRLI; break;
1371 case RISCV::SRAI: return RISCV::PseudoCCSRAI; break;
1372 case RISCV::ANDI: return RISCV::PseudoCCANDI; break;
1373 case RISCV::ORI: return RISCV::PseudoCCORI; break;
1374 case RISCV::XORI: return RISCV::PseudoCCXORI; break;
1375
1376 case RISCV::ADDW: return RISCV::PseudoCCADDW; break;
1377 case RISCV::SUBW: return RISCV::PseudoCCSUBW; break;
1378 case RISCV::SLLW: return RISCV::PseudoCCSLLW; break;
1379 case RISCV::SRLW: return RISCV::PseudoCCSRLW; break;
1380 case RISCV::SRAW: return RISCV::PseudoCCSRAW; break;
1381
1382 case RISCV::ADDIW: return RISCV::PseudoCCADDIW; break;
1383 case RISCV::SLLIW: return RISCV::PseudoCCSLLIW; break;
1384 case RISCV::SRLIW: return RISCV::PseudoCCSRLIW; break;
1385 case RISCV::SRAIW: return RISCV::PseudoCCSRAIW; break;
1386
1387 case RISCV::ANDN: return RISCV::PseudoCCANDN; break;
1388 case RISCV::ORN: return RISCV::PseudoCCORN; break;
1389 case RISCV::XNOR: return RISCV::PseudoCCXNOR; break;
1390 }
1391
1392 return RISCV::INSTRUCTION_LIST_END;
1393}
1394
1395
1396
1400 if (!Reg.isVirtual())
1401 return nullptr;
1402 if (.hasOneNonDBGUse(Reg))
1403 return nullptr;
1405 if ()
1406 return nullptr;
1407
1409 return nullptr;
1410
1411 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(1).isReg() &&
1412 MI->getOperand(1).getReg() == RISCV::X0)
1413 return nullptr;
1414
1416
1417 if (MO.isFI() || MO.isCPI() || MO.isJTI())
1418 return nullptr;
1419 if (!MO.isReg())
1420 continue;
1421
1422 if (MO.isTied())
1423 return nullptr;
1424 if (MO.isDef())
1425 return nullptr;
1426
1427 if (MO.getReg().isPhysical() && .isConstantPhysReg(MO.getReg()))
1428 return nullptr;
1429 }
1430 bool DontMoveAcrossStores = true;
1431 if (->isSafeToMove(DontMoveAcrossStores))
1432 return nullptr;
1433 return MI;
1434}
1435
1438 unsigned &TrueOp, unsigned &FalseOp,
1439 bool &Optimizable) const {
1440 assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1441 "Unknown select instruction");
1442
1443
1444
1445
1446
1447
1448
1449 TrueOp = 5;
1450 FalseOp = 4;
1451 Cond.push_back(MI.getOperand(1));
1452 Cond.push_back(MI.getOperand(2));
1453 Cond.push_back(MI.getOperand(3));
1454
1455 Optimizable = STI.hasShortForwardBranchOpt();
1456 return false;
1457}
1458
1462 bool PreferFalse) const {
1463 assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1464 "Unknown select instruction");
1465 if (.hasShortForwardBranchOpt())
1466 return nullptr;
1467
1471 bool Invert = ;
1475 return nullptr;
1476
1477
1479 Register DestReg = MI.getOperand(0).getReg();
1481 if (.constrainRegClass(DestReg, PreviousClass))
1482 return nullptr;
1483
1485 assert(PredOpc != RISCV::INSTRUCTION_LIST_END && "Unexpected opcode!");
1486
1487
1489 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(PredOpc), DestReg);
1490
1491
1492 NewMI.add(MI.getOperand(1));
1493 NewMI.add(MI.getOperand(2));
1494
1495
1497 if (Invert)
1500
1501
1502 NewMI.add(FalseReg);
1503
1504
1506 for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
1508
1509
1510 SeenMIs.insert(NewMI);
1512
1513
1514
1515
1516
1519
1520
1522 return NewMI;
1523}
1524
1526 if (MI.isMetaInstruction())
1527 return 0;
1528
1529 unsigned Opcode = MI.getOpcode();
1530
1531 if (Opcode == TargetOpcode::INLINEASM ||
1532 Opcode == TargetOpcode::INLINEASM_BR) {
1534 return getInlineAsmLength(MI.getOperand(0).getSymbolName(),
1536 }
1537
1538 if (.memoperands_empty()) {
1542 if (isCompressibleInst(MI, STI))
1543 return 4;
1544 return 6;
1545 }
1546 return 8;
1547 }
1548 }
1549
1550 if (Opcode == TargetOpcode::BUNDLE)
1551 return getInstBundleLength(MI);
1552
1553 if (MI.getParent() && MI.getParent()->getParent()) {
1554 if (isCompressibleInst(MI, STI))
1555 return 2;
1556 }
1557
1558 switch (Opcode) {
1559 case RISCV::PseudoMV_FPR16INX:
1560 case RISCV::PseudoMV_FPR32INX:
1561
1563 case TargetOpcode::STACKMAP:
1564
1566 case TargetOpcode::PATCHPOINT:
1567
1569 case TargetOpcode::STATEPOINT: {
1570
1572
1573 return std::max(NumBytes, 8U);
1574 }
1575 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1576 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1577 case TargetOpcode::PATCHABLE_TAIL_CALL: {
1580 if (Opcode == TargetOpcode::PATCHABLE_FUNCTION_ENTER &&
1581 F.hasFnAttribute("patchable-function-entry")) {
1582 unsigned Num;
1583 if (F.getFnAttribute("patchable-function-entry")
1584 .getValueAsString()
1585 .getAsInteger(10, Num))
1586 return get(Opcode).getSize();
1587
1588
1590 }
1591
1592
1594 }
1595 default:
1596 return get(Opcode).getSize();
1597 }
1598}
1599
1600unsigned RISCVInstrInfo::getInstBundleLength(const MachineInstr &MI) const {
1601 unsigned Size = 0;
1604 while (++I != E && I->isInsideBundle()) {
1605 assert(->isBundle() && "No nested bundle!");
1607 }
1608 return Size;
1609}
1610
1612 const unsigned Opcode = MI.getOpcode();
1613 switch (Opcode) {
1614 default:
1615 break;
1616 case RISCV::FSGNJ_D:
1617 case RISCV::FSGNJ_S:
1618 case RISCV::FSGNJ_H:
1619 case RISCV::FSGNJ_D_INX:
1620 case RISCV::FSGNJ_D_IN32X:
1621 case RISCV::FSGNJ_S_INX:
1622 case RISCV::FSGNJ_H_INX:
1623
1624 return MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
1625 MI.getOperand(1).getReg() == MI.getOperand(2).getReg();
1626 case RISCV::ADDI:
1627 case RISCV::ORI:
1628 case RISCV::XORI:
1629 return (MI.getOperand(1).isReg() &&
1630 MI.getOperand(1).getReg() == RISCV::X0) ||
1631 (MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0);
1632 }
1633 return MI.isAsCheapAsAMove();
1634}
1635
1636std::optional
1638 if (MI.isMoveReg())
1640 switch (MI.getOpcode()) {
1641 default:
1642 break;
1643 case RISCV::ADDI:
1644
1645 if (MI.getOperand(1).isReg() && MI.getOperand(2).isImm() &&
1646 MI.getOperand(2).getImm() == 0)
1648 break;
1649 case RISCV::FSGNJ_D:
1650 case RISCV::FSGNJ_S:
1651 case RISCV::FSGNJ_H:
1652 case RISCV::FSGNJ_D_INX:
1653 case RISCV::FSGNJ_D_IN32X:
1654 case RISCV::FSGNJ_S_INX:
1655 case RISCV::FSGNJ_H_INX:
1656
1657 if (MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
1658 MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
1660 break;
1661 }
1662 return std::nullopt;
1663}
1664
1667
1668
1669
1670 const auto &SchedModel = STI.getSchedModel();
1671 return (!SchedModel.hasInstrSchedModel() || SchedModel.isOutOfOrder())
1674 }
1675
1677}
1678
1682 int16_t FrmOpIdx =
1684 if (FrmOpIdx < 0) {
1688 RISCV::OpName::frm) < 0;
1689 }) &&
1690 "New instructions require FRM whereas the old one does not have it");
1691 return;
1692 }
1693
1696
1697 for (auto *NewMI : InsInstrs) {
1698
1700 NewMI->getOpcode(), RISCV::OpName::frm)) != NewMI->getNumOperands())
1701 continue;
1703 MIB.add(FRM);
1706 }
1707}
1708
1710 switch (Opc) {
1711 default:
1712 return false;
1713 case RISCV::FADD_H:
1714 case RISCV::FADD_S:
1715 case RISCV::FADD_D:
1716 return true;
1717 }
1718}
1719
1721 switch (Opc) {
1722 default:
1723 return false;
1724 case RISCV::FSUB_H:
1725 case RISCV::FSUB_S:
1726 case RISCV::FSUB_D:
1727 return true;
1728 }
1729}
1730
1732 switch (Opc) {
1733 default:
1734 return false;
1735 case RISCV::FMUL_H:
1736 case RISCV::FMUL_S:
1737 case RISCV::FMUL_D:
1738 return true;
1739 }
1740}
1741
1742bool RISCVInstrInfo::isVectorAssociativeAndCommutative(const MachineInstr &Inst,
1743 bool Invert) const {
1744#define OPCODE_LMUL_CASE(OPC) \
1745 case RISCV::OPC##_M1: \
1746 case RISCV::OPC##_M2: \
1747 case RISCV::OPC##_M4: \
1748 case RISCV::OPC##_M8: \
1749 case RISCV::OPC##_MF2: \
1750 case RISCV::OPC##_MF4: \
1751 case RISCV::OPC##_MF8
1752
1753#define OPCODE_LMUL_MASK_CASE(OPC) \
1754 case RISCV::OPC##_M1_MASK: \
1755 case RISCV::OPC##_M2_MASK: \
1756 case RISCV::OPC##_M4_MASK: \
1757 case RISCV::OPC##_M8_MASK: \
1758 case RISCV::OPC##_MF2_MASK: \
1759 case RISCV::OPC##_MF4_MASK: \
1760 case RISCV::OPC##_MF8_MASK
1761
1762 unsigned Opcode = Inst.getOpcode();
1763 if (Invert) {
1765 Opcode = *InvOpcode;
1766 else
1767 return false;
1768 }
1769
1770
1771 switch (Opcode) {
1772 default:
1773 return false;
1778 return true;
1779 }
1780
1781
1782#undef OPCODE_LMUL_MASK_CASE
1783#undef OPCODE_LMUL_CASE
1784}
1785
1786bool RISCVInstrInfo::areRVVInstsReassociable(const MachineInstr &Root,
1789 return false;
1790
1794
1795
1798
1799 auto checkImmOperand = [&](unsigned OpIdx) {
1801 };
1802
1803 auto checkRegOperand = [&](unsigned OpIdx) {
1805 };
1806
1807
1808
1809
1810
1811 if (!checkRegOperand(1))
1812 return false;
1813
1814
1817 return false;
1818
1819
1825
1826 bool SeenMI2 = false;
1827 for (auto End = MBB->rend(), It = It1; It != End; ++It) {
1828 if (It == It2) {
1829 SeenMI2 = true;
1831
1832
1833 break;
1834 }
1835
1836 if (It->modifiesRegister(RISCV::V0, TRI)) {
1837 Register SrcReg = It->getOperand(1).getReg();
1838
1839
1841 return false;
1842
1843 if (!MI1VReg.isValid()) {
1844
1845 MI1VReg = SrcReg;
1846 continue;
1847 }
1848
1849
1850 if (!SeenMI2)
1851 continue;
1852
1853
1854
1855 if (MI1VReg != SrcReg)
1856 return false;
1857 else
1858 break;
1859 }
1860 }
1861
1862
1863
1864 assert(SeenMI2 && "Prev is expected to appear before Root");
1865 }
1866
1867
1870 return false;
1871
1872
1878 return false;
1879 switch (Op1.getType()) {
1882 return false;
1883 break;
1886 return false;
1887 break;
1888 default:
1890 }
1891 }
1892
1893
1896 return false;
1897
1898 return true;
1899}
1900
1901
1902
1903bool RISCVInstrInfo::hasReassociableVectorSibling(const MachineInstr &Inst,
1904 bool &Commuted) const {
1908 "Expect the present of passthrough operand.");
1911
1912
1913
1914 Commuted = !areRVVInstsReassociable(Inst, *MI1) &&
1915 areRVVInstsReassociable(Inst, *MI2);
1916 if (Commuted)
1918
1919 return areRVVInstsReassociable(Inst, *MI1) &&
1920 (isVectorAssociativeAndCommutative(*MI1) ||
1921 isVectorAssociativeAndCommutative(*MI1, true)) &&
1924}
1925
1928 if (!isVectorAssociativeAndCommutative(Inst) &&
1929 !isVectorAssociativeAndCommutative(Inst, true))
1931
1935
1936
1937
1941 MI1 = MRI.getUniqueVRegDef(Op1.getReg());
1943 MI2 = MRI.getUniqueVRegDef(Op2.getReg());
1944
1945
1947}
1948
1954
1955 for (unsigned I = 0; I < 5; ++I)
1957 }
1958}
1959
1961 bool &Commuted) const {
1962 if (isVectorAssociativeAndCommutative(Inst) ||
1963 isVectorAssociativeAndCommutative(Inst, true))
1964 return hasReassociableVectorSibling(Inst, Commuted);
1965
1967 return false;
1968
1970 unsigned OperandIdx = Commuted ? 2 : 1;
1973
1974 int16_t InstFrmOpIdx =
1976 int16_t SiblingFrmOpIdx =
1978
1979 return (InstFrmOpIdx < 0 && SiblingFrmOpIdx < 0) ||
1981}
1982
1984 bool Invert) const {
1985 if (isVectorAssociativeAndCommutative(Inst, Invert))
1986 return true;
1987
1988 unsigned Opc = Inst.getOpcode();
1989 if (Invert) {
1991 if (!InverseOpcode)
1992 return false;
1993 Opc = *InverseOpcode;
1994 }
1995
1999
2000 switch (Opc) {
2001 default:
2002 return false;
2003 case RISCV::ADD:
2004 case RISCV::ADDW:
2005 case RISCV::AND:
2006 case RISCV::OR:
2007 case RISCV::XOR:
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020 case RISCV::MUL:
2021 case RISCV::MULW:
2022 case RISCV::MIN:
2023 case RISCV::MINU:
2024 case RISCV::MAX:
2025 case RISCV::MAXU:
2026 case RISCV::FMIN_H:
2027 case RISCV::FMIN_S:
2028 case RISCV::FMIN_D:
2029 case RISCV::FMAX_H:
2030 case RISCV::FMAX_S:
2031 case RISCV::FMAX_D:
2032 return true;
2033 }
2034
2035 return false;
2036}
2037
2038std::optional
2040#define RVV_OPC_LMUL_CASE(OPC, INV) \
2041 case RISCV::OPC##_M1: \
2042 return RISCV::INV##_M1; \
2043 case RISCV::OPC##_M2: \
2044 return RISCV::INV##_M2; \
2045 case RISCV::OPC##_M4: \
2046 return RISCV::INV##_M4; \
2047 case RISCV::OPC##_M8: \
2048 return RISCV::INV##_M8; \
2049 case RISCV::OPC##_MF2: \
2050 return RISCV::INV##_MF2; \
2051 case RISCV::OPC##_MF4: \
2052 return RISCV::INV##_MF4; \
2053 case RISCV::OPC##_MF8: \
2054 return RISCV::INV##_MF8
2055
2056#define RVV_OPC_LMUL_MASK_CASE(OPC, INV) \
2057 case RISCV::OPC##_M1_MASK: \
2058 return RISCV::INV##_M1_MASK; \
2059 case RISCV::OPC##_M2_MASK: \
2060 return RISCV::INV##_M2_MASK; \
2061 case RISCV::OPC##_M4_MASK: \
2062 return RISCV::INV##_M4_MASK; \
2063 case RISCV::OPC##_M8_MASK: \
2064 return RISCV::INV##_M8_MASK; \
2065 case RISCV::OPC##_MF2_MASK: \
2066 return RISCV::INV##_MF2_MASK; \
2067 case RISCV::OPC##_MF4_MASK: \
2068 return RISCV::INV##_MF4_MASK; \
2069 case RISCV::OPC##_MF8_MASK: \
2070 return RISCV::INV##_MF8_MASK
2071
2072 switch (Opcode) {
2073 default:
2074 return std::nullopt;
2075 case RISCV::FADD_H:
2076 return RISCV::FSUB_H;
2077 case RISCV::FADD_S:
2078 return RISCV::FSUB_S;
2079 case RISCV::FADD_D:
2080 return RISCV::FSUB_D;
2081 case RISCV::FSUB_H:
2082 return RISCV::FADD_H;
2083 case RISCV::FSUB_S:
2084 return RISCV::FADD_S;
2085 case RISCV::FSUB_D:
2086 return RISCV::FADD_D;
2087 case RISCV::ADD:
2088 return RISCV::SUB;
2089 case RISCV::SUB:
2090 return RISCV::ADD;
2091 case RISCV::ADDW:
2092 return RISCV::SUBW;
2093 case RISCV::SUBW:
2094 return RISCV::ADDW;
2095
2100
2101 }
2102
2103#undef RVV_OPC_LMUL_MASK_CASE
2104#undef RVV_OPC_LMUL_CASE
2105}
2106
2109 bool DoRegPressureReduce) {
2111 return false;
2114 if ( ||
(MI->getOpcode()))
2115 return false;
2116
2119 return false;
2120
2121
2122
2123
2124
2125 if (DoRegPressureReduce && .hasOneNonDBGUse(MI->getOperand(0).getReg()))
2126 return false;
2127
2128
2129 if (Root.getParent() != MI->getParent())
2130 return false;
2132}
2133
2136 bool DoRegPressureReduce) {
2137 unsigned Opc = Root.getOpcode();
2138 bool IsFAdd = isFADD(Opc);
2139 if (!IsFAdd && (Opc))
2140 return false;
2141 bool Added = false;
2143 DoRegPressureReduce)) {
2146 Added = true;
2147 }
2149 DoRegPressureReduce)) {
2152 Added = true;
2153 }
2154 return Added;
2155}
2156
2159 bool DoRegPressureReduce) {
2161}
2162
2163
2164
2167 unsigned CombineOpc) {
2170
2173
2174 if ( || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
2175 return nullptr;
2176
2177 if (.hasOneNonDBGUse(MI->getOperand(0).getReg()))
2178 return nullptr;
2179
2180 return MI;
2181}
2182
2183
2184
2185
2188 unsigned OuterShiftAmt) {
2190 if (!ShiftMI)
2191 return false;
2192
2194 if (InnerShiftAmt < OuterShiftAmt || (InnerShiftAmt - OuterShiftAmt) > 3)
2195 return false;
2196
2197 return true;
2198}
2199
2200
2201
2203 switch (Opc) {
2204 default:
2205 return 0;
2206 case RISCV::SH1ADD:
2207 return 1;
2208 case RISCV::SH2ADD:
2209 return 2;
2210 case RISCV::SH3ADD:
2211 return 3;
2212 }
2213}
2214
2215
2216
2220 if (!ShiftAmt)
2221 return false;
2222
2224
2226 if (!AddMI)
2227 return false;
2228
2229 bool Found = false;
2232 Found = true;
2233 }
2236 Found = true;
2237 }
2238
2239 return Found;
2240}
2241
2249 default:
2251 }
2252}
2253
2256 bool DoRegPressureReduce) const {
2257
2258 if (getFPPatterns(Root, Patterns, DoRegPressureReduce))
2259 return true;
2260
2262 return true;
2263
2265 DoRegPressureReduce);
2266}
2267
2269 switch (RootOpc) {
2270 default:
2272 case RISCV::FADD_H:
2273 return RISCV::FMADD_H;
2274 case RISCV::FADD_S:
2275 return RISCV::FMADD_S;
2276 case RISCV::FADD_D:
2277 return RISCV::FMADD_D;
2278 case RISCV::FSUB_H:
2280 : RISCV::FNMSUB_H;
2281 case RISCV::FSUB_S:
2283 : RISCV::FNMSUB_S;
2284 case RISCV::FSUB_D:
2286 : RISCV::FNMSUB_D;
2287 }
2288}
2289
2292 default:
2296 return 2;
2299 return 1;
2300 }
2301}
2302
2310
2315
2316 Register DstReg = Dst.getReg();
2321
2322 bool Mul1IsKill = Mul1.isKill();
2323 bool Mul2IsKill = Mul2.isKill();
2324 bool AddendIsKill = Addend.isKill();
2325
2326
2327
2328
2329 MRI.clearKillFlags(Mul1.getReg());
2330 MRI.clearKillFlags(Mul2.getReg());
2331
2333 BuildMI(*MF, MergedLoc, TII->get(FusedOpc), DstReg)
2338
2343}
2344
2345
2346
2347
2348static void
2356
2358 assert(OuterShiftAmt != 0 && "Unexpected opcode");
2359
2363
2365 assert(InnerShiftAmt >= OuterShiftAmt && "Unexpected shift amount");
2366
2367 unsigned InnerOpc;
2368 switch (InnerShiftAmt - OuterShiftAmt) {
2369 default:
2371 case 0:
2372 InnerOpc = RISCV::ADD;
2373 break;
2374 case 1:
2375 InnerOpc = RISCV::SH1ADD;
2376 break;
2377 case 2:
2378 InnerOpc = RISCV::SH2ADD;
2379 break;
2380 case 3:
2381 InnerOpc = RISCV::SH3ADD;
2382 break;
2383 }
2384
2388
2389 Register NewVR = MRI.createVirtualRegister(&RISCV::GPRRegClass);
2390
2398
2399 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
2405}
2406
2414 default:
2416 DelInstrs, InstrIdxForVirtReg);
2417 return;
2422 return;
2423 }
2428 return;
2429 }
2431 genShXAddAddShift(Root, 1, InsInstrs, DelInstrs, InstrIdxForVirtReg);
2432 return;
2434 genShXAddAddShift(Root, 2, InsInstrs, DelInstrs, InstrIdxForVirtReg);
2435 return;
2436 }
2437}
2438
2442
2443 for (const auto &[Index, Operand] : enumerate(Desc.operands())) {
2444 unsigned OpType = Operand.OperandType;
2448 if (MO.isReg()) {
2449 ErrInfo = "Expected a non-register operand.";
2450 return false;
2451 }
2452 if (MO.isImm()) {
2453 int64_t Imm = MO.getImm();
2454 bool Ok;
2455 switch (OpType) {
2456 default:
2458
2459
2460#define CASE_OPERAND_UIMM(NUM) \
2461 case RISCVOp::OPERAND_UIMM##NUM: \
2462 Ok = isUInt(Imm); \
2463 break;
2464#define CASE_OPERAND_SIMM(NUM) \
2465 case RISCVOp::OPERAND_SIMM##NUM: \
2466 Ok = isInt(Imm); \
2467 break;
2479
2481 Ok = isShiftedUInt<1, 1>(Imm);
2482 break;
2484 Ok = isShiftedUInt<4, 1>(Imm);
2485 break;
2487 Ok = isShiftedUInt<5, 1>(Imm);
2488 break;
2490 Ok = isShiftedUInt<5, 2>(Imm);
2491 break;
2493 Ok = isShiftedUInt<6, 2>(Imm);
2494 break;
2496 Ok = isShiftedUInt<5, 3>(Imm);
2497 break;
2499 Ok = isUInt<8>(Imm) && Imm >= 32;
2500 break;
2502 Ok = isShiftedUInt<6, 3>(Imm);
2503 break;
2505 Ok = isShiftedInt<6, 4>(Imm) && (Imm != 0);
2506 break;
2508 Ok = isShiftedUInt<8, 2>(Imm) && (Imm != 0);
2509 break;
2511 Ok = Imm == 0;
2512 break;
2513
2517
2519 Ok = (isInt<5>(Imm) && Imm != -16) || Imm == 16;
2520 break;
2522 Ok = Imm != 0 && isInt<6>(Imm);
2523 break;
2525 Ok = isUInt<10>(Imm);
2526 break;
2528 Ok = isUInt<11>(Imm);
2529 break;
2531 Ok = isShiftedInt<7, 5>(Imm);
2532 break;
2534 Ok = STI.is64Bit() ? isUInt<6>(Imm) : isUInt<5>(Imm);
2535 break;
2537 Ok = STI.is64Bit() ? isUInt<6>(Imm) : isUInt<5>(Imm);
2538 Ok = Ok && Imm != 0;
2539 break;
2541 Ok = (isUInt<5>(Imm) && Imm != 0) ||
2542 (Imm >= 0xfffe0 && Imm <= 0xfffff);
2543 break;
2545 Ok = Imm >= 0 && Imm <= 10;
2546 break;
2548 Ok = Imm >= 0 && Imm <= 7;
2549 break;
2551 Ok = Imm >= 1 && Imm <= 10;
2552 break;
2554 Ok = Imm >= 2 && Imm <= 14;
2555 break;
2557 Ok = (Imm & 0xf) == 0;
2558 break;
2561 break;
2564 break;
2567 break;
2570 break;
2573 break;
2575 Ok = Imm == 0;
2576 break;
2580 Ok = isUInt<2>(Imm);
2581 else
2583 break;
2584 }
2585 if (!Ok) {
2586 ErrInfo = "Invalid immediate";
2587 return false;
2588 }
2589 }
2590 }
2591 }
2592
2596 if (.isImm() &&
.isReg()) {
2597 ErrInfo = "Invalid operand type for VL operand";
2598 return false;
2599 }
2600 if (Op.isReg() && Op.getReg() != RISCV::NoRegister) {
2602 auto *RC = MRI.getRegClass(Op.getReg());
2603 if (!RISCV::GPRRegClass.hasSubClassEq(RC)) {
2604 ErrInfo = "Invalid register class for VL operand";
2605 return false;
2606 }
2607 }
2609 ErrInfo = "VL operand w/o SEW operand?";
2610 return false;
2611 }
2612 }
2615 if (.getOperand(OpIdx).isImm()) {
2616 ErrInfo = "SEW value expected to be an immediate";
2617 return false;
2618 }
2619 uint64_t Log2SEW = MI.getOperand(OpIdx).getImm();
2620 if (Log2SEW > 31) {
2621 ErrInfo = "Unexpected SEW value";
2622 return false;
2623 }
2624 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
2626 ErrInfo = "Unexpected SEW value";
2627 return false;
2628 }
2629 }
2632 if (.getOperand(OpIdx).isImm()) {
2633 ErrInfo = "Policy operand expected to be an immediate";
2634 return false;
2635 }
2636 uint64_t Policy = MI.getOperand(OpIdx).getImm();
2638 ErrInfo = "Invalid Policy Value";
2639 return false;
2640 }
2642 ErrInfo = "policy operand w/o VL operand?";
2643 return false;
2644 }
2645
2646
2647
2648
2649 unsigned UseOpIdx;
2650 if (.isRegTiedToUseOperand(0, &UseOpIdx)) {
2651 ErrInfo = "policy operand w/o tied operand?";
2652 return false;
2653 }
2654 }
2655
2658 .readsRegister(RISCV::FRM, nullptr)) {
2659 ErrInfo = "dynamic rounding mode should read FRM";
2660 return false;
2661 }
2662
2663 return true;
2664}
2665
2670 default:
2671 return false;
2672 case RISCV::LB:
2673 case RISCV::LBU:
2674 case RISCV::LH:
2675 case RISCV::LH_INX:
2676 case RISCV::LHU:
2677 case RISCV::LW:
2678 case RISCV::LW_INX:
2679 case RISCV::LWU:
2680 case RISCV::LD:
2681 case RISCV::FLH:
2682 case RISCV::FLW:
2683 case RISCV::FLD:
2684 case RISCV::SB:
2685 case RISCV::SH:
2686 case RISCV::SH_INX:
2687 case RISCV::SW:
2688 case RISCV::SW_INX:
2689 case RISCV::SD:
2690 case RISCV::FSH:
2691 case RISCV::FSW:
2692 case RISCV::FSD:
2693 break;
2694 }
2695
2697 return false;
2698
2701 return false;
2702
2705 int64_t NewOffset = OldOffset + Disp;
2707 NewOffset = SignExtend64<32>(NewOffset);
2708
2709 if (!isInt<12>(NewOffset))
2710 return false;
2711
2717 return true;
2718}
2719
2722
2725
2727 "Addressing mode not supported for folding");
2728
2736}
2737
2743 return false;
2744
2745
2747 case RISCV::LB:
2748 case RISCV::LBU:
2749 case RISCV::SB:
2750 case RISCV::LH:
2751 case RISCV::LH_INX:
2752 case RISCV::LHU:
2753 case RISCV::FLH:
2754 case RISCV::SH:
2755 case RISCV::SH_INX:
2756 case RISCV::FSH:
2757 case RISCV::LW:
2758 case RISCV::LW_INX:
2759 case RISCV::LWU:
2760 case RISCV::FLW:
2761 case RISCV::SW:
2762 case RISCV::SW_INX:
2763 case RISCV::FSW:
2764 case RISCV::LD:
2765 case RISCV::FLD:
2766 case RISCV::SD:
2767 case RISCV::FSD:
2768 break;
2769 default:
2770 return false;
2771 }
2773 OffsetIsScalable = false;
2775 return false;
2777 return true;
2778}
2779
2780
2781
2786
2787
2788
2789 if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front()))
2790 return true;
2791
2793 return false;
2794
2797 if (MO1->getAddrSpace() != MO2->getAddrSpace())
2798 return false;
2799
2800 auto Base1 = MO1->getValue();
2801 auto Base2 = MO2->getValue();
2802 if (!Base1 || !Base2)
2803 return false;
2806
2807 if (isa(Base1) || isa(Base2))
2808 return false;
2809
2810 return Base1 == Base2;
2811}
2812
2816 int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize,
2817 unsigned NumBytes) const {
2818
2819
2820 if (!BaseOps1.empty() && !BaseOps2.empty()) {
2822 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
2824 return false;
2825 } else if (!BaseOps1.empty() || !BaseOps2.empty()) {
2826
2827 return false;
2828 }
2829
2831 BaseOps1.front()->getParent()->getMF()->getSubtarget().getCacheLineSize();
2832
2834
2835
2836
2837 return ClusterSize <= 4 && std::abs(Offset1 - Offset2) < CacheLineSize;
2838}
2839
2840
2841
2842
2843
2844
2845
2846
2851 return false;
2852
2853
2854
2855
2857 return false;
2860 return false;
2861
2863 return false;
2864
2868 return true;
2869}
2870
2875
2878 return false;
2879
2880
2881
2882
2883
2884
2886 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
2887 int64_t OffsetA = 0, OffsetB = 0;
2892 int LowOffset = std::min(OffsetA, OffsetB);
2893 int HighOffset = std::max(OffsetA, OffsetB);
2894 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
2896 LowOffset + (int)LowWidth.getValue() <= HighOffset)
2897 return true;
2898 }
2899 }
2900 return false;
2901}
2902
2903std::pair<unsigned, unsigned>
2906 return std::make_pair(TF & Mask, TF & ~Mask);
2907}
2908
2911 using namespace RISCVII;
2912 static const std::pair<unsigned, const char *> TargetFlags[] = {
2913 {MO_CALL, "riscv-call"},
2914 {MO_LO, "riscv-lo"},
2915 {MO_HI, "riscv-hi"},
2916 {MO_PCREL_LO, "riscv-pcrel-lo"},
2917 {MO_PCREL_HI, "riscv-pcrel-hi"},
2918 {MO_GOT_HI, "riscv-got-hi"},
2919 {MO_TPREL_LO, "riscv-tprel-lo"},
2920 {MO_TPREL_HI, "riscv-tprel-hi"},
2921 {MO_TPREL_ADD, "riscv-tprel-add"},
2922 {MO_TLS_GOT_HI, "riscv-tls-got-hi"},
2923 {MO_TLS_GD_HI, "riscv-tls-gd-hi"},
2924 {MO_TLSDESC_HI, "riscv-tlsdesc-hi"},
2925 {MO_TLSDESC_LOAD_LO, "riscv-tlsdesc-load-lo"},
2926 {MO_TLSDESC_ADD_LO, "riscv-tlsdesc-add-lo"},
2927 {MO_TLSDESC_CALL, "riscv-tlsdesc-call"}};
2928 return ArrayRef(TargetFlags);
2929}
2931 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
2933
2934
2935 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
2936 return false;
2937
2938
2939
2940 if (F.hasSection())
2941 return false;
2942
2943
2944 return true;
2945}
2946
2948 unsigned &Flags) const {
2949
2951}
2952
2953
2958
2962}
2963
2967 return F.getFnAttribute("fentry-call").getValueAsBool() ||
2968 F.hasFnAttribute("patchable-function-entry");
2969}
2970
2972 unsigned RegNo) {
2973 return MI.readsRegister(RegNo, TRI) ||
2974 MI.getDesc().hasImplicitUseOfPhysReg(RegNo);
2975}
2976
2979 return MI.modifiesRegister(RegNo, TRI) ||
2980 MI.getDesc().hasImplicitDefOfPhysReg(RegNo);
2981}
2982
2985 return true;
2987 return true;
2988
2989
2990
2991
2993 unsigned TailExpandUseRegNo =
2997 return true;
2999 break;
3000 }
3001 return false;
3002}
3003
3004static std::optional
3006
3007
3008 if (C.back().isReturn()) {
3010 "The candidate who uses return instruction must be outlined "
3011 "using tail call");
3013 }
3014
3018 return isMIModifiesReg(MI, TRI, RISCV::X5);
3019 }))
3020 return true;
3021 return .isAvailableAcrossAndOutOfSeq(RISCV::X5, *TRI);
3022 };
3023
3024 if (!CandidateUsesX5(C))
3026
3027 return std::nullopt;
3028}
3029
3030std::optional<std::unique_ptroutliner::OutlinedFunction>
3033 std::vectoroutliner::Candidate &RepeatedSequenceLocs,
3034 unsigned MinRepeats) const {
3035
3036
3039 if (!CandidateInfo)
3040 RepeatedSequenceLocs.clear();
3041
3042
3043 if (RepeatedSequenceLocs.size() < MinRepeats)
3044 return std::nullopt;
3045
3046 unsigned InstrSizeCExt =
3048 : 4;
3049 unsigned CallOverhead = 0, FrameOverhead = 0;
3050
3052 switch (MOCI) {
3054
3055 CallOverhead = 8;
3056
3057 FrameOverhead = InstrSizeCExt;
3058 break;
3060
3061 CallOverhead = 4 + InstrSizeCExt;
3062
3063 FrameOverhead = 0;
3064 break;
3065 }
3066
3067 for (auto &C : RepeatedSequenceLocs)
3068 C.setCallInfo(MOCI, CallOverhead);
3069
3070 unsigned SequenceSize = 0;
3071 for (auto &MI : Candidate)
3073
3074 return std::make_uniqueoutliner::OutlinedFunction(
3075 RepeatedSequenceLocs, SequenceSize, FrameOverhead, MOCI);
3076}
3077
3081 unsigned Flags) const {
3086 const auto &F = MI.getMF()->getFunction();
3087
3088
3089 if (MI.isCFIInstruction())
3090
3091
3092
3095
3099
3100
3101 for (const auto &MO : MI.operands()) {
3102
3103
3104
3106 (MI.getMF()->getTarget().getFunctionSections() || F.hasComdat() ||
3107 F.hasSection() || F.getSectionPrefix()))
3109 }
3110
3112}
3113
3117
3118
3119 bool Changed = true;
3120 while (Changed) {
3121 Changed = false;
3125 if (I->isCFIInstruction()) {
3126 I->removeFromParent();
3127 Changed = true;
3128 break;
3129 }
3130 }
3131 }
3132
3134 return;
3135
3137
3138
3141 .addReg(RISCV::X5)
3142 .addImm(0));
3143}
3144
3148
3151 .addGlobalAddress(M.getNamedValue(MF.getName()),
3153 return It;
3154 }
3155
3156
3159 .addGlobalAddress(M.getNamedValue(MF.getName()), 0,
3161 return It;
3162}
3163
3166
3167
3169 if (!Op0.isReg() || Reg != Op0.getReg())
3170 return std::nullopt;
3171
3172
3173
3174 if (MI.getOpcode() == RISCV::ADDI && MI.getOperand(1).isReg() &&
3175 MI.getOperand(2).isImm())
3176 return RegImmPair{MI.getOperand(1).getReg(), MI.getOperand(2).getImm()};
3177
3178 return std::nullopt;
3179}
3180
3181
3185
3186 std::string GenericComment =
3188 if (!GenericComment.empty())
3189 return GenericComment;
3190
3191
3192 if (.isImm())
3193 return std::string();
3194
3196 if (OpIdx >= Desc.getNumOperands())
3197 return std::string();
3198
3199 std::string Comment;
3201
3203
3204
3205
3209 unsigned Imm = Op.getImm();
3211 break;
3212 }
3215 unsigned Log2SEW = Op.getImm();
3216 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
3218 OS << "e" << SEW;
3219 break;
3220 }
3222 unsigned Policy = Op.getImm();
3224 "Invalid Policy Value");
3227 break;
3228 }
3229
3230 return Comment;
3231}
3232
3233
3234#define CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL) \
3235 RISCV::Pseudo##OP##_##LMUL
3236
3237#define CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL) \
3238 RISCV::Pseudo##OP##_##LMUL##_MASK
3239
3240#define CASE_RVV_OPCODE_LMUL(OP, LMUL) \
3241 CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL): \
3242 case CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL)
3243
3244#define CASE_RVV_OPCODE_UNMASK_WIDEN(OP) \
3245 CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF8): \
3246 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF4): \
3247 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF2): \
3248 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M1): \
3249 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M2): \
3250 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M4)
3251
3252#define CASE_RVV_OPCODE_UNMASK(OP) \
3253 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
3254 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M8)
3255
3256#define CASE_RVV_OPCODE_MASK_WIDEN(OP) \
3257 CASE_RVV_OPCODE_MASK_LMUL(OP, MF8): \
3258 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF4): \
3259 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF2): \
3260 case CASE_RVV_OPCODE_MASK_LMUL(OP, M1): \
3261 case CASE_RVV_OPCODE_MASK_LMUL(OP, M2): \
3262 case CASE_RVV_OPCODE_MASK_LMUL(OP, M4)
3263
3264#define CASE_RVV_OPCODE_MASK(OP) \
3265 CASE_RVV_OPCODE_MASK_WIDEN(OP): \
3266 case CASE_RVV_OPCODE_MASK_LMUL(OP, M8)
3267
3268#define CASE_RVV_OPCODE_WIDEN(OP) \
3269 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
3270 case CASE_RVV_OPCODE_MASK_WIDEN(OP)
3271
3272#define CASE_RVV_OPCODE(OP) \
3273 CASE_RVV_OPCODE_UNMASK(OP): \
3274 case CASE_RVV_OPCODE_MASK(OP)
3275
3276
3277
3278#define CASE_VMA_OPCODE_COMMON(OP, TYPE, LMUL) \
3279 RISCV::PseudoV##OP##_##TYPE##_##LMUL
3280
3281#define CASE_VMA_OPCODE_LMULS_M1(OP, TYPE) \
3282 CASE_VMA_OPCODE_COMMON(OP, TYPE, M1): \
3283 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M2): \
3284 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M4): \
3285 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M8)
3286
3287#define CASE_VMA_OPCODE_LMULS_MF2(OP, TYPE) \
3288 CASE_VMA_OPCODE_COMMON(OP, TYPE, MF2): \
3289 case CASE_VMA_OPCODE_LMULS_M1(OP, TYPE)
3290
3291#define CASE_VMA_OPCODE_LMULS_MF4(OP, TYPE) \
3292 CASE_VMA_OPCODE_COMMON(OP, TYPE, MF4): \
3293 case CASE_VMA_OPCODE_LMULS_MF2(OP, TYPE)
3294
3295#define CASE_VMA_OPCODE_LMULS(OP, TYPE) \
3296 CASE_VMA_OPCODE_COMMON(OP, TYPE, MF8): \
3297 case CASE_VMA_OPCODE_LMULS_MF4(OP, TYPE)
3298
3299
3300#define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL, SEW) \
3301 RISCV::PseudoV##OP##_##TYPE##_##LMUL##_##SEW
3302
3303#define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW) \
3304 CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1, SEW): \
3305 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2, SEW): \
3306 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4, SEW): \
3307 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8, SEW)
3308
3309#define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW) \
3310 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2, SEW): \
3311 case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW)
3312
3313#define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE, SEW) \
3314 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4, SEW): \
3315 case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW)
3316
3317#define CASE_VFMA_OPCODE_VV(OP) \
3318 CASE_VFMA_OPCODE_LMULS_MF4(OP, VV, E16): \
3319 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VV, E32): \
3320 case CASE_VFMA_OPCODE_LMULS_M1(OP, VV, E64)
3321
3322#define CASE_VFMA_SPLATS(OP) \
3323 CASE_VFMA_OPCODE_LMULS_MF4(OP, VFPR16, E16): \
3324 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VFPR32, E32): \
3325 case CASE_VFMA_OPCODE_LMULS_M1(OP, VFPR64, E64)
3326
3327
3329 unsigned &SrcOpIdx1,
3330 unsigned &SrcOpIdx2) const {
3332 if (.isCommutable())
3333 return false;
3334
3335 switch (MI.getOpcode()) {
3336 case RISCV::TH_MVEQZ:
3337 case RISCV::TH_MVNEZ:
3338
3339
3340
3341 if (MI.getOperand(2).getReg() == RISCV::X0)
3342 return false;
3343
3344 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1, 2);
3345 case RISCV::TH_MULA:
3346 case RISCV::TH_MULAW:
3347 case RISCV::TH_MULAH:
3348 case RISCV::TH_MULS:
3349 case RISCV::TH_MULSW:
3350 case RISCV::TH_MULSH:
3351
3352 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
3353 case RISCV::PseudoCCMOVGPRNoX0:
3354 case RISCV::PseudoCCMOVGPR:
3355
3356 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 4, 5);
3382
3383 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
3402
3404 if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0)
3405 return false;
3406
3407
3408
3409 unsigned CommutableOpIdx1 = 1;
3410 unsigned CommutableOpIdx2 = 3;
3411 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
3412 CommutableOpIdx2))
3413 return false;
3414 return true;
3415 }
3422
3424 if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0)
3425 return false;
3426
3427
3428
3429
3430
3431 if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3)
3432 return false;
3433 if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3)
3434 return false;
3435
3436
3437 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
3438 SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1)
3439 return false;
3440
3441
3442
3443
3444 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
3445 SrcOpIdx2 == CommuteAnyOperandIndex) {
3446
3447
3448 unsigned CommutableOpIdx1 = SrcOpIdx1;
3449 if (SrcOpIdx1 == SrcOpIdx2) {
3450
3451
3452 CommutableOpIdx1 = 1;
3453 } else if (SrcOpIdx1 == CommuteAnyOperandIndex) {
3454
3455 CommutableOpIdx1 = SrcOpIdx2;
3456 }
3457
3458
3459
3460 unsigned CommutableOpIdx2;
3461 if (CommutableOpIdx1 != 1) {
3462
3463 CommutableOpIdx2 = 1;
3464 } else {
3465 Register Op1Reg = MI.getOperand(CommutableOpIdx1).getReg();
3466
3467
3468
3469
3470 if (Op1Reg != MI.getOperand(2).getReg())
3471 CommutableOpIdx2 = 2;
3472 else
3473 CommutableOpIdx2 = 3;
3474 }
3475
3476
3477
3478 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
3479 CommutableOpIdx2))
3480 return false;
3481 }
3482
3483 return true;
3484 }
3485 }
3486
3488}
3489
3490
3491#define CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL) \
3492 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL: \
3493 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL; \
3494 break;
3495
3496#define CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE) \
3497 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1) \
3498 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2) \
3499 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4) \
3500 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)
3501
3502#define CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE) \
3503 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2) \
3504 CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE)
3505
3506#define CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE) \
3507 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4) \
3508 CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE)
3509
3510#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE) \
3511 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8) \
3512 CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE)
3513
3514#define CASE_VMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \
3515 CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16) \
3516 CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32) \
3517 CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64)
3518
3519
3520#define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL, SEW) \
3521 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL##_##SEW: \
3522 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL##_##SEW; \
3523 break;
3524
3525#define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW) \
3526 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1, SEW) \
3527 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2, SEW) \
3528 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4, SEW) \
3529 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8, SEW)
3530
3531#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW) \
3532 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2, SEW) \
3533 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)
3534
3535#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP) \
3536 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VV, E16) \
3537 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32) \
3538 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VV, E64)
3539
3540#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW) \
3541 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4, SEW) \
3542 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW)
3543
3544#define CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE, SEW) \
3545 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8, SEW) \
3546 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW)
3547
3548#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \
3549 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16, E16) \
3550 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32, E32) \
3551 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64, E64)
3552
3554 bool NewMI,
3555 unsigned OpIdx1,
3556 unsigned OpIdx2) const {
3558 if (NewMI)
3559 return *MI.getParent()->getParent()->CloneMachineInstr(&MI);
3560 return MI;
3561 };
3562
3563 switch (MI.getOpcode()) {
3564 case RISCV::TH_MVEQZ:
3565 case RISCV::TH_MVNEZ: {
3566 auto &WorkingMI = cloneIfNew(MI);
3567 WorkingMI.setDesc(get(MI.getOpcode() == RISCV::TH_MVEQZ ? RISCV::TH_MVNEZ
3568 : RISCV::TH_MVEQZ));
3570 OpIdx2);
3571 }
3572 case RISCV::PseudoCCMOVGPRNoX0:
3573 case RISCV::PseudoCCMOVGPR: {
3574
3577 auto &WorkingMI = cloneIfNew(MI);
3578 WorkingMI.getOperand(3).setImm(CC);
3580 OpIdx1, OpIdx2);
3581 }
3600
3601
3602 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
3603 assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index");
3604 unsigned Opc;
3605 switch (MI.getOpcode()) {
3606 default:
3626 }
3627
3628 auto &WorkingMI = cloneIfNew(MI);
3629 WorkingMI.setDesc(get(Opc));
3631 OpIdx1, OpIdx2);
3632 }
3639 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
3640
3641
3642 if (OpIdx1 == 3 || OpIdx2 == 3) {
3643 unsigned Opc;
3644 switch (MI.getOpcode()) {
3645 default:
3653 }
3654
3655 auto &WorkingMI = cloneIfNew(MI);
3656 WorkingMI.setDesc(get(Opc));
3658 OpIdx1, OpIdx2);
3659 }
3660
3661 break;
3662 }
3663 }
3664
3666}
3667
3668#undef CASE_RVV_OPCODE_UNMASK_LMUL
3669#undef CASE_RVV_OPCODE_MASK_LMUL
3670#undef CASE_RVV_OPCODE_LMUL
3671#undef CASE_RVV_OPCODE_UNMASK_WIDEN
3672#undef CASE_RVV_OPCODE_UNMASK
3673#undef CASE_RVV_OPCODE_MASK_WIDEN
3674#undef CASE_RVV_OPCODE_MASK
3675#undef CASE_RVV_OPCODE_WIDEN
3676#undef CASE_RVV_OPCODE
3677
3678#undef CASE_VMA_OPCODE_COMMON
3679#undef CASE_VMA_OPCODE_LMULS_M1
3680#undef CASE_VMA_OPCODE_LMULS_MF2
3681#undef CASE_VMA_OPCODE_LMULS_MF4
3682#undef CASE_VMA_OPCODE_LMULS
3683#undef CASE_VFMA_OPCODE_COMMON
3684#undef CASE_VFMA_OPCODE_LMULS_M1
3685#undef CASE_VFMA_OPCODE_LMULS_MF2
3686#undef CASE_VFMA_OPCODE_LMULS_MF4
3687#undef CASE_VFMA_OPCODE_VV
3688#undef CASE_VFMA_SPLATS
3689
3690
3691#define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL) \
3692 RISCV::PseudoV##OP##_##LMUL##_TIED
3693
3694#define CASE_WIDEOP_OPCODE_LMULS_MF4(OP) \
3695 CASE_WIDEOP_OPCODE_COMMON(OP, MF4): \
3696 case CASE_WIDEOP_OPCODE_COMMON(OP, MF2): \
3697 case CASE_WIDEOP_OPCODE_COMMON(OP, M1): \
3698 case CASE_WIDEOP_OPCODE_COMMON(OP, M2): \
3699 case CASE_WIDEOP_OPCODE_COMMON(OP, M4)
3700
3701#define CASE_WIDEOP_OPCODE_LMULS(OP) \
3702 CASE_WIDEOP_OPCODE_COMMON(OP, MF8): \
3703 case CASE_WIDEOP_OPCODE_LMULS_MF4(OP)
3704
3705#define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL) \
3706 case RISCV::PseudoV##OP##_##LMUL##_TIED: \
3707 NewOpc = RISCV::PseudoV##OP##_##LMUL; \
3708 break;
3709
3710#define CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP) \
3711 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4) \
3712 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2) \
3713 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1) \
3714 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2) \
3715 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4)
3716
3717#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
3718 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8) \
3719 CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
3720
3721
3722#define CASE_FP_WIDEOP_OPCODE_COMMON(OP, LMUL, SEW) \
3723 RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED
3724
3725#define CASE_FP_WIDEOP_OPCODE_LMULS_MF4(OP) \
3726 CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16): \
3727 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16): \
3728 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E32): \
3729 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16): \
3730 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E32): \
3731 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16): \
3732 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E32): \
3733 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16): \
3734 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E32) \
3735
3736#define CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL, SEW) \
3737 case RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED: \
3738 NewOpc = RISCV::PseudoV##OP##_##LMUL##_##SEW; \
3739 break;
3740
3741#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP) \
3742 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16) \
3743 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16) \
3744 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E32) \
3745 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16) \
3746 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E32) \
3747 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16) \
3748 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E32) \
3749 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16) \
3750 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E32) \
3751
3752#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
3753 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
3754
3755
3760 switch (MI.getOpcode()) {
3761 default:
3762 return nullptr;
3766 MI.getNumExplicitOperands() == 7 &&
3767 "Expect 7 explicit operands rd, rs2, rs1, rm, vl, sew, policy");
3768
3770 1) == 0)
3771 return nullptr;
3772
3773 unsigned NewOpc;
3774 switch (MI.getOpcode()) {
3775 default:
3779 }
3780
3781
3792 break;
3793 }
3798
3800 MI.getNumExplicitOperands() == 6);
3801 if ((MI.getOperand(5).getImm() & 1) == 0)
3802 return nullptr;
3803
3804
3805 unsigned NewOpc;
3806 switch (MI.getOpcode()) {
3807 default:
3813 }
3814
3815
3825 break;
3826 }
3827 }
3829
3830 if (LV) {
3831 unsigned NumOps = MI.getNumOperands();
3832 for (unsigned I = 1; I < NumOps; ++I) {
3834 if (Op.isReg() && Op.isKill())
3836 }
3837 }
3838
3839 if (LIS) {
3841
3842 if (MI.getOperand(0).isEarlyClobber()) {
3843
3844
3845
3848 if (S->end == Idx.getRegSlot(true))
3849 S->end = Idx.getRegSlot();
3850 }
3851 }
3852
3853 return MIB;
3854}
3855
3856#undef CASE_WIDEOP_OPCODE_COMMON
3857#undef CASE_WIDEOP_OPCODE_LMULS_MF4
3858#undef CASE_WIDEOP_OPCODE_LMULS
3859#undef CASE_WIDEOP_CHANGE_OPCODE_COMMON
3860#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4
3861#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS
3862#undef CASE_FP_WIDEOP_OPCODE_COMMON
3863#undef CASE_FP_WIDEOP_OPCODE_LMULS_MF4
3864#undef CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON
3865#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4
3866#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS
3867
3873 if (llvm::has_single_bit<uint32_t>(Amount)) {
3875 if (ShiftAmount == 0)
3876 return;
3879 .addImm(ShiftAmount)
3881 } else if (STI.hasStdExtZba() &&
3882 ((Amount % 3 == 0 && isPowerOf2_64(Amount / 3)) ||
3883 (Amount % 5 == 0 && isPowerOf2_64(Amount / 5)) ||
3884 (Amount % 9 == 0 && isPowerOf2_64(Amount / 9)))) {
3885
3886 unsigned Opc;
3888 if (Amount % 9 == 0) {
3889 Opc = RISCV::SH3ADD;
3890 ShiftAmount = Log2_64(Amount / 9);
3891 } else if (Amount % 5 == 0) {
3892 Opc = RISCV::SH2ADD;
3893 ShiftAmount = Log2_64(Amount / 5);
3894 } else if (Amount % 3 == 0) {
3895 Opc = RISCV::SH1ADD;
3896 ShiftAmount = Log2_64(Amount / 3);
3897 } else {
3899 }
3900 if (ShiftAmount)
3903 .addImm(ShiftAmount)
3909 } else if (llvm::has_single_bit<uint32_t>(Amount - 1)) {
3910 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);
3914 .addImm(ShiftAmount)
3920 } else if (llvm::has_single_bit<uint32_t>(Amount + 1)) {
3921 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);
3925 .addImm(ShiftAmount)
3931 } else if (STI.hasStdExtZmmul()) {
3932 Register N = MRI.createVirtualRegister(&RISCV::GPRRegClass);
3938 } else {
3940 uint32_t PrevShiftAmount = 0;
3941 for (uint32_t ShiftAmount = 0; Amount >> ShiftAmount; ShiftAmount++) {
3942 if (Amount & (1U << ShiftAmount)) {
3943 if (ShiftAmount)
3946 .addImm(ShiftAmount - PrevShiftAmount)
3948 if (Amount >> (ShiftAmount + 1)) {
3949
3950 if (!Acc) {
3951 Acc = MRI.createVirtualRegister(&RISCV::GPRRegClass);
3955 } else {
3960 }
3961 }
3962 PrevShiftAmount = ShiftAmount;
3963 }
3964 }
3965 assert(Acc && "Expected valid accumulator");
3970 }
3971}
3972
3975 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
3978 return ArrayRef(TargetFlags);
3979}
3980
3984 : 2;
3985}
3986
3987
3989 return MI.getOpcode() == RISCV::ADDIW && MI.getOperand(1).isReg() &&
3990 MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0;
3991}
3992
3993
3995 return MI.getOpcode() == RISCV::ADD_UW && MI.getOperand(1).isReg() &&
3996 MI.getOperand(2).isReg() && MI.getOperand(2).getReg() == RISCV::X0;
3997}
3998
3999
4001 return MI.getOpcode() == RISCV::ANDI && MI.getOperand(1).isReg() &&
4002 MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 255;
4003}
4004
4006 switch (Opcode) {
4007 default:
4008 return false;
4009 case RISCV::VS1R_V:
4010 case RISCV::VS2R_V:
4011 case RISCV::VS4R_V:
4012 case RISCV::VS8R_V:
4013 case RISCV::VL1RE8_V:
4014 case RISCV::VL2RE8_V:
4015 case RISCV::VL4RE8_V:
4016 case RISCV::VL8RE8_V:
4017 case RISCV::VL1RE16_V:
4018 case RISCV::VL2RE16_V:
4019 case RISCV::VL4RE16_V:
4020 case RISCV::VL8RE16_V:
4021 case RISCV::VL1RE32_V:
4022 case RISCV::VL2RE32_V:
4023 case RISCV::VL4RE32_V:
4024 case RISCV::VL8RE32_V:
4025 case RISCV::VL1RE64_V:
4026 case RISCV::VL2RE64_V:
4027 case RISCV::VL4RE64_V:
4028 case RISCV::VL8RE64_V:
4029 return true;
4030 }
4031}
4032
4034
4035
4036 unsigned Opcode = MI.getOpcode();
4037 if (!RISCVVPseudosTable::getPseudoInfo(Opcode) &&
4039 return false;
4040 return true;
4041}
4042
4043std::optional<std::pair<unsigned, unsigned>>
4045 switch (Opcode) {
4046 default:
4047 return std::nullopt;
4048 case RISCV::PseudoVSPILL2_M1:
4049 case RISCV::PseudoVRELOAD2_M1:
4050 return std::make_pair(2u, 1u);
4051 case RISCV::PseudoVSPILL2_M2:
4052 case RISCV::PseudoVRELOAD2_M2:
4053 return std::make_pair(2u, 2u);
4054 case RISCV::PseudoVSPILL2_M4:
4055 case RISCV::PseudoVRELOAD2_M4:
4056 return std::make_pair(2u, 4u);
4057 case RISCV::PseudoVSPILL3_M1:
4058 case RISCV::PseudoVRELOAD3_M1:
4059 return std::make_pair(3u, 1u);
4060 case RISCV::PseudoVSPILL3_M2:
4061 case RISCV::PseudoVRELOAD3_M2:
4062 return std::make_pair(3u, 2u);
4063 case RISCV::PseudoVSPILL4_M1:
4064 case RISCV::PseudoVRELOAD4_M1:
4065 return std::make_pair(4u, 1u);
4066 case RISCV::PseudoVSPILL4_M2:
4067 case RISCV::PseudoVRELOAD4_M2:
4068 return std::make_pair(4u, 2u);
4069 case RISCV::PseudoVSPILL5_M1:
4070 case RISCV::PseudoVRELOAD5_M1:
4071 return std::make_pair(5u, 1u);
4072 case RISCV::PseudoVSPILL6_M1:
4073 case RISCV::PseudoVRELOAD6_M1:
4074 return std::make_pair(6u, 1u);
4075 case RISCV::PseudoVSPILL7_M1:
4076 case RISCV::PseudoVRELOAD7_M1:
4077 return std::make_pair(7u, 1u);
4078 case RISCV::PseudoVSPILL8_M1:
4079 case RISCV::PseudoVRELOAD8_M1:
4080 return std::make_pair(8u, 1u);
4081 }
4082}
4083
4085 return MI.getNumExplicitDefs() == 2 &&
4086 MI.modifiesRegister(RISCV::VL, nullptr) && .isInlineAsm();
4087}
4088
4090 int16_t MI1FrmOpIdx =
4092 int16_t MI2FrmOpIdx =
4094 if (MI1FrmOpIdx < 0 || MI2FrmOpIdx < 0)
4095 return false;
4099}
4100
4101std::optional
4103
4104 switch (Opcode) {
4105 default:
4106 return std::nullopt;
4107
4108
4109 case RISCV::VSLL_VX:
4110 case RISCV::VSRL_VX:
4111 case RISCV::VSRA_VX:
4112
4113 case RISCV::VSSRL_VX:
4114 case RISCV::VSSRA_VX:
4115
4116 return Log2SEW;
4117
4118
4119 case RISCV::VNSRL_WX:
4120 case RISCV::VNSRA_WX:
4121
4122 case RISCV::VNCLIPU_WX:
4123 case RISCV::VNCLIP_WX:
4124
4125 return Log2SEW + 1;
4126
4127
4128 case RISCV::VADD_VX:
4129 case RISCV::VSUB_VX:
4130 case RISCV::VRSUB_VX:
4131
4132 case RISCV::VWADDU_VX:
4133 case RISCV::VWSUBU_VX:
4134 case RISCV::VWADD_VX:
4135 case RISCV::VWSUB_VX:
4136 case RISCV::VWADDU_WX:
4137 case RISCV::VWSUBU_WX:
4138 case RISCV::VWADD_WX:
4139 case RISCV::VWSUB_WX:
4140
4141 case RISCV::VADC_VXM:
4142 case RISCV::VADC_VIM:
4143 case RISCV::VMADC_VXM:
4144 case RISCV::VMADC_VIM:
4145 case RISCV::VMADC_VX:
4146 case RISCV::VSBC_VXM:
4147 case RISCV::VMSBC_VXM:
4148 case RISCV::VMSBC_VX:
4149
4150 case RISCV::VAND_VX:
4151 case RISCV::VOR_VX:
4152 case RISCV::VXOR_VX:
4153
4154 case RISCV::VMSEQ_VX:
4155 case RISCV::VMSNE_VX:
4156 case RISCV::VMSLTU_VX:
4157 case RISCV::VMSLT_VX:
4158 case RISCV::VMSLEU_VX:
4159 case RISCV::VMSLE_VX:
4160 case RISCV::VMSGTU_VX:
4161 case RISCV::VMSGT_VX:
4162
4163 case RISCV::VMINU_VX:
4164 case RISCV::VMIN_VX:
4165 case RISCV::VMAXU_VX:
4166 case RISCV::VMAX_VX:
4167
4168 case RISCV::VMUL_VX:
4169 case RISCV::VMULH_VX:
4170 case RISCV::VMULHU_VX:
4171 case RISCV::VMULHSU_VX:
4172
4173 case RISCV::VDIVU_VX:
4174 case RISCV::VDIV_VX:
4175 case RISCV::VREMU_VX:
4176 case RISCV::VREM_VX:
4177
4178 case RISCV::VWMUL_VX:
4179 case RISCV::VWMULU_VX:
4180 case RISCV::VWMULSU_VX:
4181
4182 case RISCV::VMACC_VX:
4183 case RISCV::VNMSAC_VX:
4184 case RISCV::VMADD_VX:
4185 case RISCV::VNMSUB_VX:
4186
4187 case RISCV::VWMACCU_VX:
4188 case RISCV::VWMACC_VX:
4189 case RISCV::VWMACCSU_VX:
4190 case RISCV::VWMACCUS_VX:
4191
4192 case RISCV::VMERGE_VXM:
4193
4194 case RISCV::VMV_V_X:
4195
4196 case RISCV::VSADDU_VX:
4197 case RISCV::VSADD_VX:
4198 case RISCV::VSSUBU_VX:
4199 case RISCV::VSSUB_VX:
4200
4201 case RISCV::VAADDU_VX:
4202 case RISCV::VAADD_VX:
4203 case RISCV::VASUBU_VX:
4204 case RISCV::VASUB_VX:
4205
4206 case RISCV::VSMUL_VX:
4207
4208 case RISCV::VMV_S_X:
4209 return 1U << Log2SEW;
4210 }
4211}
4212
4215 RISCVVPseudosTable::getPseudoInfo(RVVPseudoOpcode);
4216 if ()
4217 return 0;
4218 return RVV->BaseInstr;
4219}
4220
4222 unsigned DestEEW =
4224
4225 if (DestEEW == 0)
4226 return 0;
4227
4228 unsigned Scaled = Log2SEW + (DestEEW - 1);
4231}
4232
4233
4235 if (LHS.isReg() && RHS.isReg() && LHS.getReg().isVirtual() &&
4236 LHS.getReg() == RHS.getReg())
4237 return true;
4239 return true;
4241 return false;
4242 if (.isImm() ||
.isImm())
4243 return false;
4244 return LHS.getImm() <= RHS.getImm();
4245}
4246
4247namespace {
4252
4253public:
4257
4258 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
4259
4260
4262 return true;
4264 return true;
4265 return false;
4266 }
4267
4268 std::optional createTripCountGreaterCondition(
4271
4272
4273
4274 CondParam = Cond;
4275 return {};
4276 }
4277
4279
4280 void adjustTripCount(int TripCountAdjust) override {}
4281
4282 void disposed() override {}
4283};
4284}
4285
4286std::unique_ptrTargetInstrInfo::PipelinerLoopInfo
4291 return nullptr;
4292
4293
4294 if (TBB == LoopBB && FBB == LoopBB)
4295 return nullptr;
4296
4297
4298 if (FBB == nullptr)
4299 return nullptr;
4300
4301 assert((TBB == LoopBB || FBB == LoopBB) &&
4302 "The Loop must be a single-basic-block loop");
4303
4304
4305 if (TBB == LoopBB)
4307
4310 if (.isReg())
4311 return nullptr;
4313 if (!Reg.isVirtual())
4314 return nullptr;
4315 return MRI.getVRegDef(Reg);
4316 };
4317
4321 return nullptr;
4323 return nullptr;
4324
4325 return std::make_unique(LHS, RHS, Cond);
4326}
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder MachineInstrBuilder & DefMI
static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg, unsigned NumRegs)
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc, unsigned ZeroReg=0, bool CheckZeroReg=false)
static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, SmallVectorImpl< MachineOperand > &Cond)
@ MachineOutlinerTailCall
Emit a save, restore, call, and return.
static ARCCC::CondCode getOppositeBranchCondition(ARCCC::CondCode CC)
Return the inverse of passed condition, i.e. turning COND_E to COND_NE.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
Module.h This file contains the declarations for the Module class.
static M68k::CondCode getCondFromBranchOpc(unsigned BrOpc)
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static bool cannotInsertTailCall(const MachineBasicBlock &MBB)
#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP)
#define CASE_OPERAND_SIMM(NUM)
static bool isRVVWholeLoadStore(unsigned Opcode)
#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP)
static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc, unsigned Pattern)
#define RVV_OPC_LMUL_CASE(OPC, INV)
static void combineFPFusedMultiply(MachineInstr &Root, MachineInstr &Prev, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs)
static unsigned getAddendOperandIdx(unsigned Pattern)
#define CASE_RVV_OPCODE_UNMASK(OP)
#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP)
static std::optional< MachineOutlinerConstructionID > analyzeCandidate(outliner::Candidate &C)
static cl::opt< bool > PreferWholeRegisterMove("riscv-prefer-whole-register-move", cl::init(false), cl::Hidden, cl::desc("Prefer whole register move for vector registers."))
#define CASE_VFMA_SPLATS(OP)
unsigned getPredicatedOpcode(unsigned Opcode)
static void genShXAddAddShift(MachineInstr &Root, unsigned AddOpIdx, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg)
#define CASE_WIDEOP_OPCODE_LMULS(OP)
#define OPCODE_LMUL_MASK_CASE(OPC)
static bool isFSUB(unsigned Opc)
#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE)
#define CASE_RVV_OPCODE(OP)
#define CASE_VFMA_OPCODE_VV(OP)
MachineOutlinerConstructionID
#define CASE_RVV_OPCODE_WIDEN(OP)
#define CASE_VMA_OPCODE_LMULS(OP, TYPE)
static bool isFMUL(unsigned Opc)
static bool getFPPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce)
#define OPCODE_LMUL_CASE(OPC)
#define CASE_OPERAND_UIMM(NUM)
static bool canCombineShiftIntoShXAdd(const MachineBasicBlock &MBB, const MachineOperand &MO, unsigned OuterShiftAmt)
Utility routine that checks if.
static bool isCandidatePatchable(const MachineBasicBlock &MBB)
static bool isMIReadsReg(const MachineInstr &MI, const TargetRegisterInfo *TRI, unsigned RegNo)
static bool isFADD(unsigned Opc)
#define CASE_FP_WIDEOP_OPCODE_LMULS_MF4(OP)
static bool isConvertibleToVMV_V_V(const RISCVSubtarget &STI, const MachineBasicBlock &MBB, MachineBasicBlock::const_iterator MBBI, MachineBasicBlock::const_iterator &DefMBBI, RISCVII::VLMUL LMul)
static bool isMIModifiesReg(const MachineInstr &MI, const TargetRegisterInfo *TRI, unsigned RegNo)
static MachineInstr * canFoldAsPredicatedOp(Register Reg, const MachineRegisterInfo &MRI, const TargetInstrInfo *TII)
Identify instructions that can be folded into a CCMOV instruction, and return the defining instructio...
static bool canCombineFPFusedMultiply(const MachineInstr &Root, const MachineOperand &MO, bool DoRegPressureReduce)
static bool getSHXADDPatterns(const MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static bool getFPFusedMultiplyPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce)
static cl::opt< MachineTraceStrategy > ForceMachineCombinerStrategy("riscv-force-machine-combiner-strategy", cl::Hidden, cl::desc("Force machine combiner to use a specific strategy for machine " "trace metrics evaluation."), cl::init(MachineTraceStrategy::TS_NumStrategies), cl::values(clEnumValN(MachineTraceStrategy::TS_Local, "local", "Local strategy."), clEnumValN(MachineTraceStrategy::TS_MinInstrCount, "min-instr", "MinInstrCount strategy.")))
static unsigned getSHXADDShiftAmount(unsigned Opc)
#define CASE_RVV_OPCODE_MASK(OP)
#define RVV_OPC_LMUL_MASK_CASE(OPC, INV)
#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
const SmallVectorImpl< MachineOperand > & Cond
This file declares the machine register scavenger class.
static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, ArrayRef< const MachineOperand * > BaseOps1, const MachineInstr &MI2, ArrayRef< const MachineOperand * > BaseOps2)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI)
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file defines the SmallVector class.
static cl::opt< unsigned > CacheLineSize("cache-line-size", cl::init(0), cl::Hidden, cl::desc("Use this to override the target cache line size when " "specified by the user."))
static unsigned getSize(unsigned Kind)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
static DILocation * getMergedLocation(DILocation *LocA, DILocation *LocB)
When two instructions are combined into a single instruction we also need to combine the original loc...
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
LiveInterval - This class represents the liveness of a register, or stack slot.
LiveInterval & getInterval(Register Reg)
SlotIndex ReplaceMachineInstrInMaps(MachineInstr &MI, MachineInstr &NewMI)
const Segment * getSegmentContaining(SlotIndex Idx) const
Return the segment that contains the specified index, or null if there is none.
void replaceKillInstruction(Register Reg, MachineInstr &OldMI, MachineInstr &NewMI)
replaceKillInstruction - Update register kill info by replacing a kill instruction with a new one.
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
TypeSize getValue() const
MCInstBuilder & addReg(MCRegister Reg)
Add a new register operand.
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Instances of this class represent a single low-level machine instruction.
Describe properties that are true of each instruction in the target description file.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
bool isConditionalBranch() const
Return true if this is a branch which may fall through to the next instruction or may transfer contro...
This holds information about one operand of a machine instruction, indicating the register class for ...
uint8_t OperandType
Information about the type of the operand.
Wrapper class representing physical registers. Should be passed by value.
const FeatureBitset & getFeatureBits() const
unsigned pred_size() const
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
iterator getLastNonDebugInstr(bool SkipPseudoOp=true)
Returns an iterator to the last non-debug instruction in the basic block, or end().
Instructions::const_iterator const_instr_iterator
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void setStackID(int ObjectIdx, uint8_t ID)
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & copyImplicitOps(const MachineInstr &OtherMI) const
Copy all the implicit operands from OtherMI onto this one.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool isReturn(QueryType Type=AnyInBundle) const
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
bool hasOneMemOperand() const
Return true if this instruction has exactly one MachineMemOperand.
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
void clearKillInfo()
Clears kill flags on all operands.
A description of a memory reference used in the backend.
bool isNonTemporal() const
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
static MachineOperand CreateImm(int64_t Val)
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
@ MO_Immediate
Immediate operand.
@ MO_Register
Register operand.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
A Module instance is used to store all the information related to an LLVM module.
MI-level patchpoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given patchpoint should emit.
MachineInstr * convertToThreeAddress(MachineInstr &MI, LiveVariables *LV, LiveIntervals *LIS) const override
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
std::optional< std::unique_ptr< outliner::OutlinedFunction > > getOutliningCandidateInfo(const MachineModuleInfo &MMI, std::vector< outliner::Candidate > &RepeatedSequenceLocs, unsigned MinRepeats) const override
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
void movImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, Register DstReg, uint64_t Val, MachineInstr::MIFlag Flag=MachineInstr::NoFlags, bool DstRenamable=false, bool DstIsDead=false) const
MachineInstr * emitLdStWithAddr(MachineInstr &MemI, const ExtAddrMode &AM) const override
void mulImm(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator II, const DebugLoc &DL, Register DestReg, uint32_t Amt, MachineInstr::MIFlag Flag) const
Generate code to multiply the value in DestReg by Amt - handles all the common optimizations for this...
bool isReallyTriviallyReMaterializable(const MachineInstr &MI) const override
bool isFunctionSafeToOutlineFrom(MachineFunction &MF, bool OutlineFromLinkOnceODRs) const override
std::unique_ptr< TargetInstrInfo::PipelinerLoopInfo > analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &dl, int *BytesAdded=nullptr) const override
bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) const override
RISCVInstrInfo(RISCVSubtarget &STI)
void copyPhysRegVector(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, const TargetRegisterClass *RegClass) const
void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg) const override
const MCInstrDesc & getBrCond(RISCVCC::CondCode CC, bool Imm=false) const
MachineInstr * optimizeSelect(MachineInstr &MI, SmallPtrSetImpl< MachineInstr * > &SeenMIs, bool) const override
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, bool RenamableDest=false, bool RenamableSrc=false) const override
bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg, const MachineInstr &AddrI, ExtAddrMode &AM) const override
void insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &NewDestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const override
bool isAsCheapAsAMove(const MachineInstr &MI) const override
bool verifyInstruction(const MachineInstr &MI, StringRef &ErrInfo) const override
bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset, LocationSize &Width, const TargetRegisterInfo *TRI) const
unsigned getTailDuplicateSize(CodeGenOptLevel OptLevel) const override
void getReassociateOperandIndices(const MachineInstr &Root, unsigned Pattern, std::array< unsigned, 5 > &OperandIndices) const override
const RISCVSubtarget & STI
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
std::optional< unsigned > getInverseOpcode(unsigned Opcode) const override
ArrayRef< std::pair< unsigned, const char * > > getSerializableDirectMachineOperandTargetFlags() const override
virtual outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI, MachineBasicBlock::iterator &MBBI, unsigned Flags) const override
MachineTraceStrategy getMachineCombinerTraceStrategy() const override
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
std::optional< RegImmPair > isAddImmediate(const MachineInstr &MI, Register Reg) const override
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
ArrayRef< std::pair< MachineMemOperand::Flags, const char * > > getSerializableMachineMemOperandTargetFlags() const override
MCInst getNop() const override
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, unsigned &Flags) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &MI, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width, const TargetRegisterInfo *TRI) const override
void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF, const outliner::OutlinedFunction &OF) const override
void finalizeInsInstrs(MachineInstr &Root, unsigned &Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs) const override
std::pair< unsigned, unsigned > decomposeMachineOperandsTargetFlags(unsigned TF) const override
MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const override
bool hasReassociableOperands(const MachineInstr &Inst, const MachineBasicBlock *MBB) const override
MachineBasicBlock * getBranchDestBlock(const MachineInstr &MI) const override
std::string createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx, const TargetRegisterInfo *TRI) const override
bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override
bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const override
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, outliner::Candidate &C) const override
bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DstReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
bool isAssociativeAndCommutative(const MachineInstr &Inst, bool Invert) const override
CombinerObjective getCombinerObjective(unsigned Pattern) const override
bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const override
bool optimizeCondBranch(MachineInstr &MI) const override
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
bool analyzeSelect(const MachineInstr &MI, SmallVectorImpl< MachineOperand > &Cond, unsigned &TrueOp, unsigned &FalseOp, bool &Optimizable) const override
bool shouldClusterMemOps(ArrayRef< const MachineOperand * > BaseOps1, int64_t Offset1, bool OffsetIsScalable1, ArrayRef< const MachineOperand * > BaseOps2, int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize, unsigned NumBytes) const override
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool IsKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
int getBranchRelaxationScratchFrameIndex() const
bool hasStdExtCOrZca() const
unsigned getTailDupAggressiveThreshold() const
const RISCVRegisterInfo * getRegisterInfo() const override
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
void setRegUsed(Register Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Tell the scavenger a register is used.
Register scavengeRegisterBackwards(const TargetRegisterClass &RC, MachineBasicBlock::iterator To, bool RestoreAfter, int SPAdj, bool AllowSpill=true)
Make a register of the specific register class available from the current position backwards to the p...
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
SlotIndex - An opaque wrapper around machine indexes.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MI-level stackmap operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
MI-level Statepoint operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given statepoint should emit.
StringRef - Represent a constant reference to a string, i.e.
Object returned by analyzeLoopForPipelining.
TargetInstrInfo - Interface to description of machine instruction set.
virtual bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const
Returns true iff the routine could find two commutable operands in the given machine instruction.
virtual bool hasReassociableOperands(const MachineInstr &Inst, const MachineBasicBlock *MBB) const
Return true when \P Inst has reassociable operands in the same \P MBB.
virtual void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
virtual bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, unsigned &Flags) const
Optional target hook that returns true if MBB is safe to outline from, and returns any target-specifi...
virtual void getReassociateOperandIndices(const MachineInstr &Root, unsigned Pattern, std::array< unsigned, 5 > &OperandIndices) const
The returned array encodes the operand index for each parameter because the operands may be commuted;...
virtual bool isReallyTriviallyReMaterializable(const MachineInstr &MI) const
For instructions with opcodes for which the M_REMATERIALIZABLE flag is set, this hook lets the target...
virtual CombinerObjective getCombinerObjective(unsigned Pattern) const
Return the objective of a combiner pattern.
virtual MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const
This method commutes the operands of the given machine instruction MI.
virtual bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) const
Return true when \P Inst has reassociable sibling.
virtual std::string createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx, const TargetRegisterInfo *TRI) const
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
const uint8_t TSFlags
Configurable target specific flags.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
CondCode getOppositeBranchCondition(CondCode)
unsigned getBrCond(CondCode CC, bool Imm=false)
static bool isValidRoundingMode(unsigned Mode)
static unsigned getVecPolicyOpNum(const MCInstrDesc &Desc)
static bool usesMaskPolicy(uint64_t TSFlags)
static bool hasRoundModeOp(uint64_t TSFlags)
static unsigned getVLOpNum(const MCInstrDesc &Desc)
static bool hasVLOp(uint64_t TSFlags)
static int getFRMOpNum(const MCInstrDesc &Desc)
static bool hasVecPolicyOp(uint64_t TSFlags)
static bool usesVXRM(uint64_t TSFlags)
static unsigned getTailExpandUseRegNo(const FeatureBitset &FeatureBits)
static bool isRVVWideningReduction(uint64_t TSFlags)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
static bool hasSEWOp(uint64_t TSFlags)
static bool isFirstDefTiedToFirstUse(const MCInstrDesc &Desc)
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
@ OPERAND_UIMMLOG2XLEN_NONZERO
@ OPERAND_SIMM12_LSB00000
@ OPERAND_FIRST_RISCV_IMM
@ OPERAND_UIMM10_LSB00_NONZERO
@ OPERAND_SIMM10_LSB0000_NONZERO
static RISCVII::VLMUL getLMul(uint64_t TSFlags)
static unsigned getNF(uint64_t TSFlags)
static bool isTailAgnostic(unsigned VType)
static RISCVII::VLMUL getVLMUL(unsigned VType)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
static bool isValidSEW(unsigned SEW)
void printVType(unsigned VType, raw_ostream &OS)
static unsigned getSEW(unsigned VType)
bool hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2)
bool isVLKnownLE(const MachineOperand &LHS, const MachineOperand &RHS)
Given two VL operands, do we know that LHS <= RHS?
std::optional< unsigned > getVectorLowDemandedScalarBits(uint16_t Opcode, unsigned Log2SEW)
int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex)
unsigned getRVVMCOpcode(unsigned RVVPseudoOpcode)
unsigned getDestLog2EEW(const MCInstrDesc &Desc, unsigned Log2SEW)
bool isSEXT_W(const MachineInstr &MI)
bool isFaultFirstLoad(const MachineInstr &MI)
std::optional< std::pair< unsigned, unsigned > > isRVVSpillForZvlsseg(unsigned Opcode)
bool isZEXT_B(const MachineInstr &MI)
bool isRVVSpill(const MachineInstr &MI)
static constexpr int64_t VLMaxSentinel
bool isZEXT_W(const MachineInstr &MI)
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
InstrType
Represents how an instruction should be mapped by the outliner.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
MachineTraceStrategy
Strategies for selecting traces.
@ TS_MinInstrCount
Select the trace through a block that has the fewest instructions.
@ TS_Local
Select the trace that contains only the current basic block.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static const MachineMemOperand::Flags MONontemporalBit1
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
static const MachineMemOperand::Flags MONontemporalBit0
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
unsigned getDeadRegState(bool B)
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
CombinerObjective
The combiner's goal may differ based on which pattern it is attempting to optimize.
CodeGenOptLevel
Code generation optimization level.
unsigned getKillRegState(bool B)
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
unsigned getRenamableRegState(bool B)
DWARFExpression::Operation Op
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Description of the encoding of one expression Op.
Used to describe addressing mode similar to ExtAddrMode in CodeGenPrepare.
This represents a simple continuous liveness interval for a value.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
static bool isRVVRegClass(const TargetRegisterClass *RC)
Used to describe a register and immediate addition.
An individual sequence of instructions to be replaced with a call to an outlined function.
MachineFunction * getMF() const
The information necessary to create an outlined function for some class of candidate.
unsigned FrameConstructionID
Target-defined identifier for constructing a frame for this function.