LLVM: lib/Target/X86/X86ISelDAGToDAG.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
13
22#include "llvm/Config/llvm-config.h"
27#include "llvm/IR/IntrinsicsX86.h"
34#include
35
36using namespace llvm;
37
38#define DEBUG_TYPE "x86-isel"
39#define PASS_NAME "X86 DAG->DAG Instruction Selection"
40
41STATISTIC(NumLoadMoved, "Number of loads moved below TokenFactor");
42
44 cl::desc("Enable setting constant bits to reduce size of mask immediates"),
46
48 "x86-promote-anyext-load", cl::init(true),
49 cl::desc("Enable promoting aligned anyext load to wider load"), cl::Hidden);
50
52
53
54
55
56
57namespace {
58
59
60 struct X86ISelAddressMode {
61 enum {
62 RegBase,
63 FrameIndexBase
64 } BaseType = RegBase;
65
66
68 int Base_FrameIndex = 0;
69
70 unsigned Scale = 1;
72 int32_t Disp = 0;
74 const GlobalValue *GV = nullptr;
77 const char *ES = nullptr;
79 int JT = -1;
80 Align Alignment;
82 bool NegateIndex = false;
83
84 X86ISelAddressMode() = default;
85
86 bool hasSymbolicDisplacement() const {
87 return GV != nullptr || CP != nullptr || ES != nullptr ||
88 MCSym != nullptr || JT != -1 || BlockAddr != nullptr;
89 }
90
91 bool hasBaseOrIndexReg() const {
92 return BaseType == FrameIndexBase ||
93 IndexReg.getNode() != nullptr || Base_Reg.getNode() != nullptr;
94 }
95
96
98 if (BaseType != RegBase) return false;
99 if (RegisterSDNode *RegNode =
101 return RegNode->getReg() == X86::RIP;
102 return false;
103 }
104
106 BaseType = RegBase;
107 Base_Reg = Reg;
108 }
109
110#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
111 void dump(SelectionDAG *DAG = nullptr) {
112 dbgs() << "X86ISelAddressMode " << this << '\n';
113 dbgs() << "Base_Reg ";
114 if (Base_Reg.getNode())
115 Base_Reg.getNode()->dump(DAG);
116 else
117 dbgs() << "nul\n";
118 if (BaseType == FrameIndexBase)
119 dbgs() << " Base.FrameIndex " << Base_FrameIndex << '\n';
120 dbgs() << " Scale " << Scale << '\n'
121 << "IndexReg ";
122 if (NegateIndex)
123 dbgs() << "negate ";
124 if (IndexReg.getNode())
125 IndexReg.getNode()->dump(DAG);
126 else
127 dbgs() << "nul\n";
128 dbgs() << " Disp " << Disp << '\n'
129 << "GV ";
130 if (GV)
131 GV->dump();
132 else
133 dbgs() << "nul";
134 dbgs() << " CP ";
135 if (CP)
136 CP->dump();
137 else
138 dbgs() << "nul";
139 dbgs() << '\n'
140 << "ES ";
141 if (ES)
142 dbgs() << ES;
143 else
144 dbgs() << "nul";
145 dbgs() << " MCSym ";
146 if (MCSym)
147 dbgs() << MCSym;
148 else
149 dbgs() << "nul";
150 dbgs() << " JT" << JT << " Align" << Alignment.value() << '\n';
151 }
152#endif
153 };
154}
155
156namespace {
157
158
159
160
162
163
164 const X86Subtarget *Subtarget;
165
166
167 bool OptForMinSize;
168
169
170 bool IndirectTlsSegRefs;
171
172 public:
173 X86DAGToDAGISel() = delete;
174
175 explicit X86DAGToDAGISel(X86TargetMachine &tm, CodeGenOptLevel OptLevel)
176 : SelectionDAGISel(tm, OptLevel), Subtarget(nullptr),
177 OptForMinSize(false), IndirectTlsSegRefs(false) {}
178
179 bool runOnMachineFunction(MachineFunction &MF) override {
180
181 Subtarget = &MF.getSubtarget();
183 "indirect-tls-seg-refs");
184
185
188 }
189
190 void emitFunctionEntryCode() override;
191
192 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override;
193
194 void PreprocessISelDAG() override;
195 void PostprocessISelDAG() override;
196
197
198#include "X86GenDAGISel.inc"
199
200 private:
201 void Select(SDNode *N) override;
202
203 bool foldOffsetIntoAddress(uint64_t Offset, X86ISelAddressMode &AM);
204 bool matchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM,
205 bool AllowSegmentRegForX32 = false);
206 bool matchWrapper(SDValue N, X86ISelAddressMode &AM);
207 bool matchAddress(SDValue N, X86ISelAddressMode &AM);
208 bool matchVectorAddress(SDValue N, X86ISelAddressMode &AM);
209 bool matchAdd(SDValue &N, X86ISelAddressMode &AM, unsigned Depth);
210 SDValue matchIndexRecursively(SDValue N, X86ISelAddressMode &AM,
212 bool matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
214 bool matchVectorAddressRecursively(SDValue N, X86ISelAddressMode &AM,
216 bool matchAddressBase(SDValue N, X86ISelAddressMode &AM);
220 bool selectVectorAddr(MemSDNode *Parent, SDValue BasePtr, SDValue IndexOp,
233
234 bool tryFoldLoad(SDNode *Root, SDNode *P, SDValue N,
238
239
240 bool tryFoldLoad(SDNode *P, SDValue N,
244 return tryFoldLoad(P, P, N, Base, Scale, Index, Disp, Segment);
245 }
246
247 bool tryFoldBroadcast(SDNode *Root, SDNode *P, SDValue N,
251
252 bool isProfitableToFormMaskedOp(SDNode *N) const;
253
254
255 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
257 std::vector &OutOps) override;
258
259 void emitSpecialCodeForMain();
260
261 inline void getAddressOperands(X86ISelAddressMode &AM, const SDLoc &DL,
265 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
266 Base = CurDAG->getTargetFrameIndex(
267 AM.Base_FrameIndex, TLI->getPointerTy(CurDAG->getDataLayout()));
268 else if (AM.Base_Reg.getNode())
269 Base = AM.Base_Reg;
270 else
271 Base = CurDAG->getRegister(0, VT);
272
273 Scale = getI8Imm(AM.Scale, DL);
274
275#define GET_ND_IF_ENABLED(OPC) (Subtarget->hasNDD() ? OPC##_ND : OPC)
276
277 if (AM.NegateIndex) {
278 unsigned NegOpc;
280 default:
282 case MVT::i64:
284 break;
285 case MVT::i32:
287 break;
288 case MVT::i16:
290 break;
291 case MVT::i8:
293 break;
294 }
295 SDValue Neg = SDValue(CurDAG->getMachineNode(NegOpc, DL, VT, MVT::i32,
296 AM.IndexReg), 0);
297 AM.IndexReg = Neg;
298 }
299
300 if (AM.IndexReg.getNode())
301 Index = AM.IndexReg;
302 else
303 Index = CurDAG->getRegister(0, VT);
304
305
306
307 if (AM.GV)
308 Disp = CurDAG->getTargetGlobalAddress(AM.GV, SDLoc(),
309 MVT::i32, AM.Disp,
310 AM.SymbolFlags);
311 else if (AM.CP)
312 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, AM.Alignment,
313 AM.Disp, AM.SymbolFlags);
314 else if (AM.ES) {
315 assert(!AM.Disp && "Non-zero displacement is ignored with ES.");
316 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags);
317 } else if (AM.MCSym) {
318 assert(!AM.Disp && "Non-zero displacement is ignored with MCSym.");
319 assert(AM.SymbolFlags == 0 && "oo");
320 Disp = CurDAG->getMCSymbol(AM.MCSym, MVT::i32);
321 } else if (AM.JT != -1) {
322 assert(!AM.Disp && "Non-zero displacement is ignored with JT.");
323 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags);
324 } else if (AM.BlockAddr)
325 Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp,
326 AM.SymbolFlags);
327 else
328 Disp = CurDAG->getSignedTargetConstant(AM.Disp, DL, MVT::i32);
329
330 if (AM.Segment.getNode())
331 Segment = AM.Segment;
332 else
333 Segment = CurDAG->getRegister(0, MVT::i16);
334 }
335
336
337
338 bool isAMXSDNode(SDNode *N) const {
339
340
341
342 for (unsigned Idx = 0, E = N->getNumValues(); Idx != E; ++Idx) {
343 if (N->getValueType(Idx) == MVT::x86amx)
344 return true;
345 }
346 for (unsigned Idx = 0, E = N->getNumOperands(); Idx != E; ++Idx) {
348 if (Op.getValueType() == MVT::x86amx)
349 return true;
350 }
351 return false;
352 }
353
354
355
356
357
358
359
360 bool shouldAvoidImmediateInstFormsForSize(SDNode *N) const {
361 uint32_t UseCount = 0;
362
363
364
365
366 if (!CurDAG->shouldOptForSize())
367 return false;
368
369
370 for (const SDNode *User : N->users()) {
371 if (UseCount >= 2)
372 break;
373
374
375
376 if (User->isMachineOpcode()) {
377 UseCount++;
378 continue;
379 }
380
381
382 if (User->getOpcode() == ISD::STORE &&
383 User->getOperand(1).getNode() == N) {
384 UseCount++;
385 continue;
386 }
387
388
389
390
391
392
393
394 if (User->getNumOperands() != 2)
395 continue;
396
397
398
400 if (C && isInt<8>(C->getSExtValue()))
401 continue;
402
403
404
405
406
411
412
415 OtherOp = User->getOperand(1);
416
417
418 RegisterSDNode *RegNode;
422 if ((RegNode->getReg() == X86::ESP) ||
423 (RegNode->getReg() == X86::RSP))
424 continue;
425 }
426
427
428 UseCount++;
429 }
430
431
432 return (UseCount > 1);
433 }
434
435
436 inline SDValue getI8Imm(unsigned Imm, const SDLoc &DL) {
437 return CurDAG->getTargetConstant(Imm, DL, MVT::i8);
438 }
439
440
441 inline SDValue getI32Imm(unsigned Imm, const SDLoc &DL) {
442 return CurDAG->getTargetConstant(Imm, DL, MVT::i32);
443 }
444
445
446 inline SDValue getI64Imm(uint64_t Imm, const SDLoc &DL) {
447 return CurDAG->getTargetConstant(Imm, DL, MVT::i64);
448 }
449
450 SDValue getExtractVEXTRACTImmediate(SDNode *N, unsigned VecWidth,
451 const SDLoc &DL) {
452 assert((VecWidth == 128 || VecWidth == 256) && "Unexpected vector width");
453 uint64_t Index = N->getConstantOperandVal(1);
454 MVT VecVT = N->getOperand(0).getSimpleValueType();
456 }
457
458 SDValue getInsertVINSERTImmediate(SDNode *N, unsigned VecWidth,
459 const SDLoc &DL) {
460 assert((VecWidth == 128 || VecWidth == 256) && "Unexpected vector width");
461 uint64_t Index = N->getConstantOperandVal(2);
462 MVT VecVT = N->getSimpleValueType(0);
464 }
465
466 SDValue getPermuteVINSERTCommutedImmediate(SDNode *N, unsigned VecWidth,
467 const SDLoc &DL) {
468 assert(VecWidth == 128 && "Unexpected vector width");
469 uint64_t Index = N->getConstantOperandVal(2);
470 MVT VecVT = N->getSimpleValueType(0);
472 assert((InsertIdx == 0 || InsertIdx == 1) && "Bad insertf128 index");
473
474
475 return getI8Imm(InsertIdx ? 0x02 : 0x30, DL);
476 }
477
478 SDValue getSBBZero(SDNode *N) {
479 SDLoc dl(N);
480 MVT VT = N->getSimpleValueType(0);
481
482
483 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
485 SDValue(CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, {}), 0);
486 if (VT == MVT::i64) {
488 CurDAG->getMachineNode(
489 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
490 CurDAG->getTargetConstant(0, dl, MVT::i64), Zero,
491 CurDAG->getTargetConstant(X86::sub_32bit, dl, MVT::i32)),
492 0);
493 }
494
495
496 unsigned Opcode = N->getOpcode();
498 "Unexpected opcode for SBB materialization");
499 unsigned FlagOpIndex = Opcode == X86ISD::SBB ? 2 : 1;
501 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
502 N->getOperand(FlagOpIndex), SDValue());
503
504
505
506 unsigned Opc = VT == MVT::i64 ? X86::SBB64rr : X86::SBB32rr;
507 MVT SBBVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
508 VTs = CurDAG->getVTList(SBBVT, MVT::i32);
510 CurDAG->getMachineNode(Opc, dl, VTs,
511 {Zero, Zero, EFLAGS, EFLAGS.getValue(1)}),
512 0);
513 }
514
515
516
517 bool isUnneededShiftMask(SDNode *N, unsigned Width) const {
518 assert(N->getOpcode() == ISD::AND && "Unexpected opcode");
519 const APInt &Val = N->getConstantOperandAPInt(1);
520
522 return true;
523
524 APInt Mask = Val | CurDAG->computeKnownBits(N->getOperand(0)).Zero;
525 return Mask.countr_one() >= Width;
526 }
527
528
529
530
531 SDNode *getGlobalBaseReg();
532
533
534
535 const X86TargetMachine &getTargetMachine() const {
536 return static_cast<const X86TargetMachine &>(TM);
537 }
538
539
540
541 const X86InstrInfo *getInstrInfo() const {
542 return Subtarget->getInstrInfo();
543 }
544
545
547
548
549
550
551 bool ComplexPatternFuncMutatesDAG() const override {
552 return true;
553 }
554
555 bool isSExtAbsoluteSymbolRef(unsigned Width, SDNode *N) const;
556
557
558 bool useNonTemporalLoad(LoadSDNode *N) const {
559 if (->isNonTemporal())
560 return false;
561
562 unsigned StoreSize = N->getMemoryVT().getStoreSize();
563
564 if (N->getAlign().value() < StoreSize)
565 return false;
566
567 switch (StoreSize) {
569 case 4:
570 case 8:
571 return false;
572 case 16:
573 return Subtarget->hasSSE41();
574 case 32:
575 return Subtarget->hasAVX2();
576 case 64:
577 return Subtarget->hasAVX512();
578 }
579 }
580
581 bool foldLoadStoreIntoMemOperand(SDNode *Node);
582 MachineSDNode *matchBEXTRFromAndImm(SDNode *Node);
583 bool matchBitExtract(SDNode *Node);
584 bool shrinkAndImmediate(SDNode *N);
585 bool isMaskZeroExtended(SDNode *N) const;
586 bool tryShiftAmountMod(SDNode *N);
587 bool tryShrinkShlLogicImm(SDNode *N);
588 bool tryVPTERNLOG(SDNode *N);
589 bool matchVPTERNLOG(SDNode *Root, SDNode *ParentA, SDNode *ParentB,
591 uint8_t Imm);
592 bool tryVPTESTM(SDNode *Root, SDValue Setcc, SDValue Mask);
593 bool tryMatchBitSelect(SDNode *N);
594
595 MachineSDNode *emitPCMPISTR(unsigned ROpc, unsigned MOpc, bool MayFoldLoad,
596 const SDLoc &dl, MVT VT, SDNode *Node);
597 MachineSDNode *emitPCMPESTR(unsigned ROpc, unsigned MOpc, bool MayFoldLoad,
598 const SDLoc &dl, MVT VT, SDNode *Node,
600
601 bool tryOptimizeRem8Extend(SDNode *N);
602
603 bool onlyUsesZeroFlag(SDValue Flags) const;
604 bool hasNoSignFlagUses(SDValue Flags) const;
605 bool hasNoCarryFlagUses(SDValue Flags) const;
606 };
607
609 public:
610 static char ID;
611 explicit X86DAGToDAGISelLegacy(X86TargetMachine &tm,
613 : SelectionDAGISelLegacy(
614 ID, std::make_unique(tm, OptLevel)) {}
615 };
616}
617
618char X86DAGToDAGISelLegacy::ID = 0;
619
621
622
623
625 unsigned Opcode = N->getOpcode();
629
630
631
632 EVT OpVT = N->getOperand(0).getValueType();
633
634
636 OpVT = N->getOperand(1).getValueType();
638 return Subtarget->hasVLX();
639
640 return true;
641 }
642
645 return true;
646
647 return false;
648}
649
650
651
652bool X86DAGToDAGISel::isMaskZeroExtended(SDNode *N) const {
653
654
655
659
661}
662
663bool
664X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const {
665 if (OptLevel == CodeGenOptLevel::None)
666 return false;
667
668 if (.hasOneUse())
669 return false;
670
671 if (N.getOpcode() != ISD::LOAD)
672 return true;
673
674
676 return false;
677
678
679 if (U == Root) {
680 switch (U->getOpcode()) {
681 default: break;
694 SDValue Op1 = U->getOperand(1);
695
696
697
698
699
700
701
702
703
704
705
707 if (Imm->getAPIntValue().isSignedIntN(8))
708 return false;
709
710
711
712
713
714
715 if (U->getOpcode() == ISD::AND &&
716 Imm->getAPIntValue().getBitWidth() == 64 &&
717 Imm->getAPIntValue().isIntN(32))
718 return false;
719
720
721
722
723 if (U->getOpcode() == ISD::AND &&
724 (Imm->getAPIntValue() == UINT8_MAX ||
725 Imm->getAPIntValue() == UINT16_MAX ||
726 Imm->getAPIntValue() == UINT32_MAX))
727 return false;
728
729
730
732 (-Imm->getAPIntValue()).isSignedIntN(8))
733 return false;
734
736 (-Imm->getAPIntValue()).isSignedIntN(8) &&
737 hasNoCarryFlagUses(SDValue(U, 1)))
738 return false;
739 }
740
741
742
743
744
745
746
747
748
749
750
754 return false;
755 }
756
757
758
759
760
762 if (U->getOperand(0).getOpcode() == ISD::SHL &&
764 return false;
765
766 if (U->getOperand(1).getOpcode() == ISD::SHL &&
768 return false;
769 }
770 if (U->getOpcode() == ISD::AND) {
771 SDValue U0 = U->getOperand(0);
772 SDValue U1 = U->getOperand(1);
775 if (C && C->getSExtValue() == -2)
776 return false;
777 }
778
781 if (C && C->getSExtValue() == -2)
782 return false;
783 }
784 }
785
786 break;
787 }
791
792
793
794
796 return false;
797
798 break;
799 }
800 }
801
802
803
808 return false;
809
810 return true;
811}
812
813
814
815
816bool X86DAGToDAGISel::isProfitableToFormMaskedOp(SDNode *N) const {
819 "Unexpected opcode!");
820
821
822
823
824 return N->getOperand(1).hasOneUse();
825}
826
827
828
833 if (Chain.getNode() == Load.getNode())
834 Ops.push_back(Load.getOperand(0));
835 else {
837 "Unexpected chain operand");
838 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i)
840 Ops.push_back(Load.getOperand(0));
841 else
845 Ops.clear();
846 Ops.push_back(NewChain);
847 }
851 Load.getOperand(1), Load.getOperand(2));
852
853 Ops.clear();
854 Ops.push_back(SDValue(Load.getNode(), 1));
855 Ops.append(Call->op_begin() + 1, Call->op_end());
857}
858
859
860
861
862
863
865
866
867
868
869 if (Callee.getNode() == Chain.getNode() || !Callee.hasOneUse())
870 return false;
872 if (!LD ||
873 !LD->isSimple() ||
876 return false;
877
878
879 while (HasCallSeq && Chain.getOpcode() != ISD::CALLSEQ_START) {
881 return false;
883 }
884
886 return false;
887
888
891 return false;
893 return true;
896 Callee.getValue(1).hasOneUse())
897 return true;
898 return false;
899}
900
902
903
904 if ((Imm & 0x00FFFFFF) != 0x0F1EFA)
905 return false;
906
907 uint8_t OptionalPrefixBytes [] = {0x26, 0x2e, 0x36, 0x3e, 0x64,
908 0x65, 0x66, 0x67, 0xf0, 0xf2};
909 int i = 24;
910 while (i < 64) {
911 uint8_t Byte = (Imm >> i) & 0xFF;
912 if (Byte == 0xF3)
913 return true;
915 return false;
916 i += 8;
917 }
918
919 return false;
920}
921
923 return (VT == MVT::v32i16 || VT == MVT::v32f16 || VT == MVT::v64i8);
924}
925
926void X86DAGToDAGISel::PreprocessISelDAG() {
927 bool MadeChange = false;
929 E = CurDAG->allnodes_end(); I != E; ) {
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
948 MVT VT = N->getSimpleValueType(0);
950 int32_t EndbrImm = Subtarget->is64Bit() ? 0xF30F1EFA : 0xF30F1EFB;
952
953 Metadata *CFProtectionBranch =
955 "cf-protection-branch");
957 SDLoc dl(N);
958 SDValue Complement = CurDAG->getConstant(~Imm, dl, VT, false, true);
959 Complement = CurDAG->getNOT(dl, Complement, VT);
960 --I;
961 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Complement);
962 ++I;
963 MadeChange = true;
964 continue;
965 }
966 }
967 }
968
969
970
971 if (N->getOpcode() == X86ISD::AND && ->hasAnyUseOfValue(1)) {
972 SDValue Res = CurDAG->getNode(ISD::AND, SDLoc(N), N->getValueType(0),
973 N->getOperand(0), N->getOperand(1));
974 --I;
975 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
976 ++I;
977 MadeChange = true;
978 continue;
979 }
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999 auto mayPreventLoadFold = [&]() {
1001 N->getOpcode() == ISD::ADD && Subtarget->hasAVX() &&
1002 ->getOperand(1).hasOneUse();
1003 };
1005 N->getSimpleValueType(0).isVector() && !mayPreventLoadFold()) {
1006 APInt SplatVal;
1010 SplatVal.isOne()) {
1012
1013 MVT VT = N->getSimpleValueType(0);
1016 CurDAG->getAllOnesConstant(DL, MVT::getVectorVT(MVT::i32, NumElts));
1018
1021 CurDAG->getNode(NewOpcode, DL, VT, N->getOperand(0), AllOnes);
1022 --I;
1023 CurDAG->ReplaceAllUsesWith(N, Res.getNode());
1024 ++I;
1025 MadeChange = true;
1026 continue;
1027 }
1028 }
1029
1030 switch (N->getOpcode()) {
1032 MVT VT = N->getSimpleValueType(0);
1033
1034 if (!Subtarget->hasBWI() && needBWI(VT)) {
1036 SDLoc dl(N);
1041 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
1044 CurDAG->getIntPtrConstant(Index, dl));
1045
1046 --I;
1047 CurDAG->ReplaceAllUsesWith(N, Res.getNode());
1048 ++I;
1049 MadeChange = true;
1050 continue;
1051 }
1052
1053 break;
1054 }
1056 MVT VT = N->getSimpleValueType(0);
1057
1058 if (!Subtarget->hasBWI() && needBWI(VT)) {
1061 SDLoc dl(N);
1062 SDVTList VTs = CurDAG->getVTList(NarrowVT, MVT::Other);
1063 SDValue Ops[] = {MemNode->getChain(), MemNode->getBasePtr()};
1064 SDValue NarrowBCast = CurDAG->getMemIntrinsicNode(
1066 MemNode->getMemOperand());
1069 NarrowBCast, CurDAG->getIntPtrConstant(0, dl));
1072 CurDAG->getIntPtrConstant(Index, dl));
1073
1074 --I;
1076 CurDAG->ReplaceAllUsesWith(N, To);
1077 ++I;
1078 MadeChange = true;
1079 continue;
1080 }
1081
1082 break;
1083 }
1084 case ISD::LOAD: {
1085
1086
1088 MVT VT = N->getSimpleValueType(0);
1091 break;
1092
1093 MVT MaxVT = VT;
1094 SDNode *MaxLd = nullptr;
1095 SDValue Ptr = Ld->getBasePtr();
1096 SDValue Chain = Ld->getChain();
1097 for (SDNode *User : Ptr->users()) {
1099 MVT UserVT = User->getSimpleValueType(0);
1101 UserLd->getBasePtr() == Ptr && UserLd->getChain() == Chain &&
1102 ->hasAnyUseOfValue(1) &&
1106 MaxLd = User;
1107 MaxVT = UserVT;
1108 }
1109 }
1110 if (MaxLd) {
1111 SDLoc dl(N);
1116 CurDAG->getIntPtrConstant(0, dl));
1117 SDValue Res = CurDAG->getBitcast(VT, Extract);
1118
1119 --I;
1121 CurDAG->ReplaceAllUsesWith(N, To);
1122 ++I;
1123 MadeChange = true;
1124 continue;
1125 }
1126 break;
1127 }
1129
1130 EVT EleVT = N->getOperand(0).getValueType().getVectorElementType();
1131 if (EleVT == MVT::i1)
1132 break;
1133
1134 assert(Subtarget->hasSSE41() && "Expected SSE4.1 support!");
1135 assert(N->getValueType(0).getVectorElementType() != MVT::i16 &&
1136 "We can't replace VSELECT with BLENDV in vXi16!");
1138 if (Subtarget->hasVLX() && CurDAG->ComputeNumSignBits(N->getOperand(0)) ==
1141 N->getOperand(0), N->getOperand(1), N->getOperand(2),
1142 CurDAG->getTargetConstant(0xCA, SDLoc(N), MVT::i8));
1143 } else {
1144 R = CurDAG->getNode(X86ISD::BLENDV, SDLoc(N), N->getValueType(0),
1145 N->getOperand(0), N->getOperand(1),
1146 N->getOperand(2));
1147 }
1148 --I;
1149 CurDAG->ReplaceAllUsesWith(N, R.getNode());
1150 ++I;
1151 MadeChange = true;
1152 continue;
1153 }
1160
1161
1162 if (->getSimpleValueType(0).isVector())
1163 break;
1164
1165 unsigned NewOpc;
1166 switch (N->getOpcode()) {
1174 }
1176 if (N->isStrictFPOpcode())
1177 Res =
1178 CurDAG->getNode(NewOpc, SDLoc(N), {N->getValueType(0), MVT::Other},
1179 {N->getOperand(0), N->getOperand(1)});
1180 else
1181 Res =
1182 CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0),
1183 N->getOperand(0));
1184 --I;
1185 CurDAG->ReplaceAllUsesWith(N, Res.getNode());
1186 ++I;
1187 MadeChange = true;
1188 continue;
1189 }
1193
1194
1195 if (->getValueType(0).isVector())
1196 break;
1197
1198 unsigned NewOpc;
1199 switch (N->getOpcode()) {
1204 }
1205 SDValue Res = CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0),
1206 N->getOperand(0), N->getOperand(1));
1207 --I;
1208 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
1209 ++I;
1210 MadeChange = true;
1211 continue;
1212 }
1215
1216
1217 if (->getValueType(0).isVector())
1218 break;
1219
1220 unsigned NewOpc;
1221 if (N->getOperand(0).getScalarValueSizeInBits() == 1) {
1223 "Unexpected opcode for mask vector!");
1225 } else {
1229 }
1230
1231 SDValue Res = CurDAG->getNode(NewOpc, SDLoc(N), N->getValueType(0),
1232 N->getOperand(0));
1233 --I;
1234 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
1235 ++I;
1236 MadeChange = true;
1237 continue;
1238 }
1239 case ISD::FCEIL:
1241 case ISD::FFLOOR:
1243 case ISD::FTRUNC:
1245 case ISD::FROUNDEVEN:
1247 case ISD::FNEARBYINT:
1249 case ISD::FRINT:
1251
1252
1253 unsigned Imm;
1254 switch (N->getOpcode()) {
1257 case ISD::FCEIL: Imm = 0xA; break;
1259 case ISD::FFLOOR: Imm = 0x9; break;
1261 case ISD::FTRUNC: Imm = 0xB; break;
1263 case ISD::FROUNDEVEN: Imm = 0x8; break;
1265 case ISD::FNEARBYINT: Imm = 0xC; break;
1267 case ISD::FRINT: Imm = 0x4; break;
1268 }
1269 SDLoc dl(N);
1270 bool IsStrict = N->isStrictFPOpcode();
1272 if (IsStrict)
1274 {N->getValueType(0), MVT::Other},
1275 {N->getOperand(0), N->getOperand(1),
1276 CurDAG->getTargetConstant(Imm, dl, MVT::i32)});
1277 else
1279 N->getOperand(0),
1280 CurDAG->getTargetConstant(Imm, dl, MVT::i32));
1281 --I;
1282 CurDAG->ReplaceAllUsesWith(N, Res.getNode());
1283 ++I;
1284 MadeChange = true;
1285 continue;
1286 }
1291
1292
1293 MVT VT = N->getSimpleValueType(0);
1294 if (VT.isVector() || VT == MVT::f128)
1295 break;
1296
1297 MVT VecVT = VT == MVT::f64 ? MVT::v2f64
1298 : VT == MVT::f32 ? MVT::v4f32
1299 : MVT::v8f16;
1300
1301 SDLoc dl(N);
1303 N->getOperand(0));
1305 N->getOperand(1));
1306
1308 if (Subtarget->hasSSE2()) {
1309 EVT IntVT = EVT(VecVT).changeVectorElementTypeToInteger();
1310 Op0 = CurDAG->getNode(ISD::BITCAST, dl, IntVT, Op0);
1311 Op1 = CurDAG->getNode(ISD::BITCAST, dl, IntVT, Op1);
1312 unsigned Opc;
1313 switch (N->getOpcode()) {
1319 }
1320 Res = CurDAG->getNode(Opc, dl, IntVT, Op0, Op1);
1321 Res = CurDAG->getNode(ISD::BITCAST, dl, VecVT, Res);
1322 } else {
1323 Res = CurDAG->getNode(N->getOpcode(), dl, VecVT, Op0, Op1);
1324 }
1326 CurDAG->getIntPtrConstant(0, dl));
1327 --I;
1328 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
1329 ++I;
1330 MadeChange = true;
1331 continue;
1332 }
1333 }
1334
1335 if (OptLevel != CodeGenOptLevel::None &&
1336
1337
1338 !Subtarget->useIndirectThunkCalls() &&
1339 ((N->getOpcode() == X86ISD::CALL && !Subtarget->slowTwoMemOps()) ||
1341 (Subtarget->is64Bit() ||
1342 !getTargetMachine().isPositionIndependent())))) {
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362 bool HasCallSeq = N->getOpcode() == X86ISD::CALL;
1366 continue;
1368 ++NumLoadMoved;
1369 MadeChange = true;
1370 continue;
1371 }
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381 switch (N->getOpcode()) {
1382 default: continue;
1384 case ISD::FP_EXTEND:
1385 {
1386 MVT SrcVT = N->getOperand(0).getSimpleValueType();
1387 MVT DstVT = N->getSimpleValueType(0);
1388
1389
1391 continue;
1392
1393
1394
1395 const X86TargetLowering *X86Lowering =
1396 static_cast<const X86TargetLowering *>(TLI);
1399 if (SrcIsSSE && DstIsSSE)
1400 continue;
1401
1402 if (!SrcIsSSE && !DstIsSSE) {
1403
1404 if (N->getOpcode() == ISD::FP_EXTEND)
1405 continue;
1406
1407 if (N->getConstantOperandVal(1))
1408 continue;
1409 }
1410
1411
1412
1413
1414 MVT MemVT = (N->getOpcode() == ISD::FP_ROUND) ? DstVT : SrcVT;
1415 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1417 MachinePointerInfo MPI =
1419 SDLoc dl(N);
1420
1421
1422
1424 CurDAG->getEntryNode(), dl, N->getOperand(0), MemTmp, MPI, MemVT);
1426 MemTmp, MPI, MemVT);
1427
1428
1429
1430
1431
1432 --I;
1433 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
1434 break;
1435 }
1436
1437
1438
1441 {
1442 MVT SrcVT = N->getOperand(1).getSimpleValueType();
1443 MVT DstVT = N->getSimpleValueType(0);
1444
1445
1447 continue;
1448
1449
1450
1451 const X86TargetLowering *X86Lowering =
1452 static_cast<const X86TargetLowering *>(TLI);
1455 if (SrcIsSSE && DstIsSSE)
1456 continue;
1457
1458 if (!SrcIsSSE && !DstIsSSE) {
1459
1461 continue;
1462
1463 if (N->getConstantOperandVal(2))
1464 continue;
1465 }
1466
1467
1468
1469
1471 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT);
1473 MachinePointerInfo MPI =
1475 SDLoc dl(N);
1476
1477
1478
1479
1481 if (!SrcIsSSE) {
1482 SDVTList VTs = CurDAG->getVTList(MVT::Other);
1483 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), MemTmp};
1485 MPI, std::nullopt,
1487 if (N->getFlags().hasNoFPExcept()) {
1488 SDNodeFlags Flags = Store->getFlags();
1489 Flags.setNoFPExcept(true);
1490 Store->setFlags(Flags);
1491 }
1492 } else {
1493 assert(SrcVT == MemVT && "Unexpected VT!");
1494 Store = CurDAG->getStore(N->getOperand(0), dl, N->getOperand(1), MemTmp,
1495 MPI);
1496 }
1497
1498 if (!DstIsSSE) {
1499 SDVTList VTs = CurDAG->getVTList(DstVT, MVT::Other);
1501 Result = CurDAG->getMemIntrinsicNode(
1504 if (N->getFlags().hasNoFPExcept()) {
1506 Flags.setNoFPExcept(true);
1507 Result->setFlags(Flags);
1508 }
1509 } else {
1510 assert(DstVT == MemVT && "Unexpected VT!");
1511 Result = CurDAG->getLoad(DstVT, dl, Store, MemTmp, MPI);
1512 }
1513
1514
1515
1516
1517
1518 --I;
1519 CurDAG->ReplaceAllUsesWith(N, Result.getNode());
1520 break;
1521 }
1522 }
1523
1524
1525
1526
1527 ++I;
1528 MadeChange = true;
1529 }
1530
1531
1532 if (MadeChange)
1533 CurDAG->RemoveDeadNodes();
1534}
1535
1536
1537bool X86DAGToDAGISel::tryOptimizeRem8Extend(SDNode *N) {
1538 unsigned Opc = N->getMachineOpcode();
1539 if (Opc != X86::MOVZX32rr8 && Opc != X86::MOVSX32rr8 &&
1540 Opc != X86::MOVSX64rr8)
1541 return false;
1542
1543 SDValue N0 = N->getOperand(0);
1544
1545
1549 return false;
1550
1551
1552 unsigned ExpectedOpc = Opc == X86::MOVZX32rr8 ? X86::MOVZX32rr8_NOREX
1553 : X86::MOVSX32rr8_NOREX;
1556 return false;
1557
1558 if (Opc == X86::MOVSX64rr8) {
1559
1560
1561 MachineSDNode *Extend = CurDAG->getMachineNode(X86::MOVSX64rr32, SDLoc(N),
1562 MVT::i64, N00);
1563 ReplaceUses(N, Extend);
1564 } else {
1565
1566 ReplaceUses(N, N00.getNode());
1567 }
1568
1569 return true;
1570}
1571
1572void X86DAGToDAGISel::PostprocessISelDAG() {
1573
1574 if (TM.getOptLevel() == CodeGenOptLevel::None)
1575 return;
1576
1578
1579 bool MadeChange = false;
1580 while (Position != CurDAG->allnodes_begin()) {
1581 SDNode *N = &*--Position;
1582
1583 if (N->use_empty() || ->isMachineOpcode())
1584 continue;
1585
1586 if (tryOptimizeRem8Extend(N)) {
1587 MadeChange = true;
1588 continue;
1589 }
1590
1591 unsigned Opc = N->getMachineOpcode();
1592 switch (Opc) {
1593 default:
1594 continue;
1595
1596 case X86::TEST8rr:
1597 case X86::TEST16rr:
1598 case X86::TEST32rr:
1599 case X86::TEST64rr:
1600
1601 case X86::CTEST8rr:
1602 case X86::CTEST16rr:
1603 case X86::CTEST32rr:
1604 case X86::CTEST64rr: {
1608 continue;
1610#define CASE_ND(OP) \
1611 case X86::OP: \
1612 case X86::OP##_ND:
1613 switch (And.getMachineOpcode()) {
1614 default:
1615 continue;
1620 if (And->hasAnyUseOfValue(1))
1621 continue;
1623 Ops[0] = And.getOperand(0);
1624 Ops[1] = And.getOperand(1);
1625 MachineSDNode *Test =
1626 CurDAG->getMachineNode(Opc, SDLoc(N), MVT::i32, Ops);
1628 MadeChange = true;
1629 continue;
1630 }
1635 if (And->hasAnyUseOfValue(1))
1636 continue;
1637 unsigned NewOpc;
1638 bool IsCTESTCC = X86::isCTESTCC(Opc);
1639#define FROM_TO(A, B) \
1640 CASE_ND(A) NewOpc = IsCTESTCC ? X86::C##B : X86::B; \
1641 break;
1642 switch (And.getMachineOpcode()) {
1643 FROM_TO(AND8rm, TEST8mr);
1644 FROM_TO(AND16rm, TEST16mr);
1645 FROM_TO(AND32rm, TEST32mr);
1646 FROM_TO(AND64rm, TEST64mr);
1647 }
1648#undef FROM_TO
1649#undef CASE_ND
1650
1652 And.getOperand(3), And.getOperand(4),
1653 And.getOperand(5), And.getOperand(0)};
1654
1655 if (IsCTESTCC) {
1656 Ops.push_back(N->getOperand(2));
1657 Ops.push_back(N->getOperand(3));
1658 }
1659
1660 Ops.push_back(And.getOperand(6));
1661
1662 if (IsCTESTCC)
1663 Ops.push_back(N->getOperand(4));
1664
1665 MachineSDNode *Test = CurDAG->getMachineNode(
1666 NewOpc, SDLoc(N), MVT::i32, MVT::Other, Ops);
1667 CurDAG->setNodeMemRefs(
1671 MadeChange = true;
1672 continue;
1673 }
1674 }
1675 }
1676
1677
1678
1679
1680 case X86::KORTESTBkk:
1681 case X86::KORTESTWkk:
1682 case X86::KORTESTDkk:
1683 case X86::KORTESTQkk: {
1685 if (Op0 != N->getOperand(1) || ->isOnlyUserOf(Op0.getNode()) ||
1687 continue;
1688#define CASE(A) \
1689 case X86::A: \
1690 break;
1692 default:
1693 continue;
1694 CASE(KANDBkk)
1695 CASE(KANDWkk)
1696 CASE(KANDDkk)
1697 CASE(KANDQkk)
1698 }
1699 unsigned NewOpc;
1700#define FROM_TO(A, B) \
1701 case X86::A: \
1702 NewOpc = X86::B; \
1703 break;
1704 switch (Opc) {
1705 FROM_TO(KORTESTBkk, KTESTBkk)
1706 FROM_TO(KORTESTWkk, KTESTWkk)
1707 FROM_TO(KORTESTDkk, KTESTDkk)
1708 FROM_TO(KORTESTQkk, KTESTQkk)
1709 }
1710
1711
1712 if (NewOpc == X86::KTESTWkk && !Subtarget->hasDQI())
1713 continue;
1714#undef FROM_TO
1715 MachineSDNode *KTest = CurDAG->getMachineNode(
1717 ReplaceUses(N, KTest);
1718 MadeChange = true;
1719 continue;
1720 }
1721
1722 case TargetOpcode::SUBREG_TO_REG: {
1723 unsigned SubRegIdx = N->getConstantOperandVal(2);
1724 if (SubRegIdx != X86::sub_xmm && SubRegIdx != X86::sub_ymm)
1725 continue;
1726
1727 SDValue Move = N->getOperand(1);
1729 continue;
1730
1731
1733 default:
1734 continue;
1735 CASE(VMOVAPDrr) CASE(VMOVUPDrr)
1736 CASE(VMOVAPSrr) CASE(VMOVUPSrr)
1737 CASE(VMOVDQArr) CASE(VMOVDQUrr)
1738 CASE(VMOVAPDYrr) CASE(VMOVUPDYrr)
1739 CASE(VMOVAPSYrr) CASE(VMOVUPSYrr)
1740 CASE(VMOVDQAYrr) CASE(VMOVDQUYrr)
1741 CASE(VMOVAPDZ128rr) CASE(VMOVUPDZ128rr)
1742 CASE(VMOVAPSZ128rr) CASE(VMOVUPSZ128rr)
1743 CASE(VMOVDQA32Z128rr) CASE(VMOVDQU32Z128rr)
1744 CASE(VMOVDQA64Z128rr) CASE(VMOVDQU64Z128rr)
1745 CASE(VMOVAPDZ256rr) CASE(VMOVUPDZ256rr)
1746 CASE(VMOVAPSZ256rr) CASE(VMOVUPSZ256rr)
1747 CASE(VMOVDQA32Z256rr) CASE(VMOVDQU32Z256rr)
1748 CASE(VMOVDQA64Z256rr) CASE(VMOVDQU64Z256rr)
1749 }
1750#undef CASE
1751
1753 if (.isMachineOpcode() ||
1754 In.getMachineOpcode() <= TargetOpcode::GENERIC_OP_END)
1755 continue;
1756
1757
1758
1759 uint64_t TSFlags = getInstrInfo()->get(In.getMachineOpcode()).TSFlags;
1763 continue;
1764
1765
1766
1767 CurDAG->UpdateNodeOperands(N, N->getOperand(0), In, N->getOperand(2));
1768 MadeChange = true;
1769 }
1770 }
1771 }
1772
1773 if (MadeChange)
1774 CurDAG->RemoveDeadNodes();
1775}
1776
1777
1778
1779void X86DAGToDAGISel::emitSpecialCodeForMain() {
1780 if (Subtarget->isTargetCygMing()) {
1781 TargetLowering::ArgListTy Args;
1782 auto &DL = CurDAG->getDataLayout();
1783
1784 TargetLowering::CallLoweringInfo CLI(*CurDAG);
1785 CLI.setChain(CurDAG->getRoot())
1786 .setCallee(CallingConv::C, Type::getVoidTy(*CurDAG->getContext()),
1787 CurDAG->getExternalSymbol("__main", TLI->getPointerTy(DL)),
1788 std::move(Args));
1789 const TargetLowering &TLI = CurDAG->getTargetLoweringInfo();
1791 CurDAG->setRoot(Result.second);
1792 }
1793}
1794
1795void X86DAGToDAGISel::emitFunctionEntryCode() {
1796
1798 if (F.hasExternalLinkage() && F.getName() == "main")
1799 emitSpecialCodeForMain();
1800}
1801
1803
1804
1805
1806
1807
1808
1810}
1811
1812bool X86DAGToDAGISel::foldOffsetIntoAddress(uint64_t Offset,
1813 X86ISelAddressMode &AM) {
1814
1815
1816
1817
1818 int64_t Val = AM.Disp + Offset;
1819
1820
1821 if (Val != 0 && (AM.ES || AM.MCSym))
1822 return true;
1823
1825 if (Subtarget->is64Bit()) {
1826 if (Val != 0 &&
1828 AM.hasSymbolicDisplacement()))
1829 return true;
1830
1831
1832 if (AM.BaseType == X86ISelAddressMode::FrameIndexBase &&
1834 return true;
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851 if (Subtarget->isTarget64BitILP32() &&
1853 !AM.hasBaseOrIndexReg())
1854 return true;
1856
1857
1858 return true;
1859 AM.Disp = Val;
1860 return false;
1861}
1862
1863bool X86DAGToDAGISel::matchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM,
1864 bool AllowSegmentRegForX32) {
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1877 !IndirectTlsSegRefs &&
1878 (Subtarget->isTargetGlibc() || Subtarget->isTargetAndroid() ||
1879 Subtarget->isTargetFuchsia())) {
1880 if (Subtarget->isTarget64BitILP32() && !AllowSegmentRegForX32)
1881 return true;
1882 switch (N->getPointerInfo().getAddrSpace()) {
1884 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
1885 return false;
1887 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
1888 return false;
1889
1890
1891 }
1892 }
1893
1894 return true;
1895}
1896
1897
1898
1899
1900bool X86DAGToDAGISel::matchWrapper(SDValue N, X86ISelAddressMode &AM) {
1901
1902
1903 if (AM.hasSymbolicDisplacement())
1904 return true;
1905
1906 bool IsRIPRelTLS = false;
1908 if (IsRIPRel) {
1909 SDValue Val = N.getOperand(0);
1911 IsRIPRelTLS = true;
1912 }
1913
1914
1915
1916
1917
1918
1920 if (Subtarget->is64Bit() && M == CodeModel::Large && !IsRIPRelTLS)
1921 return true;
1922
1923
1924 if (IsRIPRel && AM.hasBaseOrIndexReg())
1925 return true;
1926
1927
1928 X86ISelAddressMode Backup = AM;
1929
1933 AM.GV = G->getGlobal();
1934 AM.SymbolFlags = G->getTargetFlags();
1937 AM.CP = CP->getConstVal();
1938 AM.Alignment = CP->getAlign();
1939 AM.SymbolFlags = CP->getTargetFlags();
1942 AM.ES = S->getSymbol();
1943 AM.SymbolFlags = S->getTargetFlags();
1945 AM.MCSym = S->getMCSymbol();
1947 AM.JT = J->getIndex();
1948 AM.SymbolFlags = J->getTargetFlags();
1950 AM.BlockAddr = BA->getBlockAddress();
1951 AM.SymbolFlags = BA->getTargetFlags();
1952 Offset = BA->getOffset();
1953 } else
1955
1956
1957 if (Subtarget->is64Bit() && !IsRIPRel && AM.GV &&
1958 TM.isLargeGlobalValue(AM.GV)) {
1959 AM = Backup;
1960 return true;
1961 }
1962
1963 if (foldOffsetIntoAddress(Offset, AM)) {
1964 AM = Backup;
1965 return true;
1966 }
1967
1968 if (IsRIPRel)
1969 AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64));
1970
1971
1972 return false;
1973}
1974
1975
1976
1977bool X86DAGToDAGISel::matchAddress(SDValue N, X86ISelAddressMode &AM) {
1978 if (matchAddressRecursively(N, AM, 0))
1979 return true;
1980
1981
1982
1983
1984
1985 if (Subtarget->isTarget64BitILP32() &&
1986 AM.BaseType == X86ISelAddressMode::RegBase &&
1987 AM.Base_Reg.getNode() != nullptr && AM.IndexReg.getNode() == nullptr) {
1988 SDValue Save_Base_Reg = AM.Base_Reg;
1990 AM.Base_Reg = SDValue();
1991 if (matchLoadInAddress(LoadN, AM, true))
1992 AM.Base_Reg = Save_Base_Reg;
1993 }
1994 }
1995
1996
1997
1998 if (AM.Scale == 2 &&
1999 AM.BaseType == X86ISelAddressMode::RegBase &&
2000 AM.Base_Reg.getNode() == nullptr) {
2001 AM.Base_Reg = AM.IndexReg;
2002 AM.Scale = 1;
2003 }
2004
2005
2006
2008 (!AM.GV || !TM.isLargeGlobalValue(AM.GV)) && Subtarget->is64Bit() &&
2009 AM.Scale == 1 && AM.BaseType == X86ISelAddressMode::RegBase &&
2010 AM.Base_Reg.getNode() == nullptr && AM.IndexReg.getNode() == nullptr &&
2011 AM.SymbolFlags == X86II::MO_NO_FLAG && AM.hasSymbolicDisplacement()) {
2012
2013
2014
2015
2016
2017
2018
2020 return true;
2021
2022 AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64);
2023 }
2024
2025 return false;
2026}
2027
2028bool X86DAGToDAGISel::matchAdd(SDValue &N, X86ISelAddressMode &AM,
2029 unsigned Depth) {
2030
2031
2032 HandleSDNode Handle(N);
2033
2034 X86ISelAddressMode Backup = AM;
2035 if (!matchAddressRecursively(N.getOperand(0), AM, Depth+1) &&
2036 !matchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1))
2037 return false;
2038 AM = Backup;
2039
2040
2041 if (!matchAddressRecursively(Handle.getValue().getOperand(1), AM,
2043 !matchAddressRecursively(Handle.getValue().getOperand(0), AM, Depth + 1))
2044 return false;
2045 AM = Backup;
2046
2047
2048
2049
2050 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2051 !AM.Base_Reg.getNode() &&
2052 !AM.IndexReg.getNode()) {
2053 N = Handle.getValue();
2056 AM.Scale = 1;
2057 return false;
2058 }
2059 N = Handle.getValue();
2060 return true;
2061}
2062
2063
2064
2065
2066
2067
2069 if (N->getNodeId() == -1 ||
2073
2074
2075
2076
2079 }
2080}
2081
2082
2083
2084
2085
2089 X86ISelAddressMode &AM) {
2093 return true;
2094
2096 if (ScaleLog <= 0 || ScaleLog >= 4 ||
2097 Mask != (0xffu << ScaleLog))
2098 return true;
2099
2100 MVT XVT = X.getSimpleValueType();
2101 MVT VT = N.getSimpleValueType();
2110
2111
2112
2113
2114
2115
2125 AM.IndexReg = Ext;
2126 AM.Scale = (1 << ScaleLog);
2127 return false;
2128}
2129
2130
2131
2132
2134 X86ISelAddressMode &AM) {
2135 SDValue Shift = N.getOperand(0);
2136
2137
2138
2139
2141
2142
2143
2144
2145 bool FoundAnyExtend = false;
2149 FoundAnyExtend = true;
2151 }
2152
2155 return true;
2156
2158
2159
2160
2161
2162 if (.hasOneUse() || !Shift.hasOneUse())
2163 return true;
2164
2165
2167 if (ShiftAmt != 1 && ShiftAmt != 2 && ShiftAmt != 3)
2168 return true;
2169
2170 MVT VT = N.getSimpleValueType();
2172 if (FoundAnyExtend) {
2175 X = NewX;
2176 }
2177
2181
2182
2183
2184
2185
2186
2192
2193 AM.Scale = 1 << ShiftAmt;
2194 AM.IndexReg = NewAnd;
2195 return false;
2196}
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2228 X86ISelAddressMode &AM) {
2231 return true;
2232
2233
2234 unsigned MaskIdx, MaskLen;
2236 return true;
2237 unsigned MaskLZ = 64 - (MaskIdx + MaskLen);
2238
2240
2241
2242
2243 unsigned AMShiftAmt = MaskIdx;
2244
2245
2246
2247 if (AMShiftAmt == 0 || AMShiftAmt > 3) return true;
2248
2249
2250
2251 unsigned ScaleDown = (64 - X.getSimpleValueType().getSizeInBits()) + ShiftAmt;
2252 if (MaskLZ < ScaleDown)
2253 return true;
2254 MaskLZ -= ScaleDown;
2255
2256
2257
2258
2259
2260
2261
2262 bool ReplacingAnyExtend = false;
2264 unsigned ExtendBits = X.getSimpleValueType().getSizeInBits() -
2265 X.getOperand(0).getSimpleValueType().getSizeInBits();
2266
2267
2269 MaskLZ = ExtendBits > MaskLZ ? 0 : MaskLZ - ExtendBits;
2270 ReplacingAnyExtend = true;
2271 }
2272 APInt MaskedHighBits =
2275 return true;
2276
2277
2278
2279 MVT VT = N.getSimpleValueType();
2280 if (ReplacingAnyExtend) {
2281 assert(X.getValueType() != VT);
2282
2285 X = NewX;
2286 }
2287
2288 MVT XVT = X.getSimpleValueType();
2295
2296
2297
2298
2299
2300
2308
2309 AM.Scale = 1 << AMShiftAmt;
2310 AM.IndexReg = NewExt;
2311 return false;
2312}
2313
2314
2315
2316
2320 X86ISelAddressMode &AM,
2324 !Shift.hasOneUse() || .hasOneUse())
2325 return true;
2326
2327
2328 if (!Subtarget.hasTBM() &&
2329 !(Subtarget.hasBMI() && Subtarget.hasFastBEXTR()))
2330 return true;
2331
2332
2333 unsigned MaskIdx, MaskLen;
2335 return true;
2336
2338
2339
2340
2341 unsigned AMShiftAmt = MaskIdx;
2342
2343
2344
2345 if (AMShiftAmt == 0 || AMShiftAmt > 3) return true;
2346
2347 MVT XVT = X.getSimpleValueType();
2348 MVT VT = N.getSimpleValueType();
2357
2358
2359
2360
2361
2362
2372
2373 AM.Scale = 1 << AMShiftAmt;
2374 AM.IndexReg = NewExt;
2375 return false;
2376}
2377
2378
2379
2380SDValue X86DAGToDAGISel::matchIndexRecursively(SDValue N,
2381 X86ISelAddressMode &AM,
2382 unsigned Depth) {
2383 assert(AM.IndexReg.getNode() == nullptr && "IndexReg already matched");
2384 assert((AM.Scale == 1 || AM.Scale == 2 || AM.Scale == 4 || AM.Scale == 8) &&
2385 "Illegal index scale");
2386
2387
2389 return N;
2390
2391 EVT VT = N.getValueType();
2392 unsigned Opc = N.getOpcode();
2393
2394
2395 if (CurDAG->isBaseWithConstantOffset(N)) {
2397 uint64_t Offset = (uint64_t)AddVal->getSExtValue() * AM.Scale;
2398 if (!foldOffsetIntoAddress(Offset, AM))
2399 return matchIndexRecursively(N.getOperand(0), AM, Depth + 1);
2400 }
2401
2402
2403 if (Opc == ISD::ADD && N.getOperand(0) == N.getOperand(1)) {
2404 if (AM.Scale <= 4) {
2405 AM.Scale *= 2;
2406 return matchIndexRecursively(N.getOperand(0), AM, Depth + 1);
2407 }
2408 }
2409
2410
2412 uint64_t ShiftAmt = N.getConstantOperandVal(1);
2413 uint64_t ScaleAmt = 1ULL << ShiftAmt;
2414 if ((AM.Scale * ScaleAmt) <= 8) {
2415 AM.Scale *= ScaleAmt;
2416 return matchIndexRecursively(N.getOperand(0), AM, Depth + 1);
2417 }
2418 }
2419
2420
2421
2423 SDValue Src = N.getOperand(0);
2424 if (Src.getOpcode() == ISD::ADD && Src->getFlags().hasNoSignedWrap() &&
2425 Src.hasOneUse()) {
2426 if (CurDAG->isBaseWithConstantOffset(Src)) {
2427 SDValue AddSrc = Src.getOperand(0);
2429 int64_t Offset = AddVal->getSExtValue();
2430 if (!foldOffsetIntoAddress((uint64_t)Offset * AM.Scale, AM)) {
2432 SDValue ExtSrc = CurDAG->getNode(Opc, DL, VT, AddSrc);
2433 SDValue ExtVal = CurDAG->getSignedConstant(Offset, DL, VT);
2434 SDValue ExtAdd = CurDAG->getNode(ISD::ADD, DL, VT, ExtSrc, ExtVal);
2438 CurDAG->ReplaceAllUsesWith(N, ExtAdd);
2439 CurDAG->RemoveDeadNode(N.getNode());
2440 return ExtSrc;
2441 }
2442 }
2443 }
2444 }
2445
2446
2447
2448
2450 SDValue Src = N.getOperand(0);
2451 unsigned SrcOpc = Src.getOpcode();
2452 if (((SrcOpc == ISD::ADD && Src->getFlags().hasNoUnsignedWrap()) ||
2453 CurDAG->isADDLike(Src, true)) &&
2454 Src.hasOneUse()) {
2455 if (CurDAG->isBaseWithConstantOffset(Src)) {
2456 SDValue AddSrc = Src.getOperand(0);
2457 uint64_t Offset = Src.getConstantOperandVal(1);
2458 if (!foldOffsetIntoAddress(Offset * AM.Scale, AM)) {
2461
2466 APInt HiBits =
2468 uint64_t ScaleAmt = 1ULL << ShAmt;
2469 if ((AM.Scale * ScaleAmt) <= 8 &&
2471 CurDAG->MaskedValueIsZero(ShVal, HiBits))) {
2472 AM.Scale *= ScaleAmt;
2473 SDValue ExtShVal = CurDAG->getNode(Opc, DL, VT, ShVal);
2478 AddSrc = ExtShift;
2479 Res = ExtShVal;
2480 }
2481 }
2482 SDValue ExtSrc = CurDAG->getNode(Opc, DL, VT, AddSrc);
2484 SDValue ExtAdd = CurDAG->getNode(SrcOpc, DL, VT, ExtSrc, ExtVal);
2488 CurDAG->ReplaceAllUsesWith(N, ExtAdd);
2489 CurDAG->RemoveDeadNode(N.getNode());
2490 return Res ? Res : ExtSrc;
2491 }
2492 }
2493 }
2494 }
2495
2496
2497 return N;
2498}
2499
2500bool X86DAGToDAGISel::matchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
2501 unsigned Depth) {
2503 dbgs() << "MatchAddress: ";
2504 AM.dump(CurDAG);
2505 });
2506
2508 return matchAddressBase(N, AM);
2509
2510
2511
2512
2513 if (AM.isRIPRelative()) {
2514
2515
2516
2517 if (!(AM.ES || AM.MCSym) && AM.JT != -1)
2518 return true;
2519
2521 if (!foldOffsetIntoAddress(Cst->getSExtValue(), AM))
2522 return false;
2523 return true;
2524 }
2525
2526 switch (N.getOpcode()) {
2527 default: break;
2529 if (!AM.hasSymbolicDisplacement() && AM.Disp == 0)
2531
2532 AM.MCSym = ESNode->getMCSymbol();
2533 return false;
2534 }
2535 break;
2536 }
2539 if (!foldOffsetIntoAddress(Val, AM))
2540 return false;
2541 break;
2542 }
2543
2546 if (!matchWrapper(N, AM))
2547 return false;
2548 break;
2549
2550 case ISD::LOAD:
2552 return false;
2553 break;
2554
2556 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2557 AM.Base_Reg.getNode() == nullptr &&
2559 AM.BaseType = X86ISelAddressMode::FrameIndexBase;
2561 return false;
2562 }
2563 break;
2564
2566 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1)
2567 break;
2568
2570 unsigned Val = CN->getZExtValue();
2571
2572
2573
2574
2575 if (Val == 1 || Val == 2 || Val == 3) {
2577 AM.Scale = 1 << Val;
2578 AM.IndexReg = matchIndexRecursively(ShVal, AM, Depth + 1);
2579 return false;
2580 }
2581 }
2582 break;
2583
2585
2586 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) break;
2587
2588
2589
2590 assert(N.getSimpleValueType().getSizeInBits() <= 64 &&
2591 "Unexpected value size!");
2592
2596
2597
2598
2599
2602 break;
2603 uint64_t Mask = And.getConstantOperandVal(1) >> N.getConstantOperandVal(1);
2604
2605
2606
2608 return false;
2609 break;
2610 }
2611
2614
2615 if (N.getResNo() != 0) break;
2616 [[fallthrough]];
2619
2620 if (AM.BaseType == X86ISelAddressMode::RegBase &&
2621 AM.Base_Reg.getNode() == nullptr &&
2622 AM.IndexReg.getNode() == nullptr) {
2624 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 ||
2625 CN->getZExtValue() == 9) {
2626 AM.Scale = unsigned(CN->getZExtValue())-1;
2627
2628 SDValue MulVal = N.getOperand(0);
2630
2631
2632
2633
2638 uint64_t Disp = AddVal->getSExtValue() * CN->getZExtValue();
2639 if (foldOffsetIntoAddress(Disp, AM))
2641 } else {
2643 }
2644
2645 AM.IndexReg = AM.Base_Reg = Reg;
2646 return false;
2647 }
2648 }
2649 break;
2650
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661 HandleSDNode Handle(N);
2662
2663
2664 X86ISelAddressMode Backup = AM;
2665 if (matchAddressRecursively(N.getOperand(0), AM, Depth+1)) {
2666 N = Handle.getValue();
2667 AM = Backup;
2668 break;
2669 }
2670 N = Handle.getValue();
2671
2672 if (AM.IndexReg.getNode() || AM.isRIPRelative()) {
2673 AM = Backup;
2674 break;
2675 }
2676
2677 int Cost = 0;
2679
2680
2681
2687 RHS.getOperand(0).getValueType() == MVT::i32))
2689
2690
2691 if ((AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode() &&
2693 AM.BaseType == X86ISelAddressMode::FrameIndexBase)
2695
2696
2697 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) +
2698 ((AM.Disp != 0) && (Backup.Disp == 0)) +
2699 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2)
2701
2702 if (Cost >= 0) {
2703 AM = Backup;
2704 break;
2705 }
2706
2707
2708
2709
2710 AM.IndexReg = RHS;
2711 AM.NegateIndex = true;
2712 AM.Scale = 1;
2713 return false;
2714 }
2715
2718
2719 if (!CurDAG->isADDLike(N))
2720 break;
2721 [[fallthrough]];
2723 if (!matchAdd(N, AM, Depth))
2724 return false;
2725 break;
2726
2728
2729
2730
2731
2732 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1) break;
2733
2734
2735
2736 assert(N.getSimpleValueType().getSizeInBits() <= 64 &&
2737 "Unexpected value size!");
2738
2740 break;
2741
2742 if (N.getOperand(0).getOpcode() == ISD::SRL) {
2743 SDValue Shift = N.getOperand(0);
2745
2746 uint64_t Mask = N.getConstantOperandVal(1);
2747
2748
2750 return false;
2751
2752
2754 return false;
2755
2756
2758 return false;
2759 }
2760
2761
2762
2764 return false;
2765
2766 break;
2767 }
2769
2770
2771 if (AM.IndexReg.getNode() != nullptr || AM.Scale != 1)
2772 break;
2773
2774 SDValue Src = N.getOperand(0);
2775
2776
2777
2778 if (Src.getOpcode() == ISD::ADD || Src.getOpcode() == ISD::OR)
2779 if (SDValue Index = matchIndexRecursively(N, AM, Depth + 1))
2780 if (Index != N) {
2781 AM.IndexReg = Index;
2782 return false;
2783 }
2784
2785
2787 if (Src.getOpcode() == ISD::AND && Src.hasOneUse())
2789 Mask = MaskC->getAPIntValue();
2790 Src = Src.getOperand(0);
2791 }
2792
2793 if (Src.getOpcode() == ISD::SHL && Src.hasOneUse() && N->hasOneUse()) {
2794
2795 SDValue ShlSrc = Src.getOperand(0);
2796 SDValue ShlAmt = Src.getOperand(1);
2798 if (!ShAmtC)
2799 break;
2800 unsigned ShAmtV = ShAmtC->getZExtValue();
2801 if (ShAmtV > 3)
2802 break;
2803
2804
2805
2806 APInt HighZeros =
2808 if (!Src->getFlags().hasNoUnsignedWrap() &&
2809 !CurDAG->MaskedValueIsZero(ShlSrc, HighZeros & Mask))
2810 break;
2811
2812
2813
2814
2815
2817 MVT VT = N.getSimpleValueType();
2819
2821 if (.isAllOnes()) {
2822 Res = CurDAG->getConstant(Mask.lshr(ShAmtV), DL, SrcVT);
2824 Res = CurDAG->getNode(ISD::AND, DL, SrcVT, ShlSrc, Res);
2826 }
2829 SDValue NewShl = CurDAG->getNode(ISD::SHL, DL, VT, Zext, ShlAmt);
2831 CurDAG->ReplaceAllUsesWith(N, NewShl);
2832 CurDAG->RemoveDeadNode(N.getNode());
2833
2834
2835 AM.Scale = 1 << ShAmtV;
2836
2837
2838
2839 AM.IndexReg = matchIndexRecursively(Zext, AM, Depth + 1);
2840 return false;
2841 }
2842
2843 if (Src.getOpcode() == ISD::SRL && .isAllOnes()) {
2844
2846 Src.getOperand(0), AM))
2847 return false;
2848
2849
2851 Src.getOperand(0), AM))
2852 return false;
2853
2854
2856 Src.getOperand(0), AM, *Subtarget))
2857 return false;
2858 }
2859
2860 break;
2861 }
2862 }
2863
2864 return matchAddressBase(N, AM);
2865}
2866
2867
2868
2869bool X86DAGToDAGISel::matchAddressBase(SDValue N, X86ISelAddressMode &AM) {
2870
2871 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) {
2872
2873 if (!AM.IndexReg.getNode()) {
2874 AM.IndexReg = N;
2875 AM.Scale = 1;
2876 return false;
2877 }
2878
2879
2880 return true;
2881 }
2882
2883
2884 AM.BaseType = X86ISelAddressMode::RegBase;
2885 AM.Base_Reg = N;
2886 return false;
2887}
2888
2889bool X86DAGToDAGISel::matchVectorAddressRecursively(SDValue N,
2890 X86ISelAddressMode &AM,
2891 unsigned Depth) {
2893 dbgs() << "MatchVectorAddress: ";
2894 AM.dump(CurDAG);
2895 });
2896
2898 return matchAddressBase(N, AM);
2899
2900
2901 switch (N.getOpcode()) {
2904 if (!foldOffsetIntoAddress(Val, AM))
2905 return false;
2906 break;
2907 }
2909 if (!matchWrapper(N, AM))
2910 return false;
2911 break;
2913
2914
2915 HandleSDNode Handle(N);
2916
2917 X86ISelAddressMode Backup = AM;
2918 if (!matchVectorAddressRecursively(N.getOperand(0), AM, Depth + 1) &&
2919 !matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2921 return false;
2922 AM = Backup;
2923
2924
2925 if (!matchVectorAddressRecursively(Handle.getValue().getOperand(1), AM,
2927 !matchVectorAddressRecursively(Handle.getValue().getOperand(0), AM,
2929 return false;
2930 AM = Backup;
2931
2932 N = Handle.getValue();
2933 break;
2934 }
2935 }
2936
2937 return matchAddressBase(N, AM);
2938}
2939
2940
2941
2942
2943bool X86DAGToDAGISel::matchVectorAddress(SDValue N, X86ISelAddressMode &AM) {
2944 return matchVectorAddressRecursively(N, AM, 0);
2945}
2946
2947bool X86DAGToDAGISel::selectVectorAddr(MemSDNode *Parent, SDValue BasePtr,
2952 X86ISelAddressMode AM;
2954
2955
2956
2958 AM.IndexReg = matchIndexRecursively(IndexOp, AM, 0);
2959 else
2960 AM.IndexReg = IndexOp;
2961
2964 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
2966 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
2968 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
2969
2970 SDLoc DL(BasePtr);
2971 MVT VT = BasePtr.getSimpleValueType();
2972
2973
2974 if (matchVectorAddress(BasePtr, AM))
2975 return false;
2976
2977 getAddressOperands(AM, DL, VT, Base, Scale, Index, Disp, Segment);
2978 return true;
2979}
2980
2981
2982
2983
2984
2985
2986
2987
2988bool X86DAGToDAGISel::selectAddr(SDNode *Parent, SDValue N, SDValue &Base,
2991 X86ISelAddressMode AM;
2992
2993 if (Parent &&
2994
2995
3003 unsigned AddrSpace =
3004 cast(Parent)->getPointerInfo().getAddrSpace();
3006 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16);
3008 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16);
3010 AM.Segment = CurDAG->getRegister(X86::SS, MVT::i16);
3011 }
3012
3013
3015 MVT VT = N.getSimpleValueType();
3016
3017 if (matchAddress(N, AM))
3018 return false;
3019
3020 getAddressOperands(AM, DL, VT, Base, Scale, Index, Disp, Segment);
3021 return true;
3022}
3023
3024bool X86DAGToDAGISel::selectMOV64Imm32(SDValue N, SDValue &Imm) {
3025
3026
3029 return false;
3030
3031
3032
3034 return false;
3035
3037
3038
3039
3041 return false;
3042
3044
3045
3049 }
3050
3053 return CR->getUnsignedMax().ult(1ull << 32);
3054
3055 return !TM.isLargeGlobalValue(GV);
3056}
3057
3061
3063
3064 if (!selectLEAAddr(N, Base, Scale, Index, Disp, Segment))
3065 return false;
3066
3070 SubReg = X86::sub_8bit;
3071 else if (BaseType == MVT::i16)
3072 SubReg = X86::sub_16bit;
3073 else
3074 SubReg = X86::sub_32bit;
3075
3077 if (RN && RN->getReg() == 0)
3078 Base = CurDAG->getRegister(0, MVT::i64);
3082
3083 SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, DL,
3084 MVT::i64), 0);
3085 Base = CurDAG->getTargetInsertSubreg(SubReg, DL, MVT::i64, ImplDef, Base);
3086 }
3087
3088 [[maybe_unused]] EVT IndexType = Index.getValueType();
3090 if (RN && RN->getReg() == 0)
3091 Index = CurDAG->getRegister(0, MVT::i64);
3092 else {
3094 "Expect to be extending 8/16/32-bit registers for use in LEA");
3095 SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, DL,
3096 MVT::i64), 0);
3097 Index = CurDAG->getTargetInsertSubreg(SubReg, DL, MVT::i64, ImplDef, Index);
3098 }
3099
3100 return true;
3101}
3102
3103
3104
3105bool X86DAGToDAGISel::selectLEAAddr(SDValue N,
3109 X86ISelAddressMode AM;
3110
3111
3113 MVT VT = N.getSimpleValueType();
3114
3115
3116
3118 SDValue T = CurDAG->getRegister(0, MVT::i32);
3119 AM.Segment = T;
3120 if (matchAddress(N, AM))
3121 return false;
3122 assert (T == AM.Segment);
3123 AM.Segment = Copy;
3124
3125 unsigned Complexity = 0;
3126 if (AM.BaseType == X86ISelAddressMode::RegBase && AM.Base_Reg.getNode())
3127 Complexity = 1;
3128 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase)
3129 Complexity = 4;
3130
3131 if (AM.IndexReg.getNode())
3132 Complexity++;
3133
3134
3135
3136 if (AM.Scale > 1)
3137 Complexity++;
3138
3139
3140
3141
3142
3143
3144 if (AM.hasSymbolicDisplacement()) {
3145
3146 if (Subtarget->is64Bit())
3147 Complexity = 4;
3148 else
3149 Complexity += 2;
3150 }
3151
3152
3153
3154
3155 if (N.getOpcode() == ISD::ADD) {
3156 auto isMathWithFlags = [](SDValue V) {
3157 switch (V.getOpcode()) {
3164
3165
3166
3167
3168
3169
3170
3171 return (V.getNode(), 1).use_empty();
3172 default:
3173 return false;
3174 }
3175 };
3176
3177
3178 if (isMathWithFlags(N.getOperand(0)) || isMathWithFlags(N.getOperand(1)))
3179 Complexity++;
3180 }
3181
3182 if (AM.Disp)
3183 Complexity++;
3184
3185
3186 if (Complexity <= 2)
3187 return false;
3188
3189 getAddressOperands(AM, DL, VT, Base, Scale, Index, Disp, Segment);
3190 return true;
3191}
3192
3193
3199
3200 X86ISelAddressMode AM;
3202 AM.GV = GA->getGlobal();
3203 AM.Disp += GA->getOffset();
3204 AM.SymbolFlags = GA->getTargetFlags();
3205 } else {
3207 AM.ES = SA->getSymbol();
3208 AM.SymbolFlags = SA->getTargetFlags();
3209 }
3210
3211 if (Subtarget->is32Bit()) {
3212 AM.Scale = 1;
3213 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32);
3214 }
3215
3216 MVT VT = N.getSimpleValueType();
3217 getAddressOperands(AM, SDLoc(N), VT, Base, Scale, Index, Disp, Segment);
3218 return true;
3219}
3220
3222
3223
3224
3225 EVT VT = N.getValueType();
3226 bool WasTruncated = false;
3228 WasTruncated = true;
3230 }
3231
3233 return false;
3234
3235
3236
3237
3238 unsigned Opc = N.getOperand(0)->getOpcode();
3241
3242
3243 return !WasTruncated;
3244 }
3245
3246
3248 std::optional CR = GA->getGlobal()->getAbsoluteSymbolRange();
3249 if (!CR || CR->getUnsignedMax().uge(1ull << VT.getSizeInBits()))
3250 return false;
3251
3252
3253 Op = CurDAG->getTargetGlobalAddress(GA->getGlobal(), SDLoc(N), VT,
3254 GA->getOffset(), GA->getTargetFlags());
3255 return true;
3256}
3257
3258bool X86DAGToDAGISel::tryFoldLoad(SDNode *Root, SDNode *P, SDValue N,
3262 assert(Root && P && "Unknown root/parent nodes");
3264 !IsProfitableToFold(N, P, Root) ||
3265 !IsLegalToFold(N, P, Root, OptLevel))
3266 return false;
3267
3268 return selectAddr(N.getNode(),
3269 N.getOperand(1), Base, Scale, Index, Disp, Segment);
3270}
3271
3272bool X86DAGToDAGISel::tryFoldBroadcast(SDNode *Root, SDNode *P, SDValue N,
3276 assert(Root && P && "Unknown root/parent nodes");
3278 !IsProfitableToFold(N, P, Root) ||
3279 !IsLegalToFold(N, P, Root, OptLevel))
3280 return false;
3281
3282 return selectAddr(N.getNode(),
3283 N.getOperand(1), Base, Scale, Index, Disp, Segment);
3284}
3285
3286
3287
3288
3289SDNode *X86DAGToDAGISel::getGlobalBaseReg() {
3292 return CurDAG->getRegister(GlobalBaseReg, TLI->getPointerTy(DL)).getNode();
3293}
3294
3295bool X86DAGToDAGISel::isSExtAbsoluteSymbolRef(unsigned Width, SDNode *N) const {
3297 N = N->getOperand(0).getNode();
3299 return false;
3300
3302 if (!GA)
3303 return false;
3304
3305 auto *GV = GA->getGlobal();
3307 if (CR)
3308 return CR->getSignedMin().sge(-1ull << Width) &&
3309 CR->getSignedMax().slt(1ull << Width);
3310
3311
3312
3313
3314 return Width == 32 && !TM.isLargeGlobalValue(GV);
3315}
3316
3317X86::CondCode X86DAGToDAGISel::getCondFromNode(SDNode *N) const {
3318 assert(N->isMachineOpcode() && "Unexpected node");
3319 unsigned Opc = N->getMachineOpcode();
3320 const MCInstrDesc &MCID = getInstrInfo()->get(Opc);
3322 if (CondNo < 0)
3324
3325 return static_cast<X86::CondCode>(N->getConstantOperandVal(CondNo));
3326}
3327
3328
3329
3330bool X86DAGToDAGISel::onlyUsesZeroFlag(SDValue Flags) const {
3331
3332 for (SDUse &Use : Flags->uses()) {
3333
3334 if (Use.getResNo() != Flags.getResNo())
3335 continue;
3336 SDNode *User = Use.getUser();
3337
3340 return false;
3341
3342 for (SDUse &FlagUse : User->uses()) {
3343
3344 if (FlagUse.getResNo() != 1)
3345 continue;
3346
3347 if (!FlagUse.getUser()->isMachineOpcode())
3348 return false;
3349
3350 X86::CondCode CC = getCondFromNode(FlagUse.getUser());
3351
3352 switch (CC) {
3353
3355 continue;
3356
3357 default:
3358 return false;
3359 }
3360 }
3361 }
3362 return true;
3363}
3364
3365
3366
3367bool X86DAGToDAGISel::hasNoSignFlagUses(SDValue Flags) const {
3368
3369 for (SDUse &Use : Flags->uses()) {
3370
3371 if (Use.getResNo() != Flags.getResNo())
3372 continue;
3373 SDNode *User = Use.getUser();
3374
3377 return false;
3378
3379 for (SDUse &FlagUse : User->uses()) {
3380
3381 if (FlagUse.getResNo() != 1)
3382 continue;
3383
3384 if (!FlagUse.getUser()->isMachineOpcode())
3385 return false;
3386
3387 X86::CondCode CC = getCondFromNode(FlagUse.getUser());
3388
3389 switch (CC) {
3390
3396 continue;
3397
3398 default:
3399 return false;
3400 }
3401 }
3402 }
3403 return true;
3404}
3405
3407 switch (CC) {
3408
3415 return false;
3416
3417 default:
3418 return true;
3419 }
3420}
3421
3422
3423
3424 bool X86DAGToDAGISel::hasNoCarryFlagUses(SDValue Flags) const {
3425
3426 for (SDUse &Use : Flags->uses()) {
3427
3428 if (Use.getResNo() != Flags.getResNo())
3429 continue;
3430
3431 SDNode *User = Use.getUser();
3432 unsigned UserOpc = User->getOpcode();
3433
3435
3437 return false;
3438
3439 for (SDUse &FlagUse : User->uses()) {
3440
3441 if (FlagUse.getResNo() != 1)
3442 continue;
3443
3444 if (!FlagUse.getUser()->isMachineOpcode())
3445 return false;
3446
3447 X86::CondCode CC = getCondFromNode(FlagUse.getUser());
3448
3450 return false;
3451 }
3452
3453
3454 continue;
3455 }
3456
3457
3458
3459 unsigned CCOpNo;
3460 switch (UserOpc) {
3461 default:
3462
3463 return false;
3468 }
3469
3472 return false;
3473 }
3474 return true;
3475}
3476
3477
3478
3481 unsigned LoadOpNo,
3484
3485 if (StoredVal.getResNo() != 0) return false;
3486
3487
3489
3490
3492 return false;
3493
3495
3497
3498
3500
3501
3502 if (!Load.hasOneUse())
3503 return false;
3504
3505
3508 return false;
3509
3510 bool FoundLoad = false;
3514 const unsigned int Max = 1024;
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3554
3555
3556 if (Chain == Load.getValue(1)) {
3557 FoundLoad = true;
3558 ChainOps.push_back(Load.getOperand(0));
3560 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) {
3562 if (Op == Load.getValue(1)) {
3563 FoundLoad = true;
3564
3565 ChainOps.push_back(Load.getOperand(0));
3566 continue;
3567 }
3570 }
3571 }
3572
3573 if (!FoundLoad)
3574 return false;
3575
3576
3578 if (Op.getNode() != LoadNode)
3580
3581
3583 true))
3584 return false;
3585
3586 InputChain =
3588 return true;
3589}
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(SDNode *Node) {
3612 SDValue StoredVal = StoreNode->getOperand(1);
3614
3615
3616
3617
3618 EVT MemVT = StoreNode->getMemoryVT();
3619 if (MemVT != MVT::i64 && MemVT != MVT::i32 && MemVT != MVT::i16 &&
3620 MemVT != MVT::i8)
3621 return false;
3622
3623 bool IsCommutable = false;
3624 bool IsNegate = false;
3625 switch (Opc) {
3626 default:
3627 return false;
3630 break;
3632 break;
3638 IsCommutable = true;
3639 break;
3640 }
3641
3642 unsigned LoadOpNo = IsNegate ? 1 : 0;
3643 LoadSDNode *LoadNode = nullptr;
3646 LoadNode, InputChain)) {
3647 if (!IsCommutable)
3648 return false;
3649
3650
3651 LoadOpNo = 1;
3653 LoadNode, InputChain))
3654 return false;
3655 }
3656
3658 if (!selectAddr(LoadNode, LoadNode->getBasePtr(), Base, Scale, Index, Disp,
3659 Segment))
3660 return false;
3661
3662 auto SelectOpcode = [&](unsigned Opc64, unsigned Opc32, unsigned Opc16,
3663 unsigned Opc8) {
3665 case MVT::i64:
3666 return Opc64;
3667 case MVT::i32:
3668 return Opc32;
3669 case MVT::i16:
3670 return Opc16;
3671 case MVT::i8:
3672 return Opc8;
3673 default:
3675 }
3676 };
3677
3678 MachineSDNode *Result;
3679 switch (Opc) {
3681
3682 if (IsNegate) {
3683 unsigned NewOpc = SelectOpcode(X86::NEG64m, X86::NEG32m, X86::NEG16m,
3684 X86::NEG8m);
3686 Result = CurDAG->getMachineNode(NewOpc, SDLoc(Node), MVT::i32,
3687 MVT::Other, Ops);
3688 break;
3689 }
3690 [[fallthrough]];
3692
3693 if (!Subtarget->slowIncDec() || CurDAG->shouldOptForSize()) {
3696
3697 if ((IsOne || IsNegOne) && hasNoCarryFlagUses(StoredVal.getValue(1))) {
3698 unsigned NewOpc =
3700 ? SelectOpcode(X86::INC64m, X86::INC32m, X86::INC16m, X86::INC8m)
3701 : SelectOpcode(X86::DEC64m, X86::DEC32m, X86::DEC16m, X86::DEC8m);
3703 Result = CurDAG->getMachineNode(NewOpc, SDLoc(Node), MVT::i32,
3704 MVT::Other, Ops);
3705 break;
3706 }
3707 }
3708 [[fallthrough]];
3714 auto SelectRegOpcode = [SelectOpcode](unsigned Opc) {
3715 switch (Opc) {
3717 return SelectOpcode(X86::ADD64mr, X86::ADD32mr, X86::ADD16mr,
3718 X86::ADD8mr);
3720 return SelectOpcode(X86::ADC64mr, X86::ADC32mr, X86::ADC16mr,
3721 X86::ADC8mr);
3723 return SelectOpcode(X86::SUB64mr, X86::SUB32mr, X86::SUB16mr,
3724 X86::SUB8mr);
3726 return SelectOpcode(X86::SBB64mr, X86::SBB32mr, X86::SBB16mr,
3727 X86::SBB8mr);
3729 return SelectOpcode(X86::AND64mr, X86::AND32mr, X86::AND16mr,
3730 X86::AND8mr);
3732 return SelectOpcode(X86::OR64mr, X86::OR32mr, X86::OR16mr, X86::OR8mr);
3734 return SelectOpcode(X86::XOR64mr, X86::XOR32mr, X86::XOR16mr,
3735 X86::XOR8mr);
3736 default:
3738 }
3739 };
3740 auto SelectImmOpcode = [SelectOpcode](unsigned Opc) {
3741 switch (Opc) {
3743 return SelectOpcode(X86::ADD64mi32, X86::ADD32mi, X86::ADD16mi,
3744 X86::ADD8mi);
3746 return SelectOpcode(X86::ADC64mi32, X86::ADC32mi, X86::ADC16mi,
3747 X86::ADC8mi);
3749 return SelectOpcode(X86::SUB64mi32, X86::SUB32mi, X86::SUB16mi,
3750 X86::SUB8mi);
3752 return SelectOpcode(X86::SBB64mi32, X86::SBB32mi, X86::SBB16mi,
3753 X86::SBB8mi);
3755 return SelectOpcode(X86::AND64mi32, X86::AND32mi, X86::AND16mi,
3756 X86::AND8mi);
3758 return SelectOpcode(X86::OR64mi32, X86::OR32mi, X86::OR16mi,
3759 X86::OR8mi);
3761 return SelectOpcode(X86::XOR64mi32, X86::XOR32mi, X86::XOR16mi,
3762 X86::XOR8mi);
3763 default:
3765 }
3766 };
3767
3768 unsigned NewOpc = SelectRegOpcode(Opc);
3770
3771
3772
3774 int64_t OperandV = OperandC->getSExtValue();
3775
3776
3777
3778
3780 ((MemVT != MVT::i8 && (OperandV) && isInt<8>(-OperandV)) ||
3781 (MemVT == MVT::i64 && (OperandV) &&
3783 hasNoCarryFlagUses(StoredVal.getValue(1))) {
3784 OperandV = -OperandV;
3786 }
3787
3788 if (MemVT != MVT::i64 || isInt<32>(OperandV)) {
3789 Operand = CurDAG->getSignedTargetConstant(OperandV, SDLoc(Node), MemVT);
3790 NewOpc = SelectImmOpcode(Opc);
3791 }
3792 }
3793
3796 CurDAG->getCopyToReg(InputChain, SDLoc(Node), X86::EFLAGS,
3798
3800 Segment, Operand, CopyTo, CopyTo.getValue(1)};
3801 Result = CurDAG->getMachineNode(NewOpc, SDLoc(Node), MVT::i32, MVT::Other,
3803 } else {
3805 Segment, Operand, InputChain};
3806 Result = CurDAG->getMachineNode(NewOpc, SDLoc(Node), MVT::i32, MVT::Other,
3808 }
3809 break;
3810 }
3811 default:
3813 }
3814
3815 MachineMemOperand *MemOps[] = {StoreNode->getMemOperand(),
3817 CurDAG->setNodeMemRefs(Result, MemOps);
3818
3819
3820 ReplaceUses(SDValue(LoadNode, 1), SDValue(Result, 1));
3821 ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1));
3823 CurDAG->RemoveDeadNode(Node);
3824 return true;
3825}
3826
3827
3828
3829
3830
3831
3832
3833
3834bool X86DAGToDAGISel::matchBitExtract(SDNode *Node) {
3838 "Should be either an and-mask, or right-shift after clearing high bits.");
3839
3840
3841 if (!Subtarget->hasBMI() && !Subtarget->hasBMI2())
3842 return false;
3843
3844 MVT NVT = Node->getSimpleValueType(0);
3845
3846
3847 if (NVT != MVT::i32 && NVT != MVT::i64)
3848 return false;
3849
3851 bool NegateNBits;
3852
3853
3854
3855 const bool AllowExtraUsesByDefault = Subtarget->hasBMI2();
3856 auto checkUses = [AllowExtraUsesByDefault](
3858 std::optional AllowExtraUses) {
3859 return AllowExtraUses.value_or(AllowExtraUsesByDefault) ||
3860 Op.getNode()->hasNUsesOfValue(NUses, Op.getResNo());
3861 };
3862 auto checkOneUse = [checkUses](SDValue Op,
3863 std::optional AllowExtraUses =
3864 std::nullopt) {
3865 return checkUses(Op, 1, AllowExtraUses);
3866 };
3867 auto checkTwoUse = [checkUses](SDValue Op,
3868 std::optional AllowExtraUses =
3869 std::nullopt) {
3870 return checkUses(Op, 2, AllowExtraUses);
3871 };
3872
3873 auto peekThroughOneUseTruncation = [checkOneUse](SDValue V) {
3874 if (V->getOpcode() == ISD::TRUNCATE && checkOneUse(V)) {
3875 assert(V.getSimpleValueType() == MVT::i32 &&
3876 V.getOperand(0).getSimpleValueType() == MVT::i64 &&
3877 "Expected i64 -> i32 truncation");
3879 }
3880 return V;
3881 };
3882
3883
3884 auto matchPatternA = [checkOneUse, peekThroughOneUseTruncation, &NBits,
3886
3887 if (Mask->getOpcode() != ISD::ADD || !checkOneUse(Mask))
3888 return false;
3889
3891 return false;
3892
3893 SDValue M0 = peekThroughOneUseTruncation(Mask->getOperand(0));
3894 if (M0->getOpcode() != ISD::SHL || !checkOneUse(M0))
3895 return false;
3897 return false;
3898 NBits = M0->getOperand(1);
3899 NegateNBits = false;
3900 return true;
3901 };
3902
3903 auto isAllOnes = [this, peekThroughOneUseTruncation, NVT](SDValue V) {
3904 V = peekThroughOneUseTruncation(V);
3905 return CurDAG->MaskedValueIsAllOnes(
3908 };
3909
3910
3911 auto matchPatternB = [checkOneUse, isAllOnes, peekThroughOneUseTruncation,
3912 &NBits, &NegateNBits](SDValue Mask) -> bool {
3913
3914 if (Mask.getOpcode() != ISD::XOR || !checkOneUse(Mask))
3915 return false;
3916
3917 if (!isAllOnes(Mask->getOperand(1)))
3918 return false;
3919
3920 SDValue M0 = peekThroughOneUseTruncation(Mask->getOperand(0));
3921 if (M0->getOpcode() != ISD::SHL || !checkOneUse(M0))
3922 return false;
3923
3924 if (!isAllOnes(M0->getOperand(0)))
3925 return false;
3926 NBits = M0->getOperand(1);
3927 NegateNBits = false;
3928 return true;
3929 };
3930
3931
3932
3933 auto canonicalizeShiftAmt = [&NBits, &NegateNBits](SDValue ShiftAmt,
3934 unsigned Bitwidth) {
3935 NBits = ShiftAmt;
3936 NegateNBits = true;
3937
3939 NBits = NBits.getOperand(0);
3940
3941
3942 if (NBits.getOpcode() != ISD::SUB)
3943 return;
3945 if (!V0 || V0->getZExtValue() != Bitwidth)
3946 return;
3947 NBits = NBits.getOperand(1);
3948 NegateNBits = false;
3949 };
3950
3951
3952
3953
3954 auto matchPatternC = [checkOneUse, peekThroughOneUseTruncation, &NegateNBits,
3955 canonicalizeShiftAmt](SDValue Mask) -> bool {
3956
3957 Mask = peekThroughOneUseTruncation(Mask);
3958 unsigned Bitwidth = Mask.getSimpleValueType().getSizeInBits();
3959
3960 if (Mask.getOpcode() != ISD::SRL || !checkOneUse(Mask))
3961 return false;
3962
3964 return false;
3966
3967 if (!checkOneUse(M1))
3968 return false;
3969 canonicalizeShiftAmt(M1, Bitwidth);
3970
3971
3972
3973
3974 return !NegateNBits;
3975 };
3976
3978
3979
3980
3981
3982 auto matchPatternD = [checkOneUse, checkTwoUse, canonicalizeShiftAmt,
3983 AllowExtraUsesByDefault, &NegateNBits,
3984 &X](SDNode *Node) -> bool {
3986 return false;
3989 return false;
3993
3994 if (N1 != N01)
3995 return false;
3996 canonicalizeShiftAmt(N1, Bitwidth);
3997
3998
3999
4000 const bool AllowExtraUses = AllowExtraUsesByDefault && !NegateNBits;
4001 if (!checkOneUse(N0, AllowExtraUses) || !checkTwoUse(N1, AllowExtraUses))
4002 return false;
4004 return true;
4005 };
4006
4007 auto matchLowBitMask = [matchPatternA, matchPatternB,
4009 return matchPatternA(Mask) || matchPatternB(Mask) || matchPatternC(Mask);
4010 };
4011
4015
4016 if (matchLowBitMask(Mask)) {
4017
4018 } else {
4020 if (!matchLowBitMask(Mask))
4021 return false;
4022 }
4023 } else if (matchLowBitMask(SDValue(Node, 0))) {
4024 X = CurDAG->getAllOnesConstant(SDLoc(Node), NVT);
4025 } else if (!matchPatternD(Node))
4026 return false;
4027
4028
4029
4030 if (NegateNBits && !Subtarget->hasBMI2())
4031 return false;
4032
4033 SDLoc DL(Node);
4034
4035
4036 NBits = CurDAG->getNode(ISD::TRUNCATE, DL, MVT::i8, NBits);
4038
4039
4040
4042 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::i32), 0);
4044
4045 SDValue SRIdxVal = CurDAG->getTargetConstant(X86::sub_8bit, DL, MVT::i32);
4047 NBits = SDValue(CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
4048 MVT::i32, ImplDef, NBits, SRIdxVal),
4049 0);
4051
4052
4053
4054 if (NegateNBits) {
4057
4058 NBits = CurDAG->getNode(ISD::SUB, DL, MVT::i32, BitWidthC, NBits);
4060 }
4061
4062 if (Subtarget->hasBMI2()) {
4063
4064 if (NVT != MVT::i32) {
4065
4068 }
4069
4071 ReplaceNode(Node, Extract.getNode());
4072 SelectCode(Extract.getNode());
4073 return true;
4074 }
4075
4076
4077
4078
4079
4080 {
4081 SDValue RealX = peekThroughOneUseTruncation(X);
4082
4084 X = RealX;
4085 }
4086
4087 MVT XVT = X.getSimpleValueType();
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097 SDValue C8 = CurDAG->getConstant(8, DL, MVT::i8);
4099 SDValue Control = CurDAG->getNode(ISD::SHL, DL, MVT::i32, NBits, C8);
4101
4102
4103
4104 if (X.getOpcode() == ISD::SRL) {
4105 SDValue ShiftAmt = X.getOperand(1);
4107
4109 "Expected shift amount to be i8");
4110
4111
4112
4113 SDValue OrigShiftAmt = ShiftAmt;
4114 ShiftAmt = CurDAG->getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShiftAmt);
4116
4117
4118 Control = CurDAG->getNode(ISD::OR, DL, MVT::i32, Control, ShiftAmt);
4120 }
4121
4122
4123 if (XVT != MVT::i32) {
4126 }
4127
4128
4130
4131
4132 if (XVT != NVT) {
4134 Extract = CurDAG->getNode(ISD::TRUNCATE, DL, NVT, Extract);
4135 }
4136
4137 ReplaceNode(Node, Extract.getNode());
4138 SelectCode(Extract.getNode());
4139
4140 return true;
4141}
4142
4143
4144MachineSDNode *X86DAGToDAGISel::matchBEXTRFromAndImm(SDNode *Node) {
4145 MVT NVT = Node->getSimpleValueType(0);
4146 SDLoc dl(Node);
4147
4150
4151
4152
4153
4154
4155
4156
4157 bool PreferBEXTR =
4158 Subtarget->hasTBM() || (Subtarget->hasBMI() && Subtarget->hasFastBEXTR());
4159 if (!PreferBEXTR && !Subtarget->hasBMI2())
4160 return nullptr;
4161
4162
4164 return nullptr;
4165
4166
4168 return nullptr;
4169
4170
4171 if (NVT != MVT::i32 && NVT != MVT::i64)
4172 return nullptr;
4173
4174
4177 if (!MaskCst || !ShiftCst)
4178 return nullptr;
4179
4180
4181 uint64_t Mask = MaskCst->getZExtValue();
4183 return nullptr;
4184
4185 uint64_t Shift = ShiftCst->getZExtValue();
4187
4188
4189
4190 if (Shift == 8 && MaskSize == 8)
4191 return nullptr;
4192
4193
4194
4196 return nullptr;
4197
4198
4199
4200
4201 if (!PreferBEXTR && MaskSize <= 32)
4202 return nullptr;
4203
4205 unsigned ROpc, MOpc;
4206
4207#define GET_EGPR_IF_ENABLED(OPC) (Subtarget->hasEGPR() ? OPC##_EVEX : OPC)
4208 if (!PreferBEXTR) {
4209 assert(Subtarget->hasBMI2() && "We must have BMI2's BZHI then.");
4210
4211
4212
4213 Control = CurDAG->getTargetConstant(Shift + MaskSize, dl, NVT);
4218 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4219 Control = SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
4220 } else {
4221
4222
4223
4224
4225 Control = CurDAG->getTargetConstant(Shift | (MaskSize << 8), dl, NVT);
4226 if (Subtarget->hasTBM()) {
4227 ROpc = NVT == MVT::i64 ? X86::BEXTRI64ri : X86::BEXTRI32ri;
4228 MOpc = NVT == MVT::i64 ? X86::BEXTRI64mi : X86::BEXTRI32mi;
4229 } else {
4230 assert(Subtarget->hasBMI() && "We must have BMI1's BEXTR then.");
4231
4236 unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
4237 Control = SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
4238 }
4239 }
4240
4241 MachineSDNode *NewNode;
4243 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4244 if (tryFoldLoad(Node, N0.getNode(), Input, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4246 Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Control, Input.getOperand(0)};
4247 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
4248 NewNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4249
4251
4252 CurDAG->setNodeMemRefs(NewNode, {cast(Input)->getMemOperand()});
4253 } else {
4254 NewNode = CurDAG->getMachineNode(ROpc, dl, NVT, MVT::i32, Input, Control);
4255 }
4256
4257 if (!PreferBEXTR) {
4258
4259 SDValue ShAmt = CurDAG->getTargetConstant(Shift, dl, NVT);
4260 unsigned NewOpc = NVT == MVT::i64 ? GET_ND_IF_ENABLED(X86::SHR64ri)
4262 NewNode =
4263 CurDAG->getMachineNode(NewOpc, dl, NVT, SDValue(NewNode, 0), ShAmt);
4264 }
4265
4266 return NewNode;
4267}
4268
4269
4270MachineSDNode *X86DAGToDAGISel::emitPCMPISTR(unsigned ROpc, unsigned MOpc,
4271 bool MayFoldLoad, const SDLoc &dl,
4272 MVT VT, SDNode *Node) {
4277 Imm = CurDAG->getTargetConstant(*Val, SDLoc(Node), Imm.getValueType());
4278
4279
4280 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4281 if (MayFoldLoad && tryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4282 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm,
4284 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other);
4285 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4286
4288
4289 CurDAG->setNodeMemRefs(CNode, {cast(N1)->getMemOperand()});
4290 return CNode;
4291 }
4292
4294 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32);
4295 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
4296 return CNode;
4297}
4298
4299
4300
4301
4302MachineSDNode *X86DAGToDAGISel::emitPCMPESTR(unsigned ROpc, unsigned MOpc,
4303 bool MayFoldLoad, const SDLoc &dl,
4304 MVT VT, SDNode *Node,
4310 Imm = CurDAG->getTargetConstant(*Val, SDLoc(Node), Imm.getValueType());
4311
4312
4313 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4314 if (MayFoldLoad && tryFoldLoad(Node, N2, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4315 SDValue Ops[] = { N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm,
4317 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Other, MVT::Glue);
4318 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
4319 InGlue = SDValue(CNode, 3);
4320
4322
4323 CurDAG->setNodeMemRefs(CNode, {cast(N2)->getMemOperand()});
4324 return CNode;
4325 }
4326
4328 SDVTList VTs = CurDAG->getVTList(VT, MVT::i32, MVT::Glue);
4329 MachineSDNode *CNode = CurDAG->getMachineNode(ROpc, dl, VTs, Ops);
4330 InGlue = SDValue(CNode, 2);
4331 return CNode;
4332}
4333
4334bool X86DAGToDAGISel::tryShiftAmountMod(SDNode *N) {
4335 EVT VT = N->getValueType(0);
4336
4337
4339 return false;
4340
4341
4342 unsigned Size = VT == MVT::i64 ? 64 : 32;
4343
4345 SDValue ShiftAmt = OrigShiftAmt;
4347
4348
4351
4352
4353
4354
4362
4363
4364 if (Add1C && Add1C->getAPIntValue().urem(Size) == 0) {
4365 NewShiftAmt = Add0;
4366
4368 ((Add0C && Add0C->getAPIntValue().urem(Size) == Size - 1) ||
4369 (Add1C && Add1C->getAPIntValue().urem(Size) == Size - 1))) {
4370
4371
4372
4373 assert(Add0C == nullptr || Add1C == nullptr);
4374
4375
4377 return false;
4378
4380
4382 NewShiftAmt = CurDAG->getNode(ISD::XOR, DL, OpVT,
4383 Add0C == nullptr ? Add0 : Add1, AllOnes);
4385 insertDAGNode(*CurDAG, OrigShiftAmt, NewShiftAmt);
4386
4387
4389 Add0C->getZExtValue() != 0) {
4392 if (Add0C->getZExtValue() % Size == 0)
4393 X = Add1;
4395 Add0C->getZExtValue() % 32 == 0) {
4396
4397
4401 }
4403 Add0 = CurDAG->getZExtOrTrunc(Add0, DL, SubVT);
4405 }
4406
4407 X = CurDAG->getNode(ISD::ADD, DL, SubVT, Add1, Add0);
4409 } else
4410 return false;
4411
4412
4413
4414 SDValue Zero = CurDAG->getConstant(0, DL, SubVT);
4416 NewShiftAmt = Neg;
4417
4418
4419
4422 } else
4423 return false;
4424 } else
4425 return false;
4426
4427 if (NewShiftAmt.getValueType() != MVT::i8) {
4428
4429 NewShiftAmt = CurDAG->getNode(ISD::TRUNCATE, DL, MVT::i8, NewShiftAmt);
4430
4431 insertDAGNode(*CurDAG, OrigShiftAmt, NewShiftAmt);
4432 }
4433
4434
4435
4436 NewShiftAmt = CurDAG->getNode(ISD::AND, DL, MVT::i8, NewShiftAmt,
4437 CurDAG->getConstant(Size - 1, DL, MVT::i8));
4438
4439 insertDAGNode(*CurDAG, OrigShiftAmt, NewShiftAmt);
4440
4441 SDNode *UpdatedNode = CurDAG->UpdateNodeOperands(N, N->getOperand(0),
4442 NewShiftAmt);
4443 if (UpdatedNode != N) {
4444
4445
4446 ReplaceNode(N, UpdatedNode);
4447 return true;
4448 }
4449
4450
4451
4453 CurDAG->RemoveDeadNode(OrigShiftAmt.getNode());
4454
4455
4456
4457 SelectCode(N);
4458 return true;
4459}
4460
4461bool X86DAGToDAGISel::tryShrinkShlLogicImm(SDNode *N) {
4462 MVT NVT = N->getSimpleValueType(0);
4463 unsigned Opcode = N->getOpcode();
4464 SDLoc dl(N);
4465
4466
4467
4468 SDValue Shift = N->getOperand(0);
4470
4472 if (!Cst)
4473 return false;
4474
4475 int64_t Val = Cst->getSExtValue();
4476
4477
4478
4479
4480 bool FoundAnyExtend = false;
4484 FoundAnyExtend = true;
4486 }
4487
4489 return false;
4490
4491
4492 if (NVT != MVT::i32 && NVT != MVT::i64)
4493 return false;
4494
4496 if (!ShlCst)
4497 return false;
4498
4499 uint64_t ShAmt = ShlCst->getZExtValue();
4500
4501
4502
4503 uint64_t RemovedBitsMask = (1ULL << ShAmt) - 1;
4504 if (Opcode != ISD::AND && (Val & RemovedBitsMask) != 0)
4505 return false;
4506
4507
4508
4509 auto CanShrinkImmediate = [&](int64_t &ShiftedVal) {
4511
4512
4513 ShiftedVal = (uint64_t)Val >> ShAmt;
4515 return true;
4516
4517 if (ShiftedVal == UINT8_MAX || ShiftedVal == UINT16_MAX)
4518 return true;
4519 }
4520 ShiftedVal = Val >> ShAmt;
4523 return true;
4525
4526 ShiftedVal = (uint64_t)Val >> ShAmt;
4528 return true;
4529 }
4530 return false;
4531 };
4532
4533 int64_t ShiftedVal;
4534 if (!CanShrinkImmediate(ShiftedVal))
4535 return false;
4536
4537
4538
4539
4540
4541
4543
4544 unsigned ZExtWidth = Cst->getAPIntValue().getActiveBits();
4545 ZExtWidth = llvm::bit_ceil(std::max(ZExtWidth, 8U));
4546
4547
4549 ZExtWidth);
4550 NeededMask &= ~Cst->getAPIntValue();
4551
4552 if (CurDAG->MaskedValueIsZero(N->getOperand(0), NeededMask))
4553 return false;
4554 }
4555
4557 if (FoundAnyExtend) {
4560 X = NewX;
4561 }
4562
4563 SDValue NewCst = CurDAG->getSignedConstant(ShiftedVal, dl, NVT);
4565 SDValue NewBinOp = CurDAG->getNode(Opcode, dl, NVT, X, NewCst);
4567 SDValue NewSHL = CurDAG->getNode(ISD::SHL, dl, NVT, NewBinOp,
4569 ReplaceNode(N, NewSHL.getNode());
4570 SelectCode(NewSHL.getNode());
4571 return true;
4572}
4573
4574bool X86DAGToDAGISel::matchVPTERNLOG(SDNode *Root, SDNode *ParentA,
4575 SDNode *ParentB, SDNode *ParentC,
4577 uint8_t Imm) {
4578 assert(A.isOperandOf(ParentA) && B.isOperandOf(ParentB) &&
4579 C.isOperandOf(ParentC) && "Incorrect parent node");
4580
4581 auto tryFoldLoadOrBCast =
4584 if (tryFoldLoad(Root, P, L, Base, Scale, Index, Disp, Segment))
4585 return true;
4586
4587
4588 if (L.getOpcode() == ISD::BITCAST && L.hasOneUse()) {
4591 }
4592
4594 return false;
4595
4596
4598 unsigned Size = MemIntr->getMemoryVT().getSizeInBits();
4599 if (Size != 32 && Size != 64)
4600 return false;
4601
4602 return tryFoldBroadcast(Root, P, L, Base, Scale, Index, Disp, Segment);
4603 };
4604
4605 bool FoldedLoad = false;
4606 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
4607 if (tryFoldLoadOrBCast(Root, ParentC, C, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
4608 FoldedLoad = true;
4609 } else if (tryFoldLoadOrBCast(Root, ParentA, A, Tmp0, Tmp1, Tmp2, Tmp3,
4610 Tmp4)) {
4611 FoldedLoad = true;
4613
4614 uint8_t OldImm = Imm;
4615 Imm = OldImm & 0xa5;
4616 if (OldImm & 0x02) Imm |= 0x10;
4617 if (OldImm & 0x10) Imm |= 0x02;
4618 if (OldImm & 0x08) Imm |= 0x40;
4619 if (OldImm & 0x40) Imm |= 0x08;
4620 } else if (tryFoldLoadOrBCast(Root, ParentB, B, Tmp0, Tmp1, Tmp2, Tmp3,
4621 Tmp4)) {
4622 FoldedLoad = true;
4624
4625 uint8_t OldImm = Imm;
4626 Imm = OldImm & 0x99;
4627 if (OldImm & 0x02) Imm |= 0x04;
4628 if (OldImm & 0x04) Imm |= 0x02;
4629 if (OldImm & 0x20) Imm |= 0x40;
4630 if (OldImm & 0x40) Imm |= 0x20;
4631 }
4632
4633 SDLoc DL(Root);
4634
4635 SDValue TImm = CurDAG->getTargetConstant(Imm, DL, MVT::i8);
4636
4638
4639 MachineSDNode *MNode;
4640 if (FoldedLoad) {
4641 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
4642
4643 unsigned Opc;
4646 unsigned EltSize = MemIntr->getMemoryVT().getSizeInBits();
4647 assert((EltSize == 32 || EltSize == 64) && "Unexpected broadcast size!");
4648
4649 bool UseD = EltSize == 32;
4651 Opc = UseD ? X86::VPTERNLOGDZ128rmbi : X86::VPTERNLOGQZ128rmbi;
4653 Opc = UseD ? X86::VPTERNLOGDZ256rmbi : X86::VPTERNLOGQZ256rmbi;
4655 Opc = UseD ? X86::VPTERNLOGDZrmbi : X86::VPTERNLOGQZrmbi;
4656 else
4658 } else {
4661 Opc = UseD ? X86::VPTERNLOGDZ128rmi : X86::VPTERNLOGQZ128rmi;
4663 Opc = UseD ? X86::VPTERNLOGDZ256rmi : X86::VPTERNLOGQZ256rmi;
4665 Opc = UseD ? X86::VPTERNLOGDZrmi : X86::VPTERNLOGQZrmi;
4666 else
4668 }
4669
4671 MNode = CurDAG->getMachineNode(Opc, DL, VTs, Ops);
4672
4673
4674 ReplaceUses(C.getValue(1), SDValue(MNode, 1));
4675
4676 CurDAG->setNodeMemRefs(MNode, {cast(C)->getMemOperand()});
4677 } else {
4679 unsigned Opc;
4681 Opc = UseD ? X86::VPTERNLOGDZ128rri : X86::VPTERNLOGQZ128rri;
4683 Opc = UseD ? X86::VPTERNLOGDZ256rri : X86::VPTERNLOGQZ256rri;
4685 Opc = UseD ? X86::VPTERNLOGDZrri : X86::VPTERNLOGQZrri;
4686 else
4688
4689 MNode = CurDAG->getMachineNode(Opc, DL, NVT, {A, B, C, TImm});
4690 }
4691
4693 CurDAG->RemoveDeadNode(Root);
4694 return true;
4695}
4696
4697
4698
4699bool X86DAGToDAGISel::tryVPTERNLOG(SDNode *N) {
4700 MVT NVT = N->getSimpleValueType(0);
4701
4702
4703 if (!NVT.isVector() || !Subtarget->hasAVX512() ||
4705 return false;
4706
4707
4709 return false;
4710
4711 auto getFoldableLogicOp = [](SDValue Op) {
4712
4713 if (Op.getOpcode() == ISD::BITCAST && Op.hasOneUse())
4715
4716 if (.hasOneUse())
4718
4719 unsigned Opc = Op.getOpcode();
4722 return Op;
4723
4725 };
4726
4727 SDValue N0, N1, A, FoldableOp;
4728
4729
4730 auto tryPeelOuterNotWrappingLogic = [&](SDNode *Op) {
4731 if (Op->getOpcode() == ISD::XOR && Op->hasOneUse() &&
4733 SDValue InnerOp = getFoldableLogicOp(Op->getOperand(0));
4734
4735 if (!InnerOp)
4737
4740 if ((FoldableOp = getFoldableLogicOp(N1))) {
4741 A = N0;
4742 return InnerOp;
4743 }
4744 if ((FoldableOp = getFoldableLogicOp(N0))) {
4745 A = N1;
4746 return InnerOp;
4747 }
4748 }
4750 };
4751
4752 bool PeeledOuterNot = false;
4753 SDNode *OriN = N;
4754 if (SDValue InnerOp = tryPeelOuterNotWrappingLogic(N)) {
4755 PeeledOuterNot = true;
4757 } else {
4760
4761 if ((FoldableOp = getFoldableLogicOp(N1)))
4762 A = N0;
4763 else if ((FoldableOp = getFoldableLogicOp(N0)))
4764 A = N1;
4765 else
4766 return false;
4767 }
4768
4771 SDNode *ParentA = N;
4772 SDNode *ParentB = FoldableOp.getNode();
4773 SDNode *ParentC = FoldableOp.getNode();
4774
4775
4776
4777 uint8_t TernlogMagicA = 0xf0;
4778 uint8_t TernlogMagicB = 0xcc;
4779 uint8_t TernlogMagicC = 0xaa;
4780
4781
4782
4783
4784 auto PeekThroughNot = [](SDValue &Op, SDNode *&Parent, uint8_t &Magic) {
4785 if (Op.getOpcode() == ISD::XOR && Op.hasOneUse() &&
4788 Parent = Op.getNode();
4790 }
4791 };
4792
4793 PeekThroughNot(A, ParentA, TernlogMagicA);
4794 PeekThroughNot(B, ParentB, TernlogMagicB);
4795 PeekThroughNot(C, ParentC, TernlogMagicC);
4796
4797 uint8_t Imm;
4798 switch (FoldableOp.getOpcode()) {
4800 case ISD::AND: Imm = TernlogMagicB & TernlogMagicC; break;
4801 case ISD::OR: Imm = TernlogMagicB | TernlogMagicC; break;
4802 case ISD::XOR: Imm = TernlogMagicB ^ TernlogMagicC; break;
4803 case X86ISD::ANDNP: Imm = ~(TernlogMagicB) & TernlogMagicC; break;
4804 }
4805
4806 switch (N->getOpcode()) {
4809 if (A == N0)
4810 Imm &= ~TernlogMagicA;
4811 else
4812 Imm = ~(Imm) & TernlogMagicA;
4813 break;
4814 case ISD::AND: Imm &= TernlogMagicA; break;
4815 case ISD::OR: Imm |= TernlogMagicA; break;
4816 case ISD::XOR: Imm ^= TernlogMagicA; break;
4817 }
4818
4819 if (PeeledOuterNot)
4820 Imm = ~Imm;
4821
4822 return matchVPTERNLOG(OriN, ParentA, ParentB, ParentC, A, B, C, Imm);
4823}
4824
4825
4826
4827
4828
4829
4830
4831
4832bool X86DAGToDAGISel::shrinkAndImmediate(SDNode *And) {
4833
4834
4835 MVT VT = And->getSimpleValueType(0);
4836 if (VT != MVT::i32 && VT != MVT::i64)
4837 return false;
4838
4840 if (!And1C)
4841 return false;
4842
4843
4844
4845
4846
4847
4848 APInt MaskVal = And1C->getAPIntValue();
4849 unsigned MaskLZ = MaskVal.countl_zero();
4850 if (!MaskLZ || (VT == MVT::i64 && MaskLZ == 32))
4851 return false;
4852
4853
4854 if (VT == MVT::i64 && MaskLZ >= 32) {
4855 MaskLZ -= 32;
4856 MaskVal = MaskVal.trunc(32);
4857 }
4858
4861 APInt NegMaskVal = MaskVal | HighZeros;
4862
4863
4864
4866 if (MinWidth > 32 || (MinWidth > 8 && MaskVal.getSignificantBits() <= 32))
4867 return false;
4868
4869
4870 if (VT == MVT::i64 && MaskVal.getBitWidth() < 64) {
4871 NegMaskVal = NegMaskVal.zext(64);
4872 HighZeros = HighZeros.zext(64);
4873 }
4874
4875
4876
4877
4878 KnownBits Known0 = CurDAG->computeKnownBits(And0);
4880 return false;
4881
4882
4883
4886 return true;
4887 }
4888
4889
4890 SDValue NewMask = CurDAG->getConstant(NegMaskVal, SDLoc(And), VT);
4892 SDValue NewAnd = CurDAG->getNode(ISD::AND, SDLoc(And), VT, And0, NewMask);
4894 SelectCode(NewAnd.getNode());
4895 return true;
4896}
4897
4899 bool FoldedBCast, bool Masked) {
4900#define VPTESTM_CASE(VT, SUFFIX) \
4901case MVT::VT: \
4902 if (Masked) \
4903 return IsTestN ? X86::VPTESTNM##SUFFIX##k: X86::VPTESTM##SUFFIX##k; \
4904 return IsTestN ? X86::VPTESTNM##SUFFIX : X86::VPTESTM##SUFFIX;
4905
4906
4907#define VPTESTM_BROADCAST_CASES(SUFFIX) \
4908default: llvm_unreachable("Unexpected VT!"); \
4909VPTESTM_CASE(v4i32, DZ128##SUFFIX) \
4910VPTESTM_CASE(v2i64, QZ128##SUFFIX) \
4911VPTESTM_CASE(v8i32, DZ256##SUFFIX) \
4912VPTESTM_CASE(v4i64, QZ256##SUFFIX) \
4913VPTESTM_CASE(v16i32, DZ##SUFFIX) \
4914VPTESTM_CASE(v8i64, QZ##SUFFIX)
4915
4916#define VPTESTM_FULL_CASES(SUFFIX) \
4917VPTESTM_BROADCAST_CASES(SUFFIX) \
4918VPTESTM_CASE(v16i8, BZ128##SUFFIX) \
4919VPTESTM_CASE(v8i16, WZ128##SUFFIX) \
4920VPTESTM_CASE(v32i8, BZ256##SUFFIX) \
4921VPTESTM_CASE(v16i16, WZ256##SUFFIX) \
4922VPTESTM_CASE(v64i8, BZ##SUFFIX) \
4923VPTESTM_CASE(v32i16, WZ##SUFFIX)
4924
4925 if (FoldedBCast) {
4928 }
4929 }
4930
4931 if (FoldedLoad) {
4934 }
4935 }
4936
4939 }
4940
4941#undef VPTESTM_FULL_CASES
4942#undef VPTESTM_BROADCAST_CASES
4943#undef VPTESTM_CASE
4944}
4945
4946
4947
4948bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
4950 assert(Subtarget->hasAVX512() && "Expected AVX512!");
4952 "Unexpected VT!");
4953
4954
4957 return false;
4958
4961
4962
4965
4966
4968 return false;
4969
4971
4974
4975
4978
4979 {
4980
4984
4985
4989 }
4990 }
4991
4992
4994
4995 auto tryFoldLoadOrBCast = [&](SDNode *Root, SDNode *P, SDValue &L,
4998
5000 if (tryFoldLoad(Root, P, L, Base, Scale, Index, Disp, Segment))
5001 return true;
5002
5003
5004
5005 if (CmpSVT != MVT::i32 && CmpSVT != MVT::i64)
5006 return false;
5007
5008
5009 if (L.getOpcode() == ISD::BITCAST && L.hasOneUse()) {
5012 }
5013
5015 return false;
5016
5018 if (MemIntr->getMemoryVT().getSizeInBits() != CmpSVT.getSizeInBits())
5019 return false;
5020
5021 return tryFoldBroadcast(Root, P, L, Base, Scale, Index, Disp, Segment);
5022 };
5023
5024
5025 bool CanFoldLoads = Src0 != Src1;
5026
5027 bool FoldedLoad = false;
5028 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5029 if (CanFoldLoads) {
5030 FoldedLoad = tryFoldLoadOrBCast(Root, N0.getNode(), Src1, Tmp0, Tmp1, Tmp2,
5031 Tmp3, Tmp4);
5032 if (!FoldedLoad) {
5033
5034 FoldedLoad = tryFoldLoadOrBCast(Root, N0.getNode(), Src0, Tmp0, Tmp1,
5035 Tmp2, Tmp3, Tmp4);
5036 if (FoldedLoad)
5038 }
5039 }
5040
5042
5043 bool IsMasked = InMask.getNode() != nullptr;
5044
5045 SDLoc dl(Root);
5046
5048 MVT MaskVT = ResVT;
5050
5056 SDValue ImplDef = SDValue(CurDAG->getMachineNode(X86::IMPLICIT_DEF, dl,
5057 CmpVT), 0);
5058 Src0 = CurDAG->getTargetInsertSubreg(SubReg, dl, CmpVT, ImplDef, Src0);
5059
5060 if (!FoldedBCast)
5061 Src1 = CurDAG->getTargetInsertSubreg(SubReg, dl, CmpVT, ImplDef, Src1);
5062
5063 if (IsMasked) {
5064
5066 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
5067 InMask = SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
5068 dl, MaskVT, InMask, RC), 0);
5069 }
5070 }
5071
5073 unsigned Opc = getVPTESTMOpc(CmpVT, IsTestN, FoldedLoad, FoldedBCast,
5074 IsMasked);
5075
5076 MachineSDNode *CNode;
5077 if (FoldedLoad) {
5078 SDVTList VTs = CurDAG->getVTList(MaskVT, MVT::Other);
5079
5080 if (IsMasked) {
5081 SDValue Ops[] = { InMask, Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
5083 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5084 } else {
5085 SDValue Ops[] = { Src0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4,
5087 CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5088 }
5089
5090
5092
5093 CurDAG->setNodeMemRefs(CNode, {cast(Src1)->getMemOperand()});
5094 } else {
5095 if (IsMasked)
5096 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, InMask, Src0, Src1);
5097 else
5098 CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, Src0, Src1);
5099 }
5100
5101
5104 SDValue RC = CurDAG->getTargetConstant(RegClass, dl, MVT::i32);
5105 CNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
5106 dl, ResVT, SDValue(CNode, 0), RC);
5107 }
5108
5110 CurDAG->RemoveDeadNode(Root);
5111 return true;
5112}
5113
5114
5115
5116bool X86DAGToDAGISel::tryMatchBitSelect(SDNode *N) {
5117 assert(N->getOpcode() == ISD::OR && "Unexpected opcode!");
5118
5119 MVT NVT = N->getSimpleValueType(0);
5120
5121
5122 if (!NVT.isVector() || !Subtarget->hasAVX512())
5123 return false;
5124
5125
5127 return false;
5128
5131
5132
5135
5139 return false;
5140
5141
5144
5145
5146
5152 else
5153 return false;
5154
5155 SDLoc dl(N);
5156 SDValue Imm = CurDAG->getTargetConstant(0xCA, dl, MVT::i8);
5158 ReplaceNode(N, Ternlog.getNode());
5159
5162}
5163
5164void X86DAGToDAGISel::Select(SDNode *Node) {
5165 MVT NVT = Node->getSimpleValueType(0);
5166 unsigned Opcode = Node->getOpcode();
5167 SDLoc dl(Node);
5168
5169 if (Node->isMachineOpcode()) {
5171 Node->setNodeId(-1);
5172 return;
5173 }
5174
5175 switch (Opcode) {
5176 default: break;
5178 unsigned IntNo = Node->getConstantOperandVal(1);
5179 switch (IntNo) {
5180 default: break;
5181 case Intrinsic::x86_encodekey128:
5182 case Intrinsic::x86_encodekey256: {
5183 if (!Subtarget->hasKL())
5184 break;
5185
5186 unsigned Opcode;
5187 switch (IntNo) {
5189 case Intrinsic::x86_encodekey128:
5190 Opcode = X86::ENCODEKEY128;
5191 break;
5192 case Intrinsic::x86_encodekey256:
5193 Opcode = X86::ENCODEKEY256;
5194 break;
5195 }
5196
5198 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0, Node->getOperand(3),
5200 if (Opcode == X86::ENCODEKEY256)
5201 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1, Node->getOperand(4),
5203
5204 MachineSDNode *Res = CurDAG->getMachineNode(
5205 Opcode, dl, Node->getVTList(),
5206 {Node->getOperand(2), Chain, Chain.getValue(1)});
5207 ReplaceNode(Node, Res);
5208 return;
5209 }
5210 case Intrinsic::x86_tileloaddrs64_internal:
5211 case Intrinsic::x86_tileloaddrst164_internal:
5212 if (!Subtarget->hasAMXMOVRS())
5213 break;
5214 [[fallthrough]];
5215 case Intrinsic::x86_tileloadd64_internal:
5216 case Intrinsic::x86_tileloaddt164_internal: {
5217 if (!Subtarget->hasAMXTILE())
5218 break;
5219 auto *MFI =
5220 CurDAG->getMachineFunction().getInfo();
5221 MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA);
5222 unsigned Opc;
5223 switch (IntNo) {
5224 default:
5226 case Intrinsic::x86_tileloaddrs64_internal:
5227 Opc = X86::PTILELOADDRSV;
5228 break;
5229 case Intrinsic::x86_tileloaddrst164_internal:
5230 Opc = X86::PTILELOADDRST1V;
5231 break;
5232 case Intrinsic::x86_tileloadd64_internal:
5233 Opc = X86::PTILELOADDV;
5234 break;
5235 case Intrinsic::x86_tileloaddt164_internal:
5236 Opc = X86::PTILELOADDT1V;
5237 break;
5238 }
5239
5241 SDValue Scale = getI8Imm(1, dl);
5243 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5244 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5246 MachineSDNode *CNode;
5248 Node->getOperand(3),
5250 Scale,
5252 Disp,
5253 Segment,
5254 Chain};
5255 CNode = CurDAG->getMachineNode(Opc, dl, {MVT::x86amx, MVT::Other}, Ops);
5256 ReplaceNode(Node, CNode);
5257 return;
5258 }
5259 }
5260 break;
5261 }
5263 unsigned IntNo = Node->getConstantOperandVal(1);
5264 switch (IntNo) {
5265 default: break;
5266 case Intrinsic::x86_sse3_monitor:
5267 case Intrinsic::x86_monitorx:
5268 case Intrinsic::x86_clzero: {
5269 bool Use64BitPtr = Node->getOperand(2).getValueType() == MVT::i64;
5270
5271 unsigned Opc = 0;
5272 switch (IntNo) {
5274 case Intrinsic::x86_sse3_monitor:
5275 if (!Subtarget->hasSSE3())
5276 break;
5277 Opc = Use64BitPtr ? X86::MONITOR64rrr : X86::MONITOR32rrr;
5278 break;
5279 case Intrinsic::x86_monitorx:
5280 if (!Subtarget->hasMWAITX())
5281 break;
5282 Opc = Use64BitPtr ? X86::MONITORX64rrr : X86::MONITORX32rrr;
5283 break;
5284 case Intrinsic::x86_clzero:
5285 if (!Subtarget->hasCLZERO())
5286 break;
5287 Opc = Use64BitPtr ? X86::CLZERO64r : X86::CLZERO32r;
5288 break;
5289 }
5290
5291 if (Opc) {
5292 unsigned PtrReg = Use64BitPtr ? X86::RAX : X86::EAX;
5293 SDValue Chain = CurDAG->getCopyToReg(Node->getOperand(0), dl, PtrReg,
5296
5297 if (IntNo == Intrinsic::x86_sse3_monitor ||
5298 IntNo == Intrinsic::x86_monitorx) {
5299
5300 Chain = CurDAG->getCopyToReg(Chain, dl, X86::ECX, Node->getOperand(3),
5301 InGlue);
5303 Chain = CurDAG->getCopyToReg(Chain, dl, X86::EDX, Node->getOperand(4),
5304 InGlue);
5306 }
5307
5308 MachineSDNode *CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
5309 { Chain, InGlue});
5310 ReplaceNode(Node, CNode);
5311 return;
5312 }
5313
5314 break;
5315 }
5316 case Intrinsic::x86_tilestored64_internal: {
5317 auto *MFI =
5318 CurDAG->getMachineFunction().getInfo();
5319 MFI->setAMXProgModel(AMXProgModelEnum::ManagedRA);
5320 unsigned Opc = X86::PTILESTOREDV;
5321
5323 SDValue Scale = getI8Imm(1, dl);
5325 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5326 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5328 MachineSDNode *CNode;
5330 Node->getOperand(3),
5332 Scale,
5334 Disp,
5335 Segment,
5336 Node->getOperand(6),
5337 Chain};
5338 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5339 ReplaceNode(Node, CNode);
5340 return;
5341 }
5342 case Intrinsic::x86_tileloaddrs64:
5343 case Intrinsic::x86_tileloaddrst164:
5344 if (!Subtarget->hasAMXMOVRS())
5345 break;
5346 [[fallthrough]];
5347 case Intrinsic::x86_tileloadd64:
5348 case Intrinsic::x86_tileloaddt164:
5349 case Intrinsic::x86_tilestored64: {
5350 if (!Subtarget->hasAMXTILE())
5351 break;
5352 auto *MFI =
5353 CurDAG->getMachineFunction().getInfo();
5354 MFI->setAMXProgModel(AMXProgModelEnum::DirectReg);
5355 unsigned Opc;
5356 switch (IntNo) {
5358 case Intrinsic::x86_tileloadd64: Opc = X86::PTILELOADD; break;
5359 case Intrinsic::x86_tileloaddrs64:
5360 Opc = X86::PTILELOADDRS;
5361 break;
5362 case Intrinsic::x86_tileloaddt164: Opc = X86::PTILELOADDT1; break;
5363 case Intrinsic::x86_tileloaddrst164:
5364 Opc = X86::PTILELOADDRST1;
5365 break;
5366 case Intrinsic::x86_tilestored64: Opc = X86::PTILESTORED; break;
5367 }
5368
5369 unsigned TIndex = Node->getConstantOperandVal(2);
5372 SDValue Scale = getI8Imm(1, dl);
5374 SDValue Disp = CurDAG->getTargetConstant(0, dl, MVT::i32);
5375 SDValue Segment = CurDAG->getRegister(0, MVT::i16);
5377 MachineSDNode *CNode;
5378 if (Opc == X86::PTILESTORED) {
5380 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5381 } else {
5383 CNode = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
5384 }
5385 ReplaceNode(Node, CNode);
5386 return;
5387 }
5388 }
5389 break;
5390 }
5391 case ISD::BRIND:
5393 if (Subtarget->isTarget64BitILP32()) {
5394
5395
5396
5398 assert(Target.getValueType() == MVT::i32 && "Unexpected VT!");
5399 SDValue ZextTarget = CurDAG->getZExtOrTrunc(Target, dl, MVT::i64);
5400 SDValue Brind = CurDAG->getNode(Opcode, dl, MVT::Other,
5401 Node->getOperand(0), ZextTarget);
5402 ReplaceNode(Node, Brind.getNode());
5403 SelectCode(ZextTarget.getNode());
5404 SelectCode(Brind.getNode());
5405 return;
5406 }
5407 break;
5408 }
5410 ReplaceNode(Node, getGlobalBaseReg());
5411 return;
5412
5413 case ISD::BITCAST:
5414
5416 NVT == MVT::f128) {
5417 ReplaceUses(SDValue(Node, 0), Node->getOperand(0));
5418 CurDAG->RemoveDeadNode(Node);
5419 return;
5420 }
5421 break;
5422
5424 if (matchBitExtract(Node))
5425 return;
5426 [[fallthrough]];
5429 if (tryShiftAmountMod(Node))
5430 return;
5431 break;
5432
5434 uint8_t Imm = Node->getConstantOperandVal(3);
5435 if (matchVPTERNLOG(Node, Node, Node, Node, Node->getOperand(0),
5436 Node->getOperand(1), Node->getOperand(2), Imm))
5437 return;
5438 break;
5439 }
5440
5442 if (tryVPTERNLOG(Node))
5443 return;
5444 break;
5445
5448
5452 tryVPTESTM(Node, N0, N1))
5453 return;
5455 tryVPTESTM(Node, N1, N0))
5456 return;
5457 }
5458
5459 if (MachineSDNode *NewNode = matchBEXTRFromAndImm(Node)) {
5461 CurDAG->RemoveDeadNode(Node);
5462 return;
5463 }
5464 if (matchBitExtract(Node))
5465 return;
5466 if (AndImmShrink && shrinkAndImmediate(Node))
5467 return;
5468
5469 [[fallthrough]];
5472 if (tryShrinkShlLogicImm(Node))
5473 return;
5474 if (Opcode == ISD::OR && tryMatchBitSelect(Node))
5475 return;
5476 if (tryVPTERNLOG(Node))
5477 return;
5478
5479 [[fallthrough]];
5481 if (Opcode == ISD::ADD && matchBitExtract(Node))
5482 return;
5483 [[fallthrough]];
5485
5486
5487
5488
5489
5490
5491 if (!CurDAG->shouldOptForSize())
5492 break;
5493
5494
5495 if (NVT != MVT::i8 && NVT != MVT::i16 && NVT != MVT::i32 && NVT != MVT::i64)
5496 break;
5497
5500
5502 if (!Cst)
5503 break;
5504
5505 int64_t Val = Cst->getSExtValue();
5506
5507
5508
5510 break;
5511
5512
5513 if (Opcode == ISD::ADD && (Val == 1 || Val == -1))
5514 break;
5515
5516
5517 if (!shouldAvoidImmediateInstFormsForSize(N1.getNode()))
5518 break;
5519
5520
5521 unsigned ROpc, MOpc;
5524 case MVT::i8:
5525 switch (Opcode) {
5530 break;
5534 break;
5538 break;
5542 break;
5546 break;
5547 }
5548 break;
5549 case MVT::i16:
5550 switch (Opcode) {
5555 break;
5559 break;
5563 break;
5567 break;
5571 break;
5572 }
5573 break;
5574 case MVT::i32:
5575 switch (Opcode) {
5580 break;
5584 break;
5588 break;
5592 break;
5596 break;
5597 }
5598 break;
5599 case MVT::i64:
5600 switch (Opcode) {
5605 break;
5609 break;
5613 break;
5617 break;
5621 break;
5622 }
5623 break;
5624 }
5625
5626
5627
5628
5630 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5631 if (tryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
5633 SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
5634 MachineSDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5635
5637
5638 CurDAG->setNodeMemRefs(CNode, {cast(N0)->getMemOperand()});
5640 CurDAG->RemoveDeadNode(Node);
5641 return;
5642 }
5643 }
5644
5645 CurDAG->SelectNodeTo(Node, ROpc, NVT, MVT::i32, N0, N1);
5646 return;
5647 }
5648
5650
5651 if (NVT != MVT::i8)
5652 break;
5653 [[fallthrough]];
5657
5658 unsigned LoReg, ROpc, MOpc;
5661 case MVT::i8:
5662 LoReg = X86::AL;
5663 ROpc = Opcode == X86ISD::SMUL ? X86::IMUL8r : X86::MUL8r;
5664 MOpc = Opcode == X86ISD::SMUL ? X86::IMUL8m : X86::MUL8m;
5665 break;
5666 case MVT::i16:
5667 LoReg = X86::AX;
5668 ROpc = X86::MUL16r;
5669 MOpc = X86::MUL16m;
5670 break;
5671 case MVT::i32:
5672 LoReg = X86::EAX;
5673 ROpc = X86::MUL32r;
5674 MOpc = X86::MUL32m;
5675 break;
5676 case MVT::i64:
5677 LoReg = X86::RAX;
5678 ROpc = X86::MUL64r;
5679 MOpc = X86::MUL64m;
5680 break;
5681 }
5682
5683 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5684 bool FoldedLoad = tryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5685
5686 if (!FoldedLoad) {
5687 FoldedLoad = tryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5688 if (FoldedLoad)
5690 }
5691
5692 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
5693 N0, SDValue()).getValue(1);
5694
5695 MachineSDNode *CNode;
5696 if (FoldedLoad) {
5697
5698
5699 SDVTList VTs;
5700 if (NVT == MVT::i8)
5701 VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
5702 else
5703 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32, MVT::Other);
5704
5706 InGlue };
5707 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5708
5709
5710 ReplaceUses(N1.getValue(1), SDValue(CNode, NVT == MVT::i8 ? 2 : 3));
5711
5712 CurDAG->setNodeMemRefs(CNode, {cast(N1)->getMemOperand()});
5713 } else {
5714
5715
5716 SDVTList VTs;
5717 if (NVT == MVT::i8)
5718 VTs = CurDAG->getVTList(NVT, MVT::i32);
5719 else
5720 VTs = CurDAG->getVTList(NVT, NVT, MVT::i32);
5721
5722 CNode = CurDAG->getMachineNode(ROpc, dl, VTs, {N1, InGlue});
5723 }
5724
5726 ReplaceUses(SDValue(Node, 1), SDValue(CNode, NVT == MVT::i8 ? 1 : 2));
5727 CurDAG->RemoveDeadNode(Node);
5728 return;
5729 }
5730
5735
5736 unsigned Opc, MOpc;
5737 unsigned LoReg, HiReg;
5739 bool UseMULX = !IsSigned && Subtarget->hasBMI2();
5740 bool UseMULXHi = UseMULX && SDValue(Node, 0).use_empty();
5743 case MVT::i32:
5744 Opc = UseMULXHi ? X86::MULX32Hrr
5746 : IsSigned ? X86::IMUL32r
5747 : X86::MUL32r;
5748 MOpc = UseMULXHi ? X86::MULX32Hrm
5750 : IsSigned ? X86::IMUL32m
5751 : X86::MUL32m;
5752 LoReg = UseMULX ? X86::EDX : X86::EAX;
5753 HiReg = X86::EDX;
5754 break;
5755 case MVT::i64:
5756 Opc = UseMULXHi ? X86::MULX64Hrr
5758 : IsSigned ? X86::IMUL64r
5759 : X86::MUL64r;
5760 MOpc = UseMULXHi ? X86::MULX64Hrm
5762 : IsSigned ? X86::IMUL64m
5763 : X86::MUL64m;
5764 LoReg = UseMULX ? X86::RDX : X86::RAX;
5765 HiReg = X86::RDX;
5766 break;
5767 }
5768
5769 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5770 bool foldedLoad = tryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5771
5772 if (!foldedLoad) {
5773 foldedLoad = tryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5774 if (foldedLoad)
5776 }
5777
5778 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
5779 N0, SDValue()).getValue(1);
5781 if (foldedLoad) {
5783 MachineSDNode *CNode = nullptr;
5785 InGlue };
5786 if (UseMULXHi) {
5787 SDVTList VTs = CurDAG->getVTList(NVT, MVT::Other);
5788 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5789 ResHi = SDValue(CNode, 0);
5790 Chain = SDValue(CNode, 1);
5791 } else if (UseMULX) {
5792 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::Other);
5793 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5794 ResHi = SDValue(CNode, 0);
5795 ResLo = SDValue(CNode, 1);
5796 Chain = SDValue(CNode, 2);
5797 } else {
5798 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
5799 CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
5800 Chain = SDValue(CNode, 0);
5801 InGlue = SDValue(CNode, 1);
5802 }
5803
5804
5805 ReplaceUses(N1.getValue(1), Chain);
5806
5807 CurDAG->setNodeMemRefs(CNode, {cast(N1)->getMemOperand()});
5808 } else {
5810 if (UseMULXHi) {
5811 SDVTList VTs = CurDAG->getVTList(NVT);
5812 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5813 ResHi = SDValue(CNode, 0);
5814 } else if (UseMULX) {
5815 SDVTList VTs = CurDAG->getVTList(NVT, NVT);
5816 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5817 ResHi = SDValue(CNode, 0);
5818 ResLo = SDValue(CNode, 1);
5819 } else {
5820 SDVTList VTs = CurDAG->getVTList(MVT::Glue);
5821 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
5822 InGlue = SDValue(CNode, 0);
5823 }
5824 }
5825
5826
5827 if ((Node, 0).use_empty()) {
5828 if (!ResLo) {
5829 assert(LoReg && "Register for low half is not defined!");
5830 ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg,
5831 NVT, InGlue);
5833 }
5834 ReplaceUses(SDValue(Node, 0), ResLo);
5836 dbgs() << '\n');
5837 }
5838
5839 if ((Node, 1).use_empty()) {
5840 if (!ResHi) {
5841 assert(HiReg && "Register for high half is not defined!");
5842 ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg,
5843 NVT, InGlue);
5845 }
5846 ReplaceUses(SDValue(Node, 1), ResHi);
5848 dbgs() << '\n');
5849 }
5850
5851 CurDAG->RemoveDeadNode(Node);
5852 return;
5853 }
5854
5859
5860 unsigned ROpc, MOpc;
5865 case MVT::i8: ROpc = X86::DIV8r; MOpc = X86::DIV8m; break;
5866 case MVT::i16: ROpc = X86::DIV16r; MOpc = X86::DIV16m; break;
5867 case MVT::i32: ROpc = X86::DIV32r; MOpc = X86::DIV32m; break;
5868 case MVT::i64: ROpc = X86::DIV64r; MOpc = X86::DIV64m; break;
5869 }
5870 } else {
5873 case MVT::i8: ROpc = X86::IDIV8r; MOpc = X86::IDIV8m; break;
5874 case MVT::i16: ROpc = X86::IDIV16r; MOpc = X86::IDIV16m; break;
5875 case MVT::i32: ROpc = X86::IDIV32r; MOpc = X86::IDIV32m; break;
5876 case MVT::i64: ROpc = X86::IDIV64r; MOpc = X86::IDIV64m; break;
5877 }
5878 }
5879
5880 unsigned LoReg, HiReg, ClrReg;
5881 unsigned SExtOpcode;
5884 case MVT::i8:
5885 LoReg = X86::AL; ClrReg = HiReg = X86::AH;
5886 SExtOpcode = 0;
5887 break;
5888 case MVT::i16:
5889 LoReg = X86::AX; HiReg = X86::DX;
5890 ClrReg = X86::DX;
5891 SExtOpcode = X86::CWD;
5892 break;
5893 case MVT::i32:
5894 LoReg = X86::EAX; ClrReg = HiReg = X86::EDX;
5895 SExtOpcode = X86::CDQ;
5896 break;
5897 case MVT::i64:
5898 LoReg = X86::RAX; ClrReg = HiReg = X86::RDX;
5899 SExtOpcode = X86::CQO;
5900 break;
5901 }
5902
5903 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
5904 bool foldedLoad = tryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
5905 bool signBitIsZero = CurDAG->SignBitIsZero(N0);
5906
5908 if (NVT == MVT::i8) {
5909
5910
5911 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain;
5912 MachineSDNode *Move;
5913 if (tryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
5915 unsigned Opc = (isSigned && !signBitIsZero) ? X86::MOVSX16rm8
5916 : X86::MOVZX16rm8;
5917 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, MVT::Other, Ops);
5918 Chain = SDValue(Move, 1);
5919 ReplaceUses(N0.getValue(1), Chain);
5920
5921 CurDAG->setNodeMemRefs(Move, {cast(N0)->getMemOperand()});
5922 } else {
5923 unsigned Opc = (isSigned && !signBitIsZero) ? X86::MOVSX16rr8
5924 : X86::MOVZX16rr8;
5925 Move = CurDAG->getMachineNode(Opc, dl, MVT::i16, N0);
5926 Chain = CurDAG->getEntryNode();
5927 }
5928 Chain = CurDAG->getCopyToReg(Chain, dl, X86::AX, SDValue(Move, 0),
5931 } else {
5932 InGlue =
5933 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl,
5934 LoReg, N0, SDValue()).getValue(1);
5935 if (isSigned && !signBitIsZero) {
5936
5937 InGlue =
5938 SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InGlue),0);
5939 } else {
5940
5941 SDVTList VTs = CurDAG->getVTList(MVT::i32, MVT::i32);
5943 SDValue(CurDAG->getMachineNode(X86::MOV32r0, dl, VTs, {}), 0);
5945 case MVT::i16:
5946 ClrNode =
5947 SDValue(CurDAG->getMachineNode(
5948 TargetOpcode::EXTRACT_SUBREG, dl, MVT::i16, ClrNode,
5949 CurDAG->getTargetConstant(X86::sub_16bit, dl,
5950 MVT::i32)),
5951 0);
5952 break;
5953 case MVT::i32:
5954 break;
5955 case MVT::i64:
5956 ClrNode =
5957 SDValue(CurDAG->getMachineNode(
5958 TargetOpcode::SUBREG_TO_REG, dl, MVT::i64,
5959 CurDAG->getTargetConstant(0, dl, MVT::i64), ClrNode,
5960 CurDAG->getTargetConstant(X86::sub_32bit, dl,
5961 MVT::i32)),
5962 0);
5963 break;
5964 default:
5966 }
5967
5968 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg,
5969 ClrNode, InGlue).getValue(1);
5970 }
5971 }
5972
5973 if (foldedLoad) {
5975 InGlue };
5976 MachineSDNode *CNode =
5977 CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops);
5978 InGlue = SDValue(CNode, 1);
5979
5981
5982 CurDAG->setNodeMemRefs(CNode, {cast(N1)->getMemOperand()});
5983 } else {
5984 InGlue =
5985 SDValue(CurDAG->getMachineNode(ROpc, dl, MVT::Glue, N1, InGlue), 0);
5986 }
5987
5988
5989
5990
5991
5992
5993
5994
5995 if (HiReg == X86::AH && (Node, 1).use_empty()) {
5996 SDValue AHCopy = CurDAG->getRegister(X86::AH, MVT::i8);
5997 unsigned AHExtOpcode =
5998 isSigned ? X86::MOVSX32rr8_NOREX : X86::MOVZX32rr8_NOREX;
5999
6000 SDNode *RNode = CurDAG->getMachineNode(AHExtOpcode, dl, MVT::i32,
6001 MVT::Glue, AHCopy, InGlue);
6003 InGlue = SDValue(RNode, 1);
6004
6006 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result);
6007
6008 ReplaceUses(SDValue(Node, 1), Result);
6010 dbgs() << '\n');
6011 }
6012
6013 if ((Node, 0).use_empty()) {
6014 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
6015 LoReg, NVT, InGlue);
6016 InGlue = Result.getValue(2);
6017 ReplaceUses(SDValue(Node, 0), Result);
6019 dbgs() << '\n');
6020 }
6021
6022 if ((Node, 1).use_empty()) {
6023 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
6024 HiReg, NVT, InGlue);
6025 InGlue = Result.getValue(2);
6026 ReplaceUses(SDValue(Node, 1), Result);
6028 dbgs() << '\n');
6029 }
6030 CurDAG->RemoveDeadNode(Node);
6031 return;
6032 }
6033
6039 SDValue N0 = Node->getOperand(IsStrictCmp ? 1 : 0);
6040 SDValue N1 = Node->getOperand(IsStrictCmp ? 2 : 1);
6041
6042
6044
6045
6046 if (Subtarget->canUseCMOV())
6047 break;
6048
6050
6051 unsigned Opc;
6054 case MVT::f32:
6055 Opc = IsSignaling ? X86::COM_Fpr32 : X86::UCOM_Fpr32;
6056 break;
6057 case MVT::f64:
6058 Opc = IsSignaling ? X86::COM_Fpr64 : X86::UCOM_Fpr64;
6059 break;
6060 case MVT::f80:
6061 Opc = IsSignaling ? X86::COM_Fpr80 : X86::UCOM_Fpr80;
6062 break;
6063 }
6064
6066 IsStrictCmp ? Node->getOperand(0) : CurDAG->getEntryNode();
6068 if (IsStrictCmp) {
6069 SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
6070 Chain = SDValue(CurDAG->getMachineNode(Opc, dl, VTs, {N0, N1, Chain}), 0);
6072 } else {
6073 Glue = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, N0, N1), 0);
6074 }
6075
6076
6078 SDValue(CurDAG->getMachineNode(X86::FNSTSW16r, dl, MVT::i16, Glue), 0);
6079
6080
6082 CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl, MVT::i8, FNSTSW);
6083
6084
6085
6086 assert(Subtarget->canUseLAHFSAHF() &&
6087 "Target doesn't support SAHF or FCOMI?");
6088 SDValue AH = CurDAG->getCopyToReg(Chain, dl, X86::AH, Extract, SDValue());
6089 Chain = AH;
6091 CurDAG->getMachineNode(X86::SAHF, dl, MVT::i32, AH.getValue(1)), 0);
6092
6093 if (IsStrictCmp)
6094 ReplaceUses(SDValue(Node, 1), Chain);
6095
6096 ReplaceUses(SDValue(Node, 0), SAHF);
6097 CurDAG->RemoveDeadNode(Node);
6098 return;
6099 }
6100
6104
6105
6107 break;
6108
6109
6111
6112
6113
6114
6115
6117 if (MachineSDNode *NewNode = matchBEXTRFromAndImm(N0.getNode())) {
6118 unsigned TestOpc = CmpVT == MVT::i64 ? X86::TEST64rr
6119 : X86::TEST32rr;
6121 NewNode = CurDAG->getMachineNode(TestOpc, dl, MVT::i32, BEXTR, BEXTR);
6123 CurDAG->RemoveDeadNode(Node);
6124 return;
6125 }
6126 }
6127
6128
6131
6132
6133
6134
6138 if (!MaskC)
6139 break;
6140
6141
6142
6143 uint64_t Mask = MaskC->getZExtValue();
6145
6146
6147
6148
6150 onlyUsesZeroFlag(SDValue(Node, 0))) {
6152 unsigned ShiftAmt;
6153 unsigned SubRegIdx;
6154 MVT SubRegVT;
6155 unsigned TestOpcode;
6158
6159
6160
6161
6163 if (LeadingZeros == 0 && SavesBytes) {
6164
6165
6166
6168 ShiftAmt = TrailingZeros;
6169 SubRegIdx = 0;
6170 TestOpcode = X86::TEST64rr;
6171 } else if (TrailingZeros == 0 && SavesBytes) {
6172
6173
6174
6176 ShiftAmt = LeadingZeros;
6177 SubRegIdx = 0;
6178 TestOpcode = X86::TEST64rr;
6179 } else if (MaskC->hasOneUse() && (Mask)) {
6180
6181
6182 unsigned PopCount = 64 - LeadingZeros - TrailingZeros;
6183 if (PopCount == 8) {
6185 ShiftAmt = TrailingZeros;
6186 SubRegIdx = X86::sub_8bit;
6187 SubRegVT = MVT::i8;
6188 TestOpcode = X86::TEST8rr;
6189 } else if (PopCount == 16) {
6191 ShiftAmt = TrailingZeros;
6192 SubRegIdx = X86::sub_16bit;
6193 SubRegVT = MVT::i16;
6194 TestOpcode = X86::TEST16rr;
6195 } else if (PopCount == 32) {
6197 ShiftAmt = TrailingZeros;
6198 SubRegIdx = X86::sub_32bit;
6199 SubRegVT = MVT::i32;
6200 TestOpcode = X86::TEST32rr;
6201 }
6202 }
6204 SDValue ShiftC = CurDAG->getTargetConstant(ShiftAmt, dl, MVT::i64);
6206 CurDAG->getMachineNode(ShiftOpcode, dl, MVT::i64, MVT::i32,
6208 0);
6209 if (SubRegIdx != 0) {
6210 Shift =
6211 CurDAG->getTargetExtractSubreg(SubRegIdx, dl, SubRegVT, Shift);
6212 }
6213 MachineSDNode *Test =
6214 CurDAG->getMachineNode(TestOpcode, dl, MVT::i32, Shift, Shift);
6215 ReplaceNode(Node, Test);
6216 return;
6217 }
6218 }
6219
6220 MVT VT;
6221 int SubRegOp;
6222 unsigned ROpc, MOpc;
6223
6224
6225
6226
6227
6228
6230 (!(Mask & 0x80) || CmpVT == MVT::i8 ||
6231 hasNoSignFlagUses(SDValue(Node, 0)))) {
6232
6233 VT = MVT::i8;
6234 SubRegOp = X86::sub_8bit;
6235 ROpc = X86::TEST8ri;
6236 MOpc = X86::TEST8mi;
6237 } else if (OptForMinSize && isUInt<16>(Mask) &&
6238 (!(Mask & 0x8000) || CmpVT == MVT::i16 ||
6239 hasNoSignFlagUses(SDValue(Node, 0)))) {
6240
6241
6242
6243
6244 VT = MVT::i16;
6245 SubRegOp = X86::sub_16bit;
6246 ROpc = X86::TEST16ri;
6247 MOpc = X86::TEST16mi;
6249 ((!(Mask & 0x80000000) &&
6250
6251
6252 (CmpVT != MVT::i16 || !(Mask & 0x8000))) ||
6253 CmpVT == MVT::i32 ||
6254 hasNoSignFlagUses(SDValue(Node, 0)))) {
6255
6256
6257
6258
6259
6260 VT = MVT::i32;
6261 SubRegOp = X86::sub_32bit;
6262 ROpc = X86::TEST32ri;
6263 MOpc = X86::TEST32mi;
6264 } else {
6265
6266 break;
6267 }
6268
6269 SDValue Imm = CurDAG->getTargetConstant(Mask, dl, VT);
6271
6272
6273 MachineSDNode *NewNode;
6274 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
6275 if (tryFoldLoad(Node, N0.getNode(), Reg, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
6277 if (!LoadN->isSimple()) {
6278 unsigned NumVolBits = LoadN->getValueType(0).getSizeInBits();
6279 if ((MOpc == X86::TEST8mi && NumVolBits != 8) ||
6280 (MOpc == X86::TEST16mi && NumVolBits != 16) ||
6281 (MOpc == X86::TEST32mi && NumVolBits != 32))
6282 break;
6283 }
6284 }
6285 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Imm,
6286 Reg.getOperand(0) };
6287 NewNode = CurDAG->getMachineNode(MOpc, dl, MVT::i32, MVT::Other, Ops);
6288
6289 ReplaceUses(Reg.getValue(1), SDValue(NewNode, 1));
6290
6291 CurDAG->setNodeMemRefs(NewNode,
6293 } else {
6294
6296 Reg = CurDAG->getTargetExtractSubreg(SubRegOp, dl, VT, Reg);
6297
6298 NewNode = CurDAG->getMachineNode(ROpc, dl, MVT::i32, Reg, Imm);
6299 }
6300
6301 ReplaceNode(Node, NewNode);
6302 return;
6303 }
6304 break;
6305 }
6307 if (!Subtarget->hasSSE42())
6308 break;
6309
6310 bool NeedIndex = (Node, 0).use_empty();
6311 bool NeedMask = (Node, 1).use_empty();
6312
6313 bool MayFoldLoad = !NeedIndex || !NeedMask;
6314
6315 MachineSDNode *CNode;
6316 if (NeedMask) {
6317 unsigned ROpc =
6318 Subtarget->hasAVX() ? X86::VPCMPISTRMrri : X86::PCMPISTRMrri;
6319 unsigned MOpc =
6320 Subtarget->hasAVX() ? X86::VPCMPISTRMrmi : X86::PCMPISTRMrmi;
6321 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, MVT::v16i8, Node);
6323 }
6324 if (NeedIndex || !NeedMask) {
6325 unsigned ROpc =
6326 Subtarget->hasAVX() ? X86::VPCMPISTRIrri : X86::PCMPISTRIrri;
6327 unsigned MOpc =
6328 Subtarget->hasAVX() ? X86::VPCMPISTRIrmi : X86::PCMPISTRIrmi;
6329 CNode = emitPCMPISTR(ROpc, MOpc, MayFoldLoad, dl, MVT::i32, Node);
6331 }
6332
6333
6335 CurDAG->RemoveDeadNode(Node);
6336 return;
6337 }
6339 if (!Subtarget->hasSSE42())
6340 break;
6341
6342
6343 SDValue InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EAX,
6344 Node->getOperand(1),
6346 InGlue = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EDX,
6347 Node->getOperand(3), InGlue).getValue(1);
6348
6349 bool NeedIndex = (Node, 0).use_empty();
6350 bool NeedMask = (Node, 1).use_empty();
6351
6352 bool MayFoldLoad = !NeedIndex || !NeedMask;
6353
6354 MachineSDNode *CNode;
6355 if (NeedMask) {
6356 unsigned ROpc =
6357 Subtarget->hasAVX() ? X86::VPCMPESTRMrri : X86::PCMPESTRMrri;
6358 unsigned MOpc =
6359 Subtarget->hasAVX() ? X86::VPCMPESTRMrmi : X86::PCMPESTRMrmi;
6360 CNode =
6361 emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, MVT::v16i8, Node, InGlue);
6363 }
6364 if (NeedIndex || !NeedMask) {
6365 unsigned ROpc =
6366 Subtarget->hasAVX() ? X86::VPCMPESTRIrri : X86::PCMPESTRIrri;
6367 unsigned MOpc =
6368 Subtarget->hasAVX() ? X86::VPCMPESTRIrmi : X86::PCMPESTRIrmi;
6369 CNode = emitPCMPESTR(ROpc, MOpc, MayFoldLoad, dl, MVT::i32, Node, InGlue);
6371 }
6372
6374 CurDAG->RemoveDeadNode(Node);
6375 return;
6376 }
6377
6380 return;
6381
6382 break;
6383 }
6384
6385 case ISD::STORE:
6386 if (foldLoadStoreIntoMemOperand(Node))
6387 return;
6388 break;
6389
6391 MVT VT = Node->getSimpleValueType(0);
6393 if (Subtarget->hasSBBDepBreaking()) {
6394
6395
6396
6398 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EFLAGS,
6400
6401
6402
6403 unsigned Opc = VT == MVT::i64 ? X86::SETB_C64r : X86::SETB_C32r;
6404 MVT SetVT = VT == MVT::i64 ? MVT::i64 : MVT::i32;
6406 CurDAG->getMachineNode(Opc, dl, SetVT, EFLAGS, EFLAGS.getValue(1)),
6407 0);
6408 } else {
6409
6410
6411 Result = getSBBZero(Node);
6412 }
6413
6414
6415 if (VT == MVT::i8 || VT == MVT::i16) {
6416 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit;
6417 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result);
6418 }
6419
6420 ReplaceUses(SDValue(Node, 0), Result);
6421 CurDAG->RemoveDeadNode(Node);
6422 return;
6423 }
6428
6429
6430 ReplaceUses(SDValue(Node, 1), Result.getValue(1));
6431
6432
6433 if ((Node, 0).use_empty()) {
6434
6435 MVT VT = Node->getSimpleValueType(0);
6436 if (VT == MVT::i8 || VT == MVT::i16) {
6437 int SubIndex = VT == MVT::i16 ? X86::sub_16bit : X86::sub_8bit;
6438 Result = CurDAG->getTargetExtractSubreg(SubIndex, dl, VT, Result);
6439 }
6440 ReplaceUses(SDValue(Node, 0), Result);
6441 }
6442
6443 CurDAG->RemoveDeadNode(Node);
6444 return;
6445 }
6446 break;
6447 }
6450 SDValue IndexOp = Mgt->getIndex();
6453 MVT ValueVT = Node->getSimpleValueType(0);
6454 MVT MaskVT = Mask.getSimpleValueType();
6455
6456
6457
6458
6460 break;
6461
6464
6467
6468 unsigned Opc = 0;
6470 if (AVX512Gather) {
6471 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6472 Opc = IsFP ? X86::VGATHERDPSZ128rm : X86::VPGATHERDDZ128rm;
6473 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6474 Opc = IsFP ? X86::VGATHERDPSZ256rm : X86::VPGATHERDDZ256rm;
6475 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32)
6476 Opc = IsFP ? X86::VGATHERDPSZrm : X86::VPGATHERDDZrm;
6477 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6478 Opc = IsFP ? X86::VGATHERDPDZ128rm : X86::VPGATHERDQZ128rm;
6479 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6480 Opc = IsFP ? X86::VGATHERDPDZ256rm : X86::VPGATHERDQZ256rm;
6481 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64)
6482 Opc = IsFP ? X86::VGATHERDPDZrm : X86::VPGATHERDQZrm;
6483 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6484 Opc = IsFP ? X86::VGATHERQPSZ128rm : X86::VPGATHERQDZ128rm;
6485 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6486 Opc = IsFP ? X86::VGATHERQPSZ256rm : X86::VPGATHERQDZ256rm;
6487 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32)
6488 Opc = IsFP ? X86::VGATHERQPSZrm : X86::VPGATHERQDZrm;
6489 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6490 Opc = IsFP ? X86::VGATHERQPDZ128rm : X86::VPGATHERQQZ128rm;
6491 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6492 Opc = IsFP ? X86::VGATHERQPDZ256rm : X86::VPGATHERQQZ256rm;
6493 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64)
6494 Opc = IsFP ? X86::VGATHERQPDZrm : X86::VPGATHERQQZrm;
6495 } else {
6496 assert(EVT(MaskVT) == EVT(ValueVT).changeVectorElementTypeToInteger() &&
6497 "Unexpected mask VT!");
6498 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6499 Opc = IsFP ? X86::VGATHERDPSrm : X86::VPGATHERDDrm;
6500 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6501 Opc = IsFP ? X86::VGATHERDPSYrm : X86::VPGATHERDDYrm;
6502 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6503 Opc = IsFP ? X86::VGATHERDPDrm : X86::VPGATHERDQrm;
6504 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6505 Opc = IsFP ? X86::VGATHERDPDYrm : X86::VPGATHERDQYrm;
6506 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6507 Opc = IsFP ? X86::VGATHERQPSrm : X86::VPGATHERQDrm;
6508 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6509 Opc = IsFP ? X86::VGATHERQPSYrm : X86::VPGATHERQDYrm;
6510 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6511 Opc = IsFP ? X86::VGATHERQPDrm : X86::VPGATHERQQrm;
6512 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6513 Opc = IsFP ? X86::VGATHERQPDYrm : X86::VPGATHERQQYrm;
6514 }
6515
6516 if ()
6517 break;
6518
6520 if (!selectVectorAddr(Mgt, Mgt->getBasePtr(), IndexOp, Mgt->getScale(),
6521 Base, Scale, Index, Disp, Segment))
6522 break;
6523
6524 SDValue PassThru = Mgt->getPassThru();
6525 SDValue Chain = Mgt->getChain();
6526
6527 SDVTList VTs = CurDAG->getVTList(ValueVT, MaskVT, MVT::Other);
6528
6529 MachineSDNode *NewNode;
6530 if (AVX512Gather) {
6532 Index, Disp, Segment, Chain};
6533 NewNode = CurDAG->getMachineNode(Opc, SDLoc(dl), VTs, Ops);
6534 } else {
6536 Disp, Segment, Mask, Chain};
6537 NewNode = CurDAG->getMachineNode(Opc, SDLoc(dl), VTs, Ops);
6538 }
6539 CurDAG->setNodeMemRefs(NewNode, {Mgt->getMemOperand()});
6542 CurDAG->RemoveDeadNode(Node);
6543 return;
6544 }
6548 SDValue IndexOp = Sc->getIndex();
6550 MVT ValueVT = Value.getSimpleValueType();
6551
6552
6553
6554
6556 break;
6557
6560
6563
6564 unsigned Opc;
6565 if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 32)
6566 Opc = IsFP ? X86::VSCATTERDPSZ128mr : X86::VPSCATTERDDZ128mr;
6567 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 32)
6568 Opc = IsFP ? X86::VSCATTERDPSZ256mr : X86::VPSCATTERDDZ256mr;
6569 else if (IndexVT == MVT::v16i32 && NumElts == 16 && EltSize == 32)
6570 Opc = IsFP ? X86::VSCATTERDPSZmr : X86::VPSCATTERDDZmr;
6571 else if (IndexVT == MVT::v4i32 && NumElts == 2 && EltSize == 64)
6572 Opc = IsFP ? X86::VSCATTERDPDZ128mr : X86::VPSCATTERDQZ128mr;
6573 else if (IndexVT == MVT::v4i32 && NumElts == 4 && EltSize == 64)
6574 Opc = IsFP ? X86::VSCATTERDPDZ256mr : X86::VPSCATTERDQZ256mr;
6575 else if (IndexVT == MVT::v8i32 && NumElts == 8 && EltSize == 64)
6576 Opc = IsFP ? X86::VSCATTERDPDZmr : X86::VPSCATTERDQZmr;
6577 else if (IndexVT == MVT::v2i64 && NumElts == 4 && EltSize == 32)
6578 Opc = IsFP ? X86::VSCATTERQPSZ128mr : X86::VPSCATTERQDZ128mr;
6579 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 32)
6580 Opc = IsFP ? X86::VSCATTERQPSZ256mr : X86::VPSCATTERQDZ256mr;
6581 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 32)
6582 Opc = IsFP ? X86::VSCATTERQPSZmr : X86::VPSCATTERQDZmr;
6583 else if (IndexVT == MVT::v2i64 && NumElts == 2 && EltSize == 64)
6584 Opc = IsFP ? X86::VSCATTERQPDZ128mr : X86::VPSCATTERQQZ128mr;
6585 else if (IndexVT == MVT::v4i64 && NumElts == 4 && EltSize == 64)
6586 Opc = IsFP ? X86::VSCATTERQPDZ256mr : X86::VPSCATTERQQZ256mr;
6587 else if (IndexVT == MVT::v8i64 && NumElts == 8 && EltSize == 64)
6588 Opc = IsFP ? X86::VSCATTERQPDZmr : X86::VPSCATTERQQZmr;
6589 else
6590 break;
6591
6593 if (!selectVectorAddr(Sc, Sc->getBasePtr(), IndexOp, Sc->getScale(),
6594 Base, Scale, Index, Disp, Segment))
6595 break;
6596
6598 SDValue Chain = Sc->getChain();
6599
6600 SDVTList VTs = CurDAG->getVTList(Mask.getValueType(), MVT::Other);
6602
6603 MachineSDNode *NewNode = CurDAG->getMachineNode(Opc, SDLoc(dl), VTs, Ops);
6604 CurDAG->setNodeMemRefs(NewNode, {Sc->getMemOperand()});
6606 CurDAG->RemoveDeadNode(Node);
6607 return;
6608 }
6609 case ISD::PREALLOCATED_SETUP: {
6610 auto *MFI = CurDAG->getMachineFunction().getInfo();
6611 auto CallId = MFI->getPreallocatedIdForCallSite(
6614 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32);
6615 MachineSDNode *New = CurDAG->getMachineNode(
6616 TargetOpcode::PREALLOCATED_SETUP, dl, MVT::Other, CallIdValue, Chain);
6617 ReplaceUses(SDValue(Node, 0), SDValue(New, 0));
6618 CurDAG->RemoveDeadNode(Node);
6619 return;
6620 }
6621 case ISD::PREALLOCATED_ARG: {
6622 auto *MFI = CurDAG->getMachineFunction().getInfo();
6626 SDValue CallIdValue = CurDAG->getTargetConstant(CallId, dl, MVT::i32);
6629 Ops[0] = CallIdValue;
6630 Ops[1] = ArgIndex;
6631 Ops[2] = Chain;
6632 MachineSDNode *New = CurDAG->getMachineNode(
6633 TargetOpcode::PREALLOCATED_ARG, dl,
6634 CurDAG->getVTList(TLI->getPointerTy(CurDAG->getDataLayout()),
6635 MVT::Other),
6637 ReplaceUses(SDValue(Node, 0), SDValue(New, 0));
6638 ReplaceUses(SDValue(Node, 1), SDValue(New, 1));
6639 CurDAG->RemoveDeadNode(Node);
6640 return;
6641 }
6646 if (!Subtarget->hasWIDEKL())
6647 break;
6648
6649 unsigned Opcode;
6650 switch (Node->getOpcode()) {
6651 default:
6654 Opcode = X86::AESENCWIDE128KL;
6655 break;
6657 Opcode = X86::AESDECWIDE128KL;
6658 break;
6660 Opcode = X86::AESENCWIDE256KL;
6661 break;
6663 Opcode = X86::AESDECWIDE256KL;
6664 break;
6665 }
6666
6669
6671 if (!selectAddr(Node, Addr, Base, Scale, Index, Disp, Segment))
6672 break;
6673
6674 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM0, Node->getOperand(2),
6676 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM1, Node->getOperand(3),
6678 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM2, Node->getOperand(4),
6680 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM3, Node->getOperand(5),
6682 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM4, Node->getOperand(6),
6684 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM5, Node->getOperand(7),
6686 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM6, Node->getOperand(8),
6688 Chain = CurDAG->getCopyToReg(Chain, dl, X86::XMM7, Node->getOperand(9),
6690
6691 MachineSDNode *Res = CurDAG->getMachineNode(
6692 Opcode, dl, Node->getVTList(),
6693 {Base, Scale, Index, Disp, Segment, Chain, Chain.getValue(1)});
6694 CurDAG->setNodeMemRefs(Res, cast(Node)->getMemOperand());
6695 ReplaceNode(Node, Res);
6696 return;
6697 }
6702 if (Node->getNumValues() == 3)
6703 Glue = Node->getOperand(2);
6705 CurDAG->getCopyFromReg(Chain, dl, Reg, Node->getValueType(0), Glue);
6706 ReplaceNode(Node, Copy.getNode());
6707 return;
6708 }
6709 }
6710
6711 SelectCode(Node);
6712}
6713
6714bool X86DAGToDAGISel::SelectInlineAsmMemoryOperand(
6716 std::vector &OutOps) {
6717 SDValue Op0, Op1, Op2, Op3, Op4;
6718 switch (ConstraintID) {
6719 default:
6721 case InlineAsm::ConstraintCode::o:
6722 case InlineAsm::ConstraintCode::v:
6723 case InlineAsm::ConstraintCode::m:
6724 case InlineAsm::ConstraintCode::X:
6725 case InlineAsm::ConstraintCode::p:
6726 if (!selectAddr(nullptr, Op, Op0, Op1, Op2, Op3, Op4))
6727 return true;
6728 break;
6729 }
6730
6731 OutOps.push_back(Op0);
6732 OutOps.push_back(Op1);
6733 OutOps.push_back(Op2);
6734 OutOps.push_back(Op3);
6735 OutOps.push_back(Op4);
6736 return false;
6737}
6738
6741 std::make_unique(TM, TM.getOptLevel())) {}
6742
6743
6744
6747 return new X86DAGToDAGISelLegacy(TM, OptLevel);
6748}
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis false
#define CASE(ATTRNAME, AANAME,...)
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool isSigned(unsigned int Opcode)
Module.h This file contains the declarations for the Module class.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Promote Memory to Register
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
BaseType
A given derived pointer can have multiple base pointers through phi/selects.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, LoadSDNode *&LoadNode, SDValue &InputChain)
static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N)
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static bool isRIPRelative(const MCInst &MI, const MCInstrInfo &MCII)
Check if the instruction uses RIP relative addressing.
#define FROM_TO(FROM, TO)
#define GET_EGPR_IF_ENABLED(OPC)
static bool isLegalMaskCompare(SDNode *N, const X86Subtarget *Subtarget)
Definition X86ISelDAGToDAG.cpp:624
static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
Definition X86ISelDAGToDAG.cpp:2225
static bool foldMaskAndShiftToExtract(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)
Definition X86ISelDAGToDAG.cpp:2086
static bool needBWI(MVT VT)
Definition X86ISelDAGToDAG.cpp:922
static unsigned getVPTESTMOpc(MVT TestVT, bool IsTestN, bool FoldedLoad, bool FoldedBCast, bool Masked)
Definition X86ISelDAGToDAG.cpp:4898
static bool foldMaskedShiftToBEXTR(SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM, const X86Subtarget &Subtarget)
Definition X86ISelDAGToDAG.cpp:2317
static bool mayUseCarryFlag(X86::CondCode CC)
Definition X86ISelDAGToDAG.cpp:3406
static cl::opt< bool > EnablePromoteAnyextLoad("x86-promote-anyext-load", cl::init(true), cl::desc("Enable promoting aligned anyext load to wider load"), cl::Hidden)
static void moveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load, SDValue Call, SDValue OrigChain)
Replace the original chain operand of the call with load's chain operand and move load below the call...
Definition X86ISelDAGToDAG.cpp:829
#define GET_ND_IF_ENABLED(OPC)
#define VPTESTM_BROADCAST_CASES(SUFFIX)
static cl::opt< bool > AndImmShrink("x86-and-imm-shrink", cl::init(true), cl::desc("Enable setting constant bits to reduce size of mask immediates"), cl::Hidden)
static bool foldMaskedShiftToScaledMask(SelectionDAG &DAG, SDValue N, X86ISelAddressMode &AM)
Definition X86ISelDAGToDAG.cpp:2133
#define VPTESTM_FULL_CASES(SUFFIX)
static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq)
Return true if call address is a load and it can be moved below CALLSEQ_START and the chains leading ...
Definition X86ISelDAGToDAG.cpp:864
static bool isDispSafeForFrameIndexOrRegBase(int64_t Val)
Definition X86ISelDAGToDAG.cpp:1802
static bool isEndbrImm64(uint64_t Imm)
Definition X86ISelDAGToDAG.cpp:901
cl::opt< bool > IndirectBranchTracking("x86-indirect-branch-tracking", cl::init(false), cl::Hidden, cl::desc("Enable X86 indirect branch tracking pass."))
#define GET_ND_IF_ENABLED(OPC)
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned countl_zero() const
The APInt version of std::countl_zero.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
bool isOne() const
Determine if this is a value of 1.
unsigned countr_one() const
Count the number of trailing one bits.
FunctionPass class - This class is used to implement most global optimizations.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Module * getParent()
Get the module that this global value is contained inside of...
LLVM_ABI std::optional< ConstantRange > getAbsoluteSymbolRange() const
If this is an absolute symbol reference, returns the range of the symbol, otherwise returns std::null...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
bool is128BitVector() const
Return true if this is a 128-bit vector type.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool is512BitVector() const
Return true if this is a 512-bit vector type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool is256BitVector() const
Return true if this is a 256-bit vector type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
bool isNonTemporal() const
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
int getNodeId() const
Return the unique node id.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
const SDValue & getOperand(unsigned Num) const
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
iterator_range< user_iterator > users()
op_iterator op_end() const
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isMachineOpcode() const
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getMachineOpcode() const
unsigned getOpcode() const
unsigned getNumOperands() const
SelectionDAGISelPass(std::unique_ptr< SelectionDAGISel > Selector)
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
static int getUninvalidatedNodeId(SDNode *N)
virtual bool runOnMachineFunction(MachineFunction &mf)
static void InvalidateNodeId(SDNode *N)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
static constexpr unsigned MaxRecursionDepth
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
LLVM_ABI void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVM_ABI SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
void RepositionNode(allnodes_iterator Position, SDNode *N)
Move node N in the AllNodes list to be immediately before the given iterator Position.
ilist< SDNode >::iterator allnodes_iterator
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
unsigned getID() const
Return the register class ID number.
bool hasOneUse() const
Return true if there is exactly one use of this value.
X86ISelDAGToDAGPass(X86TargetMachine &TM)
Definition X86ISelDAGToDAG.cpp:6739
size_t getPreallocatedIdForCallSite(const Value *CS)
bool isScalarFPTypeInSSEReg(EVT VT) const
Return true if the specified scalar FP type is computed in an SSE register, not on the X87 floating p...
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ ADD
Simple integer binary arithmetic operators.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ SHL
Shift and rotation operations.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ LOCAL_RECOVER
LOCAL_RECOVER - Represents the llvm.localrecover intrinsic.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
LLVM_ABI bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
LLVM_ABI bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LLVM_ABI bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
@ GlobalBaseReg
The result of the mflr at function entry, used for PIC code.
@ X86
Windows x64, Windows Itanium (IA-64)
@ MO_NO_FLAG
MO_NO_FLAG - No flag for the operand.
@ EVEX
EVEX - Specifies that this instruction use EVEX form which provides syntax support up to 32 512-bit r...
@ VEX
VEX - encoding using 0xC4/0xC5.
@ XOP
XOP - Opcode prefix used by XOP instructions.
@ FST
This instruction implements a truncating store from FP stack slots.
@ CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ CMP
X86 compare and logical compare instructions.
@ BLENDV
Dynamic (non-constant condition) vector blend where only the sign bits of the condition elements are ...
@ STRICT_CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ NT_BRIND
BRIND node with NoTrack prefix.
@ FSETCCM
X86 FP SETCC, similar to above, but with output as an i1 mask and and a version with SAE.
@ FXOR
Bitwise logical XOR of floating point values.
@ BRCOND
X86 conditional branches.
@ CALL
These operations represent an abstract X86 call instruction, which includes a bunch of information.
@ FANDN
Bitwise logical ANDNOT of floating point values.
@ GlobalBaseReg
On Darwin, this node represents the result of the popl at function entry, used for PIC code.
@ FLD
This instruction implements an extending load to FP stack slots.
@ TC_RETURN
Tail call return.
@ FOR
Bitwise logical OR of floating point values.
@ Wrapper
A wrapper node for TargetConstantPool, TargetJumpTable, TargetExternalSymbol, TargetGlobalAddress,...
@ ANDNP
Bitwise Logical AND NOT of Packed FP values.
@ POP_FROM_X87_REG
The same as ISD::CopyFromReg except that this node makes it explicit that it may lower to an x87 FPU ...
@ FAND
Bitwise logical AND of floating point values.
@ CMOV
X86 conditional moves.
@ WrapperRIP
Special wrapper used under X86-64 PIC mode for RIP relative displacements.
int getCondSrcNoFromDesc(const MCInstrDesc &MCID)
Return the source operand # for condition code by MCID.
bool mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget, bool AssumeSingleUse=false)
Check if Op is a load operation that could be folded into some other x86 instruction as a memory oper...
bool isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, bool hasSymbolicDisplacement)
Returns true of the given offset can be fit into displacement field of the instruction.
bool isConstantSplat(SDValue Op, APInt &SplatVal, bool AllowPartialUndefs)
If Op is a constant whose elements are all the same constant or undefined, return true and return the...
initializer< Ty > init(const Ty &Val)
@ User
could "use" a pointer
NodeAddr< UseNode * > Use
NodeAddr< NodeBase * > Node
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
FunctionAddr VTableAddr Value
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
decltype(auto) dyn_cast(const From &Val)
dyn_cast - Return the argument parameter cast to the specified type.
bool isa_and_nonnull(const Y &Val)
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
unsigned M1(unsigned Val)
auto dyn_cast_or_null(const Y &Val)
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
FunctionPass * createX86ISelDag(X86TargetMachine &TM, CodeGenOptLevel OptLevel)
This pass converts a legalized DAG into a X86-specific DAG, ready for instruction scheduling.
Definition X86ISelDAGToDAG.cpp:6745
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
CodeGenOptLevel
Code generation optimization level.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa - Return true if the parameter to the template is an instance of one of the template type argu...
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
unsigned M0(unsigned Val)
decltype(auto) cast(const From &Val)
cast - Return the argument parameter cast to the specified type.
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
bool isVector() const
Return true if this is a vector value type.
bool is256BitVector() const
Return true if this is a 256-bit vector type.
bool isConstant() const
Returns true if we know the value of all bits.
LLVM_ABI unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
bool hasNoUnsignedWrap() const