LLVM: lib/Target/X86/X86FastISel.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
36#include "llvm/IR/IntrinsicsX86.h"
43using namespace llvm;
44
45namespace {
46
47class X86FastISel final : public FastISel {
48
49
50 const X86Subtarget *Subtarget;
51
52public:
53 explicit X86FastISel(FunctionLoweringInfo &funcInfo,
54 const TargetLibraryInfo *libInfo)
55 : FastISel(funcInfo, libInfo) {
56 Subtarget = &funcInfo.MF->getSubtarget();
57 }
58
59 bool fastSelectInstruction(const Instruction *I) override;
60
61
62
63
64
65 bool tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,
66 const LoadInst *LI) override;
67
68 bool fastLowerArguments() override;
69 bool fastLowerCall(CallLoweringInfo &CLI) override;
70 bool fastLowerIntrinsicCall(const IntrinsicInst *II) override;
71
72#include "X86GenFastISel.inc"
73
74private:
75 bool X86FastEmitCompare(const Value *LHS, const Value *RHS, EVT VT,
77
78 bool X86FastEmitLoad(MVT VT, X86AddressMode &AM, MachineMemOperand *MMO,
79 Register &ResultReg, unsigned Alignment = 1);
80
81 bool X86FastEmitStore(EVT VT, const Value *Val, X86AddressMode &AM,
82 MachineMemOperand *MMO = nullptr, bool Aligned = false);
83 bool X86FastEmitStore(EVT VT, Register ValReg, X86AddressMode &AM,
84 MachineMemOperand *MMO = nullptr, bool Aligned = false);
85
88
90 bool X86SelectCallAddress(const Value *V, X86AddressMode &AM);
91
92 bool X86SelectLoad(const Instruction *I);
93
94 bool X86SelectStore(const Instruction *I);
95
96 bool X86SelectRet(const Instruction *I);
97
98 bool X86SelectCmp(const Instruction *I);
99
100 bool X86SelectZExt(const Instruction *I);
101
102 bool X86SelectSExt(const Instruction *I);
103
104 bool X86SelectBranch(const Instruction *I);
105
106 bool X86SelectShift(const Instruction *I);
107
108 bool X86SelectDivRem(const Instruction *I);
109
110 bool X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I);
111
112 bool X86FastEmitSSESelect(MVT RetVT, const Instruction *I);
113
114 bool X86FastEmitPseudoSelect(MVT RetVT, const Instruction *I);
115
116 bool X86SelectSelect(const Instruction *I);
117
118 bool X86SelectTrunc(const Instruction *I);
119
120 bool X86SelectFPExtOrFPTrunc(const Instruction *I, unsigned Opc,
121 const TargetRegisterClass *RC);
122
123 bool X86SelectFPExt(const Instruction *I);
124 bool X86SelectFPTrunc(const Instruction *I);
125 bool X86SelectSIToFP(const Instruction *I);
126 bool X86SelectUIToFP(const Instruction *I);
127 bool X86SelectIntToFP(const Instruction *I, bool IsSigned);
128 bool X86SelectBitCast(const Instruction *I);
129
130 const X86InstrInfo *getInstrInfo() const {
131 return Subtarget->getInstrInfo();
132 }
133 const X86TargetMachine *getTargetMachine() const {
134 return static_cast<const X86TargetMachine *>(&TM);
135 }
136
137 bool handleConstantAddresses(const Value *V, X86AddressMode &AM);
138
139 Register X86MaterializeInt(const ConstantInt *CI, MVT VT);
140 Register X86MaterializeFP(const ConstantFP *CFP, MVT VT);
141 Register X86MaterializeGV(const GlobalValue *GV, MVT VT);
142 Register fastMaterializeConstant(const Constant *C) override;
143
144 Register fastMaterializeAlloca(const AllocaInst *C) override;
145
146 Register fastMaterializeFloatZero(const ConstantFP *CF) override;
147
148
149
150 bool isScalarFPTypeInSSEReg(EVT VT) const {
151 return (VT == MVT::f64 && Subtarget->hasSSE2()) ||
152 (VT == MVT::f32 && Subtarget->hasSSE1()) || VT == MVT::f16;
153 }
154
155 bool isTypeLegal(Type *Ty, MVT &VT, bool AllowI1 = false);
156
157 bool IsMemcpySmall(uint64_t Len);
158
159 bool TryEmitSmallMemcpy(X86AddressMode DestAM,
160 X86AddressMode SrcAM, uint64_t Len);
161
162 bool foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I,
164
165 const MachineInstrBuilder &addFullAddress(const MachineInstrBuilder &MIB,
166 X86AddressMode &AM);
167
168 Register fastEmitInst_rrrr(unsigned MachineInstOpcode,
169 const TargetRegisterClass *RC, Register Op0,
171};
172
173}
174
175static std::pair<unsigned, bool>
177 unsigned CC;
178 bool NeedSwap = false;
179
180
181
182
183
184
185
186
187
188
205 }
206
207 return std::make_pair(CC, NeedSwap);
208}
209
210
211
212
213
217
221 return ::addFullAddress(MIB, AM);
222}
223
224
225
226bool X86FastISel::foldX86XALUIntrinsic(X86::CondCode &CC, const Instruction *I,
229 return false;
230
233 return false;
234
236 MVT RetVT;
238 Type *RetTy =
240 if (!isTypeLegal(RetTy, RetVT))
241 return false;
242
243 if (RetVT != MVT::i32 && RetVT != MVT::i64)
244 return false;
245
247 switch (II->getIntrinsicID()) {
248 default: return false;
249 case Intrinsic::sadd_with_overflow:
250 case Intrinsic::ssub_with_overflow:
251 case Intrinsic::smul_with_overflow:
252 case Intrinsic::umul_with_overflow: TmpCC = X86::COND_O; break;
253 case Intrinsic::uadd_with_overflow:
254 case Intrinsic::usub_with_overflow: TmpCC = X86::COND_B; break;
255 }
256
257
258 if (II->getParent() != I->getParent())
259 return false;
260
261
264 for (auto Itr = std::prev(Start); Itr != End; --Itr) {
265
266
268 return false;
269
270
272 if (EVI->getAggregateOperand() != II)
273 return false;
274 }
275
276
277
278 auto HasPhis = [](const BasicBlock *Succ) { return !Succ->phis().empty(); };
280 return false;
281
282
283
284 if (llvm::any_of(I->operands(), [](Value *V) { return isa(V); }))
285 return false;
286
287 CC = TmpCC;
288 return true;
289}
290
291bool X86FastISel::isTypeLegal(Type *Ty, MVT &VT, bool AllowI1) {
292 EVT evt = TLI.getValueType(DL, Ty, true);
293 if (evt == MVT::Other || !evt.isSimple())
294
295 return false;
296
298
299
300 if (VT == MVT::f64 && !Subtarget->hasSSE2())
301 return false;
302 if (VT == MVT::f32 && !Subtarget->hasSSE1())
303 return false;
304
305 if (VT == MVT::f80)
306 return false;
307
308
309
310
311 return (AllowI1 && VT == MVT::i1) || TLI.isTypeLegal(VT);
312}
313
314
315
316
317bool X86FastISel::X86FastEmitLoad(MVT VT, X86AddressMode &AM,
318 MachineMemOperand *MMO, Register &ResultReg,
319 unsigned Alignment) {
320 bool HasSSE1 = Subtarget->hasSSE1();
321 bool HasSSE2 = Subtarget->hasSSE2();
322 bool HasSSE41 = Subtarget->hasSSE41();
323 bool HasAVX = Subtarget->hasAVX();
324 bool HasAVX2 = Subtarget->hasAVX2();
325 bool HasAVX512 = Subtarget->hasAVX512();
326 bool HasVLX = Subtarget->hasVLX();
327 bool IsNonTemporal = MMO && MMO->isNonTemporal();
328
329
330 if (VT == MVT::i1)
331 VT = MVT::i8;
332
333
334 unsigned Opc = 0;
336 default: return false;
337 case MVT::i8:
338 Opc = X86::MOV8rm;
339 break;
340 case MVT::i16:
341 Opc = X86::MOV16rm;
342 break;
343 case MVT::i32:
344 Opc = X86::MOV32rm;
345 break;
346 case MVT::i64:
347
348 Opc = X86::MOV64rm;
349 break;
350 case MVT::f32:
351 Opc = HasAVX512 ? X86::VMOVSSZrm_alt
352 : HasAVX ? X86::VMOVSSrm_alt
353 : HasSSE1 ? X86::MOVSSrm_alt
354 : X86::LD_Fp32m;
355 break;
356 case MVT::f64:
357 Opc = HasAVX512 ? X86::VMOVSDZrm_alt
358 : HasAVX ? X86::VMOVSDrm_alt
359 : HasSSE2 ? X86::MOVSDrm_alt
360 : X86::LD_Fp64m;
361 break;
362 case MVT::f80:
363
364 return false;
365 case MVT::v4f32:
366 if (IsNonTemporal && Alignment >= 16 && HasSSE41)
367 Opc = HasVLX ? X86::VMOVNTDQAZ128rm :
368 HasAVX ? X86::VMOVNTDQArm : X86::MOVNTDQArm;
369 else if (Alignment >= 16)
370 Opc = HasVLX ? X86::VMOVAPSZ128rm :
371 HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm;
372 else
373 Opc = HasVLX ? X86::VMOVUPSZ128rm :
374 HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm;
375 break;
376 case MVT::v2f64:
377 if (IsNonTemporal && Alignment >= 16 && HasSSE41)
378 Opc = HasVLX ? X86::VMOVNTDQAZ128rm :
379 HasAVX ? X86::VMOVNTDQArm : X86::MOVNTDQArm;
380 else if (Alignment >= 16)
381 Opc = HasVLX ? X86::VMOVAPDZ128rm :
382 HasAVX ? X86::VMOVAPDrm : X86::MOVAPDrm;
383 else
384 Opc = HasVLX ? X86::VMOVUPDZ128rm :
385 HasAVX ? X86::VMOVUPDrm : X86::MOVUPDrm;
386 break;
387 case MVT::v4i32:
388 case MVT::v2i64:
389 case MVT::v8i16:
390 case MVT::v16i8:
391 if (IsNonTemporal && Alignment >= 16 && HasSSE41)
392 Opc = HasVLX ? X86::VMOVNTDQAZ128rm :
393 HasAVX ? X86::VMOVNTDQArm : X86::MOVNTDQArm;
394 else if (Alignment >= 16)
395 Opc = HasVLX ? X86::VMOVDQA64Z128rm :
396 HasAVX ? X86::VMOVDQArm : X86::MOVDQArm;
397 else
398 Opc = HasVLX ? X86::VMOVDQU64Z128rm :
399 HasAVX ? X86::VMOVDQUrm : X86::MOVDQUrm;
400 break;
401 case MVT::v8f32:
403 if (IsNonTemporal && Alignment >= 32 && HasAVX2)
404 Opc = HasVLX ? X86::VMOVNTDQAZ256rm : X86::VMOVNTDQAYrm;
405 else if (IsNonTemporal && Alignment >= 16)
406 return false;
407 else if (Alignment >= 32)
408 Opc = HasVLX ? X86::VMOVAPSZ256rm : X86::VMOVAPSYrm;
409 else
410 Opc = HasVLX ? X86::VMOVUPSZ256rm : X86::VMOVUPSYrm;
411 break;
412 case MVT::v4f64:
414 if (IsNonTemporal && Alignment >= 32 && HasAVX2)
415 Opc = HasVLX ? X86::VMOVNTDQAZ256rm : X86::VMOVNTDQAYrm;
416 else if (IsNonTemporal && Alignment >= 16)
417 return false;
418 else if (Alignment >= 32)
419 Opc = HasVLX ? X86::VMOVAPDZ256rm : X86::VMOVAPDYrm;
420 else
421 Opc = HasVLX ? X86::VMOVUPDZ256rm : X86::VMOVUPDYrm;
422 break;
423 case MVT::v8i32:
424 case MVT::v4i64:
425 case MVT::v16i16:
426 case MVT::v32i8:
428 if (IsNonTemporal && Alignment >= 32 && HasAVX2)
429 Opc = HasVLX ? X86::VMOVNTDQAZ256rm : X86::VMOVNTDQAYrm;
430 else if (IsNonTemporal && Alignment >= 16)
431 return false;
432 else if (Alignment >= 32)
433 Opc = HasVLX ? X86::VMOVDQA64Z256rm : X86::VMOVDQAYrm;
434 else
435 Opc = HasVLX ? X86::VMOVDQU64Z256rm : X86::VMOVDQUYrm;
436 break;
437 case MVT::v16f32:
439 if (IsNonTemporal && Alignment >= 64)
440 Opc = X86::VMOVNTDQAZrm;
441 else
442 Opc = (Alignment >= 64) ? X86::VMOVAPSZrm : X86::VMOVUPSZrm;
443 break;
444 case MVT::v8f64:
446 if (IsNonTemporal && Alignment >= 64)
447 Opc = X86::VMOVNTDQAZrm;
448 else
449 Opc = (Alignment >= 64) ? X86::VMOVAPDZrm : X86::VMOVUPDZrm;
450 break;
451 case MVT::v8i64:
452 case MVT::v16i32:
453 case MVT::v32i16:
454 case MVT::v64i8:
456
457
458 if (IsNonTemporal && Alignment >= 64)
459 Opc = X86::VMOVNTDQAZrm;
460 else
461 Opc = (Alignment >= 64) ? X86::VMOVDQA64Zrm : X86::VMOVDQU64Zrm;
462 break;
463 }
464
465 const TargetRegisterClass *RC = TLI.getRegClassFor(VT);
466
467 ResultReg = createResultReg(RC);
468 MachineInstrBuilder MIB =
469 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), ResultReg);
471 if (MMO)
473 return true;
474}
475
476
477
478
479
480bool X86FastISel::X86FastEmitStore(EVT VT, Register ValReg, X86AddressMode &AM,
481 MachineMemOperand *MMO, bool Aligned) {
482 bool HasSSE1 = Subtarget->hasSSE1();
483 bool HasSSE2 = Subtarget->hasSSE2();
484 bool HasSSE4A = Subtarget->hasSSE4A();
485 bool HasAVX = Subtarget->hasAVX();
486 bool HasAVX512 = Subtarget->hasAVX512();
487 bool HasVLX = Subtarget->hasVLX();
488 bool IsNonTemporal = MMO && MMO->isNonTemporal();
489
490
491 unsigned Opc = 0;
493 case MVT::f80:
494 default: return false;
495 case MVT::i1: {
496
497 Register AndResult = createResultReg(&X86::GR8RegClass);
498 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
499 TII.get(X86::AND8ri), AndResult)
501 ValReg = AndResult;
502 [[fallthrough]];
503 }
504 case MVT::i8: Opc = X86::MOV8mr; break;
505 case MVT::i16: Opc = X86::MOV16mr; break;
506 case MVT::i32:
507 Opc = (IsNonTemporal && HasSSE2) ? X86::MOVNTImr : X86::MOV32mr;
508 break;
509 case MVT::i64:
510
511 Opc = (IsNonTemporal && HasSSE2) ? X86::MOVNTI_64mr : X86::MOV64mr;
512 break;
513 case MVT::f32:
514 if (HasSSE1) {
515 if (IsNonTemporal && HasSSE4A)
516 Opc = X86::MOVNTSS;
517 else
518 Opc = HasAVX512 ? X86::VMOVSSZmr :
519 HasAVX ? X86::VMOVSSmr : X86::MOVSSmr;
520 } else
521 Opc = X86::ST_Fp32m;
522 break;
523 case MVT::f64:
524 if (HasSSE2) {
525 if (IsNonTemporal && HasSSE4A)
526 Opc = X86::MOVNTSD;
527 else
528 Opc = HasAVX512 ? X86::VMOVSDZmr :
529 HasAVX ? X86::VMOVSDmr : X86::MOVSDmr;
530 } else
531 Opc = X86::ST_Fp64m;
532 break;
533 case MVT::x86mmx:
534 Opc = (IsNonTemporal && HasSSE1) ? X86::MMX_MOVNTQmr : X86::MMX_MOVQ64mr;
535 break;
536 case MVT::v4f32:
538 if (IsNonTemporal)
539 Opc = HasVLX ? X86::VMOVNTPSZ128mr :
540 HasAVX ? X86::VMOVNTPSmr : X86::MOVNTPSmr;
541 else
542 Opc = HasVLX ? X86::VMOVAPSZ128mr :
543 HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr;
544 } else
545 Opc = HasVLX ? X86::VMOVUPSZ128mr :
546 HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr;
547 break;
548 case MVT::v2f64:
550 if (IsNonTemporal)
551 Opc = HasVLX ? X86::VMOVNTPDZ128mr :
552 HasAVX ? X86::VMOVNTPDmr : X86::MOVNTPDmr;
553 else
554 Opc = HasVLX ? X86::VMOVAPDZ128mr :
555 HasAVX ? X86::VMOVAPDmr : X86::MOVAPDmr;
556 } else
557 Opc = HasVLX ? X86::VMOVUPDZ128mr :
558 HasAVX ? X86::VMOVUPDmr : X86::MOVUPDmr;
559 break;
560 case MVT::v4i32:
561 case MVT::v2i64:
562 case MVT::v8i16:
563 case MVT::v16i8:
565 if (IsNonTemporal)
566 Opc = HasVLX ? X86::VMOVNTDQZ128mr :
567 HasAVX ? X86::VMOVNTDQmr : X86::MOVNTDQmr;
568 else
569 Opc = HasVLX ? X86::VMOVDQA64Z128mr :
570 HasAVX ? X86::VMOVDQAmr : X86::MOVDQAmr;
571 } else
572 Opc = HasVLX ? X86::VMOVDQU64Z128mr :
573 HasAVX ? X86::VMOVDQUmr : X86::MOVDQUmr;
574 break;
575 case MVT::v8f32:
578 if (IsNonTemporal)
579 Opc = HasVLX ? X86::VMOVNTPSZ256mr : X86::VMOVNTPSYmr;
580 else
581 Opc = HasVLX ? X86::VMOVAPSZ256mr : X86::VMOVAPSYmr;
582 } else
583 Opc = HasVLX ? X86::VMOVUPSZ256mr : X86::VMOVUPSYmr;
584 break;
585 case MVT::v4f64:
588 if (IsNonTemporal)
589 Opc = HasVLX ? X86::VMOVNTPDZ256mr : X86::VMOVNTPDYmr;
590 else
591 Opc = HasVLX ? X86::VMOVAPDZ256mr : X86::VMOVAPDYmr;
592 } else
593 Opc = HasVLX ? X86::VMOVUPDZ256mr : X86::VMOVUPDYmr;
594 break;
595 case MVT::v8i32:
596 case MVT::v4i64:
597 case MVT::v16i16:
598 case MVT::v32i8:
601 if (IsNonTemporal)
602 Opc = HasVLX ? X86::VMOVNTDQZ256mr : X86::VMOVNTDQYmr;
603 else
604 Opc = HasVLX ? X86::VMOVDQA64Z256mr : X86::VMOVDQAYmr;
605 } else
606 Opc = HasVLX ? X86::VMOVDQU64Z256mr : X86::VMOVDQUYmr;
607 break;
608 case MVT::v16f32:
611 Opc = IsNonTemporal ? X86::VMOVNTPSZmr : X86::VMOVAPSZmr;
612 else
613 Opc = X86::VMOVUPSZmr;
614 break;
615 case MVT::v8f64:
618 Opc = IsNonTemporal ? X86::VMOVNTPDZmr : X86::VMOVAPDZmr;
619 } else
620 Opc = X86::VMOVUPDZmr;
621 break;
622 case MVT::v8i64:
623 case MVT::v16i32:
624 case MVT::v32i16:
625 case MVT::v64i8:
627
628
630 Opc = IsNonTemporal ? X86::VMOVNTDQZmr : X86::VMOVDQA64Zmr;
631 else
632 Opc = X86::VMOVDQU64Zmr;
633 break;
634 }
635
637
638
639
640
641
642
644 MachineInstrBuilder MIB =
645 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, Desc);
647 if (MMO)
649
650 return true;
651}
652
653bool X86FastISel::X86FastEmitStore(EVT VT, const Value *Val,
654 X86AddressMode &AM,
655 MachineMemOperand *MMO, bool Aligned) {
656
659
660
662 unsigned Opc = 0;
665 default: break;
666 case MVT::i1:
668 [[fallthrough]];
669 case MVT::i8: Opc = X86::MOV8mi; break;
670 case MVT::i16: Opc = X86::MOV16mi; break;
671 case MVT::i32: Opc = X86::MOV32mi; break;
672 case MVT::i64:
673
674 if (isInt<32>(CI->getSExtValue()))
675 Opc = X86::MOV64mi32;
676 break;
677 }
678
679 if (Opc) {
680 MachineInstrBuilder MIB =
681 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc));
683 : CI->getZExtValue());
684 if (MMO)
686 return true;
687 }
688 }
689
690 Register ValReg = getRegForValue(Val);
691 if (!ValReg)
692 return false;
693
694 return X86FastEmitStore(VT, ValReg, AM, MMO, Aligned);
695}
696
697
698
699
701 EVT SrcVT, Register &ResultReg) {
703 if (!RR)
704 return false;
705
706 ResultReg = RR;
707 return true;
708}
709
710bool X86FastISel::handleConstantAddresses(const Value *V, X86AddressMode &AM) {
711
713
716 return false;
717
718
719 if (TM.isLargeGlobalValue(GV))
720 return false;
721
722
723 if (GV->isThreadLocal())
724 return false;
725
726
727 if (GV->isAbsoluteSymbolRef())
728 return false;
729
730
731
732
733 if (!Subtarget->isPICStyleRIPRel() ||
735
736 AM.GV = GV;
737
738
739 unsigned char GVFlags = Subtarget->classifyGlobalReference(GV);
740
741
743
744 AM.Base.Reg = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
745 }
746
747
748
750 if (Subtarget->isPICStyleRIPRel()) {
751
752
755 }
757 return true;
758 }
759
760
761
762 DenseMap<const Value *, Register>::iterator I = LocalValueMap.find(V);
764 if (I != LocalValueMap.end() && I->second) {
765 LoadReg = I->second;
766 } else {
767
768 unsigned Opc = 0;
769 const TargetRegisterClass *RC = nullptr;
770 X86AddressMode StubAM;
772 StubAM.GV = GV;
774
775
776 SavePoint SaveInsertPt = enterLocalValueArea();
777
778 if (TLI.getPointerTy(DL) == MVT::i64) {
779 Opc = X86::MOV64rm;
780 RC = &X86::GR64RegClass;
781 } else {
782 Opc = X86::MOV32rm;
783 RC = &X86::GR32RegClass;
784 }
785
786 if (Subtarget->isPICStyleRIPRel() || GVFlags == X86II::MO_GOTPCREL ||
788 StubAM.Base.Reg = X86::RIP;
789
790 LoadReg = createResultReg(RC);
791 MachineInstrBuilder LoadMI =
792 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), LoadReg);
794
795
796 leaveLocalValueArea(SaveInsertPt);
797
798
799 LocalValueMap[V] = LoadReg;
800 }
801
802
803
805 AM.GV = nullptr;
806 return true;
807 }
808 }
809
810
811 if (!AM.GV || !Subtarget->isPICStyleRIPRel()) {
813 AM.Base.Reg = getRegForValue(V);
815 }
817 assert(AM.Scale == 1 && "Scale with no index!");
818 AM.IndexReg = getRegForValue(V);
820 }
821 }
822
823 return false;
824}
825
826
827
828bool X86FastISel::X86SelectAddress(const Value *V, X86AddressMode &AM) {
830redo_gep:
832 unsigned Opcode = Instruction::UserOp1;
834
835
836
837 if (FuncInfo.StaticAllocaMap.count(static_cast<const AllocaInst *>(V)) ||
838 FuncInfo.getMBB(I->getParent()) == FuncInfo.MBB) {
839 Opcode = I->getOpcode();
841 }
843 Opcode = C->getOpcode();
845 }
846
848 if (Ty->getAddressSpace() > 255)
849
850
851 return false;
852
853 switch (Opcode) {
854 default: break;
855 case Instruction::BitCast:
856
858
859 case Instruction::IntToPtr:
860
861 if (TLI.getValueType(DL, U->getOperand(0)->getType()) ==
862 TLI.getPointerTy(DL))
864 break;
865
866 case Instruction::PtrToInt:
867
868 if (TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL))
870 break;
871
872 case Instruction::Alloca: {
873
875 DenseMap<const AllocaInst *, int>::iterator SI =
876 FuncInfo.StaticAllocaMap.find(A);
877 if (SI != FuncInfo.StaticAllocaMap.end()) {
880 return true;
881 }
882 break;
883 }
884
885 case Instruction::Add: {
886
888 uint64_t Disp = (int32_t)AM.Disp + (uint64_t)CI->getSExtValue();
889
891 AM.Disp = (uint32_t)Disp;
893 }
894 }
895 break;
896 }
897
898 case Instruction::GetElementPtr: {
899 X86AddressMode SavedAM = AM;
900
901
902 uint64_t Disp = (int32_t)AM.Disp;
904 unsigned Scale = AM.Scale;
905 MVT PtrVT = TLI.getValueType(DL, U->getType()).getSimpleVT();
906
908
909
911 i != e; ++i, ++GTI) {
914 const StructLayout *SL = DL.getStructLayout(STy);
916 continue;
917 }
918
919
920
922 for (;;) {
924
925 Disp += CI->getSExtValue() * S;
926 break;
927 }
928 if (canFoldAddIntoGEP(U, Op)) {
929
930 ConstantInt *CI =
933
935 continue;
936 }
937 if (!IndexReg && (!AM.GV || !Subtarget->isPICStyleRIPRel()) &&
938 (S == 1 || S == 2 || S == 4 || S == 8)) {
939
940 Scale = S;
941 IndexReg = getRegForGEPIndex(PtrVT, Op);
942 if (!IndexReg)
943 return false;
944 break;
945 }
946
947 goto unsupported_gep;
948 }
949 }
950
951
953 break;
954
956 AM.Scale = Scale;
957 AM.Disp = (uint32_t)Disp;
959
960 if (const GetElementPtrInst *GEP =
962
963
965 goto redo_gep;
967 return true;
968 }
969
970
971
972 AM = SavedAM;
973
975 if (handleConstantAddresses(I, AM))
976 return true;
977
978 return false;
979 unsupported_gep:
980
981 break;
982 }
983 }
984
985 return handleConstantAddresses(V, AM);
986}
987
988
989
990bool X86FastISel::X86SelectCallAddress(const Value *V, X86AddressMode &AM) {
992 unsigned Opcode = Instruction::UserOp1;
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017 bool InMBB = true;
1018 if (I) {
1019 Opcode = I->getOpcode();
1021 InMBB = I->getParent() == FuncInfo.MBB->getBasicBlock();
1023 Opcode = C->getOpcode();
1025 }
1026
1027 switch (Opcode) {
1028 default: break;
1029 case Instruction::BitCast:
1030
1031 if (InMBB)
1032 return X86SelectCallAddress(U->getOperand(0), AM);
1033 break;
1034
1035 case Instruction::IntToPtr:
1036
1037 if (InMBB &&
1038 TLI.getValueType(DL, U->getOperand(0)->getType()) ==
1039 TLI.getPointerTy(DL))
1040 return X86SelectCallAddress(U->getOperand(0), AM);
1041 break;
1042
1043 case Instruction::PtrToInt:
1044
1045 if (InMBB && TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL))
1046 return X86SelectCallAddress(U->getOperand(0), AM);
1047 break;
1048 }
1049
1050
1052
1055 return false;
1056
1057
1058 if (Subtarget->isPICStyleRIPRel() &&
1060 return false;
1061
1062
1064 if (GVar->isThreadLocal())
1065 return false;
1066
1067
1068 AM.GV = GV;
1069
1070
1071
1072
1073 if (Subtarget->isPICStyleRIPRel()) {
1074
1075
1078 } else {
1079 AM.GVOpFlags = Subtarget->classifyLocalReference(nullptr);
1080 }
1081
1082 return true;
1083 }
1084
1085
1086 if (!AM.GV || !Subtarget->isPICStyleRIPRel()) {
1087 auto GetCallRegForValue = [this](const Value *V) {
1089
1090
1091 if (Reg && Subtarget->isTarget64BitILP32()) {
1092 Register CopyReg = createResultReg(&X86::GR32RegClass);
1093 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::MOV32rr),
1094 CopyReg)
1096
1097 Register ExtReg = createResultReg(&X86::GR64RegClass);
1098 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1099 TII.get(TargetOpcode::SUBREG_TO_REG), ExtReg)
1102 .addImm(X86::sub_32bit);
1103 Reg = ExtReg;
1104 }
1105
1106 return Reg;
1107 };
1108
1110 AM.Base.Reg = GetCallRegForValue(V);
1112 }
1114 assert(AM.Scale == 1 && "Scale with no index!");
1115 AM.IndexReg = GetCallRegForValue(V);
1117 }
1118 }
1119
1120 return false;
1121}
1122
1123
1124
1125bool X86FastISel::X86SelectStore(const Instruction *I) {
1126
1128
1130 return false;
1131
1132 const Value *PtrV = I->getOperand(1);
1133 if (TLI.supportSwiftError()) {
1134
1135
1137 if (Arg->hasSwiftErrorAttr())
1138 return false;
1139 }
1140
1142 if (Alloca->isSwiftError())
1143 return false;
1144 }
1145 }
1146
1149
1150 MVT VT;
1151 if (!isTypeLegal(Val->getType(), VT, true))
1152 return false;
1153
1155 Align ABIAlignment = DL.getABITypeAlign(Val->getType());
1156 bool Aligned = Alignment >= ABIAlignment;
1157
1158 X86AddressMode AM;
1160 return false;
1161
1162 return X86FastEmitStore(VT, Val, AM, createMachineMemOperandFor(I), Aligned);
1163}
1164
1165
1166bool X86FastISel::X86SelectRet(const Instruction *I) {
1168 const Function &F = *I->getParent()->getParent();
1169 const X86MachineFunctionInfo *X86MFInfo =
1170 FuncInfo.MF->getInfo();
1171
1172 if (!FuncInfo.CanLowerReturn)
1173 return false;
1174
1175 if (TLI.supportSwiftError() &&
1176 F.getAttributes().hasAttrSomewhere(Attribute::SwiftError))
1177 return false;
1178
1179 if (TLI.supportSplitCSR(FuncInfo.MF))
1180 return false;
1181
1182 CallingConv::ID CC = F.getCallingConv();
1183 if (CC != CallingConv::C &&
1184 CC != CallingConv::Fast &&
1185 CC != CallingConv::Tail &&
1186 CC != CallingConv::SwiftTail &&
1187 CC != CallingConv::X86_FastCall &&
1188 CC != CallingConv::X86_StdCall &&
1189 CC != CallingConv::X86_ThisCall &&
1190 CC != CallingConv::X86_64_SysV &&
1191 CC != CallingConv::Win64)
1192 return false;
1193
1194
1196 return false;
1197
1198
1199
1200 if ((CC == CallingConv::Fast && TM.Options.GuaranteedTailCallOpt) ||
1201 CC == CallingConv::Tail || CC == CallingConv::SwiftTail)
1202 return false;
1203
1204
1205 if (F.isVarArg())
1206 return false;
1207
1208
1210
1213 GetReturnInfo(CC, F.getReturnType(), F.getAttributes(), Outs, TLI, DL);
1214
1215
1217 CCState CCInfo(CC, F.isVarArg(), *FuncInfo.MF, ValLocs, I->getContext());
1218 CCInfo.AnalyzeReturn(Outs, RetCC_X86);
1219
1222 if ()
1223 return false;
1224
1225
1226 if (ValLocs.size() != 1)
1227 return false;
1228
1229 CCValAssign &VA = ValLocs[0];
1230
1231
1233 return false;
1234
1236 return false;
1237
1238
1239
1241 return false;
1242
1244 EVT SrcVT = TLI.getValueType(DL, RV->getType());
1246
1247 if (SrcVT != DstVT) {
1248 if (SrcVT != MVT::i1 && SrcVT != MVT::i8 && SrcVT != MVT::i16)
1249 return false;
1250
1251 if (!Outs[0].Flags.isZExt() && !Outs[0].Flags.isSExt())
1252 return false;
1253
1254 if (SrcVT == MVT::i1) {
1255 if (Outs[0].Flags.isSExt())
1256 return false;
1257 SrcReg = fastEmitZExtFromI1(MVT::i8, SrcReg);
1258 SrcVT = MVT::i8;
1259 }
1260 if (SrcVT != DstVT) {
1261 unsigned Op =
1263 SrcReg =
1265 }
1266 }
1267
1268
1270 const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
1271
1272 if (!SrcRC->contains(DstReg))
1273 return false;
1274 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1275 TII.get(TargetOpcode::COPY), DstReg).addReg(SrcReg);
1276
1277
1279 }
1280
1281
1282
1283
1284
1285
1286
1287
1288 if (F.hasStructRetAttr() && CC != CallingConv::Swift &&
1289 CC != CallingConv::SwiftTail) {
1292 "SRetReturnReg should have been set in LowerFormalArguments()!");
1293 Register RetReg = Subtarget->isTarget64BitLP64() ? X86::RAX : X86::EAX;
1294 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1297 }
1298
1299
1300 MachineInstrBuilder MIB;
1302 MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1303 TII.get(Subtarget->is64Bit() ? X86::RETI64 : X86::RETI32))
1305 } else {
1306 MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1307 TII.get(Subtarget->is64Bit() ? X86::RET64 : X86::RET32));
1308 }
1311 return true;
1312}
1313
1314
1315
1316bool X86FastISel::X86SelectLoad(const Instruction *I) {
1318
1319
1321 return false;
1322
1323 const Value *SV = I->getOperand(0);
1324 if (TLI.supportSwiftError()) {
1325
1326
1328 if (Arg->hasSwiftErrorAttr())
1329 return false;
1330 }
1331
1333 if (Alloca->isSwiftError())
1334 return false;
1335 }
1336 }
1337
1338 MVT VT;
1339 if (!isTypeLegal(LI->getType(), VT, true))
1340 return false;
1341
1343
1344 X86AddressMode AM;
1346 return false;
1347
1349 if (!X86FastEmitLoad(VT, AM, createMachineMemOperandFor(LI), ResultReg,
1351 return false;
1352
1353 updateValueMap(I, ResultReg);
1354 return true;
1355}
1356
1358 bool HasAVX512 = Subtarget->hasAVX512();
1359 bool HasAVX = Subtarget->hasAVX();
1360 bool HasSSE1 = Subtarget->hasSSE1();
1361 bool HasSSE2 = Subtarget->hasSSE2();
1362
1364 default: return 0;
1365 case MVT::i8: return X86::CMP8rr;
1366 case MVT::i16: return X86::CMP16rr;
1367 case MVT::i32: return X86::CMP32rr;
1368 case MVT::i64: return X86::CMP64rr;
1369 case MVT::f32:
1370 return HasAVX512 ? X86::VUCOMISSZrr
1371 : HasAVX ? X86::VUCOMISSrr
1372 : HasSSE1 ? X86::UCOMISSrr
1373 : 0;
1374 case MVT::f64:
1375 return HasAVX512 ? X86::VUCOMISDZrr
1376 : HasAVX ? X86::VUCOMISDrr
1377 : HasSSE2 ? X86::UCOMISDrr
1378 : 0;
1379 }
1380}
1381
1382
1383
1386
1387 default:
1388 return 0;
1389 case MVT::i8:
1390 return X86::CMP8ri;
1391 case MVT::i16:
1392 return X86::CMP16ri;
1393 case MVT::i32:
1394 return X86::CMP32ri;
1395 case MVT::i64:
1396
1397
1399 }
1400}
1401
1402bool X86FastISel::X86FastEmitCompare(const Value *Op0, const Value *Op1, EVT VT,
1404 Register Op0Reg = getRegForValue(Op0);
1405 if (!Op0Reg)
1406 return false;
1407
1408
1411
1412
1413
1414
1417 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, CurMIMD, TII.get(CompareImmOpc))
1419 .addImm(Op1C->getSExtValue());
1420 return true;
1421 }
1422 }
1423
1425 if (CompareOpc == 0) return false;
1426
1427 Register Op1Reg = getRegForValue(Op1);
1428 if (!Op1Reg)
1429 return false;
1430 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, CurMIMD, TII.get(CompareOpc))
1433
1434 return true;
1435}
1436
1437bool X86FastISel::X86SelectCmp(const Instruction *I) {
1439
1440 MVT VT;
1441 if (!isTypeLegal(I->getOperand(0)->getType(), VT))
1442 return false;
1443
1444
1446 return false;
1447
1448
1451 switch (Predicate) {
1452 default: break;
1454 ResultReg = createResultReg(&X86::GR32RegClass);
1455 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::MOV32r0),
1456 ResultReg);
1457 ResultReg = fastEmitInst_extractsubreg(MVT::i8, ResultReg, X86::sub_8bit);
1458 if (!ResultReg)
1459 return false;
1460 break;
1461 }
1463 ResultReg = createResultReg(&X86::GR8RegClass);
1464 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::MOV8ri),
1465 ResultReg).addImm(1);
1466 break;
1467 }
1468 }
1469
1470 if (ResultReg) {
1471 updateValueMap(I, ResultReg);
1472 return true;
1473 }
1474
1477
1478
1479
1480
1483 if (RHSC && RHSC->isNullValue())
1485 }
1486
1487
1488 static const uint16_t SETFOpcTable[2][3] = {
1491 };
1492 const uint16_t *SETFOpc = nullptr;
1493 switch (Predicate) {
1494 default: break;
1497 }
1498
1499 ResultReg = createResultReg(&X86::GR8RegClass);
1500 if (SETFOpc) {
1501 if (!X86FastEmitCompare(LHS, RHS, VT, I->getDebugLoc()))
1502 return false;
1503
1504 Register FlagReg1 = createResultReg(&X86::GR8RegClass);
1505 Register FlagReg2 = createResultReg(&X86::GR8RegClass);
1506 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::SETCCr),
1507 FlagReg1).addImm(SETFOpc[0]);
1508 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::SETCCr),
1509 FlagReg2).addImm(SETFOpc[1]);
1510 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(SETFOpc[2]),
1511 ResultReg).addReg(FlagReg1).addReg(FlagReg2);
1512 updateValueMap(I, ResultReg);
1513 return true;
1514 }
1515
1517 bool SwapArgs;
1520
1521 if (SwapArgs)
1523
1524
1525 if (!X86FastEmitCompare(LHS, RHS, VT, I->getDebugLoc()))
1526 return false;
1527
1528 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::SETCCr),
1529 ResultReg).addImm(CC);
1530 updateValueMap(I, ResultReg);
1531 return true;
1532}
1533
1534bool X86FastISel::X86SelectZExt(const Instruction *I) {
1535 EVT DstVT = TLI.getValueType(DL, I->getType());
1536 if (!TLI.isTypeLegal(DstVT))
1537 return false;
1538
1539 Register ResultReg = getRegForValue(I->getOperand(0));
1540 if (!ResultReg)
1541 return false;
1542
1543
1544 MVT SrcVT = TLI.getSimpleValueType(DL, I->getOperand(0)->getType());
1545 if (SrcVT == MVT::i1) {
1546
1547 ResultReg = fastEmitZExtFromI1(MVT::i8, ResultReg);
1548 SrcVT = MVT::i8;
1549
1550 if (!ResultReg)
1551 return false;
1552 }
1553
1554 if (DstVT == MVT::i64) {
1555
1556 unsigned MovInst;
1557
1559 case MVT::i8: MovInst = X86::MOVZX32rr8; break;
1560 case MVT::i16: MovInst = X86::MOVZX32rr16; break;
1561 case MVT::i32: MovInst = X86::MOV32rr; break;
1562 default: llvm_unreachable("Unexpected zext to i64 source type");
1563 }
1564
1565 Register Result32 = createResultReg(&X86::GR32RegClass);
1566 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(MovInst), Result32)
1568
1569 ResultReg = createResultReg(&X86::GR64RegClass);
1570 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::SUBREG_TO_REG),
1571 ResultReg)
1573 } else if (DstVT == MVT::i16) {
1574
1575
1576 Register Result32 = createResultReg(&X86::GR32RegClass);
1577 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::MOVZX32rr8),
1578 Result32).addReg(ResultReg);
1579
1580 ResultReg = fastEmitInst_extractsubreg(MVT::i16, Result32, X86::sub_16bit);
1581 } else if (DstVT != MVT::i8) {
1583 ResultReg);
1584 if (!ResultReg)
1585 return false;
1586 }
1587
1588 updateValueMap(I, ResultReg);
1589 return true;
1590}
1591
1592bool X86FastISel::X86SelectSExt(const Instruction *I) {
1593 EVT DstVT = TLI.getValueType(DL, I->getType());
1594 if (!TLI.isTypeLegal(DstVT))
1595 return false;
1596
1597 Register ResultReg = getRegForValue(I->getOperand(0));
1598 if (!ResultReg)
1599 return false;
1600
1601
1602 MVT SrcVT = TLI.getSimpleValueType(DL, I->getOperand(0)->getType());
1603 if (SrcVT == MVT::i1) {
1604
1605 Register ZExtReg = fastEmitZExtFromI1(MVT::i8, ResultReg);
1606 if (!ZExtReg)
1607 return false;
1608
1609
1610 ResultReg = createResultReg(&X86::GR8RegClass);
1611 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::NEG8r),
1612 ResultReg).addReg(ZExtReg);
1613
1614 SrcVT = MVT::i8;
1615 }
1616
1617 if (DstVT == MVT::i16) {
1618
1619
1620 Register Result32 = createResultReg(&X86::GR32RegClass);
1621 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::MOVSX32rr8),
1622 Result32).addReg(ResultReg);
1623
1624 ResultReg = fastEmitInst_extractsubreg(MVT::i16, Result32, X86::sub_16bit);
1625 } else if (DstVT != MVT::i8) {
1627 ResultReg);
1628 if (!ResultReg)
1629 return false;
1630 }
1631
1632 updateValueMap(I, ResultReg);
1633 return true;
1634}
1635
1636bool X86FastISel::X86SelectBranch(const Instruction *I) {
1637
1638
1640 MachineBasicBlock *TrueMBB = FuncInfo.getMBB(BI->getSuccessor(0));
1641 MachineBasicBlock *FalseMBB = FuncInfo.getMBB(BI->getSuccessor(1));
1642
1643
1644
1645
1650
1651
1653 switch (Predicate) {
1654 default: break;
1655 case CmpInst::FCMP_FALSE: fastEmitBranch(FalseMBB, MIMD.getDL()); return true;
1656 case CmpInst::FCMP_TRUE: fastEmitBranch(TrueMBB, MIMD.getDL()); return true;
1657 }
1658
1661
1662
1663
1664
1665
1668 if (CmpRHSC && CmpRHSC->isNullValue())
1669 CmpRHS = CmpLHS;
1670 }
1671
1672
1673 if (FuncInfo.MBB->isLayoutSuccessor(TrueMBB)) {
1676 }
1677
1678
1679
1680
1681
1682
1683 bool NeedExtraBranch = false;
1684 switch (Predicate) {
1685 default: break;
1688 [[fallthrough]];
1690 NeedExtraBranch = true;
1692 break;
1693 }
1694
1695 bool SwapArgs;
1698
1699 if (SwapArgs)
1701
1702
1703 if (!X86FastEmitCompare(CmpLHS, CmpRHS, VT, CI->getDebugLoc()))
1704 return false;
1705
1706 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::JCC_1))
1708
1709
1710
1711 if (NeedExtraBranch) {
1712 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::JCC_1))
1714 }
1715
1716 finishCondBranch(BI->getParent(), TrueMBB, FalseMBB);
1717 return true;
1718 }
1720
1721
1722 MVT SourceVT;
1723 if (TI->hasOneUse() && TI->getParent() == I->getParent() &&
1724 isTypeLegal(TI->getOperand(0)->getType(), SourceVT)) {
1725 unsigned TestOpc = 0;
1726 switch (SourceVT.SimpleTy) {
1727 default: break;
1728 case MVT::i8: TestOpc = X86::TEST8ri; break;
1729 case MVT::i16: TestOpc = X86::TEST16ri; break;
1730 case MVT::i32: TestOpc = X86::TEST32ri; break;
1731 case MVT::i64: TestOpc = X86::TEST64ri32; break;
1732 }
1733 if (TestOpc) {
1734 Register OpReg = getRegForValue(TI->getOperand(0));
1735 if (!OpReg)
1736 return false;
1737
1738 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TestOpc))
1740
1742 if (FuncInfo.MBB->isLayoutSuccessor(TrueMBB)) {
1745 }
1746
1747 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::JCC_1))
1749
1750 finishCondBranch(BI->getParent(), TrueMBB, FalseMBB);
1751 return true;
1752 }
1753 }
1754 } else if (foldX86XALUIntrinsic(CC, BI, BI->getCondition())) {
1755
1756
1758 if (!TmpReg)
1759 return false;
1760
1761 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::JCC_1))
1763 finishCondBranch(BI->getParent(), TrueMBB, FalseMBB);
1764 return true;
1765 }
1766
1767
1768
1769
1771 if (!OpReg)
1772 return false;
1773
1774
1775 if (MRI.getRegClass(OpReg) == &X86::VK1RegClass) {
1777 OpReg = createResultReg(&X86::GR32RegClass);
1778 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1779 TII.get(TargetOpcode::COPY), OpReg)
1781 OpReg = fastEmitInst_extractsubreg(MVT::i8, OpReg, X86::sub_8bit);
1782 }
1783 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::TEST8ri))
1786 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::JCC_1))
1788 finishCondBranch(BI->getParent(), TrueMBB, FalseMBB);
1789 return true;
1790}
1791
1792bool X86FastISel::X86SelectShift(const Instruction *I) {
1794 unsigned OpReg;
1795 const TargetRegisterClass *RC = nullptr;
1796 if (I->getType()->isIntegerTy(8)) {
1797 CReg = X86::CL;
1798 RC = &X86::GR8RegClass;
1799 switch (I->getOpcode()) {
1800 case Instruction::LShr: OpReg = X86::SHR8rCL; break;
1801 case Instruction::AShr: OpReg = X86::SAR8rCL; break;
1802 case Instruction::Shl: OpReg = X86::SHL8rCL; break;
1803 default: return false;
1804 }
1805 } else if (I->getType()->isIntegerTy(16)) {
1806 CReg = X86::CX;
1807 RC = &X86::GR16RegClass;
1808 switch (I->getOpcode()) {
1810 case Instruction::LShr: OpReg = X86::SHR16rCL; break;
1811 case Instruction::AShr: OpReg = X86::SAR16rCL; break;
1812 case Instruction::Shl: OpReg = X86::SHL16rCL; break;
1813 }
1814 } else if (I->getType()->isIntegerTy(32)) {
1815 CReg = X86::ECX;
1816 RC = &X86::GR32RegClass;
1817 switch (I->getOpcode()) {
1819 case Instruction::LShr: OpReg = X86::SHR32rCL; break;
1820 case Instruction::AShr: OpReg = X86::SAR32rCL; break;
1821 case Instruction::Shl: OpReg = X86::SHL32rCL; break;
1822 }
1823 } else if (I->getType()->isIntegerTy(64)) {
1824 CReg = X86::RCX;
1825 RC = &X86::GR64RegClass;
1826 switch (I->getOpcode()) {
1828 case Instruction::LShr: OpReg = X86::SHR64rCL; break;
1829 case Instruction::AShr: OpReg = X86::SAR64rCL; break;
1830 case Instruction::Shl: OpReg = X86::SHL64rCL; break;
1831 }
1832 } else {
1833 return false;
1834 }
1835
1836 MVT VT;
1837 if (!isTypeLegal(I->getType(), VT))
1838 return false;
1839
1840 Register Op0Reg = getRegForValue(I->getOperand(0));
1841 if (!Op0Reg)
1842 return false;
1843
1844 Register Op1Reg = getRegForValue(I->getOperand(1));
1845 if (!Op1Reg)
1846 return false;
1847 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
1848 CReg).addReg(Op1Reg);
1849
1850
1851
1852 if (CReg != X86::CL)
1853 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1854 TII.get(TargetOpcode::KILL), X86::CL)
1856
1857 Register ResultReg = createResultReg(RC);
1858 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(OpReg), ResultReg)
1860 updateValueMap(I, ResultReg);
1861 return true;
1862}
1863
1864bool X86FastISel::X86SelectDivRem(const Instruction *I) {
1865 const static unsigned NumTypes = 4;
1866 const static unsigned NumOps = 4;
1867 const static bool S = true;
1868 const static bool U = false;
1869 const static unsigned Copy = TargetOpcode::COPY;
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879 const static struct DivRemEntry {
1880
1881 const TargetRegisterClass *RC;
1882 unsigned LowInReg;
1883 unsigned HighInReg;
1884
1885 struct DivRemResult {
1886 unsigned OpDivRem;
1887 unsigned OpSignExtend;
1888
1889 unsigned OpCopy;
1890
1891 unsigned DivRemResultReg;
1892 bool IsOpSigned;
1893 } ResultTable[NumOps];
1894 } OpTable[NumTypes] = {
1895 { &X86::GR8RegClass, X86::AX, 0, {
1896 { X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AL, S },
1897 { X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AH, S },
1898 { X86::DIV8r, 0, X86::MOVZX16rr8, X86::AL, U },
1899 { X86::DIV8r, 0, X86::MOVZX16rr8, X86::AH, U },
1900 }
1901 },
1902 { &X86::GR16RegClass, X86::AX, X86::DX, {
1903 { X86::IDIV16r, X86::CWD, Copy, X86::AX, S },
1904 { X86::IDIV16r, X86::CWD, Copy, X86::DX, S },
1905 { X86::DIV16r, X86::MOV32r0, Copy, X86::AX, U },
1906 { X86::DIV16r, X86::MOV32r0, Copy, X86::DX, U },
1907 }
1908 },
1909 { &X86::GR32RegClass, X86::EAX, X86::EDX, {
1910 { X86::IDIV32r, X86::CDQ, Copy, X86::EAX, S },
1911 { X86::IDIV32r, X86::CDQ, Copy, X86::EDX, S },
1912 { X86::DIV32r, X86::MOV32r0, Copy, X86::EAX, U },
1913 { X86::DIV32r, X86::MOV32r0, Copy, X86::EDX, U },
1914 }
1915 },
1916 { &X86::GR64RegClass, X86::RAX, X86::RDX, {
1917 { X86::IDIV64r, X86::CQO, Copy, X86::RAX, S },
1918 { X86::IDIV64r, X86::CQO, Copy, X86::RDX, S },
1919 { X86::DIV64r, X86::MOV32r0, Copy, X86::RAX, U },
1920 { X86::DIV64r, X86::MOV32r0, Copy, X86::RDX, U },
1921 }
1922 },
1923 };
1924
1925 MVT VT;
1926 if (!isTypeLegal(I->getType(), VT))
1927 return false;
1928
1929 unsigned TypeIndex, OpIndex;
1931 default: return false;
1932 case MVT::i8: TypeIndex = 0; break;
1933 case MVT::i16: TypeIndex = 1; break;
1934 case MVT::i32: TypeIndex = 2; break;
1935 case MVT::i64: TypeIndex = 3;
1936 if (!Subtarget->is64Bit())
1937 return false;
1938 break;
1939 }
1940
1941 switch (I->getOpcode()) {
1943 case Instruction::SDiv: OpIndex = 0; break;
1944 case Instruction::SRem: OpIndex = 1; break;
1945 case Instruction::UDiv: OpIndex = 2; break;
1946 case Instruction::URem: OpIndex = 3; break;
1947 }
1948
1949 const DivRemEntry &TypeEntry = OpTable[TypeIndex];
1950 const DivRemEntry::DivRemResult &OpEntry = TypeEntry.ResultTable[OpIndex];
1951 Register Op0Reg = getRegForValue(I->getOperand(0));
1952 if (!Op0Reg)
1953 return false;
1954 Register Op1Reg = getRegForValue(I->getOperand(1));
1955 if (!Op1Reg)
1956 return false;
1957
1958
1959 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1961
1962 if (OpEntry.OpSignExtend) {
1963 if (OpEntry.IsOpSigned)
1964 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1965 TII.get(OpEntry.OpSignExtend));
1966 else {
1967 Register Zero32 = createResultReg(&X86::GR32RegClass);
1968 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1969 TII.get(X86::MOV32r0), Zero32);
1970
1971
1972
1973
1974 if (VT == MVT::i16) {
1975 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1977 .addReg(Zero32, 0, X86::sub_16bit);
1978 } else if (VT == MVT::i32) {
1979 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1982 } else if (VT == MVT::i64) {
1983 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1984 TII.get(TargetOpcode::SUBREG_TO_REG), TypeEntry.HighInReg)
1986 }
1987 }
1988 }
1989
1990 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1992
1993
1994
1995
1996
1997
1998
1999
2001 if ((I->getOpcode() == Instruction::SRem ||
2002 I->getOpcode() == Instruction::URem) &&
2003 OpEntry.DivRemResultReg == X86::AH && Subtarget->is64Bit()) {
2004 Register SourceSuperReg = createResultReg(&X86::GR16RegClass);
2005 Register ResultSuperReg = createResultReg(&X86::GR16RegClass);
2006 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2007 TII.get(Copy), SourceSuperReg).addReg(X86::AX);
2008
2009
2010 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::SHR16ri),
2011 ResultSuperReg).addReg(SourceSuperReg).addImm(8);
2012
2013
2014 ResultReg = fastEmitInst_extractsubreg(MVT::i8, ResultSuperReg,
2015 X86::sub_8bit);
2016 }
2017
2018 if (!ResultReg) {
2019 ResultReg = createResultReg(TypeEntry.RC);
2020 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Copy), ResultReg)
2021 .addReg(OpEntry.DivRemResultReg);
2022 }
2023 updateValueMap(I, ResultReg);
2024
2025 return true;
2026}
2027
2028
2029
2030bool X86FastISel::X86FastEmitCMoveSelect(MVT RetVT, const Instruction *I) {
2031
2032 if (!Subtarget->canUseCMOV())
2033 return false;
2034
2035
2036 if (RetVT < MVT::i16 || RetVT > MVT::i64)
2037 return false;
2038
2039 const Value *Cond = I->getOperand(0);
2040 const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
2041 bool NeedTest = true;
2043
2044
2045
2046
2048 if (CI && (CI->getParent() == I->getParent())) {
2050
2051
2052 static const uint16_t SETFOpcTable[2][3] = {
2055 };
2056 const uint16_t *SETFOpc = nullptr;
2057 switch (Predicate) {
2058 default: break;
2060 SETFOpc = &SETFOpcTable[0][0];
2062 break;
2064 SETFOpc = &SETFOpcTable[1][0];
2066 break;
2067 }
2068
2069 bool NeedSwap;
2072
2075 if (NeedSwap)
2077
2078 EVT CmpVT = TLI.getValueType(DL, CmpLHS->getType());
2079
2080 if (!X86FastEmitCompare(CmpLHS, CmpRHS, CmpVT, CI->getDebugLoc()))
2081 return false;
2082
2083 if (SETFOpc) {
2084 Register FlagReg1 = createResultReg(&X86::GR8RegClass);
2085 Register FlagReg2 = createResultReg(&X86::GR8RegClass);
2086 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::SETCCr),
2087 FlagReg1).addImm(SETFOpc[0]);
2088 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::SETCCr),
2089 FlagReg2).addImm(SETFOpc[1]);
2090 auto const &II = TII.get(SETFOpc[2]);
2091 if (II.getNumDefs()) {
2092 Register TmpReg = createResultReg(&X86::GR8RegClass);
2093 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, TmpReg)
2095 } else {
2096 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
2098 }
2099 }
2100 NeedTest = false;
2101 } else if (foldX86XALUIntrinsic(CC, I, Cond)) {
2102
2103
2105 if (!TmpReg)
2106 return false;
2107
2108 NeedTest = false;
2109 }
2110
2111 if (NeedTest) {
2112
2113
2114
2115
2116
2118 if (!CondReg)
2119 return false;
2120
2121
2122 if (MRI.getRegClass(CondReg) == &X86::VK1RegClass) {
2123 Register KCondReg = CondReg;
2124 CondReg = createResultReg(&X86::GR32RegClass);
2125 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2126 TII.get(TargetOpcode::COPY), CondReg)
2128 CondReg = fastEmitInst_extractsubreg(MVT::i8, CondReg, X86::sub_8bit);
2129 }
2130 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::TEST8ri))
2133 }
2134
2135 const Value *LHS = I->getOperand(1);
2136 const Value *RHS = I->getOperand(2);
2137
2140 if (!LHSReg || !RHSReg)
2141 return false;
2142
2143 const TargetRegisterInfo &TRI = *Subtarget->getRegisterInfo();
2145 Subtarget->hasNDD());
2146 Register ResultReg = fastEmitInst_rri(Opc, RC, RHSReg, LHSReg, CC);
2147 updateValueMap(I, ResultReg);
2148 return true;
2149}
2150
2151
2152
2153
2154
2155
2156bool X86FastISel::X86FastEmitSSESelect(MVT RetVT, const Instruction *I) {
2157
2158
2159
2161 if (!CI || (CI->getParent() != I->getParent()))
2162 return false;
2163
2165 !((Subtarget->hasSSE1() && RetVT == MVT::f32) ||
2166 (Subtarget->hasSSE2() && RetVT == MVT::f64)))
2167 return false;
2168
2172
2173
2174
2175
2178 if (CmpRHSC && CmpRHSC->isNullValue())
2179 CmpRHS = CmpLHS;
2180 }
2181
2182 unsigned CC;
2183 bool NeedSwap;
2185 if (CC > 7 && !Subtarget->hasAVX())
2186 return false;
2187
2188 if (NeedSwap)
2190
2191 const Value *LHS = I->getOperand(1);
2192 const Value *RHS = I->getOperand(2);
2193
2196 Register CmpLHSReg = getRegForValue(CmpLHS);
2197 Register CmpRHSReg = getRegForValue(CmpRHS);
2198 if (!LHSReg || !RHSReg || !CmpLHSReg || !CmpRHSReg)
2199 return false;
2200
2201 const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
2203
2204 if (Subtarget->hasAVX512()) {
2205
2206 const TargetRegisterClass *VR128X = &X86::VR128XRegClass;
2207 const TargetRegisterClass *VK1 = &X86::VK1RegClass;
2208
2209 unsigned CmpOpcode =
2210 (RetVT == MVT::f32) ? X86::VCMPSSZrri : X86::VCMPSDZrri;
2211 Register CmpReg = fastEmitInst_rri(CmpOpcode, VK1, CmpLHSReg, CmpRHSReg,
2212 CC);
2213
2214
2215
2216 Register ImplicitDefReg = createResultReg(VR128X);
2217 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2218 TII.get(TargetOpcode::IMPLICIT_DEF), ImplicitDefReg);
2219
2220
2221
2222 unsigned MovOpcode =
2223 (RetVT == MVT::f32) ? X86::VMOVSSZrrk : X86::VMOVSDZrrk;
2224 Register MovReg = fastEmitInst_rrrr(MovOpcode, VR128X, RHSReg, CmpReg,
2225 ImplicitDefReg, LHSReg);
2226
2227 ResultReg = createResultReg(RC);
2228 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2229 TII.get(TargetOpcode::COPY), ResultReg).addReg(MovReg);
2230
2231 } else if (Subtarget->hasAVX()) {
2232 const TargetRegisterClass *VR128 = &X86::VR128RegClass;
2233
2234
2235
2236
2237
2238
2239 unsigned CmpOpcode =
2240 (RetVT == MVT::f32) ? X86::VCMPSSrri : X86::VCMPSDrri;
2241 unsigned BlendOpcode =
2242 (RetVT == MVT::f32) ? X86::VBLENDVPSrrr : X86::VBLENDVPDrrr;
2243
2244 Register CmpReg = fastEmitInst_rri(CmpOpcode, RC, CmpLHSReg, CmpRHSReg,
2245 CC);
2246 Register VBlendReg = fastEmitInst_rrr(BlendOpcode, VR128, RHSReg, LHSReg,
2247 CmpReg);
2248 ResultReg = createResultReg(RC);
2249 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2250 TII.get(TargetOpcode::COPY), ResultReg).addReg(VBlendReg);
2251 } else {
2252
2253 static const uint16_t OpcTable[2][4] = {
2254 { X86::CMPSSrri, X86::ANDPSrr, X86::ANDNPSrr, X86::ORPSrr },
2255 { X86::CMPSDrri, X86::ANDPDrr, X86::ANDNPDrr, X86::ORPDrr }
2256 };
2257
2258 const uint16_t *Opc = nullptr;
2260 default: return false;
2261 case MVT::f32: Opc = &OpcTable[0][0]; break;
2262 case MVT::f64: Opc = &OpcTable[1][0]; break;
2263 }
2264
2265 const TargetRegisterClass *VR128 = &X86::VR128RegClass;
2266 Register CmpReg = fastEmitInst_rri(Opc[0], RC, CmpLHSReg, CmpRHSReg, CC);
2267 Register AndReg = fastEmitInst_rr(Opc[1], VR128, CmpReg, LHSReg);
2268 Register AndNReg = fastEmitInst_rr(Opc[2], VR128, CmpReg, RHSReg);
2269 Register OrReg = fastEmitInst_rr(Opc[3], VR128, AndNReg, AndReg);
2270 ResultReg = createResultReg(RC);
2271 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2272 TII.get(TargetOpcode::COPY), ResultReg).addReg(OrReg);
2273 }
2274 updateValueMap(I, ResultReg);
2275 return true;
2276}
2277
2278bool X86FastISel::X86FastEmitPseudoSelect(MVT RetVT, const Instruction *I) {
2279
2280
2281 unsigned Opc;
2283 default: return false;
2284 case MVT::i8: Opc = X86::CMOV_GR8; break;
2285 case MVT::i16: Opc = X86::CMOV_GR16; break;
2286 case MVT::i32: Opc = X86::CMOV_GR32; break;
2287 case MVT::f16:
2288 Opc = Subtarget->hasAVX512() ? X86::CMOV_FR16X : X86::CMOV_FR16; break;
2289 case MVT::f32:
2290 Opc = Subtarget->hasAVX512() ? X86::CMOV_FR32X : X86::CMOV_FR32; break;
2291 case MVT::f64:
2292 Opc = Subtarget->hasAVX512() ? X86::CMOV_FR64X : X86::CMOV_FR64; break;
2293 }
2294
2295 const Value *Cond = I->getOperand(0);
2297
2298
2299
2300
2302 if (CI && (CI->getParent() == I->getParent())) {
2303 bool NeedSwap;
2306 return false;
2307
2310
2311 if (NeedSwap)
2313
2314 EVT CmpVT = TLI.getValueType(DL, CmpLHS->getType());
2315 if (!X86FastEmitCompare(CmpLHS, CmpRHS, CmpVT, CI->getDebugLoc()))
2316 return false;
2317 } else {
2319 if (!CondReg)
2320 return false;
2321
2322
2323 if (MRI.getRegClass(CondReg) == &X86::VK1RegClass) {
2324 Register KCondReg = CondReg;
2325 CondReg = createResultReg(&X86::GR32RegClass);
2326 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2327 TII.get(TargetOpcode::COPY), CondReg)
2329 CondReg = fastEmitInst_extractsubreg(MVT::i8, CondReg, X86::sub_8bit);
2330 }
2331 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::TEST8ri))
2334 }
2335
2336 const Value *LHS = I->getOperand(1);
2337 const Value *RHS = I->getOperand(2);
2338
2341 if (!LHSReg || !RHSReg)
2342 return false;
2343
2344 const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
2345
2347 fastEmitInst_rri(Opc, RC, RHSReg, LHSReg, CC);
2348 updateValueMap(I, ResultReg);
2349 return true;
2350}
2351
2352bool X86FastISel::X86SelectSelect(const Instruction *I) {
2353 MVT RetVT;
2354 if (!isTypeLegal(I->getType(), RetVT))
2355 return false;
2356
2357
2360 const Value *Opnd = nullptr;
2361 switch (Predicate) {
2362 default: break;
2365 }
2366
2367 if (Opnd) {
2368 Register OpReg = getRegForValue(Opnd);
2369 if (!OpReg)
2370 return false;
2371 const TargetRegisterClass *RC = TLI.getRegClassFor(RetVT);
2372 Register ResultReg = createResultReg(RC);
2373 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2374 TII.get(TargetOpcode::COPY), ResultReg)
2376 updateValueMap(I, ResultReg);
2377 return true;
2378 }
2379 }
2380
2381
2382 if (X86FastEmitCMoveSelect(RetVT, I))
2383 return true;
2384
2385
2386 if (X86FastEmitSSESelect(RetVT, I))
2387 return true;
2388
2389
2390
2391 if (X86FastEmitPseudoSelect(RetVT, I))
2392 return true;
2393
2394 return false;
2395}
2396
2397
2398bool X86FastISel::X86SelectIntToFP(const Instruction *I, bool IsSigned) {
2399
2400
2401
2402
2403 bool HasAVX512 = Subtarget->hasAVX512();
2404 if (!Subtarget->hasAVX() || (!IsSigned && !HasAVX512))
2405 return false;
2406
2407
2408 EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
2409 if (SrcVT != MVT::i32 && SrcVT != MVT::i64)
2410 return false;
2411
2412
2413 Register OpReg = getRegForValue(I->getOperand(0));
2414 if (!OpReg)
2415 return false;
2416
2417 unsigned Opcode;
2418
2419 static const uint16_t SCvtOpc[2][2][2] = {
2420 { { X86::VCVTSI2SSrr, X86::VCVTSI642SSrr },
2421 { X86::VCVTSI2SDrr, X86::VCVTSI642SDrr } },
2422 { { X86::VCVTSI2SSZrr, X86::VCVTSI642SSZrr },
2423 { X86::VCVTSI2SDZrr, X86::VCVTSI642SDZrr } },
2424 };
2425 static const uint16_t UCvtOpc[2][2] = {
2426 { X86::VCVTUSI2SSZrr, X86::VCVTUSI642SSZrr },
2427 { X86::VCVTUSI2SDZrr, X86::VCVTUSI642SDZrr },
2428 };
2429 bool Is64Bit = SrcVT == MVT::i64;
2430
2431 if (I->getType()->isDoubleTy()) {
2432
2433 Opcode = IsSigned ? SCvtOpc[HasAVX512][1][Is64Bit] : UCvtOpc[1][Is64Bit];
2434 } else if (I->getType()->isFloatTy()) {
2435
2436 Opcode = IsSigned ? SCvtOpc[HasAVX512][0][Is64Bit] : UCvtOpc[0][Is64Bit];
2437 } else
2438 return false;
2439
2440 MVT DstVT = TLI.getValueType(DL, I->getType()).getSimpleVT();
2441 const TargetRegisterClass *RC = TLI.getRegClassFor(DstVT);
2442 Register ImplicitDefReg = createResultReg(RC);
2443 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2444 TII.get(TargetOpcode::IMPLICIT_DEF), ImplicitDefReg);
2445 Register ResultReg = fastEmitInst_rr(Opcode, RC, ImplicitDefReg, OpReg);
2446 updateValueMap(I, ResultReg);
2447 return true;
2448}
2449
2450bool X86FastISel::X86SelectSIToFP(const Instruction *I) {
2451 return X86SelectIntToFP(I, true);
2452}
2453
2454bool X86FastISel::X86SelectUIToFP(const Instruction *I) {
2455 return X86SelectIntToFP(I, false);
2456}
2457
2458
2459bool X86FastISel::X86SelectFPExtOrFPTrunc(const Instruction *I,
2460 unsigned TargetOpc,
2461 const TargetRegisterClass *RC) {
2462 assert((I->getOpcode() == Instruction::FPExt ||
2463 I->getOpcode() == Instruction::FPTrunc) &&
2464 "Instruction must be an FPExt or FPTrunc!");
2465 bool HasAVX = Subtarget->hasAVX();
2466
2467 Register OpReg = getRegForValue(I->getOperand(0));
2468 if (!OpReg)
2469 return false;
2470
2472 if (HasAVX) {
2473 ImplicitDefReg = createResultReg(RC);
2474 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2475 TII.get(TargetOpcode::IMPLICIT_DEF), ImplicitDefReg);
2476
2477 }
2478
2479 Register ResultReg = createResultReg(RC);
2480 MachineInstrBuilder MIB;
2481 MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpc),
2482 ResultReg);
2483
2484 if (HasAVX)
2485 MIB.addReg(ImplicitDefReg);
2486
2488 updateValueMap(I, ResultReg);
2489 return true;
2490}
2491
2492bool X86FastISel::X86SelectFPExt(const Instruction *I) {
2493 if (Subtarget->hasSSE2() && I->getType()->isDoubleTy() &&
2494 I->getOperand(0)->getType()->isFloatTy()) {
2495 bool HasAVX512 = Subtarget->hasAVX512();
2496
2497 unsigned Opc =
2498 HasAVX512 ? X86::VCVTSS2SDZrr
2499 : Subtarget->hasAVX() ? X86::VCVTSS2SDrr : X86::CVTSS2SDrr;
2500 return X86SelectFPExtOrFPTrunc(I, Opc, TLI.getRegClassFor(MVT::f64));
2501 }
2502
2503 return false;
2504}
2505
2506bool X86FastISel::X86SelectFPTrunc(const Instruction *I) {
2507 if (Subtarget->hasSSE2() && I->getType()->isFloatTy() &&
2508 I->getOperand(0)->getType()->isDoubleTy()) {
2509 bool HasAVX512 = Subtarget->hasAVX512();
2510
2511 unsigned Opc =
2512 HasAVX512 ? X86::VCVTSD2SSZrr
2513 : Subtarget->hasAVX() ? X86::VCVTSD2SSrr : X86::CVTSD2SSrr;
2514 return X86SelectFPExtOrFPTrunc(I, Opc, TLI.getRegClassFor(MVT::f32));
2515 }
2516
2517 return false;
2518}
2519
2520bool X86FastISel::X86SelectTrunc(const Instruction *I) {
2521 EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
2522 EVT DstVT = TLI.getValueType(DL, I->getType());
2523
2524
2525 if (DstVT != MVT::i8 && DstVT != MVT::i1)
2526 return false;
2527 if (!TLI.isTypeLegal(SrcVT))
2528 return false;
2529
2530 Register InputReg = getRegForValue(I->getOperand(0));
2531 if (!InputReg)
2532
2533 return false;
2534
2535 if (SrcVT == MVT::i8) {
2536
2537 updateValueMap(I, InputReg);
2538 return true;
2539 }
2540
2541
2542 Register ResultReg = fastEmitInst_extractsubreg(MVT::i8, InputReg,
2543 X86::sub_8bit);
2544 if (!ResultReg)
2545 return false;
2546
2547 updateValueMap(I, ResultReg);
2548 return true;
2549}
2550
2551bool X86FastISel::X86SelectBitCast(const Instruction *I) {
2552
2553 MVT SrcVT, DstVT;
2554 if (!Subtarget->hasSSE2() ||
2555 !isTypeLegal(I->getOperand(0)->getType(), SrcVT) ||
2556 !isTypeLegal(I->getType(), DstVT))
2557 return false;
2558
2559
2563 return false;
2564
2565 Register Reg = getRegForValue(I->getOperand(0));
2566 if ()
2567 return false;
2568
2569
2570
2571 const TargetRegisterClass *DstClass = TLI.getRegClassFor(DstVT);
2572 Register ResultReg = createResultReg(DstClass);
2573 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
2574 ResultReg)
2576
2577 updateValueMap(I, ResultReg);
2578 return true;
2579}
2580
2581bool X86FastISel::IsMemcpySmall(uint64_t Len) {
2582 return Len <= (Subtarget->is64Bit() ? 32 : 16);
2583}
2584
2585bool X86FastISel::TryEmitSmallMemcpy(X86AddressMode DestAM,
2586 X86AddressMode SrcAM, uint64_t Len) {
2587
2588
2589 if (!IsMemcpySmall(Len))
2590 return false;
2591
2592 bool i64Legal = Subtarget->is64Bit();
2593
2594
2595 while (Len) {
2596 MVT VT;
2597 if (Len >= 8 && i64Legal)
2598 VT = MVT::i64;
2599 else if (Len >= 4)
2600 VT = MVT::i32;
2601 else if (Len >= 2)
2602 VT = MVT::i16;
2603 else
2604 VT = MVT::i8;
2605
2607 bool RV = X86FastEmitLoad(VT, SrcAM, nullptr, Reg);
2608 RV &= X86FastEmitStore(VT, Reg, DestAM);
2609 assert(RV && "Failed to emit load or store??");
2610 (void)RV;
2611
2616 }
2617
2618 return true;
2619}
2620
2621bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
2622
2623 switch (II->getIntrinsicID()) {
2624 default: return false;
2625 case Intrinsic::convert_from_fp16:
2626 case Intrinsic::convert_to_fp16: {
2627 if (Subtarget->useSoftFloat() || !Subtarget->hasF16C())
2628 return false;
2629
2630 const Value *Op = II->getArgOperand(0);
2631 Register InputReg = getRegForValue(Op);
2632 if (!InputReg)
2633 return false;
2634
2635
2636 bool IsFloatToHalf = II->getIntrinsicID() == Intrinsic::convert_to_fp16;
2637 if (IsFloatToHalf) {
2638 if (->getType()->isFloatTy())
2639 return false;
2640 } else {
2641 if (->getType()->isFloatTy())
2642 return false;
2643 }
2644
2646 const TargetRegisterClass *RC = TLI.getRegClassFor(MVT::v8i16);
2647 if (IsFloatToHalf) {
2648
2649
2650
2651
2652
2653
2654
2655 unsigned Opc = Subtarget->hasVLX() ? X86::VCVTPS2PHZ128rr
2656 : X86::VCVTPS2PHrr;
2657 InputReg = fastEmitInst_ri(Opc, RC, InputReg, 4);
2658
2659
2660 Opc = Subtarget->hasAVX512() ? X86::VMOVPDI2DIZrr
2661 : X86::VMOVPDI2DIrr;
2662 ResultReg = createResultReg(&X86::GR32RegClass);
2663 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), ResultReg)
2665
2666
2667 unsigned RegIdx = X86::sub_16bit;
2668 ResultReg = fastEmitInst_extractsubreg(MVT::i16, ResultReg, RegIdx);
2669 } else {
2670 assert(Op->getType()->isIntegerTy(16) && "Expected a 16-bit integer!");
2671
2672 InputReg = fastEmit_r(MVT::i16, MVT::i32, ISD::ZERO_EXTEND, InputReg);
2673
2674
2676 InputReg);
2677
2678 unsigned Opc = Subtarget->hasVLX() ? X86::VCVTPH2PSZ128rr
2679 : X86::VCVTPH2PSrr;
2680 InputReg = fastEmitInst_r(Opc, RC, InputReg);
2681
2682
2683
2684 ResultReg = createResultReg(TLI.getRegClassFor(MVT::f32));
2685 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2686 TII.get(TargetOpcode::COPY), ResultReg)
2688 }
2689
2690 updateValueMap(II, ResultReg);
2691 return true;
2692 }
2693 case Intrinsic::frameaddress: {
2694 MachineFunction *MF = FuncInfo.MF;
2696 return false;
2697
2698 Type *RetTy = II->getCalledFunction()->getReturnType();
2699
2700 MVT VT;
2701 if (!isTypeLegal(RetTy, VT))
2702 return false;
2703
2704 unsigned Opc;
2705 const TargetRegisterClass *RC = nullptr;
2706
2708 default: llvm_unreachable("Invalid result type for frameaddress.");
2709 case MVT::i32: Opc = X86::MOV32rm; RC = &X86::GR32RegClass; break;
2710 case MVT::i64: Opc = X86::MOV64rm; RC = &X86::GR64RegClass; break;
2711 }
2712
2713
2714
2717
2718 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
2720 assert(((FrameReg == X86::RBP && VT == MVT::i64) ||
2721 (FrameReg == X86::EBP && VT == MVT::i32)) &&
2722 "Invalid Frame Register!");
2723
2724
2725
2726
2727 Register SrcReg = createResultReg(RC);
2728 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2729 TII.get(TargetOpcode::COPY), SrcReg).addReg(FrameReg);
2730
2731
2732
2733
2734
2735
2737 while (Depth--) {
2738 Register DestReg = createResultReg(RC);
2740 TII.get(Opc), DestReg), SrcReg);
2741 SrcReg = DestReg;
2742 }
2743
2744 updateValueMap(II, SrcReg);
2745 return true;
2746 }
2747 case Intrinsic::memcpy: {
2749
2751 return false;
2752
2754
2755
2757 if (IsMemcpySmall(Len)) {
2758 X86AddressMode DestAM, SrcAM;
2761 return false;
2762 TryEmitSmallMemcpy(DestAM, SrcAM, Len);
2763 return true;
2764 }
2765 }
2766
2767 unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32;
2769 return false;
2770
2772 return false;
2773
2774 return lowerCallTo(II, "memcpy", II->arg_size() - 1);
2775 }
2776 case Intrinsic::memset: {
2778
2780 return false;
2781
2782 unsigned SizeWidth = Subtarget->is64Bit() ? 64 : 32;
2784 return false;
2785
2787 return false;
2788
2789 return lowerCallTo(II, "memset", II->arg_size() - 1);
2790 }
2791 case Intrinsic::stackprotector: {
2792
2793 EVT PtrTy = TLI.getPointerTy(DL);
2794
2795 const Value *Op1 = II->getArgOperand(0);
2797
2799
2800
2801 X86AddressMode AM;
2803 if (!X86FastEmitStore(PtrTy, Op1, AM)) return false;
2804 return true;
2805 }
2806 case Intrinsic::dbg_declare: {
2808 X86AddressMode AM;
2809 assert(DI->getAddress() && "Null address should be checked earlier!");
2811 return false;
2812 const MCInstrDesc &II = TII.get(TargetOpcode::DBG_VALUE);
2814 "Expected inlined-at fields to agree");
2819 return true;
2820 }
2821 case Intrinsic:🪤 {
2822 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::TRAP));
2823 return true;
2824 }
2825 case Intrinsic::sqrt: {
2826 if (!Subtarget->hasSSE1())
2827 return false;
2828
2829 Type *RetTy = II->getCalledFunction()->getReturnType();
2830
2831 MVT VT;
2832 if (!isTypeLegal(RetTy, VT))
2833 return false;
2834
2835
2836
2837
2838 static const uint16_t SqrtOpc[3][2] = {
2839 { X86::SQRTSSr, X86::SQRTSDr },
2840 { X86::VSQRTSSr, X86::VSQRTSDr },
2841 { X86::VSQRTSSZr, X86::VSQRTSDZr },
2842 };
2843 unsigned AVXLevel = Subtarget->hasAVX512() ? 2 :
2844 Subtarget->hasAVX() ? 1 :
2845 0;
2846 unsigned Opc;
2848 default: return false;
2849 case MVT::f32: Opc = SqrtOpc[AVXLevel][0]; break;
2850 case MVT::f64: Opc = SqrtOpc[AVXLevel][1]; break;
2851 }
2852
2853 const Value *SrcVal = II->getArgOperand(0);
2854 Register SrcReg = getRegForValue(SrcVal);
2855
2856 if (!SrcReg)
2857 return false;
2858
2859 const TargetRegisterClass *RC = TLI.getRegClassFor(VT);
2861 if (AVXLevel > 0) {
2862 ImplicitDefReg = createResultReg(RC);
2863 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2864 TII.get(TargetOpcode::IMPLICIT_DEF), ImplicitDefReg);
2865 }
2866
2867 Register ResultReg = createResultReg(RC);
2868 MachineInstrBuilder MIB;
2869 MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc),
2870 ResultReg);
2871
2872 if (ImplicitDefReg)
2873 MIB.addReg(ImplicitDefReg);
2874
2876
2877 updateValueMap(II, ResultReg);
2878 return true;
2879 }
2880 case Intrinsic::sadd_with_overflow:
2881 case Intrinsic::uadd_with_overflow:
2882 case Intrinsic::ssub_with_overflow:
2883 case Intrinsic::usub_with_overflow:
2884 case Intrinsic::smul_with_overflow:
2885 case Intrinsic::umul_with_overflow: {
2886
2887
2890 Type *RetTy = Ty->getTypeAtIndex(0U);
2893 "Overflow value expected to be an i1");
2894
2895 MVT VT;
2896 if (!isTypeLegal(RetTy, VT))
2897 return false;
2898
2899 if (VT < MVT::i8 || VT > MVT::i64)
2900 return false;
2901
2902 const Value *LHS = II->getArgOperand(0);
2903 const Value *RHS = II->getArgOperand(1);
2904
2905
2908
2910 switch (II->getIntrinsicID()) {
2912 case Intrinsic::sadd_with_overflow:
2914 case Intrinsic::uadd_with_overflow:
2916 case Intrinsic::ssub_with_overflow:
2918 case Intrinsic::usub_with_overflow:
2920 case Intrinsic::smul_with_overflow:
2922 case Intrinsic::umul_with_overflow:
2924 }
2925
2927 if (!LHSReg)
2928 return false;
2929
2931
2933 static const uint16_t Opc[2][4] = {
2934 { X86::INC8r, X86::INC16r, X86::INC32r, X86::INC64r },
2935 { X86::DEC8r, X86::DEC16r, X86::DEC32r, X86::DEC64r }
2936 };
2937
2938 if (CI->isOne() && (BaseOpc == ISD::ADD || BaseOpc == ISD::SUB) &&
2940
2941 ResultReg = createResultReg(TLI.getRegClassFor(VT));
2942 bool IsDec = BaseOpc == ISD::SUB;
2943 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2946 } else
2947 ResultReg = fastEmit_ri(VT, VT, BaseOpc, LHSReg, CI->getZExtValue());
2948 }
2949
2951 if (!ResultReg) {
2952 RHSReg = getRegForValue(RHS);
2953 if (!RHSReg)
2954 return false;
2955 ResultReg = fastEmit_rr(VT, VT, BaseOpc, LHSReg, RHSReg);
2956 }
2957
2958
2959
2960 if (BaseOpc == X86ISD::UMUL && !ResultReg) {
2961 static const uint16_t MULOpc[] =
2962 { X86::MUL8r, X86::MUL16r, X86::MUL32r, X86::MUL64r };
2963 static const MCPhysReg Reg[] = { X86::AL, X86::AX, X86::EAX, X86::RAX };
2964
2965
2966 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2969 ResultReg = fastEmitInst_r(MULOpc[VT.SimpleTy-MVT::i8],
2970 TLI.getRegClassFor(VT), RHSReg);
2971 } else if (BaseOpc == X86ISD::SMUL && !ResultReg) {
2972 static const uint16_t MULOpc[] =
2973 { X86::IMUL8r, X86::IMUL16rr, X86::IMUL32rr, X86::IMUL64rr };
2974 if (VT == MVT::i8) {
2975
2976
2977 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
2978 TII.get(TargetOpcode::COPY), X86::AL)
2980 ResultReg = fastEmitInst_r(MULOpc[0], TLI.getRegClassFor(VT), RHSReg);
2981 } else
2982 ResultReg = fastEmitInst_rr(MULOpc[VT.SimpleTy-MVT::i8],
2983 TLI.getRegClassFor(VT), LHSReg, RHSReg);
2984 }
2985
2986 if (!ResultReg)
2987 return false;
2988
2989
2990 Register ResultReg2 = createResultReg(&X86::GR8RegClass);
2991 assert((ResultReg+1) == ResultReg2 && "Nonconsecutive result registers.");
2992 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::SETCCr),
2993 ResultReg2).addImm(CondCode);
2994
2995 updateValueMap(II, ResultReg, 2);
2996 return true;
2997 }
2998 case Intrinsic::x86_sse_cvttss2si:
2999 case Intrinsic::x86_sse_cvttss2si64:
3000 case Intrinsic::x86_sse2_cvttsd2si:
3001 case Intrinsic::x86_sse2_cvttsd2si64: {
3002 bool IsInputDouble;
3003 switch (II->getIntrinsicID()) {
3005 case Intrinsic::x86_sse_cvttss2si:
3006 case Intrinsic::x86_sse_cvttss2si64:
3007 if (!Subtarget->hasSSE1())
3008 return false;
3009 IsInputDouble = false;
3010 break;
3011 case Intrinsic::x86_sse2_cvttsd2si:
3012 case Intrinsic::x86_sse2_cvttsd2si64:
3013 if (!Subtarget->hasSSE2())
3014 return false;
3015 IsInputDouble = true;
3016 break;
3017 }
3018
3019 Type *RetTy = II->getCalledFunction()->getReturnType();
3020 MVT VT;
3021 if (!isTypeLegal(RetTy, VT))
3022 return false;
3023
3024 static const uint16_t CvtOpc[3][2][2] = {
3025 { { X86::CVTTSS2SIrr, X86::CVTTSS2SI64rr },
3026 { X86::CVTTSD2SIrr, X86::CVTTSD2SI64rr } },
3027 { { X86::VCVTTSS2SIrr, X86::VCVTTSS2SI64rr },
3028 { X86::VCVTTSD2SIrr, X86::VCVTTSD2SI64rr } },
3029 { { X86::VCVTTSS2SIZrr, X86::VCVTTSS2SI64Zrr },
3030 { X86::VCVTTSD2SIZrr, X86::VCVTTSD2SI64Zrr } },
3031 };
3032 unsigned AVXLevel = Subtarget->hasAVX512() ? 2 :
3033 Subtarget->hasAVX() ? 1 :
3034 0;
3035 unsigned Opc;
3038 case MVT::i32: Opc = CvtOpc[AVXLevel][IsInputDouble][0]; break;
3039 case MVT::i64: Opc = CvtOpc[AVXLevel][IsInputDouble][1]; break;
3040 }
3041
3042
3043 const Value *Op = II->getArgOperand(0);
3047 break;
3049
3050 if (!Idx) {
3052 break;
3053 }
3055 }
3056
3058 if ()
3059 return false;
3060
3061 Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
3062 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), ResultReg)
3064
3065 updateValueMap(II, ResultReg);
3066 return true;
3067 }
3068 case Intrinsic::x86_sse42_crc32_32_8:
3069 case Intrinsic::x86_sse42_crc32_32_16:
3070 case Intrinsic::x86_sse42_crc32_32_32:
3071 case Intrinsic::x86_sse42_crc32_64_64: {
3072 if (!Subtarget->hasCRC32())
3073 return false;
3074
3075 Type *RetTy = II->getCalledFunction()->getReturnType();
3076
3077 MVT VT;
3078 if (!isTypeLegal(RetTy, VT))
3079 return false;
3080
3081 unsigned Opc;
3082 const TargetRegisterClass *RC = nullptr;
3083
3084 switch (II->getIntrinsicID()) {
3085 default:
3087#define GET_EGPR_IF_ENABLED(OPC) Subtarget->hasEGPR() ? OPC##_EVEX : OPC
3088 case Intrinsic::x86_sse42_crc32_32_8:
3090 RC = &X86::GR32RegClass;
3091 break;
3092 case Intrinsic::x86_sse42_crc32_32_16:
3094 RC = &X86::GR32RegClass;
3095 break;
3096 case Intrinsic::x86_sse42_crc32_32_32:
3098 RC = &X86::GR32RegClass;
3099 break;
3100 case Intrinsic::x86_sse42_crc32_64_64:
3102 RC = &X86::GR64RegClass;
3103 break;
3104#undef GET_EGPR_IF_ENABLED
3105 }
3106
3107 const Value *LHS = II->getArgOperand(0);
3108 const Value *RHS = II->getArgOperand(1);
3109
3112 if (!LHSReg || !RHSReg)
3113 return false;
3114
3115 Register ResultReg = fastEmitInst_rr(Opc, RC, LHSReg, RHSReg);
3116 if (!ResultReg)
3117 return false;
3118
3119 updateValueMap(II, ResultReg);
3120 return true;
3121 }
3122 }
3123}
3124
3125bool X86FastISel::fastLowerArguments() {
3126 if (!FuncInfo.CanLowerReturn)
3127 return false;
3128
3129 const Function *F = FuncInfo.Fn;
3130 if (F->isVarArg())
3131 return false;
3132
3133 CallingConv::ID CC = F->getCallingConv();
3134 if (CC != CallingConv::C)
3135 return false;
3136
3137 if (Subtarget->isCallingConvWin64(CC))
3138 return false;
3139
3140 if (!Subtarget->is64Bit())
3141 return false;
3142
3143 if (Subtarget->useSoftFloat())
3144 return false;
3145
3146
3147 unsigned GPRCnt = 0;
3148 unsigned FPRCnt = 0;
3149 for (auto const &Arg : F->args()) {
3150 if (Arg.hasAttribute(Attribute::ByVal) ||
3151 Arg.hasAttribute(Attribute::InReg) ||
3152 Arg.hasAttribute(Attribute::StructRet) ||
3153 Arg.hasAttribute(Attribute::SwiftSelf) ||
3154 Arg.hasAttribute(Attribute::SwiftAsync) ||
3155 Arg.hasAttribute(Attribute::SwiftError) ||
3156 Arg.hasAttribute(Attribute::Nest))
3157 return false;
3158
3159 Type *ArgTy = Arg.getType();
3161 return false;
3162
3163 EVT ArgVT = TLI.getValueType(DL, ArgTy);
3164 if (!ArgVT.isSimple()) return false;
3166 default: return false;
3167 case MVT::i32:
3168 case MVT::i64:
3169 ++GPRCnt;
3170 break;
3171 case MVT::f32:
3172 case MVT::f64:
3173 if (!Subtarget->hasSSE1())
3174 return false;
3175 ++FPRCnt;
3176 break;
3177 }
3178
3179 if (GPRCnt > 6)
3180 return false;
3181
3182 if (FPRCnt > 8)
3183 return false;
3184 }
3185
3186 static const MCPhysReg GPR32ArgRegs[] = {
3187 X86::EDI, X86::ESI, X86::EDX, X86::ECX, X86::R8D, X86::R9D
3188 };
3189 static const MCPhysReg GPR64ArgRegs[] = {
3190 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8 , X86::R9
3191 };
3192 static const MCPhysReg XMMArgRegs[] = {
3193 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
3194 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
3195 };
3196
3197 unsigned GPRIdx = 0;
3198 unsigned FPRIdx = 0;
3199 for (auto const &Arg : F->args()) {
3200 MVT VT = TLI.getSimpleValueType(DL, Arg.getType());
3201 const TargetRegisterClass *RC = TLI.getRegClassFor(VT);
3202 MCRegister SrcReg;
3205 case MVT::i32: SrcReg = GPR32ArgRegs[GPRIdx++]; break;
3206 case MVT::i64: SrcReg = GPR64ArgRegs[GPRIdx++]; break;
3207 case MVT::f32: [[fallthrough]];
3208 case MVT::f64: SrcReg = XMMArgRegs[FPRIdx++]; break;
3209 }
3210 Register DstReg = FuncInfo.MF->addLiveIn(SrcReg, RC);
3211
3212
3213
3214 Register ResultReg = createResultReg(RC);
3215 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3216 TII.get(TargetOpcode::COPY), ResultReg)
3218 updateValueMap(&Arg, ResultReg);
3219 }
3220 return true;
3221}
3222
3226 if (Subtarget->is64Bit())
3227 return 0;
3229 return 0;
3233 return 0;
3234
3235 if (CB)
3238 return 0;
3239
3240 return 4;
3241}
3242
3243bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
3244 auto &OutVals = CLI.OutVals;
3245 auto &OutFlags = CLI.OutFlags;
3246 auto &OutRegs = CLI.OutRegs;
3247 auto &Ins = CLI.Ins;
3248 auto &InRegs = CLI.InRegs;
3249 CallingConv::ID CC = CLI.CallConv;
3250 bool &IsTailCall = CLI.IsTailCall;
3251 bool IsVarArg = CLI.IsVarArg;
3254 const auto *CB = CLI.CB;
3255
3256 bool Is64Bit = Subtarget->is64Bit();
3257 bool IsWin64 = Subtarget->isCallingConvWin64(CC);
3258
3259
3260
3261 if (CB && CB->doesNoCfCheck())
3262 return false;
3263
3264
3265 if ((CB && isa(CB) && CB->hasFnAttr("no_caller_saved_registers")))
3266 return false;
3267
3268
3269 if ((CB && CB->hasFnAttr("no_callee_saved_registers")))
3270 return false;
3271
3272
3273 if (CB && CB->isIndirectCall() && CB->getOperandBundle(LLVMContext::OB_kcfi))
3274 return false;
3275
3276
3277 if (Subtarget->useIndirectThunkCalls())
3278 return false;
3279
3280
3281 switch (CC) {
3282 default: return false;
3283 case CallingConv::C:
3284 case CallingConv::Fast:
3285 case CallingConv::Tail:
3286 case CallingConv::Swift:
3287 case CallingConv::SwiftTail:
3288 case CallingConv::X86_FastCall:
3289 case CallingConv::X86_StdCall:
3290 case CallingConv::X86_ThisCall:
3291 case CallingConv::Win64:
3292 case CallingConv::X86_64_SysV:
3293 case CallingConv::CFGuard_Check:
3294 break;
3295 }
3296
3297
3298 if (IsTailCall)
3299 return false;
3300
3301
3302
3303 if ((CC == CallingConv::Fast && TM.Options.GuaranteedTailCallOpt) ||
3304 CC == CallingConv::Tail || CC == CallingConv::SwiftTail)
3305 return false;
3306
3307
3308
3309 if (IsVarArg && IsWin64)
3310 return false;
3311
3312
3313 if (CLI.CB && CLI.CB->hasInAllocaArgument())
3314 return false;
3315
3316 for (auto Flag : CLI.OutFlags)
3317 if (Flag.isSwiftError() || Flag.isPreallocated())
3318 return false;
3319
3320
3321 if (Is64Bit &&
3323 return false;
3324
3327 SmallVector<Register, 16> ArgRegs;
3328
3329
3330
3331
3332 for (int i = 0, e = OutVals.size(); i != e; ++i) {
3333 Value *&Val = OutVals[i];
3334 ISD::ArgFlagsTy Flags = OutFlags[i];
3336 if (CI->getBitWidth() < 32) {
3337 if (Flags.isSExt())
3338 Val = ConstantInt::get(CI->getContext(), CI->getValue().sext(32));
3339 else
3340 Val = ConstantInt::get(CI->getContext(), CI->getValue().zext(32));
3341 }
3342 }
3343
3344
3345
3346 MVT VT;
3349 if (TI && TI->getType()->isIntegerTy(1) && CLI.CB &&
3350 (TI->getParent() == CLI.CB->getParent()) && TI->hasOneUse()) {
3351 Value *PrevVal = TI->getOperand(0);
3352 ResultReg = getRegForValue(PrevVal);
3353
3354 if (!ResultReg)
3355 return false;
3356
3357 if (!isTypeLegal(PrevVal->getType(), VT))
3358 return false;
3359
3360 ResultReg = fastEmit_ri(VT, VT, ISD::AND, ResultReg, 1);
3361 } else {
3362 if (!isTypeLegal(Val->getType(), VT) ||
3364 return false;
3365 ResultReg = getRegForValue(Val);
3366 }
3367
3368 if (!ResultReg)
3369 return false;
3370
3374 }
3375
3376
3378 CCState CCInfo(CC, IsVarArg, *FuncInfo.MF, ArgLocs, CLI.RetTy->getContext());
3379
3380
3381 if (IsWin64)
3382 CCInfo.AllocateStack(32, Align(8));
3383
3384 CCInfo.AnalyzeCallOperands(OutVTs, OutFlags, ArgTys, CC_X86);
3385
3386
3387 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
3388
3389
3391 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(AdjStackDown))
3393
3394
3395 const X86RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
3396 for (const CCValAssign &VA : ArgLocs) {
3398 MVT ArgVT = OutVTs[VA.getValNo()];
3399
3400 if (ArgVT == MVT::x86mmx)
3401 return false;
3402
3404
3405
3410 "Unexpected extend");
3411
3412 if (ArgVT == MVT::i1)
3413 return false;
3414
3416 ArgVT, ArgReg);
3417 assert(Emitted && "Failed to emit a sext!"); (void)Emitted;
3419 break;
3420 }
3423 "Unexpected extend");
3424
3425
3426 if (ArgVT == MVT::i1) {
3427
3428 ArgReg = fastEmitZExtFromI1(MVT::i8, ArgReg);
3429 ArgVT = MVT::i8;
3430
3431 if (!ArgReg)
3432 return false;
3433 }
3434
3436 ArgVT, ArgReg);
3437 assert(Emitted && "Failed to emit a zext!"); (void)Emitted;
3439 break;
3440 }
3443 "Unexpected extend");
3445 ArgVT, ArgReg);
3446 if (!Emitted)
3448 ArgVT, ArgReg);
3449 if (!Emitted)
3451 ArgVT, ArgReg);
3452
3453 assert(Emitted && "Failed to emit a aext!"); (void)Emitted;
3455 break;
3456 }
3458 ArgReg = fastEmit_r(ArgVT, VA.getLocVT(), ISD::BITCAST, ArgReg);
3459 assert(ArgReg && "Failed to emit a bitcast!");
3461 break;
3462 }
3464
3465
3466 return false;
3474
3475
3476 return false;
3477 }
3478
3480 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3482 OutRegs.push_back(VA.getLocReg());
3483 } else {
3485
3486
3488 continue;
3489
3491 X86AddressMode AM;
3493 AM.Disp = LocMemOffset;
3494 ISD::ArgFlagsTy Flags = OutFlags[VA.getValNo()];
3495 Align Alignment = DL.getABITypeAlign(ArgVal->getType());
3496 MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
3499 if (Flags.isByVal()) {
3500 X86AddressMode SrcAM;
3502 if (!TryEmitSmallMemcpy(AM, SrcAM, Flags.getByValSize()))
3503 return false;
3505
3506
3507
3508 if (!X86FastEmitStore(ArgVT, ArgVal, AM, MMO))
3509 return false;
3510 } else {
3511 if (!X86FastEmitStore(ArgVT, ArgReg, AM, MMO))
3512 return false;
3513 }
3514 }
3515 }
3516
3517
3518
3519 if (Subtarget->isPICStyleGOT()) {
3520 Register Base = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
3521 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3523 }
3524
3525 if (Is64Bit && IsVarArg && !IsWin64) {
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535 static const MCPhysReg XMMArgRegs[] = {
3536 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
3537 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
3538 };
3539 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs);
3540 assert((Subtarget->hasSSE1() || !NumXMMRegs)
3541 && "SSE registers cannot be used when SSE is disabled");
3542 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::MOV8ri),
3543 X86::AL).addImm(NumXMMRegs);
3544 }
3545
3546
3547
3548 X86AddressMode CalleeAM;
3549 if (!X86SelectCallAddress(Callee, CalleeAM))
3550 return false;
3551
3553 const GlobalValue *GV = nullptr;
3554 if (CalleeAM.GV != nullptr) {
3555 GV = CalleeAM.GV;
3556 } else if (CalleeAM.Base.Reg) {
3557 CalleeOp = CalleeAM.Base.Reg;
3558 } else
3559 return false;
3560
3561
3562 MachineInstrBuilder MIB;
3563 if (CalleeOp) {
3564
3565 unsigned CallOpc = Is64Bit ? X86::CALL64r : X86::CALL32r;
3566 MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(CallOpc))
3568 } else {
3569
3570 assert(GV && "Not a direct call");
3571
3572 unsigned char OpFlags = Subtarget->classifyGlobalFunctionReference(GV);
3577
3578
3579
3584 unsigned CallOpc = NeedLoad
3585 ? (Is64Bit ? X86::CALL64m : X86::CALL32m)
3586 : (Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32);
3587
3588 MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(CallOpc));
3589 if (NeedLoad)
3590 MIB.addReg(Is64Bit ? X86::RIP : X86::NoRegister).addImm(1).addReg(0);
3591 if (Symbol)
3592 MIB.addSym(Symbol, OpFlags);
3593 else
3595 if (NeedLoad)
3597 }
3598
3599
3600
3601 MIB.addRegMask(TRI.getCallPreservedMask(*FuncInfo.MF, CC));
3602
3603
3604 if (Subtarget->isPICStyleGOT())
3606
3607 if (Is64Bit && IsVarArg && !IsWin64)
3609
3610
3611 for (auto Reg : OutRegs)
3613
3614
3615 unsigned NumBytesForCalleeToPop =
3617 TM.Options.GuaranteedTailCallOpt)
3618 ? NumBytes
3621 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(AdjStackUp))
3622 .addImm(NumBytes).addImm(NumBytesForCalleeToPop);
3623
3624
3626 CCState CCRetInfo(CC, IsVarArg, *FuncInfo.MF, RVLocs,
3627 CLI.RetTy->getContext());
3628 CCRetInfo.AnalyzeCallResult(Ins, RetCC_X86);
3629
3630
3631 Register ResultReg = FuncInfo.CreateRegs(CLI.RetTy);
3632 for (unsigned i = 0; i != RVLocs.size(); ++i) {
3633 CCValAssign &VA = RVLocs[i];
3635 Register CopyReg = ResultReg + i;
3637
3638
3639 if ((CopyVT == MVT::f32 || CopyVT == MVT::f64) &&
3640 ((Is64Bit || Ins[i].Flags.isInReg()) && !Subtarget->hasSSE1())) {
3642 }
3643
3644
3645
3646 if ((SrcReg == X86::FP0 || SrcReg == X86::FP1) &&
3647 isScalarFPTypeInSSEReg(VA.getValVT())) {
3648 CopyVT = MVT::f80;
3649 CopyReg = createResultReg(&X86::RFP80RegClass);
3650 }
3651
3652
3653 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3654 TII.get(TargetOpcode::COPY), CopyReg).addReg(SrcReg);
3655 InRegs.push_back(VA.getLocReg());
3656
3657
3658
3659
3660 if (CopyVT != VA.getValVT()) {
3662 unsigned Opc = ResVT == MVT::f32 ? X86::ST_Fp80m32 : X86::ST_Fp80m64;
3668 Opc = ResVT == MVT::f32 ? X86::MOVSSrm_alt : X86::MOVSDrm_alt;
3670 TII.get(Opc), ResultReg + i), FI);
3671 }
3672 }
3673
3674 CLI.ResultReg = ResultReg;
3675 CLI.NumResultRegs = RVLocs.size();
3676 CLI.Call = MIB;
3677
3678
3679 if (TM.Options.EmitCallGraphSection && CB && CB->isIndirectCall()) {
3680 MachineFunction::CallSiteInfo CSInfo(*CB);
3682 }
3683
3684 return true;
3685}
3686
3687bool
3688X86FastISel::fastSelectInstruction(const Instruction *I) {
3689 switch (I->getOpcode()) {
3690 default: break;
3691 case Instruction::Load:
3692 return X86SelectLoad(I);
3693 case Instruction::Store:
3694 return X86SelectStore(I);
3695 case Instruction::Ret:
3696 return X86SelectRet(I);
3697 case Instruction::ICmp:
3698 case Instruction::FCmp:
3699 return X86SelectCmp(I);
3700 case Instruction::ZExt:
3701 return X86SelectZExt(I);
3702 case Instruction::SExt:
3703 return X86SelectSExt(I);
3704 case Instruction::Br:
3705 return X86SelectBranch(I);
3706 case Instruction::LShr:
3707 case Instruction::AShr:
3708 case Instruction::Shl:
3709 return X86SelectShift(I);
3710 case Instruction::SDiv:
3711 case Instruction::UDiv:
3712 case Instruction::SRem:
3713 case Instruction::URem:
3714 return X86SelectDivRem(I);
3715 case Instruction::Select:
3716 return X86SelectSelect(I);
3717 case Instruction::Trunc:
3718 return X86SelectTrunc(I);
3719 case Instruction::FPExt:
3720 return X86SelectFPExt(I);
3721 case Instruction::FPTrunc:
3722 return X86SelectFPTrunc(I);
3723 case Instruction::SIToFP:
3724 return X86SelectSIToFP(I);
3725 case Instruction::UIToFP:
3726 return X86SelectUIToFP(I);
3727 case Instruction::IntToPtr:
3728 case Instruction::PtrToInt: {
3729 EVT SrcVT = TLI.getValueType(DL, I->getOperand(0)->getType());
3730 EVT DstVT = TLI.getValueType(DL, I->getType());
3731 if (DstVT.bitsGT(SrcVT))
3732 return X86SelectZExt(I);
3733 if (DstVT.bitsLT(SrcVT))
3734 return X86SelectTrunc(I);
3735 Register Reg = getRegForValue(I->getOperand(0));
3736 if ()
3737 return false;
3739 return true;
3740 }
3741 case Instruction::BitCast:
3742 return X86SelectBitCast(I);
3743 }
3744
3745 return false;
3746}
3747
3748Register X86FastISel::X86MaterializeInt(const ConstantInt *CI, MVT VT) {
3749 if (VT > MVT::i64)
3751
3753 if (Imm == 0) {
3754 Register SrcReg = fastEmitInst_(X86::MOV32r0, &X86::GR32RegClass);
3757 case MVT::i1:
3758 case MVT::i8:
3759 return fastEmitInst_extractsubreg(MVT::i8, SrcReg, X86::sub_8bit);
3760 case MVT::i16:
3761 return fastEmitInst_extractsubreg(MVT::i16, SrcReg, X86::sub_16bit);
3762 case MVT::i32:
3763 return SrcReg;
3764 case MVT::i64: {
3765 Register ResultReg = createResultReg(&X86::GR64RegClass);
3766 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3767 TII.get(TargetOpcode::SUBREG_TO_REG), ResultReg)
3769 return ResultReg;
3770 }
3771 }
3772 }
3773
3774 unsigned Opc = 0;
3777 case MVT::i1:
3778 VT = MVT::i8;
3779 [[fallthrough]];
3780 case MVT::i8: Opc = X86::MOV8ri; break;
3781 case MVT::i16: Opc = X86::MOV16ri; break;
3782 case MVT::i32: Opc = X86::MOV32ri; break;
3783 case MVT::i64: {
3785 Opc = X86::MOV32ri64;
3787 Opc = X86::MOV64ri32;
3788 else
3789 Opc = X86::MOV64ri;
3790 break;
3791 }
3792 }
3793 return fastEmitInst_i(Opc, TLI.getRegClassFor(VT), Imm);
3794}
3795
3796Register X86FastISel::X86MaterializeFP(const ConstantFP *CFP, MVT VT) {
3798 return fastMaterializeFloatZero(CFP);
3799
3800
3805
3806
3807 unsigned Opc = 0;
3808 bool HasSSE1 = Subtarget->hasSSE1();
3809 bool HasSSE2 = Subtarget->hasSSE2();
3810 bool HasAVX = Subtarget->hasAVX();
3811 bool HasAVX512 = Subtarget->hasAVX512();
3813 default:
3815 case MVT::f32:
3816 Opc = HasAVX512 ? X86::VMOVSSZrm_alt
3817 : HasAVX ? X86::VMOVSSrm_alt
3818 : HasSSE1 ? X86::MOVSSrm_alt
3819 : X86::LD_Fp32m;
3820 break;
3821 case MVT::f64:
3822 Opc = HasAVX512 ? X86::VMOVSDZrm_alt
3823 : HasAVX ? X86::VMOVSDrm_alt
3824 : HasSSE2 ? X86::MOVSDrm_alt
3825 : X86::LD_Fp64m;
3826 break;
3827 case MVT::f80:
3828
3830 }
3831
3832
3833 Align Alignment = DL.getPrefTypeAlign(CFP->getType());
3834
3835
3837 unsigned char OpFlag = Subtarget->classifyLocalReference(nullptr);
3839 PICBase = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
3841 PICBase = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
3842 else if (Subtarget->is64Bit() && TM.getCodeModel() != CodeModel::Large)
3843 PICBase = X86::RIP;
3844
3845
3846 unsigned CPI = MCP.getConstantPoolIndex(CFP, Alignment);
3847 Register ResultReg = createResultReg(TLI.getRegClassFor(VT.SimpleTy));
3848
3849
3851 Register AddrReg = createResultReg(&X86::GR64RegClass);
3852 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::MOV64ri),
3853 AddrReg)
3855 MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
3857 addRegReg(MIB, AddrReg, false, X86::NoSubRegister, PICBase, false,
3858 X86::NoSubRegister);
3859 MachineMemOperand *MMO = FuncInfo.MF->getMachineMemOperand(
3863 return ResultReg;
3864 }
3865
3868 CPI, PICBase, OpFlag);
3869 return ResultReg;
3870}
3871
3872Register X86FastISel::X86MaterializeGV(const GlobalValue *GV, MVT VT) {
3873
3877 if (TM.isLargeGlobalValue(GV))
3879
3880
3881 X86AddressMode AM;
3883
3884
3886 AM.IndexReg == 0 && AM.Disp == 0 && AM.GV == nullptr)
3888
3889 Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
3891 TLI.getPointerTy(DL) == MVT::i64) {
3892
3893
3894 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(X86::MOV64ri),
3895 ResultReg)
3897 } else {
3898 unsigned Opc =
3899 TLI.getPointerTy(DL) == MVT::i32
3900 ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r)
3901 : X86::LEA64r;
3904 }
3905 return ResultReg;
3906 }
3908}
3909
3910Register X86FastISel::fastMaterializeConstant(const Constant *C) {
3911 EVT CEVT = TLI.getValueType(DL, C->getType(), true);
3912
3913
3917
3919 return X86MaterializeInt(CI, VT);
3921 return X86MaterializeFP(CFP, VT);
3923 return X86MaterializeGV(GV, VT);
3925 unsigned Opc = 0;
3927 default:
3928 break;
3929 case MVT::f32:
3930 if (!Subtarget->hasSSE1())
3931 Opc = X86::LD_Fp032;
3932 break;
3933 case MVT::f64:
3934 if (!Subtarget->hasSSE2())
3935 Opc = X86::LD_Fp064;
3936 break;
3937 case MVT::f80:
3938 Opc = X86::LD_Fp080;
3939 break;
3940 }
3941
3942 if (Opc) {
3943 Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
3944 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc),
3945 ResultReg);
3946 return ResultReg;
3947 }
3948 }
3949
3951}
3952
3953Register X86FastISel::fastMaterializeAlloca(const AllocaInst *C) {
3954
3955
3956
3957
3958
3959
3960
3961 if (!FuncInfo.StaticAllocaMap.count(C))
3963 assert(C->isStaticAlloca() && "dynamic alloca in the static alloca map?");
3964
3965 X86AddressMode AM;
3968 unsigned Opc =
3969 TLI.getPointerTy(DL) == MVT::i32
3970 ? (Subtarget->isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r)
3971 : X86::LEA64r;
3972 const TargetRegisterClass *RC = TLI.getRegClassFor(TLI.getPointerTy(DL));
3973 Register ResultReg = createResultReg(RC);
3976 return ResultReg;
3977}
3978
3979Register X86FastISel::fastMaterializeFloatZero(const ConstantFP *CF) {
3980 MVT VT;
3981 if (!isTypeLegal(CF->getType(), VT))
3983
3984
3985 bool HasSSE1 = Subtarget->hasSSE1();
3986 bool HasSSE2 = Subtarget->hasSSE2();
3987 bool HasAVX512 = Subtarget->hasAVX512();
3988 unsigned Opc = 0;
3990 default: return 0;
3991 case MVT::f16:
3992 Opc = HasAVX512 ? X86::AVX512_FsFLD0SH : X86::FsFLD0SH;
3993 break;
3994 case MVT::f32:
3995 Opc = HasAVX512 ? X86::AVX512_FsFLD0SS
3996 : HasSSE1 ? X86::FsFLD0SS
3997 : X86::LD_Fp032;
3998 break;
3999 case MVT::f64:
4000 Opc = HasAVX512 ? X86::AVX512_FsFLD0SD
4001 : HasSSE2 ? X86::FsFLD0SD
4002 : X86::LD_Fp064;
4003 break;
4004 case MVT::f80:
4005
4007 }
4008
4009 Register ResultReg = createResultReg(TLI.getRegClassFor(VT));
4010 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), ResultReg);
4011 return ResultReg;
4012}
4013
4014bool X86FastISel::tryToFoldLoadIntoMI(MachineInstr *MI, unsigned OpNo,
4015 const LoadInst *LI) {
4017 X86AddressMode AM;
4019 return false;
4020
4021 const X86InstrInfo &XII = (const X86InstrInfo &)TII;
4022
4023 unsigned Size = DL.getTypeAllocSize(LI->getType());
4024
4027
4029 *FuncInfo.MF, *MI, OpNo, AddrOps, FuncInfo.InsertPt, Size, LI->getAlign(),
4030 true);
4031 if (!Result)
4032 return false;
4033
4034
4035
4036
4037
4038
4039 unsigned OperandNo = 0;
4041 E = Result->operands_end(); I != E; ++I, ++OperandNo) {
4042 MachineOperand &MO = *I;
4044 continue;
4045
4047 MO.getReg(), OperandNo);
4048 if (IndexReg == MO.getReg())
4049 continue;
4050 MO.setReg(IndexReg);
4051 }
4052
4053 if (MI->isCall())
4054 FuncInfo.MF->moveAdditionalCallInfo(MI, Result);
4055 Result->addMemOperand(*FuncInfo.MF, createMachineMemOperandFor(LI));
4056 Result->cloneInstrSymbols(*FuncInfo.MF, *MI);
4058 removeDeadCode(I, std::next(I));
4059 return true;
4060}
4061
4062Register X86FastISel::fastEmitInst_rrrr(unsigned MachineInstOpcode,
4063 const TargetRegisterClass *RC,
4066 const MCInstrDesc &II = TII.get(MachineInstOpcode);
4067
4068 Register ResultReg = createResultReg(RC);
4073
4074 if (II.getNumDefs() >= 1)
4075 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II, ResultReg)
4080 else {
4081 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, II)
4086 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(TargetOpcode::COPY),
4087 ResultReg)
4088 .addReg(II.implicit_defs()[0]);
4089 }
4090 return ResultReg;
4091}
4092
4093namespace llvm {
4096 return new X86FastISel(funcInfo, libInfo);
4097 }
4098}
unsigned const MachineRegisterInfo * MRI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const TargetInstrInfo & TII
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file defines the FastISel class.
Module.h This file contains the declarations for the Module class.
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
Register const TargetRegisterInfo * TRI
Promote Memory to Register
uint64_t IntrinsicInst * II
const SmallVectorImpl< MachineOperand > & Cond
#define GET_EGPR_IF_ENABLED(OPC)
static unsigned X86ChooseCmpImmediateOpcode(EVT VT, const ConstantInt *RHSC)
If we have a comparison with RHS as the RHS of the comparison, return an opcode that works for the co...
Definition X86FastISel.cpp:1384
static std::pair< unsigned, bool > getX86SSEConditionCode(CmpInst::Predicate Predicate)
Definition X86FastISel.cpp:176
static unsigned computeBytesPoppedByCalleeForSRet(const X86Subtarget *Subtarget, CallingConv::ID CC, const CallBase *CB)
Definition X86FastISel.cpp:3223
static unsigned X86ChooseCmpOpcode(EVT VT, const X86Subtarget *Subtarget)
Definition X86FastISel.cpp:1357
static bool X86SelectAddress(MachineInstr &I, const X86TargetMachine &TM, const MachineRegisterInfo &MRI, const X86Subtarget &STI, X86AddressMode &AM)
InstListType::const_iterator const_iterator
BasicBlock * getSuccessor(unsigned i) const
Value * getCondition() const
Register getLocReg() const
LocInfo getLocInfo() const
int64_t getLocMemOffset() const
unsigned getValNo() const
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
LLVM_ABI bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
@ FCMP_ULT
1 1 0 0 True if unordered or less than
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Predicate getPredicate() const
Return the predicate for this instruction.
This is the shared class of boolean and integer constants.
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
LLVM_ABI bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
bool isValidLocationForIntrinsic(const DILocation *DL) const
Check that a location is valid for this variable.
Value * getAddress() const
DILocalVariable * getVariable() const
DIExpression * getExpression() const
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Module * getParent()
Get the module that this global value is contained inside of...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI bool isAtomic() const LLVM_READONLY
Return true if this instruction has an AtomicOrdering of unordered or higher.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
bool usesWindowsCFI() const
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
MVT getVectorElementType() const
MachineInstrBundleIterator< MachineInstr > iterator
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setStackProtectorIndex(int I)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Function & getFunction()
Return the LLVM function that this machine code represents.
void addCallSiteInfo(const MachineInstr *CallI, CallSiteInfo &&CallInfo)
Start tracking the arguments passed to the call CallI.
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMetadata(const MDNode *MD) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
unsigned getNumOperands() const
Retuns the total number of operands.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
MachineOperand * mop_iterator
iterator/begin/end - Iterate over all operands of a machine instruction.
LLVM_ABI void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)
Add a MachineMemOperand to the machine instruction.
bool isNonTemporal() const
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
Register getReg() const
getReg - Returns the register number.
Value * getLength() const
Value * getRawDest() const
unsigned getDestAddressSpace() const
Value * getRawSource() const
Return the arguments to the instruction.
unsigned getSourceAddressSpace() const
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
void push_back(const T &Elt)
Value * getValueOperand()
Value * getPointerOperand()
TypeSize getElementOffset(unsigned Idx) const
unsigned getCallFrameSetupOpcode() const
These methods return the opcode of the frame setup/destroy instructions if they exist (-1 otherwise).
unsigned getCallFrameDestroyOpcode() const
Provides information about what library functions are available for the current target.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
bool contains(Register Reg) const
Return true if the specified register is included in this register class.
bool isOSMSVCRT() const
Is this a "Windows" OS targeting a "MSVCRT.dll" environment.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isArrayTy() const
True if this is an instance of ArrayType.
bool isStructTy() const
True if this is an instance of StructType.
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
const Use * const_op_iterator
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
Fold a load or store of the specified stack slot into the specified machine instruction for the speci...
Register getSRetReturnReg() const
unsigned getBytesToPopOnReturn() const
Register getPtrSizedFrameRegister(const MachineFunction &MF) const
Register getStackRegister() const
const Triple & getTargetTriple() const
StructType * getStructTypeOrNull() const
TypeSize getSequentialElementStride(const DataLayout &DL) const
const ParentTy * getParent() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ HiPE
Used by the High-Performance Erlang Compiler (HiPE).
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ SwiftTail
This follows the Swift calling convention in how arguments are passed but guarantees tail calls will ...
@ C
The default llvm calling convention, compatible with C.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ ADD
Simple integer binary arithmetic operators.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ BasicBlock
Various leaf nodes.
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ AND
Bitwise operators - logical and, logical or, logical xor.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Kill
The last use of a register.
@ X86
Windows x64, Windows Itanium (IA-64)
@ MO_GOTPCREL_NORELAX
MO_GOTPCREL_NORELAX - Same as MO_GOTPCREL except that R_X86_64_GOTPCREL relocations are guaranteed to...
@ MO_GOTOFF
MO_GOTOFF - On a symbol operand this indicates that the immediate is the offset to the location of th...
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
@ MO_PLT
MO_PLT - On a symbol operand this indicates that the immediate is offset to the PLT entry of symbol n...
@ MO_NO_FLAG
MO_NO_FLAG - No flag for the operand.
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand "FOO", this indicates that the reference is actually to the "__imp...
@ MO_PIC_BASE_OFFSET
MO_PIC_BASE_OFFSET - On a symbol operand this indicates that the immediate should get the value of th...
@ MO_GOTPCREL
MO_GOTPCREL - On a symbol operand this indicates that the immediate is offset to the GOT entry for th...
std::pair< CondCode, bool > getX86ConditionCode(CmpInst::Predicate Predicate)
Return a pair of condition code for the given predicate and whether the instruction operands should b...
bool isCalleePop(CallingConv::ID CallingConv, bool is64Bit, bool IsVarArg, bool GuaranteeTCO)
Determines whether the callee is required to pop its own arguments.
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
Definition X86FastISel.cpp:4094
unsigned getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand=false, bool HasNDD=false)
Return a cmov opcode for the given register size in bytes, and operand type.
StringMapEntry< std::atomic< TypeEntryBody * > > TypeEntry
@ User
could "use" a pointer
@ Emitted
Assigned address, still materializing.
friend class Instruction
Iterator for Instructions in a `BasicBlock.
This is an optimization pass for GlobalISel generic memory operations.
static bool isGlobalStubReference(unsigned char TargetFlag)
isGlobalStubReference - Return true if the specified TargetFlag operand is a reference to a stub for ...
FunctionAddr VTableAddr Value
static bool isGlobalRelativeToPICBase(unsigned char TargetFlag)
isGlobalRelativeToPICBase - Return true if the specified global value reference is relative to a 32-b...
LLVM_ABI Register constrainOperandRegClass(const MachineFunction &MF, const TargetRegisterInfo &TRI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII, const RegisterBankInfo &RBI, MachineInstr &InsertPt, const TargetRegisterClass &RegClass, MachineOperand &RegMO)
Constrain the Register operand OpIdx, so that it is now constrained to the TargetRegisterClass passed...
LLVM_ABI void GetReturnInfo(CallingConv::ID CC, Type *ReturnType, AttributeList attr, SmallVectorImpl< ISD::OutputArg > &Outs, const TargetLowering &TLI, const DataLayout &DL)
Given an LLVM IR type and return type attributes, compute the return value EVTs and flags,...
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
decltype(auto) dyn_cast(const From &Val)
dyn_cast - Return the argument parameter cast to the specified type.
auto successors(const MachineBasicBlock *BB)
static const MachineInstrBuilder & addConstantPoolReference(const MachineInstrBuilder &MIB, unsigned CPI, Register GlobalBaseReg, unsigned char OpFlags)
addConstantPoolReference - This function is used to add a reference to the base of a constant value s...
static const MachineInstrBuilder & addRegReg(const MachineInstrBuilder &MIB, Register Reg1, bool isKill1, unsigned SubReg1, Register Reg2, bool isKill2, unsigned SubReg2)
addRegReg - This function is used to add a memory reference of the form: [Reg + Reg].
static const MachineInstrBuilder & addFrameReference(const MachineInstrBuilder &MIB, int FI, int Offset=0, bool mem=true)
addFrameReference - This function is used to add a reference to the base of an abstract object on the...
static const MachineInstrBuilder & addFullAddress(const MachineInstrBuilder &MIB, const X86AddressMode &AM)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
generic_gep_type_iterator<> gep_type_iterator
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa - Return true if the parameter to the template is an instance of one of the template type argu...
unsigned getKillRegState(bool B)
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
DWARFExpression::Operation Op
bool CC_X86(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
bool RetCC_X86(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
decltype(auto) cast(const From &Val)
cast - Return the argument parameter cast to the specified type.
gep_type_iterator gep_type_begin(const User *GEP)
static const MachineInstrBuilder & addDirectMem(const MachineInstrBuilder &MIB, Register Reg)
addDirectMem - This function is used to add a direct memory reference to the current instruction – th...
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static LLVM_ABI MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static LLVM_ABI MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
X86AddressMode - This struct holds a generalized full x86 address mode.
void getFullAddress(SmallVectorImpl< MachineOperand > &MO)
union llvm::X86AddressMode::BaseUnion Base
enum llvm::X86AddressMode::@202116273335065351270200035056227005202106004277 BaseType