LLVM: lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
80#include "llvm/IR/IntrinsicsWebAssembly.h"
104#include
105#include
106#include
107#include
108#include
109#include
110#include
111#include
112#include
113
114using namespace llvm;
115
116#define DEBUG_TYPE "isel"
117#define ISEL_DUMP_DEBUG_TYPE DEBUG_TYPE "-dump"
118
119STATISTIC(NumFastIselFailures, "Number of instructions fast isel failed on");
120STATISTIC(NumFastIselSuccess, "Number of instructions fast isel selected");
121STATISTIC(NumFastIselBlocks, "Number of blocks selected entirely by fast isel");
122STATISTIC(NumDAGBlocks, "Number of blocks selected using DAG");
123STATISTIC(NumDAGIselRetries,"Number of times dag isel has to try another path");
124STATISTIC(NumEntryBlocks, "Number of entry blocks encountered");
126 "Number of entry blocks where fast isel failed to lower arguments");
127
130 cl::desc("Enable abort calls when \"fast\" instruction selection "
131 "fails to lower an instruction: 0 disable the abort, 1 will "
132 "abort but for args, calls and terminators, 2 will also "
133 "abort for argument lowering, and 3 will never fallback "
134 "to SelectionDAG."));
135
137 "fast-isel-report-on-fallback", cl::Hidden,
138 cl::desc("Emit a diagnostic when \"fast\" instruction selection "
139 "falls back to SelectionDAG."));
140
143 cl::desc("use Machine Branch Probability Info"),
145
146#ifndef NDEBUG
149 cl::desc("Print DAGs with sorted nodes in debug dump"),
151
154 cl::desc("Only display the basic block whose name "
155 "matches this for all view-*-dags options"));
158 cl::desc("Pop up a window to show dags before the first "
159 "dag combine pass"));
162 cl::desc("Pop up a window to show dags before legalize types"));
165 cl::desc("Pop up a window to show dags before the post "
166 "legalize types dag combine pass"));
169 cl::desc("Pop up a window to show dags before legalize"));
172 cl::desc("Pop up a window to show dags before the second "
173 "dag combine pass"));
176 cl::desc("Pop up a window to show isel dags as they are selected"));
179 cl::desc("Pop up a window to show sched dags as they are processed"));
182 cl::desc("Pop up a window to show SUnit dags after they are processed"));
183#else
188#endif
189
190#ifndef NDEBUG
191#define ISEL_DUMP(X) \
192 do { \
193 if (llvm::DebugFlag && \
194 (isCurrentDebugType(DEBUG_TYPE) || \
195 (isCurrentDebugType(ISEL_DUMP_DEBUG_TYPE) && MatchFilterFuncName))) { \
196 X; \
197 } \
198 } while (false)
199#else
200#define ISEL_DUMP(X) do { } while (false)
201#endif
202
203
204
205
206
207
210
211
212
213
214
215
220 cl::desc("Instruction schedulers available (before register"
221 " allocation):"));
222
226
228
229
230
231
233 return Arg.hasAttribute(Attribute::AttrKind::SwiftAsync);
234 });
235}
236
240 return true;
246 }
247 return false;
248}
249
250namespace llvm {
251
252
253
254
258 bool SavedFastISel;
259
260 public:
262 : IS(ISel) {
263 SavedOptLevel = IS.OptLevel;
264 SavedFastISel = IS.TM.Options.EnableFastISel;
265 if (NewOptLevel != SavedOptLevel) {
266 IS.OptLevel = NewOptLevel;
267 IS.TM.setOptLevel(NewOptLevel);
268 LLVM_DEBUG(dbgs() << "\nChanging optimization level for Function "
269 << IS.MF->getFunction().getName() << "\n");
270 LLVM_DEBUG(dbgs() << "\tBefore: -O" << static_cast<int>(SavedOptLevel)
271 << " ; After: -O" << static_cast<int>(NewOptLevel)
272 << "\n");
274 IS.TM.setFastISel(IS.TM.getO0WantsFastISel());
275 }
277 IS.TM.setFastISel(false);
279 dbgs() << "\tFastISel is "
280 << (IS.TM.Options.EnableFastISel ? "enabled" : "disabled")
281 << "\n");
282 }
283
285 if (IS.OptLevel == SavedOptLevel)
286 return;
287 LLVM_DEBUG(dbgs() << "\nRestoring optimization level for Function "
288 << IS.MF->getFunction().getName() << "\n");
289 LLVM_DEBUG(dbgs() << "\tBefore: -O" << static_cast<int>(IS.OptLevel)
290 << " ; After: -O" << static_cast<int>(SavedOptLevel) << "\n");
291 IS.OptLevel = SavedOptLevel;
292 IS.TM.setOptLevel(SavedOptLevel);
293 IS.TM.setFastISel(SavedFastISel);
294 }
295 };
296
297
298
299
304
305
306 if (auto *SchedulerCtor = ST.getDAGScheduler(OptLevel)) {
307 return SchedulerCtor(IS, OptLevel);
308 }
309
311 (ST.enableMachineScheduler() && ST.enableMachineSchedDefaultSched()) ||
325 "Unknown sched type!");
327 }
328
329}
330
334#ifndef NDEBUG
335 dbgs() << "If a target marks an instruction with "
336 "'usesCustomInserter', it must implement "
337 "TargetLowering::EmitInstrWithCustomInserter!\n";
338#endif
340}
341
344 assert(.hasPostISelHook() &&
345 "If a target marks an instruction with 'hasPostISelHook', "
346 "it must implement TargetLowering::AdjustInstrPostInstrSelection!");
347}
348
349
350
351
352
354 char &ID, std::unique_ptr S)
361}
362
364
366 return false;
367
368
371
372
373
375
376
377
378
379
380
381 Selector->TM.resetTargetOptions(MF.getFunction());
382
385 : Selector->OptLevel;
386
387 Selector->MF = &MF;
389 Selector->initializeAnalysisResults(*this);
390 return Selector->runOnMachineFunction(MF);
391}
392
406
408
411 bool RegisterPGOPasses = maintainPGOProfile(Selector->TM, Selector->OptLevel);
420 if (UseMBPI && RegisterPGOPasses)
423
424
427 if (RegisterPGOPasses)
430}
431
435
438
439
442
443
444
446
447
448
449
450
451
452 Selector->TM.resetTargetOptions(MF.getFunction());
453
454
455 Selector->MF = &MF;
456
459 : Selector->OptLevel;
460
462 Selector->initializeAnalysisResults(MFAM);
463 Selector->runOnMachineFunction(MF);
464
466}
467
471 .getManager();
474#ifndef NDEBUG
477#else
479#endif
480
482 TII = MF->getSubtarget().getInstrInfo();
483 TLI = MF->getSubtarget().getTargetLowering();
487 ORE = std::make_unique(&Fn);
492 if (PSI && PSI->hasProfileSummary() && RegisterPGOPasses)
494
498
502
504
505
506
507
508
509
510 if (UseMBPI && RegisterPGOPasses)
512 else
514
517 else
519
521
523}
524
527#ifndef NDEBUG
530#else
532#endif
533
535 TII = MF->getSubtarget().getInstrInfo();
536 TLI = MF->getSubtarget().getTargetLowering();
540 : nullptr;
541 ORE = std::make_unique(&Fn);
545 if (PSI && PSI->hasProfileSummary() && RegisterPGOPasses)
547
551
554 UA = &UAPass->getUniformityInfo();
555
558
560
561
562
563
564
565
566 if (UseMBPI && RegisterPGOPasses)
569 else
571
574 else
576
578
580}
581
585
587
589
591
593
594 MF->setHasInlineAsm(false);
595
597
598
599
602
603
606 continue;
607
608 const Instruction *Term = BB.getTerminator();
610 continue;
611
612
614 break;
615 }
616 }
617
620
621 TLI->initializeSplitCSR(EntryMBB);
622
623 SelectAllBasicBlocks(Fn);
627 }
628
629
630
631
632
633
634
637 E = FuncInfo->RegFixups.end();
641
642
643 while (true) {
645 if (J == E)
646 break;
647 To = J->second;
648 }
649
651 MRI.constrainRegClass(To, MRI.getRegClass(From));
652
653
654
655
656
657 if (.use_empty(To))
658 MRI.clearKillFlags(From);
659 MRI.replaceRegWith(From, To);
660 }
661
662
663
664
667
668
671
673 if (.succ_empty())
674 continue;
675
677 if (Term != MBB.end() && Term->isReturn()) {
679 continue;
680 }
681 }
682 TLI->insertCopiesSplitCSR(EntryMBB, Returns);
683 }
684
686 if (->ArgDbgValues.empty())
687 for (std::pair<MCRegister, Register> LI : RegInfo->liveins())
688 if (LI.second)
689 LiveInMap.insert(LI);
690
691
692 for (unsigned i = 0, e = FuncInfo->ArgDbgValues.size(); i != e; ++i) {
694 assert(MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
695 "Function parameters should not be described by DBG_VALUE_LIST.");
696 bool hasFI = MI->getDebugOperand(0).isFI();
698 hasFI ? TRI.getFrameRegister(*MF) : MI->getDebugOperand(0).getReg();
699 if (Reg.isPhysical())
701 else {
703 if (Def) {
705
706 Def->getParent()->insert(std::next(InsertPos), MI);
707 } else
708 LLVM_DEBUG(dbgs() << "Dropping debug info for dead vreg"
710 }
711
712
713 if (InstrRef)
714 continue;
715
716
717 if (!Reg.isPhysical())
718 continue;
720 if (LDI != LiveInMap.end()) {
721 assert(!hasFI && "There's no handling of frame pointer updating here yet "
722 "- add if needed");
725 const MDNode *Variable = MI->getDebugVariable();
726 const MDNode *Expr = MI->getDebugExpression();
728 bool IsIndirect = MI->isIndirectDebugValue();
729 if (IsIndirect)
730 assert(MI->getDebugOffset().getImm() == 0 &&
731 "DBG_VALUE with nonzero offset");
733 "Expected inlined-at fields to agree");
734 assert(MI->getOpcode() != TargetOpcode::DBG_VALUE_LIST &&
735 "Didn't expect to see a DBG_VALUE_LIST here");
736
737 BuildMI(*EntryMBB, ++InsertPos, DL, TII->get(TargetOpcode::DBG_VALUE),
738 IsIndirect, LDI->second, Variable, Expr);
739
740
741
742
745 if (UseMI.isDebugValue())
746 continue;
747 if (UseMI.isCopy() && !CopyUseMI && UseMI.getParent() == EntryMBB) {
748 CopyUseMI = &UseMI;
749 continue;
750 }
751
752 CopyUseMI = nullptr;
753 break;
754 }
755 if (CopyUseMI &&
756 TRI.getRegSizeInBits(LDI->second, MRI) ==
758
759
761 BuildMI(*MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
765 }
766 }
767 }
768
769
770
771 if (MF->useDebugInstrRef())
772 MF->finalizeDebugInstrRefs();
773
774
776 for (const auto &MBB : *MF) {
777 if (MFI.hasCalls() && MF->hasInlineAsm())
778 break;
779
780 for (const auto &MI : MBB) {
782 if ((MCID.isCall() && .isReturn()) ||
783 MI.isStackAligningInlineAsm()) {
785 }
786 if (MI.isInlineAsm()) {
787 MF->setHasInlineAsm(true);
788 }
789 }
790 }
791
792
793
795
796 ISEL_DUMP(dbgs() << "*** MachineFunction at end of ISel ***\n");
798
799 return true;
800}
801
805 bool ShouldAbort) {
806
807
808 if (!R.getLocation().isValid() || ShouldAbort)
809 R << (" (in function: " + MF.getName() + ")").str();
810
811 if (ShouldAbort)
813
816}
817
818
819
820
825 return;
826
827 bool HaveFakeUse = false;
828 bool HaveTailCall = false;
829 do {
831 if (CI->isTailCall()) {
832 HaveTailCall = true;
833 break;
834 }
836 if (II->getIntrinsicID() == Intrinsic::fake_use)
837 HaveFakeUse = true;
838 } while (I != Begin);
839
840
841 if (!HaveTailCall || !HaveFakeUse)
842 return;
843
845
846
847
850 FakeUse && FakeUse->getIntrinsicID() == Intrinsic::fake_use) {
852 !UsedDef || UsedDef->getParent() != I->getParent() ||
853 UsedDef->comesBefore(&*I))
855 }
856 }
857
858 for (auto *Inst : FakeUses)
859 Inst->moveBefore(*Inst->getParent(), I);
860}
861
864 bool &HadTailCall) {
865
866 CurDAG->NewNodesMustHaveLegalTypes = false;
867
868
869
870
874 else
876 }
877
878
879 CurDAG->setRoot(SDB->getControlRoot());
880 HadTailCall = SDB->HasTailCall;
881 SDB->resolveOrClearDbgInfo();
882 SDB->clear();
883
884
885 CodeGenAndEmitDAG();
886}
887
888void SelectionDAGISel::ComputeLiveOutVRegInfo() {
889 SmallPtrSet<SDNode *, 16> Added;
891
893 Added.insert(CurDAG->getRoot().getNode());
894
895 KnownBits Known;
896
897 do {
899
900
901 for (const SDValue &Op : N->op_values())
902 if (Op.getValueType() == MVT::Other && Added.insert(Op.getNode()).second)
904
905
907 continue;
908
911 continue;
912
913
914 SDValue Src = N->getOperand(2);
915 EVT SrcVT = Src.getValueType();
917 continue;
918
919 unsigned NumSignBits = CurDAG->ComputeNumSignBits(Src);
920 Known = CurDAG->computeKnownBits(Src);
921 FuncInfo->AddLiveOutRegInfo(DestReg, NumSignBits, Known);
922 } while (!Worklist.empty());
923}
924
925void SelectionDAGISel::CodeGenAndEmitDAG() {
926 StringRef GroupName = "sdag";
927 StringRef GroupDescription = "Instruction Selection and Scheduling";
928 std::string BlockName;
929 bool MatchFilterBB = false;
930 (void)MatchFilterBB;
931
932
933 CurDAG->NewNodesMustHaveLegalTypes = false;
934
935#ifndef NDEBUG
938 FuncInfo->MBB->getBasicBlock()->getName());
939#endif
940#ifdef NDEBUG
944#endif
945 {
946 BlockName =
947 (MF->getName() + ":" + FuncInfo->MBB->getBasicBlock()->getName()).str();
948 }
951 << "'\n";
953
954#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
955 if (TTI->hasBranchDivergence())
956 CurDAG->VerifyDAGDivergence();
957#endif
958
960 CurDAG->viewGraph("dag-combine1 input for " + BlockName);
961
962
963 {
964 NamedRegionTimer T("combine1", "DAG Combining 1", GroupName,
967 }
968
969 ISEL_DUMP(dbgs() << "\nOptimized lowered selection DAG: "
971 << "'\n";
973
974#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
975 if (TTI->hasBranchDivergence())
976 CurDAG->VerifyDAGDivergence();
977#endif
978
979
980
982 CurDAG->viewGraph("legalize-types input for " + BlockName);
983
985 {
986 NamedRegionTimer T("legalize_types", "Type Legalization", GroupName,
989 }
990
991 ISEL_DUMP(dbgs() << "\nType-legalized selection DAG: "
993 << "'\n";
995
996#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
997 if (TTI->hasBranchDivergence())
998 CurDAG->VerifyDAGDivergence();
999#endif
1000
1001
1002 CurDAG->NewNodesMustHaveLegalTypes = true;
1003
1006 CurDAG->viewGraph("dag-combine-lt input for " + BlockName);
1007
1008
1009 {
1010 NamedRegionTimer T("combine_lt", "DAG Combining after legalize types",
1013 }
1014
1015 ISEL_DUMP(dbgs() << "\nOptimized type-legalized selection DAG: "
1017 << "'\n";
1019
1020#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
1021 if (TTI->hasBranchDivergence())
1022 CurDAG->VerifyDAGDivergence();
1023#endif
1024 }
1025
1026 {
1027 NamedRegionTimer T("legalize_vec", "Vector Legalization", GroupName,
1030 }
1031
1033 ISEL_DUMP(dbgs() << "\nVector-legalized selection DAG: "
1035 << "'\n";
1037
1038#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
1039 if (TTI->hasBranchDivergence())
1040 CurDAG->VerifyDAGDivergence();
1041#endif
1042
1043 {
1044 NamedRegionTimer T("legalize_types2", "Type Legalization 2", GroupName,
1046 CurDAG->LegalizeTypes();
1047 }
1048
1049 ISEL_DUMP(dbgs() << "\nVector/type-legalized selection DAG: "
1051 << "'\n";
1053
1054#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
1055 if (TTI->hasBranchDivergence())
1056 CurDAG->VerifyDAGDivergence();
1057#endif
1058
1060 CurDAG->viewGraph("dag-combine-lv input for " + BlockName);
1061
1062
1063 {
1064 NamedRegionTimer T("combine_lv", "DAG Combining after legalize vectors",
1067 }
1068
1069 ISEL_DUMP(dbgs() << "\nOptimized vector-legalized selection DAG: "
1071 << "'\n";
1073
1074#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
1075 if (TTI->hasBranchDivergence())
1076 CurDAG->VerifyDAGDivergence();
1077#endif
1078 }
1079
1081 CurDAG->viewGraph("legalize input for " + BlockName);
1082
1083 {
1084 NamedRegionTimer T("legalize", "DAG Legalization", GroupName,
1087 }
1088
1089 ISEL_DUMP(dbgs() << "\nLegalized selection DAG: "
1091 << "'\n";
1093
1094#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
1095 if (TTI->hasBranchDivergence())
1096 CurDAG->VerifyDAGDivergence();
1097#endif
1098
1100 CurDAG->viewGraph("dag-combine2 input for " + BlockName);
1101
1102
1103 {
1104 NamedRegionTimer T("combine2", "DAG Combining 2", GroupName,
1107 }
1108
1109 ISEL_DUMP(dbgs() << "\nOptimized legalized selection DAG: "
1111 << "'\n";
1113
1114#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
1115 if (TTI->hasBranchDivergence())
1116 CurDAG->VerifyDAGDivergence();
1117#endif
1118
1120 ComputeLiveOutVRegInfo();
1121
1123 CurDAG->viewGraph("isel input for " + BlockName);
1124
1125
1126
1127 {
1128 NamedRegionTimer T("isel", "Instruction Selection", GroupName,
1130 DoInstructionSelection();
1131 }
1132
1133 ISEL_DUMP(dbgs() << "\nSelected selection DAG: "
1135 << "'\n";
1137
1139 CurDAG->viewGraph("scheduler input for " + BlockName);
1140
1141
1142 ScheduleDAGSDNodes *Scheduler = CreateScheduler();
1143 {
1144 NamedRegionTimer T("sched", "Instruction Scheduling", GroupName,
1147 }
1148
1151
1152
1153
1154 MachineBasicBlock *FirstMBB = FuncInfo->MBB, *LastMBB;
1155 {
1156 NamedRegionTimer T("emit", "Instruction Creation", GroupName,
1158
1159
1160
1162 }
1163
1164
1165
1166 if (FirstMBB != LastMBB)
1167 SDB->UpdateSplitBlock(FirstMBB, LastMBB);
1168
1169
1170 {
1171 NamedRegionTimer T("cleanup", "Instruction Scheduling Cleanup", GroupName,
1174 }
1175
1176
1178}
1179
1180namespace {
1181
1182
1183
1186
1187public:
1189 : SelectionDAG::DAGUpdateListener(DAG), ISelPosition(isp) {}
1190
1191
1192
1193
1194 void NodeDeleted(SDNode *N, SDNode *E) override {
1196 ++ISelPosition;
1197 }
1198
1199
1200
1201
1202 void NodeInserted(SDNode *N) override {
1203 SDNode *CurNode = &*ISelPosition;
1204 if (MDNode *MD = DAG.getPCSections(CurNode))
1205 DAG.addPCSections(N, MD);
1206 if (MDNode *MMRA = DAG.getMMRAMetadata(CurNode))
1207 DAG.addMMRAMetadata(N, MMRA);
1208 }
1209};
1210
1211}
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1236
1237 while (!Nodes.empty()) {
1239 for (auto *U : N->users()) {
1240 auto UId = U->getNodeId();
1241 if (UId > 0) {
1244 }
1245 }
1246 }
1247}
1248
1249
1250
1251
1253 int InvalidId = -(N->getNodeId() + 1);
1254 N->setNodeId(InvalidId);
1255}
1256
1257
1259 int Id = N->getNodeId();
1260 if (Id < -1)
1261 return -(Id + 1);
1262 return Id;
1263}
1264
1265void SelectionDAGISel::DoInstructionSelection() {
1266 LLVM_DEBUG(dbgs() << "===== Instruction selection begins: "
1268 << FuncInfo->MBB->getName() << "'\n");
1269
1271
1272
1273 {
1274
1276
1277
1278
1279
1282 ++ISelPosition;
1283
1284
1285
1286 ISelUpdater ISU(*CurDAG, ISelPosition);
1287
1288
1289
1290
1291
1294
1295
1296
1297 if (Node->use_empty())
1298 continue;
1299
1300#ifndef NDEBUG
1303
1304 while (!Nodes.empty()) {
1307 continue;
1308 for (const SDValue &Op : N->op_values()) {
1311 else {
1312
1313
1314
1315
1316
1317
1318
1319
1320 assert(Op->getNodeId() != -1 &&
1321 "Node has already selected predecessor node");
1322 }
1323 }
1324 }
1325#endif
1326
1327
1328
1329
1330
1331
1332
1333 if (->isStrictFPEnabled() && Node->isStrictFPOpcode()) {
1334
1335
1336
1337 EVT ActionVT;
1338 switch (Node->getOpcode()) {
1347 ActionVT = Node->getOperand(1).getValueType();
1348 break;
1349 default:
1350 ActionVT = Node->getValueType(0);
1351 break;
1352 }
1353 if (TLI->getOperationAction(Node->getOpcode(), ActionVT)
1355 Node = CurDAG->mutateStrictFPToFP(Node);
1356 }
1357
1358 LLVM_DEBUG(dbgs() << "\nISEL: Starting selection on root node: ";
1360
1362 }
1363
1364 CurDAG->setRoot(Dummy.getValue());
1365 }
1366
1367 LLVM_DEBUG(dbgs() << "\n===== Instruction selection ends:\n");
1368
1370}
1371
1373 for (const User *U : CPI->users()) {
1375 Intrinsic::ID IID = EHPtrCall->getIntrinsicID();
1376 if (IID == Intrinsic::eh_exceptionpointer ||
1377 IID == Intrinsic::eh_exceptioncode)
1378 return true;
1379 }
1380 }
1381 return false;
1382}
1383
1384
1385
1386
1390
1391
1392 bool IsSingleCatchAllClause =
1395
1396
1397 bool IsCatchLongjmp = CPI->arg_size() == 0;
1398 if (!IsSingleCatchAllClause && !IsCatchLongjmp) {
1399
1400 bool IntrFound = false;
1401 for (const User *U : CPI->users()) {
1404 if (IID == Intrinsic::wasm_landingpad_index) {
1405 Value *IndexArg = Call->getArgOperand(1);
1408 IntrFound = true;
1409 break;
1410 }
1411 }
1412 }
1413 assert(IntrFound && "wasm.landingpad.index intrinsic not found!");
1414 (void)IntrFound;
1415 }
1416}
1417
1418
1419
1420bool SelectionDAGISel::PrepareEHLandingPad() {
1422 const Constant *PersonalityFn = FuncInfo->Fn->getPersonalityFn();
1424 const TargetRegisterClass *PtrRC =
1425 TLI->getRegClassFor(TLI->getPointerTy(CurDAG->getDataLayout()));
1426
1428
1429
1430
1434
1435
1436 MCRegister EHPhysReg = TLI->getExceptionPointerRegister(PersonalityFn);
1437 assert(EHPhysReg && "target lacks exception pointer register");
1439 Register VReg = FuncInfo->getCatchPadExceptionPointerVReg(CPI, PtrRC);
1441 TII->get(TargetOpcode::COPY), VReg)
1443 }
1444 }
1445 return true;
1446 }
1447
1448
1449
1451
1452 const MCInstrDesc &II = TII->get(TargetOpcode::EH_LABEL);
1455
1456
1457
1458 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
1459 if (auto *RegMask = TRI.getCustomEHPadPreservedMask(*MF))
1460 MF->getRegInfo().addPhysRegsUsedFromRegMask(RegMask);
1461
1465 } else {
1466
1467 MF->setCallSiteLandingPad(Label, SDB->LPadToCallSiteMap[MBB]);
1468
1469 if (MCRegister Reg = TLI->getExceptionPointerRegister(PersonalityFn))
1471
1472 if (MCRegister Reg = TLI->getExceptionSelectorRegister(PersonalityFn))
1474 }
1475
1476 return true;
1477}
1478
1479
1480void SelectionDAGISel::reportIPToStateForBlocks(MachineFunction *MF) {
1481 llvm::WinEHFuncInfo *EHInfo = MF->getWinEHFuncInfo();
1482 if (!EHInfo)
1483 return;
1484 for (MachineBasicBlock &MBB : *MF) {
1488
1490
1492 continue;
1493
1494 MachineInstr *MIb = &*MBBb;
1496 continue;
1497
1498
1499 MCSymbol *BeginLabel = MF->getContext().createTempSymbol();
1500 MCSymbol *EndLabel = MF->getContext().createTempSymbol();
1503 TII->get(TargetOpcode::EH_LABEL))
1504 .addSym(BeginLabel);
1506 MachineInstr *MIe = &*(--MBBe);
1507
1509 MIe = &*(--MBBe);
1510 ++MBBe;
1512 TII->get(TargetOpcode::EH_LABEL))
1514 }
1515 }
1516}
1517
1518
1519
1520
1523 return ->mayWriteToMemory() &&
1524 ->isTerminator() &&
1525 ->isEHPad() &&
1526 !FuncInfo.isExportedInst(I);
1527}
1528
1534 return false;
1535
1536 auto ArgIt = FuncInfo.ValueMap.find(Arg);
1537 if (ArgIt == FuncInfo.ValueMap.end())
1538 return false;
1539 Register ArgVReg = ArgIt->getSecond();
1540
1541
1542 for (auto [PhysReg, VirtReg] : FuncInfo.RegInfo->liveins())
1543 if (VirtReg == ArgVReg) {
1544
1547 LLVM_DEBUG(dbgs() << "processDbgDeclare: setVariableDbgInfo Var=" << *Var
1548 << ", Expr=" << *Expr << ", MCRegister=" << PhysReg
1549 << ", DbgLoc=" << DbgLoc << "\n");
1550 return true;
1551 }
1552 return false;
1553}
1554
1558 if (!Address) {
1559 LLVM_DEBUG(dbgs() << "processDbgDeclares skipping " << *Var
1560 << " (bad address)\n");
1561 return false;
1562 }
1563
1565 return true;
1566
1567 if (!Address->getType()->isPointerTy())
1568 return false;
1569
1572
1573 assert(Var && "Missing variable");
1574 assert(DbgLoc && "Missing location");
1575
1576
1577
1578 APInt Offset(DL.getIndexTypeSizeInBits(Address->getType()), 0);
1579 Address = Address->stripAndAccumulateInBoundsConstantOffsets(DL, Offset);
1580
1581
1582
1583
1584 int FI = std::numeric_limits::max();
1588 FI = SI->second;
1591
1592 if (FI == std::numeric_limits::max())
1593 return false;
1594
1595 if (Offset.getBoolValue())
1597 Offset.getZExtValue());
1598
1599 LLVM_DEBUG(dbgs() << "processDbgDeclare: setVariableDbgInfo Var=" << *Var
1600 << ", Expr=" << *Expr << ", FI=" << FI
1601 << ", DbgLoc=" << DbgLoc << "\n");
1603 return true;
1604}
1605
1606
1607
1613 DVR.getExpression(), DVR.getVariable(),
1614 DVR.getDebugLoc()))
1616 }
1617 }
1618}
1619
1620
1621
1622
1627 It != End; ++It) {
1628 assert(!It->Values.hasArgList() && "Single loc variadic ops not supported");
1629 processDbgDeclare(FuncInfo, It->Values.getVariableLocationOp(0), It->Expr,
1631 }
1632}
1633
1634void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) {
1636
1637 FastISel *FastIS = nullptr;
1638 if (TM.Options.EnableFastISel) {
1641 }
1642
1643 ReversePostOrderTraversal<const Function*> RPOT(&Fn);
1644
1645
1646
1648 ++NumEntryBlocks;
1649
1650
1653
1655
1656 if (!FastIS) {
1657 LowerArguments(Fn);
1658 } else {
1659
1663
1664 ++NumFastIselFailLowerArguments;
1665
1666 OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1669 R << "FastISel didn't lower all arguments: "
1672
1673
1674 LowerArguments(Fn);
1675 CurDAG->setRoot(SDB->getControlRoot());
1676 SDB->clear();
1677 CodeGenAndEmitDAG();
1678 }
1679
1680
1681
1682
1685 else
1687 }
1688
1690
1691 if (FastIS && Inserted)
1693
1696 "expected AssignmentTrackingAnalysis pass results");
1698 } else {
1700 }
1701
1702
1704 for (const BasicBlock *LLVMBB : RPOT) {
1706 bool AllPredsVisited = true;
1707 for (const BasicBlock *Pred : predecessors(LLVMBB)) {
1708 if (->VisitedBBs[Pred->getNumber()]) {
1709 AllPredsVisited = false;
1710 break;
1711 }
1712 }
1713
1714 if (AllPredsVisited) {
1715 for (const PHINode &PN : LLVMBB->phis())
1716 FuncInfo->ComputePHILiveOutRegInfo(&PN);
1717 } else {
1718 for (const PHINode &PN : LLVMBB->phis())
1719 FuncInfo->InvalidatePHILiveOutRegInfo(&PN);
1720 }
1721
1723 }
1724
1725
1726
1727
1728 {
1730 const_cast<BasicBlock *>(LLVMBB)->getFirstNonPHIIt();
1733 }
1734
1738
1741 continue;
1742
1743
1745
1746
1749 if (LLVMBB->isEHPad()) {
1750 if (!PrepareEHLandingPad())
1751 continue;
1752
1753 if (!FastIS) {
1756 if (NewRoot && NewRoot != CurDAG->getRoot())
1757 CurDAG->setRoot(NewRoot);
1758 }
1759 }
1760
1761
1762 if (FastIS) {
1765
1766 unsigned NumFastIselRemaining = std::distance(Begin, End);
1767
1768
1770
1771
1772 for (; BI != Begin; --BI) {
1773 const Instruction *Inst = &*std::prev(BI);
1774
1775
1778 --NumFastIselRemaining;
1780 continue;
1781 }
1782
1783
1784
1786
1787
1789 --NumFastIselRemaining;
1790 ++NumFastIselSuccess;
1791
1793
1794
1795
1797 while (BeforeInst != &*Begin) {
1800 break;
1801 }
1802 if (BeforeInst != Inst && isa(BeforeInst) &&
1805
1807 << "FastISel folded load: " << *BeforeInst << "\n");
1810 --NumFastIselRemaining;
1811 ++NumFastIselSuccess;
1812 }
1813 continue;
1814 }
1815
1817
1818
1819
1820
1821
1822
1825 OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1827
1828 R << "FastISel missed call";
1829
1831 std::string InstStrStorage;
1832 raw_string_ostream InstStr(InstStrStorage);
1833 InstStr << *Inst;
1834
1835 R << ": " << InstStrStorage;
1836 }
1837
1839
1843 if (!R)
1845 }
1846
1847 bool HadTailCall = false;
1849 SelectBasicBlock(Inst->getIterator(), BI, HadTailCall);
1850
1851
1852
1853 if (HadTailCall) {
1855 --BI;
1856 break;
1857 }
1858
1859
1860
1861 unsigned RemainingNow = std::distance(Begin, BI);
1862 NumFastIselFailures += NumFastIselRemaining - RemainingNow;
1863 NumFastIselRemaining = RemainingNow;
1864 continue;
1865 }
1866
1867 OptimizationRemarkMissed R("sdagisel", "FastISelFailure",
1869
1872
1873 R << "FastISel missed terminator";
1874
1876 } else {
1877 R << "FastISel missed";
1878 }
1879
1881 std::string InstStrStorage;
1882 raw_string_ostream InstStr(InstStrStorage);
1883 InstStr << *Inst;
1884 R << ": " << InstStrStorage;
1885 }
1886
1888
1889 NumFastIselFailures += NumFastIselRemaining;
1890 break;
1891 }
1892
1894 }
1895
1896 if (SP->shouldEmitSDCheck(*LLVMBB)) {
1897 bool FunctionBasedInstrumentation =
1899 SDB->SPDescriptor.initialize(LLVMBB, FuncInfo->getMBB(LLVMBB),
1900 FunctionBasedInstrumentation);
1901 }
1902
1903 if (Begin != BI)
1904 ++NumDAGBlocks;
1905 else
1906 ++NumFastIselBlocks;
1907
1908 if (Begin != BI) {
1909
1910
1911
1912 bool HadTailCall;
1913 SelectBasicBlock(Begin, BI, HadTailCall);
1914
1915
1916
1917
1918 if (FastIS && HadTailCall && FuncInfo->InsertPt != FuncInfo->MBB->end())
1920 }
1921
1922 if (FastIS)
1924 FinishBasicBlock();
1925 FuncInfo->PHINodesToUpdate.clear();
1927 }
1928
1929
1931 reportIPToStateForBlocks(MF);
1932
1933 SP->copyToMachineFrameInfo(MF->getFrameInfo());
1934
1936
1937 delete FastIS;
1938 SDB->clearDanglingDebugInfo();
1939 SDB->SPDescriptor.resetPerFunctionState();
1940}
1941
1942void
1943SelectionDAGISel::FinishBasicBlock() {
1944 LLVM_DEBUG(dbgs() << "Total amount of phi nodes to update: "
1945 << FuncInfo->PHINodesToUpdate.size() << "\n";
1946 for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e;
1948 << "Node " << i << " : (" << FuncInfo->PHINodesToUpdate[i].first
1950 << ")\n");
1951
1952
1953
1954 for (unsigned i = 0, e = FuncInfo->PHINodesToUpdate.size(); i != e; ++i) {
1955 MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[i].first);
1957 "This is not a machine PHI node that we are updating!");
1958 if (->MBB->isSuccessor(PHI->getParent()))
1959 continue;
1961 }
1962
1963
1964 if (SDB->SPDescriptor.shouldEmitFunctionBasedCheckStackProtector()) {
1965
1966
1967 MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();
1968
1969
1972 SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB);
1973 CurDAG->setRoot(SDB->getRoot());
1974 SDB->clear();
1975 CodeGenAndEmitDAG();
1976
1977
1978 SDB->SPDescriptor.resetPerBBState();
1979 } else if (SDB->SPDescriptor.shouldEmitStackProtector()) {
1980 MachineBasicBlock *ParentMBB = SDB->SPDescriptor.getParentMBB();
1981 MachineBasicBlock *SuccessMBB = SDB->SPDescriptor.getSuccessMBB();
1982
1983
1984
1985
1986
1987
1988
1991
1992
1993 SuccessMBB->splice(SuccessMBB->end(), ParentMBB, SplitPoint,
1994 ParentMBB->end());
1995
1996
1999 SDB->visitSPDescriptorParent(SDB->SPDescriptor, ParentMBB);
2000 CurDAG->setRoot(SDB->getRoot());
2001 SDB->clear();
2002 CodeGenAndEmitDAG();
2003
2004
2005 MachineBasicBlock *FailureMBB = SDB->SPDescriptor.getFailureMBB();
2006 if (FailureMBB->empty()) {
2009 SDB->visitSPDescriptorFailure(SDB->SPDescriptor);
2010 CurDAG->setRoot(SDB->getRoot());
2011 SDB->clear();
2012 CodeGenAndEmitDAG();
2013 }
2014
2015
2016 SDB->SPDescriptor.resetPerBBState();
2017 }
2018
2019
2020 for (auto &BTB : SDB->SL->BitTestCases) {
2021
2022 if (!BTB.Emitted) {
2023
2026
2027 SDB->visitBitTestHeader(BTB, FuncInfo->MBB);
2028 CurDAG->setRoot(SDB->getRoot());
2029 SDB->clear();
2030 CodeGenAndEmitDAG();
2031 }
2032
2033 BranchProbability UnhandledProb = BTB.Prob;
2034 for (unsigned j = 0, ej = BTB.Cases.size(); j != ej; ++j) {
2035 UnhandledProb -= BTB.Cases[j].ExtraProb;
2036
2037 FuncInfo->MBB = BTB.Cases[j].ThisBB;
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049 MachineBasicBlock *NextMBB;
2050 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
2051
2052
2053 NextMBB = BTB.Cases[j + 1].TargetBB;
2054 } else if (j + 1 == ej) {
2055
2056 NextMBB = BTB.Default;
2057 } else {
2058
2059 NextMBB = BTB.Cases[j + 1].ThisBB;
2060 }
2061
2062 SDB->visitBitTestCase(BTB, NextMBB, UnhandledProb, BTB.Reg, BTB.Cases[j],
2064
2065 CurDAG->setRoot(SDB->getRoot());
2066 SDB->clear();
2067 CodeGenAndEmitDAG();
2068
2069 if ((BTB.ContiguousRange || BTB.FallthroughUnreachable) && j + 2 == ej) {
2070
2071 BTB.Cases.pop_back();
2072 break;
2073 }
2074 }
2075
2076
2077 for (const std::pair<MachineInstr *, Register> &P :
2078 FuncInfo->PHINodesToUpdate) {
2079 MachineInstrBuilder PHI(*MF, P.first);
2080 MachineBasicBlock *PHIBB = PHI->getParent();
2082 "This is not a machine PHI node that we are updating!");
2083
2084
2085 if (PHIBB == BTB.Default) {
2086 PHI.addReg(P.second).addMBB(BTB.Parent);
2087 if (!BTB.ContiguousRange) {
2088 PHI.addReg(P.second).addMBB(BTB.Cases.back().ThisBB);
2089 }
2090 }
2091
2092 for (const SwitchCG::BitTestCase &BT : BTB.Cases) {
2093 MachineBasicBlock* cBB = BT.ThisBB;
2095 PHI.addReg(P.second).addMBB(cBB);
2096 }
2097 }
2098 }
2099 SDB->SL->BitTestCases.clear();
2100
2101
2102
2103
2104 for (unsigned i = 0, e = SDB->SL->JTCases.size(); i != e; ++i) {
2105
2106 if (->SL->JTCases[i].first.Emitted) {
2107
2108 FuncInfo->MBB = SDB->SL->JTCases[i].first.HeaderBB;
2110
2111 SDB->visitJumpTableHeader(SDB->SL->JTCases[i].second,
2112 SDB->SL->JTCases[i].first, FuncInfo->MBB);
2113 CurDAG->setRoot(SDB->getRoot());
2114 SDB->clear();
2115 CodeGenAndEmitDAG();
2116 }
2117
2118
2119 FuncInfo->MBB = SDB->SL->JTCases[i].second.MBB;
2121
2122 SDB->visitJumpTable(SDB->SL->JTCases[i].second);
2123 CurDAG->setRoot(SDB->getRoot());
2124 SDB->clear();
2125 CodeGenAndEmitDAG();
2126
2127
2128 for (unsigned pi = 0, pe = FuncInfo->PHINodesToUpdate.size();
2129 pi != pe; ++pi) {
2130 MachineInstrBuilder PHI(*MF, FuncInfo->PHINodesToUpdate[pi].first);
2131 MachineBasicBlock *PHIBB = PHI->getParent();
2133 "This is not a machine PHI node that we are updating!");
2134
2135 if (PHIBB == SDB->SL->JTCases[i].second.Default)
2136 PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second)
2137 .addMBB(SDB->SL->JTCases[i].first.HeaderBB);
2138
2139 if (FuncInfo->MBB->isSuccessor(PHIBB))
2140 PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(FuncInfo->MBB);
2141 }
2142 }
2143 SDB->SL->JTCases.clear();
2144
2145
2146
2147 for (unsigned i = 0, e = SDB->SL->SwitchCases.size(); i != e; ++i) {
2148
2149 FuncInfo->MBB = SDB->SL->SwitchCases[i].ThisBB;
2151
2152
2154 Succs.push_back(SDB->SL->SwitchCases[i].TrueBB);
2155 if (SDB->SL->SwitchCases[i].TrueBB != SDB->SL->SwitchCases[i].FalseBB)
2156 Succs.push_back(SDB->SL->SwitchCases[i].FalseBB);
2157
2158
2159 SDB->visitSwitchCase(SDB->SL->SwitchCases[i], FuncInfo->MBB);
2160 CurDAG->setRoot(SDB->getRoot());
2161 SDB->clear();
2162 CodeGenAndEmitDAG();
2163
2164
2165
2166 MachineBasicBlock *ThisBB = FuncInfo->MBB;
2167
2168
2169
2170
2171
2172 for (MachineBasicBlock *Succ : Succs) {
2175
2176
2181 MachineInstrBuilder PHI(*MF, MBBI);
2182
2183 for (unsigned pn = 0; ; ++pn) {
2185 "Didn't find PHI entry!");
2186 if (FuncInfo->PHINodesToUpdate[pn].first == PHI) {
2187 PHI.addReg(FuncInfo->PHINodesToUpdate[pn].second).addMBB(ThisBB);
2188 break;
2189 }
2190 }
2191 }
2192 }
2193 }
2194 }
2195 SDB->SL->SwitchCases.clear();
2196}
2197
2198
2199
2200
2201
2204}
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2216 int64_t DesiredMaskS) const {
2217 const APInt &ActualMask = RHS->getAPIntValue();
2218
2219
2220 const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS,
2221 false, true);
2222
2223
2224 if (ActualMask == DesiredMask)
2225 return true;
2226
2227
2228 if (!ActualMask.isSubsetOf(DesiredMask))
2229 return false;
2230
2231
2232
2233 APInt NeededMask = DesiredMask & ~ActualMask;
2234 if (CurDAG->MaskedValueIsZero(LHS, NeededMask))
2235 return true;
2236
2237
2238
2239
2240 return false;
2241}
2242
2243
2244
2245
2246
2248 int64_t DesiredMaskS) const {
2249 const APInt &ActualMask = RHS->getAPIntValue();
2250
2251
2252 const APInt &DesiredMask = APInt(LHS.getValueSizeInBits(), DesiredMaskS,
2253 false, true);
2254
2255
2256 if (ActualMask == DesiredMask)
2257 return true;
2258
2259
2260 if (!ActualMask.isSubsetOf(DesiredMask))
2261 return false;
2262
2263
2264
2265 APInt NeededMask = DesiredMask & ~ActualMask;
2267
2268
2270 return true;
2271
2272
2273
2274
2275 return false;
2276}
2277
2278
2279
2282
2283
2284
2285 std::list Handles;
2286
2290 Handles.emplace_back(
2292
2295 --e;
2296
2297 while (i != e) {
2299 if (!Flags.isMemKind() && !Flags.isFuncKind()) {
2300
2301 Handles.insert(Handles.end(), Ops.begin() + i,
2302 Ops.begin() + i + Flags.getNumOperandRegisters() + 1);
2303 i += Flags.getNumOperandRegisters() + 1;
2304 } else {
2305 assert(Flags.getNumOperandRegisters() == 1 &&
2306 "Memory operand with multiple values?");
2307
2308 unsigned TiedToOperand;
2309 if (Flags.isUseOperandTiedToDef(TiedToOperand)) {
2310
2313 for (; TiedToOperand; --TiedToOperand) {
2314 CurOp += Flags.getNumOperandRegisters() + 1;
2316 }
2317 }
2318
2319
2320 std::vector SelOps;
2322 Flags.getMemoryConstraintID();
2325 " failure!");
2326
2327
2330 SelOps.size());
2331 Flags.setMemConstraint(ConstraintID);
2332 Handles.emplace_back(CurDAG->getTargetConstant(Flags, DL, MVT::i32));
2334 i += 2;
2335 }
2336 }
2337
2338
2339 if (e != Ops.size())
2340 Handles.emplace_back(Ops.back());
2341
2342 Ops.clear();
2343 for (auto &handle : Handles)
2344 Ops.push_back(handle.getValue());
2345}
2346
2347
2348
2350 bool IgnoreChains) {
2353
2355 return false;
2356
2357
2358
2359 Visited.insert(ImmedUse);
2362
2363
2364 if ((Op.getValueType() == MVT::Other && IgnoreChains) || N == Def)
2365 continue;
2366 if (!Visited.insert(N).second)
2367 continue;
2369 }
2370
2371
2372 if (Root != ImmedUse) {
2375
2376 if ((Op.getValueType() == MVT::Other && IgnoreChains) || N == Def)
2377 continue;
2378 if (!Visited.insert(N).second)
2379 continue;
2381 }
2382 }
2383
2385}
2386
2387
2388
2390 SDNode *Root) const {
2392 return false;
2393 return N.hasOneUse();
2394}
2395
2396
2397
2400 bool IgnoreChains) {
2402 return false;
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2449 while (VT == MVT::Glue) {
2451 if (!GU)
2452 break;
2453 Root = GU;
2455
2456
2457
2458
2459
2460 IgnoreChains = false;
2461 }
2462
2463 return (Root, N.getNode(), U, IgnoreChains);
2464}
2465
2466void SelectionDAGISel::Select_INLINEASM(SDNode *N) {
2468
2469 std::vector Ops(N->op_begin(), N->op_end());
2471
2472 const EVT VTs[] = {MVT::Other, MVT::Glue};
2474 New->setNodeId(-1);
2477}
2478
2479void SelectionDAGISel::Select_READ_REGISTER(SDNode *Op) {
2480 SDLoc dl(Op);
2483
2484 EVT VT = Op->getValueType(0);
2486
2487 const MachineFunction &MF = CurDAG->getMachineFunction();
2489
2492 const Function &Fn = MF.getFunction();
2494 "invalid register \"" + Twine(RegStr->getString().data()) +
2495 "\" for llvm.read_register",
2496 Fn, Op->getDebugLoc()));
2498 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0);
2500 } else {
2502 CurDAG->getCopyFromReg(Op->getOperand(0), dl, Reg, Op->getValueType(0));
2503 }
2504
2505 New->setNodeId(-1);
2508}
2509
2510void SelectionDAGISel::Select_WRITE_REGISTER(SDNode *Op) {
2511 SDLoc dl(Op);
2514
2515 EVT VT = Op->getOperand(2).getValueType();
2517
2518 const MachineFunction &MF = CurDAG->getMachineFunction();
2520
2522 const Function &Fn = MF.getFunction();
2524 "invalid register \"" + Twine(RegStr->getString().data()) +
2525 "\" for llvm.write_register",
2526 Fn, Op->getDebugLoc()));
2528 } else {
2530 CurDAG->getCopyToReg(Op->getOperand(0), dl, Reg, Op->getOperand(2));
2531 New->setNodeId(-1);
2533 }
2534
2536}
2537
2538void SelectionDAGISel::Select_UNDEF(SDNode *N) {
2539 CurDAG->SelectNodeTo(N, TargetOpcode::IMPLICIT_DEF, N->getValueType(0));
2540}
2541
2542
2543
2544void SelectionDAGISel::Select_FAKE_USE(SDNode *N) {
2545 CurDAG->SelectNodeTo(N, TargetOpcode::FAKE_USE, N->getValueType(0),
2546 N->getOperand(1), N->getOperand(0));
2547}
2548
2549void SelectionDAGISel::Select_RELOC_NONE(SDNode *N) {
2550 CurDAG->SelectNodeTo(N, TargetOpcode::RELOC_NONE, N->getValueType(0),
2551 N->getOperand(1), N->getOperand(0));
2552}
2553
2554void SelectionDAGISel::Select_FREEZE(SDNode *N) {
2555
2556
2557
2558 CurDAG->SelectNodeTo(N, TargetOpcode::COPY, N->getValueType(0),
2559 N->getOperand(0));
2560}
2561
2562void SelectionDAGISel::Select_ARITH_FENCE(SDNode *N) {
2563 CurDAG->SelectNodeTo(N, TargetOpcode::ARITH_FENCE, N->getValueType(0),
2564 N->getOperand(0));
2565}
2566
2567void SelectionDAGISel::Select_MEMBARRIER(SDNode *N) {
2568 CurDAG->SelectNodeTo(N, TargetOpcode::MEMBARRIER, N->getValueType(0),
2569 N->getOperand(0));
2570}
2571
2572void SelectionDAGISel::Select_CONVERGENCECTRL_ANCHOR(SDNode *N) {
2573 CurDAG->SelectNodeTo(N, TargetOpcode::CONVERGENCECTRL_ANCHOR,
2574 N->getValueType(0));
2575}
2576
2577void SelectionDAGISel::Select_CONVERGENCECTRL_ENTRY(SDNode *N) {
2578 CurDAG->SelectNodeTo(N, TargetOpcode::CONVERGENCECTRL_ENTRY,
2579 N->getValueType(0));
2580}
2581
2582void SelectionDAGISel::Select_CONVERGENCECTRL_LOOP(SDNode *N) {
2583 CurDAG->SelectNodeTo(N, TargetOpcode::CONVERGENCECTRL_LOOP,
2584 N->getValueType(0), N->getOperand(0));
2585}
2586
2589 SDNode *OpNode = OpVal.getNode();
2590
2591
2592
2594
2596 Ops.push_back(
2597 CurDAG->getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
2600 } else {
2601 Ops.push_back(OpVal);
2602 }
2603}
2604
2605void SelectionDAGISel::Select_STACKMAP(SDNode *N) {
2607 auto *It = N->op_begin();
2609
2610
2613
2614
2616 assert(ID.getValueType() == MVT::i64);
2618
2619
2622 Ops.push_back(Shad);
2623
2624
2625 for (; It != N->op_end(); It++)
2626 pushStackMapLiveVariable(Ops, *It, DL);
2627
2628 Ops.push_back(Chain);
2629 Ops.push_back(InGlue);
2630
2631 SDVTList NodeTys = CurDAG->getVTList(MVT::Other, MVT::Glue);
2632 CurDAG->SelectNodeTo(N, TargetOpcode::STACKMAP, NodeTys, Ops);
2633}
2634
2635void SelectionDAGISel::Select_PATCHPOINT(SDNode *N) {
2637 auto *It = N->op_begin();
2639
2640
2642 std::optional Glue;
2643 if (It->getValueType() == MVT::Glue)
2644 Glue = *It++;
2645 SDValue RegMask = *It++;
2646
2647
2649 assert(ID.getValueType() == MVT::i64);
2651
2652
2655 Ops.push_back(Shad);
2656
2657
2658 Ops.push_back(*It++);
2659
2660
2661 SDValue NumArgs = *It++;
2663 Ops.push_back(NumArgs);
2664
2665
2666 Ops.push_back(*It++);
2667
2668
2670 Ops.push_back(*It++);
2671
2672
2673 for (; It != N->op_end(); It++)
2674 pushStackMapLiveVariable(Ops, *It, DL);
2675
2676
2677 Ops.push_back(RegMask);
2678 Ops.push_back(Chain);
2679 if (Glue.has_value())
2680 Ops.push_back(*Glue);
2681
2682 SDVTList NodeTys = N->getVTList();
2683 CurDAG->SelectNodeTo(N, TargetOpcode::PATCHPOINT, NodeTys, Ops);
2684}
2685
2686
2688GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx) {
2689 assert(Val >= 128 && "Not a VBR");
2690 Val &= 127;
2691
2692 unsigned Shift = 7;
2694 do {
2695 NextBits = MatcherTable[Idx++];
2696 Val |= (NextBits&127) << Shift;
2697 Shift += 7;
2698 } while (NextBits & 128);
2699
2700 return Val;
2701}
2702
2703
2704
2706getSimpleVT(const unsigned char *MatcherTable, unsigned &MatcherIndex) {
2707 unsigned SimpleVT = MatcherTable[MatcherIndex++];
2708 if (SimpleVT & 128)
2709 SimpleVT = GetVBR(SimpleVT, MatcherTable, MatcherIndex);
2710
2712}
2713
2714void SelectionDAGISel::Select_JUMP_TABLE_DEBUG_INFO(SDNode *N) {
2715 SDLoc dl(N);
2716 CurDAG->SelectNodeTo(N, TargetOpcode::JUMP_TABLE_DEBUG_INFO, MVT::Glue,
2717 CurDAG->getTargetConstant(N->getConstantOperandVal(1),
2718 dl, MVT::i64, true));
2719}
2720
2721
2722
2723void SelectionDAGISel::UpdateChains(
2727
2728
2729
2730 if (!ChainNodesMatched.empty()) {
2732 "Matched input chains but didn't produce a chain");
2733
2734
2735 for (unsigned i = 0, e = ChainNodesMatched.size(); i != e; ++i) {
2736 SDNode *ChainNode = ChainNodesMatched[i];
2737
2738
2739 if (!ChainNode)
2740 continue;
2741
2743 "Deleted node left in chain");
2744
2745
2746
2747 if (ChainNode == NodeToMatch && isMorphNodeTo)
2748 continue;
2749
2754 SelectionDAG::DAGNodeDeletedListener NDL(
2755 *CurDAG, [&](SDNode *N, SDNode *E) {
2756 llvm::replace(ChainNodesMatched, N, static_cast<SDNode *>(nullptr));
2757 });
2760
2761
2762 if (ChainNode != NodeToMatch && ChainNode->use_empty() &&
2764 NowDeadNodes.push_back(ChainNode);
2765 }
2766 }
2767
2768 if (!NowDeadNodes.empty())
2769 CurDAG->RemoveDeadNodes(NowDeadNodes);
2770
2772}
2773
2774
2775
2776
2777
2778
2779
2783
2787 unsigned int Max = 8192;
2788
2789
2790 if (ChainNodesMatched.size() == 1)
2791 return ChainNodesMatched[0]->getOperand(0);
2792
2793
2794
2795 std::function<void(const SDValue)> AddChains = [&](const SDValue V) {
2796 if (V.getValueType() != MVT::Other)
2797 return;
2799 return;
2800 if (!Visited.insert(V.getNode()).second)
2801 return;
2803 for (const SDValue &Op : V->op_values())
2804 AddChains(Op);
2805 } else
2807 };
2808
2809 for (auto *N : ChainNodesMatched) {
2812 }
2813
2814 while (!Worklist.empty())
2815 AddChains(Worklist.pop_back_val()->getOperand(0));
2816
2817
2818 if (InputChains.size() == 0)
2820
2821
2822
2823
2824 Visited.clear();
2825 for (SDValue V : InputChains) {
2826
2827
2828
2829 if (InputChains.size() != 1 &&
2830 V->getValueType(V->getNumValues() - 1) == MVT::Glue &&
2831 InputGlue.getNode() == V.getNode())
2833 Worklist.push_back(V.getNode());
2834 }
2835
2836 for (auto *N : ChainNodesMatched)
2839
2840
2841 if (InputChains.size() == 1)
2842 return InputChains[0];
2844 MVT::Other, InputChains);
2845}
2846
2847
2848SDNode *SelectionDAGISel::
2849MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList,
2851
2852
2853
2854
2855
2856
2857 int OldGlueResultNo = -1, OldChainResultNo = -1;
2858
2859 unsigned NTMNumResults = Node->getNumValues();
2860 if (Node->getValueType(NTMNumResults-1) == MVT::Glue) {
2861 OldGlueResultNo = NTMNumResults-1;
2862 if (NTMNumResults != 1 &&
2863 Node->getValueType(NTMNumResults-2) == MVT::Other)
2864 OldChainResultNo = NTMNumResults-2;
2865 } else if (Node->getValueType(NTMNumResults-1) == MVT::Other)
2866 OldChainResultNo = NTMNumResults-1;
2867
2868
2869
2870 SDNode *Res = CurDAG->MorphNodeTo(Node, ~TargetOpc, VTList, Ops);
2871
2872
2873
2874
2875 if (Res == Node) {
2876
2877
2879 }
2880
2881 unsigned ResNumResults = Res->getNumValues();
2882
2883 if ((EmitNodeInfo & OPFL_GlueOutput) && OldGlueResultNo != -1 &&
2884 static_cast<unsigned>(OldGlueResultNo) != ResNumResults - 1)
2886 SDValue(Res, ResNumResults - 1));
2887
2889 --ResNumResults;
2890
2891
2892 if ((EmitNodeInfo & OPFL_Chain) && OldChainResultNo != -1 &&
2893 static_cast<unsigned>(OldChainResultNo) != ResNumResults - 1)
2895 SDValue(Res, ResNumResults - 1));
2896
2897
2898
2899 if (Res != Node) {
2901 } else {
2903 }
2904
2905 return Res;
2906}
2907
2908
2911 const SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes) {
2912
2913 unsigned RecNo = MatcherTable[MatcherIndex++];
2914 assert(RecNo < RecordedNodes.size() && "Invalid CheckSame");
2915 return N == RecordedNodes[RecNo].first;
2916}
2917
2918
2920 const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N,
2921 const SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes,
2922 unsigned ChildNo) {
2923 if (ChildNo >= N.getNumOperands())
2924 return false;
2925 return ::CheckSame(MatcherTable, MatcherIndex, N.getOperand(ChildNo),
2926 RecordedNodes);
2927}
2928
2929
2933 bool TwoBytePredNo =
2935 unsigned PredNo =
2937 ? MatcherTable[MatcherIndex++]
2939 if (TwoBytePredNo)
2940 PredNo |= MatcherTable[MatcherIndex++] << 8;
2942}
2943
2944
2950 ? MatcherTable[MatcherIndex++]
2953}
2954
2956CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex,
2958 uint16_t Opc = MatcherTable[MatcherIndex++];
2959 Opc |= static_cast<uint16_t>(MatcherTable[MatcherIndex++]) << 8;
2960 return N->getOpcode() == Opc;
2961}
2962
2967 if (N.getValueType() == VT)
2968 return true;
2969
2970
2971 return VT == MVT::iPTR && N.getValueType() == TLI->getPointerTy(DL);
2972}
2973
2977 if (ChildNo >= N.getNumOperands())
2978 return false;
2979 return ::CheckType(VT, N.getOperand(ChildNo), TLI, DL);
2980}
2981
2988
2992 if (2 >= N.getNumOperands())
2993 return false;
2994 return ::CheckCondCode(MatcherTable, MatcherIndex, N.getOperand(2));
2995}
2996
2998CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex,
3002 return true;
3003
3004
3006}
3007
3008
3009
3011 if ((V & 1) == 0)
3012 return V >> 1;
3013 if (V != 1)
3014 return -(V >> 1);
3015
3016 return 1ULL << 63;
3017}
3018
3020CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex,
3022 int64_t Val = MatcherTable[MatcherIndex++];
3023 if (Val & 128)
3024 Val = GetVBR(Val, MatcherTable, MatcherIndex);
3025
3027
3029 return C && C->getAPIntValue().trySExtValue() == Val;
3030}
3031
3034 SDValue N, unsigned ChildNo) {
3035 if (ChildNo >= N.getNumOperands())
3036 return false;
3037 return ::CheckInteger(MatcherTable, MatcherIndex, N.getOperand(ChildNo));
3038}
3039
3041CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex,
3043 int64_t Val = MatcherTable[MatcherIndex++];
3044 if (Val & 128)
3045 Val = GetVBR(Val, MatcherTable, MatcherIndex);
3046
3047 if (N->getOpcode() != ISD::AND) return false;
3048
3050 return C && SDISel.CheckAndMask(N.getOperand(0), C, Val);
3051}
3052
3056 int64_t Val = MatcherTable[MatcherIndex++];
3057 if (Val & 128)
3058 Val = GetVBR(Val, MatcherTable, MatcherIndex);
3059
3060 if (N->getOpcode() != ISD::OR) return false;
3061
3063 return C && SDISel.CheckOrMask(N.getOperand(0), C, Val);
3064}
3065
3066
3067
3068
3069
3070
3071
3074 bool &Result,
3076 SmallVectorImpl<std::pair<SDValue, SDNode*>> &RecordedNodes) {
3077 unsigned Opcode = Table[Index++];
3078 switch (Opcode) {
3079 default:
3080 Result = false;
3081 return Index-1;
3083 Result = (Table, Index, N, RecordedNodes);
3084 return Index;
3091 return Index;
3103 return Index;
3114 return Index;
3117 return Index;
3122 switch (Opcode) {
3124 VT = MVT::i32;
3125 break;
3127 VT = MVT::i64;
3128 break;
3129 default:
3131 break;
3132 }
3134 return Index;
3135 }
3137 unsigned Res = Table[Index++];
3140 return Index;
3141 }
3167 unsigned ChildNo;
3170 VT = MVT::i32;
3174 VT = MVT::i64;
3176 } else {
3179 }
3182 return Index;
3183 }
3186 return Index;
3189 return Index;
3193 return Index;
3196 return Index;
3204 return Index;
3207 return Index;
3209 Result = (Table, Index, N, SDISel);
3210 return Index;
3211 }
3212}
3213
3214namespace {
3215
3216struct MatchScope {
3217
3218 unsigned FailIndex;
3219
3220
3222
3223
3224 unsigned NumRecordedNodes;
3225
3226
3227 unsigned NumMatchedMemRefs;
3228
3229
3230 SDValue InputChain, InputGlue;
3231
3232
3233 bool HasChainNodesMatched;
3234};
3235
3236
3237
3238
3239
3241{
3242 SDNode **NodeToMatch;
3243 SmallVectorImpl<std::pair<SDValue, SDNode *>> &RecordedNodes;
3244 SmallVectorImpl &MatchScopes;
3245
3246public:
3247 MatchStateUpdater(SelectionDAG &DAG, SDNode **NodeToMatch,
3248 SmallVectorImpl<std::pair<SDValue, SDNode *>> &RN,
3249 SmallVectorImpl &MS)
3250 : SelectionDAG::DAGUpdateListener(DAG), NodeToMatch(NodeToMatch),
3251 RecordedNodes(RN), MatchScopes(MS) {}
3252
3253 void NodeDeleted(SDNode *N, SDNode *E) override {
3254
3255
3256
3257
3258
3259 if ( || E->isMachineOpcode())
3260 return;
3261
3262 if (N == *NodeToMatch)
3263 *NodeToMatch = E;
3264
3265
3266
3267 for (auto &I : RecordedNodes)
3268 if (I.first.getNode() == N)
3270
3271 for (auto &I : MatchScopes)
3272 for (auto &J : I.NodeStack)
3273 if (J.getNode() == N)
3274 J.setNode(E);
3275 }
3276};
3277
3278}
3279
3281 const unsigned char *MatcherTable,
3282 unsigned TableSize) {
3283
3284 switch (NodeToMatch->getOpcode()) {
3285 default:
3286 break;
3291 case ISD::HANDLENODE:
3292 case ISD::MDNODE_SDNODE:
3306 case ISD::EH_LABEL:
3307 case ISD::ANNOTATION_LABEL:
3308 case ISD::LIFETIME_START:
3309 case ISD::LIFETIME_END:
3310 case ISD::PSEUDO_PROBE:
3312 NodeToMatch->setNodeId(-1);
3313 return;
3319 CurDAG->RemoveDeadNode(NodeToMatch);
3320 return;
3321 case ISD::INLINEASM:
3322 case ISD::INLINEASM_BR:
3323 Select_INLINEASM(NodeToMatch);
3324 return;
3326 Select_READ_REGISTER(NodeToMatch);
3327 return;
3329 Select_WRITE_REGISTER(NodeToMatch);
3330 return;
3333 Select_UNDEF(NodeToMatch);
3334 return;
3335 case ISD::FAKE_USE:
3336 Select_FAKE_USE(NodeToMatch);
3337 return;
3338 case ISD::RELOC_NONE:
3339 Select_RELOC_NONE(NodeToMatch);
3340 return;
3342 Select_FREEZE(NodeToMatch);
3343 return;
3344 case ISD::ARITH_FENCE:
3345 Select_ARITH_FENCE(NodeToMatch);
3346 return;
3347 case ISD::MEMBARRIER:
3348 Select_MEMBARRIER(NodeToMatch);
3349 return;
3350 case ISD::STACKMAP:
3351 Select_STACKMAP(NodeToMatch);
3352 return;
3353 case ISD::PATCHPOINT:
3354 Select_PATCHPOINT(NodeToMatch);
3355 return;
3356 case ISD::JUMP_TABLE_DEBUG_INFO:
3357 Select_JUMP_TABLE_DEBUG_INFO(NodeToMatch);
3358 return;
3359 case ISD::CONVERGENCECTRL_ANCHOR:
3360 Select_CONVERGENCECTRL_ANCHOR(NodeToMatch);
3361 return;
3362 case ISD::CONVERGENCECTRL_ENTRY:
3363 Select_CONVERGENCECTRL_ENTRY(NodeToMatch);
3364 return;
3365 case ISD::CONVERGENCECTRL_LOOP:
3366 Select_CONVERGENCECTRL_LOOP(NodeToMatch);
3367 return;
3368 }
3369
3371
3372
3376
3377
3378
3380
3381
3382
3383
3385
3386
3387
3389
3390
3391
3392
3393 SDValue InputChain, InputGlue, DeactivationSymbol;
3394
3395
3396
3397
3398
3400
3401 LLVM_DEBUG(dbgs() << "ISEL: Starting pattern match\n");
3402
3403
3404
3405
3406
3407 unsigned MatcherIndex = 0;
3408
3409 if (!OpcodeOffset.empty()) {
3410
3411 if (N.getOpcode() < OpcodeOffset.size())
3412 MatcherIndex = OpcodeOffset[N.getOpcode()];
3413 LLVM_DEBUG(dbgs() << " Initial Opcode index to " << MatcherIndex << "\n");
3414
3416
3417
3418
3419 unsigned Idx = 1;
3420 while (true) {
3421
3422 unsigned CaseSize = MatcherTable[Idx++];
3423 if (CaseSize & 128)
3424 CaseSize = GetVBR(CaseSize, MatcherTable, Idx);
3425 if (CaseSize == 0) break;
3426
3427
3429 Opc |= static_cast<uint16_t>(MatcherTable[Idx++]) << 8;
3430 if (Opc >= OpcodeOffset.size())
3431 OpcodeOffset.resize((Opc+1)*2);
3432 OpcodeOffset[Opc] = Idx;
3433 Idx += CaseSize;
3434 }
3435
3436
3437 if (N.getOpcode() < OpcodeOffset.size())
3438 MatcherIndex = OpcodeOffset[N.getOpcode()];
3439 }
3440
3441 while (true) {
3442 assert(MatcherIndex < TableSize && "Invalid index");
3443#ifndef NDEBUG
3444 unsigned CurrentOpcodeIndex = MatcherIndex;
3445#endif
3447 static_cast<BuiltinOpcodes>(MatcherTable[MatcherIndex++]);
3448 switch (Opcode) {
3450
3451
3452
3453
3454
3455 unsigned FailIndex;
3456
3457 while (true) {
3458 unsigned NumToSkip = MatcherTable[MatcherIndex++];
3459 if (NumToSkip & 128)
3460 NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
3461
3462 if (NumToSkip == 0) {
3463 FailIndex = 0;
3464 break;
3465 }
3466
3467 FailIndex = MatcherIndex+NumToSkip;
3468
3469 unsigned MatcherIndexOfPredicate = MatcherIndex;
3470 (void)MatcherIndexOfPredicate;
3471
3472
3473
3474
3475 bool Result;
3477 Result, *this, RecordedNodes);
3478 if (!Result)
3479 break;
3480
3482 dbgs() << " Skipped scope entry (due to false predicate) at "
3483 << "index " << MatcherIndexOfPredicate << ", continuing at "
3484 << FailIndex << "\n");
3485 ++NumDAGIselRetries;
3486
3487
3488
3489 MatcherIndex = FailIndex;
3490 }
3491
3492
3493 if (FailIndex == 0) break;
3494
3495
3496
3497 MatchScope NewEntry;
3498 NewEntry.FailIndex = FailIndex;
3499 NewEntry.NodeStack.append(NodeStack.begin(), NodeStack.end());
3500 NewEntry.NumRecordedNodes = RecordedNodes.size();
3501 NewEntry.NumMatchedMemRefs = MatchedMemRefs.size();
3502 NewEntry.InputChain = InputChain;
3503 NewEntry.InputGlue = InputGlue;
3504 NewEntry.HasChainNodesMatched = !ChainNodesMatched.empty();
3505 MatchScopes.push_back(NewEntry);
3506 continue;
3507 }
3509
3510 SDNode *Parent = nullptr;
3511 if (NodeStack.size() > 1)
3512 Parent = NodeStack[NodeStack.size()-2].getNode();
3513 RecordedNodes.push_back(std::make_pair(N, Parent));
3514 continue;
3515 }
3516
3522 if (ChildNo >= N.getNumOperands())
3523 break;
3524
3525 RecordedNodes.push_back(std::make_pair(N->getOperand(ChildNo),
3526 N.getNode()));
3527 continue;
3528 }
3531 MatchedMemRefs.push_back(MN->getMemOperand());
3532 else {
3534 dbgs() << '\n');
3535 }
3536
3537 continue;
3538
3540
3541 if (N->getNumOperands() != 0 &&
3542 N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Glue)
3543 InputGlue = N->getOperand(N->getNumOperands()-1);
3544 continue;
3545
3547
3548
3549 if (N->getNumOperands() != 0 &&
3550 N->getOperand(N->getNumOperands() - 1).getOpcode() ==
3552 DeactivationSymbol = N->getOperand(N->getNumOperands() - 1);
3553 continue;
3554
3556 unsigned ChildNo = MatcherTable[MatcherIndex++];
3557 if (ChildNo >= N.getNumOperands())
3558 break;
3559 N = N.getOperand(ChildNo);
3561 continue;
3562 }
3563
3569 if (ChildNo >= N.getNumOperands())
3570 break;
3571 N = N.getOperand(ChildNo);
3573 continue;
3574 }
3575
3585
3587 assert(!NodeStack.empty() && "Node stack imbalance!");
3589
3591 ? MatcherTable[MatcherIndex++]
3593 if (SiblingNo >= N.getNumOperands())
3594 break;
3595 N = N.getOperand(SiblingNo);
3597 continue;
3598 }
3600
3602 assert(!NodeStack.empty() && "Node stack imbalance!");
3604 continue;
3605
3607 if ((MatcherTable, MatcherIndex, N, RecordedNodes)) break;
3608 continue;
3609
3612 if ((MatcherTable, MatcherIndex, N, RecordedNodes,
3614 break;
3615 continue;
3616
3628 break;
3629 continue;
3640 break;
3641 continue;
3643 unsigned OpNum = MatcherTable[MatcherIndex++];
3645
3646 for (unsigned i = 0; i < OpNum; ++i)
3647 Operands.push_back(RecordedNodes[MatcherTable[MatcherIndex++]].first);
3648
3649 unsigned PredNo = MatcherTable[MatcherIndex++];
3651 break;
3652 continue;
3653 }
3664 ? MatcherTable[MatcherIndex++]
3666 unsigned RecNo = MatcherTable[MatcherIndex++];
3667 assert(RecNo < RecordedNodes.size() && "Invalid CheckComplexPat");
3668
3669
3670
3671 std::unique_ptr MSU;
3673 MSU.reset(new MatchStateUpdater(*CurDAG, &NodeToMatch, RecordedNodes,
3674 MatchScopes));
3675
3677 RecordedNodes[RecNo].first, CPNum,
3678 RecordedNodes))
3679 break;
3680 continue;
3681 }
3683 if ((MatcherTable, MatcherIndex, N.getNode())) break;
3684 continue;
3685
3690 switch (Opcode) {
3692 VT = MVT::i32;
3693 break;
3695 VT = MVT::i64;
3696 break;
3697 default:
3698 VT = getSimpleVT(MatcherTable, MatcherIndex);
3699 break;
3700 }
3702 break;
3703 continue;
3704
3706 unsigned Res = MatcherTable[MatcherIndex++];
3709 break;
3710 continue;
3711 }
3712
3714 unsigned CurNodeOpcode = N.getOpcode();
3715 unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3716 unsigned CaseSize;
3717 while (true) {
3718
3719 CaseSize = MatcherTable[MatcherIndex++];
3720 if (CaseSize & 128)
3721 CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
3722 if (CaseSize == 0) break;
3723
3724 uint16_t Opc = MatcherTable[MatcherIndex++];
3725 Opc |= static_cast<uint16_t>(MatcherTable[MatcherIndex++]) << 8;
3726
3727
3728 if (CurNodeOpcode == Opc)
3729 break;
3730
3731
3732 MatcherIndex += CaseSize;
3733 }
3734
3735
3736 if (CaseSize == 0) break;
3737
3738
3739 LLVM_DEBUG(dbgs() << " OpcodeSwitch from " << SwitchStart << " to "
3740 << MatcherIndex << "\n");
3741 continue;
3742 }
3743
3745 MVT CurNodeVT = N.getSimpleValueType();
3746 unsigned SwitchStart = MatcherIndex-1; (void)SwitchStart;
3747 unsigned CaseSize;
3748 while (true) {
3749
3750 CaseSize = MatcherTable[MatcherIndex++];
3751 if (CaseSize & 128)
3752 CaseSize = GetVBR(CaseSize, MatcherTable, MatcherIndex);
3753 if (CaseSize == 0) break;
3754
3755 MVT CaseVT = getSimpleVT(MatcherTable, MatcherIndex);
3756 if (CaseVT == MVT::iPTR)
3757 CaseVT = TLI->getPointerTy(CurDAG->getDataLayout());
3758
3759
3760 if (CurNodeVT == CaseVT)
3761 break;
3762
3763
3764 MatcherIndex += CaseSize;
3765 }
3766
3767
3768 if (CaseSize == 0) break;
3769
3770
3772 << "] from " << SwitchStart << " to " << MatcherIndex
3773 << '\n');
3774 continue;
3775 }
3801 unsigned ChildNo;
3804 VT = MVT::i32;
3808 VT = MVT::i64;
3810 } else {
3811 VT = getSimpleVT(MatcherTable, MatcherIndex);
3813 }
3815 break;
3816 continue;
3817 }
3820 continue;
3823 continue;
3826 CurDAG->getDataLayout()))
3827 break;
3828 continue;
3830 if ((MatcherTable, MatcherIndex, N)) break;
3831 continue;
3837 continue;
3839 if ((MatcherTable, MatcherIndex, N, *this)) break;
3840 continue;
3842 if ((MatcherTable, MatcherIndex, N, *this)) break;
3843 continue;
3846 break;
3847 continue;
3850 break;
3851 continue;
3852
3854 assert(NodeStack.size() != 1 && "No parent node");
3855
3856
3857 bool HasMultipleUses = false;
3858 for (unsigned i = 1, e = NodeStack.size()-1; i != e; ++i) {
3859 unsigned NNonChainUses = 0;
3860 SDNode *NS = NodeStack[i].getNode();
3861 for (const SDUse &U : NS->uses())
3862 if (U.getValueType() != MVT::Other)
3863 if (++NNonChainUses > 1) {
3864 HasMultipleUses = true;
3865 break;
3866 }
3867 if (HasMultipleUses) break;
3868 }
3869 if (HasMultipleUses) break;
3870
3871
3872
3874 NodeToMatch) ||
3877 true))
3878 break;
3879
3880 continue;
3881 }
3890 switch (Opcode) {
3892 VT = MVT::i8;
3893 break;
3895 VT = MVT::i16;
3896 break;
3899 VT = MVT::i32;
3900 break;
3902 VT = MVT::i64;
3903 break;
3904 default:
3905 VT = getSimpleVT(MatcherTable, MatcherIndex);
3906 break;
3907 }
3908 int64_t Val = MatcherTable[MatcherIndex++];
3909 if (Val & 128)
3910 Val = GetVBR(Val, MatcherTable, MatcherIndex);
3913 RecordedNodes.push_back(std::pair<SDValue, SDNode *>(
3914 CurDAG->getSignedConstant(Val, SDLoc(NodeToMatch), VT,
3915 true),
3916 nullptr));
3917 continue;
3918 }
3923 switch (Opcode) {
3925 VT = MVT::i32;
3926 break;
3928 VT = MVT::i64;
3929 break;
3930 default:
3931 VT = getSimpleVT(MatcherTable, MatcherIndex);
3932 break;
3933 }
3934 unsigned RegNo = MatcherTable[MatcherIndex++];
3935 RecordedNodes.push_back(std::pair<SDValue, SDNode *>(
3936 CurDAG->getRegister(RegNo, VT), nullptr));
3937 continue;
3938 }
3940
3941
3942
3944 unsigned RegNo = MatcherTable[MatcherIndex++];
3945 RegNo |= MatcherTable[MatcherIndex++] << 8;
3946 RecordedNodes.push_back(std::pair<SDValue, SDNode*>(
3947 CurDAG->getRegister(RegNo, VT), nullptr));
3948 continue;
3949 }
3950
3960
3962 ? MatcherTable[MatcherIndex++]
3964 assert(RecNo < RecordedNodes.size() && "Invalid EmitConvertToTarget");
3965 SDValue Imm = RecordedNodes[RecNo].first;
3966
3969 Imm = CurDAG->getTargetConstant(*Val, SDLoc(NodeToMatch),
3970 Imm.getValueType());
3973 Imm = CurDAG->getTargetConstantFP(*Val, SDLoc(NodeToMatch),
3974 Imm.getValueType());
3975 }
3976
3977 RecordedNodes.push_back(std::make_pair(Imm, RecordedNodes[RecNo].second));
3978 continue;
3979 }
3980
3984
3986 "EmitMergeInputChains should be the first chain producing node");
3988 "Should only have one EmitMergeInputChains per match");
3989
3990
3992 assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains");
3993 ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
3994
3995
3996
3997
3998
3999 if (ChainNodesMatched.back() != NodeToMatch &&
4000 !RecordedNodes[RecNo].first.hasOneUse()) {
4001 ChainNodesMatched.clear();
4002 break;
4003 }
4004
4005
4007
4008 if (!InputChain.getNode())
4009 break;
4010 continue;
4011 }
4012
4015 "EmitMergeInputChains should be the first chain producing node");
4016
4017
4018
4019
4020
4021
4022 unsigned NumChains = MatcherTable[MatcherIndex++];
4023 assert(NumChains != 0 && "Can't TF zero chains");
4024
4026 "Should only have one EmitMergeInputChains per match");
4027
4028
4029 for (unsigned i = 0; i != NumChains; ++i) {
4030 unsigned RecNo = MatcherTable[MatcherIndex++];
4031 assert(RecNo < RecordedNodes.size() && "Invalid EmitMergeInputChains");
4032 ChainNodesMatched.push_back(RecordedNodes[RecNo].first.getNode());
4033
4034
4035
4036
4037
4038 if (ChainNodesMatched.back() != NodeToMatch &&
4039 !RecordedNodes[RecNo].first.hasOneUse()) {
4040 ChainNodesMatched.clear();
4041 break;
4042 }
4043 }
4044
4045
4046 if (ChainNodesMatched.empty())
4047 break;
4048
4049
4051
4052 if (!InputChain.getNode())
4053 break;
4054
4055 continue;
4056 }
4057
4068 unsigned RecNo =
4071 : MatcherTable[MatcherIndex++];
4072 assert(RecNo < RecordedNodes.size() && "Invalid EmitCopyToReg");
4073 unsigned DestPhysReg = MatcherTable[MatcherIndex++];
4075 DestPhysReg |= MatcherTable[MatcherIndex++] << 8;
4076
4077 if (!InputChain.getNode())
4078 InputChain = CurDAG->getEntryNode();
4079
4080 InputChain = CurDAG->getCopyToReg(InputChain, SDLoc(NodeToMatch),
4081 DestPhysReg, RecordedNodes[RecNo].first,
4082 InputGlue);
4083
4084 InputGlue = InputChain.getValue(1);
4085 continue;
4086 }
4087
4089 unsigned XFormNo = MatcherTable[MatcherIndex++];
4090 unsigned RecNo = MatcherTable[MatcherIndex++];
4091 assert(RecNo < RecordedNodes.size() && "Invalid EmitNodeXForm");
4093 RecordedNodes.push_back(std::pair<SDValue,SDNode*>(Res, nullptr));
4094 continue;
4095 }
4097
4098
4099 unsigned index = MatcherTable[MatcherIndex++];
4100 index |= (MatcherTable[MatcherIndex++] << 8);
4101 index |= (MatcherTable[MatcherIndex++] << 16);
4102 index |= (MatcherTable[MatcherIndex++] << 24);
4105 continue;
4106 }
4107
4134 uint16_t TargetOpc = MatcherTable[MatcherIndex++];
4135 TargetOpc |= static_cast<uint16_t>(MatcherTable[MatcherIndex++]) << 8;
4136 unsigned EmitNodeInfo;
4140 else
4152 else
4154 } else
4155 EmitNodeInfo = MatcherTable[MatcherIndex++];
4156
4157 unsigned NumVTs;
4158
4159
4179 else
4180 NumVTs = MatcherTable[MatcherIndex++];
4182 for (unsigned i = 0; i != NumVTs; ++i) {
4184 if (VT == MVT::iPTR)
4185 VT = TLI->getPointerTy(CurDAG->getDataLayout()).SimpleTy;
4187 }
4188
4193
4194
4195
4197 if (VTs.size() == 1)
4198 VTList = CurDAG->getVTList(VTs[0]);
4199 else if (VTs.size() == 2)
4200 VTList = CurDAG->getVTList(VTs[0], VTs[1]);
4201 else
4202 VTList = CurDAG->getVTList(VTs);
4203
4204
4205 unsigned NumOps = MatcherTable[MatcherIndex++];
4207 for (unsigned i = 0; i != NumOps; ++i) {
4208 unsigned RecNo = MatcherTable[MatcherIndex++];
4209 if (RecNo & 128)
4210 RecNo = GetVBR(RecNo, MatcherTable, MatcherIndex);
4211
4212 assert(RecNo < RecordedNodes.size() && "Invalid EmitNode");
4213 Ops.push_back(RecordedNodes[RecNo].first);
4214 }
4215
4216
4218
4220 FirstOpToCopy += (EmitNodeInfo & OPFL_Chain) ? 1 : 0;
4222 "Invalid variadic node");
4223
4224
4225 for (unsigned i = FirstOpToCopy, e = NodeToMatch->getNumOperands();
4226 i != e; ++i) {
4228 if (V.getValueType() == MVT::Glue) break;
4229 Ops.push_back(V);
4230 }
4231 }
4232
4233
4235 Ops.push_back(InputChain);
4236 if (DeactivationSymbol.getNode() != nullptr)
4237 Ops.push_back(DeactivationSymbol);
4239 Ops.push_back(InputGlue);
4240
4241
4242
4243
4244
4245 bool MayRaiseFPException =
4248 });
4249
4250
4252 bool IsMorphNodeTo =
4255 if (!IsMorphNodeTo) {
4256
4257
4258 Res = CurDAG->getMachineNode(TargetOpc, SDLoc(NodeToMatch),
4259 VTList, Ops);
4260
4261
4262 for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
4263 if (VTs[i] == MVT::Other || VTs[i] == MVT::Glue) break;
4264 RecordedNodes.push_back(std::pair<SDValue,SDNode*>(SDValue(Res, i),
4265 nullptr));
4266 }
4267 } else {
4269 "NodeToMatch was removed partway through selection");
4272 CurDAG->salvageDebugInfo(*N);
4273 auto &Chain = ChainNodesMatched;
4275 "Chain node replaced during MorphNode");
4277 });
4279 Ops, EmitNodeInfo));
4280 }
4281
4282
4283
4286
4287
4288
4292 InputChain = SDValue(Res, VTs.size()-2);
4293 } else if (EmitNodeInfo & OPFL_Chain)
4294 InputChain = SDValue(Res, VTs.size()-1);
4295
4296
4297
4298
4299
4300
4301
4303
4304
4306 bool mayLoad = MCID.mayLoad();
4307 bool mayStore = MCID.mayStore();
4308
4309
4310
4313 if (MMO->isLoad()) {
4314 if (mayLoad)
4316 } else if (MMO->isStore()) {
4317 if (mayStore)
4319 } else {
4321 }
4322 }
4323
4324 CurDAG->setNodeMemRefs(Res, FilteredMemRefs);
4325 }
4326
4328 if (!MatchedMemRefs.empty() && Res->memoperands_empty())
4329 dbgs() << " Dropping mem operands\n";
4330 dbgs() << " " << (IsMorphNodeTo ? "Morphed" : "Created") << " node: ";
4332 });
4333
4334
4335 if (IsMorphNodeTo) {
4336
4337 UpdateChains(Res, InputChain, ChainNodesMatched, true);
4338 return;
4339 }
4340 continue;
4341 }
4342
4344
4345
4346
4347 unsigned NumResults = MatcherTable[MatcherIndex++];
4348
4349 for (unsigned i = 0; i != NumResults; ++i) {
4350 unsigned ResSlot = MatcherTable[MatcherIndex++];
4351 if (ResSlot & 128)
4352 ResSlot = GetVBR(ResSlot, MatcherTable, MatcherIndex);
4353
4354 assert(ResSlot < RecordedNodes.size() && "Invalid CompleteMatch");
4355 SDValue Res = RecordedNodes[ResSlot].first;
4356
4357 assert(i < NodeToMatch->getNumValues() &&
4358 NodeToMatch->getValueType(i) != MVT::Other &&
4359 NodeToMatch->getValueType(i) != MVT::Glue &&
4360 "Invalid number of results to complete!");
4362 NodeToMatch->getValueType(i) == MVT::iPTR ||
4366 "invalid replacement");
4368 }
4369
4370
4371 UpdateChains(NodeToMatch, InputChain, ChainNodesMatched, false);
4372
4373
4374
4375
4376
4378 MVT::Glue &&
4381 InputGlue);
4382
4384 "Didn't replace all uses of the node?");
4385 CurDAG->RemoveDeadNode(NodeToMatch);
4386
4387 return;
4388 }
4389 }
4390
4391
4392
4393
4394 LLVM_DEBUG(dbgs() << " Match failed at index " << CurrentOpcodeIndex
4395 << "\n");
4396 ++NumDAGIselRetries;
4397 while (true) {
4398 if (MatchScopes.empty()) {
4399 CannotYetSelect(NodeToMatch);
4400 return;
4401 }
4402
4403
4404
4405 MatchScope &LastScope = MatchScopes.back();
4406 RecordedNodes.resize(LastScope.NumRecordedNodes);
4407 NodeStack.clear();
4408 NodeStack.append(LastScope.NodeStack.begin(), LastScope.NodeStack.end());
4410
4411 if (LastScope.NumMatchedMemRefs != MatchedMemRefs.size())
4412 MatchedMemRefs.resize(LastScope.NumMatchedMemRefs);
4413 MatcherIndex = LastScope.FailIndex;
4414
4415 LLVM_DEBUG(dbgs() << " Continuing at " << MatcherIndex << "\n");
4416
4417 InputChain = LastScope.InputChain;
4418 InputGlue = LastScope.InputGlue;
4419 if (!LastScope.HasChainNodesMatched)
4420 ChainNodesMatched.clear();
4421
4422
4423
4424
4425 unsigned NumToSkip = MatcherTable[MatcherIndex++];
4426 if (NumToSkip & 128)
4427 NumToSkip = GetVBR(NumToSkip, MatcherTable, MatcherIndex);
4428
4429
4430
4431 if (NumToSkip != 0) {
4432 LastScope.FailIndex = MatcherIndex+NumToSkip;
4433 break;
4434 }
4435
4436
4437
4439 }
4440 }
4441}
4442
4443
4445
4446 if (N->isMachineOpcode()) {
4448 return MCID.mayRaiseFPException();
4449 }
4450
4451
4452
4453 if (N->isTargetOpcode()) {
4456 }
4457 return N->isStrictFPOpcode();
4458}
4459
4461 assert(N->getOpcode() == ISD::OR && "Unexpected opcode");
4463 if ()
4464 return false;
4465
4466
4470 int32_t Off = C->getSExtValue();
4471
4472
4473 return (Off >= 0) && (((A.value() - 1) & Off) == unsigned(Off));
4474 }
4475 return false;
4476}
4477
4478void SelectionDAGISel::CannotYetSelect(SDNode *N) {
4479 std::string msg;
4481 Msg << "Cannot select: ";
4482
4483 Msg.enable_colors(errs().has_colors());
4484
4488 N->printrFull(Msg, CurDAG);
4489 Msg << "\nIn function: " << MF->getName();
4490 } else {
4491 bool HasInputChain = N->getOperand(0).getValueType() == MVT::Other;
4492 unsigned iid = N->getConstantOperandVal(HasInputChain);
4493 if (iid < Intrinsic::num_intrinsics)
4495 else
4496 Msg << "unknown intrinsic #" << iid;
4497 }
4499}
unsigned const MachineRegisterInfo * MRI
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
MachineInstrBuilder & UseMI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Register Bank Select
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Expand Atomic instructions
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_ATTRIBUTE_ALWAYS_INLINE
LLVM_ATTRIBUTE_ALWAYS_INLINE - On compilers where we have a directive to do so, mark a method "always...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file defines the DenseMap class.
This file defines the FastISel class.
Module.h This file contains the declarations for the Module class.
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Machine Instruction Scheduler
Register const TargetRegisterInfo * TRI
Promote Memory to Register
uint64_t IntrinsicInst * II
FunctionAnalysisManager FAM
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
static Type * getValueType(Value *V)
Returns the type of the given value/instruction V.
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
Definition SelectionDAGISel.cpp:2998
static cl::opt< bool > ViewSUnitDAGs("view-sunit-dags", cl::Hidden, cl::desc("Pop up a window to show SUnit dags after they are processed"))
static cl::opt< bool > ViewDAGCombineLT("view-dag-combine-lt-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the post " "legalize types dag combine pass"))
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckPatternPredicate(unsigned Opcode, const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel)
CheckPatternPredicate - Implements OP_CheckPatternPredicate.
Definition SelectionDAGISel.cpp:2931
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes, unsigned ChildNo)
CheckChildSame - Implements OP_CheckChildXSame.
Definition SelectionDAGISel.cpp:2919
static uint64_t decodeSignRotatedValue(uint64_t V)
Decode a signed value stored with the sign bit in the LSB for dense VBR encoding.
Definition SelectionDAGISel.cpp:3010
static cl::opt< bool > ViewISelDAGs("view-isel-dags", cl::Hidden, cl::desc("Pop up a window to show isel dags as they are selected"))
static cl::opt< bool > DumpSortedDAG("dump-sorted-dags", cl::Hidden, cl::desc("Print DAGs with sorted nodes in debug dump"), cl::init(false))
static LLVM_ATTRIBUTE_ALWAYS_INLINE uint64_t GetVBR(uint64_t Val, const unsigned char *MatcherTable, unsigned &Idx)
GetVBR - decode a vbr encoding whose top bit is set.
Definition SelectionDAGISel.cpp:2688
static void reportFastISelFailure(MachineFunction &MF, OptimizationRemarkEmitter &ORE, OptimizationRemarkMissed &R, bool ShouldAbort)
Definition SelectionDAGISel.cpp:802
static cl::opt< bool > ViewDAGCombine2("view-dag-combine2-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the second " "dag combine pass"))
static RegisterScheduler defaultListDAGScheduler("default", "Best scheduler for the target", createDefaultScheduler)
static unsigned IsPredicateKnownToFail(const unsigned char *Table, unsigned Index, SDValue N, bool &Result, const SelectionDAGISel &SDISel, SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes)
IsPredicateKnownToFail - If we know how and can do so without pushing a scope, evaluate the current n...
Definition SelectionDAGISel.cpp:3072
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckNodePredicate(unsigned Opcode, const unsigned char *MatcherTable, unsigned &MatcherIndex, const SelectionDAGISel &SDISel, SDValue Op)
CheckNodePredicate - Implements OP_CheckNodePredicate.
Definition SelectionDAGISel.cpp:2946
static cl::opt< int > EnableFastISelAbort("fast-isel-abort", cl::Hidden, cl::desc("Enable abort calls when \"fast\" instruction selection " "fails to lower an instruction: 0 disable the abort, 1 will " "abort but for args, calls and terminators, 2 will also " "abort for argument lowering, and 3 will never fallback " "to SelectionDAG."))
static void mapWasmLandingPadIndex(MachineBasicBlock *MBB, const CatchPadInst *CPI)
Definition SelectionDAGISel.cpp:1387
#define ISEL_DUMP(X)
Definition SelectionDAGISel.cpp:191
static void processSingleLocVars(FunctionLoweringInfo &FuncInfo, FunctionVarLocs const *FnVarLocs)
Collect single location variable information generated with assignment tracking.
Definition SelectionDAGISel.cpp:1623
static LLVM_ATTRIBUTE_ALWAYS_INLINE MVT::SimpleValueType getSimpleVT(const unsigned char *MatcherTable, unsigned &MatcherIndex)
getSimpleVT - Decode a value in MatcherTable, if it's a VBR encoded value, use GetVBR to decode it.
Definition SelectionDAGISel.cpp:2706
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
Definition SelectionDAGISel.cpp:3020
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckAndImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
Definition SelectionDAGISel.cpp:3041
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOrImm(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SelectionDAGISel &SDISel)
Definition SelectionDAGISel.cpp:3054
static cl::opt< bool > UseMBPI("use-mbpi", cl::desc("use Machine Branch Probability Info"), cl::init(true), cl::Hidden)
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildType(MVT::SimpleValueType VT, SDValue N, const TargetLowering *TLI, const DataLayout &DL, unsigned ChildNo)
Definition SelectionDAGISel.cpp:2975
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const SmallVectorImpl< std::pair< SDValue, SDNode * > > &RecordedNodes)
CheckSame - Implements OP_CheckSame.
Definition SelectionDAGISel.cpp:2910
static bool dontUseFastISelFor(const Function &Fn)
Definition SelectionDAGISel.cpp:227
static bool findNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse, bool IgnoreChains)
findNonImmUse - Return true if "Def" is a predecessor of "Root" via a path beyond "ImmedUse".
Definition SelectionDAGISel.cpp:2349
static cl::opt< bool > ViewDAGCombine1("view-dag-combine1-dags", cl::Hidden, cl::desc("Pop up a window to show dags before the first " "dag combine pass"))
static bool processIfEntryValueDbgDeclare(FunctionLoweringInfo &FuncInfo, const Value *Arg, DIExpression *Expr, DILocalVariable *Var, DebugLoc DbgLoc)
Definition SelectionDAGISel.cpp:1529
static cl::opt< bool > ViewSchedDAGs("view-sched-dags", cl::Hidden, cl::desc("Pop up a window to show sched dags as they are processed"))
static void processDbgDeclares(FunctionLoweringInfo &FuncInfo)
Collect llvm.dbg.declare information.
Definition SelectionDAGISel.cpp:1608
static void preserveFakeUses(BasicBlock::iterator Begin, BasicBlock::iterator End)
Definition SelectionDAGISel.cpp:821
static SDValue HandleMergeInputChains(const SmallVectorImpl< SDNode * > &ChainNodesMatched, SDValue InputGlue, SelectionDAG *CurDAG)
HandleMergeInputChains - This implements the OPC_EmitMergeInputChains operation for when the pattern ...
Definition SelectionDAGISel.cpp:2781
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckType(MVT::SimpleValueType VT, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
Definition SelectionDAGISel.cpp:2963
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
Definition SelectionDAGISel.cpp:2983
static bool hasExceptionPointerOrCodeUser(const CatchPadInst *CPI)
Definition SelectionDAGISel.cpp:1372
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N)
Definition SelectionDAGISel.cpp:2990
static cl::opt< bool > ViewLegalizeDAGs("view-legalize-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize"))
static cl::opt< bool > ViewLegalizeTypesDAGs("view-legalize-types-dags", cl::Hidden, cl::desc("Pop up a window to show dags before legalize types"))
static cl::opt< RegisterScheduler::FunctionPassCtor, false, RegisterPassParser< RegisterScheduler > > ISHeuristic("pre-RA-sched", cl::init(&createDefaultScheduler), cl::Hidden, cl::desc("Instruction schedulers available (before register" " allocation):"))
ISHeuristic command line option for instruction schedulers.
static bool maintainPGOProfile(const TargetMachine &TM, CodeGenOptLevel OptLevel)
Definition SelectionDAGISel.cpp:237
static cl::opt< bool > EnableFastISelFallbackReport("fast-isel-report-on-fallback", cl::Hidden, cl::desc("Emit a diagnostic when \"fast\" instruction selection " "falls back to SelectionDAG."))
static bool processDbgDeclare(FunctionLoweringInfo &FuncInfo, const Value *Address, DIExpression *Expr, DILocalVariable *Var, DebugLoc DbgLoc)
Definition SelectionDAGISel.cpp:1555
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckOpcode(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDNode *N)
Definition SelectionDAGISel.cpp:2956
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckChildInteger(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, unsigned ChildNo)
Definition SelectionDAGISel.cpp:3033
static cl::opt< std::string > FilterDAGBasicBlockName("filter-view-dags", cl::Hidden, cl::desc("Only display the basic block whose name " "matches this for all view-*-dags options"))
static bool isFoldedOrDeadInstruction(const Instruction *I, const FunctionLoweringInfo &FuncInfo)
isFoldedOrDeadInstruction - Return true if the specified instruction is side-effect free and is eithe...
Definition SelectionDAGISel.cpp:1521
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file describes how to lower LLVM code to machine code.
This pass exposes codegen information to IR-level passes.
LLVM IR instance of the generic uniformity analysis.
A manager for alias analyses.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
AAResults & getAAResults()
Class for arbitrary precision integers.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
A function analysis which provides an AssumptionCache.
An immutable pass that tracks lazily created AssumptionCache objects.
LLVM Basic Block Representation.
unsigned getNumber() const
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
InstListType::const_iterator const_iterator
InstListType::iterator iterator
Instruction iterators...
bool isEHPad() const
Return true if this basic block is an exception handling block.
LLVM_ABI const Instruction * getFirstMayFaultInst() const
Returns the first potential AsynchEH faulty instruction currently it checks for loads/stores (which m...
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Analysis pass which computes BranchProbabilityInfo.
Legacy analysis pass which computes BranchProbabilityInfo.
This class represents a function call, abstracting a target machine's calling convention.
ConstantFP - Floating Point Values [float, double].
This is the shared class of boolean and integer constants.
LLVM_ABI bool isEntryValue() const
Check if the expression consists of exactly one entry value operand.
static LLVM_ABI DIExpression * append(const DIExpression *Expr, ArrayRef< uint64_t > Ops)
Append the opcodes Ops to DIExpr.
static LLVM_ABI DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
A parsed version of the target data layout string in and methods for querying it.
Record of a variable value-assignment, aka a non instruction representation of the dbg....
iterator find(const_arg_type_t< KeyT > Val)
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT > iterator
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Diagnostic information for ISel fallback path.
void setLastLocalValue(MachineInstr *I)
Update the position of the last instruction emitted for materializing constants for use in the curren...
void handleDbgInfo(const Instruction *II)
Target-independent lowering of non-instruction debug info associated with this instruction.
bool tryToFoldLoad(const LoadInst *LI, const Instruction *FoldInst)
We're checking to see if we can fold LI into FoldInst.
void removeDeadCode(MachineBasicBlock::iterator I, MachineBasicBlock::iterator E)
Remove all dead instructions between the I and E.
void startNewBlock()
Set the current block to which generated machine instructions will be appended.
bool selectInstruction(const Instruction *I)
Do "fast" instruction selection for the given LLVM IR instruction and append the generated machine in...
void finishBasicBlock()
Flush the local value map.
void recomputeInsertPt()
Reset InsertPt to prepare for inserting instructions into the current block.
bool lowerArguments()
Do "fast" instruction selection for function arguments and append the machine instructions to the cur...
unsigned arg_size() const
arg_size - Return the number of funcletpad arguments.
Value * getArgOperand(unsigned i) const
getArgOperand/setArgOperand - Return/set the i-th funcletpad argument.
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
SmallPtrSet< const DbgVariableRecord *, 8 > PreprocessedDVRDeclares
Collection of dbg_declare instructions handled after argument lowering and before ISel proper.
DenseMap< const AllocaInst *, int > StaticAllocaMap
StaticAllocaMap - Keep track of frame indices for fixed sized allocas in the entry block.
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
bool isExportedInst(const Value *V) const
isExportedInst - Return true if the specified value is an instruction exported from its block.
DenseMap< const Value *, Register > ValueMap
ValueMap - Since we emit code for the function a basic block at a time, we must remember which virtua...
MachineRegisterInfo * RegInfo
bool skipFunction(const Function &F) const
Optional passes call this function to check whether the pass should be skipped.
Data structure describing the variable locations in a function.
const VarLocInfo * single_locs_begin() const
DILocalVariable * getDILocalVariable(const VarLocInfo *Loc) const
Return the DILocalVariable for the location definition represented by ID.
const VarLocInfo * single_locs_end() const
One past the last single-location variable location definition.
const BasicBlock & getEntryBlock() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
unsigned getMaxBlockNumber() const
Return a value larger than the largest block number.
iterator_range< arg_iterator > args()
DISubprogram * getSubprogram() const
Get the attached subprogram.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
bool hasOptNone() const
Do not optimize this function (-O0).
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
An analysis pass which caches information about the Function.
An analysis pass which caches information about the entire Module.
Module * getParent()
Get the module that this global value is contained inside of...
This class is used to form a handle around another node that is persistent and is updated across invo...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
bool isTerminator() const
A wrapper class for inspecting calls to intrinsic functions.
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This is an alternative analysis pass to BlockFrequencyInfoWrapperPass.
static void getLazyBFIAnalysisUsage(AnalysisUsage &AU)
Helper for client passes to set up the analysis usage on behalf of this pass.
Describe properties that are true of each instruction in the target description file.
const MDNode * getMD() const
const MDOperand & getOperand(unsigned I) const
LLVM_ABI StringRef getString() const
LLVM_ABI instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
instr_iterator instr_end()
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
LLVM_ABI bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool hasCalls() const
Return true if the current function has any function calls.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
MachineFunctionPass(char &ID)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
bool useDebugInstrRef() const
Returns true if the function's variable locations are tracked with instruction referencing.
void setWasmLandingPadIndex(const MachineBasicBlock *LPad, unsigned Index)
Map the landing pad to its index. Used for Wasm exception handling.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
void setUseDebugInstrRef(bool UseInstrRef)
Set whether this function will use instruction referencing or not.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
bool shouldUseDebugInstrRef() const
Determine whether, in the current machine configuration, we should use instruction referencing or not...
const MachineFunctionProperties & getProperties() const
Get the function properties.
void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr, int Slot, const DILocation *Loc)
Collect information used to emit debugging information of a variable in a stack slot.
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
bool isTerminator(QueryType Type=AnyInBundle) const
Returns true if this instruction part of the terminator for a basic block.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
An analysis that produces MachineModuleInfo for a module.
This class contains meta information specific to a module.
Register getReg() const
getReg - Returns the register number.
MachinePassRegistry - Track the registration of machine passes.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
ArrayRef< std::pair< MCRegister, Register > > liveins() const
An SDNode that represents everything that will be needed to construct a MachineInstr.
Metadata * getModuleFlag(StringRef Key) const
Return the corresponding value if Key appears in module flags, otherwise return null.
This class is used by SelectionDAGISel to temporarily override the optimization level on a per-functi...
Definition SelectionDAGISel.cpp:255
~OptLevelChanger()
Definition SelectionDAGISel.cpp:284
OptLevelChanger(SelectionDAGISel &ISel, CodeGenOptLevel NewOptLevel)
Definition SelectionDAGISel.cpp:261
static LLVM_ABI PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
AnalysisType & getAnalysis() const
getAnalysis() - This function is used by subclasses to get to the analysis information ...
AnalysisType * getAnalysisIfAvailable() const
getAnalysisIfAvailable() - Subclasses use this function to get analysis information tha...
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
RegisterPassParser class - Handle the addition of new machine passes.
ScheduleDAGSDNodes *(*)(SelectionDAGISel *, CodeGenOptLevel) FunctionPassCtor
static LLVM_ABI MachinePassRegistry< FunctionPassCtor > Registry
RegisterScheduler class - Track the registration of instruction schedulers.
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDNode * getGluedUser() const
If this node has a glue value with a user, return the user (there is at most one).
LLVM_ABI bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< value_op_iterator > op_values() const
iterator_range< use_iterator > uses()
void setNodeId(int Id)
Set unique node id.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
ScheduleDAGSDNodes - A ScheduleDAG for scheduling SDNode-based DAGs.
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
Definition SelectionDAGISel.cpp:363
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition SelectionDAGISel.cpp:409
SelectionDAGISelLegacy(char &ID, std::unique_ptr< SelectionDAGISel > S)
Definition SelectionDAGISel.cpp:353
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
Definition SelectionDAGISel.cpp:433
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
std::optional< BatchAAResults > BatchAA
std::unique_ptr< FunctionLoweringInfo > FuncInfo
SmallPtrSet< const Instruction *, 4 > ElidedArgCopyInstrs
virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps)
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
bool CheckOrMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckOrMask - The isel is trying to match something like (or X, 255).
Definition SelectionDAGISel.cpp:2247
void initializeAnalysisResults(MachineFunctionAnalysisManager &MFAM)
Definition SelectionDAGISel.cpp:468
const TargetTransformInfo * TTI
virtual bool CheckNodePredicate(SDValue Op, unsigned PredNo) const
CheckNodePredicate - This function is generated by tblgen in the target.
virtual bool CheckNodePredicateWithOperands(SDValue Op, unsigned PredNo, ArrayRef< SDValue > Operands) const
CheckNodePredicateWithOperands - This function is generated by tblgen in the target.
const TargetLowering * TLI
virtual void PostprocessISelDAG()
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
std::unique_ptr< OptimizationRemarkEmitter > ORE
Current optimization remark emitter.
MachineRegisterInfo * RegInfo
unsigned DAGSize
DAGSize - Size of DAG being instruction selected.
@ OPC_MorphNodeTo2GlueOutput
@ OPC_CheckPatternPredicate5
@ OPC_EmitCopyToRegTwoByte
@ OPC_MorphNodeTo2GlueInput
@ OPC_CheckChild2CondCode
@ OPC_CheckPatternPredicateTwoByte
@ OPC_CheckPatternPredicate1
@ OPC_MorphNodeTo1GlueOutput
@ OPC_CaptureDeactivationSymbol
@ OPC_EmitMergeInputChains1_1
@ OPC_CheckPatternPredicate2
@ OPC_EmitConvertToTarget2
@ OPC_EmitConvertToTarget0
@ OPC_CheckPatternPredicate4
@ OPC_EmitConvertToTarget1
@ OPC_CheckPatternPredicate
@ OPC_MorphNodeTo0GlueInput
@ OPC_CheckPatternPredicate6
@ OPC_MorphNodeTo0GlueOutput
@ OPC_CheckPatternPredicate7
@ OPC_EmitMergeInputChains
@ OPC_EmitMergeInputChains1_0
@ OPC_CheckFoldableChainNode
@ OPC_EmitConvertToTarget3
@ OPC_CheckPredicateWithOperands
@ OPC_EmitConvertToTarget4
@ OPC_EmitStringInteger32
@ OPC_EmitConvertToTarget7
@ OPC_EmitMergeInputChains1_2
@ OPC_EmitConvertToTarget5
@ OPC_CheckPatternPredicate0
@ OPC_MorphNodeTo1GlueInput
@ OPC_CheckPatternPredicate3
@ OPC_EmitConvertToTarget
@ OPC_EmitConvertToTarget6
bool isOrEquivalentToAdd(const SDNode *N) const
Definition SelectionDAGISel.cpp:4460
virtual bool CheckComplexPattern(SDNode *Root, SDNode *Parent, SDValue N, unsigned PatternNo, SmallVectorImpl< std::pair< SDValue, SDNode * > > &Result)
virtual bool CheckPatternPredicate(unsigned PredNo) const
CheckPatternPredicate - This function is generated by tblgen in the target.
static int getNumFixedFromVariadicInfo(unsigned Flags)
getNumFixedFromVariadicInfo - Transform an EmitNode flags word into the number of fixed arity values ...
const TargetLibraryInfo * LibInfo
static int getUninvalidatedNodeId(SDNode *N)
Definition SelectionDAGISel.cpp:1258
const TargetInstrInfo * TII
std::unique_ptr< SwiftErrorValueTracking > SwiftError
void SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable, unsigned TableSize)
Definition SelectionDAGISel.cpp:3280
static void EnforceNodeIdInvariant(SDNode *N)
Definition SelectionDAGISel.cpp:1233
void ReplaceUses(SDValue F, SDValue T)
ReplaceUses - replace all uses of the old node F with the use of the new node T.
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
Definition SelectionDAGISel.cpp:2389
virtual SDValue RunSDNodeXForm(SDValue V, unsigned XFormNo)
bool MatchFilterFuncName
True if the function currently processing is in the function printing list (i.e.
void SelectInlineAsmMemoryOperands(std::vector< SDValue > &Ops, const SDLoc &DL)
SelectInlineAsmMemoryOperands - Calls to this are automatically generated by tblgen.
Definition SelectionDAGISel.cpp:2280
static bool IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, CodeGenOptLevel OptLevel, bool IgnoreChains=false)
IsLegalToFold - Returns true if the specific operand node N of U can be folded during instruction sel...
Definition SelectionDAGISel.cpp:2398
virtual bool ComplexPatternFuncMutatesDAG() const
Return true if complex patterns for this target can mutate the DAG.
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
BatchAAResults * getBatchAA() const
Returns a (possibly null) pointer to the current BatchAAResults.
bool CheckAndMask(SDValue LHS, ConstantSDNode *RHS, int64_t DesiredMaskS) const
CheckAndMask - The isel is trying to match something like (and X, 255).
Definition SelectionDAGISel.cpp:2215
virtual ~SelectionDAGISel()
Definition SelectionDAGISel.cpp:407
virtual StringRef getPatternForIndex(unsigned index)
getPatternForIndex - Patterns selected by tablegen during ISEL
bool mayRaiseFPException(SDNode *Node) const
Return whether the node may raise an FP exception.
Definition SelectionDAGISel.cpp:4444
std::unique_ptr< SelectionDAGBuilder > SDB
void ReplaceNode(SDNode *F, SDNode *T)
Replace all uses of F with T, then remove F from the DAG.
SelectionDAGISel(TargetMachine &tm, CodeGenOptLevel OL=CodeGenOptLevel::Default)
Definition SelectionDAGISel.cpp:393
virtual bool runOnMachineFunction(MachineFunction &mf)
Definition SelectionDAGISel.cpp:582
static void InvalidateNodeId(SDNode *N)
Definition SelectionDAGISel.cpp:1252
virtual StringRef getIncludePathForIndex(unsigned index)
getIncludePathForIndex - get the td source location of pattern instantiation
Targets can subclass this to parameterize the SelectionDAG lowering and instruction selection process...
virtual bool mayRaiseFPException(unsigned Opcode) const
Returns true if a node with the given target-specific opcode may raise a floating-point exception.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
allnodes_const_iterator allnodes_begin() const
const DataLayout & getDataLayout() const
LLVM_ABI void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
ilist< SDNode >::iterator allnodes_iterator
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Sched::Preference getSchedulingPreference() const
Return target scheduling preference.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
Definition SelectionDAGISel.cpp:342
virtual MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Definition SelectionDAGISel.cpp:332
Primary interface to the complete machine description for the target machine.
const std::optional< PGOOptions > & getPGOOption() const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
Wrapper pass for TargetTransformInfo.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
bool isTokenTy() const
Return true if this is 'token'.
bool isVoidTy() const
Return true if this is 'void'.
Analysis pass which computes UniformityInfo.
Legacy analysis pass which computes a CycleInfo.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
iterator_range< user_iterator > users()
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
self_iterator getIterator()
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
LLVM_ABI bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ POISON
POISON - A poison node.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ UNDEF
UNDEF - An undefined node.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ BasicBlock
Various leaf nodes.
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
@ AssertNoFPClass
AssertNoFPClass - These nodes record if a register contains a float value that is known to be not som...
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
LLVM_ABI bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LLVM_ABI StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
@ Kill
The last use of a register.
initializer< Ty > init(const Ty &Val)
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< NodeBase * > Node
friend class Instruction
Iterator for Instructions in a `BasicBlock.
This is an optimization pass for GlobalISel generic memory operations.
GenericUniformityInfo< SSAContext > UniformityInfo
LLVM_ABI ScheduleDAGSDNodes * createDefaultScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createDefaultScheduler - This creates an instruction scheduler appropriate for the target.
Definition SelectionDAGISel.cpp:300
OuterAnalysisManagerProxy< ModuleAnalysisManager, MachineFunction > ModuleAnalysisManagerMachineFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
bool succ_empty(const Instruction *I)
LLVM_ABI ScheduleDAGSDNodes * createBURRListDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createBURRListDAGScheduler - This creates a bottom up register usage reduction list scheduler.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
decltype(auto) dyn_cast(const From &Val)
dyn_cast - Return the argument parameter cast to the specified type.
LLVM_ABI ScheduleDAGSDNodes * createHybridListDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel)
createHybridListDAGScheduler - This creates a bottom up register pressure aware list scheduler that m...
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
MachineBasicBlock::iterator findSplitPointForStackProtector(MachineBasicBlock *BB, const TargetInstrInfo &TII)
Find the split point at which to splice the end of BB into its success stack protector check machine ...
LLVM_ABI bool TimePassesIsEnabled
If the user specifies the -time-passes argument on an LLVM tool command line then the value of this b...
LLVM_ABI LLT getLLTForMVT(MVT Ty)
Get a rough equivalent of an LLT for a given MVT.
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
LLVM_ABI void initializeGCModuleInfoPass(PassRegistry &)
LLVM_ABI ScheduleDAGSDNodes * createFastDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createFastDAGScheduler - This creates a "fast" scheduler.
LLVM_ABI PreservedAnalyses getMachineFunctionPassPreservedAnalyses()
Returns the minimum set of Analyses that all machine function passes must preserve.
void erase(Container &C, ValueType V)
Wrapper function to remove a value from a container:
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI ScheduleDAGSDNodes * createDAGLinearizer(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createDAGLinearizer - This creates a "no-scheduling" scheduler which linearize the DAG using topologi...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isFunctionInPrintList(StringRef FunctionName)
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
LLVM_ABI EHPersonality classifyEHPersonality(const Value *Pers)
See if the given exception handling personality function is one that we understand.
CodeGenOptLevel
Code generation optimization level.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
bool isFuncletEHPersonality(EHPersonality Pers)
Returns true if this is a personality function that invokes handler funclets (which must return to it...
LLVM_ABI ScheduleDAGSDNodes * createSourceListDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createSourceListDAGScheduler - This creates a bottom up list scheduler that schedules nodes in source...
LLVM_ABI bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
void replace(R &&Range, const T &OldValue, const T &NewValue)
Provide wrappers to std::replace which take ranges instead of having to pass begin/end explicitly.
DWARFExpression::Operation Op
LLVM_ABI void initializeAAResultsWrapperPassPass(PassRegistry &)
LLVM_ABI void initializeTargetLibraryInfoWrapperPassPass(PassRegistry &)
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI ScheduleDAGSDNodes * createILPListDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel)
createILPListDAGScheduler - This creates a bottom up register pressure aware list scheduler that trie...
decltype(auto) cast(const From &Val)
cast - Return the argument parameter cast to the specified type.
auto predecessors(const MachineBasicBlock *BB)
LLVM_ABI void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry &)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
LLVM_ABI ScheduleDAGSDNodes * createVLIWDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel)
createVLIWDAGScheduler - Scheduler for VLIW targets.
static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)
Filter the DbgRecord range to DbgVariableRecord types only and downcast.
LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
LLVM_ABI Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Implement std::hash so that hash_code can be used in STL containers.
This struct is a compact representation of a valid (non-zero power of two) alignment.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
A struct capturing PGO tunables.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Clients of various APIs that cause global effects on the DAG can optionally implement this interface.
void addIPToStateRange(const InvokeInst *II, MCSymbol *InvokeBegin, MCSymbol *InvokeEnd)
DenseMap< const BasicBlock *, int > BlockToStateMap