LLVM: lib/Target/AMDGPU/GCNIterativeScheduler.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
13
18
19using namespace llvm;
20
21#define DEBUG_TYPE "machine-scheduler"
22
23namespace llvm {
24
27
30}
31
32
42
43#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
49 unsigned MaxInstNum =
50 std::numeric_limits::max()) {
51 auto *BB = Begin->getParent();
52 OS << BB->getParent()->getName() << ":" << printMBBReference(*BB) << ' '
53 << BB->getName() << ":\n";
54 auto I = Begin;
55 MaxInstNum = std::max(MaxInstNum, 1u);
56 for (; I != End && MaxInstNum; ++I, --MaxInstNum) {
57 if (->isDebugInstr() && LIS)
59 OS << '\t' << *I;
60 }
61 if (I != End) {
62 OS << "\t...\n";
63 I = std::prev(End);
64 if (->isDebugInstr() && LIS)
66 OS << '\t' << *I;
67 }
68 if (End != BB->end()) {
69 OS << "----\n";
71 OS << *End;
72 }
73}
74
80 auto *const BB = Begin->getParent();
81 const auto &MRI = BB->getParent()->getRegInfo();
82
85
86 const auto BottomMI = End == BB->end() ? std::prev(End) : End;
89}
90
94 for (auto *const R : Regions) {
95 OS << "Region to schedule ";
98 OS << "Max RP: " << print(R->MaxPressure, &ST);
99 }
100}
101
106 OS << "\nAfter scheduling ";
109 OS << '\n';
110}
111
117 OS << "RP before: " << print(Before, &ST)
118 << "RP after: " << print(After, &ST);
119}
120#endif
121
123 bool HasIGLPInstrs = false;
127 HasIGLPInstrs = true;
128 break;
129 }
130 }
131
132 if (HasIGLPInstrs) {
137
139 }
140}
141
142
146
148public:
150 : Sch(_Sch) {
151 auto *BB = R.Begin->getParent();
152 Sch.BaseClass::startBlock(BB);
153 Sch.BaseClass::enterRegion(BB, R.Begin, R.End, R.NumRegionInstrs);
154 Sch.swapIGLPMutations(R, IsReentry);
155 Sch.buildSchedGraph(Sch.AA, nullptr, nullptr, nullptr,
156 true);
157 Sch.postProcessDAG();
158 Sch.Topo.InitDAGTopologicalSorting();
159 Sch.findRootsAndBiasEdges(TopRoots, BotRoots);
160 }
161
163 Sch.BaseClass::exitRegion();
164 Sch.BaseClass::finishBlock();
165 }
166
168 return TopRoots;
169 }
171 return BotRoots;
172 }
173};
174
178 std::unique_ptr SaveSchedImpl;
180
181public:
185 : Sch(_Sch)
186 , Rgn(R)
188 , SaveMaxRP(R.MaxPressure) {
189 Sch.SchedImpl.reset(&OverrideStrategy);
190 auto *BB = R.Begin->getParent();
191 Sch.BaseClass::startBlock(BB);
192 Sch.BaseClass::enterRegion(BB, R.Begin, R.End, R.NumRegionInstrs);
193 }
194
196 Sch.BaseClass::exitRegion();
197 Sch.BaseClass::finishBlock();
198 Sch.SchedImpl.release();
199 Sch.SchedImpl = std::move(SaveSchedImpl);
200 }
201
203 assert(Sch.RegionBegin == Rgn.Begin && Sch.RegionEnd == Rgn.End);
206 Sch.BaseClass::schedule();
207
208
209 Sch.RegionEnd = Rgn.End;
210
211 Rgn.Begin = Sch.RegionBegin;
212 Rgn.MaxPressure.clear();
213 }
214
216 assert(Sch.RegionBegin == Rgn.Begin && Sch.RegionEnd == Rgn.End);
217
218
219 Sch.scheduleRegion(Rgn, Sch.SUnits, SaveMaxRP);
220 }
221};
222
223namespace {
224
225
227public:
228 bool shouldTrackPressure() const override { return false; }
229 bool shouldTrackLaneMasks() const override { return false; }
230 void initialize(ScheduleDAGMI *DAG) override {}
231 SUnit *pickNode(bool &IsTopNode) override { return nullptr; }
232 void schedNode(SUnit *SU, bool IsTopNode) override {}
233 void releaseTopNode(SUnit *SU) override {}
234 void releaseBottomNode(SUnit *SU) override {}
235};
236
237}
238
241 : BaseClass(C, std::make_unique())
245}
246
247
251 const {
252
253
254
255 auto const BBEnd = Begin->getParent()->end();
256 auto const BottomMI = End == BBEnd ? std::prev(End) : End;
257
258
259
260 auto AfterBottomMI = std::next(BottomMI);
261 if (AfterBottomMI == BBEnd ||
262 &*AfterBottomMI != UPTracker.getLastTrackedMI()) {
264 } else {
266 }
267
268 for (auto I = BottomMI; I != Begin; --I)
270
272
274 (dbgs() << "Tracked region ",
276 return UPTracker.getMaxPressureAndReset();
277}
278
279
282 Range &&Schedule) const {
283 auto const BBEnd = R.Begin->getParent()->end();
285 if (R.End != BBEnd) {
286
287
290 } else {
291
292 RPTracker.reset(*std::prev(BBEnd));
293 }
294 for (auto I = Schedule.end(), B = Schedule.begin(); I != B;) {
296 }
297 return RPTracker.getMaxPressureAndReset();
298}
299
307 new (Alloc.Allocate())
308 Region { Begin, End, NumRegionInstrs,
309 getRegionPressure(Begin, End), nullptr });
310 }
311}
312
314
317 dbgs() << "Max RP: "
318 << print(Regions.back()->MaxPressure,
319 &MF.getSubtarget());
321 << '\n';);
322}
323
334
335
336
337std::vector<MachineInstr*>
339 std::vector<MachineInstr*> Res;
340 Res.reserve(Schedule.size() * 2);
341
344
346 for (const auto *SU : Schedule) {
347 Res.push_back(SU->getInstr());
348 const auto &D = std::find_if(DbgB, DbgE, [SU](decltype(*DbgB) &P) {
349 return P.second == SU->getInstr();
350 });
351 if (D != DbgE)
352 Res.push_back(D->first);
353 }
354 return Res;
355}
356
360 R.BestSchedule.reset(
362}
363
365 assert(R.BestSchedule.get() && "No schedule specified");
366 scheduleRegion(R, R.BestSchedule->Schedule, R.BestSchedule->MaxPressure);
367 R.BestSchedule.reset();
368}
369
370
371
372template
377#ifndef NDEBUG
379#endif
380 auto *BB = R.Begin->getParent();
381 auto Top = R.Begin;
382 for (const auto &I : Schedule) {
384 if (MI != &*Top) {
387 if (->isDebugInstr())
388 LIS->handleMove(*MI, true);
389 }
390 if (->isDebugInstr()) {
391
392 for (auto &Op : MI->all_defs())
393 Op.setIsUndef(false);
394
396 RegOpers.collect(*MI, *TRI, MRI, true,
397 false);
398
399 auto SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
401 }
402 Top = std::next(MI->getIterator());
403 }
405
406
407
408 if (!std::is_same_v<decltype(*Schedule.begin()), MachineInstr*>) {
410
411
413 }
414
416 R.MaxPressure = MaxRP;
417
418#ifndef NDEBUG
421#endif
423 (SchedMaxRP == RegionMaxRP && (MaxRP.empty() || SchedMaxRP == MaxRP)) ||
424 (dbgs() << "Max RP mismatch!!!\n"
425 "RP for schedule (calculated): "
426 << print(SchedMaxRP, &ST)
427 << "RP for schedule (reported): " << print(MaxRP, &ST)
428 << "RP after scheduling: " << print(RegionMaxRP, &ST),
429 false));
430}
431
432
435 return R2->MaxPressure.less(MF, R1->MaxPressure, TargetOcc);
436 });
437}
438
439
440
441
442
443
444
445
446
447
449
451 const unsigned DynamicVGPRBlockSize =
453 const auto Occ =
454 Regions.front()->MaxPressure.getOccupancy(ST, DynamicVGPRBlockSize);
455 LLVM_DEBUG(dbgs() << "Trying to improve occupancy, target = " << TargetOcc
456 << ", current = " << Occ << '\n');
457
458 auto NewOcc = TargetOcc;
459 for (auto *R : Regions) {
460
462
463 if (R->MaxPressure.getOccupancy(ST, DynamicVGPRBlockSize) >= NewOcc)
464 continue;
465
468
471 LLVM_DEBUG(dbgs() << "Occupancy improvement attempt:\n";
473
474 NewOcc = std::min(NewOcc, MaxRP.getOccupancy(ST, DynamicVGPRBlockSize));
475 if (NewOcc <= Occ)
476 break;
477
479 }
481 << ", prev occupancy = " << Occ << '\n');
482 if (NewOcc > Occ) {
484 MFI->increaseOccupancy(MF, NewOcc);
485 }
486
487 return std::max(NewOcc, Occ);
488}
489
491 bool TryMaximizeOccupancy) {
494 auto TgtOcc = MFI->getMinAllowedOccupancy();
495 unsigned DynamicVGPRBlockSize = MFI->getDynamicVGPRBlockSize();
496
498 auto Occ =
499 Regions.front()->MaxPressure.getOccupancy(ST, DynamicVGPRBlockSize);
500
501 bool IsReentry = false;
502 if (TryMaximizeOccupancy && Occ < TgtOcc) {
504 IsReentry = true;
505 }
506
507
508
509 const int NumPasses = Occ < TgtOcc ? 2 : 1;
510
511 TgtOcc = std::min(Occ, TgtOcc);
512 LLVM_DEBUG(dbgs() << "Scheduling using default scheduler, "
513 "target occupancy = "
514 << TgtOcc << '\n');
516 unsigned FinalOccupancy = std::min(Occ, MFI->getOccupancy());
517
518 for (int I = 0; I < NumPasses; ++I) {
519
520
522 for (auto *R : Regions) {
524 IsReentry |= I > 0;
529
530 if (RP.getOccupancy(ST, DynamicVGPRBlockSize) < TgtOcc) {
531 LLVM_DEBUG(dbgs() << "Didn't fit into target occupancy O" << TgtOcc);
532 if (R->BestSchedule.get() && R->BestSchedule->MaxPressure.getOccupancy(
533 ST, DynamicVGPRBlockSize) >= TgtOcc) {
534 LLVM_DEBUG(dbgs() << ", scheduling minimal register\n");
536 } else {
539 assert(R->MaxPressure.getOccupancy(ST, DynamicVGPRBlockSize) >=
540 TgtOcc);
541 }
542 }
543 FinalOccupancy =
544 std::min(FinalOccupancy, RP.getOccupancy(ST, DynamicVGPRBlockSize));
545 }
546 }
547 MFI->limitOccupancy(FinalOccupancy);
548}
549
550
551
552
555 const auto TgtOcc = MFI->getOccupancy();
557
558 auto MaxPressure = Regions.front()->MaxPressure;
559 for (auto *R : Regions) {
560 if (!force && R->MaxPressure.less(MF, MaxPressure, TgtOcc))
561 break;
562
565
567 LLVM_DEBUG(if (R->MaxPressure.less(MF, RP, TgtOcc)) {
568 dbgs() << "\nWarning: Pressure becomes worse after minreg!";
569 printSchedRP(dbgs(), R->MaxPressure, RP);
570 });
571
572 if (!force && MaxPressure.less(MF, RP, TgtOcc))
573 break;
574
577
578 MaxPressure = RP;
579 }
580}
581
582
583
584
586 bool TryMaximizeOccupancy) {
589 auto TgtOcc = MFI->getMinAllowedOccupancy();
590 unsigned DynamicVGPRBlockSize = MFI->getDynamicVGPRBlockSize();
591
593 auto Occ =
594 Regions.front()->MaxPressure.getOccupancy(ST, DynamicVGPRBlockSize);
595
596 bool IsReentry = false;
597 if (TryMaximizeOccupancy && Occ < TgtOcc) {
599 IsReentry = true;
600 }
601
602 TgtOcc = std::min(Occ, TgtOcc);
603 LLVM_DEBUG(dbgs() << "Scheduling using default scheduler, "
604 "target occupancy = "
605 << TgtOcc << '\n');
606
607 unsigned FinalOccupancy = std::min(Occ, MFI->getOccupancy());
608 for (auto *R : Regions) {
609 BuildDAG DAG(*R, *this, IsReentry);
611
614
615 if (RP.getOccupancy(ST, DynamicVGPRBlockSize) < TgtOcc) {
616 LLVM_DEBUG(dbgs() << "Didn't fit into target occupancy O" << TgtOcc);
617 if (R->BestSchedule.get() && R->BestSchedule->MaxPressure.getOccupancy(
618 ST, DynamicVGPRBlockSize) >= TgtOcc) {
619 LLVM_DEBUG(dbgs() << ", scheduling minimal register\n");
621 }
622 } else {
625 FinalOccupancy =
626 std::min(FinalOccupancy, RP.getOccupancy(ST, DynamicVGPRBlockSize));
627 }
628 }
629 MFI->limitOccupancy(FinalOccupancy);
630}
unsigned const MachineRegisterInfo * MRI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
static LLVM_DUMP_METHOD void printLivenessInfo(raw_ostream &OS, MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, const LiveIntervals *LIS)
Definition GCNIterativeScheduler.cpp:76
static MachineInstr * getMachineInstr(MachineInstr *MI)
Definition GCNIterativeScheduler.cpp:33
static LLVM_DUMP_METHOD void printRegion(raw_ostream &OS, MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, const LiveIntervals *LIS, unsigned MaxInstNum=std::numeric_limits< unsigned >::max())
Definition GCNIterativeScheduler.cpp:45
This file defines the class GCNIterativeScheduler, which uses an iterative approach to find a best sc...
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, const llvm::StringTable &StandardNames, VectorLibrary VecLib)
Initialize the set of available library functions based on the specified target triple.
ArrayRef< SUnit * > getBottomRoots() const
Definition GCNIterativeScheduler.cpp:170
~BuildDAG()
Definition GCNIterativeScheduler.cpp:162
ArrayRef< const SUnit * > getTopRoots() const
Definition GCNIterativeScheduler.cpp:167
BuildDAG(const Region &R, GCNIterativeScheduler &_Sch, bool IsReentry=false)
Definition GCNIterativeScheduler.cpp:149
~OverrideLegacyStrategy()
Definition GCNIterativeScheduler.cpp:195
void schedule()
Definition GCNIterativeScheduler.cpp:202
void restoreOrder()
Definition GCNIterativeScheduler.cpp:215
OverrideLegacyStrategy(Region &R, MachineSchedStrategy &OverrideStrategy, GCNIterativeScheduler &_Sch)
Definition GCNIterativeScheduler.cpp:182
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
SpecificBumpPtrAllocator< Region > Alloc
void printSchedRP(raw_ostream &OS, const GCNRegPressure &Before, const GCNRegPressure &After) const
Definition GCNIterativeScheduler.cpp:113
void enterRegion(MachineBasicBlock *BB, MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, unsigned RegionInstrs) override
Initialize the DAG and common scheduler state for a new scheduling region.
Definition GCNIterativeScheduler.cpp:300
void sortRegionsByPressure(unsigned TargetOcc)
Definition GCNIterativeScheduler.cpp:433
std::vector< Region * > Regions
void scheduleILP(bool TryMaximizeOccupancy=true)
Definition GCNIterativeScheduler.cpp:585
GCNUpwardRPTracker UPTracker
void swapIGLPMutations(const Region &R, bool IsReentry)
Definition GCNIterativeScheduler.cpp:122
void scheduleBest(Region &R)
Definition GCNIterativeScheduler.cpp:364
MachineSchedContext * Context
GCNIterativeScheduler(MachineSchedContext *C, StrategyKind S)
Definition GCNIterativeScheduler.cpp:239
void printSchedResult(raw_ostream &OS, const Region *R, const GCNRegPressure &RP) const
Definition GCNIterativeScheduler.cpp:103
unsigned tryMaximizeOccupancy(unsigned TargetOcc=std::numeric_limits< unsigned >::max())
Definition GCNIterativeScheduler.cpp:448
void printRegions(raw_ostream &OS) const
Definition GCNIterativeScheduler.cpp:92
@ SCHEDULE_LEGACYMAXOCCUPANCY
void setBestSchedule(Region &R, ScheduleRef Schedule, const GCNRegPressure &MaxRP=GCNRegPressure())
Definition GCNIterativeScheduler.cpp:357
void finalizeSchedule() override
Allow targets to perform final scheduling actions at the level of the whole MachineFunction.
Definition GCNIterativeScheduler.cpp:324
void scheduleLegacyMaxOccupancy(bool TryMaximizeOccupancy=true)
Definition GCNIterativeScheduler.cpp:490
std::vector< std::unique_ptr< ScheduleDAGMutation > > SavedMutations
void schedule() override
Orders nodes according to selected style.
Definition GCNIterativeScheduler.cpp:313
GCNRegPressure getSchedulePressure(const Region &R, Range &&Schedule) const
Definition GCNIterativeScheduler.cpp:281
void scheduleMinReg(bool force=false)
Definition GCNIterativeScheduler.cpp:553
GCNRegPressure getRegionPressure(MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End) const
Definition GCNIterativeScheduler.cpp:249
ArrayRef< const SUnit * > ScheduleRef
void scheduleRegion(Region &R, Range &&Schedule, const GCNRegPressure &MaxRP=GCNRegPressure())
Definition GCNIterativeScheduler.cpp:373
const StrategyKind Strategy
std::vector< MachineInstr * > detachSchedule(ScheduleRef Schedule) const
Definition GCNIterativeScheduler.cpp:338
The goal of this scheduling strategy is to maximize kernel occupancy (i.e.
void setTargetOccupancy(unsigned Occ)
SlotIndex getInstructionIndex(const MachineInstr &Instr) const
Returns the base index of the given instruction.
MachineInstrBundleIterator< MachineInstr > iterator
Representation of each machine instruction.
MachineSchedStrategy - Interface to the scheduling algorithm used by ScheduleDAGMI.
List of registers defined and used by a machine instruction.
LLVM_ABI void collect(const MachineInstr &MI, const TargetRegisterInfo &TRI, const MachineRegisterInfo &MRI, bool TrackLaneMasks, bool IgnoreDead)
Analyze the given instruction MI and fill in the Uses, Defs and DeadDefs list based on the MachineOpe...
LLVM_ABI void adjustLaneLiveness(const LiveIntervals &LIS, const MachineRegisterInfo &MRI, SlotIndex Pos, MachineInstr *AddFlagsMI=nullptr)
Use liveness information to find out which uses/defs are partially undefined/dead and adjust the VReg...
bool isIGLPMutationOnly(unsigned Opcode) const
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
Scheduling unit. This is a node in the scheduling DAG.
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
MachineBasicBlock * BB
The block in which to insert instructions.
MachineInstr * FirstDbgValue
MachineBasicBlock::iterator RegionEnd
The end of the range to be scheduled.
DbgValueVector DbgValues
Remember instruction that precedes DBG_VALUE.
MachineBasicBlock::iterator RegionBegin
The beginning of the range to be scheduled.
unsigned NumRegionInstrs
Instructions in this region (distance(RegionBegin, RegionEnd)).
const MachineFrameInfo & MFI
void enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, MachineBasicBlock::iterator end, unsigned regioninstrs) override
Implement the ScheduleDAGInstrs interface for handling the next scheduling region.
RegPressureTracker RPTracker
std::unique_ptr< MachineSchedStrategy > SchedImpl
void addMutation(std::unique_ptr< ScheduleDAGMutation > Mutation)
Add a postprocessing step to the DAG builder.
void placeDebugValues()
Reinsert debug_values recorded in ScheduleDAGInstrs::DbgValues.
std::vector< std::unique_ptr< ScheduleDAGMutation > > Mutations
Ordered list of DAG postprocessing steps.
MachineRegisterInfo & MRI
Virtual/real register map.
const TargetInstrInfo * TII
Target instruction information.
const TargetRegisterInfo * TRI
Target processor register info.
MachineFunction & MF
Machine function.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class implements an extremely fast bulk output stream that can only output to a stream.
@ C
The default llvm calling convention, compatible with C.
This is an optimization pass for GlobalISel generic memory operations.
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)
GCNRegPressure getRegPressure(const MachineRegisterInfo &MRI, Range &&LiveRegs)
std::unique_ptr< ScheduleDAGMutation > createIGroupLPDAGMutation(AMDGPU::SchedulingPhase Phase)
Phase specifes whether or not this is a reentry into the IGroupLPDAGMutation.
GCNRPTracker::LiveRegSet getLiveRegsAfter(const MachineInstr &MI, const LiveIntervals &LIS)
void sort(IteratorTy Start, IteratorTy End)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
DWARFExpression::Operation Op
std::vector< const SUnit * > makeGCNILPScheduler(ArrayRef< const SUnit * > BotRoots, const ScheduleDAG &DAG)
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
std::vector< const SUnit * > makeMinRegSchedule(ArrayRef< const SUnit * > TopRoots, const ScheduleDAG &DAG)
GCNRPTracker::LiveRegSet getLiveRegsBefore(const MachineInstr &MI, const LiveIntervals &LIS)
LLVM_ABI Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Implement std::hash so that hash_code can be used in STL containers.
GCNRegPressure MaxPressure
MachineSchedContext provides enough context from the MachineScheduler pass for the target to instanti...