SIMachineScheduler.h Source File (original) (raw)

14#ifndef LLVM_LIB_TARGET_AMDGPU_SIMACHINESCHEDULER_H

15#define LLVM_LIB_TARGET_AMDGPU_SIMACHINESCHEDULER_H

20#include

21#include

22#include

24namespace llvm {

62 std::vector<SUnit*> SUnits;

63 std::map<unsigned, unsigned> NodeNum2Index;

64 std::vector<SUnit*> TopReadySUs;

65 std::vector<SUnit*> ScheduledSUnits;

75 std::vector InternalAdditionalPressure;

77 std::vector LiveInPressure;

78 std::vector LiveOutPressure;

84 std::set LiveInRegs;

85 std::set LiveOutRegs;

87 bool Scheduled = false;

88 bool HighLatencyBlock = false;

90 std::vector HasLowLatencyNonWaitedParent;

93 unsigned ID;

95 std::vector<SIScheduleBlock*> Preds;

97 std::vector<std::pair<SIScheduleBlock*, SIScheduleBlockLinkKind>> Succs;

98 unsigned NumHighLatencySuccessors = 0;

100public:

102 unsigned ID):

103 DAG(DAG), BC(BC), TopRPTracker(TopPressure), ID(ID) {}

104

106

107 unsigned getID() const { return ID; }

108

109

111

112

114

115

118

119 const std::vector<SIScheduleBlock*>& getPreds() const { return Preds; }

122

123 unsigned Height = 0;

124 unsigned Depth = 0;

125

127 return NumHighLatencySuccessors;

128 }

129

131

132

133

134

135 int getCost() { return SUnits.size(); }

136

137

138

139

141

143

144

145

146

147

148

151

153

154

155

157 return InternalAdditionalPressure;

158 }

159

160 std::set &getInRegs() { return LiveInRegs; }

161 std::set &getOutRegs() { return LiveOutRegs; }

162

164

165private:

167

168 SUnit *SU = nullptr;

169

170 unsigned SGPRUsage;

171 unsigned VGPRUsage;

172 bool IsLowLatency;

173 unsigned LowLatencyOffset;

174 bool HasLowLatencyNonWaitedParent;

175

176 SISchedCandidate() = default;

177

178 bool isValid() const { return SU; }

179

180

181 void setBest(SISchedCandidate &Best) {

182 assert(Best.Reason != NoCand && "uninitialized Sched candidate");

183 SU = Best.SU;

184 Reason = Best.Reason;

185 SGPRUsage = Best.SGPRUsage;

186 VGPRUsage = Best.VGPRUsage;

187 IsLowLatency = Best.IsLowLatency;

188 LowLatencyOffset = Best.LowLatencyOffset;

189 HasLowLatencyNonWaitedParent = Best.HasLowLatencyNonWaitedParent;

190 }

191 };

192

193 void undoSchedule();

194

195 void undoReleaseSucc(SUnit *SU, SDep *SuccEdge);

196 void releaseSucc(SUnit *SU, SDep *SuccEdge);

197

198

199 void releaseSuccessors(SUnit *SU, bool InOrOutBlock);

200

201 void nodeScheduled(SUnit *SU);

202 void tryCandidateTopDown(SISchedCandidate &Cand, SISchedCandidate &TryCand);

203 void tryCandidateBottomUp(SISchedCandidate &Cand, SISchedCandidate &TryCand);

204 SUnit* pickNode();

205 void traceCandidate(const SISchedCandidate &Cand);

208};

209

211 std::vector<SIScheduleBlock*> Blocks;

214};

215

221

224

225 std::vector<std::unique_ptr> BlockPtrs;

228 std::vector<SIScheduleBlock*> CurrentBlocks;

229 std::vector Node2CurrentBlock;

230

231

232

233 std::vector TopDownIndex2Block;

234 std::vector TopDownBlock2Index;

235 std::vector BottomUpIndex2Block;

236

237

238

239

240 int NextReservedID;

241 int NextNonReservedID;

242 std::vector CurrentColoring;

243 std::vector CurrentTopDownReservedDependencyColoring;

244 std::vector CurrentBottomUpReservedDependencyColoring;

245

246public:

248

251

253

254private:

255

256 void colorHighLatenciesAlone();

257

258

259 void colorHighLatenciesGroups();

260

261

262

263 void colorComputeReservedDependencies();

264

265

266 void colorAccordingToReservedDependencies();

267

268

269

270

271 void colorEndsAccordingToDependencies();

272

273

274 void colorForceConsecutiveOrderInGroup();

275

276

277

278 void colorMergeConstantLoadsNextGroup();

279

280

281 void colorMergeIfPossibleNextGroup();

282

283

284

285 void colorMergeIfPossibleNextGroupOnlyForReserved();

286

287

288

289 void colorMergeIfPossibleSmallGroupsToNextGroup();

290

291

292

293

294 void cutHugeBlocks();

295

296

297

298 void regroupNoUserInstructions();

299

300

301 void colorExports();

302

304

305 void topologicalSort();

306

307 void scheduleInsideBlocks();

308

309 void fillStats();

310};

311

317

321 std::vector<SIScheduleBlock*> Blocks;

322

323 std::vector<std::map<Register, unsigned>> LiveOutRegsNumUsages;

324 std::set LiveRegs;

325

326 std::map<Register, unsigned> LiveRegsConsumers;

327

328 std::vector LastPosHighLatencyParentScheduled;

329 int LastPosWaitedHighLatency;

330

331 std::vector<SIScheduleBlock*> BlocksScheduled;

332 unsigned NumBlockScheduled;

333 std::vector<SIScheduleBlock*> ReadyBlocks;

334

335 unsigned VregCurrentUsage;

336 unsigned SregCurrentUsage;

337

338

339 unsigned maxVregUsage;

340 unsigned maxSregUsage;

341

342 std::vector BlockNumPredsLeft;

343 std::vector BlockNumSuccsLeft;

344

345public:

350

351 std::vector<SIScheduleBlock*> getBlocks() { return BlocksScheduled; }

352

355

356private:

358

360

361 bool IsHighLatency;

362 int VGPRUsageDiff;

363 unsigned NumSuccessors;

364 unsigned NumHighLatencySuccessors;

365 unsigned LastPosHighLatParentScheduled;

366 unsigned Height;

367

368 SIBlockSchedCandidate() = default;

369

371

372

373 void setBest(SIBlockSchedCandidate &Best) {

374 assert(Best.Reason != NoCand && "uninitialized Sched candidate");

375 Block = Best.Block;

376 Reason = Best.Reason;

377 IsHighLatency = Best.IsHighLatency;

378 VGPRUsageDiff = Best.VGPRUsageDiff;

379 NumSuccessors = Best.NumSuccessors;

380 NumHighLatencySuccessors = Best.NumHighLatencySuccessors;

381 LastPosHighLatParentScheduled = Best.LastPosHighLatParentScheduled;

382 Height = Best.Height;

383 }

384 };

385

386 bool tryCandidateLatency(SIBlockSchedCandidate &Cand,

387 SIBlockSchedCandidate &TryCand);

388 bool tryCandidateRegUsage(SIBlockSchedCandidate &Cand,

389 SIBlockSchedCandidate &TryCand);

390 SIScheduleBlock *pickBlock();

391

392 void addLiveRegs(std::set &Regs);

393 void decreaseLiveRegs(SIScheduleBlock *Block, std::set &Regs);

394 void releaseBlockSuccs(SIScheduleBlock *Parent);

395 void blockScheduled(SIScheduleBlock *Block);

396

397

398

399 std::vector checkRegUsageImpact(std::set &InRegs,

400 std::set &OutRegs);

401

402 void schedule();

403};

404

406 std::vector SUs;

409};

410

414

415public:

417

419

423};

424

428

429 std::vector SUnitsLinksBackup;

430

431

432 std::vector ScheduledSUnits;

433 std::vector ScheduledSUnitsInv;

434

435public:

437

439

440

442

443

447

457

459

461 _Iterator End,

462 unsigned &VgprUsage,

463 unsigned &SgprUsage);

464

466 std::set InRegs;

467 for (const auto &RegMaskPair : RPTracker.getPressure().LiveInRegs) {

468 InRegs.insert(RegMaskPair.VRegOrUnit);

469 }

470 return InRegs;

471 }

472

474 std::set OutRegs;

475 for (const auto &RegMaskPair : RPTracker.getPressure().LiveOutRegs) {

476 OutRegs.insert(RegMaskPair.VRegOrUnit);

477 }

478 return OutRegs;

479 };

480

481private:

482 void topologicalSort();

483

484 void moveLowLatencies();

485

486public:

487

491

492

495};

496

497}

498

499#endif

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static bool isValid(const char C)

Returns true if C is a valid mangled character: <0-9a-zA-Z_>.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

MachineInstrBundleIterator< MachineInstr > iterator

MachineRegisterInfo - Keep track of information for virtual and physical registers,...

Track the current register pressure at some position in the instruction stream, and remember the high...

bool isSUInBlock(SUnit *SU, unsigned ID)

SIScheduleBlockCreator(SIScheduleDAGMI *DAG)

SIScheduleBlocks getBlocks(SISchedulerBlockCreatorVariant BlockVariant)

SIScheduleBlockScheduler(SIScheduleDAGMI *DAG, SISchedulerBlockSchedulerVariant Variant, SIScheduleBlocks BlocksStruct)

unsigned getVGPRUsage()

Definition SIMachineScheduler.h:353

~SIScheduleBlockScheduler()=default

unsigned getSGPRUsage()

Definition SIMachineScheduler.h:354

std::vector< SIScheduleBlock * > getBlocks()

Definition SIMachineScheduler.h:351

unsigned getNumHighLatencySuccessors() const

Definition SIMachineScheduler.h:126

SIScheduleBlock(SIScheduleDAGMI *DAG, SIScheduleBlockCreator *BC, unsigned ID)

Definition SIMachineScheduler.h:101

ArrayRef< std::pair< SIScheduleBlock *, SIScheduleBlockLinkKind > > getSuccs() const

Definition SIMachineScheduler.h:121

int getCost()

Definition SIMachineScheduler.h:135

~SIScheduleBlock()=default

std::vector< unsigned > & getInternalAdditionalRegUsage()

Definition SIMachineScheduler.h:156

const std::vector< SIScheduleBlock * > & getPreds() const

Definition SIMachineScheduler.h:119

void addPred(SIScheduleBlock *Pred)

std::set< Register > & getOutRegs()

Definition SIMachineScheduler.h:161

void printDebug(bool Full)

std::vector< SUnit * > getScheduledUnits()

Definition SIMachineScheduler.h:142

void addSucc(SIScheduleBlock *Succ, SIScheduleBlockLinkKind Kind)

unsigned Depth

Definition SIMachineScheduler.h:124

unsigned getID() const

Definition SIMachineScheduler.h:107

bool isScheduled()

Definition SIMachineScheduler.h:152

void schedule(MachineBasicBlock::iterator BeginBlock, MachineBasicBlock::iterator EndBlock)

void addUnit(SUnit *SU)

Functions for Block construction.

std::set< Register > & getInRegs()

Definition SIMachineScheduler.h:160

unsigned Height

Definition SIMachineScheduler.h:123

bool isHighLatencyBlock()

Definition SIMachineScheduler.h:130

SUnit & getEntrySU()

Definition SIMachineScheduler.h:455

void restoreSULinksLeft()

std::vector< int > BottomUpIndex2SU

Definition SIMachineScheduler.h:494

std::vector< unsigned > IsHighLatencySU

Definition SIMachineScheduler.h:490

std::vector< unsigned > LowLatencyOffset

Definition SIMachineScheduler.h:489

std::vector< int > TopDownIndex2SU

Definition SIMachineScheduler.h:493

void schedule() override

Implement ScheduleDAGInstrs interface for scheduling a sequence of reorderable instructions.

LiveIntervals * getLIS()

Definition SIMachineScheduler.h:451

ScheduleDAGTopologicalSort * GetTopo()

Definition SIMachineScheduler.h:454

void fillVgprSgprCost(_Iterator First, _Iterator End, unsigned &VgprUsage, unsigned &SgprUsage)

MachineRegisterInfo * getMRI()

Definition SIMachineScheduler.h:452

SIScheduleDAGMI(MachineSchedContext *C)

MachineBasicBlock::iterator getCurrentBottom()

Definition SIMachineScheduler.h:450

std::vector< unsigned > IsLowLatencySU

Definition SIMachineScheduler.h:488

MachineBasicBlock::iterator getCurrentTop()

Definition SIMachineScheduler.h:449

SUnit & getExitSU()

Definition SIMachineScheduler.h:456

MachineBasicBlock * getBB()

Definition SIMachineScheduler.h:448

void initRPTracker(RegPressureTracker &RPTracker)

Definition SIMachineScheduler.h:444

std::set< VirtRegOrUnit > getOutRegs()

Definition SIMachineScheduler.h:473

~SIScheduleDAGMI() override

const TargetRegisterInfo * getTRI()

Definition SIMachineScheduler.h:453

std::set< VirtRegOrUnit > getInRegs()

Definition SIMachineScheduler.h:465

struct SIScheduleBlockResult scheduleVariant(SISchedulerBlockCreatorVariant BlockVariant, SISchedulerBlockSchedulerVariant ScheduleVariant)

SIScheduler(SIScheduleDAGMI *DAG)

Definition SIMachineScheduler.h:416

Scheduling unit. This is a node in the scheduling DAG.

MachineBasicBlock * BB

The block in which to insert instructions.

ScheduleDAGTopologicalSort Topo

Topo - A topological ordering for SUnits which permits fast IsReachable and similar queries.

MachineBasicBlock::iterator RegionBegin

The beginning of the range to be scheduled.

ScheduleDAGMILive(MachineSchedContext *C, std::unique_ptr< MachineSchedStrategy > S)

RegisterClassInfo * RegClassInfo

RegPressureTracker RPTracker

MachineBasicBlock::iterator CurrentBottom

The bottom of the unscheduled zone.

MachineBasicBlock::iterator CurrentTop

The top of the unscheduled zone.

This class can compute a topological ordering for SUnits and provides methods for dynamically updatin...

MachineRegisterInfo & MRI

Virtual/real register map.

const TargetRegisterInfo * TRI

Target processor register info.

SUnit EntrySU

Special node for the region entry.

MachineFunction & MF

Machine function.

SUnit ExitSU

Special node for the region exit.

TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

@ C

The default llvm calling convention, compatible with C.

This is an optimization pass for GlobalISel generic memory operations.

SIScheduleCandReason

Definition SIMachineScheduler.h:31

@ Latency

Definition SIMachineScheduler.h:34

@ NodeOrder

Definition SIMachineScheduler.h:37

@ Successor

Definition SIMachineScheduler.h:35

@ RegUsage

Definition SIMachineScheduler.h:33

@ NoCand

Definition SIMachineScheduler.h:32

@ Depth

Definition SIMachineScheduler.h:36

SISchedulerBlockSchedulerVariant

Definition SIMachineScheduler.h:312

@ BlockLatencyRegUsage

Definition SIMachineScheduler.h:313

@ BlockRegUsageLatency

Definition SIMachineScheduler.h:314

@ BlockRegUsage

Definition SIMachineScheduler.h:315

SISchedulerBlockCreatorVariant

Definition SIMachineScheduler.h:216

@ LatenciesAlonePlusConsecutive

Definition SIMachineScheduler.h:219

@ LatenciesAlone

Definition SIMachineScheduler.h:217

@ LatenciesGrouped

Definition SIMachineScheduler.h:218

@ First

Helpers to iterate all locations in the MemoryEffectsBase class.

FunctionAddr VTableAddr uintptr_t uintptr_t Data

SIScheduleBlockLinkKind

Definition SIMachineScheduler.h:53

@ NoData

Definition SIMachineScheduler.h:54

RegisterPressure computed within a region of instructions delimited by TopIdx and BottomIdx.

MachineSchedContext provides enough context from the MachineScheduler pass for the target to instanti...

unsigned MaxVGPRUsage

Definition SIMachineScheduler.h:408

unsigned MaxSGPRUsage

Definition SIMachineScheduler.h:407

std::vector< unsigned > SUs

Definition SIMachineScheduler.h:406

std::vector< int > TopDownIndex2Block

Definition SIMachineScheduler.h:212

std::vector< SIScheduleBlock * > Blocks

Definition SIMachineScheduler.h:211

std::vector< int > TopDownBlock2Index

Definition SIMachineScheduler.h:213

SISchedulerCandidate()=default

SIScheduleCandReason Reason

Definition SIMachineScheduler.h:42

bool isRepeat(SIScheduleCandReason R)

Definition SIMachineScheduler.h:49

uint32_t RepeatReasonSet

Definition SIMachineScheduler.h:45

void setRepeat(SIScheduleCandReason R)

Definition SIMachineScheduler.h:50