ResourceManager.cpp Source File (original) (raw)

20namespace llvm {

21namespace mca {

23#define DEBUG_TYPE "llvm-mca"

27 uint64_t &NextInSequenceMask) {

30 NextInSequenceMask &= (CandidateMask | (CandidateMask - 1));

31 return CandidateMask;

32}

36 uint64_t CandidateMask = ReadyMask & NextInSequenceMask;

37 if (CandidateMask)

38 return selectImpl(CandidateMask, NextInSequenceMask);

40 NextInSequenceMask = ResourceUnitMask ^ RemovedFromNextInSequence;

41 RemovedFromNextInSequence = 0;

42 CandidateMask = ReadyMask & NextInSequenceMask;

43 if (CandidateMask)

44 return selectImpl(CandidateMask, NextInSequenceMask);

46 NextInSequenceMask = ResourceUnitMask;

47 CandidateMask = ReadyMask & NextInSequenceMask;

48 return selectImpl(CandidateMask, NextInSequenceMask);

49}

52 if (Mask > NextInSequenceMask) {

53 RemovedFromNextInSequence |= Mask;

54 return;

55 }

57 NextInSequenceMask &= (~Mask);

58 if (NextInSequenceMask)

59 return;

61 NextInSequenceMask = ResourceUnitMask ^ RemovedFromNextInSequence;

62 RemovedFromNextInSequence = 0;

63}

67 : ProcResourceDescIndex(Index), ResourceMask(Mask),

68 BufferSize(Desc.BufferSize), IsAGroup(llvm::popcount(ResourceMask) > 1) {

69 if (IsAGroup) {

70 ResourceSizeMask =

72 } else {

73 ResourceSizeMask = (1ULL << Desc.NumUnits) - 1;

74 }

75 ReadyMask = ResourceSizeMask;

76 AvailableSlots = BufferSize == -1 ? 0U : static_cast<unsigned>(BufferSize);

78}

80bool ResourceState::isReady(unsigned NumUnits) const {

81 return (!isReserved() || isADispatchHazard()) &&

83}

86 if (isADispatchHazard() && isReserved())

88 if (!isBuffered() || AvailableSlots)

91}

93#ifndef NDEBUG

94void ResourceState::dump() const {

96 << ", SZMASK=" << format_hex(ResourceSizeMask, 16)

97 << ", RDYMASK=" << format_hex(ReadyMask, 16)

98 << ", BufferSize=" << BufferSize

99 << ", AvailableSlots=" << AvailableSlots

101}

102#endif

103

104static std::unique_ptr

106 if (RS.isAResourceGroup() || RS.getNumUnits() > 1)

107 return std::make_unique(RS.getReadyMask());

108 return std::unique_ptr(nullptr);

109}

110

112 : Resources(SM.getNumProcResourceKinds() - 1),

113 Strategies(SM.getNumProcResourceKinds() - 1),

114 Resource2Groups(SM.getNumProcResourceKinds() - 1, 0),

115 ProcResID2Mask(SM.getNumProcResourceKinds(), 0),

116 ResIndex2ProcResID(SM.getNumProcResourceKinds() - 1, 0),

117 ProcResUnitMask(0), ReservedResourceGroups(0), AvailableBuffers(~0ULL),

118 ReservedBuffers(0) {

120

121

124 ResIndex2ProcResID[Index] = I;

125 }

126

128 uint64_t Mask = ProcResID2Mask[I];

130 Resources[Index] =

131 std::make_unique(*SM.getProcResource(I), I, Mask);

132 Strategies[Index] = getStrategyFor(*Resources[Index]);

133 }

134

136 uint64_t Mask = ProcResID2Mask[I];

139 if (.isAResourceGroup()) {

140 ProcResUnitMask |= Mask;

141 continue;

142 }

143

144 uint64_t GroupMaskIdx = 1ULL << Index;

145 Mask -= GroupMaskIdx;

146 while (Mask) {

147

150 Resource2Groups[IndexUnit] |= GroupMaskIdx;

152 }

153 }

154

155 AvailableProcResUnits = ProcResUnitMask;

156}

157

158void ResourceManager::setCustomStrategyImpl(std::unique_ptr S,

159 uint64_t ResourceMask) {

161 assert(Index < Resources.size() && "Invalid processor resource index!");

162 assert(S && "Unexpected null strategy in input!");

163 Strategies[Index] = std::move(S);

164}

165

166unsigned ResourceManager::resolveResourceMask(uint64_t Mask) const {

168}

169

170unsigned ResourceManager::getNumUnits(uint64_t ResourceID) const {

172}

173

174

175

176

177ResourceRef ResourceManager::selectPipe(uint64_t ResourceID) {

179 assert(Index < Resources.size() && "Invalid resource use!");

180 ResourceState &RS = *Resources[Index];

181 assert(RS.isReady() && "No available units to select!");

182

183

184

185 if (.isAResourceGroup() && RS.getNumUnits() == 1)

186 return std::make_pair(ResourceID, RS.getReadyMask());

187

188 uint64_t SubResourceID = Strategies[Index]->select(RS.getReadyMask());

189 if (RS.isAResourceGroup())

190 return selectPipe(SubResourceID);

191 return std::make_pair(ResourceID, SubResourceID);

192}

193

194void ResourceManager::use(const ResourceRef &RR) {

195

197 ResourceState &RS = *Resources[RSID];

198 RS.markSubResourceAsUsed(RR.second);

199

200

201 if (RS.getNumUnits() > 1)

202 Strategies[RSID]->used(RR.second);

203

204

205

206 if (RS.isReady())

207 return;

208

209 AvailableProcResUnits ^= RR.first;

210

211

212 uint64_t Users = Resource2Groups[RSID];

214

216 ResourceState &CurrentUser = *Resources[GroupIndex];

217 CurrentUser.markSubResourceAsUsed(RR.first);

218 Strategies[GroupIndex]->used(RR.first);

219

221 }

222}

223

224void ResourceManager::release(const ResourceRef &RR) {

226 ResourceState &RS = *Resources[RSID];

227 bool WasFullyUsed = .isReady();

228 RS.releaseSubResource(RR.second);

229 if (!WasFullyUsed)

230 return;

231

232 AvailableProcResUnits ^= RR.first;

233

234

235 uint64_t Users = Resource2Groups[RSID];

238 ResourceState &CurrentUser = *Resources[GroupIndex];

239 CurrentUser.releaseSubResource(RR.first);

241 }

242}

243

245ResourceManager::canBeDispatched(uint64_t ConsumedBuffers) const {

246 if (ConsumedBuffers & ReservedBuffers)

247 return ResourceStateEvent::RS_RESERVED;

248 if (ConsumedBuffers & (~AvailableBuffers))

249 return ResourceStateEvent::RS_BUFFER_UNAVAILABLE;

250 return ResourceStateEvent::RS_BUFFER_AVAILABLE;

251}

252

253void ResourceManager::reserveBuffers(uint64_t ConsumedBuffers) {

254 while (ConsumedBuffers) {

255 uint64_t CurrentBuffer = ConsumedBuffers & (-ConsumedBuffers);

257 ConsumedBuffers ^= CurrentBuffer;

258 assert(RS.isBufferAvailable() == ResourceStateEvent::RS_BUFFER_AVAILABLE);

259 if (.reserveBuffer())

260 AvailableBuffers ^= CurrentBuffer;

261 if (RS.isADispatchHazard()) {

262

263

264

265 ReservedBuffers ^= CurrentBuffer;

266 }

267 }

268}

269

270void ResourceManager::releaseBuffers(uint64_t ConsumedBuffers) {

271 AvailableBuffers |= ConsumedBuffers;

272 while (ConsumedBuffers) {

273 uint64_t CurrentBuffer = ConsumedBuffers & (-ConsumedBuffers);

275 ConsumedBuffers ^= CurrentBuffer;

276 RS.releaseBuffer();

277

278

279 }

280}

281

282uint64_t ResourceManager::checkAvailability(const InstrDesc &Desc) const {

283 uint64_t BusyResourceMask = 0;

284 uint64_t ConsumedResourceMask = 0;

285 DenseMap<uint64_t, unsigned> AvailableUnits;

286

287 for (const std::pair<uint64_t, ResourceUsage> &E : Desc.Resources) {

288 unsigned NumUnits = E.second.isReserved() ? 0U : E.second.NumUnits;

290 if (.isReady(NumUnits)) {

291 BusyResourceMask |= E.first;

292 continue;

293 }

294

295 if (Desc.HasPartiallyOverlappingGroups && .isAResourceGroup()) {

296 unsigned NumAvailableUnits = llvm::popcount(RS.getReadyMask());

297 NumAvailableUnits -= NumUnits;

298 AvailableUnits[E.first] = NumAvailableUnits;

299 if (!NumAvailableUnits)

300 ConsumedResourceMask |= E.first;

301 }

302 }

303

304 BusyResourceMask &= ProcResUnitMask;

305 if (BusyResourceMask)

306 return BusyResourceMask;

307

308 BusyResourceMask = Desc.UsedProcResGroups & ReservedResourceGroups;

309 if ( Desc .HasPartiallyOverlappingGroups || BusyResourceMask)

310 return BusyResourceMask;

311

312

313

314 for (const std::pair<uint64_t, ResourceUsage> &E : Desc.Resources) {

316 if (.second.isReserved() && RS.isAResourceGroup()) {

317 uint64_t ReadyMask = RS.getReadyMask() & ~ConsumedResourceMask;

318 if (!ReadyMask) {

319 BusyResourceMask |= RS.getReadyMask();

320 continue;

321 }

322

324

325 auto [it, Inserted] = AvailableUnits.try_emplace(ResourceMask);

326 if (Inserted) {

328 unsigned NumUnits = llvm::popcount(Resources[Index]->getReadyMask());

329 it->second = NumUnits;

330 }

331

332 if (!it->second) {

333 BusyResourceMask |= it->first;

334 continue;

335 }

336

337 it->second--;

338 if (!it->second)

339 ConsumedResourceMask |= it->first;

340 }

341 }

342

343 return BusyResourceMask;

344}

345

346void ResourceManager::issueInstructionImpl(

348

349

350

351

352

353

355

356 using ResourceWithUsage = std::pair<uint64_t, ResourceUsage>;

357

358 for (const ResourceWithUsage &R : Desc.Resources) {

359 const CycleSegment &CS = R.second.CS;

360 if (!CS.size()) {

361 releaseResource(R.first);

362 continue;

363 }

364

365 assert(CS.begin() == 0 && "Invalid {Start, End} cycles!");

366 if (R.second.isReserved()) {

368

369 assert(R.second.isReserved());

370 reserveResource(R.first);

371 BusyResources[ResourceRef(R.first, R.first)] += CS.size();

372 continue;

373 }

374

376 if (RS.isAResourceGroup() && RS.getNumReadyUnits() > 1) {

377 Worklist.push_back(R);

378 continue;

379 }

380

382 use(Pipe);

383 BusyResources[Pipe] += CS.size();

384 Pipes.emplace_back(std::make_pair(Pipe, ReleaseAtCycles(CS.size())));

385 }

386

387

388

389

390

391

392

393

394 while (!Worklist.empty()) {

395 sort(Worklist, [&](const ResourceWithUsage &Lhs,

396 const ResourceWithUsage &Rhs) {

399 uint64_t LhsReadyUnits = LhsRS.getNumReadyUnits();

400 uint64_t RhsReadyUnits = RhsRS.getNumReadyUnits();

401 if (LhsReadyUnits == RhsReadyUnits)

402 return Lhs.first < Rhs.first;

403 return LhsReadyUnits < RhsReadyUnits;

404 });

405

407

408 for (unsigned I = 0, E = Worklist.size(); I < E; ++I) {

409 const auto &Elt = Worklist[I];

411

412 if (I == 0 || RS.getNumReadyUnits() == 1) {

414 use(Pipe);

415 const CycleSegment &CS = Elt.second.CS;

416 BusyResources[Pipe] += CS.size();

417 Pipes.emplace_back(std::make_pair(Pipe, ReleaseAtCycles(CS.size())));

418 continue;

419 }

420

421 NewWorklist.push_back(Elt);

422 }

423

424 swap(NewWorklist, Worklist);

425 };

426}

427

428void ResourceManager::fastIssueInstruction(

430 for (const std::pair<uint64_t, ResourceUsage> &R : Desc.Resources) {

431 const CycleSegment &CS = R.second.CS;

432 if (!CS.size()) {

433 releaseResource(R.first);

434 continue;

435 }

436

437 assert(CS.begin() == 0 && "Invalid {Start, End} cycles!");

438 if (.second.isReserved()) {

440 use(Pipe);

441 BusyResources[Pipe] += CS.size();

442 Pipes.emplace_back(std::pair<ResourceRef, ReleaseAtCycles>(

443 Pipe, ReleaseAtCycles(CS.size())));

444 } else {

446

447 assert(R.second.isReserved());

448 reserveResource(R.first);

449 BusyResources[ResourceRef(R.first, R.first)] += CS.size();

450 }

451 }

452}

453

455 for (std::pair<ResourceRef, unsigned> &BR : BusyResources) {

456 if (BR.second)

457 BR.second--;

458 if (!BR.second) {

459

461

464 releaseResource(RR.first);

465 ResourcesFreed.push_back(RR);

466 }

467 }

468

469 for (const ResourceRef &RF : ResourcesFreed)

470 BusyResources.erase(RF);

471}

472

473void ResourceManager::reserveResource(uint64_t ResourceID) {

475 ResourceState &Resource = *Resources[Index];

476 assert(Resource.isAResourceGroup() && !Resource.isReserved() &&

477 "Unexpected resource state found!");

478 Resource.setReserved();

479 ReservedResourceGroups ^= 1ULL << Index;

480}

481

482void ResourceManager::releaseResource(uint64_t ResourceID) {

484 ResourceState &Resource = *Resources[Index];

485 Resource.clearReserved();

486 if (Resource.isAResourceGroup())

487 ReservedResourceGroups ^= 1ULL << Index;

488

489 if (Resource.isADispatchHazard())

490 ReservedBuffers ^= 1ULL << Index;

491}

492

493}

494}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

@ Unavailable

We know the block is not fully available. This is a fixpoint.

iv Induction Variable Users

Move duplicate certain instructions close to their use

The classes here represent processor resource units and their management strategy.

ResourceManager(const TargetSubtargetInfo *ST, ScheduleDAGInstrs *DAG)

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

void used(uint64_t Mask) override

Called by the ResourceManager when a processor resource group, or a processor resource with multiple ...

Definition ResourceManager.cpp:51

uint64_t select(uint64_t ReadyMask) override

Selects a processor resource unit from a ReadyMask.

Definition ResourceManager.cpp:34

A processor resource descriptor.

virtual ~ResourceStrategy()

Helper functions used by various pipeline components.

constexpr std::underlying_type_t< E > Mask()

Get a bitmask with 1s in all places up to the high-order bit of E's largest value.

static std::unique_ptr< ResourceStrategy > getStrategyFor(const ResourceState &RS)

Definition ResourceManager.cpp:105

ResourceStateEvent

Used to notify the internal state of a processor resource.

std::pair< uint64_t, uint64_t > ResourceRef

static uint64_t selectImpl(uint64_t CandidateMask, uint64_t &NextInSequenceMask)

Definition ResourceManager.cpp:26

LLVM_ABI void computeProcResourceMasks(const MCSchedModel &SM, MutableArrayRef< uint64_t > Masks)

Populates vector Masks with processor resource masks.

unsigned getResourceStateIndex(uint64_t Mask)

This is an optimization pass for GlobalISel generic memory operations.

constexpr int popcount(T Value) noexcept

Count the number of set bits in a value.

void sort(IteratorTy Start, IteratorTy End)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

FormattedNumber format_hex(uint64_t N, unsigned Width, bool Upper=false)

format_hex - Output N as a fixed width hexadecimal.

T bit_floor(T Value)

Returns the largest integral power of two no greater than Value if Value is nonzero.

void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)

Implement std::swap in terms of BitVector swap.

Define a kind of processor resource that will be modeled by the scheduler.

Machine model for scheduling, bundling, and heuristics.

unsigned getNumProcResourceKinds() const

const MCProcResourceDesc * getProcResource(unsigned ProcResourceIdx) const