ResourceManager.cpp Source File (original) (raw)

20namespace llvm {

21namespace mca {

23#define DEBUG_TYPE "llvm-mca"

27 uint64_t &NextInSequenceMask) {

30 NextInSequenceMask &= (CandidateMask | (CandidateMask - 1));

31 return CandidateMask;

32}

36 uint64_t CandidateMask = ReadyMask & NextInSequenceMask;

37 if (CandidateMask)

38 return selectImpl(CandidateMask, NextInSequenceMask);

40 NextInSequenceMask = ResourceUnitMask ^ RemovedFromNextInSequence;

41 RemovedFromNextInSequence = 0;

42 CandidateMask = ReadyMask & NextInSequenceMask;

43 if (CandidateMask)

44 return selectImpl(CandidateMask, NextInSequenceMask);

46 NextInSequenceMask = ResourceUnitMask;

47 CandidateMask = ReadyMask & NextInSequenceMask;

48 return selectImpl(CandidateMask, NextInSequenceMask);

49}

52 if (Mask > NextInSequenceMask) {

53 RemovedFromNextInSequence |= Mask;

54 return;

55 }

57 NextInSequenceMask &= (~Mask);

58 if (NextInSequenceMask)

59 return;

61 NextInSequenceMask = ResourceUnitMask ^ RemovedFromNextInSequence;

62 RemovedFromNextInSequence = 0;

63}

67 : ProcResourceDescIndex(Index), ResourceMask(Mask),

68 BufferSize(Desc.BufferSize), IsAGroup(llvm::popcount(ResourceMask) > 1) {

69 if (IsAGroup) {

70 ResourceSizeMask =

72 } else {

73 ResourceSizeMask = (1ULL << Desc.NumUnits) - 1;

74 }

75 ReadyMask = ResourceSizeMask;

76 AvailableSlots = BufferSize == -1 ? 0U : static_cast<unsigned>(BufferSize);

77 Unavailable = false;

78}

83}

91}

93#ifndef NDEBUG

96 << ", SZMASK=" << format_hex(ResourceSizeMask, 16)

97 << ", RDYMASK=" << format_hex(ReadyMask, 16)

98 << ", BufferSize=" << BufferSize

99 << ", AvailableSlots=" << AvailableSlots

100 << ", Reserved=" << Unavailable << '\n';

101}

102#endif

103

104static std::unique_ptr

107 return std::make_unique(RS.getReadyMask());

108 return std::unique_ptr(nullptr);

109}

110

112 : Resources(SM.getNumProcResourceKinds() - 1),

113 Strategies(SM.getNumProcResourceKinds() - 1),

114 Resource2Groups(SM.getNumProcResourceKinds() - 1, 0),

115 ProcResID2Mask(SM.getNumProcResourceKinds(), 0),

116 ResIndex2ProcResID(SM.getNumProcResourceKinds() - 1, 0),

117 ProcResUnitMask(0), ReservedResourceGroups(0), AvailableBuffers(~0ULL),

118 ReservedBuffers(0) {

120

121

124 ResIndex2ProcResID[Index] = I;

125 }

126

128 uint64_t Mask = ProcResID2Mask[I];

130 Resources[Index] =

131 std::make_unique(*SM.getProcResource(I), I, Mask);

133 }

134

136 uint64_t Mask = ProcResID2Mask[I];

140 ProcResUnitMask |= Mask;

141 continue;

142 }

143

145 Mask -= GroupMaskIdx;

146 while (Mask) {

147

148 uint64_t Unit = Mask & (-Mask);

150 Resource2Groups[IndexUnit] |= GroupMaskIdx;

151 Mask ^= Unit;

152 }

153 }

154

155 AvailableProcResUnits = ProcResUnitMask;

156}

157

158void ResourceManager::setCustomStrategyImpl(std::unique_ptr S,

161 assert(Index < Resources.size() && "Invalid processor resource index!");

162 assert(S && "Unexpected null strategy in input!");

163 Strategies[Index] = std::move(S);

164}

165

168}

169

170unsigned ResourceManager::getNumUnits(uint64_t ResourceID) const {

172}

173

174

175

176

179 assert(Index < Resources.size() && "Invalid resource use!");

180 ResourceState &RS = *Resources[Index];

181 assert(RS.isReady() && "No available units to select!");

182

183

184

185 if (!RS.isAResourceGroup() && RS.getNumUnits() == 1)

186 return std::make_pair(ResourceID, RS.getReadyMask());

187

188 uint64_t SubResourceID = Strategies[Index]->select(RS.getReadyMask());

189 if (RS.isAResourceGroup())

190 return selectPipe(SubResourceID);

191 return std::make_pair(ResourceID, SubResourceID);

192}

193

194void ResourceManager::use(const ResourceRef &RR) {

195

197 ResourceState &RS = *Resources[RSID];

198 RS.markSubResourceAsUsed(RR.second);

199

200

201 if (RS.getNumUnits() > 1)

202 Strategies[RSID]->used(RR.second);

203

204

205

206 if (RS.isReady())

207 return;

208

209 AvailableProcResUnits ^= RR.first;

210

211

214

216 ResourceState &CurrentUser = *Resources[GroupIndex];

217 CurrentUser.markSubResourceAsUsed(RR.first);

218 Strategies[GroupIndex]->used(RR.first);

219

221 }

222}

223

224void ResourceManager::release(const ResourceRef &RR) {

226 ResourceState &RS = *Resources[RSID];

227 bool WasFullyUsed = !RS.isReady();

228 RS.releaseSubResource(RR.second);

229 if (!WasFullyUsed)

230 return;

231

232 AvailableProcResUnits ^= RR.first;

233

234

238 ResourceState &CurrentUser = *Resources[GroupIndex];

239 CurrentUser.releaseSubResource(RR.first);

241 }

242}

243

246 if (ConsumedBuffers & ReservedBuffers)

248 if (ConsumedBuffers & (~AvailableBuffers))

251}

252

254 while (ConsumedBuffers) {

255 uint64_t CurrentBuffer = ConsumedBuffers & (-ConsumedBuffers);

257 ConsumedBuffers ^= CurrentBuffer;

259 if (!RS.reserveBuffer())

260 AvailableBuffers ^= CurrentBuffer;

261 if (RS.isADispatchHazard()) {

262

263

264

265 ReservedBuffers ^= CurrentBuffer;

266 }

267 }

268}

269

271 AvailableBuffers |= ConsumedBuffers;

272 while (ConsumedBuffers) {

273 uint64_t CurrentBuffer = ConsumedBuffers & (-ConsumedBuffers);

275 ConsumedBuffers ^= CurrentBuffer;

276 RS.releaseBuffer();

277

278

279 }

280}

281

283 uint64_t BusyResourceMask = 0;

284 uint64_t ConsumedResourceMask = 0;

286

287 for (const std::pair<uint64_t, ResourceUsage> &E : Desc.Resources) {

288 unsigned NumUnits = E.second.isReserved() ? 0U : E.second.NumUnits;

290 if (!RS.isReady(NumUnits)) {

291 BusyResourceMask |= E.first;

292 continue;

293 }

294

297 NumAvailableUnits -= NumUnits;

298 AvailableUnits[E.first] = NumAvailableUnits;

299 if (!NumAvailableUnits)

300 ConsumedResourceMask |= E.first;

301 }

302 }

303

304 BusyResourceMask &= ProcResUnitMask;

305 if (BusyResourceMask)

306 return BusyResourceMask;

307

308 BusyResourceMask = Desc.UsedProcResGroups & ReservedResourceGroups;

309 if ( Desc .HasPartiallyOverlappingGroups || BusyResourceMask)

310 return BusyResourceMask;

311

312

313

314 for (const std::pair<uint64_t, ResourceUsage> &E : Desc.Resources) {

318 if (!ReadyMask) {

320 continue;

321 }

322

324

325 auto [it, Inserted] = AvailableUnits.try_emplace(ResourceMask);

326 if (Inserted) {

329 it->second = NumUnits;

330 }

331

332 if (!it->second) {

333 BusyResourceMask |= it->first;

334 continue;

335 }

336

337 it->second--;

338 if (!it->second)

339 ConsumedResourceMask |= it->first;

340 }

341 }

342

343 return BusyResourceMask;

344}

345

348

349

350

351

352

353

355

356 using ResourceWithUsage = std::pair<uint64_t, ResourceUsage>;

357

358 for (const ResourceWithUsage &R : Desc.Resources) {

360 if (!CS.size()) {

362 continue;

363 }

364

365 assert(CS.begin() == 0 && "Invalid {Start, End} cycles!");

366 if (R.second.isReserved()) {

368

369 assert(R.second.isReserved());

371 BusyResources[ResourceRef(R.first, R.first)] += CS.size();

372 continue;

373 }

374

378 continue;

379 }

380

382 use(Pipe);

383 BusyResources[Pipe] += CS.size();

385 }

386

387

388

389

390

391

392

393

394 while (!Worklist.empty()) {

395 sort(Worklist, [&](const ResourceWithUsage &Lhs,

396 const ResourceWithUsage &Rhs) {

401 if (LhsReadyUnits == RhsReadyUnits)

402 return Lhs.first < Rhs.first;

403 return LhsReadyUnits < RhsReadyUnits;

404 });

405

407

408 for (unsigned I = 0, E = Worklist.size(); I < E; ++I) {

409 const auto &Elt = Worklist[I];

411

413 ResourceRef Pipe = selectPipe(Elt.first);

414 use(Pipe);

416 BusyResources[Pipe] += CS.size();

418 continue;

419 }

420

421 NewWorklist.push_back(Elt);

422 }

423

424 swap(NewWorklist, Worklist);

425 };

426}

427

430 for (const std::pair<uint64_t, ResourceUsage> &R : Desc.Resources) {

432 if (!CS.size()) {

434 continue;

435 }

436

437 assert(CS.begin() == 0 && "Invalid {Start, End} cycles!");

438 if (!R.second.isReserved()) {

440 use(Pipe);

441 BusyResources[Pipe] += CS.size();

442 Pipes.emplace_back(std::pair<ResourceRef, ReleaseAtCycles>(

444 } else {

446

447 assert(R.second.isReserved());

449 BusyResources[ResourceRef(R.first, R.first)] += CS.size();

450 }

451 }

452}

453

455 for (std::pair<ResourceRef, unsigned> &BR : BusyResources) {

456 if (BR.second)

457 BR.second--;

458 if (!BR.second) {

459

461

466 }

467 }

468

469 for (const ResourceRef &RF : ResourcesFreed)

470 BusyResources.erase(RF);

471}

472

477 "Unexpected resource state found!");

479 ReservedResourceGroups ^= 1ULL << Index;

480}

481

487 ReservedResourceGroups ^= 1ULL << Index;

488

490 ReservedBuffers ^= 1ULL << Index;

491}

492

493}

494}

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

iv Induction Variable Users

Move duplicate certain instructions close to their use

The classes here represent processor resource units and their management strategy.

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

reference emplace_back(ArgTypes &&... Args)

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

void used(uint64_t Mask) override

Called by the ResourceManager when a processor resource group, or a processor resource with multiple ...

uint64_t select(uint64_t ReadyMask) override

Selects a processor resource unit from a ReadyMask.

This class represents the number of cycles per resource (fractions of cycles).

void reserveBuffers(uint64_t ConsumedBuffers)

void releaseBuffers(uint64_t ConsumedBuffers)

void fastIssueInstruction(const InstrDesc &Desc, SmallVectorImpl< ResourceWithCycles > &Pipes)

void releaseResource(uint64_t ResourceID)

ResourceManager(const MCSchedModel &SM)

void reserveResource(uint64_t ResourceID)

void issueInstructionImpl(const InstrDesc &Desc, SmallVectorImpl< ResourceWithCycles > &Pipes)

unsigned resolveResourceMask(uint64_t Mask) const

void cycleEvent(SmallVectorImpl< ResourceRef > &ResourcesFreed)

ResourceStateEvent canBeDispatched(uint64_t ConsumedBuffers) const

uint64_t checkAvailability(const InstrDesc &Desc) const

A processor resource descriptor.

ResourceStateEvent isBufferAvailable() const

Checks if there is an available slot in the resource buffer.

ResourceState(const MCProcResourceDesc &Desc, unsigned Index, uint64_t Mask)

uint64_t getReadyMask() const

bool isADispatchHazard() const

Returns true if this is an in-order dispatch/issue resource.

bool isReady(unsigned NumUnits=1) const

Returs true if this resource is not reserved, and if there are at least NumUnits available units.

uint64_t getNumReadyUnits() const

unsigned getNumUnits() const

bool isAResourceGroup() const

virtual ~ResourceStrategy()

Helper functions used by various pipeline components.

static std::unique_ptr< ResourceStrategy > getStrategyFor(const ResourceState &RS)

ResourceStateEvent

Used to notify the internal state of a processor resource.

static uint64_t selectImpl(uint64_t CandidateMask, uint64_t &NextInSequenceMask)

std::pair< uint64_t, uint64_t > ResourceRef

A resource unit identifier.

void computeProcResourceMasks(const MCSchedModel &SM, MutableArrayRef< uint64_t > Masks)

Populates vector Masks with processor resource masks.

unsigned getResourceStateIndex(uint64_t Mask)

This is an optimization pass for GlobalISel generic memory operations.

int popcount(T Value) noexcept

Count the number of set bits in a value.

void sort(IteratorTy Start, IteratorTy End)

raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

FormattedNumber format_hex(uint64_t N, unsigned Width, bool Upper=false)

format_hex - Output N as a fixed width hexadecimal.

T bit_floor(T Value)

Returns the largest integral power of two no greater than Value if Value is nonzero.

Description of the encoding of one expression Op.

Define a kind of processor resource that will be modeled by the scheduler.

Machine model for scheduling, bundling, and heuristics.

unsigned getNumProcResourceKinds() const

const MCProcResourceDesc * getProcResource(unsigned ProcResourceIdx) const

An instruction descriptor.