AMDGPUTargetMachine.cpp Source File (original) (raw)

91#include "llvm/IR/IntrinsicsAMDGPU.h"

126#include

127

128using namespace llvm;

130

131namespace {

132

133

134

135

136class AMDGPUCodeGenPassBuilder

137 : public CodeGenPassBuilder<AMDGPUCodeGenPassBuilder, GCNTargetMachine> {

138 using Base = CodeGenPassBuilder<AMDGPUCodeGenPassBuilder, GCNTargetMachine>;

139

140public:

141 AMDGPUCodeGenPassBuilder(GCNTargetMachine &TM,

142 const CGPassBuilderOption &Opts,

143 PassInstrumentationCallbacks *PIC);

144

145 void addIRPasses(AddIRPass &) const;

146 void addCodeGenPrepare(AddIRPass &) const;

147 void addPreISel(AddIRPass &addPass) const;

148 void addILPOpts(AddMachinePass &) const;

149 void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const;

150 Error addInstSelector(AddMachinePass &) const;

151 void addPreRewrite(AddMachinePass &) const;

152 void addMachineSSAOptimization(AddMachinePass &) const;

153 void addPostRegAlloc(AddMachinePass &) const;

154 void addPreEmitPass(AddMachinePass &) const;

155 void addPreEmitRegAlloc(AddMachinePass &) const;

156 Error addRegAssignmentOptimized(AddMachinePass &) const;

157 void addPreRegAlloc(AddMachinePass &) const;

158 void addOptimizedRegAlloc(AddMachinePass &) const;

159 void addPreSched2(AddMachinePass &) const;

160

161

162

163

166 void addEarlyCSEOrGVNPass(AddIRPass &) const;

167 void addStraightLineScalarOptimizationPasses(AddIRPass &) const;

168};

169

170class SGPRRegisterRegAlloc : public RegisterRegAllocBase {

171public:

172 SGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)

173 : RegisterRegAllocBase(N, D, C) {}

174};

175

176class VGPRRegisterRegAlloc : public RegisterRegAllocBase {

177public:

178 VGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)

179 : RegisterRegAllocBase(N, D, C) {}

180};

181

182class WWMRegisterRegAlloc : public RegisterRegAllocBase {

183public:

184 WWMRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)

185 : RegisterRegAllocBase(N, D, C) {}

186};

187

193}

194

199 return !static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC);

200}

201

208 return !static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(RC) &&

210}

211

212

214

215

216

217static llvm::once_flag InitializeDefaultSGPRRegisterAllocatorFlag;

218static llvm::once_flag InitializeDefaultVGPRRegisterAllocatorFlag;

219static llvm::once_flag InitializeDefaultWWMRegisterAllocatorFlag;

220

221static SGPRRegisterRegAlloc

222defaultSGPRRegAlloc("default",

223 "pick SGPR register allocator based on -O option",

225

226static cl::opt<SGPRRegisterRegAlloc::FunctionPassCtor, false,

229 cl::desc("Register allocator to use for SGPRs"));

230

231static cl::opt<VGPRRegisterRegAlloc::FunctionPassCtor, false,

234 cl::desc("Register allocator to use for VGPRs"));

235

236static cl::opt<WWMRegisterRegAlloc::FunctionPassCtor, false,

238 WWMRegAlloc("wwm-regalloc", cl::Hidden,

240 cl::desc("Register allocator to use for WWM registers"));

241

242static void initializeDefaultSGPRRegisterAllocatorOnce() {

244

245 if (!Ctor) {

246 Ctor = SGPRRegAlloc;

247 SGPRRegisterRegAlloc::setDefault(SGPRRegAlloc);

248 }

249}

250

251static void initializeDefaultVGPRRegisterAllocatorOnce() {

253

254 if (!Ctor) {

255 Ctor = VGPRRegAlloc;

256 VGPRRegisterRegAlloc::setDefault(VGPRRegAlloc);

257 }

258}

259

260static void initializeDefaultWWMRegisterAllocatorOnce() {

262

263 if (!Ctor) {

264 Ctor = WWMRegAlloc;

265 WWMRegisterRegAlloc::setDefault(WWMRegAlloc);

266 }

267}

268

269static FunctionPass *createBasicSGPRRegisterAllocator() {

271}

272

273static FunctionPass *createGreedySGPRRegisterAllocator() {

275}

276

277static FunctionPass *createFastSGPRRegisterAllocator() {

279}

280

281static FunctionPass *createBasicVGPRRegisterAllocator() {

283}

284

285static FunctionPass *createGreedyVGPRRegisterAllocator() {

287}

288

289static FunctionPass *createFastVGPRRegisterAllocator() {

291}

292

293static FunctionPass *createBasicWWMRegisterAllocator() {

295}

296

297static FunctionPass *createGreedyWWMRegisterAllocator() {

299}

300

301static FunctionPass *createFastWWMRegisterAllocator() {

303}

304

305static SGPRRegisterRegAlloc basicRegAllocSGPR(

306 "basic", "basic register allocator", createBasicSGPRRegisterAllocator);

307static SGPRRegisterRegAlloc greedyRegAllocSGPR(

308 "greedy", "greedy register allocator", createGreedySGPRRegisterAllocator);

309

310static SGPRRegisterRegAlloc fastRegAllocSGPR(

311 "fast", "fast register allocator", createFastSGPRRegisterAllocator);

312

313

314static VGPRRegisterRegAlloc basicRegAllocVGPR(

315 "basic", "basic register allocator", createBasicVGPRRegisterAllocator);

316static VGPRRegisterRegAlloc greedyRegAllocVGPR(

317 "greedy", "greedy register allocator", createGreedyVGPRRegisterAllocator);

318

319static VGPRRegisterRegAlloc fastRegAllocVGPR(

320 "fast", "fast register allocator", createFastVGPRRegisterAllocator);

321static WWMRegisterRegAlloc basicRegAllocWWMReg("basic",

322 "basic register allocator",

323 createBasicWWMRegisterAllocator);

324static WWMRegisterRegAlloc

325 greedyRegAllocWWMReg("greedy", "greedy register allocator",

326 createGreedyWWMRegisterAllocator);

327static WWMRegisterRegAlloc fastRegAllocWWMReg("fast", "fast register allocator",

328 createFastWWMRegisterAllocator);

329

333}

334}

335

338 cl::desc("Run early if-conversion"),

340

343 cl::desc("Run pre-RA exec mask optimizations"),

345

348 cl::desc("Lower GPU ctor / dtors to globals on the device."),

350

351

353 "amdgpu-load-store-vectorizer",

354 cl::desc("Enable load store vectorizer"),

357

358

360 "amdgpu-scalarize-global-loads",

361 cl::desc("Enable global load scalarization"),

364

365

367 "amdgpu-internalize-symbols",

368 cl::desc("Enable elimination of non-kernel functions and unused globals"),

371

372

374 "amdgpu-early-inline-all",

375 cl::desc("Inline all functions early"),

378

380 "amdgpu-enable-remove-incompatible-functions", cl::Hidden,

381 cl::desc("Enable removal of functions when they"

382 "use features not supported by the target GPU"),

384

386 "amdgpu-sdwa-peephole",

387 cl::desc("Enable SDWA peepholer"),

389

391 "amdgpu-dpp-combine",

392 cl::desc("Enable DPP combiner"),

394

395

397 cl::desc("Enable AMDGPU Alias Analysis"),

399

400

402 "amdgpu-simplify-libcall",

403 cl::desc("Enable amdgpu library simplifications"),

406

408 "amdgpu-ir-lower-kernel-arguments",

409 cl::desc("Lower kernel argument loads in IR pass"),

412

414 "amdgpu-reassign-regs",

415 cl::desc("Enable register reassign optimizations on gfx10+"),

418

420 "amdgpu-opt-vgpr-liverange",

421 cl::desc("Enable VGPR liverange optimizations for if-else structure"),

423

425 "amdgpu-atomic-optimizer-strategy",

426 cl::desc("Select DPP or Iterative strategy for scan"),

431 "Use Iterative approach for scan"),

433

434

436 "amdgpu-mode-register",

437 cl::desc("Enable mode register pass"),

440

441

444 cl::desc("Enable s_delay_alu insertion"),

446

447

450 cl::desc("Enable VOPD, dual issue of VALU in wave32"),

452

453

457 cl::desc("Enable machine DCE inside regalloc"));

458

460 cl::desc("Adjust wave priority"),

462

464 "amdgpu-scalar-ir-passes",

465 cl::desc("Enable scalar IR passes"),

468

470 "amdgpu-enable-lower-exec-sync",

471 cl::desc("Enable lowering of execution synchronization."), cl::init(true),

473

476 cl::desc("Enable lowering of lds to global memory pass "

477 "and asan instrument resulting IR."),

479

481 "amdgpu-enable-lower-module-lds", cl::desc("Enable lower module lds pass"),

484

486 "amdgpu-enable-pre-ra-optimizations",

487 cl::desc("Enable Pre-RA optimizations pass"), cl::init(true),

489

491 "amdgpu-enable-promote-kernel-arguments",

492 cl::desc("Enable promotion of flat kernel pointer arguments to global"),

494

496 "amdgpu-enable-image-intrinsic-optimizer",

497 cl::desc("Enable image intrinsic optimizer pass"), cl::init(true),

499

502 cl::desc("Enable loop data prefetch on AMDGPU"),

504

507 cl::desc("Select custom AMDGPU scheduling strategy."),

509

511 "amdgpu-enable-rewrite-partial-reg-uses",

512 cl::desc("Enable rewrite partial reg uses pass"), cl::init(true),

514

516 "amdgpu-enable-hipstdpar",

517 cl::desc("Enable HIP Standard Parallelism Offload support"), cl::init(false),

519

522 cl::desc("Enable AMDGPUAttributorPass"),

524

526 "new-reg-bank-select",

527 cl::desc("Run amdgpu-regbankselect and amdgpu-regbanklegalize instead of "

528 "regbankselect"),

530

532 "amdgpu-link-time-closed-world",

533 cl::desc("Whether has closed-world assumption at link time"),

535

537 "amdgpu-enable-uniform-intrinsic-combine",

538 cl::desc("Enable/Disable the Uniform Intrinsic Combine Pass"),

540

631

632static std::unique_ptr createTLOF(const Triple &TT) {

633 return std::make_unique();

634}

635

639

646 if (ST.shouldClusterStores())

652 return DAG;

653}

654

662

667 C, std::make_unique(C));

669 if (ST.shouldClusterStores())

673 return DAG;

674}

675

682 if (ST.shouldClusterStores())

685 return DAG;

686}

687

694

700 if (ST.shouldClusterStores())

704 return DAG;

705}

706

710

713 "Run GCN scheduler to maximize occupancy",

715

719

721 "gcn-max-memory-clause", "Run GCN scheduler to maximize memory clause",

723

725 "gcn-iterative-max-occupancy-experimental",

726 "Run GCN scheduler to maximize occupancy (experimental)",

728

730 "gcn-iterative-minreg",

731 "Run GCN iterative scheduler for minimal register usage (experimental)",

733

735 "gcn-iterative-ilp",

736 "Run GCN iterative scheduler for ILP scheduling (experimental)",

738

741 if (!GPU.empty())

742 return GPU;

743

744

745 if (TT.isAMDGCN())

746 return TT.getOS() == Triple::AMDHSA ? "generic-hsa" : "generic";

747

748 return "r600";

749}

750

756

760 std::optionalReloc::Model RM,

761 std::optionalCodeModel::Model CM,

769 if (TT.isAMDGCN()) {

774 }

775}

776

779

781

783 Attribute GPUAttr = F.getFnAttribute("target-cpu");

785}

786

788 Attribute FSAttr = F.getFnAttribute("target-features");

789

792}

793

799 if (ST.shouldClusterStores())

801 return DAG;

802}

803

804

807 return F->isDeclaration() || F->getName().starts_with("__asan_") ||

808 F->getName().starts_with("__sanitizer_") ||

810

813}

814

818

821 if (Params.empty())

829 if (Result)

830 return *Result;

832}

833

837 while (!Params.empty()) {

839 std::tie(ParamName, Params) = Params.split(';');

840 if (ParamName == "closed-world") {

841 Result.IsClosedWorld = true;

842 } else {

844 formatv("invalid AMDGPUAttributor pass parameter '{0}' ", ParamName)

845 .str(),

847 }

848 }

849 return Result;

850}

851

853

854#define GET_PASS_REGISTRY "AMDGPUPassRegistry.def"

856

857 PB.registerScalarOptimizerLateEPCallback(

860 return;

861

863 });

864

865 PB.registerVectorizerEndEPCallback(

868 return;

869

871 });

872

873 PB.registerPipelineEarlySimplificationEPCallback(

877

878

879

883 }

885 }

886

888 return;

889

890

894 }

895

898 });

899

900 PB.registerPeepholeEPCallback(

903 return;

904

908

911 });

912

913 PB.registerCGSCCOptimizerLateEPCallback(

916 return;

917

919

920

921

922

926

927

928

930

931

932

934

936

937

938

940 }

941

943 });

944

945

954 }

955 }

956 }

957 });

958

959 PB.registerFullLinkTimeOptimizationLastEPCallback(

961

962

963

964

968 }

969

970

971

979

980

984 }

985

989 }

996 }

997 }

1002 }

1003 });

1004

1005 PB.registerRegClassFilterParsingCallback(

1007 if (FilterName == "sgpr")

1008 return onlyAllocateSGPRs;

1009 if (FilterName == "vgpr")

1010 return onlyAllocateVGPRs;

1011 if (FilterName == "wwm")

1012 return onlyAllocateWWMRegs;

1013 return nullptr;

1014 });

1015}

1016

1021 ? -1

1022 : 0;

1023}

1024

1026 unsigned DestAS) const {

1029}

1030

1033 Arg &&

1035 !Arg->hasByRefAttr())

1037

1039 if (!LD)

1041

1042

1044

1045 const auto *Ptr = LD->getPointerOperand();

1048

1049

1050

1051

1053}

1054

1055std::pair<const Value *, unsigned>

1058 switch (II->getIntrinsicID()) {

1059 case Intrinsic::amdgcn_is_shared:

1061 case Intrinsic::amdgcn_is_private:

1063 default:

1064 break;

1065 }

1066 return std::pair(nullptr, -1);

1067 }

1068

1069

1070

1073 const_cast<Value *>(V),

1078

1079 return std::pair(nullptr, -1);

1080}

1081

1082unsigned

1084 switch (Kind) {

1094 }

1096}

1097

1099 Module &M, unsigned NumParts,

1100 function_ref<void(std::unique_ptr MPart)> ModuleCallback) {

1101

1102

1103

1104

1109

1111 PB.registerModuleAnalyses(MAM);

1112 PB.registerFunctionAnalyses(FAM);

1114

1118 return true;

1119}

1120

1121

1122

1123

1124

1128 std::optionalReloc::Model RM,

1129 std::optionalCodeModel::Model CM,

1132

1137

1139 SubtargetKey.append(FS);

1140

1141 auto &I = SubtargetMap[SubtargetKey];

1142 if () {

1143

1144

1145

1147 I = std::make_unique(TargetTriple, GPU, FS, *this);

1148 }

1149

1151

1152 return I.get();

1153}

1154

1159

1164 AMDGPUCodeGenPassBuilder CGPB(*this, Opts, PIC);

1165 return CGPB.buildPipeline(MPM, Out, DwoOut, FileType);

1166}

1167

1171 if (ST.enableSIScheduler())

1173

1175 C->MF->getFunction().getFnAttribute("amdgpu-sched-strategy");

1179

1180 if (SchedStrategy == "max-ilp")

1182

1183 if (SchedStrategy == "max-memory-clause")

1185

1186 if (SchedStrategy == "iterative-ilp")

1188

1189 if (SchedStrategy == "iterative-minreg")

1191

1192 if (SchedStrategy == "iterative-maxocc")

1194

1196}

1197

1202 true);

1205 if (ST.shouldClusterStores())

1208 if ((EnableVOPD.getNumOccurrences() ||

1214 return DAG;

1215}

1216

1217

1218

1219

1223

1224namespace {

1225

1227public:

1231 }

1232

1235 }

1236

1237 bool addPreISel() override;

1238 void addMachineSSAOptimization() override;

1239 bool addILPOpts() override;

1240 bool addInstSelector() override;

1241 bool addIRTranslator() override;

1242 void addPreLegalizeMachineIR() override;

1243 bool addLegalizeMachineIR() override;

1244 void addPreRegBankSelect() override;

1245 bool addRegBankSelect() override;

1246 void addPreGlobalInstructionSelect() override;

1247 bool addGlobalInstructionSelect() override;

1248 void addPreRegAlloc() override;

1249 void addFastRegAlloc() override;

1250 void addOptimizedRegAlloc() override;

1251

1252 FunctionPass *createSGPRAllocPass(bool Optimized);

1253 FunctionPass *createVGPRAllocPass(bool Optimized);

1254 FunctionPass *createWWMRegAllocPass(bool Optimized);

1255 FunctionPass *createRegAllocPass(bool Optimized) override;

1256

1257 bool addRegAssignAndRewriteFast() override;

1258 bool addRegAssignAndRewriteOptimized() override;

1259

1260 bool addPreRewrite() override;

1261 void addPostRegAlloc() override;

1262 void addPreSched2() override;

1263 void addPreEmitPass() override;

1264 void addPostBBSections() override;

1265};

1266

1267}

1268

1279

1286

1291

1292

1294

1295

1297

1299

1300

1302}

1303

1306

1309

1310

1314

1318

1319 if (TM.getTargetTriple().isAMDGCN() &&

1322

1325

1326

1327

1329

1330

1333

1334

1337

1338

1340

1341

1344

1345

1348

1349

1352 }

1353

1354

1355 if ((TM.getTargetTriple().isAMDGCN()) &&

1359 }

1360

1362

1365

1368

1374 AAR.addAAResult(WrapperPass->getResult());

1375 }));

1376 }

1377

1378 if (TM.getTargetTriple().isAMDGCN()) {

1379

1381 }

1382

1383

1384

1387 }

1388

1390

1391

1392

1393

1394

1395

1396

1397

1398

1399

1400

1401

1402

1405}

1406

1408 if (TM->getTargetTriple().isAMDGCN() &&

1411

1414

1416

1419

1420 if (TM->getTargetTriple().isAMDGCN()) {

1421

1422

1423

1424

1425

1426

1427

1430 }

1431

1432

1433

1434

1435

1437}

1438

1444

1449

1451

1452 return false;

1453}

1454

1455

1456

1457

1458

1459bool GCNPassConfig::addPreISel() {

1461

1464

1467

1468

1469

1474

1477

1478

1479

1481

1482

1483

1487

1490

1491 return false;

1492}

1493

1494void GCNPassConfig::addMachineSSAOptimization() {

1496

1497

1498

1499

1500

1501

1502

1503

1513 }

1516}

1517

1518bool GCNPassConfig::addILPOpts() {

1521

1523 return false;

1524}

1525

1526bool GCNPassConfig::addInstSelector() {

1530 return false;

1531}

1532

1533bool GCNPassConfig::addIRTranslator() {

1535 return false;

1536}

1537

1538void GCNPassConfig::addPreLegalizeMachineIR() {

1542}

1543

1544bool GCNPassConfig::addLegalizeMachineIR() {

1546 return false;

1547}

1548

1549void GCNPassConfig::addPreRegBankSelect() {

1553}

1554

1555bool GCNPassConfig::addRegBankSelect() {

1559 } else {

1561 }

1562 return false;

1563}

1564

1565void GCNPassConfig::addPreGlobalInstructionSelect() {

1568}

1569

1570bool GCNPassConfig::addGlobalInstructionSelect() {

1572 return false;

1573}

1574

1575void GCNPassConfig::addFastRegAlloc() {

1576

1577

1578

1579

1580

1581

1583

1585

1587}

1588

1589void GCNPassConfig::addPreRegAlloc() {

1592}

1593

1594void GCNPassConfig::addOptimizedRegAlloc() {

1597

1598

1599

1600

1601

1604

1605

1606

1607

1609

1612

1615

1616

1617

1619

1622

1623

1624

1627

1629}

1630

1631bool GCNPassConfig::addPreRewrite() {

1634

1636 return true;

1637}

1638

1639FunctionPass *GCNPassConfig::createSGPRAllocPass(bool Optimized) {

1640

1641 llvm::call_once(InitializeDefaultSGPRRegisterAllocatorFlag,

1642 initializeDefaultSGPRRegisterAllocatorOnce);

1643

1646 return Ctor();

1647

1648 if (Optimized)

1650

1652}

1653

1654FunctionPass *GCNPassConfig::createVGPRAllocPass(bool Optimized) {

1655

1656 llvm::call_once(InitializeDefaultVGPRRegisterAllocatorFlag,

1657 initializeDefaultVGPRRegisterAllocatorOnce);

1658

1661 return Ctor();

1662

1663 if (Optimized)

1664 return createGreedyVGPRRegisterAllocator();

1665

1666 return createFastVGPRRegisterAllocator();

1667}

1668

1669FunctionPass *GCNPassConfig::createWWMRegAllocPass(bool Optimized) {

1670

1671 llvm::call_once(InitializeDefaultWWMRegisterAllocatorFlag,

1672 initializeDefaultWWMRegisterAllocatorOnce);

1673

1676 return Ctor();

1677

1678 if (Optimized)

1679 return createGreedyWWMRegisterAllocator();

1680

1681 return createFastWWMRegisterAllocator();

1682}

1683

1684FunctionPass *GCNPassConfig::createRegAllocPass(bool Optimized) {

1686}

1687

1689 "-regalloc not supported with amdgcn. Use -sgpr-regalloc, -wwm-regalloc, "

1690 "and -vgpr-regalloc";

1691

1692bool GCNPassConfig::addRegAssignAndRewriteFast() {

1693 if (!usingDefaultRegAlloc())

1695

1697

1698 addPass(createSGPRAllocPass(false));

1699

1700

1702

1703

1705

1706

1707 addPass(createWWMRegAllocPass(false));

1708

1711

1712

1713 addPass(createVGPRAllocPass(false));

1714

1715 return true;

1716}

1717

1718bool GCNPassConfig::addRegAssignAndRewriteOptimized() {

1719 if (!usingDefaultRegAlloc())

1721

1723

1724 addPass(createSGPRAllocPass(true));

1725

1726

1727

1728

1729

1731

1732

1733

1734

1736

1737

1739

1740

1742

1743

1744 addPass(createWWMRegAllocPass(true));

1748

1749

1750 addPass(createVGPRAllocPass(true));

1751

1752 addPreRewrite();

1754

1756

1757 return true;

1758}

1759

1760void GCNPassConfig::addPostRegAlloc() {

1765}

1766

1767void GCNPassConfig::addPreSched2() {

1771}

1772

1773void GCNPassConfig::addPreEmitPass() {

1778

1780

1783

1789

1790

1791

1792

1793

1794

1795

1796

1798

1800

1802

1805

1807}

1808

1809void GCNPassConfig::addPostBBSections() {

1810

1811

1813}

1814

1816 return new GCNPassConfig(*this, PM);

1817}

1818

1824

1831

1835

1842

1851

1853 return true;

1854

1855 if (MFI->Occupancy == 0) {

1856

1857 MFI->Occupancy = ST.getOccupancyWithWorkGroupSizes(MF).second;

1858 }

1859

1863 SourceRange = RegName.SourceRange;

1864 return true;

1865 }

1866 RegVal = TempReg;

1867

1868 return false;

1869 };

1870

1873 return RegName .Value.empty() && parseRegister(RegName, RegVal);

1874 };

1875

1876 if (parseOptionalRegister(YamlMFI.VGPRForAGPRCopy, MFI->VGPRForAGPRCopy))

1877 return true;

1878

1879 if (parseOptionalRegister(YamlMFI.SGPRForEXECCopy, MFI->SGPRForEXECCopy))

1880 return true;

1881

1883 MFI->LongBranchReservedReg))

1884 return true;

1885

1887

1892 "incorrect register class for field", RegName.Value,

1893 {}, {});

1894 SourceRange = RegName.SourceRange;

1895 return true;

1896 };

1897

1898 if (parseRegister(YamlMFI.ScratchRSrcReg, MFI->ScratchRSrcReg) ||

1899 parseRegister(YamlMFI.FrameOffsetReg, MFI->FrameOffsetReg) ||

1900 parseRegister(YamlMFI.StackPtrOffsetReg, MFI->StackPtrOffsetReg))

1901 return true;

1902

1903 if (MFI->ScratchRSrcReg != AMDGPU::PRIVATE_RSRC_REG &&

1904 !AMDGPU::SGPR_128RegClass.contains(MFI->ScratchRSrcReg)) {

1905 return diagnoseRegisterClass(YamlMFI.ScratchRSrcReg);

1906 }

1907

1908 if (MFI->FrameOffsetReg != AMDGPU::FP_REG &&

1909 !AMDGPU::SGPR_32RegClass.contains(MFI->FrameOffsetReg)) {

1910 return diagnoseRegisterClass(YamlMFI.FrameOffsetReg);

1911 }

1912

1913 if (MFI->StackPtrOffsetReg != AMDGPU::SP_REG &&

1914 !AMDGPU::SGPR_32RegClass.contains(MFI->StackPtrOffsetReg)) {

1916 }

1917

1920 if (parseRegister(YamlReg, ParsedReg))

1921 return true;

1922

1924 }

1925

1927 MFI->setFlag(Info->VReg, Info->Flags);

1928 }

1929 for (const auto &[_, Info] : PFS.VRegInfos) {

1930 MFI->setFlag(Info->VReg, Info->Flags);

1931 }

1932

1933 for (const auto &YamlRegStr : YamlMFI.SpillPhysVGPRS) {

1935 if (parseRegister(YamlRegStr, ParsedReg))

1936 return true;

1937 MFI->SpillPhysVGPRs.push_back(ParsedReg);

1938 }

1939

1940 auto parseAndCheckArgument = [&](const std::optionalyaml::SIArgument &A,

1943 unsigned SystemSGPRs) {

1944

1945 if ()

1946 return false;

1947

1948 if (A->IsRegister) {

1951 SourceRange = A->RegisterName.SourceRange;

1952 return true;

1953 }

1954 if (!RC.contains(Reg))

1955 return diagnoseRegisterClass(A->RegisterName);

1957 } else

1959

1960 if (A->Mask)

1962

1963 MFI->NumUserSGPRs += UserSGPRs;

1964 MFI->NumSystemSGPRs += SystemSGPRs;

1965 return false;

1966 };

1967

1969 (parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentBuffer,

1970 AMDGPU::SGPR_128RegClass,

1972 parseAndCheckArgument(YamlMFI.ArgInfo->DispatchPtr,

1973 AMDGPU::SReg_64RegClass, MFI->ArgInfo.DispatchPtr,

1974 2, 0) ||

1975 parseAndCheckArgument(YamlMFI.ArgInfo->QueuePtr, AMDGPU::SReg_64RegClass,

1976 MFI->ArgInfo.QueuePtr, 2, 0) ||

1977 parseAndCheckArgument(YamlMFI.ArgInfo->KernargSegmentPtr,

1978 AMDGPU::SReg_64RegClass,

1980 parseAndCheckArgument(YamlMFI.ArgInfo->DispatchID,

1981 AMDGPU::SReg_64RegClass, MFI->ArgInfo.DispatchID,

1982 2, 0) ||

1983 parseAndCheckArgument(YamlMFI.ArgInfo->FlatScratchInit,

1984 AMDGPU::SReg_64RegClass,

1986 parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentSize,

1987 AMDGPU::SGPR_32RegClass,

1989 parseAndCheckArgument(YamlMFI.ArgInfo->LDSKernelId,

1990 AMDGPU::SGPR_32RegClass,

1992 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDX,

1993 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDX,

1994 0, 1) ||

1995 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDY,

1996 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDY,

1997 0, 1) ||

1998 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupIDZ,

1999 AMDGPU::SGPR_32RegClass, MFI->ArgInfo.WorkGroupIDZ,

2000 0, 1) ||

2001 parseAndCheckArgument(YamlMFI.ArgInfo->WorkGroupInfo,

2002 AMDGPU::SGPR_32RegClass,

2004 parseAndCheckArgument(YamlMFI.ArgInfo->PrivateSegmentWaveByteOffset,

2005 AMDGPU::SGPR_32RegClass,

2007 parseAndCheckArgument(YamlMFI.ArgInfo->ImplicitArgPtr,

2008 AMDGPU::SReg_64RegClass,

2010 parseAndCheckArgument(YamlMFI.ArgInfo->ImplicitBufferPtr,

2011 AMDGPU::SReg_64RegClass,

2013 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDX,

2014 AMDGPU::VGPR_32RegClass,

2016 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDY,

2017 AMDGPU::VGPR_32RegClass,

2019 parseAndCheckArgument(YamlMFI.ArgInfo->WorkItemIDZ,

2020 AMDGPU::VGPR_32RegClass,

2022 return true;

2023

2024

2025

2026 if (YamlMFI.ArgInfo && YamlMFI.ArgInfo->FirstKernArgPreloadReg) {

2028

2029 if (.IsRegister) {

2030

2031

2034

2037

2040 "firstKernArgPreloadReg must be a register, not a stack location", "",

2041 {}, {});

2042

2043 SourceRange = Range;

2044 return true;

2045 }

2046

2049 SourceRange = A.RegisterName.SourceRange;

2050 return true;

2051 }

2052

2053 if (!AMDGPU::SGPR_32RegClass.contains(Reg))

2054 return diagnoseRegisterClass(A.RegisterName);

2055

2058 }

2059

2060 if (ST.hasIEEEMode())

2062 if (ST.hasDX10ClampMode())

2064

2065

2072

2079

2082

2083 return false;

2084}

2085

2086

2087

2088

2089

2090AMDGPUCodeGenPassBuilder::AMDGPUCodeGenPassBuilder(

2094 Opt.MISchedPostRA = true;

2095 Opt.RequiresCodeGenSCCOrder = true;

2096

2097

2098

2099 disablePass<StackMapLivenessPass, FuncletLayoutPass,

2101}

2102

2103void AMDGPUCodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const {

2106

2110

2113

2116

2117

2119

2122

2124

2127

2130

2131

2134

2135

2139

2141

2145 addStraightLineScalarOptimizationPasses(addPass);

2146

2147

2148

2149

2151

2152

2153

2156 true));

2157 }

2158 }

2159

2160 Base::addIRPasses(addPass);

2161

2162

2163

2164

2165

2166

2167

2168

2169

2170

2171

2172

2173

2175 addEarlyCSEOrGVNPass(addPass);

2176}

2177

2178void AMDGPUCodeGenPassBuilder::addCodeGenPrepare(AddIRPass &addPass) const {

2181

2184

2185 Base::addCodeGenPrepare(addPass);

2186

2189

2190

2191

2192

2193

2194

2195

2196

2198 addPass.requireCGSCCOrder();

2199

2201

2202

2203

2204

2205

2207}

2208

2209void AMDGPUCodeGenPassBuilder::addPreISel(AddIRPass &addPass) const {

2210

2211

2213

2218 }

2219

2220

2221

2222

2227

2229

2231

2232

2233

2234

2236

2240

2243

2244

2245

2247 true);

2248}

2249

2250void AMDGPUCodeGenPassBuilder::addILPOpts(AddMachinePass &addPass) const {

2253

2254 Base::addILPOpts(addPass);

2255}

2256

2257void AMDGPUCodeGenPassBuilder::addAsmPrinter(AddMachinePass &addPass,

2258 CreateMCStreamer) const {

2259

2260}

2261

2262Error AMDGPUCodeGenPassBuilder::addInstSelector(AddMachinePass &addPass) const {

2267}

2268

2269void AMDGPUCodeGenPassBuilder::addPreRewrite(AddMachinePass &addPass) const {

2272 }

2273}

2274

2275void AMDGPUCodeGenPassBuilder::addMachineSSAOptimization(

2276 AddMachinePass &addPass) const {

2277 Base::addMachineSSAOptimization(addPass);

2278

2282 }

2289 }

2292}

2293

2294void AMDGPUCodeGenPassBuilder::addOptimizedRegAlloc(

2295 AddMachinePass &addPass) const {

2298

2299

2300

2301

2302

2304 insertPass<RequireAnalysisPass<LiveVariablesAnalysis, MachineFunction>>(

2306

2307

2308

2309

2311

2314

2317

2318

2319

2321

2324

2325

2326

2329

2330 Base::addOptimizedRegAlloc(addPass);

2331}

2332

2333void AMDGPUCodeGenPassBuilder::addPreRegAlloc(AddMachinePass &addPass) const {

2336}

2337

2338Error AMDGPUCodeGenPassBuilder::addRegAssignmentOptimized(

2339 AddMachinePass &addPass) const {

2340

2341

2343

2344 addPass(RAGreedyPass({onlyAllocateSGPRs, "sgpr"}));

2345

2346

2347

2348

2349

2351

2352

2353

2354

2356

2357

2359

2360

2362

2363

2364 addPass(RAGreedyPass({onlyAllocateWWMRegs, "wwm"}));

2368

2369

2370 addPass(RAGreedyPass({onlyAllocateVGPRs, "vgpr"}));

2371

2372

2373 addPreRewrite(addPass);

2375

2378}

2379

2380void AMDGPUCodeGenPassBuilder::addPostRegAlloc(AddMachinePass &addPass) const {

2384 Base::addPostRegAlloc(addPass);

2385}

2386

2387void AMDGPUCodeGenPassBuilder::addPreSched2(AddMachinePass &addPass) const {

2391}

2392

2393void AMDGPUCodeGenPassBuilder::addPreEmitPass(AddMachinePass &addPass) const {

2396 }

2397

2400

2401

2402

2404

2405 }

2406

2408

2411

2414

2415

2416

2417

2418

2419

2420

2421

2422

2426

2429 }

2430

2432}

2433

2434bool AMDGPUCodeGenPassBuilder::isPassEnabled(const cl::opt &Opt,

2437 return Opt;

2438 if (TM.getOptLevel() < Level)

2439 return false;

2440 return Opt;

2441}

2442

2443void AMDGPUCodeGenPassBuilder::addEarlyCSEOrGVNPass(AddIRPass &addPass) const {

2446 else

2448}

2449

2450void AMDGPUCodeGenPassBuilder::addStraightLineScalarOptimizationPasses(

2451 AddIRPass &addPass) const {

2454

2456

2457

2458

2460

2461

2462

2463 addEarlyCSEOrGVNPass(addPass);

2464

2465

2467

2468

2469

2471}

unsigned const MachineRegisterInfo * MRI

aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static cl::opt< bool > EnableEarlyIfConversion("aarch64-enable-early-ifcvt", cl::Hidden, cl::desc("Run early if-conversion"), cl::init(true))

static std::unique_ptr< TargetLoweringObjectFile > createTLOF(const Triple &TT)

This is the AMGPU address space based alias analysis pass.

Defines an instruction selector for the AMDGPU target.

Analyzes if a function potentially memory bound and if a kernel kernel may benefit from limiting numb...

Analyzes how many registers and other resources are used by functions.

static cl::opt< bool > EnableDCEInRA("amdgpu-dce-in-ra", cl::init(true), cl::Hidden, cl::desc("Enable machine DCE inside regalloc"))

static cl::opt< bool, true > EnableLowerModuleLDS("amdgpu-enable-lower-module-lds", cl::desc("Enable lower module lds pass"), cl::location(AMDGPUTargetMachine::EnableLowerModuleLDS), cl::init(true), cl::Hidden)

static MachineSchedRegistry GCNMaxMemoryClauseSchedRegistry("gcn-max-memory-clause", "Run GCN scheduler to maximize memory clause", createGCNMaxMemoryClauseMachineScheduler)

static Reloc::Model getEffectiveRelocModel()

Definition AMDGPUTargetMachine.cpp:751

static cl::opt< bool > EnableUniformIntrinsicCombine("amdgpu-enable-uniform-intrinsic-combine", cl::desc("Enable/Disable the Uniform Intrinsic Combine Pass"), cl::init(true), cl::Hidden)

static MachineSchedRegistry SISchedRegistry("si", "Run SI's custom scheduler", createSIMachineScheduler)

static ScheduleDAGInstrs * createIterativeILPMachineScheduler(MachineSchedContext *C)

Definition AMDGPUTargetMachine.cpp:696

static cl::opt< bool > EarlyInlineAll("amdgpu-early-inline-all", cl::desc("Inline all functions early"), cl::init(false), cl::Hidden)

static cl::opt< bool > EnableSwLowerLDS("amdgpu-enable-sw-lower-lds", cl::desc("Enable lowering of lds to global memory pass " "and asan instrument resulting IR."), cl::init(true), cl::Hidden)

static cl::opt< bool > EnableLowerKernelArguments("amdgpu-ir-lower-kernel-arguments", cl::desc("Lower kernel argument loads in IR pass"), cl::init(true), cl::Hidden)

static ScheduleDAGInstrs * createGCNMaxILPMachineScheduler(MachineSchedContext *C)

Definition AMDGPUTargetMachine.cpp:656

static cl::opt< bool > EnableSDWAPeephole("amdgpu-sdwa-peephole", cl::desc("Enable SDWA peepholer"), cl::init(true))

static MachineSchedRegistry GCNMinRegSchedRegistry("gcn-iterative-minreg", "Run GCN iterative scheduler for minimal register usage (experimental)", createMinRegScheduler)

static cl::opt< bool > EnableImageIntrinsicOptimizer("amdgpu-enable-image-intrinsic-optimizer", cl::desc("Enable image intrinsic optimizer pass"), cl::init(true), cl::Hidden)

static cl::opt< bool > HasClosedWorldAssumption("amdgpu-link-time-closed-world", cl::desc("Whether has closed-world assumption at link time"), cl::init(false), cl::Hidden)

static ScheduleDAGInstrs * createGCNMaxMemoryClauseMachineScheduler(MachineSchedContext *C)

Definition AMDGPUTargetMachine.cpp:664

static cl::opt< bool > EnableSIModeRegisterPass("amdgpu-mode-register", cl::desc("Enable mode register pass"), cl::init(true), cl::Hidden)

static cl::opt< std::string > AMDGPUSchedStrategy("amdgpu-sched-strategy", cl::desc("Select custom AMDGPU scheduling strategy."), cl::Hidden, cl::init(""))

static cl::opt< bool > EnableDPPCombine("amdgpu-dpp-combine", cl::desc("Enable DPP combiner"), cl::init(true))

static MachineSchedRegistry IterativeGCNMaxOccupancySchedRegistry("gcn-iterative-max-occupancy-experimental", "Run GCN scheduler to maximize occupancy (experimental)", createIterativeGCNMaxOccupancyMachineScheduler)

static cl::opt< bool > EnableSetWavePriority("amdgpu-set-wave-priority", cl::desc("Adjust wave priority"), cl::init(false), cl::Hidden)

static cl::opt< bool > LowerCtorDtor("amdgpu-lower-global-ctor-dtor", cl::desc("Lower GPU ctor / dtors to globals on the device."), cl::init(true), cl::Hidden)

static cl::opt< bool > OptExecMaskPreRA("amdgpu-opt-exec-mask-pre-ra", cl::Hidden, cl::desc("Run pre-RA exec mask optimizations"), cl::init(true))

static cl::opt< bool > EnablePromoteKernelArguments("amdgpu-enable-promote-kernel-arguments", cl::desc("Enable promotion of flat kernel pointer arguments to global"), cl::Hidden, cl::init(true))

LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget()

Definition AMDGPUTargetMachine.cpp:541

static cl::opt< bool > EnableRewritePartialRegUses("amdgpu-enable-rewrite-partial-reg-uses", cl::desc("Enable rewrite partial reg uses pass"), cl::init(true), cl::Hidden)

static cl::opt< bool > EnableLibCallSimplify("amdgpu-simplify-libcall", cl::desc("Enable amdgpu library simplifications"), cl::init(true), cl::Hidden)

static MachineSchedRegistry GCNMaxILPSchedRegistry("gcn-max-ilp", "Run GCN scheduler to maximize ilp", createGCNMaxILPMachineScheduler)

static cl::opt< bool > InternalizeSymbols("amdgpu-internalize-symbols", cl::desc("Enable elimination of non-kernel functions and unused globals"), cl::init(false), cl::Hidden)

static cl::opt< bool > EnableAMDGPUAttributor("amdgpu-attributor-enable", cl::desc("Enable AMDGPUAttributorPass"), cl::init(true), cl::Hidden)

static LLVM_READNONE StringRef getGPUOrDefault(const Triple &TT, StringRef GPU)

Definition AMDGPUTargetMachine.cpp:740

Expected< AMDGPUAttributorOptions > parseAMDGPUAttributorPassOptions(StringRef Params)

Definition AMDGPUTargetMachine.cpp:835

static cl::opt< bool > EnableAMDGPUAliasAnalysis("enable-amdgpu-aa", cl::Hidden, cl::desc("Enable AMDGPU Alias Analysis"), cl::init(true))

static Expected< ScanOptions > parseAMDGPUAtomicOptimizerStrategy(StringRef Params)

Definition AMDGPUTargetMachine.cpp:820

static ScheduleDAGInstrs * createMinRegScheduler(MachineSchedContext *C)

Definition AMDGPUTargetMachine.cpp:688

static cl::opt< bool > EnableHipStdPar("amdgpu-enable-hipstdpar", cl::desc("Enable HIP Standard Parallelism Offload support"), cl::init(false), cl::Hidden)

static cl::opt< bool > EnableInsertDelayAlu("amdgpu-enable-delay-alu", cl::desc("Enable s_delay_alu insertion"), cl::init(true), cl::Hidden)

static ScheduleDAGInstrs * createIterativeGCNMaxOccupancyMachineScheduler(MachineSchedContext *C)

Definition AMDGPUTargetMachine.cpp:677

static cl::opt< bool > EnableLoadStoreVectorizer("amdgpu-load-store-vectorizer", cl::desc("Enable load store vectorizer"), cl::init(true), cl::Hidden)

static bool mustPreserveGV(const GlobalValue &GV)

Predicate for Internalize pass.

Definition AMDGPUTargetMachine.cpp:805

static cl::opt< bool > EnableLoopPrefetch("amdgpu-loop-prefetch", cl::desc("Enable loop data prefetch on AMDGPU"), cl::Hidden, cl::init(false))

static cl::opt< bool > NewRegBankSelect("new-reg-bank-select", cl::desc("Run amdgpu-regbankselect and amdgpu-regbanklegalize instead of " "regbankselect"), cl::init(false), cl::Hidden)

static cl::opt< bool > RemoveIncompatibleFunctions("amdgpu-enable-remove-incompatible-functions", cl::Hidden, cl::desc("Enable removal of functions when they" "use features not supported by the target GPU"), cl::init(true))

static cl::opt< bool > EnableScalarIRPasses("amdgpu-scalar-ir-passes", cl::desc("Enable scalar IR passes"), cl::init(true), cl::Hidden)

static cl::opt< bool > EnableRegReassign("amdgpu-reassign-regs", cl::desc("Enable register reassign optimizations on gfx10+"), cl::init(true), cl::Hidden)

static cl::opt< bool > OptVGPRLiveRange("amdgpu-opt-vgpr-liverange", cl::desc("Enable VGPR liverange optimizations for if-else structure"), cl::init(true), cl::Hidden)

static ScheduleDAGInstrs * createSIMachineScheduler(MachineSchedContext *C)

Definition AMDGPUTargetMachine.cpp:636

static cl::opt< bool > EnablePreRAOptimizations("amdgpu-enable-pre-ra-optimizations", cl::desc("Enable Pre-RA optimizations pass"), cl::init(true), cl::Hidden)

static cl::opt< ScanOptions > AMDGPUAtomicOptimizerStrategy("amdgpu-atomic-optimizer-strategy", cl::desc("Select DPP or Iterative strategy for scan"), cl::init(ScanOptions::Iterative), cl::values(clEnumValN(ScanOptions::DPP, "DPP", "Use DPP operations for scan"), clEnumValN(ScanOptions::Iterative, "Iterative", "Use Iterative approach for scan"), clEnumValN(ScanOptions::None, "None", "Disable atomic optimizer")))

static cl::opt< bool > EnableVOPD("amdgpu-enable-vopd", cl::desc("Enable VOPD, dual issue of VALU in wave32"), cl::init(true), cl::Hidden)

static cl::opt< bool > EnableEarlyIfConversion("amdgpu-early-ifcvt", cl::Hidden, cl::desc("Run early if-conversion"), cl::init(false))

static ScheduleDAGInstrs * createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C)

Definition AMDGPUTargetMachine.cpp:641

static cl::opt< bool > EnableLowerExecSync("amdgpu-enable-lower-exec-sync", cl::desc("Enable lowering of execution synchronization."), cl::init(true), cl::Hidden)

static MachineSchedRegistry GCNILPSchedRegistry("gcn-iterative-ilp", "Run GCN iterative scheduler for ILP scheduling (experimental)", createIterativeILPMachineScheduler)

static cl::opt< bool > ScalarizeGlobal("amdgpu-scalarize-global-loads", cl::desc("Enable global load scalarization"), cl::init(true), cl::Hidden)

static const char RegAllocOptNotSupportedMessage[]

Definition AMDGPUTargetMachine.cpp:1688

static MachineSchedRegistry GCNMaxOccupancySchedRegistry("gcn-max-occupancy", "Run GCN scheduler to maximize occupancy", createGCNMaxOccupancyMachineScheduler)

The AMDGPU TargetMachine interface definition for hw codegen targets.

This file declares the AMDGPU-specific subclass of TargetLoweringObjectFile.

This file a TargetTransformInfoImplBase conforming object specific to the AMDGPU target machine.

Provides passes to inlining "always_inline" functions.

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")

This header provides classes for managing passes over SCCs of the call graph.

Provides analysis for continuously CSEing during GISel passes.

Interfaces for producing common pass manager configurations.

#define clEnumValN(ENUMVAL, FLAGNAME, DESC)

#define LLVM_EXTERNAL_VISIBILITY

This file provides the interface for a simple, fast CSE pass.

This file defines the class GCNIterativeScheduler, which uses an iterative approach to find a best sc...

This file provides the interface for LLVM's Global Value Numbering pass which eliminates fully redund...

AcceleratorCodeSelection - Identify all functions reachable from a kernel, removing those that are un...

This file declares the IRTranslator pass.

This header defines various interfaces for pass management in LLVM.

This file provides the interface for LLVM's Loop Data Prefetching Pass.

This header provides classes for managing a pipeline of passes over loops in LLVM IR.

ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))

uint64_t IntrinsicInst * II

CGSCCAnalysisManager CGAM

FunctionAnalysisManager FAM

ModuleAnalysisManager MAM

PassInstrumentationCallbacks PIC

PassBuilder PB(Machine, PassOpts->PTO, std::nullopt, &PIC)

static bool isLTOPreLink(ThinOrFullLTOPhase Phase)

The AMDGPU TargetMachine interface definition for hw codegen targets.

This file describes the interface of the MachineFunctionPass responsible for assigning the generic vi...

const GCNTargetMachine & getTM(const GCNSubtarget *STI)

SI Machine Scheduler interface.

static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)

static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")

static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")

static FunctionPass * useDefaultRegisterAllocator()

-regalloc=... command line option.

static cl::opt< cl::boolOrDefault > EnableGlobalISelOption("global-isel", cl::Hidden, cl::desc("Enable the \"global\" instruction selector"))

Target-Independent Code Generator Pass Configuration Options pass.

LLVM IR instance of the generic uniformity analysis.

static std::unique_ptr< TargetLoweringObjectFile > createTLOF()

A manager for alias analyses.

void registerFunctionAnalysis()

void addAAResult(AAResultT &AAResult)

Legacy wrapper pass to provide the AMDGPUAAResult object.

Analysis pass providing a never-invalidated alias analysis result.

Lower llvm.global_ctors and llvm.global_dtors to special kernels.

AMDGPUTargetMachine & getAMDGPUTargetMachine() const

std::unique_ptr< CSEConfigBase > getCSEConfig() const override

Returns the CSEConfig object to use for the current optimization level.

Definition AMDGPUTargetMachine.cpp:1220

bool isPassEnabled(const cl::opt< bool > &Opt, CodeGenOptLevel Level=CodeGenOptLevel::Default) const

Check if a pass is enabled given Opt option.

bool addPreISel() override

Methods with trivial inline returns are convenient points in the common codegen pass pipeline where t...

Definition AMDGPUTargetMachine.cpp:1439

bool addInstSelector() override

addInstSelector - This method should install an instruction selector pass, which converts from LLVM c...

Definition AMDGPUTargetMachine.cpp:1445

bool addGCPasses() override

addGCPasses - Add late codegen passes that analyze code for garbage collection.

Definition AMDGPUTargetMachine.cpp:1450

void addStraightLineScalarOptimizationPasses()

Definition AMDGPUTargetMachine.cpp:1287

AMDGPUPassConfig(TargetMachine &TM, PassManagerBase &PM)

Definition AMDGPUTargetMachine.cpp:1269

void addIRPasses() override

Add common target configurable passes that perform LLVM IR to IR transforms following machine indepen...

Definition AMDGPUTargetMachine.cpp:1304

void addEarlyCSEOrGVNPass()

Definition AMDGPUTargetMachine.cpp:1280

void addCodeGenPrepare() override

Add pass to prepare the LLVM IR for code generation.

Definition AMDGPUTargetMachine.cpp:1407

Splits the module M into N linkable partitions.

std::unique_ptr< TargetLoweringObjectFile > TLOF

static int64_t getNullPointerValue(unsigned AddrSpace)

Get the integer value of a null pointer in the given address space.

Definition AMDGPUTargetMachine.cpp:1017

unsigned getAddressSpaceForPseudoSourceKind(unsigned Kind) const override

getAddressSpaceForPseudoSourceKind - Given the kind of memory (e.g.

Definition AMDGPUTargetMachine.cpp:1083

const TargetSubtargetInfo * getSubtargetImpl() const

void registerDefaultAliasAnalyses(AAManager &) override

Allow the target to register alias analyses with the AAManager for use with the new pass manager.

Definition AMDGPUTargetMachine.cpp:815

~AMDGPUTargetMachine() override

std::pair< const Value *, unsigned > getPredicatedAddrSpace(const Value *V) const override

If the specified predicate checks whether a generic pointer falls within a specified address space,...

Definition AMDGPUTargetMachine.cpp:1056

StringRef getFeatureString(const Function &F) const

Definition AMDGPUTargetMachine.cpp:787

ScheduleDAGInstrs * createMachineScheduler(MachineSchedContext *C) const override

Create an instance of ScheduleDAGInstrs to be run within the standard MachineScheduler pass for this ...

Definition AMDGPUTargetMachine.cpp:795

static bool EnableFunctionCalls

AMDGPUTargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, std::optional< Reloc::Model > RM, std::optional< CodeModel::Model > CM, CodeGenOptLevel OL)

Definition AMDGPUTargetMachine.cpp:757

bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override

Returns true if a cast between SrcAS and DestAS is a noop.

Definition AMDGPUTargetMachine.cpp:1025

void registerPassBuilderCallbacks(PassBuilder &PB) override

Allow the target to modify the pass pipeline.

Definition AMDGPUTargetMachine.cpp:852

static bool EnableLowerModuleLDS

StringRef getGPUName(const Function &F) const

Definition AMDGPUTargetMachine.cpp:782

unsigned getAssumedAddrSpace(const Value *V) const override

If the specified generic pointer could be assumed as a pointer to a specific address space,...

Definition AMDGPUTargetMachine.cpp:1031

bool splitModule(Module &M, unsigned NumParts, function_ref< void(std::unique_ptr< Module > MPart)> ModuleCallback) override

Entry point for module splitting.

Definition AMDGPUTargetMachine.cpp:1098

Inlines functions marked as "always_inline".

Functions, function parameters, and return types can have attributes to indicate how they should be t...

LLVM_ABI StringRef getValueAsString() const

Return the attribute's value as a string.

bool isValid() const

Return true if the attribute is any kind of attribute.

This class provides access to building LLVM's passes.

CodeGenTargetMachineImpl(const Target &T, StringRef DataLayoutString, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, Reloc::Model RM, CodeModel::Model CM, CodeGenOptLevel OL)

LLVM_ABI void removeDeadConstantUsers() const

If there are any dead constant users dangling off of this constant, remove them.

Lightweight error class with error context and mandatory checking.

static ErrorSuccess success()

Create a success value.

Tagged union holding either a T or a Error.

FunctionPass class - This class is used to implement most global optimizations.

@ SCHEDULE_LEGACYMAXOCCUPANCY

const SIRegisterInfo * getRegisterInfo() const override

TargetTransformInfo getTargetTransformInfo(const Function &F) const override

Get a TargetTransformInfo implementation for the target.

Definition AMDGPUTargetMachine.cpp:1156

ScheduleDAGInstrs * createPostMachineScheduler(MachineSchedContext *C) const override

Similar to createMachineScheduler but used when postRA machine scheduling is enabled.

Definition AMDGPUTargetMachine.cpp:1199

ScheduleDAGInstrs * createMachineScheduler(MachineSchedContext *C) const override

Create an instance of ScheduleDAGInstrs to be run within the standard MachineScheduler pass for this ...

Definition AMDGPUTargetMachine.cpp:1169

void registerMachineRegisterInfoCallback(MachineFunction &MF) const override

Definition AMDGPUTargetMachine.cpp:1819

bool parseMachineFunctionInfo(const yaml::MachineFunctionInfo &, PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange) const override

Parse out the target's MachineFunctionInfo from the YAML reprsentation.

Definition AMDGPUTargetMachine.cpp:1843

yaml::MachineFunctionInfo * convertFuncInfoToYAML(const MachineFunction &MF) const override

Allocate and initialize an instance of the YAML representation of the MachineFunctionInfo.

Definition AMDGPUTargetMachine.cpp:1837

Error buildCodeGenPipeline(ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType, const CGPassBuilderOption &Opts, PassInstrumentationCallbacks *PIC) override

Definition AMDGPUTargetMachine.cpp:1160

yaml::MachineFunctionInfo * createDefaultFuncInfoYAML() const override

Allocate and return a default initialized instance of the YAML representation for the MachineFunction...

Definition AMDGPUTargetMachine.cpp:1832

TargetPassConfig * createPassConfig(PassManagerBase &PM) override

Create a pass configuration object to be used by addPassToEmitX methods for generating a pipeline of ...

Definition AMDGPUTargetMachine.cpp:1815

GCNTargetMachine(const Target &T, const Triple &TT, StringRef CPU, StringRef FS, const TargetOptions &Options, std::optional< Reloc::Model > RM, std::optional< CodeModel::Model > CM, CodeGenOptLevel OL, bool JIT)

Definition AMDGPUTargetMachine.cpp:1125

MachineFunctionInfo * createMachineFunctionInfo(BumpPtrAllocator &Allocator, const Function &F, const TargetSubtargetInfo *STI) const override

Create the target's instance of MachineFunctionInfo.

Definition AMDGPUTargetMachine.cpp:1825

The core GVN pass object.

Pass to remove unused function declarations.

This pass is responsible for selecting generic machine instructions to target-specific instructions.

A pass that internalizes all functions and variables other than those that must be preserved accordin...

Converts loops into loop-closed SSA form.

Performs Loop Invariant Code Motion Pass.

This pass implements the localization mechanism described at the top of this file.

An optimization pass inserting data prefetches in loops.

const TargetSubtargetInfo & getSubtarget() const

getSubtarget - Return the subtarget for which this machine code is being compiled.

MachineRegisterInfo & getRegInfo()

getRegInfo - Return information about the registers currently in use.

Ty * getInfo()

getInfo - Keep track of various per-function pieces of information for backends that would like to do...

MachineRegisterInfo - Keep track of information for virtual and physical registers,...

void addDelegate(Delegate *delegate)

MachineSchedRegistry provides a selection of available machine instruction schedulers.

This interface provides simple read-only access to a block of memory, and provides simple methods for...

virtual StringRef getBufferIdentifier() const

Return an identifier for this buffer, typically the filename it was read from.

const char * getBufferStart() const

A Module instance is used to store all the information related to an LLVM module.

static LLVM_ABI const OptimizationLevel O0

Disable as many optimizations as possible.

static LLVM_ABI const OptimizationLevel O1

Optimize quickly without destroying debuggability.

This class provides access to building LLVM's passes.

This class manages callbacks registration, as well as provides a way for PassInstrumentation to pass ...

LLVM_ATTRIBUTE_MINSIZE std::enable_if_t<!std::is_same_v< PassT, PassManager > > addPass(PassT &&Pass)

PreservedAnalyses run(IRUnitT &IR, AnalysisManagerT &AM, ExtraArgTs... ExtraArgs)

Run all of the passes in this manager over the given unit of IR.

PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...

static LLVM_ABI PassRegistry * getPassRegistry()

getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...

Pass interface - Implemented by all 'passes'.

@ ExternalSymbolCallEntry

This pass implements the reg bank selector pass used in the GlobalISel pipeline.

RegisterPassParser class - Handle the addition of new machine passes.

RegisterRegAllocBase class - Track the registration of register allocators.

FunctionPass *(*)() FunctionPassCtor

Wrapper class representing virtual and physical registers.

This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...

bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF, PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange)

void setFlag(Register Reg, uint8_t Flag)

bool checkFlag(Register Reg, uint8_t Flag) const

void reserveWWMRegister(Register Reg)

Instances of this class encapsulate one diagnostic report, allowing printing to a raw_ostream as a ca...

Represents a location in source code.

static SMLoc getFromPointer(const char *Ptr)

Represents a range in source code.

A ScheduleDAG for scheduling lists of MachineInstr.

ScheduleDAGMILive is an implementation of ScheduleDAGInstrs that schedules machine instructions while...

ScheduleDAGMI is an implementation of ScheduleDAGInstrs that simply schedules machine instructions ac...

void addMutation(std::unique_ptr< ScheduleDAGMutation > Mutation)

Add a postprocessing step to the DAG builder.

const TargetInstrInfo * TII

Target instruction information.

const TargetRegisterInfo * TRI

Target processor register info.

Move instructions into successor blocks when possible.

SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...

void append(StringRef RHS)

Append from a StringRef.

unsigned getMainFileID() const

const MemoryBuffer * getMemoryBuffer(unsigned i) const

StringRef - Represent a constant reference to a string, i.e.

std::pair< StringRef, StringRef > split(char Separator) const

Split into two substrings around the first occurrence of a separator character.

constexpr bool empty() const

empty - Check if the string is empty.

bool consume_front(StringRef Prefix)

Returns true if this StringRef has the given prefix and removes that prefix.

A switch()-like statement whose cases are string literals.

StringSwitch & Cases(std::initializer_list< StringLiteral > CaseStrings, T Value)

Primary interface to the complete machine description for the target machine.

CodeGenOptLevel getOptLevel() const

Returns the optimization level: None, Less, Default, or Aggressive.

Triple TargetTriple

Triple string, CPU name, and target feature strings the TargetMachine instance is created with.

const Triple & getTargetTriple() const

const MCSubtargetInfo * getMCSubtargetInfo() const

StringRef getTargetFeatureString() const

StringRef getTargetCPU() const

std::unique_ptr< const MCSubtargetInfo > STI

void resetTargetOptions(const Function &F) const

Reset the target options based on the function's attributes.

std::unique_ptr< const MCRegisterInfo > MRI

Target-Independent Code Generator Pass Configuration Options.

virtual void addCodeGenPrepare()

Add pass to prepare the LLVM IR for code generation.

virtual bool addILPOpts()

Add passes that optimize instruction level parallelism for out-of-order targets.

virtual void addPostRegAlloc()

This method may be implemented by targets that want to run passes after register allocation pass pipe...

CodeGenOptLevel getOptLevel() const

virtual void addOptimizedRegAlloc()

addOptimizedRegAlloc - Add passes related to register allocation.

virtual void addIRPasses()

Add common target configurable passes that perform LLVM IR to IR transforms following machine indepen...

virtual void addFastRegAlloc()

addFastRegAlloc - Add the minimum set of target-independent passes that are required for fast registe...

virtual void addMachineSSAOptimization()

addMachineSSAOptimization - Add standard passes that optimize machine instructions in SSA form.

void disablePass(AnalysisID PassID)

Allow the target to disable a specific standard pass by default.

AnalysisID addPass(AnalysisID PassID)

Utilities for targets to add passes to the pass manager.

TargetPassConfig(TargetMachine &TM, PassManagerBase &PM)

TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...

TargetSubtargetInfo - Generic base class for all target subtargets.

This pass provides access to the codegen interfaces that are needed for IR-level transformations.

Target - Wrapper for Target specific information.

Triple - Helper class for working with autoconf configuration names.

LLVM Value Representation.

int getNumOccurrences() const

An efficient, type-erasing, non-owning reference to a callable.

PassManagerBase - An abstract interface to allow code to add passes to a pass manager without having ...

An abstract base class for streams implementations that also support a pwrite operation.

Interfaces for registering analysis passes, producing common pass manager configurations,...

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

@ REGION_ADDRESS

Address space for region memory. (GDS)

@ LOCAL_ADDRESS

Address space for local memory.

@ CONSTANT_ADDRESS

Address space for constant memory (VTX2).

@ FLAT_ADDRESS

Address space for flat memory.

@ GLOBAL_ADDRESS

Address space for global memory (RAT0, VTX0).

@ PRIVATE_ADDRESS

Address space for private memory.

bool isFlatGlobalAddrSpace(unsigned AS)

LLVM_READNONE constexpr bool isModuleEntryFunctionCC(CallingConv::ID CC)

LLVM_READNONE constexpr bool isEntryFunctionCC(CallingConv::ID CC)

@ C

The default llvm calling convention, compatible with C.

BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)

Matches a register not-ed by a G_XOR.

BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)

Matches an And with LHS and RHS in either order.

bool match(Val *V, const Pattern &P)

IntrinsicID_match m_Intrinsic()

Match intrinsic calls like this: m_IntrinsicIntrinsic::fabs(m_Value(X))

deferredval_ty< Value > m_Deferred(Value *const &V)

Like m_Specific(), but works if the specific value to match is determined as part of the same match()...

class_match< Value > m_Value()

Match an arbitrary value and ignore it.

template class LLVM_TEMPLATE_ABI opt< bool >

ValuesClass values(OptsTy... Options)

Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...

initializer< Ty > init(const Ty &Val)

LocationClass< Ty > location(Ty &L)

This is an optimization pass for GlobalISel generic memory operations.

ScheduleDAGMILive * createSchedLive(MachineSchedContext *C)

Create the standard converging machine scheduler.

LLVM_ABI FunctionPass * createFlattenCFGPass()

std::unique_ptr< ScheduleDAGMutation > createAMDGPUBarrierLatencyDAGMutation(MachineFunction *MF)

LLVM_ABI FunctionPass * createFastRegisterAllocator()

FastRegisterAllocation Pass - This pass register allocates as fast as possible.

LLVM_ABI char & EarlyMachineLICMID

This pass performs loop invariant code motion on machine instructions.

ImmutablePass * createAMDGPUAAWrapperPass()

LLVM_ABI char & PostRAHazardRecognizerID

PostRAHazardRecognizer - This pass runs the post-ra hazard recognizer.

std::function< bool(const TargetRegisterInfo &TRI, const MachineRegisterInfo &MRI, const Register Reg)> RegAllocFilterFunc

Filter function for register classes during regalloc.

FunctionPass * createAMDGPUSetWavePriorityPass()

LLVM_ABI Pass * createLCSSAPass()

void initializeAMDGPUMarkLastScratchLoadLegacyPass(PassRegistry &)

void initializeAMDGPUInsertDelayAluLegacyPass(PassRegistry &)

void initializeSIOptimizeExecMaskingPreRALegacyPass(PassRegistry &)

char & GCNPreRAOptimizationsID

LLVM_ABI char & GCLoweringID

GCLowering Pass - Used by gc.root to perform its default lowering operations.

void initializeSIInsertHardClausesLegacyPass(PassRegistry &)

ModulePass * createExpandVariadicsPass(ExpandVariadicsMode)

FunctionPass * createSIAnnotateControlFlowLegacyPass()

Create the annotation pass.

FunctionPass * createSIModeRegisterPass()

void initializeGCNPreRAOptimizationsLegacyPass(PassRegistry &)

void initializeSILowerWWMCopiesLegacyPass(PassRegistry &)

LLVM_ABI FunctionPass * createGreedyRegisterAllocator()

Greedy register allocation pass - This pass implements a global register allocator for optimized buil...

void initializeAMDGPUAAWrapperPassPass(PassRegistry &)

void initializeSIShrinkInstructionsLegacyPass(PassRegistry &)

ModulePass * createAMDGPULowerBufferFatPointersPass()

void initializeR600ClauseMergePassPass(PassRegistry &)

ModulePass * createAMDGPUCtorDtorLoweringLegacyPass()

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

ModuleToFunctionPassAdaptor createModuleToFunctionPassAdaptor(FunctionPassT &&Pass, bool EagerlyInvalidate=false)

A function to deduce a function pass type and wrap it in the templated adaptor.

ModulePass * createAMDGPUSwLowerLDSLegacyPass(const AMDGPUTargetMachine *TM=nullptr)

void initializeGCNRewritePartialRegUsesLegacyPass(llvm::PassRegistry &)

void initializeAMDGPURewriteUndefForPHILegacyPass(PassRegistry &)

char & GCNRewritePartialRegUsesID

void initializeAMDGPUSwLowerLDSLegacyPass(PassRegistry &)

LLVM_ABI std::error_code inconvertibleErrorCode()

The value returned by this function can be returned from convertToErrorCode for Error values where no...

void initializeAMDGPULowerVGPREncodingLegacyPass(PassRegistry &)

char & AMDGPUWaitSGPRHazardsLegacyID

void initializeSILowerSGPRSpillsLegacyPass(PassRegistry &)

LLVM_ABI Pass * createLoadStoreVectorizerPass()

Create a legacy pass manager instance of the LoadStoreVectorizer pass.

std::unique_ptr< ScheduleDAGMutation > createIGroupLPDAGMutation(AMDGPU::SchedulingPhase Phase)

Phase specifes whether or not this is a reentry into the IGroupLPDAGMutation.

void initializeAMDGPUDAGToDAGISelLegacyPass(PassRegistry &)

FunctionPass * createAMDGPURegBankCombiner(bool IsOptNone)

LLVM_ABI FunctionPass * createNaryReassociatePass()

char & AMDGPUReserveWWMRegsLegacyID

void initializeAMDGPUWaitSGPRHazardsLegacyPass(PassRegistry &)

LLVM_ABI char & PatchableFunctionID

This pass implements the "patchable-function" attribute.

char & SIOptimizeExecMaskingLegacyID

LLVM_ABI char & PostRASchedulerID

PostRAScheduler - This pass performs post register allocation scheduling.

void initializeR600ExpandSpecialInstrsPassPass(PassRegistry &)

void initializeR600PacketizerPass(PassRegistry &)

std::unique_ptr< ScheduleDAGMutation > createVOPDPairingMutation()

ModulePass * createAMDGPUExportKernelRuntimeHandlesLegacyPass()

ModulePass * createAMDGPUAlwaysInlinePass(bool GlobalOpt=true)

void initializeAMDGPUAsmPrinterPass(PassRegistry &)

void initializeSIFoldOperandsLegacyPass(PassRegistry &)

char & SILoadStoreOptimizerLegacyID

void initializeAMDGPUGlobalISelDivergenceLoweringPass(PassRegistry &)

PassManager< LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, CGSCCUpdateResult & > CGSCCPassManager

The CGSCC pass manager.

LLVM_ABI std::unique_ptr< CSEConfigBase > getStandardCSEConfigForOpt(CodeGenOptLevel Level)

Target & getTheR600Target()

The target for R600 GPUs.

LLVM_ABI char & MachineSchedulerID

MachineScheduler - This pass schedules machine instructions.

LLVM_ABI Pass * createStructurizeCFGPass(bool SkipUniformRegions=false)

When SkipUniformRegions is true the structizer will not structurize regions that only contain uniform...

LLVM_ABI char & PostMachineSchedulerID

PostMachineScheduler - This pass schedules machine instructions postRA.

LLVM_ABI Pass * createLICMPass()

char & SIFormMemoryClausesID

void initializeSILoadStoreOptimizerLegacyPass(PassRegistry &)

void initializeAMDGPULowerModuleLDSLegacyPass(PassRegistry &)

AnalysisManager< LazyCallGraph::SCC, LazyCallGraph & > CGSCCAnalysisManager

The CGSCC analysis manager.

void initializeAMDGPUCtorDtorLoweringLegacyPass(PassRegistry &)

LLVM_ABI char & EarlyIfConverterLegacyID

EarlyIfConverter - This pass performs if-conversion on SSA form by inserting cmov instructions.

AnalysisManager< Loop, LoopStandardAnalysisResults & > LoopAnalysisManager

The loop analysis manager.

FunctionPass * createAMDGPUUniformIntrinsicCombineLegacyPass()

void initializeAMDGPURegBankCombinerPass(PassRegistry &)

ThinOrFullLTOPhase

This enumerates the LLVM full LTO or ThinLTO optimization phases.

@ FullLTOPreLink

Full LTO prelink phase.

@ FullLTOPostLink

Full LTO postlink (backend compile) phase.

@ ThinLTOPreLink

ThinLTO prelink (summary) phase.

char & AMDGPUUnifyDivergentExitNodesID

void initializeAMDGPUPrepareAGPRAllocLegacyPass(PassRegistry &)

FunctionPass * createAMDGPUAtomicOptimizerPass(ScanOptions ScanStrategy)

FunctionPass * createAMDGPUPreloadKernArgPrologLegacyPass()

char & SIOptimizeVGPRLiveRangeLegacyID

LLVM_ABI char & ShadowStackGCLoweringID

ShadowStackGCLowering - Implements the custom lowering mechanism used by the shadow stack GC.

void initializeAMDGPURewriteOutArgumentsPass(PassRegistry &)

static Reloc::Model getEffectiveRelocModel(std::optional< Reloc::Model > RM)

void initializeAMDGPUExternalAAWrapperPass(PassRegistry &)

auto formatv(bool Validate, const char *Fmt, Ts &&...Vals)

void initializeAMDGPULowerKernelArgumentsPass(PassRegistry &)

void initializeSIModeRegisterLegacyPass(PassRegistry &)

CodeModel::Model getEffectiveCodeModel(std::optional< CodeModel::Model > CM, CodeModel::Model Default)

Helper method for getting the code model, returning Default if CM does not have a value.

void initializeAMDGPUPreloadKernelArgumentsLegacyPass(PassRegistry &)

char & SILateBranchLoweringPassID

FunctionToLoopPassAdaptor createFunctionToLoopPassAdaptor(LoopPassT &&Pass, bool UseMemorySSA=false)

A function to deduce a loop pass type and wrap it in the templated adaptor.

LLVM_ABI char & BranchRelaxationPassID

BranchRelaxation - This pass replaces branches that need to jump further than is supported by a branc...

LLVM_ABI FunctionPass * createSinkingPass()

CGSCCToFunctionPassAdaptor createCGSCCToFunctionPassAdaptor(FunctionPassT &&Pass, bool EagerlyInvalidate=false, bool NoRerun=false)

A function to deduce a function pass type and wrap it in the templated adaptor.

void initializeSIMemoryLegalizerLegacyPass(PassRegistry &)

ModulePass * createAMDGPULowerIntrinsicsLegacyPass()

void initializeR600MachineCFGStructurizerPass(PassRegistry &)

CodeGenFileType

These enums are meant to be passed into addPassesToEmitFile to indicate what type of file to emit,...

char & GCNDPPCombineLegacyID

PassManager< Module > ModulePassManager

Convenience typedef for a pass manager over modules.

LLVM_ABI std::unique_ptr< ScheduleDAGMutation > createStoreClusterDAGMutation(const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, bool ReorderWhileClustering=false)

If ReorderWhileClustering is set to true, no attempt will be made to reduce reordering due to store c...

LLVM_ABI FunctionPass * createLoopDataPrefetchPass()

FunctionPass * createAMDGPULowerKernelArgumentsPass()

char & AMDGPUInsertDelayAluID

std::unique_ptr< ScheduleDAGMutation > createAMDGPUMacroFusionDAGMutation()

Note that you have to add: DAG.addMutation(createAMDGPUMacroFusionDAGMutation()); to AMDGPUTargetMach...

LLVM_ABI char & StackMapLivenessID

StackMapLiveness - This pass analyses the register live-out set of stackmap/patchpoint intrinsics and...

void initializeGCNPreRALongBranchRegLegacyPass(PassRegistry &)

char & SILowerWWMCopiesLegacyID

LLVM_ABI FunctionPass * createUnifyLoopExitsPass()

char & SIOptimizeExecMaskingPreRAID

LLVM_ABI FunctionPass * createFixIrreduciblePass()

void initializeR600EmitClauseMarkersPass(PassRegistry &)

LLVM_ABI char & FuncletLayoutID

This pass lays out funclets contiguously.

LLVM_ABI char & DetectDeadLanesID

This pass adds dead/undef flags after analyzing subregister lanes.

void initializeAMDGPULowerExecSyncLegacyPass(PassRegistry &)

void initializeAMDGPUPostLegalizerCombinerPass(PassRegistry &)

void initializeAMDGPUExportKernelRuntimeHandlesLegacyPass(PassRegistry &)

CodeGenOptLevel

Code generation optimization level.

void initializeSIInsertWaitcntsLegacyPass(PassRegistry &)

ModulePass * createAMDGPUPreloadKernelArgumentsLegacyPass(const TargetMachine *)

ModulePass * createAMDGPUPrintfRuntimeBinding()

LLVM_ABI char & StackSlotColoringID

StackSlotColoring - This pass performs stack slot coloring.

LLVM_ABI Pass * createAlwaysInlinerLegacyPass(bool InsertLifetime=true)

Create a legacy pass manager instance of a pass to inline and remove functions marked as "always_inli...

void initializeR600ControlFlowFinalizerPass(PassRegistry &)

void initializeAMDGPUImageIntrinsicOptimizerPass(PassRegistry &)

void initializeSILateBranchLoweringLegacyPass(PassRegistry &)

void initializeSILowerControlFlowLegacyPass(PassRegistry &)

void initializeSIFormMemoryClausesLegacyPass(PassRegistry &)

char & SIPreAllocateWWMRegsLegacyID

Error make_error(ArgTs &&... Args)

Make a Error instance representing failure using the given error info type.

ModulePass * createAMDGPULowerModuleLDSLegacyPass(const AMDGPUTargetMachine *TM=nullptr)

void initializeAMDGPUPreLegalizerCombinerPass(PassRegistry &)

FunctionPass * createAMDGPUPromoteAlloca()

void initializeAMDGPUArgumentUsageInfoWrapperLegacyPass(PassRegistry &)

LLVM_ABI FunctionPass * createSeparateConstOffsetFromGEPPass(bool LowerGEP=false)

void initializeAMDGPUReserveWWMRegsLegacyPass(PassRegistry &)

char & SIPreEmitPeepholeID

char & SIPostRABundlerLegacyID

ModulePass * createAMDGPURemoveIncompatibleFunctionsPass(const TargetMachine *)

void initializeGCNRegPressurePrinterPass(PassRegistry &)

void initializeSILowerI1CopiesLegacyPass(PassRegistry &)

char & SILowerSGPRSpillsLegacyID

LLVM_ABI FunctionPass * createBasicRegisterAllocator()

BasicRegisterAllocation Pass - This pass implements a degenerate global register allocator using the ...

LLVM_ABI void initializeGlobalISel(PassRegistry &)

Initialize all passes linked into the GlobalISel library.

char & SILowerControlFlowLegacyID

ModulePass * createR600OpenCLImageTypeLoweringPass()

FunctionPass * createAMDGPUCodeGenPreparePass()

void initializeSIAnnotateControlFlowLegacyPass(PassRegistry &)

FunctionPass * createAMDGPUISelDag(TargetMachine &TM, CodeGenOptLevel OptLevel)

This pass converts a legalized DAG into a AMDGPU-specific.

void initializeGCNCreateVOPDLegacyPass(PassRegistry &)

void initializeAMDGPUUniformIntrinsicCombineLegacyPass(PassRegistry &)

void initializeSIPreAllocateWWMRegsLegacyPass(PassRegistry &)

void initializeSIFixVGPRCopiesLegacyPass(PassRegistry &)

Target & getTheGCNTarget()

The target for GCN GPUs.

void initializeSIFixSGPRCopiesLegacyPass(PassRegistry &)

void initializeAMDGPUAtomicOptimizerPass(PassRegistry &)

void initializeAMDGPULowerIntrinsicsLegacyPass(PassRegistry &)

LLVM_ABI FunctionPass * createGVNPass()

Create a legacy GVN pass.

void initializeAMDGPURewriteAGPRCopyMFMALegacyPass(PassRegistry &)

void initializeSIPostRABundlerLegacyPass(PassRegistry &)

FunctionPass * createAMDGPURegBankSelectPass()

FunctionPass * createAMDGPURegBankLegalizePass()

LLVM_ABI char & MachineCSELegacyID

MachineCSE - This pass performs global CSE on machine instructions.

LLVM_ABI std::unique_ptr< ScheduleDAGMutation > createLoadClusterDAGMutation(const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, bool ReorderWhileClustering=false)

If ReorderWhileClustering is set to true, no attempt will be made to reduce reordering due to store c...

PassManager< Function > FunctionPassManager

Convenience typedef for a pass manager over functions.

LLVM_ABI char & LiveVariablesID

LiveVariables pass - This pass computes the set of blocks in which each variable is life and sets mac...

void initializeAMDGPUCodeGenPreparePass(PassRegistry &)

FunctionPass * createAMDGPURewriteUndefForPHILegacyPass()

void initializeSIOptimizeExecMaskingLegacyPass(PassRegistry &)

void call_once(once_flag &flag, Function &&F, Args &&... ArgList)

Execute the function specified as a parameter once.

FunctionPass * createSILowerI1CopiesLegacyPass()

FunctionPass * createAMDGPUPostLegalizeCombiner(bool IsOptNone)

void initializeAMDGPULowerKernelAttributesPass(PassRegistry &)

char & SIInsertHardClausesID

char & SIFixSGPRCopiesLegacyID

void initializeGCNDPPCombineLegacyPass(PassRegistry &)

char & SIPeepholeSDWALegacyID

LLVM_ABI char & VirtRegRewriterID

VirtRegRewriter pass.

char & SIFoldOperandsLegacyID

void initializeGCNNSAReassignLegacyPass(PassRegistry &)

char & AMDGPUPrepareAGPRAllocLegacyID

LLVM_ABI FunctionPass * createLowerSwitchPass()

void initializeAMDGPUPreloadKernArgPrologLegacyPass(PassRegistry &)

LLVM_ABI FunctionPass * createVirtRegRewriter(bool ClearVirtRegs=true)

void initializeR600VectorRegMergerPass(PassRegistry &)

char & AMDGPURewriteAGPRCopyMFMALegacyID

ModulePass * createAMDGPULowerExecSyncLegacyPass()

char & AMDGPULowerVGPREncodingLegacyID

FunctionPass * createAMDGPUGlobalISelDivergenceLoweringPass()

FunctionPass * createSIMemoryLegalizerPass()

void initializeAMDGPULateCodeGenPrepareLegacyPass(PassRegistry &)

void initializeSIOptimizeVGPRLiveRangeLegacyPass(PassRegistry &)

void initializeSIPeepholeSDWALegacyPass(PassRegistry &)

void initializeAMDGPURegBankLegalizePass(PassRegistry &)

LLVM_ABI char & TwoAddressInstructionPassID

TwoAddressInstruction - This pass reduces two-address instructions to use two operands.

AnalysisManager< Function > FunctionAnalysisManager

Convenience typedef for the Function analysis manager.

FunctionPass * createAMDGPUPreLegalizeCombiner(bool IsOptNone)

void initializeAMDGPURegBankSelectPass(PassRegistry &)

FunctionPass * createAMDGPULateCodeGenPrepareLegacyPass()

LLVM_ABI FunctionPass * createAtomicExpandLegacyPass()

AtomicExpandPass - At IR level this pass replace atomic instructions with __atomic_* library calls,...

MCRegisterInfo * createGCNMCRegisterInfo(AMDGPUDwarfFlavour DwarfFlavour)

LLVM_ABI FunctionPass * createStraightLineStrengthReducePass()

BumpPtrAllocatorImpl<> BumpPtrAllocator

The standard BumpPtrAllocator which just uses the default template parameters.

FunctionPass * createAMDGPUImageIntrinsicOptimizerPass(const TargetMachine *)

void initializeAMDGPUUnifyDivergentExitNodesPass(PassRegistry &)

void initializeAMDGPULowerBufferFatPointersPass(PassRegistry &)

FunctionPass * createSIInsertWaitcntsPass()

FunctionPass * createAMDGPUAnnotateUniformValuesLegacy()

LLVM_ABI FunctionPass * createEarlyCSEPass(bool UseMemorySSA=false)

void initializeSIWholeQuadModeLegacyPass(PassRegistry &)

LLVM_ABI char & PHIEliminationID

PHIElimination - This pass eliminates machine instruction PHI nodes by inserting copy instructions.

LLVM_ABI llvm:🆑:opt< bool > NoKernelInfoEndLTO

bool parseNamedRegisterReference(PerFunctionMIParsingState &PFS, Register &Reg, StringRef Src, SMDiagnostic &Error)

void initializeAMDGPUResourceUsageAnalysisWrapperPassPass(PassRegistry &)

FunctionPass * createSIShrinkInstructionsLegacyPass()

char & AMDGPUMarkLastScratchLoadID

LLVM_ABI char & RenameIndependentSubregsID

This pass detects subregister lanes in a virtual register that are used independently of other lanes ...

void initializeAMDGPUAnnotateUniformValuesLegacyPass(PassRegistry &)

std::unique_ptr< ScheduleDAGMutation > createAMDGPUExportClusteringDAGMutation()

void initializeAMDGPUPrintfRuntimeBindingPass(PassRegistry &)

void initializeAMDGPUPromoteAllocaPass(PassRegistry &)

void initializeAMDGPURemoveIncompatibleFunctionsLegacyPass(PassRegistry &)

void initializeAMDGPUAlwaysInlinePass(PassRegistry &)

LLVM_ABI char & DeadMachineInstructionElimID

DeadMachineInstructionElim - This pass removes dead machine instructions.

void initializeSIPreEmitPeepholeLegacyPass(PassRegistry &)

AnalysisManager< Module > ModuleAnalysisManager

Convenience typedef for the Module analysis manager.

char & AMDGPUPerfHintAnalysisLegacyID

LLVM_ABI ImmutablePass * createExternalAAWrapperPass(std::function< void(Pass &, Function &, AAResults &)> Callback)

A wrapper pass around a callback which can be used to populate the AAResults in the AAResultsWrapperP...

char & GCNPreRALongBranchRegID

LLVM_ABI CGPassBuilderOption getCGPassBuilderOption()

LLVM_ABI void reportFatalUsageError(Error Err)

Report a fatal error that does not indicate a bug in LLVM.

void initializeAMDGPUPromoteKernelArgumentsPass(PassRegistry &)

ArgDescriptor PrivateSegmentBuffer

ArgDescriptor WorkGroupIDY

ArgDescriptor WorkGroupIDZ

ArgDescriptor PrivateSegmentSize

ArgDescriptor ImplicitArgPtr

ArgDescriptor PrivateSegmentWaveByteOffset

ArgDescriptor WorkGroupInfo

ArgDescriptor WorkItemIDZ

ArgDescriptor WorkItemIDY

ArgDescriptor LDSKernelId

ArgDescriptor KernargSegmentPtr

ArgDescriptor WorkItemIDX

ArgDescriptor FlatScratchInit

ArgDescriptor DispatchPtr

ArgDescriptor ImplicitBufferPtr

ArgDescriptor WorkGroupIDX

static ArgDescriptor createStack(unsigned Offset, unsigned Mask=~0u)

static ArgDescriptor createArg(const ArgDescriptor &Arg, unsigned Mask)

static ArgDescriptor createRegister(Register Reg, unsigned Mask=~0u)

DenormalModeKind Input

Denormal treatment kind for floating point instruction inputs in the default floating-point environme...

@ PreserveSign

The sign of a flushed-to-zero number is preserved in the sign of 0.

@ IEEE

IEEE-754 denormal numbers preserved.

DenormalModeKind Output

Denormal flushing mode for floating point instruction results in the default floating point environme...

A simple and fast domtree-based CSE pass.

MachineFunctionInfo - This class can be derived from and used by targets to hold private target-speci...

static FuncInfoTy * create(BumpPtrAllocator &Allocator, const Function &F, const SubtargetTy *STI)

Factory function: default behavior is to call new using the supplied allocator.

MachineSchedContext provides enough context from the MachineScheduler pass for the target to instanti...

StringMap< VRegInfo * > VRegInfosNamed

DenseMap< Register, VRegInfo * > VRegInfos

RegisterTargetMachine - Helper template for registering a target machine implementation,...

A utility pass template to force an analysis result to be available.

bool DX10Clamp

Used by the vector ALU to force DX10-style treatment of NaNs: when set, clamp NaN to zero; otherwise,...

DenormalMode FP64FP16Denormals

If this is set, neither input or output denormals are flushed for both f64 and f16/v2f16 instructions...

bool IEEE

Floating point opcodes that support exception flag gathering quiet and propagate signaling NaN inputs...

DenormalMode FP32Denormals

If this is set, neither input or output denormals are flushed for most f32 instructions.

The llvm::once_flag structure.

Targets should override this in a way that mirrors the implementation of llvm::MachineFunctionInfo.

StringValue SGPRForEXECCopy

SmallVector< StringValue > WWMReservedRegs

StringValue FrameOffsetReg

StringValue LongBranchReservedReg

unsigned NumKernargPreloadSGPRs

StringValue VGPRForAGPRCopy

std::optional< SIArgumentInfo > ArgInfo

SmallVector< StringValue, 2 > SpillPhysVGPRS

StringValue ScratchRSrcReg

StringValue StackPtrOffsetReg

bool FP64FP16OutputDenormals

bool FP64FP16InputDenormals

A wrapper around std::string which contains a source range that's being set during parsing.