SPIRVBuiltins.cpp Source File (original) (raw)

20#include "llvm/IR/IntrinsicsSPIRV.h"

21#include

22#include

23#include

25#define DEBUG_TYPE "spirv-builtins"

27namespace llvm {

29#define GET_BuiltinGroup_DECL

30#include "SPIRVGenTables.inc"

34 InstructionSet::InstructionSet Set;

38};

40#define GET_DemangledBuiltins_DECL

41#define GET_DemangledBuiltins_IMPL

63 InstructionSet::InstructionSet Set;

65};

67#define GET_NativeBuiltins_DECL

68#define GET_NativeBuiltins_IMPL

86#define GET_GroupBuiltins_DECL

87#define GET_GroupBuiltins_IMPL

97#define GET_IntelSubgroupsBuiltins_DECL

98#define GET_IntelSubgroupsBuiltins_IMPL

104

105#define GET_AtomicFloatingBuiltins_DECL

106#define GET_AtomicFloatingBuiltins_IMPL

112

113#define GET_GroupUniformBuiltins_DECL

114#define GET_GroupUniformBuiltins_IMPL

115

118 InstructionSet::InstructionSet Set;

120};

121

123#define GET_GetBuiltins_DECL

124#define GET_GetBuiltins_IMPL

125

128 InstructionSet::InstructionSet Set;

130};

131

132#define GET_ImageQueryBuiltins_DECL

133#define GET_ImageQueryBuiltins_IMPL

134

140

141#define GET_IntegerDotProductBuiltins_DECL

142#define GET_IntegerDotProductBuiltins_IMPL

143

146 InstructionSet::InstructionSet Set;

153};

154

157 InstructionSet::InstructionSet Set;

162};

163

165#define GET_ConvertBuiltins_DECL

166#define GET_ConvertBuiltins_IMPL

167

168using namespace InstructionSet;

169#define GET_VectorLoadStoreBuiltins_DECL

170#define GET_VectorLoadStoreBuiltins_IMPL

171

172#define GET_CLMemoryScope_DECL

173#define GET_CLSamplerAddressingMode_DECL

174#define GET_CLMemoryFenceFlags_DECL

175#define GET_ExtendedBuiltins_DECL

176#include "SPIRVGenTables.inc"

177}

178

179

180

181

182

183

185

188 StringRef PassPrefix = "(anonymous namespace)::";

189 std::string BuiltinName;

190

191 if (DemangledCall.starts_with(PassPrefix))

192 BuiltinName = DemangledCall.substr(PassPrefix.size());

193 else

194 BuiltinName = DemangledCall;

195

196

197 BuiltinName = BuiltinName.substr(0, BuiltinName.find('('));

198

199

200 if (BuiltinName.rfind("__spirv_ocl_", 0) == 0)

201 BuiltinName = BuiltinName.substr(12);

202

203

204

205

206 std::size_t Pos1 = BuiltinName.rfind('<');

207 if (Pos1 != std:🧵:npos && BuiltinName.back() == '>') {

208 std::size_t Pos2 = BuiltinName.rfind(' ', Pos1);

209 if (Pos2 == std:🧵:npos)

210 Pos2 = 0;

211 else

212 ++Pos2;

213 BuiltinName = BuiltinName.substr(Pos2, Pos1 - Pos2);

214 BuiltinName = BuiltinName.substr(BuiltinName.find_last_of(' ') + 1);

215 }

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241 static const std::regex SpvWithR(

242 "(__spirv_(ImageSampleExplicitLod|ImageRead|ImageWrite|ImageQuerySizeLod|"

243 "UDotKHR|"

245 "ReadClockKHR|SubgroupBlockReadINTEL|SubgroupImageBlockReadINTEL|"

246 "SubgroupImageMediaBlockReadINTEL|SubgroupImageMediaBlockWriteINTEL|"

247 "Convert|Round|"

248 "UConvert|SConvert|FConvert|SatConvert)[^_]*)(_R[^_]*_?(\\w+)?.*)?");

249 std::smatch Match;

250 if (std::regex_match(BuiltinName, Match, SpvWithR) && Match.size() > 1) {

251 std::ssub_match SubMatch;

252 if (DecorationId && Match.size() > 3) {

253 SubMatch = Match[4];

255 }

256 SubMatch = Match[1];

257 BuiltinName = SubMatch.str();

258 }

259

260 return BuiltinName;

261}

262}

263

264

265

266

267

268

269

270

271

272

273

274static std::unique_ptr

276 SPIRV::InstructionSet::InstructionSet Set,

280

283 DemangledCall.slice(DemangledCall.find('(') + 1, DemangledCall.find(')'));

284 BuiltinArgs.split(BuiltinArgumentTypes, ',', -1, false);

285

286

287

289 if ((Builtin = SPIRV::lookupBuiltin(BuiltinName, Set)))

290 return std::make_uniqueSPIRV::IncomingCall(

291 BuiltinName, Builtin, ReturnRegister, ReturnType, Arguments);

292

293

294

295

296 if (BuiltinArgumentTypes.size() >= 1) {

297 char FirstArgumentType = BuiltinArgumentTypes[0][0];

298

299

300 std::string Prefix;

301

302 switch (FirstArgumentType) {

303

304 case 'u':

305 if (Set == SPIRV::InstructionSet::OpenCL_std)

306 Prefix = "u_";

307 else if (Set == SPIRV::InstructionSet::GLSL_std_450)

308 Prefix = "u";

309 break;

310

311 case 'c':

312 case 's':

313 case 'i':

314 case 'l':

315 if (Set == SPIRV::InstructionSet::OpenCL_std)

316 Prefix = "s_";

317 else if (Set == SPIRV::InstructionSet::GLSL_std_450)

318 Prefix = "s";

319 break;

320

321 case 'f':

322 case 'd':

323 case 'h':

324 if (Set == SPIRV::InstructionSet::OpenCL_std ||

325 Set == SPIRV::InstructionSet::GLSL_std_450)

326 Prefix = "f";

327 break;

328 }

329

330

331 if (!Prefix.empty() &&

332 (Builtin = SPIRV::lookupBuiltin(Prefix + BuiltinName, Set)))

333 return std::make_uniqueSPIRV::IncomingCall(

334 BuiltinName, Builtin, ReturnRegister, ReturnType, Arguments);

335

336

337

338

339 std::string Suffix;

340

341 switch (FirstArgumentType) {

342

343 case 'u':

344 Suffix = "u";

345 break;

346

347 case 'c':

348 case 's':

349 case 'i':

350 case 'l':

351 Suffix = "s";

352 break;

353

354 case 'f':

355 case 'd':

356 case 'h':

357 Suffix = "f";

358 break;

359 }

360

361

362 if (!Suffix.empty() &&

363 (Builtin = SPIRV::lookupBuiltin(BuiltinName + Suffix, Set)))

364 return std::make_uniqueSPIRV::IncomingCall(

365 BuiltinName, Builtin, ReturnRegister, ReturnType, Arguments);

366 }

367

368

369 return nullptr;

370}

371

374

375

376

377

378

380 assert(MI->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST &&

381 MI->getOperand(1).isReg());

382 Register BitcastReg = MI->getOperand(1).getReg();

383 MachineInstr *BitcastMI = MRI->getUniqueVRegDef(BitcastReg);

388 return ValueMI;

389}

390

391

392

393

396 assert(DefMI->getOpcode() == TargetOpcode::G_CONSTANT &&

397 DefMI->getOperand(1).isCImm());

398 return DefMI->getOperand(1).getCImm()->getValue().getZExtValue();

399}

400

401

402

405 if (!NextMI)

406 return nullptr;

407 if (isSpvIntrinsic(*NextMI, Intrinsic::spv_assign_name))

408 if ((NextMI = NextMI->getNextNode()) == nullptr)

409 return nullptr;

410 Register ValueReg = MI->getOperand(0).getReg();

411 if (( isSpvIntrinsic (*NextMI, Intrinsic::spv_assign_type) &&

412 isSpvIntrinsic (*NextMI, Intrinsic::spv_assign_ptr_type)) ||

414 return nullptr;

416 assert(Ty && "Type is expected");

417 return Ty;

418}

419

422

423

424

425

426

428 if (MI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE)

429 return MI->getOperand(1).getGlobal()->getType();

431 "Blocks in OpenCL C must be traceable to allocation site");

433}

434

435

436

437

438

439

440

441

442

443static std::tuple<Register, SPIRVType *>

448

449 if (ResultType->getOpcode() == SPIRV::OpTypeVector) {

450 unsigned VectorElements = ResultType->getOperand(2).getImm();

452 MIRBuilder, true);

456 } else {

458 }

459

464 return std::make_tuple(ResultRegister, BoolType);

465}

466

467

468

473 Register TrueConst, FalseConst;

474

475 if (ReturnType->getOpcode() == SPIRV::OpTypeVector) {

478 TrueConst =

481 } else {

482 TrueConst = GR->buildConstantInt(1, MIRBuilder, ReturnType, true);

483 FalseConst = GR->buildConstantInt(0, MIRBuilder, ReturnType, true);

484 }

485

486 return MIRBuilder.buildSelect(ReturnRegister, SourceRegister, TrueConst,

487 FalseConst);

488}

489

490

491

496 if (!DestinationReg.isValid())

498

499

501 MIRBuilder.buildLoad(DestinationReg, PtrRegister, PtrInfo, Align());

502 return DestinationReg;

503}

504

505

506

511 const std::optionalSPIRV::LinkageType::LinkageType &LinkageTy = {

512 SPIRV::LinkageType::Import}) {

516 NewRegister,

520 VariableType, MIRBuilder, SPIRV::StorageClass::Input);

522

523

526 SPIRV::StorageClass::Input, nullptr, isConst, LinkageTy,

527 MIRBuilder, false);

528

529

531 buildLoadInst(VariableType, Variable, MIRBuilder, GR, LLType, Reg);

532 MIRBuilder.getMRI()->setType(LoadedRegister, LLType);

533 return LoadedRegister;

534}

535

536

537

541

542

543static SPIRV::MemorySemantics::MemorySemantics

545 switch (MemOrder) {

546 case std::memory_order_relaxed:

547 return SPIRV::MemorySemantics::None;

548 case std::memory_order_acquire:

549 return SPIRV::MemorySemantics::Acquire;

550 case std::memory_order_release:

551 return SPIRV::MemorySemantics::Release;

552 case std::memory_order_acq_rel:

553 return SPIRV::MemorySemantics::AcquireRelease;

554 case std::memory_order_seq_cst:

555 return SPIRV::MemorySemantics::SequentiallyConsistent;

556 default:

558 }

559}

560

561static SPIRV::Scope::Scope getSPIRVScope(SPIRV::CLMemoryScope ClScope) {

562 switch (ClScope) {

563 case SPIRV::CLMemoryScope::memory_scope_work_item:

564 return SPIRV::Scope::Invocation;

565 case SPIRV::CLMemoryScope::memory_scope_work_group:

566 return SPIRV::Scope::Workgroup;

567 case SPIRV::CLMemoryScope::memory_scope_device:

568 return SPIRV::Scope::Device;

569 case SPIRV::CLMemoryScope::memory_scope_all_svm_devices:

570 return SPIRV::Scope::CrossDevice;

571 case SPIRV::CLMemoryScope::memory_scope_sub_group:

572 return SPIRV::Scope::Subgroup;

573 }

575}

576

583

585 SPIRV::Scope::Scope Scope,

589 if (CLScopeRegister.isValid()) {

590 auto CLScope =

591 static_castSPIRV::CLMemoryScope\(getIConstVal(CLScopeRegister, MRI));

593

594 if (CLScope == static_cast<unsigned>(Scope)) {

595 MRI->setRegClass(CLScopeRegister, &SPIRV::iIDRegClass);

596 return CLScopeRegister;

597 }

598 }

600}

601

604 if (MRI->getRegClassOrNull(Reg))

605 return;

607 MRI->setRegClass(Reg,

608 SpvType ? GR->getRegClass(SpvType) : &SPIRV::iIDRegClass);

609}

610

612 Register PtrRegister, unsigned &Semantics,

615 if (SemanticsRegister.isValid()) {

617 std::memory_order Order =

618 static_caststd::memory\_order\(getIConstVal(SemanticsRegister, MRI));

619 Semantics =

622 if (static_cast<unsigned>(Order) == Semantics) {

623 MRI->setRegClass(SemanticsRegister, &SPIRV::iIDRegClass);

624 return SemanticsRegister;

625 }

626 }

628}

629

634 auto MIB = MIRBuilder.buildInstr(Opcode);

637 unsigned Sz = Call->Arguments.size() - ImmArgs.size();

638 for (unsigned i = 0; i < Sz; ++i)

639 MIB.addUse(Call->Arguments[i]);

640 for (uint32_t ImmArg : ImmArgs)

641 MIB.addImm(ImmArg);

642 return true;

643}

644

645

648 if (Call->isSpirvOp())

650

651 assert(Call->Arguments.size() == 2 &&

652 "Need 2 arguments for atomic init translation");

653 MIRBuilder.buildInstr(SPIRV::OpStore)

656 return true;

657}

658

659

664 if (Call->isSpirvOp())

666

668

669

670

672 Call->Arguments.size() > 1

673 ? Call->Arguments[1]

676 if (Call->Arguments.size() > 2) {

677

678 MemSemanticsReg = Call->Arguments[2];

679 } else {

680 int Semantics =

681 SPIRV::MemorySemantics::SequentiallyConsistent |

684 }

685

686 MIRBuilder.buildInstr(SPIRV::OpAtomicLoad)

690 .addUse(ScopeRegister)

691 .addUse(MemSemanticsReg);

692 return true;

693}

694

695

699 if (Call->isSpirvOp())

702

706 int Semantics =

707 SPIRV::MemorySemantics::SequentiallyConsistent |

710 MIRBuilder.buildInstr(SPIRV::OpAtomicStore)

712 .addUse(ScopeRegister)

713 .addUse(MemSemanticsReg)

715 return true;

716}

717

718

722 if (Call->isSpirvOp())

725

726 bool IsCmpxchg = Call->Builtin->Name.contains("cmpxchg");

728

729 Register ObjectPtr = Call->Arguments[0];

730 Register ExpectedArg = Call->Arguments[1];

731 Register Desired = Call->Arguments[2];

733 LLT DesiredLLT = MRI->getType(Desired);

734

736 SPIRV::OpTypePointer);

738 (void)ExpectedType;

739 assert(IsCmpxchg ? ExpectedType == SPIRV::OpTypeInt

740 : ExpectedType == SPIRV::OpTypePointer);

742

745 auto StorageClass = static_castSPIRV::StorageClass::StorageClass\(

748

752 IsCmpxchg

753 ? SPIRV::MemorySemantics::None

754 : SPIRV::MemorySemantics::SequentiallyConsistent | MemSemStorage;

756 IsCmpxchg

757 ? SPIRV::MemorySemantics::None

758 : SPIRV::MemorySemantics::SequentiallyConsistent | MemSemStorage;

759 if (Call->Arguments.size() >= 4) {

760 assert(Call->Arguments.size() >= 5 &&

761 "Need 5+ args for explicit atomic cmpxchg");

762 auto MemOrdEq =

764 auto MemOrdNeq =

768 if (static_cast<unsigned>(MemOrdEq) == MemSemEqual)

769 MemSemEqualReg = Call->Arguments[3];

770 if (static_cast<unsigned>(MemOrdNeq) == MemSemEqual)

771 MemSemUnequalReg = Call->Arguments[4];

772 }

773 if (!MemSemEqualReg.isValid())

775 if (!MemSemUnequalReg.isValid())

777

779 auto Scope = IsCmpxchg ? SPIRV::Scope::Workgroup : SPIRV::Scope::Device;

780 if (Call->Arguments.size() >= 6) {

781 assert(Call->Arguments.size() == 6 &&

782 "Extra args for explicit atomic cmpxchg");

783 auto ClScope = static_castSPIRV::CLMemoryScope\(

786 if (ClScope == static_cast<unsigned>(Scope))

787 ScopeReg = Call->Arguments[5];

788 }

791

793 ? ExpectedArg

794 : buildLoadInst(SpvDesiredTy, ExpectedArg, MIRBuilder,

797 Register Tmp = !IsCmpxchg ? MRI->createGenericVirtualRegister(DesiredLLT)

798 : Call->ReturnRegister;

799 if ( MRI ->getRegClassOrNull(Tmp))

800 MRI->setRegClass(Tmp, GR->getRegClass(SpvDesiredTy));

802

808 .addUse(MemSemEqualReg)

809 .addUse(MemSemUnequalReg)

812 if (!IsCmpxchg) {

815 }

816 return true;

817}

818

819

823 if (Call->isSpirvOp())

826

829 Call->Arguments.size() >= 4 ? Call->Arguments[3] : Register();

830

831 assert(Call->Arguments.size() <= 4 &&

832 "Too many args for explicit atomic RMW");

833 ScopeRegister = buildScopeReg(ScopeRegister, SPIRV::Scope::Workgroup,

834 MIRBuilder, GR, MRI);

835

837 unsigned Semantics = SPIRV::MemorySemantics::None;

839 Call->Arguments.size() >= 3 ? Call->Arguments[2] : Register();

841 Semantics, MIRBuilder, GR);

844

845 if (Call->ReturnType->getOpcode() == SPIRV::OpTypeFloat) {

846 if (Opcode == SPIRV::OpAtomicIAdd) {

847 Opcode = SPIRV::OpAtomicFAddEXT;

848 } else if (Opcode == SPIRV::OpAtomicISub) {

849

850

851 Opcode = SPIRV::OpAtomicFAddEXT;

853 MRI->createGenericVirtualRegister(MRI->getType(ValueReg));

856 MIRBuilder.getMF());

857 MIRBuilder.buildInstr(TargetOpcode::G_FNEG)

860 updateRegType(NegValueReg, nullptr, Call->ReturnType, GR, MIRBuilder,

862 ValueReg = NegValueReg;

863 }

864 }

867 .addUse(ValueTypeReg)

869 .addUse(ScopeRegister)

870 .addUse(MemSemanticsReg)

872 return true;

873}

874

875

877 unsigned Opcode,

880 assert(Call->Arguments.size() == 4 &&

881 "Wrong number of atomic floating-type builtin");

884 Register MemSemanticsReg = Call->Arguments[2];

891 .addUse(MemSemanticsReg)

893 return true;

894}

895

896

897

901 bool IsSet = Opcode == SPIRV::OpAtomicFlagTestAndSet;

903 if (Call->isSpirvOp())

905 IsSet ? TypeReg : Register(0));

906

909 unsigned Semantics = SPIRV::MemorySemantics::SequentiallyConsistent;

911 Call->Arguments.size() >= 2 ? Call->Arguments[1] : Register();

913 Semantics, MIRBuilder, GR);

914

915 assert((Opcode != SPIRV::OpAtomicFlagClear ||

916 (Semantics != SPIRV::MemorySemantics::Acquire &&

917 Semantics != SPIRV::MemorySemantics::AcquireRelease)) &&

918 "Invalid memory order argument!");

919

921 Call->Arguments.size() >= 3 ? Call->Arguments[2] : Register();

922 ScopeRegister =

923 buildScopeReg(ScopeRegister, SPIRV::Scope::Device, MIRBuilder, GR, MRI);

924

925 auto MIB = MIRBuilder.buildInstr(Opcode);

926 if (IsSet)

928

929 MIB.addUse(PtrRegister).addUse(ScopeRegister).addUse(MemSemanticsReg);

930 return true;

931}

932

933

934

939 const auto *ST =

941 if ((Opcode == SPIRV::OpControlBarrierArriveINTEL ||

942 Opcode == SPIRV::OpControlBarrierWaitINTEL) &&

943 !ST->canUseExtension(SPIRV::Extension::SPV_INTEL_split_barrier)) {

944 std::string DiagMsg = std::string(Builtin->Name) +

945 ": the builtin requires the following SPIR-V "

946 "extension: SPV_INTEL_split_barrier";

948 }

949

950 if (Call->isSpirvOp())

952

955 unsigned MemSemantics = SPIRV::MemorySemantics::None;

956

957 if (MemFlags & SPIRV::CLK_LOCAL_MEM_FENCE)

958 MemSemantics |= SPIRV::MemorySemantics::WorkgroupMemory;

959

960 if (MemFlags & SPIRV::CLK_GLOBAL_MEM_FENCE)

961 MemSemantics |= SPIRV::MemorySemantics::CrossWorkgroupMemory;

962

963 if (MemFlags & SPIRV::CLK_IMAGE_MEM_FENCE)

964 MemSemantics |= SPIRV::MemorySemantics::ImageMemory;

965

966 if (Opcode == SPIRV::OpMemoryBarrier)

969 MemSemantics;

970 else if (Opcode == SPIRV::OpControlBarrierArriveINTEL)

971 MemSemantics |= SPIRV::MemorySemantics::Release;

972 else if (Opcode == SPIRV::OpControlBarrierWaitINTEL)

973 MemSemantics |= SPIRV::MemorySemantics::Acquire;

974 else

975 MemSemantics |= SPIRV::MemorySemantics::SequentiallyConsistent;

976

978 MemFlags == MemSemantics

979 ? Call->Arguments[0]

982 SPIRV::Scope::Scope Scope = SPIRV::Scope::Workgroup;

983 SPIRV::Scope::Scope MemScope = Scope;

984 if (Call->Arguments.size() >= 2) {

986 ((Opcode != SPIRV::OpMemoryBarrier && Call->Arguments.size() == 2) ||

987 (Opcode == SPIRV::OpMemoryBarrier && Call->Arguments.size() == 3)) &&

988 "Extra args for explicitly scoped barrier");

989 Register ScopeArg = (Opcode == SPIRV::OpMemoryBarrier) ? Call->Arguments[2]

990 : Call->Arguments[1];

991 SPIRV::CLMemoryScope CLScope =

992 static_castSPIRV::CLMemoryScope\(getIConstVal(ScopeArg, MRI));

994 if (!(MemFlags & SPIRV::CLK_LOCAL_MEM_FENCE) ||

995 (Opcode == SPIRV::OpMemoryBarrier))

996 Scope = MemScope;

997 if (CLScope == static_cast<unsigned>(Scope))

998 ScopeReg = Call->Arguments[1];

999 }

1000

1001 if (!ScopeReg.isValid())

1003

1005 if (Opcode != SPIRV::OpMemoryBarrier)

1007 MIB.addUse(MemSemanticsReg);

1008 return true;

1009}

1010

1011

1013 unsigned Opcode,

1017 const auto *ST =

1018 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());

1019 if ((Opcode == SPIRV::OpBitFieldInsert ||

1020 Opcode == SPIRV::OpBitFieldSExtract ||

1021 Opcode == SPIRV::OpBitFieldUExtract || Opcode == SPIRV::OpBitReverse) &&

1022 !ST->canUseExtension(SPIRV::Extension::SPV_KHR_bit_instructions)) {

1023 std::string DiagMsg = std::string(Builtin->Name) +

1024 ": the builtin requires the following SPIR-V "

1025 "extension: SPV_KHR_bit_instructions";

1027 }

1028

1029

1030 if (Call->isSpirvOp())

1033

1034 auto MIB = MIRBuilder.buildInstr(Opcode)

1037 for (unsigned i = 0; i < Call->Arguments.size(); ++i)

1039

1040 return true;

1041}

1042

1043

1045 unsigned Opcode,

1048

1049 if (Call->isSpirvOp())

1052

1057

1058 return true;

1059}

1060

1061

1065

1066 if (Call->isSpirvOp())

1069

1070 auto MIB = MIRBuilder.buildInstr(Opcode)

1073 for (unsigned i = 0; i < Call->Arguments.size(); ++i)

1075

1076 return true;

1077}

1078

1079

1081 unsigned Opcode,

1084

1085 if (Call->isSpirvOp())

1087

1088 auto MIB = MIRBuilder.buildInstr(Opcode)

1091 for (unsigned i = 0; i < Call->Arguments.size(); ++i)

1093

1094 return true;

1095}

1096

1100 switch (Opcode) {

1101 case SPIRV::OpCommitReadPipe:

1102 case SPIRV::OpCommitWritePipe:

1104 case SPIRV::OpGroupCommitReadPipe:

1105 case SPIRV::OpGroupCommitWritePipe:

1106 case SPIRV::OpGroupReserveReadPipePackets:

1107 case SPIRV::OpGroupReserveWritePipePackets: {

1111 MRI->setRegClass(ScopeConstReg, &SPIRV::iIDRegClass);

1113 MIB = MIRBuilder.buildInstr(Opcode);

1114

1115 if (Opcode == SPIRV::OpGroupReserveReadPipePackets ||

1116 Opcode == SPIRV::OpGroupReserveWritePipePackets)

1119

1120 MIB.addUse(ScopeConstReg);

1121 for (unsigned int i = 0; i < Call->Arguments.size(); ++i)

1123

1124 return true;

1125 }

1126 default:

1129 }

1130}

1131

1133 switch (dim) {

1134 case SPIRV::Dim::DIM_1D:

1135 case SPIRV::Dim::DIM_Buffer:

1136 return 1;

1137 case SPIRV::Dim::DIM_2D:

1138 case SPIRV::Dim::DIM_Cube:

1139 case SPIRV::Dim::DIM_Rect:

1140 return 2;

1141 case SPIRV::Dim::DIM_3D:

1142 return 3;

1143 default:

1145 }

1146}

1147

1148

1151 auto dim = static_castSPIRV::Dim::Dim\(imgType->getOperand(2).getImm());

1154 return arrayed ? numComps + 1 : numComps;

1155}

1156

1158 switch (BuiltinNumber) {

1159 case SPIRV::OpenCLExtInst::s_min:

1160 case SPIRV::OpenCLExtInst::u_min:

1161 case SPIRV::OpenCLExtInst::s_max:

1162 case SPIRV::OpenCLExtInst::u_max:

1163 case SPIRV::OpenCLExtInst::fmax:

1164 case SPIRV::OpenCLExtInst::fmin:

1165 case SPIRV::OpenCLExtInst::fmax_common:

1166 case SPIRV::OpenCLExtInst::fmin_common:

1167 case SPIRV::OpenCLExtInst::s_clamp:

1168 case SPIRV::OpenCLExtInst::fclamp:

1169 case SPIRV::OpenCLExtInst::u_clamp:

1170 case SPIRV::OpenCLExtInst::mix:

1171 case SPIRV::OpenCLExtInst:🪜

1172 case SPIRV::OpenCLExtInst::smoothstep:

1173 return true;

1174 default:

1175 break;

1176 }

1177 return false;

1178}

1179

1180

1181

1182

1183

1187

1189 unsigned ResultElementCount =

1191 bool MayNeedPromotionToVec =

1193

1194 if (!MayNeedPromotionToVec)

1195 return {Call->Arguments.begin(), Call->Arguments.end()};

1196

1201 if (ArgumentType != Call->ReturnType) {

1203 auto VecSplat = MIRBuilder.buildInstr(SPIRV::OpCompositeConstruct)

1205 .addUse(ReturnTypeId);

1206 for (unsigned I = 0; I != ResultElementCount; ++I)

1208 }

1210 }

1212}

1213

1217

1220 SPIRV::lookupExtendedBuiltin(Builtin->Name, Builtin->Set)->Number;

1221

1222

1223

1227 if (ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2) &&

1228 (Number == SPIRV::OpenCLExtInst::fmin_common ||

1229 Number == SPIRV::OpenCLExtInst::fmax_common)) {

1230 Number = (Number == SPIRV::OpenCLExtInst::fmin_common)

1231 ? SPIRV::OpenCLExtInst::fmin

1232 : SPIRV::OpenCLExtInst::fmax;

1233 }

1234

1238

1239

1240 auto MIB =

1241 MIRBuilder.buildInstr(SPIRV::OpExtInst)

1243 .addUse(ReturnTypeId)

1244 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))

1246

1249

1251 if (OrigNumber == SPIRV::OpenCLExtInst::fmin_common ||

1252 OrigNumber == SPIRV::OpenCLExtInst::fmax_common) {

1253

1256 }

1257 return true;

1258}

1259

1263

1265 unsigned Opcode =

1266 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

1267

1270 std::tie(CompareRegister, RelationType) =

1272

1273

1274 auto MIB = MIRBuilder.buildInstr(Opcode)

1275 .addDef(CompareRegister)

1277

1280

1281

1282 return buildSelectInst(MIRBuilder, Call->ReturnRegister, CompareRegister,

1283 Call->ReturnType, GR);

1284}

1285

1291 SPIRV::lookupGroupBuiltin(Builtin->Name);

1292

1294 if (Call->isSpirvOp()) {

1297 if (GroupBuiltin->Opcode ==

1298 SPIRV::OpSubgroupMatrixMultiplyAccumulateINTEL &&

1299 Call->Arguments.size() > 4)

1303 }

1304

1305

1308 if ( || MI->getOpcode() != TargetOpcode::G_CONSTANT)

1310 "Group Operation parameter must be an integer constant");

1311 uint64_t GrpOp = MI->getOperand(1).getCImm()->getValue().getZExtValue();

1318 for (unsigned i = 2; i < Call->Arguments.size(); ++i)

1320 return true;

1321 }

1322

1328 if (!BoolRegType)

1331 if (ArgInstruction->getOpcode() == TargetOpcode::G_CONSTANT) {

1332 if (BoolRegType->getOpcode() != SPIRV::OpTypeBool)

1334 BoolType, true);

1335 } else {

1336 if (BoolRegType->getOpcode() == SPIRV::OpTypeInt) {

1337 Arg0 = MRI->createGenericVirtualRegister(LLT::scalar(1));

1338 MRI->setRegClass(Arg0, &SPIRV::iIDRegClass);

1343 updateRegType(Arg0, nullptr, BoolType, GR, MIRBuilder,

1345 } else if (BoolRegType->getOpcode() != SPIRV::OpTypeBool) {

1347 }

1348

1349 }

1350 }

1351

1352 Register GroupResultRegister = Call->ReturnRegister;

1354

1355

1356

1357 const bool HasBoolReturnTy =

1361

1362 if (HasBoolReturnTy)

1363 std::tie(GroupResultRegister, GroupResultType) =

1365

1366 auto Scope = Builtin->Name.starts_with("sub_group") ? SPIRV::Scope::Subgroup

1367 : SPIRV::Scope::Workgroup;

1369

1371 if (GroupBuiltin->Opcode == SPIRV::OpGroupBroadcast &&

1372 Call->Arguments.size() > 2) {

1373

1374

1375

1376

1377

1380 if (!ElemType || ElemType->getOpcode() != SPIRV::OpTypeInt)

1382 unsigned VecLen = Call->Arguments.size() - 1;

1383 VecReg = MRI->createGenericVirtualRegister(

1385 MRI->setRegClass(VecReg, &SPIRV::vIDRegClass);

1389 auto MIB =

1390 MIRBuilder.buildInstr(TargetOpcode::G_BUILD_VECTOR).addDef(VecReg);

1391 for (unsigned i = 1; i < Call->Arguments.size(); i++) {

1392 MIB.addUse(Call->Arguments[i]);

1394 }

1395 updateRegType(VecReg, nullptr, VecType, GR, MIRBuilder,

1397 }

1398

1399

1401 .addDef(GroupResultRegister)

1403 .addUse(ScopeRegister);

1404

1407 if (Call->Arguments.size() > 0) {

1408 MIB.addUse(Arg0.isValid() ? Arg0 : Call->Arguments[0]);

1411 MIB.addUse(VecReg);

1412 else

1413 for (unsigned i = 1; i < Call->Arguments.size(); i++)

1414 MIB.addUse(Call->Arguments[i]);

1415 }

1416

1417

1418 if (HasBoolReturnTy)

1420 Call->ReturnType, GR);

1421 return true;

1422}

1423

1431 SPIRV::lookupIntelSubgroupsBuiltin(Builtin->Name);

1432

1433 if (IntelSubgroups->IsMedia &&

1434 !ST->canUseExtension(SPIRV::Extension::SPV_INTEL_media_block_io)) {

1435 std::string DiagMsg = std::string(Builtin->Name) +

1436 ": the builtin requires the following SPIR-V "

1437 "extension: SPV_INTEL_media_block_io";

1439 } else if (!IntelSubgroups->IsMedia &&

1440 !ST->canUseExtension(SPIRV::Extension::SPV_INTEL_subgroups)) {

1441 std::string DiagMsg = std::string(Builtin->Name) +

1442 ": the builtin requires the following SPIR-V "

1443 "extension: SPV_INTEL_subgroups";

1445 }

1446

1448 if (Call->isSpirvOp()) {

1449 bool IsSet = OpCode != SPIRV::OpSubgroupBlockWriteINTEL &&

1450 OpCode != SPIRV::OpSubgroupImageBlockWriteINTEL &&

1451 OpCode != SPIRV::OpSubgroupImageMediaBlockWriteINTEL;

1455 }

1456

1457 if (IntelSubgroups->IsBlock) {

1458

1460 if (Arg0Type->getOpcode() == SPIRV::OpTypeImage) {

1461

1462

1463

1464

1465 switch (OpCode) {

1466 case SPIRV::OpSubgroupBlockReadINTEL:

1467 OpCode = SPIRV::OpSubgroupImageBlockReadINTEL;

1468 break;

1469 case SPIRV::OpSubgroupBlockWriteINTEL:

1470 OpCode = SPIRV::OpSubgroupImageBlockWriteINTEL;

1471 break;

1472 }

1473 }

1474 }

1475 }

1476

1477

1478

1479

1480

1481

1482

1483

1484

1485

1486

1493 for (size_t i = 0; i < Call->Arguments.size(); ++i)

1495 return true;

1496}

1497

1504 if (!ST->canUseExtension(

1505 SPIRV::Extension::SPV_KHR_uniform_group_instructions)) {

1506 std::string DiagMsg = std::string(Builtin->Name) +

1507 ": the builtin requires the following SPIR-V "

1508 "extension: SPV_KHR_uniform_group_instructions";

1510 }

1512 SPIRV::lookupGroupUniformBuiltin(Builtin->Name);

1514

1515 Register GroupResultReg = Call->ReturnRegister;

1518

1519

1520 Register ConstGroupOpReg = Call->Arguments[1];

1522 if (!Const || Const->getOpcode() != TargetOpcode::G_CONSTANT)

1524 "expect a constant group operation for a uniform group instruction",

1525 false);

1526 const MachineOperand &ConstOperand = Const->getOperand(1);

1527 if (!ConstOperand.isCImm())

1528 report_fatal_error("uniform group instructions: group operation must be an "

1529 "integer constant",

1530 false);

1531

1533 .addDef(GroupResultReg)

1537 MIB.addUse(ValueReg);

1538

1539 return true;

1540}

1541

1548 if (!ST->canUseExtension(SPIRV::Extension::SPV_KHR_shader_clock)) {

1549 std::string DiagMsg = std::string(Builtin->Name) +

1550 ": the builtin requires the following SPIR-V "

1551 "extension: SPV_KHR_shader_clock";

1553 }

1554

1556

1557 if (Builtin->Name == "__spirv_ReadClockKHR") {

1558 MIRBuilder.buildInstr(SPIRV::OpReadClockKHR)

1562 } else {

1563

1564 SPIRV::Scope::Scope ScopeArg =

1566 .EndsWith("device", SPIRV::Scope::Scope::Device)

1567 .EndsWith("work_group", SPIRV::Scope::Scope::Workgroup)

1568 .EndsWith("sub_group", SPIRV::Scope::Scope::Subgroup);

1570

1571 MIRBuilder.buildInstr(SPIRV::OpReadClockKHR)

1575 }

1576

1577 return true;

1578}

1579

1580

1581

1582

1583

1584

1585

1586

1587

1588

1589

1590

1591

1592

1593

1594

1595

1596

1597

1598

1599

1600

1601

1602

1603

1604

1605

1609 SPIRV::BuiltIn::BuiltIn BuiltinValue,

1611 Register IndexRegister = Call->Arguments[0];

1612 const unsigned ResultWidth = Call->ReturnType->getOperand(1).getImm();

1614 const SPIRVType *PointerSizeType =

1618

1619

1621

1622

1623 bool IsConstantIndex =

1624 IndexInstruction->getOpcode() == TargetOpcode::G_CONSTANT;

1625

1626

1627

1628 if (IsConstantIndex && getIConstVal(IndexRegister, MRI) >= 3) {

1630 if (PointerSize != ResultWidth) {

1631 DefaultReg = MRI->createGenericVirtualRegister(LLT::scalar(PointerSize));

1632 MRI->setRegClass(DefaultReg, &SPIRV::iIDRegClass);

1634 MIRBuilder.getMF());

1635 ToTruncate = DefaultReg;

1636 }

1637 auto NewRegister =

1638 GR->buildConstantInt(DefaultValue, MIRBuilder, PointerSizeType, true);

1639 MIRBuilder.buildCopy(DefaultReg, NewRegister);

1640 } else {

1641 auto Vec3Ty =

1646

1648 if (!IsConstantIndex || PointerSize != ResultWidth) {

1649 Extracted = MRI->createGenericVirtualRegister(LLT::scalar(PointerSize));

1650 MRI->setRegClass(Extracted, &SPIRV::iIDRegClass);

1652 }

1653

1654

1656 Intrinsic::spv_extractelt, ArrayRef{Extracted}, true, false);

1657 ExtractInst.addUse(LoadedVector).addUse(IndexRegister);

1658

1659

1660 if (!IsConstantIndex) {

1661 updateRegType(Extracted, nullptr, PointerSizeType, GR, MIRBuilder, *MRI);

1662

1665

1668 MRI->setRegClass(CompareRegister, &SPIRV::iIDRegClass);

1670

1671

1675

1676

1677

1679 GR->buildConstantInt(DefaultValue, MIRBuilder, PointerSizeType, true);

1680

1681

1682 Register SelectionResult = Call->ReturnRegister;

1683 if (PointerSize != ResultWidth) {

1684 SelectionResult =

1685 MRI->createGenericVirtualRegister(LLT::scalar(PointerSize));

1686 MRI->setRegClass(SelectionResult, &SPIRV::iIDRegClass);

1688 MIRBuilder.getMF());

1689 }

1690

1691 MIRBuilder.buildSelect(SelectionResult, CompareRegister, Extracted,

1692 DefaultRegister);

1693 ToTruncate = SelectionResult;

1694 } else {

1695 ToTruncate = Extracted;

1696 }

1697 }

1698

1699 if (PointerSize != ResultWidth)

1701 return true;

1702}

1703

1707

1709 SPIRV::BuiltIn::BuiltIn Value =

1710 SPIRV::lookupGetBuiltin(Builtin->Name, Builtin->Set)->Value;

1711

1712 if (Value == SPIRV::BuiltIn::GlobalInvocationId)

1714

1715

1717 LLT LLType;

1718 if (Call->ReturnType->getOpcode() == SPIRV::OpTypeVector)

1719 LLType =

1721 else

1723

1725 LLType, Call->ReturnRegister);

1726}

1727

1731

1733 unsigned Opcode =

1734 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

1735

1736 switch (Opcode) {

1737 case SPIRV::OpStore:

1739 case SPIRV::OpAtomicLoad:

1741 case SPIRV::OpAtomicStore:

1743 case SPIRV::OpAtomicCompareExchange:

1744 case SPIRV::OpAtomicCompareExchangeWeak:

1746 GR);

1747 case SPIRV::OpAtomicIAdd:

1748 case SPIRV::OpAtomicISub:

1749 case SPIRV::OpAtomicOr:

1750 case SPIRV::OpAtomicXor:

1751 case SPIRV::OpAtomicAnd:

1752 case SPIRV::OpAtomicExchange:

1754 case SPIRV::OpMemoryBarrier:

1756 case SPIRV::OpAtomicFlagTestAndSet:

1757 case SPIRV::OpAtomicFlagClear:

1759 default:

1760 if (Call->isSpirvOp())

1763 return false;

1764 }

1765}

1766

1770

1772 unsigned Opcode = SPIRV::lookupAtomicFloatingBuiltin(Builtin->Name)->Opcode;

1773

1774 switch (Opcode) {

1775 case SPIRV::OpAtomicFAddEXT:

1776 case SPIRV::OpAtomicFMinEXT:

1777 case SPIRV::OpAtomicFMaxEXT:

1779 default:

1780 return false;

1781 }

1782}

1783

1787

1789 unsigned Opcode =

1790 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

1791

1793}

1794

1798

1800 unsigned Opcode =

1801 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

1802

1803 if (Opcode == SPIRV::OpGenericCastToPtrExplicit) {

1804 SPIRV::StorageClass::StorageClass ResSC =

1807 return false;

1808

1814 } else {

1815 MIRBuilder.buildInstr(TargetOpcode::G_ADDRSPACE_CAST)

1818 }

1819 return true;

1820}

1821

1826 if (Call->isSpirvOp())

1829

1831 SPIRV::OpTypeVector;

1832

1833 uint32_t OC = IsVec ? SPIRV::OpDot : SPIRV::OpFMulS;

1834 bool IsSwapReq = false;

1835

1836 const auto *ST =

1837 static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());

1839 (ST->canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product) ||

1840 ST->isAtLeastSPIRVVer(VersionTuple(1, 6)))) {

1843 SPIRV::lookupIntegerDotProductBuiltin(Builtin->Name);

1844 if (IntDot) {

1845 OC = IntDot->Opcode;

1847 } else if (IsVec) {

1848

1849

1853 bool IsFirstSigned = TypeStrs[0].trim()[0] != 'u';

1854 bool IsSecondSigned = TypeStrs[1].trim()[0] != 'u';

1855

1856 if (Call->BuiltinName == "dot") {

1857 if (IsFirstSigned && IsSecondSigned)

1858 OC = SPIRV::OpSDot;

1859 else if (!IsFirstSigned && !IsSecondSigned)

1860 OC = SPIRV::OpUDot;

1861 else {

1862 OC = SPIRV::OpSUDot;

1863 if (!IsFirstSigned)

1864 IsSwapReq = true;

1865 }

1866 } else if (Call->BuiltinName == "dot_acc_sat") {

1867 if (IsFirstSigned && IsSecondSigned)

1868 OC = SPIRV::OpSDotAccSat;

1869 else if (!IsFirstSigned && !IsSecondSigned)

1870 OC = SPIRV::OpUDotAccSat;

1871 else {

1872 OC = SPIRV::OpSUDotAccSat;

1873 if (!IsFirstSigned)

1874 IsSwapReq = true;

1875 }

1876 }

1877 }

1878 }

1879

1883

1884 if (IsSwapReq) {

1887

1888 for (size_t i = 2; i < Call->Arguments.size(); ++i)

1890 } else {

1891 for (size_t i = 0; i < Call->Arguments.size(); ++i)

1893 }

1894

1895

1896

1897 if (!IsVec && OC != SPIRV::OpFMulS)

1898 MIB.addImm(SPIRV::PackedVectorFormat4x8Bit);

1899

1900 return true;

1901}

1902

1907 SPIRV::BuiltIn::BuiltIn Value =

1908 SPIRV::lookupGetBuiltin(Builtin->Name, Builtin->Set)->Value;

1909

1910

1911 assert(Call->ReturnType->getOpcode() == SPIRV::OpTypeInt);

1913

1915 MIRBuilder, Call->ReturnType, GR, Value, LLType, Call->ReturnRegister,

1916 false, std::nullopt);

1917}

1918

1919

1920

1921

1922

1923

1924

1929 unsigned Opcode =

1930 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

1931

1935 if (!RetType)

1937 if (RetType->getOpcode() != SPIRV::OpTypeStruct)

1938 report_fatal_error("Expected struct type result for the arithmetic with "

1939 "overflow builtins");

1940

1943 if (!OpType1 || !OpType2 || OpType1 != OpType2)

1945 if (OpType1->getOpcode() == SPIRV::OpTypeVector)

1946 switch (Opcode) {

1947 case SPIRV::OpIAddCarryS:

1948 Opcode = SPIRV::OpIAddCarryV;

1949 break;

1950 case SPIRV::OpISubBorrowS:

1951 Opcode = SPIRV::OpISubBorrowV;

1952 break;

1953 }

1954

1956 Register ResReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);

1958 MRI->getRegClassOrNull(Call->Arguments[1])) {

1959 MRI->setRegClass(ResReg, DstRC);

1960 MRI->setType(ResReg, MRI->getType(Call->Arguments[1]));

1961 } else {

1963 }

1971 return true;

1972}

1973

1977

1978 SPIRV::BuiltIn::BuiltIn Value =

1979 SPIRV::lookupGetBuiltin(Call->Builtin->Name, Call->Builtin->Set)->Value;

1980 const bool IsDefaultOne = (Value == SPIRV::BuiltIn::GlobalSize ||

1981 Value == SPIRV::BuiltIn::NumWorkgroups ||

1982 Value == SPIRV::BuiltIn::WorkgroupSize ||

1983 Value == SPIRV::BuiltIn::EnqueuedWorkgroupSize);

1985}

1986

1990

1993 SPIRV::lookupImageQueryBuiltin(Builtin->Name, Builtin->Set)->Component;

1994

1995

1996

1997 unsigned NumExpectedRetComponents =

1998 Call->ReturnType->getOpcode() == SPIRV::OpTypeVector

1999 ? Call->ReturnType->getOperand(2).getImm()

2000 : 1;

2001

2004 Register QueryResult = Call->ReturnRegister;

2006 if (NumExpectedRetComponents != NumActualRetComponents) {

2007 unsigned Bitwidth = Call->ReturnType->getOpcode() == SPIRV::OpTypeInt

2008 ? Call->ReturnType->getOperand(1).getImm()

2009 : 32;

2012 MIRBuilder.getMRI()->setRegClass(QueryResult, &SPIRV::vIDRegClass);

2015 IntTy, NumActualRetComponents, MIRBuilder, true);

2017 }

2018 bool IsDimBuf = ImgType->getOperand(2).getImm() == SPIRV::Dim::DIM_Buffer;

2019 unsigned Opcode =

2020 IsDimBuf ? SPIRV::OpImageQuerySize : SPIRV::OpImageQuerySizeLod;

2021 auto MIB = MIRBuilder.buildInstr(Opcode)

2022 .addDef(QueryResult)

2025 if (!IsDimBuf)

2027 if (NumExpectedRetComponents == NumActualRetComponents)

2028 return true;

2029 if (NumExpectedRetComponents == 1) {

2030

2031 unsigned ExtractedComposite =

2032 Component == 3 ? NumActualRetComponents - 1 : Component;

2033 assert(ExtractedComposite < NumActualRetComponents &&

2034 "Invalid composite index!");

2037 if (QueryResultType->getOpcode() == SPIRV::OpTypeVector) {

2039 if (TypeReg != NewTypeReg &&

2041 TypeReg = NewTypeReg;

2042 }

2043 MIRBuilder.buildInstr(SPIRV::OpCompositeExtract)

2046 .addUse(QueryResult)

2047 .addImm(ExtractedComposite);

2048 if (NewType != nullptr)

2049 updateRegType(Call->ReturnRegister, nullptr, NewType, GR, MIRBuilder,

2051 } else {

2052

2053 auto MIB = MIRBuilder.buildInstr(SPIRV::OpVectorShuffle)

2056 .addUse(QueryResult)

2057 .addUse(QueryResult);

2058 for (unsigned i = 0; i < NumExpectedRetComponents; ++i)

2059 MIB.addImm(i < NumActualRetComponents ? i : 0xffffffff);

2060 }

2061 return true;

2062}

2063

2067 assert(Call->ReturnType->getOpcode() == SPIRV::OpTypeInt &&

2068 "Image samples query result must be of int type!");

2069

2070

2072 unsigned Opcode =

2073 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

2074

2076 SPIRV::Dim::Dim ImageDimensionality = static_castSPIRV::Dim::Dim\(

2078 (void)ImageDimensionality;

2079

2080 switch (Opcode) {

2081 case SPIRV::OpImageQuerySamples:

2082 assert(ImageDimensionality == SPIRV::Dim::DIM_2D &&

2083 "Image must be of 2D dimensionality");

2084 break;

2085 case SPIRV::OpImageQueryLevels:

2086 assert((ImageDimensionality == SPIRV::Dim::DIM_1D ||

2087 ImageDimensionality == SPIRV::Dim::DIM_2D ||

2088 ImageDimensionality == SPIRV::Dim::DIM_3D ||

2089 ImageDimensionality == SPIRV::Dim::DIM_Cube) &&

2090 "Image must be of 1D/2D/3D/Cube dimensionality");

2091 break;

2092 }

2093

2098 return true;

2099}

2100

2101

2102static SPIRV::SamplerAddressingMode::SamplerAddressingMode

2104 switch (Bitmask & SPIRV::CLK_ADDRESS_MODE_MASK) {

2105 case SPIRV::CLK_ADDRESS_CLAMP:

2106 return SPIRV::SamplerAddressingMode::Clamp;

2107 case SPIRV::CLK_ADDRESS_CLAMP_TO_EDGE:

2108 return SPIRV::SamplerAddressingMode::ClampToEdge;

2109 case SPIRV::CLK_ADDRESS_REPEAT:

2110 return SPIRV::SamplerAddressingMode::Repeat;

2111 case SPIRV::CLK_ADDRESS_MIRRORED_REPEAT:

2112 return SPIRV::SamplerAddressingMode::RepeatMirrored;

2113 case SPIRV::CLK_ADDRESS_NONE:

2114 return SPIRV::SamplerAddressingMode::None;

2115 default:

2117 }

2118}

2119

2121 return (Bitmask & SPIRV::CLK_NORMALIZED_COORDS_TRUE) ? 1 : 0;

2122}

2123

2124static SPIRV::SamplerFilterMode::SamplerFilterMode

2126 if (Bitmask & SPIRV::CLK_FILTER_LINEAR)

2127 return SPIRV::SamplerFilterMode::Linear;

2128 if (Bitmask & SPIRV::CLK_FILTER_NEAREST)

2129 return SPIRV::SamplerFilterMode::Nearest;

2130 return SPIRV::SamplerFilterMode::Nearest;

2131}

2132

2137 if (Call->isSpirvOp())

2144 if (HasOclSampler) {

2146

2147 if (!GR->isScalarOfType(Sampler, SPIRV::OpTypeSampler) &&

2154 }

2158 Register SampledImage = MRI->createVirtualRegister(&SPIRV::iIDRegClass);

2159

2160 MIRBuilder.buildInstr(SPIRV::OpSampledImage)

2161 .addDef(SampledImage)

2165

2167 MIRBuilder);

2168

2169 if (Call->ReturnType->getOpcode() != SPIRV::OpTypeVector) {

2173 MRI->createGenericVirtualRegister(GR->getRegType(TempType));

2174 MRI->setRegClass(TempRegister, GR->getRegClass(TempType));

2176 MIRBuilder.buildInstr(SPIRV::OpImageSampleExplicitLod)

2177 .addDef(TempRegister)

2179 .addUse(SampledImage)

2180 .addUse(Call->Arguments[2])

2181 .addImm(SPIRV::ImageOperand::Lod)

2183 MIRBuilder.buildInstr(SPIRV::OpCompositeExtract)

2186 .addUse(TempRegister)

2188 } else {

2189 MIRBuilder.buildInstr(SPIRV::OpImageSampleExplicitLod)

2192 .addUse(SampledImage)

2193 .addUse(Call->Arguments[2])

2194 .addImm(SPIRV::ImageOperand::Lod)

2196 }

2197 } else if (HasMsaa) {

2198 MIRBuilder.buildInstr(SPIRV::OpImageRead)

2202 .addUse(Call->Arguments[1])

2203 .addImm(SPIRV::ImageOperand::Sample)

2205 } else {

2206 MIRBuilder.buildInstr(SPIRV::OpImageRead)

2210 .addUse(Call->Arguments[1]);

2211 }

2212 return true;

2213}

2214

2227

2233 if (Call->Builtin->Name.contains_insensitive(

2234 "__translate_sampler_initializer")) {

2235

2241 return Sampler.isValid();

2242 } else if (Call->Builtin->Name.contains_insensitive("__spirv_SampledImage")) {

2243

2249 Call->ReturnRegister.isValid()

2250 ? Call->ReturnRegister

2251 : MRI->createVirtualRegister(&SPIRV::iIDRegClass);

2252 MIRBuilder.buildInstr(SPIRV::OpSampledImage)

2253 .addDef(SampledImage)

2256 .addUse(Call->Arguments[1]);

2257 return true;

2258 } else if (Call->Builtin->Name.contains_insensitive(

2259 "__spirv_ImageSampleExplicitLod")) {

2260

2261 std::string ReturnType = DemangledCall.str();

2262 if (DemangledCall.contains("_R")) {

2263 ReturnType = ReturnType.substr(ReturnType.find("_R") + 2);

2264 ReturnType = ReturnType.substr(0, ReturnType.find('('));

2265 }

2267 Call->ReturnType

2268 ? Call->ReturnType

2270 if ( Type ) {

2271 std::string DiagMsg =

2272 "Unable to recognize SPIRV type name: " + ReturnType;

2274 }

2275 MIRBuilder.buildInstr(SPIRV::OpImageSampleExplicitLod)

2278 .addUse(Call->Arguments[0])

2279 .addUse(Call->Arguments[1])

2280 .addImm(SPIRV::ImageOperand::Lod)

2282 return true;

2283 }

2284 return false;

2285}

2286

2290 Call->Arguments[1], Call->Arguments[2]);

2291 return true;

2292}

2293

2298 SPIRV::OpCompositeConstructContinuedINTEL,

2299 Call->Arguments, Call->ReturnRegister,

2301 return true;

2302}

2303

2308 unsigned Opcode =

2309 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

2310 bool IsSet = Opcode != SPIRV::OpCooperativeMatrixStoreKHR &&

2311 Opcode != SPIRV::OpCooperativeMatrixStoreCheckedINTEL &&

2312 Opcode != SPIRV::OpCooperativeMatrixPrefetchINTEL;

2313 unsigned ArgSz = Call->Arguments.size();

2314 unsigned LiteralIdx = 0;

2315 switch (Opcode) {

2316

2317 case SPIRV::OpCooperativeMatrixLoadKHR:

2318 LiteralIdx = ArgSz > 3 ? 3 : 0;

2319 break;

2320 case SPIRV::OpCooperativeMatrixStoreKHR:

2321 LiteralIdx = ArgSz > 4 ? 4 : 0;

2322 break;

2323 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:

2324 LiteralIdx = ArgSz > 7 ? 7 : 0;

2325 break;

2326 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:

2327 LiteralIdx = ArgSz > 8 ? 8 : 0;

2328 break;

2329

2330 case SPIRV::OpCooperativeMatrixMulAddKHR:

2331 LiteralIdx = ArgSz > 3 ? 3 : 0;

2332 break;

2333 };

2334

2337 if (Opcode == SPIRV::OpCooperativeMatrixPrefetchINTEL) {

2339 auto MIB = MIRBuilder.buildInstr(SPIRV::OpCooperativeMatrixPrefetchINTEL)

2340 .addUse(Call->Arguments[0])

2342 .addUse(Call->Arguments[2])

2343 .addImm(CacheLevel)

2344 .addUse(Call->Arguments[4]);

2345 if (ArgSz > 5)

2346 MIB.addUse(Call->Arguments[5]);

2347 if (ArgSz > 6) {

2349 MIB.addImm(MemOp);

2350 }

2351 return true;

2352 }

2353 if (LiteralIdx > 0)

2356 if (Opcode == SPIRV::OpCooperativeMatrixLengthKHR) {

2358 if (!CoopMatrType)

2364 return true;

2365 }

2367 IsSet ? TypeReg : Register(0), ImmArgs);

2368}

2369

2373

2375 unsigned Opcode =

2376 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

2378

2379 switch (Opcode) {

2380 case SPIRV::OpSpecConstant: {

2381

2382 unsigned SpecId =

2384 buildOpDecorate(Call->ReturnRegister, MIRBuilder, SPIRV::Decoration::SpecId,

2385 {SpecId});

2386

2387 Register ConstRegister = Call->Arguments[1];

2390 (Const->getOpcode() == TargetOpcode::G_CONSTANT ||

2391 Const->getOpcode() == TargetOpcode::G_FCONSTANT) &&

2392 "Argument should be either an int or floating-point constant");

2393

2394 const MachineOperand &ConstOperand = Const->getOperand(1);

2395 if (Call->ReturnType->getOpcode() == SPIRV::OpTypeBool) {

2396 assert(ConstOperand.isCImm() && "Int constant operand is expected");

2398 ? SPIRV::OpSpecConstantTrue

2399 : SPIRV::OpSpecConstantFalse;

2400 }

2401 auto MIB = MIRBuilder.buildInstr(Opcode)

2404

2405 if (Call->ReturnType->getOpcode() != SPIRV::OpTypeBool) {

2406 if (Const->getOpcode() == TargetOpcode::G_CONSTANT)

2408 else

2410 }

2411 return true;

2412 }

2413 case SPIRV::OpSpecConstantComposite: {

2415 SPIRV::OpSpecConstantCompositeContinuedINTEL,

2416 Call->Arguments, Call->ReturnRegister,

2418 return true;

2419 }

2420 default:

2421 return false;

2422 }

2423}

2424

2428

2430 unsigned Opcode =

2431 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

2432

2434}

2435

2439

2441 unsigned Opcode =

2442 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

2443

2445}

2446

2451 unsigned Opcode =

2452 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

2454}

2455

2463 bool IsSRet = RetTy->isVoidTy();

2464

2465 if (IsSRet) {

2466 const LLT ValTy = MRI->getType(InputReg);

2467 Register ActualRetValReg = MRI->createGenericVirtualRegister(ValTy);

2470 InputReg = Call->Arguments[1];

2473 if (InputType->getTypeID() == llvm::Type::TypeID::TypedPointerTyID) {

2474 LLT InputLLT = MRI->getType(InputReg);

2475 PtrInputReg = MRI->createGenericVirtualRegister(InputLLT);

2481 MIRBuilder.buildLoad(PtrInputReg, InputReg, *MMO1);

2482 MRI->setRegClass(PtrInputReg, &SPIRV::iIDRegClass);

2484 }

2485

2486 for (unsigned index = 2; index < 7; index++) {

2488 }

2489

2490

2491 auto MIB = MIRBuilder.buildInstr(Opcode)

2492 .addDef(ActualRetValReg)

2494 if (PtrInputReg)

2495 MIB.addUse(PtrInputReg);

2496 else

2497 MIB.addUse(InputReg);

2498

2499 for (uint32_t Imm : ImmArgs)

2501 unsigned Size = ValTy.getSizeInBytes();

2502

2505 MRI->setRegClass(ActualRetValReg, &SPIRV::pIDRegClass);

2506 MIRBuilder.buildStore(ActualRetValReg, Call->Arguments[0], *MMO);

2507 return true;

2508 } else {

2509 for (unsigned index = 1; index < 6; index++)

2511

2514 }

2515}

2516

2521 unsigned Opcode =

2522 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

2523

2525}

2526

2527static bool

2531

2533 unsigned Opcode =

2534 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

2535

2537}

2538

2542

2544 unsigned Opcode =

2545 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

2546

2548}

2549

2554 unsigned Opcode =

2555 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

2556

2557 unsigned Scope = SPIRV::Scope::Workgroup;

2559 Scope = SPIRV::Scope::Subgroup;

2560

2562}

2563

2568 unsigned Opcode =

2569 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

2570

2571 bool IsSet = Opcode != SPIRV::OpPredicatedStoreINTEL;

2572 unsigned ArgSz = Call->Arguments.size();

2575

2576 if (ArgSz > 3)

2579

2582 IsSet ? TypeReg : Register(0), ImmArgs);

2583}

2584

2595 Register TmpReg = MRI->createVirtualRegister(&SPIRV::iIDRegClass);

2597

2598

2599 unsigned NumArgs = Call->Arguments.size();

2600 assert(NumArgs >= 2);

2601 Register GlobalWorkSize = Call->Arguments[NumArgs < 4 ? 1 : 2];

2603 NumArgs == 2 ? Register(0) : Call->Arguments[NumArgs < 4 ? 2 : 3];

2604 Register GlobalWorkOffset = NumArgs <= 3 ? Register(0) : Call->Arguments[1];

2605 if (NumArgs < 4) {

2608 if (SpvTy->getOpcode() == SPIRV::OpTypePointer) {

2609 MachineInstr *DefInstr = MRI->getUniqueVRegDef(GlobalWorkSize);

2613

2614 unsigned Size = Call->Builtin->Name == "ndrange_3D" ? 3 : 2;

2619 FieldTy, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true);

2620 GlobalWorkSize = MRI->createVirtualRegister(&SPIRV::iIDRegClass);

2622 MIRBuilder.buildInstr(SPIRV::OpLoad)

2623 .addDef(GlobalWorkSize)

2629 SpvFieldTy, *ST.getInstrInfo());

2630 } else {

2632 }

2633 if (!LocalWorkSize.isValid())

2634 LocalWorkSize = Const;

2635 if (!GlobalWorkOffset.isValid())

2636 GlobalWorkOffset = Const;

2637 }

2639 MIRBuilder.buildInstr(SPIRV::OpBuildNDRange)

2642 .addUse(GlobalWorkSize)

2643 .addUse(LocalWorkSize)

2644 .addUse(GlobalWorkOffset);

2645 return MIRBuilder.buildInstr(SPIRV::OpStore)

2648}

2649

2650

2658 SPIRV::AccessQualifier::ReadWrite, true);

2659}

2660

2666 bool IsSpirvOp = Call->isSpirvOp();

2667 bool HasEvents = Call->Builtin->Name.contains("events") || IsSpirvOp;

2669

2670

2671

2672

2674 if (Call->Builtin->Name.contains("_varargs") || IsSpirvOp) {

2675 const unsigned LocalSizeArrayIdx = HasEvents ? 9 : 6;

2676 Register GepReg = Call->Arguments[LocalSizeArrayIdx];

2681 MachineInstr *ArrayMI = MRI->getUniqueVRegDef(ArrayReg);

2683 assert(LocalSizeTy && "Local size type is expected");

2684 const uint64_t LocalSizeNum =

2689 Int32Ty, MIRBuilder, SPIRV::StorageClass::Function);

2690 for (unsigned I = 0; I < LocalSizeNum; ++I) {

2691 Register Reg = MRI->createVirtualRegister(&SPIRV::pIDRegClass);

2692 MRI->setType(Reg, LLType);

2696 GEPInst

2702 }

2703 }

2704

2705

2706 auto MIB = MIRBuilder.buildInstr(SPIRV::OpEnqueueKernel)

2709

2710

2711 const unsigned BlockFIdx = HasEvents ? 6 : 3;

2712 for (unsigned i = 0; i < BlockFIdx; i++)

2713 MIB.addUse(Call->Arguments[i]);

2714

2715

2716 if (!HasEvents) {

2720 MIB.addUse(NullPtr);

2721 MIB.addUse(NullPtr);

2722 }

2723

2725 assert(BlockMI->getOpcode() == TargetOpcode::G_GLOBAL_VALUE);

2726

2728

2729 Register BlockLiteralReg = Call->Arguments[BlockFIdx + 1];

2730

2731 MIB.addUse(BlockLiteralReg);

2732

2734

2735

2737

2739 MIRBuilder, GR));

2740

2741 for (unsigned i = 0; i < LocalSizes.size(); i++)

2742 MIB.addUse(LocalSizes[i]);

2743 return true;

2744}

2745

2749

2751 unsigned Opcode =

2752 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

2753

2754 switch (Opcode) {

2755 case SPIRV::OpRetainEvent:

2756 case SPIRV::OpReleaseEvent:

2758 case SPIRV::OpCreateUserEvent:

2759 case SPIRV::OpGetDefaultQueue:

2760 return MIRBuilder.buildInstr(Opcode)

2763 case SPIRV::OpIsValidEvent:

2764 return MIRBuilder.buildInstr(Opcode)

2768 case SPIRV::OpSetUserEventStatus:

2769 return MIRBuilder.buildInstr(Opcode)

2772 case SPIRV::OpCaptureEventProfilingInfo:

2773 return MIRBuilder.buildInstr(Opcode)

2777 case SPIRV::OpBuildNDRange:

2779 case SPIRV::OpEnqueueKernel:

2781 default:

2782 return false;

2783 }

2784}

2785

2789

2791 unsigned Opcode =

2792 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

2793

2794 bool IsSet = Opcode == SPIRV::OpGroupAsyncCopy;

2796 if (Call->isSpirvOp())

2798 IsSet ? TypeReg : Register(0));

2799

2801

2802 switch (Opcode) {

2803 case SPIRV::OpGroupAsyncCopy: {

2805 Call->ReturnType->getOpcode() == SPIRV::OpTypeEvent

2806 ? nullptr

2809 unsigned NumArgs = Call->Arguments.size();

2810 Register EventReg = Call->Arguments[NumArgs - 1];

2811 bool Res = MIRBuilder.buildInstr(Opcode)

2819 ? Call->Arguments[3]

2822 if (NewType != nullptr)

2823 updateRegType(Call->ReturnRegister, nullptr, NewType, GR, MIRBuilder,

2825 return Res;

2826 }

2827 case SPIRV::OpGroupWaitEvents:

2828 return MIRBuilder.buildInstr(Opcode)

2832 default:

2833 return false;

2834 }

2835}

2836

2841

2843 SPIRV::lookupConvertBuiltin(Call->Builtin->Name, Call->Builtin->Set);

2844

2845 if (!Builtin && Call->isSpirvOp()) {

2847 unsigned Opcode =

2848 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

2851 }

2852

2853 assert(Builtin && "Conversion builtin not found.");

2856 SPIRV::Decoration::SaturatedConversion, {});

2859 SPIRV::Decoration::FPRoundingMode,

2860 {(unsigned)Builtin->RoundingMode});

2861

2862 std::string NeedExtMsg;

2863 bool IsRightComponentsNumber = true;

2864 unsigned Opcode = SPIRV::OpNop;

2866

2868

2871 : SPIRV::OpSatConvertSToU;

2872 else

2874 : SPIRV::OpSConvert;

2876 SPIRV::OpTypeFloat)) {

2877

2879 const auto *ST = static_cast<const SPIRVSubtarget *>(

2880 &MIRBuilder.getMF().getSubtarget());

2881 if (!ST->canUseExtension(

2882 SPIRV::Extension::SPV_INTEL_bfloat16_conversion))

2883 NeedExtMsg = "SPV_INTEL_bfloat16_conversion";

2884 IsRightComponentsNumber =

2887 Opcode = SPIRV::OpConvertBF16ToFINTEL;

2888 } else {

2889 bool IsSourceSigned =

2890 DemangledCall[DemangledCall.find_first_of('(') + 1] != 'u';

2891 Opcode = IsSourceSigned ? SPIRV::OpConvertSToF : SPIRV::OpConvertUToF;

2892 }

2893 }

2895 SPIRV::OpTypeFloat)) {

2896

2898

2900 const auto *ST = static_cast<const SPIRVSubtarget *>(

2901 &MIRBuilder.getMF().getSubtarget());

2902 if (!ST->canUseExtension(

2903 SPIRV::Extension::SPV_INTEL_bfloat16_conversion))

2904 NeedExtMsg = "SPV_INTEL_bfloat16_conversion";

2905 IsRightComponentsNumber =

2908 Opcode = SPIRV::OpConvertFToBF16INTEL;

2909 } else {

2911 : SPIRV::OpConvertFToU;

2912 }

2914 SPIRV::OpTypeFloat)) {

2915 if (Builtin->IsTF32) {

2916 const auto *ST = static_cast<const SPIRVSubtarget *>(

2917 &MIRBuilder.getMF().getSubtarget());

2918 if (!ST->canUseExtension(

2919 SPIRV::Extension::SPV_INTEL_tensor_float32_conversion))

2920 NeedExtMsg = "SPV_INTEL_tensor_float32_conversion";

2921 IsRightComponentsNumber =

2924 Opcode = SPIRV::OpRoundFToTF32INTEL;

2925 } else {

2926

2927 Opcode = SPIRV::OpFConvert;

2928 }

2929 }

2930 }

2931

2932 if (!NeedExtMsg.empty()) {

2933 std::string DiagMsg = std::string(Builtin->Name) +

2934 ": the builtin requires the following SPIR-V "

2935 "extension: " +

2936 NeedExtMsg;

2938 }

2939 if (!IsRightComponentsNumber) {

2940 std::string DiagMsg =

2941 std::string(Builtin->Name) +

2942 ": result and argument must have the same number of components";

2944 }

2945 assert(Opcode != SPIRV::OpNop &&

2946 "Conversion between the types not implemented!");

2947

2952 return true;

2953}

2954

2958

2960 SPIRV::lookupVectorLoadStoreBuiltin(Call->Builtin->Name,

2961 Call->Builtin->Set);

2962

2963 auto MIB =

2964 MIRBuilder.buildInstr(SPIRV::OpExtInst)

2967 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::OpenCL_std))

2973

2974

2975

2978 return true;

2979}

2980

2984

2986 unsigned Opcode =

2987 SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;

2988 bool IsLoad = Opcode == SPIRV::OpLoad;

2989

2990 auto MIB = MIRBuilder.buildInstr(Opcode);

2991 if (IsLoad) {

2992 MIB.addDef(Call->ReturnRegister);

2994 }

2995

2998

2999 if (!IsLoad)

3000 MIB.addUse(Call->Arguments[1]);

3001

3002 unsigned NumArgs = Call->Arguments.size();

3003 if ((IsLoad && NumArgs >= 2) || NumArgs >= 3)

3005 if ((IsLoad && NumArgs >= 3) || NumArgs >= 4)

3007 return true;

3008}

3009

3010namespace SPIRV {

3011

3012

3013

3014

3015

3016

3017

3018std::tuple<int, unsigned, unsigned>

3020 SPIRV::InstructionSet::InstructionSet Set) {

3023 std::unique_ptr Call =

3026 return std::make_tuple(-1, 0, 0);

3027

3028 switch (Call->Builtin->Group) {

3029 case SPIRV::Relational:

3030 case SPIRV::Atomic:

3031 case SPIRV::Barrier:

3032 case SPIRV::CastToPtr:

3033 case SPIRV::ImageMiscQuery:

3034 case SPIRV::SpecConstant:

3035 case SPIRV::Enqueue:

3036 case SPIRV::AsyncCopy:

3037 case SPIRV::LoadStore:

3038 case SPIRV::CoopMatr:

3039 if (const auto *R =

3040 SPIRV::lookupNativeBuiltin(Call->Builtin->Name, Call->Builtin->Set))

3041 return std::make_tuple(Call->Builtin->Group, R->Opcode, 0);

3042 break;

3043 case SPIRV::Extended:

3044 if (const auto *R = SPIRV::lookupExtendedBuiltin(Call->Builtin->Name,

3045 Call->Builtin->Set))

3046 return std::make_tuple(Call->Builtin->Group, 0, R->Number);

3047 break;

3048 case SPIRV::VectorLoadStore:

3049 if (const auto *R = SPIRV::lookupVectorLoadStoreBuiltin(Call->Builtin->Name,

3050 Call->Builtin->Set))

3051 return std::make_tuple(SPIRV::Extended, 0, R->Number);

3052 break;

3053 case SPIRV::Group:

3054 if (const auto *R = SPIRV::lookupGroupBuiltin(Call->Builtin->Name))

3055 return std::make_tuple(Call->Builtin->Group, R->Opcode, 0);

3056 break;

3057 case SPIRV::AtomicFloating:

3058 if (const auto *R = SPIRV::lookupAtomicFloatingBuiltin(Call->Builtin->Name))

3059 return std::make_tuple(Call->Builtin->Group, R->Opcode, 0);

3060 break;

3061 case SPIRV::IntelSubgroups:

3062 if (const auto *R = SPIRV::lookupIntelSubgroupsBuiltin(Call->Builtin->Name))

3063 return std::make_tuple(Call->Builtin->Group, R->Opcode, 0);

3064 break;

3065 case SPIRV::GroupUniform:

3066 if (const auto *R = SPIRV::lookupGroupUniformBuiltin(Call->Builtin->Name))

3067 return std::make_tuple(Call->Builtin->Group, R->Opcode, 0);

3068 break;

3069 case SPIRV::IntegerDot:

3070 if (const auto *R =

3071 SPIRV::lookupIntegerDotProductBuiltin(Call->Builtin->Name))

3072 return std::make_tuple(Call->Builtin->Group, R->Opcode, 0);

3073 break;

3074 case SPIRV::WriteImage:

3075 return std::make_tuple(Call->Builtin->Group, SPIRV::OpImageWrite, 0);

3076 case SPIRV::Select:

3077 return std::make_tuple(Call->Builtin->Group, TargetOpcode::G_SELECT, 0);

3078 case SPIRV::Construct:

3079 return std::make_tuple(Call->Builtin->Group, SPIRV::OpCompositeConstruct,

3080 0);

3081 case SPIRV::KernelClock:

3082 return std::make_tuple(Call->Builtin->Group, SPIRV::OpReadClockKHR, 0);

3083 default:

3084 return std::make_tuple(-1, 0, 0);

3085 }

3086 return std::make_tuple(-1, 0, 0);

3087}

3088

3090 SPIRV::InstructionSet::InstructionSet Set,

3092 const Register OrigRet, const Type *OrigRetTy,

3095 LLVM_DEBUG(dbgs() << "Lowering builtin call: " << DemangledCall << "\n");

3096

3097

3099 assert(SpvType && "Inconsistent return register: expected valid type info");

3100 std::unique_ptr Call =

3101 lookupBuiltin(DemangledCall, Set, OrigRet, SpvType, Args);

3102

3103 if ( Call ) {

3104 LLVM_DEBUG(dbgs() << "Builtin record was not found!\n");

3105 return std::nullopt;

3106 }

3107

3108

3109 assert(Args.size() >= Call->Builtin->MinNumArgs &&

3110 "Too few arguments to generate the builtin");

3111 if (Call->Builtin->MaxNumArgs && Args.size() > Call->Builtin->MaxNumArgs)

3112 LLVM_DEBUG(dbgs() << "More arguments provided than required!\n");

3113

3114

3115 switch (Call->Builtin->Group) {

3116 case SPIRV::Extended:

3118 case SPIRV::Relational:

3120 case SPIRV::Group:

3122 case SPIRV::Variable:

3124 case SPIRV::Atomic:

3126 case SPIRV::AtomicFloating:

3128 case SPIRV::Barrier:

3130 case SPIRV::CastToPtr:

3132 case SPIRV::Dot:

3133 case SPIRV::IntegerDot:

3135 case SPIRV::Wave:

3137 case SPIRV::ICarryBorrow:

3139 case SPIRV::GetQuery:

3141 case SPIRV::ImageSizeQuery:

3143 case SPIRV::ImageMiscQuery:

3145 case SPIRV::ReadImage:

3147 case SPIRV::WriteImage:

3149 case SPIRV::SampleImage:

3151 case SPIRV::Select:

3153 case SPIRV::Construct:

3155 case SPIRV::SpecConstant:

3157 case SPIRV::Enqueue:

3159 case SPIRV::AsyncCopy:

3161 case SPIRV::Convert:

3163 case SPIRV::VectorLoadStore:

3165 case SPIRV::LoadStore:

3167 case SPIRV::IntelSubgroups:

3169 case SPIRV::GroupUniform:

3171 case SPIRV::KernelClock:

3173 case SPIRV::CoopMatr:

3175 case SPIRV::ExtendedBitOps:

3177 case SPIRV::BindlessINTEL:

3179 case SPIRV::TernaryBitwiseINTEL:

3181 case SPIRV::Block2DLoadStore:

3183 case SPIRV::Pipe:

3185 case SPIRV::PredicatedLoadStore:

3187 case SPIRV::BlockingPipes:

3189 case SPIRV::ArbitraryPrecisionFixedPoint:

3191 }

3192 return false;

3193}

3194

3196

3198

3199

3200 [[maybe_unused]] bool IsOCLBuiltinType = TypeStr.consume_front("ocl_");

3201 assert(IsOCLBuiltinType && "Invalid OpenCL builtin prefix");

3202

3203

3204

3205

3206

3207

3210

3212 Ctx);

3213 }

3214

3215

3216

3218 unsigned VecElts = 0;

3219

3222

3223 return nullptr;

3224

3225

3227

3229 TypeStr = TypeStr.substr(0, TypeStr.find(']'));

3230

3232 if (VecElts > 0)

3235

3237}

3238

3241 auto Pos1 = DemangledCall.find('(');

3243 return false;

3244 auto Pos2 = DemangledCall.find(')');

3246 return false;

3247 DemangledCall.slice(Pos1 + 1, Pos2)

3248 .split(BuiltinArgsTypeStrs, ',', -1, false);

3249 return true;

3250}

3251

3256 if (ArgIdx >= BuiltinArgsTypeStrs.size())

3257 return nullptr;

3258 StringRef TypeStr = BuiltinArgsTypeStrs[ArgIdx].trim();

3260}

3261

3266

3267#define GET_BuiltinTypes_DECL

3268#define GET_BuiltinTypes_IMPL

3269

3274

3275#define GET_OpenCLTypes_DECL

3276#define GET_OpenCLTypes_IMPL

3277

3278#include "SPIRVGenTables.inc"

3279}

3280

3281

3282

3283

3284

3286 if (Name.starts_with("void"))

3288 else if (Name.starts_with("int") || Name.starts_with("uint"))

3290 else if (Name.starts_with("float"))

3292 else if (Name.starts_with("half"))

3295}

3296

3297

3298

3299

3300

3305 unsigned Opcode = TypeRecord->Opcode;

3306

3308}

3309

3315

3320 "Invalid number of parameters for SPIR-V pipe builtin!");

3321

3323 SPIRV::AccessQualifier::AccessQualifier(

3325}

3326

3331 "Invalid number of parameters for SPIR-V coop matrices builtin!");

3333 "SPIR-V coop matrices builtin type must have a type parameter!");

3336 SPIRV::AccessQualifier::ReadWrite, true);

3337

3339 MIRBuilder, ExtensionType, ElemType, ExtensionType->getIntParameter(0),

3342}

3343

3348 OpaqueType, SPIRV::AccessQualifier::ReadOnly, MIRBuilder);

3349

3351}

3352

3357 "Inline SPIR-V type builtin takes an opcode, size, and alignment "

3358 "parameter");

3360

3364 if (ParamEType->getName() == "spirv.IntegralConstant") {

3365 assert(ParamEType->getNumTypeParameters() == 1 &&

3366 "Inline SPIR-V integral constant builtin must have a type "

3367 "parameter");

3368 assert(ParamEType->getNumIntParameters() == 1 &&

3369 "Inline SPIR-V integral constant builtin must have a "

3370 "value parameter");

3371

3372 auto OperandValue = ParamEType->getIntParameter(0);

3373 auto *OperandType = ParamEType->getTypeParameter(0);

3374

3376 OperandType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true);

3377

3379 OperandValue, MIRBuilder, OperandSPIRVType, true)));

3380 continue;

3381 } else if (ParamEType->getName() == "spirv.Literal") {

3382 assert(ParamEType->getNumTypeParameters() == 0 &&

3383 "Inline SPIR-V literal builtin does not take type "

3384 "parameters");

3385 assert(ParamEType->getNumIntParameters() == 1 &&

3386 "Inline SPIR-V literal builtin must have an integer "

3387 "parameter");

3388

3389 auto OperandValue = ParamEType->getIntParameter(0);

3390

3392 continue;

3393 }

3394 }

3396 Param, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, true);

3398 }

3399

3401 Operands);

3402}

3403

3408 "Vulkan buffers have exactly one type for the type of the buffer.");

3410 "Vulkan buffer have 2 integer parameters: storage class and is "

3411 "writable.");

3412

3414 auto SC = static_castSPIRV::StorageClass::StorageClass\(

3418}

3419

3425

3426namespace SPIRV {

3429 StringRef NameWithParameters = TypeName;

3430

3431

3432

3433

3434 if (NameWithParameters.starts_with("opencl.")) {

3436 SPIRV::lookupOpenCLType(NameWithParameters);

3437 if (!OCLTypeRecord)

3439 NameWithParameters);

3441

3442 }

3443

3444

3445

3447 "Unknown builtin opaque type!");

3448

3449

3450

3451 if (!NameWithParameters.contains('_'))

3453

3455 unsigned BaseNameLength = NameWithParameters.find('_') - 1;

3456 SplitString(NameWithParameters.substr(BaseNameLength + 1), Parameters, "_");

3457

3459 bool HasTypeParameter = isDigit (Parameters[0][0]);

3460 if (HasTypeParameter)

3463 for (unsigned i = HasTypeParameter ? 1 : 0; i < Parameters.size(); i++) {

3464 unsigned IntParameter = 0;

3465 bool ValidLiteral = !Parameters[i].getAsInteger(10, IntParameter);

3466 (void)ValidLiteral;

3467 assert(ValidLiteral &&

3468 "Invalid format of SPIR-V builtin parameter literal!");

3469 IntParameters.push_back(IntParameter);

3470 }

3472 NameWithParameters.substr(0, BaseNameLength),

3473 TypeParameters, IntParameters);

3474}

3475

3477 SPIRV::AccessQualifier::AccessQualifier AccessQual,

3480

3481

3482

3483

3484

3485

3486

3487

3492

3494

3496 LLVM_DEBUG(dbgs() << "Lowering builtin type: " << Name << "\n");

3497

3499 if (Name == "spirv.Type") {

3501 } else if (Name == "spirv.VulkanBuffer") {

3503 } else if (Name == "spirv.Padding") {

3505 } else if (Name == "spirv.Layout") {

3507 } else {

3508

3509 const SPIRV::BuiltinType *TypeRecord = SPIRV::lookupBuiltinType(Name);

3510 if (!TypeRecord)

3511 report_fatal_error("Missing TableGen record for builtin type: " + Name);

3512

3513

3514

3515

3516

3517

3518

3519 switch (TypeRecord->Opcode) {

3520 case SPIRV::OpTypeImage:

3522 break;

3523 case SPIRV::OpTypePipe:

3525 break;

3526 case SPIRV::OpTypeDeviceEvent:

3528 break;

3529 case SPIRV::OpTypeSampler:

3531 break;

3532 case SPIRV::OpTypeSampledImage:

3534 break;

3535 case SPIRV::OpTypeCooperativeMatrixKHR:

3537 break;

3538 default:

3539 TargetType =

3541 break;

3542 }

3543 }

3544

3545

3546

3549

3550 return TargetType;

3551}

3552}

3553}

unsigned const MachineRegisterInfo * MRI

MachineInstrBuilder MachineInstrBuilder & DefMI

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

AMDGPU Lower Kernel Arguments

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

Promote Memory to Register

BaseType

A given derived pointer can have multiple base pointers through phi/selects.

static const fltSemantics & IEEEsingle()

APInt bitcastToAPInt() const

static APFloat getZero(const fltSemantics &Sem, bool Negative=false)

Factory for Positive and Negative Zero.

static APInt getAllOnes(unsigned numBits)

Return an APInt of a specified width with all bits set.

uint64_t getZExtValue() const

Get zero extended value.

This class represents an incoming formal argument to a Function.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)

This static method is the primary way to construct an ArrayType.

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

@ ICMP_ULT

unsigned less than

const APFloat & getValueAPF() const

const APInt & getValue() const

Return the constant as an APInt value reference.

A parsed version of the target data layout string in and methods for querying it.

Tagged union holding either a T or a Error.

Class to represent fixed width SIMD vectors.

LLVMContext & getContext() const

getContext - Return a reference to the LLVMContext associated with this function.

static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)

This static method is the primary way of constructing an IntegerType.

static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)

Get a low-level vector of some number of elements and element width.

static constexpr LLT scalar(unsigned SizeInBits)

Get a low-level scalar or aggregate "bag of bits".

static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)

Get a low-level pointer in the given address space.

static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)

Get a low-level fixed-width vector of some number of elements and element width.

constexpr TypeSize getSizeInBytes() const

Returns the total size of the type in bytes, i.e.

This is an important class for using LLVM in a threaded context.

static MCOperand createReg(MCRegister Reg)

static MCOperand createImm(int64_t Val)

const TargetSubtargetInfo & getSubtarget() const

getSubtarget - Return the subtarget for which this machine code is being compiled.

MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)

getMachineMemOperand - Allocate a new MachineMemOperand.

MachineRegisterInfo & getRegInfo()

getRegInfo - Return information about the registers currently in use.

Function & getFunction()

Return the LLVM function that this machine code represents.

Helper class to build MachineInstr.

LLVMContext & getContext() const

MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)

Build and insert a Res = G_SELECT Tst, Op0, Op1.

MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)

Build and insert a Res = G_ICMP Pred, Op0, Op1.

MachineBasicBlock::iterator getInsertPt()

Current insertion point for new instructions.

MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)

Build and insert a G_INTRINSIC instruction.

MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)

Build and insert Res = G_LOAD Addr, MMO.

MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)

Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...

MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)

Build and insert G_STORE Val, Addr, MMO.

MachineInstrBuilder buildInstr(unsigned Opcode)

Build and insert = Opcode .

MachineFunction & getMF()

Getter for the function we currently build.

MachineRegisterInfo * getMRI()

Getter for MRI.

MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)

Build and insert Res = COPY Op.

const DataLayout & getDataLayout() const

virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)

Build and insert Res = G_CONSTANT Val.

Get the register for the operand index.

const MachineInstrBuilder & addImm(int64_t Val) const

Add a new immediate operand.

const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const

Add a virtual register use operand.

MachineInstr * getInstr() const

If conversion operators fail, use this method to get the MachineInstr explicitly.

const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const

Add a virtual register definition operand.

Representation of each machine instruction.

unsigned getOpcode() const

Returns the opcode of this MachineInstr.

LLVM_ABI void copyIRFlags(const Instruction &I)

Copy all flags to MachineInst MIFlags.

const MachineOperand & getOperand(unsigned i) const

A description of a memory reference used in the backend.

@ MOLoad

The memory access reads data.

@ MOStore

The memory access writes data.

MachineOperand class - Representation of each machine instruction operand.

const GlobalValue * getGlobal() const

const ConstantInt * getCImm() const

bool isCImm() const

isCImm - Test if this is a MO_CImmediate operand.

bool isReg() const

isReg - Tests if this is a MO_Register operand.

const MDNode * getMetadata() const

getReg - Returns the register number.

const ConstantFP * getFPImm() const

MachineRegisterInfo - Keep track of information for virtual and physical registers,...

LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")

createVirtualRegister - Create and return a new virtual register in the function with the specified r...

LLVM_ABI void setType(Register VReg, LLT Ty)

Set the low-level type of VReg to Ty.

LLVM_ABI void setRegClass(Register Reg, const TargetRegisterClass *RC)

setRegClass - Set the register class of the specified virtual register.

LLVM_ABI Register createGenericVirtualRegister(LLT Ty, StringRef Name="")

Create and return a new generic virtual register with low-level type Ty.

unsigned getNumVirtRegs() const

getNumVirtRegs - Return the number of virtual registers created.

static LLVM_ABI PointerType * get(Type *ElementType, unsigned AddressSpace)

This constructs a pointer to an object of the specified type in a numbered address space.

Wrapper class representing virtual and physical registers.

constexpr bool isValid() const

SPIRVType * getOrCreateOpTypePipe(MachineIRBuilder &MIRBuilder, SPIRV::AccessQualifier::AccessQualifier AccQual)

SPIRVType * getSPIRVTypeForVReg(Register VReg, const MachineFunction *MF=nullptr) const

SPIRVType * getOrCreateSPIRVBoolType(MachineIRBuilder &MIRBuilder, bool EmitIR)

SPIRVType * getOrCreatePaddingType(MachineIRBuilder &MIRBuilder)

void assignSPIRVTypeToVReg(SPIRVType *Type, Register VReg, const MachineFunction &MF)

Register buildGlobalVariable(Register Reg, SPIRVType *BaseType, StringRef Name, const GlobalValue *GV, SPIRV::StorageClass::StorageClass Storage, const MachineInstr *Init, bool IsConst, const std::optional< SPIRV::LinkageType::LinkageType > &LinkageType, MachineIRBuilder &MIRBuilder, bool IsInstSelector)

const Type * getTypeForSPIRVType(const SPIRVType *Ty) const

SPIRVType * getOrCreateUnknownType(const Type *Ty, MachineIRBuilder &MIRBuilder, unsigned Opcode, const ArrayRef< MCOperand > Operands)

unsigned getScalarOrVectorComponentCount(Register VReg) const

SPIRVType * getOrCreateSPIRVType(const Type *Type, MachineInstr &I, SPIRV::AccessQualifier::AccessQualifier AQ, bool EmitIR)

unsigned getPointerSize() const

SPIRVType * getOrCreateSPIRVPointerType(const Type *BaseType, MachineIRBuilder &MIRBuilder, SPIRV::StorageClass::StorageClass SC)

SPIRVType * getOrCreateOpTypeByOpcode(const Type *Ty, MachineIRBuilder &MIRBuilder, unsigned Opcode)

SPIRVType * getPointeeType(SPIRVType *PtrType)

bool isScalarOfType(Register VReg, unsigned TypeOpcode) const

SPIRVType * getOrCreateOpTypeSampledImage(SPIRVType *ImageType, MachineIRBuilder &MIRBuilder)

SPIRVType * getOrCreateVulkanBufferType(MachineIRBuilder &MIRBuilder, Type *ElemType, SPIRV::StorageClass::StorageClass SC, bool IsWritable, bool EmitIr=false)

SPIRVType * getOrCreateSPIRVTypeByName(StringRef TypeStr, MachineIRBuilder &MIRBuilder, bool EmitIR, SPIRV::StorageClass::StorageClass SC=SPIRV::StorageClass::Function, SPIRV::AccessQualifier::AccessQualifier AQ=SPIRV::AccessQualifier::ReadWrite)

SPIRVType * getOrCreateLayoutType(MachineIRBuilder &MIRBuilder, const TargetExtType *T, bool EmitIr=false)

const TargetRegisterClass * getRegClass(SPIRVType *SpvType) const

SPIRVType * getOrCreateSPIRVVectorType(SPIRVType *BaseType, unsigned NumElements, MachineIRBuilder &MIRBuilder, bool EmitIR)

SPIRVType * getOrCreateOpTypeCoopMatr(MachineIRBuilder &MIRBuilder, const TargetExtType *ExtensionType, const SPIRVType *ElemType, uint32_t Scope, uint32_t Rows, uint32_t Columns, uint32_t Use, bool EmitIR)

bool isScalarOrVectorOfType(Register VReg, unsigned TypeOpcode) const

Register getOrCreateConstIntArray(uint64_t Val, size_t Num, MachineInstr &I, SPIRVType *SpvType, const SPIRVInstrInfo &TII)

SPIRVType * getOrCreateOpTypeDeviceEvent(MachineIRBuilder &MIRBuilder)

SPIRVType * getImageType(const TargetExtType *ExtensionType, const SPIRV::AccessQualifier::AccessQualifier Qualifier, MachineIRBuilder &MIRBuilder)

SPIRVType * getOrCreateSPIRVIntegerType(unsigned BitWidth, MachineIRBuilder &MIRBuilder)

Register buildConstantInt(uint64_t Val, MachineIRBuilder &MIRBuilder, SPIRVType *SpvType, bool EmitIR, bool ZeroAsNull=true)

LLT getRegType(SPIRVType *SpvType) const

SPIRV::StorageClass::StorageClass getPointerStorageClass(Register VReg) const

SPIRVType * getOrCreateOpTypeSampler(MachineIRBuilder &MIRBuilder)

Register buildConstantSampler(Register Res, unsigned AddrMode, unsigned Param, unsigned FilerMode, MachineIRBuilder &MIRBuilder)

unsigned getScalarOrVectorBitWidth(const SPIRVType *Type) const

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

std::pair< StringRef, StringRef > split(char Separator) const

Split into two substrings around the first occurrence of a separator character.

static constexpr size_t npos

bool consume_back(StringRef Suffix)

Returns true if this StringRef has the given suffix and removes that suffix.

bool getAsInteger(unsigned Radix, T &Result) const

Parse the current string as an integer of the specified radix.

std::string str() const

str - Get the contents as an std::string.

constexpr StringRef substr(size_t Start, size_t N=npos) const

Return a reference to the substring from [Start, Start + N).

bool starts_with(StringRef Prefix) const

Check if this string starts with the given Prefix.

bool contains_insensitive(StringRef Other) const

Return true if the given string is a substring of *this, and false otherwise.

StringRef slice(size_t Start, size_t End) const

Return a reference to the substring from [Start, End).

constexpr size_t size() const

size - Get the string size.

bool contains(StringRef Other) const

Return true if the given string is a substring of *this, and false otherwise.

bool consume_front(StringRef Prefix)

Returns true if this StringRef has the given prefix and removes that prefix.

size_t find_first_of(char C, size_t From=0) const

Find the first character in the string that is C, or npos if not found.

size_t rfind(char C, size_t From=npos) const

Search for the last character C in the string.

size_t find(char C, size_t From=0) const

Search for the first character C in the string.

bool ends_with(StringRef Suffix) const

Check if this string ends with the given Suffix.

A switch()-like statement whose cases are string literals.

StringSwitch & EndsWith(StringLiteral S, T Value)

Class to represent struct types.

Class to represent target extensions types, which are generally unintrospectable from target-independ...

ArrayRef< Type * > type_params() const

Return the type parameters for this particular target extension type.

unsigned getNumIntParameters() const

static LLVM_ABI TargetExtType * get(LLVMContext &Context, StringRef Name, ArrayRef< Type * > Types={}, ArrayRef< unsigned > Ints={})

Return a target extension type having the specified name and optional type and integer parameters.

Type * getTypeParameter(unsigned i) const

unsigned getNumTypeParameters() const

unsigned getIntParameter(unsigned i) const

The instances of the Type class are immutable: once they are created, they are never changed.

static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)

LLVM_ABI StringRef getStructName() const

static LLVM_ABI Type * getVoidTy(LLVMContext &C)

static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)

static LLVM_ABI Type * getFloatTy(LLVMContext &C)

static LLVM_ABI Type * getHalfTy(LLVMContext &C)

bool isVoidTy() const

Return true if this is 'void'.

LLVM Value Representation.

LLVM_ABI Value(Type *Ty, unsigned scid)

static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)

This static method is the primary way to construct an VectorType.

Represents a version number in the form major[.minor[.subminor[.build]]].

NodeTy * getNextNode()

Get the next node, or nullptr for the list tail.

LLVM_C_ABI LLVMTypeRef LLVMVectorType(LLVMTypeRef ElementType, unsigned ElementCount)

Create a vector type that contains a defined type and has a specific number of elements.

std::string lookupBuiltinNameHelper(StringRef DemangledCall, FPDecorationId *DecorationId)

Parses the name part of the demangled builtin call.

Definition SPIRVBuiltins.cpp:186

Type * parseBuiltinCallArgumentType(StringRef TypeStr, LLVMContext &Ctx)

Definition SPIRVBuiltins.cpp:3195

bool parseBuiltinTypeStr(SmallVector< StringRef, 10 > &BuiltinArgsTypeStrs, const StringRef DemangledCall, LLVMContext &Ctx)

Definition SPIRVBuiltins.cpp:3239

std::optional< bool > lowerBuiltin(const StringRef DemangledCall, SPIRV::InstructionSet::InstructionSet Set, MachineIRBuilder &MIRBuilder, const Register OrigRet, const Type *OrigRetTy, const SmallVectorImpl< Register > &Args, SPIRVGlobalRegistry *GR, const CallBase &CB)

Definition SPIRVBuiltins.cpp:3089

std::tuple< int, unsigned, unsigned > mapBuiltinToOpcode(const StringRef DemangledCall, SPIRV::InstructionSet::InstructionSet Set)

Helper function for finding a builtin function attributes by a demangled function name.

Definition SPIRVBuiltins.cpp:3019

Type * parseBuiltinCallArgumentBaseType(const StringRef DemangledCall, unsigned ArgIdx, LLVMContext &Ctx)

Parses the provided ArgIdx argument base type in the DemangledCall skeleton.

Definition SPIRVBuiltins.cpp:3252

TargetExtType * parseBuiltinTypeNameToTargetExtType(std::string TypeName, LLVMContext &Context)

Translates a string representing a SPIR-V or OpenCL builtin type to a TargetExtType that can be furth...

Definition SPIRVBuiltins.cpp:3427

SPIRVType * lowerBuiltinType(const Type *OpaqueType, SPIRV::AccessQualifier::AccessQualifier AccessQual, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:3476

This is an optimization pass for GlobalISel generic memory operations.

void buildOpName(Register Target, const StringRef &Name, MachineIRBuilder &MIRBuilder)

static bool build2DBlockIOINTELInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Helper function for building Intel's 2d block io instructions.

Definition SPIRVBuiltins.cpp:1080

static SPIRVType * getVulkanBufferType(const TargetExtType *ExtensionType, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:3404

static bool generateExtInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR, const CallBase &CB)

Definition SPIRVBuiltins.cpp:1214

static bool generateBindlessImageINTELInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2436

static bool generateGetQueryInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:1974

static bool generateLoadStoreInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2981

static SPIRVType * getInlineSpirvType(const TargetExtType *ExtensionType, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:3353

static bool generateConstructInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2294

static bool buildAtomicFlagInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Helper function for building atomic flag instructions (e.g.

Definition SPIRVBuiltins.cpp:898

static bool generateImageSizeQueryInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:1987

static SPIRV::SamplerFilterMode::SamplerFilterMode getSamplerFilterModeFromBitmask(unsigned Bitmask)

Definition SPIRVBuiltins.cpp:2125

static bool buildAtomicStoreInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Helper function for building an atomic store instruction.

Definition SPIRVBuiltins.cpp:696

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

FunctionAddr VTableAddr uintptr_t uintptr_t Int32Ty

void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB)

static bool buildExtendedBitOpsInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Helper function for building extended bit operations.

Definition SPIRVBuiltins.cpp:1012

static const Type * getBlockStructType(Register ParamReg, MachineRegisterInfo *MRI)

Definition SPIRVBuiltins.cpp:420

static bool generateGroupInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:1286

FPDecorationId demangledPostfixToDecorationId(const std::string &S)

void updateRegType(Register Reg, Type *Ty, SPIRVType *SpirvTy, SPIRVGlobalRegistry *GR, MachineIRBuilder &MIB, MachineRegisterInfo &MRI)

Helper external function for assigning SPIRVType to a register, ensuring the register class and type ...

static unsigned getNumComponentsForDim(SPIRV::Dim::Dim dim)

Definition SPIRVBuiltins.cpp:1132

static bool builtinMayNeedPromotionToVec(uint32_t BuiltinNumber)

Definition SPIRVBuiltins.cpp:1157

static bool generateICarryBorrowInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:1925

static Register buildScopeReg(Register CLScopeRegister, SPIRV::Scope::Scope Scope, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI)

Definition SPIRVBuiltins.cpp:584

static std::tuple< Register, SPIRVType * > buildBoolRegister(MachineIRBuilder &MIRBuilder, const SPIRVType *ResultType, SPIRVGlobalRegistry *GR)

Helper function building either a resulting scalar or vector bool register depending on the expected ...

Definition SPIRVBuiltins.cpp:444

static unsigned getNumSizeComponents(SPIRVType *imgType)

Helper function for obtaining the number of size components.

Definition SPIRVBuiltins.cpp:1149

uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI)

static Register buildConstantIntReg32(uint64_t Val, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:577

static SPIRVType * getSampledImageType(const TargetExtType *OpaqueType, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:3344

SmallVector< MachineInstr *, 4 > createContinuedInstructions(MachineIRBuilder &MIRBuilder, unsigned Opcode, unsigned MinWC, unsigned ContinuedOpcode, ArrayRef< Register > Args, Register ReturnRegister, Register TypeID)

SPIRV::MemorySemantics::MemorySemantics getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC)

constexpr unsigned storageClassToAddressSpace(SPIRV::StorageClass::StorageClass SC)

static bool generateSampleImageInst(const StringRef DemangledCall, const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2228

static bool generateBarrierInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:1784

LLVM_ABI void SplitString(StringRef Source, SmallVectorImpl< StringRef > &OutFragments, StringRef Delimiters=" \t\n\v\f\r")

SplitString - Split up the specified string according to the specified delimiters,...

static SPIRVType * getCoopMatrType(const TargetExtType *ExtensionType, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:3327

static bool generateKernelClockInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:1542

static void setRegClassIfNull(Register Reg, MachineRegisterInfo *MRI, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:602

static bool generateGroupUniformInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:1498

static bool generateWaveInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:1903

void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder, SPIRV::Decoration::Decoration Dec, const std::vector< uint32_t > &DecArgs, StringRef StrImm)

Register createVirtualRegister(SPIRVType *SpvType, SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI, const MachineFunction &MF)

MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)

static bool buildBarrierInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Helper function for building barriers, i.e., memory/control ordering operations.

Definition SPIRVBuiltins.cpp:935

static bool generateAsyncCopy(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2786

static SPIRV::Scope::Scope getSPIRVScope(SPIRV::CLMemoryScope ClScope)

Definition SPIRVBuiltins.cpp:561

static bool buildAPFixedPointInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2456

static SPIRVType * getSamplerType(MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:3310

static bool generateBlockingPipesInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2447

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

static Register buildLoadInst(SPIRVType *BaseType, Register PtrRegister, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR, LLT LowLevelType, Register DestinationReg=Register(0))

Helper function for building a load instruction loading into the DestinationReg.

Definition SPIRVBuiltins.cpp:492

static bool generateEnqueueInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2746

LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)

static bool buildSelectInst(MachineIRBuilder &MIRBuilder, Register ReturnRegister, Register SourceRegister, const SPIRVType *ReturnType, SPIRVGlobalRegistry *GR)

Helper function for building either a vector or scalar select instruction depending on the expected R...

Definition SPIRVBuiltins.cpp:469

static const Type * getMachineInstrType(MachineInstr *MI)

Definition SPIRVBuiltins.cpp:403

bool isDigit(char C)

Checks if character C is one of the 10 decimal digits.

static SPIRV::SamplerAddressingMode::SamplerAddressingMode getSamplerAddressingModeFromBitmask(unsigned Bitmask)

Definition SPIRVBuiltins.cpp:2103

static bool generateAtomicInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:1728

const MachineInstr SPIRVType

static SPIRVType * getLayoutType(const TargetExtType *ExtensionType, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:3420

static bool generateDotOrFMulInst(const StringRef DemangledCall, const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:1822

static Register buildBuiltinVariableLoad(MachineIRBuilder &MIRBuilder, SPIRVType *VariableType, SPIRVGlobalRegistry *GR, SPIRV::BuiltIn::BuiltIn BuiltinValue, LLT LLType, Register Reg=Register(0), bool isConst=true, const std::optional< SPIRV::LinkageType::LinkageType > &LinkageTy={ SPIRV::LinkageType::Import})

Helper function for building a load instruction for loading a builtin global variable of BuiltinValue...

Definition SPIRVBuiltins.cpp:507

static bool generateConvertInst(const StringRef DemangledCall, const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2837

static bool generateTernaryBitwiseFunctionINTELInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2528

static bool generateCastToPtrInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:1795

constexpr bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC)

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

static Register buildMemSemanticsReg(Register SemanticsRegister, Register PtrRegister, unsigned &Semantics, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:611

static unsigned getConstFromIntrinsic(Register Reg, MachineRegisterInfo *MRI)

Definition SPIRVBuiltins.cpp:394

static bool generateImageMiscQueryInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2064

static bool generateSelectInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder)

Definition SPIRVBuiltins.cpp:2287

static bool buildAtomicLoadInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Helper function for building an atomic load instruction.

Definition SPIRVBuiltins.cpp:660

static bool generateIntelSubgroupsInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:1424

static bool generateExtendedBitOpsInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2425

static bool buildPipeInst(const SPIRV::IncomingCall *Call, unsigned Opcode, unsigned Scope, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:1097

static bool generateSpecConstantInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2370

static SPIRVType * getOrCreateSPIRVDeviceEventPointer(MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2652

Type * parseBasicTypeName(StringRef &TypeName, LLVMContext &Ctx)

static bool generateVectorLoadStoreInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2955

static bool genWorkgroupQuery(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR, SPIRV::BuiltIn::BuiltIn BuiltinValue, uint64_t DefaultValue)

Definition SPIRVBuiltins.cpp:1606

static bool generateCoopMatrInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2304

static SmallVector< Register > getBuiltinCallArguments(const SPIRV::IncomingCall *Call, uint32_t BuiltinNumber, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:1185

static bool buildBindlessImageINTELInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Helper function for building Intel's bindless image instructions.

Definition SPIRVBuiltins.cpp:1044

static std::unique_ptr< const SPIRV::IncomingCall > lookupBuiltin(StringRef DemangledCall, SPIRV::InstructionSet::InstructionSet Set, Register ReturnRegister, const SPIRVType *ReturnType, const SmallVectorImpl< Register > &Arguments)

Looks up the demangled builtin call in the SPIRVBuiltins.td records using the provided DemangledCall ...

Definition SPIRVBuiltins.cpp:275

static bool buildAtomicFloatingRMWInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Helper function for building an atomic floating-type instruction.

Definition SPIRVBuiltins.cpp:876

MachineInstr * getDefInstrMaybeConstant(Register &ConstReg, const MachineRegisterInfo *MRI)

constexpr unsigned BitWidth

OutputIt move(R &&Range, OutputIt Out)

Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.

static bool generate2DBlockIOINTELInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2539

static bool generateReadImageInst(const StringRef DemangledCall, const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2133

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

bool hasBuiltinTypePrefix(StringRef Name)

static bool buildEnqueueKernel(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2661

Type * getMDOperandAsType(const MDNode *N, unsigned I)

static bool generatePipeInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2550

static bool buildTernaryBitwiseFunctionINTELInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Helper function for building Intel's OpBitwiseFunctionINTEL instruction.

Definition SPIRVBuiltins.cpp:1062

static bool generateAPFixedPointInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2517

static bool buildAtomicRMWInst(const SPIRV::IncomingCall *Call, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Helper function for building atomic instructions.

Definition SPIRVBuiltins.cpp:820

static SPIRV::MemorySemantics::MemorySemantics getSPIRVMemSemantics(std::memory_order MemOrder)

Definition SPIRVBuiltins.cpp:544

static bool generateRelationalInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:1260

static bool buildAtomicInitInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder)

Helper function for translating atomic init to OpStore.

Definition SPIRVBuiltins.cpp:646

static bool generateWriteImageInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2215

static SPIRVType * getPipeType(const TargetExtType *ExtensionType, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:3316

static Type * parseTypeString(const StringRef Name, LLVMContext &Context)

Definition SPIRVBuiltins.cpp:3285

bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID)

static bool generatePredicatedLoadStoreInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2564

static bool generateAtomicFloatingInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:1767

static bool buildNDRange(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:2585

static SPIRVType * getNonParameterizedType(const TargetExtType *ExtensionType, const SPIRV::BuiltinType *TypeRecord, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:3301

static MachineInstr * getBlockStructInstr(Register ParamReg, MachineRegisterInfo *MRI)

Definition SPIRVBuiltins.cpp:372

static bool buildOpFromWrapper(MachineIRBuilder &MIRBuilder, unsigned Opcode, const SPIRV::IncomingCall *Call, Register TypeReg, ArrayRef< uint32_t > ImmArgs={})

Definition SPIRVBuiltins.cpp:630

static unsigned getSamplerParamFromBitmask(unsigned Bitmask)

Definition SPIRVBuiltins.cpp:2120

static bool buildAtomicCompareExchangeInst(const SPIRV::IncomingCall *Call, const SPIRV::DemangledBuiltin *Builtin, unsigned Opcode, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Helper function for building an atomic compare-exchange instruction.

Definition SPIRVBuiltins.cpp:719

std::string getLinkStringForBuiltIn(SPIRV::BuiltIn::BuiltIn BuiltInValue)

static bool generateBuiltinVar(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR)

Definition SPIRVBuiltins.cpp:1704

Implement std::hash so that hash_code can be used in STL containers.

This struct is a compact representation of a valid (non-zero power of two) alignment.

This class contains a discriminated union of information about pointers in memory operands,...

Definition SPIRVBuiltins.cpp:100

StringRef Name

Definition SPIRVBuiltins.cpp:101

uint32_t Opcode

Definition SPIRVBuiltins.cpp:102

Definition SPIRVBuiltins.cpp:3262

StringRef Name

Definition SPIRVBuiltins.cpp:3263

uint32_t Opcode

Definition SPIRVBuiltins.cpp:3264

Definition SPIRVBuiltins.cpp:144

bool IsSaturated

Definition SPIRVBuiltins.cpp:148

FPRoundingMode::FPRoundingMode RoundingMode

Definition SPIRVBuiltins.cpp:152

bool IsTF32

Definition SPIRVBuiltins.cpp:151

bool IsDestinationSigned

Definition SPIRVBuiltins.cpp:147

bool IsRounded

Definition SPIRVBuiltins.cpp:149

StringRef Name

Definition SPIRVBuiltins.cpp:145

bool IsBfloat16

Definition SPIRVBuiltins.cpp:150

InstructionSet::InstructionSet Set

Definition SPIRVBuiltins.cpp:146

Definition SPIRVBuiltins.cpp:32

InstructionSet::InstructionSet Set

Definition SPIRVBuiltins.cpp:34

uint8_t MaxNumArgs

Definition SPIRVBuiltins.cpp:37

uint8_t MinNumArgs

Definition SPIRVBuiltins.cpp:36

StringRef Name

Definition SPIRVBuiltins.cpp:33

BuiltinGroup Group

Definition SPIRVBuiltins.cpp:35

Definition SPIRVBuiltins.cpp:116

InstructionSet::InstructionSet Set

Definition SPIRVBuiltins.cpp:118

StringRef Name

Definition SPIRVBuiltins.cpp:117

BuiltIn::BuiltIn Value

Definition SPIRVBuiltins.cpp:119

Definition SPIRVBuiltins.cpp:70

bool IsBallotFindBit

Definition SPIRVBuiltins.cpp:80

bool IsBallot

Definition SPIRVBuiltins.cpp:77

StringRef Name

Definition SPIRVBuiltins.cpp:71

bool HasBoolArg

Definition SPIRVBuiltins.cpp:83

bool IsAllEqual

Definition SPIRVBuiltins.cpp:76

bool IsAllOrAny

Definition SPIRVBuiltins.cpp:75

bool IsBallotBitExtract

Definition SPIRVBuiltins.cpp:79

bool NoGroupOperation

Definition SPIRVBuiltins.cpp:82

bool IsInverseBallot

Definition SPIRVBuiltins.cpp:78

bool IsLogical

Definition SPIRVBuiltins.cpp:81

uint32_t Opcode

Definition SPIRVBuiltins.cpp:72

bool IsElect

Definition SPIRVBuiltins.cpp:74

uint32_t GroupOperation

Definition SPIRVBuiltins.cpp:73

Definition SPIRVBuiltins.cpp:107

uint32_t Opcode

Definition SPIRVBuiltins.cpp:109

StringRef Name

Definition SPIRVBuiltins.cpp:108

bool IsLogical

Definition SPIRVBuiltins.cpp:110

Definition SPIRVBuiltins.cpp:126

StringRef Name

Definition SPIRVBuiltins.cpp:127

uint32_t Component

Definition SPIRVBuiltins.cpp:129

InstructionSet::InstructionSet Set

Definition SPIRVBuiltins.cpp:128

Definition SPIRVBuiltins.cpp:43

const SmallVectorImpl< Register > & Arguments

Definition SPIRVBuiltins.cpp:49

bool isSpirvOp() const

Definition SPIRVBuiltins.cpp:58

const std::string BuiltinName

Definition SPIRVBuiltins.cpp:44

const SPIRVType * ReturnType

Definition SPIRVBuiltins.cpp:48

const Register ReturnRegister

Definition SPIRVBuiltins.cpp:47

const DemangledBuiltin * Builtin

Definition SPIRVBuiltins.cpp:45

IncomingCall(const std::string BuiltinName, const DemangledBuiltin *Builtin, const Register ReturnRegister, const SPIRVType *ReturnType, const SmallVectorImpl< Register > &Arguments)

Definition SPIRVBuiltins.cpp:51

Definition SPIRVBuiltins.cpp:135

uint32_t Opcode

Definition SPIRVBuiltins.cpp:137

bool IsSwapReq

Definition SPIRVBuiltins.cpp:138

StringRef Name

Definition SPIRVBuiltins.cpp:136

Definition SPIRVBuiltins.cpp:89

uint32_t Opcode

Definition SPIRVBuiltins.cpp:91

bool IsWrite