LLVM: lib/Target/SPIRV/SPIRVModuleAnalysis.cpp Source File (original) (raw)
63 const auto &Op = MdNode->getOperand(OpIndex);
65 }
67}
68
70getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
73
75 if (.isShader())
76 AvoidCaps.S.insert(SPIRV::Capability::Shader);
77 else
78 AvoidCaps.S.insert(SPIRV::Capability::Kernel);
79
83 bool MinVerOK = SPIRVVersion.empty() || SPIRVVersion >= ReqMinVer;
84 bool MaxVerOK =
85 ReqMaxVer.empty() || SPIRVVersion.empty() || SPIRVVersion <= ReqMaxVer;
88 if (ReqCaps.empty()) {
89 if (ReqExts.empty()) {
90 if (MinVerOK && MaxVerOK)
91 return {true, {}, {}, ReqMinVer, ReqMaxVer};
93 }
94 } else if (MinVerOK && MaxVerOK) {
95 if (ReqCaps.size() == 1) {
96 auto Cap = ReqCaps[0];
99 SPIRV::OperandCategory::CapabilityOperand, Cap));
100 return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
101 }
102 } else {
103
104
105
106
107
108
110 for (auto Cap : ReqCaps)
113 for (size_t i = 0, Sz = UseCaps.size(); i < Sz; ++i) {
114 auto Cap = UseCaps[i];
115 if (i == Sz - 1 || !AvoidCaps.S.contains(Cap)) {
117 SPIRV::OperandCategory::CapabilityOperand, Cap));
118 return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
119 }
120 }
121 }
122 }
123
124
125
126 if (llvm::all_of(ReqExts, [&ST](const SPIRV::Extension::Extension &Ext) {
127 return ST.canUseExtension(Ext);
128 })) {
129 return {true,
130 {},
131 std::move(ReqExts),
133 VersionTuple()};
134 }
136}
137
138void SPIRVModuleAnalysis::setBaseInfo(const Module &M) {
139 MAI.MaxID = 0;
141 MAI.MS[i].clear();
142 MAI.RegisterAliasTable.clear();
143 MAI.InstrsToDelete.clear();
144 MAI.FuncMap.clear();
145 MAI.GlobalVarList.clear();
146 MAI.ExtInstSetMap.clear();
147 MAI.Reqs.clear();
148 MAI.Reqs.initAvailableCapabilities(*ST);
149
150
151 if (auto MemModel = M.getNamedMetadata("spirv.MemoryModel")) {
152 auto MemMD = MemModel->getOperand(0);
153 MAI.Addr = static_castSPIRV::AddressingModel::AddressingModel\(
154 getMetadataUInt(MemMD, 0));
155 MAI.Mem =
156 static_castSPIRV::MemoryModel::MemoryModel\(getMetadataUInt(MemMD, 1));
157 } else {
158
159 MAI.Mem = ST->isShader() ? SPIRV::MemoryModel::GLSL450
160 : SPIRV::MemoryModel::OpenCL;
161 if (MAI.Mem == SPIRV::MemoryModel::OpenCL) {
162 unsigned PtrSize = ST->getPointerSize();
163 MAI.Addr = PtrSize == 32 ? SPIRV::AddressingModel::Physical32
164 : PtrSize == 64 ? SPIRV::AddressingModel::Physical64
165 : SPIRV::AddressingModel::Logical;
166 } else {
167
168 MAI.Addr = SPIRV::AddressingModel::Logical;
169 }
170 }
171
172
173 if (auto VerNode = M.getNamedMetadata("opencl.ocl.version")) {
174 MAI.SrcLang = SPIRV::SourceLanguage::OpenCL_C;
175
176
177 assert(VerNode->getNumOperands() > 0 && "Invalid SPIR");
178 auto VersionMD = VerNode->getOperand(0);
179 unsigned MajorNum = getMetadataUInt(VersionMD, 0, 2);
180 unsigned MinorNum = getMetadataUInt(VersionMD, 1);
181 unsigned RevNum = getMetadataUInt(VersionMD, 2);
182
183 MAI.SrcLangVersion =
184 (std::max(1U, MajorNum) * 100 + MinorNum) * 1000 + RevNum;
185 } else {
186
187
188
189
190 if (!ST->isShader()) {
191 MAI.SrcLang = SPIRV::SourceLanguage::OpenCL_CPP;
192 MAI.SrcLangVersion = 100000;
193 } else {
194 MAI.SrcLang = SPIRV::SourceLanguage::Unknown;
195 MAI.SrcLangVersion = 0;
196 }
197 }
198
199 if (auto ExtNode = M.getNamedMetadata("opencl.used.extensions")) {
200 for (unsigned I = 0, E = ExtNode->getNumOperands(); I != E; ++I) {
201 MDNode *MD = ExtNode->getOperand(I);
203 continue;
206 }
207 }
208
209
210
211 MAI.Reqs.getAndAddRequirements(SPIRV::OperandCategory::MemoryModelOperand,
212 MAI.Mem, *ST);
213 MAI.Reqs.getAndAddRequirements(SPIRV::OperandCategory::SourceLanguageOperand,
214 MAI.SrcLang, *ST);
215 MAI.Reqs.getAndAddRequirements(SPIRV::OperandCategory::AddressingModelOperand,
216 MAI.Addr, *ST);
217
218 if (!ST->isShader()) {
219
220 MAI.ExtInstSetMap[static_cast<unsigned>(
221 SPIRV::InstructionSet::OpenCL_std)] = MAI.getNextIDRegister();
222 }
223}
224
225
226
230
231
232 if (UseMI.getOpcode() != SPIRV::OpDecorate &&
233 UseMI.getOpcode() != SPIRV::OpMemberDecorate)
234 continue;
235
236 for (unsigned I = 0; I < UseMI.getNumOperands(); ++I) {
239 continue;
241 }
242 }
243}
244
245
246
247
248
251 bool UseDefReg) {
254 for (unsigned i = 0; i < MI.getNumOperands(); ++i) {
255
256
257
258
259
260
261
262
263 unsigned Opcode = MI.getOpcode();
264 if ((Opcode == SPIRV::OpDecorate) && i >= 2) {
265 unsigned DecorationID = MI.getOperand(1).getImm();
266 if (DecorationID != SPIRV::Decoration::FuncParamAttr &&
267 DecorationID != SPIRV::Decoration::UserSemantic &&
268 DecorationID != SPIRV::Decoration::CacheControlLoadINTEL &&
269 DecorationID != SPIRV::Decoration::CacheControlStoreINTEL)
270 continue;
271 }
273 size_t h;
274 if (MO.isReg()) {
275 if (!UseDefReg && MO.isDef()) {
276 assert(!DefReg.isValid() && "Multiple def registers.");
277 DefReg = MO.getReg();
278 continue;
279 }
281 if (!RegAlias.isValid()) {
283 dbgs() << "Unexpectedly, no global id found for the operand ";
285 dbgs() << "\nInstruction: ";
287 dbgs() << "\n";
288 });
289 report_fatal_error("All v-regs must have been mapped to global id's");
290 }
291
294 } else {
296 }
298 }
299
301
302
303
304 appendDecorationsForReg(MI.getMF()->getRegInfo(), DefReg, Signature);
305 }
306 return Signature;
307}
308
311 unsigned Opcode = MI.getOpcode();
312 switch (Opcode) {
313 case SPIRV::OpTypeForwardPointer:
314
315 return false;
316 case SPIRV::OpVariable:
317 return static_castSPIRV::StorageClass::StorageClass\(
318 MI.getOperand(2).getImm()) != SPIRV::StorageClass::Function;
319 case SPIRV::OpFunction:
320 case SPIRV::OpFunctionParameter:
321 return true;
322 }
323 if (GR->hasConstFunPtr() && Opcode == SPIRV::OpUndef) {
324 Register DefReg = MI.getOperand(0).getReg();
325 for (MachineInstr &UseMI : MRI.use_instructions(DefReg)) {
326 if (UseMI.getOpcode() != SPIRV::OpConstantFunctionPointerINTEL)
327 continue;
328
329
333 return false;
334 }
335 }
336 return TII->isTypeDeclInstr(MI) || TII->isConstantInstr(MI) ||
337 TII->isInlineAsmDefInstr(MI);
338}
339
340
341
342
343void SPIRVModuleAnalysis::visitFunPtrUse(
345 std::map<const Value *, unsigned> &GlobalToGReg, const MachineFunction *MF,
347 const MachineOperand *OpFunDef =
348 GR->getFunctionDefinitionByUse(&MI.getOperand(2));
350
351 const MachineInstr *OpDefMI = OpFunDef->getParent();
352 assert(OpDefMI && OpDefMI->getOpcode() == SPIRV::OpFunction);
354 const MachineRegisterInfo &FunDefMRI = FunDefMF->getRegInfo();
355 do {
356 visitDecl(FunDefMRI, SignatureToGReg, GlobalToGReg, FunDefMF, *OpDefMI);
358 } while (OpDefMI && (OpDefMI->getOpcode() == SPIRV::OpFunction ||
359 OpDefMI->getOpcode() == SPIRV::OpFunctionParameter));
360
361 MCRegister GlobalFunDefReg =
362 MAI.getRegisterAlias(FunDefMF, OpFunDef->getReg());
364 "Function definition must refer to a global register");
365 MAI.setRegisterAlias(MF, OpReg, GlobalFunDefReg);
366}
367
368
369
370void SPIRVModuleAnalysis::visitDecl(
372 std::map<const Value *, unsigned> &GlobalToGReg, const MachineFunction *MF,
374 unsigned Opcode = MI.getOpcode();
375
376
377 for (const MachineOperand &MO : MI.operands()) {
379 continue;
381
382 if (Opcode == SPIRV::OpConstantFunctionPointerINTEL &&
383 MRI.getRegClass(OpReg) == &SPIRV::pIDRegClass) {
384 visitFunPtrUse(OpReg, SignatureToGReg, GlobalToGReg, MF, MI);
385 continue;
386 }
387
388 if (MAI.hasRegisterAlias(MF, MO.getReg()))
389 continue;
390
391 if (const MachineInstr *OpDefMI = MRI.getUniqueVRegDef(OpReg)) {
392 if (isDeclSection(MRI, *OpDefMI))
393 visitDecl(MRI, SignatureToGReg, GlobalToGReg, MF, *OpDefMI);
394 continue;
395 }
396
397
399 dbgs() << "Unexpectedly, no unique definition for the operand ";
401 dbgs() << "\nInstruction: ";
403 dbgs() << "\n";
404 });
406 "No unique definition is found for the virtual register");
407 }
408
409 MCRegister GReg;
410 bool IsFunDef = false;
411 if (TII->isSpecConstantInstr(MI)) {
412 GReg = MAI.getNextIDRegister();
414 } else if (Opcode == SPIRV::OpFunction ||
415 Opcode == SPIRV::OpFunctionParameter) {
416 GReg = handleFunctionOrParameter(MF, MI, GlobalToGReg, IsFunDef);
417 } else if (Opcode == SPIRV::OpTypeStruct ||
418 Opcode == SPIRV::OpConstantComposite) {
419 GReg = handleTypeDeclOrConstant(MI, SignatureToGReg);
420 const MachineInstr *NextInstr = MI.getNextNode();
421 while (NextInstr &&
422 ((Opcode == SPIRV::OpTypeStruct &&
423 NextInstr->getOpcode() == SPIRV::OpTypeStructContinuedINTEL) ||
424 (Opcode == SPIRV::OpConstantComposite &&
426 SPIRV::OpConstantCompositeContinuedINTEL))) {
427 MCRegister Tmp = handleTypeDeclOrConstant(*NextInstr, SignatureToGReg);
429 MAI.setSkipEmission(NextInstr);
431 }
432 } else if (TII->isTypeDeclInstr(MI) || TII->isConstantInstr(MI) ||
433 TII->isInlineAsmDefInstr(MI)) {
434 GReg = handleTypeDeclOrConstant(MI, SignatureToGReg);
435 } else if (Opcode == SPIRV::OpVariable) {
436 GReg = handleVariable(MF, MI, GlobalToGReg);
437 } else {
439 dbgs() << "\nInstruction: ";
441 dbgs() << "\n";
442 });
444 }
445 MAI.setRegisterAlias(MF, MI.getOperand(0).getReg(), GReg);
446 if (!IsFunDef)
448}
449
450MCRegister SPIRVModuleAnalysis::handleFunctionOrParameter(
452 std::map<const Value *, unsigned> &GlobalToGReg, bool &IsFunDef) {
453 const Value *GObj = GR->getGlobalObject(MF, MI.getOperand(0).getReg());
454 assert(GObj && "Unregistered global definition");
456 if ()
458 assert(F && "Expected a reference to a function or an argument");
459 IsFunDef = ->isDeclaration();
460 auto [It, Inserted] = GlobalToGReg.try_emplace(GObj);
461 if (!Inserted)
462 return It->second;
463 MCRegister GReg = MAI.getNextIDRegister();
464 It->second = GReg;
465 if (!IsFunDef)
467 return GReg;
468}
469
471SPIRVModuleAnalysis::handleTypeDeclOrConstant(const MachineInstr &MI,
474 auto [It, Inserted] = SignatureToGReg.try_emplace(MISign);
475 if (!Inserted)
476 return It->second;
477 MCRegister GReg = MAI.getNextIDRegister();
478 It->second = GReg;
480 return GReg;
481}
482
483MCRegister SPIRVModuleAnalysis::handleVariable(
485 std::map<const Value *, unsigned> &GlobalToGReg) {
486 MAI.GlobalVarList.push_back(&MI);
487 const Value *GObj = GR->getGlobalObject(MF, MI.getOperand(0).getReg());
488 assert(GObj && "Unregistered global definition");
489 auto [It, Inserted] = GlobalToGReg.try_emplace(GObj);
490 if (!Inserted)
491 return It->second;
492 MCRegister GReg = MAI.getNextIDRegister();
493 It->second = GReg;
495 return GReg;
496}
497
498void SPIRVModuleAnalysis::collectDeclarations(const Module &M) {
500 std::map<const Value *, unsigned> GlobalToGReg;
501 for (const Function &F : M) {
502 MachineFunction *MF = MMI->getMachineFunction(F);
503 if (!MF)
504 continue;
505 const MachineRegisterInfo &MRI = MF->getRegInfo();
506 unsigned PastHeader = 0;
507 for (MachineBasicBlock &MBB : *MF) {
508 for (MachineInstr &MI : MBB) {
509 if (MI.getNumOperands() == 0)
510 continue;
511 unsigned Opcode = MI.getOpcode();
512 if (Opcode == SPIRV::OpFunction) {
513 if (PastHeader == 0) {
514 PastHeader = 1;
515 continue;
516 }
517 } else if (Opcode == SPIRV::OpFunctionParameter) {
518 if (PastHeader < 2)
519 continue;
520 } else if (PastHeader > 0) {
521 PastHeader = 2;
522 }
523
524 const MachineOperand &DefMO = MI.getOperand(0);
525 switch (Opcode) {
526 case SPIRV::OpExtension:
527 MAI.Reqs.addExtension(SPIRV::Extension::Extension(DefMO.getImm()));
529 break;
530 case SPIRV::OpCapability:
531 MAI.Reqs.addCapability(SPIRV::Capability::Capability(DefMO.getImm()));
533 if (PastHeader > 0)
534 PastHeader = 2;
535 break;
536 default:
537 if (DefMO.isReg() && isDeclSection(MRI, MI) &&
538 .hasRegisterAlias(MF, DefMO.getReg()))
539 visitDecl(MRI, SignatureToGReg, GlobalToGReg, MF, MI);
540 }
541 }
542 }
543 }
544}
545
546
547
548
549
550void SPIRVModuleAnalysis::collectFuncNames(MachineInstr &MI,
552 if (MI.getOpcode() == SPIRV::OpDecorate) {
553
554 auto Dec = MI.getOperand(1).getImm();
555 if (Dec == SPIRV::Decoration::LinkageAttributes) {
556 auto Lnk = MI.getOperand(MI.getNumOperands() - 1).getImm();
557 if (Lnk == SPIRV::LinkageType::Import) {
558
559 const Function *ImportedFunc =
562 MAI.FuncMap[ImportedFunc] = MAI.getRegisterAlias(MI.getMF(), Target);
563 }
564 }
565 } else if (MI.getOpcode() == SPIRV::OpFunction) {
566
568 MCRegister GlobalReg = MAI.getRegisterAlias(MI.getMF(), Reg);
570 MAI.FuncMap[F] = GlobalReg;
571 }
572}
573
574
575
576
579 bool Append = true) {
582 auto FoundMI = IS.insert(std::move(MISign));
583 if (!FoundMI.second) {
584 if (MI.getOpcode() == SPIRV::OpDecorate) {
585 assert(MI.getNumOperands() >= 2 &&
586 "Decoration instructions must have at least 2 operands");
588 "Only OpDecorate instructions can be duplicates");
589
590
591
592
593 if (MI.getOperand(1).getImm() != SPIRV::Decoration::FPFastMathMode)
594 return;
595
597 for (const MachineInstr *OrigMI : Decorations) {
598 if (instrToSignature(*OrigMI, MAI, true) == MISign) {
599 assert(OrigMI->getNumOperands() == MI.getNumOperands() &&
600 "Original instruction must have the same number of operands");
602 OrigMI->getNumOperands() == 3 &&
603 "FPFastMathMode decoration must have 3 operands for OpDecorate");
604 unsigned OrigFlags = OrigMI->getOperand(2).getImm();
605 unsigned NewFlags = MI.getOperand(2).getImm();
606 if (OrigFlags == NewFlags)
607 return;
608
609
610 unsigned FinalFlags = OrigFlags | NewFlags;
612 << "Warning: Conflicting FPFastMathMode decoration flags "
613 "in instruction: "
614 << *OrigMI << "Original flags: " << OrigFlags
615 << ", new flags: " << NewFlags
616 << ". They will be merged on a best effort basis, but not "
617 "validated. Final flags: "
618 << FinalFlags << "\n";
622 return;
623 }
624 }
625 assert(false && "No original instruction found for the duplicate "
626 "OpDecorate, but we found one in IS.");
627 }
628 return;
629 }
630
631 if (Append)
633 else
635}
636
637
638
639void SPIRVModuleAnalysis::processOtherInstrs(const Module &M) {
641 for (const Function &F : M) {
642 if (F.isDeclaration())
643 continue;
644 MachineFunction *MF = MMI->getMachineFunction(F);
646
647 for (MachineBasicBlock &MBB : *MF)
648 for (MachineInstr &MI : MBB) {
649 if (MAI.getSkipEmission(&MI))
650 continue;
651 const unsigned OpCode = MI.getOpcode();
652 if (OpCode == SPIRV::OpString) {
654 } else if (OpCode == SPIRV::OpExtInst && MI.getOperand(2).isImm() &&
655 MI.getOperand(2).getImm() ==
656 SPIRV::InstructionSet::
657 NonSemantic_Shader_DebugInfo_100) {
658 MachineOperand Ins = MI.getOperand(3);
659 namespace NS = SPIRV::NonSemanticExtInst;
660 static constexpr int64_t GlobalNonSemanticDITy[] = {
661 NS::DebugSource, NS::DebugCompilationUnit, NS::DebugInfoNone,
662 NS::DebugTypeBasic, NS::DebugTypePointer};
663 bool IsGlobalDI = false;
664 for (unsigned Idx = 0; Idx < std::size(GlobalNonSemanticDITy); ++Idx)
665 IsGlobalDI |= Ins.getImm() == GlobalNonSemanticDITy[Idx];
666 if (IsGlobalDI)
668 } else if (OpCode == SPIRV::OpName || OpCode == SPIRV::OpMemberName) {
670 } else if (OpCode == SPIRV::OpEntryPoint) {
672 } else if (TII->isAliasingInstr(MI)) {
674 } else if (TII->isDecorationInstr(MI)) {
677 } else if (TII->isConstantInstr(MI)) {
678
679
681 } else if (OpCode == SPIRV::OpFunction) {
683 } else if (OpCode == SPIRV::OpTypeForwardPointer) {
685 }
686 }
687 }
688}
689
690
691
692
693void SPIRVModuleAnalysis::numberRegistersGlobally(const Module &M) {
694 for (const Function &F : M) {
695 if (F.isDeclaration())
696 continue;
697 MachineFunction *MF = MMI->getMachineFunction(F);
699 for (MachineBasicBlock &MBB : *MF) {
700 for (MachineInstr &MI : MBB) {
701 for (MachineOperand &Op : MI.operands()) {
702 if (.isReg())
703 continue;
705 if (MAI.hasRegisterAlias(MF, Reg))
706 continue;
707 MCRegister NewReg = MAI.getNextIDRegister();
708 MAI.setRegisterAlias(MF, Reg, NewReg);
709 }
710 if (MI.getOpcode() != SPIRV::OpExtInst)
711 continue;
712 auto Set = MI.getOperand(2).getImm();
713 auto [It, Inserted] = MAI.ExtInstSetMap.try_emplace(Set);
714 if (Inserted)
715 It->second = MAI.getNextIDRegister();
716 }
717 }
718 }
719}
720
721
723 SPIRV::OperandCategory::OperandCategory Category, uint32_t i,
725 addRequirements(getSymbolicOperandRequirements(Category, i, ST, *this));
726}
727
728void SPIRV::RequirementHandler::recursiveAddCapabilities(
730 for (const auto &Cap : ToPrune) {
731 AllCaps.insert(Cap);
734 recursiveAddCapabilities(ImplicitDecls);
735 }
736}
737
739 for (const auto &Cap : ToAdd) {
740 bool IsNewlyInserted = AllCaps.insert(Cap).second;
741 if (!IsNewlyInserted)
742 continue;
745 recursiveAddCapabilities(ImplicitDecls);
746 MinimalCaps.push_back(Cap);
747 }
748}
749
751 const SPIRV::Requirements &Req) {
753 report_fatal_error("Adding SPIR-V requirements this target can't satisfy.");
754
755 if (Req.Cap.has_value())
756 addCapabilities({Req.Cap.value()});
757
758 addExtensions(Req.Exts);
759
761 if (!MaxVersion.empty() && Req.MinVer > MaxVersion) {
763 << " and <= " << MaxVersion << "\n");
764 report_fatal_error("Adding SPIR-V requirements that can't be satisfied.");
765 }
766
767 if (MinVersion.empty() || Req.MinVer > MinVersion)
768 MinVersion = Req.MinVer;
769 }
770
772 if (!MinVersion.empty() && Req.MaxVer < MinVersion) {
774 << " and >= " << MinVersion << "\n");
775 report_fatal_error("Adding SPIR-V requirements that can't be satisfied.");
776 }
777
778 if (MaxVersion.empty() || Req.MaxVer < MaxVersion)
779 MaxVersion = Req.MaxVer;
780 }
781}
782
784 const SPIRVSubtarget &ST) const {
785
786 bool IsSatisfiable = true;
787 auto TargetVer = ST.getSPIRVVersion();
788
789 if (!MaxVersion.empty() && !TargetVer.empty() && MaxVersion < TargetVer) {
791 dbgs() << "Target SPIR-V version too high for required features\n"
792 << "Required max version: " << MaxVersion << " target version "
793 << TargetVer << "\n");
794 IsSatisfiable = false;
795 }
796
797 if (!MinVersion.empty() && !TargetVer.empty() && MinVersion > TargetVer) {
798 LLVM_DEBUG(dbgs() << "Target SPIR-V version too low for required features\n"
799 << "Required min version: " << MinVersion
800 << " target version " << TargetVer << "\n");
801 IsSatisfiable = false;
802 }
803
804 if (!MinVersion.empty() && !MaxVersion.empty() && MinVersion > MaxVersion) {
807 << "Version is too low for some features and too high for others.\n"
808 << "Required SPIR-V min version: " << MinVersion
809 << " required SPIR-V max version " << MaxVersion << "\n");
810 IsSatisfiable = false;
811 }
812
813 AvoidCapabilitiesSet AvoidCaps;
814 if (.isShader())
815 AvoidCaps.S.insert(SPIRV::Capability::Shader);
816 else
817 AvoidCaps.S.insert(SPIRV::Capability::Kernel);
818
819 for (auto Cap : MinimalCaps) {
820 if (AvailableCaps.contains(Cap) && !AvoidCaps.S.contains(Cap))
821 continue;
824 OperandCategory::CapabilityOperand, Cap)
825 << "\n");
826 IsSatisfiable = false;
827 }
828
829 for (auto Ext : AllExtensions) {
830 if (ST.canUseExtension(Ext))
831 continue;
834 OperandCategory::ExtensionOperand, Ext)
835 << "\n");
836 IsSatisfiable = false;
837 }
838
839 if (!IsSatisfiable)
840 report_fatal_error("Unable to meet SPIR-V requirements for this target.");
841}
842
843
845 for (const auto Cap : ToAdd)
846 if (AvailableCaps.insert(Cap).second)
848 SPIRV::OperandCategory::CapabilityOperand, Cap));
849}
850
852 const Capability::Capability ToRemove,
853 const Capability::Capability IfPresent) {
854 if (AllCaps.contains(IfPresent))
856}
857
858namespace llvm {
861
862 addAvailableCaps({Capability::Shader, Capability::Linkage, Capability::Int8,
863 Capability::Int16});
864
865 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 3)))
867 Capability::GroupNonUniformVote,
868 Capability::GroupNonUniformArithmetic,
869 Capability::GroupNonUniformBallot,
870 Capability::GroupNonUniformClustered,
871 Capability::GroupNonUniformShuffle,
872 Capability::GroupNonUniformShuffleRelative});
873
874 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 6)))
875 addAvailableCaps({Capability::DotProduct, Capability::DotProductInputAll,
876 Capability::DotProductInput4x8Bit,
877 Capability::DotProductInput4x8BitPacked,
878 Capability::DemoteToHelperInvocation});
879
880
881 for (auto Extension : ST.getAllAvailableExtensions()) {
885 }
886
887 if (.isShader()) {
888 initAvailableCapabilitiesForOpenCL(ST);
889 return;
890 }
891
892 if (ST.isShader()) {
893 initAvailableCapabilitiesForVulkan(ST);
894 return;
895 }
896
897 report_fatal_error("Unimplemented environment for SPIR-V generation.");
898}
899
900void RequirementHandler::initAvailableCapabilitiesForOpenCL(
901 const SPIRVSubtarget &ST) {
902
903 addAvailableCaps({Capability::Addresses, Capability::Float16Buffer,
904 Capability::Kernel, Capability::Vector16,
905 Capability::Groups, Capability::GenericPointer,
906 Capability::StorageImageWriteWithoutFormat,
907 Capability::StorageImageReadWithoutFormat});
908 if (ST.hasOpenCLFullProfile())
909 addAvailableCaps({Capability::Int64, Capability::Int64Atomics});
910 if (ST.hasOpenCLImageSupport()) {
911 addAvailableCaps({Capability::ImageBasic, Capability::LiteralSampler,
912 Capability::Image1D, Capability::SampledBuffer,
913 Capability::ImageBuffer});
914 if (ST.isAtLeastOpenCLVer(VersionTuple(2, 0)))
916 }
917 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 1)) &&
918 ST.isAtLeastOpenCLVer(VersionTuple(2, 2)))
919 addAvailableCaps({Capability::SubgroupDispatch, Capability::PipeStorage});
920 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 4)))
921 addAvailableCaps({Capability::DenormPreserve, Capability::DenormFlushToZero,
922 Capability::SignedZeroInfNanPreserve,
923 Capability::RoundingModeRTE,
924 Capability::RoundingModeRTZ});
925
926 addAvailableCaps({Capability::Float16, Capability::Float64});
927
928
929}
930
931void RequirementHandler::initAvailableCapabilitiesForVulkan(
932 const SPIRVSubtarget &ST) {
933
934
935 addAvailableCaps({Capability::Int64, Capability::Float16, Capability::Float64,
936 Capability::GroupNonUniform, Capability::Image1D,
937 Capability::SampledBuffer, Capability::ImageBuffer,
938 Capability::UniformBufferArrayDynamicIndexing,
939 Capability::SampledImageArrayDynamicIndexing,
940 Capability::StorageBufferArrayDynamicIndexing,
941 Capability::StorageImageArrayDynamicIndexing,
942 Capability::DerivativeControl});
943
944
945 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 5))) {
947 {Capability::ShaderNonUniformEXT, Capability::RuntimeDescriptorArrayEXT,
948 Capability::InputAttachmentArrayDynamicIndexingEXT,
949 Capability::UniformTexelBufferArrayDynamicIndexingEXT,
950 Capability::StorageTexelBufferArrayDynamicIndexingEXT,
951 Capability::UniformBufferArrayNonUniformIndexingEXT,
952 Capability::SampledImageArrayNonUniformIndexingEXT,
953 Capability::StorageBufferArrayNonUniformIndexingEXT,
954 Capability::StorageImageArrayNonUniformIndexingEXT,
955 Capability::InputAttachmentArrayNonUniformIndexingEXT,
956 Capability::UniformTexelBufferArrayNonUniformIndexingEXT,
957 Capability::StorageTexelBufferArrayNonUniformIndexingEXT});
958 }
959
960
961 if (ST.isAtLeastSPIRVVer(VersionTuple(1, 6)))
962 addAvailableCaps({Capability::StorageImageWriteWithoutFormat,
963 Capability::StorageImageReadWithoutFormat});
964}
965
966}
967}
968
969
970
971static void addOpDecorateReqs(const MachineInstr &MI, unsigned DecIndex,
974 int64_t DecOp = MI.getOperand(DecIndex).getImm();
975 auto Dec = static_castSPIRV::Decoration::Decoration\(DecOp);
977 SPIRV::OperandCategory::DecorationOperand, Dec, ST, Reqs));
978
979 if (Dec == SPIRV::Decoration::BuiltIn) {
980 int64_t BuiltInOp = MI.getOperand(DecIndex + 1).getImm();
981 auto BuiltIn = static_castSPIRV::BuiltIn::BuiltIn\(BuiltInOp);
983 SPIRV::OperandCategory::BuiltInOperand, BuiltIn, ST, Reqs));
984 } else if (Dec == SPIRV::Decoration::LinkageAttributes) {
985 int64_t LinkageOp = MI.getOperand(MI.getNumOperands() - 1).getImm();
986 SPIRV::LinkageType::LinkageType LnkType =
987 static_castSPIRV::LinkageType::LinkageType\(LinkageOp);
988 if (LnkType == SPIRV::LinkageType::LinkOnceODR)
989 Reqs.addExtension(SPIRV::Extension::SPV_KHR_linkonce_odr);
990 } else if (Dec == SPIRV::Decoration::CacheControlLoadINTEL ||
991 Dec == SPIRV::Decoration::CacheControlStoreINTEL) {
992 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_cache_controls);
993 } else if (Dec == SPIRV::Decoration::HostAccessINTEL) {
994 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_global_variable_host_access);
995 } else if (Dec == SPIRV::Decoration::InitModeINTEL ||
996 Dec == SPIRV::Decoration::ImplementInRegisterMapINTEL) {
998 SPIRV::Extension::SPV_INTEL_global_variable_fpga_decorations);
999 } else if (Dec == SPIRV::Decoration::NonUniformEXT) {
1000 Reqs.addRequirements(SPIRV::Capability::ShaderNonUniformEXT);
1001 } else if (Dec == SPIRV::Decoration::FPMaxErrorDecorationINTEL) {
1002 Reqs.addRequirements(SPIRV::Capability::FPMaxErrorINTEL);
1003 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_fp_max_error);
1004 } else if (Dec == SPIRV::Decoration::FPFastMathMode) {
1005 if (ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2)) {
1006 Reqs.addRequirements(SPIRV::Capability::FloatControls2);
1007 Reqs.addExtension(SPIRV::Extension::SPV_KHR_float_controls2);
1008 }
1009 }
1010}
1011
1012
1013static void addOpTypeImageReqs(const MachineInstr &MI,
1016 assert(MI.getNumOperands() >= 8 && "Insufficient operands for OpTypeImage");
1017
1018
1019 int64_t ImgFormatOp = MI.getOperand(7).getImm();
1020 auto ImgFormat = static_castSPIRV::ImageFormat::ImageFormat\(ImgFormatOp);
1022 ImgFormat, ST);
1023
1024 bool IsArrayed = MI.getOperand(4).getImm() == 1;
1025 bool IsMultisampled = MI.getOperand(5).getImm() == 1;
1026 bool NoSampler = MI.getOperand(6).getImm() == 2;
1027
1028 assert(MI.getOperand(2).isImm());
1029 switch (MI.getOperand(2).getImm()) {
1030 case SPIRV::Dim::DIM_1D:
1031 Reqs.addRequirements(NoSampler ? SPIRV::Capability::Image1D
1032 : SPIRV::Capability::Sampled1D);
1033 break;
1034 case SPIRV::Dim::DIM_2D:
1035 if (IsMultisampled && NoSampler)
1037 break;
1038 case SPIRV::Dim::DIM_Cube:
1040 if (IsArrayed)
1041 Reqs.addRequirements(NoSampler ? SPIRV::Capability::ImageCubeArray
1042 : SPIRV::Capability::SampledCubeArray);
1043 break;
1044 case SPIRV::Dim::DIM_Rect:
1045 Reqs.addRequirements(NoSampler ? SPIRV::Capability::ImageRect
1046 : SPIRV::Capability::SampledRect);
1047 break;
1048 case SPIRV::Dim::DIM_Buffer:
1049 Reqs.addRequirements(NoSampler ? SPIRV::Capability::ImageBuffer
1050 : SPIRV::Capability::SampledBuffer);
1051 break;
1052 case SPIRV::Dim::DIM_SubpassData:
1053 Reqs.addRequirements(SPIRV::Capability::InputAttachment);
1054 break;
1055 }
1056
1057
1058 if (.isShader()) {
1059 if (MI.getNumOperands() > 8 &&
1060 MI.getOperand(8).getImm() == SPIRV::AccessQualifier::ReadWrite)
1061 Reqs.addRequirements(SPIRV::Capability::ImageReadWrite);
1062 else
1064 }
1065}
1066
1067static bool isBFloat16Type(const SPIRVType *TypeDef) {
1069 TypeDef->getOpcode() == SPIRV::OpTypeFloat &&
1071 TypeDef->getOperand(2).getImm() == SPIRV::FPEncoding::BFloat16KHR;
1072}
1073
1074
1075#define ATOM_FLT_REQ_EXT_MSG(ExtName) \
1076 "The atomic float instruction requires the following SPIR-V " \
1077 "extension: SPV_EXT_shader_atomic_float" ExtName
1078static void AddAtomicVectorFloatRequirements(const MachineInstr &MI,
1082 MI.getMF()->getRegInfo().getVRegDef(MI.getOperand(1).getReg());
1083
1085 if (Rank != 2 && Rank != 4)
1087 "must be a 2-component or 4 component vector");
1088
1090 MI.getMF()->getRegInfo().getVRegDef(VecTypeDef->getOperand(1).getReg());
1091
1092 if (EltTypeDef->getOpcode() != SPIRV::OpTypeFloat ||
1095 "The element type for the result type of an atomic vector float "
1096 "instruction must be a 16-bit floating-point scalar");
1097
1098 if (isBFloat16Type(EltTypeDef))
1100 "The element type for the result type of an atomic vector float "
1101 "instruction cannot be a bfloat16 scalar");
1102 if (.canUseExtension(SPIRV::Extension::SPV_NV_shader_atomic_fp16_vector))
1104 "The atomic float16 vector instruction requires the following SPIR-V "
1105 "extension: SPV_NV_shader_atomic_fp16_vector");
1106
1107 Reqs.addExtension(SPIRV::Extension::SPV_NV_shader_atomic_fp16_vector);
1108 Reqs.addCapability(SPIRV::Capability::AtomicFloat16VectorNV);
1109}
1110
1111static void AddAtomicFloatRequirements(const MachineInstr &MI,
1114 assert(MI.getOperand(1).isReg() &&
1115 "Expect register operand in atomic float instruction");
1116 Register TypeReg = MI.getOperand(1).getReg();
1117 SPIRVType *TypeDef = MI.getMF()->getRegInfo().getVRegDef(TypeReg);
1118
1119 if (TypeDef->getOpcode() == SPIRV::OpTypeVector)
1120 return AddAtomicVectorFloatRequirements(MI, Reqs, ST);
1121
1122 if (TypeDef->getOpcode() != SPIRV::OpTypeFloat)
1123 report_fatal_error("Result type of an atomic float instruction must be a "
1124 "floating-point type scalar");
1125
1127 unsigned Op = MI.getOpcode();
1128 if (Op == SPIRV::OpAtomicFAddEXT) {
1129 if (.canUseExtension(SPIRV::Extension::SPV_EXT_shader_atomic_float_add))
1131 Reqs.addExtension(SPIRV::Extension::SPV_EXT_shader_atomic_float_add);
1133 case 16:
1134 if (isBFloat16Type(TypeDef)) {
1135 if (.canUseExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics))
1137 "The atomic bfloat16 instruction requires the following SPIR-V "
1138 "extension: SPV_INTEL_16bit_atomics",
1139 false);
1140 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics);
1141 Reqs.addCapability(SPIRV::Capability::AtomicBFloat16AddINTEL);
1142 } else {
1143 if (.canUseExtension(
1144 SPIRV::Extension::SPV_EXT_shader_atomic_float16_add))
1146 Reqs.addExtension(SPIRV::Extension::SPV_EXT_shader_atomic_float16_add);
1147 Reqs.addCapability(SPIRV::Capability::AtomicFloat16AddEXT);
1148 }
1149 break;
1150 case 32:
1151 Reqs.addCapability(SPIRV::Capability::AtomicFloat32AddEXT);
1152 break;
1153 case 64:
1154 Reqs.addCapability(SPIRV::Capability::AtomicFloat64AddEXT);
1155 break;
1156 default:
1158 "Unexpected floating-point type width in atomic float instruction");
1159 }
1160 } else {
1161 if (.canUseExtension(
1162 SPIRV::Extension::SPV_EXT_shader_atomic_float_min_max))
1164 Reqs.addExtension(SPIRV::Extension::SPV_EXT_shader_atomic_float_min_max);
1166 case 16:
1167 if (isBFloat16Type(TypeDef)) {
1168 if (.canUseExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics))
1170 "The atomic bfloat16 instruction requires the following SPIR-V "
1171 "extension: SPV_INTEL_16bit_atomics",
1172 false);
1173 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_16bit_atomics);
1174 Reqs.addCapability(SPIRV::Capability::AtomicBFloat16MinMaxINTEL);
1175 } else {
1176 Reqs.addCapability(SPIRV::Capability::AtomicFloat16MinMaxEXT);
1177 }
1178 break;
1179 case 32:
1180 Reqs.addCapability(SPIRV::Capability::AtomicFloat32MinMaxEXT);
1181 break;
1182 case 64:
1183 Reqs.addCapability(SPIRV::Capability::AtomicFloat64MinMaxEXT);
1184 break;
1185 default:
1187 "Unexpected floating-point type width in atomic float instruction");
1188 }
1189 }
1190}
1191
1192bool isUniformTexelBuffer(MachineInstr *ImageInst) {
1193 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1194 return false;
1197 return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 1;
1198}
1199
1200bool isStorageTexelBuffer(MachineInstr *ImageInst) {
1201 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1202 return false;
1205 return Dim == SPIRV::Dim::DIM_Buffer && Sampled == 2;
1206}
1207
1208bool isSampledImage(MachineInstr *ImageInst) {
1209 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1210 return false;
1213 return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 1;
1214}
1215
1216bool isInputAttachment(MachineInstr *ImageInst) {
1217 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1218 return false;
1221 return Dim == SPIRV::Dim::DIM_SubpassData && Sampled == 2;
1222}
1223
1224bool isStorageImage(MachineInstr *ImageInst) {
1225 if (ImageInst->getOpcode() != SPIRV::OpTypeImage)
1226 return false;
1229 return Dim != SPIRV::Dim::DIM_Buffer && Sampled == 2;
1230}
1231
1232bool isCombinedImageSampler(MachineInstr *SampledImageInst) {
1233 if (SampledImageInst->getOpcode() != SPIRV::OpTypeSampledImage)
1234 return false;
1235
1238 auto *ImageInst = MRI.getUniqueVRegDef(ImageReg);
1239 return isSampledImage(ImageInst);
1240}
1241
1243 for (const auto &MI : MRI.reg_instructions(Reg)) {
1244 if (MI.getOpcode() != SPIRV::OpDecorate)
1245 continue;
1246
1247 uint32_t Dec = MI.getOperand(1).getImm();
1248 if (Dec == SPIRV::Decoration::NonUniformEXT)
1249 return true;
1250 }
1251 return false;
1252}
1253
1254void addOpAccessChainReqs(const MachineInstr &Instr,
1258
1259
1260
1261 Register ResTypeReg = Instr.getOperand(1).getReg();
1262 MachineInstr *ResTypeInst = MRI.getUniqueVRegDef(ResTypeReg);
1263
1264 assert(ResTypeInst->getOpcode() == SPIRV::OpTypePointer);
1266 if (StorageClass != SPIRV::StorageClass::StorageClass::UniformConstant &&
1267 StorageClass != SPIRV::StorageClass::StorageClass::Uniform &&
1268 StorageClass != SPIRV::StorageClass::StorageClass::StorageBuffer) {
1269 return;
1270 }
1271
1272 bool IsNonUniform =
1273 hasNonUniformDecoration(Instr.getOperand(0).getReg(), MRI);
1274
1275 auto FirstIndexReg = Instr.getOperand(3).getReg();
1276 bool FirstIndexIsConstant =
1278
1279 if (StorageClass == SPIRV::StorageClass::StorageClass::StorageBuffer) {
1280 if (IsNonUniform)
1282 SPIRV::Capability::StorageBufferArrayNonUniformIndexingEXT);
1283 else if (!FirstIndexIsConstant)
1285 SPIRV::Capability::StorageBufferArrayDynamicIndexing);
1286 return;
1287 }
1288
1290 MachineInstr *PointeeType = MRI.getUniqueVRegDef(PointeeTypeReg);
1291 if (PointeeType->getOpcode() != SPIRV::OpTypeImage &&
1292 PointeeType->getOpcode() != SPIRV::OpTypeSampledImage &&
1293 PointeeType->getOpcode() != SPIRV::OpTypeSampler) {
1294 return;
1295 }
1296
1297 if (isUniformTexelBuffer(PointeeType)) {
1298 if (IsNonUniform)
1300 SPIRV::Capability::UniformTexelBufferArrayNonUniformIndexingEXT);
1301 else if (!FirstIndexIsConstant)
1303 SPIRV::Capability::UniformTexelBufferArrayDynamicIndexingEXT);
1304 } else if (isInputAttachment(PointeeType)) {
1305 if (IsNonUniform)
1307 SPIRV::Capability::InputAttachmentArrayNonUniformIndexingEXT);
1308 else if (!FirstIndexIsConstant)
1310 SPIRV::Capability::InputAttachmentArrayDynamicIndexingEXT);
1311 } else if (isStorageTexelBuffer(PointeeType)) {
1312 if (IsNonUniform)
1314 SPIRV::Capability::StorageTexelBufferArrayNonUniformIndexingEXT);
1315 else if (!FirstIndexIsConstant)
1317 SPIRV::Capability::StorageTexelBufferArrayDynamicIndexingEXT);
1318 } else if (isSampledImage(PointeeType) ||
1319 isCombinedImageSampler(PointeeType) ||
1320 PointeeType->getOpcode() == SPIRV::OpTypeSampler) {
1321 if (IsNonUniform)
1323 SPIRV::Capability::SampledImageArrayNonUniformIndexingEXT);
1324 else if (!FirstIndexIsConstant)
1326 SPIRV::Capability::SampledImageArrayDynamicIndexing);
1327 } else if (isStorageImage(PointeeType)) {
1328 if (IsNonUniform)
1330 SPIRV::Capability::StorageImageArrayNonUniformIndexingEXT);
1331 else if (!FirstIndexIsConstant)
1333 SPIRV::Capability::StorageImageArrayDynamicIndexing);
1334 }
1335}
1336
1337static bool isImageTypeWithUnknownFormat(SPIRVType *TypeInst) {
1338 if (TypeInst->getOpcode() != SPIRV::OpTypeImage)
1339 return false;
1342}
1343
1344static void AddDotProductRequirements(const MachineInstr &MI,
1347 if (ST.canUseExtension(SPIRV::Extension::SPV_KHR_integer_dot_product))
1348 Reqs.addExtension(SPIRV::Extension::SPV_KHR_integer_dot_product);
1349 Reqs.addCapability(SPIRV::Capability::DotProduct);
1350
1352 assert(MI.getOperand(2).isReg() && "Unexpected operand in dot");
1353
1354
1356 assert(Input->getOperand(1).isReg() && "Unexpected operand in dot input");
1357 Register InputReg = Input->getOperand(1).getReg();
1358
1359 SPIRVType *TypeDef = MRI.getVRegDef(InputReg);
1360 if (TypeDef->getOpcode() == SPIRV::OpTypeInt) {
1362 Reqs.addCapability(SPIRV::Capability::DotProductInput4x8BitPacked);
1363 } else if (TypeDef->getOpcode() == SPIRV::OpTypeVector) {
1368 "Dot operand of 8-bit integer type requires 4 components");
1369 Reqs.addCapability(SPIRV::Capability::DotProductInput4x8Bit);
1370 } else {
1371 Reqs.addCapability(SPIRV::Capability::DotProductInputAll);
1372 }
1373 }
1374}
1375
1381 if (PtrType) {
1383 if (ASOp.isImm()) {
1384 unsigned AddrSpace = ASOp.getImm();
1385 if (AddrSpace != SPIRV::StorageClass::UniformConstant) {
1386 if (.canUseExtension(
1388 SPV_EXT_relaxed_printf_string_address_space)) {
1389 report_fatal_error("SPV_EXT_relaxed_printf_string_address_space is "
1390 "required because printf uses a format string not "
1391 "in constant address space.",
1392 false);
1393 }
1395 SPIRV::Extension::SPV_EXT_relaxed_printf_string_address_space);
1396 }
1397 }
1398 }
1399}
1400
1405 switch (MI.getOpcode()) {
1406 case SPIRV::OpMemoryModel: {
1407 int64_t Addr = MI.getOperand(0).getImm();
1409 Addr, ST);
1410 int64_t Mem = MI.getOperand(1).getImm();
1412 ST);
1413 break;
1414 }
1415 case SPIRV::OpEntryPoint: {
1416 int64_t Exe = MI.getOperand(0).getImm();
1418 Exe, ST);
1419 break;
1420 }
1421 case SPIRV::OpExecutionMode:
1422 case SPIRV::OpExecutionModeId: {
1423 int64_t Exe = MI.getOperand(1).getImm();
1425 Exe, ST);
1426 break;
1427 }
1428 case SPIRV::OpTypeMatrix:
1430 break;
1431 case SPIRV::OpTypeInt: {
1432 unsigned BitWidth = MI.getOperand(1).getImm();
1440 ST.canUseExtension(SPIRV::Extension::SPV_INTEL_int4)) {
1441 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_int4);
1442 Reqs.addCapability(SPIRV::Capability::Int4TypeINTEL);
1443 } else if (BitWidth != 32) {
1444 if (.canUseExtension(
1445 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers))
1447 "OpTypeInt type with a width other than 8, 16, 32 or 64 bits "
1448 "requires the following SPIR-V extension: "
1449 "SPV_ALTERA_arbitrary_precision_integers");
1451 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers);
1452 Reqs.addCapability(SPIRV::Capability::ArbitraryPrecisionIntegersALTERA);
1453 }
1454 break;
1455 }
1456 case SPIRV::OpDot: {
1458 SPIRVType *TypeDef = MRI.getVRegDef(MI.getOperand(1).getReg());
1459 if (isBFloat16Type(TypeDef))
1460 Reqs.addCapability(SPIRV::Capability::BFloat16DotProductKHR);
1461 break;
1462 }
1463 case SPIRV::OpTypeFloat: {
1464 unsigned BitWidth = MI.getOperand(1).getImm();
1468 if (isBFloat16Type(&MI)) {
1469 if (.canUseExtension(SPIRV::Extension::SPV_KHR_bfloat16))
1471 "following SPIR-V extension: SPV_KHR_bfloat16",
1472 false);
1473 Reqs.addExtension(SPIRV::Extension::SPV_KHR_bfloat16);
1474 Reqs.addCapability(SPIRV::Capability::BFloat16TypeKHR);
1475 } else {
1477 }
1478 }
1479 break;
1480 }
1481 case SPIRV::OpTypeVector: {
1482 unsigned NumComponents = MI.getOperand(2).getImm();
1483 if (NumComponents == 8 || NumComponents == 16)
1484 Reqs.addCapability(SPIRV::Capability::Vector16);
1485 break;
1486 }
1487 case SPIRV::OpTypePointer: {
1488 auto SC = MI.getOperand(1).getImm();
1490 ST);
1491
1492
1493 if (ST.isShader())
1494 break;
1495 assert(MI.getOperand(2).isReg());
1497 SPIRVType *TypeDef = MRI.getVRegDef(MI.getOperand(2).getReg());
1499 (TypeDef->getOpcode() == SPIRV::OpTypeFloat) &&
1501 Reqs.addCapability(SPIRV::Capability::Float16Buffer);
1502 break;
1503 }
1504 case SPIRV::OpExtInst: {
1505 if (MI.getOperand(2).getImm() ==
1506 static_cast<int64_t>(
1507 SPIRV::InstructionSet::NonSemantic_Shader_DebugInfo_100)) {
1508 Reqs.addExtension(SPIRV::Extension::SPV_KHR_non_semantic_info);
1509 break;
1510 }
1511 if (MI.getOperand(3).getImm() ==
1512 static_cast<int64_t>(SPIRV::OpenCLExtInst::printf)) {
1513 addPrintfRequirements(MI, Reqs, ST);
1514 break;
1515 }
1516
1517
1518 break;
1519 }
1520 case SPIRV::OpAliasDomainDeclINTEL:
1521 case SPIRV::OpAliasScopeDeclINTEL:
1522 case SPIRV::OpAliasScopeListDeclINTEL: {
1523 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_memory_access_aliasing);
1524 Reqs.addCapability(SPIRV::Capability::MemoryAccessAliasingINTEL);
1525 break;
1526 }
1527 case SPIRV::OpBitReverse:
1528 case SPIRV::OpBitFieldInsert:
1529 case SPIRV::OpBitFieldSExtract:
1530 case SPIRV::OpBitFieldUExtract:
1531 if (.canUseExtension(SPIRV::Extension::SPV_KHR_bit_instructions)) {
1533 break;
1534 }
1535 Reqs.addExtension(SPIRV::Extension::SPV_KHR_bit_instructions);
1536 Reqs.addCapability(SPIRV::Capability::BitInstructions);
1537 break;
1538 case SPIRV::OpTypeRuntimeArray:
1540 break;
1541 case SPIRV::OpTypeOpaque:
1542 case SPIRV::OpTypeEvent:
1544 break;
1545 case SPIRV::OpTypePipe:
1546 case SPIRV::OpTypeReserveId:
1548 break;
1549 case SPIRV::OpTypeDeviceEvent:
1550 case SPIRV::OpTypeQueue:
1551 case SPIRV::OpBuildNDRange:
1552 Reqs.addCapability(SPIRV::Capability::DeviceEnqueue);
1553 break;
1554 case SPIRV::OpDecorate:
1555 case SPIRV::OpDecorateId:
1556 case SPIRV::OpDecorateString:
1557 addOpDecorateReqs(MI, 1, Reqs, ST);
1558 break;
1559 case SPIRV::OpMemberDecorate:
1560 case SPIRV::OpMemberDecorateString:
1561 addOpDecorateReqs(MI, 2, Reqs, ST);
1562 break;
1563 case SPIRV::OpInBoundsPtrAccessChain:
1564 Reqs.addCapability(SPIRV::Capability::Addresses);
1565 break;
1566 case SPIRV::OpConstantSampler:
1567 Reqs.addCapability(SPIRV::Capability::LiteralSampler);
1568 break;
1569 case SPIRV::OpInBoundsAccessChain:
1570 case SPIRV::OpAccessChain:
1571 addOpAccessChainReqs(MI, Reqs, ST);
1572 break;
1573 case SPIRV::OpTypeImage:
1574 addOpTypeImageReqs(MI, Reqs, ST);
1575 break;
1576 case SPIRV::OpTypeSampler:
1577 if (.isShader()) {
1578 Reqs.addCapability(SPIRV::Capability::ImageBasic);
1579 }
1580 break;
1581 case SPIRV::OpTypeForwardPointer:
1582
1583 Reqs.addCapability(SPIRV::Capability::Addresses);
1584 break;
1585 case SPIRV::OpAtomicFlagTestAndSet:
1586 case SPIRV::OpAtomicLoad:
1587 case SPIRV::OpAtomicStore:
1588 case SPIRV::OpAtomicExchange:
1589 case SPIRV::OpAtomicCompareExchange:
1590 case SPIRV::OpAtomicIIncrement:
1591 case SPIRV::OpAtomicIDecrement:
1592 case SPIRV::OpAtomicIAdd:
1593 case SPIRV::OpAtomicISub:
1594 case SPIRV::OpAtomicUMin:
1595 case SPIRV::OpAtomicUMax:
1596 case SPIRV::OpAtomicSMin:
1597 case SPIRV::OpAtomicSMax:
1598 case SPIRV::OpAtomicAnd:
1599 case SPIRV::OpAtomicOr:
1600 case SPIRV::OpAtomicXor: {
1603 if (MI.getOpcode() == SPIRV::OpAtomicStore) {
1604 assert(MI.getOperand(3).isReg());
1605 InstrPtr = MRI.getVRegDef(MI.getOperand(3).getReg());
1606 assert(InstrPtr && "Unexpected type instruction for OpAtomicStore");
1607 }
1610 SPIRVType *TypeDef = MRI.getVRegDef(TypeReg);
1611 if (TypeDef->getOpcode() == SPIRV::OpTypeInt) {
1614 Reqs.addCapability(SPIRV::Capability::Int64Atomics);
1615 }
1616 break;
1617 }
1618 case SPIRV::OpGroupNonUniformIAdd:
1619 case SPIRV::OpGroupNonUniformFAdd:
1620 case SPIRV::OpGroupNonUniformIMul:
1621 case SPIRV::OpGroupNonUniformFMul:
1622 case SPIRV::OpGroupNonUniformSMin:
1623 case SPIRV::OpGroupNonUniformUMin:
1624 case SPIRV::OpGroupNonUniformFMin:
1625 case SPIRV::OpGroupNonUniformSMax:
1626 case SPIRV::OpGroupNonUniformUMax:
1627 case SPIRV::OpGroupNonUniformFMax:
1628 case SPIRV::OpGroupNonUniformBitwiseAnd:
1629 case SPIRV::OpGroupNonUniformBitwiseOr:
1630 case SPIRV::OpGroupNonUniformBitwiseXor:
1631 case SPIRV::OpGroupNonUniformLogicalAnd:
1632 case SPIRV::OpGroupNonUniformLogicalOr:
1633 case SPIRV::OpGroupNonUniformLogicalXor: {
1634 assert(MI.getOperand(3).isImm());
1635 int64_t GroupOp = MI.getOperand(3).getImm();
1636 switch (GroupOp) {
1637 case SPIRV::GroupOperation::Reduce:
1638 case SPIRV::GroupOperation::InclusiveScan:
1639 case SPIRV::GroupOperation::ExclusiveScan:
1640 Reqs.addCapability(SPIRV::Capability::GroupNonUniformArithmetic);
1641 break;
1642 case SPIRV::GroupOperation::ClusteredReduce:
1643 Reqs.addCapability(SPIRV::Capability::GroupNonUniformClustered);
1644 break;
1645 case SPIRV::GroupOperation::PartitionedReduceNV:
1646 case SPIRV::GroupOperation::PartitionedInclusiveScanNV:
1647 case SPIRV::GroupOperation::PartitionedExclusiveScanNV:
1648 Reqs.addCapability(SPIRV::Capability::GroupNonUniformPartitionedNV);
1649 break;
1650 }
1651 break;
1652 }
1653 case SPIRV::OpGroupNonUniformShuffle:
1654 case SPIRV::OpGroupNonUniformShuffleXor:
1655 Reqs.addCapability(SPIRV::Capability::GroupNonUniformShuffle);
1656 break;
1657 case SPIRV::OpGroupNonUniformShuffleUp:
1658 case SPIRV::OpGroupNonUniformShuffleDown:
1659 Reqs.addCapability(SPIRV::Capability::GroupNonUniformShuffleRelative);
1660 break;
1661 case SPIRV::OpGroupAll:
1662 case SPIRV::OpGroupAny:
1663 case SPIRV::OpGroupBroadcast:
1664 case SPIRV::OpGroupIAdd:
1665 case SPIRV::OpGroupFAdd:
1666 case SPIRV::OpGroupFMin:
1667 case SPIRV::OpGroupUMin:
1668 case SPIRV::OpGroupSMin:
1669 case SPIRV::OpGroupFMax:
1670 case SPIRV::OpGroupUMax:
1671 case SPIRV::OpGroupSMax:
1673 break;
1674 case SPIRV::OpGroupNonUniformElect:
1675 Reqs.addCapability(SPIRV::Capability::GroupNonUniform);
1676 break;
1677 case SPIRV::OpGroupNonUniformAll:
1678 case SPIRV::OpGroupNonUniformAny:
1679 case SPIRV::OpGroupNonUniformAllEqual:
1680 Reqs.addCapability(SPIRV::Capability::GroupNonUniformVote);
1681 break;
1682 case SPIRV::OpGroupNonUniformBroadcast:
1683 case SPIRV::OpGroupNonUniformBroadcastFirst:
1684 case SPIRV::OpGroupNonUniformBallot:
1685 case SPIRV::OpGroupNonUniformInverseBallot:
1686 case SPIRV::OpGroupNonUniformBallotBitExtract:
1687 case SPIRV::OpGroupNonUniformBallotBitCount:
1688 case SPIRV::OpGroupNonUniformBallotFindLSB:
1689 case SPIRV::OpGroupNonUniformBallotFindMSB:
1690 Reqs.addCapability(SPIRV::Capability::GroupNonUniformBallot);
1691 break;
1692 case SPIRV::OpSubgroupShuffleINTEL:
1693 case SPIRV::OpSubgroupShuffleDownINTEL:
1694 case SPIRV::OpSubgroupShuffleUpINTEL:
1695 case SPIRV::OpSubgroupShuffleXorINTEL:
1696 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_subgroups)) {
1697 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_subgroups);
1698 Reqs.addCapability(SPIRV::Capability::SubgroupShuffleINTEL);
1699 }
1700 break;
1701 case SPIRV::OpSubgroupBlockReadINTEL:
1702 case SPIRV::OpSubgroupBlockWriteINTEL:
1703 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_subgroups)) {
1704 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_subgroups);
1705 Reqs.addCapability(SPIRV::Capability::SubgroupBufferBlockIOINTEL);
1706 }
1707 break;
1708 case SPIRV::OpSubgroupImageBlockReadINTEL:
1709 case SPIRV::OpSubgroupImageBlockWriteINTEL:
1710 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_subgroups)) {
1711 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_subgroups);
1712 Reqs.addCapability(SPIRV::Capability::SubgroupImageBlockIOINTEL);
1713 }
1714 break;
1715 case SPIRV::OpSubgroupImageMediaBlockReadINTEL:
1716 case SPIRV::OpSubgroupImageMediaBlockWriteINTEL:
1717 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_media_block_io)) {
1718 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_media_block_io);
1719 Reqs.addCapability(SPIRV::Capability::SubgroupImageMediaBlockIOINTEL);
1720 }
1721 break;
1722 case SPIRV::OpAssumeTrueKHR:
1723 case SPIRV::OpExpectKHR:
1724 if (ST.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) {
1725 Reqs.addExtension(SPIRV::Extension::SPV_KHR_expect_assume);
1726 Reqs.addCapability(SPIRV::Capability::ExpectAssumeKHR);
1727 }
1728 break;
1729 case SPIRV::OpPtrCastToCrossWorkgroupINTEL:
1730 case SPIRV::OpCrossWorkgroupCastToPtrINTEL:
1731 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_usm_storage_classes)) {
1732 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_usm_storage_classes);
1733 Reqs.addCapability(SPIRV::Capability::USMStorageClassesINTEL);
1734 }
1735 break;
1736 case SPIRV::OpConstantFunctionPointerINTEL:
1737 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers)) {
1738 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_function_pointers);
1739 Reqs.addCapability(SPIRV::Capability::FunctionPointersINTEL);
1740 }
1741 break;
1742 case SPIRV::OpGroupNonUniformRotateKHR:
1743 if (.canUseExtension(SPIRV::Extension::SPV_KHR_subgroup_rotate))
1744 report_fatal_error("OpGroupNonUniformRotateKHR instruction requires the "
1745 "following SPIR-V extension: SPV_KHR_subgroup_rotate",
1746 false);
1747 Reqs.addExtension(SPIRV::Extension::SPV_KHR_subgroup_rotate);
1748 Reqs.addCapability(SPIRV::Capability::GroupNonUniformRotateKHR);
1749 Reqs.addCapability(SPIRV::Capability::GroupNonUniform);
1750 break;
1751 case SPIRV::OpFixedCosALTERA:
1752 case SPIRV::OpFixedSinALTERA:
1753 case SPIRV::OpFixedCosPiALTERA:
1754 case SPIRV::OpFixedSinPiALTERA:
1755 case SPIRV::OpFixedExpALTERA:
1756 case SPIRV::OpFixedLogALTERA:
1757 case SPIRV::OpFixedRecipALTERA:
1758 case SPIRV::OpFixedSqrtALTERA:
1759 case SPIRV::OpFixedSinCosALTERA:
1760 case SPIRV::OpFixedSinCosPiALTERA:
1761 case SPIRV::OpFixedRsqrtALTERA:
1762 if (.canUseExtension(
1763 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_fixed_point))
1765 "following SPIR-V extension: "
1766 "SPV_ALTERA_arbitrary_precision_fixed_point",
1767 false);
1769 SPIRV::Extension::SPV_ALTERA_arbitrary_precision_fixed_point);
1770 Reqs.addCapability(SPIRV::Capability::ArbitraryPrecisionFixedPointALTERA);
1771 break;
1772 case SPIRV::OpGroupIMulKHR:
1773 case SPIRV::OpGroupFMulKHR:
1774 case SPIRV::OpGroupBitwiseAndKHR:
1775 case SPIRV::OpGroupBitwiseOrKHR:
1776 case SPIRV::OpGroupBitwiseXorKHR:
1777 case SPIRV::OpGroupLogicalAndKHR:
1778 case SPIRV::OpGroupLogicalOrKHR:
1779 case SPIRV::OpGroupLogicalXorKHR:
1780 if (ST.canUseExtension(
1781 SPIRV::Extension::SPV_KHR_uniform_group_instructions)) {
1782 Reqs.addExtension(SPIRV::Extension::SPV_KHR_uniform_group_instructions);
1783 Reqs.addCapability(SPIRV::Capability::GroupUniformArithmeticKHR);
1784 }
1785 break;
1786 case SPIRV::OpReadClockKHR:
1787 if (.canUseExtension(SPIRV::Extension::SPV_KHR_shader_clock))
1789 "following SPIR-V extension: SPV_KHR_shader_clock",
1790 false);
1791 Reqs.addExtension(SPIRV::Extension::SPV_KHR_shader_clock);
1792 Reqs.addCapability(SPIRV::Capability::ShaderClockKHR);
1793 break;
1794 case SPIRV::OpFunctionPointerCallINTEL:
1795 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers)) {
1796 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_function_pointers);
1797 Reqs.addCapability(SPIRV::Capability::FunctionPointersINTEL);
1798 }
1799 break;
1800 case SPIRV::OpAtomicFAddEXT:
1801 case SPIRV::OpAtomicFMinEXT:
1802 case SPIRV::OpAtomicFMaxEXT:
1803 AddAtomicFloatRequirements(MI, Reqs, ST);
1804 break;
1805 case SPIRV::OpConvertBF16ToFINTEL:
1806 case SPIRV::OpConvertFToBF16INTEL:
1807 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_bfloat16_conversion)) {
1808 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_bfloat16_conversion);
1809 Reqs.addCapability(SPIRV::Capability::BFloat16ConversionINTEL);
1810 }
1811 break;
1812 case SPIRV::OpRoundFToTF32INTEL:
1813 if (ST.canUseExtension(
1814 SPIRV::Extension::SPV_INTEL_tensor_float32_conversion)) {
1815 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_tensor_float32_conversion);
1816 Reqs.addCapability(SPIRV::Capability::TensorFloat32RoundingINTEL);
1817 }
1818 break;
1819 case SPIRV::OpVariableLengthArrayINTEL:
1820 case SPIRV::OpSaveMemoryINTEL:
1821 case SPIRV::OpRestoreMemoryINTEL:
1822 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) {
1823 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_variable_length_array);
1824 Reqs.addCapability(SPIRV::Capability::VariableLengthArrayINTEL);
1825 }
1826 break;
1827 case SPIRV::OpAsmTargetINTEL:
1828 case SPIRV::OpAsmINTEL:
1829 case SPIRV::OpAsmCallINTEL:
1830 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_inline_assembly)) {
1831 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_inline_assembly);
1832 Reqs.addCapability(SPIRV::Capability::AsmINTEL);
1833 }
1834 break;
1835 case SPIRV::OpTypeCooperativeMatrixKHR: {
1836 if (.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix))
1838 "OpTypeCooperativeMatrixKHR type requires the "
1839 "following SPIR-V extension: SPV_KHR_cooperative_matrix",
1840 false);
1841 Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);
1842 Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);
1844 SPIRVType *TypeDef = MRI.getVRegDef(MI.getOperand(1).getReg());
1845 if (isBFloat16Type(TypeDef))
1846 Reqs.addCapability(SPIRV::Capability::BFloat16CooperativeMatrixKHR);
1847 break;
1848 }
1849 case SPIRV::OpArithmeticFenceEXT:
1850 if (.canUseExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence))
1852 "following SPIR-V extension: SPV_EXT_arithmetic_fence",
1853 false);
1854 Reqs.addExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence);
1855 Reqs.addCapability(SPIRV::Capability::ArithmeticFenceEXT);
1856 break;
1857 case SPIRV::OpControlBarrierArriveINTEL:
1858 case SPIRV::OpControlBarrierWaitINTEL:
1859 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_split_barrier)) {
1860 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_split_barrier);
1861 Reqs.addCapability(SPIRV::Capability::SplitBarrierINTEL);
1862 }
1863 break;
1864 case SPIRV::OpCooperativeMatrixMulAddKHR: {
1865 if (.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix))
1867 "following SPIR-V extension: "
1868 "SPV_KHR_cooperative_matrix",
1869 false);
1870 Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);
1871 Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);
1872 constexpr unsigned MulAddMaxSize = 6;
1873 if (MI.getNumOperands() != MulAddMaxSize)
1874 break;
1875 const int64_t CoopOperands = MI.getOperand(MulAddMaxSize - 1).getImm();
1876 if (CoopOperands &
1877 SPIRV::CooperativeMatrixOperands::MatrixAAndBTF32ComponentsINTEL) {
1878 if (.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
1879 report_fatal_error("MatrixAAndBTF32ComponentsINTEL type interpretation "
1880 "require the following SPIR-V extension: "
1881 "SPV_INTEL_joint_matrix",
1882 false);
1883 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
1885 SPIRV::Capability::CooperativeMatrixTF32ComponentTypeINTEL);
1886 }
1888 MatrixAAndBBFloat16ComponentsINTEL ||
1889 CoopOperands &
1890 SPIRV::CooperativeMatrixOperands::MatrixCBFloat16ComponentsINTEL ||
1892 MatrixResultBFloat16ComponentsINTEL) {
1893 if (.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
1895 "require the following SPIR-V extension: "
1896 "SPV_INTEL_joint_matrix",
1897 false);
1898 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
1900 SPIRV::Capability::CooperativeMatrixBFloat16ComponentTypeINTEL);
1901 }
1902 break;
1903 }
1904 case SPIRV::OpCooperativeMatrixLoadKHR:
1905 case SPIRV::OpCooperativeMatrixStoreKHR:
1906 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
1907 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
1908 case SPIRV::OpCooperativeMatrixPrefetchINTEL: {
1909 if (.canUseExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix))
1911 "following SPIR-V extension: "
1912 "SPV_KHR_cooperative_matrix",
1913 false);
1914 Reqs.addExtension(SPIRV::Extension::SPV_KHR_cooperative_matrix);
1915 Reqs.addCapability(SPIRV::Capability::CooperativeMatrixKHR);
1916
1917
1918
1919 std::unordered_map<unsigned, unsigned> LayoutToInstMap = {
1920 {SPIRV::OpCooperativeMatrixLoadKHR, 3},
1921 {SPIRV::OpCooperativeMatrixStoreKHR, 2},
1922 {SPIRV::OpCooperativeMatrixLoadCheckedINTEL, 5},
1923 {SPIRV::OpCooperativeMatrixStoreCheckedINTEL, 4},
1924 {SPIRV::OpCooperativeMatrixPrefetchINTEL, 4}};
1925
1926 const auto OpCode = MI.getOpcode();
1927 const unsigned LayoutNum = LayoutToInstMap[OpCode];
1928 Register RegLayout = MI.getOperand(LayoutNum).getReg();
1930 MachineInstr *MILayout = MRI.getUniqueVRegDef(RegLayout);
1931 if (MILayout->getOpcode() == SPIRV::OpConstantI) {
1932 const unsigned LayoutVal = MILayout->getOperand(2).getImm();
1933 if (LayoutVal ==
1934 static_cast<unsigned>(SPIRV::CooperativeMatrixLayout::PackedINTEL)) {
1935 if (.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
1936 report_fatal_error("PackedINTEL layout require the following SPIR-V "
1937 "extension: SPV_INTEL_joint_matrix",
1938 false);
1939 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
1940 Reqs.addCapability(SPIRV::Capability::PackedCooperativeMatrixINTEL);
1941 }
1942 }
1943
1944
1945 if (OpCode == SPIRV::OpCooperativeMatrixLoadKHR ||
1946 OpCode == SPIRV::OpCooperativeMatrixStoreKHR)
1947 break;
1948
1949 std::string InstName;
1950 switch (OpCode) {
1951 case SPIRV::OpCooperativeMatrixPrefetchINTEL:
1952 InstName = "OpCooperativeMatrixPrefetchINTEL";
1953 break;
1954 case SPIRV::OpCooperativeMatrixLoadCheckedINTEL:
1955 InstName = "OpCooperativeMatrixLoadCheckedINTEL";
1956 break;
1957 case SPIRV::OpCooperativeMatrixStoreCheckedINTEL:
1958 InstName = "OpCooperativeMatrixStoreCheckedINTEL";
1959 break;
1960 }
1961
1962 if (.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix)) {
1963 const std::string ErrorMsg =
1964 InstName + " instruction requires the "
1965 "following SPIR-V extension: SPV_INTEL_joint_matrix";
1967 }
1968 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
1969 if (OpCode == SPIRV::OpCooperativeMatrixPrefetchINTEL) {
1970 Reqs.addCapability(SPIRV::Capability::CooperativeMatrixPrefetchINTEL);
1971 break;
1972 }
1974 SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
1975 break;
1976 }
1977 case SPIRV::OpCooperativeMatrixConstructCheckedINTEL:
1978 if (.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
1980 "instructions require the following SPIR-V extension: "
1981 "SPV_INTEL_joint_matrix",
1982 false);
1983 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
1985 SPIRV::Capability::CooperativeMatrixCheckedInstructionsINTEL);
1986 break;
1987 case SPIRV::OpReadPipeBlockingALTERA:
1988 case SPIRV::OpWritePipeBlockingALTERA:
1989 if (ST.canUseExtension(SPIRV::Extension::SPV_ALTERA_blocking_pipes)) {
1990 Reqs.addExtension(SPIRV::Extension::SPV_ALTERA_blocking_pipes);
1991 Reqs.addCapability(SPIRV::Capability::BlockingPipesALTERA);
1992 }
1993 break;
1994 case SPIRV::OpCooperativeMatrixGetElementCoordINTEL:
1995 if (.canUseExtension(SPIRV::Extension::SPV_INTEL_joint_matrix))
1996 report_fatal_error("OpCooperativeMatrixGetElementCoordINTEL requires the "
1997 "following SPIR-V extension: SPV_INTEL_joint_matrix",
1998 false);
1999 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_joint_matrix);
2001 SPIRV::Capability::CooperativeMatrixInvocationInstructionsINTEL);
2002 break;
2003 case SPIRV::OpConvertHandleToImageINTEL:
2004 case SPIRV::OpConvertHandleToSamplerINTEL:
2005 case SPIRV::OpConvertHandleToSampledImageINTEL: {
2006 if (.canUseExtension(SPIRV::Extension::SPV_INTEL_bindless_images))
2007 report_fatal_error("OpConvertHandleTo[Image/Sampler/SampledImage]INTEL "
2008 "instructions require the following SPIR-V extension: "
2009 "SPV_INTEL_bindless_images",
2010 false);
2012 SPIRV::AddressingModel::AddressingModel AddrModel = MAI.Addr;
2014 if (MI.getOpcode() == SPIRV::OpConvertHandleToImageINTEL &&
2015 TyDef->getOpcode() != SPIRV::OpTypeImage) {
2017 "OpConvertHandleToImageINTEL",
2018 false);
2019 } else if (MI.getOpcode() == SPIRV::OpConvertHandleToSamplerINTEL &&
2020 TyDef->getOpcode() != SPIRV::OpTypeSampler) {
2022 "OpConvertHandleToSamplerINTEL",
2023 false);
2024 } else if (MI.getOpcode() == SPIRV::OpConvertHandleToSampledImageINTEL &&
2025 TyDef->getOpcode() != SPIRV::OpTypeSampledImage) {
2027 "OpConvertHandleToSampledImageINTEL",
2028 false);
2029 }
2032 if (!(Bitwidth == 32 && AddrModel == SPIRV::AddressingModel::Physical32) &&
2033 !(Bitwidth == 64 && AddrModel == SPIRV::AddressingModel::Physical64)) {
2035 "Parameter value must be a 32-bit scalar in case of "
2036 "Physical32 addressing model or a 64-bit scalar in case of "
2037 "Physical64 addressing model",
2038 false);
2039 }
2040 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_bindless_images);
2041 Reqs.addCapability(SPIRV::Capability::BindlessImagesINTEL);
2042 break;
2043 }
2044 case SPIRV::OpSubgroup2DBlockLoadINTEL:
2045 case SPIRV::OpSubgroup2DBlockLoadTransposeINTEL:
2046 case SPIRV::OpSubgroup2DBlockLoadTransformINTEL:
2047 case SPIRV::OpSubgroup2DBlockPrefetchINTEL:
2048 case SPIRV::OpSubgroup2DBlockStoreINTEL: {
2049 if (.canUseExtension(SPIRV::Extension::SPV_INTEL_2d_block_io))
2050 report_fatal_error("OpSubgroup2DBlock[Load/LoadTranspose/LoadTransform/"
2051 "Prefetch/Store]INTEL instructions require the "
2052 "following SPIR-V extension: SPV_INTEL_2d_block_io",
2053 false);
2054 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_2d_block_io);
2055 Reqs.addCapability(SPIRV::Capability::Subgroup2DBlockIOINTEL);
2056
2057 const auto OpCode = MI.getOpcode();
2058 if (OpCode == SPIRV::OpSubgroup2DBlockLoadTransposeINTEL) {
2059 Reqs.addCapability(SPIRV::Capability::Subgroup2DBlockTransposeINTEL);
2060 break;
2061 }
2062 if (OpCode == SPIRV::OpSubgroup2DBlockLoadTransformINTEL) {
2063 Reqs.addCapability(SPIRV::Capability::Subgroup2DBlockTransformINTEL);
2064 break;
2065 }
2066 break;
2067 }
2068 case SPIRV::OpKill: {
2070 } break;
2071 case SPIRV::OpDemoteToHelperInvocation:
2072 Reqs.addCapability(SPIRV::Capability::DemoteToHelperInvocation);
2073
2074 if (ST.canUseExtension(
2075 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation)) {
2078 SPIRV::Extension::SPV_EXT_demote_to_helper_invocation);
2079 }
2080 break;
2081 case SPIRV::OpSDot:
2082 case SPIRV::OpUDot:
2083 case SPIRV::OpSUDot:
2084 case SPIRV::OpSDotAccSat:
2085 case SPIRV::OpUDotAccSat:
2086 case SPIRV::OpSUDotAccSat:
2087 AddDotProductRequirements(MI, Reqs, ST);
2088 break;
2089 case SPIRV::OpImageRead: {
2090 Register ImageReg = MI.getOperand(2).getReg();
2091 SPIRVType *TypeDef = ST.getSPIRVGlobalRegistry()->getResultType(
2093
2094
2095
2096
2097
2098
2099 if (isImageTypeWithUnknownFormat(TypeDef) && ST.isShader())
2100 Reqs.addCapability(SPIRV::Capability::StorageImageReadWithoutFormat);
2101 break;
2102 }
2103 case SPIRV::OpImageWrite: {
2104 Register ImageReg = MI.getOperand(0).getReg();
2105 SPIRVType *TypeDef = ST.getSPIRVGlobalRegistry()->getResultType(
2107
2108
2109
2110
2111
2112
2113 if (isImageTypeWithUnknownFormat(TypeDef) && ST.isShader())
2114 Reqs.addCapability(SPIRV::Capability::StorageImageWriteWithoutFormat);
2115 break;
2116 }
2117 case SPIRV::OpTypeStructContinuedINTEL:
2118 case SPIRV::OpConstantCompositeContinuedINTEL:
2119 case SPIRV::OpSpecConstantCompositeContinuedINTEL:
2120 case SPIRV::OpCompositeConstructContinuedINTEL: {
2121 if (.canUseExtension(SPIRV::Extension::SPV_INTEL_long_composites))
2123 "Continued instructions require the "
2124 "following SPIR-V extension: SPV_INTEL_long_composites",
2125 false);
2126 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_long_composites);
2127 Reqs.addCapability(SPIRV::Capability::LongCompositesINTEL);
2128 break;
2129 }
2130 case SPIRV::OpSubgroupMatrixMultiplyAccumulateINTEL: {
2131 if (.canUseExtension(
2132 SPIRV::Extension::SPV_INTEL_subgroup_matrix_multiply_accumulate))
2134 "OpSubgroupMatrixMultiplyAccumulateINTEL instruction requires the "
2135 "following SPIR-V "
2136 "extension: SPV_INTEL_subgroup_matrix_multiply_accumulate",
2137 false);
2139 SPIRV::Extension::SPV_INTEL_subgroup_matrix_multiply_accumulate);
2141 SPIRV::Capability::SubgroupMatrixMultiplyAccumulateINTEL);
2142 break;
2143 }
2144 case SPIRV::OpBitwiseFunctionINTEL: {
2145 if (.canUseExtension(
2146 SPIRV::Extension::SPV_INTEL_ternary_bitwise_function))
2148 "OpBitwiseFunctionINTEL instruction requires the following SPIR-V "
2149 "extension: SPV_INTEL_ternary_bitwise_function",
2150 false);
2151 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_ternary_bitwise_function);
2152 Reqs.addCapability(SPIRV::Capability::TernaryBitwiseFunctionINTEL);
2153 break;
2154 }
2155 case SPIRV::OpCopyMemorySized: {
2156 Reqs.addCapability(SPIRV::Capability::Addresses);
2157
2158 break;
2159 }
2160 case SPIRV::OpPredicatedLoadINTEL:
2161 case SPIRV::OpPredicatedStoreINTEL: {
2162 if (.canUseExtension(SPIRV::Extension::SPV_INTEL_predicated_io))
2164 "OpPredicated[Load/Store]INTEL instructions require "
2165 "the following SPIR-V extension: SPV_INTEL_predicated_io",
2166 false);
2167 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_predicated_io);
2168 Reqs.addCapability(SPIRV::Capability::PredicatedIOINTEL);
2169 break;
2170 }
2171 case SPIRV::OpFAddS:
2172 case SPIRV::OpFSubS:
2173 case SPIRV::OpFMulS:
2174 case SPIRV::OpFDivS:
2175 case SPIRV::OpFRemS:
2176 case SPIRV::OpFMod:
2177 case SPIRV::OpFNegate:
2178 case SPIRV::OpFAddV:
2179 case SPIRV::OpFSubV:
2180 case SPIRV::OpFMulV:
2181 case SPIRV::OpFDivV:
2182 case SPIRV::OpFRemV:
2183 case SPIRV::OpFNegateV: {
2185 SPIRVType *TypeDef = MRI.getVRegDef(MI.getOperand(1).getReg());
2186 if (TypeDef->getOpcode() == SPIRV::OpTypeVector)
2188 if (isBFloat16Type(TypeDef)) {
2189 if (.canUseExtension(SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic))
2191 "Arithmetic instructions with bfloat16 arguments require the "
2192 "following SPIR-V extension: SPV_INTEL_bfloat16_arithmetic",
2193 false);
2194 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic);
2195 Reqs.addCapability(SPIRV::Capability::BFloat16ArithmeticINTEL);
2196 }
2197 break;
2198 }
2199 case SPIRV::OpOrdered:
2200 case SPIRV::OpUnordered:
2201 case SPIRV::OpFOrdEqual:
2202 case SPIRV::OpFOrdNotEqual:
2203 case SPIRV::OpFOrdLessThan:
2204 case SPIRV::OpFOrdLessThanEqual:
2205 case SPIRV::OpFOrdGreaterThan:
2206 case SPIRV::OpFOrdGreaterThanEqual:
2207 case SPIRV::OpFUnordEqual:
2208 case SPIRV::OpFUnordNotEqual:
2209 case SPIRV::OpFUnordLessThan:
2210 case SPIRV::OpFUnordLessThanEqual:
2211 case SPIRV::OpFUnordGreaterThan:
2212 case SPIRV::OpFUnordGreaterThanEqual: {
2214 MachineInstr *OperandDef = MRI.getVRegDef(MI.getOperand(2).getReg());
2216 if (TypeDef->getOpcode() == SPIRV::OpTypeVector)
2218 if (isBFloat16Type(TypeDef)) {
2219 if (.canUseExtension(SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic))
2221 "Relational instructions with bfloat16 arguments require the "
2222 "following SPIR-V extension: SPV_INTEL_bfloat16_arithmetic",
2223 false);
2224 Reqs.addExtension(SPIRV::Extension::SPV_INTEL_bfloat16_arithmetic);
2225 Reqs.addCapability(SPIRV::Capability::BFloat16ArithmeticINTEL);
2226 }
2227 break;
2228 }
2229 case SPIRV::OpDPdxCoarse:
2230 case SPIRV::OpDPdyCoarse:
2231 case SPIRV::OpDPdxFine:
2232 case SPIRV::OpDPdyFine: {
2233 Reqs.addCapability(SPIRV::Capability::DerivativeControl);
2234 break;
2235 }
2236
2237 default:
2238 break;
2239 }
2240
2241
2242
2243
2245 SPIRV::Capability::Shader);
2246}
2247
2250
2253 if (!MF)
2254 continue;
2257 addInstrRequirements(MI, MAI, ST);
2258 }
2259
2260 auto Node = M.getNamedMetadata("spirv.ExecutionMode");
2262 bool RequireFloatControls = false, RequireIntelFloatControls2 = false,
2263 RequireKHRFloatControls2 = false,
2264 VerLower14 = .isAtLeastSPIRVVer(VersionTuple(1, 4));
2265 bool HasIntelFloatControls2 =
2266 ST.canUseExtension(SPIRV::Extension::SPV_INTEL_float_controls2);
2267 bool HasKHRFloatControls2 =
2268 ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2);
2269 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
2273 Constant *C = CMeta->getValue();
2275 auto EM = Const->getZExtValue();
2276
2277
2278 switch (EM) {
2279 case SPIRV::ExecutionMode::DenormPreserve:
2280 case SPIRV::ExecutionMode::DenormFlushToZero:
2281 case SPIRV::ExecutionMode::RoundingModeRTE:
2282 case SPIRV::ExecutionMode::RoundingModeRTZ:
2283 RequireFloatControls = VerLower14;
2285 SPIRV::OperandCategory::ExecutionModeOperand, EM, ST);
2286 break;
2287 case SPIRV::ExecutionMode::RoundingModeRTPINTEL:
2288 case SPIRV::ExecutionMode::RoundingModeRTNINTEL:
2289 case SPIRV::ExecutionMode::FloatingPointModeALTINTEL:
2290 case SPIRV::ExecutionMode::FloatingPointModeIEEEINTEL:
2291 if (HasIntelFloatControls2) {
2292 RequireIntelFloatControls2 = true;
2294 SPIRV::OperandCategory::ExecutionModeOperand, EM, ST);
2295 }
2296 break;
2297 case SPIRV::ExecutionMode::FPFastMathDefault: {
2298 if (HasKHRFloatControls2) {
2299 RequireKHRFloatControls2 = true;
2301 SPIRV::OperandCategory::ExecutionModeOperand, EM, ST);
2302 }
2303 break;
2304 }
2305 case SPIRV::ExecutionMode::ContractionOff:
2306 case SPIRV::ExecutionMode::SignedZeroInfNanPreserve:
2307 if (HasKHRFloatControls2) {
2308 RequireKHRFloatControls2 = true;
2310 SPIRV::OperandCategory::ExecutionModeOperand,
2311 SPIRV::ExecutionMode::FPFastMathDefault, ST);
2312 } else {
2314 SPIRV::OperandCategory::ExecutionModeOperand, EM, ST);
2315 }
2316 break;
2317 default:
2319 SPIRV::OperandCategory::ExecutionModeOperand, EM, ST);
2320 }
2321 }
2322 }
2323 }
2324 if (RequireFloatControls &&
2325 ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls))
2326 MAI.Reqs.addExtension(SPIRV::Extension::SPV_KHR_float_controls);
2327 if (RequireIntelFloatControls2)
2328 MAI.Reqs.addExtension(SPIRV::Extension::SPV_INTEL_float_controls2);
2329 if (RequireKHRFloatControls2)
2330 MAI.Reqs.addExtension(SPIRV::Extension::SPV_KHR_float_controls2);
2331 }
2333 if (F.isDeclaration())
2334 continue;
2335 if (F.getMetadata("reqd_work_group_size"))
2337 SPIRV::OperandCategory::ExecutionModeOperand,
2338 SPIRV::ExecutionMode::LocalSize, ST);
2339 if (F.getFnAttribute("hlsl.numthreads").isValid()) {
2341 SPIRV::OperandCategory::ExecutionModeOperand,
2342 SPIRV::ExecutionMode::LocalSize, ST);
2343 }
2344 if (F.getFnAttribute("enable-maximal-reconvergence").getValueAsBool()) {
2345 MAI.Reqs.addExtension(SPIRV::Extension::SPV_KHR_maximal_reconvergence);
2346 }
2347 if (F.getMetadata("work_group_size_hint"))
2349 SPIRV::OperandCategory::ExecutionModeOperand,
2350 SPIRV::ExecutionMode::LocalSizeHint, ST);
2351 if (F.getMetadata("intel_reqd_sub_group_size"))
2353 SPIRV::OperandCategory::ExecutionModeOperand,
2354 SPIRV::ExecutionMode::SubgroupSize, ST);
2355 if (F.getMetadata("max_work_group_size"))
2357 SPIRV::OperandCategory::ExecutionModeOperand,
2358 SPIRV::ExecutionMode::MaxWorkgroupSizeINTEL, ST);
2359 if (F.getMetadata("vec_type_hint"))
2361 SPIRV::OperandCategory::ExecutionModeOperand,
2362 SPIRV::ExecutionMode::VecTypeHint, ST);
2363
2364 if (F.hasOptNone()) {
2365 if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_optnone)) {
2368 } else if (ST.canUseExtension(SPIRV::Extension::SPV_EXT_optnone)) {
2371 }
2372 }
2373 }
2374}
2375
2376static unsigned getFastMathFlags(const MachineInstr &I,
2378 unsigned Flags = SPIRV::FPFastMathMode::None;
2379 bool CanUseKHRFloatControls2 =
2380 ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2);
2382 Flags |= SPIRV::FPFastMathMode::NotNaN;
2384 Flags |= SPIRV::FPFastMathMode::NotInf;
2386 Flags |= SPIRV::FPFastMathMode::NSZ;
2388 Flags |= SPIRV::FPFastMathMode::AllowRecip;
2390 Flags |= SPIRV::FPFastMathMode::AllowContract;
2392 if (CanUseKHRFloatControls2)
2393
2394
2395
2396
2397
2398
2399
2400 Flags |= SPIRV::FPFastMathMode::NotNaN | SPIRV::FPFastMathMode::NotInf |
2401 SPIRV::FPFastMathMode::NSZ | SPIRV::FPFastMathMode::AllowRecip |
2402 SPIRV::FPFastMathMode::AllowTransform |
2403 SPIRV::FPFastMathMode::AllowReassoc |
2404 SPIRV::FPFastMathMode::AllowContract;
2405 else
2406 Flags |= SPIRV::FPFastMathMode::Fast;
2407 }
2408
2409 if (CanUseKHRFloatControls2) {
2410
2411 assert(!(Flags & SPIRV::FPFastMathMode::Fast) &&
2412 "SPIRV::FPFastMathMode::Fast is deprecated and should not be used "
2413 "anymore.");
2414
2415
2416
2417 assert((!(Flags & SPIRV::FPFastMathMode::AllowTransform) ||
2418 ((Flags & SPIRV::FPFastMathMode::AllowReassoc &&
2419 Flags & SPIRV::FPFastMathMode::AllowContract))) &&
2420 "SPIRV::FPFastMathMode::AllowTransform requires AllowReassoc and "
2421 "AllowContract flags to be enabled as well.");
2422 }
2423
2424 return Flags;
2425}
2426
2427static bool isFastMathModeAvailable(const SPIRVSubtarget &ST) {
2428 if (ST.isKernel())
2429 return true;
2431 return false;
2432 return ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2);
2433}
2434
2435static void handleMIFlagDecoration(
2440 getSymbolicOperandRequirements(SPIRV::OperandCategory::DecorationOperand,
2441 SPIRV::Decoration::NoSignedWrap, ST, Reqs)
2442 .IsSatisfiable) {
2444 SPIRV::Decoration::NoSignedWrap, {});
2445 }
2447 getSymbolicOperandRequirements(SPIRV::OperandCategory::DecorationOperand,
2448 SPIRV::Decoration::NoUnsignedWrap, ST,
2449 Reqs)
2450 .IsSatisfiable) {
2452 SPIRV::Decoration::NoUnsignedWrap, {});
2453 }
2454 if (.canUseFastMathFlags(
2455 I, ST.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2)))
2456 return;
2457
2458 unsigned FMFlags = getFastMathFlags(I, ST);
2459 if (FMFlags == SPIRV::FPFastMathMode::None) {
2460
2461
2462 if (FPFastMathDefaultInfoVec.empty())
2463 return;
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478 assert(I.getNumOperands() >= 3 && "Expected at least 3 operands");
2479 Register ResReg = I.getOpcode() == SPIRV::OpExtInst
2480 ? I.getOperand(1).getReg()
2481 : I.getOperand(2).getReg();
2485
2486
2487 bool Emit = false;
2489 if (Ty == Elem.Ty) {
2490 FMFlags = Elem.FastMathFlags;
2491 Emit = Elem.ContractionOff || Elem.SignedZeroInfNanPreserve ||
2492 Elem.FPFastMathDefault;
2493 break;
2494 }
2495 }
2496
2497 if (FMFlags == SPIRV::FPFastMathMode::None && !Emit)
2498 return;
2499 }
2500 if (isFastMathModeAvailable(ST)) {
2501 Register DstReg = I.getOperand(0).getReg();
2503 {FMFlags});
2504 }
2505}
2506
2507
2514 if (!MF)
2515 continue;
2516
2517 for (auto &MBB : *MF)
2519 handleMIFlagDecoration(MI, ST, TII, MAI.Reqs, GR,
2521 }
2522}
2523
2529 if (!MF)
2530 continue;
2532 for (auto &MBB : *MF) {
2533 if (.hasName() || MBB.empty())
2534 continue;
2535
2537 MRI.setRegClass(Reg, &SPIRV::IDRegClass);
2541 }
2542 }
2543}
2544
2545
2550 if (!MF)
2551 continue;
2552 for (auto &MBB : *MF) {
2554 MI.setDesc(TII.get(SPIRV::OpPhi));
2557 MI.insert(MI.operands_begin() + 1,
2558 {MachineOperand::CreateReg(ResTypeReg, false)});
2559 }
2560 }
2561
2563 }
2564}
2565
2570 return it->second;
2571
2572
2573
2574
2577 SPIRV::FPFastMathMode::None);
2579 SPIRV::FPFastMathMode::None);
2581 SPIRV::FPFastMathMode::None);
2583}
2584
2587 const Type *Ty) {
2588 size_t BitWidth = Ty->getScalarSizeInBits();
2589 int Index =
2592 assert(Index >= 0 && Index < 3 &&
2593 "Expected FPFastMathDefaultInfo for half, float, or double");
2594 assert(FPFastMathDefaultInfoVec.size() == 3 &&
2595 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
2596 return FPFastMathDefaultInfoVec[Index];
2597}
2598
2599static void collectFPFastMathDefaults(const Module &M,
2602 if (.canUseExtension(SPIRV::Extension::SPV_KHR_float_controls2))
2603 return;
2604
2605
2606
2607
2608
2609
2610
2611 auto Node = M.getNamedMetadata("spirv.ExecutionMode");
2613 return;
2614
2615 for (unsigned i = 0; i < Node->getNumOperands(); i++) {
2620 const auto EM =
2623 ->getZExtValue();
2624 if (EM == SPIRV::ExecutionMode::FPFastMathDefault) {
2626 "Expected 4 operands for FPFastMathDefault");
2627
2629 unsigned Flags =
2632 ->getZExtValue();
2637 Info.FastMathFlags = Flags;
2638 Info.FPFastMathDefault = true;
2639 } else if (EM == SPIRV::ExecutionMode::ContractionOff) {
2641 "Expected no operands for ContractionOff");
2642
2643
2644
2648 Info.ContractionOff = true;
2649 }
2650 } else if (EM == SPIRV::ExecutionMode::SignedZeroInfNanPreserve) {
2652 "Expected 1 operand for SignedZeroInfNanPreserve");
2653 unsigned TargetWidth =
2656 ->getZExtValue();
2657
2662 assert(Index >= 0 && Index < 3 &&
2663 "Expected FPFastMathDefaultInfo for half, float, or double");
2664 assert(FPFastMathDefaultInfoVec.size() == 3 &&
2665 "Expected FPFastMathDefaultInfoVec to have exactly 3 elements");
2666 FPFastMathDefaultInfoVec[Index].SignedZeroInfNanPreserve = true;
2667 }
2668 }
2669}
2683 TII = ST->getInstrInfo();
2684
2686
2687 setBaseInfo(M);
2688
2689 patchPhis(M, GR, *TII, MMI);
2690
2691 addMBBNames(M, *TII, MMI, *ST, MAI);
2692 collectFPFastMathDefaults(M, MAI, *ST);
2693 addDecorations(M, *TII, MMI, *ST, MAI, GR);
2694
2695 collectReqs(M, MAI, MMI, *ST);
2696
2697
2698
2699 collectReqs(M, MAI, MMI, *ST);
2700 collectDeclarations(M);
2701
2702
2703 numberRegistersGlobally(M);
2704
2705
2706 processOtherInstrs(M);
2707
2708
2710 MAI.Reqs.addCapability(SPIRV::Capability::Linkage);
2711
2712
2713 GR->setBound(MAI.MaxID);
2714
2715 return false;
2716}