SPIRVUtils.cpp Source File (original) (raw)

26#include "llvm/IR/IntrinsicsSPIRV.h"

27#include

28#include

30namespace llvm {

39 if (!NMD)

40 return FTy;

44 return nullptr;

47 return nullptr;

48 };

51 if (auto *MDS = dyn_cast_or_null(N->getOperand(0)))

52 return MDS->getString() == Name;

53 return false;

54 });

56 if (It == NMD->op_end())

57 return FTy;

62 for (unsigned I = 1; I != (*It)->getNumOperands(); ++I) {

64 assert(MD && "MDNode operand is expected");

68 assert(CMeta && "ConstantAsMetadata operand is expected");

69 assert(Const->getSExtValue() >= -1);

72 if (Const->getSExtValue() == -1)

73 RetTy = CMeta->getType();

74 else

75 PTys[Const->getSExtValue()] = CMeta->getType();

76 }

77 }

80}

84 F.getParent()->getNamedMetadata("spv.cloned_funcs"), F.getFunctionType(),

85 F.getName());

86}

93}

100 uint32_t Word = 0u;

101 for (unsigned WordIndex = 0; WordIndex < 4; ++WordIndex) {

102 unsigned StrIndex = i + WordIndex;

103 uint8_t CharToAdd = 0;

104 if (StrIndex < Str.size()) {

105 CharToAdd = Str[StrIndex];

106 }

107 Word |= (CharToAdd << (WordIndex * 8));

108 }

109 return Word;

110}

111

112

114 return (Str.size() + 4) & ~3;

115}

116

119 for (unsigned i = 0; i < PaddedLen; i += 4) {

120

122 }

123}

124

127 for (unsigned i = 0; i < PaddedLen; i += 4) {

128

130 }

131}

132

134 std::vector<Value *> &Args) {

136 for (unsigned i = 0; i < PaddedLen; i += 4) {

137

139 }

140}

141

145

148 assert(Def && Def->getOpcode() == TargetOpcode::G_GLOBAL_VALUE &&

149 "Expected G_GLOBAL_VALUE");

150 const GlobalValue *GV = Def->getOperand(1).getGlobal();

154}

155

157 const auto Bitwidth = Imm.getBitWidth();

158 if (Bitwidth == 1)

159 return;

160 else if (Bitwidth <= 32) {

161 MIB.addImm(Imm.getZExtValue());

162

163 if (Bitwidth == 16)

165 return;

166 } else if (Bitwidth <= 64) {

167 uint64_t FullImm = Imm.getZExtValue();

168 uint32_t LowBits = FullImm & 0xffffffff;

169 uint32_t HighBits = (FullImm >> 32) & 0xffffffff;

171

173 return;

174 } else if (Bitwidth <= 128) {

175 uint32_t LowBits = Imm.getRawData()[0] & 0xffffffff;

176 uint32_t MidBits0 = (Imm.getRawData()[0] >> 32) & 0xffffffff;

177 uint32_t MidBits1 = Imm.getRawData()[1] & 0xffffffff;

178 uint32_t HighBits = (Imm.getRawData()[1] >> 32) & 0xffffffff;

180 return;

181 }

183}

184

187 if (!Name.empty()) {

190 }

191}

192

195 if (!Name.empty()) {

196 auto MIB =

197 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpName))

200 }

201}

202

204 const std::vector<uint32_t> &DecArgs,

206 if (!StrImm.empty())

208 for (const auto &DecArg : DecArgs)

210}

211

213 SPIRV::Decoration::Decoration Dec,

214 const std::vector<uint32_t> &DecArgs, StringRef StrImm) {

215 auto MIB = MIRBuilder.buildInstr(SPIRV::OpDecorate)

219}

220

222 SPIRV::Decoration::Decoration Dec,

223 const std::vector<uint32_t> &DecArgs, StringRef StrImm) {

225 auto MIB = BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpDecorate))

229}

230

232 SPIRV::Decoration::Decoration Dec, uint32_t Member,

233 const std::vector<uint32_t> &DecArgs,

235 auto MIB = MIRBuilder.buildInstr(SPIRV::OpMemberDecorate)

240}

241

244 SPIRV::Decoration::Decoration Dec, uint32_t Member,

245 const std::vector<uint32_t> &DecArgs,

248 auto MIB = BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemberDecorate))

253}

254

259 if (!OpMD)

261 if (OpMD->getNumOperands() == 0)

265 if (!DecorationId)

266 report_fatal_error("Expect SPIR-V operand to be the first "

267 "element of the decoration");

268

269

270

271

272

273

274

275

277 static_cast<uint32_t>(SPIRV::Decoration::NoContraction) ||

279 static_cast<uint32_t>(SPIRV::Decoration::FPFastMathMode)) {

280 continue;

281 }

282 auto MIB = MIRBuilder.buildInstr(SPIRV::OpDecorate)

285 for (unsigned OpI = 1, OpE = OpMD->getNumOperands(); OpI != OpE; ++OpI) {

288 MIB.addImm(static_cast<uint32_t>(OpV->getZExtValue()));

291 else

293 }

294 }

295}

296

301 E = MBB->end();

302 bool IsHeader = false;

303 unsigned Opcode;

304 for (; It != E && It != I; ++It) {

305 Opcode = It->getOpcode();

306 if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) {

307 IsHeader = true;

308 } else if (IsHeader &&

309 !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) {

310 ++It;

311 break;

312 }

313 }

314 return It;

315}

316

319 if (I == MBB->begin())

320 return I;

321 --I;

322 while (I->isTerminator() || I->isDebugValue()) {

323 if (I == MBB->begin())

324 break;

325 --I;

326 }

327 return I;

328}

329

330SPIRV::StorageClass::StorageClass

332 switch (AddrSpace) {

333 case 0:

334 return SPIRV::StorageClass::Function;

335 case 1:

336 return SPIRV::StorageClass::CrossWorkgroup;

337 case 2:

338 return SPIRV::StorageClass::UniformConstant;

339 case 3:

340 return SPIRV::StorageClass::Workgroup;

341 case 4:

342 return SPIRV::StorageClass::Generic;

343 case 5:

344 return STI.canUseExtension(SPIRV::Extension::SPV_INTEL_usm_storage_classes)

345 ? SPIRV::StorageClass::DeviceOnlyINTEL

346 : SPIRV::StorageClass::CrossWorkgroup;

347 case 6:

348 return STI.canUseExtension(SPIRV::Extension::SPV_INTEL_usm_storage_classes)

349 ? SPIRV::StorageClass::HostOnlyINTEL

350 : SPIRV::StorageClass::CrossWorkgroup;

351 case 7:

352 return SPIRV::StorageClass::Input;

353 case 8:

354 return SPIRV::StorageClass::Output;

355 case 9:

356 return SPIRV::StorageClass::CodeSectionINTEL;

357 case 10:

358 return SPIRV::StorageClass::Private;

359 case 11:

360 return SPIRV::StorageClass::StorageBuffer;

361 case 12:

362 return SPIRV::StorageClass::Uniform;

363 default:

365 }

366}

367

368SPIRV::MemorySemantics::MemorySemantics

370 switch (SC) {

371 case SPIRV::StorageClass::StorageBuffer:

372 case SPIRV::StorageClass::Uniform:

373 return SPIRV::MemorySemantics::UniformMemory;

374 case SPIRV::StorageClass::Workgroup:

375 return SPIRV::MemorySemantics::WorkgroupMemory;

376 case SPIRV::StorageClass::CrossWorkgroup:

377 return SPIRV::MemorySemantics::CrossWorkgroupMemory;

378 case SPIRV::StorageClass::AtomicCounter:

379 return SPIRV::MemorySemantics::AtomicCounterMemory;

380 case SPIRV::StorageClass::Image:

381 return SPIRV::MemorySemantics::ImageMemory;

382 default:

383 return SPIRV::MemorySemantics::None;

384 }

385}

386

388 switch (Ord) {

390 return SPIRV::MemorySemantics::Acquire;

392 return SPIRV::MemorySemantics::Release;

394 return SPIRV::MemorySemantics::AcquireRelease;

396 return SPIRV::MemorySemantics::SequentiallyConsistent;

400 return SPIRV::MemorySemantics::None;

401 }

403}

404

406

407

408

409

410

412 Ctx.getOrInsertSyncScopeID("subgroup");

414 Ctx.getOrInsertSyncScopeID("workgroup");

416 Ctx.getOrInsertSyncScopeID("device");

417

419 return SPIRV::Scope::Invocation;

421 return SPIRV::Scope::CrossDevice;

422 else if (Id == SubGroup)

423 return SPIRV::Scope::Subgroup;

424 else if (Id == WorkGroup)

425 return SPIRV::Scope::Workgroup;

426 else if (Id == Device)

427 return SPIRV::Scope::Device;

428 return SPIRV::Scope::CrossDevice;

429}

430

435 MI->getOpcode() == SPIRV::G_TRUNC || MI->getOpcode() == SPIRV::G_ZEXT

436 ? MRI->getVRegDef(MI->getOperand(1).getReg())

437 : MI;

439 if (GI->is(Intrinsic::spv_track_constant)) {

441 return MRI->getVRegDef(ConstReg);

442 }

443 } else if (ConstInstr->getOpcode() == SPIRV::ASSIGN_TYPE) {

445 return MRI->getVRegDef(ConstReg);

446 } else if (ConstInstr->getOpcode() == TargetOpcode::G_CONSTANT ||

447 ConstInstr->getOpcode() == TargetOpcode::G_FCONSTANT) {

449 return ConstInstr;

450 }

451 return MRI->getVRegDef(ConstReg);

452}

453

456 assert(MI && MI->getOpcode() == TargetOpcode::G_CONSTANT);

457 return MI->getOperand(1).getCImm()->getValue().getZExtValue();

458}

459

462 assert(MI && MI->getOpcode() == TargetOpcode::G_CONSTANT);

463 return MI->getOperand(1).getCImm()->getSExtValue();

464}

465

468 return GI->is(IntrinsicID);

469 return false;

470}

471

476

477

478

480 return MangledName == "write_pipe_2" || MangledName == "read_pipe_2" ||

481 MangledName == "write_pipe_2_bl" || MangledName == "read_pipe_2_bl" ||

482 MangledName == "write_pipe_4" || MangledName == "read_pipe_4" ||

483 MangledName == "reserve_write_pipe" ||

484 MangledName == "reserve_read_pipe" ||

485 MangledName == "commit_write_pipe" ||

486 MangledName == "commit_read_pipe" ||

487 MangledName == "work_group_reserve_write_pipe" ||

488 MangledName == "work_group_reserve_read_pipe" ||

489 MangledName == "work_group_commit_write_pipe" ||

490 MangledName == "work_group_commit_read_pipe" ||

491 MangledName == "get_pipe_num_packets_ro" ||

492 MangledName == "get_pipe_max_packets_ro" ||

493 MangledName == "get_pipe_num_packets_wo" ||

494 MangledName == "get_pipe_max_packets_wo" ||

495 MangledName == "sub_group_reserve_write_pipe" ||

496 MangledName == "sub_group_reserve_read_pipe" ||

497 MangledName == "sub_group_commit_write_pipe" ||

498 MangledName == "sub_group_commit_read_pipe" ||

499 MangledName == "to_global" || MangledName == "to_local" ||

500 MangledName == "to_private";

501}

502

504 return MangledName == "__enqueue_kernel_basic" ||

505 MangledName == "__enqueue_kernel_basic_events" ||

506 MangledName == "__enqueue_kernel_varargs" ||

507 MangledName == "__enqueue_kernel_events_varargs";

508}

509

511 return MangledName == "__get_kernel_work_group_size_impl" ||

512 MangledName == "__get_kernel_sub_group_count_for_ndrange_impl" ||

513 MangledName == "__get_kernel_max_sub_group_size_for_ndrange_impl" ||

514 MangledName == "__get_kernel_preferred_work_group_size_multiple_impl";

515}

516

518 if (!Name.starts_with("__"))

519 return false;

520

523 Name == "__translate_sampler_initializer";

524}

525

528 bool IsNonMangledSPIRV = Name.starts_with("__spirv_");

529 bool IsNonMangledHLSL = Name.starts_with("__hlsl_");

530 bool IsMangled = Name.starts_with("_Z");

531

532

533 if (IsNonMangledOCL || IsNonMangledSPIRV || IsNonMangledHLSL || !IsMangled)

534 return Name.str();

535

536

537 if (char *DemangledName = itaniumDemangle(Name.data())) {

538 std::string Result = DemangledName;

539 free(DemangledName);

540 return Result;

541 }

542

543

544

545

546

547 size_t Start, Len = 0;

548 size_t DemangledNameLenStart = 2;

549 if (Name.starts_with("_ZN")) {

550

551 size_t NameSpaceStart = Name.find_first_not_of("rVKRO", 3);

552

553 if (Name.substr(NameSpaceStart, 11) != "2cl7__spirv")

554 return std::string();

555 DemangledNameLenStart = NameSpaceStart + 11;

556 }

557 Start = Name.find_first_not_of("0123456789", DemangledNameLenStart);

558 [[maybe_unused]] bool Error =

559 Name.substr(DemangledNameLenStart, Start - DemangledNameLenStart)

560 .getAsInteger(10, Len);

561 assert( Error && "Failed to parse demangled name length");

562 return Name.substr(Start, Len).str();

563}

564

566 if (Name.starts_with("opencl.") || Name.starts_with("ocl_") ||

567 Name.starts_with("spirv."))

568 return true;

569 return false;

570}

571

577

578 return false;

579}

580

582

583

585 return true;

586

587

588

589 if (F.getFnAttribute("hlsl.shader").isValid())

590 return true;

591

592 return false;

593}

594

596 TypeName.consume_front("atomic_");

597 if (TypeName.consume_front("void"))

599 else if (TypeName.consume_front("bool") || TypeName.consume_front("_Bool"))

601 else if (TypeName.consume_front("char") ||

602 TypeName.consume_front("signed char") ||

603 TypeName.consume_front("unsigned char") ||

604 TypeName.consume_front("uchar"))

606 else if (TypeName.consume_front("short") ||

607 TypeName.consume_front("signed short") ||

608 TypeName.consume_front("unsigned short") ||

609 TypeName.consume_front("ushort"))

611 else if (TypeName.consume_front("int") ||

612 TypeName.consume_front("signed int") ||

613 TypeName.consume_front("unsigned int") ||

614 TypeName.consume_front("uint"))

616 else if (TypeName.consume_front("long") ||

617 TypeName.consume_front("signed long") ||

618 TypeName.consume_front("unsigned long") ||

619 TypeName.consume_front("ulong"))

621 else if (TypeName.consume_front("half") ||

622 TypeName.consume_front("_Float16") ||

623 TypeName.consume_front("__fp16"))

625 else if (TypeName.consume_front("float"))

627 else if (TypeName.consume_front("double"))

629

630

631 return nullptr;

632}

633

634std::unordered_set<BasicBlock *>

635PartialOrderingVisitor::getReachableFrom(BasicBlock *Start) {

636 std::queue<BasicBlock *> ToVisit;

637 ToVisit.push(Start);

638

639 std::unordered_set<BasicBlock *> Output;

640 while (ToVisit.size() != 0) {

641 BasicBlock *BB = ToVisit.front();

642 ToVisit.pop();

643

644 if (Output.count(BB) != 0)

645 continue;

646 Output.insert(BB);

647

650 continue;

652 }

653 }

654

655 return Output;

656}

657

658bool PartialOrderingVisitor::CanBeVisited(BasicBlock *BB) const {

660

661 if (DT.dominates(BB, P))

662 continue;

663

664

665 if (BlockToOrder.count(P) == 0)

666 return false;

667

668

669

670 Loop *L = LI.getLoopFor(P);

671 if (L == nullptr || L->contains(BB))

672 continue;

673

674

675

676

677 assert(L->getNumBackEdges() <= 1);

678

679

680

683 if (Latch == nullptr)

684 continue;

685

686

687 if (BlockToOrder.count(Latch) == 0)

688 return false;

689 }

690

691 return true;

692}

693

695 auto It = BlockToOrder.find(BB);

696 if (It != BlockToOrder.end())

697 return It->second.Rank;

698

699 size_t result = 0;

701

702 if (DT.dominates(BB, P))

703 continue;

704

705 auto Iterator = BlockToOrder.end();

706 Loop *L = LI.getLoopFor(P);

707 BasicBlock *Latch = L ? L->getLoopLatch() : nullptr;

708

709

710

711 if (L == nullptr || L->contains(BB) || Latch == nullptr) {

712 Iterator = BlockToOrder.find(P);

713 } else {

714

715

716

717 Iterator = BlockToOrder.find(Latch);

718 }

719

720 assert(Iterator != BlockToOrder.end());

721 result = std::max(result, Iterator->second.Rank + 1);

722 }

723

724 return result;

725}

726

727size_t PartialOrderingVisitor::visit(BasicBlock *BB, size_t Unused) {

728 ToVisit.push(BB);

729 Queued.insert(BB);

730

731 size_t QueueIndex = 0;

732 while (ToVisit.size() != 0) {

734 ToVisit.pop();

735

736 if (!CanBeVisited(BB)) {

737 ToVisit.push(BB);

738 if (QueueIndex >= ToVisit.size())

740 "No valid candidate in the queue. Is the graph reducible?");

741 QueueIndex++;

742 continue;

743 }

744

745 QueueIndex = 0;

747 OrderInfo Info = {Rank, BlockToOrder.size()};

748 BlockToOrder.emplace(BB, Info);

749

750 for (BasicBlock *S : successors(BB)) {

751 if (Queued.count(S) != 0)

752 continue;

753 ToVisit.push(S);

754 Queued.insert(S);

755 }

756 }

757

758 return 0;

759}

760

762 DT.recalculate(F);

764

765 visit(&*F.begin(), 0);

766

767 Order.reserve(F.size());

768 for (auto &[BB, Info] : BlockToOrder)

769 Order.emplace_back(BB);

770

771 std::sort(Order.begin(), Order.end(), [&](const auto &LHS, const auto &RHS) {

772 return compare(LHS, RHS);

773 });

774}

775

778 const OrderInfo &InfoLHS = BlockToOrder.at(const_cast<BasicBlock *>(LHS));

779 const OrderInfo &InfoRHS = BlockToOrder.at(const_cast<BasicBlock *>(RHS));

780 if (InfoLHS.Rank != InfoRHS.Rank)

781 return InfoLHS.Rank < InfoRHS.Rank;

782 return InfoLHS.TraversalIndex < InfoRHS.TraversalIndex;

783}

784

787 std::unordered_set<BasicBlock *> Reachable = getReachableFrom(&Start);

788 assert(BlockToOrder.count(&Start) != 0);

789

790

791 auto It = Order.begin();

792 while (It != Order.end() && *It != &Start)

793 ++It;

794

795

796

797 assert(It != Order.end());

798

799

800 std::optional<size_t> EndRank = std::nullopt;

801 for (; It != Order.end(); ++It) {

802 if (EndRank.has_value() && BlockToOrder[*It].Rank > *EndRank)

803 break;

804

805 if (Reachable.count(*It) == 0) {

806 continue;

807 }

808

809 if ((*It)) {

810 EndRank = BlockToOrder[*It].Rank;

811 }

812 }

813}

814

816 if (F.size() == 0)

817 return false;

818

820 std::vector<BasicBlock *> Order;

821 Order.reserve(F.size());

822

825

826 assert(&*F.begin() == Order[0]);

829 if (BB != LastBlock && &*LastBlock->getNextNode() != BB) {

832 }

833 LastBlock = BB;

834 }

835

837}

838

841 if (MaybeDef && MaybeDef->getOpcode() == SPIRV::ASSIGN_TYPE)

843 return MaybeDef;

844}

845

847

848

849 constexpr unsigned MaxIters = 1024;

850 for (unsigned I = 0; I < MaxIters; ++I) {

851 std::string OrdName = Name + Twine(I).str();

852 if (!M.getFunction(OrdName)) {

853 Name = std::move(OrdName);

854 return true;

855 }

856 }

857 return false;

858}

859

860

861

864 bool Force) {

866 if ( MRI ->getRegClassOrNull(Reg) || Force) {

869 }

870}

871

872

873

874

877 SPIRV::AccessQualifier::AccessQualifier AccessQual,

878 bool EmitIR, bool Force) {

881 GR, MIRBuilder.getMRI(), MIRBuilder.getMF(), Force);

882}

883

884

885

894

895

896

900 MIRBuilder.getMF());

901}

902

903

904

907 SPIRV::AccessQualifier::AccessQualifier AccessQual, bool EmitIR) {

910 MIRBuilder);

911}

912

917 Args.push_back(Arg2);

918 Args.push_back(buildMD(Arg));

920 return B.CreateIntrinsic(IntrID, {Types}, Args);

921}

922

923

925 if (Ty->isPtrOrPtrVectorTy())

926 return true;

929 return true;

930 for (const Type *ArgTy : RefTy->params())

932 return true;

933 return false;

934 }

937 return false;

938}

939

942 if (Function *F = II->getCalledFunction())

943 if (F->getName().starts_with("llvm.spv."))

944 return true;

945 return false;

946}

947

948

949

950SmallVector<MachineInstr *, 4>

952 unsigned MinWC, unsigned ContinuedOpcode,

955

957 constexpr unsigned MaxWordCount = UINT16_MAX;

958 const size_t NumElements = Args.size();

959 size_t MaxNumElements = MaxWordCount - MinWC;

960 size_t SPIRVStructNumElements = NumElements;

961

962 if (NumElements > MaxNumElements) {

963

964

965 SPIRVStructNumElements = MaxNumElements;

966 MaxNumElements = MaxWordCount - 1;

967 }

968

969 auto MIB =

971

972 for (size_t I = 0; I < SPIRVStructNumElements; ++I)

974

975 Instructions.push_back(MIB.getInstr());

976

977 for (size_t I = SPIRVStructNumElements; I < NumElements;

978 I += MaxNumElements) {

979 auto MIB = MIRBuilder.buildInstr(ContinuedOpcode);

980 for (size_t J = I; J < std::min(I + MaxNumElements, NumElements); ++J)

982 Instructions.push_back(MIB.getInstr());

983 }

984 return Instructions;

985}

986

988 unsigned LC = SPIRV::LoopControl::None;

989

990

991

992 std::vector<std::pair<unsigned, unsigned>> MaskToValueMap;

994 LC |= SPIRV::LoopControl::DontUnroll;

995 } else {

998 LC |= SPIRV::LoopControl::Unroll;

999 }

1000 std::optional Count =

1003 LC |= SPIRV::LoopControl::PartialCount;

1004 MaskToValueMap.emplace_back(

1005 std::make_pair(SPIRV::LoopControl::PartialCount, *Count));

1006 }

1007 }

1009 for (auto &[Mask, Val] : MaskToValueMap)

1010 Result.push_back(Val);

1011 return Result;

1012}

1013

1015

1016 static const std::set TypeFoldingSupportingOpcs = {

1017 TargetOpcode::G_ADD,

1018 TargetOpcode::G_FADD,

1019 TargetOpcode::G_STRICT_FADD,

1020 TargetOpcode::G_SUB,

1021 TargetOpcode::G_FSUB,

1022 TargetOpcode::G_STRICT_FSUB,

1023 TargetOpcode::G_MUL,

1024 TargetOpcode::G_FMUL,

1025 TargetOpcode::G_STRICT_FMUL,

1026 TargetOpcode::G_SDIV,

1027 TargetOpcode::G_UDIV,

1028 TargetOpcode::G_FDIV,

1029 TargetOpcode::G_STRICT_FDIV,

1030 TargetOpcode::G_SREM,

1031 TargetOpcode::G_UREM,

1032 TargetOpcode::G_FREM,

1033 TargetOpcode::G_STRICT_FREM,

1034 TargetOpcode::G_FNEG,

1035 TargetOpcode::G_CONSTANT,

1036 TargetOpcode::G_FCONSTANT,

1037 TargetOpcode::G_AND,

1038 TargetOpcode::G_OR,

1039 TargetOpcode::G_XOR,

1040 TargetOpcode::G_SHL,

1041 TargetOpcode::G_ASHR,

1042 TargetOpcode::G_LSHR,

1043 TargetOpcode::G_SELECT,

1044 TargetOpcode::G_EXTRACT_VECTOR_ELT,

1045 };

1046

1047 return TypeFoldingSupportingOpcs;

1048}

1049

1053

1054

1056 return (Def->getOpcode() == SPIRV::ASSIGN_TYPE ||

1057 Def->getOpcode() == TargetOpcode::COPY)

1058 ? MRI->getVRegDef(Def->getOperand(1).getReg())

1059 : Def;

1060}

1061

1065 return nullptr;

1066}

1067

1070 if (Def->getOpcode() == TargetOpcode::G_CONSTANT ||

1071 Def->getOpcode() == SPIRV::OpConstantI)

1072 return Def;

1073 }

1074 return nullptr;

1075}

1076

1079 if (Def->getOpcode() == SPIRV::OpConstantI)

1080 return Def->getOperand(2).getImm();

1081 if (Def->getOpcode() == TargetOpcode::G_CONSTANT)

1082 return Def->getOperand(1).getCImm()->getZExtValue();

1083 }

1085}

1086

1091

1094

1095

1096

1098 while (VarPos != BB.end() && VarPos->getOpcode() != SPIRV::OpFunction) {

1099 ++VarPos;

1100 }

1101

1102

1103

1104 ++VarPos;

1105 while (VarPos != BB.end() &&

1106 VarPos->getOpcode() == SPIRV::OpFunctionParameter) {

1107 ++VarPos;

1108 }

1109

1110

1111 return VarPos != BB.end() && VarPos->getOpcode() == SPIRV::OpLabel ? ++VarPos

1112 : VarPos;

1113}

1114

1117

1118 if (Ty->getStructNumElements() != 2)

1119 return false;

1120

1123

1125 return false;

1126

1128 if (!ArrayElementType->isStructTy() ||

1130 return false;

1131

1133 if (T_in_struct != SecondElement)

1134 return false;

1135

1136 auto *Padding_in_struct =

1138 if (!Padding_in_struct || Padding_in_struct->getName() != "spirv.Padding")

1139 return false;

1140

1142 TotalSize = ArraySize + 1;

1143 OriginalElementType = ArrayElementType;

1144 return true;

1145}

1146

1148 if (!Ty->isStructTy())

1149 return Ty;

1150

1152 Type *OriginalElementType = nullptr;

1157 return ResultTy;

1158 }

1159

1162 for (Type *ElementTy : STy->elements()) {

1164 if (NewElementTy != ElementTy)

1166 NewElementTypes.push_back(NewElementTy);

1167 }

1168

1170 return Ty;

1171

1172 Type *ResultTy;

1173 if (STy->isLiteral())

1174 ResultTy =

1175 StructType::get(STy->getContext(), NewElementTypes, STy->isPacked());

1176 else {

1177 auto *NewTy = StructType::create(STy->getContext(), STy->getName());

1178 NewTy->setBody(NewElementTypes, STy->isPacked());

1179 ResultTy = NewTy;

1180 }

1181 return ResultTy;

1182}

1183

1184std::optionalSPIRV::LinkageType::LinkageType

1187 return std::nullopt;

1188

1190 return SPIRV::LinkageType::Import;

1191

1193 ST.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr))

1194 return SPIRV::LinkageType::LinkOnceODR;

1195

1196 return SPIRV::LinkageType::Export;

1197}

1198

1199}

unsigned const MachineRegisterInfo * MRI

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

Analysis containing CSE Info

Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...

const HexagonInstrInfo * TII

This file declares the MachineIRBuilder class.

uint64_t IntrinsicInst * II

static ConstantInt * getConstInt(MDNode *MD, unsigned NumOp)

Class for arbitrary precision integers.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

Class to represent array types.

static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)

This static method is the primary way to construct an ArrayType.

LLVM Basic Block Representation.

LLVM_ABI void moveAfter(BasicBlock *MovePos)

Unlink this basic block from its current function and insert it right after MovePos in the function M...

const Instruction & front() const

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

FunctionType * getFunctionType() const

This class represents a function call, abstracting a target machine's calling convention.

An array constant whose element type is a simple 1/2/4/8-byte integer or float/double,...

StringRef getAsCString() const

If this array is isCString(), then this method returns the array (without the trailing null byte) as ...

This is the shared class of boolean and integer constants.

uint64_t getZExtValue() const

Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...

bool dominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const

dominates - Returns true iff A dominates B.

Lightweight error class with error context and mandatory checking.

Class to represent function types.

ArrayRef< Type * > params() const

Type * getReturnType() const

static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)

This static method is the primary way of constructing a FunctionType.

bool hasLocalLinkage() const

bool hasHiddenVisibility() const

bool isDeclarationForLinker() const

bool hasLinkOnceODRLinkage() const

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

LLVM_ABI const Module * getModule() const

Return the module owning the function this instruction belongs to or nullptr it the function does not...

This is an important class for using LLVM in a threaded context.

Represents a single loop in the control flow graph.

Instances of this class represent a single low-level machine instruction.

void addOperand(const MCOperand Op)

static MCOperand createImm(int64_t Val)

const MDOperand & getOperand(unsigned I) const

unsigned getNumOperands() const

Return number of MDNode operands.

MachineInstrBundleIterator< MachineInstr > iterator

const MachineBasicBlock & front() const

Helper class to build MachineInstr.

MachineInstrBuilder buildInstr(unsigned Opcode)

Build and insert = Opcode .

MachineFunction & getMF()

Getter for the function we currently build.

MachineRegisterInfo * getMRI()

Getter for MRI.

const MachineInstrBuilder & addImm(int64_t Val) const

Add a new immediate operand.

const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const

Add a virtual register use operand.

MachineInstr * getInstr() const

If conversion operators fail, use this method to get the MachineInstr explicitly.

const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const

Add a virtual register definition operand.

Representation of each machine instruction.

unsigned getOpcode() const

Returns the opcode of this MachineInstr.

void setAsmPrinterFlag(uint8_t Flag)

Set a flag for the AsmPrinter.

const MachineOperand & getOperand(unsigned i) const

MachineOperand class - Representation of each machine instruction operand.

getReg - Returns the register number.

MachineRegisterInfo - Keep track of information for virtual and physical registers,...

A Module instance is used to store all the information related to an LLVM module.

NamedMDNode * getNamedMetadata(StringRef Name) const

Return the first NamedMDNode in the module with the specified name.

iterator_range< op_iterator > operands()

size_t GetNodeRank(BasicBlock *BB) const

Definition SPIRVUtils.cpp:694

void partialOrderVisit(BasicBlock &Start, std::function< bool(BasicBlock *)> Op)

Definition SPIRVUtils.cpp:785

bool compare(const BasicBlock *LHS, const BasicBlock *RHS) const

Definition SPIRVUtils.cpp:776

PartialOrderingVisitor(Function &F)

Definition SPIRVUtils.cpp:761

Wrapper class representing virtual and physical registers.

void assignSPIRVTypeToVReg(SPIRVType *Type, Register VReg, const MachineFunction &MF)

SPIRVType * getOrCreateSPIRVType(const Type *Type, MachineInstr &I, SPIRV::AccessQualifier::AccessQualifier AQ, bool EmitIR)

const TargetRegisterClass * getRegClass(SPIRVType *SpvType) const

LLT getRegType(SPIRVType *SpvType) const

bool canUseExtension(SPIRV::Extension::Extension E) const

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

std::string str() const

str - Get the contents as an std::string.

constexpr bool empty() const

empty - Check if the string is empty.

Class to represent struct types.

static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)

This static method is the primary way to create a literal StructType.

static LLVM_ABI StructType * create(LLVMContext &Context, StringRef Name)

This creates an identified struct.

Class to represent target extensions types, which are generally unintrospectable from target-independ...

Target - Wrapper for Target specific information.

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

LLVM_ABI std::string str() const

Return the twine contents as a std::string.

The instances of the Type class are immutable: once they are created, they are never changed.

static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)

LLVM_ABI Type * getStructElementType(unsigned N) const

bool isArrayTy() const

True if this is an instance of ArrayType.

static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)

Type * getArrayElementType() const

LLVM_ABI unsigned getStructNumElements() const

LLVM_ABI uint64_t getArrayNumElements() const

static LLVM_ABI Type * getVoidTy(LLVMContext &C)

static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)

bool isStructTy() const

True if this is an instance of StructType.

static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)

static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)

static LLVM_ABI Type * getDoubleTy(LLVMContext &C)

static LLVM_ABI Type * getFloatTy(LLVMContext &C)

static LLVM_ABI Type * getHalfTy(LLVMContext &C)

Value * getOperand(unsigned i) const

LLVM Value Representation.

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

NodeTy * getNextNode()

Get the next node, or nullptr for the list tail.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

@ SPIR_KERNEL

Used for SPIR kernel functions.

@ BasicBlock

Various leaf nodes.

FunctionType * getOriginalFunctionType(const Function &F)

Definition SPIRVUtils.cpp:82

static FunctionType * extractFunctionTypeFromMetadata(NamedMDNode *NMD, FunctionType *FTy, StringRef Name)

Definition SPIRVUtils.cpp:36

@ SingleThread

Synchronized with respect to signal handlers executing in the same thread.

@ System

Synchronized with respect to all concurrently executing threads.

std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)

Extract a Value from Metadata, if any.

This is an optimization pass for GlobalISel generic memory operations.

void buildOpName(Register Target, const StringRef &Name, MachineIRBuilder &MIRBuilder)

Definition SPIRVUtils.cpp:185

bool getVacantFunctionName(Module &M, std::string &Name)

Definition SPIRVUtils.cpp:846

std::string getStringImm(const MachineInstr &MI, unsigned StartIndex)

Definition SPIRVUtils.cpp:142

LLVM_ABI bool getBooleanLoopAttribute(const Loop *TheLoop, StringRef Name)

Returns true if Name is applied to TheLoop and enabled.

int64_t getIConstValSext(Register ConstReg, const MachineRegisterInfo *MRI)

Definition SPIRVUtils.cpp:460

bool isTypedPointerWrapper(const TargetExtType *ExtTy)

MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)

Builder interface. Specify how to create the initial instruction itself.

static void finishBuildOpDecorate(MachineInstrBuilder &MIB, const std::vector< uint32_t > &DecArgs, StringRef StrImm)

Definition SPIRVUtils.cpp:203

bool isTypeFoldingSupported(unsigned Opcode)

Definition SPIRVUtils.cpp:1050

static uint32_t convertCharsToWord(const StringRef &Str, unsigned i)

Definition SPIRVUtils.cpp:99

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

MachineInstr * getDef(const MachineOperand &MO, const MachineRegisterInfo *MRI)

Definition SPIRVUtils.cpp:1062

void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB)

Definition SPIRVUtils.cpp:156

auto successors(const MachineBasicBlock *BB)

CallInst * buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef< Type * > Types, Value *Arg, Value *Arg2, ArrayRef< Constant * > Imms, IRBuilder<> &B)

Definition SPIRVUtils.cpp:913

bool matchPeeledArrayPattern(const StructType *Ty, Type *&OriginalElementType, uint64_t &TotalSize)

Definition SPIRVUtils.cpp:1115

void append_range(Container &C, Range &&R)

Wrapper function to append range R to container C.

unsigned getArrayComponentCount(const MachineRegisterInfo *MRI, const MachineInstr *ResType)

Definition SPIRVUtils.cpp:1087

bool sortBlocks(Function &F)

Definition SPIRVUtils.cpp:815

SmallVector< unsigned, 1 > getSpirvLoopControlOperandsFromLoopMetadata(Loop *L)

Definition SPIRVUtils.cpp:987

uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI)

Definition SPIRVUtils.cpp:454

SmallVector< MachineInstr *, 4 > createContinuedInstructions(MachineIRBuilder &MIRBuilder, unsigned Opcode, unsigned MinWC, unsigned ContinuedOpcode, ArrayRef< Register > Args, Register ReturnRegister, Register TypeID)

Definition SPIRVUtils.cpp:951

SPIRV::MemorySemantics::MemorySemantics getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC)

Definition SPIRVUtils.cpp:369

MachineBasicBlock::iterator getFirstValidInstructionInsertPoint(MachineBasicBlock &BB)

Definition SPIRVUtils.cpp:1093

bool isNestedPointer(const Type *Ty)

Definition SPIRVUtils.cpp:924

MetadataAsValue * buildMD(Value *Arg)

std::string getOclOrSpirvBuiltinDemangledName(StringRef Name)

Definition SPIRVUtils.cpp:526

void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder, SPIRV::Decoration::Decoration Dec, const std::vector< uint32_t > &DecArgs, StringRef StrImm)

Definition SPIRVUtils.cpp:212

MachineBasicBlock::iterator getOpVariableMBBIt(MachineInstr &I)

Definition SPIRVUtils.cpp:297

Register createVirtualRegister(SPIRVType *SpvType, SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI, const MachineFunction &MF)

Definition SPIRVUtils.cpp:886

MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)

Definition SPIRVUtils.cpp:1068

std::string getSPIRVStringOperand(const InstType &MI, unsigned StartIndex)

void buildOpMemberDecorate(Register Reg, MachineIRBuilder &MIRBuilder, SPIRV::Decoration::Decoration Dec, uint32_t Member, const std::vector< uint32_t > &DecArgs, StringRef StrImm)

Definition SPIRVUtils.cpp:231

Type * toTypedPointer(Type *Ty)

DEMANGLE_ABI char * itaniumDemangle(std::string_view mangled_name, bool ParseParams=true)

Returns a non-NULL pointer to a NUL-terminated C style string that should be explicitly freed,...

bool isSpecialOpaqueType(const Type *Ty)

Definition SPIRVUtils.cpp:572

void setRegClassType(Register Reg, SPIRVType *SpvType, SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI, const MachineFunction &MF, bool Force)

Definition SPIRVUtils.cpp:862

LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)

MachineBasicBlock::iterator getInsertPtValidEnd(MachineBasicBlock *MBB)

Definition SPIRVUtils.cpp:317

FunctionAddr VTableAddr Count

const MachineInstr SPIRVType

static bool isNonMangledOCLBuiltin(StringRef Name)

Definition SPIRVUtils.cpp:517

MachineInstr * passCopy(MachineInstr *Def, const MachineRegisterInfo *MRI)

Definition SPIRVUtils.cpp:1055

std::optional< SPIRV::LinkageType::LinkageType > getSpirvLinkageTypeFor(const SPIRVSubtarget &ST, const GlobalValue &GV)

Definition SPIRVUtils.cpp:1185

bool isEntryPoint(const Function &F)

Definition SPIRVUtils.cpp:581

const std::set< unsigned > & getTypeFoldingSupportedOpcodes()

Definition SPIRVUtils.cpp:1014

SPIRV::StorageClass::StorageClass addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI)

Definition SPIRVUtils.cpp:331

LLVM_ABI std::optional< int > getOptionalIntLoopAttribute(const Loop *TheLoop, StringRef Name)

Find named metadata for a loop with an integer value.

AtomicOrdering

Atomic ordering for LLVM's memory model.

SPIRV::Scope::Scope getMemScope(LLVMContext &Ctx, SyncScope::ID Id)

Definition SPIRVUtils.cpp:405

static bool isPipeOrAddressSpaceCastBI(const StringRef MangledName)

Definition SPIRVUtils.cpp:479

void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder, const MDNode *GVarMD, const SPIRVSubtarget &ST)

Definition SPIRVUtils.cpp:255

std::string getStringValueFromReg(Register Reg, MachineRegisterInfo &MRI)

Definition SPIRVUtils.cpp:146

int64_t foldImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)

Definition SPIRVUtils.cpp:1077

Type * parseBasicTypeName(StringRef &TypeName, LLVMContext &Ctx)

Definition SPIRVUtils.cpp:595

DWARFExpression::Operation Op

MachineInstr * getDefInstrMaybeConstant(Register &ConstReg, const MachineRegisterInfo *MRI)

Definition SPIRVUtils.cpp:431

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

bool hasBuiltinTypePrefix(StringRef Name)

Definition SPIRVUtils.cpp:565

Type * getMDOperandAsType(const MDNode *N, unsigned I)

Definition SPIRVUtils.cpp:472

auto find_if(R &&Range, UnaryPredicate P)

Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.

auto predecessors(const MachineBasicBlock *BB)

static size_t getPaddedLen(const StringRef &Str)

Definition SPIRVUtils.cpp:113

bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID)

Definition SPIRVUtils.cpp:466

void addStringImm(const StringRef &Str, MCInst &Inst)

Definition SPIRVUtils.cpp:117

static bool isKernelQueryBI(const StringRef MangledName)

Definition SPIRVUtils.cpp:510

MachineInstr * getVRegDef(MachineRegisterInfo &MRI, Register Reg)

Definition SPIRVUtils.cpp:839

static bool isEnqueueKernelBI(const StringRef MangledName)

Definition SPIRVUtils.cpp:503

Type * reconstitutePeeledArrayType(Type *Ty)

Definition SPIRVUtils.cpp:1147

SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord)

Definition SPIRVUtils.cpp:387