SPIRVUtils.cpp Source File (original) (raw)

26#include "llvm/IR/IntrinsicsSPIRV.h"

27#include

28#include

30namespace llvm {

39 if (!NMD)

40 return FTy;

44 return nullptr;

47 return nullptr;

48 };

51 if (auto *MDS = dyn_cast_or_null(N->getOperand(0)))

52 return MDS->getString() == Name;

53 return false;

54 });

56 if (It == NMD->op_end())

57 return FTy;

62 for (unsigned I = 1; I != (*It)->getNumOperands(); ++I) {

64 assert(MD && "MDNode operand is expected");

68 assert(CMeta && "ConstantAsMetadata operand is expected");

69 assert(Const->getSExtValue() >= -1);

72 if (Const->getSExtValue() == -1)

73 RetTy = CMeta->getType();

74 else

75 PTys[Const->getSExtValue()] = CMeta->getType();

76 }

77 }

80}

84 F.getParent()->getNamedMetadata("spv.cloned_funcs"), F.getFunctionType(),

85 F.getName());

86}

93}

100 uint32_t Word = 0u;

101 for (unsigned WordIndex = 0; WordIndex < 4; ++WordIndex) {

102 unsigned StrIndex = i + WordIndex;

103 uint8_t CharToAdd = 0;

104 if (StrIndex < Str.size()) {

105 CharToAdd = Str[StrIndex];

106 }

107 Word |= (CharToAdd << (WordIndex * 8));

108 }

109 return Word;

110}

111

112

114 return (Str.size() + 4) & ~3;

115}

116

119 for (unsigned i = 0; i < PaddedLen; i += 4) {

120

122 }

123}

124

127 for (unsigned i = 0; i < PaddedLen; i += 4) {

128

130 }

131}

132

134 std::vector<Value *> &Args) {

136 for (unsigned i = 0; i < PaddedLen; i += 4) {

137

139 }

140}

141

145

148 assert(Def && Def->getOpcode() == TargetOpcode::G_GLOBAL_VALUE &&

149 "Expected G_GLOBAL_VALUE");

150 const GlobalValue *GV = Def->getOperand(1).getGlobal();

154}

155

157 const auto Bitwidth = Imm.getBitWidth();

158 if (Bitwidth == 1)

159 return;

160 else if (Bitwidth <= 32) {

161 MIB.addImm(Imm.getZExtValue());

162

163 if (Bitwidth == 16)

165 return;

166 } else if (Bitwidth <= 64) {

167 uint64_t FullImm = Imm.getZExtValue();

168 uint32_t LowBits = FullImm & 0xffffffff;

169 uint32_t HighBits = (FullImm >> 32) & 0xffffffff;

171

173 return;

174 }

176}

177

180 if (!Name.empty()) {

183 }

184}

185

188 if (!Name.empty()) {

189 auto MIB =

190 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpName))

193 }

194}

195

197 const std::vector<uint32_t> &DecArgs,

199 if (!StrImm.empty())

201 for (const auto &DecArg : DecArgs)

203}

204

206 SPIRV::Decoration::Decoration Dec,

207 const std::vector<uint32_t> &DecArgs, StringRef StrImm) {

208 auto MIB = MIRBuilder.buildInstr(SPIRV::OpDecorate)

212}

213

215 SPIRV::Decoration::Decoration Dec,

216 const std::vector<uint32_t> &DecArgs, StringRef StrImm) {

218 auto MIB = BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpDecorate))

222}

223

225 SPIRV::Decoration::Decoration Dec, uint32_t Member,

226 const std::vector<uint32_t> &DecArgs,

228 auto MIB = MIRBuilder.buildInstr(SPIRV::OpMemberDecorate)

233}

234

237 SPIRV::Decoration::Decoration Dec, uint32_t Member,

238 const std::vector<uint32_t> &DecArgs,

241 auto MIB = BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemberDecorate))

246}

247

252 if (!OpMD)

254 if (OpMD->getNumOperands() == 0)

258 if (!DecorationId)

259 report_fatal_error("Expect SPIR-V operand to be the first "

260 "element of the decoration");

261

262

263

264

265

266

267

268

270 static_cast<uint32_t>(SPIRV::Decoration::NoContraction) ||

272 static_cast<uint32_t>(SPIRV::Decoration::FPFastMathMode)) {

273 continue;

274 }

275 auto MIB = MIRBuilder.buildInstr(SPIRV::OpDecorate)

278 for (unsigned OpI = 1, OpE = OpMD->getNumOperands(); OpI != OpE; ++OpI) {

281 MIB.addImm(static_cast<uint32_t>(OpV->getZExtValue()));

284 else

286 }

287 }

288}

289

294 E = MBB->end();

295 bool IsHeader = false;

296 unsigned Opcode;

297 for (; It != E && It != I; ++It) {

298 Opcode = It->getOpcode();

299 if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) {

300 IsHeader = true;

301 } else if (IsHeader &&

302 !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) {

303 ++It;

304 break;

305 }

306 }

307 return It;

308}

309

312 if (I == MBB->begin())

313 return I;

314 --I;

315 while (I->isTerminator() || I->isDebugValue()) {

316 if (I == MBB->begin())

317 break;

318 --I;

319 }

320 return I;

321}

322

323SPIRV::StorageClass::StorageClass

325 switch (AddrSpace) {

326 case 0:

327 return SPIRV::StorageClass::Function;

328 case 1:

329 return SPIRV::StorageClass::CrossWorkgroup;

330 case 2:

331 return SPIRV::StorageClass::UniformConstant;

332 case 3:

333 return SPIRV::StorageClass::Workgroup;

334 case 4:

335 return SPIRV::StorageClass::Generic;

336 case 5:

337 return STI.canUseExtension(SPIRV::Extension::SPV_INTEL_usm_storage_classes)

338 ? SPIRV::StorageClass::DeviceOnlyINTEL

339 : SPIRV::StorageClass::CrossWorkgroup;

340 case 6:

341 return STI.canUseExtension(SPIRV::Extension::SPV_INTEL_usm_storage_classes)

342 ? SPIRV::StorageClass::HostOnlyINTEL

343 : SPIRV::StorageClass::CrossWorkgroup;

344 case 7:

345 return SPIRV::StorageClass::Input;

346 case 8:

347 return SPIRV::StorageClass::Output;

348 case 9:

349 return SPIRV::StorageClass::CodeSectionINTEL;

350 case 10:

351 return SPIRV::StorageClass::Private;

352 case 11:

353 return SPIRV::StorageClass::StorageBuffer;

354 case 12:

355 return SPIRV::StorageClass::Uniform;

356 default:

358 }

359}

360

361SPIRV::MemorySemantics::MemorySemantics

363 switch (SC) {

364 case SPIRV::StorageClass::StorageBuffer:

365 case SPIRV::StorageClass::Uniform:

366 return SPIRV::MemorySemantics::UniformMemory;

367 case SPIRV::StorageClass::Workgroup:

368 return SPIRV::MemorySemantics::WorkgroupMemory;

369 case SPIRV::StorageClass::CrossWorkgroup:

370 return SPIRV::MemorySemantics::CrossWorkgroupMemory;

371 case SPIRV::StorageClass::AtomicCounter:

372 return SPIRV::MemorySemantics::AtomicCounterMemory;

373 case SPIRV::StorageClass::Image:

374 return SPIRV::MemorySemantics::ImageMemory;

375 default:

376 return SPIRV::MemorySemantics::None;

377 }

378}

379

381 switch (Ord) {

383 return SPIRV::MemorySemantics::Acquire;

385 return SPIRV::MemorySemantics::Release;

387 return SPIRV::MemorySemantics::AcquireRelease;

389 return SPIRV::MemorySemantics::SequentiallyConsistent;

393 return SPIRV::MemorySemantics::None;

394 }

396}

397

399

400

401

402

403

405 Ctx.getOrInsertSyncScopeID("subgroup");

407 Ctx.getOrInsertSyncScopeID("workgroup");

409 Ctx.getOrInsertSyncScopeID("device");

410

412 return SPIRV::Scope::Invocation;

414 return SPIRV::Scope::CrossDevice;

415 else if (Id == SubGroup)

416 return SPIRV::Scope::Subgroup;

417 else if (Id == WorkGroup)

418 return SPIRV::Scope::Workgroup;

419 else if (Id == Device)

420 return SPIRV::Scope::Device;

421 return SPIRV::Scope::CrossDevice;

422}

423

428 MI->getOpcode() == SPIRV::G_TRUNC || MI->getOpcode() == SPIRV::G_ZEXT

429 ? MRI->getVRegDef(MI->getOperand(1).getReg())

430 : MI;

432 if (GI->is(Intrinsic::spv_track_constant)) {

434 return MRI->getVRegDef(ConstReg);

435 }

436 } else if (ConstInstr->getOpcode() == SPIRV::ASSIGN_TYPE) {

438 return MRI->getVRegDef(ConstReg);

439 } else if (ConstInstr->getOpcode() == TargetOpcode::G_CONSTANT ||

440 ConstInstr->getOpcode() == TargetOpcode::G_FCONSTANT) {

442 return ConstInstr;

443 }

444 return MRI->getVRegDef(ConstReg);

445}

446

449 assert(MI && MI->getOpcode() == TargetOpcode::G_CONSTANT);

450 return MI->getOperand(1).getCImm()->getValue().getZExtValue();

451}

452

455 assert(MI && MI->getOpcode() == TargetOpcode::G_CONSTANT);

456 return MI->getOperand(1).getCImm()->getSExtValue();

457}

458

461 return GI->is(IntrinsicID);

462 return false;

463}

464

469

470

471

473 return MangledName == "write_pipe_2" || MangledName == "read_pipe_2" ||

474 MangledName == "write_pipe_2_bl" || MangledName == "read_pipe_2_bl" ||

475 MangledName == "write_pipe_4" || MangledName == "read_pipe_4" ||

476 MangledName == "reserve_write_pipe" ||

477 MangledName == "reserve_read_pipe" ||

478 MangledName == "commit_write_pipe" ||

479 MangledName == "commit_read_pipe" ||

480 MangledName == "work_group_reserve_write_pipe" ||

481 MangledName == "work_group_reserve_read_pipe" ||

482 MangledName == "work_group_commit_write_pipe" ||

483 MangledName == "work_group_commit_read_pipe" ||

484 MangledName == "get_pipe_num_packets_ro" ||

485 MangledName == "get_pipe_max_packets_ro" ||

486 MangledName == "get_pipe_num_packets_wo" ||

487 MangledName == "get_pipe_max_packets_wo" ||

488 MangledName == "sub_group_reserve_write_pipe" ||

489 MangledName == "sub_group_reserve_read_pipe" ||

490 MangledName == "sub_group_commit_write_pipe" ||

491 MangledName == "sub_group_commit_read_pipe" ||

492 MangledName == "to_global" || MangledName == "to_local" ||

493 MangledName == "to_private";

494}

495

497 return MangledName == "__enqueue_kernel_basic" ||

498 MangledName == "__enqueue_kernel_basic_events" ||

499 MangledName == "__enqueue_kernel_varargs" ||

500 MangledName == "__enqueue_kernel_events_varargs";

501}

502

504 return MangledName == "__get_kernel_work_group_size_impl" ||

505 MangledName == "__get_kernel_sub_group_count_for_ndrange_impl" ||

506 MangledName == "__get_kernel_max_sub_group_size_for_ndrange_impl" ||

507 MangledName == "__get_kernel_preferred_work_group_size_multiple_impl";

508}

509

511 if (!Name.starts_with("__"))

512 return false;

513

516 Name == "__translate_sampler_initializer";

517}

518

521 bool IsNonMangledSPIRV = Name.starts_with("__spirv_");

522 bool IsNonMangledHLSL = Name.starts_with("__hlsl_");

523 bool IsMangled = Name.starts_with("_Z");

524

525

526 if (IsNonMangledOCL || IsNonMangledSPIRV || IsNonMangledHLSL || !IsMangled)

527 return Name.str();

528

529

530 if (char *DemangledName = itaniumDemangle(Name.data())) {

531 std::string Result = DemangledName;

532 free(DemangledName);

533 return Result;

534 }

535

536

537

538

539

540 size_t Start, Len = 0;

541 size_t DemangledNameLenStart = 2;

542 if (Name.starts_with("_ZN")) {

543

544 size_t NameSpaceStart = Name.find_first_not_of("rVKRO", 3);

545

546 if (Name.substr(NameSpaceStart, 11) != "2cl7__spirv")

547 return std::string();

548 DemangledNameLenStart = NameSpaceStart + 11;

549 }

550 Start = Name.find_first_not_of("0123456789", DemangledNameLenStart);

551 [[maybe_unused]] bool Error =

552 Name.substr(DemangledNameLenStart, Start - DemangledNameLenStart)

553 .getAsInteger(10, Len);

554 assert( Error && "Failed to parse demangled name length");

555 return Name.substr(Start, Len).str();

556}

557

559 if (Name.starts_with("opencl.") || Name.starts_with("ocl_") ||

560 Name.starts_with("spirv."))

561 return true;

562 return false;

563}

564

570

571 return false;

572}

573

575

576

578 return true;

579

580

581

582 if (F.getFnAttribute("hlsl.shader").isValid())

583 return true;

584

585 return false;

586}

587

589 TypeName.consume_front("atomic_");

590 if (TypeName.consume_front("void"))

592 else if (TypeName.consume_front("bool") || TypeName.consume_front("_Bool"))

594 else if (TypeName.consume_front("char") ||

595 TypeName.consume_front("signed char") ||

596 TypeName.consume_front("unsigned char") ||

597 TypeName.consume_front("uchar"))

599 else if (TypeName.consume_front("short") ||

600 TypeName.consume_front("signed short") ||

601 TypeName.consume_front("unsigned short") ||

602 TypeName.consume_front("ushort"))

604 else if (TypeName.consume_front("int") ||

605 TypeName.consume_front("signed int") ||

606 TypeName.consume_front("unsigned int") ||

607 TypeName.consume_front("uint"))

609 else if (TypeName.consume_front("long") ||

610 TypeName.consume_front("signed long") ||

611 TypeName.consume_front("unsigned long") ||

612 TypeName.consume_front("ulong"))

614 else if (TypeName.consume_front("half") ||

615 TypeName.consume_front("_Float16") ||

616 TypeName.consume_front("__fp16"))

618 else if (TypeName.consume_front("float"))

620 else if (TypeName.consume_front("double"))

622

623

624 return nullptr;

625}

626

627std::unordered_set<BasicBlock *>

628PartialOrderingVisitor::getReachableFrom(BasicBlock *Start) {

629 std::queue<BasicBlock *> ToVisit;

630 ToVisit.push(Start);

631

632 std::unordered_set<BasicBlock *> Output;

633 while (ToVisit.size() != 0) {

634 BasicBlock *BB = ToVisit.front();

635 ToVisit.pop();

636

637 if (Output.count(BB) != 0)

638 continue;

639 Output.insert(BB);

640

643 continue;

645 }

646 }

647

648 return Output;

649}

650

651bool PartialOrderingVisitor::CanBeVisited(BasicBlock *BB) const {

653

654 if (DT.dominates(BB, P))

655 continue;

656

657

658 if (BlockToOrder.count(P) == 0)

659 return false;

660

661

662

663 Loop *L = LI.getLoopFor(P);

664 if (L == nullptr || L->contains(BB))

665 continue;

666

667

668

669

670 assert(L->getNumBackEdges() <= 1);

671

672

673

676 if (Latch == nullptr)

677 continue;

678

679

680 if (BlockToOrder.count(Latch) == 0)

681 return false;

682 }

683

684 return true;

685}

686

688 auto It = BlockToOrder.find(BB);

689 if (It != BlockToOrder.end())

690 return It->second.Rank;

691

692 size_t result = 0;

694

695 if (DT.dominates(BB, P))

696 continue;

697

698 auto Iterator = BlockToOrder.end();

699 Loop *L = LI.getLoopFor(P);

700 BasicBlock *Latch = L ? L->getLoopLatch() : nullptr;

701

702

703

704 if (L == nullptr || L->contains(BB) || Latch == nullptr) {

705 Iterator = BlockToOrder.find(P);

706 } else {

707

708

709

710 Iterator = BlockToOrder.find(Latch);

711 }

712

713 assert(Iterator != BlockToOrder.end());

714 result = std::max(result, Iterator->second.Rank + 1);

715 }

716

717 return result;

718}

719

720size_t PartialOrderingVisitor::visit(BasicBlock *BB, size_t Unused) {

721 ToVisit.push(BB);

722 Queued.insert(BB);

723

724 size_t QueueIndex = 0;

725 while (ToVisit.size() != 0) {

727 ToVisit.pop();

728

729 if (!CanBeVisited(BB)) {

730 ToVisit.push(BB);

731 if (QueueIndex >= ToVisit.size())

733 "No valid candidate in the queue. Is the graph reducible?");

734 QueueIndex++;

735 continue;

736 }

737

738 QueueIndex = 0;

740 OrderInfo Info = {Rank, BlockToOrder.size()};

741 BlockToOrder.emplace(BB, Info);

742

743 for (BasicBlock *S : successors(BB)) {

744 if (Queued.count(S) != 0)

745 continue;

746 ToVisit.push(S);

747 Queued.insert(S);

748 }

749 }

750

751 return 0;

752}

753

755 DT.recalculate(F);

757

758 visit(&*F.begin(), 0);

759

760 Order.reserve(F.size());

761 for (auto &[BB, Info] : BlockToOrder)

762 Order.emplace_back(BB);

763

764 std::sort(Order.begin(), Order.end(), [&](const auto &LHS, const auto &RHS) {

765 return compare(LHS, RHS);

766 });

767}

768

771 const OrderInfo &InfoLHS = BlockToOrder.at(const_cast<BasicBlock *>(LHS));

772 const OrderInfo &InfoRHS = BlockToOrder.at(const_cast<BasicBlock *>(RHS));

773 if (InfoLHS.Rank != InfoRHS.Rank)

774 return InfoLHS.Rank < InfoRHS.Rank;

775 return InfoLHS.TraversalIndex < InfoRHS.TraversalIndex;

776}

777

780 std::unordered_set<BasicBlock *> Reachable = getReachableFrom(&Start);

781 assert(BlockToOrder.count(&Start) != 0);

782

783

784 auto It = Order.begin();

785 while (It != Order.end() && *It != &Start)

786 ++It;

787

788

789

790 assert(It != Order.end());

791

792

793 std::optional<size_t> EndRank = std::nullopt;

794 for (; It != Order.end(); ++It) {

795 if (EndRank.has_value() && BlockToOrder[*It].Rank > *EndRank)

796 break;

797

798 if (Reachable.count(*It) == 0) {

799 continue;

800 }

801

802 if ((*It)) {

803 EndRank = BlockToOrder[*It].Rank;

804 }

805 }

806}

807

809 if (F.size() == 0)

810 return false;

811

813 std::vector<BasicBlock *> Order;

814 Order.reserve(F.size());

815

818

819 assert(&*F.begin() == Order[0]);

822 if (BB != LastBlock && &*LastBlock->getNextNode() != BB) {

825 }

826 LastBlock = BB;

827 }

828

830}

831

834 if (MaybeDef && MaybeDef->getOpcode() == SPIRV::ASSIGN_TYPE)

836 return MaybeDef;

837}

838

840

841

842 constexpr unsigned MaxIters = 1024;

843 for (unsigned I = 0; I < MaxIters; ++I) {

844 std::string OrdName = Name + Twine(I).str();

845 if (!M.getFunction(OrdName)) {

846 Name = std::move(OrdName);

847 return true;

848 }

849 }

850 return false;

851}

852

853

854

857 bool Force) {

859 if ( MRI ->getRegClassOrNull(Reg) || Force) {

862 }

863}

864

865

866

867

870 SPIRV::AccessQualifier::AccessQualifier AccessQual,

871 bool EmitIR, bool Force) {

874 GR, MIRBuilder.getMRI(), MIRBuilder.getMF(), Force);

875}

876

877

878

887

888

889

893 MIRBuilder.getMF());

894}

895

896

897

900 SPIRV::AccessQualifier::AccessQualifier AccessQual, bool EmitIR) {

903 MIRBuilder);

904}

905

910 Args.push_back(Arg2);

911 Args.push_back(buildMD(Arg));

913 return B.CreateIntrinsic(IntrID, {Types}, Args);

914}

915

916

918 if (Ty->isPtrOrPtrVectorTy())

919 return true;

922 return true;

923 for (const Type *ArgTy : RefTy->params())

925 return true;

926 return false;

927 }

930 return false;

931}

932

935 if (Function *F = II->getCalledFunction())

936 if (F->getName().starts_with("llvm.spv."))

937 return true;

938 return false;

939}

940

941

942

943SmallVector<MachineInstr *, 4>

945 unsigned MinWC, unsigned ContinuedOpcode,

948

950 constexpr unsigned MaxWordCount = UINT16_MAX;

951 const size_t NumElements = Args.size();

952 size_t MaxNumElements = MaxWordCount - MinWC;

953 size_t SPIRVStructNumElements = NumElements;

954

955 if (NumElements > MaxNumElements) {

956

957

958 SPIRVStructNumElements = MaxNumElements;

959 MaxNumElements = MaxWordCount - 1;

960 }

961

962 auto MIB =

964

965 for (size_t I = 0; I < SPIRVStructNumElements; ++I)

967

968 Instructions.push_back(MIB.getInstr());

969

970 for (size_t I = SPIRVStructNumElements; I < NumElements;

971 I += MaxNumElements) {

972 auto MIB = MIRBuilder.buildInstr(ContinuedOpcode);

973 for (size_t J = I; J < std::min(I + MaxNumElements, NumElements); ++J)

975 Instructions.push_back(MIB.getInstr());

976 }

977 return Instructions;

978}

979

981 unsigned LC = SPIRV::LoopControl::None;

982

983

984

985 std::vector<std::pair<unsigned, unsigned>> MaskToValueMap;

987 LC |= SPIRV::LoopControl::DontUnroll;

988 } else {

991 LC |= SPIRV::LoopControl::Unroll;

992 }

993 std::optional Count =

996 LC |= SPIRV::LoopControl::PartialCount;

997 MaskToValueMap.emplace_back(

998 std::make_pair(SPIRV::LoopControl::PartialCount, *Count));

999 }

1000 }

1002 for (auto &[Mask, Val] : MaskToValueMap)

1003 Result.push_back(Val);

1004 return Result;

1005}

1006

1008

1009 static const std::set TypeFoldingSupportingOpcs = {

1010 TargetOpcode::G_ADD,

1011 TargetOpcode::G_FADD,

1012 TargetOpcode::G_STRICT_FADD,

1013 TargetOpcode::G_SUB,

1014 TargetOpcode::G_FSUB,

1015 TargetOpcode::G_STRICT_FSUB,

1016 TargetOpcode::G_MUL,

1017 TargetOpcode::G_FMUL,

1018 TargetOpcode::G_STRICT_FMUL,

1019 TargetOpcode::G_SDIV,

1020 TargetOpcode::G_UDIV,

1021 TargetOpcode::G_FDIV,

1022 TargetOpcode::G_STRICT_FDIV,

1023 TargetOpcode::G_SREM,

1024 TargetOpcode::G_UREM,

1025 TargetOpcode::G_FREM,

1026 TargetOpcode::G_STRICT_FREM,

1027 TargetOpcode::G_FNEG,

1028 TargetOpcode::G_CONSTANT,

1029 TargetOpcode::G_FCONSTANT,

1030 TargetOpcode::G_AND,

1031 TargetOpcode::G_OR,

1032 TargetOpcode::G_XOR,

1033 TargetOpcode::G_SHL,

1034 TargetOpcode::G_ASHR,

1035 TargetOpcode::G_LSHR,

1036 TargetOpcode::G_SELECT,

1037 TargetOpcode::G_EXTRACT_VECTOR_ELT,

1038 };

1039

1040 return TypeFoldingSupportingOpcs;

1041}

1042

1046

1047

1049 return (Def->getOpcode() == SPIRV::ASSIGN_TYPE ||

1050 Def->getOpcode() == TargetOpcode::COPY)

1051 ? MRI->getVRegDef(Def->getOperand(1).getReg())

1052 : Def;

1053}

1054

1058 return nullptr;

1059}

1060

1063 if (Def->getOpcode() == TargetOpcode::G_CONSTANT ||

1064 Def->getOpcode() == SPIRV::OpConstantI)

1065 return Def;

1066 }

1067 return nullptr;

1068}

1069

1072 if (Def->getOpcode() == SPIRV::OpConstantI)

1073 return Def->getOperand(2).getImm();

1074 if (Def->getOpcode() == TargetOpcode::G_CONSTANT)

1075 return Def->getOperand(1).getCImm()->getZExtValue();

1076 }

1078}

1079

1084

1087

1088

1089

1091 while (VarPos != BB.end() && VarPos->getOpcode() != SPIRV::OpFunction) {

1092 ++VarPos;

1093 }

1094

1095

1096

1097 ++VarPos;

1098 while (VarPos != BB.end() &&

1099 VarPos->getOpcode() == SPIRV::OpFunctionParameter) {

1100 ++VarPos;

1101 }

1102

1103

1104 return VarPos != BB.end() && VarPos->getOpcode() == SPIRV::OpLabel ? ++VarPos

1105 : VarPos;

1106}

1107

1110

1111 if (Ty->getStructNumElements() != 2)

1112 return false;

1113

1116

1118 return false;

1119

1121 if (!ArrayElementType->isStructTy() ||

1123 return false;

1124

1126 if (T_in_struct != SecondElement)

1127 return false;

1128

1129 auto *Padding_in_struct =

1131 if (!Padding_in_struct || Padding_in_struct->getName() != "spirv.Padding")

1132 return false;

1133

1135 TotalSize = ArraySize + 1;

1136 OriginalElementType = ArrayElementType;

1137 return true;

1138}

1139

1141 if (!Ty->isStructTy())

1142 return Ty;

1143

1145 Type *OriginalElementType = nullptr;

1150 return ResultTy;

1151 }

1152

1155 for (Type *ElementTy : STy->elements()) {

1157 if (NewElementTy != ElementTy)

1159 NewElementTypes.push_back(NewElementTy);

1160 }

1161

1163 return Ty;

1164

1165 Type *ResultTy;

1166 if (STy->isLiteral())

1167 ResultTy =

1168 StructType::get(STy->getContext(), NewElementTypes, STy->isPacked());

1169 else {

1170 auto *NewTy = StructType::create(STy->getContext(), STy->getName());

1171 NewTy->setBody(NewElementTypes, STy->isPacked());

1172 ResultTy = NewTy;

1173 }

1174 return ResultTy;

1175}

1176

1177std::optionalSPIRV::LinkageType::LinkageType

1180 return std::nullopt;

1181

1183 return SPIRV::LinkageType::Import;

1184

1186 ST.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr))

1187 return SPIRV::LinkageType::LinkOnceODR;

1188

1189 return SPIRV::LinkageType::Export;

1190}

1191

1192}

unsigned const MachineRegisterInfo * MRI

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

const TargetInstrInfo & TII

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

Analysis containing CSE Info

Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...

This file declares the MachineIRBuilder class.

uint64_t IntrinsicInst * II

static ConstantInt * getConstInt(MDNode *MD, unsigned NumOp)

Class for arbitrary precision integers.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

Class to represent array types.

static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)

This static method is the primary way to construct an ArrayType.

LLVM Basic Block Representation.

LLVM_ABI void moveAfter(BasicBlock *MovePos)

Unlink this basic block from its current function and insert it right after MovePos in the function M...

const Instruction & front() const

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

FunctionType * getFunctionType() const

This class represents a function call, abstracting a target machine's calling convention.

An array constant whose element type is a simple 1/2/4/8-byte integer or float/double,...

StringRef getAsCString() const

If this array is isCString(), then this method returns the array (without the trailing null byte) as ...

This is the shared class of boolean and integer constants.

uint64_t getZExtValue() const

Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...

bool dominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const

dominates - Returns true iff A dominates B.

Lightweight error class with error context and mandatory checking.

Class to represent function types.

ArrayRef< Type * > params() const

Type * getReturnType() const

static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)

This static method is the primary way of constructing a FunctionType.

bool hasLocalLinkage() const

bool hasHiddenVisibility() const

bool isDeclarationForLinker() const

bool hasLinkOnceODRLinkage() const

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

LLVM_ABI const Module * getModule() const

Return the module owning the function this instruction belongs to or nullptr it the function does not...

This is an important class for using LLVM in a threaded context.

Represents a single loop in the control flow graph.

Instances of this class represent a single low-level machine instruction.

void addOperand(const MCOperand Op)

static MCOperand createImm(int64_t Val)

const MDOperand & getOperand(unsigned I) const

unsigned getNumOperands() const

Return number of MDNode operands.

MachineInstrBundleIterator< MachineInstr > iterator

const MachineBasicBlock & front() const

Helper class to build MachineInstr.

MachineInstrBuilder buildInstr(unsigned Opcode)

Build and insert = Opcode .

MachineFunction & getMF()

Getter for the function we currently build.

MachineRegisterInfo * getMRI()

Getter for MRI.

const MachineInstrBuilder & addImm(int64_t Val) const

Add a new immediate operand.

const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const

Add a virtual register use operand.

MachineInstr * getInstr() const

If conversion operators fail, use this method to get the MachineInstr explicitly.

const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const

Add a virtual register definition operand.

Representation of each machine instruction.

unsigned getOpcode() const

Returns the opcode of this MachineInstr.

void setAsmPrinterFlag(uint8_t Flag)

Set a flag for the AsmPrinter.

const MachineOperand & getOperand(unsigned i) const

MachineOperand class - Representation of each machine instruction operand.

getReg - Returns the register number.

MachineRegisterInfo - Keep track of information for virtual and physical registers,...

A Module instance is used to store all the information related to an LLVM module.

NamedMDNode * getNamedMetadata(StringRef Name) const

Return the first NamedMDNode in the module with the specified name.

iterator_range< op_iterator > operands()

size_t GetNodeRank(BasicBlock *BB) const

Definition SPIRVUtils.cpp:687

void partialOrderVisit(BasicBlock &Start, std::function< bool(BasicBlock *)> Op)

Definition SPIRVUtils.cpp:778

bool compare(const BasicBlock *LHS, const BasicBlock *RHS) const

Definition SPIRVUtils.cpp:769

PartialOrderingVisitor(Function &F)

Definition SPIRVUtils.cpp:754

Wrapper class representing virtual and physical registers.

void assignSPIRVTypeToVReg(SPIRVType *Type, Register VReg, const MachineFunction &MF)

SPIRVType * getOrCreateSPIRVType(const Type *Type, MachineInstr &I, SPIRV::AccessQualifier::AccessQualifier AQ, bool EmitIR)

const TargetRegisterClass * getRegClass(SPIRVType *SpvType) const

LLT getRegType(SPIRVType *SpvType) const

bool canUseExtension(SPIRV::Extension::Extension E) const

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

std::string str() const

str - Get the contents as an std::string.

constexpr bool empty() const

empty - Check if the string is empty.

Class to represent struct types.

static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)

This static method is the primary way to create a literal StructType.

static LLVM_ABI StructType * create(LLVMContext &Context, StringRef Name)

This creates an identified struct.

Class to represent target extensions types, which are generally unintrospectable from target-independ...

Target - Wrapper for Target specific information.

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

LLVM_ABI std::string str() const

Return the twine contents as a std::string.

The instances of the Type class are immutable: once they are created, they are never changed.

static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)

LLVM_ABI Type * getStructElementType(unsigned N) const

bool isArrayTy() const

True if this is an instance of ArrayType.

static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)

Type * getArrayElementType() const

LLVM_ABI unsigned getStructNumElements() const

LLVM_ABI uint64_t getArrayNumElements() const

static LLVM_ABI Type * getVoidTy(LLVMContext &C)

static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)

bool isStructTy() const

True if this is an instance of StructType.

static LLVM_ABI IntegerType * getInt16Ty(LLVMContext &C)

static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)

static LLVM_ABI Type * getDoubleTy(LLVMContext &C)

static LLVM_ABI Type * getFloatTy(LLVMContext &C)

static LLVM_ABI Type * getHalfTy(LLVMContext &C)

Value * getOperand(unsigned i) const

LLVM Value Representation.

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

NodeTy * getNextNode()

Get the next node, or nullptr for the list tail.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

@ SPIR_KERNEL

Used for SPIR kernel functions.

@ BasicBlock

Various leaf nodes.

FunctionType * getOriginalFunctionType(const Function &F)

Definition SPIRVUtils.cpp:82

static FunctionType * extractFunctionTypeFromMetadata(NamedMDNode *NMD, FunctionType *FTy, StringRef Name)

Definition SPIRVUtils.cpp:36

@ SingleThread

Synchronized with respect to signal handlers executing in the same thread.

@ System

Synchronized with respect to all concurrently executing threads.

std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)

Extract a Value from Metadata, if any.

This is an optimization pass for GlobalISel generic memory operations.

void buildOpName(Register Target, const StringRef &Name, MachineIRBuilder &MIRBuilder)

Definition SPIRVUtils.cpp:178

bool getVacantFunctionName(Module &M, std::string &Name)

Definition SPIRVUtils.cpp:839

std::string getStringImm(const MachineInstr &MI, unsigned StartIndex)

Definition SPIRVUtils.cpp:142

LLVM_ABI bool getBooleanLoopAttribute(const Loop *TheLoop, StringRef Name)

Returns true if Name is applied to TheLoop and enabled.

int64_t getIConstValSext(Register ConstReg, const MachineRegisterInfo *MRI)

Definition SPIRVUtils.cpp:453

bool isTypedPointerWrapper(const TargetExtType *ExtTy)

MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)

Builder interface. Specify how to create the initial instruction itself.

static void finishBuildOpDecorate(MachineInstrBuilder &MIB, const std::vector< uint32_t > &DecArgs, StringRef StrImm)

Definition SPIRVUtils.cpp:196

bool isTypeFoldingSupported(unsigned Opcode)

Definition SPIRVUtils.cpp:1043

static uint32_t convertCharsToWord(const StringRef &Str, unsigned i)

Definition SPIRVUtils.cpp:99

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

MachineInstr * getDef(const MachineOperand &MO, const MachineRegisterInfo *MRI)

Definition SPIRVUtils.cpp:1055

void addNumImm(const APInt &Imm, MachineInstrBuilder &MIB)

Definition SPIRVUtils.cpp:156

auto successors(const MachineBasicBlock *BB)

CallInst * buildIntrWithMD(Intrinsic::ID IntrID, ArrayRef< Type * > Types, Value *Arg, Value *Arg2, ArrayRef< Constant * > Imms, IRBuilder<> &B)

Definition SPIRVUtils.cpp:906

bool matchPeeledArrayPattern(const StructType *Ty, Type *&OriginalElementType, uint64_t &TotalSize)

Definition SPIRVUtils.cpp:1108

void append_range(Container &C, Range &&R)

Wrapper function to append range R to container C.

unsigned getArrayComponentCount(const MachineRegisterInfo *MRI, const MachineInstr *ResType)

Definition SPIRVUtils.cpp:1080

bool sortBlocks(Function &F)

Definition SPIRVUtils.cpp:808

SmallVector< unsigned, 1 > getSpirvLoopControlOperandsFromLoopMetadata(Loop *L)

Definition SPIRVUtils.cpp:980

uint64_t getIConstVal(Register ConstReg, const MachineRegisterInfo *MRI)

Definition SPIRVUtils.cpp:447

SmallVector< MachineInstr *, 4 > createContinuedInstructions(MachineIRBuilder &MIRBuilder, unsigned Opcode, unsigned MinWC, unsigned ContinuedOpcode, ArrayRef< Register > Args, Register ReturnRegister, Register TypeID)

Definition SPIRVUtils.cpp:944

SPIRV::MemorySemantics::MemorySemantics getMemSemanticsForStorageClass(SPIRV::StorageClass::StorageClass SC)

Definition SPIRVUtils.cpp:362

MachineBasicBlock::iterator getFirstValidInstructionInsertPoint(MachineBasicBlock &BB)

Definition SPIRVUtils.cpp:1086

bool isNestedPointer(const Type *Ty)

Definition SPIRVUtils.cpp:917

MetadataAsValue * buildMD(Value *Arg)

std::string getOclOrSpirvBuiltinDemangledName(StringRef Name)

Definition SPIRVUtils.cpp:519

void buildOpDecorate(Register Reg, MachineIRBuilder &MIRBuilder, SPIRV::Decoration::Decoration Dec, const std::vector< uint32_t > &DecArgs, StringRef StrImm)

Definition SPIRVUtils.cpp:205

MachineBasicBlock::iterator getOpVariableMBBIt(MachineInstr &I)

Definition SPIRVUtils.cpp:290

Register createVirtualRegister(SPIRVType *SpvType, SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI, const MachineFunction &MF)

Definition SPIRVUtils.cpp:879

MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)

Definition SPIRVUtils.cpp:1061

std::string getSPIRVStringOperand(const InstType &MI, unsigned StartIndex)

void buildOpMemberDecorate(Register Reg, MachineIRBuilder &MIRBuilder, SPIRV::Decoration::Decoration Dec, uint32_t Member, const std::vector< uint32_t > &DecArgs, StringRef StrImm)

Definition SPIRVUtils.cpp:224

Type * toTypedPointer(Type *Ty)

DEMANGLE_ABI char * itaniumDemangle(std::string_view mangled_name, bool ParseParams=true)

Returns a non-NULL pointer to a NUL-terminated C style string that should be explicitly freed,...

bool isSpecialOpaqueType(const Type *Ty)

Definition SPIRVUtils.cpp:565

void setRegClassType(Register Reg, SPIRVType *SpvType, SPIRVGlobalRegistry *GR, MachineRegisterInfo *MRI, const MachineFunction &MF, bool Force)

Definition SPIRVUtils.cpp:855

LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)

MachineBasicBlock::iterator getInsertPtValidEnd(MachineBasicBlock *MBB)

Definition SPIRVUtils.cpp:310

FunctionAddr VTableAddr Count

const MachineInstr SPIRVType

static bool isNonMangledOCLBuiltin(StringRef Name)

Definition SPIRVUtils.cpp:510

MachineInstr * passCopy(MachineInstr *Def, const MachineRegisterInfo *MRI)

Definition SPIRVUtils.cpp:1048

std::optional< SPIRV::LinkageType::LinkageType > getSpirvLinkageTypeFor(const SPIRVSubtarget &ST, const GlobalValue &GV)

Definition SPIRVUtils.cpp:1178

bool isEntryPoint(const Function &F)

Definition SPIRVUtils.cpp:574

const std::set< unsigned > & getTypeFoldingSupportedOpcodes()

Definition SPIRVUtils.cpp:1007

SPIRV::StorageClass::StorageClass addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI)

Definition SPIRVUtils.cpp:324

LLVM_ABI std::optional< int > getOptionalIntLoopAttribute(const Loop *TheLoop, StringRef Name)

Find named metadata for a loop with an integer value.

AtomicOrdering

Atomic ordering for LLVM's memory model.

SPIRV::Scope::Scope getMemScope(LLVMContext &Ctx, SyncScope::ID Id)

Definition SPIRVUtils.cpp:398

static bool isPipeOrAddressSpaceCastBI(const StringRef MangledName)

Definition SPIRVUtils.cpp:472

void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder, const MDNode *GVarMD, const SPIRVSubtarget &ST)

Definition SPIRVUtils.cpp:248

std::string getStringValueFromReg(Register Reg, MachineRegisterInfo &MRI)

Definition SPIRVUtils.cpp:146

int64_t foldImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)

Definition SPIRVUtils.cpp:1070

Type * parseBasicTypeName(StringRef &TypeName, LLVMContext &Ctx)

Definition SPIRVUtils.cpp:588

DWARFExpression::Operation Op

MachineInstr * getDefInstrMaybeConstant(Register &ConstReg, const MachineRegisterInfo *MRI)

Definition SPIRVUtils.cpp:424

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

bool hasBuiltinTypePrefix(StringRef Name)

Definition SPIRVUtils.cpp:558

Type * getMDOperandAsType(const MDNode *N, unsigned I)

Definition SPIRVUtils.cpp:465

auto find_if(R &&Range, UnaryPredicate P)

Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.

auto predecessors(const MachineBasicBlock *BB)

static size_t getPaddedLen(const StringRef &Str)

Definition SPIRVUtils.cpp:113

bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID)

Definition SPIRVUtils.cpp:459

void addStringImm(const StringRef &Str, MCInst &Inst)

Definition SPIRVUtils.cpp:117

static bool isKernelQueryBI(const StringRef MangledName)

Definition SPIRVUtils.cpp:503

MachineInstr * getVRegDef(MachineRegisterInfo &MRI, Register Reg)

Definition SPIRVUtils.cpp:832

static bool isEnqueueKernelBI(const StringRef MangledName)

Definition SPIRVUtils.cpp:496

Type * reconstitutePeeledArrayType(Type *Ty)

Definition SPIRVUtils.cpp:1140

SPIRV::MemorySemantics::MemorySemantics getMemSemantics(AtomicOrdering Ord)

Definition SPIRVUtils.cpp:380