LLVM: include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h Source File (original) (raw)

46 &ExecInfo,

50 const PredicateBitset &AvailableFeatures,

52

56

58

59 bool NoFPException = !State.MIs[0]->getDesc().mayRaiseFPException();

60

61 const uint32_t Flags = State.MIs[0]->getFlags();

62

63 enum RejectAction { RejectAndGiveUp, RejectAndResume };

64 auto handleReject = [&]() -> RejectAction {

66 dbgs() << CurrentIdx << ": Rejected\n");

67 if (OnFailResumeAt.empty())

68 return RejectAndGiveUp;

71 dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("

72 << OnFailResumeAt.size() << " try-blocks remain)\n");

73 return RejectAndResume;

74 };

75

76 const auto propagateFlags = [&]() {

77 for (auto MIB : OutMIs) {

78

79

80 uint32_t MIBFlags = Flags | MIB.getInstr()->getFlags();

81 if (NoFPException && MIB->mayRaiseFPException())

83 if (Observer)

85 MIB.setMIFlags(MIBFlags);

86 if (Observer)

88 }

89 };

90

91

92

93 const auto getTypeFromIdx = [&](int64_t Idx) -> LLT {

94 if (Idx >= 0)

96 return State.RecordedTypes[1 - Idx];

97 };

98

99 const auto readULEB = [&]() {

101 };

102

103

104

105

106

107

108 const auto readS8 = [&]() { return (int8_t)MatchTable[CurrentIdx++]; };

109

110 const auto readU16 = [&]() {

112 CurrentIdx += 2;

113 return V;

114 };

115

116 const auto readU32 = [&]() {

118 CurrentIdx += 4;

119 return V;

120 };

121

122 const auto readU64 = [&]() {

124 CurrentIdx += 8;

125 return V;

126 };

127

129

130

131 if (Builder.getInsertPt() == MI)

132 Builder.setInsertPt(*MI->getParent(), ++MI->getIterator());

133 if (Observer)

135 MI->eraseFromParent();

136 };

137

138 while (true) {

139 assert(CurrentIdx != ~0u && "Invalid MatchTable index");

140 uint8_t MatcherOpcode = MatchTable[CurrentIdx++];

141 switch (MatcherOpcode) {

144 dbgs() << CurrentIdx << ": Begin try-block\n");

145 OnFailResumeAt.push_back(readU32());

146 break;

147 }

148

151 uint64_t NewInsnID = readULEB();

152 uint64_t InsnID = readULEB();

154

155

156

157 assert(NewInsnID != 0 && "Refusing to modify MIs[0]");

158

160 if (!MO.isReg()) {

162 dbgs() << CurrentIdx << ": Not a register\n");

163 if (handleReject() == RejectAndGiveUp)

164 return false;

165 break;

166 }

169 dbgs() << CurrentIdx << ": Is a physical register\n");

170 if (handleReject() == RejectAndGiveUp)

171 return false;

172 break;

173 }

174

178 else

179 NewMI = MRI.getVRegDef(MO.getReg());

180

181 if ((size_t)NewInsnID < State.MIs.size())

182 State.MIs[NewInsnID] = NewMI;

183 else {

184 assert((size_t)NewInsnID == State.MIs.size() &&

185 "Expected to store MIs in order");

186 State.MIs.push_back(NewMI);

187 }

189 dbgs() << CurrentIdx << ": MIs[" << NewInsnID

190 << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx

191 << ")\n");

192 break;

193 }

194

196 uint16_t ExpectedBitsetID = readU16();

198 dbgs() << CurrentIdx

199 << ": GIM_CheckFeatures(ExpectedBitsetID="

200 << ExpectedBitsetID << ")\n");

201 if ((AvailableFeatures & ExecInfo.FeatureBitsets[ExpectedBitsetID]) !=

203 if (handleReject() == RejectAndGiveUp)

204 return false;

205 }

206 break;

207 }

210 uint64_t InsnID = readULEB();

211 uint16_t Expected0 = readU16();

214 Expected1 = readU16();

215

216 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

217 unsigned Opcode = State.MIs[InsnID]->getOpcode();

218

220 dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID

221 << "], ExpectedOpcode=" << Expected0;

223 dbgs() << " || " << Expected1;

224 dbgs() << ") // Got=" << Opcode << "\n";

225 });

226

227 if (Opcode != Expected0 && Opcode != Expected1) {

228 if (handleReject() == RejectAndGiveUp)

229 return false;

230 }

231 break;

232 }

234 uint64_t InsnID = readULEB();

235 uint16_t LowerBound = readU16();

236 uint16_t UpperBound = readU16();

238

239 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

240 const int64_t Opcode = State.MIs[InsnID]->getOpcode();

241

243 dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["

244 << LowerBound << ", " << UpperBound << "), Default=" << Default

245 << ", JumpTable...) // Got=" << Opcode << "\n";

246 });

247 if (Opcode < LowerBound || UpperBound <= Opcode) {

249 break;

250 }

251 const auto EntryIdx = (Opcode - LowerBound);

252

253 CurrentIdx =

255 if (!CurrentIdx) {

257 break;

258 }

260 break;

261 }

262

264 uint64_t InsnID = readULEB();

266 uint16_t LowerBound = readU16();

267 uint16_t UpperBound = readU16();

268 int64_t Default = readU32();

269

270 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

272

274 dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID

275 << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "

276 << UpperBound << "), Default=" << Default

277 << ", JumpTable...) // Got=";

279 dbgs() << "Not a VReg\n";

280 else

282 });

283 if (!MO.isReg()) {

285 break;

286 }

288 const auto TyI = ExecInfo.TypeIDMap.find(Ty);

289 if (TyI == ExecInfo.TypeIDMap.end()) {

291 break;

292 }

293 const int64_t TypeID = TyI->second;

294 if (TypeID < LowerBound || UpperBound <= TypeID) {

296 break;

297 }

298 const auto NumEntry = (TypeID - LowerBound);

299

300 CurrentIdx =

302 if (!CurrentIdx) {

304 break;

305 }

307 break;

308 }

309

312 uint64_t InsnID = readULEB();

316 dbgs() << CurrentIdx << ": GIM_CheckNumOperands"

317 << (IsLE ? "LE" : "GE") << "(MIs[" << InsnID

318 << "], Expected=" << Expected << ")\n");

319 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

320 const unsigned NumOps = State.MIs[InsnID]->getNumOperands();

322 if (handleReject() == RejectAndGiveUp)

323 return false;

324 }

325 break;

326 }

328 uint64_t InsnID = readULEB();

331 dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["

332 << InsnID << "], Expected=" << Expected << ")\n");

333 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

334 if (State.MIs[InsnID]->getNumOperands() != Expected) {

335 if (handleReject() == RejectAndGiveUp)

336 return false;

337 }

338 break;

339 }

342 uint64_t InsnID = readULEB();

347 dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs["

348 << InsnID << "]->getOperand(" << OpIdx

349 << "), Predicate=" << Predicate << ")\n");

350 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

351 assert((State.MIs[InsnID]->getOperand(OpIdx).isImm() ||

352 State.MIs[InsnID]->getOperand(OpIdx).isCImm()) &&

353 "Expected immediate operand");

355 int64_t Value = 0;

356 if (State.MIs[InsnID]->getOperand(OpIdx).isCImm())

357 Value = State.MIs[InsnID]->getOperand(OpIdx).getCImm()->getSExtValue();

358 else if (State.MIs[InsnID]->getOperand(OpIdx).isImm())

359 Value = State.MIs[InsnID]->getOperand(OpIdx).getImm();

360 else

362

364 if (handleReject() == RejectAndGiveUp)

365 return false;

366 break;

367 }

369 uint64_t InsnID = readULEB();

373 << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["

374 << InsnID << "], Predicate=" << Predicate << ")\n");

375 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

376 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&

377 "Expected G_CONSTANT");

379 if (!State.MIs[InsnID]->getOperand(1).isCImm())

381

383 State.MIs[InsnID]->getOperand(1).getCImm()->getValue();

385 if (handleReject() == RejectAndGiveUp)

386 return false;

387 break;

388 }

390 uint64_t InsnID = readULEB();

394 << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["

395 << InsnID << "], Predicate=" << Predicate << ")\n");

396 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

397 assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&

398 "Expected G_FCONSTANT");

399 assert(State.MIs[InsnID]->getOperand(1).isFPImm() &&

400 "Expected FPImm operand");

403 State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();

404

406 if (handleReject() == RejectAndGiveUp)

407 return false;

408 break;

409 }

411 uint64_t InsnID = readULEB();

415 dbgs() << CurrentIdx

416 << ": GIM_CheckLeafOperandPredicate(MIs[" << InsnID

417 << "]->getOperand(" << OpIdx

418 << "), Predicate=" << Predicate << ")\n");

419 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

420 assert(State.MIs[InsnID]->getOperand(OpIdx).isReg() &&

421 "Expected register operand");

424

426 if (handleReject() == RejectAndGiveUp)

427 return false;

428 break;

429 }

432 uint64_t InsnID = readULEB();

433

435 dbgs() << CurrentIdx

436 << ": GIM_CheckBuildVectorAll{Zeros|Ones}(MIs["

437 << InsnID << "])\n");

438 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

439

441 assert((MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR ||

442 MI->getOpcode() == TargetOpcode::G_BUILD_VECTOR_TRUNC) &&

443 "Expected G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC");

444

447 if (handleReject() == RejectAndGiveUp)

448 return false;

449 }

450 } else {

452 if (handleReject() == RejectAndGiveUp)

453 return false;

454 }

455 }

456

457 break;

458 }

460

461

462

465 dbgs() << CurrentIdx

466 << ": GIM_CheckSimplePredicate(Predicate="

470 if (handleReject() == RejectAndGiveUp)

471 return false;

472 }

473 break;

474 }

476 uint64_t InsnID = readULEB();

480 << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["

481 << InsnID << "], Predicate=" << Predicate << ")\n");

482 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

484

486 if (handleReject() == RejectAndGiveUp)

487 return false;

488 break;

489 }

491 uint64_t InsnID = readULEB();

492

494 dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs["

495 << InsnID << "]\n");

496

498 assert(MI && "Used insn before defined");

499 assert(MI->getNumDefs() > 0 && "No defs");

500 const Register Res = MI->getOperand(0).getReg();

501

502 if (MRI.use_nodbg_empty(Res)) {

503 if (handleReject() == RejectAndGiveUp)

504 return false;

505 }

506 break;

507 }

509 uint64_t InsnID = readULEB();

510

512 dbgs() << CurrentIdx << ": GIM_CheckHasOneUse(MIs["

513 << InsnID << "]\n");

514

516 assert(MI && "Used insn before defined");

517 assert(MI->getNumDefs() > 0 && "No defs");

518 const Register Res = MI->getOperand(0).getReg();

519

520 if (MRI.hasOneNonDBGUse(Res)) {

521 if (handleReject() == RejectAndGiveUp)

522 return false;

523 }

524 break;

525 }

527 uint64_t InsnID = readULEB();

528 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];

530 dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["

531 << InsnID << "], " << (uint64_t)Ordering << ")\n");

532 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

533 if (!State.MIs[InsnID]->hasOneMemOperand())

534 if (handleReject() == RejectAndGiveUp)

535 return false;

536

537 for (const auto &MMO : State.MIs[InsnID]->memoperands())

538 if (MMO->getMergedOrdering() != Ordering)

539 if (handleReject() == RejectAndGiveUp)

540 return false;

541 break;

542 }

544 uint64_t InsnID = readULEB();

545 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];

547 dbgs() << CurrentIdx

548 << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["

549 << InsnID << "], " << (uint64_t)Ordering << ")\n");

550 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

551 if (!State.MIs[InsnID]->hasOneMemOperand())

552 if (handleReject() == RejectAndGiveUp)

553 return false;

554

555 for (const auto &MMO : State.MIs[InsnID]->memoperands())

557 if (handleReject() == RejectAndGiveUp)

558 return false;

559 break;

560 }

562 uint64_t InsnID = readULEB();

563 auto Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];

565 dbgs() << CurrentIdx

566 << ": GIM_CheckAtomicOrderingWeakerThan(MIs["

567 << InsnID << "], " << (uint64_t)Ordering << ")\n");

568 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

569 if (!State.MIs[InsnID]->hasOneMemOperand())

570 if (handleReject() == RejectAndGiveUp)

571 return false;

572

573 for (const auto &MMO : State.MIs[InsnID]->memoperands())

574 if (isStrongerThan(Ordering, MMO->getMergedOrdering()))

575 if (handleReject() == RejectAndGiveUp)

576 return false;

577 break;

578 }

580 uint64_t InsnID = readULEB();

581 uint64_t MMOIdx = readULEB();

582

583 const uint64_t NumAddrSpace = MatchTable[CurrentIdx++];

584

585 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {

586 if (handleReject() == RejectAndGiveUp)

587 return false;

588 break;

589 }

590

591

592

593 const uint64_t LastIdx = CurrentIdx + NumAddrSpace;

594

596 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);

597 const unsigned MMOAddrSpace = MMO->getAddrSpace();

598

600 for (unsigned I = 0; I != NumAddrSpace; ++I) {

601 uint64_t AddrSpace = readULEB();

603 dbgs() << "addrspace(" << MMOAddrSpace << ") vs "

604 << AddrSpace << '\n');

605

606 if (AddrSpace == MMOAddrSpace) {

608 break;

609 }

610 }

611

612 CurrentIdx = LastIdx;

613 if (Success && handleReject() == RejectAndGiveUp)

614 return false;

615 break;

616 }

618 uint64_t InsnID = readULEB();

619 uint64_t MMOIdx = readULEB();

621

622 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

623

624 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {

625 if (handleReject() == RejectAndGiveUp)

626 return false;

627 break;

628 }

629

631 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);

633 dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"

634 << "(MIs[" << InsnID << "]->memoperands() + "

635 << MMOIdx << ")->getAlignment() >= " << MinAlign

636 << ")\n");

637 if (MMO->getAlign() < MinAlign && handleReject() == RejectAndGiveUp)

638 return false;

639

640 break;

641 }

643 uint64_t InsnID = readULEB();

644 uint64_t MMOIdx = readULEB();

646

648 dbgs() << CurrentIdx << ": GIM_CheckMemorySizeEqual(MIs["

649 << InsnID << "]->memoperands() + " << MMOIdx

650 << ", Size=" << Size << ")\n");

651 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

652

653 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {

654 if (handleReject() == RejectAndGiveUp)

655 return false;

656 break;

657 }

658

660 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);

661

663 << " bytes vs " << Size

664 << " bytes\n");

666 if (handleReject() == RejectAndGiveUp)

667 return false;

668

669 break;

670 }

674 uint64_t InsnID = readULEB();

675 uint64_t MMOIdx = readULEB();

677

679 TgtExecutor::getName(),

680 dbgs() << CurrentIdx << ": GIM_CheckMemorySize"

683 ? "GreaterThan"

684 : "LessThan")

685 << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx

686 << ", OpIdx=" << OpIdx << ")\n");

687 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

688

690 if (!MO.isReg()) {

692 dbgs() << CurrentIdx << ": Not a register\n");

693 if (handleReject() == RejectAndGiveUp)

694 return false;

695 break;

696 }

697

698 if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {

699 if (handleReject() == RejectAndGiveUp)

700 return false;

701 break;

702 }

703

705 *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);

706

710 if (handleReject() == RejectAndGiveUp)

711 return false;

714 if (handleReject() == RejectAndGiveUp)

715 return false;

718 if (handleReject() == RejectAndGiveUp)

719 return false;

720

721 break;

722 }

727 int TypeID = readS8();

729 dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID

730 << "]->getOperand(" << OpIdx

731 << "), TypeID=" << TypeID << ")\n");

732 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

735 if (handleReject() == RejectAndGiveUp)

736 return false;

737 }

738 break;

739 }

741 uint64_t InsnID = readULEB();

743 uint64_t SizeInBits = readULEB();

744

746 dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["

747 << InsnID << "]->getOperand(" << OpIdx

748 << "), SizeInBits=" << SizeInBits << ")\n");

749 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

752

753

754 if (SizeInBits == 0) {

755 MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();

756 const unsigned AddrSpace = Ty.getAddressSpace();

757 SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);

758 }

759

760 assert(SizeInBits != 0 && "Pointer size must be known");

761

762 if (MO.isReg()) {

763 if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)

764 if (handleReject() == RejectAndGiveUp)

765 return false;

766 } else if (handleReject() == RejectAndGiveUp)

767 return false;

768

769 break;

770 }

772 uint64_t InsnID = readULEB();

774 uint64_t StoreIdx = readULEB();

775

777 dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs["

778 << InsnID << "]->getOperand(" << OpIdx

779 << "), StoreIdx=" << StoreIdx << ")\n");

780 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

781 assert(StoreIdx < State.RecordedOperands.size() && "Index out of range");

782 State.RecordedOperands[StoreIdx] = &State.MIs[InsnID]->getOperand(OpIdx);

783 break;

784 }

786 uint64_t InsnID = readULEB();

788 int TypeIdx = readS8();

789

791 dbgs() << CurrentIdx << ": GIM_RecordRegType(MIs["

792 << InsnID << "]->getOperand(" << OpIdx

793 << "), TypeIdx=" << TypeIdx << ")\n");

794 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

795 assert(TypeIdx < 0 && "Temp types always have negative indexes!");

796

797 TypeIdx = 1 - TypeIdx;

798 const auto &Op = State.MIs[InsnID]->getOperand(OpIdx);

799 if (State.RecordedTypes.size() <= (uint64_t)TypeIdx)

800 State.RecordedTypes.resize(TypeIdx + 1, LLT());

801 State.RecordedTypes[TypeIdx] = MRI.getType(Op.getReg());

802 break;

803 }

804

812 dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["

813 << InsnID << "]->getOperand(" << OpIdx

814 << "), RCEnum=" << RCEnum << ")\n");

815 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

817 if (!MO.isReg() ||

821 if (handleReject() == RejectAndGiveUp)

822 return false;

823 }

824 break;

825 }

826

828 uint64_t InsnID = readULEB();

830 uint16_t RendererID = readU16();

831 uint16_t ComplexPredicateID = readU16();

833 dbgs() << CurrentIdx << ": State.Renderers[" << RendererID

834 << "] = GIM_CheckComplexPattern(MIs[" << InsnID

835 << "]->getOperand(" << OpIdx

836 << "), ComplexPredicateID=" << ComplexPredicateID

837 << ")\n");

838 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

839

842 State.MIs[InsnID]->getOperand(OpIdx));

843 if (Renderer)

844 State.Renderers[RendererID] = *Renderer;

845 else if (handleReject() == RejectAndGiveUp)

846 return false;

847 break;

848 }

849

853

854 uint64_t InsnID = readULEB();

856 uint64_t Value = IsInt8 ? (int64_t)readS8() : readU64();

858 dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["

859 << InsnID << "]->getOperand(" << OpIdx

860 << "), Value=" << Value << ")\n");

861 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

863 if (MO.isReg()) {

864

866

867

868 if (Ty.getScalarSizeInBits() > 64) {

869 if (handleReject() == RejectAndGiveUp)

870 return false;

871 break;

872 }

873

876 if (handleReject() == RejectAndGiveUp)

877 return false;

878 }

879 } else if (handleReject() == RejectAndGiveUp)

880 return false;

881

882 break;

883 }

884

886 uint64_t InsnID = readULEB();

888 int64_t Value = readU64();

890 dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["

891 << InsnID << "]->getOperand(" << OpIdx

892 << "), Value=" << Value << ")\n");

893 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

896 break;

897

899 break;

900

901 if (handleReject() == RejectAndGiveUp)

902 return false;

903

904 break;

905 }

906

908 uint64_t InsnID = readULEB();

912 dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["

913 << InsnID << "]->getOperand(" << OpIdx

914 << "), Value=" << Value << ")\n");

915 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

918 if (handleReject() == RejectAndGiveUp)

919 return false;

920 break;

921 }

923 uint64_t InsnID = readULEB();

927 dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["

928 << InsnID << "]->getOperand(" << OpIdx

929 << "), Value=" << Value << ")\n");

930 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

933 if (handleReject() == RejectAndGiveUp)

934 return false;

935 break;

936 }

938 uint64_t InsnID = readULEB();

941 dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID

942 << "]->getOperand(" << OpIdx << "))\n");

943 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

944 if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {

945 if (handleReject() == RejectAndGiveUp)

946 return false;

947 }

948 break;

949 }

951 uint64_t InsnID = readULEB();

954 dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID

955 << "]->getOperand(" << OpIdx << "))\n");

956 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

957 if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {

958 if (handleReject() == RejectAndGiveUp)

959 return false;

960 }

961 break;

962 }

964 uint64_t NumInsn = MatchTable[CurrentIdx++];

966 dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(N = "

967 << NumInsn << ")\n");

969 for (unsigned K = 1, E = NumInsn + 1; K < E; ++K) {

971 if (handleReject() == RejectAndGiveUp)

972 return false;

973 }

974 }

975 break;

976 }

979 uint64_t InsnID = readULEB();

981 uint64_t OtherInsnID = readULEB();

982 uint64_t OtherOpIdx = readULEB();

984 dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["

985 << InsnID << "][" << OpIdx << "], MIs["

986 << OtherInsnID << "][" << OtherOpIdx << "])\n");

987 assert(State.MIs[InsnID] != nullptr && "Used insn before defined");

988 assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");

989

991 MachineOperand &OtherOp = State.MIs[OtherInsnID]->getOperand(OtherOpIdx);

992

994 if (Op.isReg() && OtherOp.isReg()) {

997 break;

998 }

999 }

1000

1001 if (Op.isIdenticalTo(OtherOp)) {

1002 if (handleReject() == RejectAndGiveUp)

1003 return false;

1004 }

1005 break;

1006 }

1008 uint64_t OldInsnID = readULEB();

1009 uint64_t OldOpIdx = readULEB();

1010 uint64_t NewInsnID = readULEB();

1011 uint64_t NewOpIdx = readULEB();

1012

1014 dbgs() << CurrentIdx << ": GIM_CheckCanReplaceReg(MIs["

1015 << OldInsnID << "][" << OldOpIdx << "] = MIs["

1016 << NewInsnID << "][" << NewOpIdx << "])\n");

1017

1018 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();

1019 Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg();

1021 if (handleReject() == RejectAndGiveUp)

1022 return false;

1023 }

1024 break;

1025 }

1027 uint64_t InsnID = readULEB();

1029

1031 dbgs() << CurrentIdx << ": GIM_MIFlags(MIs[" << InsnID

1032 << "], " << Flags << ")\n");

1033 if ((State.MIs[InsnID]->getFlags() & Flags) != Flags) {

1034 if (handleReject() == RejectAndGiveUp)

1035 return false;

1036 }

1037 break;

1038 }

1040 uint64_t InsnID = readULEB();

1042

1044 dbgs() << CurrentIdx << ": GIM_MIFlagsNot(MIs[" << InsnID

1045 << "], " << Flags << ")\n");

1046 if ((State.MIs[InsnID]->getFlags() & Flags)) {

1047 if (handleReject() == RejectAndGiveUp)

1048 return false;

1049 }

1050 break;

1051 }

1054 dbgs() << CurrentIdx << ": GIM_Reject\n");

1055 if (handleReject() == RejectAndGiveUp)

1056 return false;

1057 break;

1059 uint64_t OldInsnID = readULEB();

1060 uint64_t NewInsnID = readULEB();

1061 uint16_t NewOpcode = readU16();

1062 if (NewInsnID >= OutMIs.size())

1063 OutMIs.resize(NewInsnID + 1);

1064

1066 if (Observer)

1069 OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));

1070 if (Observer)

1073 dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["

1074 << NewInsnID << "], MIs[" << OldInsnID << "], "

1075 << NewOpcode << ")\n");

1076 break;

1077 }

1078

1082 uint16_t Opcode = readU16();

1083 if (NewInsnID >= OutMIs.size())

1084 OutMIs.resize(NewInsnID + 1);

1085

1086 OutMIs[NewInsnID] = Builder.buildInstr(Opcode);

1088 dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["

1089 << NewInsnID << "], " << Opcode << ")\n");

1090 break;

1091 }

1092

1094 uint64_t TempRegID = readULEB();

1096 Builder.buildConstant(State.TempRegisters[TempRegID], Imm);

1098 dbgs() << CurrentIdx << ": GIR_BuildConstant(TempReg["

1099 << TempRegID << "], Imm=" << Imm << ")\n");

1100 break;

1101 }

1102

1110 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");

1111 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));

1114 << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID

1115 << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");

1116 break;

1117 }

1118

1120 uint64_t NewInsnID = readULEB();

1121 uint64_t OldInsnID = readULEB();

1123 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");

1124 MachineInstr &OldMI = *State.MIs[OldInsnID];

1129 dbgs() << CurrentIdx << ": GIR_CopyRemaining(OutMIs["

1130 << NewInsnID << "], MIs[" << OldInsnID

1131 << "], /*start=*/" << OpIdx << ")\n");

1132 break;

1133 }

1134

1136 uint64_t NewInsnID = readULEB();

1137 uint64_t OldInsnID = readULEB();

1139 uint16_t ZeroReg = readU16();

1140 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");

1143 OutMIs[NewInsnID].addReg(ZeroReg);

1144 else

1145 OutMIs[NewInsnID].add(MO);

1147 dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["

1148 << NewInsnID << "], MIs[" << OldInsnID << "], "

1149 << OpIdx << ", " << ZeroReg << ")\n");

1150 break;

1151 }

1152

1154 uint64_t NewInsnID = readULEB();

1155 uint64_t OldInsnID = readULEB();

1157 uint16_t SubRegIdx = readU16();

1158 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");

1159 OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),

1160 0, SubRegIdx);

1162 dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["

1163 << NewInsnID << "], MIs[" << OldInsnID << "], "

1164 << OpIdx << ", " << SubRegIdx << ")\n");

1165 break;

1166 }

1167

1169 uint64_t InsnID = readULEB();

1170 uint16_t RegNum = readU16();

1172 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");

1174 OutMIs[InsnID].addDef(RegNum, Flags);

1176 dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["

1177 << InsnID << "], " << RegNum << ")\n");

1178 break;

1179 }

1180

1182 uint64_t InsnID = readULEB();

1183 uint16_t RegNum = readU16();

1184 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");

1187 dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["

1188 << InsnID << "], " << RegNum << ")\n");

1189 break;

1190 }

1191

1193 uint64_t InsnID = readULEB();

1194 uint16_t RegNum = readU16();

1195 uint16_t RegFlags = readU16();

1196 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");

1197 OutMIs[InsnID].addReg(RegNum, RegFlags);

1200 << CurrentIdx << ": GIR_AddRegister(OutMIs[" << InsnID

1201 << "], " << RegNum << ", " << RegFlags << ")\n");

1202 break;

1203 }

1205 uint64_t InsnID = readULEB();

1207 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");

1210 dbgs() << CurrentIdx << ": GIR_AddIntrinsicID(OutMIs["

1211 << InsnID << "], " << Value << ")\n");

1212 break;

1213 }

1215 uint64_t InsnID = readULEB();

1218 dbgs() << CurrentIdx << ": GIR_SetImplicitDefDead(OutMIs["

1219 << InsnID << "], OpIdx=" << OpIdx << ")\n");

1221 assert(MI && "Modifying undefined instruction");

1222 MI->getOperand(MI->getNumExplicitOperands() + OpIdx).setIsDead();

1223 break;

1224 }

1226 uint64_t InsnID = readULEB();

1228

1230 dbgs() << CurrentIdx << ": GIR_SetMIFlags(OutMIs["

1231 << InsnID << "], " << Flags << ")\n");

1233 MI->setFlags(MI->getFlags() | Flags);

1234 break;

1235 }

1237 uint64_t InsnID = readULEB();

1239

1241 dbgs() << CurrentIdx << ": GIR_UnsetMIFlags(OutMIs["

1242 << InsnID << "], " << Flags << ")\n");

1244 MI->setFlags(MI->getFlags() & ~Flags);

1245 break;

1246 }

1248 uint64_t InsnID = readULEB();

1249 uint64_t OldInsnID = readULEB();

1250

1252 dbgs() << CurrentIdx << ": GIR_CopyMIFlags(OutMIs["

1253 << InsnID << "], MIs[" << OldInsnID << "])\n");

1255 MI->setFlags(MI->getFlags() | State.MIs[OldInsnID]->getFlags());

1256 break;

1257 }

1261 uint64_t InsnID = readULEB();

1262 uint64_t TempRegID = readULEB();

1265 TempRegFlags = readU16();

1269

1270 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");

1271

1272 OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags,

1275 TgtExecutor::getName(),

1276 dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs[" << InsnID

1277 << "], TempRegisters[" << TempRegID << "]";

1279 dbgs() << ", " << TempRegFlags << ")\n");

1280 break;

1281 }

1282

1285 const bool IsAdd8 = (MatcherOpcode == GIR_AddImm8);

1286 uint64_t InsnID = readULEB();

1287 uint64_t Imm = IsAdd8 ? (int64_t)readS8() : readU64();

1288 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");

1289 OutMIs[InsnID].addImm(Imm);

1291 dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID

1292 << "], " << Imm << ")\n");

1293 break;

1294 }

1295

1297 uint64_t InsnID = readULEB();

1298 int TypeID = readS8();

1300 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");

1301

1303 LLVMContext &Ctx = MF->getFunction().getContext();

1304 OutMIs[InsnID].addCImm(

1305 ConstantInt::get(IntegerType::get(Ctx, Width), Imm, true));

1307 dbgs() << CurrentIdx << ": GIR_AddCImm(OutMIs[" << InsnID

1308 << "], TypeID=" << TypeID << ", Imm=" << Imm

1309 << ")\n");

1310 break;

1311 }

1312

1314 uint64_t InsnID = readULEB();

1315 uint16_t RendererID = readU16();

1316 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");

1317 for (const auto &RenderOpFn : State.Renderers[RendererID])

1318 RenderOpFn(OutMIs[InsnID]);

1320 dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["

1321 << InsnID << "], " << RendererID << ")\n");

1322 break;

1323 }

1325 uint64_t InsnID = readULEB();

1326 uint16_t RendererID = readU16();

1327 uint64_t RenderOpID = readULEB();

1328 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");

1329 State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);

1331 dbgs() << CurrentIdx

1332 << ": GIR_ComplexSubOperandRenderer(OutMIs["

1333 << InsnID << "], " << RendererID << ", "

1334 << RenderOpID << ")\n");

1335 break;

1336 }

1338 uint64_t InsnID = readULEB();

1339 uint16_t RendererID = readU16();

1340 uint64_t RenderOpID = readULEB();

1341 uint16_t SubRegIdx = readU16();

1343 assert(MI && "Attempted to add to undefined instruction");

1344 State.Renderers[RendererID][RenderOpID](MI);

1345 MI->getOperand(MI->getNumOperands() - 1).setSubReg(SubRegIdx);

1347 dbgs() << CurrentIdx

1348 << ": GIR_ComplexSubOperandSubRegRenderer(OutMIs["

1349 << InsnID << "], " << RendererID << ", "

1350 << RenderOpID << ", " << SubRegIdx << ")\n");

1351 break;

1352 }

1353

1355 uint64_t NewInsnID = readULEB();

1356 uint64_t OldInsnID = readULEB();

1357 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");

1358 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&

1359 "Expected G_CONSTANT");

1360 if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {

1361 OutMIs[NewInsnID].addImm(

1362 State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());

1363 } else if (State.MIs[OldInsnID]->getOperand(1).isImm())

1364 OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));

1365 else

1368 dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["

1369 << NewInsnID << "], MIs[" << OldInsnID << "])\n");

1370 break;

1371 }

1372

1373

1375 uint64_t NewInsnID = readULEB();

1376 uint64_t OldInsnID = readULEB();

1377 assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");

1378 assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&

1379 "Expected G_FCONSTANT");

1380 if (State.MIs[OldInsnID]->getOperand(1).isFPImm())

1381 OutMIs[NewInsnID].addFPImm(

1382 State.MIs[OldInsnID]->getOperand(1).getFPImm());

1383 else

1387 << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["

1388 << NewInsnID << "], MIs[" << OldInsnID << "])\n");

1389 break;

1390 }

1391

1393 uint64_t InsnID = readULEB();

1394 uint64_t OldInsnID = readULEB();

1395 uint16_t RendererFnID = readU16();

1396 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");

1398 dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["

1399 << InsnID << "], MIs[" << OldInsnID << "], "

1400 << RendererFnID << ")\n");

1402 OutMIs[InsnID], *State.MIs[OldInsnID],

1403 -1);

1404 break;

1405 }

1409 dbgs() << CurrentIdx << ": GIR_DoneWithCustomAction(FnID="

1410 << FnID << ")\n");

1413 propagateFlags();

1414 return true;

1415 }

1416

1417 if (handleReject() == RejectAndGiveUp)

1418 return false;

1419 break;

1420 }

1422 uint64_t InsnID = readULEB();

1423 uint64_t OldInsnID = readULEB();

1425 uint16_t RendererFnID = readU16();

1426 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");

1427

1429 dbgs() << CurrentIdx

1430 << ": GIR_CustomOperandRenderer(OutMIs[" << InsnID

1431 << "], MIs[" << OldInsnID << "]->getOperand("

1432 << OpIdx << "), " << RendererFnID << ")\n");

1434 OutMIs[InsnID], *State.MIs[OldInsnID], OpIdx);

1435 break;

1436 }

1438 uint64_t InsnID = readULEB();

1440 uint16_t RCEnum = readU16();

1441 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");

1449 dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["

1450 << InsnID << "], " << OpIdx << ", " << RCEnum

1451 << ")\n");

1452 break;

1453 }

1454

1458 ? 0

1459 : readULEB();

1460 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");

1462 RBI);

1464 dbgs() << CurrentIdx

1465 << ": GIR_ConstrainSelectedInstOperands(OutMIs["

1466 << InsnID << "])\n");

1467 break;

1468 }

1470 uint64_t InsnID = readULEB();

1471 uint64_t NumInsn = MatchTable[CurrentIdx++];

1472 assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");

1473

1475 dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["

1476 << InsnID << "]");

1477 for (unsigned K = 0; K < NumInsn; ++K) {

1478 uint64_t NextID = readULEB();

1480 dbgs() << ", MIs[" << NextID << "]");

1481 for (const auto &MMO : State.MIs[NextID]->memoperands())

1482 OutMIs[InsnID].addMemOperand(MMO);

1483 }

1485 break;

1486 }

1488 uint64_t InsnID = readULEB();

1490 assert(MI && "Attempted to erase an undefined instruction");

1492 dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["

1493 << InsnID << "])\n");

1494 eraseImpl(MI);

1495 break;

1496 }

1500 << CurrentIdx << ": GIR_EraseRootFromParent_Done\n");

1501 eraseImpl(State.MIs[0]);

1502 propagateFlags();

1503 return true;

1504 }

1506 uint64_t TempRegID = readULEB();

1507 int TypeID = readS8();

1508

1509 State.TempRegisters[TempRegID] =

1510 MRI.createGenericVirtualRegister(getTypeFromIdx(TypeID));

1512 dbgs() << CurrentIdx << ": TempRegs[" << TempRegID

1513 << "] = GIR_MakeTempReg(" << TypeID << ")\n");

1514 break;

1515 }

1517 uint64_t OldInsnID = readULEB();

1518 uint64_t OldOpIdx = readULEB();

1519 uint64_t NewInsnID = readULEB();

1520 uint64_t NewOpIdx = readULEB();

1521

1523 dbgs() << CurrentIdx << ": GIR_ReplaceReg(MIs["

1524 << OldInsnID << "][" << OldOpIdx << "] = MIs["

1525 << NewInsnID << "][" << NewOpIdx << "])\n");

1526

1527 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();

1528 Register New = State.MIs[NewInsnID]->getOperand(NewOpIdx).getReg();

1529 if (Observer)

1531 MRI.replaceRegWith(Old, New);

1532 if (Observer)

1534 break;

1535 }

1537 uint64_t OldInsnID = readULEB();

1538 uint64_t OldOpIdx = readULEB();

1539 uint64_t TempRegID = readULEB();

1540

1542 dbgs() << CurrentIdx << ": GIR_ReplaceRegWithTempReg(MIs["

1543 << OldInsnID << "][" << OldOpIdx << "] = TempRegs["

1544 << TempRegID << "])\n");

1545

1546 Register Old = State.MIs[OldInsnID]->getOperand(OldOpIdx).getReg();

1547 Register New = State.TempRegisters[TempRegID];

1548 if (Observer)

1550 MRI.replaceRegWith(Old, New);

1551 if (Observer)

1553 break;

1554 }

1556 uint32_t RuleID = readU32();

1559

1561 << ": GIR_Coverage("

1562 << RuleID << ")");

1563 break;

1564 }

1565

1568 dbgs() << CurrentIdx << ": GIR_Done\n");

1569 propagateFlags();

1570 return true;

1571 default:

1573 }

1574 }

1575}