CombinerHelper.cpp Source File (original) (raw)

64 RBI(Builder.getMF().getSubtarget().getRegBankInfo()),

65 TRI(Builder.getMF().getSubtarget().getRegisterInfo()) {

66 (void)this->VT;

67}

70 return *Builder.getMF().getSubtarget().getTargetLowering();

71}

88 assert(I < ByteWidth && "I must be in [0, ByteWidth)");

89 return I;

90}

96 LLT Ty = MRI.getType(V);

100}

106static unsigned bigEndianByteAt(const unsigned ByteWidth, const unsigned I) {

107 assert(I < ByteWidth && "I must be in [0, ByteWidth)");

108 return ByteWidth - I - 1;

109}

128static std::optional

130 int64_t LowestIdx) {

131

132 unsigned Width = MemOffset2Idx.size();

133 if (Width < 2)

134 return std::nullopt;

135 bool BigEndian = true, LittleEndian = true;

136 for (unsigned MemOffset = 0; MemOffset < Width; ++ MemOffset) {

137 auto MemOffsetAndIdx = MemOffset2Idx.find(MemOffset);

138 if (MemOffsetAndIdx == MemOffset2Idx.end())

139 return std::nullopt;

140 const int64_t Idx = MemOffsetAndIdx->second - LowestIdx;

141 assert(Idx >= 0 && "Expected non-negative byte offset?");

144 if (!BigEndian && !LittleEndian)

145 return std::nullopt;

146 }

147

148 assert((BigEndian != LittleEndian) &&

149 "Pattern cannot be both big and little endian!");

150 return BigEndian;

151}

152

154

156 assert(LI && "Must have LegalizerInfo to query isLegal!");

158}

159

164

169

171 if (!Ty.isVector())

173

175 return true;

177 return isLegal({TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}}) &&

178 isLegal({TargetOpcode::G_CONSTANT, {EltTy}});

179}

180

183 Observer.changingAllUsesOfReg(MRI, FromReg);

184

185 if (MRI.constrainRegAttrs(ToReg, FromReg))

186 MRI.replaceRegWith(FromReg, ToReg);

187 else

188 Builder.buildCopy(FromReg, ToReg);

189

190 Observer.finishedChangingAllUsesOfReg();

191}

192

196 assert(FromRegOp.getParent() && "Expected an operand in an MI");

198

199 FromRegOp.setReg(ToReg);

200

202}

203

205 unsigned ToOpcode) const {

206 Observer.changingInstr(FromMI);

207

209

210 Observer.changedInstr(FromMI);

211}

212

214 return RBI->getRegBank(Reg, MRI, *TRI);

215}

216

219 if (RegBank)

220 MRI.setRegBank(Reg, *RegBank);

221}

222

226 return true;

227 }

228 return false;

229}

231 if (MI.getOpcode() != TargetOpcode::COPY)

232 return false;

233 Register DstReg = MI.getOperand(0).getReg();

234 Register SrcReg = MI.getOperand(1).getReg();

236}

238 Register DstReg = MI.getOperand(0).getReg();

239 Register SrcReg = MI.getOperand(1).getReg();

241 MI.eraseFromParent();

242}

243

246

248 Register OrigOp = MI.getOperand(1).getReg();

249

250 if ( MRI .hasOneNonDBGUse(OrigOp))

251 return false;

252

254

255

256

257

258

259

260

261

263 return false;

264

266 false))

267 return false;

268

269 std::optional MaybePoisonOperand;

271 if (!Operand.isReg())

272 return false;

273

275 continue;

276

277 if (!MaybePoisonOperand)

278 MaybePoisonOperand = Operand;

279 else {

280

281

282 return false;

283 }

284 }

285

286

287 if (!MaybePoisonOperand) {

289 Observer.changingInstr(*OrigDef);

291 Observer.changedInstr(*OrigDef);

292 B.buildCopy(DstOp, OrigOp);

293 };

294 return true;

295 }

296

297 Register MaybePoisonOperandReg = MaybePoisonOperand->getReg();

298 LLT MaybePoisonOperandRegTy = MRI.getType(MaybePoisonOperandReg);

299

301 Observer.changingInstr(*OrigDef);

303 Observer.changedInstr(*OrigDef);

305 auto Freeze = B.buildFreeze(MaybePoisonOperandRegTy, MaybePoisonOperandReg);

310 };

311 return true;

312}

313

316 assert(MI.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&

317 "Invalid instruction");

318 bool IsUndef = true;

320

321

322

323

327 assert(Def && "Operand not defined");

328 if ( MRI .hasOneNonDBGUse(Reg))

329 return false;

330 switch (Def->getOpcode()) {

331 case TargetOpcode::G_BUILD_VECTOR:

332 IsUndef = false;

333

334

336 Ops.push_back(BuildVecMO.getReg());

337 break;

338 case TargetOpcode::G_IMPLICIT_DEF: {

339 LLT OpType = MRI.getType(Reg);

340

341 if (!Undef) {

342 Builder.setInsertPt(*MI.getParent(), MI);

343 Undef = Builder.buildUndef(OpType.getScalarType());

344 }

345 assert(MRI.getType(Undef->getOperand(0).getReg()) ==

346 OpType.getScalarType() &&

347 "All undefs should have the same type");

348

349

350 for (unsigned EltIdx = 0, EltEnd = OpType.getNumElements();

351 EltIdx != EltEnd; ++EltIdx)

352 Ops.push_back(Undef->getOperand(0).getReg());

353 break;

354 }

355 default:

356 return false;

357 }

358 }

359

360

361 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());

363 {TargetOpcode::G_BUILD_VECTOR, {DstTy, MRI.getType(Ops[0])}})) {

364 return false;

365 }

366

367 if (IsUndef)

368 Ops.clear();

369

370 return true;

371}

374

375

376 Register DstReg = MI.getOperand(0).getReg();

377 Builder.setInsertPt(*MI.getParent(), MI);

378 Register NewDstReg = MRI.cloneVirtualRegister(DstReg);

379

380

381

382

383

384

385

386 if (Ops.empty())

387 Builder.buildUndef(NewDstReg);

388 else

389 Builder.buildBuildVector(NewDstReg, Ops);

391 MI.eraseFromParent();

392}

393

396

397 Register SrcVec1 = Shuffle.getSrc1Reg();

398 Register SrcVec2 = Shuffle.getSrc2Reg();

399 LLT EltTy = MRI.getType(SrcVec1).getElementType();

400 int Width = MRI.getType(SrcVec1).getNumElements();

401

402 auto Unmerge1 = Builder.buildUnmerge(EltTy, SrcVec1);

403 auto Unmerge2 = Builder.buildUnmerge(EltTy, SrcVec2);

404

406

407 for (int Val : Shuffle.getMask()) {

408 if (Val == -1)

410 else if (Val < Width)

411 Extracts.push_back(Unmerge1.getReg(Val));

412 else

413 Extracts.push_back(Unmerge2.getReg(Val - Width));

414 }

415 assert(Extracts.size() > 0 && "Expected at least one element in the shuffle");

416 if (Extracts.size() == 1)

417 Builder.buildCopy(MI.getOperand(0).getReg(), Extracts[0]);

418 else

419 Builder.buildBuildVector(MI.getOperand(0).getReg(), Extracts);

420 MI.eraseFromParent();

421}

422

426 auto ConcatMI1 =

428 auto ConcatMI2 =

430 if (!ConcatMI1 || !ConcatMI2)

431 return false;

432

433

434 if (MRI.getType(ConcatMI1->getSourceReg(0)) !=

435 MRI.getType(ConcatMI2->getSourceReg(0)))

436 return false;

437

438 LLT ConcatSrcTy = MRI.getType(ConcatMI1->getReg(1));

439 LLT ShuffleSrcTy1 = MRI.getType(MI.getOperand(1).getReg());

440 unsigned ConcatSrcNumElt = ConcatSrcTy.getNumElements();

441 for (unsigned i = 0; i < Mask.size(); i += ConcatSrcNumElt) {

442

443

444 if (Mask[i] == -1) {

445 for (unsigned j = 1; j < ConcatSrcNumElt; j++) {

446 if (i + j >= Mask.size())

447 return false;

448 if (Mask[i + j] != -1)

449 return false;

450 }

452 {TargetOpcode::G_IMPLICIT_DEF, {ConcatSrcTy}}))

453 return false;

454 Ops.push_back(0);

455 } else if (Mask[i] % ConcatSrcNumElt == 0) {

456 for (unsigned j = 1; j < ConcatSrcNumElt; j++) {

457 if (i + j >= Mask.size())

458 return false;

459 if (Mask[i + j] != Mask[i] + static_cast<int>(j))

460 return false;

461 }

462

463

465 Ops.push_back(ConcatMI1->getSourceReg(Mask[i] / ConcatSrcNumElt));

466 } else {

467 Ops.push_back(ConcatMI2->getSourceReg(Mask[i] / ConcatSrcNumElt -

468 ConcatMI1->getNumSources()));

469 }

470 } else {

471 return false;

472 }

473 }

474

476 {TargetOpcode::G_CONCAT_VECTORS,

477 {MRI.getType(MI.getOperand(0).getReg()), ConcatSrcTy}}))

478 return false;

479

480 return Ops .empty();

481}

482

485 LLT SrcTy;

487 if (Reg != 0)

488 SrcTy = MRI.getType(Reg);

489 }

490 assert(SrcTy.isValid() && "Unexpected full undef vector in concat combine");

491

493

495 if (Reg == 0) {

496 if (UndefReg == 0)

497 UndefReg = Builder.buildUndef(SrcTy).getReg(0);

498 Reg = UndefReg;

499 }

500 }

501

502 if (Ops.size() > 1)

503 Builder.buildConcatVectors(MI.getOperand(0).getReg(), Ops);

504 else

505 Builder.buildCopy(MI.getOperand(0).getReg(), Ops[0]);

506 MI.eraseFromParent();

507}

508

513 return true;

514 }

515 return false;

516}

517

520 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&

521 "Invalid instruction kind");

522 LLT DstType = MRI.getType(MI.getOperand(0).getReg());

523 Register Src1 = MI.getOperand(1).getReg();

524 LLT SrcType = MRI.getType(Src1);

525

526 unsigned DstNumElts = DstType.getNumElements();

527 unsigned SrcNumElts = SrcType.getNumElements();

528

529

530

531

532

533

534

535

536

537

538

539

540

541

542

543

544 if (DstNumElts < 2 * SrcNumElts)

545 return false;

546

547

548

549 if (DstNumElts % SrcNumElts != 0)

550 return false;

551

552

553

554

555 unsigned NumConcat = DstNumElts / SrcNumElts;

558 for (unsigned i = 0; i != DstNumElts; ++i) {

559 int Idx = Mask[i];

560

561 if (Idx < 0)

562 continue;

563

564

565 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||

566 (ConcatSrcs[i / SrcNumElts] >= 0 &&

567 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts)))

568 return false;

569

570 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;

571 }

572

573

574

576 Register Src2 = MI.getOperand(2).getReg();

577 for (auto Src : ConcatSrcs) {

578 if (Src < 0) {

579 if (!UndefReg) {

580 Builder.setInsertPt(*MI.getParent(), MI);

581 UndefReg = Builder.buildUndef(SrcType).getReg(0);

582 }

583 Ops.push_back(UndefReg);

584 } else if (Src == 0)

585 Ops.push_back(Src1);

586 else

587 Ops.push_back(Src2);

588 }

589 return true;

590}

591

594 Register DstReg = MI.getOperand(0).getReg();

595 Builder.setInsertPt(*MI.getParent(), MI);

596 Register NewDstReg = MRI.cloneVirtualRegister(DstReg);

597

598 if (Ops.size() == 1)

599 Builder.buildCopy(NewDstReg, Ops[0]);

600 else

601 Builder.buildMergeLikeInstr(NewDstReg, Ops);

602

604 MI.eraseFromParent();

605}

606

607namespace {

608

609

610

613 const LLT TyForCandidate,

614 unsigned OpcodeForCandidate,

617 if (CurrentUse.ExtendOpcode == OpcodeForCandidate ||

618 CurrentUse.ExtendOpcode == TargetOpcode::G_ANYEXT)

619 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};

620 return CurrentUse;

621 }

622

623

624

625

626

627

628

629

630 if (OpcodeForCandidate == TargetOpcode::G_ANYEXT &&

631 CurrentUse.ExtendOpcode != TargetOpcode::G_ANYEXT)

632 return CurrentUse;

633 else if (CurrentUse.ExtendOpcode == TargetOpcode::G_ANYEXT &&

634 OpcodeForCandidate != TargetOpcode::G_ANYEXT)

635 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};

636

637

638

639

640

641 if ( isa<GZExtLoad> (LoadMI) && CurrentUse.Ty == TyForCandidate) {

642 if (CurrentUse.ExtendOpcode == TargetOpcode::G_SEXT &&

643 OpcodeForCandidate == TargetOpcode::G_ZEXT)

644 return CurrentUse;

645 else if (CurrentUse.ExtendOpcode == TargetOpcode::G_ZEXT &&

646 OpcodeForCandidate == TargetOpcode::G_SEXT)

647 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};

648 }

649

650

651

652

653

654

656 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};

657 }

658 return CurrentUse;

659}

660

661

662

663

664

665

666

667static void InsertInsnsWithoutSideEffectsBeforeUse(

671 Inserter) {

673

675

676

677 if (UseMI.isPHI()) {

679 InsertBB = PredBB->getMBB();

680 }

681

682

683

684 if (InsertBB == DefMI.getParent()) {

686 Inserter(InsertBB, std::next(InsertPt), UseMO);

687 return;

688 }

689

690

691 Inserter(InsertBB, InsertBB->getFirstNonPHI(), UseMO);

692}

693}

694

699 return true;

700 }

701 return false;

702}

703

705 unsigned CandidateLoadOpc;

706 switch (ExtOpc) {

707 case TargetOpcode::G_ANYEXT:

708 CandidateLoadOpc = TargetOpcode::G_LOAD;

709 break;

710 case TargetOpcode::G_SEXT:

711 CandidateLoadOpc = TargetOpcode::G_SEXTLOAD;

712 break;

713 case TargetOpcode::G_ZEXT:

714 CandidateLoadOpc = TargetOpcode::G_ZEXTLOAD;

715 break;

716 default:

718 }

719 return CandidateLoadOpc;

720}

721

724

725

726

727

728

729

731 if (!LoadMI)

732 return false;

733

735

736 LLT LoadValueTy = MRI.getType(LoadReg);

738 return false;

739

740

741

742

743

744

746 return false;

747

748

749

751 return false;

752

753

754

755

756

757

758 unsigned PreferredOpcode =

760 ? TargetOpcode::G_ANYEXT

761 : isa(&MI) ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;

762 Preferred = {LLT(), PreferredOpcode, nullptr};

763 for (auto &UseMI : MRI.use_nodbg_instructions(LoadReg)) {

764 if (UseMI.getOpcode() == TargetOpcode::G_SEXT ||

765 UseMI.getOpcode() == TargetOpcode::G_ZEXT ||

766 (UseMI.getOpcode() == TargetOpcode::G_ANYEXT)) {

767 const auto &MMO = LoadMI->getMMO();

768

769 if (MMO.isAtomic())

770 continue;

771

775 LLT UseTy = MRI.getType(UseMI.getOperand(0).getReg());

777 if (LI->getAction({CandidateLoadOpc, {UseTy, SrcTy}, {MMDesc}})

779 continue;

780 }

781 Preferred = ChoosePreferredUse(MI, Preferred,

782 MRI.getType(UseMI.getOperand(0).getReg()),

784 }

785 }

786

787

788 if (!Preferred.MI)

789 return false;

790

791

792 assert(Preferred.Ty != LoadValueTy && "Extending to same type?");

793

794 LLVM_DEBUG(dbgs() << "Preferred use is: " << *Preferred.MI);

795 return true;

796}

797

800

802

803

804

810 if (PreviouslyEmitted) {

814 return;

815 }

816

817 Builder.setInsertPt(*InsertIntoBB, InsertBefore);

818 Register NewDstReg = MRI.cloneVirtualRegister(MI.getOperand(0).getReg());

820 EmittedInsns[InsertIntoBB] = NewMI;

822 };

823

826 MI.setDesc(Builder.getTII().get(LoadOpc));

827

828

832

833 for (auto *UseMO : Uses) {

835

836

837

839 UseMI->getOpcode() == TargetOpcode::G_ANYEXT) {

840 Register UseDstReg = UseMI->getOperand(0).getReg();

842 const LLT UseDstTy = MRI.getType(UseDstReg);

843 if (UseDstReg != ChosenDstReg) {

844 if (Preferred.Ty == UseDstTy) {

845

846

847

848

849

850

851

852

853

858

859

860

861

862

863

864

865

866

867

869 } else {

870

871

872

873

874

875

876

877

878

879

880

881 InsertInsnsWithoutSideEffectsBeforeUse(Builder, MI, *UseMO,

882 InsertTruncAt);

883 }

884 continue;

885 }

886

887

888

891 continue;

892 }

893

894

895

896 InsertInsnsWithoutSideEffectsBeforeUse(Builder, MI, *UseMO, InsertTruncAt);

897 }

898

899 MI.getOperand(0).setReg(ChosenDstReg);

901}

902

905 assert(MI.getOpcode() == TargetOpcode::G_AND);

906

907

908

909

910

911

912

913

914

915 Register Dst = MI.getOperand(0).getReg();

916 if (MRI.getType(Dst).isVector())

917 return false;

918

919 auto MaybeMask =

921 if (!MaybeMask)

922 return false;

923

924 APInt MaskVal = MaybeMask->Value;

925

926 if (!MaskVal.isMask())

927 return false;

928

929 Register SrcReg = MI.getOperand(1).getReg();

930

931

933 if (!LoadMI || MRI .hasOneNonDBGUse(LoadMI->getDstReg()))

934 return false;

935

937 LLT RegTy = MRI.getType(LoadReg);

941 unsigned MaskSizeBits = MaskVal.countr_one();

942

943

944

945 if (MaskSizeBits > LoadSizeBits.getValue())

946 return false;

947

948

949

950 if (MaskSizeBits >= RegSize)

951 return false;

952

953

954

955 if (MaskSizeBits < 8 || $isPowerOf2\_32$ (MaskSizeBits))

956 return false;

957

960

961

962

965 else if (LoadSizeBits.getValue() > MaskSizeBits ||

967 return false;

968

969

971 {TargetOpcode::G_ZEXTLOAD, {RegTy, MRI.getType(PtrReg)}, {MemDesc}}))

972 return false;

973

975 B.setInstrAndDebugLoc(*LoadMI);

976 auto &MF = B.getMF();

978 auto *NewMMO = MF.getMachineMemOperand(&MMO, PtrInfo, MemDesc.MemoryTy);

979 B.buildLoadInstr(TargetOpcode::G_ZEXTLOAD, Dst, PtrReg, *NewMMO);

981 };

982 return true;

983}

984

988 "shouldn't consider debug uses");

991 return true;

995 });

996 if (DefOrUse == MBB.end())

997 llvm_unreachable("Block must contain both DefMI and UseMI!");

998 return &*DefOrUse == &DefMI;

999}

1000

1004 "shouldn't consider debug uses");

1005 if (MDT)

1007 else if (DefMI.getParent() != UseMI.getParent())

1008 return false;

1009

1011}

1012

1014 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);

1015 Register SrcReg = MI.getOperand(1).getReg();

1017

1018 if (MRI.getType(SrcReg).isVector())

1019 return false;

1020

1023 LoadUser = TruncSrc;

1024

1025 uint64_t SizeInBits = MI.getOperand(2).getImm();

1026

1027

1029

1030 auto LoadSizeBits = LoadMI->getMemSizeInBits();

1031 if (TruncSrc &&

1032 MRI.getType(TruncSrc).getSizeInBits() < LoadSizeBits.getValue())

1033 return false;

1034 if (LoadSizeBits == SizeInBits)

1035 return true;

1036 }

1037 return false;

1038}

1039

1041 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);

1042 Builder.buildCopy(MI.getOperand(0).getReg(), MI.getOperand(1).getReg());

1043 MI.eraseFromParent();

1044}

1045

1047 MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) const {

1048 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);

1049

1050 Register DstReg = MI.getOperand(0).getReg();

1051 LLT RegTy = MRI.getType(DstReg);

1052

1053

1055 return false;

1056

1057 Register SrcReg = MI.getOperand(1).getReg();

1059 if (!LoadDef || MRI .hasOneNonDBGUse(SrcReg))

1060 return false;

1061

1062 uint64_t MemBits = LoadDef->getMemSizeInBits().getValue();

1063

1064

1065

1066

1067 unsigned NewSizeBits = std::min((uint64_t)MI.getOperand(2).getImm(), MemBits);

1068

1069

1070 if (NewSizeBits < 8)

1071 return false;

1072

1073

1075 return false;

1076

1079

1080

1081

1082 if (LoadDef->isSimple())

1084 else if (MemBits > NewSizeBits || MemBits == RegTy.getSizeInBits())

1085 return false;

1086

1087

1089 {MRI.getType(LoadDef->getDstReg()),

1090 MRI.getType(LoadDef->getPointerReg())},

1091 {MMDesc}}))

1092 return false;

1093

1094 MatchInfo = std::make_tuple(LoadDef->getDstReg(), NewSizeBits);

1095 return true;

1096}

1097

1099 MachineInstr &MI, std::tuple<Register, unsigned> &MatchInfo) const {

1100 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);

1102 unsigned ScalarSizeBits;

1103 std::tie(LoadReg, ScalarSizeBits) = MatchInfo;

1105

1106

1107

1108

1109

1110

1111

1112 auto &MMO = LoadDef->getMMO();

1113 Builder.setInstrAndDebugLoc(*LoadDef);

1114 auto &MF = Builder.getMF();

1115 auto PtrInfo = MMO.getPointerInfo();

1116 auto *NewMMO = MF.getMachineMemOperand(&MMO, PtrInfo, ScalarSizeBits / 8);

1117 Builder.buildLoadInstr(TargetOpcode::G_SEXTLOAD, MI.getOperand(0).getReg(),

1119 MI.eraseFromParent();

1120

1121

1123}

1124

1125

1126

1130 auto *MF = MI->getMF();

1132 if (!Addr)

1133 return false;

1134

1137 AM.BaseOffs = CstOff->getSExtValue();

1138 else

1139 AM.Scale = 1;

1140

1142 MF->getDataLayout(), AM,

1144 MF->getFunction().getContext()),

1145 MI->getMMO().getAddrSpace());

1146}

1147

1149 switch (LdStOpc) {

1150 case TargetOpcode::G_LOAD:

1151 return TargetOpcode::G_INDEXED_LOAD;

1152 case TargetOpcode::G_STORE:

1153 return TargetOpcode::G_INDEXED_STORE;

1154 case TargetOpcode::G_ZEXTLOAD:

1155 return TargetOpcode::G_INDEXED_ZEXTLOAD;

1156 case TargetOpcode::G_SEXTLOAD:

1157 return TargetOpcode::G_INDEXED_SEXTLOAD;

1158 default:

1160 }

1161}

1162

1163bool CombinerHelper::isIndexedLoadStoreLegal(GLoadStore &LdSt) const {

1164

1166 LLT Ty = MRI.getType(LdSt.getReg(0));

1173 if (IndexedOpc == TargetOpcode::G_INDEXED_STORE)

1174 OpTys = {PtrTy, Ty, Ty};

1175 else

1176 OpTys = {Ty, PtrTy};

1177

1178 LegalityQuery Q(IndexedOpc, OpTys, MemDescrs);

1180}

1181

1184 cl::desc("Number of uses of a base pointer to check before it is no longer "

1185 "considered for post-indexing."));

1186

1187bool CombinerHelper::findPostIndexCandidate(GLoadStore &LdSt, Register &Addr,

1189 bool &RematOffset) const {

1190

1191

1192

1193

1194

1196

1198

1199 if (MRI.hasOneNonDBGUse(Ptr))

1200 return false;

1201

1202 if (!isIndexedLoadStoreLegal(LdSt))

1203 return false;

1204

1205 if (getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Ptr, MRI))

1206 return false;

1207

1209 auto *PtrDef = MRI.getVRegDef(Ptr);

1210

1211 unsigned NumUsesChecked = 0;

1212 for (auto &Use : MRI.use_nodbg_instructions(Ptr)) {

1214 return false;

1215

1217

1218

1219 if (!PtrAdd || MRI.use_nodbg_empty(PtrAdd->getReg(0)))

1220 continue;

1221

1222

1223

1224 if (StoredValDef == &Use)

1225 continue;

1226

1227 Offset = PtrAdd->getOffsetReg();

1229 !TLI.isIndexingLegal(LdSt, PtrAdd->getBaseReg(), Offset,

1230 false, MRI))

1231 continue;

1232

1233

1235 RematOffset = false;

1236 if ( dominates (*OffsetDef, LdSt)) {

1237

1238

1239 if (OffsetDef->getOpcode() != TargetOpcode::G_CONSTANT)

1240 continue;

1241 RematOffset = true;

1242 }

1243

1244 for (auto &BasePtrUse : MRI.use_nodbg_instructions(PtrAdd->getBaseReg())) {

1245 if (&BasePtrUse == PtrDef)

1246 continue;

1247

1248

1249

1251 if (BasePtrLdSt && BasePtrLdSt != &LdSt &&

1252 dominates(LdSt, *BasePtrLdSt) &&

1253 isIndexedLoadStoreLegal(*BasePtrLdSt))

1254 return false;

1255

1256

1257

1259 Register PtrAddDefReg = BasePtrUseDef->getReg(0);

1260 for (auto &BaseUseUse : MRI.use_nodbg_instructions(PtrAddDefReg)) {

1261

1262

1263 if (BaseUseUse.getParent() != LdSt.getParent())

1264 return false;

1265

1268 return false;

1269 }

1270 if ( dominates (LdSt, BasePtrUse))

1271 return false;

1272 }

1273 }

1274

1275 Addr = PtrAdd->getReg(0);

1276 Base = PtrAdd->getBaseReg();

1277 return true;

1278 }

1279

1280 return false;

1281}

1282

1283bool CombinerHelper::findPreIndexCandidate(GLoadStore &LdSt, Register &Addr,

1288

1291 MRI.hasOneNonDBGUse(Addr))

1292 return false;

1293

1295 !TLI.isIndexingLegal(LdSt, Base, Offset, true, MRI))

1296 return false;

1297

1298 if (!isIndexedLoadStoreLegal(LdSt))

1299 return false;

1300

1302 if (BaseDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)

1303 return false;

1304

1306

1307 if (Base == St->getValueReg())

1308 return false;

1309

1310

1311

1312 if (St->getValueReg() == Addr)

1313 return false;

1314 }

1315

1316

1317 for (auto &AddrUse : MRI.use_nodbg_instructions(Addr))

1318 if (AddrUse.getParent() != LdSt.getParent())

1319 return false;

1320

1321

1322

1323 bool RealUse = false;

1324 for (auto &AddrUse : MRI.use_nodbg_instructions(Addr)) {

1326 return false;

1327

1328

1329

1332 RealUse = true;

1333 } else {

1334 RealUse = true;

1335 }

1336 }

1337 return RealUse;

1338}

1339

1342 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);

1343

1344

1346 if (!LoadMI)

1347 return false;

1348

1350 LLT VecEltTy = MRI.getType(Vector).getElementType();

1351

1352 assert(MRI.getType(MI.getOperand(0).getReg()) == VecEltTy);

1353

1354

1355 if ( MRI .hasOneNonDBGUse(Vector))

1356 return false;

1357

1358

1359 if (!LoadMI->isSimple())

1360 return false;

1361

1362

1363

1364

1366 return false;

1367

1368

1369 if (MI.getParent() != LoadMI->getParent())

1370 return false;

1371 const unsigned MaxIter = 20;

1372 unsigned Iter = 0;

1373 for (auto II = LoadMI->getIterator(), IE = MI.getIterator(); II != IE; ++II) {

1374 if (II->isLoadFoldBarrier())

1375 return false;

1376 if (Iter++ == MaxIter)

1377 return false;

1378 }

1379

1380

1381

1386

1387

1388

1389

1390

1392 int Elt = CVal->getZExtValue();

1393

1396 } else {

1397

1398

1401 }

1402

1404

1405 Register VecPtr = LoadMI->getPointerReg();

1406 LLT PtrTy = MRI.getType(VecPtr);

1407

1410

1412

1414 {TargetOpcode::G_LOAD, {VecEltTy, PtrTy}, {MMDesc}}))

1415 return false;

1416

1417

1420 unsigned Fast = 0;

1424 return false;

1425

1426 Register Result = MI.getOperand(0).getReg();

1427 Register Index = MI.getOperand(2).getReg();

1428

1432

1435 Index);

1436

1437 B.buildLoad(Result, finalPtr, PtrInfo, Alignment);

1438

1440 };

1441

1442 return true;

1443}

1444

1448

1450 return false;

1451

1452 MatchInfo.IsPre = findPreIndexCandidate(LdSt, MatchInfo.Addr, MatchInfo.Base,

1454 if (!MatchInfo.IsPre &&

1455 !findPostIndexCandidate(LdSt, MatchInfo.Addr, MatchInfo.Base,

1457 return false;

1458

1459 return true;

1460}

1461

1465 unsigned Opcode = MI.getOpcode();

1466 bool IsStore = Opcode == TargetOpcode::G_STORE;

1468

1469

1470

1472 auto *OldCst = MRI.getVRegDef(MatchInfo.Offset);

1473 auto NewCst = Builder.buildConstant(MRI.getType(MatchInfo.Offset),

1474 *OldCst->getOperand(1).getCImm());

1475 MatchInfo.Offset = NewCst.getReg(0);

1476 }

1477

1478 auto MIB = Builder.buildInstr(NewOpcode);

1479 if (IsStore) {

1480 MIB.addDef(MatchInfo.Addr);

1481 MIB.addUse(MI.getOperand(0).getReg());

1482 } else {

1483 MIB.addDef(MI.getOperand(0).getReg());

1484 MIB.addDef(MatchInfo.Addr);

1485 }

1486

1487 MIB.addUse(MatchInfo.Base);

1488 MIB.addUse(MatchInfo.Offset);

1489 MIB.addImm(MatchInfo.IsPre);

1490 MIB->cloneMemRefs(*MI.getMF(), MI);

1491 MI.eraseFromParent();

1493

1494 LLVM_DEBUG(dbgs() << " Combinined to indexed operation");

1495}

1496

1499 unsigned Opcode = MI.getOpcode();

1500 bool IsDiv, IsSigned;

1501

1502 switch (Opcode) {

1503 default:

1505 case TargetOpcode::G_SDIV:

1506 case TargetOpcode::G_UDIV: {

1507 IsDiv = true;

1508 IsSigned = Opcode == TargetOpcode::G_SDIV;

1509 break;

1510 }

1511 case TargetOpcode::G_SREM:

1512 case TargetOpcode::G_UREM: {

1513 IsDiv = false;

1514 IsSigned = Opcode == TargetOpcode::G_SREM;

1515 break;

1516 }

1517 }

1518

1519 Register Src1 = MI.getOperand(1).getReg();

1520 unsigned DivOpcode, RemOpcode, DivremOpcode;

1521 if (IsSigned) {

1522 DivOpcode = TargetOpcode::G_SDIV;

1523 RemOpcode = TargetOpcode::G_SREM;

1524 DivremOpcode = TargetOpcode::G_SDIVREM;

1525 } else {

1526 DivOpcode = TargetOpcode::G_UDIV;

1527 RemOpcode = TargetOpcode::G_UREM;

1528 DivremOpcode = TargetOpcode::G_UDIVREM;

1529 }

1530

1532 return false;

1533

1534

1535

1536

1537

1538

1539

1540

1541

1542

1543

1544

1545

1546 for (auto &UseMI : MRI.use_nodbg_instructions(Src1)) {

1547 if (MI.getParent() == UseMI.getParent() &&

1548 ((IsDiv && UseMI.getOpcode() == RemOpcode) ||

1549 (!IsDiv && UseMI.getOpcode() == DivOpcode)) &&

1552 OtherMI = &UseMI;

1553 return true;

1554 }

1555 }

1556

1557 return false;

1558}

1559

1562 unsigned Opcode = MI.getOpcode();

1563 assert(OtherMI && "OtherMI shouldn't be empty.");

1564

1565 Register DestDivReg, DestRemReg;

1566 if (Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_UDIV) {

1567 DestDivReg = MI.getOperand(0).getReg();

1569 } else {

1571 DestRemReg = MI.getOperand(0).getReg();

1572 }

1573

1574 bool IsSigned =

1575 Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM;

1576

1577

1578

1579

1580

1582 Builder.setInstrAndDebugLoc(*FirstInst);

1583

1584 Builder.buildInstr(IsSigned ? TargetOpcode::G_SDIVREM

1585 : TargetOpcode::G_UDIVREM,

1586 {DestDivReg, DestRemReg},

1588 MI.eraseFromParent();

1590}

1591

1594 assert(MI.getOpcode() == TargetOpcode::G_BR);

1595

1596

1597

1598

1599

1600

1601

1602

1603

1604

1605

1606

1607

1608

1611 if (BrIt == MBB->begin())

1612 return false;

1613 assert(std::next(BrIt) == MBB->end() && "expected G_BR to be a terminator");

1614

1615 BrCond = &*std::prev(BrIt);

1616 if (BrCond->getOpcode() != TargetOpcode::G_BRCOND)

1617 return false;

1618

1619

1620

1622 return BrCondTarget != MI.getOperand(0).getMBB() &&

1623 MBB->isLayoutSuccessor(BrCondTarget);

1624}

1625

1629 Builder.setInstrAndDebugLoc(*BrCond);

1631

1632

1633

1634 auto True = Builder.buildConstant(

1637

1640 MI.getOperand(0).setMBB(FallthroughBB);

1642

1643

1644

1645 Observer.changingInstr(*BrCond);

1648 Observer.changedInstr(*BrCond);

1649}

1650

1654 LegalizerHelper Helper(HelperBuilder.getMF(), DummyObserver, HelperBuilder);

1655 return Helper.lowerMemcpyInline(MI) ==

1657}

1658

1660 unsigned MaxLen) const {

1663 LegalizerHelper Helper(HelperBuilder.getMF(), DummyObserver, HelperBuilder);

1666}

1667

1672 switch (MI.getOpcode()) {

1673 default:

1675 case TargetOpcode::G_FNEG: {

1676 Result.changeSign();

1677 return Result;

1678 }

1679 case TargetOpcode::G_FABS: {

1680 Result.clearSign();

1681 return Result;

1682 }

1683 case TargetOpcode::G_FPEXT:

1684 case TargetOpcode::G_FPTRUNC: {

1685 bool Unused;

1686 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());

1688 &Unused);

1689 return Result;

1690 }

1691 case TargetOpcode::G_FSQRT: {

1692 bool Unused;

1694 &Unused);

1695 Result = APFloat(sqrt(Result.convertToDouble()));

1696 break;

1697 }

1698 case TargetOpcode::G_FLOG2: {

1699 bool Unused;

1701 &Unused);

1702 Result = APFloat(log2(Result.convertToDouble()));

1703 break;

1704 }

1705 }

1706

1707

1708

1709 bool Unused;

1711 return Result;

1712}

1713

1717 const ConstantFP *NewCst = ConstantFP::get(Builder.getContext(), Folded);

1718 Builder.buildFConstant(MI.getOperand(0), *NewCst);

1719 MI.eraseFromParent();

1720}

1721

1724

1725

1726

1727

1728

1729

1730 if (MI.getOpcode() != TargetOpcode::G_PTR_ADD)

1731 return false;

1732

1733 Register Add2 = MI.getOperand(1).getReg();

1734 Register Imm1 = MI.getOperand(2).getReg();

1736 if (!MaybeImmVal)

1737 return false;

1738

1740 if (!Add2Def || Add2Def->getOpcode() != TargetOpcode::G_PTR_ADD)

1741 return false;

1742

1746 if (!MaybeImm2Val)

1747 return false;

1748

1749

1750

1751

1752

1753 Type *AccessTy = nullptr;

1754 auto &MF = *MI.getMF();

1755 for (auto &UseMI : MRI.use_nodbg_instructions(MI.getOperand(0).getReg())) {

1758 MF.getFunction().getContext());

1759 break;

1760 }

1761 }

1763 APInt CombinedImm = MaybeImmVal->Value + MaybeImm2Val->Value;

1765 if (AccessTy) {

1768 AMOld.BaseOffs = MaybeImmVal->Value.getSExtValue();

1770 unsigned AS = MRI.getType(Add2).getAddressSpace();

1771 const auto &TLI = *MF.getSubtarget().getTargetLowering();

1772 if (TLI.isLegalAddressingMode(MF.getDataLayout(), AMOld, AccessTy, AS) &&

1773 !TLI.isLegalAddressingMode(MF.getDataLayout(), AMNew, AccessTy, AS))

1774 return false;

1775 }

1776

1777

1778

1779

1780

1781

1782 unsigned PtrAddFlags = MI.getFlags();

1783 unsigned LHSPtrAddFlags = Add2Def->getFlags();

1785 bool IsInBounds =

1787 unsigned Flags = 0;

1788 if (IsNoUWrap)

1790 if (IsInBounds) {

1793 }

1794

1795

1799 MatchInfo.Flags = Flags;

1800 return true;

1801}

1802

1805 assert(MI.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD");

1807 LLT OffsetTy = MRI.getType(MI.getOperand(2).getReg());

1808 auto NewOffset = MIB.buildConstant(OffsetTy, MatchInfo.Imm);

1811 MI.getOperand(1).setReg(MatchInfo.Base);

1812 MI.getOperand(2).setReg(NewOffset.getReg(0));

1813 MI.setFlags(MatchInfo.Flags);

1815}

1816

1819

1820

1821

1822

1823

1824

1825

1826 unsigned Opcode = MI.getOpcode();

1827 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||

1828 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||

1829 Opcode == TargetOpcode::G_USHLSAT) &&

1830 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");

1831

1832 Register Shl2 = MI.getOperand(1).getReg();

1833 Register Imm1 = MI.getOperand(2).getReg();

1835 if (!MaybeImmVal)

1836 return false;

1837

1839 if (Shl2Def->getOpcode() != Opcode)

1840 return false;

1841

1845 if (!MaybeImm2Val)

1846 return false;

1847

1848

1849 MatchInfo.Imm =

1850 (MaybeImmVal->Value.getZExtValue() + MaybeImm2Val->Value).getZExtValue();

1852

1853

1854

1855 if (Opcode == TargetOpcode::G_USHLSAT &&

1856 MatchInfo.Imm >= MRI.getType(Shl2).getScalarSizeInBits())

1857 return false;

1858

1859 return true;

1860}

1861

1864 unsigned Opcode = MI.getOpcode();

1865 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||

1866 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||

1867 Opcode == TargetOpcode::G_USHLSAT) &&

1868 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");

1869

1870 LLT Ty = MRI.getType(MI.getOperand(1).getReg());

1871 unsigned const ScalarSizeInBits = Ty.getScalarSizeInBits();

1872 auto Imm = MatchInfo.Imm;

1873

1874 if (Imm >= ScalarSizeInBits) {

1875

1876 if (Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR) {

1877 Builder.buildConstant(MI.getOperand(0), 0);

1878 MI.eraseFromParent();

1879 return;

1880 }

1881

1882

1883 Imm = ScalarSizeInBits - 1;

1884 }

1885

1886 LLT ImmTy = MRI.getType(MI.getOperand(2).getReg());

1887 Register NewImm = Builder.buildConstant(ImmTy, Imm).getReg(0);

1889 MI.getOperand(1).setReg(MatchInfo.Reg);

1890 MI.getOperand(2).setReg(NewImm);

1892}

1893

1896

1897

1898

1899

1900

1901

1902

1903

1904

1905

1906 unsigned ShiftOpcode = MI.getOpcode();

1907 assert((ShiftOpcode == TargetOpcode::G_SHL ||

1908 ShiftOpcode == TargetOpcode::G_ASHR ||

1909 ShiftOpcode == TargetOpcode::G_LSHR ||

1910 ShiftOpcode == TargetOpcode::G_USHLSAT ||

1911 ShiftOpcode == TargetOpcode::G_SSHLSAT) &&

1912 "Expected G_SHL, G_ASHR, G_LSHR, G_USHLSAT and G_SSHLSAT");

1913

1914

1915 Register LogicDest = MI.getOperand(1).getReg();

1916 if ( MRI .hasOneNonDBGUse(LogicDest))

1917 return false;

1918

1919 MachineInstr *LogicMI = MRI.getUniqueVRegDef(LogicDest);

1920 unsigned LogicOpcode = LogicMI->getOpcode();

1921 if (LogicOpcode != TargetOpcode::G_AND && LogicOpcode != TargetOpcode::G_OR &&

1922 LogicOpcode != TargetOpcode::G_XOR)

1923 return false;

1924

1925

1926 const Register C1 = MI.getOperand(2).getReg();

1928 if (!MaybeImmVal || MaybeImmVal->Value == 0)

1929 return false;

1930

1931 const uint64_t C1Val = MaybeImmVal->Value.getZExtValue();

1932

1934

1935 if (MI->getOpcode() != ShiftOpcode ||

1936 MRI .hasOneNonDBGUse(MI->getOperand(0).getReg()))

1937 return false;

1938

1939

1940 auto MaybeImmVal =

1942 if (!MaybeImmVal)

1943 return false;

1944

1945 ShiftVal = MaybeImmVal->Value.getSExtValue();

1946 return true;

1947 };

1948

1949

1951 MachineInstr *LogicMIOp1 = MRI.getUniqueVRegDef(LogicMIReg1);

1953 MachineInstr *LogicMIOp2 = MRI.getUniqueVRegDef(LogicMIReg2);

1955

1956 if (matchFirstShift(LogicMIOp1, C0Val)) {

1958 MatchInfo.Shift2 = LogicMIOp1;

1959 } else if (matchFirstShift(LogicMIOp2, C0Val)) {

1961 MatchInfo.Shift2 = LogicMIOp2;

1962 } else

1963 return false;

1964

1965 MatchInfo.ValSum = C0Val + C1Val;

1966

1967

1968 if (MatchInfo.ValSum >= MRI.getType(LogicDest).getScalarSizeInBits())

1969 return false;

1970

1971 MatchInfo.Logic = LogicMI;

1972 return true;

1973}

1974

1977 unsigned Opcode = MI.getOpcode();

1978 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||

1979 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_USHLSAT ||

1980 Opcode == TargetOpcode::G_SSHLSAT) &&

1981 "Expected G_SHL, G_ASHR, G_LSHR, G_USHLSAT and G_SSHLSAT");

1982

1983 LLT ShlType = MRI.getType(MI.getOperand(2).getReg());

1984 LLT DestType = MRI.getType(MI.getOperand(0).getReg());

1985

1987

1990 Builder.buildInstr(Opcode, {DestType}, {Shift1Base, Const}).getReg(0);

1991

1992

1993

1994

1995

1996

1998

1999 Register Shift2Const = MI.getOperand(2).getReg();

2001 .buildInstr(Opcode, {DestType},

2004

2005 Register Dest = MI.getOperand(0).getReg();

2007

2008

2010

2011 MI.eraseFromParent();

2012}

2013

2016 assert(MI.getOpcode() == TargetOpcode::G_SHL && "Expected G_SHL");

2017

2018

2020 Register DstReg = Shl.getReg(0);

2021 Register SrcReg = Shl.getReg(1);

2022 Register ShiftReg = Shl.getReg(2);

2024

2026 return false;

2027

2031 return false;

2032

2033 APInt C1Val, C2Val;

2036 return false;

2037

2038 auto *SrcDef = MRI.getVRegDef(SrcReg);

2039 assert((SrcDef->getOpcode() == TargetOpcode::G_ADD ||

2040 SrcDef->getOpcode() == TargetOpcode::G_OR) && "Unexpected op");

2041 LLT SrcTy = MRI.getType(SrcReg);

2043 auto S1 = B.buildShl(SrcTy, X, ShiftReg);

2044 auto S2 = B.buildShl(SrcTy, C1, ShiftReg);

2045 B.buildInstr(SrcDef->getOpcode(), {DstReg}, {S1, S2});

2046 };

2047 return true;

2048}

2049

2053 assert(MI.getOpcode() == TargetOpcode::G_LSHR && "Expected a G_LSHR");

2054

2055 Register N0 = MI.getOperand(1).getReg();

2056 Register N1 = MI.getOperand(2).getReg();

2057 unsigned OpSizeInBits = MRI.getType(N0).getScalarSizeInBits();

2058

2059 APInt N1C, N001C;

2061 return false;

2064 return false;

2065

2068 else

2070

2072 LLT InnerShiftTy = MRI.getType(InnerShift);

2074 if ((N1C + N001C).ult(InnerShiftSize)) {

2076 MatchInfo.ShiftAmt = N1C + N001C;

2079

2080 if ((N001C + OpSizeInBits) == InnerShiftSize)

2081 return true;

2082 if (MRI.hasOneUse(N0) && MRI.hasOneUse(InnerShift)) {

2083 MatchInfo.Mask = true;

2085 return true;

2086 }

2087 }

2088 return false;

2089}

2090

2093 assert(MI.getOpcode() == TargetOpcode::G_LSHR && "Expected a G_LSHR");

2094

2095 Register Dst = MI.getOperand(0).getReg();

2096 auto ShiftAmt =

2098 auto Shift =

2100 if (MatchInfo.Mask == true) {

2107 } else

2108 Builder.buildTrunc(Dst, Shift);

2109 MI.eraseFromParent();

2110}

2111

2113 unsigned &ShiftVal) const {

2114 assert(MI.getOpcode() == TargetOpcode::G_MUL && "Expected a G_MUL");

2115 auto MaybeImmVal =

2117 if (!MaybeImmVal)

2118 return false;

2119

2120 ShiftVal = MaybeImmVal->Value.exactLogBase2();

2121 return (static_cast<int32_t>(ShiftVal) != -1);

2122}

2123

2125 unsigned &ShiftVal) const {

2126 assert(MI.getOpcode() == TargetOpcode::G_MUL && "Expected a G_MUL");

2128 LLT ShiftTy = MRI.getType(MI.getOperand(0).getReg());

2129 auto ShiftCst = MIB.buildConstant(ShiftTy, ShiftVal);

2131 MI.setDesc(MIB.getTII().get(TargetOpcode::G_SHL));

2132 MI.getOperand(2).setReg(ShiftCst.getReg(0));

2136}

2137

2141

2142 LLT Ty = MRI.getType(Sub.getReg(0));

2143

2145 return false;

2146

2148 return false;

2149

2151

2153 auto NegCst = B.buildConstant(Ty, -Imm);

2155 MI.setDesc(B.getTII().get(TargetOpcode::G_ADD));

2156 MI.getOperand(2).setReg(NegCst.getReg(0));

2158 if (Imm.isMinSignedValue())

2161 };

2162 return true;

2163}

2164

2165

2168 assert(MI.getOpcode() == TargetOpcode::G_SHL && VT);

2170 return false;

2171

2172 Register LHS = MI.getOperand(1).getReg();

2173

2178 return false;

2179

2180 Register RHS = MI.getOperand(2).getReg();

2183 if (!MaybeShiftAmtVal)

2184 return false;

2185

2186 if (LI) {

2187 LLT SrcTy = MRI.getType(ExtSrc);

2188

2189

2190

2191

2194 return false;

2195 }

2196

2197 int64_t ShiftAmt = MaybeShiftAmtVal->getSExtValue();

2198 MatchData.Reg = ExtSrc;

2199 MatchData.Imm = ShiftAmt;

2200

2201 unsigned MinLeadingZeros = VT->getKnownZeroes(ExtSrc).countl_one();

2202 unsigned SrcTySize = MRI.getType(ExtSrc).getScalarSizeInBits();

2203 return MinLeadingZeros >= ShiftAmt && ShiftAmt < SrcTySize;

2204}

2205

2209 int64_t ShiftAmtVal = MatchData.Imm;

2210

2211 LLT ExtSrcTy = MRI.getType(ExtSrcReg);

2212 auto ShiftAmt = Builder.buildConstant(ExtSrcTy, ShiftAmtVal);

2213 auto NarrowShift =

2214 Builder.buildShl(ExtSrcTy, ExtSrcReg, ShiftAmt, MI.getFlags());

2215 Builder.buildZExt(MI.getOperand(0), NarrowShift);

2216 MI.eraseFromParent();

2217}

2218

2220 Register &MatchInfo) const {

2223 for (unsigned I = 0; I < Merge.getNumSources(); ++I)

2225

2227 if (!Unmerge || Unmerge->getNumDefs() != Merge.getNumSources())

2228 return false;

2229

2230 for (unsigned I = 0; I < MergedValues.size(); ++I)

2231 if (MergedValues[I] != Unmerge->getReg(I))

2232 return false;

2233

2234 MatchInfo = Unmerge->getSourceReg();

2235 return true;

2236}

2237

2241 ;

2242

2243 return Reg;

2244}

2245

2248 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&

2249 "Expected an unmerge");

2252

2254 if (!SrcInstr)

2255 return false;

2256

2257

2258 LLT SrcMergeTy = MRI.getType(SrcInstr->getSourceReg(0));

2259 LLT Dst0Ty = MRI.getType(Unmerge.getReg(0));

2261 if (SrcMergeTy != Dst0Ty && !SameSize)

2262 return false;

2263

2264

2265 for (unsigned Idx = 0; Idx < SrcInstr->getNumSources(); ++Idx)

2266 Operands.push_back(SrcInstr->getSourceReg(Idx));

2267 return true;

2268}

2269

2272 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&

2273 "Expected an unmerge");

2274 assert((MI.getNumOperands() - 1 == Operands.size()) &&

2275 "Not enough operands to replace all defs");

2276 unsigned NumElems = MI.getNumOperands() - 1;

2277

2278 LLT SrcTy = MRI.getType(Operands[0]);

2279 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());

2280 bool CanReuseInputDirectly = DstTy == SrcTy;

2281 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {

2282 Register DstReg = MI.getOperand(Idx).getReg();

2283 Register SrcReg = Operands[Idx];

2284

2285

2286

2287 const auto &DstCB = MRI.getRegClassOrRegBank(DstReg);

2288 if (!DstCB.isNull() && DstCB != MRI.getRegClassOrRegBank(SrcReg)) {

2289 SrcReg = Builder.buildCopy(MRI.getType(SrcReg), SrcReg).getReg(0);

2290 MRI.setRegClassOrRegBank(SrcReg, DstCB);

2291 }

2292

2293 if (CanReuseInputDirectly)

2295 else

2296 Builder.buildCast(DstReg, SrcReg);

2297 }

2298 MI.eraseFromParent();

2299}

2300

2303 unsigned SrcIdx = MI.getNumOperands() - 1;

2304 Register SrcReg = MI.getOperand(SrcIdx).getReg();

2306 if (SrcInstr->getOpcode() != TargetOpcode::G_CONSTANT &&

2307 SrcInstr->getOpcode() != TargetOpcode::G_FCONSTANT)

2308 return false;

2309

2311 APInt Val = SrcInstr->getOpcode() == TargetOpcode::G_CONSTANT

2314

2315 LLT Dst0Ty = MRI.getType(MI.getOperand(0).getReg());

2317

2318 for (unsigned Idx = 0; Idx != SrcIdx; ++Idx) {

2320 Val = Val.lshr(ShiftAmt);

2321 }

2322

2323 return true;

2324}

2325

2328 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&

2329 "Expected an unmerge");

2330 assert((MI.getNumOperands() - 1 == Csts.size()) &&

2331 "Not enough operands to replace all defs");

2332 unsigned NumElems = MI.getNumOperands() - 1;

2333 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {

2334 Register DstReg = MI.getOperand(Idx).getReg();

2335 Builder.buildConstant(DstReg, Csts[Idx]);

2336 }

2337

2338 MI.eraseFromParent();

2339}

2340

2344 unsigned SrcIdx = MI.getNumOperands() - 1;

2345 Register SrcReg = MI.getOperand(SrcIdx).getReg();

2347 unsigned NumElems = MI.getNumOperands() - 1;

2348 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {

2349 Register DstReg = MI.getOperand(Idx).getReg();

2350 B.buildUndef(DstReg);

2351 }

2352 };

2354}

2355

2358 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&

2359 "Expected an unmerge");

2360 if (MRI.getType(MI.getOperand(0).getReg()).isVector() ||

2361 MRI.getType(MI.getOperand(MI.getNumDefs()).getReg()).isVector())

2362 return false;

2363

2364 for (unsigned Idx = 1, EndIdx = MI.getNumDefs(); Idx != EndIdx; ++Idx) {

2365 if ( MRI .use_nodbg_empty(MI.getOperand(Idx).getReg()))

2366 return false;

2367 }

2368 return true;

2369}

2370

2373 Register SrcReg = MI.getOperand(MI.getNumDefs()).getReg();

2374 Register Dst0Reg = MI.getOperand(0).getReg();

2375 Builder.buildTrunc(Dst0Reg, SrcReg);

2376 MI.eraseFromParent();

2377}

2378

2380 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&

2381 "Expected an unmerge");

2382 Register Dst0Reg = MI.getOperand(0).getReg();

2383 LLT Dst0Ty = MRI.getType(Dst0Reg);

2384

2385

2386

2388 return false;

2389 Register SrcReg = MI.getOperand(MI.getNumDefs()).getReg();

2390 LLT SrcTy = MRI.getType(SrcReg);

2391 if (SrcTy.isVector())

2392 return false;

2393

2396 return false;

2397

2398

2399

2400

2401 LLT ZExtSrcTy = MRI.getType(ZExtSrcReg);

2403}

2404

2406 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&

2407 "Expected an unmerge");

2408

2409 Register Dst0Reg = MI.getOperand(0).getReg();

2410

2412 MRI.getVRegDef(MI.getOperand(MI.getNumDefs()).getReg());

2413 assert(ZExtInstr && ZExtInstr->getOpcode() == TargetOpcode::G_ZEXT &&

2414 "Expecting a G_ZEXT");

2415

2417 LLT Dst0Ty = MRI.getType(Dst0Reg);

2418 LLT ZExtSrcTy = MRI.getType(ZExtSrcReg);

2419

2421 Builder.buildZExt(Dst0Reg, ZExtSrcReg);

2422 } else {

2424 "ZExt src doesn't fit in destination");

2426 }

2427

2429 for (unsigned Idx = 1, EndIdx = MI.getNumDefs(); Idx != EndIdx; ++Idx) {

2430 if (!ZeroReg)

2431 ZeroReg = Builder.buildConstant(Dst0Ty, 0).getReg(0);

2433 }

2434 MI.eraseFromParent();

2435}

2436

2438 unsigned TargetShiftSize,

2439 unsigned &ShiftVal) const {

2440 assert((MI.getOpcode() == TargetOpcode::G_SHL ||

2441 MI.getOpcode() == TargetOpcode::G_LSHR ||

2442 MI.getOpcode() == TargetOpcode::G_ASHR) && "Expected a shift");

2443

2444 LLT Ty = MRI.getType(MI.getOperand(0).getReg());

2445 if (Ty.isVector())

2446 return false;

2447

2448

2449 unsigned Size = Ty.getSizeInBits();

2450 if (Size <= TargetShiftSize)

2451 return false;

2452

2453 auto MaybeImmVal =

2455 if (!MaybeImmVal)

2456 return false;

2457

2458 ShiftVal = MaybeImmVal->Value.getSExtValue();

2459 return ShiftVal >= Size / 2 && ShiftVal < Size;

2460}

2461

2464 Register DstReg = MI.getOperand(0).getReg();

2465 Register SrcReg = MI.getOperand(1).getReg();

2466 LLT Ty = MRI.getType(SrcReg);

2467 unsigned Size = Ty.getSizeInBits();

2468 unsigned HalfSize = Size / 2;

2469 assert(ShiftVal >= HalfSize);

2470

2472

2473 auto Unmerge = Builder.buildUnmerge(HalfTy, SrcReg);

2474 unsigned NarrowShiftAmt = ShiftVal - HalfSize;

2475

2476 if (MI.getOpcode() == TargetOpcode::G_LSHR) {

2477 Register Narrowed = Unmerge.getReg(1);

2478

2479

2480

2481

2482

2483

2484 if (NarrowShiftAmt != 0) {

2485 Narrowed = Builder.buildLShr(HalfTy, Narrowed,

2486 Builder.buildConstant(HalfTy, NarrowShiftAmt)).getReg(0);

2487 }

2488

2489 auto Zero = Builder.buildConstant(HalfTy, 0);

2490 Builder.buildMergeLikeInstr(DstReg, {Narrowed, Zero});

2491 } else if (MI.getOpcode() == TargetOpcode::G_SHL) {

2492 Register Narrowed = Unmerge.getReg(0);

2493

2494

2495

2496

2497 if (NarrowShiftAmt != 0) {

2498 Narrowed = Builder.buildShl(HalfTy, Narrowed,

2499 Builder.buildConstant(HalfTy, NarrowShiftAmt)).getReg(0);

2500 }

2501

2502 auto Zero = Builder.buildConstant(HalfTy, 0);

2503 Builder.buildMergeLikeInstr(DstReg, {Zero, Narrowed});

2504 } else {

2505 assert(MI.getOpcode() == TargetOpcode::G_ASHR);

2507 HalfTy, Unmerge.getReg(1),

2508 Builder.buildConstant(HalfTy, HalfSize - 1));

2509

2510 if (ShiftVal == HalfSize) {

2511

2512

2513 Builder.buildMergeLikeInstr(DstReg, {Unmerge.getReg(1), Hi});

2514 } else if (ShiftVal == Size - 1) {

2515

2516

2517

2518

2519 Builder.buildMergeLikeInstr(DstReg, {Hi, Hi});

2520 } else {

2522 HalfTy, Unmerge.getReg(1),

2523 Builder.buildConstant(HalfTy, ShiftVal - HalfSize));

2524

2525

2526

2527 Builder.buildMergeLikeInstr(DstReg, {Lo, Hi});

2528 }

2529 }

2530

2531 MI.eraseFromParent();

2532}

2533

2535 MachineInstr &MI, unsigned TargetShiftAmount) const {

2536 unsigned ShiftAmt;

2539 return true;

2540 }

2541

2542 return false;

2543}

2544

2547 assert(MI.getOpcode() == TargetOpcode::G_INTTOPTR && "Expected a G_INTTOPTR");

2548 Register DstReg = MI.getOperand(0).getReg();

2549 LLT DstTy = MRI.getType(DstReg);

2550 Register SrcReg = MI.getOperand(1).getReg();

2553}

2554

2557 assert(MI.getOpcode() == TargetOpcode::G_INTTOPTR && "Expected a G_INTTOPTR");

2558 Register DstReg = MI.getOperand(0).getReg();

2559 Builder.buildCopy(DstReg, Reg);

2560 MI.eraseFromParent();

2561}

2562

2565 assert(MI.getOpcode() == TargetOpcode::G_PTRTOINT && "Expected a G_PTRTOINT");

2566 Register DstReg = MI.getOperand(0).getReg();

2567 Builder.buildZExtOrTrunc(DstReg, Reg);

2568 MI.eraseFromParent();

2569}

2570

2572 MachineInstr &MI, std::pair<Register, bool> &PtrReg) const {

2573 assert(MI.getOpcode() == TargetOpcode::G_ADD);

2574 Register LHS = MI.getOperand(1).getReg();

2575 Register RHS = MI.getOperand(2).getReg();

2576 LLT IntTy = MRI.getType(LHS);

2577

2578

2579

2580 PtrReg.second = false;

2581 for (Register SrcReg : {LHS, RHS}) {

2583

2584

2585 LLT PtrTy = MRI.getType(PtrReg.first);

2587 return true;

2588 }

2589

2590 PtrReg.second = true;

2591 }

2592

2593 return false;

2594}

2595

2597 MachineInstr &MI, std::pair<Register, bool> &PtrReg) const {

2598 Register Dst = MI.getOperand(0).getReg();

2599 Register LHS = MI.getOperand(1).getReg();

2600 Register RHS = MI.getOperand(2).getReg();

2601

2602 const bool DoCommute = PtrReg.second;

2603 if (DoCommute)

2605 LHS = PtrReg.first;

2606

2607 LLT PtrTy = MRI.getType(LHS);

2608

2609 auto PtrAdd = Builder.buildPtrAdd(PtrTy, LHS, RHS);

2610 Builder.buildPtrToInt(Dst, PtrAdd);

2611 MI.eraseFromParent();

2612}

2613

2615 APInt &NewCst) const {

2617 Register LHS = PtrAdd.getBaseReg();

2618 Register RHS = PtrAdd.getOffsetReg();

2620

2624 auto DstTy = MRI.getType(PtrAdd.getReg(0));

2625

2626 NewCst = Cst.zextOrTrunc(DstTy.getSizeInBits());

2627 NewCst += RHSCst->sextOrTrunc(DstTy.getSizeInBits());

2628 return true;

2629 }

2630 }

2631

2632 return false;

2633}

2634

2636 APInt &NewCst) const {

2638 Register Dst = PtrAdd.getReg(0);

2639

2640 Builder.buildConstant(Dst, NewCst);

2641 PtrAdd.eraseFromParent();

2642}

2643

2646 assert(MI.getOpcode() == TargetOpcode::G_ANYEXT && "Expected a G_ANYEXT");

2647 Register DstReg = MI.getOperand(0).getReg();

2648 Register SrcReg = MI.getOperand(1).getReg();

2650 if (OriginalSrcReg.isValid())

2651 SrcReg = OriginalSrcReg;

2652 LLT DstTy = MRI.getType(DstReg);

2656}

2657

2660 assert(MI.getOpcode() == TargetOpcode::G_ZEXT && "Expected a G_ZEXT");

2661 Register DstReg = MI.getOperand(0).getReg();

2662 Register SrcReg = MI.getOperand(1).getReg();

2663 LLT DstTy = MRI.getType(DstReg);

2668 unsigned SrcSize = MRI.getType(SrcReg).getScalarSizeInBits();

2669 return VT->getKnownBits(Reg).countMinLeadingZeros() >= DstSize - SrcSize;

2670 }

2671 return false;

2672}

2673

2677

2678

2679 if (ShiftSize > 32 && TruncSize < 32)

2681

2682

2683

2684

2685

2686

2687

2688 return ShiftTy;

2689}

2690

2692 MachineInstr &MI, std::pair<MachineInstr *, LLT> &MatchInfo) const {

2693 assert(MI.getOpcode() == TargetOpcode::G_TRUNC && "Expected a G_TRUNC");

2694 Register DstReg = MI.getOperand(0).getReg();

2695 Register SrcReg = MI.getOperand(1).getReg();

2696

2697 if ( MRI .hasOneNonDBGUse(SrcReg))

2698 return false;

2699

2700 LLT SrcTy = MRI.getType(SrcReg);

2701 LLT DstTy = MRI.getType(DstReg);

2702

2705

2706 LLT NewShiftTy;

2708 default:

2709 return false;

2710 case TargetOpcode::G_SHL: {

2711 NewShiftTy = DstTy;

2712

2713

2716 return false;

2717 break;

2718 }

2719 case TargetOpcode::G_LSHR:

2720 case TargetOpcode::G_ASHR: {

2721

2722

2723

2724

2725

2726 for (auto &User : MRI.use_instructions(DstReg))

2727 if (User.getOpcode() == TargetOpcode::G_STORE)

2728 return false;

2729

2731 if (NewShiftTy == SrcTy)

2732 return false;

2733

2734

2738 return false;

2739 break;

2740 }

2741 }

2742

2745 {NewShiftTy, TL.getPreferredShiftAmountTy(NewShiftTy)}}))

2746 return false;

2747

2748 MatchInfo = std::make_pair(SrcMI, NewShiftTy);

2749 return true;

2750}

2751

2753 MachineInstr &MI, std::pair<MachineInstr *, LLT> &MatchInfo) const {

2755 LLT NewShiftTy = MatchInfo.second;

2756

2757 Register Dst = MI.getOperand(0).getReg();

2758 LLT DstTy = MRI.getType(Dst);

2759

2762 ShiftSrc = Builder.buildTrunc(NewShiftTy, ShiftSrc).getReg(0);

2763

2766 .buildInstr(ShiftMI->getOpcode(), {NewShiftTy}, {ShiftSrc, ShiftAmt})

2767 .getReg(0);

2768

2769 if (NewShiftTy == DstTy)

2771 else

2772 Builder.buildTrunc(Dst, NewShift);

2773

2775}

2776

2779 return MO.isReg() &&

2780 getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MO.getReg(), MRI);

2781 });

2782}

2783

2786 return !MO.isReg() ||

2787 getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MO.getReg(), MRI);

2788 });

2789}

2790

2792 assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);

2794 return all_of(Mask, [](int Elt) { return Elt < 0; });

2795}

2796

2798 assert(MI.getOpcode() == TargetOpcode::G_STORE);

2799 return getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MI.getOperand(0).getReg(),

2801}

2802

2804 assert(MI.getOpcode() == TargetOpcode::G_SELECT);

2805 return getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MI.getOperand(1).getReg(),

2807}

2808

2811 assert((MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT ||

2812 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT) &&

2813 "Expected an insert/extract element op");

2814 LLT VecTy = MRI.getType(MI.getOperand(1).getReg());

2816 return false;

2817

2818 unsigned IdxIdx =

2819 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;

2821 if (!Idx)

2822 return false;

2823 return Idx->getZExtValue() >= VecTy.getNumElements();

2824}

2825

2827 unsigned &OpIdx) const {

2829 auto Cst =

2831 if (!Cst)

2832 return false;

2833 OpIdx = Cst->isZero() ? 3 : 2;

2834 return true;

2835}

2836

2838

2842 return false;

2844 if (!InstAndDef1)

2845 return false;

2847 if (!InstAndDef2)

2848 return false;

2851

2852

2853

2854

2855

2856

2857

2858 if (I1 == I2)

2860

2861

2862

2863

2864

2865

2866

2867

2868

2869

2870

2871

2872

2873

2874

2875

2876

2877

2878 if (I1->mayLoadOrStore() && !I1->isDereferenceableInvariantLoad())

2879 return false;

2880

2881

2882

2886 if (!LS1 || !LS2)

2887 return false;

2888

2891 return false;

2892 }

2893

2894

2895

2896

2897

2898

2899

2900

2901

2902

2903

2905 return MO.isReg() && MO.getReg().isPhysical();

2906 })) {

2907

2908

2909

2910

2911

2912

2913

2914

2915 return I1->isIdenticalTo(*I2);

2916 }

2917

2918

2919

2920

2921

2922

2923 if (Builder.getTII().produceSameValue(*I1, *I2, &MRI)) {

2924

2925

2926

2927

2928

2929

2930 return I1->findRegisterDefOperandIdx(InstAndDef1->Reg, nullptr) ==

2932 }

2933 return false;

2934}

2935

2937 int64_t C) const {

2938 if (!MOP.isReg())

2939 return false;

2940 auto *MI = MRI.getVRegDef(MOP.getReg());

2942 return MaybeCst && MaybeCst->getBitWidth() <= 64 &&

2943 MaybeCst->getSExtValue() == C;

2944}

2945

2947 double C) const {

2948 if (!MOP.isReg())

2949 return false;

2950 std::optional MaybeCst;

2952 return false;

2953

2954 return MaybeCst->Value.isExactlyValue(C);

2955}

2956

2958 unsigned OpIdx) const {

2959 assert(MI.getNumExplicitDefs() == 1 && "Expected one explicit def?");

2960 Register OldReg = MI.getOperand(0).getReg();

2964 MI.eraseFromParent();

2965}

2966

2968 Register Replacement) const {

2969 assert(MI.getNumExplicitDefs() == 1 && "Expected one explicit def?");

2970 Register OldReg = MI.getOperand(0).getReg();

2973 MI.eraseFromParent();

2974}

2975

2977 unsigned ConstIdx) const {

2978 Register ConstReg = MI.getOperand(ConstIdx).getReg();

2979 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());

2980

2981

2983 if (!VRegAndVal)

2984 return false;

2985

2986

2987 return (VRegAndVal->Value.uge(DstTy.getSizeInBits()));

2988}

2989

2991 assert((MI.getOpcode() == TargetOpcode::G_FSHL ||

2992 MI.getOpcode() == TargetOpcode::G_FSHR) &&

2993 "This is not a funnel shift operation");

2994

2995 Register ConstReg = MI.getOperand(3).getReg();

2996 LLT ConstTy = MRI.getType(ConstReg);

2997 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());

2998

3000 assert((VRegAndVal) && "Value is not a constant");

3001

3002

3003 APInt NewConst = VRegAndVal->Value.urem(

3005

3006 auto NewConstInstr = Builder.buildConstant(ConstTy, NewConst.getZExtValue());

3008 MI.getOpcode(), {MI.getOperand(0)},

3009 {MI.getOperand(1), MI.getOperand(2), NewConstInstr.getReg(0)});

3010

3011 MI.eraseFromParent();

3012}

3013

3015 assert(MI.getOpcode() == TargetOpcode::G_SELECT);

3016

3018 canReplaceReg(MI.getOperand(0).getReg(), MI.getOperand(2).getReg(),

3020}

3021

3027

3029 unsigned OpIdx) const {

3033}

3034

3036 unsigned OpIdx) const {

3038 return MO.isReg() &&

3040}

3041

3043 unsigned OpIdx) const {

3046}

3047

3049 double C) const {

3050 assert(MI.getNumDefs() == 1 && "Expected only one def?");

3051 Builder.buildFConstant(MI.getOperand(0), C);

3052 MI.eraseFromParent();

3053}

3054

3056 int64_t C) const {

3057 assert(MI.getNumDefs() == 1 && "Expected only one def?");

3058 Builder.buildConstant(MI.getOperand(0), C);

3059 MI.eraseFromParent();

3060}

3061

3063 assert(MI.getNumDefs() == 1 && "Expected only one def?");

3064 Builder.buildConstant(MI.getOperand(0), C);

3065 MI.eraseFromParent();

3066}

3067

3070 assert(MI.getNumDefs() == 1 && "Expected only one def?");

3072 MI.eraseFromParent();

3073}

3074

3076 assert(MI.getNumDefs() == 1 && "Expected only one def?");

3077 Builder.buildUndef(MI.getOperand(0));

3078 MI.eraseFromParent();

3079}

3080

3082 MachineInstr &MI, std::tuple<Register, Register> &MatchInfo) const {

3083 Register LHS = MI.getOperand(1).getReg();

3084 Register RHS = MI.getOperand(2).getReg();

3085 Register &NewLHS = std::get<0>(MatchInfo);

3086 Register &NewRHS = std::get<1>(MatchInfo);

3087

3088

3089

3090

3091 auto CheckFold = [&](Register &MaybeSub, Register &MaybeNewLHS) {

3093 return false;

3094 NewLHS = MaybeNewLHS;

3095 return true;

3096 };

3097

3098 return CheckFold(LHS, RHS) || CheckFold(RHS, LHS);

3099}

3100

3103 assert(MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT &&

3104 "Invalid opcode");

3105 Register DstReg = MI.getOperand(0).getReg();

3106 LLT DstTy = MRI.getType(DstReg);

3107 assert(DstTy.isVector() && "Invalid G_INSERT_VECTOR_ELT?");

3108

3110 return false;

3111

3113

3114

3115 if (MRI.hasOneUse(DstReg) && MRI.use_instr_begin(DstReg)->getOpcode() ==

3116 TargetOpcode::G_INSERT_VECTOR_ELT)

3117 return false;

3120 int64_t IntImm;

3122 MatchInfo.resize(NumElts);

3126 if (IntImm >= NumElts || IntImm < 0)

3127 return false;

3128 if (!MatchInfo[IntImm])

3129 MatchInfo[IntImm] = TmpReg;

3130 CurrInst = TmpInst;

3131 }

3132

3133 if (CurrInst->getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)

3134 return false;

3135 if (TmpInst->getOpcode() == TargetOpcode::G_BUILD_VECTOR) {

3137 if (!MatchInfo[I - 1].isValid())

3139 }

3140 return true;

3141 }

3142

3143

3144 return TmpInst->getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||

3145 all_of(MatchInfo, [](Register Reg) { return !!Reg; });

3146}

3147

3151 auto GetUndef = [&]() {

3152 if (UndefReg)

3153 return UndefReg;

3154 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());

3156 return UndefReg;

3157 };

3158 for (Register &Reg : MatchInfo) {

3159 if (!Reg)

3160 Reg = GetUndef();

3161 }

3162 Builder.buildBuildVector(MI.getOperand(0).getReg(), MatchInfo);

3163 MI.eraseFromParent();

3164}

3165

3167 MachineInstr &MI, std::tuple<Register, Register> &MatchInfo) const {

3169 std::tie(SubLHS, SubRHS) = MatchInfo;

3170 Builder.buildSub(MI.getOperand(0).getReg(), SubLHS, SubRHS);

3171 MI.eraseFromParent();

3172}

3173

3176

3177

3178

3179

3180

3181

3182 unsigned LogicOpcode = MI.getOpcode();

3183 assert(LogicOpcode == TargetOpcode::G_AND ||

3184 LogicOpcode == TargetOpcode::G_OR ||

3185 LogicOpcode == TargetOpcode::G_XOR);

3187 Register Dst = MI.getOperand(0).getReg();

3188 Register LHSReg = MI.getOperand(1).getReg();

3189 Register RHSReg = MI.getOperand(2).getReg();

3190

3191

3192 if ( MRI .hasOneNonDBGUse(LHSReg) || MRI .hasOneNonDBGUse(RHSReg))

3193 return false;

3194

3195

3198 if (!LeftHandInst || !RightHandInst)

3199 return false;

3200 unsigned HandOpcode = LeftHandInst->getOpcode();

3201 if (HandOpcode != RightHandInst->getOpcode())

3202 return false;

3207 return false;

3208

3209

3210

3213 LLT XTy = MRI.getType(X);

3214 LLT YTy = MRI.getType(Y);

3215 if (!XTy.isValid() || XTy != YTy)

3216 return false;

3217

3218

3220 switch (HandOpcode) {

3221 default:

3222 return false;

3223 case TargetOpcode::G_ANYEXT:

3224 case TargetOpcode::G_SEXT:

3225 case TargetOpcode::G_ZEXT: {

3226

3227 break;

3228 }

3229 case TargetOpcode::G_TRUNC: {

3230

3233

3234 LLT DstTy = MRI.getType(Dst);

3236

3237

3238

3240 return false;

3241 break;

3242 }

3243 case TargetOpcode::G_AND:

3244 case TargetOpcode::G_ASHR:

3245 case TargetOpcode::G_LSHR:

3246 case TargetOpcode::G_SHL: {

3247

3250 return false;

3251 ExtraHandOpSrcReg = ZOp.getReg();

3252 break;

3253 }

3254 }

3255

3257 return false;

3258

3259

3260

3261

3262 auto NewLogicDst = MRI.createGenericVirtualRegister(XTy);

3268

3269

3273 if (ExtraHandOpSrcReg.isValid())

3277

3279 return true;

3280}

3281

3285 "Expected at least one instr to build?");

3286 for (auto &InstrToBuild : MatchInfo.InstrsToBuild) {

3287 assert(InstrToBuild.Opcode && "Expected a valid opcode?");

3288 assert(InstrToBuild.OperandFns.size() && "Expected at least one operand?");

3290 for (auto &OperandFn : InstrToBuild.OperandFns)

3291 OperandFn(Instr);

3292 }

3293 MI.eraseFromParent();

3294}

3295

3297 MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const {

3298 assert(MI.getOpcode() == TargetOpcode::G_ASHR);

3299 int64_t ShlCst, AshrCst;

3304 return false;

3305 if (ShlCst != AshrCst)

3306 return false;

3308 {TargetOpcode::G_SEXT_INREG, {MRI.getType(Src)}}))

3309 return false;

3310 MatchInfo = std::make_tuple(Src, ShlCst);

3311 return true;

3312}

3313

3315 MachineInstr &MI, std::tuple<Register, int64_t> &MatchInfo) const {

3316 assert(MI.getOpcode() == TargetOpcode::G_ASHR);

3318 int64_t ShiftAmt;

3319 std::tie(Src, ShiftAmt) = MatchInfo;

3320 unsigned Size = MRI.getType(Src).getScalarSizeInBits();

3321 Builder.buildSExtInReg(MI.getOperand(0).getReg(), Src, Size - ShiftAmt);

3322 MI.eraseFromParent();

3323}

3324

3325

3329 assert(MI.getOpcode() == TargetOpcode::G_AND);

3330

3331 Register Dst = MI.getOperand(0).getReg();

3332 LLT Ty = MRI.getType(Dst);

3333

3335 int64_t C1;

3336 int64_t C2;

3338 Dst, MRI,

3340 return false;

3341

3343 if (C1 & C2) {

3344 B.buildAnd(Dst, R, B.buildConstant(Ty, C1 & C2));

3345 return;

3346 }

3347 auto Zero = B.buildConstant(Ty, 0);

3349 };

3350 return true;

3351}

3352

3354 Register &Replacement) const {

3355

3356

3357

3358

3359

3360

3361

3362

3363

3364

3365

3366

3367

3368

3369

3370 assert(MI.getOpcode() == TargetOpcode::G_AND);

3371 if ()

3372 return false;

3373

3374 Register AndDst = MI.getOperand(0).getReg();

3375 Register LHS = MI.getOperand(1).getReg();

3376 Register RHS = MI.getOperand(2).getReg();

3377

3378

3379

3380

3381 KnownBits RHSBits = VT->getKnownBits(RHS);

3383 return false;

3384

3385 KnownBits LHSBits = VT->getKnownBits(LHS);

3386

3387

3388

3389

3390

3391

3392

3394 (LHSBits.Zero | RHSBits.One).isAllOnes()) {

3395 Replacement = LHS;

3396 return true;

3397 }

3398

3399

3401 (LHSBits.One | RHSBits.Zero).isAllOnes()) {

3402 Replacement = RHS;

3403 return true;

3404 }

3405

3406 return false;

3407}

3408

3410 Register &Replacement) const {

3411

3412

3413

3414

3415

3416

3417

3418 assert(MI.getOpcode() == TargetOpcode::G_OR);

3419 if ()

3420 return false;

3421

3422 Register OrDst = MI.getOperand(0).getReg();

3423 Register LHS = MI.getOperand(1).getReg();

3424 Register RHS = MI.getOperand(2).getReg();

3425

3426 KnownBits LHSBits = VT->getKnownBits(LHS);

3427 KnownBits RHSBits = VT->getKnownBits(RHS);

3428

3429

3430

3431

3432

3433

3434

3436 (LHSBits.One | RHSBits.Zero).isAllOnes()) {

3437 Replacement = LHS;

3438 return true;

3439 }

3440

3441

3443 (LHSBits.Zero | RHSBits.One).isAllOnes()) {

3444 Replacement = RHS;

3445 return true;

3446 }

3447

3448 return false;

3449}

3450

3452

3453 Register Src = MI.getOperand(1).getReg();

3454 unsigned ExtBits = MI.getOperand(2).getImm();

3455 unsigned TypeSize = MRI.getType(Src).getScalarSizeInBits();

3456 return VT->computeNumSignBits(Src) >= (TypeSize - ExtBits + 1);

3457}

3458

3460 int64_t Cst, bool IsVector, bool IsFP) {

3461

3462 return (ScalarSizeBits == 1 && Cst == -1) ||

3464}

3465

3466

3467

3468

3469

3470

3471

3472

3473

3474

3475

3476

3477

3478

3481 Register &UnmergeSrc) const {

3483

3484 unsigned BuildUseCount = BV.getNumSources();

3485 if (BuildUseCount % 2 != 0)

3486 return false;

3487

3488 unsigned NumUnmerge = BuildUseCount / 2;

3489

3491

3492

3493

3494 if (!Unmerge || Unmerge->getNumDefs() != NumUnmerge)

3495 return false;

3496

3497 UnmergeSrc = Unmerge->getSourceReg();

3498

3499 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());

3500 LLT UnmergeSrcTy = MRI.getType(UnmergeSrc);

3501

3502 if (!UnmergeSrcTy.isVector())

3503 return false;

3504

3505

3507 isLegal ({TargetOpcode::G_CONCAT_VECTORS, {DstTy, UnmergeSrcTy}}))

3508 return false;

3509

3510

3511

3512 for (unsigned I = 0; I < NumUnmerge; ++I) {

3513 auto MaybeUnmergeReg = BV.getSourceReg(I);

3515

3516 if (!LoopUnmerge || LoopUnmerge != Unmerge)

3517 return false;

3518

3519 if (LoopUnmerge->getOperand(I).getReg() != MaybeUnmergeReg)

3520 return false;

3521 }

3522

3523

3524 if (Unmerge->getNumDefs() != NumUnmerge)

3525 return false;

3526

3527

3528 for (unsigned I = NumUnmerge; I < BuildUseCount; ++I) {

3530

3531 if (Undef->getOpcode() != TargetOpcode::G_IMPLICIT_DEF)

3532 return false;

3533 }

3534

3535 return true;

3536}

3537

3541 Register &UnmergeSrc) const {

3542 assert(UnmergeSrc && "Expected there to be one matching G_UNMERGE_VALUES");

3543 B.setInstrAndDebugLoc(MI);

3544

3545 Register UndefVec = B.buildUndef(MRI.getType(UnmergeSrc)).getReg(0);

3546 B.buildConcatVectors(MI.getOperand(0), {UnmergeSrc, UndefVec});

3547

3548 MI.eraseFromParent();

3549}

3550

3551

3552

3553

3554

3555

3556

3557

3558

3559

3560

3561

3562

3563

3564

3565

3566

3568 Register &MatchInfo) const {

3570 unsigned NumOperands = BuildMI->getNumSources();

3572

3573

3574 unsigned I;

3576

3577

3578 for (I = 0; I < NumOperands; ++I) {

3579 auto SrcMI = MRI.getVRegDef(BuildMI->getSourceReg(I));

3580 auto SrcMIOpc = SrcMI->getOpcode();

3581

3582

3583 if (SrcMIOpc == TargetOpcode::G_TRUNC) {

3584 if (!UnmergeMI) {

3585 UnmergeMI = MRI.getVRegDef(SrcMI->getOperand(1).getReg());

3586 if (UnmergeMI->getOpcode() != TargetOpcode::G_UNMERGE_VALUES)

3587 return false;

3588 } else {

3589 auto UnmergeSrcMI = MRI.getVRegDef(SrcMI->getOperand(1).getReg());

3590 if (UnmergeMI != UnmergeSrcMI)

3591 return false;

3592 }

3593 } else {

3594 break;

3595 }

3596 }

3597 if (I < 2)

3598 return false;

3599

3600

3601 for (; I < NumOperands; ++I) {

3602 auto SrcMI = MRI.getVRegDef(BuildMI->getSourceReg(I));

3603 auto SrcMIOpc = SrcMI->getOpcode();

3604

3605 if (SrcMIOpc != TargetOpcode::G_IMPLICIT_DEF)

3606 return false;

3607 }

3608

3609

3610 MatchInfo = cast(UnmergeMI)->getSourceReg();

3611 LLT UnmergeSrcTy = MRI.getType(MatchInfo);

3613 return false;

3614

3615

3618 LLT UnmergeDstEltTy = MRI.getType(UnmergeDstReg);

3619 if (UnmergeSrcEltTy != UnmergeDstEltTy)

3620 return false;

3621

3622

3625

3627 isLegal ({TargetOpcode::G_CONCAT_VECTORS, {MidTy, UnmergeSrcTy}}))

3628 return false;

3629

3630 if ( isLegal ({TargetOpcode::G_TRUNC, {DstTy, MidTy}}))

3631 return false;

3632 }

3633

3634 return true;

3635}

3636

3638 Register &MatchInfo) const {

3642 LLT DstTy = MRI.getType(DstReg);

3643 LLT UnmergeSrcTy = MRI.getType(MatchInfo);

3645 unsigned UnmergeSrcTyNumElt = UnmergeSrcTy.getNumElements();

3646

3647

3648 if (DstTyNumElt / UnmergeSrcTyNumElt == 1) {

3649 MidReg = MatchInfo;

3650 } else {

3651 Register UndefReg = Builder.buildUndef(UnmergeSrcTy).getReg(0);

3653 for (unsigned I = 1; I < DstTyNumElt / UnmergeSrcTyNumElt; ++I)

3655

3657 MidReg = Builder.buildConcatVectors(MidTy, ConcatRegs).getReg(0);

3658 }

3659

3660 Builder.buildTrunc(DstReg, MidReg);

3661 MI.eraseFromParent();

3662}

3663

3666 assert(MI.getOpcode() == TargetOpcode::G_XOR);

3667 LLT Ty = MRI.getType(MI.getOperand(0).getReg());

3668 const auto &TLI = *Builder.getMF().getSubtarget().getTargetLowering();

3671

3674 return false;

3675

3676 if ( MRI .hasOneNonDBGUse(XorSrc))

3677 return false;

3678

3679

3680

3681

3683

3684 bool IsInt = false;

3685 bool IsFP = false;

3686 for (unsigned I = 0; I < RegsToNegate.size(); ++I) {

3688 if ( MRI .hasOneNonDBGUse(Reg))

3689 return false;

3691 switch (Def->getOpcode()) {

3692 default:

3693

3694

3695 return false;

3696 case TargetOpcode::G_ICMP:

3697 if (IsFP)

3698 return false;

3699 IsInt = true;

3700

3701 break;

3702 case TargetOpcode::G_FCMP:

3703 if (IsInt)

3704 return false;

3705 IsFP = true;

3706

3707 break;

3708 case TargetOpcode::G_AND:

3709 case TargetOpcode::G_OR:

3710

3711

3712

3713

3714

3715 RegsToNegate.push_back(Def->getOperand(1).getReg());

3716 RegsToNegate.push_back(Def->getOperand(2).getReg());

3717 break;

3718 }

3719 }

3720

3721

3722

3723 int64_t Cst;

3724 if (Ty.isVector()) {

3727 if (!MaybeCst)

3728 return false;

3729 if ( isConstValidTrue (TLI, Ty.getScalarSizeInBits(), *MaybeCst, true, IsFP))

3730 return false;

3731 } else {

3733 return false;

3734 if ( isConstValidTrue (TLI, Ty.getSizeInBits(), Cst, false, IsFP))

3735 return false;

3736 }

3737

3738 return true;

3739}

3740

3743 for (Register Reg : RegsToNegate) {

3745 Observer.changingInstr(*Def);

3746

3747

3748 switch (Def->getOpcode()) {

3749 default:

3751 case TargetOpcode::G_ICMP:

3752 case TargetOpcode::G_FCMP: {

3757 break;

3758 }

3759 case TargetOpcode::G_AND:

3760 Def->setDesc(Builder.getTII().get(TargetOpcode::G_OR));

3761 break;

3762 case TargetOpcode::G_OR:

3763 Def->setDesc(Builder.getTII().get(TargetOpcode::G_AND));

3764 break;

3765 }

3766 Observer.changedInstr(*Def);

3767 }

3768

3770 MI.eraseFromParent();

3771}

3772

3774 MachineInstr &MI, std::pair<Register, Register> &MatchInfo) const {

3775

3776 assert(MI.getOpcode() == TargetOpcode::G_XOR);

3779 Register AndReg = MI.getOperand(1).getReg();

3780 Register SharedReg = MI.getOperand(2).getReg();

3781

3782

3783

3784

3785

3786

3790 return false;

3791 }

3792

3793

3794 if ( MRI .hasOneNonDBGUse(AndReg))

3795 return false;

3796

3797

3798

3799 if (Y != SharedReg)

3801 return Y == SharedReg;

3802}

3803

3805 MachineInstr &MI, std::pair<Register, Register> &MatchInfo) const {

3806

3808 std::tie(X, Y) = MatchInfo;

3809 auto Not = Builder.buildNot(MRI.getType(X), X);

3811 MI.setDesc(Builder.getTII().get(TargetOpcode::G_AND));

3812 MI.getOperand(1).setReg(Not->getOperand(0).getReg());

3813 MI.getOperand(2).setReg(Y);

3815}

3816

3819 Register DstReg = PtrAdd.getReg(0);

3820 LLT Ty = MRI.getType(DstReg);

3822

3823 if (DL.isNonIntegralAddressSpace(Ty.getScalarType().getAddressSpace()))

3824 return false;

3825

3826 if (Ty.isPointer()) {

3828 return ConstVal && *ConstVal == 0;

3829 }

3830

3831 assert(Ty.isVector() && "Expecting a vector type");

3832 const MachineInstr *VecMI = MRI.getVRegDef(PtrAdd.getBaseReg());

3834}

3835

3838 Builder.buildIntToPtr(PtrAdd.getReg(0), PtrAdd.getOffsetReg());

3839 PtrAdd.eraseFromParent();

3840}

3841

3842

3844 Register DstReg = MI.getOperand(0).getReg();

3845 Register Src0 = MI.getOperand(1).getReg();

3846 Register Pow2Src1 = MI.getOperand(2).getReg();

3847 LLT Ty = MRI.getType(DstReg);

3848

3849

3850 auto NegOne = Builder.buildConstant(Ty, -1);

3851 auto Add = Builder.buildAdd(Ty, Pow2Src1, NegOne);

3852 Builder.buildAnd(DstReg, Src0, Add);

3853 MI.eraseFromParent();

3854}

3855

3857 unsigned &SelectOpNo) const {

3858 Register LHS = MI.getOperand(1).getReg();

3859 Register RHS = MI.getOperand(2).getReg();

3860

3861 Register OtherOperandReg = RHS;

3862 SelectOpNo = 1;

3864

3865

3866

3867 if (Select->getOpcode() != TargetOpcode::G_SELECT ||

3868 MRI .hasOneNonDBGUse(LHS)) {

3869 OtherOperandReg = LHS;

3870 SelectOpNo = 2;

3872 if (Select->getOpcode() != TargetOpcode::G_SELECT ||

3873 MRI .hasOneNonDBGUse(RHS))

3874 return false;

3875 }

3876

3879

3881 true,

3882 false))

3883 return false;

3885 true,

3886 false))

3887 return false;

3888

3889 unsigned BinOpcode = MI.getOpcode();

3890

3891

3892

3893

3894 bool CanFoldNonConst =

3895 (BinOpcode == TargetOpcode::G_AND || BinOpcode == TargetOpcode::G_OR) &&

3900 if (CanFoldNonConst)

3901 return true;

3902

3904 true,

3905 false);

3906}

3907

3908

3909

3911 MachineInstr &MI, const unsigned &SelectOperand) const {

3912 Register Dst = MI.getOperand(0).getReg();

3913 Register LHS = MI.getOperand(1).getReg();

3914 Register RHS = MI.getOperand(2).getReg();

3916

3917 Register SelectCond = Select->getOperand(1).getReg();

3918 Register SelectTrue = Select->getOperand(2).getReg();

3919 Register SelectFalse = Select->getOperand(3).getReg();

3920

3921 LLT Ty = MRI.getType(Dst);

3922 unsigned BinOpcode = MI.getOpcode();

3923

3924 Register FoldTrue, FoldFalse;

3925

3926

3927

3928

3929 if (SelectOperand == 1) {

3930

3931

3932

3933 FoldTrue = Builder.buildInstr(BinOpcode, {Ty}, {SelectTrue, RHS}).getReg(0);

3934 FoldFalse =

3935 Builder.buildInstr(BinOpcode, {Ty}, {SelectFalse, RHS}).getReg(0);

3936 } else {

3937 FoldTrue = Builder.buildInstr(BinOpcode, {Ty}, {LHS, SelectTrue}).getReg(0);

3938 FoldFalse =

3939 Builder.buildInstr(BinOpcode, {Ty}, {LHS, SelectFalse}).getReg(0);

3940 }

3941

3942 Builder.buildSelect(Dst, SelectCond, FoldTrue, FoldFalse, MI.getFlags());

3943 MI.eraseFromParent();

3944}

3945

3946std::optional<SmallVector<Register, 8>>

3947CombinerHelper::findCandidatesForLoadOrCombine(const MachineInstr *Root) const {

3948 assert(Root->getOpcode() == TargetOpcode::G_OR && "Expected G_OR only!");

3949

3950

3951

3952

3953

3954

3955

3956

3957

3958

3959

3960

3961

3962

3963

3964

3965

3966

3967

3968

3969

3970

3971

3974

3975

3976

3977 const unsigned MaxIter =

3979 for (unsigned Iter = 0; Iter < MaxIter; ++Iter) {

3980 if (Ors.empty())

3981 break;

3985

3986

3987 if ( MRI .hasOneNonDBGUse(OrLHS) || MRI .hasOneNonDBGUse(OrRHS))

3988 return std::nullopt;

3989

3990

3991

3994 else

3998 else

4000 }

4001

4002

4003

4004 if (RegsToVisit.empty() || RegsToVisit.size() % 2 != 0)

4005 return std::nullopt;

4006 return RegsToVisit;

4007}

4008

4009

4010

4011

4012

4013

4014

4015

4016

4017static std::optional<std::pair<GZExtLoad *, int64_t>>

4021 "Expected Reg to only have one non-debug use?");

4023 int64_t Shift;

4026 Shift = 0;

4027 MaybeLoad = Reg;

4028 }

4029

4030 if (Shift % MemSizeInBits != 0)

4031 return std::nullopt;

4032

4033

4035 if (!Load)

4036 return std::nullopt;

4037

4038 if (!Load->isUnordered() || Load->getMemSizeInBits() != MemSizeInBits)

4039 return std::nullopt;

4040

4041 return std::make_pair(Load, Shift / MemSizeInBits);

4042}

4043

4044std::optional<std::tuple<GZExtLoad *, int64_t, GZExtLoad *>>

4045CombinerHelper::findLoadOffsetsForLoadOrCombine(

4048 const unsigned MemSizeInBits) const {

4049

4050

4051 SmallSetVector<const MachineInstr *, 8> Loads;

4052

4053

4055

4056

4057 GZExtLoad *LowestIdxLoad = nullptr;

4058

4059

4060 SmallSet<int64_t, 8> SeenIdx;

4061

4062

4063

4064 MachineBasicBlock *MBB = nullptr;

4065 const MachineMemOperand *MMO = nullptr;

4066

4067

4068 GZExtLoad *EarliestLoad = nullptr;

4069

4070

4071 GZExtLoad *LatestLoad = nullptr;

4072

4073

4075

4076

4077

4078

4079

4080 for (auto Reg : RegsToVisit) {

4081

4082

4084 if (!LoadAndPos)

4085 return std::nullopt;

4086 GZExtLoad *Load;

4087 int64_t DstPos;

4088 std::tie(Load, DstPos) = *LoadAndPos;

4089

4090

4091

4092 MachineBasicBlock *LoadMBB = Load->getParent();

4093 if ( MBB )

4094 MBB = LoadMBB;

4095 if (LoadMBB != MBB)

4096 return std::nullopt;

4097

4098

4099 auto &LoadMMO = Load->getMMO();

4100 if (!MMO)

4101 MMO = &LoadMMO;

4102 if (MMO->getAddrSpace() != LoadMMO.getAddrSpace())

4103 return std::nullopt;

4104

4105

4107 int64_t Idx;

4110 LoadPtr = Load->getOperand(1).getReg();

4111 Idx = 0;

4112 }

4113

4114

4115 if (!SeenIdx.insert(Idx).second)

4116 return std::nullopt;

4117

4118

4119

4120

4123 if (BasePtr != LoadPtr)

4124 return std::nullopt;

4125

4126 if (Idx < LowestIdx) {

4127 LowestIdx = Idx;

4128 LowestIdxLoad = Load;

4129 }

4130

4131

4132

4133

4134

4135 if (!MemOffset2Idx.try_emplace(DstPos, Idx).second)

4136 return std::nullopt;

4138

4139

4140

4141

4142

4143

4144 if (!EarliestLoad || dominates(*Load, *EarliestLoad))

4145 EarliestLoad = Load;

4146 if (!LatestLoad || dominates(*LatestLoad, *Load))

4147 LatestLoad = Load;

4148 }

4149

4150

4151

4152 assert(Loads.size() == RegsToVisit.size() &&

4153 "Expected to find a load for each register?");

4154 assert(EarliestLoad != LatestLoad && EarliestLoad &&

4155 LatestLoad && "Expected at least two loads?");

4156

4157

4158

4159

4160

4161

4162

4163

4164 const unsigned MaxIter = 20;

4165 unsigned Iter = 0;

4169 continue;

4170 if (MI.isLoadFoldBarrier())

4171 return std::nullopt;

4172 if (Iter++ == MaxIter)

4173 return std::nullopt;

4174 }

4175

4176 return std::make_tuple(LowestIdxLoad, LowestIdx, LatestLoad);

4177}

4178

4182 assert(MI.getOpcode() == TargetOpcode::G_OR);

4184

4185

4186

4187

4188

4189

4190

4191

4192

4193

4194 Register Dst = MI.getOperand(0).getReg();

4195 LLT Ty = MRI.getType(Dst);

4196 if (Ty.isVector())

4197 return false;

4198

4199

4200

4201 const unsigned WideMemSizeInBits = Ty.getSizeInBits();

4202 if (WideMemSizeInBits < 16 || WideMemSizeInBits % 8 != 0)

4203 return false;

4204

4205

4206 auto RegsToVisit = findCandidatesForLoadOrCombine(&MI);

4207 if (!RegsToVisit)

4208 return false;

4209

4210

4211

4212

4213 const unsigned NarrowMemSizeInBits = WideMemSizeInBits / RegsToVisit->size();

4214 if (NarrowMemSizeInBits % 8 != 0)

4215 return false;

4216

4217

4218

4219

4220

4221

4222

4223

4225 GZExtLoad *LowestIdxLoad, *LatestLoad;

4226 int64_t LowestIdx;

4227 auto MaybeLoadInfo = findLoadOffsetsForLoadOrCombine(

4228 MemOffset2Idx, *RegsToVisit, NarrowMemSizeInBits);

4229 if (!MaybeLoadInfo)

4230 return false;

4231 std::tie(LowestIdxLoad, LowestIdx, LatestLoad) = *MaybeLoadInfo;

4232

4233

4234

4235

4236

4238 std::optional IsBigEndian = isBigEndian(MemOffset2Idx, LowestIdx);

4239 if (!IsBigEndian)

4240 return false;

4241 bool NeedsBSwap = IsBigEndianTarget != *IsBigEndian;

4243 return false;

4244

4245

4246

4247

4248

4249

4250

4251

4252

4253 const unsigned NumLoadsInTy = WideMemSizeInBits / NarrowMemSizeInBits;

4254 const unsigned ZeroByteOffset =

4255 *IsBigEndian

4258 auto ZeroOffsetIdx = MemOffset2Idx.find(ZeroByteOffset);

4259 if (ZeroOffsetIdx == MemOffset2Idx.end() ||

4260 ZeroOffsetIdx->second != LowestIdx)

4261 return false;

4262

4263

4264

4270 {TargetOpcode::G_LOAD, {Ty, MRI.getType(Ptr)}, {MMDesc}}))

4271 return false;

4273 auto *NewMMO = MF.getMachineMemOperand(&MMO, PtrInfo, WideMemSizeInBits / 8);

4274

4275

4278 unsigned Fast = 0;

4281 return false;

4282

4284 MIB.setInstrAndDebugLoc(*LatestLoad);

4285 Register LoadDst = NeedsBSwap ? MRI.cloneVirtualRegister(Dst) : Dst;

4286 MIB.buildLoad(LoadDst, Ptr, *NewMMO);

4287 if (NeedsBSwap)

4288 MIB.buildBSwap(Dst, LoadDst);

4289 };

4290 return true;

4291}

4292

4297

4298

4299

4300 if (MRI.getType(DstReg).isVector())

4301 return false;

4302

4303

4304 if ( MRI .hasOneNonDBGUse(DstReg))

4305 return false;

4306 ExtMI = &*MRI.use_instr_nodbg_begin(DstReg);

4308 case TargetOpcode::G_ANYEXT:

4309 return true;

4310 case TargetOpcode::G_ZEXT:

4311 case TargetOpcode::G_SEXT:

4312 break;

4313 default:

4314 return false;

4315 }

4316

4317

4318 if (Builder.getTII().isExtendLikelyToBeFolded(*ExtMI, MRI))

4319 return false;

4320

4321

4322

4323

4325 for (unsigned I = 0; I < PHI.getNumIncomingValues(); ++I) {

4327 switch (DefMI->getOpcode()) {

4328 case TargetOpcode::G_LOAD:

4329 case TargetOpcode::G_TRUNC:

4330 case TargetOpcode::G_SEXT:

4331 case TargetOpcode::G_ZEXT:

4332 case TargetOpcode::G_ANYEXT:

4333 case TargetOpcode::G_CONSTANT:

4335

4336

4337 if (InSrcs.size() > 2)

4338 return false;

4339 break;

4340 default:

4341 return false;

4342 }

4343 }

4344 return true;

4345}

4346

4351 LLT ExtTy = MRI.getType(DstReg);

4352

4353

4354

4355

4358 for (unsigned I = 0; I < PHI.getNumIncomingValues(); ++I) {

4359 auto SrcReg = PHI.getIncomingValue(I);

4360 auto *SrcMI = MRI.getVRegDef(SrcReg);

4361 if (!SrcMIs.insert(SrcMI))

4362 continue;

4363

4364

4365 auto *MBB = SrcMI->getParent();

4367 if (InsertPt != MBB->end() && InsertPt->isPHI())

4368 InsertPt = MBB->getFirstNonPHI();

4369

4370 Builder.setInsertPt(*SrcMI->getParent(), InsertPt);

4371 Builder.setDebugLoc(MI.getDebugLoc());

4372 auto NewExt = Builder.buildExtOrTrunc(ExtMI->getOpcode(), ExtTy, SrcReg);

4373 OldToNewSrcMap[SrcMI] = NewExt;

4374 }

4375

4376

4377 Builder.setInstrAndDebugLoc(MI);

4378 auto NewPhi = Builder.buildInstrNoInsert(TargetOpcode::G_PHI);

4379 NewPhi.addDef(DstReg);

4381 if (!MO.isReg()) {

4382 NewPhi.addMBB(MO.getMBB());

4383 continue;

4384 }

4385 auto *NewSrc = OldToNewSrcMap[MRI.getVRegDef(MO.getReg())];

4386 NewPhi.addUse(NewSrc->getOperand(0).getReg());

4387 }

4388 Builder.insertInstr(NewPhi);

4390}

4391

4394 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);

4395

4396

4397 Register SrcVec = MI.getOperand(1).getReg();

4398 LLT SrcTy = MRI.getType(SrcVec);

4399 if (SrcTy.isScalableVector())

4400 return false;

4401

4403 if (!Cst || Cst->Value.getZExtValue() >= SrcTy.getNumElements())

4404 return false;

4405

4406 unsigned VecIdx = Cst->Value.getZExtValue();

4407

4408

4409

4411 if (SrcVecMI->getOpcode() == TargetOpcode::G_TRUNC) {

4413 }

4414

4415 if (SrcVecMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR &&

4416 SrcVecMI->getOpcode() != TargetOpcode::G_BUILD_VECTOR_TRUNC)

4417 return false;

4418

4420 if ( MRI .hasOneNonDBGUse(SrcVec) &&

4422 return false;

4423

4425 return true;

4426}

4427

4430

4431

4432 LLT ScalarTy = MRI.getType(Reg);

4433 Register DstReg = MI.getOperand(0).getReg();

4434 LLT DstTy = MRI.getType(DstReg);

4435

4436 if (ScalarTy != DstTy) {

4438 Builder.buildTrunc(DstReg, Reg);

4439 MI.eraseFromParent();

4440 return;

4441 }

4443}

4444

4447 SmallVectorImpl<std::pair<Register, MachineInstr *>> &SrcDstPairs) const {

4448 assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);

4449

4450

4451

4452

4453

4454

4455

4456

4457

4458

4459

4460

4461

4462

4463

4464

4465 Register DstReg = MI.getOperand(0).getReg();

4466 LLT DstTy = MRI.getType(DstReg);

4468

4471 if (II.getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT)

4472 return false;

4474 if (!Cst)

4475 return false;

4476 unsigned Idx = Cst->getZExtValue();

4477 if (Idx >= NumElts)

4478 return false;

4479 ExtractedElts.set(Idx);

4480 SrcDstPairs.emplace_back(

4481 std::make_pair(MI.getOperand(Idx + 1).getReg(), &II));

4482 }

4483

4484 return ExtractedElts.all();

4485}

4486

4489 SmallVectorImpl<std::pair<Register, MachineInstr *>> &SrcDstPairs) const {

4490 assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);

4491 for (auto &Pair : SrcDstPairs) {

4492 auto *ExtMI = Pair.second;

4494 ExtMI->eraseFromParent();

4495 }

4496 MI.eraseFromParent();

4497}

4498

4503 MI.eraseFromParent();

4504}

4505

4511

4513 bool AllowScalarConstants,

4515 assert(MI.getOpcode() == TargetOpcode::G_OR);

4516

4517 Register Dst = MI.getOperand(0).getReg();

4518 LLT Ty = MRI.getType(Dst);

4519 unsigned BitWidth = Ty.getScalarSizeInBits();

4520

4521 Register ShlSrc, ShlAmt, LShrSrc, LShrAmt, Amt;

4522 unsigned FshOpc = 0;

4523

4524

4526

4529 return false;

4530

4531

4532

4533 int64_t CstShlAmt = 0, CstLShrAmt;

4536 CstShlAmt + CstLShrAmt == BitWidth) {

4537 FshOpc = TargetOpcode::G_FSHR;

4538 Amt = LShrAmt;

4541 ShlAmt == Amt) {

4542

4543 FshOpc = TargetOpcode::G_FSHL;

4546 LShrAmt == Amt) {

4547

4548 FshOpc = TargetOpcode::G_FSHR;

4549 } else {

4550 return false;

4551 }

4552

4553 LLT AmtTy = MRI.getType(Amt);

4555 (!AllowScalarConstants || CstShlAmt == 0 || !Ty.isScalar()))

4556 return false;

4557

4559 B.buildInstr(FshOpc, {Dst}, {ShlSrc, LShrSrc, Amt});

4560 };

4561 return true;

4562}

4563

4564

4566 unsigned Opc = MI.getOpcode();

4567 assert(Opc == TargetOpcode::G_FSHL || Opc == TargetOpcode::G_FSHR);

4570 if (X != Y)

4571 return false;

4572 unsigned RotateOpc =

4573 Opc == TargetOpcode::G_FSHL ? TargetOpcode::G_ROTL : TargetOpcode::G_ROTR;

4575}

4576

4578 unsigned Opc = MI.getOpcode();

4579 assert(Opc == TargetOpcode::G_FSHL || Opc == TargetOpcode::G_FSHR);

4580 bool IsFSHL = Opc == TargetOpcode::G_FSHL;

4582 MI.setDesc(Builder.getTII().get(IsFSHL ? TargetOpcode::G_ROTL

4583 : TargetOpcode::G_ROTR));

4584 MI.removeOperand(2);

4586}

4587

4588

4590 assert(MI.getOpcode() == TargetOpcode::G_ROTL ||

4591 MI.getOpcode() == TargetOpcode::G_ROTR);

4592 unsigned Bitsize =

4593 MRI.getType(MI.getOperand(0).getReg()).getScalarSizeInBits();

4594 Register AmtReg = MI.getOperand(2).getReg();

4595 bool OutOfRange = false;

4596 auto MatchOutOfRange = [Bitsize, &OutOfRange](const Constant *C) {

4598 OutOfRange |= CI->getValue().uge(Bitsize);

4599 return true;

4600 };

4602}

4603

4605 assert(MI.getOpcode() == TargetOpcode::G_ROTL ||

4606 MI.getOpcode() == TargetOpcode::G_ROTR);

4607 unsigned Bitsize =

4608 MRI.getType(MI.getOperand(0).getReg()).getScalarSizeInBits();

4609 Register Amt = MI.getOperand(2).getReg();

4610 LLT AmtTy = MRI.getType(Amt);

4611 auto Bits = Builder.buildConstant(AmtTy, Bitsize);

4612 Amt = Builder.buildURem(AmtTy, MI.getOperand(2).getReg(), Bits).getReg(0);

4614 MI.getOperand(2).setReg(Amt);

4616}

4617

4619 int64_t &MatchInfo) const {

4620 assert(MI.getOpcode() == TargetOpcode::G_ICMP);

4621 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());

4622

4623

4624

4625

4626

4627

4628

4629

4630

4631 auto KnownRHS = VT->getKnownBits(MI.getOperand(3).getReg());

4632 if (KnownRHS.isUnknown())

4633 return false;

4634

4635 std::optional KnownVal;

4636 if (KnownRHS.isZero()) {

4637

4638

4640 KnownVal = true;

4642 KnownVal = false;

4643 }

4644

4645 if (!KnownVal) {

4646 auto KnownLHS = VT->getKnownBits(MI.getOperand(2).getReg());

4648 }

4649

4650 if (!KnownVal)

4651 return false;

4652 MatchInfo =

4653 *KnownVal

4655

4656 MRI.getType(MI.getOperand(0).getReg()).isVector(),

4657 false)

4658 : 0;

4659 return true;

4660}

4661

4665 assert(MI.getOpcode() == TargetOpcode::G_ICMP);

4666

4667

4668

4669

4670

4671

4672

4673

4674

4675

4676

4677 auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());

4679 return false;

4680 Register Dst = MI.getOperand(0).getReg();

4681 LLT DstTy = MRI.getType(Dst);

4683 false) != 1)

4684 return false;

4687 return false;

4688 Register LHS = MI.getOperand(2).getReg();

4689 auto KnownLHS = VT->getKnownBits(LHS);

4690 if (KnownLHS.getMinValue() != 0 || KnownLHS.getMaxValue() != 1)

4691 return false;

4692

4693 LLT LHSTy = MRI.getType(LHS);

4696 unsigned Op = TargetOpcode::COPY;

4697 if (DstSize != LHSSize)

4698 Op = DstSize < LHSSize ? TargetOpcode::G_TRUNC : TargetOpcode::G_ZEXT;

4700 return false;

4702 return true;

4703}

4704

4705

4709 assert(MI.getOpcode() == TargetOpcode::G_AND);

4710

4711

4712

4713 LLT Ty = MRI.getType(MI.getOperand(0).getReg());

4714 if (Ty.isVector())

4715 return false;

4716

4719 int64_t AndMaskBits;

4720 int64_t OrMaskBits;

4724 return false;

4725

4726

4727 if (AndMaskBits & OrMaskBits)

4728 return false;

4729

4732

4733 if (MI.getOperand(1).getReg() == AndMaskReg)

4734 MI.getOperand(2).setReg(AndMaskReg);

4735 MI.getOperand(1).setReg(Src);

4737 };

4738 return true;

4739}

4740

4741

4745 assert(MI.getOpcode() == TargetOpcode::G_SEXT_INREG);

4746 Register Dst = MI.getOperand(0).getReg();

4747 Register Src = MI.getOperand(1).getReg();

4748 LLT Ty = MRI.getType(Src);

4750 if ( || ->isLegalOrCustom({TargetOpcode::G_SBFX, {Ty, ExtractTy}}))

4751 return false;

4752 int64_t Width = MI.getOperand(2).getImm();

4754 int64_t ShiftImm;

4756 Src, MRI,

4759 return false;

4760 if (ShiftImm < 0 || ShiftImm + Width > Ty.getScalarSizeInBits())

4761 return false;

4762

4764 auto Cst1 = B.buildConstant(ExtractTy, ShiftImm);

4765 auto Cst2 = B.buildConstant(ExtractTy, Width);

4766 B.buildSbfx(Dst, ShiftSrc, Cst1, Cst2);

4767 };

4768 return true;

4769}

4770

4771

4776 LLT Ty = MRI.getType(Dst);

4778

4779

4780 if (LI && ->isLegalOrCustom({TargetOpcode::G_UBFX, {Ty, ExtractTy}}))

4781 return false;

4782

4783 int64_t AndImm, LSBImm;

4785 const unsigned Size = Ty.getScalarSizeInBits();

4789 return false;

4790

4791

4792 auto MaybeMask = static_cast<uint64_t>(AndImm);

4793 if (MaybeMask & (MaybeMask + 1))

4794 return false;

4795

4796

4798 return false;

4799

4802 auto WidthCst = B.buildConstant(ExtractTy, Width);

4803 auto LSBCst = B.buildConstant(ExtractTy, LSBImm);

4804 B.buildInstr(TargetOpcode::G_UBFX, {Dst}, {ShiftSrc, LSBCst, WidthCst});

4805 };

4806 return true;

4807}

4808

4812 const unsigned Opcode = MI.getOpcode();

4813 assert(Opcode == TargetOpcode::G_ASHR || Opcode == TargetOpcode::G_LSHR);

4814

4815 const Register Dst = MI.getOperand(0).getReg();

4816

4817 const unsigned ExtrOpcode = Opcode == TargetOpcode::G_ASHR

4818 ? TargetOpcode::G_SBFX

4819 : TargetOpcode::G_UBFX;

4820

4821

4822 LLT Ty = MRI.getType(Dst);

4824 if ( || ->isLegalOrCustom({ExtrOpcode, {Ty, ExtractTy}}))

4825 return false;

4826

4828 int64_t ShrAmt;

4829 int64_t ShlAmt;

4830 const unsigned Size = Ty.getScalarSizeInBits();

4831

4832

4837 return false;

4838

4839

4840 if (ShlAmt < 0 || ShlAmt > ShrAmt || ShrAmt >= Size)

4841 return false;

4842

4843

4844 if (Opcode == TargetOpcode::G_ASHR && ShlAmt == ShrAmt)

4845 return false;

4846

4847

4848 const int64_t Pos = ShrAmt - ShlAmt;

4849 const int64_t Width = Size - ShrAmt;

4850

4852 auto WidthCst = B.buildConstant(ExtractTy, Width);

4853 auto PosCst = B.buildConstant(ExtractTy, Pos);

4854 B.buildInstr(ExtrOpcode, {Dst}, {ShlSrc, PosCst, WidthCst});

4855 };

4856 return true;

4857}

4858

4862 const unsigned Opcode = MI.getOpcode();

4863 assert(Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_ASHR);

4864

4865 const Register Dst = MI.getOperand(0).getReg();

4866 LLT Ty = MRI.getType(Dst);

4868 if (LI && ->isLegalOrCustom({TargetOpcode::G_UBFX, {Ty, ExtractTy}}))

4869 return false;

4870

4871

4873 int64_t ShrAmt;

4874 int64_t SMask;

4879 return false;

4880

4881 const unsigned Size = Ty.getScalarSizeInBits();

4882 if (ShrAmt < 0 || ShrAmt >= Size)

4883 return false;

4884

4885

4886 if (0 == (SMask >> ShrAmt)) {

4888 B.buildConstant(Dst, 0);

4889 };

4890 return true;

4891 }

4892

4893

4894 uint64_t UMask = SMask;

4898 return false;

4899

4900

4901 const int64_t Pos = ShrAmt;

4903

4904

4905

4906 if (Opcode == TargetOpcode::G_ASHR && Width + ShrAmt == Size)

4907 return false;

4908

4910 auto WidthCst = B.buildConstant(ExtractTy, Width);

4911 auto PosCst = B.buildConstant(ExtractTy, Pos);

4912 B.buildInstr(TargetOpcode::G_UBFX, {Dst}, {AndSrc, PosCst, WidthCst});

4913 };

4914 return true;

4915}

4916

4917bool CombinerHelper::reassociationCanBreakAddressingModePattern(

4920

4921 Register Src1Reg = PtrAdd.getBaseReg();

4923 if (!Src1Def)

4924 return false;

4925

4926 Register Src2Reg = PtrAdd.getOffsetReg();

4927

4928 if (MRI.hasOneNonDBGUse(Src1Reg))

4929 return false;

4930

4932 if (!C1)

4933 return false;

4935 if (!C2)

4936 return false;

4937

4938 const APInt &C1APIntVal = *C1;

4939 const APInt &C2APIntVal = *C2;

4940 const int64_t CombinedValue = (C1APIntVal + C2APIntVal).getSExtValue();

4941

4942 for (auto &UseMI : MRI.use_nodbg_instructions(PtrAdd.getReg(0))) {

4943

4944

4945 MachineInstr *ConvUseMI = &UseMI;

4946 unsigned ConvUseOpc = ConvUseMI->getOpcode();

4947 while (ConvUseOpc == TargetOpcode::G_INTTOPTR ||

4948 ConvUseOpc == TargetOpcode::G_PTRTOINT) {

4950 if ( MRI .hasOneNonDBGUse(DefReg))

4951 break;

4952 ConvUseMI = &*MRI.use_instr_nodbg_begin(DefReg);

4953 ConvUseOpc = ConvUseMI->getOpcode();

4954 }

4956 if (!LdStMI)

4957 continue;

4958

4959

4960

4961 TargetLoweringBase::AddrMode AM;

4964 unsigned AS = MRI.getType(LdStMI->getPointerReg()).getAddressSpace();

4965 Type *AccessTy = getTypeForLLT(LdStMI->getMMO().getMemoryType(),

4966 PtrAdd.getMF()->getFunction().getContext());

4967 const auto &TLI = *PtrAdd.getMF()->getSubtarget().getTargetLowering();

4968 if (!TLI.isLegalAddressingMode(PtrAdd.getMF()->getDataLayout(), AM,

4969 AccessTy, AS))

4970 continue;

4971

4972

4973 AM.BaseOffs = CombinedValue;

4974 if (!TLI.isLegalAddressingMode(PtrAdd.getMF()->getDataLayout(), AM,

4975 AccessTy, AS))

4976 return true;

4977 }

4978

4979 return false;

4980}

4981

4985

4986 Register Src1Reg = MI.getOperand(1).getReg();

4987 if (RHS->getOpcode() != TargetOpcode::G_ADD)

4988 return false;

4990 if (!C2)

4991 return false;

4992

4993

4994

4995

4996

4997

4998

4999 unsigned PtrAddFlags = MI.getFlags();

5000 unsigned AddFlags = RHS->getFlags();

5004 unsigned Flags = 0;

5005 if (IsNoUWrap)

5007 if (IsNoUSWrap)

5009 if (IsInBounds)

5011

5013 LLT PtrTy = MRI.getType(MI.getOperand(0).getReg());

5014

5015 auto NewBase =

5016 Builder.buildPtrAdd(PtrTy, Src1Reg, RHS->getOperand(1).getReg(), Flags);

5018 MI.getOperand(1).setReg(NewBase.getReg(0));

5019 MI.getOperand(2).setReg(RHS->getOperand(2).getReg());

5020 MI.setFlags(Flags);

5022 };

5023 return !reassociationCanBreakAddressingModePattern(MI);

5024}

5025

5030

5031

5033 std::optional LHSCstOff;

5036 return false;

5037

5039

5040

5041

5042

5043 unsigned PtrAddFlags = MI.getFlags();

5044 unsigned LHSPtrAddFlags = LHSPtrAdd->getFlags();

5046 bool IsNoUSWrap = IsNoUWrap && (PtrAddFlags & LHSPtrAddFlags &

5048 bool IsInBounds = IsNoUWrap && (PtrAddFlags & LHSPtrAddFlags &

5050 unsigned Flags = 0;

5051 if (IsNoUWrap)

5053 if (IsNoUSWrap)

5055 if (IsInBounds)

5057

5059

5060

5061

5062 LHSPtrAdd->moveBefore(&MI);

5063 Register RHSReg = MI.getOffsetReg();

5064

5065 auto NewCst = B.buildConstant(MRI.getType(RHSReg), LHSCstOff->Value);

5067 MI.getOperand(2).setReg(NewCst.getReg(0));

5068 MI.setFlags(Flags);

5070 Observer.changingInstr(*LHSPtrAdd);

5071 LHSPtrAdd->getOperand(2).setReg(RHSReg);

5072 LHSPtrAdd->setFlags(Flags);

5073 Observer.changedInstr(*LHSPtrAdd);

5074 };

5075 return !reassociationCanBreakAddressingModePattern(MI);

5076}

5077

5081

5083 if (!LHSPtrAdd)

5084 return false;

5085

5086 Register Src2Reg = MI.getOperand(2).getReg();

5087 Register LHSSrc1 = LHSPtrAdd->getBaseReg();

5088 Register LHSSrc2 = LHSPtrAdd->getOffsetReg();

5090 if (!C1)

5091 return false;

5093 if (!C2)

5094 return false;

5095

5096

5097

5098

5099

5100

5101 unsigned PtrAddFlags = MI.getFlags();

5102 unsigned LHSPtrAddFlags = LHSPtrAdd->getFlags();

5104 bool IsInBounds =

5106 unsigned Flags = 0;

5107 if (IsNoUWrap)

5109 if (IsInBounds) {

5112 }

5113

5115 auto NewCst = B.buildConstant(MRI.getType(Src2Reg), *C1 + *C2);

5117 MI.getOperand(1).setReg(LHSSrc1);

5118 MI.getOperand(2).setReg(NewCst.getReg(0));

5119 MI.setFlags(Flags);

5121 };

5122 return !reassociationCanBreakAddressingModePattern(MI);

5123}

5124

5128

5129

5130

5131

5132

5133

5134

5135

5136

5137

5138

5139

5140 MachineInstr *LHS = MRI.getVRegDef(PtrAdd.getBaseReg());

5141 MachineInstr *RHS = MRI.getVRegDef(PtrAdd.getOffsetReg());

5142

5143

5145 return true;

5146

5147

5149 return true;

5150

5151

5153 return true;

5154

5155 return false;

5156}

5160 LLT OpRHSTy = MRI.getType(OpRHS);

5162

5164 return false;

5165

5169

5170

5171

5172

5173

5177

5179 auto NewCst = B.buildInstr(Opc, {OpRHSTy}, {OpLHSRHS, OpRHS});

5180 B.buildInstr(Opc, {DstReg}, {OpLHSLHS, NewCst});

5181 };

5182 return true;

5183 }

5185

5186

5188 auto NewLHSLHS = B.buildInstr(Opc, {OpRHSTy}, {OpLHSLHS, OpRHS});

5189 B.buildInstr(Opc, {DstReg}, {NewLHSLHS, OpLHSRHS});

5190 };

5191 return true;

5192 }

5193 }

5194

5195 return false;

5196}

5197

5200

5201

5202 unsigned Opc = MI.getOpcode();

5203 Register DstReg = MI.getOperand(0).getReg();

5204 Register LHSReg = MI.getOperand(1).getReg();

5205 Register RHSReg = MI.getOperand(2).getReg();

5206

5208 return true;

5210 return true;

5211 return false;

5212}

5213

5215 APInt &MatchInfo) const {

5216 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());

5218

5220 MatchInfo = *MaybeCst;

5221 return true;

5222 }

5223

5224 return false;

5225}

5226

5228 APInt &MatchInfo) const {

5229 Register Op1 = MI.getOperand(1).getReg();

5230 Register Op2 = MI.getOperand(2).getReg();

5232 if (!MaybeCst)

5233 return false;

5234 MatchInfo = *MaybeCst;

5235 return true;

5236}

5237

5240 Register Op1 = MI.getOperand(1).getReg();

5241 Register Op2 = MI.getOperand(2).getReg();

5243 if (!MaybeCst)

5244 return false;

5245 MatchInfo =

5246 ConstantFP::get(MI.getMF()->getFunction().getContext(), *MaybeCst);

5247 return true;

5248}

5249

5252 assert(MI.getOpcode() == TargetOpcode::G_FMA ||

5253 MI.getOpcode() == TargetOpcode::G_FMAD);

5254 auto [_, Op1, Op2, Op3] = MI.getFirst4Regs();

5255

5257 if (!Op3Cst)

5258 return false;

5259

5261 if (!Op2Cst)

5262 return false;

5263

5265 if (!Op1Cst)

5266 return false;

5267

5271 MatchInfo = ConstantFP::get(MI.getMF()->getFunction().getContext(), Op1F);

5272 return true;

5273}

5274

5278

5279

5280

5281

5282

5283

5284

5285

5286

5287

5288

5289

5290

5291

5292

5293

5294 assert(MI.getOpcode() == TargetOpcode::G_AND);

5295 Register Dst = MI.getOperand(0).getReg();

5296 Register AndLHS = MI.getOperand(1).getReg();

5297 Register AndRHS = MI.getOperand(2).getReg();

5298 LLT WideTy = MRI.getType(Dst);

5299

5300

5301

5302 if (!WideTy.isScalar() || MRI .hasOneNonDBGUse(AndLHS))

5303 return false;

5304

5305

5306

5307

5308

5309

5310

5312 if (!LHSInst)

5313 return false;

5314 unsigned LHSOpc = LHSInst->getOpcode();

5315 switch (LHSOpc) {

5316 default:

5317 return false;

5318 case TargetOpcode::G_ADD:

5319 case TargetOpcode::G_SUB:

5320 case TargetOpcode::G_MUL:

5321 case TargetOpcode::G_AND:

5322 case TargetOpcode::G_OR:

5323 case TargetOpcode::G_XOR:

5324 break;

5325 }

5326

5327

5329 if (!Cst)

5330 return false;

5331 auto Mask = Cst->Value;

5332 if (!Mask.isMask())

5333 return false;

5334

5335

5336 unsigned NarrowWidth = Mask.countr_one();

5338 return false;

5340

5341

5342 auto &MF = *MI.getMF();

5344 LLVMContext &Ctx = MF.getFunction().getContext();

5345 if (!TLI.isTruncateFree(WideTy, NarrowTy, Ctx) ||

5346 !TLI.isZExtFree(NarrowTy, WideTy, Ctx))

5347 return false;

5350 return false;

5354 auto NarrowLHS = Builder.buildTrunc(NarrowTy, BinOpLHS);

5355 auto NarrowRHS = Builder.buildTrunc(NarrowTy, BinOpRHS);

5356 auto NarrowBinOp =

5357 Builder.buildInstr(LHSOpc, {NarrowTy}, {NarrowLHS, NarrowRHS});

5358 auto Ext = Builder.buildZExt(WideTy, NarrowBinOp);

5360 MI.getOperand(1).setReg(Ext.getReg(0));

5362 };

5363 return true;

5364}

5365

5368 unsigned Opc = MI.getOpcode();

5369 assert(Opc == TargetOpcode::G_UMULO || Opc == TargetOpcode::G_SMULO);

5370

5372 return false;

5373

5376 unsigned NewOpc = Opc == TargetOpcode::G_UMULO ? TargetOpcode::G_UADDO

5377 : TargetOpcode::G_SADDO;

5378 MI.setDesc(Builder.getTII().get(NewOpc));

5379 MI.getOperand(3).setReg(MI.getOperand(2).getReg());

5381 };

5382 return true;

5383}

5384

5387

5388 assert(MI.getOpcode() == TargetOpcode::G_UMULO ||

5389 MI.getOpcode() == TargetOpcode::G_SMULO);

5391 return false;

5392 Register Dst = MI.getOperand(0).getReg();

5393 Register Carry = MI.getOperand(1).getReg();

5396 return false;

5398 B.buildConstant(Dst, 0);

5399 B.buildConstant(Carry, 0);

5400 };

5401 return true;

5402}

5403

5406

5407

5408 assert(MI.getOpcode() == TargetOpcode::G_UADDE ||

5409 MI.getOpcode() == TargetOpcode::G_SADDE ||

5410 MI.getOpcode() == TargetOpcode::G_USUBE ||

5411 MI.getOpcode() == TargetOpcode::G_SSUBE);

5413 return false;

5415 unsigned NewOpcode;

5416 switch (MI.getOpcode()) {

5417 case TargetOpcode::G_UADDE:

5418 NewOpcode = TargetOpcode::G_UADDO;

5419 break;

5420 case TargetOpcode::G_SADDE:

5421 NewOpcode = TargetOpcode::G_SADDO;

5422 break;

5423 case TargetOpcode::G_USUBE:

5424 NewOpcode = TargetOpcode::G_USUBO;

5425 break;

5426 case TargetOpcode::G_SSUBE:

5427 NewOpcode = TargetOpcode::G_SSUBO;

5428 break;

5429 }

5431 MI.setDesc(B.getTII().get(NewOpcode));

5432 MI.removeOperand(4);

5434 };

5435 return true;

5436}

5437

5440 assert(MI.getOpcode() == TargetOpcode::G_SUB);

5441 Register Dst = MI.getOperand(0).getReg();

5442

5443

5447 int64_t CstX, CstY;

5450 ReplaceReg = X;

5453 ReplaceReg = Y;

5454 if (ReplaceReg) {

5455 MatchInfo = [=](MachineIRBuilder &B) { B.buildCopy(Dst, ReplaceReg); };

5456 return true;

5457 }

5458 }

5459

5460

5461

5464 int64_t CstX;

5467 ReplaceReg = Y;

5470 ReplaceReg = Z;

5471 if (ReplaceReg) {

5473 auto Zero = B.buildConstant(MRI.getType(Dst), 0);

5474 B.buildSub(Dst, Zero, ReplaceReg);

5475 };

5476 return true;

5477 }

5478 }

5479 return false;

5480}

5481

5483 unsigned Opcode = MI.getOpcode();

5484 assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);

5486 Register Dst = UDivorRem.getReg(0);

5487 Register LHS = UDivorRem.getReg(1);

5488 Register RHS = UDivorRem.getReg(2);

5489 LLT Ty = MRI.getType(Dst);

5494

5496

5497 bool UseSRL = false;

5501

5502 auto BuildExactUDIVPattern = [&](const Constant *C) {

5503

5504 if (IsSplat && !Factors.empty()) {

5507 return true;

5508 }

5509

5511 APInt Divisor = CI->getValue();

5513 if (Shift) {

5515 UseSRL = true;

5516 }

5517

5518

5520 Shifts.push_back(MIB.buildConstant(ScalarShiftAmtTy, Shift).getReg(0));

5521 Factors.push_back(MIB.buildConstant(ScalarTy, Factor).getReg(0));

5522 return true;

5523 };

5524

5526

5528 llvm_unreachable("Expected unary predicate match to succeed");

5529

5531 if (Ty.isVector()) {

5532 Shift = MIB.buildBuildVector(ShiftAmtTy, Shifts).getReg(0);

5533 Factor = MIB.buildBuildVector(Ty, Factors).getReg(0);

5534 } else {

5535 Shift = Shifts[0];

5536 Factor = Factors[0];

5537 }

5538

5540

5541 if (UseSRL)

5543

5544 return MIB.buildMul(Ty, Res, Factor);

5545 }

5546

5547 unsigned KnownLeadingZeros =

5548 VT ? VT->getKnownBits(LHS).countMinLeadingZeros() : 0;

5549

5550 bool UseNPQ = false;

5552 auto BuildUDIVPattern = [&](const Constant *C) {

5554 const APInt &Divisor = CI->getValue();

5555

5556 bool SelNPQ = false;

5558 unsigned PreShift = 0, PostShift = 0;

5559

5560

5561

5562

5563 if (!Divisor.isOne()) {

5564

5565

5566

5569 Divisor, std::min(KnownLeadingZeros, Divisor.countl_zero()));

5570

5571 Magic = std::move(magics.Magic);

5572

5574 "We shouldn't generate an undefined shift!");

5576 "We shouldn't generate an undefined shift!");

5577 assert((!magics.IsAdd || magics.PreShift == 0) && "Unexpected pre-shift");

5580 SelNPQ = magics.IsAdd;

5581 }

5582

5584 MIB.buildConstant(ScalarShiftAmtTy, PreShift).getReg(0));

5585 MagicFactors.push_back(MIB.buildConstant(ScalarTy, Magic).getReg(0));

5587 MIB.buildConstant(ScalarTy,

5590 .getReg(0));

5592 MIB.buildConstant(ScalarShiftAmtTy, PostShift).getReg(0));

5593 UseNPQ |= SelNPQ;

5594 return true;

5595 };

5596

5597

5599 (void)Matched;

5600 assert(Matched && "Expected unary predicate match to succeed");

5601

5602 Register PreShift, PostShift, MagicFactor, NPQFactor;

5604 if (RHSDef) {

5605 PreShift = MIB.buildBuildVector(ShiftAmtTy, PreShifts).getReg(0);

5606 MagicFactor = MIB.buildBuildVector(Ty, MagicFactors).getReg(0);

5607 NPQFactor = MIB.buildBuildVector(Ty, NPQFactors).getReg(0);

5608 PostShift = MIB.buildBuildVector(ShiftAmtTy, PostShifts).getReg(0);

5609 } else {

5610 assert(MRI.getType(RHS).isScalar() &&

5611 "Non-build_vector operation should have been a scalar");

5612 PreShift = PreShifts[0];

5613 MagicFactor = MagicFactors[0];

5614 PostShift = PostShifts[0];

5615 }

5616

5618 Q = MIB.buildLShr(Ty, Q, PreShift).getReg(0);

5619

5620

5621 Q = MIB.buildUMulH(Ty, Q, MagicFactor).getReg(0);

5622

5623 if (UseNPQ) {

5624 Register NPQ = MIB.buildSub(Ty, LHS, Q).getReg(0);

5625

5626

5627

5628 if (Ty.isVector())

5629 NPQ = MIB.buildUMulH(Ty, NPQ, NPQFactor).getReg(0);

5630 else

5631 NPQ = MIB.buildLShr(Ty, NPQ, MIB.buildConstant(ShiftAmtTy, 1)).getReg(0);

5632

5633 Q = MIB.buildAdd(Ty, NPQ, Q).getReg(0);

5634 }

5635

5636 Q = MIB.buildLShr(Ty, Q, PostShift).getReg(0);

5637 auto One = MIB.buildConstant(Ty, 1);

5638 auto IsOne = MIB.buildICmp(

5640 Ty.isScalar() ? LLT::scalar(1) : Ty.changeElementSize(1), RHS, One);

5641 auto ret = MIB.buildSelect(Ty, IsOne, LHS, Q);

5642

5643 if (Opcode == TargetOpcode::G_UREM) {

5644 auto Prod = MIB.buildMul(Ty, ret, RHS);

5645 return MIB.buildSub(Ty, LHS, Prod);

5646 }

5647 return ret;

5648}

5649

5651 unsigned Opcode = MI.getOpcode();

5652 assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);

5653 Register Dst = MI.getOperand(0).getReg();

5654 Register RHS = MI.getOperand(2).getReg();

5655 LLT DstTy = MRI.getType(Dst);

5656

5657 auto &MF = *MI.getMF();

5658 AttributeList Attr = MF.getFunction().getAttributes();

5660 LLVMContext &Ctx = MF.getFunction().getContext();

5662 return false;

5663

5664

5665

5666 if (MF.getFunction().hasMinSize())

5667 return false;

5668

5669 if (Opcode == TargetOpcode::G_UDIV &&

5672 MRI, RHS, [](const Constant *C) { return C && ->isNullValue(); });

5673 }

5674

5675 auto *RHSDef = MRI.getVRegDef(RHS);

5677 return false;

5678

5679

5680 if (LI) {

5682 return false;

5684 return false;

5686 {TargetOpcode::G_ICMP,

5688 DstTy}}))

5689 return false;

5690 if (Opcode == TargetOpcode::G_UREM &&

5692 return false;

5693 }

5694

5696 MRI, RHS, [](const Constant *C) { return C && ->isNullValue(); });

5697}

5698

5703

5705 unsigned Opcode = MI.getOpcode();

5706 assert(Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM);

5707 Register Dst = MI.getOperand(0).getReg();

5708 Register RHS = MI.getOperand(2).getReg();

5709 LLT DstTy = MRI.getType(Dst);

5712

5713 auto &MF = *MI.getMF();

5714 AttributeList Attr = MF.getFunction().getAttributes();

5716 LLVMContext &Ctx = MF.getFunction().getContext();

5718 return false;

5719

5720

5721

5722 if (MF.getFunction().hasMinSize())

5723 return false;

5724

5725

5726 if (Opcode == TargetOpcode::G_SDIV &&

5729 MRI, RHS, [](const Constant *C) { return C && ->isNullValue(); });

5730 }

5731

5732 auto *RHSDef = MRI.getVRegDef(RHS);

5734 return false;

5735

5736

5737 if (LI) {

5739 return false;

5740 if ( isLegal ({TargetOpcode::G_SMULH, {DstTy}}) &&

5742 return false;

5743 if (Opcode == TargetOpcode::G_SREM &&

5745 return false;

5746 }

5747

5749 MRI, RHS, [](const Constant *C) { return C && ->isNullValue(); });

5750}

5751

5756

5758 unsigned Opcode = MI.getOpcode();

5759 assert(MI.getOpcode() == TargetOpcode::G_SDIV ||

5760 Opcode == TargetOpcode::G_SREM);

5762 Register Dst = SDivorRem.getReg(0);

5763 Register LHS = SDivorRem.getReg(1);

5764 Register RHS = SDivorRem.getReg(2);

5765 LLT Ty = MRI.getType(Dst);

5771

5772 bool UseSRA = false;

5774

5777

5778 auto BuildExactSDIVPattern = [&](const Constant *C) {

5779

5780 if (IsSplat && !ExactFactors.empty()) {

5781 ExactShifts.push_back(ExactShifts[0]);

5782 ExactFactors.push_back(ExactFactors[0]);

5783 return true;

5784 }

5785

5787 APInt Divisor = CI->getValue();

5789 if (Shift) {

5791 UseSRA = true;

5792 }

5793

5794

5795

5797 ExactShifts.push_back(MIB.buildConstant(ScalarShiftAmtTy, Shift).getReg(0));

5798 ExactFactors.push_back(MIB.buildConstant(ScalarTy, Factor).getReg(0));

5799 return true;

5800 };

5801

5803

5805 (void)Matched;

5806 assert(Matched && "Expected unary predicate match to succeed");

5807

5809 if (Ty.isVector()) {

5810 Shift = MIB.buildBuildVector(ShiftAmtTy, ExactShifts).getReg(0);

5811 Factor = MIB.buildBuildVector(Ty, ExactFactors).getReg(0);

5812 } else {

5813 Shift = ExactShifts[0];

5814 Factor = ExactFactors[0];

5815 }

5816

5818

5819 if (UseSRA)

5821

5822 return MIB.buildMul(Ty, Res, Factor);

5823 }

5824

5826

5827 auto BuildSDIVPattern = [&](const Constant *C) {

5829 const APInt &Divisor = CI->getValue();

5830

5833 int NumeratorFactor = 0;

5834 int ShiftMask = -1;

5835

5837

5839 Magics.Magic = 0;

5841 ShiftMask = 0;

5843

5844 NumeratorFactor = 1;

5846

5847 NumeratorFactor = -1;

5848 }

5849

5850 MagicFactors.push_back(MIB.buildConstant(ScalarTy, Magics.Magic).getReg(0));

5851 Factors.push_back(MIB.buildConstant(ScalarTy, NumeratorFactor).getReg(0));

5853 MIB.buildConstant(ScalarShiftAmtTy, Magics.ShiftAmount).getReg(0));

5854 ShiftMasks.push_back(MIB.buildConstant(ScalarTy, ShiftMask).getReg(0));

5855

5856 return true;

5857 };

5858

5859

5861 (void)Matched;

5862 assert(Matched && "Expected unary predicate match to succeed");

5863

5864 Register MagicFactor, Factor, Shift, ShiftMask;

5866 if (RHSDef) {

5867 MagicFactor = MIB.buildBuildVector(Ty, MagicFactors).getReg(0);

5868 Factor = MIB.buildBuildVector(Ty, Factors).getReg(0);

5869 Shift = MIB.buildBuildVector(ShiftAmtTy, Shifts).getReg(0);

5870 ShiftMask = MIB.buildBuildVector(Ty, ShiftMasks).getReg(0);

5871 } else {

5872 assert(MRI.getType(RHS).isScalar() &&

5873 "Non-build_vector operation should have been a scalar");

5874 MagicFactor = MagicFactors[0];

5875 Factor = Factors[0];

5876 Shift = Shifts[0];

5877 ShiftMask = ShiftMasks[0];

5878 }

5879

5881 Q = MIB.buildSMulH(Ty, LHS, MagicFactor).getReg(0);

5882

5883

5884 Factor = MIB.buildMul(Ty, LHS, Factor).getReg(0);

5885 Q = MIB.buildAdd(Ty, Q, Factor).getReg(0);

5886

5887

5888 Q = MIB.buildAShr(Ty, Q, Shift).getReg(0);

5889

5890

5891 auto SignShift = MIB.buildConstant(ShiftAmtTy, EltBits - 1);

5892 auto T = MIB.buildLShr(Ty, Q, SignShift);

5893 T = MIB.buildAnd(Ty, T, ShiftMask);

5894 auto ret = MIB.buildAdd(Ty, Q, T);

5895

5896 if (Opcode == TargetOpcode::G_SREM) {

5897 auto Prod = MIB.buildMul(Ty, ret, RHS);

5898 return MIB.buildSub(Ty, LHS, Prod);

5899 }

5900 return ret;

5901}

5902

5904 assert((MI.getOpcode() == TargetOpcode::G_SDIV ||

5905 MI.getOpcode() == TargetOpcode::G_UDIV) &&

5906 "Expected SDIV or UDIV");

5908 Register RHS = Div.getReg(2);

5909 auto MatchPow2 = [&](const Constant *C) {

5911 return CI && (CI->getValue().isPowerOf2() ||

5912 (IsSigned && CI->getValue().isNegatedPowerOf2()));

5913 };

5915}

5916

5918 assert(MI.getOpcode() == TargetOpcode::G_SDIV && "Expected SDIV");

5920 Register Dst = SDiv.getReg(0);

5921 Register LHS = SDiv.getReg(1);

5922 Register RHS = SDiv.getReg(2);

5923 LLT Ty = MRI.getType(Dst);

5925 LLT CCVT =

5927

5928

5929

5930

5931

5932

5933

5934

5935

5936

5937

5938

5939

5940

5941

5942

5943 unsigned BitWidth = Ty.getScalarSizeInBits();

5944 auto Zero = Builder.buildConstant(Ty, 0);

5945

5946 auto Bits = Builder.buildConstant(ShiftAmtTy, BitWidth);

5947 auto C1 = Builder.buildCTTZ(ShiftAmtTy, RHS);

5948 auto Inexact = Builder.buildSub(ShiftAmtTy, Bits, C1);

5949

5950 auto Sign = Builder.buildAShr(

5951 Ty, LHS, Builder.buildConstant(ShiftAmtTy, BitWidth - 1));

5952

5953

5954 auto LSrl = Builder.buildLShr(Ty, Sign, Inexact);

5955 auto Add = Builder.buildAdd(Ty, LHS, LSrl);

5956 auto AShr = Builder.buildAShr(Ty, Add, C1);

5957

5958

5959

5960 auto One = Builder.buildConstant(Ty, 1);

5961 auto MinusOne = Builder.buildConstant(Ty, -1);

5963 auto IsMinusOne =

5965 auto IsOneOrMinusOne = Builder.buildOr(CCVT, IsOne, IsMinusOne);

5966 AShr = Builder.buildSelect(Ty, IsOneOrMinusOne, LHS, AShr);

5967

5968

5969

5970 auto Neg = Builder.buildNeg(Ty, AShr);

5972 Builder.buildSelect(MI.getOperand(0).getReg(), IsNeg, Neg, AShr);

5973 MI.eraseFromParent();

5974}

5975

5977 assert(MI.getOpcode() == TargetOpcode::G_UDIV && "Expected UDIV");

5979 Register Dst = UDiv.getReg(0);

5980 Register LHS = UDiv.getReg(1);

5981 Register RHS = UDiv.getReg(2);

5982 LLT Ty = MRI.getType(Dst);

5984

5985 auto C1 = Builder.buildCTTZ(ShiftAmtTy, RHS);

5986 Builder.buildLShr(MI.getOperand(0).getReg(), LHS, C1);

5987 MI.eraseFromParent();

5988}

5989

5991 assert(MI.getOpcode() == TargetOpcode::G_UMULH);

5992 Register RHS = MI.getOperand(2).getReg();

5993 Register Dst = MI.getOperand(0).getReg();

5994 LLT Ty = MRI.getType(Dst);

5995 LLT RHSTy = MRI.getType(RHS);

5997 auto MatchPow2ExceptOne = [&](const Constant *C) {

5999 return CI->getValue().isPowerOf2() && !CI->getValue().isOne();

6000 return false;

6001 };

6003 return false;

6004

6005

6008}

6009

6011 Register LHS = MI.getOperand(1).getReg();

6012 Register RHS = MI.getOperand(2).getReg();

6013 Register Dst = MI.getOperand(0).getReg();

6014 LLT Ty = MRI.getType(Dst);

6017

6019 auto ShiftAmt =

6020 Builder.buildSub(Ty, Builder.buildConstant(Ty, NumEltBits), LogBase2);

6021 auto Trunc = Builder.buildZExtOrTrunc(ShiftAmtTy, ShiftAmt);

6022 Builder.buildLShr(Dst, LHS, Trunc);

6023 MI.eraseFromParent();

6024}

6025

6027 Register &MatchInfo) const {

6028 Register Dst = MI.getOperand(0).getReg();

6029 Register Src = MI.getOperand(1).getReg();

6030 LLT DstTy = MRI.getType(Dst);

6031 LLT SrcTy = MRI.getType(Src);

6033 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();

6034 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");

6035

6036 if ( || isLegal ({TargetOpcode::G_TRUNC_SSAT_S, {DstTy, SrcTy}}))

6037 return false;

6038

6049}

6050

6052 Register &MatchInfo) const {

6053 Register Dst = MI.getOperand(0).getReg();

6054 Builder.buildTruncSSatS(Dst, MatchInfo);

6055 MI.eraseFromParent();

6056}

6057

6059 Register &MatchInfo) const {

6060 Register Dst = MI.getOperand(0).getReg();

6061 Register Src = MI.getOperand(1).getReg();

6062 LLT DstTy = MRI.getType(Dst);

6063 LLT SrcTy = MRI.getType(Src);

6065 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();

6066 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");

6067

6068 if ( || isLegal ({TargetOpcode::G_TRUNC_SSAT_U, {DstTy, SrcTy}}))

6069 return false;

6081}

6082

6084 Register &MatchInfo) const {

6085 Register Dst = MI.getOperand(0).getReg();

6086 Builder.buildTruncSSatU(Dst, MatchInfo);

6087 MI.eraseFromParent();

6088}

6089

6094 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());

6095 LLT SrcTy = MRI.getType(Val);

6097 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();

6098 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");

6099

6100 if ( || isLegal ({TargetOpcode::G_TRUNC_SSAT_U, {DstTy, SrcTy}}))

6101 return false;

6105}

6106

6109 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());

6111

6112 return LI &&

6114}

6115

6118 unsigned Opc = MI.getOpcode();

6119 assert(Opc == TargetOpcode::G_FADD || Opc == TargetOpcode::G_FSUB ||

6120 Opc == TargetOpcode::G_FMUL || Opc == TargetOpcode::G_FDIV ||

6121 Opc == TargetOpcode::G_FMAD || Opc == TargetOpcode::G_FMA);

6122

6123 Register Dst = MI.getOperand(0).getReg();

6127

6128

6129

6130

6133 Opc = TargetOpcode::G_FSUB;

6134 }

6135

6138 Opc = TargetOpcode::G_FADD;

6139 }

6140

6141

6142

6143

6144 else if ((Opc == TargetOpcode::G_FMUL || Opc == TargetOpcode::G_FDIV ||

6145 Opc == TargetOpcode::G_FMAD || Opc == TargetOpcode::G_FMA) &&

6148

6149 } else

6150 return false;

6151

6154 MI.setDesc(B.getTII().get(Opc));

6155 MI.getOperand(1).setReg(X);

6156 MI.getOperand(2).setReg(Y);

6158 };

6159 return true;

6160}

6161

6163 Register &MatchInfo) const {

6164 assert(MI.getOpcode() == TargetOpcode::G_FSUB);

6165

6166 Register LHS = MI.getOperand(1).getReg();

6167 MatchInfo = MI.getOperand(2).getReg();

6168 LLT Ty = MRI.getType(MI.getOperand(0).getReg());

6169

6170 const auto LHSCst = Ty.isVector()

6173 if (!LHSCst)

6174 return false;

6175

6176

6177 if (LHSCst->Value.isNegZero())

6178 return true;

6179

6180

6181 if (LHSCst->Value.isPosZero())

6183

6184 return false;

6185}

6186

6188 Register &MatchInfo) const {

6189 Register Dst = MI.getOperand(0).getReg();

6191 Dst, Builder.buildFCanonicalize(MRI.getType(Dst), MatchInfo).getReg(0));

6193}

6194

6195

6196

6198 if (MI.getOpcode() != TargetOpcode::G_FMUL)

6199 return false;

6201}

6202

6205 return std::distance(MRI.use_instr_nodbg_begin(MI0.getOperand(0).getReg()),

6206 MRI.use_instr_nodbg_end()) >

6208 MRI.use_instr_nodbg_end());

6209}

6210

6212 bool &AllowFusionGlobally,

6214 bool CanReassociate) const {

6215

6216 auto *MF = MI.getMF();

6217 const auto &TLI = *MF->getSubtarget().getTargetLowering();

6219 LLT DstType = MRI.getType(MI.getOperand(0).getReg());

6220

6222 return false;

6223

6224

6225 HasFMAD = ( isPreLegalize () && TLI.isFMADLegal(MI, DstType));

6226

6227 bool HasFMA = TLI.isFMAFasterThanFMulAndFAdd(*MF, DstType) &&

6229

6230 if (!HasFMAD && !HasFMA)

6231 return false;

6232

6234

6236 return false;

6237

6238 Aggressive = TLI.enableAggressiveFMAFusion(DstType);

6239 return true;

6240}

6241

6245 assert(MI.getOpcode() == TargetOpcode::G_FADD);

6246

6247 bool AllowFusionGlobally, HasFMAD, Aggressive;

6249 return false;

6250

6251 Register Op1 = MI.getOperand(1).getReg();

6252 Register Op2 = MI.getOperand(2).getReg();

6255 unsigned PreferredFusedOpcode =

6256 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;

6257

6258

6259

6264 }

6265

6266

6270 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},

6271 {LHS.MI->getOperand(1).getReg(),

6272 LHS.MI->getOperand(2).getReg(), RHS.Reg});

6273 };

6274 return true;

6275 }

6276

6277

6281 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},

6282 {RHS.MI->getOperand(1).getReg(),

6283 RHS.MI->getOperand(2).getReg(), LHS.Reg});

6284 };

6285 return true;

6286 }

6287

6288 return false;

6289}

6290

6294 assert(MI.getOpcode() == TargetOpcode::G_FADD);

6295

6296 bool AllowFusionGlobally, HasFMAD, Aggressive;

6298 return false;

6299

6300 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();

6301 Register Op1 = MI.getOperand(1).getReg();

6302 Register Op2 = MI.getOperand(2).getReg();

6305 LLT DstType = MRI.getType(MI.getOperand(0).getReg());

6306

6307 unsigned PreferredFusedOpcode =

6308 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;

6309

6310

6311

6316 }

6317

6318

6322 TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,

6325 auto FpExtX = B.buildFPExt(DstType, FpExtSrc->getOperand(1).getReg());

6326 auto FpExtY = B.buildFPExt(DstType, FpExtSrc->getOperand(2).getReg());

6327 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},

6328 {FpExtX.getReg(0), FpExtY.getReg(0), RHS.Reg});

6329 };

6330 return true;

6331 }

6332

6333

6334

6337 TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,

6340 auto FpExtX = B.buildFPExt(DstType, FpExtSrc->getOperand(1).getReg());

6341 auto FpExtY = B.buildFPExt(DstType, FpExtSrc->getOperand(2).getReg());

6342 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},

6343 {FpExtX.getReg(0), FpExtY.getReg(0), LHS.Reg});

6344 };

6345 return true;

6346 }

6347

6348 return false;

6349}

6350

6354 assert(MI.getOpcode() == TargetOpcode::G_FADD);

6355

6356 bool AllowFusionGlobally, HasFMAD, Aggressive;

6358 return false;

6359

6360 Register Op1 = MI.getOperand(1).getReg();

6361 Register Op2 = MI.getOperand(2).getReg();

6364 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());

6365

6366 unsigned PreferredFusedOpcode =

6367 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;

6368

6369

6370

6375 }

6376

6379

6380 if (LHS.MI->getOpcode() == PreferredFusedOpcode &&

6381 (MRI.getVRegDef(LHS.MI->getOperand(3).getReg())->getOpcode() ==

6382 TargetOpcode::G_FMUL) &&

6383 MRI.hasOneNonDBGUse(LHS.MI->getOperand(0).getReg()) &&

6384 MRI.hasOneNonDBGUse(LHS.MI->getOperand(3).getReg())) {

6385 FMA = LHS.MI;

6386 Z = RHS.Reg;

6387 }

6388

6389 else if (RHS.MI->getOpcode() == PreferredFusedOpcode &&

6390 (MRI.getVRegDef(RHS.MI->getOperand(3).getReg())->getOpcode() ==

6391 TargetOpcode::G_FMUL) &&

6392 MRI.hasOneNonDBGUse(RHS.MI->getOperand(0).getReg()) &&

6393 MRI.hasOneNonDBGUse(RHS.MI->getOperand(3).getReg())) {

6394 Z = LHS.Reg;

6395 FMA = RHS.MI;

6396 }

6397

6398 if (FMA) {

6399 MachineInstr *FMulMI = MRI.getVRegDef(FMA->getOperand(3).getReg());

6400 Register X = FMA->getOperand(1).getReg();

6401 Register Y = FMA->getOperand(2).getReg();

6404

6406 Register InnerFMA = MRI.createGenericVirtualRegister(DstTy);

6407 B.buildInstr(PreferredFusedOpcode, {InnerFMA}, {U, V, Z});

6408 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},

6409 {X, Y, InnerFMA});

6410 };

6411 return true;

6412 }

6413

6414 return false;

6415}

6416

6420 assert(MI.getOpcode() == TargetOpcode::G_FADD);

6421

6422 bool AllowFusionGlobally, HasFMAD, Aggressive;

6424 return false;

6425

6427 return false;

6428

6429 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();

6430 LLT DstType = MRI.getType(MI.getOperand(0).getReg());

6431 Register Op1 = MI.getOperand(1).getReg();

6432 Register Op2 = MI.getOperand(2).getReg();

6435

6436 unsigned PreferredFusedOpcode =

6437 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;

6438

6439

6440

6445 }

6446

6447

6450 Register FpExtU = B.buildFPExt(DstType, U).getReg(0);

6451 Register FpExtV = B.buildFPExt(DstType, V).getReg(0);

6453 B.buildInstr(PreferredFusedOpcode, {DstType}, {FpExtU, FpExtV, Z})

6455 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},

6456 {X, Y, InnerFMA});

6457 };

6458

6460

6461

6462 if (LHS.MI->getOpcode() == PreferredFusedOpcode &&

6463 mi_match(LHS.MI->getOperand(3).getReg(), MRI,

6466 TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,

6471 LHS.MI->getOperand(1).getReg(),

6472 LHS.MI->getOperand(2).getReg(), B);

6473 };

6474 return true;

6475 }

6476

6477

6478

6479

6480

6481

6483 FMAMI->getOpcode() == PreferredFusedOpcode) {

6486 TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,

6491 X = B.buildFPExt(DstType, X).getReg(0);

6492 Y = B.buildFPExt(DstType, Y).getReg(0);

6495 };

6496

6497 return true;

6498 }

6499 }

6500

6501

6502

6503 if (RHS.MI->getOpcode() == PreferredFusedOpcode &&

6504 mi_match(RHS.MI->getOperand(3).getReg(), MRI,

6507 TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,

6512 RHS.MI->getOperand(1).getReg(),

6513 RHS.MI->getOperand(2).getReg(), B);

6514 };

6515 return true;

6516 }

6517

6518

6519

6520

6521

6522

6524 FMAMI->getOpcode() == PreferredFusedOpcode) {

6527 TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstType,

6532 X = B.buildFPExt(DstType, X).getReg(0);

6533 Y = B.buildFPExt(DstType, Y).getReg(0);

6536 };

6537 return true;

6538 }

6539 }

6540

6541 return false;

6542}

6543

6547 assert(MI.getOpcode() == TargetOpcode::G_FSUB);

6548

6549 bool AllowFusionGlobally, HasFMAD, Aggressive;

6551 return false;

6552

6553 Register Op1 = MI.getOperand(1).getReg();

6554 Register Op2 = MI.getOperand(2).getReg();

6557 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());

6558

6559

6560

6561 int FirstMulHasFewerUses = true;

6565 FirstMulHasFewerUses = false;

6566

6567 unsigned PreferredFusedOpcode =

6568 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;

6569

6570

6571 if (FirstMulHasFewerUses &&

6573 (Aggressive || MRI.hasOneNonDBGUse(LHS.Reg)))) {

6575 Register NegZ = B.buildFNeg(DstTy, RHS.Reg).getReg(0);

6576 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},

6577 {LHS.MI->getOperand(1).getReg(),

6578 LHS.MI->getOperand(2).getReg(), NegZ});

6579 };

6580 return true;

6581 }

6582

6584 (Aggressive || MRI.hasOneNonDBGUse(RHS.Reg)))) {

6587 B.buildFNeg(DstTy, RHS.MI->getOperand(1).getReg()).getReg(0);

6588 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},

6589 {NegY, RHS.MI->getOperand(2).getReg(), LHS.Reg});

6590 };

6591 return true;

6592 }

6593

6594 return false;

6595}

6596

6600 assert(MI.getOpcode() == TargetOpcode::G_FSUB);

6601

6602 bool AllowFusionGlobally, HasFMAD, Aggressive;

6604 return false;

6605

6606 Register LHSReg = MI.getOperand(1).getReg();

6607 Register RHSReg = MI.getOperand(2).getReg();

6608 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());

6609

6610 unsigned PreferredFusedOpcode =

6611 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;

6612

6614

6622 Register NegZ = B.buildFNeg(DstTy, RHSReg).getReg(0);

6623 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},

6625 };

6626 return true;

6627 }

6628

6629

6635 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},

6638 };

6639 return true;

6640 }

6641

6642 return false;

6643}

6644

6648 assert(MI.getOpcode() == TargetOpcode::G_FSUB);

6649

6650 bool AllowFusionGlobally, HasFMAD, Aggressive;

6652 return false;

6653

6654 Register LHSReg = MI.getOperand(1).getReg();

6655 Register RHSReg = MI.getOperand(2).getReg();

6656 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());

6657

6658 unsigned PreferredFusedOpcode =

6659 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;

6660

6662

6671 Register NegZ = B.buildFNeg(DstTy, RHSReg).getReg(0);

6672 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},

6673 {FpExtX, FpExtY, NegZ});

6674 };

6675 return true;

6676 }

6677

6678

6685 Register NegY = B.buildFNeg(DstTy, FpExtY).getReg(0);

6688 B.buildInstr(PreferredFusedOpcode, {MI.getOperand(0).getReg()},

6689 {NegY, FpExtZ, LHSReg});

6690 };

6691 return true;

6692 }

6693

6694 return false;

6695}

6696

6700 assert(MI.getOpcode() == TargetOpcode::G_FSUB);

6701

6702 bool AllowFusionGlobally, HasFMAD, Aggressive;

6704 return false;

6705

6706 const auto &TLI = *MI.getMF()->getSubtarget().getTargetLowering();

6707 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());

6708 Register LHSReg = MI.getOperand(1).getReg();

6709 Register RHSReg = MI.getOperand(2).getReg();

6710

6711 unsigned PreferredFusedOpcode =

6712 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;

6713

6716 Register FpExtX = B.buildFPExt(DstTy, X).getReg(0);

6717 Register FpExtY = B.buildFPExt(DstTy, Y).getReg(0);

6718 B.buildInstr(PreferredFusedOpcode, {Dst}, {FpExtX, FpExtY, Z});

6719 };

6720

6722

6723

6724

6725

6729 TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstTy,

6732 Register FMAReg = MRI.createGenericVirtualRegister(DstTy);

6735 B.buildFNeg(MI.getOperand(0).getReg(), FMAReg);

6736 };

6737 return true;

6738 }

6739

6740

6741

6745 TLI.isFPExtFoldable(MI, PreferredFusedOpcode, DstTy,

6748 buildMatchInfo(MI.getOperand(0).getReg(), FMulMI->getOperand(1).getReg(),

6750 };

6751 return true;

6752 }

6753

6754 return false;

6755}

6756

6758 unsigned &IdxToPropagate) const {

6759 bool PropagateNaN;

6760 switch (MI.getOpcode()) {

6761 default:

6762 return false;

6763 case TargetOpcode::G_FMINNUM:

6764 case TargetOpcode::G_FMAXNUM:

6765 PropagateNaN = false;

6766 break;

6767 case TargetOpcode::G_FMINIMUM:

6768 case TargetOpcode::G_FMAXIMUM:

6769 PropagateNaN = true;

6770 break;

6771 }

6772

6773 auto MatchNaN = [&](unsigned Idx) {

6774 Register MaybeNaNReg = MI.getOperand(Idx).getReg();

6777 return false;

6778 IdxToPropagate = PropagateNaN ? Idx : (Idx == 1 ? 2 : 1);

6779 return true;

6780 };

6781

6782 return MatchNaN(1) || MatchNaN(2);

6783}

6784

6785

6786

6787

6790 assert(MI.getOpcode() == TargetOpcode::G_FDIV);

6791

6794

6796 return false;

6797

6798

6800 if (N0CFP && (N0CFP->isExactlyValue(1.0) || N0CFP->isExactlyValue(-1.0)))

6801 return false;

6802

6803

6804

6806 if (!MinUses)

6807 return false;

6808

6809

6810

6811

6813 for (auto &U : MRI.use_nodbg_instructions(Y)) {

6814 if (&U == &MI || U.getParent() != MI.getParent())

6815 continue;

6816 if (U.getOpcode() == TargetOpcode::G_FDIV &&

6817 U.getOperand(2).getReg() == Y && U.getOperand(1).getReg() != Y) {

6818

6819

6822 if (dominates(U, *MatchInfo[0]))

6824 }

6825 }

6826 }

6827

6828

6829

6830 return MatchInfo.size() >= MinUses;

6831}

6832

6835

6836

6837 Builder.setInsertPt(*MatchInfo[0]->getParent(), MatchInfo[0]);

6838 LLT Ty = MRI.getType(MatchInfo[0]->getOperand(0).getReg());

6839 auto Div = Builder.buildFDiv(Ty, Builder.buildFConstant(Ty, 1.0),

6840 MatchInfo[0]->getOperand(2).getReg(),

6841 MatchInfo[0]->getFlags());

6842

6843

6845 Builder.setInsertPt(*MI->getParent(), MI);

6846 Builder.buildFMul(MI->getOperand(0).getReg(), MI->getOperand(1).getReg(),

6847 Div->getOperand(0).getReg(), MI->getFlags());

6848 MI->eraseFromParent();

6849 }

6850}

6851

6853 assert(MI.getOpcode() == TargetOpcode::G_ADD && "Expected a G_ADD");

6854 Register LHS = MI.getOperand(1).getReg();

6855 Register RHS = MI.getOperand(2).getReg();

6856

6857

6858

6859

6860 auto CheckFold = [&](Register MaybeSub, Register MaybeSameReg) {

6863 Reg == MaybeSameReg;

6864 };

6865 return CheckFold(LHS, RHS) || CheckFold(RHS, LHS);

6866}

6867

6869 Register &MatchInfo) const {

6870

6871

6872

6873

6874

6875

6876

6877

6878

6879

6880

6881

6882

6883

6884

6885

6886 LLT DstVecTy = MRI.getType(MI.getOperand(0).getReg());

6888

6890

6894 MatchInfo = Lo;

6895 return MRI.getType(MatchInfo) == DstVecTy;

6896 }

6897

6898 std::optional ShiftAmount;

6905 if (Lo == Hi && ShiftAmount->Value == DstEltTy.getSizeInBits()) {

6906 MatchInfo = Lo;

6907 return MRI.getType(MatchInfo) == DstVecTy;

6908 }

6909 }

6910

6911 return false;

6912}

6913

6915 Register &MatchInfo) const {

6916

6917

6920 return false;

6921

6922 return MRI.getType(MatchInfo) == MRI.getType(MI.getOperand(0).getReg());

6923}

6924

6926 Register &MatchInfo) const {

6927

6928

6929 std::optional ShiftAmt;

6933 return false;

6934

6935 LLT MatchTy = MRI.getType(MatchInfo);

6936 return ShiftAmt->Value.getZExtValue() == MatchTy.getSizeInBits() &&

6937 MatchTy == MRI.getType(MI.getOperand(0).getReg());

6938}

6939

6940unsigned CombinerHelper::getFPMinMaxOpcForSelect(

6942 SelectPatternNaNBehaviour VsNaNRetVal) const {

6943 assert(VsNaNRetVal != SelectPatternNaNBehaviour::NOT_APPLICABLE &&

6944 "Expected a NaN behaviour?");

6945

6946

6947 switch (Pred) {

6948 default:

6949 return 0;

6954 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)

6955 return TargetOpcode::G_FMAXNUM;

6956 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)

6957 return TargetOpcode::G_FMAXIMUM;

6958 if (isLegal({TargetOpcode::G_FMAXNUM, {DstTy}}))

6959 return TargetOpcode::G_FMAXNUM;

6960 if (isLegal({TargetOpcode::G_FMAXIMUM, {DstTy}}))

6961 return TargetOpcode::G_FMAXIMUM;

6962 return 0;

6967 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)

6968 return TargetOpcode::G_FMINNUM;

6969 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)

6970 return TargetOpcode::G_FMINIMUM;

6971 if (isLegal({TargetOpcode::G_FMINNUM, {DstTy}}))

6972 return TargetOpcode::G_FMINNUM;

6973 if ( isLegal ({TargetOpcode::G_FMINIMUM, {DstTy}}))

6974 return 0;

6975 return TargetOpcode::G_FMINIMUM;

6976 }

6977}

6978

6979CombinerHelper::SelectPatternNaNBehaviour

6981 bool IsOrderedComparison) const {

6984

6985 if (!LHSSafe && !RHSSafe)

6986 return SelectPatternNaNBehaviour::NOT_APPLICABLE;

6987 if (LHSSafe && RHSSafe)

6988 return SelectPatternNaNBehaviour::RETURNS_ANY;

6989

6990

6991 if (IsOrderedComparison)

6992 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_NAN

6993 : SelectPatternNaNBehaviour::RETURNS_OTHER;

6994

6995

6996 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_OTHER

6997 : SelectPatternNaNBehaviour::RETURNS_NAN;

6998}

6999

7003

7004

7005

7006 LLT DstTy = MRI.getType(Dst);

7007

7009 return false;

7010

7011

7018 return false;

7019 SelectPatternNaNBehaviour ResWithKnownNaNInfo =

7021 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::NOT_APPLICABLE)

7022 return false;

7023 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {

7026 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_NAN)

7027 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_OTHER;

7028 else if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_OTHER)

7029 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_NAN;

7030 }

7031 if (TrueVal != CmpLHS || FalseVal != CmpRHS)

7032 return false;

7033

7034 unsigned Opc = getFPMinMaxOpcForSelect(Pred, DstTy, ResWithKnownNaNInfo);

7036 return false;

7037

7038

7039 if (Opc != TargetOpcode::G_FMAXIMUM && Opc != TargetOpcode::G_FMINIMUM) {

7040

7041

7042

7044 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero()) {

7046 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero())

7047 return false;

7048 }

7049 }

7050 MatchInfo = [=](MachineIRBuilder &B) {

7051 B.buildInstr(Opc, {Dst}, {CmpLHS, CmpRHS});

7052 };

7053 return true;

7054}

7055

7058

7059 assert(MI.getOpcode() == TargetOpcode::G_SELECT);

7060

7064 Cond = MaybeTrunc;

7065 Register Dst = MI.getOperand(0).getReg();

7066 Register TrueVal = MI.getOperand(2).getReg();

7067 Register FalseVal = MI.getOperand(3).getReg();

7068 return matchFPSelectToMinMax(Dst, Cond, TrueVal, FalseVal, MatchInfo);

7069}

7070

7073 assert(MI.getOpcode() == TargetOpcode::G_ICMP);

7074

7075

7076

7077

7078

7079

7080 Register Dst = MI.getOperand(0).getReg();

7084 Dst, MRI,

7086 if (MatchedSub && X != OpLHS)

7087 return false;

7088 if (!MatchedSub) {

7093 return false;

7094 Y = X == OpLHS ? OpRHS : X == OpRHS ? OpLHS : Register();

7095 }

7097 auto Zero = B.buildConstant(MRI.getType(Y), 0);

7098 B.buildICmp(Pred, Dst, Y, Zero);

7099 };

7101}

7102

7103

7104

7105static std::optional

7107 std::optional<int64_t> &Result) {

7108 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR ||

7109 Opcode == TargetOpcode::G_ASHR) &&

7110 "Expect G_SHL, G_LSHR or G_ASHR.");

7111 auto SignificantBits = 0;

7112 switch (Opcode) {

7113 case TargetOpcode::G_SHL:

7115 Result = 0;

7116 break;

7117 case TargetOpcode::G_LSHR:

7118 Result = 0;

7120 break;

7121 case TargetOpcode::G_ASHR:

7124 Result = 0;

7127 Result = -1;

7128 } else {

7129

7130 Result = std::nullopt;

7131 }

7132 break;

7133 default:

7134 break;

7135 }

7136 return ValueKB.getBitWidth() - SignificantBits;

7137}

7138

7140 MachineInstr &MI, std::optional<int64_t> &MatchInfo) const {

7141 Register ShiftVal = MI.getOperand(1).getReg();

7142 Register ShiftReg = MI.getOperand(2).getReg();

7143 LLT ResTy = MRI.getType(MI.getOperand(0).getReg());

7144 auto IsShiftTooBig = [&](const Constant *C) {

7146 if (!CI)

7147 return false;

7149 MatchInfo = std::nullopt;

7150 return true;

7151 }

7153 MI.getOpcode(), MatchInfo);

7154 return OptMaxUsefulShift && CI->uge(*OptMaxUsefulShift);

7155 };

7157}

7158

7160 unsigned LHSOpndIdx = 1;

7161 unsigned RHSOpndIdx = 2;

7162 switch (MI.getOpcode()) {

7163 case TargetOpcode::G_UADDO:

7164 case TargetOpcode::G_SADDO:

7165 case TargetOpcode::G_UMULO:

7166 case TargetOpcode::G_SMULO:

7167 LHSOpndIdx = 2;

7168 RHSOpndIdx = 3;

7169 break;

7170 default:

7171 break;

7172 }

7173 Register LHS = MI.getOperand(LHSOpndIdx).getReg();

7174 Register RHS = MI.getOperand(RHSOpndIdx).getReg();

7176

7177

7178

7179 if (MRI.getVRegDef(LHS)->getOpcode() !=

7180 TargetOpcode::G_CONSTANT_FOLD_BARRIER)

7181 return false;

7182 }

7183

7184 return MRI.getVRegDef(RHS)->getOpcode() !=

7185 TargetOpcode::G_CONSTANT_FOLD_BARRIER &&

7187}

7188

7190 Register LHS = MI.getOperand(1).getReg();

7191 Register RHS = MI.getOperand(2).getReg();

7192 std::optional ValAndVReg;

7194 return false;

7196}

7197

7200 unsigned LHSOpndIdx = 1;

7201 unsigned RHSOpndIdx = 2;

7202 switch (MI.getOpcode()) {

7203 case TargetOpcode::G_UADDO:

7204 case TargetOpcode::G_SADDO:

7205 case TargetOpcode::G_UMULO:

7206 case TargetOpcode::G_SMULO:

7207 LHSOpndIdx = 2;

7208 RHSOpndIdx = 3;

7209 break;

7210 default:

7211 break;

7212 }

7213 Register LHSReg = MI.getOperand(LHSOpndIdx).getReg();

7214 Register RHSReg = MI.getOperand(RHSOpndIdx).getReg();

7215 MI.getOperand(LHSOpndIdx).setReg(RHSReg);

7216 MI.getOperand(RHSOpndIdx).setReg(LHSReg);

7218}

7219

7220bool CombinerHelper::isOneOrOneSplat(Register Src, bool AllowUndefs) const {

7221 LLT SrcTy = MRI.getType(Src);

7222 if (SrcTy.isFixedVector())

7223 return isConstantSplatVector(Src, 1, AllowUndefs);

7224 if (SrcTy.isScalar()) {

7226 return true;

7228 return IConstant && IConstant->Value == 1;

7229 }

7230 return false;

7231}

7232

7233bool CombinerHelper::isZeroOrZeroSplat(Register Src, bool AllowUndefs) const {

7234 LLT SrcTy = MRI.getType(Src);

7236 return isConstantSplatVector(Src, 0, AllowUndefs);

7239 return true;

7241 return IConstant && IConstant->Value == 0;

7242 }

7243 return false;

7244}

7245

7246

7247

7248bool CombinerHelper::isConstantSplatVector(Register Src, int64_t SplatValue,

7249 bool AllowUndefs) const {

7251 if (!BuildVector)

7252 return false;

7253 unsigned NumSources = BuildVector->getNumSources();

7254

7255 for (unsigned I = 0; I < NumSources; ++I) {

7256 GImplicitDef *ImplicitDef =

7258 if (ImplicitDef && AllowUndefs)

7259 continue;

7260 if (ImplicitDef && !AllowUndefs)

7261 return false;

7262 std::optional IConstant =

7264 if (IConstant && IConstant->Value == SplatValue)

7265 continue;

7266 return false;

7267 }

7268 return true;

7269}

7270

7271

7272

7273std::optional

7274CombinerHelper::getConstantOrConstantSplatVector(Register Src) const {

7276 if (IConstant)

7277 return IConstant->Value;

7278

7280 if (!BuildVector)

7281 return std::nullopt;

7282 unsigned NumSources = BuildVector->getNumSources();

7283

7284 std::optional Value = std::nullopt;

7285 for (unsigned I = 0; I < NumSources; ++I) {

7286 std::optional IConstant =

7288 if (!IConstant)

7289 return std::nullopt;

7291 Value = IConstant->Value;

7292 else if (*Value != IConstant->Value)

7293 return std::nullopt;

7294 }

7296}

7297

7298

7299bool CombinerHelper::isConstantOrConstantVectorI(Register Src) const {

7301 if (IConstant)

7302 return true;

7303

7305 if (!BuildVector)

7306 return false;

7307

7308 unsigned NumSources = BuildVector->getNumSources();

7309 for (unsigned I = 0; I < NumSources; ++I) {

7310 std::optional IConstant =

7312 if (!IConstant)

7313 return false;

7314 }

7315 return true;

7316}

7317

7318

7319bool CombinerHelper::tryFoldSelectOfConstants(GSelect *Select,

7326 LLT CondTy = MRI.getType(Select->getCondReg());

7327 LLT TrueTy = MRI.getType(Select->getTrueReg());

7328

7329

7331 return false;

7332

7334 return false;

7335

7336

7337 std::optional TrueOpt =

7339 std::optional FalseOpt =

7341

7342 if (!TrueOpt || !FalseOpt)

7343 return false;

7344

7345 APInt TrueValue = TrueOpt->Value;

7346 APInt FalseValue = FalseOpt->Value;

7347

7348

7349 if (TrueValue.isOne() && FalseValue.isZero()) {

7350 MatchInfo = [=](MachineIRBuilder &B) {

7351 B.setInstrAndDebugLoc(*Select);

7352 B.buildZExtOrTrunc(Dest, Cond);

7353 };

7354 return true;

7355 }

7356

7357

7359 MatchInfo = [=](MachineIRBuilder &B) {

7360 B.setInstrAndDebugLoc(*Select);

7361 B.buildSExtOrTrunc(Dest, Cond);

7362 };

7363 return true;

7364 }

7365

7366

7367 if (TrueValue.isZero() && FalseValue.isOne()) {

7368 MatchInfo = [=](MachineIRBuilder &B) {

7369 B.setInstrAndDebugLoc(*Select);

7370 Register Inner = MRI.createGenericVirtualRegister(CondTy);

7371 B.buildNot(Inner, Cond);

7372 B.buildZExtOrTrunc(Dest, Inner);

7373 };

7374 return true;

7375 }

7376

7377

7379 MatchInfo = [=](MachineIRBuilder &B) {

7380 B.setInstrAndDebugLoc(*Select);

7381 Register Inner = MRI.createGenericVirtualRegister(CondTy);

7382 B.buildNot(Inner, Cond);

7383 B.buildSExtOrTrunc(Dest, Inner);

7384 };

7385 return true;

7386 }

7387

7388

7389 if (TrueValue - 1 == FalseValue) {

7390 MatchInfo = [=](MachineIRBuilder &B) {

7391 B.setInstrAndDebugLoc(*Select);

7392 Register Inner = MRI.createGenericVirtualRegister(TrueTy);

7393 B.buildZExtOrTrunc(Inner, Cond);

7394 B.buildAdd(Dest, Inner, False);

7395 };

7396 return true;

7397 }

7398

7399

7400 if (TrueValue + 1 == FalseValue) {

7401 MatchInfo = [=](MachineIRBuilder &B) {

7402 B.setInstrAndDebugLoc(*Select);

7403 Register Inner = MRI.createGenericVirtualRegister(TrueTy);

7404 B.buildSExtOrTrunc(Inner, Cond);

7405 B.buildAdd(Dest, Inner, False);

7406 };

7407 return true;

7408 }

7409

7410

7412 MatchInfo = [=](MachineIRBuilder &B) {

7413 B.setInstrAndDebugLoc(*Select);

7414 Register Inner = MRI.createGenericVirtualRegister(TrueTy);

7415 B.buildZExtOrTrunc(Inner, Cond);

7416

7418 auto ShAmtC = B.buildConstant(ShiftTy, TrueValue.exactLogBase2());

7419 B.buildShl(Dest, Inner, ShAmtC, Flags);

7420 };

7421 return true;

7422 }

7423

7424

7426 MatchInfo = [=](MachineIRBuilder &B) {

7427 B.setInstrAndDebugLoc(*Select);

7428 Register Not = MRI.createGenericVirtualRegister(CondTy);

7429 B.buildNot(Not, Cond);

7430 Register Inner = MRI.createGenericVirtualRegister(TrueTy);

7431 B.buildZExtOrTrunc(Inner, Not);

7432

7434 auto ShAmtC = B.buildConstant(ShiftTy, FalseValue.exactLogBase2());

7435 B.buildShl(Dest, Inner, ShAmtC, Flags);

7436 };

7437 return true;

7438 }

7439

7440

7442 MatchInfo = [=](MachineIRBuilder &B) {

7443 B.setInstrAndDebugLoc(*Select);

7444 Register Inner = MRI.createGenericVirtualRegister(TrueTy);

7445 B.buildSExtOrTrunc(Inner, Cond);

7446 B.buildOr(Dest, Inner, False, Flags);

7447 };

7448 return true;

7449 }

7450

7451

7453 MatchInfo = [=](MachineIRBuilder &B) {

7454 B.setInstrAndDebugLoc(*Select);

7455 Register Not = MRI.createGenericVirtualRegister(CondTy);

7456 B.buildNot(Not, Cond);

7457 Register Inner = MRI.createGenericVirtualRegister(TrueTy);

7458 B.buildSExtOrTrunc(Inner, Not);

7459 B.buildOr(Dest, Inner, True, Flags);

7460 };

7461 return true;

7462 }

7463

7464 return false;

7465}

7466

7467

7468bool CombinerHelper::tryFoldBoolSelectToLogic(GSelect *Select,

7475 LLT CondTy = MRI.getType(Select->getCondReg());

7476 LLT TrueTy = MRI.getType(Select->getTrueReg());

7477

7478

7483 return false;

7484

7485 if (CondTy != TrueTy)

7486 return false;

7487

7488

7489

7490 if ((Cond == True) || isOneOrOneSplat(True, true)) {

7491 MatchInfo = [=](MachineIRBuilder &B) {

7492 B.setInstrAndDebugLoc(*Select);

7493 Register Ext = MRI.createGenericVirtualRegister(TrueTy);

7494 B.buildZExtOrTrunc(Ext, Cond);

7495 auto FreezeFalse = B.buildFreeze(TrueTy, False);

7496 B.buildOr(DstReg, Ext, FreezeFalse, Flags);

7497 };

7498 return true;

7499 }

7500

7501

7502

7503 if ((Cond == False) || isZeroOrZeroSplat(False, true)) {

7504 MatchInfo = [=](MachineIRBuilder &B) {

7505 B.setInstrAndDebugLoc(*Select);

7506 Register Ext = MRI.createGenericVirtualRegister(TrueTy);

7507 B.buildZExtOrTrunc(Ext, Cond);

7508 auto FreezeTrue = B.buildFreeze(TrueTy, True);

7509 B.buildAnd(DstReg, Ext, FreezeTrue);

7510 };

7511 return true;

7512 }

7513

7514

7515 if (isOneOrOneSplat(False, true)) {

7516 MatchInfo = [=](MachineIRBuilder &B) {

7517 B.setInstrAndDebugLoc(*Select);

7518

7519 Register Inner = MRI.createGenericVirtualRegister(CondTy);

7520 B.buildNot(Inner, Cond);

7521

7522 Register Ext = MRI.createGenericVirtualRegister(TrueTy);

7523 B.buildZExtOrTrunc(Ext, Inner);

7524 auto FreezeTrue = B.buildFreeze(TrueTy, True);

7525 B.buildOr(DstReg, Ext, FreezeTrue, Flags);

7526 };

7527 return true;

7528 }

7529

7530

7531 if (isZeroOrZeroSplat(True, true)) {

7532 MatchInfo = [=](MachineIRBuilder &B) {

7533 B.setInstrAndDebugLoc(*Select);

7534

7535 Register Inner = MRI.createGenericVirtualRegister(CondTy);

7536 B.buildNot(Inner, Cond);

7537

7538 Register Ext = MRI.createGenericVirtualRegister(TrueTy);

7539 B.buildZExtOrTrunc(Ext, Inner);

7540 auto FreezeFalse = B.buildFreeze(TrueTy, False);

7541 B.buildAnd(DstReg, Ext, FreezeFalse);

7542 };

7543 return true;

7544 }

7545

7546 return false;

7547}

7548

7553

7557 LLT DstTy = MRI.getType(DstReg);

7558

7560 return false;

7561

7562

7563 if ( MRI .hasOneNonDBGUse(Cmp->getReg(0)))

7564 return false;

7565

7567

7568

7570 return false;

7571

7572 Register CmpLHS = Cmp->getLHSReg();

7573 Register CmpRHS = Cmp->getRHSReg();

7574

7575

7576 if (True == CmpRHS && False == CmpLHS) {

7579 }

7580

7581

7582

7583

7584 if (True != CmpLHS || False != CmpRHS)

7585 return false;

7586

7587 switch (Pred) {

7591 return false;

7592 MatchInfo = [=](MachineIRBuilder &B) { B.buildUMax(DstReg, True, False); };

7593 return true;

7594 }

7598 return false;

7599 MatchInfo = [=](MachineIRBuilder &B) { B.buildSMax(DstReg, True, False); };

7600 return true;

7601 }

7605 return false;

7606 MatchInfo = [=](MachineIRBuilder &B) { B.buildUMin(DstReg, True, False); };

7607 return true;

7608 }

7612 return false;

7613 MatchInfo = [=](MachineIRBuilder &B) { B.buildSMin(DstReg, True, False); };

7614 return true;

7615 }

7616 default:

7617 return false;

7618 }

7619}

7620

7621

7624 assert(MI.getOpcode() == TargetOpcode::G_SUB);

7625 Register DestReg = MI.getOperand(0).getReg();

7626 LLT DestTy = MRI.getType(DestReg);

7627

7636 MachineInstr *MinMaxMI = MRI.getVRegDef(MI.getOperand(2).getReg());

7638 if (isLegal({NewOpc, {DestTy}})) {

7640 B.buildInstr(NewOpc, {DestReg}, {X, Sub0});

7641 };

7642 return true;

7643 }

7644 }

7645

7646 return false;

7647}

7648

7651

7652 if (tryFoldSelectOfConstants(Select, MatchInfo))

7653 return true;

7654

7655 if (tryFoldBoolSelectToLogic(Select, MatchInfo))

7656 return true;

7657

7658 return false;

7659}

7660

7661

7662

7663

7664

7665bool CombinerHelper::tryFoldAndOrOrICmpsUsingRanges(

7667 assert(Logic->getOpcode() != TargetOpcode::G_XOR && "unexpected xor");

7668 bool IsAnd = Logic->getOpcode() == TargetOpcode::G_AND;

7672 unsigned Flags = Logic->getFlags();

7673

7674

7676 if (!Cmp1)

7677 return false;

7678

7679

7681 if (!Cmp2)

7682 return false;

7683

7684

7685 if ( MRI .hasOneNonDBGUse(Cmp1->getReg(0)) ||

7686 MRI .hasOneNonDBGUse(Cmp2->getReg(0)))

7687 return false;

7688

7691 std::optional MaybeC1 =

7693 if (!MaybeC1)

7694 return false;

7695 C1 = MaybeC1->Value;

7696

7697 std::optional MaybeC2 =

7699 if (!MaybeC2)

7700 return false;

7701 C2 = MaybeC2->Value;

7702

7708 LLT CmpOperandTy = MRI.getType(R1);

7709

7711 return false;

7712

7713

7714

7718 return false;

7719

7720

7721

7722 std::optional Offset1;

7723 std::optional Offset2;

7724 if (R1 != R2) {

7726 std::optional MaybeOffset1 =

7728 if (MaybeOffset1) {

7729 R1 = Add->getLHSReg();

7730 Offset1 = MaybeOffset1->Value;

7731 }

7732 }

7734 std::optional MaybeOffset2 =

7736 if (MaybeOffset2) {

7737 R2 = Add->getLHSReg();

7738 Offset2 = MaybeOffset2->Value;

7739 }

7740 }

7741 }

7742

7743 if (R1 != R2)

7744 return false;

7745

7746

7749 if (Offset1)

7750 CR1 = CR1.subtract(*Offset1);

7751

7754 if (Offset2)

7755 CR2 = CR2.subtract(*Offset2);

7756

7757 bool CreateMask = false;

7758 APInt LowerDiff;

7759 std::optional CR = CR1.exactUnionWith(CR2);

7760 if (!CR) {

7761

7763 return false;

7764

7765

7766

7768 APInt UpperDiff = (CR1.getUpper() - 1) ^ (CR2.getUpper() - 1);

7770 if (!LowerDiff.isPowerOf2() || LowerDiff != UpperDiff ||

7772 return false;

7773

7775 CreateMask = true;

7776 }

7777

7778 if (IsAnd)

7779 CR = CR->inverse();

7780

7783 CR->getEquivalentICmp(NewPred, NewC, Offset);

7784

7785

7786

7787

7788

7789

7790

7791

7792 MatchInfo = [=](MachineIRBuilder &B) {

7793 if (CreateMask && Offset != 0) {

7794 auto TildeLowerDiff = B.buildConstant(CmpOperandTy, ~LowerDiff);

7795 auto And = B.buildAnd(CmpOperandTy, R1, TildeLowerDiff);

7796 auto OffsetC = B.buildConstant(CmpOperandTy, Offset);

7797 auto Add = B.buildAdd(CmpOperandTy, And, OffsetC, Flags);

7798 auto NewCon = B.buildConstant(CmpOperandTy, NewC);

7799 auto ICmp = B.buildICmp(NewPred, CmpTy, Add, NewCon);

7800 B.buildZExtOrTrunc(DstReg, ICmp);

7801 } else if (CreateMask && Offset == 0) {

7802 auto TildeLowerDiff = B.buildConstant(CmpOperandTy, ~LowerDiff);

7803 auto And = B.buildAnd(CmpOperandTy, R1, TildeLowerDiff);

7804 auto NewCon = B.buildConstant(CmpOperandTy, NewC);

7805 auto ICmp = B.buildICmp(NewPred, CmpTy, And, NewCon);

7806 B.buildZExtOrTrunc(DstReg, ICmp);

7807 } else if (!CreateMask && Offset != 0) {

7808 auto OffsetC = B.buildConstant(CmpOperandTy, Offset);

7809 auto Add = B.buildAdd(CmpOperandTy, R1, OffsetC, Flags);

7810 auto NewCon = B.buildConstant(CmpOperandTy, NewC);

7811 auto ICmp = B.buildICmp(NewPred, CmpTy, Add, NewCon);

7812 B.buildZExtOrTrunc(DstReg, ICmp);

7813 } else if (!CreateMask && Offset == 0) {

7814 auto NewCon = B.buildConstant(CmpOperandTy, NewC);

7815 auto ICmp = B.buildICmp(NewPred, CmpTy, R1, NewCon);

7816 B.buildZExtOrTrunc(DstReg, ICmp);

7817 } else {

7818 llvm_unreachable("unexpected configuration of CreateMask and Offset");

7819 }

7820 };

7821 return true;

7822}

7823

7824bool CombinerHelper::tryFoldLogicOfFCmps(GLogicalBinOp *Logic,

7826 assert(Logic->getOpcode() != TargetOpcode::G_XOR && "unexpecte xor");

7830 bool IsAnd = Logic->getOpcode() == TargetOpcode::G_AND;

7831

7832

7834 if (!Cmp1)

7835 return false;

7836

7837

7839 if (!Cmp2)

7840 return false;

7841

7842 LLT CmpTy = MRI.getType(Cmp1->getReg(0));

7843 LLT CmpOperandTy = MRI.getType(Cmp1->getLHSReg());

7844

7845

7846

7848 {TargetOpcode::G_FCMP, {CmpTy, CmpOperandTy}}) ||

7849 MRI .hasOneNonDBGUse(Logic->getReg(0)) ||

7850 MRI .hasOneNonDBGUse(Cmp1->getReg(0)) ||

7851 MRI .hasOneNonDBGUse(Cmp2->getReg(0)) ||

7853 return false;

7854

7861

7862 if (LHS0 == RHS1 && LHS1 == RHS0) {

7863

7866 }

7867

7868 if (LHS0 == RHS0 && LHS1 == RHS1) {

7869

7872 unsigned NewPred = IsAnd ? CmpCodeL & CmpCodeR : CmpCodeL | CmpCodeR;

7874 MatchInfo = [=](MachineIRBuilder &B) {

7875

7879 auto False = B.buildConstant(CmpTy, 0);

7880 B.buildZExtOrTrunc(DestReg, False);

7883 auto True =

7885 CmpTy.isVector() ,

7886 true ));

7887 B.buildZExtOrTrunc(DestReg, True);

7888 } else {

7889 auto Cmp = B.buildFCmp(Pred, CmpTy, LHS0, LHS1, Flags);

7890 B.buildZExtOrTrunc(DestReg, Cmp);

7891 }

7892 };

7893 return true;

7894 }

7895

7896 return false;

7897}

7898

7901

7902 if (tryFoldAndOrOrICmpsUsingRanges(And, MatchInfo))

7903 return true;

7904

7905 if (tryFoldLogicOfFCmps(And, MatchInfo))

7906 return true;

7907

7908 return false;

7909}

7910

7913

7914 if (tryFoldAndOrOrICmpsUsingRanges(Or, MatchInfo))

7915 return true;

7916

7917 if (tryFoldLogicOfFCmps(Or, MatchInfo))

7918 return true;

7919

7920 return false;

7921}

7922

7926

7927

7932 bool IsSigned = Add->isSigned();

7933 LLT DstTy = MRI.getType(Dst);

7934 LLT CarryTy = MRI.getType(Carry);

7935

7936

7937 if (MRI.use_nodbg_empty(Carry) &&

7940 B.buildAdd(Dst, LHS, RHS);

7941 B.buildUndef(Carry);

7942 };

7943 return true;

7944 }

7945

7946

7947 if (isConstantOrConstantVectorI(LHS) && !isConstantOrConstantVectorI(RHS)) {

7948 if (IsSigned) {

7950 B.buildSAddo(Dst, Carry, RHS, LHS);

7951 };

7952 return true;

7953 }

7954

7956 B.buildUAddo(Dst, Carry, RHS, LHS);

7957 };

7958 return true;

7959 }

7960

7961 std::optional MaybeLHS = getConstantOrConstantSplatVector(LHS);

7962 std::optional MaybeRHS = getConstantOrConstantSplatVector(RHS);

7963

7964

7967 bool Overflow;

7968 APInt Result = IsSigned ? MaybeLHS->sadd_ov(*MaybeRHS, Overflow)

7969 : MaybeLHS->uadd_ov(*MaybeRHS, Overflow);

7971 B.buildConstant(Dst, Result);

7972 B.buildConstant(Carry, Overflow);

7973 };

7974 return true;

7975 }

7976

7977

7980 B.buildCopy(Dst, LHS);

7981 B.buildConstant(Carry, 0);

7982 };

7983 return true;

7984 }

7985

7986

7987

7988

7990 if (MaybeRHS && AddLHS && MRI.hasOneNonDBGUse(Add->getReg(0)) &&

7993 std::optional MaybeAddRHS =

7994 getConstantOrConstantSplatVector(AddLHS->getRHSReg());

7995 if (MaybeAddRHS) {

7996 bool Overflow;

7997 APInt NewC = IsSigned ? MaybeAddRHS->sadd_ov(*MaybeRHS, Overflow)

7998 : MaybeAddRHS->uadd_ov(*MaybeRHS, Overflow);

8000 if (IsSigned) {

8002 auto ConstRHS = B.buildConstant(DstTy, NewC);

8003 B.buildSAddo(Dst, Carry, AddLHS->getLHSReg(), ConstRHS);

8004 };

8005 return true;

8006 }

8007

8009 auto ConstRHS = B.buildConstant(DstTy, NewC);

8010 B.buildUAddo(Dst, Carry, AddLHS->getLHSReg(), ConstRHS);

8011 };

8012 return true;

8013 }

8014 }

8015 };

8016

8017

8020 return false;

8021

8022

8023 if (!IsSigned) {

8028

8031 return false;

8035 B.buildConstant(Carry, 0);

8036 };

8037 return true;

8038 }

8042 B.buildAdd(Dst, LHS, RHS);

8043 B.buildConstant(Carry, 1);

8044 };

8045 return true;

8046 }

8047 }

8048 return false;

8049 }

8050

8051

8052

8053

8054

8055 if (VT->computeNumSignBits(RHS) > 1 && VT->computeNumSignBits(LHS) > 1) {

8058 B.buildConstant(Carry, 0);

8059 };

8060 return true;

8061 }

8062

8067

8070 return false;

8074 B.buildConstant(Carry, 0);

8075 };

8076 return true;

8077 }

8081 B.buildAdd(Dst, LHS, RHS);

8082 B.buildConstant(Carry, 1);

8083 };

8084 return true;

8085 }

8086 }

8087

8088 return false;

8089}

8090

8097

8100 bool OptForSize = MI.getMF()->getFunction().hasOptSize();

8102}

8103

8106 auto [Dst, Base] = MI.getFirst2Regs();

8107 LLT Ty = MRI.getType(Dst);

8109

8110 if (ExpVal == 0) {

8111 Builder.buildFConstant(Dst, 1.0);

8112 MI.removeFromParent();

8113 return;

8114 }

8115

8116 if (ExpVal < 0)

8117 ExpVal = -ExpVal;

8118

8119

8120

8121

8122

8123

8124 std::optional Res;

8126 while (ExpVal > 0) {

8127 if (ExpVal & 1) {

8128 if (!Res)

8129 Res = CurSquare;

8130 else

8131 Res = Builder.buildFMul(Ty, *Res, CurSquare);

8132 }

8133

8134 CurSquare = Builder.buildFMul(Ty, CurSquare, CurSquare);

8135 ExpVal >>= 1;

8136 }

8137

8138

8139

8141 Res = Builder.buildFDiv(Ty, Builder.buildFConstant(Ty, 1.0), *Res,

8142 MI.getFlags());

8143

8144 Builder.buildCopy(Dst, *Res);

8145 MI.eraseFromParent();

8146}

8147

8150

8153

8154 if ( MRI .hasOneNonDBGUse(Add->getReg(0)))

8155 return false;

8156

8159

8161 LLT DstTy = MRI.getType(Dst);

8162

8164 auto Const = B.buildConstant(DstTy, C1 - C2);

8165 B.buildAdd(Dst, Add->getLHSReg(), Const);

8166 };

8167

8168 return true;

8169}

8170

8173

8176

8177 if ( MRI .hasOneNonDBGUse(Add->getReg(0)))

8178 return false;

8179

8182

8184 LLT DstTy = MRI.getType(Dst);

8185

8187 auto Const = B.buildConstant(DstTy, C2 - C1);

8188 B.buildSub(Dst, Const, Add->getLHSReg());

8189 };

8190

8191 return true;

8192}

8193

8196

8199

8200 if ( MRI .hasOneNonDBGUse(Sub2->getReg(0)))

8201 return false;

8202

8205

8207 LLT DstTy = MRI.getType(Dst);

8208

8210 auto Const = B.buildConstant(DstTy, C1 + C2);

8211 B.buildSub(Dst, Sub2->getLHSReg(), Const);

8212 };

8213

8214 return true;

8215}

8216

8219

8222

8223 if ( MRI .hasOneNonDBGUse(Sub2->getReg(0)))

8224 return false;

8225

8228

8230 LLT DstTy = MRI.getType(Dst);

8231

8233 auto Const = B.buildConstant(DstTy, C1 - C2);

8234 B.buildSub(Dst, Const, Sub2->getRHSReg());

8235 };

8236

8237 return true;

8238}

8239

8242

8245

8246 if ( MRI .hasOneNonDBGUse(Sub->getReg(0)))

8247 return false;

8248

8251

8253 LLT DstTy = MRI.getType(Dst);

8254

8256 auto Const = B.buildConstant(DstTy, C2 - C1);

8257 B.buildAdd(Dst, Sub->getLHSReg(), Const);

8258 };

8259

8260 return true;

8261}

8262

8266

8268 return false;

8269

8271

8272 LLT DstTy = MRI.getType(Unmerge->getReg(0));

8273

8274

8275

8276

8277

8278

8279

8280

8281

8282

8283

8284

8285

8286

8287

8288

8289

8290

8291

8293 return false;

8294

8296 if ( Any )

8297 return false;

8298

8300

8302

8303

8304 if ( MRI .hasOneNonDBGUse(BV->getReg(0)))

8305 return false;

8306

8307

8308 if (BV->getNumSources() % Unmerge->getNumDefs() != 0)

8309 return false;

8310

8311 LLT BigBvTy = MRI.getType(BV->getReg(0));

8312 LLT SmallBvTy = DstTy;

8314

8316 {TargetOpcode::G_BUILD_VECTOR, {SmallBvTy, SmallBvElemenTy}}))

8317 return false;

8318

8319

8321 {TargetOpcode::G_ANYEXT,

8323 return false;

8324

8326

8327

8328 for (unsigned I = 0; I < Unmerge->getNumDefs(); ++I) {

8330 for (unsigned J = 0; J < SmallBvTy.getNumElements(); ++J) {

8333 auto AnyExt = B.buildAnyExt(SmallBvElemenTy, SourceArray);

8334 Ops.push_back(AnyExt.getReg(0));

8335 }

8337 };

8338 };

8339 return true;

8340 };

8341

8342 return false;

8343}

8344

8347

8352 const LLT SrcTy = MRI.getType(Shuffle.getSrc1Reg());

8353 const unsigned NumSrcElems = SrcTy.isVector() ? SrcTy.getNumElements() : 1;

8354 const unsigned NumDstElts = OrigMask.size();

8355 for (unsigned i = 0; i != NumDstElts; ++i) {

8356 int Idx = OrigMask[i];

8357 if (Idx >= (int)NumSrcElems) {

8358 Idx = -1;

8360 }

8362 }

8363

8365 return false;

8366

8367 MatchInfo = [&, NewMask = std::move(NewMask)](MachineIRBuilder &B) {

8368 B.buildShuffleVector(MI.getOperand(0), MI.getOperand(1), MI.getOperand(2),

8369 std::move(NewMask));

8370 };

8371

8372 return true;

8373}

8374

8376 const unsigned MaskSize = Mask.size();

8377 for (unsigned I = 0; I < MaskSize; ++I) {

8378 int Idx = Mask[I];

8379 if (Idx < 0)

8380 continue;

8381

8382 if (Idx < (int)NumElems)

8383 Mask[I] = Idx + NumElems;

8384 else

8385 Mask[I] = Idx - NumElems;

8386 }

8387}

8388

8391

8393

8394

8395 if (getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, Shuffle.getSrc1Reg(), MRI))

8396 return false;

8397

8398 if (getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, Shuffle.getSrc2Reg(), MRI))

8399 return false;

8400

8401 const LLT DstTy = MRI.getType(Shuffle.getReg(0));

8402 const LLT Src1Ty = MRI.getType(Shuffle.getSrc1Reg());

8404 {TargetOpcode::G_SHUFFLE_VECTOR, {DstTy, Src1Ty}}))

8405 return false;

8406

8408 const unsigned NumSrcElems = Src1Ty.getNumElements();

8409

8410 bool TouchesSrc1 = false;

8411 bool TouchesSrc2 = false;

8412 const unsigned NumElems = Mask.size();

8413 for (unsigned Idx = 0; Idx < NumElems; ++Idx) {

8414 if (Mask[Idx] < 0)

8415 continue;

8416

8417 if (Mask[Idx] < (int)NumSrcElems)

8418 TouchesSrc1 = true;

8419 else

8420 TouchesSrc2 = true;

8421 }

8422

8423 if (TouchesSrc1 == TouchesSrc2)

8424 return false;

8425

8426 Register NewSrc1 = Shuffle.getSrc1Reg();

8428 if (TouchesSrc2) {

8429 NewSrc1 = Shuffle.getSrc2Reg();

8431 }

8432

8434 auto Undef = B.buildUndef(Src1Ty);

8435 B.buildShuffleVector(Shuffle.getReg(0), NewSrc1, Undef, NewMask);

8436 };

8437

8438 return true;

8439}

8440

8444

8449 LLT DstTy = MRI.getType(Dst);

8450 LLT CarryTy = MRI.getType(Carry);

8451

8452

8455 return false;

8456

8459 Subo->isSigned());

8462 Subo->isSigned());

8463

8465

8468 return false;

8472 B.buildConstant(Carry, 0);

8473 };

8474 return true;

8475 }

8479 B.buildSub(Dst, LHS, RHS);

8481 CarryTy.isVector(),

8482 false));

8483 };

8484 return true;

8485 }

8486 }

8487 return false;

8488 }

8489

8490

8493 return false;

8497 B.buildConstant(Carry, 0);

8498 };

8499 return true;

8500 }

8504 B.buildSub(Dst, LHS, RHS);

8506 CarryTy.isVector(),

8507 false));

8508 };

8509 return true;

8510 }

8511 }

8512

8513 return false;

8514}