LegalizationArtifactCombiner.h Source File (original) (raw)

41 static bool isArtifactCast(unsigned Opc) {

42 switch (Opc) {

43 case TargetOpcode::G_TRUNC:

44 case TargetOpcode::G_SEXT:

45 case TargetOpcode::G_ZEXT:

46 case TargetOpcode::G_ANYEXT:

47 return true;

48 default:

49 return false;

50 }

51 }

53public:

57 : Builder(B), MRI(MRI), LI(LI), VT(VT) {}

64 assert(MI.getOpcode() == TargetOpcode::G_ANYEXT);

66 Builder.setInstrAndDebugLoc(MI);

67 Register DstReg = MI.getOperand(0).getReg();

68 Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg());

74 if (MRI.getType(DstReg) == MRI.getType(TruncSrc))

76 Observer);

77 else

78 Builder.buildAnyExtOrTrunc(DstReg, TruncSrc);

80 markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts);

81 return true;

82 }

91 Builder.buildInstr(ExtMI->getOpcode(), {DstReg}, {ExtSrc});

93 markInstAndDefDead(MI, *ExtMI, DeadInsts);

94 return true;

95 }

98 auto *SrcMI = MRI.getVRegDef(SrcReg);

99 if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) {

100 const LLT DstTy = MRI.getType(DstReg);

101 if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) {

102 auto &CstVal = SrcMI->getOperand(1);

103 auto MergedLocation =

105

106

107 Builder.setDebugLoc(MergedLocation);

108 Builder.buildConstant(

109 DstReg, CstVal.getCImm()->getValue().sext(DstTy.getSizeInBits()));

111 markInstAndDefDead(MI, *SrcMI, DeadInsts);

112 return true;

113 }

114 }

116 }

117

123 assert(MI.getOpcode() == TargetOpcode::G_ZEXT);

124

125 Builder.setInstrAndDebugLoc(MI);

126 Register DstReg = MI.getOperand(0).getReg();

127 Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg());

128

129

130

135 LLT DstTy = MRI.getType(DstReg);

136 if (isInstUnsupported({TargetOpcode::G_AND, {DstTy}}) ||

137 isConstantUnsupported(DstTy))

138 return false;

140 LLT SrcTy = MRI.getType(SrcReg);

142 if (SextSrc && (DstTy != MRI.getType(SextSrc)))

143 SextSrc = Builder.buildSExtOrTrunc(DstTy, SextSrc).getReg(0);

144 if (TruncSrc && (DstTy != MRI.getType(TruncSrc)))

145 TruncSrc = Builder.buildAnyExtOrTrunc(DstTy, TruncSrc).getReg(0);

147 Register AndSrc = SextSrc ? SextSrc : TruncSrc;

148

149

150

151

152

153

154 if (VT && (VT->getKnownZeroes(AndSrc) | ExtMaskVal).isAllOnes()) {

156 Observer);

157 } else {

158 auto Mask = Builder.buildConstant(DstTy, ExtMaskVal);

159 Builder.buildAnd(DstReg, AndSrc, Mask);

160 }

161 markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts);

162 return true;

163 }

164

165

170 MI.getOperand(1).setReg(ZextSrc);

173 markDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts);

174 return true;

175 }

176

177

178 auto *SrcMI = MRI.getVRegDef(SrcReg);

179 if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) {

180 const LLT DstTy = MRI.getType(DstReg);

181 if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) {

182 auto &CstVal = SrcMI->getOperand(1);

183 Builder.buildConstant(

184 DstReg, CstVal.getCImm()->getValue().zext(DstTy.getSizeInBits()));

186 markInstAndDefDead(MI, *SrcMI, DeadInsts);

187 return true;

188 }

189 }

191 }

192

198 assert(MI.getOpcode() == TargetOpcode::G_SEXT);

199

200 Builder.setInstrAndDebugLoc(MI);

201 Register DstReg = MI.getOperand(0).getReg();

202 Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg());

203

204

207 LLT DstTy = MRI.getType(DstReg);

208 if (isInstUnsupported({TargetOpcode::G_SEXT_INREG, {DstTy}}))

209 return false;

211 LLT SrcTy = MRI.getType(SrcReg);

212 uint64_t SizeInBits = SrcTy.getScalarSizeInBits();

213 if (DstTy != MRI.getType(TruncSrc))

214 TruncSrc = Builder.buildAnyExtOrTrunc(DstTy, TruncSrc).getReg(0);

215

216

217 if (VT && VT->computeNumSignBits(TruncSrc) >

220 Observer);

221 else

222 Builder.buildSExtInReg(DstReg, TruncSrc, SizeInBits);

223 markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts);

224 return true;

225 }

226

227

228

235 Builder.buildInstr(ExtMI->getOpcode(), {DstReg}, {ExtSrc});

237 markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts);

238 return true;

239 }

240

241

242 auto *SrcMI = MRI.getVRegDef(SrcReg);

243 if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) {

244 const LLT DstTy = MRI.getType(DstReg);

245 if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) {

246 auto &CstVal = SrcMI->getOperand(1);

247 Builder.buildConstant(

248 DstReg, CstVal.getCImm()->getValue().sext(DstTy.getSizeInBits()));

250 markInstAndDefDead(MI, *SrcMI, DeadInsts);

251 return true;

252 }

253 }

254

256 }

257

263 assert(MI.getOpcode() == TargetOpcode::G_TRUNC);

264

265 Builder.setInstr(MI);

266 Register DstReg = MI.getOperand(0).getReg();

267 const LLT DstTy = MRI.getType(DstReg);

268 Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg());

269

270

271 auto *SrcMI = MRI.getVRegDef(SrcReg);

272 if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) {

273 if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) {

274 auto &CstVal = SrcMI->getOperand(1);

275 Builder.buildConstant(

276 DstReg, CstVal.getCImm()->getValue().trunc(DstTy.getSizeInBits()));

278 markInstAndDefDead(MI, *SrcMI, DeadInsts);

279 return true;

280 }

281 }

282

283

284

286 const Register MergeSrcReg = SrcMerge->getSourceReg(0);

287 const LLT MergeSrcTy = MRI.getType(MergeSrcReg);

288

289

291 const unsigned MergeSrcSize = MergeSrcTy.getSizeInBits();

293 return false;

294

295 if (DstSize < MergeSrcSize) {

296

297

298 if (isInstUnsupported({TargetOpcode::G_TRUNC, {DstTy, MergeSrcTy}}))

299 return false;

300

301 LLVM_DEBUG(dbgs() << "Combining G_TRUNC(G_MERGE_VALUES) to G_TRUNC: "

302 << MI);

303

304 Builder.buildTrunc(DstReg, MergeSrcReg);

306 } else if (DstSize == MergeSrcSize) {

307

309 dbgs() << "Replacing G_TRUNC(G_MERGE_VALUES) with merge input: "

310 << MI);

312 Observer);

313 } else if (DstSize % MergeSrcSize == 0) {

314

315

316 if (isInstUnsupported(

317 {TargetOpcode::G_MERGE_VALUES, {DstTy, MergeSrcTy}}))

318 return false;

319

321 dbgs() << "Combining G_TRUNC(G_MERGE_VALUES) to G_MERGE_VALUES: "

322 << MI);

323

324 const unsigned NumSrcs = DstSize / MergeSrcSize;

325 assert(NumSrcs < SrcMI->getNumOperands() - 1 &&

326 "trunc(merge) should require less inputs than merge");

328 for (unsigned i = 0; i < NumSrcs; ++i)

329 SrcRegs[i] = SrcMerge->getSourceReg(i);

330

331 Builder.buildMergeValues(DstReg, SrcRegs);

333 } else {

334

335 return false;

336 }

337

338 markInstAndDefDead(MI, *SrcMerge, DeadInsts);

339 return true;

340 }

341

342

345

346

347

349

350 Builder.buildTrunc(DstReg, TruncSrc);

352 markInstAndDefDead(MI, *MRI.getVRegDef(TruncSrc), DeadInsts);

353 return true;

354 }

355

356

360 LLT FoundRegTy = MRI.getType(FoundReg);

361 if (DstTy == FoundRegTy) {

362 LLVM_DEBUG(dbgs() << ".. Combine G_TRUNC(G_[S,Z,ANY]EXT/G_TRUNC...): "

363 << MI);

364

366 Observer);

368 markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts);

369 return true;

370 }

371 }

372

373 return false;

374 }

375

376

381 unsigned Opcode = MI.getOpcode();

382 assert(Opcode == TargetOpcode::G_ANYEXT || Opcode == TargetOpcode::G_ZEXT ||

383 Opcode == TargetOpcode::G_SEXT);

384

386 MI.getOperand(1).getReg(), MRI)) {

387 Builder.setInstr(MI);

388 Register DstReg = MI.getOperand(0).getReg();

389 LLT DstTy = MRI.getType(DstReg);

390

391 if (Opcode == TargetOpcode::G_ANYEXT) {

392

393 if (!isInstLegal({TargetOpcode::G_IMPLICIT_DEF, {DstTy}}))

394 return false;

395 LLVM_DEBUG(dbgs() << ".. Combine G_ANYEXT(G_IMPLICIT_DEF): " << MI);

396 auto Impl = Builder.buildUndef(DstTy);

398 Observer);

400 } else {

401

402

403 if (isConstantUnsupported(DstTy))

404 return false;

405 LLVM_DEBUG(dbgs() << ".. Combine G_[SZ]EXT(G_IMPLICIT_DEF): " << MI);

406 auto Cnst = Builder.buildConstant(DstTy, 0);

408 Observer);

410 }

411

412 markInstAndDefDead(MI, *DefMI, DeadInsts);

413 return true;

414 }

415 return false;

416 }

417

421

422 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);

423

424 const unsigned CastOpc = CastMI.getOpcode();

425

426 if (!isArtifactCast(CastOpc))

427 return false;

428

429 const unsigned NumDefs = MI.getNumOperands() - 1;

430

432 const LLT CastSrcTy = MRI.getType(CastSrcReg);

433 const LLT DestTy = MRI.getType(MI.getOperand(0).getReg());

434 const LLT SrcTy = MRI.getType(MI.getOperand(NumDefs).getReg());

435

436 const unsigned CastSrcSize = CastSrcTy.getSizeInBits();

437 const unsigned DestSize = DestTy.getSizeInBits();

438

439 if (CastOpc == TargetOpcode::G_TRUNC) {

440 if (SrcTy.isVector() && SrcTy.getScalarType() == DestTy.getScalarType()) {

441

442

443

444

445

446

447

448

449

450 unsigned UnmergeNumElts =

454 LLT SrcWideTy =

456

457 if (isInstUnsupported(

458 {TargetOpcode::G_UNMERGE_VALUES, {UnmergeTy, CastSrcTy}}) ||

459 LI.getAction({TargetOpcode::G_TRUNC, {SrcWideTy, UnmergeTy}})

461 return false;

462

463 Builder.setInstr(MI);

464 auto NewUnmerge = Builder.buildUnmerge(UnmergeTy, CastSrcReg);

465

466 for (unsigned I = 0; I != NumDefs; ++I) {

467 Register DefReg = MI.getOperand(I).getReg();

469 Builder.buildTrunc(DefReg, NewUnmerge.getReg(I));

470 }

471

472 markInstAndDefDead(MI, CastMI, DeadInsts);

473 return true;

474 }

475

476 if (CastSrcTy.isScalar() && SrcTy.isScalar() && !DestTy.isVector()) {

477

478

479

480

481

482

483

484 if (CastSrcSize % DestSize != 0)

485 return false;

486

487

488 if (isInstUnsupported(

489 {TargetOpcode::G_UNMERGE_VALUES, {DestTy, CastSrcTy}}))

490 return false;

491

492

493

494 const unsigned NewNumDefs = CastSrcSize / DestSize;

496 for (unsigned Idx = 0; Idx < NewNumDefs; ++Idx) {

497 if (Idx < NumDefs)

498 DstRegs[Idx] = MI.getOperand(Idx).getReg();

499 else

500 DstRegs[Idx] = MRI.createGenericVirtualRegister(DestTy);

501 }

502

503

504 Builder.setInstr(MI);

505 Builder.buildUnmerge(DstRegs, CastSrcReg);

506 UpdatedDefs.append(DstRegs.begin(), DstRegs.begin() + NewNumDefs);

507 markInstAndDefDead(MI, CastMI, DeadInsts);

508 return true;

509 }

510 }

511

512

513 return false;

514 }

515

517 LLT OpTy, LLT DestTy) {

518

519 switch (MergeOp) {

520 default:

521 return false;

522 case TargetOpcode::G_BUILD_VECTOR:

523 case TargetOpcode::G_MERGE_VALUES:

524

525

526

527

528

529

530

531

532

533

534

535

536

537

538

539

540

541

542

543

544

545

546 if (ConvertOp == 0)

547 return true;

550 case TargetOpcode::G_CONCAT_VECTORS: {

551 if (ConvertOp == 0)

552 return true;

554 return false;

555

557

558

559

560

561 if (ConvertOp == TargetOpcode::G_TRUNC)

564 }

565 }

566 }

567

568

569

576 Builder.buildCopy(DstReg, SrcReg);

578 return;

579 }

581

582 for (auto &UseMI : MRI.use_instructions(DstReg)) {

585 }

586

587 MRI.replaceRegWith(DstReg, SrcReg);

589

590 for (auto *UseMI : UseMIs)

592 }

593

594

596 unsigned DefIdx = 0;

598 if (Def.getReg() == SearchDef)

599 break;

600 ++DefIdx;

601 }

602

603 return DefIdx;

604 }

605

606

607

608

609

614

615

617

618

619

620

621

622

624 unsigned Size) {

626

627

629 unsigned SrcSize = MRI.getType(Src1Reg).getSizeInBits();

630

631

632 unsigned StartSrcIdx = (StartBit / SrcSize) + 1;

633

634 unsigned InRegOffset = StartBit % SrcSize;

635

636

637

638 if (InRegOffset + Size > SrcSize)

639 return CurrentBest;

640

642 if (InRegOffset == 0 && Size == SrcSize) {

643 CurrentBest = SrcReg;

644 return findValueFromDefImpl(SrcReg, 0, Size, MRI.getType(SrcReg));

645 }

646

647 return findValueFromDefImpl(SrcReg, InRegOffset, Size,

648 MRI.getType(SrcReg));

649 }

650

651

652

653

654

655

657 unsigned Size) {

659

660

662 unsigned SrcSize = MRI.getType(Src1Reg).getSizeInBits();

663

664

665 unsigned StartSrcIdx = (StartBit / SrcSize) + 1;

666

667 unsigned InRegOffset = StartBit % SrcSize;

668

669 if (InRegOffset != 0)

670 return CurrentBest;

671 if (Size < SrcSize)

672 return CurrentBest;

673

674

675

676 if (Size > SrcSize) {

677 if (Size % SrcSize > 0)

678 return CurrentBest;

679

680 unsigned NumSrcsUsed = Size / SrcSize;

681

684

685 LLT SrcTy = MRI.getType(Src1Reg);

687

688

690 LI.getAction({TargetOpcode::G_BUILD_VECTOR, {NewBVTy, SrcTy}});

692 return CurrentBest;

693

695 for (unsigned SrcIdx = StartSrcIdx; SrcIdx < StartSrcIdx + NumSrcsUsed;

696 ++SrcIdx)

698 MIB.setInstrAndDebugLoc(BV);

699 return MIB.buildBuildVector(NewBVTy, NewSrcs).getReg(0);

700 }

701

702 return BV.getReg(StartSrcIdx);

703 }

704

705

706

707

708

709

711 unsigned Size) {

712 assert(MI.getOpcode() == TargetOpcode::G_INSERT);

714

715 Register ContainerSrcReg = MI.getOperand(1).getReg();

716 Register InsertedReg = MI.getOperand(2).getReg();

717 LLT InsertedRegTy = MRI.getType(InsertedReg);

718 unsigned InsertOffset = MI.getOperand(3).getImm();

719

720

721

722

723

724

725

726

727

728

729

730

731

732

733

734

735

736

737

738

739

740

741

742

743

744

745

746

747

748

749

750

751

752

753

754

755

756 unsigned InsertedEndBit = InsertOffset + InsertedRegTy.getSizeInBits();

757 unsigned EndBit = StartBit + Size;

758 unsigned NewStartBit;

760 if (EndBit <= InsertOffset || InsertedEndBit <= StartBit) {

761 SrcRegToUse = ContainerSrcReg;

762 NewStartBit = StartBit;

763 return findValueFromDefImpl(SrcRegToUse, NewStartBit, Size,

764 MRI.getType(SrcRegToUse));

765 }

766 if (InsertOffset <= StartBit && EndBit <= InsertedEndBit) {

767 SrcRegToUse = InsertedReg;

768 NewStartBit = StartBit - InsertOffset;

769 if (NewStartBit == 0 &&

770 Size == MRI.getType(SrcRegToUse).getSizeInBits())

771 CurrentBest = SrcRegToUse;

772 return findValueFromDefImpl(SrcRegToUse, NewStartBit, Size,

773 MRI.getType(SrcRegToUse));

774 }

775

777 }

778

779

780

781

782

783

784

786 unsigned Size) {

787 assert(MI.getOpcode() == TargetOpcode::G_SEXT ||

788 MI.getOpcode() == TargetOpcode::G_ZEXT ||

789 MI.getOpcode() == TargetOpcode::G_ANYEXT);

791

792 Register SrcReg = MI.getOperand(1).getReg();

793 LLT SrcType = MRI.getType(SrcReg);

795

796

797 if (!SrcType.isScalar())

798 return CurrentBest;

799

800 if (StartBit + Size > SrcSize)

801 return CurrentBest;

802

803 if (StartBit == 0 && SrcType.getSizeInBits() == Size)

804 CurrentBest = SrcReg;

805 return findValueFromDefImpl(SrcReg, StartBit, Size, SrcType);

806 }

807

808

809

810

811

812

814 unsigned Size) {

815 assert(MI.getOpcode() == TargetOpcode::G_TRUNC);

817

818 Register SrcReg = MI.getOperand(1).getReg();

819 LLT SrcType = MRI.getType(SrcReg);

820

821

822 if (!SrcType.isScalar())

823 return CurrentBest;

824

825 return findValueFromDefImpl(SrcReg, StartBit, Size, SrcType);

826 }

827

828

829

830

831 Register findValueFromDefImpl(Register DefReg, unsigned StartBit,

832 unsigned Size, LLT DstTy) {

833 std::optional DefSrcReg =

836 DefReg = DefSrcReg->Reg;

837

838

839

840 switch (Def->getOpcode()) {

841 case TargetOpcode::G_CONCAT_VECTORS:

843 case TargetOpcode::G_UNMERGE_VALUES: {

844 unsigned DefStartBit = 0;

845 unsigned DefSize = MRI.getType(DefReg).getSizeInBits();

846 for (const auto &MO : Def->defs()) {

847 if (MO.getReg() == DefReg)

848 break;

849 DefStartBit += DefSize;

850 }

851 Register SrcReg = Def->getOperand(Def->getNumOperands() - 1).getReg();

853 findValueFromDefImpl(SrcReg, StartBit + DefStartBit, Size, DstTy);

854 if (SrcOriginReg)

855 return SrcOriginReg;

856

857

858

859 if (StartBit == 0 && Size == DefSize)

860 return DefReg;

861 return CurrentBest;

862 }

863 case TargetOpcode::G_BUILD_VECTOR:

866 case TargetOpcode::G_INSERT:

867 return findValueFromInsert(*Def, StartBit, Size);

868 case TargetOpcode::G_TRUNC:

869 return findValueFromTrunc(*Def, StartBit, Size);

870 case TargetOpcode::G_SEXT:

871 case TargetOpcode::G_ZEXT:

872 case TargetOpcode::G_ANYEXT:

873 return findValueFromExt(*Def, StartBit, Size);

874 case TargetOpcode::G_IMPLICIT_DEF: {

875 if (MRI.getType(DefReg) == DstTy)

876 return DefReg;

877 MIB.setInstrAndDebugLoc(*Def);

878 return MIB.buildUndef(DstTy).getReg(0);

879 }

880 default:

881 return CurrentBest;

882 }

883 }

884

885 public:

888 : MRI(Mri), MIB(Builder), LI(Info) {}

889

890

891

892

893

895 LLT DstTy) {

897 Register FoundReg = findValueFromDefImpl(DefReg, StartBit, Size, DstTy);

898 return FoundReg != DefReg ? FoundReg : Register();

899 }

900

901

902

903

906 unsigned NumDefs = MI.getNumDefs();

907 LLT DestTy = MRI.getType(MI.getReg(0));

908

910 for (unsigned DefIdx = 0; DefIdx < NumDefs; ++DefIdx) {

912 if (MRI.use_nodbg_empty(DefReg)) {

913 DeadDefs[DefIdx] = true;

914 continue;

915 }

918 if (!FoundVal)

919 continue;

920 if (MRI.getType(FoundVal) != DestTy)

921 continue;

922

924 Observer);

925

927 MI.getOperand(DefIdx).setReg(DefReg);

929 DeadDefs[DefIdx] = true;

930 }

931 return DeadDefs.all();

932 }

933

935 unsigned &DefOperandIdx) {

936 if (Register Def = findValueFromDefImpl(Reg, 0, Size, MRI.getType(Reg))) {

938 DefOperandIdx =

939 Unmerge->findRegisterDefOperandIdx(Def, nullptr);

940 return Unmerge;

941 }

942 }

943 return nullptr;

944 }

945

946

947

948

950 GUnmerge *Unmerge, unsigned UnmergeIdxStart,

951 unsigned NumElts, unsigned EltSize,

952 bool AllowUndef) {

953 assert(MergeStartIdx + NumElts <= MI.getNumSources());

954 for (unsigned i = MergeStartIdx; i < MergeStartIdx + NumElts; ++i) {

955 unsigned EltUnmergeIdx;

957 MI.getSourceReg(i), EltSize, EltUnmergeIdx);

958

959 if (EltUnmerge == Unmerge) {

960

961 if (i - MergeStartIdx != EltUnmergeIdx - UnmergeIdxStart)

962 return false;

963 } else if (!AllowUndef ||

964 MRI.getVRegDef(MI.getSourceReg(i))->getOpcode() !=

965 TargetOpcode::G_IMPLICIT_DEF)

966 return false;

967 }

968 return true;

969 }

970

976 LLT EltTy = MRI.getType(Elt0);

978

979 unsigned Elt0UnmergeIdx;

980

982 if (!Unmerge)

983 return false;

984

985 unsigned NumMIElts = MI.getNumSources();

987 LLT DstTy = MRI.getType(Dst);

988 Register UnmergeSrc = Unmerge->getSourceReg();

989 LLT UnmergeSrcTy = MRI.getType(UnmergeSrc);

990

991

992

993

994

995

996

997

998 if ((DstTy == UnmergeSrcTy) && (Elt0UnmergeIdx == 0)) {

1000 DstTy.isVector()))

1001 return false;

1002

1005 return true;

1006 }

1007

1008

1009

1010

1011

1012

1013

1014

1015

1016

1017

1018

1022 (Elt0UnmergeIdx % NumMIElts == 0) &&

1023 getCoverTy(UnmergeSrcTy, DstTy) == UnmergeSrcTy) {

1025 EltSize, false))

1026 return false;

1027 MIB.setInstrAndDebugLoc(MI);

1028 auto NewUnmerge = MIB.buildUnmerge(DstTy, Unmerge->getSourceReg());

1029 unsigned DstIdx = (Elt0UnmergeIdx * EltSize) / DstTy.getSizeInBits();

1031 UpdatedDefs, Observer);

1033 return true;

1034 }

1035

1036

1037

1038

1039

1040

1041

1042

1043

1044

1045

1047 getCoverTy(DstTy, UnmergeSrcTy) == DstTy) {

1049 unsigned NumElts = Unmerge->getNumDefs();

1050 for (unsigned i = 0; i < MI.getNumSources(); i += NumElts) {

1051 unsigned EltUnmergeIdx;

1053 EltSize, EltUnmergeIdx);

1054

1055 if ((!UnmergeI) || (UnmergeI->getNumDefs() != NumElts) ||

1056 (EltUnmergeIdx != 0))

1057 return false;

1059 false))

1060 return false;

1061 ConcatSources.push_back(UnmergeI->getSourceReg());

1062 }

1063

1064 MIB.setInstrAndDebugLoc(MI);

1065 MIB.buildMergeLikeInstr(Dst, ConcatSources);

1067 return true;

1068 }

1069

1070 return false;

1071 }

1072 };

1073

1078 unsigned NumDefs = MI.getNumDefs();

1079 Register SrcReg = MI.getSourceReg();

1080 std::optional DefSrcReg =

1082 if (!DefSrcReg)

1083 return false;

1085

1086 LLT OpTy = MRI.getType(SrcReg);

1087 LLT DestTy = MRI.getType(MI.getReg(0));

1088 unsigned SrcDefIdx = getDefIndex(*SrcDef, DefSrcReg->Reg);

1089

1090 Builder.setInstrAndDebugLoc(MI);

1091

1094 markInstAndDefDead(MI, *SrcDef, DeadInsts, SrcDefIdx);

1095 return true;

1096 }

1097

1099

1100

1101

1102

1103

1104 Register SrcUnmergeSrc = SrcUnmerge->getSourceReg();

1105 LLT SrcUnmergeSrcTy = MRI.getType(SrcUnmergeSrc);

1106

1107

1108

1109

1111 {TargetOpcode::G_UNMERGE_VALUES, {OpTy, SrcUnmergeSrcTy}});

1112 switch (ActionStep.Action) {

1114 if (!OpTy.isVector() || !LI.isLegal({TargetOpcode::G_UNMERGE_VALUES,

1115 {DestTy, SrcUnmergeSrcTy}}))

1116 return false;

1117 break;

1120 break;

1123 if (ActionStep.TypeIdx == 1)

1124 return false;

1125 break;

1126 default:

1127 return false;

1128 }

1129

1130 auto NewUnmerge = Builder.buildUnmerge(DestTy, SrcUnmergeSrc);

1131

1132

1133

1134

1135 for (unsigned I = 0; I != NumDefs; ++I) {

1138 MRI, Builder, UpdatedDefs, Observer);

1139 }

1140

1141 markInstAndDefDead(MI, *SrcUnmerge, DeadInsts, SrcDefIdx);

1142 return true;

1143 }

1144

1146 unsigned ConvertOp = 0;

1147

1148

1149 unsigned SrcOp = SrcDef->getOpcode();

1150 if (isArtifactCast(SrcOp)) {

1151 ConvertOp = SrcOp;

1153 }

1154

1156 ConvertOp, OpTy, DestTy)) {

1157

1158

1160 }

1161

1162 const unsigned NumMergeRegs = MergeI->getNumOperands() - 1;

1163

1164 if (NumMergeRegs < NumDefs) {

1165 if (NumDefs % NumMergeRegs != 0)

1166 return false;

1167

1168 Builder.setInstr(MI);

1169

1170

1171

1172

1173

1174

1175

1176 const unsigned NewNumDefs = NumDefs / NumMergeRegs;

1177 for (unsigned Idx = 0; Idx < NumMergeRegs; ++Idx) {

1179 for (unsigned j = 0, DefIdx = Idx * NewNumDefs; j < NewNumDefs;

1180 ++j, ++DefIdx)

1182

1183 if (ConvertOp) {

1184 LLT MergeDstTy = MRI.getType(SrcDef->getOperand(0).getReg());

1185

1186

1187

1188

1189 LLT MergeEltTy = MergeDstTy.divide(NumMergeRegs);

1190

1191

1192

1193

1194

1195

1196

1197

1198

1199

1200

1201

1202

1203

1204 Register TmpReg = MRI.createGenericVirtualRegister(MergeEltTy);

1205 Builder.buildInstr(ConvertOp, {TmpReg},

1207 Builder.buildUnmerge(DstRegs, TmpReg);

1208 } else {

1209 Builder.buildUnmerge(DstRegs, MergeI->getOperand(Idx + 1).getReg());

1210 }

1211 UpdatedDefs.append(DstRegs.begin(), DstRegs.end());

1212 }

1213

1214 } else if (NumMergeRegs > NumDefs) {

1215 if (ConvertOp != 0 || NumMergeRegs % NumDefs != 0)

1216 return false;

1217

1218 Builder.setInstr(MI);

1219

1220

1221

1222

1223

1224

1225

1226 const unsigned NumRegs = NumMergeRegs / NumDefs;

1227 for (unsigned DefIdx = 0; DefIdx < NumDefs; ++DefIdx) {

1229 for (unsigned j = 0, Idx = NumRegs * DefIdx + 1; j < NumRegs;

1230 ++j, ++Idx)

1232

1233 Register DefReg = MI.getReg(DefIdx);

1234 Builder.buildMergeLikeInstr(DefReg, Regs);

1235 UpdatedDefs.push_back(DefReg);

1236 }

1237

1238 } else {

1239 LLT MergeSrcTy = MRI.getType(MergeI->getOperand(1).getReg());

1240

1241 if (!ConvertOp && DestTy != MergeSrcTy) {

1242 if (DestTy.isPointer())

1243 ConvertOp = TargetOpcode::G_INTTOPTR;

1244 else if (MergeSrcTy.isPointer())

1245 ConvertOp = TargetOpcode::G_PTRTOINT;

1246 else

1247 ConvertOp = TargetOpcode::G_BITCAST;

1248 }

1249

1250 if (ConvertOp) {

1251 Builder.setInstr(MI);

1252

1253 for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {

1254 Register DefReg = MI.getOperand(Idx).getReg();

1256

1257 if (!MRI.use_empty(DefReg)) {

1258 Builder.buildInstr(ConvertOp, {DefReg}, {MergeSrc});

1259 UpdatedDefs.push_back(DefReg);

1260 }

1261 }

1262

1263 markInstAndDefDead(MI, *MergeI, DeadInsts);

1264 return true;

1265 }

1266

1267 assert(DestTy == MergeSrcTy &&

1268 "Bitcast and the other kinds of conversions should "

1269 "have happened earlier");

1270

1271 Builder.setInstr(MI);

1272 for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {

1273 Register DstReg = MI.getOperand(Idx).getReg();

1276 Observer);

1277 }

1278 }

1279

1280 markInstAndDefDead(MI, *MergeI, DeadInsts);

1281 return true;

1282 }

1283

1287 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT);

1288

1289

1290

1291

1292

1293

1294

1295

1296

1297

1298

1299

1300

1301 Register DstReg = MI.getOperand(0).getReg();

1302 Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg());

1303 MachineInstr *MergeI = MRI.getVRegDef(SrcReg);

1304 if (MergeI && MergeI->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) {

1305 Builder.setInstrAndDebugLoc(MI);

1306 Builder.buildUndef(DstReg);

1308 markInstAndDefDead(MI, *MergeI, DeadInsts);

1309 return true;

1310 }

1312 return false;

1313

1314 LLT DstTy = MRI.getType(DstReg);

1315 LLT SrcTy = MRI.getType(SrcReg);

1316

1317

1318 unsigned ExtractDstSize = DstTy.getSizeInBits();

1319 unsigned Offset = MI.getOperand(2).getImm();

1320 unsigned NumMergeSrcs = MergeI->getNumOperands() - 1;

1321 unsigned MergeSrcSize = SrcTy.getSizeInBits() / NumMergeSrcs;

1322 unsigned MergeSrcIdx = Offset / MergeSrcSize;

1323

1324

1325 unsigned EndMergeSrcIdx = (Offset + ExtractDstSize - 1) / MergeSrcSize;

1326

1327

1328 if (MergeSrcIdx != EndMergeSrcIdx)

1329 return false;

1330

1331

1332 Builder.setInstr(MI);

1333 Builder.buildExtract(DstReg, MergeI->getOperand(MergeSrcIdx + 1).getReg(),

1334 Offset - MergeSrcIdx * MergeSrcSize);

1336 markInstAndDefDead(MI, *MergeI, DeadInsts);

1337 return true;

1338 }

1339

1340

1341

1342

1343

1348

1349

1350

1351

1352 if (!DeadInsts.empty())

1353 deleteMarkedDeadInsts(DeadInsts, WrapperObserver);

1354

1355

1356

1357

1358

1361 switch (MI.getOpcode()) {

1362 default:

1363 return false;

1364 case TargetOpcode::G_ANYEXT:

1366 break;

1367 case TargetOpcode::G_ZEXT:

1369 break;

1370 case TargetOpcode::G_SEXT:

1372 break;

1373 case TargetOpcode::G_UNMERGE_VALUES:

1375 UpdatedDefs, WrapperObserver);

1376 break;

1377 case TargetOpcode::G_MERGE_VALUES:

1378 case TargetOpcode::G_BUILD_VECTOR:

1379 case TargetOpcode::G_CONCAT_VECTORS:

1380

1381

1382 for (MachineInstr &U : MRI.use_instructions(MI.getOperand(0).getReg())) {

1383 if (U.getOpcode() == TargetOpcode::G_UNMERGE_VALUES ||

1384 U.getOpcode() == TargetOpcode::G_TRUNC) {

1385 UpdatedDefs.push_back(MI.getOperand(0).getReg());

1386 break;

1387 }

1388 }

1390 UpdatedDefs, WrapperObserver);

1391 break;

1392 case TargetOpcode::G_EXTRACT:

1394 break;

1395 case TargetOpcode::G_TRUNC:

1398

1399

1400

1401

1402 UpdatedDefs.push_back(MI.getOperand(0).getReg());

1403 }

1404 break;

1405 }

1406

1407

1408

1409

1410 while (!UpdatedDefs.empty()) {

1412 assert(NewDef.isVirtual() && "Unexpected redefinition of a physreg");

1413 for (MachineInstr &Use : MRI.use_instructions(NewDef)) {

1414 switch (Use.getOpcode()) {

1415

1416 case TargetOpcode::G_ANYEXT:

1417 case TargetOpcode::G_ZEXT:

1418 case TargetOpcode::G_SEXT:

1419 case TargetOpcode::G_UNMERGE_VALUES:

1420 case TargetOpcode::G_EXTRACT:

1421 case TargetOpcode::G_TRUNC:

1422 case TargetOpcode::G_BUILD_VECTOR:

1423

1425 break;

1426 case TargetOpcode::G_ASSERT_SEXT:

1427 case TargetOpcode::G_ASSERT_ZEXT:

1428 case TargetOpcode::G_ASSERT_ALIGN:

1429 case TargetOpcode::COPY: {

1430 Register Copy = Use.getOperand(0).getReg();

1431 if (Copy.isVirtual())

1433 break;

1434 }

1435 default:

1436

1437

1438

1439 break;

1440 }

1441 }

1442 }

1444 }

1445

1446private:

1448 switch (MI.getOpcode()) {

1449 case TargetOpcode::COPY:

1450 case TargetOpcode::G_TRUNC:

1451 case TargetOpcode::G_ZEXT:

1452 case TargetOpcode::G_ANYEXT:

1453 case TargetOpcode::G_SEXT:

1454 case TargetOpcode::G_EXTRACT:

1455 case TargetOpcode::G_ASSERT_SEXT:

1456 case TargetOpcode::G_ASSERT_ZEXT:

1457 case TargetOpcode::G_ASSERT_ALIGN:

1458 return MI.getOperand(1).getReg();

1459 case TargetOpcode::G_UNMERGE_VALUES:

1460 return MI.getOperand(MI.getNumOperands() - 1).getReg();

1461 default:

1463 }

1464 }

1465

1466

1467

1468

1469

1470

1473 unsigned DefIdx = 0) {

1474

1475

1476

1477

1478

1479

1480

1481

1482

1484 while (PrevMI != &DefMI) {

1485 Register PrevRegSrc = getArtifactSrcReg(*PrevMI);

1486

1488 if (MRI.hasOneUse(PrevRegSrc)) {

1489 if (TmpDef != &DefMI) {

1491 isArtifactCast(TmpDef->getOpcode()) ||

1493 "Expecting copy or artifact cast here");

1494

1496 }

1497 } else

1498 break;

1499 PrevMI = TmpDef;

1500 }

1501

1502 if (PrevMI == &DefMI) {

1503 unsigned I = 0;

1506 if (I != DefIdx) {

1507 if ( MRI .use_empty(Def.getReg())) {

1509 break;

1510 }

1511 } else {

1512 if ( MRI .hasOneUse(DefMI.getOperand(DefIdx).getReg()))

1513 break;

1514 }

1515

1516 ++I;

1517 }

1518

1521 }

1522 }

1523

1524

1525

1526

1527

1528

1531 unsigned DefIdx = 0) {

1533 markDefDead(MI, DefMI, DeadInsts, DefIdx);

1534 }

1535

1536

1537

1538

1539

1540

1541

1544 for (auto *DeadMI : DeadInsts) {

1545 LLVM_DEBUG(dbgs() << *DeadMI << "Is dead, eagerly deleting\n");

1547 DeadMI->eraseFromParent();

1548 }

1549 DeadInsts.clear();

1550 }

1551

1552

1553

1554 bool isInstUnsupported(const LegalityQuery &Query) const {

1556 auto Step = LI.getAction(Query);

1558 }

1559

1560 bool isInstLegal(const LegalityQuery &Query) const {

1562 }

1563

1564 bool isConstantUnsupported(LLT Ty) const {

1566 return isInstUnsupported({TargetOpcode::G_CONSTANT, {Ty}});

1567

1569 return isInstUnsupported({TargetOpcode::G_CONSTANT, {EltTy}}) ||

1570 isInstUnsupported({TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}});

1571 }

1572

1573

1574

1577 return TmpReg.isValid() ? TmpReg : Reg;

1578 }

1579};