SystemZISelDAGToDAG.cpp Source File (original) (raw)

21using namespace llvm;

23#define DEBUG_TYPE "systemz-isel"

24#define PASS_NAME "SystemZ DAG->DAG Pattern Instruction Selection"

26namespace {

28struct SystemZAddressingMode {

30 enum AddrForm {

32 FormBD,

35 FormBDXNormal,

38 FormBDXLA,

41 FormBDXDynAlloc

42 };

43 AddrForm Form;

48 enum DispRange {

49 Disp12Only,

50 Disp12Pair,

51 Disp20Only,

52 Disp20Only128,

53 Disp20Pair

54 };

55 DispRange DR;

61 int64_t Disp;

63 bool IncludesDynAlloc;

65 SystemZAddressingMode(AddrForm form, DispRange dr)

66 : Form(form), DR(dr), Disp(0), IncludesDynAlloc(false) {}

69 bool hasIndexField() { return Form != FormBD; }

72 bool isDynAlloc() { return Form == FormBDXDynAlloc; }

75 errs() << "SystemZAddressingMode " << this << '\n';

77 errs() << " Base ";

78 if (Base.getNode())

79 Base.getNode()->dump(DAG);

80 else

81 errs() << "null\n";

83 if (hasIndexField()) {

84 errs() << " Index ";

85 if (Index.getNode())

86 Index.getNode()->dump(DAG);

87 else

88 errs() << "null\n";

89 }

91 errs() << " Disp " << Disp;

92 if (IncludesDynAlloc)

93 errs() << " + ADJDYNALLOC";

94 errs() << '\n';

95 }

96};

104}

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119struct RxSBGOperands {

120 RxSBGOperands(unsigned Op, SDValue N)

121 : Opcode(Op), BitSize(N.getValueSizeInBits()),

122 Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63),

123 Rotate(0) {}

124

125 unsigned Opcode;

126 unsigned BitSize;

129 unsigned Start;

130 unsigned End;

131 unsigned Rotate;

132};

133

136

137

139 return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0));

140 }

141

144 }

145

148 }

149

150

151

152 bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const;

153

154

155 bool selectAddress(SDValue N, SystemZAddressingMode &AM) const;

156

157

158 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,

160 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT,

162

163

164

165

166 bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,

168

169

170

171

172 bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr,

174

175

176

177

178 bool selectBDXAddr(SystemZAddressingMode::AddrForm Form,

179 SystemZAddressingMode::DispRange DR, SDValue Addr,

181

182

186 return true;

187 }

188 return false;

189 }

190

191

193 return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp);

194 }

196 return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);

197 }

199 return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp);

200 }

202 return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);

203 }

204

205

207 return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp);

208 }

210 return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp);

211 }

212

213

216 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,

217 SystemZAddressingMode::Disp12Only,

218 Addr, Base, Disp, Index);

219 }

222 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,

223 SystemZAddressingMode::Disp12Pair,

224 Addr, Base, Disp, Index);

225 }

228 return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc,

229 SystemZAddressingMode::Disp12Only,

230 Addr, Base, Disp, Index);

231 }

234 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,

235 SystemZAddressingMode::Disp20Only,

236 Addr, Base, Disp, Index);

237 }

240 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,

241 SystemZAddressingMode::Disp20Only128,

242 Addr, Base, Disp, Index);

243 }

246 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal,

247 SystemZAddressingMode::Disp20Pair,

248 Addr, Base, Disp, Index);

249 }

252 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,

253 SystemZAddressingMode::Disp12Pair,

254 Addr, Base, Disp, Index);

255 }

258 return selectBDXAddr(SystemZAddressingMode::FormBDXLA,

259 SystemZAddressingMode::Disp20Pair,

260 Addr, Base, Disp, Index);

261 }

262

263

264

265

266

269

270

271

272

273 bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const;

274

275

276

277 bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const;

278

279

280

281 bool expandRxSBG(RxSBGOperands &RxSBG) const;

282

283

285

286

288

289

290

291 bool tryRISBGZero(SDNode *N);

292

293

294

295 bool tryRxSBG(SDNode *N, unsigned Opcode);

296

297

298

299

300

301

302

303

304 void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0,

306

309

311

312

313 bool tryGather(SDNode *N, unsigned Opcode);

314

315

316 bool tryScatter(StoreSDNode *Store, unsigned Opcode);

317

318

319

320

321 bool tryFoldLoadStoreIntoMemOperand(SDNode *Node);

322

323

324

325

326

327

328

329

330

331

333

334

335

336 bool storeLoadCanUseMVC(SDNode *N) const;

337

338

339

340

341 bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const;

342

343

344

345 bool storeLoadIsAligned(SDNode *N) const;

346

347

349

350

352

353

354

355

356 bool shouldSelectForReassoc(SDNode *N) const;

357

358public:

359 SystemZDAGToDAGISel() = delete;

360

363

364 bool runOnMachineFunction(MachineFunction &MF) override {

366 if (F.getFnAttribute("fentry-call").getValueAsString() != "true") {

367 if (F.hasFnAttribute("mnop-mcount"))

369 if (F.hasFnAttribute("mrecord-mcount"))

370 report_fatal_error("mrecord-mcount only supported with fentry-call");

371 }

372

375 }

376

377

379 bool SelectInlineAsmMemoryOperand(const SDValue &Op,

381 std::vector &OutOps) override;

383 void PreprocessISelDAG() override;

384

385

386 #include "SystemZGenDAGISel.inc"

387};

388

390public:

391 static char ID;

395 ID, std::make_unique(TM, OptLevel)) {}

396};

397}

398

399char SystemZDAGToDAGISelLegacy::ID = 0;

400

402

405 return new SystemZDAGToDAGISelLegacy(TM, OptLevel);

406}

407

408

409

410

411static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {

412 switch (DR) {

413 case SystemZAddressingMode::Disp12Only:

415

416 case SystemZAddressingMode::Disp12Pair:

417 case SystemZAddressingMode::Disp20Only:

418 case SystemZAddressingMode::Disp20Pair:

420

421 case SystemZAddressingMode::Disp20Only128:

423 }

425}

426

427

428

431 if (IsBase)

433 else

434 AM.Index = Value;

435}

436

437

438

439

442 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) {

444 AM.IncludesDynAlloc = true;

445 return true;

446 }

447 return false;

448}

449

450

451

454 if (AM.hasIndexField() && !AM.Index.getNode()) {

455 AM.Base = Base;

456 AM.Index = Index;

457 return true;

458 }

459 return false;

460}

461

462

463

464static bool expandDisp(SystemZAddressingMode &AM, bool IsBase,

466

467 int64_t TestDisp = AM.Disp + Op1;

470 AM.Disp = TestDisp;

471 return true;

472 }

473

474

475

476 return false;

477}

478

479bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM,

480 bool IsBase) const {

483

484 if (Opcode == ISD::TRUNCATE && N.getOperand(0).getValueSizeInBits() <= 64) {

485 N = N.getOperand(0);

486 Opcode = N.getOpcode();

487 }

488 if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(N)) {

489 SDValue Op0 = N.getOperand(0);

490 SDValue Op1 = N.getOperand(1);

491

492 unsigned Op0Code = Op0->getOpcode();

493 unsigned Op1Code = Op1->getOpcode();

494

495 if (Op0Code == SystemZISD::ADJDYNALLOC)

497 if (Op1Code == SystemZISD::ADJDYNALLOC)

499

506

508 return true;

509 }

510 if (Opcode == SystemZISD::PCREL_OFFSET) {

517 }

518 return false;

519}

520

521

522

523static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) {

525 switch (DR) {

526 case SystemZAddressingMode::Disp12Only:

527 case SystemZAddressingMode::Disp20Only:

528 case SystemZAddressingMode::Disp20Only128:

529 return true;

530

531 case SystemZAddressingMode::Disp12Pair:

532

534

535 case SystemZAddressingMode::Disp20Pair:

536

538 }

540}

541

542

544

546 return false;

547

548

549

550

552 return true;

553

554 if (Disp) {

555

556 if (Index)

557 return true;

558

559

560

562 return true;

563

564

565

567 return true;

568 } else {

569

570 if (!Index)

571 return false;

572

573

574

575 if (Index->hasOneUse())

576 return false;

577

578

579

580 unsigned IndexOpcode = Index->getOpcode();

583 return false;

584 }

585

586

587

588 if (Base->hasOneUse())

589 return false;

590

591 return true;

592}

593

594

595bool SystemZDAGToDAGISel::selectAddress(SDValue Addr,

596 SystemZAddressingMode &AM) const {

597

598

599 AM.Base = Addr;

600

601

605 ;

606

607 else if (Addr.getOpcode() == SystemZISD::ADJDYNALLOC &&

609 ;

610 else

611

612 while (expandAddress(AM, true) ||

613 (AM.Index.getNode() && expandAddress(AM, false)))

614 continue;

615

616

617 if (AM.Form == SystemZAddressingMode::FormBDXLA &&

619 return false;

620

621

623 return false;

624

625

626 if (AM.isDynAlloc() && !AM.IncludesDynAlloc)

627 return false;

628

630 return true;

631}

632

633

634

635

636

637

639 if (N->getNodeId() == -1 ||

643

644

645

646

649 }

650}

651

652void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,

655 Base = AM.Base;

656 if ( Base .getNode())

657

658 Base = CurDAG->getRegister(0, VT);

660

662 Base = CurDAG->getTargetFrameIndex(FrameIndex, VT);

663 } else if (Base.getValueType() != VT) {

664

665 assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 &&

666 "Unexpected truncation");

670 Base = Trunc;

671 }

672

673

674 Disp = CurDAG->getSignedTargetConstant(AM.Disp, SDLoc(Base), VT);

675}

676

677void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM,

681 getAddressOperands(AM, VT, Base, Disp);

682

683 Index = AM.Index;

684 if ( Index .getNode())

685

686 Index = CurDAG->getRegister(0, VT);

687}

688

689bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR,

692 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR);

693 if (!selectAddress(Addr, AM))

694 return false;

695

697 return true;

698}

699

700bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR,

703 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR);

704 if (!selectAddress(Addr, AM) || AM.Index.getNode())

705 return false;

706

708 return true;

709}

710

711bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form,

712 SystemZAddressingMode::DispRange DR,

715 SystemZAddressingMode AM(Form, DR);

716 if (!selectAddress(Addr, AM))

717 return false;

718

719 getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index);

720 return true;

721}

722

723bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem,

728 if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) &&

730 for (unsigned int I = 0; I < 2; ++I) {

733

734

738 Index.getOperand(1) == Elem) {

740 return true;

741 }

742 }

743 }

744 return false;

745}

746

747bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,

748 uint64_t InsertMask) const {

749

750

752 return false;

753

754

756 if (!MaskNode)

757 return false;

758

759

760 uint64_t AndMask = MaskNode->getZExtValue();

761 if (InsertMask & AndMask)

762 return false;

763

764

765

766 uint64_t Used = allOnes(Op.getValueSizeInBits());

767 if (Used != (AndMask | InsertMask)) {

768 KnownBits Known = CurDAG->computeKnownBits(Op.getOperand(0));

769 if (Used != (AndMask | InsertMask | Known.Zero.getZExtValue()))

770 return false;

771 }

772

773 Op = Op.getOperand(0);

774 return true;

775}

776

777bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG,

778 uint64_t Mask) const {

779 const SystemZInstrInfo *TII = getInstrInfo();

780 if (RxSBG.Rotate != 0)

781 Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate));

782 Mask &= RxSBG.Mask;

783 if (TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) {

784 RxSBG.Mask = Mask;

785 return true;

786 }

787 return false;

788}

789

790

792

793 if (RxSBG.Rotate != 0)

794 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)));

795 return (Mask & RxSBG.Mask) != 0;

796}

797

798bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {

800 unsigned Opcode = N.getOpcode();

801 switch (Opcode) {

803 if (RxSBG.Opcode == SystemZ::RNSBG)

804 return false;

805 if (N.getOperand(0).getValueSizeInBits() > 64)

806 return false;

807 uint64_t BitSize = N.getValueSizeInBits();

809 if (!refineRxSBGMask(RxSBG, Mask))

810 return false;

812 return true;

813 }

815 if (RxSBG.Opcode == SystemZ::RNSBG)

816 return false;

817

819 if (!MaskNode)

820 return false;

821

822 SDValue Input = N.getOperand(0);

823 uint64_t Mask = MaskNode->getZExtValue();

824 if (!refineRxSBGMask(RxSBG, Mask)) {

825

826

827

828 KnownBits Known = CurDAG->computeKnownBits(Input);

830 if (!refineRxSBGMask(RxSBG, Mask))

831 return false;

832 }

833 RxSBG.Input = Input;

834 return true;

835 }

836

838 if (RxSBG.Opcode != SystemZ::RNSBG)

839 return false;

840

842 if (!MaskNode)

843 return false;

844

846 uint64_t Mask = ~MaskNode->getZExtValue();

847 if (!refineRxSBGMask(RxSBG, Mask)) {

848

849

850

851 KnownBits Known = CurDAG->computeKnownBits(Input);

852 Mask &= ~Known.One.getZExtValue();

853 if (!refineRxSBGMask(RxSBG, Mask))

854 return false;

855 }

856 RxSBG.Input = Input;

857 return true;

858 }

859

861

862 if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64)

863 return false;

865 if (!CountNode)

866 return false;

867

868 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63;

870 return true;

871 }

872

874

876 return true;

877

879 if (RxSBG.Opcode != SystemZ::RNSBG) {

880

881 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();

882 if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize)))

883 return false;

884

886 return true;

887 }

888 [[fallthrough]];

889

891

892

893 unsigned BitSize = N.getValueSizeInBits();

894 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits();

896

897

898 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1)

899 RxSBG.Rotate += (BitSize - InnerBitSize);

900 else

901 return false;

902 }

903

905 return true;

906 }

907

910 if (!CountNode)

911 return false;

912

913 uint64_t Count = CountNode->getZExtValue();

914 unsigned BitSize = N.getValueSizeInBits();

916 return false;

917

918 if (RxSBG.Opcode == SystemZ::RNSBG) {

919

920

922 return false;

923 } else {

924

926 return false;

927 }

928

929 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63;

931 return true;

932 }

933

937 if (!CountNode)

938 return false;

939

940 uint64_t Count = CountNode->getZExtValue();

941 unsigned BitSize = N.getValueSizeInBits();

943 return false;

944

945 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) {

946

947

949 return false;

950 } else {

951

952

953 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count)))

954 return false;

955 }

956

957 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63;

959 return true;

960 }

961 default:

962 return false;

963 }

964}

965

966SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const {

967 SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);

969}

970

971SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT,

973 if (N.getValueType() == MVT::i32 && VT == MVT::i64)

974 return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32,

975 DL, VT, getUNDEF(DL, MVT::i64), N);

976 if (N.getValueType() == MVT::i64 && VT == MVT::i32)

977 return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N);

978 assert(N.getValueType() == VT && "Unexpected value types");

979 return N;

980}

981

982bool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) {

983 SDLoc DL(N);

984 EVT VT = N->getValueType(0);

986 return false;

987 RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0));

988 unsigned Count = 0;

989 while (expandRxSBG(RISBG))

990

991

992

997 return false;

998

999

1000

1002 return false;

1003

1004

1005

1006 if (Subtarget->hasMiscellaneousExtensions4() &&

1007 RISBG.Rotate >= 1 && RISBG.Rotate <= 4 &&

1008 RISBG.Mask == (((uint64_t)1 << 32) - 1) << RISBG.Rotate &&

1009 RISBG.Input.getOpcode() == ISD::ADD)

1012 return false;

1013

1014

1015

1016

1017 if (RISBG.Rotate == 0) {

1018 bool PreferAnd = false;

1019

1020 if (VT == MVT::i32)

1021 PreferAnd = true;

1022

1023

1024 else if (RISBG.Mask == 0xff ||

1025 RISBG.Mask == 0xffff ||

1026 RISBG.Mask == 0x7fffffff ||

1029 PreferAnd = true;

1030

1031

1033 if (Load->getMemoryVT() == MVT::i32 &&

1036 RISBG.Mask == 0xffffff00 &&

1037 Subtarget->hasLoadAndZeroRightmostByte())

1038 PreferAnd = true;

1039 }

1040 if (PreferAnd) {

1041

1042

1043

1044 SDValue In = convertTo(DL, VT, RISBG.Input);

1045 SDValue Mask = CurDAG->getConstant(RISBG.Mask, DL, VT);

1047 if (N != New.getNode()) {

1050 ReplaceNode(N, New.getNode());

1051 N = New.getNode();

1052 }

1053

1054 if (->isMachineOpcode())

1055 SelectCode(N);

1056 return true;

1057 }

1058 }

1059

1060 unsigned Opcode = SystemZ::RISBG;

1061

1062 if (Subtarget->hasMiscellaneousExtensions())

1063 Opcode = SystemZ::RISBGN;

1064 EVT OpcodeVT = MVT::i64;

1065 if (VT == MVT::i32 && Subtarget->hasHighWord() &&

1066

1067

1068

1069

1070 RISBG.Start >= 32 && RISBG.End >= RISBG.Start &&

1071 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 &&

1072 ((RISBG.End + RISBG.Rotate) & 63) >=

1073 ((RISBG.Start + RISBG.Rotate) & 63)) {

1074 Opcode = SystemZ::RISBMux;

1075 OpcodeVT = MVT::i32;

1076 RISBG.Start &= 31;

1077 RISBG.End &= 31;

1078 }

1080 getUNDEF(DL, OpcodeVT),

1081 convertTo(DL, OpcodeVT, RISBG.Input),

1082 CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32),

1083 CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32),

1084 CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32)

1085 };

1087 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops), 0));

1088 ReplaceNode(N, New.getNode());

1089 return true;

1090}

1091

1092bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) {

1093 SDLoc DL(N);

1094 EVT VT = N->getValueType(0);

1096 return false;

1097

1098

1099 RxSBGOperands RxSBG[] = {

1100 RxSBGOperands(Opcode, N->getOperand(0)),

1101 RxSBGOperands(Opcode, N->getOperand(1))

1102 };

1103 unsigned Count[] = { 0, 0 };

1104 for (unsigned I = 0; I < 2; ++I)

1105 while (RxSBG[I].Input->hasOneUse() && expandRxSBG(RxSBG[I]))

1106

1107

1108

1109

1110

1111

1115

1116

1118 return false;

1119

1120

1121 unsigned I = Count[0] > Count[1] ? 0 : 1;

1123

1124

1125 if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0)

1127 if (Load->getMemoryVT() == MVT::i8)

1128 return false;

1129

1130

1131

1132 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[I].Mask)) {

1133 Opcode = SystemZ::RISBG;

1134

1135 if (Subtarget->hasMiscellaneousExtensions())

1136 Opcode = SystemZ::RISBGN;

1137 }

1138

1140 convertTo(DL, MVT::i64, Op0),

1141 convertTo(DL, MVT::i64, RxSBG[I].Input),

1142 CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32),

1143 CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32),

1144 CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32)

1145 };

1147 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops), 0));

1148 ReplaceNode(N, New.getNode());

1149 return true;

1150}

1151

1152void SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node,

1153 SDValue Op0, uint64_t UpperVal,

1154 uint64_t LowerVal) {

1155 EVT VT = Node->getValueType(0);

1156 SDLoc DL(Node);

1157 SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT);

1159 Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper);

1160

1161 {

1162

1163

1164

1165

1166

1167

1168

1169

1170

1171

1172

1173

1174 HandleSDNode Handle(Upper);

1175 SelectCode(Upper.getNode());

1176 Upper = Handle.getValue();

1177 }

1178

1179 SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT);

1181

1182 ReplaceNode(Node, Or.getNode());

1183

1184 SelectCode(Or.getNode());

1185}

1186

1187void SystemZDAGToDAGISel::loadVectorConstant(

1188 const SystemZVectorConstantInfo &VCI, SDNode *Node) {

1189 assert((VCI.Opcode == SystemZISD::BYTE_MASK ||

1190 VCI.Opcode == SystemZISD::REPLICATE ||

1191 VCI.Opcode == SystemZISD::ROTATE_MASK) &&

1192 "Bad opcode!");

1194 EVT VT = Node->getValueType(0);

1195 SDLoc DL(Node);

1197 for (unsigned OpVal : VCI.OpVals)

1198 Ops.push_back(CurDAG->getTargetConstant(OpVal, DL, MVT::i32));

1200

1202 ReplaceNode(Node, Op.getNode());

1204 SDValue BitCast = CurDAG->getNode(ISD::BITCAST, DL, VT, Op);

1205 ReplaceNode(Node, BitCast.getNode());

1206 SelectCode(BitCast.getNode());

1207 } else {

1208 unsigned SubRegIdx = (VT.getSizeInBits() == 16 ? SystemZ::subreg_h16

1209 : VT.getSizeInBits() == 32 ? SystemZ::subreg_h32

1210 : SystemZ::subreg_h64);

1211 ReplaceNode(

1212 Node, CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, Op).getNode());

1213 }

1214 SelectCode(Op.getNode());

1215}

1216

1217SDNode *SystemZDAGToDAGISel::loadPoolVectorConstant(APInt Val, EVT VT, SDLoc DL) {

1218 SDNode *ResNode;

1220

1221 SDValue CP = CurDAG->getTargetConstantPool(

1222 ConstantInt::get(Type::getInt128Ty(*CurDAG->getContext()), Val),

1223 TLI->getPointerTy(CurDAG->getDataLayout()));

1224

1225 EVT PtrVT = CP.getValueType();

1227 SDValue(CurDAG->getMachineNode(SystemZ::LARL, DL, PtrVT, CP), 0),

1228 CurDAG->getTargetConstant(0, DL, PtrVT),

1229 CurDAG->getRegister(0, PtrVT),

1230 CurDAG->getEntryNode()

1231 };

1232 ResNode = CurDAG->getMachineNode(SystemZ::VL, DL, VT, MVT::Other, Ops);

1233

1234

1235

1236

1237 MachineFunction& MF = CurDAG->getMachineFunction();

1238 MachineMemOperand *MemOp =

1241

1243 return ResNode;

1244}

1245

1246bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {

1249 if (!ElemN)

1250 return false;

1251

1252 unsigned Elem = ElemN->getZExtValue();

1253 EVT VT = N->getValueType(0);

1255 return false;

1256

1258 if (!Load || Load ->hasNUsesOfValue(1, 0))

1259 return false;

1260 if (Load->getMemoryVT().getSizeInBits() !=

1261 Load->getValueType(0).getSizeInBits())

1262 return false;

1263

1265 if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) ||

1267 return false;

1268

1269 SDLoc DL(Load);

1271 N->getOperand(0), Base, Disp, Index,

1272 CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain()

1273 };

1274 SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops);

1276 ReplaceNode(N, Res);

1277 return true;

1278}

1279

1280bool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) {

1283 return false;

1284 if (Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits())

1285 return false;

1286

1289 if (!ElemN)

1290 return false;

1291

1294 unsigned Elem = ElemN->getZExtValue();

1296 return false;

1297

1299 if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) ||

1301 return false;

1302

1303 SDLoc DL(Store);

1305 Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32),

1306 Store->getChain()

1307 };

1308 ReplaceNode(Store, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));

1309 return true;

1310}

1311

1312

1313

1318

1319 if (StoredVal.getResNo() != 0)

1320 return false;

1321

1322

1324 return false;

1325

1326

1328 return false;

1329

1331

1333 return false;

1334

1335

1337

1338

1339 if (!Load.hasOneUse())

1340 return false;

1341

1342

1345 return false;

1346

1347

1348

1350

1351 bool ChainCheck = false;

1352 if (Chain == Load.getValue(1)) {

1353 ChainCheck = true;

1354 InputChain = LoadNode->getChain();

1359 const unsigned int Max = 1024;

1360 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) {

1362 if (Op == Load.getValue(1)) {

1363 ChainCheck = true;

1364

1365 ChainOps.push_back(Load.getOperand(0));

1366 continue;

1367 }

1370 }

1371

1372 if (ChainCheck) {

1373

1375 if (Op.getNode() != LoadNode)

1377

1378

1380 true))

1381 return false;

1382

1383

1384

1386 MVT::Other, ChainOps);

1387 }

1388 }

1389 if (!ChainCheck)

1390 return false;

1391

1392 return true;

1393}

1394

1395

1396

1397

1398

1399

1400

1401

1402

1403bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(SDNode *Node) {

1407 SDLoc DL(StoreNode);

1408

1409

1410

1411

1413 unsigned NewOpc = 0;

1414 bool NegateOperand = false;

1415 switch (Opc) {

1416 default:

1417 return false;

1418 case SystemZISD::SSUBO:

1419 NegateOperand = true;

1420 [[fallthrough]];

1421 case SystemZISD::SADDO:

1422 if (MemVT == MVT::i32)

1423 NewOpc = SystemZ::ASI;

1424 else if (MemVT == MVT::i64)

1425 NewOpc = SystemZ::AGSI;

1426 else

1427 return false;

1428 break;

1429 case SystemZISD::USUBO:

1430 NegateOperand = true;

1431 [[fallthrough]];

1432 case SystemZISD::UADDO:

1433 if (MemVT == MVT::i32)

1434 NewOpc = SystemZ::ALSI;

1435 else if (MemVT == MVT::i64)

1436 NewOpc = SystemZ::ALGSI;

1437 else

1438 return false;

1439 break;

1440 }

1441

1442 LoadSDNode *LoadNode = nullptr;

1445 InputChain))

1446 return false;

1447

1450 if (!OperandC)

1451 return false;

1452 auto OperandV = OperandC->getAPIntValue();

1453 if (NegateOperand)

1454 OperandV = -OperandV;

1455 if (OperandV.getSignificantBits() > 8)

1456 return false;

1457 Operand = CurDAG->getTargetConstant(OperandV, DL, MemVT);

1458

1460 if (!selectBDAddr20Only(StoreNode->getBasePtr(), Base, Disp))

1461 return false;

1462

1463 SDValue Ops[] = { Base, Disp, Operand, InputChain };

1464 MachineSDNode *Result =

1465 CurDAG->getMachineNode(NewOpc, DL, MVT::i32, MVT::Other, Ops);

1466 CurDAG->setNodeMemRefs(

1468

1469 ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1));

1471 CurDAG->RemoveDeadNode(Node);

1472 return true;

1473}

1474

1475bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store,

1476 LoadSDNode *Load) const {

1477

1478 if (Load->getMemoryVT() != Store->getMemoryVT())

1479 return false;

1480

1481

1482 if (Load->isVolatile() || Store->isVolatile())

1483 return false;

1484

1485

1486 if (Load->isInvariant() && Load->isDereferenceable())

1487 return true;

1488

1489

1490 const Value *V1 = Load->getMemOperand()->getValue();

1491 const Value *V2 = Store->getMemOperand()->getValue();

1492 if (!V1 || !V2)

1493 return false;

1494

1495

1496 uint64_t Size = Load->getMemoryVT().getStoreSize();

1497 int64_t End1 = Load->getSrcValueOffset() + Size;

1498 int64_t End2 = Store->getSrcValueOffset() + Size;

1499 if (V1 == V2 && End1 == End2)

1500 return false;

1501

1502 return BatchAA->isNoAlias(MemoryLocation(V1, End1, Load->getAAInfo()),

1503 MemoryLocation(V2, End2, Store->getAAInfo()));

1504}

1505

1506bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {

1509

1510

1511

1512 uint64_t Size = Load->getMemoryVT().getStoreSize();

1513 if (Size > 1 && Size <= 8) {

1514

1516 return false;

1517

1519 return false;

1520 }

1521

1522 return canUseBlockOperation(Store, Load);

1523}

1524

1525bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N,

1526 unsigned I) const {

1528 auto *LoadA = cast(StoreA->getValue().getOperand(1 - I));

1529 auto *LoadB = cast(StoreA->getValue().getOperand(I));

1530 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() &&

1531 canUseBlockOperation(StoreA, LoadB);

1532}

1533

1534bool SystemZDAGToDAGISel::storeLoadIsAligned(SDNode *N) const {

1535

1538 TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize();

1540 MachineMemOperand *MMO = MemAccess->getMemOperand();

1541 assert(MMO && "Expected a memory operand.");

1542

1543

1544

1545 if (MemAccess->getAlign().value() < StoreSize ||

1546 (LdSt && !LdSt->getOffset().isUndef()))

1547 return false;

1548

1549

1550 if (MMO->getOffset() % StoreSize != 0)

1551 return false;

1552

1553

1554 if (const PseudoSourceValue *PSV = MMO->getPseudoValue())

1555 if ((PSV->isGOT() || PSV->isConstantPool()))

1556 return true;

1557

1558

1559 if (BasePtr.getNumOperands())

1560 if (GlobalAddressSDNode *GA =

1562

1563 if (GA->getOffset() % StoreSize != 0)

1564 return false;

1565

1566

1567 const GlobalValue *GV = GA->getGlobal();

1570 return false;

1571 }

1572

1573 return true;

1574}

1575

1576ISD::LoadExtType SystemZDAGToDAGISel::getLoadExtType(SDNode *N) const {

1579 ETy = L->getExtensionType();

1581 ETy = AL->getExtensionType();

1582 else

1584 return ETy;

1585}

1586

1587void SystemZDAGToDAGISel::Select(SDNode *Node) {

1588

1589 if (Node->isMachineOpcode()) {

1591 Node->setNodeId(-1);

1592 return;

1593 }

1594

1595 unsigned Opcode = Node->getOpcode();

1596 switch (Opcode) {

1599 if (tryRxSBG(Node, SystemZ::ROSBG))

1600 return;

1601 goto or_xor;

1602

1605 if (tryRxSBG(Node, SystemZ::RXSBG))

1606 return;

1607

1608 or_xor:

1609

1610

1611

1612 if (Node->getValueType(0) == MVT::i64 &&

1615 uint64_t Val = Op1->getZExtValue();

1616

1617

1618 if (Subtarget->hasMiscellaneousExtensions3()) {

1619 unsigned ChildOpcode = Node->getOperand(0).getOpcode();

1620

1621 if (Val == (uint64_t)-1 && Opcode == ISD::XOR)

1622 if (ChildOpcode == ISD::AND || ChildOpcode == ISD::OR ||

1624 break;

1625

1626

1627 if (ChildOpcode == ISD::XOR) {

1628 auto Op0 = Node->getOperand(0);

1630 if (Op0Op1->getZExtValue() == (uint64_t)-1)

1631 break;

1632 }

1633 }

1634

1635 if (Opcode == ISD::XOR && Op1->isAllOnes())

1636 break;

1638 splitLargeImmediate(Opcode, Node, Node->getOperand(0),

1639 Val - uint32_t(Val), uint32_t(Val));

1640 return;

1641 }

1642 }

1643 break;

1644

1647 if (tryRxSBG(Node, SystemZ::RNSBG))

1648 return;

1649 [[fallthrough]];

1654 if (tryRISBGZero(Node))

1655 return;

1656 break;

1657

1659 if (Node->getValueType(0) == MVT::i128) {

1660 SDLoc DL(Node);

1662 Src = CurDAG->getNode(ISD::BITCAST, DL, MVT::v16i8, Src);

1663

1664 uint64_t Bytes[2] = { 0x0706050403020100ULL, 0x0f0e0d0c0b0a0908ULL };

1665 SDNode *Mask = loadPoolVectorConstant(APInt(128, Bytes), MVT::v16i8, DL);

1667 SDValue Res = SDValue(CurDAG->getMachineNode(SystemZ::VPERM, DL,

1668 MVT::v16i8, Ops), 0);

1669

1670 Res = CurDAG->getNode(ISD::BITCAST, DL, MVT::i128, Res);

1671 SDNode *ResNode = Res.getNode();

1672 ReplaceNode(Node, ResNode);

1673 SelectCode(Src.getNode());

1674 SelectCode(ResNode);

1675 return;

1676 }

1677 break;

1678

1680

1681

1682 if (Node->getValueType(0) == MVT::i64) {

1683 uint64_t Val = Node->getAsZExtVal();

1685 splitLargeImmediate(ISD::OR, Node, SDValue(), Val - uint32_t(Val),

1686 uint32_t(Val));

1687 return;

1688 }

1689 }

1690 if (Node->getValueType(0) == MVT::i128) {

1691 const APInt &Val = Node->getAsAPIntVal();

1692 SystemZVectorConstantInfo VCI(Val);

1694 loadVectorConstant(VCI, Node);

1695 return;

1696 }

1697

1698 SDNode *ResNode = loadPoolVectorConstant(Val, MVT::i128, SDLoc(Node));

1699 ReplaceNode(Node, ResNode);

1700 return;

1701 }

1702 break;

1703

1704 case SystemZISD::SELECT_CCMASK: {

1707

1708

1709 if ((Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) ||

1710 (Subtarget->hasLoadStoreOnCond2() &&

1711 Node->getValueType(0).isInteger() &&

1712 Node->getValueType(0).getSizeInBits() <= 64 &&

1721

1722 CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask,

1725 SDNode *UpdatedNode =

1726 CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4);

1727 if (UpdatedNode != Node) {

1728

1729 ReplaceNode(Node, UpdatedNode);

1730 Node = UpdatedNode;

1731 }

1732 }

1733 break;

1734 }

1735

1737 EVT VT = Node->getValueType(0);

1739 if (ElemBitSize == 32) {

1740 if (tryGather(Node, SystemZ::VGEF))

1741 return;

1742 } else if (ElemBitSize == 64) {

1743 if (tryGather(Node, SystemZ::VGEG))

1744 return;

1745 }

1746 break;

1747 }

1748

1751 SystemZVectorConstantInfo VCI(BVN);

1753 loadVectorConstant(VCI, Node);

1754 return;

1755 }

1756 break;

1757 }

1758

1761 if (Imm.isZero() || Imm.isNegZero())

1762 break;

1763 SystemZVectorConstantInfo VCI(Imm);

1765 assert(Success && "Expected legal FP immediate");

1766 loadVectorConstant(VCI, Node);

1767 return;

1768 }

1769

1770 case ISD::STORE: {

1771 if (tryFoldLoadStoreIntoMemOperand(Node))

1772 return;

1774 unsigned ElemBitSize = Store->getValue().getValueSizeInBits();

1775 if (ElemBitSize == 32) {

1776 if (tryScatter(Store, SystemZ::VSCEF))

1777 return;

1778 } else if (ElemBitSize == 64) {

1779 if (tryScatter(Store, SystemZ::VSCEG))

1780 return;

1781 }

1782 break;

1783 }

1784

1785 case ISD::ATOMIC_STORE: {

1787

1788

1789

1791 AtomOp->getChain(), SDLoc(AtomOp), AtomOp->getVal(),

1792 AtomOp->getBasePtr(), AtomOp->getMemoryVT(), AtomOp->getMemOperand()));

1794 SDNode *Chain = St;

1795

1796

1797 if (AtomOp->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent)

1798 Chain = CurDAG->getMachineNode(SystemZ::Serialize, SDLoc(AtomOp),

1799 MVT::Other, SDValue(Chain, 0));

1800 ReplaceNode(Node, Chain);

1801 SelectCode(St);

1802 return;

1803 }

1804 }

1805

1806 SelectCode(Node);

1807}

1808

1809bool SystemZDAGToDAGISel::SelectInlineAsmMemoryOperand(

1811 std::vector &OutOps) {

1812 SystemZAddressingMode::AddrForm Form;

1813 SystemZAddressingMode::DispRange DispRange;

1815

1816 switch(ConstraintID) {

1817 default:

1819 case InlineAsm::ConstraintCode::i:

1820 case InlineAsm::ConstraintCode::Q:

1821 case InlineAsm::ConstraintCode::ZQ:

1822

1823 Form = SystemZAddressingMode::FormBD;

1824 DispRange = SystemZAddressingMode::Disp12Only;

1825 break;

1826 case InlineAsm::ConstraintCode::R:

1827 case InlineAsm::ConstraintCode::ZR:

1828

1829 Form = SystemZAddressingMode::FormBDXNormal;

1830 DispRange = SystemZAddressingMode::Disp12Only;

1831 break;

1832 case InlineAsm::ConstraintCode::S:

1833 case InlineAsm::ConstraintCode::ZS:

1834

1835 Form = SystemZAddressingMode::FormBD;

1836 DispRange = SystemZAddressingMode::Disp20Only;

1837 break;

1838 case InlineAsm::ConstraintCode::T:

1839 case InlineAsm::ConstraintCode::m:

1840 case InlineAsm::ConstraintCode::o:

1841 case InlineAsm::ConstraintCode::p:

1842 case InlineAsm::ConstraintCode::ZT:

1843

1844

1845

1846

1847 Form = SystemZAddressingMode::FormBDXNormal;

1848 DispRange = SystemZAddressingMode::Disp20Only;

1849 break;

1850 }

1851

1852 if (selectBDXAddr(Form, DispRange, Op, Base, Disp, Index)) {

1853 const TargetRegisterClass *TRC =

1856 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), DL, MVT::i32);

1857

1858

1859

1863 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,

1864 DL, Base.getValueType(),

1865 Base, RC), 0);

1866 }

1867

1868

1871 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,

1872 DL, Index.getValueType(),

1873 Index, RC), 0);

1874 }

1875

1876 OutOps.push_back(Base);

1877 OutOps.push_back(Disp);

1878 OutOps.push_back(Index);

1879 return false;

1880 }

1881

1882 return true;

1883}

1884

1885

1886

1887bool

1888SystemZDAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U,

1889 SDNode *Root) const {

1890

1891

1892 if (N.getOpcode() == ISD::LOAD && U->getOpcode() == SystemZISD::ICMP) {

1893 if (.hasOneUse() || ->hasOneUse())

1894 return false;

1895

1896

1897

1898

1899

1900 SDNode *CCUser = *U->user_begin();

1901 SDNode *CCRegUser = nullptr;

1904 for (auto *U : CCUser->users()) {

1905 if (CCRegUser == nullptr)

1906 CCRegUser = U;

1907 else if (CCRegUser != U)

1908 return false;

1909 }

1910 }

1911 if (CCRegUser == nullptr)

1912 return false;

1913

1914

1915

1919

1920

1921

1922

1923

1924 if (!IsLegalToFold(N, U, CCRegUser, OptLevel, false))

1925 return false;

1926 }

1927

1928 return true;

1929}

1930

1931namespace {

1932

1933

1934struct IPMConversion {

1935 IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit)

1936 : XORValue(xorValue), AddValue(addValue), Bit(bit) {}

1937

1938 int64_t XORValue;

1939 int64_t AddValue;

1940 unsigned Bit;

1941};

1942}

1943

1944

1945

1946

1948

1949

1954

1955

1956

1957

1958

1959

1960

1961

1972 return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31);

1976 return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31);

1977

1978

1979

1982

1983

1984

1989

1990

1991

1992

2007

2009}

2010

2011SDValue SystemZDAGToDAGISel::expandSelectBoolean(SDNode *Node) {

2014 if (!TrueOp || !FalseOp)

2016 if (FalseOp->getZExtValue() != 0)

2018 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1)

2020

2023 if (!CCValidOp || !CCMaskOp)

2025 int CCValid = CCValidOp->getZExtValue();

2026 int CCMask = CCMaskOp->getZExtValue();

2027

2028 SDLoc DL(Node);

2031 SDValue Result = CurDAG->getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);

2032

2033 if (IPM.XORValue)

2035 CurDAG->getConstant(IPM.XORValue, DL, MVT::i32));

2036

2037 if (IPM.AddValue)

2039 CurDAG->getNode(ISD::ADD, DL, MVT::i32, Result,

2040 CurDAG->getSignedConstant(IPM.AddValue, DL, MVT::i32));

2041

2042 EVT VT = Node->getValueType(0);

2043 if (VT == MVT::i32 && IPM.Bit == 31) {

2044 unsigned ShiftOp = TrueOp->getSExtValue() == 1 ? ISD::SRL : ISD::SRA;

2045 Result = CurDAG->getNode(ShiftOp, DL, MVT::i32, Result,

2046 CurDAG->getConstant(IPM.Bit, DL, MVT::i32));

2047 } else {

2048 if (VT != MVT::i32)

2050

2051 if (TrueOp->getSExtValue() == 1) {

2052

2054 CurDAG->getConstant(IPM.Bit, DL, MVT::i32));

2056 CurDAG->getConstant(1, DL, VT));

2057 } else {

2058

2062 CurDAG->getConstant(ShlAmt, DL, MVT::i32));

2064 CurDAG->getConstant(SraAmt, DL, MVT::i32));

2065 }

2066 }

2067

2069}

2070

2071bool SystemZDAGToDAGISel::shouldSelectForReassoc(SDNode *N) const {

2072 EVT VT = N->getValueType(0);

2074 return N->getFlags().hasAllowReassociation() &&

2075 N->getFlags().hasNoSignedZeros() && Subtarget->hasVector() &&

2076 (VT != MVT::f32 || Subtarget->hasVectorEnhancements1()) &&

2077 ->isStrictFPOpcode();

2078}

2079

2080void SystemZDAGToDAGISel::PreprocessISelDAG() {

2081

2082

2083 if (Subtarget->hasLoadStoreOnCond2())

2084 return;

2085

2086 bool MadeChange = false;

2087

2089 E = CurDAG->allnodes_end();

2090 I != E;) {

2091 SDNode *N = &*I++;

2092 if (N->use_empty())

2093 continue;

2094

2096 switch (N->getOpcode()) {

2097 default: break;

2098 case SystemZISD::SELECT_CCMASK:

2099 Res = expandSelectBoolean(N);

2100 break;

2101 }

2102

2103 if (Res) {

2104 LLVM_DEBUG(dbgs() << "SystemZ DAG preprocessing replacing:\nOld: ");

2109

2110 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);

2111 MadeChange = true;

2112 }

2113 }

2114

2115 if (MadeChange)

2116 CurDAG->RemoveDeadNodes();

2117}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

const TargetInstrInfo & TII

static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)

AMDGPU Register Bank Select

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]

#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)

static void changeComponent(SystemZAddressingMode &AM, bool IsBase, SDValue Value)

Definition SystemZISelDAGToDAG.cpp:429

static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask)

Definition SystemZISelDAGToDAG.cpp:1947

static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val)

Definition SystemZISelDAGToDAG.cpp:411

static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, SDValue Value)

Definition SystemZISelDAGToDAG.cpp:440

static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, SDValue StoredVal, SelectionDAG *CurDAG, LoadSDNode *&LoadNode, SDValue &InputChain)

Definition SystemZISelDAGToDAG.cpp:1314

static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val)

Definition SystemZISelDAGToDAG.cpp:523

static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, SDValue Index)

Definition SystemZISelDAGToDAG.cpp:452

static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask)

Definition SystemZISelDAGToDAG.cpp:791

static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, SDValue Op0, uint64_t Op1)

Definition SystemZISelDAGToDAG.cpp:464

static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index)

Definition SystemZISelDAGToDAG.cpp:543

static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N)

Definition SystemZISelDAGToDAG.cpp:638

static uint64_t allOnes(unsigned int Count)

The Input class is used to parse a yaml document into in-memory structs and vectors.

Class for arbitrary precision integers.

uint64_t getZExtValue() const

Get zero extended value.

FunctionPass class - This class is used to implement most global optimizations.

LLVM_ABI const DataLayout & getDataLayout() const

Get the data layout of the module this global belongs to.

This class is used to represent ISD::LOAD nodes.

const SDValue & getBasePtr() const

const SDValue & getOffset() const

TypeSize getSizeInBits() const

Returns the size of the specified MVT in bits.

const TargetSubtargetInfo & getSubtarget() const

getSubtarget - Return the subtarget for which this machine code is being compiled.

MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)

getMachineMemOperand - Allocate a new MachineMemOperand.

Function & getFunction()

Return the LLVM function that this machine code represents.

const PseudoSourceValue * getPseudoValue() const

bool isAtomic() const

Returns true if this operation has an atomic ordering requirement of unordered or higher,...

@ MOLoad

The memory access reads data.

int64_t getOffset() const

For normal values, this is a byte offset added to the base address.

MachineMemOperand * getMemOperand() const

Return a MachineMemOperand object describing the memory reference performed by operation.

const SDValue & getChain() const

bool isNonTemporal() const

EVT getMemoryVT() const

Return the type of the in-memory value.

Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...

Represents one node in the SelectionDAG.

ArrayRef< SDUse > ops() const

bool isMachineOpcode() const

Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.

int getNodeId() const

Return the unique node id.

LLVM_ABI void dump() const

Dump this node, for debugging.

unsigned getOpcode() const

Return the SelectionDAG opcode value for this node.

bool hasOneUse() const

Return true if there is exactly one use of this node.

static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)

Returns true if N is a predecessor of any node in Worklist.

uint64_t getAsZExtVal() const

Helper method returns the zero-extended integer value of a ConstantSDNode.

unsigned getMachineOpcode() const

This may only be called if isMachineOpcode returns true.

const SDValue & getOperand(unsigned Num) const

bool hasNUsesOfValue(unsigned NUses, unsigned Value) const

Return true if there are exactly NUSES uses of the indicated value.

iterator_range< user_iterator > users()

Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.

SDNode * getNode() const

get the SDNode which holds the desired result

EVT getValueType() const

Return the ValueType of the referenced return value.

const SDValue & getOperand(unsigned i) const

unsigned getResNo() const

get the index which selects a specific result in the SDNode

unsigned getOpcode() const

unsigned getNumOperands() const

SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...

static int getUninvalidatedNodeId(SDNode *N)

virtual bool runOnMachineFunction(MachineFunction &mf)

static void InvalidateNodeId(SDNode *N)

This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...

LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)

Gets or creates the specified node.

void RepositionNode(allnodes_iterator Position, SDNode *N)

Move node N in the AllNodes list to be immediately before the given iterator Position.

ilist< SDNode >::iterator allnodes_iterator

SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

This class is used to represent ISD::STORE nodes.

const SDValue & getBasePtr() const

const SDValue & getOffset() const

const SystemZInstrInfo * getInstrInfo() const override

const SystemZRegisterInfo * getRegisterInfo() const override

unsigned getID() const

Return the register class ID number.

Target - Wrapper for Target specific information.

LLVM Value Representation.

LLVM_ABI Align getPointerAlignment(const DataLayout &DL) const

Returns an alignment of the pointer value.

self_iterator getIterator()

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

constexpr char Align[]

Key for Kernel::Arg::Metadata::mAlign.

constexpr std::underlying_type_t< E > Mask()

Get a bitmask with 1s in all places up to the high-order bit of E's largest value.

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

@ C

The default llvm calling convention, compatible with C.

@ BSWAP

Byte Swap and Counting operators.

@ ADD

Simple integer binary arithmetic operators.

@ ANY_EXTEND

ANY_EXTEND - Used for integer types. The high bits are undefined.

@ SIGN_EXTEND

Conversion operators.

@ SHL

Shift and rotation operations.

@ EXTRACT_VECTOR_ELT

EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...

@ CopyToReg

CopyToReg - This node has three operands: a chain, a register number to set to this value,...

@ ZERO_EXTEND

ZERO_EXTEND - Used for integer types, zeroing the new bits.

@ SIGN_EXTEND_INREG

SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...

@ AND

Bitwise operators - logical and, logical or, logical xor.

@ INSERT_VECTOR_ELT

INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.

@ TokenFactor

TokenFactor - This node takes multiple tokens as input and produces a single token result.

@ TRUNCATE

TRUNCATE - Completely drop the high bits.

@ BUILD_VECTOR

BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...

bool isNormalStore(const SDNode *N)

Returns true if the specified node is a non-truncating and unindexed store.

LoadExtType

LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).

bool isNormalLoad(const SDNode *N)

Returns true if the specified node is a non-extending and unindexed load.

bool isPCREL(unsigned Opcode)

static bool isImmHF(uint64_t Val)

static bool isImmLF(uint64_t Val)

NodeAddr< NodeBase * > Node

This is an optimization pass for GlobalISel generic memory operations.

void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)

FunctionAddr VTableAddr Value

FunctionPass * createSystemZISelDag(SystemZTargetMachine &TM, CodeGenOptLevel OptLevel)

Definition SystemZISelDAGToDAG.cpp:403

constexpr bool isInt(int64_t x)

Checks if an integer fits into the given bit width.

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)

FunctionAddr VTableAddr Count

constexpr bool isUInt(uint64_t x)

Checks if an unsigned integer fits into the given bit width.

CodeGenOptLevel

Code generation optimization level.

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

LLVM_ABI raw_fd_ostream & errs()

This returns a reference to a raw_ostream for standard error.

@ Or

Bitwise or logical OR of integers.

DWARFExpression::Operation Op

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

constexpr uint64_t value() const

This is a hole in the type system and should not be abused.

EVT changeVectorElementTypeToInteger() const

Return a vector with the same number of elements as this vector, but with the element type converted ...

bool isFloatingPoint() const

Return true if this is a FP or a vector FP type.

TypeSize getSizeInBits() const

Return the size of the specified value type in bits.

uint64_t getScalarSizeInBits() const

MVT getSimpleVT() const

Return the SimpleValueType held in the specified simple EVT.

unsigned getVectorNumElements() const

Given a vector type, return the number of elements it contains.

bool isInteger() const

Return true if this is an integer or a vector integer type.

static LLVM_ABI MachinePointerInfo getConstantPool(MachineFunction &MF)

Return a MachinePointerInfo record that refers to the constant pool.

const TargetRegisterClass * getPointerRegClass(unsigned Kind=0) const override

getPointerRegClass - Return the register class to use to hold pointers.

SmallVector< unsigned, 2 > OpVals

bool isVectorConstantLegal(const SystemZSubtarget &Subtarget)