RISCVInsertVSETVLI.cpp Source File (original) (raw)

35#include

36using namespace llvm;

38#define DEBUG_TYPE "riscv-insert-vsetvli"

39#define RISCV_INSERT_VSETVLI_NAME "RISC-V Insert VSETVLI pass"

41 STATISTIC(NumInsertedVSETVL, "Number of VSETVL inst inserted");

42 STATISTIC(NumCoalescedVSETVL, "Number of VSETVL inst coalesced");

46 cl::desc("Insert vsetvlis before vmvNr.vs to ensure vtype is valid and "

47 "vill is cleared"),

50namespace {

58 if (!LIS)

59 return nullptr;

62 return LI.getVNInfoBefore(SI);

63}

67}

71}

73static unsigned getVecPolicyOpNum(const MachineInstr &MI) {

75}

79static std::optional getEEWForLoadStore(const MachineInstr &MI) {

81 default:

82 return std::nullopt;

83 case RISCV::VLE8_V:

84 case RISCV::VLSE8_V:

85 case RISCV::VSE8_V:

86 case RISCV::VSSE8_V:

87 return 8;

88 case RISCV::VLE16_V:

89 case RISCV::VLSE16_V:

90 case RISCV::VSE16_V:

91 case RISCV::VSSE16_V:

92 return 16;

93 case RISCV::VLE32_V:

94 case RISCV::VLSE32_V:

95 case RISCV::VSE32_V:

96 case RISCV::VSSE32_V:

97 return 32;

98 case RISCV::VLE64_V:

99 case RISCV::VLSE64_V:

100 case RISCV::VSE64_V:

101 case RISCV::VSSE64_V:

102 return 64;

103 }

104}

105

106

107

110 return false;

111 const unsigned Log2SEW = MI.getOperand(getSEWOpNum(MI)).getImm();

112

113 return Log2SEW == 0;

114}

115

116

117

118

119

120static bool hasUndefinedPassthru(const MachineInstr &MI) {

121

122 unsigned UseOpIdx;

123 if (.isRegTiedToUseOperand(0, &UseOpIdx))

124

125

126 return true;

127

128

129

132}

133

134

137 return MI.isCopy() && MI.getOperand(0).getReg().isPhysical() &&

139 TRI->getMinimalPhysRegClass(MI.getOperand(0).getReg()));

140}

141

142

143struct DemandedFields {

144

145

146 bool VLAny = false;

147

148 bool VLZeroness = false;

149

150 enum : uint8_t {

151 SEWEqual = 3,

152 SEWGreaterThanOrEqualAndLessThan64 =

153 2,

154

155

156 SEWGreaterThanOrEqual = 1,

157

158 SEWNone = 0

159 } SEW = SEWNone;

160 enum : uint8_t {

161 LMULEqual = 2,

162 LMULLessThanOrEqualToM1 = 1,

163 LMULNone = 0

164 } LMUL = LMULNone;

165 bool SEWLMULRatio = false;

166 bool TailPolicy = false;

167 bool MaskPolicy = false;

168

169

170 bool VILL = false;

171 bool TWiden = false;

172 bool AltFmt = false;

173

174

175 bool usedVTYPE() const {

176 return SEW || LMUL || SEWLMULRatio || TailPolicy || MaskPolicy || VILL ||

177 TWiden || AltFmt;

178 }

179

180

181 bool usedVL() {

182 return VLAny || VLZeroness;

183 }

184

185

186 void demandVTYPE() {

187 SEW = SEWEqual;

188 LMUL = LMULEqual;

189 SEWLMULRatio = true;

190 TailPolicy = true;

191 MaskPolicy = true;

192 VILL = true;

193 TWiden = true;

194 AltFmt = true;

195 }

196

197

198 void demandVL() {

199 VLAny = true;

200 VLZeroness = true;

201 }

202

203 static DemandedFields all() {

204 DemandedFields DF;

205 DF.demandVTYPE();

206 DF.demandVL();

207 return DF;

208 }

209

210

211 void doUnion(const DemandedFields &B) {

212 VLAny |= B.VLAny;

213 VLZeroness |= B.VLZeroness;

214 SEW = std::max(SEW, B.SEW);

215 LMUL = std::max(LMUL, B.LMUL);

216 SEWLMULRatio |= B.SEWLMULRatio;

217 TailPolicy |= B.TailPolicy;

218 MaskPolicy |= B.MaskPolicy;

219 VILL |= B.VILL;

220 AltFmt |= B.AltFmt;

221 TWiden |= B.TWiden;

222 }

223

224#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

225

228 dbgs() << "\n";

229 }

230

231

232 void print(raw_ostream &OS) const {

233 OS << "{";

234 OS << "VLAny=" << VLAny << ", ";

235 OS << "VLZeroness=" << VLZeroness << ", ";

236 OS << "SEW=";

237 switch (SEW) {

238 case SEWEqual:

239 OS << "SEWEqual";

240 break;

241 case SEWGreaterThanOrEqual:

242 OS << "SEWGreaterThanOrEqual";

243 break;

244 case SEWGreaterThanOrEqualAndLessThan64:

245 OS << "SEWGreaterThanOrEqualAndLessThan64";

246 break;

247 case SEWNone:

248 OS << "SEWNone";

249 break;

250 };

251 OS << ", ";

252 OS << "LMUL=";

253 switch (LMUL) {

254 case LMULEqual:

255 OS << "LMULEqual";

256 break;

257 case LMULLessThanOrEqualToM1:

258 OS << "LMULLessThanOrEqualToM1";

259 break;

260 case LMULNone:

261 OS << "LMULNone";

262 break;

263 };

264 OS << ", ";

265 OS << "SEWLMULRatio=" << SEWLMULRatio << ", ";

266 OS << "TailPolicy=" << TailPolicy << ", ";

267 OS << "MaskPolicy=" << MaskPolicy << ", ";

268 OS << "VILL=" << VILL << ", ";

269 OS << "AltFmt=" << AltFmt << ", ";

270 OS << "TWiden=" << TWiden;

271 OS << "}";

272 }

273#endif

274};

275

276#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

279 DF.print(OS);

280 return OS;

281}

282#endif

283

286 return Fractional || LMul == 1;

287}

288

289

290

291

292static bool areCompatibleVTYPEs(uint64_t CurVType, uint64_t NewVType,

293 const DemandedFields &Used) {

294 switch (Used.SEW) {

295 case DemandedFields::SEWNone:

296 break;

297 case DemandedFields::SEWEqual:

299 return false;

300 break;

301 case DemandedFields::SEWGreaterThanOrEqual:

303 return false;

304 break;

305 case DemandedFields::SEWGreaterThanOrEqualAndLessThan64:

308 return false;

309 break;

310 }

311

312 switch (Used.LMUL) {

313 case DemandedFields::LMULNone:

314 break;

315 case DemandedFields::LMULEqual:

317 return false;

318 break;

319 case DemandedFields::LMULLessThanOrEqualToM1:

321 return false;

322 break;

323 }

324

325 if (Used.SEWLMULRatio) {

330 if (Ratio1 != Ratio2)

331 return false;

332 }

333

336 return false;

339 return false;

345 return false;

346 if (Used.AltFmt &&

348 return false;

349 return true;

350}

351

352

354

355

356

357

358

359

360 DemandedFields Res;

361

362 if (MI.isCall() || MI.isInlineAsm() ||

363 MI.readsRegister(RISCV::VL, nullptr))

364 Res.demandVL();

365 if (MI.isCall() || MI.isInlineAsm() ||

366 MI.readsRegister(RISCV::VTYPE, nullptr))

367 Res.demandVTYPE();

368

369 uint64_t TSFlags = MI.getDesc().TSFlags;

371 Res.demandVTYPE();

374 !VLOp.isReg() || !VLOp.isUndef())

375 Res.demandVL();

376

377

379 Res.MaskPolicy = false;

380 }

381

382

383

384

385

386

387

388 if (getEEWForLoadStore(MI)) {

389 Res.SEW = DemandedFields::SEWNone;

390 Res.LMUL = DemandedFields::LMULNone;

391 }

392

393

395 Res.TailPolicy = false;

396 Res.MaskPolicy = false;

397 }

398

399

400

401

402

403 if (isMaskRegOp(MI)) {

404 Res.SEW = DemandedFields::SEWNone;

405 Res.LMUL = DemandedFields::LMULNone;

406 }

407

408

409 if (RISCVInstrInfo::isScalarInsertInstr(MI)) {

410 Res.LMUL = DemandedFields::LMULNone;

411 Res.SEWLMULRatio = false;

412 Res.VLAny = false;

413

414

415

416

417

418

419 if (hasUndefinedPassthru(MI)) {

420 if (RISCVInstrInfo::isFloatScalarMoveOrScalarSplatInstr(MI) &&

421 ->hasVInstructionsF64())

422 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;

423 else

424 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;

425 Res.TailPolicy = false;

426 }

427 }

428

429

430 if (RISCVInstrInfo::isScalarExtractInstr(MI)) {

432 Res.LMUL = DemandedFields::LMULNone;

433 Res.SEWLMULRatio = false;

434 Res.TailPolicy = false;

435 Res.MaskPolicy = false;

436 }

437

440

441

442

443

444

445

446

447

448

449 if (RISCVInstrInfo::isVSlideInstr(MI) && VLOp.isImm() &&

450 VLOp.getImm() == 1 && hasUndefinedPassthru(MI) &&

451 ->hasVLDependentLatency()) {

452 Res.VLAny = false;

453 Res.VLZeroness = true;

454 Res.LMUL = DemandedFields::LMULLessThanOrEqualToM1;

455 Res.TailPolicy = false;

456 }

457

458

459

460

461

462

463

464 if (RISCVInstrInfo::isScalarSplatInstr(MI) && VLOp.isImm() &&

465 VLOp.getImm() == 1 && hasUndefinedPassthru(MI) &&

466 ->hasVLDependentLatency()) {

467 Res.LMUL = DemandedFields::LMULLessThanOrEqualToM1;

468 Res.SEWLMULRatio = false;

469 Res.VLAny = false;

470 if (RISCVInstrInfo::isFloatScalarMoveOrScalarSplatInstr(MI) &&

471 ->hasVInstructionsF64())

472 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;

473 else

474 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;

475 Res.TailPolicy = false;

476 }

477 }

478

479

480

481

482

483

484

485

486 if (isVectorCopy(ST->getRegisterInfo(), MI)) {

487 Res.LMUL = DemandedFields::LMULNone;

488 Res.SEW = DemandedFields::SEWNone;

489 Res.SEWLMULRatio = false;

490 Res.TailPolicy = false;

491 Res.MaskPolicy = false;

492 }

493

494 if (RISCVInstrInfo::isVExtractInstr(MI)) {

496

497 Res.TailPolicy = false;

498 }

499

503 RISCVInstrInfo::isXSfmmVectorConfigInstr(MI);

504

505 return Res;

506}

507

508

509

510class VSETVLIInfo {

511 struct AVLDef {

512

513

514 const VNInfo *ValNo;

516 };

517 union {

518 AVLDef AVLRegDef;

519 unsigned AVLImm;

520 };

521

522 enum class AVLState : uint8_t {

524 AVLIsReg,

525 AVLIsImm,

526 AVLIsVLMAX,

527 Unknown,

528 } State = AVLState::Uninitialized;

529

530

532 uint8_t SEW = 0;

533 uint8_t TailAgnostic : 1;

534 uint8_t MaskAgnostic : 1;

535 uint8_t SEWLMULRatioOnly : 1;

536 uint8_t AltFmt : 1;

537 uint8_t TWiden : 3;

538

539public:

540 VSETVLIInfo()

541 : AVLImm(0), TailAgnostic(false), MaskAgnostic(false),

542 SEWLMULRatioOnly(false), AltFmt(false), TWiden(0) {}

543

544 static VSETVLIInfo getUnknown() {

545 VSETVLIInfo Info;

546 Info.setUnknown();

548 }

549

550 bool isValid() const { return State != AVLState::Uninitialized; }

551 void setUnknown() { State = AVLState::Unknown; }

552 bool isUnknown() const { return State == AVLState::Unknown; }

553

554 void setAVLRegDef(const VNInfo *VNInfo, Register AVLReg) {

556 AVLRegDef.ValNo = VNInfo;

557 AVLRegDef.DefReg = AVLReg;

558 State = AVLState::AVLIsReg;

559 }

560

561 void setAVLImm(unsigned Imm) {

562 AVLImm = Imm;

563 State = AVLState::AVLIsImm;

564 }

565

566 void setAVLVLMAX() { State = AVLState::AVLIsVLMAX; }

567

568 bool hasAVLImm() const { return State == AVLState::AVLIsImm; }

569 bool hasAVLReg() const { return State == AVLState::AVLIsReg; }

570 bool hasAVLVLMAX() const { return State == AVLState::AVLIsVLMAX; }

572 assert(hasAVLReg() && AVLRegDef.DefReg.isVirtual());

573 return AVLRegDef.DefReg;

574 }

575 unsigned getAVLImm() const {

577 return AVLImm;

578 }

579 const VNInfo *getAVLVNInfo() const {

581 return AVLRegDef.ValNo;

582 }

583

584

585

586

587 const MachineInstr *getAVLDefMI(const LiveIntervals *LIS) const {

589 if (!LIS || getAVLVNInfo()->isPHIDef())

590 return nullptr;

593 return MI;

594 }

595

596 void setAVL(const VSETVLIInfo &Info) {

598 if (Info.isUnknown())

599 setUnknown();

600 else if (Info.hasAVLReg())

601 setAVLRegDef(Info.getAVLVNInfo(), Info.getAVLReg());

602 else if (Info.hasAVLVLMAX())

603 setAVLVLMAX();

604 else {

606 setAVLImm(Info.getAVLImm());

607 }

608 }

609

610 bool hasSEWLMULRatioOnly() const { return SEWLMULRatioOnly; }

611

612 unsigned getSEW() const {

613 assert(isValid() && !isUnknown() && !hasSEWLMULRatioOnly() &&

614 "Can't use VTYPE for uninitialized or unknown");

615 return SEW;

616 }

618 assert(isValid() && !isUnknown() && !hasSEWLMULRatioOnly() &&

619 "Can't use VTYPE for uninitialized or unknown");

620 return VLMul;

621 }

622 bool getTailAgnostic() const {

624 "Can't use VTYPE for uninitialized or unknown");

625 return TailAgnostic;

626 }

627 bool getMaskAgnostic() const {

629 "Can't use VTYPE for uninitialized or unknown");

630 return MaskAgnostic;

631 }

632 bool getAltFmt() const {

634 "Can't use VTYPE for uninitialized or unknown");

635 return AltFmt;

636 }

637 unsigned getTWiden() const {

639 "Can't use VTYPE for uninitialized or unknown");

640 return TWiden;

641 }

642

643 bool hasNonZeroAVL(const LiveIntervals *LIS) const {

644 if (hasAVLImm())

645 return getAVLImm() > 0;

646 if (hasAVLReg()) {

647 if (auto *DefMI = getAVLDefMI(LIS))

648 return RISCVInstrInfo::isNonZeroLoadImmediate(*DefMI);

649 }

650 if (hasAVLVLMAX())

651 return true;

652 return false;

653 }

654

655 bool hasEquallyZeroAVL(const VSETVLIInfo &Other,

656 const LiveIntervals *LIS) const {

657 if (hasSameAVL(Other))

658 return true;

659 return (hasNonZeroAVL(LIS) && Other.hasNonZeroAVL(LIS));

660 }

661

662 bool hasSameAVLLatticeValue(const VSETVLIInfo &Other) const {

663 if (hasAVLReg() && Other.hasAVLReg()) {

664 assert(!getAVLVNInfo() == Other .getAVLVNInfo() &&

665 "we either have intervals or we don't");

666 if (!getAVLVNInfo())

667 return getAVLReg() == Other.getAVLReg();

668 return getAVLVNInfo()->id == Other.getAVLVNInfo()->id &&

669 getAVLReg() == Other.getAVLReg();

670 }

671

672 if (hasAVLImm() && Other.hasAVLImm())

673 return getAVLImm() == Other.getAVLImm();

674

675 if (hasAVLVLMAX())

676 return Other.hasAVLVLMAX() && hasSameVLMAX(Other);

677

678 return false;

679 }

680

681

682

683 bool hasSameAVL(const VSETVLIInfo &Other) const {

684

685

686

687 if (hasAVLReg() && Other.hasAVLReg()) {

688 assert(!getAVLVNInfo() == Other .getAVLVNInfo() &&

689 "we either have intervals or we don't");

690 if (!getAVLVNInfo())

691 return false;

692 }

693 return hasSameAVLLatticeValue(Other);

694 }

695

696 void setVTYPE(unsigned VType) {

698 "Can't set VTYPE for uninitialized or unknown");

704 TWiden =

706 }

707 void setVTYPE(RISCVVType::VLMUL L, unsigned S, bool TA, bool MA, bool Altfmt,

708 unsigned W) {

710 "Can't set VTYPE for uninitialized or unknown");

711 VLMul = L;

712 SEW = S;

713 TailAgnostic = TA;

714 MaskAgnostic = MA;

715 AltFmt = Altfmt;

716 TWiden = W;

717 }

718

719 void setAltFmt(bool AF) { AltFmt = AF; }

720

722

724 assert(isValid() && !isUnknown() && !SEWLMULRatioOnly &&

725 "Can't encode VTYPE for uninitialized or unknown");

726 if (TWiden != 0)

729 AltFmt);

730 }

731

732 bool hasSameVTYPE(const VSETVLIInfo &Other) const {

734 "Can't compare invalid VSETVLIInfos");

735 assert(!isUnknown() && Other .isUnknown() &&

736 "Can't compare VTYPE in unknown state");

737 assert(!SEWLMULRatioOnly && Other .SEWLMULRatioOnly &&

738 "Can't compare when only LMUL/SEW ratio is valid.");

739 return std::tie(VLMul, SEW, TailAgnostic, MaskAgnostic, AltFmt, TWiden) ==

742 }

743

746 "Can't use VTYPE for uninitialized or unknown");

748 }

749

750

751

752

753

754 bool hasSameVLMAX(const VSETVLIInfo &Other) const {

756 "Can't compare invalid VSETVLIInfos");

757 assert(!isUnknown() && Other .isUnknown() &&

758 "Can't compare VTYPE in unknown state");

760 }

761

762 bool hasCompatibleVTYPE(const DemandedFields &Used,

763 const VSETVLIInfo &Require) const {

764 return areCompatibleVTYPEs(Require.encodeVTYPE(), encodeVTYPE(), Used);

765 }

766

767

768

769

770 bool isCompatible(const DemandedFields &Used, const VSETVLIInfo &Require,

771 const LiveIntervals *LIS) const {

773 "Can't compare invalid VSETVLIInfos");

774

775 if (isUnknown() || Require.isUnknown())

776 return false;

777

778

779 if (SEWLMULRatioOnly || Require.SEWLMULRatioOnly)

780 return false;

781

782 if (Used.VLAny && !(hasSameAVL(Require) && hasSameVLMAX(Require)))

783 return false;

784

785 if (Used.VLZeroness && !hasEquallyZeroAVL(Require, LIS))

786 return false;

787

788 return hasCompatibleVTYPE(Used, Require);

789 }

790

792

794 return Other .isValid();

795 if ( Other .isValid())

797

798

799 if (isUnknown())

800 return Other.isUnknown();

801 if (Other.isUnknown())

802 return isUnknown();

803

804 if (!hasSameAVLLatticeValue(Other))

805 return false;

806

807

808 if (SEWLMULRatioOnly != Other.SEWLMULRatioOnly)

809 return false;

810

811

812 if (SEWLMULRatioOnly)

813 return hasSameVLMAX(Other);

814

815

816 return hasSameVTYPE(Other);

817 }

818

820 return !(*this == Other);

821 }

822

823

824

825 VSETVLIInfo intersect(const VSETVLIInfo &Other) const {

826

827 if ( Other .isValid())

828 return *this;

829

830

833

834

835 if (isUnknown() || Other.isUnknown())

836 return VSETVLIInfo::getUnknown();

837

838

839 if (*this == Other)

840 return *this;

841

842

843

844 if (hasSameAVL(Other) && hasSameVLMAX(Other)) {

845 VSETVLIInfo MergeInfo = *this;

846 MergeInfo.SEWLMULRatioOnly = true;

847 return MergeInfo;

848 }

849

850

851 return VSETVLIInfo::getUnknown();

852 }

853

854#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

855

858 dbgs() << "\n";

859 }

860

861

862

863 void print(raw_ostream &OS) const {

864 OS << '{';

865 switch (State) {

866 case AVLState::Uninitialized:

867 OS << "Uninitialized";

868 break;

869 case AVLState::Unknown:

870 OS << "unknown";

871 break;

872 case AVLState::AVLIsReg:

874 break;

875 case AVLState::AVLIsImm:

876 OS << "AVLImm=" << (unsigned)AVLImm;

877 break;

878 case AVLState::AVLIsVLMAX:

879 OS << "AVLVLMAX";

880 break;

881 }

882 if (isValid() && !isUnknown()) {

883 OS << ", ";

884

885 unsigned LMul;

886 bool Fractional;

887 std::tie(LMul, Fractional) = decodeVLMUL(VLMul);

888

889 OS << "VLMul=m";

890 if (Fractional)

891 OS << 'f';

892 OS << LMul << ", "

893 << "SEW=e" << (unsigned)SEW << ", "

894 << "TailAgnostic=" << (bool)TailAgnostic << ", "

895 << "MaskAgnostic=" << (bool)MaskAgnostic << ", "

896 << "SEWLMULRatioOnly=" << (bool)SEWLMULRatioOnly << ", "

897 << "TWiden=" << (unsigned)TWiden << ", "

898 << "AltFmt=" << (bool)AltFmt;

899 }

900

901 OS << '}';

902 }

903#endif

904};

905

906#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

909 V.print(OS);

910 return OS;

911}

912#endif

913

915

916

917 VSETVLIInfo Exit;

918

919

920

921 VSETVLIInfo Pred;

922

923

924 bool InQueue = false;

925

927};

928

929enum TKTMMode {

930 VSETTK = 0,

931 VSETTM = 1,

932};

933

935 const RISCVSubtarget *ST;

936 const TargetInstrInfo *TII;

937 MachineRegisterInfo *MRI;

938

939 LiveIntervals *LIS;

940

941 std::vector BlockInfo;

942 std::queue<const MachineBasicBlock *> WorkList;

943

944public:

945 static char ID;

946

947 RISCVInsertVSETVLI() : MachineFunctionPass(ID) {}

948 bool runOnMachineFunction(MachineFunction &MF) override;

949

950 void getAnalysisUsage(AnalysisUsage &AU) const override {

952

956 AU.addPreserved();

958

960 }

961

963

964private:

965 bool needVSETVLI(const DemandedFields &Used, const VSETVLIInfo &Require,

966 const VSETVLIInfo &CurInfo) const;

967 bool needVSETVLIPHI(const VSETVLIInfo &Require,

968 const MachineBasicBlock &MBB) const;

969 void insertVSETVLI(MachineBasicBlock &MBB,

971 const VSETVLIInfo &Info, const VSETVLIInfo &PrevInfo);

972

973 void transferBefore(VSETVLIInfo &Info, const MachineInstr &MI) const;

974 void transferAfter(VSETVLIInfo &Info, const MachineInstr &MI) const;

975 bool computeVLVTYPEChanges(const MachineBasicBlock &MBB,

976 VSETVLIInfo &Info) const;

977 void computeIncomingVLVTYPE(const MachineBasicBlock &MBB);

978 void emitVSETVLIs(MachineBasicBlock &MBB);

979 void doPRE(MachineBasicBlock &MBB);

980 void insertReadVL(MachineBasicBlock &MBB);

981

982 bool canMutatePriorConfig(const MachineInstr &PrevMI, const MachineInstr &MI,

983 const DemandedFields &Used) const;

984 void coalesceVSETVLIs(MachineBasicBlock &MBB) const;

985

986 VSETVLIInfo getInfoForVSETVLI(const MachineInstr &MI) const;

987 VSETVLIInfo computeInfoForInstr(const MachineInstr &MI) const;

988 void forwardVSETVLIAVL(VSETVLIInfo &Info) const;

989 bool insertVSETMTK(MachineBasicBlock &MBB, TKTMMode Mode) const;

990};

991

992}

993

994char RISCVInsertVSETVLI::ID = 0;

996

998 false, false)

999

1000

1001

1002

1003

1004

1005

1006

1007

1008void RISCVInsertVSETVLI::forwardVSETVLIAVL(VSETVLIInfo &Info) const {

1009 if ( Info .hasAVLReg())

1010 return;

1012 if ( DefMI || !RISCVInstrInfo::isVectorConfigInstr(*DefMI))

1013 return;

1014 VSETVLIInfo DefInstrInfo = getInfoForVSETVLI(*DefMI);

1015 if (!DefInstrInfo.hasSameVLMAX(Info))

1016 return;

1017 Info.setAVL(DefInstrInfo);

1018}

1019

1020

1021

1022VSETVLIInfo

1023RISCVInsertVSETVLI::getInfoForVSETVLI(const MachineInstr &MI) const {

1024 VSETVLIInfo NewInfo;

1025 if (MI.getOpcode() == RISCV::PseudoVSETIVLI) {

1026 NewInfo.setAVLImm(MI.getOperand(1).getImm());

1027 } else if (RISCVInstrInfo::isXSfmmVectorConfigTNInstr(MI)) {

1028 assert(MI.getOpcode() == RISCV::PseudoSF_VSETTNT ||

1029 MI.getOpcode() == RISCV::PseudoSF_VSETTNTX0);

1030 switch (MI.getOpcode()) {

1031 case RISCV::PseudoSF_VSETTNTX0:

1032 NewInfo.setAVLVLMAX();

1033 break;

1034 case RISCV::PseudoSF_VSETTNT:

1035 Register ATNReg = MI.getOperand(1).getReg();

1036 NewInfo.setAVLRegDef(getVNInfoFromReg(ATNReg, MI, LIS), ATNReg);

1037 break;

1038 }

1039 } else {

1040 assert(MI.getOpcode() == RISCV::PseudoVSETVLI ||

1041 MI.getOpcode() == RISCV::PseudoVSETVLIX0);

1042 if (MI.getOpcode() == RISCV::PseudoVSETVLIX0)

1043 NewInfo.setAVLVLMAX();

1044 else if (MI.getOperand(1).isUndef())

1045

1046 NewInfo.setAVLImm(1);

1047 else {

1048 Register AVLReg = MI.getOperand(1).getReg();

1049 VNInfo *VNI = getVNInfoFromReg(AVLReg, MI, LIS);

1050 NewInfo.setAVLRegDef(VNI, AVLReg);

1051 }

1052 }

1053 NewInfo.setVTYPE(MI.getOperand(2).getImm());

1054

1055 forwardVSETVLIAVL(NewInfo);

1056

1057 return NewInfo;

1058}

1059

1060static unsigned computeVLMAX(unsigned VLEN, unsigned SEW,

1063 if (Fractional)

1064 VLEN = VLEN / LMul;

1065 else

1066 VLEN = VLEN * LMul;

1067 return VLEN/SEW;

1068}

1069

1070VSETVLIInfo

1071RISCVInsertVSETVLI::computeInfoForInstr(const MachineInstr &MI) const {

1072 VSETVLIInfo InstrInfo;

1073 const uint64_t TSFlags = MI.getDesc().TSFlags;

1074

1075 bool TailAgnostic = true;

1076 bool MaskAgnostic = true;

1077 if (!hasUndefinedPassthru(MI)) {

1078

1079 TailAgnostic = false;

1080 MaskAgnostic = false;

1081

1082

1084 const MachineOperand &Op = MI.getOperand(getVecPolicyOpNum(MI));

1085 uint64_t Policy = Op.getImm();

1088 "Invalid Policy Value");

1091 }

1092

1094 MaskAgnostic = true;

1095 }

1096

1098

1100 InstrInfo.setAltFmt(AltFmt);

1101

1102 unsigned Log2SEW = MI.getOperand(getSEWOpNum(MI)).getImm();

1103

1104 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;

1106

1108 const MachineOperand &TWidenOp =

1109 MI.getOperand(MI.getNumExplicitOperands() - 1);

1110 unsigned TWiden = TWidenOp.getImm();

1111

1112 InstrInfo.setAVLVLMAX();

1114 const MachineOperand &TNOp =

1116

1118 InstrInfo.setAVLRegDef(getVNInfoFromReg(TNOp.getReg(), MI, LIS),

1120 }

1121

1122 InstrInfo.setVTYPE(VLMul, SEW, TailAgnostic, MaskAgnostic, AltFmt, TWiden);

1123

1124 return InstrInfo;

1125 }

1126

1128 const MachineOperand &VLOp = MI.getOperand(getVLOpNum(MI));

1129 if (VLOp.isImm()) {

1131

1133

1134

1135 const unsigned VLMAX = computeVLMAX(ST->getRealMaxVLen(), SEW, VLMul);

1136 if (ST->getRealMinVLen() == ST->getRealMaxVLen() && VLMAX <= 31)

1137 InstrInfo.setAVLImm(VLMAX);

1138 else

1139 InstrInfo.setAVLVLMAX();

1140 }

1141 else

1142 InstrInfo.setAVLImm(Imm);

1143 } else if (VLOp.isUndef()) {

1144

1145 InstrInfo.setAVLImm(1);

1146 } else {

1147 VNInfo *VNI = getVNInfoFromReg(VLOp.getReg(), MI, LIS);

1148 InstrInfo.setAVLRegDef(VNI, VLOp.getReg());

1149 }

1150 } else {

1151 assert(RISCVInstrInfo::isScalarExtractInstr(MI) ||

1152 RISCVInstrInfo::isVExtractInstr(MI));

1153

1154

1155 InstrInfo.setAVLImm(1);

1156 }

1157#ifndef NDEBUG

1158 if (std::optional EEW = getEEWForLoadStore(MI)) {

1159 assert(SEW == EEW && "Initial SEW doesn't match expected EEW");

1160 }

1161#endif

1162

1163 InstrInfo.setVTYPE(VLMul, SEW, TailAgnostic, MaskAgnostic, AltFmt,

1164 0);

1165

1166 forwardVSETVLIAVL(InstrInfo);

1167

1168 return InstrInfo;

1169}

1170

1171void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,

1174 const VSETVLIInfo &PrevInfo) {

1175 ++NumInsertedVSETVL;

1176

1177 if (Info.getTWiden()) {

1178 if (Info.hasAVLVLMAX()) {

1179 Register DestReg = MRI->createVirtualRegister(&RISCV::GPRNoX0RegClass);

1180 auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoSF_VSETTNTX0))

1184 if (LIS) {

1187 }

1188 } else {

1189 auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoSF_VSETTNT))

1193 if (LIS)

1195 }

1196 return;

1197 }

1198

1199 if (PrevInfo.isValid() && !PrevInfo.isUnknown()) {

1200

1201

1202 if (Info.hasSameAVL(PrevInfo) && Info.hasSameVLMAX(PrevInfo)) {

1203 auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0X0))

1208 if (LIS)

1210 return;

1211 }

1212

1213

1214

1215

1216 if (Info.hasSameVLMAX(PrevInfo) && Info.hasAVLReg()) {

1217 if (const MachineInstr *DefMI = Info.getAVLDefMI(LIS);

1218 DefMI && RISCVInstrInfo::isVectorConfigInstr(*DefMI)) {

1219 VSETVLIInfo DefInfo = getInfoForVSETVLI(*DefMI);

1220 if (DefInfo.hasSameAVL(PrevInfo) && DefInfo.hasSameVLMAX(PrevInfo)) {

1221 auto MI =

1222 BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0X0))

1227 if (LIS)

1229 return;

1230 }

1231 }

1232 }

1233 }

1234

1235 if (Info.hasAVLImm()) {

1236 auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETIVLI))

1240 if (LIS)

1242 return;

1243 }

1244

1245 if (Info.hasAVLVLMAX()) {

1246 Register DestReg = MRI->createVirtualRegister(&RISCV::GPRNoX0RegClass);

1247 auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))

1251 if (LIS) {

1254 }

1255 return;

1256 }

1257

1259 MRI->constrainRegClass(AVLReg, &RISCV::GPRNoX0RegClass);

1260 auto MI = BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLI))

1264 if (LIS) {

1266 LiveInterval &LI = LIS->getInterval(AVLReg);

1268 const VNInfo *CurVNI = Info.getAVLVNInfo();

1269

1270

1274 else {

1276 MRI->createVirtualRegister(&RISCV::GPRNoX0RegClass);

1281 else {

1283 II = std::next(II);

1284 }

1286 auto AVLCopy = BuildMI(*MBB, II, DL, TII->get(RISCV::COPY), AVLCopyReg)

1289 MI->getOperand(1).setReg(AVLCopyReg);

1291 }

1292 }

1293 }

1294}

1295

1296

1297

1298bool RISCVInsertVSETVLI::needVSETVLI(const DemandedFields &Used,

1299 const VSETVLIInfo &Require,

1300 const VSETVLIInfo &CurInfo) const {

1301 if (!CurInfo.isValid() || CurInfo.isUnknown() || CurInfo.hasSEWLMULRatioOnly())

1302 return true;

1303

1304 if (CurInfo.isCompatible(Used, Require, LIS))

1305 return false;

1306

1307 return true;

1308}

1309

1310

1311

1312

1314 const VSETVLIInfo &NewInfo,

1315 DemandedFields &Demanded) {

1316 VSETVLIInfo Info = NewInfo;

1317

1318 if (!Demanded.LMUL && !Demanded.SEWLMULRatio && PrevInfo.isValid() &&

1319 !PrevInfo.isUnknown()) {

1321 Info.getSEW()))

1322 Info.setVLMul(*NewVLMul);

1323 Demanded.LMUL = DemandedFields::LMULEqual;

1324 }

1325

1326 return Info;

1327}

1328

1329

1330

1331

1332void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,

1333 const MachineInstr &MI) const {

1334 if (isVectorCopy(ST->getRegisterInfo(), MI) &&

1335 (Info.isUnknown() || Info .isValid() || Info.hasSEWLMULRatioOnly())) {

1336

1337

1338 VSETVLIInfo NewInfo;

1339 NewInfo.setAVLImm(1);

1340 NewInfo.setVTYPE(RISCVVType::LMUL_1, 8, true, true,

1341 false, 0);

1342 Info = NewInfo;

1343 return;

1344 }

1345

1347 return;

1348

1349 DemandedFields Demanded = getDemanded(MI, ST);

1350

1351 const VSETVLIInfo NewInfo = computeInfoForInstr(MI);

1352 assert(NewInfo.isValid() && !NewInfo.isUnknown());

1353 if (Info.isValid() && !needVSETVLI(Demanded, NewInfo, Info))

1354 return;

1355

1356 const VSETVLIInfo PrevInfo = Info;

1357 if ( Info .isValid() || Info.isUnknown())

1358 Info = NewInfo;

1359

1360 const VSETVLIInfo IncomingInfo = adjustIncoming(PrevInfo, NewInfo, Demanded);

1361

1362

1363

1364

1365

1366

1367

1368

1369 bool EquallyZero = IncomingInfo.hasEquallyZeroAVL(PrevInfo, LIS) &&

1370 IncomingInfo.hasSameVLMAX(PrevInfo);

1371 if (Demanded.VLAny || (Demanded.VLZeroness && !EquallyZero))

1372 Info.setAVL(IncomingInfo);

1373

1374

1375 if (Info.hasSEWLMULRatioOnly()) {

1376 VSETVLIInfo RatiolessInfo = IncomingInfo;

1377 RatiolessInfo.setAVL(Info);

1378 Info = RatiolessInfo;

1379 } else {

1380 Info.setVTYPE(

1381 ((Demanded.LMUL || Demanded.SEWLMULRatio) ? IncomingInfo : Info)

1382 .getVLMUL(),

1383 ((Demanded.SEW || Demanded.SEWLMULRatio) ? IncomingInfo : Info)

1384 .getSEW(),

1385

1386

1387 (Demanded.TailPolicy ? IncomingInfo : Info).getTailAgnostic() ||

1388 IncomingInfo.getTailAgnostic(),

1389 (Demanded.MaskPolicy ? IncomingInfo : Info).getMaskAgnostic() ||

1390 IncomingInfo.getMaskAgnostic(),

1391 (Demanded.AltFmt ? IncomingInfo : Info).getAltFmt(),

1392 Demanded.TWiden ? IncomingInfo.getTWiden() : 0);

1393 }

1394}

1395

1396

1397

1398

1399void RISCVInsertVSETVLI::transferAfter(VSETVLIInfo &Info,

1400 const MachineInstr &MI) const {

1401 if (RISCVInstrInfo::isVectorConfigInstr(MI)) {

1402 Info = getInfoForVSETVLI(MI);

1403 return;

1404 }

1405

1406 if (RISCVInstrInfo::isFaultOnlyFirstLoad(MI)) {

1407

1408 assert(MI.getOperand(1).getReg().isVirtual());

1409 if (LIS) {

1410 auto &LI = LIS->getInterval(MI.getOperand(1).getReg());

1411 SlotIndex SI =

1414 Info.setAVLRegDef(VNI, MI.getOperand(1).getReg());

1415 } else

1416 Info.setAVLRegDef(nullptr, MI.getOperand(1).getReg());

1417 return;

1418 }

1419

1420

1421

1422 if (MI.isCall() || MI.isInlineAsm() ||

1423 MI.modifiesRegister(RISCV::VL, nullptr) ||

1424 MI.modifiesRegister(RISCV::VTYPE, nullptr))

1425 Info = VSETVLIInfo::getUnknown();

1426}

1427

1428bool RISCVInsertVSETVLI::computeVLVTYPEChanges(const MachineBasicBlock &MBB,

1429 VSETVLIInfo &Info) const {

1430 bool HadVectorOp = false;

1431

1433 for (const MachineInstr &MI : MBB) {

1434 transferBefore(Info, MI);

1435

1436 if (RISCVInstrInfo::isVectorConfigInstr(MI) ||

1438 isVectorCopy(ST->getRegisterInfo(), MI) ||

1439 RISCVInstrInfo::isXSfmmVectorConfigInstr(MI))

1440 HadVectorOp = true;

1441

1442 transferAfter(Info, MI);

1443 }

1444

1445 return HadVectorOp;

1446}

1447

1448void RISCVInsertVSETVLI::computeIncomingVLVTYPE(const MachineBasicBlock &MBB) {

1449

1450 BlockData &BBInfo = BlockInfo[MBB.getNumber()];

1451

1452 BBInfo.InQueue = false;

1453

1454

1455

1456 VSETVLIInfo InInfo = BBInfo.Pred;

1458

1459 InInfo.setUnknown();

1460 } else {

1462 InInfo = InInfo.intersect(BlockInfo[P->getNumber()].Exit);

1463 }

1464

1465

1466 if (!InInfo.isValid())

1467 return;

1468

1469

1470 if (InInfo == BBInfo.Pred)

1471 return;

1472

1473 BBInfo.Pred = InInfo;

1475 << " changed to " << BBInfo.Pred << "\n");

1476

1477

1478

1479

1480

1481 VSETVLIInfo TmpStatus;

1482 computeVLVTYPEChanges(MBB, TmpStatus);

1483

1484

1485

1486 if (BBInfo.Exit == TmpStatus)

1487 return;

1488

1489 BBInfo.Exit = TmpStatus;

1491 << " changed to " << BBInfo.Exit << "\n");

1492

1493

1494

1496 if (!BlockInfo[S->getNumber()].InQueue) {

1497 BlockInfo[S->getNumber()].InQueue = true;

1498 WorkList.push(S);

1499 }

1500}

1501

1502

1503

1504

1505bool RISCVInsertVSETVLI::needVSETVLIPHI(const VSETVLIInfo &Require,

1506 const MachineBasicBlock &MBB) const {

1507 if (!Require.hasAVLReg())

1508 return true;

1509

1510 if (!LIS)

1511 return true;

1512

1513

1514 const VNInfo *Valno = Require.getAVLVNInfo();

1516 return true;

1517

1519

1521 const VSETVLIInfo &PBBExit = BlockInfo[PBB->getNumber()].Exit;

1522

1523

1526 return true;

1528 if ( DefMI || !RISCVInstrInfo::isVectorConfigInstr(*DefMI))

1529 return true;

1530

1531

1532

1533 VSETVLIInfo DefInfo = getInfoForVSETVLI(*DefMI);

1534 if (DefInfo != PBBExit)

1535 return true;

1536

1537

1538

1539

1540 if (PBBExit.isUnknown() || !PBBExit.hasSameVTYPE(Require))

1541 return true;

1542 }

1543

1544

1545

1546 return false;

1547}

1548

1549void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) {

1550 VSETVLIInfo CurInfo = BlockInfo[MBB.getNumber()].Pred;

1551

1552

1553 bool PrefixTransparent = true;

1554 for (MachineInstr &MI : MBB) {

1555 const VSETVLIInfo PrevInfo = CurInfo;

1556 transferBefore(CurInfo, MI);

1557

1558

1559 if (RISCVInstrInfo::isVectorConfigInstr(MI)) {

1560

1561 assert(MI.getOperand(3).getReg() == RISCV::VL &&

1562 MI.getOperand(4).getReg() == RISCV::VTYPE &&

1563 "Unexpected operands where VL and VTYPE should be");

1564 MI.getOperand(3).setIsDead(false);

1565 MI.getOperand(4).setIsDead(false);

1566 PrefixTransparent = false;

1567 }

1568

1570 isVectorCopy(ST->getRegisterInfo(), MI)) {

1571 if (!PrevInfo.isCompatible(DemandedFields::all(), CurInfo, LIS)) {

1572 insertVSETVLI(MBB, MI, MI.getDebugLoc(), CurInfo, PrevInfo);

1573 PrefixTransparent = false;

1574 }

1576 true));

1577 }

1578

1579 uint64_t TSFlags = MI.getDesc().TSFlags;

1581 if (!PrevInfo.isCompatible(DemandedFields::all(), CurInfo, LIS)) {

1582

1583

1584

1585

1586

1587

1588

1589 if (!PrefixTransparent || needVSETVLIPHI(CurInfo, MBB))

1590 insertVSETVLI(MBB, MI, MI.getDebugLoc(), CurInfo, PrevInfo);

1591 PrefixTransparent = false;

1592 }

1593

1595 MachineOperand &VLOp = MI.getOperand(getVLOpNum(MI));

1596 if (VLOp.isReg()) {

1598

1599

1602 if (LIS) {

1606

1607

1610

1611

1612

1613

1614

1615 for (MachineInstr *DeadMI : DeadMIs) {

1616 if ( TII ->isAddImmediate(*DeadMI, Reg))

1617 continue;

1619 Register AddReg = DeadMI->getOperand(1).getReg();

1620 DeadMI->eraseFromParent();

1623 }

1624 }

1625 }

1627 true));

1628 }

1630 true));

1631 }

1632

1633 if (MI.isInlineAsm()) {

1635 true));

1637 true));

1638 }

1639

1640 if (MI.isCall() || MI.isInlineAsm() ||

1641 MI.modifiesRegister(RISCV::VL, nullptr) ||

1642 MI.modifiesRegister(RISCV::VTYPE, nullptr))

1643 PrefixTransparent = false;

1644

1645 transferAfter(CurInfo, MI);

1646 }

1647

1649 if (CurInfo != Info.Exit) {

1653 LLVM_DEBUG(dbgs() << " actual end state: " << CurInfo << "\n");

1654 }

1655 assert(CurInfo == Info.Exit && "InsertVSETVLI dataflow invariant violated");

1656}

1657

1658

1659

1660

1661

1662

1663void RISCVInsertVSETVLI::doPRE(MachineBasicBlock &MBB) {

1664 if (!BlockInfo[MBB.getNumber()].Pred.isUnknown())

1665 return;

1666

1667 MachineBasicBlock *UnavailablePred = nullptr;

1668 VSETVLIInfo AvailableInfo;

1670 const VSETVLIInfo &PredInfo = BlockInfo[P->getNumber()].Exit;

1671 if (PredInfo.isUnknown()) {

1672 if (UnavailablePred)

1673 return;

1674 UnavailablePred = P;

1675 } else if (!AvailableInfo.isValid()) {

1676 AvailableInfo = PredInfo;

1677 } else if (AvailableInfo != PredInfo) {

1678 return;

1679 }

1680 }

1681

1682

1683

1684 if (!UnavailablePred || !AvailableInfo.isValid())

1685 return;

1686

1687 if (!LIS)

1688 return;

1689

1690

1691

1692 if (AvailableInfo.hasSEWLMULRatioOnly())

1693 return;

1694

1695

1696 if (UnavailablePred->succ_size() != 1)

1697 return;

1698

1699

1700

1701

1702 if (AvailableInfo.hasAVLReg()) {

1703 SlotIndex SI = AvailableInfo.getAVLVNInfo()->def;

1704

1705

1707 return;

1710 return;

1711 }

1712

1713

1714

1715

1716

1717

1718

1719

1720

1721 VSETVLIInfo OldInfo = BlockInfo[MBB.getNumber()].Pred;

1722 VSETVLIInfo CurInfo = AvailableInfo;

1723 int TransitionsRemoved = 0;

1724 for (const MachineInstr &MI : MBB) {

1725 const VSETVLIInfo LastInfo = CurInfo;

1726 const VSETVLIInfo LastOldInfo = OldInfo;

1727 transferBefore(CurInfo, MI);

1728 transferBefore(OldInfo, MI);

1729 if (CurInfo == LastInfo)

1730 TransitionsRemoved++;

1731 if (LastOldInfo == OldInfo)

1732 TransitionsRemoved--;

1733 transferAfter(CurInfo, MI);

1734 transferAfter(OldInfo, MI);

1735 if (CurInfo == OldInfo)

1736

1737 break;

1738 }

1739 if (CurInfo != OldInfo || TransitionsRemoved <= 0)

1740

1741 return;

1742

1743

1744

1745

1746 auto OldExit = BlockInfo[UnavailablePred->getNumber()].Exit;

1748 << UnavailablePred->getName() << " with state "

1749 << AvailableInfo << "\n");

1750 BlockInfo[UnavailablePred->getNumber()].Exit = AvailableInfo;

1751 BlockInfo[MBB.getNumber()].Pred = AvailableInfo;

1752

1753

1754

1756 insertVSETVLI(*UnavailablePred, InsertPt,

1758 AvailableInfo, OldExit);

1759}

1760

1761

1762

1763bool RISCVInsertVSETVLI::canMutatePriorConfig(

1764 const MachineInstr &PrevMI, const MachineInstr &MI,

1765 const DemandedFields &Used) const {

1766

1767

1768

1769 if (!RISCVInstrInfo::isVLPreservingConfig(MI)) {

1770 if (Used.VLAny)

1771 return false;

1772

1773 if (Used.VLZeroness) {

1774 if (RISCVInstrInfo::isVLPreservingConfig(PrevMI))

1775 return false;

1776 if (!getInfoForVSETVLI(PrevMI).hasEquallyZeroAVL(getInfoForVSETVLI(MI),

1777 LIS))

1778 return false;

1779 }

1780

1781 auto &AVL = MI.getOperand(1);

1782

1783

1784

1785 if (AVL.isReg() && AVL.getReg() != RISCV::X0) {

1786 VNInfo *VNI = getVNInfoFromReg(AVL.getReg(), MI, LIS);

1787 VNInfo *PrevVNI = getVNInfoFromReg(AVL.getReg(), PrevMI, LIS);

1788 if (!VNI || !PrevVNI || VNI != PrevVNI)

1789 return false;

1790 }

1791

1792

1793

1794 Register VL = MI.getOperand(0).getReg();

1798 return false;

1799 }

1800

1803 auto VType = MI.getOperand(2).getImm();

1804 return areCompatibleVTYPEs(PriorVType, VType, Used);

1805}

1806

1807void RISCVInsertVSETVLI::coalesceVSETVLIs(MachineBasicBlock &MBB) const {

1808 MachineInstr *NextMI = nullptr;

1809

1810

1811 DemandedFields Used;

1812 Used.demandVL();

1813 Used.demandVTYPE();

1815

1816 auto dropAVLUse = [&](MachineOperand &MO) {

1817 if (!MO.isReg() || !MO.getReg().isVirtual())

1818 return;

1819 Register OldVLReg = MO.getReg();

1821

1822 if (LIS)

1824

1825 MachineInstr *VLOpDef = MRI->getUniqueVRegDef(OldVLReg);

1826 if (VLOpDef && TII->isAddImmediate(*VLOpDef, OldVLReg) &&

1827 MRI->use_nodbg_empty(OldVLReg))

1829 };

1830

1832

1834 RISCVInstrInfo::isXSfmmVectorConfigInstr(MI)) {

1835 NextMI = nullptr;

1836 continue;

1837 }

1838

1839 if (!RISCVInstrInfo::isVectorConfigInstr(MI)) {

1840 Used.doUnion(getDemanded(MI, ST));

1841 if (MI.isCall() || MI.isInlineAsm() ||

1842 MI.modifiesRegister(RISCV::VL, nullptr) ||

1843 MI.modifiesRegister(RISCV::VTYPE, nullptr))

1844 NextMI = nullptr;

1845 continue;

1846 }

1847

1848 if (.getOperand(0).isDead())

1849 Used.demandVL();

1850

1851 if (NextMI) {

1852 if ( Used .usedVL() && Used .usedVTYPE()) {

1853 dropAVLUse(MI.getOperand(1));

1854 if (LIS)

1856 MI.eraseFromParent();

1857 NumCoalescedVSETVL++;

1858

1859 continue;

1860 }

1861

1862 if (canMutatePriorConfig(MI, *NextMI, Used)) {

1863 if (!RISCVInstrInfo::isVLPreservingConfig(*NextMI)) {

1865

1866 MI.getOperand(0).setReg(DefReg);

1867 MI.getOperand(0).setIsDead(false);

1868

1869

1870 dropAVLUse(MI.getOperand(1));

1873 else

1875 false);

1877

1878

1879

1880 if (DefReg.isVirtual() && LIS) {

1881 LiveInterval &DefLI = LIS->getInterval(DefReg);

1883 SlotIndex NextMISlot =

1885 VNInfo *DefVNI = DefLI.getVNInfoAt(NextMISlot);

1886 LiveInterval::Segment S(MISlot, NextMISlot, DefVNI);

1888 DefVNI->def = MISlot;

1889

1891

1892

1893

1895 }

1896

1898 }

1900

1902 if (LIS)

1905 NumCoalescedVSETVL++;

1906

1907 }

1908 }

1909 NextMI = &MI;

1910 Used = getDemanded(MI, ST);

1911 }

1912

1913

1914

1915 for (auto *MI : ToDelete) {

1916 assert(MI->getOpcode() == RISCV::ADDI);

1917 Register AddReg = MI->getOperand(1).getReg();

1918 if (LIS) {

1921 }

1922 MI->eraseFromParent();

1925 }

1926}

1927

1928void RISCVInsertVSETVLI::insertReadVL(MachineBasicBlock &MBB) {

1930 MachineInstr &MI = *I++;

1931 if (RISCVInstrInfo::isFaultOnlyFirstLoad(MI)) {

1932 Register VLOutput = MI.getOperand(1).getReg();

1934 if (.getOperand(1).isDead()) {

1935 auto ReadVLMI = BuildMI(MBB, I, MI.getDebugLoc(),

1936 TII->get(RISCV::PseudoReadVL), VLOutput);

1937

1938 if (LIS) {

1939 SlotIndex NewDefSI =

1941 LiveInterval &DefLI = LIS->getInterval(VLOutput);

1945 DefVNI->def = NewDefSI;

1946 }

1947 }

1948

1949 MI.getOperand(1).setReg(RISCV::X0);

1950 MI.addRegisterDefined(RISCV::VL, MRI->getTargetRegisterInfo());

1951 }

1952 }

1953}

1954

1955bool RISCVInsertVSETVLI::insertVSETMTK(MachineBasicBlock &MBB,

1956 TKTMMode Mode) const {

1957

1959 for (auto &MI : MBB) {

1960 uint64_t TSFlags = MI.getDesc().TSFlags;

1961 if (RISCVInstrInfo::isXSfmmVectorConfigTMTKInstr(MI) ||

1963 continue;

1964

1965 VSETVLIInfo CurrInfo = computeInfoForInstr(MI);

1966

1968 continue;

1969

1971 continue;

1972

1973 unsigned OpNum = 0;

1974 unsigned Opcode = 0;

1975 switch (Mode) {

1976 case VSETTK:

1978 Opcode = RISCV::PseudoSF_VSETTK;

1979 break;

1980 case VSETTM:

1982 Opcode = RISCV::PseudoSF_VSETTM;

1983 break;

1984 }

1985

1986 assert(OpNum && Opcode && "Invalid OpNum or Opcode");

1987

1988 MachineOperand &Op = MI.getOperand(OpNum);

1989

1990 auto TmpMI = BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(Opcode))

1994 .addImm(CurrInfo.getTWiden());

1995

1999 Op.setIsKill(false);

2000 if (LIS) {

2003

2004

2006

2007

2008

2009 }

2010 }

2012}

2013

2014bool RISCVInsertVSETVLI::runOnMachineFunction(MachineFunction &MF) {

2015

2017 if (->hasVInstructions())

2018 return false;

2019

2021

2022 TII = ST->getInstrInfo();

2024 auto *LISWrapper = getAnalysisIfAvailable();

2025 LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;

2026

2027 assert(BlockInfo.empty() && "Expect empty block infos");

2029

2030 bool HaveVectorOp = false;

2031

2032

2033 for (const MachineBasicBlock &MBB : MF) {

2034 VSETVLIInfo TmpStatus;

2035 HaveVectorOp |= computeVLVTYPEChanges(MBB, TmpStatus);

2036

2037 BlockData &BBInfo = BlockInfo[MBB.getNumber()];

2038 BBInfo.Exit = TmpStatus;

2040 << " is " << BBInfo.Exit << "\n");

2041

2042 }

2043

2044

2045 if (!HaveVectorOp) {

2046 BlockInfo.clear();

2047 return false;

2048 }

2049

2050

2051

2052

2053 for (const MachineBasicBlock &MBB : MF) {

2054 WorkList.push(&MBB);

2056 }

2057 while (!WorkList.empty()) {

2058 const MachineBasicBlock &MBB = *WorkList.front();

2059 WorkList.pop();

2060 computeIncomingVLVTYPE(MBB);

2061 }

2062

2063

2064 for (MachineBasicBlock &MBB : MF)

2065 doPRE(MBB);

2066

2067

2068

2069

2070

2071 for (MachineBasicBlock &MBB : MF)

2072 emitVSETVLIs(MBB);

2073

2074

2075

2076

2077

2078

2079

2080

2081

2082

2084 coalesceVSETVLIs(*MBB);

2085

2086

2087

2088 for (MachineBasicBlock &MBB : MF)

2089 insertReadVL(MBB);

2090

2091 for (MachineBasicBlock &MBB : MF) {

2092 insertVSETMTK(MBB, VSETTM);

2093 insertVSETMTK(MBB, VSETTK);

2094 }

2095

2096 BlockInfo.clear();

2097 return HaveVectorOp;

2098}

2099

2100

2102 return new RISCVInsertVSETVLI();

2103}

unsigned const MachineRegisterInfo * MRI

MachineInstrBuilder MachineInstrBuilder & DefMI

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

Function Alias Analysis false

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

Analysis containing CSE Info

#define LLVM_ATTRIBUTE_USED

#define LLVM_DUMP_METHOD

Mark debug helper function definitions like dump() that should not be stripped from debug builds.

static RegisterPass< DebugifyFunctionPass > DF("debugify-function", "Attach debug info to a function")

const HexagonInstrInfo * TII

static Interval intersect(const Interval &I1, const Interval &I2)

Promote Memory to Register

uint64_t IntrinsicInst * II

#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)

This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.

static cl::opt< bool > EnsureWholeVectorRegisterMoveValidVTYPE(DEBUG_TYPE "-whole-vector-register-move-valid-vtype", cl::Hidden, cl::desc("Insert vsetvlis before vmvNr.vs to ensure vtype is valid and " "vill is cleared"), cl::init(true))

static VSETVLIInfo adjustIncoming(const VSETVLIInfo &PrevInfo, const VSETVLIInfo &NewInfo, DemandedFields &Demanded)

Definition RISCVInsertVSETVLI.cpp:1313

#define RISCV_INSERT_VSETVLI_NAME

Definition RISCVInsertVSETVLI.cpp:39

static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))

static bool isValid(const char C)

Returns true if C is a valid mangled character: <0-9a-zA-Z_>.

SI Optimize VGPR LiveRange

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

AnalysisUsage & addUsedIfAvailable()

Add the specified Pass class to the set of analyses used by this pass.

AnalysisUsage & addPreserved()

Add the specified Pass class to the set of analyses preserved by this pass.

LLVM_ABI void setPreservesCFG()

This function should be called by the pass, iff they do not:

FunctionPass class - This class is used to implement most global optimizations.

void setWeight(float Value)

MachineInstr * getInstructionFromIndex(SlotIndex index) const

Returns the instruction associated with the given index.

SlotIndex InsertMachineInstrInMaps(MachineInstr &MI)

SlotIndexes * getSlotIndexes() const

SlotIndex getInstructionIndex(const MachineInstr &Instr) const

Returns the base index of the given instruction.

void RemoveMachineInstrFromMaps(MachineInstr &MI)

SlotIndex getMBBEndIdx(const MachineBasicBlock *mbb) const

Return the last index in the given basic block.

LiveInterval & getInterval(Register Reg)

void removeInterval(Register Reg)

Interval removal.

LLVM_ABI bool shrinkToUses(LiveInterval *li, SmallVectorImpl< MachineInstr * > *dead=nullptr)

After removing some uses of a register, shrink its live range to just the remaining uses.

LLVM_ABI void extendToIndices(LiveRange &LR, ArrayRef< SlotIndex > Indices, ArrayRef< SlotIndex > Undefs)

Extend the live range LR to reach all points in Indices.

LLVM_ABI void splitSeparateComponents(LiveInterval &LI, SmallVectorImpl< LiveInterval * > &SplitLIs)

Split separate components in LiveInterval LI into separate intervals.

MachineBasicBlock * getMBBFromIndex(SlotIndex index) const

LiveInterval & createAndComputeVirtRegInterval(Register Reg)

LLVM_ABI iterator addSegment(Segment S)

Add the specified Segment to this range, merging segments as appropriate.

const Segment * getSegmentContaining(SlotIndex Idx) const

Return the segment that contains the specified index, or null if there is none.

bool liveAt(SlotIndex index) const

bool overlaps(const LiveRange &other) const

overlaps - Return true if the intersection of the two live ranges is not empty.

VNInfo * getVNInfoBefore(SlotIndex Idx) const

getVNInfoBefore - Return the VNInfo that is live up to but not necessarily including Idx,...

bool containsOneValue() const

LLVM_ABI void removeSegment(SlotIndex Start, SlotIndex End, bool RemoveDeadValNo=false)

Remove the specified interval from this live range.

VNInfo * getVNInfoAt(SlotIndex Idx) const

getVNInfoAt - Return the VNInfo that is live at Idx, or NULL.

int getNumber() const

MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...

LLVM_ABI iterator getFirstTerminator()

Returns an iterator to the first terminator instruction of this basic block.

unsigned succ_size() const

LLVM_ABI iterator getFirstNonPHI()

Returns a pointer to the first instruction in this block that is not a PHINode instruction.

LLVM_ABI DebugLoc findDebugLoc(instr_iterator MBBI)

Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.

iterator_range< iterator > terminators()

iterator_range< succ_iterator > successors()

LLVM_ABI instr_iterator getFirstInstrTerminator()

Same getFirstTerminator but it ignores bundles and return an instr_iterator instead.

iterator_range< pred_iterator > predecessors()

MachineInstrBundleIterator< MachineInstr > iterator

LLVM_ABI StringRef getName() const

Return the name of the corresponding LLVM basic block, or an empty string.

MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...

void getAnalysisUsage(AnalysisUsage &AU) const override

getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.

const TargetSubtargetInfo & getSubtarget() const

getSubtarget - Return the subtarget for which this machine code is being compiled.

StringRef getName() const

getName - Return the name of the corresponding LLVM function.

MachineRegisterInfo & getRegInfo()

getRegInfo - Return information about the registers currently in use.

unsigned getNumBlockIDs() const

getNumBlockIDs - Return the number of MBB ID's allocated.

const MachineInstrBuilder & addImm(int64_t Val) const

Add a new immediate operand.

const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const

Add a new virtual register operand.

Representation of each machine instruction.

const MCInstrDesc & getDesc() const

Returns the target instruction descriptor of this MachineInstr.

LLVM_ABI void eraseFromParent()

Unlink 'this' from the containing basic block and delete it.

const MachineOperand & getOperand(unsigned i) const

MachineOperand class - Representation of each machine instruction operand.

bool isReg() const

isReg - Tests if this is a MO_Register operand.

LLVM_ABI void setReg(Register Reg)

Change the register this operand corresponds to.

bool isImm() const

isImm - Tests if this is a MO_Immediate operand.

void setIsKill(bool Val=true)

getReg - Returns the register number.

static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)

Wrapper class representing virtual and physical registers.

constexpr bool isValid() const

constexpr bool isVirtual() const

Return true if the specified register number is in the virtual register namespace.

SlotIndex - An opaque wrapper around machine indexes.

SlotIndex getRegSlot(bool EC=false) const

Returns the register use/def slot in the current instruction for a normal or early-clobber def.

SlotIndex getInstructionIndex(const MachineInstr &MI, bool IgnoreBundle=false) const

Returns the base index for the given instruction.

void push_back(const T &Elt)

TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...

VNInfo - Value Number Information.

SlotIndex def

The index of the defining instruction.

bool isPHIDef() const

Returns true if this value is defined by a PHI instruction (or was, PHI instructions may have been el...

This class implements an extremely fast bulk output stream that can only output to a stream.

Predicate all(Predicate P0, Predicate P1)

True iff P0 and P1 are true.

static unsigned getVecPolicyOpNum(const MCInstrDesc &Desc)

static unsigned getTMOpNum(const MCInstrDesc &Desc)

static bool usesMaskPolicy(uint64_t TSFlags)

static bool hasTWidenOp(uint64_t TSFlags)

static RISCVVType::VLMUL getLMul(uint64_t TSFlags)

static unsigned getTKOpNum(const MCInstrDesc &Desc)

static unsigned getVLOpNum(const MCInstrDesc &Desc)

static AltFmtType getAltFmtType(uint64_t TSFlags)

static bool hasTKOp(uint64_t TSFlags)

static bool hasVLOp(uint64_t TSFlags)

static bool hasTMOp(uint64_t TSFlags)

static unsigned getTNOpNum(const MCInstrDesc &Desc)

static bool hasVecPolicyOp(uint64_t TSFlags)

static unsigned getSEWOpNum(const MCInstrDesc &Desc)

static bool hasSEWOp(uint64_t TSFlags)

static bool isTailAgnostic(unsigned VType)

LLVM_ABI unsigned encodeXSfmmVType(unsigned SEW, unsigned Widen, bool AltFmt)

static unsigned getXSfmmWiden(unsigned VType)

LLVM_ABI std::optional< VLMUL > getSameRatioLMUL(unsigned Ratio, unsigned EEW)

static bool isMaskAgnostic(unsigned VType)

LLVM_ABI std::pair< unsigned, bool > decodeVLMUL(VLMUL VLMul)

static bool hasXSfmmWiden(unsigned VType)

LLVM_ABI unsigned getSEWLMULRatio(unsigned SEW, VLMUL VLMul)

static bool isValidSEW(unsigned SEW)

static bool isAltFmt(unsigned VType)

LLVM_ABI unsigned encodeVTYPE(VLMUL VLMUL, unsigned SEW, bool TailAgnostic, bool MaskAgnostic, bool AltFmt=false)

static unsigned getSEW(unsigned VType)

static VLMUL getVLMUL(unsigned VType)

unsigned getRVVMCOpcode(unsigned RVVPseudoOpcode)

static constexpr int64_t VLMaxSentinel

@ Implicit

Not emitted register (e.g. carry, or temporary result).

@ Define

@ Kill

The last use of a register.

initializer< Ty > init(const Ty &Val)

This is an optimization pass for GlobalISel generic memory operations.

void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)

FunctionAddr VTableAddr Value

Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)

MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)

Builder interface. Specify how to create the initial instruction itself.

bool operator!=(uint64_t V1, const APInt &V2)

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

iterator_range< po_iterator< T > > post_order(const T &G)

bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)

unsigned Log2_32(uint32_t Value)

Return the floor log base 2 of the specified value, -1 if the value is zero.

auto reverse(ContainerTy &&C)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

FunctionPass * createRISCVInsertVSETVLIPass()

Returns an instance of the Insert VSETVLI pass.

Definition RISCVInsertVSETVLI.cpp:2101

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

DWARFExpression::Operation Op

raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)

char & RISCVInsertVSETVLIID

Definition RISCVInsertVSETVLI.cpp:995

LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)

Prints virtual and physical registers with or without a TRI instance.

LLVM_ABI Printable printMBBReference(const MachineBasicBlock &MBB)

Prints a machine basic block reference.

static bool isRVVRegClass(const TargetRegisterClass *RC)