RISCVLoadStoreOptimizer.cpp Source File (original) (raw)

36using namespace llvm;

38#define DEBUG_TYPE "riscv-load-store-opt"

39#define RISCV_LOAD_STORE_OPT_NAME "RISC-V Load / Store Optimizer"

45 STATISTIC(NumLD2LW, "Number of LD instructions split back to LW");

46 STATISTIC(NumSD2SW, "Number of SD instructions split back to SW");

48namespace {

51 static char ID;

56 MachineFunctionProperties getRequiredProperties() const override {

57 return MachineFunctionProperties().setNoVRegs();

58 }

60 void getAnalysisUsage(AnalysisUsage &AU) const override {

63 }

73 bool tryConvertToXqcilsmLdStPair(MachineFunction *MF,

76 bool tryConvertToMIPSLdStPair(MachineFunction *MF,

84 bool &MergeForward);

91 bool fixInvalidRegPairOp(MachineBasicBlock &MBB,

94 void splitLdSdIntoTwo(MachineBasicBlock &MBB,

97private:

99 MachineRegisterInfo *MRI;

100 const RISCVInstrInfo *TII;

101 const RISCVRegisterInfo *TRI;

102 LiveRegUnits ModifiedRegUnits, UsedRegUnits;

103};

104}

105

106char RISCVLoadStoreOpt::ID = 0;

108 false)

109

111 if (skipFunction(Fn.getFunction()))

112 return false;

114

115 bool MadeChange = false;

116 TII = Subtarget.getInstrInfo();

117 TRI = Subtarget.getRegisterInfo();

118 MRI = &Fn.getRegInfo();

119 AA = &getAnalysis().getAAResults();

120 ModifiedRegUnits.init(*TRI);

121 UsedRegUnits.init(*TRI);

122

123 if (Subtarget.useMIPSLoadStorePairs() || Subtarget.hasVendorXqcilsm()) {

124 for (MachineBasicBlock &MBB : Fn) {

125 LLVM_DEBUG(dbgs() << "MBB: " << MBB.getName() << "\n");

126

127 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();

128 MBBI != E;) {

129 if (TII->isPairableLdStInstOpc(MBBI->getOpcode()) &&

130 tryToPairLdStInst(MBBI))

131 MadeChange = true;

132 else

133 ++MBBI;

134 }

135 }

136 }

137

138 if (!Subtarget.is64Bit() && Subtarget.hasStdExtZilsd()) {

139 for (auto &MBB : Fn) {

140 for (auto MBBI = MBB.begin(), E = MBB.end(); MBBI != E;) {

141 if (fixInvalidRegPairOp(MBB, MBBI)) {

142 MadeChange = true;

143

144 } else {

145 ++MBBI;

146 }

147 }

148 }

149 }

150

151 return MadeChange;

152}

153

154

155

157 MachineInstr &MI = *MBBI;

158

159

160 if (MI.hasOrderedMemoryRef())

161 return false;

162

163 if ( TII ->isLdStSafeToPair(MI, TRI))

164 return false;

165

166

168 bool MergeForward;

170 if (Paired != E) {

171 MBBI = mergePairedInsns(MBBI, Paired, MergeForward);

172 return true;

173 }

174 return false;

175}

176

177bool RISCVLoadStoreOpt::tryConvertToXqcilsmLdStPair(

180 unsigned Opc = First->getOpcode();

181 if ((Opc != RISCV::LW && Opc != RISCV::SW) || Second->getOpcode() != Opc)

182 return false;

183

184 const auto &FirstOp1 = First->getOperand(1);

185 const auto &SecondOp1 = Second->getOperand(1);

186 const auto &FirstOp2 = First->getOperand(2);

187 const auto &SecondOp2 = Second->getOperand(2);

188

189

190 if (!FirstOp1.isReg() || !SecondOp1.isReg() || !FirstOp2.isImm() ||

191 !SecondOp2.isImm())

192 return false;

193

194 Register Base1 = FirstOp1.getReg();

195 Register Base2 = SecondOp1.getReg();

196

197 if (Base1 != Base2)

198 return false;

199

200 const MachineMemOperand *MMO = *First->memoperands_begin();

202

203 if (MMOAlign < Align(4))

204 return false;

205

206 auto &FirstOp0 = First->getOperand(0);

207 auto &SecondOp0 = Second->getOperand(0);

208

209 int64_t Off1 = FirstOp2.getImm();

210 int64_t Off2 = SecondOp2.getImm();

211

212 if (Off2 < Off1) {

215 }

216

217 Register StartReg = FirstOp0.getReg();

218 Register NextReg = SecondOp0.getReg();

219

220 if (StartReg == RISCV::X0 || NextReg == RISCV::X0)

221 return false;

222

223

224 if (Opc == RISCV::LW && (StartReg == Base1 || NextReg == Base1))

225 return false;

226

228 return false;

229

230 if (NextReg != StartReg + 1)

231 return false;

232

233 unsigned XqciOpc = (Opc == RISCV::LW) ? RISCV::QC_LWMI : RISCV::QC_SWMI;

234

237 auto NextRegState =

238 (Opc == RISCV::LW)

241

243 First->getDebugLoc() ? First->getDebugLoc() : Second->getDebugLoc();

244 MachineInstrBuilder MIB = BuildMI(*MF, DL, TII->get(XqciOpc));

245 MIB.addReg(StartReg, StartRegState)

250 .addReg(NextReg, NextRegState);

251

253 First->removeFromParent();

254 Second->removeFromParent();

255

256 return true;

257}

258

259bool RISCVLoadStoreOpt::tryConvertToMIPSLdStPair(

262

263

264 unsigned PairOpc;

265 Align RequiredAlignment;

266 switch (First->getOpcode()) {

267 default:

268 llvm_unreachable("Unsupported load/store instruction for pairing");

269 case RISCV::SW:

270 PairOpc = RISCV::MIPS_SWP;

271 RequiredAlignment = Align(8);

272 break;

273 case RISCV::LW:

274 PairOpc = RISCV::MIPS_LWP;

275 RequiredAlignment = Align(8);

276 break;

277 case RISCV::SD:

278 PairOpc = RISCV::MIPS_SDP;

279 RequiredAlignment = Align(16);

280 break;

281 case RISCV::LD:

282 PairOpc = RISCV::MIPS_LDP;

283 RequiredAlignment = Align(16);

284 break;

285 }

286

287 const MachineMemOperand *MMO = *First->memoperands_begin();

289

290 if (MMOAlign < RequiredAlignment)

291 return false;

292

293 int64_t Offset = First->getOperand(2).getImm();

295 return false;

296

297 MachineInstrBuilder MIB = BuildMI(

298 *MF, First->getDebugLoc() ? First->getDebugLoc() : Second->getDebugLoc(),

299 TII->get(PairOpc));

300 MIB.add(First->getOperand(0))

301 .add(Second->getOperand(0))

305

307

308 First->removeFromParent();

309 Second->removeFromParent();

310

311 return true;

312}

313

314

315

316

317

318

319bool RISCVLoadStoreOpt::tryConvertToLdStPair(

321 MachineFunction *MF = First->getMF();

322 const RISCVSubtarget &STI = MF->getSubtarget();

323

324

325 if (!STI.is64Bit() && STI.hasVendorXqcilsm())

326 return tryConvertToXqcilsmLdStPair(MF, First, Second);

327

328

329 return tryConvertToMIPSLdStPair(MF, First, Second);

330}

331

336 if (MIa.mayAlias(AA, *MIb, false))

337 return true;

338

339 return false;

340}

341

342

343

344

345

346

347

350 bool &MergeForward) {

353 MachineInstr &FirstMI = *I;

355

360 int64_t OffsetStride = (*FirstMI.memoperands_begin())->getSize().getValue();

361

362 MergeForward = false;

363

364

365

366 ModifiedRegUnits.clear();

367 UsedRegUnits.clear();

368

369

370 SmallVector<MachineInstr *, 4> MemInsns;

371

374 MachineInstr &MI = *MBBI;

375

376

377

378 if (.isTransient())

380

381 if (MI.getOpcode() == FirstMI.getOpcode() &&

382 TII->isLdStSafeToPair(MI, TRI)) {

383 Register MIBaseReg = MI.getOperand(1).getReg();

384 int64_t MIOffset = MI.getOperand(2).getImm();

385

386 if (BaseReg == MIBaseReg) {

387 if ((Offset != MIOffset + OffsetStride) &&

388 (Offset + OffsetStride != MIOffset)) {

392 continue;

393 }

394

395

396

397 if (MayLoad &&

398 TRI->isSuperOrSubRegisterEq(Reg, MI.getOperand(0).getReg())) {

402 continue;

403 }

404

405

406 if (!ModifiedRegUnits.available(BaseReg))

407 return E;

408

409

410

411

412

413 if (ModifiedRegUnits.available(MI.getOperand(0).getReg()) &&

414 !(MI.mayLoad() &&

415 !UsedRegUnits.available(MI.getOperand(0).getReg())) &&

417

418 MergeForward = false;

420 }

421

422

423

424

425

426 if (!(MayLoad &&

428 mayAlias (FirstMI, MemInsns, AA)) {

429

431 MergeForward = true;

433 }

434 }

435

436

437 }

438 }

439

440

441

442 if (MI.isCall())

443 return E;

444

445

447

448

449

450 if (!ModifiedRegUnits.available(BaseReg))

451 return E;

452

453

454 if (MI.mayLoadOrStore())

456 }

457 return E;

458}

459

463 bool MergeForward) {

466

467

468

469

470 if (NextI == Paired)

472

473

474

477 int Offset = I->getOperand(2).getImm();

478 int PairedOffset = Paired->getOperand(2).getImm();

479 bool InsertAfter = (Offset < PairedOffset) ^ MergeForward;

480

481 if (!MergeForward)

482 Paired->getOperand(1).setIsKill(false);

483

484

485 if (I->getOperand(0).isUse()) {

486 if (!MergeForward) {

487

488

489 MachineOperand &PairedRegOp = Paired->getOperand(0);

490 if (PairedRegOp.isKill()) {

491 for (auto It = std::next(I); It != Paired; ++It) {

492 if (It->readsRegister(PairedRegOp.getReg(), TRI)) {

494 break;

495 }

496 }

497 }

498 } else {

499

500

502 for (MachineInstr &MI : make_range(std::next(I), std::next(Paired)))

503 MI.clearRegisterKills(Reg, TRI);

504 }

505 }

506

507 MachineInstr *ToInsert = DeletionPoint->removeFromParent();

508 MachineBasicBlock &MBB = *InsertionPoint->getParent();

510

511 if (!InsertAfter) {

513 Second = InsertionPoint;

514 } else {

516 First = InsertionPoint;

517 }

518

519 if (tryConvertToLdStPair(First, Second)) {

522 }

523

524 return NextI;

525}

526

527

528

529

530

532

533

534

535

536

537

538 if (First == RISCV::X0)

539 return Second == RISCV::X0;

540

541

542 unsigned FirstNum = TRI->getEncodingValue(First);

543 unsigned SecondNum = TRI->getEncodingValue(Second);

544

545

546 return (FirstNum % 2 == 0) && (SecondNum == FirstNum + 1);

547}

548

549void RISCVLoadStoreOpt::splitLdSdIntoTwo(MachineBasicBlock &MBB,

551 bool IsLoad) {

552 MachineInstr *MI = &*MBBI;

554

555 const MachineOperand &FirstOp = MI->getOperand(0);

556 const MachineOperand &SecondOp = MI->getOperand(1);

557 const MachineOperand &BaseOp = MI->getOperand(2);

561

562

563 const MachineOperand &OffsetOp = MI->getOperand(3);

564 int BaseOffset;

566 BaseOffset = OffsetOp.getImm();

567 else

568

569 BaseOffset = OffsetOp.getOffset();

570

571 unsigned Opc = IsLoad ? RISCV::LW : RISCV::SW;

572 MachineInstrBuilder MIB1, MIB2;

573

574

575 if (IsLoad) {

576

577

578

579

580

581

582

583

584

585 if (FirstReg == BaseReg) {

594

595 } else {

600

605 }

606

607 ++NumLD2LW;

608 LLVM_DEBUG(dbgs() << "Split LD back to two LW instructions\n");

609 } else {

611 FirstReg != SecondReg &&

612 "First register and second register is impossible to be same register");

616

620

621 ++NumSD2SW;

622 LLVM_DEBUG(dbgs() << "Split SD back to two SW instructions\n");

623 }

624

625

628 MIB2.addImm(BaseOffset + 4);

629 else if (OffsetOp.isGlobal())

635 else if (OffsetOp.isBlockAddress())

638

639

640

641

644

645

647}

648

649bool RISCVLoadStoreOpt::fixInvalidRegPairOp(MachineBasicBlock &MBB,

651 MachineInstr *MI = &*MBBI;

652 unsigned Opcode = MI->getOpcode();

653

654

655 if (Opcode != RISCV::PseudoLD_RV32_OPT && Opcode != RISCV::PseudoSD_RV32_OPT)

656 return false;

657

658 bool IsLoad = Opcode == RISCV::PseudoLD_RV32_OPT;

659

660 const MachineOperand &FirstOp = MI->getOperand(0);

661 const MachineOperand &SecondOp = MI->getOperand(1);

664

665 if (!isValidZilsdRegPair(FirstReg, SecondReg)) {

666

667 splitLdSdIntoTwo(MBB, MBBI, IsLoad);

668 return true;

669 }

670

671

672 const MachineOperand &BaseOp = MI->getOperand(2);

675

676 const MachineOperand &OffsetOp = MI->getOperand(3);

677

678 unsigned RealOpc = IsLoad ? RISCV::LD_RV32 : RISCV::SD_RV32;

679

680

681 unsigned RegPair = TRI->getMatchingSuperReg(FirstReg, RISCV::sub_gpr_even,

682 &RISCV::GPRPairRegClass);

683

685

686 if (IsLoad) {

687

690 } else {

691

693 }

694

698

699 LLVM_DEBUG(dbgs() << "Converted pseudo to real instruction: " << *MIB

700 << "\n");

701

702

704

705 return true;

706}

707

708

710 return new RISCVLoadStoreOpt();

711}

unsigned const MachineRegisterInfo * MRI

static bool mayAlias(MachineInstr &MIa, SmallVectorImpl< MachineInstr * > &MemInsns, AliasAnalysis *AA)

static cl::opt< unsigned > LdStLimit("aarch64-load-store-scan-limit", cl::init(20), cl::Hidden)

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

MachineBasicBlock MachineBasicBlock::iterator MBBI

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

const HexagonInstrInfo * TII

std::pair< Instruction::BinaryOps, Value * > OffsetOp

Find all possible pairs (BinOp, RHS) that BinOp V, RHS can be simplified.

Promote Memory to Register

#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)

#define RISCV_LOAD_STORE_OPT_NAME

Definition RISCVLoadStoreOptimizer.cpp:39

static cl::opt< unsigned > LdStLimit("riscv-load-store-scan-limit", cl::init(128), cl::Hidden)

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

AnalysisUsage & addRequired()

FunctionPass class - This class is used to implement most global optimizations.

static void accumulateUsedDefed(const MachineInstr &MI, LiveRegUnits &ModifiedRegUnits, LiveRegUnits &UsedRegUnits, const TargetRegisterInfo *TRI)

For a machine instruction MI, adds all register units used in UsedRegUnits and defined or clobbered i...

bool available(MCRegister Reg) const

Returns true if no part of physical register Reg is live.

void clear()

Clears the set.

LLVM_ABI instr_iterator insert(instr_iterator I, MachineInstr *M)

Insert MI into the instruction list before I, possibly inside a bundle.

LLVM_ABI instr_iterator erase(instr_iterator I)

Remove an instruction from the instruction list and delete it.

iterator insertAfter(iterator I, MachineInstr *MI)

Insert MI into the instruction list after I.

MachineInstrBundleIterator< MachineInstr > iterator

MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...

void getAnalysisUsage(AnalysisUsage &AU) const override

getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.

const TargetSubtargetInfo & getSubtarget() const

getSubtarget - Return the subtarget for which this machine code is being compiled.

const MachineInstrBuilder & cloneMergedMemRefs(ArrayRef< const MachineInstr * > OtherMIs) const

const MachineInstrBuilder & addImm(int64_t Val) const

Add a new immediate operand.

const MachineInstrBuilder & addBlockAddress(const BlockAddress *BA, int64_t Offset=0, unsigned TargetFlags=0) const

const MachineInstrBuilder & add(const MachineOperand &MO) const

const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const

const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const

const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const

Add a new virtual register operand.

const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const

Representation of each machine instruction.

unsigned getOpcode() const

Returns the opcode of this MachineInstr.

const MachineBasicBlock * getParent() const

LLVM_ABI bool mayAlias(BatchAAResults *AA, const MachineInstr &Other, bool UseTBAA) const

Returns true if this instruction's memory access aliases the memory access of Other.

bool mayLoad(QueryType Type=AnyInBundle) const

Return true if this instruction could possibly read memory.

mmo_iterator memoperands_begin() const

Access to memory operands of the instruction.

const MachineOperand & getOperand(unsigned i) const

LLVM_ABI Align getAlign() const

Return the minimum known alignment in bytes of the actual memory reference.

void setIsKill(bool Val=true)

getReg - Returns the register number.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

void push_back(const T &Elt)

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

Abstract Attribute helper functions.

constexpr char Align[]

Key for Kernel::Arg::Metadata::mAlign.

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

@ Implicit

Not emitted register (e.g. carry, or temporary result).

@ Define

initializer< Ty > init(const Ty &Val)

BaseReg

Stack frame base register. Bit 0 of FREInfo.Info.

This is an optimization pass for GlobalISel generic memory operations.

IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)

Increment It, then continue incrementing it while it points to a debug instruction.

FunctionPass * createRISCVLoadStoreOptPass()

Definition RISCVLoadStoreOptimizer.cpp:709

MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)

Builder interface. Specify how to create the initial instruction itself.

iterator_range< T > make_range(T x, T y)

Convenience function for iterating over sub-ranges.

unsigned getDeadRegState(bool B)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

FunctionAddr VTableAddr Count

constexpr bool isUInt(uint64_t x)

Checks if an unsigned integer fits into the given bit width.

@ First

Helpers to iterate all locations in the MemoryEffectsBase class.

unsigned getKillRegState(bool B)

AAResults AliasAnalysis

Temporary typedef for legacy code that uses a generic AliasAnalysis pointer or reference.

IterT prev_nodbg(IterT It, IterT Begin, bool SkipPseudoOp=true)

Decrement It, then continue decrementing it while it points to a debug instruction.

constexpr bool isShiftedUInt(uint64_t x)

Checks if a unsigned integer is an N bit number shifted left by S.

void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)

Implement std::swap in terms of BitVector swap.