FixupStatepointCallerSaved.cpp Source File (original) (raw)

34using namespace llvm;

36#define DEBUG_TYPE "fixup-statepoint-caller-saved"

37 STATISTIC(NumSpilledRegisters, "Number of spilled register");

38 STATISTIC(NumSpillSlotsAllocated, "Number of spill slots allocated");

39 STATISTIC(NumSpillSlotsExtended, "Number of spill slots extended");

43 cl::desc("Allow spill in spill slot of greater size than register size"),

48 cl::desc("Allow passing GC Pointer arguments in callee saved registers"));

52 cl::desc("Enable simple copy propagation during register reloading"));

57 "fixup-max-csr-statepoints", cl::Hidden,

58 cl::desc("Max number of statepoints allowed to pass GC Ptrs in registers"));

60namespace {

62struct FixupStatepointCallerSavedImpl {

64};

67public:

68 static char ID;

70 FixupStatepointCallerSavedLegacy() : MachineFunctionPass(ID) {

73 }

74 void getAnalysisUsage(AnalysisUsage &AU) const override {

77 }

79 StringRef getPassName() const override {

80 return "Fixup Statepoint Caller Saved";

81 }

83 bool runOnMachineFunction(MachineFunction &MF) override;

84};

86}

88char FixupStatepointCallerSavedLegacy::ID = 0;

92 "Fixup Statepoint Caller Saved", false, false)

94 "Fixup Statepoint Caller Saved", false, false)

99 return TRI.getSpillSize(*RC);

100}

101

102

103

104

105

106

107

108

109

110

111

112

113

114

119

120 int Idx = RI->findRegisterUseOperandIdx(Reg, &TRI, false);

121 if (Idx >= 0 && (unsigned)Idx < StatepointOpers(&*RI).getNumDeoptArgsIdx()) {

122 IsKill = false;

123 return Reg;

124 }

125

127 return Reg;

128

132 for (auto It = ++(RI.getReverse()); It != E; ++It) {

133 if (It->readsRegister(Reg, &TRI) && Use )

134 Use = &*It;

135 if (It->modifiesRegister(Reg, &TRI)) {

136 Def = &*It;

137 break;

138 }

139 }

140

141 if (!Def)

142 return Reg;

143

144 auto DestSrc = TII.isCopyInstr(*Def);

145 if (!DestSrc || DestSrc->Destination->getReg() != Reg)

146 return Reg;

147

148 Register SrcReg = DestSrc->Source->getReg();

149

151 return Reg;

152

153 LLVM_DEBUG(dbgs() << "spillRegisters: perform copy propagation "

155 << "\n");

156

157

159 IsKill = DestSrc->Source->isKill();

160

161 if ( Use ) {

162

163

164 LLVM_DEBUG(dbgs() << "spillRegisters: removing dead copy " << *Def);

165 Def->eraseFromParent();

166 } else if (IsKill) {

167

168

169 const_cast<MachineOperand *>(DestSrc->Source)->setIsKill(false);

170 }

171

172 return SrcReg;

173}

174

175namespace {

176

177using RegSlotPair = std::pair<Register, int>;

178

179

180class RegReloadCache {

181 using ReloadSet = SmallSet<RegSlotPair, 8>;

182 DenseMap<const MachineBasicBlock *, ReloadSet> Reloads;

183

184public:

185 RegReloadCache() = default;

186

187

188

189 bool tryRecordReload(Register Reg, int FI, const MachineBasicBlock *MBB) {

190 RegSlotPair RSP(Reg, FI);

191 return Reloads[MBB].insert(RSP).second;

192 }

193};

194

195

196

197

198

199

200class FrameIndexesCache {

201private:

202 struct FrameIndexesPerSize {

203

204 SmallVector<int, 8> Slots;

205

206 unsigned Index = 0;

207 };

208 MachineFrameInfo &MFI;

209 const TargetRegisterInfo &TRI;

210

211

212

213

214 DenseMap<unsigned, FrameIndexesPerSize> Cache;

215

216

217

218 SmallSet<int, 8> ReservedSlots;

219

220

221

222

223 DenseMap<const MachineBasicBlock *, SmallVector<RegSlotPair, 8>>

224 GlobalIndices;

225

226 FrameIndexesPerSize &getCacheBucket(unsigned Size) {

227

228

230 }

231

232public:

233 FrameIndexesCache(MachineFrameInfo &MFI, const TargetRegisterInfo &TRI)

234 : MFI(MFI), TRI(TRI) {}

235

236

237

238 void reset(const MachineBasicBlock *EHPad) {

239 for (auto &It : Cache)

240 It.second.Index = 0;

241

242 ReservedSlots.clear();

243 if (EHPad)

244 if (auto It = GlobalIndices.find(EHPad); It != GlobalIndices.end())

246 }

247

248

249 int getFrameIndex(Register Reg, MachineBasicBlock *EHPad) {

250

251 auto It = GlobalIndices.find(EHPad);

252 if (It != GlobalIndices.end()) {

253 auto &Vec = It->second;

255 Vec, [Reg](RegSlotPair &RSP) { return Reg == RSP.first; });

256 if (Idx != Vec.end()) {

257 int FI = Idx->second;

258 LLVM_DEBUG(dbgs() << "Found global FI " << FI << " for register "

261 assert(ReservedSlots.count(FI) && "using unreserved slot");

262 return FI;

263 }

264 }

265

267 FrameIndexesPerSize &Line = getCacheBucket(Size);

268 while (Line.Index < Line.Slots.size()) {

269 int FI = Line.Slots[Line.Index++];

270 if (ReservedSlots.count(FI))

271 continue;

272

273

274 if (MFI.getObjectSize(FI) < Size) {

275 MFI.setObjectSize(FI, Size);

276 MFI.setObjectAlignment(FI, Align(Size));

277 NumSpillSlotsExtended++;

278 }

279 return FI;

280 }

281 int FI = MFI.CreateSpillStackObject(Size, Align(Size));

282 NumSpillSlotsAllocated++;

283 Line.Slots.push_back(FI);

284 ++Line.Index;

285

286

287 if (EHPad) {

288 GlobalIndices[EHPad].push_back(std::make_pair(Reg, FI));

289 LLVM_DEBUG(dbgs() << "Reserved FI " << FI << " for spilling reg "

290 << printReg(Reg, &TRI) << " at landing pad "

292 }

293

294 return FI;

295 }

296

297

298

299

300 void sortRegisters(SmallVectorImpl &Regs) {

302 return;

305 });

306 }

307};

308

309

310class StatepointState {

311private:

312

313 MachineInstr &MI;

314 MachineFunction &MF;

315

316 MachineBasicBlock *EHPad;

317 const TargetRegisterInfo &TRI;

318 const TargetInstrInfo &TII;

319 MachineFrameInfo &MFI;

320

321 const uint32_t *Mask;

322

323 FrameIndexesCache &CacheFI;

324 bool AllowGCPtrInCSR;

325

326 SmallVector<unsigned, 8> OpsToSpill;

327

329

331

332 DenseMap<Register, int> RegToSlotIdx;

333

334public:

335 StatepointState(MachineInstr &MI, const uint32_t *Mask,

336 FrameIndexesCache &CacheFI, bool AllowGCPtrInCSR)

337 : MI(MI), MF(*MI.getMF()), TRI(*MF.getSubtarget().getRegisterInfo()),

338 TII(*MF.getSubtarget().getInstrInfo()), MFI(MF.getFrameInfo()),

339 Mask(Mask), CacheFI(CacheFI), AllowGCPtrInCSR(AllowGCPtrInCSR) {

340

341

342 EHPad = nullptr;

344

346 [](MachineInstr &I) {

347 return I.getOpcode() == TargetOpcode::STATEPOINT;

348 });

349

351 return;

352

353 auto IsEHPad = [](MachineBasicBlock *B) { return B->isEHPad(); };

354

356

359 EHPad = *It;

360 }

361

362 MachineBasicBlock *getEHPad() const { return EHPad; }

363

364

366 return (Mask[Reg.id() / 32] >> (Reg.id() % 32)) & 1;

367 }

368

369

370

371

372 bool findRegistersToSpill() {

373 SmallSet<Register, 8> GCRegs;

374

375

376 for (const auto &Def : MI.defs())

378

379 SmallSet<Register, 8> VisitedRegs;

380 for (unsigned Idx = StatepointOpers(&MI).getVarIdx(),

381 EndIdx = MI.getNumOperands();

382 Idx < EndIdx; ++Idx) {

383 MachineOperand &MO = MI.getOperand(Idx);

385 continue;

388

389 if (isCalleeSaved(Reg) && (AllowGCPtrInCSR || !GCRegs.contains(Reg)))

390 continue;

391

393 << Idx << "\n");

394

395 if (VisitedRegs.insert(Reg).second)

396 RegsToSpill.push_back(Reg);

397 OpsToSpill.push_back(Idx);

398 }

399 CacheFI.sortRegisters(RegsToSpill);

400 return !RegsToSpill.empty();

401 }

402

403

404

405 void spillRegisters() {

407 int FI = CacheFI.getFrameIndex(Reg, EHPad);

408

409 NumSpilledRegisters++;

410 RegToSlotIdx[Reg] = FI;

411

413 << "\n");

414

415

416 bool IsKill = true;

419 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);

420

421 LLVM_DEBUG(dbgs() << "Insert spill before " << *InsertBefore);

422 TII.storeRegToStackSlot(*MI.getParent(), InsertBefore, Reg, IsKill, FI,

424 }

425 }

426

428 MachineBasicBlock *MBB) {

429 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);

430 int FI = RegToSlotIdx[Reg];

431 if (It != MBB->end()) {

432 TII.loadRegFromStackSlot(*MBB, It, Reg, FI, RC, Register());

433 return;

434 }

435

436

437

439 --It;

440 TII.loadRegFromStackSlot(*MBB, It, Reg, FI, RC, Register());

441 MachineInstr *Reload = It->getPrevNode();

442 int Dummy = 0;

443 (void)Dummy;

444 assert(TII.isLoadFromStackSlot(*Reload, Dummy) == Reg);

448 }

449

450

451 void insertReloads(MachineInstr *NewStatepoint, RegReloadCache &RC) {

452 MachineBasicBlock *MBB = NewStatepoint->getParent();

453 auto InsertPoint = std::next(NewStatepoint->getIterator());

454

455 for (auto Reg : RegsToReload) {

456 insertReloadBefore(Reg, InsertPoint, MBB);

458 << RegToSlotIdx[Reg] << " after statepoint\n");

459

460 if (EHPad && RC.tryRecordReload(Reg, RegToSlotIdx[Reg], EHPad)) {

461 auto EHPadInsertPoint =

462 EHPad->SkipPHIsLabelsAndDebug(EHPad->begin(), Reg);

463 insertReloadBefore(Reg, EHPadInsertPoint, EHPad);

466 }

467 }

468 }

469

470

471

472 MachineInstr *rewriteStatepoint() {

473 MachineInstr *NewMI =

474 MF.CreateMachineInstr(TII.get(MI.getOpcode()), MI.getDebugLoc(), true);

475 MachineInstrBuilder MIB(MF, NewMI);

476

477 unsigned NumOps = MI.getNumOperands();

478

479

480 SmallVector<unsigned, 8> NewIndices;

481 unsigned NumDefs = MI.getNumDefs();

482 for (unsigned I = 0; I < NumDefs; ++I) {

483 MachineOperand &DefMO = MI.getOperand(I);

484 assert(DefMO.isReg() && DefMO.isDef() && "Expected Reg Def operand");

486 assert(DefMO.isTied() && "Def is expected to be tied");

487

488

489 if (MI.getOperand(MI.findTiedOperandIdx(I)).isUndef()) {

490 if (AllowGCPtrInCSR) {

493 }

494 continue;

495 }

496 if (!AllowGCPtrInCSR) {

498 RegsToReload.push_back(Reg);

499 } else {

500 if (isCalleeSaved(Reg)) {

503 } else {

505 RegsToReload.push_back(Reg);

506 }

507 }

508 }

509

510

511 OpsToSpill.push_back(MI.getNumOperands());

512 unsigned CurOpIdx = 0;

513

514 for (unsigned I = NumDefs; I < MI.getNumOperands(); ++I) {

515 MachineOperand &MO = MI.getOperand(I);

516 if (I == OpsToSpill[CurOpIdx]) {

517 int FI = RegToSlotIdx[MO.getReg()];

518 MIB.addImm(StackMaps::IndirectMemRefOp);

520 assert(MO.isReg() && "Should be register");

522 MIB.addFrameIndex(FI);

523 MIB.addImm(0);

524 ++CurOpIdx;

525 } else {

526 MIB.add(MO);

527 unsigned OldDef;

528 if (AllowGCPtrInCSR && MI.isRegTiedToDefOperand(I, &OldDef)) {

529 assert(OldDef < NumDefs);

531 MIB->tieOperands(NewIndices[OldDef], MIB->getNumOperands() - 1);

532 }

533 }

534 }

535 assert(CurOpIdx == (OpsToSpill.size() - 1) && "Not all operands processed");

536

537 NewMI->setMemRefs(MF, MI.memoperands());

538 for (auto It : RegToSlotIdx) {

545 auto *MMO =

546 MF.getMachineMemOperand(PtrInfo, Flags, getRegisterSize(TRI, R),

547 MFI.getObjectAlign(FrameIndex));

549 }

550

551

552 MI.getParent()->insert(MI, NewMI);

553

554 LLVM_DEBUG(dbgs() << "rewritten statepoint to : " << *NewMI << "\n");

555 MI.eraseFromParent();

556 return NewMI;

557 }

558};

559

560class StatepointProcessor {

561private:

562 MachineFunction &MF;

563 const TargetRegisterInfo &TRI;

564 FrameIndexesCache CacheFI;

565 RegReloadCache ReloadCache;

566

567public:

568 StatepointProcessor(MachineFunction &MF)

569 : MF(MF), TRI(*MF.getSubtarget().getRegisterInfo()),

570 CacheFI(MF.getFrameInfo(), TRI) {}

571

572 bool process(MachineInstr &MI, bool AllowGCPtrInCSR) {

573 StatepointOpers SO(&MI);

574 uint64_t Flags = SO.getFlags();

575

576 if (Flags & (uint64_t)StatepointFlags::DeoptLiveIn)

577 return false;

578 LLVM_DEBUG(dbgs() << "\nMBB " << MI.getParent()->getNumber() << " "

579 << MI.getParent()->getName() << " : process statepoint "

580 << MI);

581 CallingConv::ID CC = SO.getCallingConv();

582 const uint32_t *Mask = TRI.getCallPreservedMask(MF, CC);

583 StatepointState SS(MI, Mask, CacheFI, AllowGCPtrInCSR);

584 CacheFI.reset(SS.getEHPad());

585

586 if (.findRegistersToSpill())

587 return false;

588

589 SS.spillRegisters();

590 auto *NewStatepoint = SS.rewriteStatepoint();

591 SS.insertReloads(NewStatepoint, ReloadCache);

592 return true;

593 }

594};

595}

596

597bool FixupStatepointCallerSavedImpl::run(MachineFunction &MF) {

599 if (.hasGC())

600 return false;

601

603 for (MachineBasicBlock &BB : MF)

604 for (MachineInstr &I : BB)

605 if (I.getOpcode() == TargetOpcode::STATEPOINT)

607

608 if (Statepoints.empty())

609 return false;

610

612 StatepointProcessor SPP(MF);

613 unsigned NumStatepoints = 0;

615 for (MachineInstr *I : Statepoints) {

616 ++NumStatepoints;

619 AllowGCPtrInCSR = false;

620 Changed |= SPP.process(*I, AllowGCPtrInCSR);

621 }

623}

624

625bool FixupStatepointCallerSavedLegacy::runOnMachineFunction(

626 MachineFunction &MF) {

628 return false;

629

630 return FixupStatepointCallerSavedImpl().run(MF);

631}

632

633PreservedAnalyses

636

637 if (!FixupStatepointCallerSavedImpl().run(MF))

639

642 return PA;

643}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

const TargetInstrInfo & TII

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

static Register performCopyPropagation(Register Reg, MachineBasicBlock::iterator &RI, bool &IsKill, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI)

Definition FixupStatepointCallerSaved.cpp:115

static cl::opt< bool > PassGCPtrInCSR("fixup-allow-gcptr-in-csr", cl::Hidden, cl::init(false), cl::desc("Allow passing GC Pointer arguments in callee saved registers"))

static cl::opt< unsigned > MaxStatepointsWithRegs("fixup-max-csr-statepoints", cl::Hidden, cl::desc("Max number of statepoints allowed to pass GC Ptrs in registers"))

Fixup Statepoint Caller static false unsigned getRegisterSize(const TargetRegisterInfo &TRI, Register Reg)

Definition FixupStatepointCallerSaved.cpp:97

static cl::opt< bool > FixupSCSExtendSlotSize("fixup-scs-extend-slot-size", cl::Hidden, cl::init(false), cl::desc("Allow spill in spill slot of greater size than register size"), cl::Hidden)

static cl::opt< bool > EnableCopyProp("fixup-scs-enable-copy-propagation", cl::Hidden, cl::init(true), cl::desc("Enable simple copy propagation during register reloading"))

const size_t AbstractManglingParser< Derived, Alloc >::NumOps

Promote Memory to Register

#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)

#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)

This file defines the SmallSet class.

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

LLVM_ABI void setPreservesCFG()

This function should be called by the pass, iff they do not:

Represents analyses that only rely on functions' control flow.

PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)

Definition FixupStatepointCallerSaved.cpp:634

MachineInstr * remove(MachineInstr *I)

Remove the unbundled instruction from the instruction list without deleting it.

MachineInstrBundleIterator< MachineInstr, true > reverse_iterator

const MachineFunction * getParent() const

Return the MachineFunction containing this basic block.

iterator_range< succ_iterator > successors()

iterator insertAfter(iterator I, MachineInstr *MI)

Insert MI into the instruction list after I.

MachineInstrBundleIterator< MachineInstr > iterator

MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...

void getAnalysisUsage(AnalysisUsage &AU) const override

getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.

Function & getFunction()

Return the LLVM function that this machine code represents.

reverse_iterator getReverse() const

Get a reverse iterator to the same node.

instr_iterator getInstrIterator() const

Representation of each machine instruction.

const MachineBasicBlock * getParent() const

unsigned getNumOperands() const

Retuns the total number of operands.

LLVM_ABI void setMemRefs(MachineFunction &MF, ArrayRef< MachineMemOperand * > MemRefs)

Assign this MachineInstr's memory reference descriptor list.

LLVM_ABI void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)

Add a MachineMemOperand to the machine instruction.

Flags

Flags values. These may be or'd together.

@ MOLoad

The memory access reads data.

@ MOStore

The memory access writes data.

MachineOperand class - Representation of each machine instruction operand.

bool isReg() const

isReg - Tests if this is a MO_Register operand.

getReg - Returns the register number.

static LLVM_ABI PassRegistry * getPassRegistry()

getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

Wrapper class representing virtual and physical registers.

constexpr unsigned id() const

constexpr bool isPhysical() const

Return true if the specified register number is in the physical register namespace.

bool contains(const T &V) const

Check if the SmallSet contains the given element.

std::pair< const_iterator, bool > insert(const T &V)

insert - Insert an element into the set if it isn't already there.

void push_back(const T &Elt)

MI-level Statepoint operands.

TargetInstrInfo - Interface to description of machine instruction set.

TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...

A Use represents the edge between a Value definition and its users.

self_iterator getIterator()

constexpr char Align[]

Key for Kernel::Arg::Metadata::mAlign.

constexpr std::underlying_type_t< E > Mask()

Get a bitmask with 1s in all places up to the high-order bit of E's largest value.

@ Define

initializer< Ty > init(const Ty &Val)

NodeAddr< DefNode * > Def

This is an optimization pass for GlobalISel generic memory operations.

LLVM_ABI char & FixupStatepointCallerSavedID

The pass fixups statepoint machine instruction to replace usage of caller saved registers with stack ...

Definition FixupStatepointCallerSaved.cpp:89

AnalysisManager< MachineFunction > MachineFunctionAnalysisManager

LLVM_ABI PreservedAnalyses getMachineFunctionPassPreservedAnalyses()

Returns the minimum set of Analyses that all machine function passes must preserve.

void sort(IteratorTy Start, IteratorTy End)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

auto make_second_range(ContainerTy &&c)

Given a container of pairs, return a range over the second elements.

auto count_if(R &&Range, UnaryPredicate P)

Wrapper function around std::count_if to count the number of times an element satisfying a given pred...

auto find_if(R &&Range, UnaryPredicate P)

Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.

LLVM_ABI void initializeFixupStatepointCallerSavedLegacyPass(PassRegistry &)

bool is_contained(R &&Range, const E &Element)

Returns true if Element is found in Range.

LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)

Prints virtual and physical registers with or without a TRI instance.

LLVM_ABI Printable printMBBReference(const MachineBasicBlock &MBB)

Prints a machine basic block reference.

static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)

Return a MachinePointerInfo record that refers to the specified FrameIndex.