FixupStatepointCallerSaved.cpp Source File (original) (raw)

33using namespace llvm;

35#define DEBUG_TYPE "fixup-statepoint-caller-saved"

36 STATISTIC(NumSpilledRegisters, "Number of spilled register");

37 STATISTIC(NumSpillSlotsAllocated, "Number of spill slots allocated");

38 STATISTIC(NumSpillSlotsExtended, "Number of spill slots extended");

42 cl::desc("Allow spill in spill slot of greater size than register size"),

47 cl::desc("Allow passing GC Pointer arguments in callee saved registers"));

51 cl::desc("Enable simple copy propagation during register reloading"));

56 "fixup-max-csr-statepoints", cl::Hidden,

57 cl::desc("Max number of statepoints allowed to pass GC Ptrs in registers"));

59namespace {

62public:

63 static char ID;

67 }

72 }

75 return "Fixup Statepoint Caller Saved";

76 }

79};

81}

83char FixupStatepointCallerSaved::ID = 0;

87 "Fixup Statepoint Caller Saved", false, false)

94 return TRI.getSpillSize(*RC);

95}

100

101

102

103

104

105

106

107

108

109

114

115 int Idx = RI->findRegisterUseOperandIdx(Reg, &TRI, false);

117 IsKill = false;

118 return Reg;

119 }

120

122 return Reg;

123

127 for (auto It = ++(RI.getReverse()); It != E; ++It) {

128 if (It->readsRegister(Reg, &TRI) && Use )

129 Use = &*It;

130 if (It->modifiesRegister(Reg, &TRI)) {

131 Def = &*It;

132 break;

133 }

134 }

135

136 if (!Def)

137 return Reg;

138

139 auto DestSrc = TII.isCopyInstr(*Def);

140 if (!DestSrc || DestSrc->Destination->getReg() != Reg)

141 return Reg;

142

143 Register SrcReg = DestSrc->Source->getReg();

144

146 return Reg;

147

148 LLVM_DEBUG(dbgs() << "spillRegisters: perform copy propagation "

150 << "\n");

151

152

154 IsKill = DestSrc->Source->isKill();

155

156 if ( Use ) {

157

158

159 LLVM_DEBUG(dbgs() << "spillRegisters: removing dead copy " << *Def);

160 Def->eraseFromParent();

161 } else if (IsKill) {

162

163

164 const_cast<MachineOperand *>(DestSrc->Source)->setIsKill(false);

165 }

166

167 return SrcReg;

168}

169

170namespace {

171

172using RegSlotPair = std::pair<Register, int>;

173

174

175class RegReloadCache {

178

179public:

180 RegReloadCache() = default;

181

182

184 RegSlotPair RSP(Reg, FI);

185 auto Res = Reloads[MBB].insert(RSP);

186 (void)Res;

187 assert(Res.second && "reload already exists");

188 }

189

190

192 RegSlotPair RSP(Reg, FI);

193 auto It = Reloads.find(MBB);

194 return It != Reloads.end() && It->second.count(RSP);

195 }

196};

197

198

199

200

201

202

203class FrameIndexesCache {

204private:

205 struct FrameIndexesPerSize {

206

208

209 unsigned Index = 0;

210 };

213

214

215

216

218

219

220

222

223

224

225

227 GlobalIndices;

228

229 FrameIndexesPerSize &getCacheBucket(unsigned Size) {

230

231

233 }

234

235public:

237 : MFI(MFI), TRI(TRI) {}

238

239

240

242 for (auto &It : Cache)

243 It.second.Index = 0;

244

245 ReservedSlots.clear();

246 if (EHPad)

247 if (auto It = GlobalIndices.find(EHPad); It != GlobalIndices.end())

248 for (auto &RSP : It->second)

249 ReservedSlots.insert(RSP.second);

250 }

251

252

254

255 auto It = GlobalIndices.find(EHPad);

256 if (It != GlobalIndices.end()) {

257 auto &Vec = It->second;

259 Vec, [Reg](RegSlotPair &RSP) { return Reg == RSP.first; });

260 if (Idx != Vec.end()) {

261 int FI = Idx->second;

262 LLVM_DEBUG(dbgs() << "Found global FI " << FI << " for register "

263 << printReg(Reg, &TRI) << " at "

265 assert(ReservedSlots.count(FI) && "using unreserved slot");

266 return FI;

267 }

268 }

269

271 FrameIndexesPerSize &Line = getCacheBucket(Size);

272 while (Line.Index < Line.Slots.size()) {

273 int FI = Line.Slots[Line.Index++];

274 if (ReservedSlots.count(FI))

275 continue;

276

277

281 NumSpillSlotsExtended++;

282 }

283 return FI;

284 }

286 NumSpillSlotsAllocated++;

287 Line.Slots.push_back(FI);

288 ++Line.Index;

289

290

291 if (EHPad) {

292 GlobalIndices[EHPad].push_back(std::make_pair(Reg, FI));

293 LLVM_DEBUG(dbgs() << "Reserved FI " << FI << " for spilling reg "

294 << printReg(Reg, &TRI) << " at landing pad "

296 }

297

298 return FI;

299 }

300

301

302

303

306 return;

309 });

310 }

311};

312

313

314class StatepointState {

315private:

316

319

324

326

327 FrameIndexesCache &CacheFI;

328 bool AllowGCPtrInCSR;

329

331

333

335

337

338public:

340 FrameIndexesCache &CacheFI, bool AllowGCPtrInCSR)

341 : MI(MI), MF(*MI.getMF()), TRI(*MF.getSubtarget().getRegisterInfo()),

342 TII(*MF.getSubtarget().getInstrInfo()), MFI(MF.getFrameInfo()),

343 Mask(Mask), CacheFI(CacheFI), AllowGCPtrInCSR(AllowGCPtrInCSR) {

344

345

346 EHPad = nullptr;

348

351 return I.getOpcode() == TargetOpcode::STATEPOINT;

352 });

353

355 return;

356

358

360

363 EHPad = *It;

364 }

365

367

368

369 bool isCalleeSaved(Register Reg) { return (Mask[Reg / 32] >> Reg % 32) & 1; }

370

371

372

373

374 bool findRegistersToSpill() {

376

377

378 for (const auto &Def : MI.defs())

380

383 EndIdx = MI.getNumOperands();

384 Idx < EndIdx; ++Idx) {

387 continue;

389 assert(Reg.isPhysical() && "Only physical regs are expected");

390

391 if (isCalleeSaved(Reg) && (AllowGCPtrInCSR || !GCRegs.contains(Reg)))

392 continue;

393

395 << Idx << "\n");

396

397 if (VisitedRegs.insert(Reg).second)

400 }

401 CacheFI.sortRegisters(RegsToSpill);

402 return !RegsToSpill.empty();

403 }

404

405

406

407 void spillRegisters() {

408 for (Register Reg : RegsToSpill) {

409 int FI = CacheFI.getFrameIndex(Reg, EHPad);

410

411 NumSpilledRegisters++;

412 RegToSlotIdx[Reg] = FI;

413

415 << "\n");

416

417

418 bool IsKill = true;

422

423 LLVM_DEBUG(dbgs() << "Insert spill before " << *InsertBefore);

426 }

427 }

428

432 int FI = RegToSlotIdx[Reg];

433 if (It != MBB->end()) {

435 return;

436 }

437

438

439

441 --It;

445 (void)Dummy;

450 }

451

452

453 void insertReloads(MachineInstr *NewStatepoint, RegReloadCache &RC) {

455 auto InsertPoint = std::next(NewStatepoint->getIterator());

456

457 for (auto Reg : RegsToReload) {

458 insertReloadBefore(Reg, InsertPoint, MBB);

460 << RegToSlotIdx[Reg] << " after statepoint\n");

461

462 if (EHPad && !RC.hasReload(Reg, RegToSlotIdx[Reg], EHPad)) {

463 RC.recordReload(Reg, RegToSlotIdx[Reg], EHPad);

464 auto EHPadInsertPoint =

466 insertReloadBefore(Reg, EHPadInsertPoint, EHPad);

469 }

470 }

471 }

472

473

474

479

480 unsigned NumOps = MI.getNumOperands();

481

482

484 unsigned NumDefs = MI.getNumDefs();

485 for (unsigned I = 0; I < NumDefs; ++I) {

487 assert(DefMO.isReg() && DefMO.isDef() && "Expected Reg Def operand");

489 assert(DefMO.isTied() && "Def is expected to be tied");

490

491

492 if (MI.getOperand(MI.findTiedOperandIdx(I)).isUndef()) {

493 if (AllowGCPtrInCSR) {

496 }

497 continue;

498 }

499 if (!AllowGCPtrInCSR) {

501 RegsToReload.push_back(Reg);

502 } else {

503 if (isCalleeSaved(Reg)) {

506 } else {

508 RegsToReload.push_back(Reg);

509 }

510 }

511 }

512

513

514 OpsToSpill.push_back(MI.getNumOperands());

515 unsigned CurOpIdx = 0;

516

517 for (unsigned I = NumDefs; I < MI.getNumOperands(); ++I) {

519 if (I == OpsToSpill[CurOpIdx]) {

520 int FI = RegToSlotIdx[MO.getReg()];

521 MIB.addImm(StackMaps::IndirectMemRefOp);

523 assert(MO.isReg() && "Should be register");

525 MIB.addFrameIndex(FI);

526 MIB.addImm(0);

527 ++CurOpIdx;

528 } else {

529 MIB.add(MO);

530 unsigned OldDef;

531 if (AllowGCPtrInCSR && MI.isRegTiedToDefOperand(I, &OldDef)) {

532 assert(OldDef < NumDefs);

533 assert(NewIndices[OldDef] < NumOps);

534 MIB->tieOperands(NewIndices[OldDef], MIB->getNumOperands() - 1);

535 }

536 }

537 }

538 assert(CurOpIdx == (OpsToSpill.size() - 1) && "Not all operands processed");

539

541 for (auto It : RegToSlotIdx) {

548 auto *MMO =

552 }

553

554

555 MI.getParent()->insert(MI, NewMI);

556

557 LLVM_DEBUG(dbgs() << "rewritten statepoint to : " << *NewMI << "\n");

558 MI.eraseFromParent();

559 return NewMI;

560 }

561};

562

563class StatepointProcessor {

564private:

567 FrameIndexesCache CacheFI;

568 RegReloadCache ReloadCache;

569

570public:

572 : MF(MF), TRI(*MF.getSubtarget().getRegisterInfo()),

573 CacheFI(MF.getFrameInfo(), TRI) {}

574

575 bool process(MachineInstr &MI, bool AllowGCPtrInCSR) {

578

579 if (Flags & (uint64_t)StatepointFlags::DeoptLiveIn)

580 return false;

581 LLVM_DEBUG(dbgs() << "\nMBB " << MI.getParent()->getNumber() << " "

582 << MI.getParent()->getName() << " : process statepoint "

583 << MI);

586 StatepointState SS(MI, Mask, CacheFI, AllowGCPtrInCSR);

587 CacheFI.reset(SS.getEHPad());

588

589 if (.findRegistersToSpill())

590 return false;

591

592 SS.spillRegisters();

593 auto *NewStatepoint = SS.rewriteStatepoint();

594 SS.insertReloads(NewStatepoint, ReloadCache);

595 return true;

596 }

597};

598}

599

600bool FixupStatepointCallerSaved::runOnMachineFunction(MachineFunction &MF) {

602 return false;

603

605 if (.hasGC())

606 return false;

607

611 if (I.getOpcode() == TargetOpcode::STATEPOINT)

613

614 if (Statepoints.empty())

615 return false;

616

617 bool Changed = false;

618 StatepointProcessor SPP(MF);

619 unsigned NumStatepoints = 0;

622 ++NumStatepoints;

625 AllowGCPtrInCSR = false;

626 Changed |= SPP.process(*I, AllowGCPtrInCSR);

627 }

628 return Changed;

629}

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx

static Register performCopyPropagation(Register Reg, MachineBasicBlock::iterator &RI, bool &IsKill, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI)

static cl::opt< bool > PassGCPtrInCSR("fixup-allow-gcptr-in-csr", cl::Hidden, cl::init(false), cl::desc("Allow passing GC Pointer arguments in callee saved registers"))

static cl::opt< unsigned > MaxStatepointsWithRegs("fixup-max-csr-statepoints", cl::Hidden, cl::desc("Max number of statepoints allowed to pass GC Ptrs in registers"))

Fixup Statepoint Caller static false unsigned getRegisterSize(const TargetRegisterInfo &TRI, Register Reg)

static cl::opt< bool > FixupSCSExtendSlotSize("fixup-scs-extend-slot-size", cl::Hidden, cl::init(false), cl::desc("Allow spill in spill slot of greater size than register size"), cl::Hidden)

Fixup Statepoint Caller Saved

static cl::opt< bool > EnableCopyProp("fixup-scs-enable-copy-propagation", cl::Hidden, cl::init(true), cl::desc("Enable simple copy propagation during register reloading"))

const HexagonInstrInfo * TII

unsigned const TargetRegisterInfo * TRI

PowerPC TLS Dynamic Call Fixup

#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)

#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

This file defines the SmallSet class.

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

Represent the analysis usage information of a pass.

void setPreservesCFG()

This function should be called by the pass, iff they do not:

iterator find(const_arg_type_t< KeyT > Val)

std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)

void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override

Store the specified register of the given register class to the specified stack frame index.

TargetInstrInfo overrides.

void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override

Load the specified register of the given register class from the specified stack frame index.

iterator SkipPHIsLabelsAndDebug(iterator I, Register Reg=Register(), bool SkipPseudoOp=true)

Return the first instruction in MBB after I that is not a PHI, label or debug.

MachineInstr * remove(MachineInstr *I)

Remove the unbundled instruction from the instruction list without deleting it.

iterator_range< succ_iterator > successors()

iterator insertAfter(iterator I, MachineInstr *MI)

Insert MI into the instruction list after I.

MachineInstrBundleIterator< MachineInstr > iterator

The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.

int CreateSpillStackObject(uint64_t Size, Align Alignment)

Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...

void setObjectSize(int ObjectIdx, int64_t Size)

Change the size of the specified stack object.

Align getObjectAlign(int ObjectIdx) const

Return the alignment of the specified stack object.

int64_t getObjectSize(int ObjectIdx) const

Return the size of the specified object.

void setObjectAlignment(int ObjectIdx, Align Alignment)

setObjectAlignment - Change the alignment of the specified stack object.

MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...

void getAnalysisUsage(AnalysisUsage &AU) const override

getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.

virtual bool runOnMachineFunction(MachineFunction &MF)=0

runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...

MachineInstr * CreateMachineInstr(const MCInstrDesc &MCID, DebugLoc DL, bool NoImplicit=false)

CreateMachineInstr - Allocate a new MachineInstr.

MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)

getMachineMemOperand - Allocate a new MachineMemOperand.

Function & getFunction()

Return the LLVM function that this machine code represents.

reverse_iterator getReverse() const

Get a reverse iterator to the same node.

instr_iterator getInstrIterator() const

Representation of each machine instruction.

const MachineBasicBlock * getParent() const

unsigned getNumOperands() const

Retuns the total number of operands.

void setMemRefs(MachineFunction &MF, ArrayRef< MachineMemOperand * > MemRefs)

Assign this MachineInstr's memory reference descriptor list.

void addMemOperand(MachineFunction &MF, MachineMemOperand *MO)

Add a MachineMemOperand to the machine instruction.

Flags

Flags values. These may be or'd together.

@ MOLoad

The memory access reads data.

@ MOStore

The memory access writes data.

MachineOperand class - Representation of each machine instruction operand.

bool isReg() const

isReg - Tests if this is a MO_Register operand.

getReg - Returns the register number.

static PassRegistry * getPassRegistry()

getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...

virtual StringRef getPassName() const

getPassName - Return a nice clean name for a pass.

Wrapper class representing virtual and physical registers.

constexpr bool isPhysical() const

Return true if the specified register number is in the physical register namespace.

SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...

size_type count(const T &V) const

count - Return 1 if the element is in the set, 0 otherwise.

bool contains(const T &V) const

Check if the SmallSet contains the given element.

std::pair< const_iterator, bool > insert(const T &V)

insert - Insert an element into the set if it isn't already there.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

MI-level Statepoint operands.

StringRef - Represent a constant reference to a string, i.e.

TargetInstrInfo - Interface to description of machine instruction set.

TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...

A Use represents the edge between a Value definition and its users.

self_iterator getIterator()

constexpr std::underlying_type_t< E > Mask()

Get a bitmask with 1s in all places up to the high-order bit of E's largest value.

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

@ Define

Reg

All possible values of the reg field in the ModR/M byte.

initializer< Ty > init(const Ty &Val)

NodeAddr< DefNode * > Def

This is an optimization pass for GlobalISel generic memory operations.

char & FixupStatepointCallerSavedID

The pass fixups statepoint machine instruction to replace usage of caller saved registers with stack ...

void initializeFixupStatepointCallerSavedPass(PassRegistry &)

void sort(IteratorTy Start, IteratorTy End)

raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

auto count_if(R &&Range, UnaryPredicate P)

Wrapper function around std::count_if to count the number of times an element satisfying a given pred...

auto find_if(R &&Range, UnaryPredicate P)

Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.

bool is_contained(R &&Range, const E &Element)

Returns true if Element is found in Range.

Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)

Prints virtual and physical registers with or without a TRI instance.

Printable printMBBReference(const MachineBasicBlock &MBB)

Prints a machine basic block reference.

This struct is a compact representation of a valid (non-zero power of two) alignment.

static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)

Return a MachinePointerInfo record that refers to the specified FrameIndex.