AArch64StackTaggingPreRA.cpp Source File (original) (raw)

28using namespace llvm;

30#define DEBUG_TYPE "aarch64-stack-tagging-pre-ra"

37 "Unconditionally apply unchecked-ld-st optimization (even for large "

38 "stack frames, or in the presence of variable sized allocas)."),

43 "apply unchecked-ld-st when the target is definitely within range"),

48 cl::desc("Apply first slot optimization for stack tagging "

49 "(eliminate ADDG Rt, Rn, 0, 0)."));

51namespace {

63public:

64 static char ID;

67 bool mayUseUncheckedLoadStore();

68 void uncheckUsesOf(unsigned TaggedReg, int FI);

69 void uncheckLoadsAndStores();

70 std::optional findFirstSlotCandidate();

72 bool runOnMachineFunction(MachineFunction &Func) override;

73 StringRef getPassName() const override {

74 return "AArch64 Stack Tagging PreRA";

75 }

77 void getAnalysisUsage(AnalysisUsage &AU) const override {

80 }

81};

82}

84char AArch64StackTaggingPreRA::ID = 0;

87 "AArch64 Stack Tagging PreRA Pass", false, false)

89 "AArch64 Stack Tagging PreRA Pass", false, false)

92 return new AArch64StackTaggingPreRA();

93}

96 switch (Opcode) {

97 case AArch64::LDRBBui:

98 case AArch64::LDRHHui:

99 case AArch64::LDRWui:

100 case AArch64::LDRXui:

101

102 case AArch64::LDRBui:

103 case AArch64::LDRHui:

104 case AArch64::LDRSui:

105 case AArch64::LDRDui:

106 case AArch64::LDRQui:

107

108 case AArch64::LDRSHWui:

109 case AArch64::LDRSHXui:

110

111 case AArch64::LDRSBWui:

112 case AArch64::LDRSBXui:

113

114 case AArch64::LDRSWui:

115

116 case AArch64::STRBBui:

117 case AArch64::STRHHui:

118 case AArch64::STRWui:

119 case AArch64::STRXui:

120

121 case AArch64::STRBui:

122 case AArch64::STRHui:

123 case AArch64::STRSui:

124 case AArch64::STRDui:

125 case AArch64::STRQui:

126

127 case AArch64::LDPWi:

128 case AArch64::LDPXi:

129 case AArch64::LDPSi:

130 case AArch64::LDPDi:

131 case AArch64::LDPQi:

132

133 case AArch64::LDPSWi:

134

135 case AArch64::STPWi:

136 case AArch64::STPXi:

137 case AArch64::STPSi:

138 case AArch64::STPDi:

139 case AArch64::STPQi:

140 return true;

141 default:

142 return false;

143 }

144}

145

146bool AArch64StackTaggingPreRA::mayUseUncheckedLoadStore() {

148 return false;

150 return true;

151

152

153

154

155

156

157

158

159

160

161

162

163 unsigned FrameSize = 0;

166 bool EntireFrameReachableFromSP = FrameSize < 0xf00;

168}

169

170void AArch64StackTaggingPreRA::uncheckUsesOf(unsigned TaggedReg, int FI) {

171 for (MachineInstr &UseI :

174

175 unsigned OpIdx = TII->getLoadStoreImmIdx(UseI.getOpcode()) - 1;

176 if (UseI.getOperand(OpIdx).isReg() &&

177 UseI.getOperand(OpIdx).getReg() == TaggedReg) {

178 UseI.getOperand(OpIdx).ChangeToFrameIndex(FI);

180 }

181 } else if (UseI.isCopy() && UseI.getOperand(0).getReg().isVirtual()) {

182 uncheckUsesOf(UseI.getOperand(0).getReg(), FI);

183 }

184 }

185}

186

187void AArch64StackTaggingPreRA::uncheckLoadsAndStores() {

188 for (auto *I : ReTags) {

189 Register TaggedReg = I->getOperand(0).getReg();

190 int FI = I->getOperand(1).getIndex();

191 uncheckUsesOf(TaggedReg, FI);

192 }

193}

194

195namespace {

196struct SlotWithTag {

197 int FI;

199 SlotWithTag(int FI, int Tag) : FI(FI), Tag(Tag) {}

200 explicit SlotWithTag(const MachineInstr &MI)

201 : FI(MI.getOperand(1).getIndex()), Tag(MI.getOperand(4).getImm()) {}

204 }

205};

206}

207

208namespace llvm {

210 static inline SlotWithTag getEmptyKey() { return {-2, -2}; }

216 static bool isEqual(const SlotWithTag &A, const SlotWithTag &B) {

217 return A == B;

218 }

219};

220}

221

226

227

228

229

230

231

232

233std::optional AArch64StackTaggingPreRA::findFirstSlotCandidate() {

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248 LLVM_DEBUG(dbgs() << "AArch64StackTaggingPreRA::findFirstSlotCandidate\n");

250 return std::nullopt;

251

252 DenseMap<SlotWithTag, int> RetagScore;

253 SlotWithTag MaxScoreST{-1, -1};

254 int MaxScore = -1;

255 for (auto *I : ReTags) {

256 SlotWithTag ST{*I};

258 continue;

259

260 Register RetagReg = I->getOperand(0).getReg();

262 continue;

263

264 int Score = 0;

267

268 while (!WorkList.empty()) {

270 for (auto &UseI : MRI->use_instructions(UseReg)) {

271 unsigned Opcode = UseI.getOpcode();

272 if (Opcode == AArch64::STGi || Opcode == AArch64::ST2Gi ||

273 Opcode == AArch64::STZGi || Opcode == AArch64::STZ2Gi ||

274 Opcode == AArch64::STGPi || Opcode == AArch64::STGloop ||

275 Opcode == AArch64::STZGloop || Opcode == AArch64::STGloop_wback ||

276 Opcode == AArch64::STZGloop_wback)

277 continue;

278 if (UseI.isCopy()) {

279 Register DstReg = UseI.getOperand(0).getReg();

282 continue;

283 }

286 Score++;

287 }

288 }

289

290 int TotalScore = RetagScore[ST] += Score;

291 if (TotalScore > MaxScore ||

292 (TotalScore == MaxScore && ST.FI > MaxScoreST.FI)) {

293 MaxScore = TotalScore;

294 MaxScoreST = ST;

295 }

296 }

297

298 if (MaxScoreST.FI < 0)

299 return std::nullopt;

300

301

302 if (MaxScoreST.Tag == 0)

303 return MaxScoreST.FI;

304

305

306 SlotWithTag SwapST{-1, -1};

307 for (auto *I : ReTags) {

308 SlotWithTag ST{*I};

309 if (ST.Tag == 0) {

310 SwapST = ST;

311 break;

312 }

313 }

314

315

316

317

318 for (auto *&I : ReTags) {

319 SlotWithTag ST{*I};

320 MachineOperand &TagOp = I->getOperand(4);

321 if (ST == MaxScoreST) {

323 } else if (ST == SwapST) {

324 TagOp.setImm(MaxScoreST.Tag);

325 }

326 }

327 return MaxScoreST.FI;

328}

329

330bool AArch64StackTaggingPreRA::runOnMachineFunction(MachineFunction &Func) {

333 AFI = MF->getInfo();

334 TII = static_cast<const AArch64InstrInfo *>(MF->getSubtarget().getInstrInfo());

335 TRI = static_cast<const AArch64RegisterInfo *>(

338 ReTags.clear();

339

341

342 LLVM_DEBUG(dbgs() << "********** AArch64 Stack Tagging PreRA **********\n"

343 << "********** Function: " << MF->getName() << '\n');

344

345 SmallSetVector<int, 8> TaggedSlots;

346 for (auto &BB : *MF) {

347 for (auto &I : BB) {

348 if (I.getOpcode() == AArch64::TAGPstack) {

349 ReTags.push_back(&I);

350 int FI = I.getOperand(1).getIndex();

351 TaggedSlots.insert(FI);

352

353 assert(I.getOperand(2).getImm() == 0);

354 }

355 }

356 }

357

358

359

360 for (int FI : TaggedSlots)

362

363 if (ReTags.empty())

364 return false;

365

366 if (mayUseUncheckedLoadStore())

367 uncheckLoadsAndStores();

368

369

370

371

372 std::optional BaseSlot = findFirstSlotCandidate();

373 if (BaseSlot)

375

376 for (auto *I : ReTags) {

377 int FI = I->getOperand(1).getIndex();

378 int Tag = I->getOperand(4).getImm();

380 if (Tag == 0 && FI == BaseSlot) {

382 I->getOperand(0).getReg())

385 }

386 }

387

388 return true;

389}

unsigned const MachineRegisterInfo * MRI

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

const TargetInstrInfo & TII

static bool isSlotPreAllocated(MachineFrameInfo *MFI, int FI)

Definition AArch64StackTaggingPreRA.cpp:222

static cl::opt< UncheckedLdStMode > ClUncheckedLdSt("stack-tagging-unchecked-ld-st", cl::Hidden, cl::init(UncheckedSafe), cl::desc("Unconditionally apply unchecked-ld-st optimization (even for large " "stack frames, or in the presence of variable sized allocas)."), cl::values(clEnumValN(UncheckedNever, "never", "never apply unchecked-ld-st"), clEnumValN(UncheckedSafe, "safe", "apply unchecked-ld-st when the target is definitely within range"), clEnumValN(UncheckedAlways, "always", "always apply unchecked-ld-st")))

UncheckedLdStMode

Definition AArch64StackTaggingPreRA.cpp:32

@ UncheckedAlways

Definition AArch64StackTaggingPreRA.cpp:32

@ UncheckedSafe

Definition AArch64StackTaggingPreRA.cpp:32

@ UncheckedNever

Definition AArch64StackTaggingPreRA.cpp:32

static cl::opt< bool > ClFirstSlot("stack-tagging-first-slot-opt", cl::Hidden, cl::init(true), cl::desc("Apply first slot optimization for stack tagging " "(eliminate ADDG Rt, Rn, 0, 0)."))

static bool isUncheckedLoadOrStoreOpcode(unsigned Opcode)

Definition AArch64StackTaggingPreRA.cpp:95

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

#define clEnumValN(ENUMVAL, FLAGNAME, DESC)

static Register UseReg(const MachineOperand &MO)

Promote Memory to Register

MachineInstr unsigned OpIdx

#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)

#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)

This file implements a set that has insertion order iteration characteristics.

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...

void setTaggedBasePointerIndex(int Index)

LLVM_ABI void setPreservesCFG()

This function should be called by the pass, iff they do not:

FunctionPass class - This class is used to implement most global optimizations.

const MCInstrDesc & get(unsigned Opcode) const

Return the machine instruction descriptor that corresponds to the specified instruction opcode.

The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.

bool hasVarSizedObjects() const

This method may be called any time after instruction selection is complete to determine if the stack ...

bool isObjectPreAllocated(int ObjectIdx) const

Return true if the object was pre-allocated into the local block.

@ SSPLK_None

Did not trigger a stack protector.

void setObjectSSPLayout(int ObjectIdx, SSPLayoutKind Kind)

bool getUseLocalStackAllocationBlock() const

Get whether the local allocation blob should be allocated together or let PEI allocate the locals in ...

int64_t getObjectSize(int ObjectIdx) const

Return the size of the specified object.

int getObjectIndexEnd() const

Return one past the maximum frame object index.

MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...

void getAnalysisUsage(AnalysisUsage &AU) const override

getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.

const TargetSubtargetInfo & getSubtarget() const

getSubtarget - Return the subtarget for which this machine code is being compiled.

StringRef getName() const

getName - Return the name of the corresponding LLVM function.

MachineFrameInfo & getFrameInfo()

getFrameInfo - Return the frame info object for the current function.

MachineRegisterInfo & getRegInfo()

getRegInfo - Return information about the registers currently in use.

Ty * getInfo()

getInfo - Keep track of various per-function pieces of information for backends that would like to do...

const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const

Add a new virtual register operand.

LLVM_ABI void eraseFromParent()

Unlink 'this' from the containing basic block and delete it.

void setImm(int64_t immVal)

MachineRegisterInfo - Keep track of information for virtual and physical registers,...

constexpr bool isVirtual() const

Return true if the specified register number is in the virtual register namespace.

bool insert(const value_type &X)

Insert a new element into the SetVector.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

@ MO_TAGGED

MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

ValuesClass values(OptsTy... Options)

Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...

initializer< Ty > init(const Ty &Val)

NodeAddr< FuncNode * > Func

This is an optimization pass for GlobalISel generic memory operations.

MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)

Builder interface. Specify how to create the initial instruction itself.

FunctionPass * createAArch64StackTaggingPreRAPass()

Definition AArch64StackTaggingPreRA.cpp:91

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)

MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

hash_code hash_combine(const Ts &...args)

Combine values into a single hash_code.

LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)

Prints virtual and physical registers with or without a TRI instance.

static bool isEqual(const SlotWithTag &A, const SlotWithTag &B)

Definition AArch64StackTaggingPreRA.cpp:216

static SlotWithTag getTombstoneKey()

Definition AArch64StackTaggingPreRA.cpp:211

static SlotWithTag getEmptyKey()

Definition AArch64StackTaggingPreRA.cpp:210

static unsigned getHashValue(const SlotWithTag &V)

Definition AArch64StackTaggingPreRA.cpp:212

An information struct used to provide DenseMap with the various necessary components for a given valu...