AArch64StackTagging.cpp Source File (original) (raw)

37#include "llvm/IR/IntrinsicsAArch64.h"

47#include

48#include

50using namespace llvm;

52#define DEBUG_TYPE "aarch64-stack-tagging"

56 cl::desc("merge stack variable initializers with tagging when possible"));

61 cl::desc("Use Stack Safety analysis results"));

73 cl::desc("How many lifetime ends to handle for a single alloca."),

88 "stack-tagging-record-stack-history",

89 cl::desc("Record stack frames with tagged allocations in a thread-local "

90 "ring buffer"),

92 clEnumVal(instr, "Insert instructions into the prologue for "

93 "storing into the stack ring buffer")),

98namespace {

100class InitializerBuilder {

107

108

112 };

114

115

116 std::map<uint64_t, Value *> Out;

117

118public:

119 InitializerBuilder(uint64_t Size, const DataLayout *DL, Value *BasePtr,

120 Function *SetTagFn, Function *SetTagZeroFn,

121 Function *StgpFn)

122 : Size(Size), DL(DL), BasePtr(BasePtr), SetTagFn(SetTagFn),

123 SetTagZeroFn(SetTagZeroFn), StgpFn(StgpFn) {}

124

125 bool addRange(uint64_t Start, uint64_t End, Instruction *Inst) {

126 auto I =

128 return LHS.End <= RHS;

129 });

130 if (I != Ranges.end() && End > I->Start) {

131

132 return false;

133 }

134 Ranges.insert(I, {Start, End, Inst});

135 return true;

136 }

137

138 bool addStore(uint64_t Offset, StoreInst *SI, const DataLayout *DL) {

139 int64_t StoreSize = DL->getTypeStoreSize(SI->getOperand(0)->getType());

141 return false;

143 applyStore(IRB, Offset, Offset + StoreSize, SI->getOperand(0));

144 return true;

145 }

146

147 bool addMemSet(uint64_t Offset, MemSetInst *MSI) {

150 return false;

154 return true;

155 }

156

157 void applyMemSet(IRBuilder<> &IRB, int64_t Start, int64_t End,

158 ConstantInt *V) {

159

160

161

162 if (V->isZero())

163 return;

164 for (int64_t Offset = Start - Start % 8; Offset < End; Offset += 8) {

165 uint64_t Cst = 0x0101010101010101UL;

167 if (LowBits)

168 Cst = (Cst >> LowBits) << LowBits;

169 int HighBits = End - Offset < 8 ? (8 - (End - Offset)) * 8 : 0;

170 if (HighBits)

171 Cst = (Cst << HighBits) >> HighBits;

172 ConstantInt *C =

173 ConstantInt::get(IRB.getInt64Ty(), Cst * V->getZExtValue());

174

176 if (!CurrentV) {

177 CurrentV = C;

178 } else {

179 CurrentV = IRB.CreateOr(CurrentV, C);

180 }

181 }

182 }

183

184

185

190 } else if (Offset < 0) {

193 } else {

195 }

196 return V;

197 }

198

199 void applyStore(IRBuilder<> &IRB, int64_t Start, int64_t End,

200 Value *StoredValue) {

201 StoredValue = flatten(IRB, StoredValue);

202 for (int64_t Offset = Start - Start % 8; Offset < End; Offset += 8) {

203 Value *V = sliceValue(IRB, StoredValue, Offset - Start);

205 if (!CurrentV) {

206 CurrentV = V;

207 } else {

208 CurrentV = IRB.CreateOr(CurrentV, V);

209 }

210 }

211 }

212

215

216 if (Ranges.empty()) {

217 emitUndef(IRB, 0, Size);

218 return;

219 }

220

221

222

223

224 uint64_t LastOffset = 0;

227 auto I2 = Out.find(Offset + 8);

228 if (I1 == Out.end() && I2 == Out.end())

229 continue;

230

231 if (Offset > LastOffset)

232 emitZeroes(IRB, LastOffset, Offset - LastOffset);

233

235 : I1->second;

237 : I2->second;

238 emitPair(IRB, Offset, Store1, Store2);

239 LastOffset = Offset + 16;

240 }

241

242

243

244 if (LastOffset < Size)

245 emitZeroes(IRB, LastOffset, Size - LastOffset);

246

247 for (const auto &R : Ranges) {

248 R.Inst->eraseFromParent();

249 }

250 }

251

252 void emitZeroes(IRBuilder<> &IRB, uint64_t Offset, uint64_t Size) {

254 << ") zero\n");

255 Value *Ptr = BasePtr;

259 {Ptr, ConstantInt::get(IRB.getInt64Ty(), Size)});

260 }

261

262 void emitUndef(IRBuilder<> &IRB, uint64_t Offset, uint64_t Size) {

264 << ") undef\n");

265 Value *Ptr = BasePtr;

269 }

270

274 Value *Ptr = BasePtr;

278 }

279

281 if (V->getType()->isIntegerTy())

282 return V;

283

286 Type *EltTy = VecTy->getElementType();

288 uint32_t EltSize = DL->getTypeSizeInBits(EltTy);

293 }

294 }

296 V, IRB.getIntNTy(DL->getTypeStoreSize(V->getType()) * 8));

297 }

298};

299

300class AArch64StackTagging : public FunctionPass {

301 const bool MergeInit;

302 const bool UseStackSafety;

303

304public:

305 static char ID;

306

307 AArch64StackTagging(bool IsOptNone = false)

308 : FunctionPass(ID),

311 : !IsOptNone) {}

312

313 void tagAlloca(AllocaInst *AI, Instruction *InsertBefore, Value *Ptr,

314 uint64_t Size);

315 void untagAlloca(AllocaInst *AI, Instruction *InsertBefore, uint64_t Size);

316

317 Instruction *collectInitializers(Instruction *StartInst, Value *StartPtr,

318 uint64_t Size, InitializerBuilder &IB);

319

322 const MapVector<AllocaInst *, memtag::AllocaInfo> &Allocas,

323 const DominatorTree *DT);

325

326 StringRef getPassName() const override { return "AArch64 Stack Tagging"; }

327

328private:

330 Function *SetTagFunc = nullptr;

331 const DataLayout *DL = nullptr;

332 AAResults *AA = nullptr;

333 const StackSafetyGlobalInfo *SSI = nullptr;

334

335 void getAnalysisUsage(AnalysisUsage &AU) const override {

337 if (UseStackSafety)

338 AU.addRequired();

339 if (MergeInit)

341 AU.addRequired();

342 }

343};

344

345}

346

347char AArch64StackTagging::ID = 0;

348

350 false, false)

356

358 return new AArch64StackTagging(IsOptNone);

359}

360

364 InitializerBuilder &IB) {

368

369 unsigned Count = 0;

372

374 continue;

375

377

378

379

380 if (BI->mayWriteToMemory() || BI->mayReadFromMemory())

381 break;

382 continue;

383 }

384

386 if (!NextStore->isSimple())

387 break;

388

389

390 std::optional<int64_t> Offset =

391 NextStore->getPointerOperand()->getPointerOffsetFrom(StartPtr, *DL);

393 break;

394

395 if (.addStore(*Offset, NextStore, DL))

396 break;

397 LastInst = NextStore;

398 } else {

400

402 break;

403

405 break;

406

407

408 std::optional<int64_t> Offset =

411 break;

412

413 if (.addMemSet(*Offset, MSI))

414 break;

415 LastInst = MSI;

416 }

417 }

418 return LastInst;

419}

420

421void AArch64StackTagging::tagAlloca(AllocaInst *AI, Instruction *InsertBefore,

424 F->getParent(), Intrinsic::aarch64_settag_zero);

426 Intrinsic::aarch64_stgp);

427

428 InitializerBuilder IB(Size, DL, Ptr, SetTagFunc, SetTagZeroFunc, StgpFunc);

430

431 if (MergeInit && ->hasOptNone() && LittleEndian &&

433 LLVM_DEBUG(dbgs() << "collecting initializers for " << *AI

434 << ", size = " << Size << "\n");

435 InsertBefore = collectInitializers(InsertBefore, Ptr, Size, IB);

436 }

437

439 IB.generate(IRB);

440}

441

442void AArch64StackTagging::untagAlloca(AllocaInst *AI, Instruction *InsertBefore,

443 uint64_t Size) {

447}

448

449Instruction *AArch64StackTagging::insertBaseTaggedPointer(

451 const MapVector<AllocaInst *, memtag::AllocaInfo> &AllocasToInstrument,

452 const DominatorTree *DT) {

454

455 for (auto &I : AllocasToInstrument) {

456 const memtag::AllocaInfo &Info = I.second;

457 AllocaInst *AI = Info.AI;

458 if (!PrologueBB) {

460 continue;

461 }

463 }

465

471 const Triple &TargetTriple = M.getTargetTriple();

472

473

474

477 !AllocasToInstrument.empty()) {

478 constexpr int StackMteSlot = -3;

479 constexpr uint64_t TagMask = 0xFULL << 56;

480

481 auto *IntptrTy = IRB.getIntPtrTy(M.getDataLayout());

483 auto *ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);

491

493 }

495}

496

497

498bool AArch64StackTagging::runOnFunction(Function &Fn) {

500 return false;

501

502 if (UseStackSafety)

503 SSI = &getAnalysis().getResult();

504 F = &Fn;

506 if (MergeInit)

507 AA = &getAnalysis().getAAResults();

508 OptimizationRemarkEmitter &ORE =

509 getAnalysis().getORE();

510

511 memtag::StackInfoBuilder SIB(SSI, DEBUG_TYPE);

513 SIB.visit(ORE, I);

514 memtag::StackInfo &SInfo = SIB.get();

515

517 return false;

518

519 std::unique_ptr DeleteDT;

520 DominatorTree *DT = nullptr;

521 if (auto *P = getAnalysisIfAvailable())

522 DT = &P->getDomTree();

523

524 if (DT == nullptr) {

525 DeleteDT = std::make_unique(*F);

526 DT = DeleteDT.get();

527 }

528

529 std::unique_ptr DeletePDT;

530 PostDominatorTree *PDT = nullptr;

531 if (auto *P = getAnalysisIfAvailable())

532 PDT = &P->getPostDomTree();

533

534 if (PDT == nullptr) {

535 DeletePDT = std::make_unique(*F);

536 PDT = DeletePDT.get();

537 }

538

539 std::unique_ptr DeleteLI;

540 LoopInfo *LI = nullptr;

541 if (auto *LIWP = getAnalysisIfAvailable()) {

542 LI = &LIWP->getLoopInfo();

543 } else {

544 DeleteLI = std::make_unique(*DT);

545 LI = DeleteLI.get();

546 }

547

549 Intrinsic::aarch64_settag);

550

553

554 unsigned int NextTag = 0;

556 memtag::AllocaInfo &Info = I.second;

557 assert(Info.AI && SIB.getAllocaInterestingness(*Info.AI) ==

560 AllocaInst *AI = Info.AI;

561 unsigned int Tag = NextTag;

562 NextTag = (NextTag + 1) % 16;

563

569 if (Info.AI->hasName())

570 TagPCall->setName(Info.AI->getName() + ".tag");

571

572 Info.AI->replaceUsesWithIf(TagPCall, [&](const Use &U) {

574 });

576

577

578

579

580

581 bool StandardLifetime =

585 if (StandardLifetime) {

586 IntrinsicInst *Start = Info.LifetimeStart[0];

587 uint64_t Size = *Info.AI->getAllocationSize(*DL);

589 tagAlloca(AI, Start->getNextNode(), TagPCall, Size);

590

592 if (!DT || !PDT ||

594 SInfo.RetVec, TagEnd)) {

595 for (auto *End : Info.LifetimeEnd)

596 End->eraseFromParent();

597 }

598 } else {

599 uint64_t Size = *Info.AI->getAllocationSize(*DL);

602 for (auto *RI : SInfo.RetVec) {

603 untagAlloca(AI, RI, Size);

604 }

605

606

607 for (auto *II : Info.LifetimeStart)

608 II->eraseFromParent();

609 for (auto *II : Info.LifetimeEnd)

610 II->eraseFromParent();

611 }

612

614 }

615

616 return true;

617}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static cl::opt< bool > ClMergeInit("stack-tagging-merge-init", cl::Hidden, cl::init(true), cl::desc("merge stack variable initializers with tagging when possible"))

StackTaggingRecordStackHistoryMode

Definition AArch64StackTagging.cpp:78

@ none

Definition AArch64StackTagging.cpp:80

@ instr

Definition AArch64StackTagging.cpp:84

static cl::opt< unsigned > ClMergeInitSizeLimit("stack-tagging-merge-init-size-limit", cl::init(272), cl::Hidden)

static cl::opt< unsigned > ClScanLimit("stack-tagging-merge-init-scan-limit", cl::init(40), cl::Hidden)

static cl::opt< bool > ClUseStackSafety("stack-tagging-use-stack-safety", cl::Hidden, cl::init(true), cl::desc("Use Stack Safety analysis results"))

static cl::opt< size_t > ClMaxLifetimes("stack-tagging-max-lifetimes-for-alloca", cl::Hidden, cl::init(3), cl::ReallyHidden, cl::desc("How many lifetime ends to handle for a single alloca."), cl::Optional)

static const Align kTagGranuleSize

Definition AArch64StackTagging.cpp:96

static cl::opt< StackTaggingRecordStackHistoryMode > ClRecordStackHistory("stack-tagging-record-stack-history", cl::desc("Record stack frames with tagged allocations in a thread-local " "ring buffer"), cl::values(clEnumVal(none, "Do not record stack ring history"), clEnumVal(instr, "Insert instructions into the prologue for " "storing into the stack ring buffer")), cl::Hidden, cl::init(none))

This file implements a class to represent arbitrary precision integral constant values and operations...

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

Expand Atomic instructions

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

Analysis containing CSE Info

#define clEnumVal(ENUMVAL, DESC)

This file contains constants used for implementing Dwarf debug support.

static bool runOnFunction(Function &F, bool PostInlining)

This header defines various interfaces for pass management in LLVM.

Machine Check Debug Module

This file implements a map that provides insertion order iteration.

ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))

uint64_t IntrinsicInst * II

#define INITIALIZE_PASS_DEPENDENCY(depName)

#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)

#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)

static unsigned getNumElements(Type *Ty)

This file defines the SmallVector class.

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.

ModRefInfo getModRefInfo(const Instruction *I, const std::optional< MemoryLocation > &OptLoc)

Check whether or not an instruction may read or write the optionally specified memory location.

AnalysisUsage & addRequired()

LLVM_ABI void setPreservesCFG()

This function should be called by the pass, iff they do not:

const Instruction & front() const

InstListType::iterator iterator

Instruction iterators...

static LLVM_ABI Constant * getNullValue(Type *Ty)

Constructor to create a '0' constant of arbitrary type.

A parsed version of the target data layout string in and methods for querying it.

LLVM_ABI Instruction * findNearestCommonDominator(Instruction *I1, Instruction *I2) const

Find the nearest instruction I that dominates both I1 and I2, in the sense that a result produced bef...

static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)

FunctionPass class - This class is used to implement most global optimizations.

const DataLayout & getDataLayout() const

Get the data layout of the module this function belongs to.

bool hasFnAttribute(Attribute::AttrKind Kind) const

Return true if the function has the attribute.

Module * getParent()

Get the module that this global value is contained inside of...

Value * CreateConstGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0, const Twine &Name="")

Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")

IntegerType * getIntNTy(unsigned N)

Fetch the type representing an N-bit integer.

Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")

Create a ZExt or Trunc from the integer value V to DestTy.

Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")

BasicBlock::iterator GetInsertPoint() const

Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")

Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)

IntegerType * getIntPtrTy(const DataLayout &DL, unsigned AddrSpace=0)

Fetch the type of an integer with size at least as big as that of a pointer in the given address spac...

IntegerType * getInt64Ty()

Fetch the type representing a 64-bit integer.

LLVM_ABI CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")

Create a call to intrinsic ID with Args, mangled using Types.

Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")

LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)

Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...

Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

LLVMContext & getContext() const

Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")

StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)

Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")

CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)

PointerType * getPtrTy(unsigned AddrSpace=0)

Fetch the type representing a pointer.

Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="", bool IsDisjoint=false)

IntegerType * getInt8Ty()

Fetch the type representing an 8-bit integer.

LLVM_ABI const Module * getModule() const

Return the module owning the function this instruction belongs to or nullptr it the function does not...

static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)

This static method is the primary way of constructing an IntegerType.

Value * getLength() const

Value * getDest() const

This is just like getRawDest, but it strips off any cast instructions (including addrspacecast) that ...

Representation for a specific memory location.

const Triple & getTargetTriple() const

Get the target triple which is a string describing the target host.

This pass performs the global (interprocedural) stack safety analysis (legacy pass manager).

bool isAndroidVersionLT(unsigned Major) const

bool isAndroid() const

Tests whether the target is Android.

LLVM_ABI bool isLittleEndian() const

Tests whether the target triple is little endian.

bool isAArch64() const

Tests whether the target is AArch64 (little and big endian).

bool isPointerTy() const

True if this is an instance of PointerType.

void setOperand(unsigned i, Value *Val)

LLVM Value Representation.

LLVM_ABI void setName(const Twine &Name)

Change the name of the value.

LLVM_ABI std::optional< int64_t > getPointerOffsetFrom(const Value *Other, const DataLayout &DL) const

If this ptr is provably equal to Other plus a constant offset, return that offset in bytes.

const ParentTy * getParent() const

@ C

The default llvm calling convention, compatible with C.

@ BasicBlock

Various leaf nodes.

LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})

Look up the Function declaration of the intrinsic id in the Module M.

ValuesClass values(OptsTy... Options)

Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...

initializer< Ty > init(const Ty &Val)

Value * getFP(IRBuilder<> &IRB)

bool isStandardLifetime(const SmallVectorImpl< IntrinsicInst * > &LifetimeStart, const SmallVectorImpl< IntrinsicInst * > &LifetimeEnd, const DominatorTree *DT, const LoopInfo *LI, size_t MaxLifetimes)

bool forAllReachableExits(const DominatorTree &DT, const PostDominatorTree &PDT, const LoopInfo &LI, const Instruction *Start, const SmallVectorImpl< IntrinsicInst * > &Ends, const SmallVectorImpl< Instruction * > &RetVec, llvm::function_ref< void(Instruction *)> Callback)

Value * getAndroidSlotPtr(IRBuilder<> &IRB, int Slot)

Value * incrementThreadLong(IRBuilder<> &IRB, Value *ThreadLong, unsigned int Inc)

void annotateDebugRecords(AllocaInfo &Info, unsigned int Tag)

void alignAndPadAlloca(memtag::AllocaInfo &Info, llvm::Align Align)

Value * getPC(const Triple &TargetTriple, IRBuilder<> &IRB)

NodeAddr< NodeBase * > Node

friend class Instruction

Iterator for Instructions in a `BasicBlock.

This is an optimization pass for GlobalISel generic memory operations.

FunctionAddr VTableAddr Value

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

FunctionAddr VTableAddr Count

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

IRBuilder(LLVMContext &, FolderTy, InserterTy, MDNode *, ArrayRef< OperandBundleDef >) -> IRBuilder< FolderTy, InserterTy >

auto lower_bound(R &&Range, T &&Value)

Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...

uint64_t alignTo(uint64_t Size, Align A)

Returns a multiple of A needed to store Size bytes.

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

FunctionPass * createAArch64StackTaggingPass(bool IsOptNone)

Definition AArch64StackTagging.cpp:357

bool isNoModRef(const ModRefInfo MRI)

This struct is a compact representation of a valid (non-zero power of two) alignment.

MapVector< AllocaInst *, AllocaInfo > AllocasToInstrument

SmallVector< Instruction *, 8 > RetVec