MemoryProfileInfo.cpp Source File (original) (raw)

20using namespace llvm;

23#define DEBUG_TYPE "memory-profile-info"

25namespace llvm {

29 cl::desc("Report total allocation sizes of hinted allocations"));

36 cl::desc("Keep all non-cold contexts (increases cloning overheads)"));

40 cl::desc("Min percent of cold bytes to hint alloc cold during cloning"));

48 cl::desc("Min percent of cold bytes at a callsite to discard non-cold "

49 "contexts"));

55 cl::desc("Min percent of max cold bytes for critical cold context"));

59 cl::desc("Apply ambiguous memprof attribute to ambiguous allocations"));

61}

81 auto *StackValMD =

84 }

86}

101 if (MDS->getString() == "cold") {

103 } else if (MDS->getString() == "hot") {

105 }

107}

108

110 switch (Type) {

112 return "notcold";

113 break;

115 return "cold";

116 break;

118 return "hot";

119 break;

120 default:

121 assert(false && "Unexpected alloc type");

122 }

124}

125

127 const unsigned NumAllocTypes = llvm::popcount(AllocTypes);

128 assert(NumAllocTypes != 0);

129 return NumAllocTypes == 1;

130}

131

138

141 return;

142

143

146 } else {

149 }

150}

151

154 std::vector ContextSizeInfo) {

155 bool First = true;

156 CallStackTrieNode *Curr = nullptr;

157 for (auto StackId : StackIds) {

158

161 if (Alloc) {

162 assert(AllocStackId == StackId);

163 Alloc->addAllocType(AllocType);

164 } else {

165 AllocStackId = StackId;

166 Alloc = new CallStackTrieNode(AllocType);

167 }

168 Curr = Alloc;

169 continue;

170 }

171

172 auto [Next, Inserted] = Curr->Callers.try_emplace(StackId);

173 if (!Inserted) {

174 Curr = Next->second;

175 Curr->addAllocType(AllocType);

176 continue;

177 }

178

179 auto *New = new CallStackTrieNode(AllocType);

180 Next->second = New;

181 Curr = New;

182 }

185}

186

188

189 BuiltFromExistingMetadata = true;

194 for (const auto &MIBStackIter : StackMD->operands()) {

197 CallStack.push_back(StackId->getZExtValue());

198 }

199 std::vector ContextSizeInfo;

200

207 ->getZExtValue();

210 ->getZExtValue();

211 ContextSizeInfo.push_back({FullStackId, TotalSize});

212 }

213 }

215}

216

221 bool BuiltFromExistingMetadata,

227

228 if (ContextSizeInfo.empty()) {

229

230

231

232

235 }

236

237 for (const auto &[FullStackId, TotalSize] : ContextSizeInfo) {

238 TotalBytes += TotalSize;

239 bool LargeColdContext = false;

241 ColdBytes += TotalSize;

242

243

244

245

246

247

248 if (BuiltFromExistingMetadata ||

251 LargeColdContext = true;

252 }

253

254

255

256

262 auto *ContextSizeMD = MDNode::get(Ctx, {FullStackIdMD, TotalSizeMD});

263 MIBPayload.push_back(ContextSizeMD);

264 }

265 }

266 assert(TotalBytes > 0);

268}

269

270void CallStackTrie::collectContextSizeInfo(

271 CallStackTrieNode *Node, std::vector &ContextSizeInfo) {

273 for (auto &Caller : Node->Callers)

274 collectContextSizeInfo(Caller.second, ContextSizeInfo);

275}

276

277void CallStackTrie::convertHotToNotCold(CallStackTrieNode *Node) {

281 }

282 for (auto &Caller : Node->Callers)

283 convertHotToNotCold(Caller.second);

284}

285

286

287

289 std::vector<Metadata *> &SavedMIBNodes,

290 unsigned CallerContextLength,

292 bool BuiltFromExistingMetadata) {

293 const bool MostlyCold =

294

295

296

297

298

299

300

301

303 ColdBytes > 0 &&

305

306

309 return;

310 }

311

312 auto EmitMessageForRemovedContexts = [](const MDNode *MIBMD, StringRef Tag,

320 ->getZExtValue();

323 ->getZExtValue();

324 errs() << "MemProf hinting: Total size for " << Tag

325 << " non-cold full allocation context hash " << FullStackId

326 << Extra << ": " << TS << "\n";

327 }

328 };

329

330

331

332

333 if (MostlyCold) {

334 auto NewColdMIBNodes =

337

339 return true;

341 const float PercentCold = ColdBytes * 100.0 / TotalBytes;

342 std::string PercentStr;

344 OS << format(" for %5.2f%% cold bytes", PercentCold);

345 EmitMessageForRemovedContexts(MIBMD, "discarded", OS.str());

346 }

347 return false;

348 });

349 for (auto *M : NewColdMIBNodes)

350 SavedMIBNodes.push_back(M);

351 return;

352 }

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381 bool LongerNotColdContextKept = false;

382 for (auto *MIB : NewMIBNodes) {

385 continue;

388 if (StackMD->getNumOperands() > CallerContextLength) {

389 LongerNotColdContextKept = true;

390 break;

391 }

392 }

393

394

395 bool KeepFirstNewNotCold = !LongerNotColdContextKept;

398

402

404 return true;

405

406

407 if (KeepFirstNewNotCold) {

408 KeepFirstNewNotCold = false;

409 return true;

410 }

412 EmitMessageForRemovedContexts(MIBMD, "pruned", "");

413 return false;

414 }

415 return true;

416 });

417 for (auto *M : NewColdMIBNodes)

418 SavedMIBNodes.push_back(M);

419}

420

421

422

423

424

425bool CallStackTrie::buildMIBNodes(CallStackTrieNode *Node, LLVMContext &Ctx,

426 std::vector<uint64_t> &MIBCallStack,

427 std::vector<Metadata *> &MIBNodes,

428 bool CalleeHasAmbiguousCallerContext,

429 uint64_t &TotalBytes, uint64_t &ColdBytes) {

430

431

433 std::vector ContextSizeInfo;

434 collectContextSizeInfo(Node, ContextSizeInfo);

436 Ctx, MIBCallStack, (AllocationType)Node->AllocTypes, ContextSizeInfo,

437 MaxColdSize, BuiltFromExistingMetadata, TotalBytes, ColdBytes));

438 return true;

439 }

440

441

442

443 if ( Node ->Callers.empty()) {

444 bool NodeHasAmbiguousCallerContext = Node->Callers.size() > 1;

445 bool AddedMIBNodesForAllCallerContexts = true;

446

447

448

449 std::vector<Metadata *> NewMIBNodes;

450

451

452 uint64_t CallerTotalBytes = 0;

453 uint64_t CallerColdBytes = 0;

454 for (auto &Caller : Node->Callers) {

455 MIBCallStack.push_back(Caller.first);

456 AddedMIBNodesForAllCallerContexts &= buildMIBNodes(

457 Caller.second, Ctx, MIBCallStack, NewMIBNodes,

458 NodeHasAmbiguousCallerContext, CallerTotalBytes, CallerColdBytes);

459

460 MIBCallStack.pop_back();

461 }

462

463

465 CallerTotalBytes, CallerColdBytes,

466 BuiltFromExistingMetadata);

467 TotalBytes += CallerTotalBytes;

468 ColdBytes += CallerColdBytes;

469

470 if (AddedMIBNodesForAllCallerContexts)

471 return true;

472

473

474 assert(!NodeHasAmbiguousCallerContext);

475 }

476

477

478

479

480

481

482

483

484

485

486

487 if (!CalleeHasAmbiguousCallerContext)

488 return false;

489 std::vector ContextSizeInfo;

490 collectContextSizeInfo(Node, ContextSizeInfo);

493 BuiltFromExistingMetadata, TotalBytes, ColdBytes));

494 return true;

495}

496

501

502

506 std::vector ContextSizeInfo;

507 collectContextSizeInfo(Alloc, ContextSizeInfo);

508 for (const auto &[FullStackId, TotalSize] : ContextSizeInfo) {

509 errs() << "MemProf hinting: Total size for full allocation context hash "

510 << FullStackId << " and " << Descriptor << " alloc type "

512 }

513 }

514 if (ORE)

516 << ore::NV("AllocationCall", CI) << " in function "

518 << " marked with memprof allocation attribute "

519 << ore::NV("Attribute", AllocTypeString));

520}

521

522

523

524

528 "single");

529 return false;

530 }

531

532

533

534

535

536

537

539 convertHotToNotCold(Alloc);

540

543 "single");

544 return false;

545 }

546 }

548 std::vector<uint64_t> MIBCallStack;

549 MIBCallStack.push_back(AllocStackId);

550 std::vector<Metadata *> MIBNodes;

553 assert(!Alloc->Callers.empty() && "addCallStack has not been called yet");

554

555

556

557 if (buildMIBNodes(Alloc, Ctx, MIBCallStack, MIBNodes,

558 false, TotalBytes,

559 ColdBytes)) {

560 assert(MIBCallStack.size() == 1 &&

561 "Should only be left with Alloc's location in stack");

564 return true;

565 }

566

567

568

569

570

572 return false;

573}

574

575template <>

577 const MDNode *N, bool End)

578 : N(N) {

579 if ()

580 return;

581 Iter = End ? N->op_end() : N->op_begin();

582}

583

584template <>

591}

592

596 ->getZExtValue();

597}

598

600

601

602

603 if (A)

604 return A;

605 return B;

606}

607

609

610

611 if (A)

612 return A;

613 return B;

614}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

This file contains the declarations for the subclasses of Constant, which represent the different fla...

static MDNode * createMIBNode(LLVMContext &Ctx, ArrayRef< uint64_t > MIBCallStack, AllocationType AllocType, ArrayRef< ContextTotalSize > ContextSizeInfo, const uint64_t MaxColdSize, bool BuiltFromExistingMetadata, uint64_t &TotalBytes, uint64_t &ColdBytes)

Definition MemoryProfileInfo.cpp:217

static void saveFilteredNewMIBNodes(std::vector< Metadata * > &NewMIBNodes, std::vector< Metadata * > &SavedMIBNodes, unsigned CallerContextLength, uint64_t TotalBytes, uint64_t ColdBytes, bool BuiltFromExistingMetadata)

Definition MemoryProfileInfo.cpp:288

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

bool empty() const

empty - Check if the array is empty.

static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)

Return a uniquified Attribute object.

LLVM_ABI StringRef getValueAsString() const

Return the attribute's value as a string.

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

void addFnAttr(Attribute::AttrKind Kind)

Adds the attribute to the function.

bool hasFnAttr(Attribute::AttrKind Kind) const

Determine whether this call has the given attribute.

Attribute getFnAttr(StringRef Kind) const

Get the attribute of a given kind for the function.

void removeFnAttr(Attribute::AttrKind Kind)

Removes the attribute from the function.

This is the shared class of boolean and integer constants.

uint64_t getZExtValue() const

Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...

LLVM_ABI const Function * getFunction() const

Return the function this instruction belongs to.

LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)

Set the metadata of the specified kind to the specified node.

This is an important class for using LLVM in a threaded context.

static LLVM_ABI MDNode * getMergedCallsiteMetadata(MDNode *A, MDNode *B)

Definition MemoryProfileInfo.cpp:608

const MDOperand & getOperand(unsigned I) const

ArrayRef< MDOperand > operands() const

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

unsigned getNumOperands() const

Return number of MDNode operands.

LLVM_ABI MDNode(LLVMContext &Context, unsigned ID, StorageType Storage, ArrayRef< Metadata * > Ops1, ArrayRef< Metadata * > Ops2={})

static LLVM_ABI MDNode * getMergedMemProfMetadata(MDNode *A, MDNode *B)

Definition MemoryProfileInfo.cpp:599

static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)

void push_back(Metadata *MD)

Append an element to the tuple. This will resize the node.

void reserve(size_type N)

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

The instances of the Type class are immutable: once they are created, they are never changed.

static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)

LLVM_ABI LLVMContext & getContext() const

All values hold a context through their type.

LLVM_ABI void addCallStack(AllocationType AllocType, ArrayRef< uint64_t > StackIds, std::vector< ContextTotalSize > ContextSizeInfo={})

Add a call stack context with the given allocation type to the Trie.

Definition MemoryProfileInfo.cpp:152

LLVM_ABI void addSingleAllocTypeAttribute(CallBase *CI, AllocationType AT, StringRef Descriptor)

Add an attribute for the given allocation type to the call instruction.

Definition MemoryProfileInfo.cpp:497

LLVM_ABI bool buildAndAttachMIBMetadata(CallBase *CI)

Build and attach the minimal necessary MIB metadata.

Definition MemoryProfileInfo.cpp:525

Helper class to iterate through stack ids in both metadata (memprof MIB and callsite) and the corresp...

A raw_ostream that writes to an std::string.

std::string & str()

Returns the string's reference.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

initializer< Ty > init(const Ty &Val)

std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)

Extract a Value from Metadata, if any.

LLVM_ABI MDNode * buildCallstackMetadata(ArrayRef< uint64_t > CallStack, LLVMContext &Ctx)

Build callstack metadata from the provided list of call stack ids.

Definition MemoryProfileInfo.cpp:76

LLVM_ABI bool recordContextSizeInfoForAnalysis()

Whether we need to record the context size info in the alloc trie used to build metadata.

Definition MemoryProfileInfo.cpp:71

LLVM_ABI bool metadataIncludesAllContextSizeInfo()

Whether the alloc memeprof metadata will include context size info for all MIBs.

Definition MemoryProfileInfo.cpp:63

LLVM_ABI AllocationType getMIBAllocType(const MDNode *MIB)

Returns the allocation type from an MIB metadata node.

Definition MemoryProfileInfo.cpp:94

LLVM_ABI bool metadataMayIncludeContextSizeInfo()

Whether the alloc memprof metadata may include context size info for some MIBs (but possibly not all)...

Definition MemoryProfileInfo.cpp:67

LLVM_ABI bool hasSingleAllocType(uint8_t AllocTypes)

True if the AllocTypes bitmask contains just a single type.

Definition MemoryProfileInfo.cpp:126

LLVM_ABI std::string getAllocTypeAttributeString(AllocationType Type)

Returns the string to use in attributes with the given type.

Definition MemoryProfileInfo.cpp:109

LLVM_ABI MDNode * getMIBStackNode(const MDNode *MIB)

Returns the stack node from an MIB metadata node.

Definition MemoryProfileInfo.cpp:88

LLVM_ABI void removeAnyExistingAmbiguousAttribute(CallBase *CB)

Removes any existing "ambiguous" memprof attribute.

Definition MemoryProfileInfo.cpp:132

LLVM_ABI void addAmbiguousAttribute(CallBase *CB)

Adds an "ambiguous" memprof attribute to call with a matched allocation profile but that we haven't y...

Definition MemoryProfileInfo.cpp:139

DiagnosticInfoOptimizationBase::Argument NV

NodeAddr< NodeBase * > Node

This is an optimization pass for GlobalISel generic memory operations.

cl::opt< unsigned > MinClonedColdBytePercent("memprof-cloning-cold-threshold", cl::init(100), cl::Hidden, cl::desc("Min percent of cold bytes to hint alloc cold during cloning"))

cl::opt< bool > MemProfReportHintedSizes("memprof-report-hinted-sizes", cl::init(false), cl::Hidden, cl::desc("Report total allocation sizes of hinted allocations"))

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

void append_range(Container &C, Range &&R)

Wrapper function to append range R to container C.

constexpr int popcount(T Value) noexcept

Count the number of set bits in a value.

LLVM_ABI cl::opt< bool > MemProfKeepAllNotColdContexts("memprof-keep-all-not-cold-contexts", cl::init(false), cl::Hidden, cl::desc("Keep all non-cold contexts (increases cloning overheads)"))

LLVM_ABI cl::opt< bool > MemProfUseAmbiguousAttributes("memprof-ambiguous-attributes", cl::init(true), cl::Hidden, cl::desc("Apply ambiguous memprof attribute to ambiguous allocations"))

cl::opt< unsigned > MinCallsiteColdBytePercent("memprof-callsite-cold-threshold", cl::init(100), cl::Hidden, cl::desc("Min percent of cold bytes at a callsite to discard non-cold " "contexts"))

iterator_range< filter_iterator< detail::IterOfRange< RangeT >, PredicateT > > make_filter_range(RangeT &&Range, PredicateT Pred)

Convenience function that takes a range of elements and a predicate, and return a new filter_iterator...

format_object< Ts... > format(const char *Fmt, const Ts &... Vals)

These are helper functions used to produce formatted output.

LLVM_ABI raw_fd_ostream & errs()

This returns a reference to a raw_ostream for standard error.

@ First

Helpers to iterate all locations in the MemoryEffectsBase class.

FunctionAddr VTableAddr Next

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

cl::opt< unsigned > MinPercentMaxColdSize("memprof-min-percent-max-cold-size", cl::init(100), cl::Hidden, cl::desc("Min percent of max cold bytes for critical cold context"))

CallStackIterator(const NodeT *N, bool End)