MemProfUse.cpp Source File (original) (raw)

36#include

37#include

39using namespace llvm;

42#define DEBUG_TYPE "memprof"

44namespace llvm {

48}

54 "memprof-match-hot-cold-new",

56 "Match allocation profiles onto existing hot/cold operator new calls"),

61 cl::desc("Print matching stats for each allocation "

62 "context in this module's profiles"),

66 "memprof-print-matched-alloc-stack",

67 cl::desc("Print full stack context for matched "

68 "allocations with -memprof-print-match-info."),

73 cl::desc("Print function GUIDs computed for matching"),

78 cl::desc("Salvage stale MemProf profile"),

82 "memprof-attach-calleeguids",

84 "Attach calleeguids as value profile metadata for indirect calls."),

89 cl::desc("Min percent of cold bytes matched to hint allocation cold"));

93 cl::desc("If true, annotate the static data section prefix"));

96 STATISTIC(NumOfMemProfMissing, "Number of functions without memory profile.");

98 "Number of functions having mismatched memory profile hash.");

99 STATISTIC(NumOfMemProfFunc, "Number of functions having valid memory profile.");

101 "Number of alloc contexts in memory profile.");

103 "Number of callsites in memory profile.");

105 "Number of matched memory profile alloc contexts.");

107 "Number of matched memory profile allocs.");

109 "Number of matched memory profile callsites.");

111 "Number of global vars annotated with 'hot' section prefix.");

113 "Number of global vars annotated with 'unlikely' section prefix.");

115 "Number of global vars with unknown hotness (no section prefix).");

116 STATISTIC(NumOfMemProfExplicitSectionGlobalVars,

117 "Number of global vars with user-specified section (not annotated).");

118

122 I.setMetadata(LLVMContext::MD_callsite,

124}

125

133 std::memcpy(&Id, Hash.data(), sizeof(Hash));

134 return Id;

135}

136

140

144 AllocInfo->Info.getTotalLifetime());

145}

146

151 for (const auto &StackFrame : AllocInfo->CallStack)

154 std::vector ContextSizeInfo;

156 auto TotalSize = AllocInfo->Info.getTotalSize();

158 assert(FullStackId != 0);

159 ContextSizeInfo.push_back({FullStackId, TotalSize});

160 }

163}

164

165

166

167

168static bool

171 return ProfileCallStack.size() >= InlinedCallStack.size() &&

173 InlinedCallStack, [](const Frame &F, uint64_t StackId) {

174 return computeStackId(F) == StackId;

175 });

176}

177

180 if (!Callee)

181 return false;

182 LibFunc Func;

184 return false;

185 switch (Func) {

186 case LibFunc_Znwm:

187 case LibFunc_ZnwmRKSt9nothrow_t:

188 case LibFunc_ZnwmSt11align_val_t:

189 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:

190 case LibFunc_Znam:

191 case LibFunc_ZnamRKSt9nothrow_t:

192 case LibFunc_ZnamSt11align_val_t:

193 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:

194 case LibFunc_size_returning_new:

195 case LibFunc_size_returning_new_aligned:

196 return true;

197 case LibFunc_Znwm12__hot_cold_t:

198 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:

199 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:

200 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:

201 case LibFunc_Znam12__hot_cold_t:

202 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:

203 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:

204 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:

205 case LibFunc_size_returning_new_hot_cold:

206 case LibFunc_size_returning_new_aligned_hot_cold:

208 default:

209 return false;

210 }

211}

212

216 "Should not handle AnnotationOK here");

218 switch (Kind) {

220 ++NumOfMemProfExplicitSectionGlobalVars;

221 Reason.append("explicit section name");

222 break;

224 Reason.append("linker declaration");

225 break;

227 Reason.append("name starts with `llvm.`");

228 break;

229 default:

231 }

233 << Reason << ".\n");

234}

235

236

237

239

241

243

244

245

246

248

249

251

252

253

256};

257

262

263 auto GetOffset = [](const DILocation *DIL) {

264 return (DIL->getLine() - DIL->getScope()->getSubprogram()->getLine()) &

265 0xffff;

266 };

267

269 if (F.isDeclaration())

270 continue;

271

272 for (auto &BB : F) {

273 for (auto &I : BB) {

275 continue;

276

278 auto *CalledFunction = CB->getCalledFunction();

279

280 if (!CalledFunction || CalledFunction->isIntrinsic())

281 continue;

282

283 StringRef CalleeName = CalledFunction->getName();

284

285

287

288

289 bool IsLeaf = true;

290 for (const DILocation *DIL = I.getDebugLoc(); DIL;

291 DIL = DIL->getInlinedAt()) {

292 StringRef CallerName = DIL->getSubprogramLinkageName();

294 "Be sure to enable -fdebug-info-for-profiling");

297

298

299 if (IsAlloc) {

300 if (IsLeaf) {

301

302

303 CalleeGUID = 0;

304 } else if (!IsPresentInProfile(CalleeGUID)) {

305

306

307 CalleeGUID = 0;

308 } else {

309

310

311 IsAlloc = false;

312 }

313 }

314

316 Calls[CallerGUID].emplace_back(Loc, CalleeGUID);

317 CalleeName = CallerName;

318 IsLeaf = false;

319 }

320 }

321 }

322 }

323

324

325 for (auto &[CallerGUID, CallList] : Calls) {

327 CallList.erase(llvm::unique(CallList), CallList.end());

328 }

329

330 return Calls;

331}

332

337

342 return CallsFromProfile.contains(GUID);

343 });

344

345

346 for (const auto &[CallerGUID, IRAnchors] : CallsFromIR) {

347 auto It = CallsFromProfile.find(CallerGUID);

348 if (It == CallsFromProfile.end())

349 continue;

350 const auto &ProfileAnchors = It->second;

351

354 ProfileAnchors, IRAnchors, std::equal_toGlobalValue::GUID(),

356 [[maybe_unused]] bool Inserted =

357 UndriftMaps.try_emplace(CallerGUID, std::move(Matchings)).second;

358

359

361 }

362

363 return UndriftMaps;

364}

365

366

367

368static void

371

372 auto UndriftCallStack = [&](std::vector &CallStack) {

374 auto I = UndriftMaps.find(F.Function);

375 if (I == UndriftMaps.end())

376 continue;

377 auto J = I->second.find(LineLocation(F.LineOffset, F.Column));

378 if (J == I->second.end())

379 continue;

380 auto &NewLoc = J->second;

381 F.LineOffset = NewLoc.LineOffset;

382 F.Column = NewLoc.Column;

383 }

384 };

385

386 for (auto &AS : MemProfRec.AllocSites)

387 UndriftCallStack(AS.CallStack);

388

389 for (auto &CS : MemProfRec.CallSites)

390 UndriftCallStack(CS.Frames);

391}

392

393

397 return;

398

399 if (I.getMetadata(LLVMContext::MD_prof)) {

401

402

403

405 1, Unused);

406

407 if (!ExistingVD.empty()) {

408 return;

409 }

410 }

411

414

416 InstrProfValueData VD;

417 VD.Value = CalleeGUID;

418

419

420

421

422 VD.Count = 1;

424 TotalCount += VD.Count;

425 }

426

427 if (!VDs.empty()) {

430 }

431}

432

436 const std::set<const AllocationInfo *> &AllocInfoSet,

437 std::map<uint64_t, AllocMatchInfo> &FullStackIdToAllocMatchInfo) {

438

439

440

442 for (auto *AllocInfo : AllocInfoSet) {

444 auto [It, Inserted] =

445 UniqueFullContextIdAllocInfo.insert({FullStackId, AllocInfo});

446

447 if (Inserted)

448 continue;

449

450 auto CurSize = It->second->Info.getTotalSize();

451 auto NewSize = AllocInfo->Info.getTotalSize();

452 if ((CurSize > NewSize) ||

453 (CurSize == NewSize &&

455 continue;

457 }

458

459

460

464 for (auto &[FullStackId, AllocInfo] : UniqueFullContextIdAllocInfo) {

465

466

467

469 InlinedCallStack)) {

470 NumOfMemProfMatchedAllocContexts++;

472 TotalSize += AllocInfo->Info.getTotalSize();

474 TotalColdSize += AllocInfo->Info.getTotalSize();

475

476

478 assert(FullStackId != 0);

479 auto [Iter, Inserted] = FullStackIdToAllocMatchInfo.try_emplace(

480 FullStackId,

482

483 Iter->second.MatchedFramesSet.insert(InlinedCallStack.size());

485 Iter->second.CallStack.insert(Iter->second.CallStack.begin(),

488 }

491 << ore::NV("AllocationCall", CI) << " in function "

493 << " matched alloc context with alloc type "

495 << " total size " << ore::NV("Size", AllocInfo->Info.getTotalSize())

496 << " full context id " << ore::NV("Context", FullStackId)

497 << " frame count " << ore::NV("Frames", InlinedCallStack.size()));

498 }

499 }

500

501

502

506 return;

507 }

508

509

510

511

512 if (!AllocTrie.empty()) {

513 NumOfMemProfMatchedAllocs++;

514

515

517 assert(MemprofMDAttached == I.hasMetadata(LLVMContext::MD_memprof));

518 if (MemprofMDAttached) {

519

520

521

522

523

524

525

527 }

528 }

529}

530

531

532

533

540

543 const std::vector &CallSiteEntries,

545 std::set<std::vector<uint64_t>> &MatchedCallSites,

547 auto &Ctx = M.getContext();

548

549

550

551

553 bool CallsiteMDAdded = false;

554 for (const auto &CallSiteEntry : CallSiteEntries) {

555

556

558 InlinedCallStack)) {

559 NumOfMemProfMatchedCallSites++;

560

561

562 if (!CallsiteMDAdded) {

564

565

569 MatchedCallSites.insert(std::move(CallStack));

570 }

572 << ore::NV("CallSite", &I) << " in function "

573 << ore::NV("Caller", I.getFunction())

574 << " matched callsite with frame count "

575 << ore::NV("Frames", InlinedCallStack.size()));

576

577

578 if (CalledFunction)

579 break;

580 CallsiteMDAdded = true;

581 }

582

583 assert(!CalledFunction && "Didn't expect direct call");

584

585

588 }

589 }

590

592}

593

594static void

597 std::map<uint64_t, AllocMatchInfo> &FullStackIdToAllocMatchInfo,

598 std::set<std::vector<uint64_t>> &MatchedCallSites,

601 auto &Ctx = M.getContext();

602

603

604

605

606

607

608

609 auto FuncName = F.getName();

612 errs() << "MemProf: Function GUID " << FuncGUID << " is " << FuncName

613 << "\n";

614 std::optionalmemprof::MemProfRecord MemProfRec;

615 auto Err = MemProfReader->getMemProfRecord(FuncGUID).moveInto(MemProfRec);

616 if (Err) {

618 auto Err = IPE.get();

619 bool SkipWarning = false;

620 LLVM_DEBUG(dbgs() << "Error in reading profile for Func " << FuncName

621 << ": ");

623 NumOfMemProfMissing++;

627 NumOfMemProfMismatch++;

628 SkipWarning =

631 (F.hasComdat() ||

633 LLVM_DEBUG(dbgs() << "hash mismatch (skip=" << SkipWarning << ")");

634 }

635

636 if (SkipWarning)

637 return;

638

639 std::string Msg = (IPE.message() + Twine(" ") + F.getName().str() +

640 Twine(" Hash = ") + std::to_string(FuncGUID))

641 .str();

642

643 Ctx.diagnose(

645 });

646 return;

647 }

648

649 NumOfMemProfFunc++;

650

651

652

655

656

657

658

659

660 bool ProfileHasColumns = false;

661

662

663

664 std::map<uint64_t, std::set<const AllocationInfo *>> LocHashToAllocInfo;

665

666

667

668 std::map<uint64_t, std::vector> LocHashToCallSites;

669 for (auto &AI : MemProfRec->AllocSites) {

670 NumOfMemProfAllocContextProfiles++;

671

672

673

675 LocHashToAllocInfo[StackId].insert(&AI);

676 ProfileHasColumns |= AI.CallStack[0].Column;

677 }

678 for (auto &CS : MemProfRec->CallSites) {

679 NumOfMemProfCallSiteProfiles++;

680

681

682 unsigned Idx = 0;

683 for (auto &StackFrame : CS.Frames) {

687 LocHashToCallSites[StackId].push_back({FrameSlice, CalleeGuids});

688

689 ProfileHasColumns |= StackFrame.Column;

690

691 if (StackFrame.Function == FuncGUID)

692 break;

693 }

694 assert(Idx <= CS.Frames.size() && CS.Frames[Idx - 1].Function == FuncGUID);

695 }

696

697 auto GetOffset = [](const DILocation *DIL) {

698 return (DIL->getLine() - DIL->getScope()->getSubprogram()->getLine()) &

699 0xffff;

700 };

701

702

703

704 for (auto &BB : F) {

705 for (auto &I : BB) {

706 if (I.isDebugOrPseudoInst())

707 continue;

708

709

711 if (!CI)

712 continue;

713 auto *CalledFunction = CI->getCalledFunction();

714 if (CalledFunction && CalledFunction->isIntrinsic())

715 continue;

716

717

719

720 bool LeafFound = false;

721

722

723

724

725

726 auto AllocInfoIter = LocHashToAllocInfo.end();

727 auto CallSitesIter = LocHashToCallSites.end();

728 for (const DILocation *DIL = I.getDebugLoc(); DIL != nullptr;

729 DIL = DIL->getInlinedAt()) {

730

731

732 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();

733 if (Name.empty())

734 Name = DIL->getScope()->getSubprogram()->getName();

736 auto StackId = computeStackId(CalleeGUID, GetOffset(DIL),

737 ProfileHasColumns ? DIL->getColumn() : 0);

738

739

740

741

742 if (!LeafFound) {

743 AllocInfoIter = LocHashToAllocInfo.find(StackId);

744 CallSitesIter = LocHashToCallSites.find(StackId);

745 if (AllocInfoIter != LocHashToAllocInfo.end() ||

746 CallSitesIter != LocHashToCallSites.end())

747 LeafFound = true;

748 }

749 if (LeafFound)

750 InlinedCallStack.push_back(StackId);

751 }

752

753 if (!LeafFound)

754 continue;

755

756

757

758

759

760 if (AllocInfoIter != LocHashToAllocInfo.end() &&

761

763 handleAllocSite(I, CI, InlinedCallStack, Ctx, ORE, MaxColdSize,

764 AllocInfoIter->second, FullStackIdToAllocMatchInfo);

765 else if (CallSitesIter != LocHashToCallSites.end())

766

767

768

770 CallSitesIter->second, M, MatchedCallSites, ORE);

771 }

772 }

773}

774

777 : MemoryProfileFileName(MemoryProfileFile), FS(FS) {

778 if (!FS)

780}

781

783

784

785 if (M.empty() && M.globals().empty())

787

789 auto &Ctx = M.getContext();

791 if (Error E = ReaderOrErr.takeError()) {

793 Ctx.diagnose(

795 });

797 }

798

799 std::unique_ptr MemProfReader =

800 std::move(ReaderOrErr.get());

803 MemoryProfileFileName.data(), StringRef("Cannot get MemProfReader")));

805 }

806

809 "Not a memory profile"));

811 }

812

814 annotateGlobalVariables(M, MemProfReader->getDataAccessProfileData());

815

816

817

818 if (M.empty())

820

822

827

828

829

830

831

832 std::map<uint64_t, AllocMatchInfo> FullStackIdToAllocMatchInfo;

833

834

835

836 std::set<std::vector<uint64_t>> MatchedCallSites;

837

839 if (auto *MemProfSum = MemProfReader->getMemProfSummary())

840 MaxColdSize = MemProfSum->getMaxColdTotalSize();

841

842 for (auto &F : M) {

843 if (F.isDeclaration())

844 continue;

845

849 MatchedCallSites, UndriftMaps, ORE, MaxColdSize);

850 }

851

853 for (const auto &[Id, Info] : FullStackIdToAllocMatchInfo) {

854 for (auto Frames : Info.MatchedFramesSet) {

855

856

857

859 << " context with id " << Id << " has total profiled size "

860 << Info.TotalSize << " is matched with " << Frames << " frames";

862 errs() << " and call stack";

863 for (auto &F : Info.CallStack)

865 }

866 errs() << "\n";

867 }

868 }

869

870 for (const auto &CallStack : MatchedCallSites) {

871 errs() << "MemProf callsite match for inline call stack";

873 errs() << " " << StackId;

874 errs() << "\n";

875 }

876 }

877

879}

880

881bool MemProfUsePass::annotateGlobalVariables(

884 return false;

885

886 if (!DataAccessProf) {

887 M.addModuleFlag(Module::Warning, "EnableDataAccessProf", 0U);

889 MemoryProfileFileName.data(),

890 StringRef("Data access profiles not found in memprof. Ignore "

891 "-memprof-annotate-static-data-prefix."),

893 return false;

894 }

895 M.addModuleFlag(Module::Warning, "EnableDataAccessProf", 1U);

896

898

899

900

901

902

904 assert(!GVar.getSectionPrefix().has_value() &&

905 "GVar shouldn't have section prefix yet");

909 continue;

910 }

911

912 StringRef Name = GVar.getName();

913

914

915

916

917 if (Name.starts_with(".str")) {

918 LLVM_DEBUG(dbgs() << "Skip annotating string literal " << Name << "\n");

919 continue;

920 }

921

922

923

924 std::optional Record =

926

927

928

929

930

931

932 if (Record && Record->AccessCount > 0) {

933 ++NumOfMemProfHotGlobalVars;

934 Changed |= GVar.setSectionPrefix("hot");

936 << " is annotated as hot\n");

938 ++NumOfMemProfColdGlobalVars;

939 Changed |= GVar.setSectionPrefix("unlikely");

942 << " is annotated as unlikely\n");

943 } else {

944 ++NumOfMemProfUnknownGlobalVars;

945 LLVM_DEBUG(dbgs() << "Global variable " << Name << " is not annotated\n");

946 }

947 }

948

950}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

Module.h This file contains the declarations for the Module class.

static void addCallsiteMetadata(Instruction &I, ArrayRef< uint64_t > InlinedCallStack, LLVMContext &Ctx)

Definition MemProfUse.cpp:119

static bool isAllocationWithHotColdVariant(const Function *Callee, const TargetLibraryInfo &TLI)

Definition MemProfUse.cpp:178

static cl::opt< bool > ClMemProfAttachCalleeGuids("memprof-attach-calleeguids", cl::desc("Attach calleeguids as value profile metadata for indirect calls."), cl::init(true), cl::Hidden)

static void HandleUnsupportedAnnotationKinds(GlobalVariable &GVar, AnnotationKind Kind)

Definition MemProfUse.cpp:213

static void undriftMemProfRecord(const DenseMap< uint64_t, LocToLocMap > &UndriftMaps, memprof::MemProfRecord &MemProfRec)

Definition MemProfUse.cpp:369

static uint64_t computeStackId(GlobalValue::GUID Function, uint32_t LineOffset, uint32_t Column)

Definition MemProfUse.cpp:126

static cl::opt< bool > PrintMatchedAllocStack("memprof-print-matched-alloc-stack", cl::desc("Print full stack context for matched " "allocations with -memprof-print-match-info."), cl::Hidden, cl::init(false))

static void handleCallSite(Instruction &I, const Function *CalledFunction, ArrayRef< uint64_t > InlinedCallStack, const std::vector< CallSiteEntry > &CallSiteEntries, Module &M, std::set< std::vector< uint64_t > > &MatchedCallSites, OptimizationRemarkEmitter &ORE)

Definition MemProfUse.cpp:541

static cl::opt< bool > ClPrintMemProfMatchInfo("memprof-print-match-info", cl::desc("Print matching stats for each allocation " "context in this module's profiles"), cl::Hidden, cl::init(false))

static void addVPMetadata(Module &M, Instruction &I, ArrayRef< GlobalValue::GUID > CalleeGuids)

Definition MemProfUse.cpp:394

static cl::opt< bool > PrintFunctionGuids("memprof-print-function-guids", cl::desc("Print function GUIDs computed for matching"), cl::Hidden, cl::init(false))

static cl::opt< bool > AnnotateStaticDataSectionPrefix("memprof-annotate-static-data-prefix", cl::init(false), cl::Hidden, cl::desc("If true, annotate the static data section prefix"))

static void handleAllocSite(Instruction &I, CallBase *CI, ArrayRef< uint64_t > InlinedCallStack, LLVMContext &Ctx, OptimizationRemarkEmitter &ORE, uint64_t MaxColdSize, const std::set< const AllocationInfo * > &AllocInfoSet, std::map< uint64_t, AllocMatchInfo > &FullStackIdToAllocMatchInfo)

Definition MemProfUse.cpp:433

static cl::opt< bool > SalvageStaleProfile("memprof-salvage-stale-profile", cl::desc("Salvage stale MemProf profile"), cl::init(false), cl::Hidden)

static cl::opt< unsigned > MinMatchedColdBytePercent("memprof-matching-cold-threshold", cl::init(100), cl::Hidden, cl::desc("Min percent of cold bytes matched to hint allocation cold"))

static void readMemprof(Module &M, Function &F, IndexedInstrProfReader *MemProfReader, const TargetLibraryInfo &TLI, std::map< uint64_t, AllocMatchInfo > &FullStackIdToAllocMatchInfo, std::set< std::vector< uint64_t > > &MatchedCallSites, DenseMap< uint64_t, LocToLocMap > &UndriftMaps, OptimizationRemarkEmitter &ORE, uint64_t MaxColdSize)

Definition MemProfUse.cpp:595

static cl::opt< bool > ClMemProfMatchHotColdNew("memprof-match-hot-cold-new", cl::desc("Match allocation profiles onto existing hot/cold operator new calls"), cl::Hidden, cl::init(false))

static AllocationType addCallStack(CallStackTrie &AllocTrie, const AllocationInfo *AllocInfo, uint64_t FullStackId)

Definition MemProfUse.cpp:147

static bool stackFrameIncludesInlinedCallStack(ArrayRef< Frame > ProfileCallStack, ArrayRef< uint64_t > InlinedCallStack)

Definition MemProfUse.cpp:169

FunctionAnalysisManager FAM

This file defines the SmallVector class.

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

Defines the virtual file system interface vfs::FileSystem.

PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)

Get the result of an analysis pass for a given IR unit.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

ArrayRef< T > take_front(size_t N=1) const

Return a copy of *this with only the first N elements.

ArrayRef< T > drop_front(size_t N=1) const

Drop the first N elements of the array.

size_t size() const

size - Get the array size.

bool empty() const

empty - Check if the array is empty.

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

iterator find(const_arg_type_t< KeyT > Val)

std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)

bool contains(const_arg_type_t< KeyT > Val) const

Return true if the specified key is in the map, false otherwise.

std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)

Diagnostic information for the PGO profiler.

Base class for error info classes.

virtual std::string message() const

Return the error message as a string.

Lightweight error class with error context and mandatory checking.

static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)

Return a 64-bit global unique ID constructed from the name of a global symbol.

uint64_t GUID

Declare a type to represent a global unique identifier for a global value.

@ AvailableExternallyLinkage

Available for inspection, not emission.

HashResultTy< HasherT_ > final()

Forward to HasherT::final() if available.

Interface to help hash various types through a hasher type.

std::enable_if_t< hashbuilder_detail::IsHashableData< T >::value, HashBuilder & > add(T Value)

Implement hashing for hashable data types, e.g. integral or enum values.

Reader for the indexed binary instrprof format.

static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const Twine &RemappingPath="")

Factory method to create an indexed reader.

instrprof_error get() const

std::string message() const override

Return the error message as a string.

LLVM_ABI const Function * getFunction() const

Return the function this instruction belongs to.

A smart pointer to a reference-counted object that inherits from RefCountedBase or ThreadSafeRefCount...

This is an important class for using LLVM in a threaded context.

LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)

Definition MemProfUse.cpp:782

LLVM_ABI MemProfUsePass(std::string MemoryProfileFile, IntrusiveRefCntPtr< vfs::FileSystem > FS=nullptr)

Definition MemProfUse.cpp:775

A Module instance is used to store all the information related to an LLVM module.

@ Warning

Emits a warning if two values disagree.

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

A vector that has set insertion semantics.

ArrayRef< value_type > getArrayRef() const

bool insert(const value_type &X)

Insert a new element into the SetVector.

SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...

void append(StringRef RHS)

Append from a StringRef.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

constexpr bool empty() const

empty - Check if the string is empty.

Analysis pass providing the TargetLibraryInfo.

Provides information about what library functions are available for the current target.

bool getLibFunc(StringRef funcName, LibFunc &F) const

Searches for a particular function name.

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

An efficient, type-erasing, non-owning reference to a callable.

Class to build a trie of call stack contexts for a particular profiled allocation call,...

LLVM_ABI void addCallStack(AllocationType AllocType, ArrayRef< uint64_t > StackIds, std::vector< ContextTotalSize > ContextSizeInfo={})

Add a call stack context with the given allocation type to the Trie.

LLVM_ABI void addSingleAllocTypeAttribute(CallBase *CI, AllocationType AT, StringRef Descriptor)

Add an attribute for the given allocation type to the call instruction.

LLVM_ABI bool buildAndAttachMIBMetadata(CallBase *CI)

Build and attach the minimal necessary MIB metadata.

Helper class to iterate through stack ids in both metadata (memprof MIB and callsite) and the corresp...

Encapsulates the data access profile data and the methods to operate on it.

LLVM_ABI std::optional< DataAccessProfRecord > getProfileRecord(const SymbolHandleRef SymID) const

Returns a profile record for SymbolID, or std::nullopt if there isn't a record.

LLVM_ABI bool isKnownColdSymbol(const SymbolHandleRef SymID) const

Returns true if SymID is seen in profiled binaries and cold.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

initializer< Ty > init(const Ty &Val)

LLVM_ABI DenseMap< uint64_t, LocToLocMap > computeUndriftMap(Module &M, IndexedInstrProfReader *MemProfReader, const TargetLibraryInfo &TLI)

Definition MemProfUse.cpp:334

LLVM_ABI MDNode * buildCallstackMetadata(ArrayRef< uint64_t > CallStack, LLVMContext &Ctx)

Build callstack metadata from the provided list of call stack ids.

LLVM_ABI AllocationType getAllocType(uint64_t TotalLifetimeAccessDensity, uint64_t AllocCount, uint64_t TotalLifetime)

Return the allocation type for a given set of memory profile values.

LLVM_ABI bool recordContextSizeInfoForAnalysis()

Whether we need to record the context size info in the alloc trie used to build metadata.

std::unordered_map< LineLocation, LineLocation, LineLocationHash > LocToLocMap

LLVM_ABI uint64_t computeFullStackId(ArrayRef< Frame > CallStack)

Helper to generate a single hash id for a given callstack, used for emitting matching statistics and ...

LLVM_ABI DenseMap< uint64_t, SmallVector< CallEdgeTy, 0 > > extractCallsFromIR(Module &M, const TargetLibraryInfo &TLI, function_ref< bool(uint64_t)> IsPresentInProfile=[](uint64_t) { return true;})

Definition MemProfUse.cpp:259

AnnotationKind getAnnotationKind(const GlobalVariable &GV)

Returns the annotation kind of the global variable GV.

LLVM_ABI GlobalValue::GUID getGUID(const StringRef FunctionName)

LLVM_ABI std::string getAllocTypeAttributeString(AllocationType Type)

Returns the string to use in attributes with the given type.

DiagnosticInfoOptimizationBase::Argument NV

LLVM_ABI IntrusiveRefCntPtr< FileSystem > getRealFileSystem()

Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.

This is an optimization pass for GlobalISel generic memory operations.

std::array< uint8_t, NumBytes > BLAKE3Result

The constant LLVM_BLAKE3_OUT_LEN provides the default output length, 32 bytes, which is recommended f...

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

void handleAllErrors(Error E, HandlerTs &&... Handlers)

Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...

void append_range(Container &C, Range &&R)

Wrapper function to append range R to container C.

InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy

Provide the FunctionAnalysisManager to Module proxy.

cl::opt< bool > PGOWarnMissing

auto unique(Range &&R, Predicate P)

LLVM_ABI void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)

Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...

void sort(IteratorTy Start, IteratorTy End)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

LLVM_ABI SmallVector< InstrProfValueData, 4 > getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)

Extract the value profile data from Inst and returns them if Inst is annotated with value profile dat...

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

cl::opt< bool > NoPGOWarnMismatch

Definition MemProfUse.cpp:46

LLVM_ABI raw_fd_ostream & errs()

This returns a reference to a raw_ostream for standard error.

cl::opt< bool > SalvageStaleProfile("salvage-stale-profile", cl::Hidden, cl::init(false), cl::desc("Salvage stale profile by fuzzy matching and use the remapped " "location for sample profile query."))

void longestCommonSequence(AnchorList AnchorList1, AnchorList AnchorList2, llvm::function_ref< bool(const Function &, const Function &)> FunctionMatchesProfile, llvm::function_ref< void(Loc, Loc)> InsertMatching)

ArrayRef(const T &OneElt) -> ArrayRef< T >

bool equal(L &&LRange, R &&RRange)

Wrapper function around std::equal to detect if pair-wise elements between two ranges are the same.

AnalysisManager< Module > ModuleAnalysisManager

Convenience typedef for the Module analysis manager.

cl::opt< bool > NoPGOWarnMismatchComdatWeak

Definition MemProfUse.cpp:238

std::set< unsigned > MatchedFramesSet

Definition MemProfUse.cpp:247

uint64_t TotalSize

Definition MemProfUse.cpp:240

std::vector< Frame > CallStack