MemProfiler.cpp Source File (original) (raw)

47#include

48#include

50using namespace llvm;

53#define DEBUG_TYPE "memprof"

55namespace llvm {

59}

78 "__memprof_version_mismatch_check_v";

81 "__memprof_shadow_memory_dynamic_address";

90 "memprof-guard-against-version-mismatch",

91 cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,

96 cl::desc("instrument read instructions"),

103

105 "memprof-instrument-atomics",

106 cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,

108

110 "memprof-use-callbacks",

111 cl::desc("Use callbacks instead of inline instrumentation sequences."),

113

116 cl::desc("Prefix for memory access callbacks"),

118

119

120

121

122

124 cl::desc("scale of memprof shadow mapping"),

126

129 cl::desc("granularity of memprof shadow mapping"),

131

133 cl::desc("Instrument scalar stack variables"),

135

136

137

140

143

146

149

150

151

152

154 "memprof-match-hot-cold-new",

156 "Match allocation profiles onto existing hot/cold operator new calls"),

158

160 cl::desc("Collect access count histograms"),

162

165 cl::desc("Print matching stats for each allocation "

166 "context in this module's profiles"),

168

171 cl::desc("The default memprof options"),

173

176 cl::desc("Salvage stale MemProf profile"),

178

181 cl::desc("Min percent of cold bytes to hint alloc cold during cloning"));

182

184

187 cl::desc("Min percent of cold bytes matched to hint allocation cold"));

188

189

190 STATISTIC(NumInstrumentedReads, "Number of instrumented reads");

191 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");

192 STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads");

193 STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes");

194

195

196 STATISTIC(NumOfMemProfMissing, "Number of functions without memory profile.");

198 "Number of functions having mismatched memory profile hash.");

199 STATISTIC(NumOfMemProfFunc, "Number of functions having valid memory profile.");

201 "Number of alloc contexts in memory profile.");

203 "Number of callsites in memory profile.");

205 "Number of matched memory profile alloc contexts.");

207 "Number of matched memory profile allocs.");

209 "Number of matched memory profile callsites.");

210

211namespace {

212

213

214

215struct ShadowMapping {

216 ShadowMapping() {

219 Mask = ~(Granularity - 1);

220 }

221

222 int Scale;

223 int Granularity;

224 uint64_t Mask;

225};

226

227static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {

230}

231

232struct InterestingMemoryAccess {

234 bool IsWrite;

235 Type *AccessTy;

236 Value *MaybeMask = nullptr;

237};

238

239

240class MemProfiler {

241public:

242 MemProfiler(Module &M) {

243 C = &(M.getContext());

244 LongSize = M.getDataLayout().getPointerSizeInBits();

246 PtrTy = PointerType::getUnqual(*C);

247 }

248

249

250

251

252 std::optional

253 isInterestingMemoryAccess(Instruction *I) const;

254

256 InterestingMemoryAccess &Access);

259 void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,

261 bool IsWrite);

264 bool instrumentFunction(Function &F);

265 bool maybeInsertMemProfInitAtFunctionEntry(Function &F);

266 bool insertDynamicShadowAtFunctionEntry(Function &F);

267

268private:

269 void initializeCallbacks(Module &M);

270

272 int LongSize;

273 Type *IntptrTy;

275 ShadowMapping Mapping;

276

277

279

280 FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;

281 Value *DynamicShadowOffset = nullptr;

282};

283

284class ModuleMemProfiler {

285public:

286 ModuleMemProfiler(Module &M) { TargetTriple = Triple(M.getTargetTriple()); }

287

288 bool instrumentModule(Module &);

289

290private:

292 ShadowMapping Mapping;

293 Function *MemProfCtorFunction = nullptr;

294};

295

296}

297

299

303 "Memprof with histogram only supports default mapping granularity");

304 Module &M = *F.getParent();

305 MemProfiler Profiler(M);

306 if (Profiler.instrumentFunction(F))

309}

310

312

315

316 ModuleMemProfiler Profiler(M);

317 if (Profiler.instrumentModule(M))

320}

321

323

324 Shadow = IRB.CreateAnd(Shadow, Mapping.Mask);

325 Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);

326

327 assert(DynamicShadowOffset);

328 return IRB.CreateAdd(Shadow, DynamicShadowOffset);

329}

330

331

332void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {

334 if (isa(MI)) {

335 IRB.CreateCall(isa(MI) ? MemProfMemmove : MemProfMemcpy,

336 {MI->getOperand(0), MI->getOperand(1),

338 } else if (isa(MI)) {

340 MemProfMemset,

341 {MI->getOperand(0),

344 }

345 MI->eraseFromParent();

346}

347

348std::optional

349MemProfiler::isInterestingMemoryAccess(Instruction *I) const {

350

351 if (DynamicShadowOffset == I)

352 return std::nullopt;

353

354 InterestingMemoryAccess Access;

355

356 if (LoadInst *LI = dyn_cast(I)) {

358 return std::nullopt;

359 Access.IsWrite = false;

360 Access.AccessTy = LI->getType();

361 Access.Addr = LI->getPointerOperand();

362 } else if (StoreInst *SI = dyn_cast(I)) {

364 return std::nullopt;

365 Access.IsWrite = true;

366 Access.AccessTy = SI->getValueOperand()->getType();

367 Access.Addr = SI->getPointerOperand();

368 } else if (AtomicRMWInst *RMW = dyn_cast(I)) {

370 return std::nullopt;

371 Access.IsWrite = true;

372 Access.AccessTy = RMW->getValOperand()->getType();

373 Access.Addr = RMW->getPointerOperand();

374 } else if (AtomicCmpXchgInst *XCHG = dyn_cast(I)) {

376 return std::nullopt;

377 Access.IsWrite = true;

378 Access.AccessTy = XCHG->getCompareOperand()->getType();

379 Access.Addr = XCHG->getPointerOperand();

380 } else if (auto *CI = dyn_cast(I)) {

381 auto *F = CI->getCalledFunction();

382 if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||

383 F->getIntrinsicID() == Intrinsic::masked_store)) {

384 unsigned OpOffset = 0;

385 if (F->getIntrinsicID() == Intrinsic::masked_store) {

387 return std::nullopt;

388

389 OpOffset = 1;

390 Access.AccessTy = CI->getArgOperand(0)->getType();

391 Access.IsWrite = true;

392 } else {

394 return std::nullopt;

395 Access.AccessTy = CI->getType();

396 Access.IsWrite = false;

397 }

398

399 auto *BasePtr = CI->getOperand(0 + OpOffset);

400 Access.MaybeMask = CI->getOperand(2 + OpOffset);

402 }

403 }

404

406 return std::nullopt;

407

408

409

410 Type *PtrTy = cast(Access.Addr->getType()->getScalarType());

412 return std::nullopt;

413

414

415

416

417

418 if (Access.Addr->isSwiftError())

419 return std::nullopt;

420

421

422 auto *Addr = Access.Addr->stripInBoundsOffsets();

423

425

426 if (GV->hasSection()) {

428

432 return std::nullopt;

433 }

434

435

436 if (GV->getName().starts_with("__llvm"))

437 return std::nullopt;

438 }

439

441}

442

443void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,

445 Type *AccessTy, bool IsWrite) {

446 auto *VTy = cast(AccessTy);

447 unsigned Num = VTy->getNumElements();

448 auto *Zero = ConstantInt::get(IntptrTy, 0);

449 for (unsigned Idx = 0; Idx < Num; ++Idx) {

450 Value *InstrumentedAddress = nullptr;

452 if (auto *Vector = dyn_cast(Mask)) {

453

454 if (auto *Masked = dyn_cast(Vector->getOperand(Idx))) {

455 if (Masked->isZero())

456

457 continue;

458

459

460 }

461 } else {

465 InsertBefore = ThenTerm;

466 }

467

469 InstrumentedAddress =

472 }

473}

474

476 InterestingMemoryAccess &Access) {

477

480 ++NumSkippedStackWrites;

481 else

482 ++NumSkippedStackReads;

483 return;

484 }

485

487 NumInstrumentedWrites++;

488 else

489 NumInstrumentedReads++;

490

491 if (Access.MaybeMask) {

492 instrumentMaskedLoadOrStore(DL, Access.MaybeMask, I, Access.Addr,

494 } else {

495

496

497

499 }

500}

501

502void MemProfiler::instrumentAddress(Instruction *OrigIns,

504 bool IsWrite) {

507

509 IRB.CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);

510 return;

511 }

512

515

516 Value *ShadowPtr = memToShadow(AddrLong, IRB);

518 Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);

519

526 }

527 Value *Inc = ConstantInt::get(ShadowTy, 1);

528 ShadowValue = IRB.CreateAdd(ShadowValue, Inc);

529 IRB.CreateStore(ShadowValue, ShadowAddr);

530}

531

532

534 const MDString *MemProfFilename =

535 dyn_cast_or_null(M.getModuleFlag("MemProfProfileFilename"));

536 if (!MemProfFilename)

537 return;

539 "Unexpected MemProfProfileFilename metadata with empty string");

541 M.getContext(), MemProfFilename->getString(), true);

543 M, ProfileNameConst->getType(), true,

545 Triple TT(M.getTargetTriple());

546 if (TT.supportsCOMDAT()) {

549 }

550}

551

552

553

560 Triple TT(M.getTargetTriple());

561 if (TT.supportsCOMDAT()) {

563 MemprofHistogramFlag->setComdat(M.getOrInsertComdat(VarName));

564 }

566}

567

574 "__memprof_default_options_str");

575 Triple TT(M.getTargetTriple());

576 if (TT.supportsCOMDAT()) {

578 OptionsVar->setComdat(M.getOrInsertComdat(OptionsVar->getName()));

579 }

580}

581

582bool ModuleMemProfiler::instrumentModule(Module &M) {

583

584

586 std::string VersionCheckName =

588 : "";

589 std::tie(MemProfCtorFunction, std::ignore) =

592 {}, VersionCheckName);

593

594 const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);

596

598

600

602

603 return true;

604}

605

606void MemProfiler::initializeCallbacks(Module &M) {

608

609 for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {

610 const std::string TypeStr = AccessIsWrite ? "store" : "load";

611 const std::string HistPrefix = ClHistogram ? "hist_" : "";

612

614 MemProfMemoryAccessCallback[AccessIsWrite] = M.getOrInsertFunction(

617 }

618 MemProfMemmove = M.getOrInsertFunction(

621 PtrTy, PtrTy, PtrTy, IntptrTy);

622 MemProfMemset =

625}

626

627bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {

628

629

630

631

632

633

634

635 if (F.getName().contains(" load]")) {

639 IRB.CreateCall(MemProfInitFunction, {});

640 return true;

641 }

642 return false;

643}

644

645bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {

647 Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(

650 cast(GlobalDynamicAddress)->setDSOLocal(true);

651 DynamicShadowOffset = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);

652 return true;

653}

654

655bool MemProfiler::instrumentFunction(Function &F) {

657 return false;

659 return false;

660 if (F.getName().starts_with("__memprof_"))

661 return false;

662

663 bool FunctionModified = false;

664

665

666

667

668 if (maybeInsertMemProfInitAtFunctionEntry(F))

669 FunctionModified = true;

670

671 LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");

672

673 initializeCallbacks(*F.getParent());

674

676

677

678 for (auto &BB : F) {

679 for (auto &Inst : BB) {

680 if (isInterestingMemoryAccess(&Inst) || isa(Inst))

682 }

683 }

684

685 if (ToInstrument.empty()) {

686 LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified

687 << " " << F << "\n");

688

689 return FunctionModified;

690 }

691

692 FunctionModified |= insertDynamicShadowAtFunctionEntry(F);

693

694 int NumInstrumented = 0;

695 for (auto *Inst : ToInstrument) {

698 std::optional Access =

699 isInterestingMemoryAccess(Inst);

701 instrumentMop(Inst, F.getDataLayout(), *Access);

702 else

703 instrumentMemIntrinsic(cast(Inst));

704 }

705 NumInstrumented++;

706 }

707

708 if (NumInstrumented > 0)

709 FunctionModified = true;

710

711 LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "

712 << F << "\n");

713

714 return FunctionModified;

715}

716

720 I.setMetadata(LLVMContext::MD_callsite,

722}

723

731 std::memcpy(&Id, Hash.data(), sizeof(Hash));

732 return Id;

733}

734

737}

738

739

740

748 std::memcpy(&Id, Hash.data(), sizeof(Hash));

749 return Id;

750}

751

756 for (const auto &StackFrame : AllocInfo->CallStack)

760 AllocInfo->Info.getTotalLifetime());

761 std::vector ContextSizeInfo;

763 auto TotalSize = AllocInfo->Info.getTotalSize();

765 assert(FullStackId != 0);

766 ContextSizeInfo.push_back({FullStackId, TotalSize});

767 }

770}

771

772

773

774

775

776static bool

779 auto StackFrame = ProfileCallStack.begin();

780 auto InlCallStackIter = InlinedCallStack.begin();

781 for (; StackFrame != ProfileCallStack.end() &&

782 InlCallStackIter != InlinedCallStack.end();

783 ++StackFrame, ++InlCallStackIter) {

785 if (StackId != *InlCallStackIter)

786 return false;

787 }

788

789

790 return InlCallStackIter == InlinedCallStack.end();

791}

792

795 if (!Callee)

796 return false;

799 return false;

800 switch (Func) {

801 case LibFunc_Znwm:

802 case LibFunc_ZnwmRKSt9nothrow_t:

803 case LibFunc_ZnwmSt11align_val_t:

804 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t:

805 case LibFunc_Znam:

806 case LibFunc_ZnamRKSt9nothrow_t:

807 case LibFunc_ZnamSt11align_val_t:

808 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t:

809 case LibFunc_size_returning_new:

810 case LibFunc_size_returning_new_aligned:

811 return true;

812 case LibFunc_Znwm12__hot_cold_t:

813 case LibFunc_ZnwmRKSt9nothrow_t12__hot_cold_t:

814 case LibFunc_ZnwmSt11align_val_t12__hot_cold_t:

815 case LibFunc_ZnwmSt11align_val_tRKSt9nothrow_t12__hot_cold_t:

816 case LibFunc_Znam12__hot_cold_t:

817 case LibFunc_ZnamRKSt9nothrow_t12__hot_cold_t:

818 case LibFunc_ZnamSt11align_val_t12__hot_cold_t:

819 case LibFunc_ZnamSt11align_val_tRKSt9nothrow_t12__hot_cold_t:

820 case LibFunc_size_returning_new_hot_cold:

821 case LibFunc_size_returning_new_aligned_hot_cold:

823 default:

824 return false;

825 }

826}

827

831 bool Matched = false;

832};

833

838

839 auto GetOffset = [](const DILocation *DIL) {

840 return (DIL->getLine() - DIL->getScope()->getSubprogram()->getLine()) &

841 0xffff;

842 };

843

845 if (F.isDeclaration())

846 continue;

847

848 for (auto &BB : F) {

849 for (auto &I : BB) {

850 if (!isa(&I) || isa(&I))

851 continue;

852

853 auto *CB = dyn_cast(&I);

854 auto *CalledFunction = CB->getCalledFunction();

855

856 if (!CalledFunction || CalledFunction->isIntrinsic())

857 continue;

858

859 StringRef CalleeName = CalledFunction->getName();

860

861

863

864

865 bool IsLeaf = true;

866 for (const DILocation *DIL = I.getDebugLoc(); DIL;

867 DIL = DIL->getInlinedAt()) {

868 StringRef CallerName = DIL->getSubprogramLinkageName();

870 "Be sure to enable -fdebug-info-for-profiling");

873

874

875 if (IsAlloc) {

876 if (IsLeaf) {

877

878

879 CalleeGUID = 0;

880 } else if (!IsPresentInProfile(CalleeGUID)) {

881

882

883 CalleeGUID = 0;

884 } else {

885

886

887 IsAlloc = false;

888 }

889 }

890

891 LineLocation Loc = {GetOffset(DIL), DIL->getColumn()};

892 Calls[CallerGUID].emplace_back(Loc, CalleeGUID);

893 CalleeName = CallerName;

894 IsLeaf = false;

895 }

896 }

897 }

898 }

899

900

901 for (auto &[CallerGUID, CallList] : Calls) {

903 CallList.erase(llvm::unique(CallList), CallList.end());

904 }

905

906 return Calls;

907}

908

913

918 return CallsFromProfile.contains(GUID);

919 });

920

921

922 for (const auto &[CallerGUID, IRAnchors] : CallsFromIR) {

923 auto It = CallsFromProfile.find(CallerGUID);

924 if (It == CallsFromProfile.end())

925 continue;

926 const auto &ProfileAnchors = It->second;

927

929 longestCommonSequence<LineLocation, GlobalValue::GUID>(

930 ProfileAnchors, IRAnchors, std::equal_toGlobalValue::GUID(),

932 bool Inserted = UndriftMaps.try_emplace(CallerGUID, Matchings).second;

933

934

936 (void)Inserted;

937 }

938

939 return UndriftMaps;

940}

941

942

943

944static void

947

948 auto UndriftCallStack = [&](std::vector &CallStack) {

950 auto I = UndriftMaps.find(F.Function);

951 if (I == UndriftMaps.end())

952 continue;

953 auto J = I->second.find(LineLocation(F.LineOffset, F.Column));

954 if (J == I->second.end())

955 continue;

956 auto &NewLoc = J->second;

957 F.LineOffset = NewLoc.LineOffset;

958 F.Column = NewLoc.Column;

959 }

960 };

961

962 for (auto &AS : MemProfRec.AllocSites)

963 UndriftCallStack(AS.CallStack);

964

965 for (auto &CS : MemProfRec.CallSites)

966 UndriftCallStack(CS);

967}

968

969static void

972 std::map<uint64_t, AllocMatchInfo> &FullStackIdToAllocMatchInfo,

974 auto &Ctx = M.getContext();

975

976

977

978

979

980

981

982 auto FuncName = F.getName();

984 std::optionalmemprof::MemProfRecord MemProfRec;

985 auto Err = MemProfReader->getMemProfRecord(FuncGUID).moveInto(MemProfRec);

986 if (Err) {

988 auto Err = IPE.get();

989 bool SkipWarning = false;

990 LLVM_DEBUG(dbgs() << "Error in reading profile for Func " << FuncName

991 << ": ");

993 NumOfMemProfMissing++;

997 NumOfMemProfMismatch++;

998 SkipWarning =

1001 (F.hasComdat() ||

1003 LLVM_DEBUG(dbgs() << "hash mismatch (skip=" << SkipWarning << ")");

1004 }

1005

1006 if (SkipWarning)

1007 return;

1008

1009 std::string Msg = (IPE.message() + Twine(" ") + F.getName().str() +

1010 Twine(" Hash = ") + std::to_string(FuncGUID))

1011 .str();

1012

1013 Ctx.diagnose(

1015 });

1016 return;

1017 }

1018

1019 NumOfMemProfFunc++;

1020

1021

1022

1025

1026

1027

1028

1029

1030 bool ProfileHasColumns = false;

1031

1032

1033

1034 std::map<uint64_t, std::set<const AllocationInfo *>> LocHashToAllocInfo;

1035

1036 struct CallStackHash {

1039 }

1040 };

1041

1042

1043 std::map<uint64_t, std::unordered_set<ArrayRef, CallStackHash>>

1044 LocHashToCallSites;

1045 for (auto &AI : MemProfRec->AllocSites) {

1046 NumOfMemProfAllocContextProfiles++;

1047

1048

1049

1051 LocHashToAllocInfo[StackId].insert(&AI);

1052 ProfileHasColumns |= AI.CallStack[0].Column;

1053 }

1054 for (auto &CS : MemProfRec->CallSites) {

1055 NumOfMemProfCallSiteProfiles++;

1056

1057

1058 unsigned Idx = 0;

1059 for (auto &StackFrame : CS) {

1061 LocHashToCallSites[StackId].insert(ArrayRef(CS).drop_front(Idx++));

1062 ProfileHasColumns |= StackFrame.Column;

1063

1064 if (StackFrame.Function == FuncGUID)

1065 break;

1066 }

1067 assert(Idx <= CS.size() && CS[Idx - 1].Function == FuncGUID);

1068 }

1069

1070 auto GetOffset = [](const DILocation *DIL) {

1071 return (DIL->getLine() - DIL->getScope()->getSubprogram()->getLine()) &

1072 0xffff;

1073 };

1074

1075

1076

1077 for (auto &BB : F) {

1078 for (auto &I : BB) {

1079 if (I.isDebugOrPseudoInst())

1080 continue;

1081

1082

1083 auto *CI = dyn_cast(&I);

1084 if (!CI)

1085 continue;

1086 auto *CalledFunction = CI->getCalledFunction();

1087 if (CalledFunction && CalledFunction->isIntrinsic())

1088 continue;

1089

1090

1092

1093 bool LeafFound = false;

1094

1095

1096

1097

1098

1099 std::map<uint64_t, std::set<const AllocationInfo *>>::iterator

1100 AllocInfoIter;

1101 decltype(LocHashToCallSites)::iterator CallSitesIter;

1102 for (const DILocation *DIL = I.getDebugLoc(); DIL != nullptr;

1103 DIL = DIL->getInlinedAt()) {

1104

1105

1106 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();

1107 if (Name.empty())

1108 Name = DIL->getScope()->getSubprogram()->getName();

1110 auto StackId = computeStackId(CalleeGUID, GetOffset(DIL),

1111 ProfileHasColumns ? DIL->getColumn() : 0);

1112

1113

1114

1115

1116 if (!LeafFound) {

1117 AllocInfoIter = LocHashToAllocInfo.find(StackId);

1118 CallSitesIter = LocHashToCallSites.find(StackId);

1119 if (AllocInfoIter != LocHashToAllocInfo.end() ||

1120 CallSitesIter != LocHashToCallSites.end())

1121 LeafFound = true;

1122 }

1123 if (LeafFound)

1124 InlinedCallStack.push_back(StackId);

1125 }

1126

1127 if (!LeafFound)

1128 continue;

1129

1130

1131

1132

1133

1134 if (AllocInfoIter != LocHashToAllocInfo.end()) {

1135

1137 continue;

1138

1139

1140

1144 for (auto *AllocInfo : AllocInfoIter->second) {

1145

1146

1147

1149 InlinedCallStack)) {

1150 NumOfMemProfMatchedAllocContexts++;

1156 TotalSize += AllocInfo->Info.getTotalSize();

1158 TotalColdSize += AllocInfo->Info.getTotalSize();

1159

1160

1162 assert(FullStackId != 0);

1163 FullStackIdToAllocMatchInfo[FullStackId] = {

1165 }

1166 }

1167 }

1168

1169

1170

1174 "dominant");

1175 continue;

1176 }

1177

1178

1179

1180

1181 if (!AllocTrie.empty()) {

1182 NumOfMemProfMatchedAllocs++;

1183

1184

1186 assert(MemprofMDAttached == I.hasMetadata(LLVMContext::MD_memprof));

1187 if (MemprofMDAttached) {

1188

1189

1190

1191

1192

1193

1194

1196 }

1197 }

1198 continue;

1199 }

1200

1201

1202

1203

1204 assert(CallSitesIter != LocHashToCallSites.end());

1205 for (auto CallStackIdx : CallSitesIter->second) {

1206

1207

1209 InlinedCallStack)) {

1210 NumOfMemProfMatchedCallSites++;

1212

1213

1214 break;

1215 }

1216 }

1217 }

1218 }

1219}

1220

1223 : MemoryProfileFileName(MemoryProfileFile), FS(FS) {

1224 if (!FS)

1226}

1227

1229

1230 if (M.empty())

1232

1234 auto &Ctx = M.getContext();

1236 if (Error E = ReaderOrErr.takeError()) {

1238 Ctx.diagnose(

1240 });

1242 }

1243

1244 std::unique_ptr MemProfReader =

1245 std::move(ReaderOrErr.get());

1248 MemoryProfileFileName.data(), StringRef("Cannot get MemProfReader")));

1250 }

1251

1254 "Not a memory profile"));

1256 }

1257

1259

1264

1265

1266

1267

1268 std::map<uint64_t, AllocMatchInfo> FullStackIdToAllocMatchInfo;

1269

1270 for (auto &F : M) {

1271 if (F.isDeclaration())

1272 continue;

1273

1276 UndriftMaps);

1277 }

1278

1280 for (const auto &[Id, Info] : FullStackIdToAllocMatchInfo)

1282 << " context with id " << Id << " has total profiled size "

1283 << Info.TotalSize << (Info.Matched ? " is" : " not")

1284 << " matched\n";

1285 }

1286

1288}

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

static cl::opt< int > ClDebugMin("asan-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1))

static cl::opt< std::string > ClMemoryAccessCallbackPrefix("asan-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__asan_"))

static cl::opt< bool > ClInsertVersionCheck("asan-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true))

static cl::opt< bool > ClInstrumentWrites("asan-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true))

static cl::opt< int > ClDebugMax("asan-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1))

static cl::opt< bool > ClStack("asan-stack", cl::desc("Handle stack memory"), cl::Hidden, cl::init(true))

static cl::opt< bool > ClInstrumentAtomics("asan-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true))

static cl::opt< int > ClMappingScale("asan-mapping-scale", cl::desc("scale of asan shadow mapping"), cl::Hidden, cl::init(0))

static cl::opt< std::string > ClDebugFunc("asan-debug-func", cl::Hidden, cl::desc("Debug func"))

static cl::opt< bool > ClInstrumentReads("asan-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true))

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

Analysis containing CSE Info

Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx

Module.h This file contains the declarations for the Module class.

static cl::opt< int > ClMappingGranularity("memprof-mapping-granularity", cl::desc("granularity of memprof shadow mapping"), cl::Hidden, cl::init(DefaultMemGranularity))

constexpr char MemProfVersionCheckNamePrefix[]

static cl::opt< int > ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1))

void createMemprofHistogramFlagVar(Module &M)

constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority

static uint64_t computeFullStackId(ArrayRef< Frame > CallStack)

static cl::opt< std::string > MemprofRuntimeDefaultOptions("memprof-runtime-default-options", cl::desc("The default memprof options"), cl::Hidden, cl::init(""))

static cl::opt< std::string > ClDebugFunc("memprof-debug-func", cl::Hidden, cl::desc("Debug func"))

constexpr char MemProfShadowMemoryDynamicAddress[]

static void addCallsiteMetadata(Instruction &I, ArrayRef< uint64_t > InlinedCallStack, LLVMContext &Ctx)

static bool isAllocationWithHotColdVariant(const Function *Callee, const TargetLibraryInfo &TLI)

constexpr uint64_t MemProfCtorAndDtorPriority

constexpr int LLVM_MEM_PROFILER_VERSION

static cl::opt< bool > ClUseCalls("memprof-use-callbacks", cl::desc("Use callbacks instead of inline instrumentation sequences."), cl::Hidden, cl::init(false))

static cl::opt< bool > ClInstrumentAtomics("memprof-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true))

static cl::opt< bool > ClInsertVersionCheck("memprof-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true))

constexpr char MemProfInitName[]

constexpr char MemProfFilenameVar[]

static void undriftMemProfRecord(const DenseMap< uint64_t, LocToLocMap > &UndriftMaps, memprof::MemProfRecord &MemProfRec)

static uint64_t computeStackId(GlobalValue::GUID Function, uint32_t LineOffset, uint32_t Column)

static cl::opt< bool > ClStack("memprof-instrument-stack", cl::desc("Instrument scalar stack variables"), cl::Hidden, cl::init(false))

static cl::opt< bool > ClHistogram("memprof-histogram", cl::desc("Collect access count histograms"), cl::Hidden, cl::init(false))

constexpr uint64_t DefaultMemGranularity

void createMemprofDefaultOptionsVar(Module &M)

static cl::opt< bool > ClPrintMemProfMatchInfo("memprof-print-match-info", cl::desc("Print matching stats for each allocation " "context in this module's profiles"), cl::Hidden, cl::init(false))

constexpr uint64_t HistogramGranularity

constexpr uint64_t DefaultShadowScale

cl::opt< bool > MemProfReportHintedSizes

static cl::opt< std::string > ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__memprof_"))

constexpr char MemProfModuleCtorName[]

static cl::opt< bool > SalvageStaleProfile("memprof-salvage-stale-profile", cl::desc("Salvage stale MemProf profile"), cl::init(false), cl::Hidden)

static cl::opt< unsigned > MinMatchedColdBytePercent("memprof-matching-cold-threshold", cl::init(100), cl::Hidden, cl::desc("Min percent of cold bytes matched to hint allocation cold"))

static cl::opt< bool > ClInstrumentReads("memprof-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true))

static cl::opt< int > ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1))

static cl::opt< bool > ClInstrumentWrites("memprof-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true))

static cl::opt< int > ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden, cl::init(0))

static cl::opt< int > ClMappingScale("memprof-mapping-scale", cl::desc("scale of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowScale))

static void readMemprof(Module &M, Function &F, IndexedInstrProfReader *MemProfReader, const TargetLibraryInfo &TLI, std::map< uint64_t, AllocMatchInfo > &FullStackIdToAllocMatchInfo, DenseMap< uint64_t, LocToLocMap > &UndriftMaps)

static cl::opt< bool > ClMemProfMatchHotColdNew("memprof-match-hot-cold-new", cl::desc("Match allocation profiles onto existing hot/cold operator new calls"), cl::Hidden, cl::init(false))

static AllocationType addCallStack(CallStackTrie &AllocTrie, const AllocationInfo *AllocInfo, uint64_t FullStackId)

constexpr char MemProfHistogramFlagVar[]

cl::opt< unsigned > MinClonedColdBytePercent("memprof-cloning-cold-threshold", cl::init(100), cl::Hidden, cl::desc("Min percent of cold bytes to hint alloc cold during cloning"))

static bool stackFrameIncludesInlinedCallStack(ArrayRef< Frame > ProfileCallStack, ArrayRef< uint64_t > InlinedCallStack)

cl::opt< bool > MemProfReportHintedSizes("memprof-report-hinted-sizes", cl::init(false), cl::Hidden, cl::desc("Report total allocation sizes of hinted allocations"))

FunctionAnalysisManager FAM

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

This file defines the SmallVector class.

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

Defines the virtual file system interface vfs::FileSystem.

Class for arbitrary precision integers.

A container for analyses that lazily runs them and caches their results.

PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)

Get the result of an analysis pass for a given IR unit.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

An instruction that atomically checks whether a specified value is in a memory location,...

an instruction that atomically reads a memory location, combines it with another value,...

static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)

This method constructs a CDS and initializes it with a text string.

This is an important base class in LLVM.

static Constant * getIntegerValue(Type *Ty, const APInt &V)

Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...

A parsed version of the target data layout string in and methods for querying it.

iterator find(const_arg_type_t< KeyT > Val)

std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)

bool contains(const_arg_type_t< KeyT > Val) const

Return true if the specified key is in the map, false otherwise.

Diagnostic information for the PGO profiler.

Base class for error info classes.

virtual std::string message() const

Return the error message as a string.

Lightweight error class with error context and mandatory checking.

A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...

static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)

This static method is the primary way of constructing a FunctionType.

void setComdat(Comdat *C)

void setLinkage(LinkageTypes LT)

GUID getGUID() const

Return a 64-bit global unique ID constructed from global value name (i.e.

@ ExternalLinkage

Externally visible function.

@ WeakAnyLinkage

Keep one copy of named function when linking (weak)

@ AvailableExternallyLinkage

Available for inspection, not emission.

HashResultTy< HasherT_ > final()

Forward to HasherT::final() if available.

Interface to help hash various types through a hasher type.

std::enable_if_t< hashbuilder_detail::IsHashableData< T >::value, HashBuilder & > add(T Value)

Implement hashing for hashable data types, e.g. integral or enum values.

Value * CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")

Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")

Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")

Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)

IntegerType * getInt32Ty()

Fetch the type representing a 32-bit integer.

Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())

LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)

Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...

Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")

StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)

Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)

Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")

void SetInsertPoint(BasicBlock *TheBB)

This specifies that created instructions should be appended to the end of the specified block.

Type * getVoidTy()

Fetch the type representing void.

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

Reader for the indexed binary instrprof format.

static Expected< std::unique_ptr< IndexedInstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const Twine &RemappingPath="")

Factory method to create an indexed reader.

An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...

instrprof_error get() const

std::string message() const override

Return the error message as a string.

A smart pointer to a reference-counted object that inherits from RefCountedBase or ThreadSafeRefCount...

This is an important class for using LLVM in a threaded context.

An instruction for reading from memory.

StringRef getString() const

This is the common base class for memset/memcpy/memmove.

PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)

MemProfUsePass(std::string MemoryProfileFile, IntrusiveRefCntPtr< vfs::FileSystem > FS=nullptr)

PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)

PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)

A Module instance is used to store all the information related to an LLVM module.

static PointerType * get(Type *ElementType, unsigned AddressSpace)

This constructs a pointer to an object of the specified type in a numbered address space.

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

An instruction for storing to memory.

StringRef - Represent a constant reference to a string, i.e.

constexpr bool empty() const

empty - Check if the string is empty.

Analysis pass providing the TargetLibraryInfo.

Provides information about what library functions are available for the current target.

bool getLibFunc(StringRef funcName, LibFunc &F) const

Searches for a particular function name.

Triple - Helper class for working with autoconf configuration names.

ObjectFormatType getObjectFormat() const

Get the object format for this triple.

bool isOSEmscripten() const

Tests whether the OS is Emscripten.

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

The instances of the Type class are immutable: once they are created, they are never changed.

static IntegerType * getInt1Ty(LLVMContext &C)

unsigned getPointerAddressSpace() const

Get the address space of this pointer or pointer vector type.

static IntegerType * getIntNTy(LLVMContext &C, unsigned N)

static IntegerType * getInt8Ty(LLVMContext &C)

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

StringRef getName() const

Return a constant reference to the value's name.

An efficient, type-erasing, non-owning reference to a callable.

Class to build a trie of call stack contexts for a particular profiled allocation call,...

void addCallStack(AllocationType AllocType, ArrayRef< uint64_t > StackIds, std::vector< ContextTotalSize > ContextSizeInfo={})

Add a call stack context with the given allocation type to the Trie.

void addSingleAllocTypeAttribute(CallBase *CI, AllocationType AT, StringRef Descriptor)

Add an attribute for the given allocation type to the call instruction.

bool buildAndAttachMIBMetadata(CallBase *CI)

Build and attach the minimal necessary MIB metadata.

Helper class to iterate through stack ids in both metadata (memprof MIB and callsite) and the corresp...

void instrumentAddress(Module &M, IRBuilder<> &IRB, Instruction *OrigIns, Instruction *InsertBefore, Value *Addr, Align Alignment, TypeSize TypeStoreSize, bool IsWrite, Value *SizeArgument, bool UseCalls, bool Recover, int AsanScale, int AsanOffset)

Instrument the memory operand Addr.

constexpr std::underlying_type_t< E > Mask()

Get a bitmask with 1s in all places up to the high-order bit of E's largest value.

@ C

The default llvm calling convention, compatible with C.

initializer< Ty > init(const Ty &Val)

DenseMap< uint64_t, LocToLocMap > computeUndriftMap(Module &M, IndexedInstrProfReader *MemProfReader, const TargetLibraryInfo &TLI)

MDNode * buildCallstackMetadata(ArrayRef< uint64_t > CallStack, LLVMContext &Ctx)

Build callstack metadata from the provided list of call stack ids.

AllocationType getAllocType(uint64_t TotalLifetimeAccessDensity, uint64_t AllocCount, uint64_t TotalLifetime)

Return the allocation type for a given set of memory profile values.

DenseMap< uint64_t, SmallVector< CallEdgeTy, 0 > > extractCallsFromIR(Module &M, const TargetLibraryInfo &TLI, function_ref< bool(uint64_t)> IsPresentInProfile=[](uint64_t) { return true;})

std::unordered_map< LineLocation, LineLocation, LineLocationHash > LocToLocMap

std::string getAllocTypeAttributeString(AllocationType Type)

Returns the string to use in attributes with the given type.

IntrusiveRefCntPtr< FileSystem > getRealFileSystem()

Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.

This is an optimization pass for GlobalISel generic memory operations.

void handleAllErrors(Error E, HandlerTs &&... Handlers)

Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...

const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)

This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....

cl::opt< bool > PGOWarnMissing

auto unique(Range &&R, Predicate P)

std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)

Return the name of the profile section corresponding to IPSK.

std::array< uint8_t, NumBytes > BLAKE3Result

The constant LLVM_BLAKE3_OUT_LEN provides the default output length, 32 bytes, which is recommended f...

FunctionCallee declareSanitizerInitFunction(Module &M, StringRef InitName, ArrayRef< Type * > InitArgTypes, bool Weak=false)

std::pair< Function *, FunctionCallee > createSanitizerCtorAndInitFunctions(Module &M, StringRef CtorName, StringRef InitName, ArrayRef< Type * > InitArgTypes, ArrayRef< Value * > InitArgs, StringRef VersionCheckName=StringRef(), bool Weak=false)

Creates sanitizer constructor function, and calls sanitizer's init function from it.

void sort(IteratorTy Start, IteratorTy End)

raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

cl::opt< bool > NoPGOWarnMismatch

raw_fd_ostream & errs()

This returns a reference to a raw_ostream for standard error.

void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)

Adds global values to the llvm.compiler.used list.

void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)

void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)

Append F to the list of global ctors of module M with the given Priority.

Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)

Split the containing block at the specified instruction - everything before SplitBefore stays in the ...

cl::opt< bool > NoPGOWarnMismatchComdatWeak

Summary of memprof metadata on allocations.

GlobalValue::GUID Function

static GlobalValue::GUID getGUID(const StringRef FunctionName)

llvm::SmallVector< std::vector< Frame > > CallSites

llvm::SmallVector< AllocationInfo > AllocSites