SampleProf.cpp Source File (original) (raw)

15#include "llvm/Config/llvm-config.h"

25#include

26#include

27#include

28#include <system_error>

30using namespace llvm;

35 cl::desc("Cutoff value about how many symbols in profile symbol list "

36 "will be used. This is very useful for performance debugging"));

39 "generate-merged-base-profiles",

40 cl::desc("When generating nested context-sensitive profiles, always "

41 "generate extra base profile for function with all its context "

42 "profiles merged into it."));

44namespace llvm {

53std::error_code

58 for (const auto &[TypeName, SampleCount] : Map) {

59 if (auto NameIndexIter = NameTable.find(TypeName);

60 NameIndexIter != NameTable.end()) {

62 } else {

65 }

67 }

69}

70}

71}

73namespace {

78class SampleProfErrorCategoryType : public std::error_category {

79 const char *name() const noexcept override { return "llvm.sampleprof"; }

81 std::string message(int IE) const override {

83 switch (E) {

84 case sampleprof_error::success:

85 return "Success";

86 case sampleprof_error::bad_magic:

87 return "Invalid sample profile data (bad magic)";

88 case sampleprof_error::unsupported_version:

89 return "Unsupported sample profile format version";

90 case sampleprof_error::too_large:

91 return "Too much profile data";

92 case sampleprof_error::truncated:

93 return "Truncated profile data";

94 case sampleprof_error::malformed:

95 return "Malformed sample profile data";

96 case sampleprof_error::unrecognized_format:

97 return "Unrecognized sample profile encoding format";

98 case sampleprof_error::unsupported_writing_format:

99 return "Profile encoding format unsupported for writing operations";

100 case sampleprof_error::truncated_name_table:

101 return "Truncated function name table";

102 case sampleprof_error::not_implemented:

103 return "Unimplemented feature";

104 case sampleprof_error::counter_overflow:

105 return "Counter overflow";

106 case sampleprof_error::ostream_seek_unsupported:

107 return "Ostream does not support seek";

108 case sampleprof_error::uncompress_failed:

109 return "Uncompress failure";

110 case sampleprof_error::zlib_unavailable:

111 return "Zlib is unavailable";

112 case sampleprof_error::hash_mismatch:

113 return "Function hash mismatch";

114 case sampleprof_error::illegal_line_offset:

115 return "Illegal line offset in sample profile data";

116 }

117 llvm_unreachable("A value of sampleprof_error has no message.");

118 }

119};

120

121}

122

124 static SampleProfErrorCategoryType ErrorCategory;

125 return ErrorCategory;

126}

127

133

136 Loc.print(OS);

137 return OS;

138}

139

140

141

146 for (const auto &I : Other.getCallTargets()) {

148 }

149 return Result;

150}

151

158 uint64_t CalleeSamples = J.second;

159 if (auto NameIndexIter = NameTable.find(Callee);

160 NameIndexIter != NameTable.end()) {

162 } else {

163

165 }

167 }

169}

170

171#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

173#endif

174

179

180

182 OS << NumSamples;

184 OS << ", calls:";

186 OS << " " << I.first << ":" << I.second;

187 }

188 OS << "\n";

189}

190

191#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

193#endif

194

197 Sample.print(OS, 0);

198 return OS;

199}

200

204 return;

205 }

206 OS << Loc << ": vtables: ";

208 OS << Type << ":" << Count << " ";

209 OS << "\n";

210}

211

212

216

217 OS << TotalSamples << ", " << TotalHeadSamples << ", " << BodySamples.size()

218 << " sampled lines\n";

219

221 if (!BodySamples.empty()) {

222 OS << "Samples collected in the function's body {\n";

224 for (const auto &SI : SortedBodySamples.get()) {

225 OS.indent(Indent + 2);

226 const auto &Loc = SI->first;

227 OS << SI->first << ": " << SI->second;

230 OS.indent(Indent + 2);

232 }

233 }

235 OS << "}\n";

236 } else {

237 OS << "No samples collected in the function's body\n";

238 }

239

241 if (!CallsiteSamples.empty()) {

242 OS << "Samples collected in inlined callsites {\n";

244 CallsiteSamples);

245 for (const auto *Element : SortedCallsiteSamples.get()) {

246

247 const auto &[Loc, FunctionSampleMap] = *Element;

250 OS.indent(Indent + 2);

251 OS << Loc << ": inlined callee: " << FuncSample.getFunction() << ": ";

252 FuncSample.print(OS, Indent + 4);

253 }

254 auto TypeSamplesIter = VirtualCallsiteTypeCounts.find(Loc);

255 if (TypeSamplesIter != VirtualCallsiteTypeCounts.end()) {

256 OS.indent(Indent + 2);

258 }

259 }

261 OS << "}\n";

262 } else {

263 OS << "No inlined callsites in this function\n";

264 }

265}

266

269 FS.print(OS);

270 return OS;

271}

272

275 std::vector &SortedProfiles) {

276 for (const auto &I : ProfileMap) {

277 SortedProfiles.push_back(std::make_pair(I.first, &I.second));

278 }

281 if (A.second->getTotalSamples() == B.second->getTotalSamples())

282 return A.second->getContext() < B.second->getContext();

283 return A.second->getTotalSamples() > B.second->getTotalSamples();

284 });

285}

286

288 return (DIL->getLine() - DIL->getScope()->getSubprogram()->getLine()) &

289 0xffff;

290}

291

295

296

297

298

300 DIL->getDiscriminator()),

301 0);

302 } else {

303 unsigned Discriminator =

306 }

307}

308

312 *FuncNameToProfNameMap) const {

315

317 for (DIL = DIL->getInlinedAt(); DIL; DIL = DIL->getInlinedAt()) {

318

319 StringRef Name = PrevDIL->getScope()->getSubprogram()->getLinkageName();

320 if (Name.empty())

321 Name = PrevDIL->getScope()->getSubprogram()->getName();

324 Name);

325 PrevDIL = DIL;

326 }

327

328 if (S.size() == 0)

329 return this;

331 for (int i = S.size() - 1; i >= 0 && FS != nullptr; i--) {

332 FS = FS->findFunctionSamplesAt(S[i].first, S[i].second, Remapper,

333 FuncNameToProfNameMap);

334 }

335 return FS;

336}

337

340 for (const auto &BS : BodySamples)

342

343 for (const auto &CS : CallsiteSamples) {

344 for (const auto &NameFS : CS.second) {

345 NameSet.insert(NameFS.first);

346 NameFS.second.findAllNames(NameSet);

347 }

348 }

349}

350

355 *FuncNameToProfNameMap) const {

357

359 if (I == CallsiteSamples.end())

360 return nullptr;

362 if (FS != I->second.end())

363 return &FS->second;

364

365 if (FuncNameToProfNameMap && !FuncNameToProfNameMap->empty()) {

366 auto R = FuncNameToProfNameMap->find(FunctionId(CalleeName));

367 if (R != FuncNameToProfNameMap->end()) {

368 CalleeName = R->second.stringRef();

370 if (FS != I->second.end())

371 return &FS->second;

372 }

373 }

374

375 if (Remapper) {

377 auto FS = I->second.find(getRepInFormat(*NameInProfile));

378 if (FS != I->second.end())

379 return &FS->second;

380 }

381 }

382

383

384

385 if (!CalleeName.empty())

386 return nullptr;

387 uint64_t MaxTotalSamples = 0;

389 for (const auto &NameFS : I->second)

390 if (NameFS.second.getTotalSamples() >= MaxTotalSamples) {

391 MaxTotalSamples = NameFS.second.getTotalSamples();

392 R = &NameFS.second;

393 }

394 return R;

395}

396

397#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

399#endif

400

403

405 std::count(Data, Data + ListSize, 0)));

406

407 const char *ListStart = reinterpret_cast<const char *>(Data);

412 add(Str);

413 Size += Str.size() + 1;

414 StrNum++;

415 }

419}

420

423 uint32_t ColdContextFrameLength, bool TrimBaseProfileOnly) {

424 if (!TrimColdContext && !MergeColdContext)

425 return;

426

427

429 return;

430

431

432

433

434 if (MergeColdContext)

435 TrimBaseProfileOnly = false;

436

437

438

439 std::vector<std::pair<hash_code, const FunctionSamples *>> ColdProfiles;

440 for (const auto &I : ProfileMap) {

441 const SampleContext &Context = I.second.getContext();

444 (!TrimBaseProfileOnly || Context.isBaseContext()))

445 ColdProfiles.emplace_back(I.first, &I.second);

446 }

447

448

449

451 for (const auto &I : ColdProfiles) {

452 if (MergeColdContext) {

453 auto MergedContext = I.second->getContext().getContextFrames();

454 if (ColdContextFrameLength < MergedContext.size())

456

458 MergedProfile.merge(*I.second);

459 }

460 ProfileMap.erase(I.first);

461 }

462

463

464 for (const auto &I : MergedProfileMap) {

465

466 if (TrimColdContext && I.second.getTotalSamples() < ColdCountThreshold &&

467 ProfileMap.find(I.second.getContext()) == ProfileMap.end())

468 continue;

469

470

471

472 auto Ret = ProfileMap.emplace(I.second.getContext(), FunctionSamples());

474 OrigProfile.merge(I.second);

475 }

476}

477

479

480

481 std::vector SortedList(Syms.begin(), Syms.end());

483

484 std::string OutputString;

485 for (auto &Sym : SortedList) {

486 OutputString.append(Sym.str());

487 OutputString.append(1, '\0');

488 }

489

490 OS << OutputString;

492}

493

495 OS << "======== Dump profile symbol list ========\n";

496 std::vector SortedList(Syms.begin(), Syms.end());

498

499 for (auto &Sym : SortedList)

500 OS << Sym << "\n";

501}

502

509 assert(It->second.FuncName == CalleeName &&

510 "Hash collision for child context node");

511 return &It->second;

512 }

513

516}

517

519 : ProfileMap(Profiles) {

520 for (auto &FuncSample : Profiles) {

522 auto *NewNode = getOrCreateContextPath(FSamples->getContext());

523 assert(!NewNode->FuncSamples && "New node cannot have sample profile");

524 NewNode->FuncSamples = FSamples;

525 }

526}

527

529ProfileConverter::getOrCreateContextPath(const SampleContext &Context) {

530 auto Node = &RootFrame;

532 for (auto &Callsite : Context.getContextFrames()) {

533 Node = Node->getOrCreateChildFrame(CallSiteLoc, Callsite.Func);

534 CallSiteLoc = Callsite.Location;

535 }

537}

538

540

541

542

543 auto *NodeProfile = Node.FuncSamples;

544 for (auto &It : Node.AllChildFrames) {

545 auto &ChildNode = It.second;

547 auto *ChildProfile = ChildNode.FuncSamples;

548 if (!ChildProfile)

549 continue;

550 SampleContext OrigChildContext = ChildProfile->getContext();

552

553 ChildProfile->getContext().setFunction(OrigChildContext.getFunction());

554 if (NodeProfile) {

555

556 auto &SamplesMap = NodeProfile->functionSamplesAt(ChildNode.CallSiteLoc);

557 SamplesMap.emplace(OrigChildContext.getFunction(), *ChildProfile);

558 NodeProfile->addTotalSamples(ChildProfile->getTotalSamples());

559

560

561 auto Count = NodeProfile->removeCalledTargetAndBodySample(

562 ChildNode.CallSiteLoc.LineOffset, ChildNode.CallSiteLoc.Discriminator,

564 NodeProfile->removeTotalSamples(Count);

565 }

566

567 uint64_t NewChildProfileHash = 0;

568

569

570

571

572

573

574 if (!NodeProfile) {

575 ProfileMap[ChildProfile->getContext()].merge(*ChildProfile);

576 NewChildProfileHash = ChildProfile->getContext().getHashCode();

578 ProfileMap[ChildProfile->getContext()].merge(*ChildProfile);

579 NewChildProfileHash = ChildProfile->getContext().getHashCode();

580 auto &SamplesMap = NodeProfile->functionSamplesAt(ChildNode.CallSiteLoc);

581 SamplesMap[ChildProfile->getFunction()].getContext().setAttribute(

583 }

584

585

586

587

588 if (NewChildProfileHash != OrigChildContextHash)

589 ProfileMap.erase(OrigChildContextHash);

590 }

591}

592

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

#define LLVM_DUMP_METHOD

Mark debug helper function definitions like dump() that should not be stripped from debug builds.

static cl::opt< unsigned > ColdCountThreshold("mfs-count-threshold", cl::desc("Minimum number of times a block must be executed to be retained."), cl::init(1), cl::Hidden)

static void printTypeCountMap(raw_ostream &OS, LineLocation Loc, const TypeCountMap &TypeCountMap)

Definition SampleProf.cpp:201

static cl::opt< bool > GenerateMergedBaseProfiles("generate-merged-base-profiles", cl::desc("When generating nested context-sensitive profiles, always " "generate extra base profile for function with all its context " "profiles merged into it."))

static cl::opt< uint64_t > ProfileSymbolListCutOff("profile-symbol-list-cutoff", cl::Hidden, cl::init(-1), cl::desc("Cutoff value about how many symbols in profile symbol list " "will be used. This is very useful for performance debugging"))

unsigned getBaseDiscriminator() const

Returns the base discriminator stored in the discriminator.

Implements a dense probed hash-table based set.

This class implements a map that also provides access to all stored values in a deterministic order.

iterator find(const KeyT &Key)

reference emplace_back(ArgTypes &&... Args)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

constexpr bool empty() const

empty - Check if the string is empty.

The instances of the Type class are immutable: once they are created, they are never changed.

std::pair< iterator, bool > insert(const ValueT &V)

void insert_range(Range &&R)

This class implements an extremely fast bulk output stream that can only output to a stream.

raw_ostream & indent(unsigned NumSpaces)

indent - Insert 'NumSpaces' spaces.

This class represents a function that is read from a sample profile.

Representation of the samples collected for a function.

static LLVM_ABI bool ProfileIsPreInlined

LLVM_ABI const FunctionSamples * findFunctionSamplesAt(const LineLocation &Loc, StringRef CalleeName, SampleProfileReaderItaniumRemapper *Remapper, const HashKeyMap< std::unordered_map, FunctionId, FunctionId > *FuncNameToProfNameMap=nullptr) const

Returns a pointer to FunctionSamples at the given callsite location Loc with callee CalleeName.

Definition SampleProf.cpp:351

static uint64_t getCallSiteHash(FunctionId Callee, const LineLocation &Callsite)

Returns a unique hash code for a combination of a callsite location and the callee function name.

static LLVM_ABI bool ProfileIsCS

const LineLocation & mapIRLocToProfileLoc(const LineLocation &IRLoc) const

FunctionId getFunction() const

Return the function name.

uint64_t getFunctionHash() const

const TypeCountMap * findCallsiteTypeSamplesAt(const LineLocation &Loc) const

Returns the TypeCountMap for inlined callsites at the given Loc.

LLVM_ABI const FunctionSamples * findFunctionSamples(const DILocation *DIL, SampleProfileReaderItaniumRemapper *Remapper=nullptr, const HashKeyMap< std::unordered_map, FunctionId, FunctionId > *FuncNameToProfNameMap=nullptr) const

Get the FunctionSamples of the inline instance where DIL originates from.

Definition SampleProf.cpp:309

static LLVM_ABI bool ProfileIsProbeBased

static StringRef getCanonicalFnName(const Function &F)

Return the canonical name for a function, taking into account suffix elision policy attributes.

LLVM_ABI void findAllNames(DenseSet< FunctionId > &NameSet) const

Definition SampleProf.cpp:338

static LLVM_ABI unsigned getOffset(const DILocation *DIL)

Returns the line offset to the start line of the subprogram.

Definition SampleProf.cpp:287

static LLVM_ABI bool ProfileIsFS

If this profile uses flow sensitive discriminators.

SampleContext & getContext() const

static LLVM_ABI bool HasUniqSuffix

Whether the profile contains any ".__uniq." suffix in a name.

uint64_t getTotalSamples() const

Return the total number of samples collected inside the function.

LLVM_ABI void print(raw_ostream &OS=dbgs(), unsigned Indent=0) const

Print the samples collected for a function on stream OS.

Definition SampleProf.cpp:213

sampleprof_error merge(const FunctionSamples &Other, uint64_t Weight=1)

Merge the samples in Other into this one.

FunctionSamples()=default

LLVM_ABI void dump() const

Definition SampleProf.cpp:398

static LLVM_ABI LineLocation getCallSiteIdentifier(const DILocation *DIL, bool ProfileIsFS=false)

Returns a unique call site identifier for a given debug location of a call instruction.

Definition SampleProf.cpp:292

static LLVM_ABI bool UseMD5

Whether the profile uses MD5 to represent string.

This class is a wrapper to associative container MapT<KeyT, ValueT> using the hash value of the origi...

iterator find(const original_key_type &Key)

LLVM_ABI ProfileConverter(SampleProfileMap &Profiles)

Definition SampleProf.cpp:518

LLVM_ABI void convertCSProfiles()

Definition SampleProf.cpp:593

void add(StringRef Name, bool Copy=false)

copy indicates whether we need to copy the underlying memory for the input Name.

LLVM_ABI std::error_code write(raw_ostream &OS)

Definition SampleProf.cpp:478

void reserve(size_t Size)

LLVM_ABI void dump(raw_ostream &OS=dbgs()) const

Definition SampleProf.cpp:494

LLVM_ABI std::error_code read(const uint8_t *Data, uint64_t ListSize)

Definition SampleProf.cpp:401

LLVM_ABI void trimAndMergeColdContextProfiles(uint64_t ColdCountThreshold, bool TrimColdContext, bool MergeColdContext, uint32_t ColdContextFrameLength, bool TrimBaseProfileOnly)

Definition SampleProf.cpp:421

uint64_t getHashCode() const

FunctionId getFunction() const

This class provides operator overloads to the map container using MD5 as the key type,...

mapped_type & create(const SampleContext &Ctx)

SampleProfileReaderItaniumRemapper remaps the profile data from a sample profile data reader,...

LLVM_ABI std::optional< StringRef > lookUpNameInProfile(StringRef FunctionName)

Return the equivalent name in the profile for FunctionName if it exists.

Representation of a single sample record.

LLVM_ABI std::error_code serialize(raw_ostream &OS, const MapVector< FunctionId, uint32_t > &NameTable) const

Serialize the sample record to the output stream using ULEB128 encoding.

Definition SampleProf.cpp:152

LLVM_ABI void dump() const

Definition SampleProf.cpp:192

bool hasCalls() const

Return true if this sample record contains function calls.

LLVM_ABI sampleprof_error merge(const SampleRecord &Other, uint64_t Weight=1)

Merge the samples in Other into this record.

Definition SampleProf.cpp:142

const CallTargetMap & getCallTargets() const

uint64_t getSamples() const

sampleprof_error addSamples(uint64_t S, uint64_t Weight=1)

Increment the number of samples for this record by S.

const SortedCallTargetSet getSortedCallTargets() const

LLVM_ABI void print(raw_ostream &OS, unsigned Indent) const

Print the sample record to the stream OS indented by Indent.

Definition SampleProf.cpp:181

sampleprof_error addCalledTarget(FunctionId F, uint64_t S, uint64_t Weight=1)

Add called function F with samples S.

Sort a LocationT->SampleT map by LocationT.

const SamplesWithLocList & get() const

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

initializer< Ty > init(const Ty &Val)

static FunctionId getRepInFormat(StringRef Name)

Get the proper representation of a string according to whether the current Format uses MD5 to represe...

LLVM_ABI void sortFuncProfiles(const SampleProfileMap &ProfileMap, std::vector< NameFunctionSamples > &SortedProfiles)

Definition SampleProf.cpp:273

std::pair< hash_code, const FunctionSamples * > NameFunctionSamples

@ ContextDuplicatedIntoBase

raw_ostream & operator<<(raw_ostream &OS, const FunctionId &Obj)

std::map< FunctionId, uint64_t > TypeCountMap

Key represents type of a C++ polymorphic class type by its vtable and value represents its counter.

std::error_code serializeTypeMap(const TypeCountMap &Map, const MapVector< FunctionId, uint32_t > &NameTable, raw_ostream &OS)

Write Map to the output stream.

Definition SampleProf.cpp:54

This is an optimization pass for GlobalISel generic memory operations.

void stable_sort(R &&Range)

auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)

Get the size of a range.

sampleprof_error mergeSampleProfErrors(sampleprof_error &Accumulator, sampleprof_error Result)

void sort(IteratorTy Start, IteratorTy End)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

auto make_first_range(ContainerTy &&c)

Given a container of pairs, return a range over the first elements.

FunctionAddr VTableAddr Count

LLVM_ABI const std::error_category & sampleprof_category()

Definition SampleProf.cpp:123

FunctionAddr VTableAddr uintptr_t uintptr_t Data

auto make_second_range(ContainerTy &&c)