LLVM: lib/ProfileData/InstrProf.cpp Source File (original) (raw)

1

2

3

4

5

6

7

8

9

10

11

12

13

19#include "llvm/Config/config.h"

48#include

49#include

50#include

51#include

52#include

53#include

54#include

55#include <system_error>

56#include <type_traits>

57#include

58#include

59

60using namespace llvm;

61

62#define DEBUG_TYPE "instrprof"

63

66 cl::desc("Use full module build paths in the profile counter names for "

67 "static functions."));

68

69

70

71

72

73

74

75

76

79 cl::desc("Strip specified level of directory name from source path in "

80 "the profile counter name for static functions."));

81

83 const std::string &ErrMsg = "") {

84 std::string Msg;

86

87 switch (Err) {

89 OS << "success";

90 break;

92 OS << "end of File";

93 break;

95 OS << "unrecognized instrumentation profile encoding format";

96 break;

98 OS << "invalid instrumentation profile data (bad magic)";

99 break;

101 OS << "invalid instrumentation profile data (file header is corrupt)";

102 break;

104 OS << "unsupported instrumentation profile format version";

105 break;

107 OS << "unsupported instrumentation profile hash type";

108 break;

110 OS << "too much profile data";

111 break;

113 OS << "truncated profile data";

114 break;

116 OS << "malformed instrumentation profile data";

117 break;

119 OS << "debug info/binary for correlation is required";

120 break;

122 OS << "debug info/binary for correlation is not necessary";

123 break;

125 OS << "unable to correlate profile";

126 break;

128 OS << "invalid profile created. Please file a bug "

129 "at: " BUG_REPORT_URL

130 " and include the profraw files that caused this error.";

131 break;

133 OS << "no profile data available for function";

134 break;

136 OS << "function control flow change detected (hash mismatch)";

137 break;

139 OS << "function basic block count change detected (counter mismatch)";

140 break;

142 OS << "function bitmap size change detected (bitmap size mismatch)";

143 break;

145 OS << "counter overflow";

146 break;

148 OS << "function value site count change detected (counter mismatch)";

149 break;

151 OS << "failed to compress data (zlib)";

152 break;

154 OS << "failed to uncompress data (zlib)";

155 break;

157 OS << "empty raw profile file";

158 break;

160 OS << "profile uses zlib compression but the profile reader was built "

161 "without zlib support";

162 break;

164 OS << "raw profile version mismatch";

165 break;

167 OS << "excessively large counter value suggests corrupted profile data";

168 break;

169 }

170

171

172 if (!ErrMsg.empty())

173 OS << ": " << ErrMsg;

174

175 return OS.str();

176}

177

178namespace {

179

180

181

182

183class InstrProfErrorCategoryType : public std::error_category {

184 const char *name() const noexcept override { return "llvm.instrprof"; }

185

186 std::string message(int IE) const override {

188 }

189};

190

191}

192

194 static InstrProfErrorCategoryType ErrorCategory;

195 return ErrorCategory;

196}

197

198namespace {

199

200const char *InstrProfSectNameCommon[] = {

201#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \

202 SectNameCommon,

204};

205

206const char *InstrProfSectNameCoff[] = {

207#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \

208 SectNameCoff,

210};

211

212const char *InstrProfSectNamePrefix[] = {

213#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \

214 Prefix,

216};

217

218}

219

220namespace llvm {

221

223 "enable-name-compression",

224 cl::desc("Enable name/filename string compression"), cl::init(true));

225

227 "enable-vtable-value-profiling", cl::init(false),

228 cl::desc("If true, the virtual table address will be instrumented to know "

229 "the types of a C++ pointer. The information is used in indirect "

230 "call promotion to do selective vtable-based comparison."));

231

233 "enable-vtable-profile-use", cl::init(false),

234 cl::desc("If ThinLTO and WPD is enabled and this option is true, vtable "

235 "profiles will be used by ICP pass for more efficient indirect "

236 "call sequence. If false, type profiles won't be used."));

237

240 bool AddSegmentInfo) {

241 std::string SectName;

242

244 SectName = InstrProfSectNamePrefix[IPSK];

245

247 SectName += InstrProfSectNameCoff[IPSK];

248 else

249 SectName += InstrProfSectNameCommon[IPSK];

250

251 if (OF == Triple::MachO && IPSK == IPSK_data && AddSegmentInfo)

252 SectName += ",regular,live_support";

253

254 return SectName;

255}

256

260

262

265

268

273

276

280 for (const auto &K : P) {

281 FDOStream.seek(K.Pos);

284 }

285

286

287

288 FDOStream.seek(LastPos);

289 } else {

291 std::string &Data = SOStream.str();

292 for (const auto &K : P) {

293 for (int I = 0, E = K.D.size(); I != E; I++) {

297 (const char *)&Bytes, sizeof(uint64_t));

298 }

299 }

300 }

301}

302

306

307

308

309 if (Name[0] == '\1')

310 Name = Name.substr(1);

311

312 std::string NewName = std::string(Name);

314

315

316

317

318 if (FileName.empty())

319 NewName = NewName.insert(0, ":");

320 else

321 NewName = NewName.insert(0, FileName.str() + ":");

322 }

323 return NewName;

324}

325

326

327

328

331 uint32_t Pos = 0, LastPos = 0;

332 for (const auto &CI : PathNameStr) {

333 ++Pos;

335 LastPos = Pos;

337 }

339 break;

340 }

341 return PathNameStr.substr(LastPos);

342}

343

349 if (StripLevel)

351 return FileName;

352}

353

354

355

356

357

358

359

360

361

362

363static std::string

369

371 if (MD != nullptr) {

373 return S.str();

374 }

375 return {};

376}

377

378

379

380

381

382

383

384

385

386

387

388

389

390

392 MDNode *PGONameMetadata) {

393 if (!InLTO) {

396 }

397

398

400 return *IRPGOFuncName;

401

402

403

404

406}

407

408

409

413

414

415

416

417

418

419

420

422 if (!InLTO) {

425 }

426

427

429 return *PGOFuncName;

430

431

432

433

435}

436

443

444

447 if (MangledName.empty())

448 return std::make_pair(StringRef(), IRPGOName);

449 return std::make_pair(FileName, MangledName);

450}

451

453 if (FileName.empty())

454 return PGOFuncName;

455

456

458 PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1);

459 return PGOFuncName;

460}

461

462

463

467 VarName += FuncName;

468

470 return VarName;

471

472

473 const char InvalidChars[] = "-:;<>/\"'";

474 size_t FoundPos = VarName.find_first_of(InvalidChars);

475 while (FoundPos != std:🧵:npos) {

476 VarName[FoundPos] = '_';

477 FoundPos = VarName.find_first_of(InvalidChars, FoundPos + 1);

478 }

479 return VarName;

480}

481

483 const Triple &T = M.getTargetTriple();

484 return T.isGPU();

485}

486

496

500

503

504

505

513

516 auto *FuncNameVar =

519

521 return FuncNameVar;

522}

523

527

530

531

532 if (F.hasName())

533 continue;

535 return E;

536

538 return E;

539 }

540

542 if (G.hasName() || G.hasMetadata(LLVMContext::MD_type))

543 continue;

545 return E;

546 }

547

548 Sorted = false;

549 finalizeSymtab();

551}

552

557 return E;

558

559 bool Inserted = true;

560 std::tie(std::ignore, Inserted) = MD5VTableMap.try_emplace(

562 if (!Inserted)

563 LLVM_DEBUG(dbgs() << "GUID conflict within one module");

565 };

566 if (Error E = NameToGUIDMap(VTablePGOName))

567 return E;

568

570 if (CanonicalName != VTablePGOName)

571 return NameToGUIDMap(CanonicalName);

580 while (P < EndP) {

585 P += N;

586 const bool IsCompressed = (CompressedSize != 0);

589 if (IsCompressed) {

592

594 UncompressedNameStrings,

595 UncompressedSize)) {

598 }

599 P += CompressedSize;

600 NameStrings = toStringRef(UncompressedNameStrings);

601 } else {

602 NameStrings =

603 StringRef(reinterpret_cast<const char *>(P), UncompressedSize);

604 P += UncompressedSize;

605 }

606

610 if (Error E = NameCallback(Name))

611 return E;

612

613 while (P < EndP && *P == 0)

614 P++;

615 }

617}

618

623

633

635 StringRef CompressedVTableStrings) {

638}

639

641

642

643

644

645

646

647

648

649

650 const std::string UniqSuffix = ".__uniq.";

651 size_t Pos = PGOName.find(UniqSuffix);

653 Pos += UniqSuffix.length();

654 else

655 Pos = 0;

656

657

658

659 Pos = PGOName.find('.', Pos);

661 return PGOName.substr(0, Pos);

662

663 return PGOName;

664}

665

667 bool AddCanonical) {

670 return E;

673 };

674 if (Error E = NameToGUIDMap(PGOFuncName))

675 return E;

676

677 if (!AddCanonical)

679

681 if (CanonicalFuncName != PGOFuncName)

682 return NameToGUIDMap(CanonicalFuncName);

683

685}

686

688

689

690 return VTableAddrMap.lookup(Address, 0);

691}

692

694 finalizeSymtab();

695 auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {

696 return A.first < Address;

697 });

698

699

700

701

702 if (It != AddrToMD5Map.end() && It->first == Address)

704 return 0;

705}

706

711 OS << S << '\n';

712}

713

715 bool DoCompression, std::string &Result) {

716 assert(!NameStrs.empty() && "No name data to emit");

717

718 uint8_t Header[20], *P = Header;

719 std::string UncompressedNameStrings =

721

724 "PGO name is invalid (contains separator token)");

725

726 unsigned EncLen = encodeULEB128(UncompressedNameStrings.length(), P);

727 P += EncLen;

728

729 auto WriteStringToResult = [&](size_t CompressedLen, StringRef InputStr) {

731 P += EncLen;

732 char *HeaderStr = reinterpret_cast<char *>(&Header[0]);

733 unsigned HeaderLen = P - &Header[0];

734 Result.append(HeaderStr, HeaderLen);

735 Result += InputStr;

737 };

738

739 if (!DoCompression) {

740 return WriteStringToResult(0, UncompressedNameStrings);

741 }

742

745 CompressedNameStrings,

747

748 return WriteStringToResult(CompressedNameStrings.size(),

750}

751

755 Arr->isCString() ? Arr->getAsCString() : Arr->getAsString();

756 return NameStr;

757}

758

760 std::string &Result, bool DoCompression) {

761 std::vectorstd::string NameStrs;

762 for (auto *NameVar : NameVars) {

764 }

767}

768

770 std::string &Result, bool DoCompression) {

771 std::vectorstd::string VTableNameStrs;

772 for (auto *VTable : VTables)

773 VTableNameStrs.push_back(getPGOName(*VTable));

776 Result);

777}

778

783 FuncSum += Count;

785

786 for (uint32_t VK = IPVK_First; VK <= IPVK_Last; ++VK) {

791 KindSum += V.Count;

792 }

794 }

795}

796

803 double Score = 0.0f, FuncLevelScore = 0.0f;

806 auto J = Input.ValueData.begin();

807 auto JE = Input.ValueData.end();

808 while (I != IE && J != JE) {

809 if (I->Value == J->Value) {

814 I->Count, J->Count, FuncLevelOverlap.Base.ValueCounts[ValueKind],

816 ++I;

817 } else if (I->Value < J->Value) {

818 ++I;

819 continue;

820 }

821 ++J;

822 }

825}

826

827

833 assert(ThisNumValueSites == Other.getNumValueSites(ValueKind));

834 if (!ThisNumValueSites)

835 return;

836

837 std::vector &ThisSiteRecords =

838 getOrCreateValueSitesForKind(ValueKind);

840 Other.getValueSitesForKind(ValueKind);

841 for (uint32_t I = 0; I < ThisNumValueSites; I++)

842 ThisSiteRecords[I].overlap(OtherSiteRecords[I], ValueKind, Overlap,

843 FuncLevelOverlap);

844}

845

849

852 bool Mismatch = (Counts.size() != Other.Counts.size());

853

854

855 if (!Mismatch) {

856 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {

858 uint32_t OtherNumValueSites = Other.getNumValueSites(Kind);

859 if (ThisNumValueSites != OtherNumValueSites) {

860 Mismatch = true;

861 break;

862 }

863 }

864 }

865 if (Mismatch) {

867 return;

868 }

869

870

871 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)

873

874 double Score = 0.0;

876

877 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {

880 MaxCount = std::max(Other.Counts[I], MaxCount);

881 }

884

885 if (MaxCount >= ValueCutoff) {

886 double FuncScore = 0.0;

887 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I)

893 FuncLevelOverlap.Valid = true;

894 }

895}

896

904 std::vector Merged;

905 Merged.reserve(std::max(ValueData.size(), Input.ValueData.size()));

906 for (const InstrProfValueData &J : Input.ValueData) {

907 while (I != IE && I->Value < J.Value) {

908 Merged.push_back(*I);

909 ++I;

910 }

911 if (I != IE && I->Value == J.Value) {

912 bool Overflowed;

914 if (Overflowed)

916 Merged.push_back(*I);

917 ++I;

918 continue;

919 }

920 Merged.push_back(J);

921 }

922 Merged.insert(Merged.end(), I, IE);

924}

925

928 for (InstrProfValueData &I : ValueData) {

929 bool Overflowed;

931 if (Overflowed)

933 }

934}

935

936

937

938void InstrProfRecord::mergeValueProfData(

942 uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);

943 if (ThisNumValueSites != OtherNumValueSites) {

945 return;

946 }

947 if (!ThisNumValueSites)

948 return;

949 std::vector &ThisSiteRecords =

950 getOrCreateValueSitesForKind(ValueKind);

952 Src.getValueSitesForKind(ValueKind);

953 for (uint32_t I = 0; I < ThisNumValueSites; I++)

954 ThisSiteRecords[I].merge(OtherSiteRecords[I], Weight, Warn);

955}

956

959

960

961 if (Counts.size() != Other.Counts.size()) {

963 return;

964 }

965

966

970

971

972

975 return;

976 }

979 else

981 return;

982 }

983

984 for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {

985 bool Overflowed;

990 Overflowed = true;

991 }

993 if (Overflowed)

995 }

996

997

998

1001 return;

1002 }

1003

1004

1005 for (size_t I = 0, E = Other.BitmapBytes.size(); I < E; ++I) {

1007 }

1008

1009 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)

1010 mergeValueProfData(Kind, Other, Weight, Warn);

1011}

1012

1013void InstrProfRecord::scaleValueProfData(

1016 for (auto &R : getValueSitesForKind(ValueKind))

1017 R.scale(N, D, Warn);

1018}

1019

1022 assert(D != 0 && "D cannot be 0");

1024 bool Overflowed;

1028 Overflowed = true;

1029 }

1030 if (Overflowed)

1032 }

1033 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)

1034 scaleValueProfData(Kind, N, D, Warn);

1035}

1036

1037

1040 if (!SymTab)

1042

1043 if (ValueKind == IPVK_IndirectCallTarget)

1045

1046 if (ValueKind == IPVK_VTableTarget)

1048

1050}

1051

1055

1056 std::vector RemappedVD;

1057 RemappedVD.reserve(VData.size());

1058 for (const auto &V : VData) {

1059 uint64_t NewValue = remapValue(V.Value, ValueKind, ValueMap);

1060 RemappedVD.push_back({NewValue, V.Count});

1061 }

1062

1063 std::vector &ValueSites =

1064 getOrCreateValueSitesForKind(ValueKind);

1065 assert(ValueSites.size() == Site);

1066

1067

1068 ValueSites.emplace_back(std::move(RemappedVD));

1069}

1070

1073 bool RemoveOutlierUNs) {

1076 UtilityNodeT MaxUN = 0;

1080

1081

1082 for (auto &Trace : Traces) {

1083 size_t CutoffTimestamp = 1;

1084 for (size_t Timestamp = 0; Timestamp < Trace.FunctionNameRefs.size();

1085 Timestamp++) {

1086 IDT Id = Trace.FunctionNameRefs[Timestamp];

1087 auto [It, WasInserted] = IdToFirstTimestamp.try_emplace(Id, Timestamp);

1088 if (!WasInserted)

1089 It->getSecond() = std::min<size_t>(It->getSecond(), Timestamp);

1090 if (Timestamp >= CutoffTimestamp) {

1091 ++MaxUN;

1092 CutoffTimestamp = 2 * Timestamp;

1093 }

1095 }

1096 for (auto &[Id, FirstUN] : IdToFirstUN)

1097 for (auto UN = FirstUN; UN <= MaxUN; ++UN)

1098 IdToUNs[Id].push_back(UN);

1099 ++MaxUN;

1100 IdToFirstUN.clear();

1101 }

1102

1103 if (RemoveOutlierUNs) {

1105 for (auto &[Id, UNs] : IdToUNs)

1106 for (auto &UN : UNs)

1107 ++UNFrequency[UN];

1108

1109

1110 for (auto &[Id, UNs] : IdToUNs)

1112 unsigned Freq = UNFrequency[UN];

1113 return Freq <= 1 || 2 * Freq > IdToUNs.size();

1114 });

1115 }

1116

1117 for (auto &[Id, UNs] : IdToUNs)

1118 Nodes.emplace_back(Id, UNs);

1119

1120

1121

1122 llvm::sort(Nodes, [&](auto &L, auto &R) {

1123 return std::make_pair(IdToFirstTimestamp[L.Id], L.Id) <

1124 std::make_pair(IdToFirstTimestamp[R.Id], R.Id);

1125 });

1126}

1127

1128#define INSTR_PROF_COMMON_API_IMPL

1130

1131

1132

1133

1134

1135

1139

1142 ->getNumValueSites(VKind);

1143}

1144

1147 ->getNumValueData(VKind);

1148}

1149

1155

1158 const auto *IPR = reinterpret_cast<const InstrProfRecord *>(R);

1159 llvm::copy(IPR->getValueArrayForSite(K, S), Dst);

1160}

1161

1164 memset(VD, 0, TotalSizeInBytes);

1165 return VD;

1166}

1167

1169 nullptr,

1174 nullptr,

1177

1178

1181 Closure.Record = &Record;

1183}

1184

1185

1186std::unique_ptr

1189

1190 std::unique_ptr VPD(

1192 return VPD;

1193}

1194

1198

1200 for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {

1201 uint8_t ValueDataCount = this->SiteCountArray[VSite];

1203 Record.addValueData(Kind, VSite, VDs, SymTab);

1204 ValueData += ValueDataCount;

1205 }

1206}

1207

1208

1209

1210

1212 using namespace support;

1213

1214 if (Old == New)

1215 return;

1216

1220 }

1223

1224

1225 for (uint32_t I = 0; I < ND; I++) {

1228 }

1232 }

1233}

1234

1238 return;

1239

1242 VR->deserializeTo(Record, SymTab);

1244 }

1245}

1246

1248 return std::unique_ptr(new (::operator new(TotalSize))

1250}

1251

1252Error ValueProfData::checkIntegrity() {

1256

1260

1262 for (uint32_t K = 0; K < this->NumValueKinds; K++) {

1263 if (VR->Kind > IPVK_Last)

1265 "value kind is invalid");

1270 "value profile address is greater than total size");

1271 }

1273}

1274

1276ValueProfData::getValueProfData(const unsigned char *D,

1277 const unsigned char *const BufferEnd,

1279 using namespace support;

1280

1283

1284 const unsigned char *Header = D;

1286

1289

1292

1293 VPD->swapBytesToHost(Endianness);

1294

1295 Error E = VPD->checkIntegrity();

1296 if (E)

1297 return std::move(E);

1298

1299 return std::move(VPD);

1300}

1301

1302void ValueProfData::swapBytesToHost(llvm::endianness Endianness) {

1303 using namespace support;

1304

1306 return;

1307

1310

1315 }

1316}

1317

1318void ValueProfData::swapBytesFromHost(llvm::endianness Endianness) {

1319 using namespace support;

1320

1322 return;

1323

1328 VR = NVR;

1329 }

1332}

1333

1339 if (VDs.empty())

1340 return;

1342 for (const InstrProfValueData &V : VDs)

1345}

1346

1351 if (VDs.empty())

1352 return;

1356

1358

1361

1364

1365

1366 uint32_t MDCount = MaxMDCount;

1367 for (const auto &VD : VDs) {

1372 if (--MDCount == 0)

1373 break;

1374 }

1376}

1377

1381 if (!MD)

1382 return nullptr;

1383

1385 return nullptr;

1386

1389 return nullptr;

1390

1391

1393 if (!KindInt)

1394 return nullptr;

1396 return nullptr;

1397

1398 return MD;

1399}

1400

1404 bool GetNoICPValue) {

1405

1406

1409 if (!MD)

1410 return ValueData;

1412

1414 if (!TotalCInt)

1415 return ValueData;

1417

1418 ValueData.reserve((NOps - 3) / 2);

1419 for (unsigned I = 3; I < NOps; I += 2) {

1420 if (ValueData.size() >= MaxNumValueData)

1421 break;

1426 ValueData.clear();

1427 return ValueData;

1428 }

1431 continue;

1432 InstrProfValueData V;

1433 V.Value = Value->getZExtValue();

1434 V.Count = CntValue;

1436 }

1437 return ValueData;

1438}

1439

1443

1446

1447

1448 if (GO.getName() == PGOName)

1449 return;

1450

1451

1453 return;

1454

1458}

1459

1463

1467

1470 return true;

1471

1472 if (!M.getTargetTriple().supportsCOMDAT())

1473 return false;

1474

1475

1476

1477

1478

1479

1480

1481

1482

1483

1484

1488 return false;

1489

1490 return true;

1491}

1492

1493

1498 return false;

1499

1500

1501

1503 return true;

1504

1505

1507 return false;

1508

1510 if (!InitVal)

1511 return false;

1513}

1514

1515

1517 if (F.getName().empty())

1518 return false;

1520 return false;

1521

1522

1523 if (CheckAddressTaken && F.hasAddressTaken())

1524 return false;

1525

1526

1528 return false;

1529

1530

1531 if (F.hasComdat()) {

1533 return true;

1534 }

1535 return true;

1536}

1537

1538

1540 if (InstrProfileOutput.empty())

1541 return;

1542 Constant *ProfileNameConst =

1548 Triple TT(M.getTargetTriple());

1549 if (TT.supportsCOMDAT()) {

1551 ProfileNameVar->setComdat(M.getOrInsertComdat(

1553 }

1554}

1555

1558 bool IsCS) {

1559 auto GetProfileSum = [IsCS](const std::string &Filename,

1561

1562

1565 if (Error E = ReaderOrErr.takeError()) {

1566 return E;

1567 }

1568 auto Reader = std::move(ReaderOrErr.get());

1569 Reader->accumulateCounts(Sum, IsCS);

1571 };

1573 if (Ret)

1574 return Ret;

1576 if (Ret)

1577 return Ret;

1582}

1583

1587 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {

1588 if (Test.ValueCounts[I] >= 1.0f)

1591 }

1592}

1593

1595 Unique.NumEntries += 1;

1597 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {

1598 if (Test.ValueCounts[I] >= 1.0f)

1600 }

1601}

1602

1605 return;

1606

1607 const char *EntryName =

1610 OS << "Profile overlap information for base_profile: " << *BaseFilename

1611 << " and test_profile: " << *TestFilename << "\nProgram level:\n";

1612 } else {

1613 OS << "Function level:\n"

1614 << " Function: " << FuncName << " (Hash=" << FuncHash << ")\n";

1615 }

1616

1617 OS << " # of " << EntryName << " overlap: " << Overlap.NumEntries << "\n";

1619 OS << " # of " << EntryName << " mismatch: " << Mismatch.NumEntries

1620 << "\n";

1621 if (Unique.NumEntries)

1622 OS << " # of " << EntryName

1623 << " only in test_profile: " << Unique.NumEntries << "\n";

1624

1625 OS << " Edge profile overlap: " << format("%.3f%%", Overlap.CountSum * 100)

1626 << "\n";

1628 OS << " Mismatched count percentage (Edge): "

1629 << format("%.3f%%", Mismatch.CountSum * 100) << "\n";

1630 if (Unique.NumEntries)

1631 OS << " Percentage of Edge profile only in test_profile: "

1632 << format("%.3f%%", Unique.CountSum * 100) << "\n";

1633 OS << " Edge profile base count sum: " << format("%.0f", Base.CountSum)

1634 << "\n"

1635 << " Edge profile test count sum: " << format("%.0f", Test.CountSum)

1636 << "\n";

1637

1638 for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) {

1639 if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f)

1640 continue;

1641 char ProfileKindName[20] = {0};

1642 switch (I) {

1643 case IPVK_IndirectCallTarget:

1644 strncpy(ProfileKindName, "IndirectCall", 19);

1645 break;

1646 case IPVK_MemOPSize:

1647 strncpy(ProfileKindName, "MemOP", 19);

1648 break;

1649 case IPVK_VTableTarget:

1650 strncpy(ProfileKindName, "VTable", 19);

1651 break;

1652 default:

1653 snprintf(ProfileKindName, 19, "VP[%d]", I);

1654 break;

1655 }

1656 OS << " " << ProfileKindName

1657 << " profile overlap: " << format("%.3f%%", Overlap.ValueCounts[I] * 100)

1658 << "\n";

1660 OS << " Mismatched count percentage (" << ProfileKindName

1661 << "): " << format("%.3f%%", Mismatch.ValueCounts[I] * 100) << "\n";

1662 if (Unique.NumEntries)

1663 OS << " Percentage of " << ProfileKindName

1664 << " profile only in test_profile: "

1665 << format("%.3f%%", Unique.ValueCounts[I] * 100) << "\n";

1666 OS << " " << ProfileKindName

1667 << " profile base count sum: " << format("%.0f", Base.ValueCounts[I])

1668 << "\n"

1669 << " " << ProfileKindName

1670 << " profile test count sum: " << format("%.0f", Test.ValueCounts[I])

1671 << "\n";

1672 }

1673}

1674

1677 using namespace support;

1678 static_assert(std::is_standard_layout_v

,

1679 "Use standard layout for Header for simplicity");

1681

1683

1686

1687

1689 if (H.getIndexedProfileVersion() >

1692

1694 "Please update the reader as needed when a new field is added "

1695 "or when indexed profile version gets bumped.");

1696

1697 Buffer += sizeof(uint64_t);

1700 if (H.getIndexedProfileVersion() >= 8)

1701 H.MemProfOffset =

1703 if (H.getIndexedProfileVersion() >= 9)

1704 H.BinaryIdOffset =

1706

1707 if (H.getIndexedProfileVersion() >= 10)

1708 H.TemporalProfTracesOffset =

1710 if (H.getIndexedProfileVersion() >= 12)

1711 H.VTableNamesOffset =

1713 return H;

1714}

1715

1719

1722

1723

1724

1725 static_assert(

1727 "Please update the size computation below if a new field has "

1728 "been added to the header; for a version bump without new "

1729 "fields, add a case statement to fall through to the latest version.");

1730 case 13ull:

1731 case 12ull:

1732 return 72;

1733 case 11ull:

1734 [[fallthrough]];

1735 case 10ull:

1736 return 64;

1737 case 9ull:

1738 return 56;

1739 case 8ull:

1740 return 48;

1741 default:

1742 return 40;

1743 }

1744}

1745

1746}

1747

1748}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

This file contains the declarations for the subclasses of Constant, which represent the different fla...

Module.h This file contains the declarations for the Module class.

static cl::opt< bool > StaticFuncFullModulePrefix("static-func-full-module-prefix", cl::init(true), cl::Hidden, cl::desc("Use full module build paths in the profile counter names for " "static functions."))

static cl::opt< unsigned > StaticFuncStripDirNamePrefix("static-func-strip-dirname-prefix", cl::init(0), cl::Hidden, cl::desc("Strip specified level of directory name from source path in " "the profile counter name for static functions."))

static std::string getInstrProfErrString(instrprof_error Err, const std::string &ErrMsg="")

Definition InstrProf.cpp:82

#define INSTR_PROF_QUOTE(x)

#define INSTR_PROF_PROFILE_NAME_VAR

#define INSTR_PROF_RAW_VERSION_VAR

#define VARIANT_MASK_IR_PROF

This file contains the declarations for profiling metadata utility functions.

This file defines the SmallVector class.

Defines the virtual file system interface vfs::FileSystem.

The Input class is used to parse a yaml document into in-memory structs and vectors.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

bool empty() const

empty - Check if the array is empty.

static LLVM_ABI Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)

This method constructs a CDS and initializes it with a text string.

This is the shared class of boolean and integer constants.

uint64_t getZExtValue() const

Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...

This is an important base class in LLVM.

std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)

Lightweight error class with error context and mandatory checking.

static ErrorSuccess success()

Create a success value.

Tagged union holding either a T or a Error.

LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)

Set a particular kind of metadata attachment.

LLVM_ABI void setComdat(Comdat *C)

MDNode * getMetadata(unsigned KindID) const

Get the current metadata attachments for the given kind, if any.

static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)

Return a 64-bit global unique ID constructed from the name of a global symbol.

static bool isLocalLinkage(LinkageTypes Linkage)

LLVM_ABI bool isDeclaration() const

Return true if the primary definition of this global value is outside of the current translation unit...

LinkageTypes getLinkage() const

bool hasLocalLinkage() const

void setLinkage(LinkageTypes LT)

Module * getParent()

Get the module that this global value is contained inside of...

@ HiddenVisibility

The GV is hidden.

@ ProtectedVisibility

The GV is protected.

static LLVM_ABI std::string getGlobalIdentifier(StringRef Name, GlobalValue::LinkageTypes Linkage, StringRef FileName)

Return the modified name for a global value suitable to be used as the key for a global lookup (e....

void setVisibility(VisibilityTypes V)

static bool isDiscardableIfUnused(LinkageTypes Linkage)

Whether the definition of this global may be discarded if it is not used in its compilation unit.

LinkageTypes

An enumeration for the kinds of linkage for global values.

@ PrivateLinkage

Like Internal, but omit from symbol table.

@ InternalLinkage

Rename collisions when linking (static functions).

@ LinkOnceAnyLinkage

Keep one copy of function when linking (inline)

@ ExternalLinkage

Externally visible function.

@ WeakAnyLinkage

Keep one copy of named function when linking (weak)

@ AvailableExternallyLinkage

Available for inspection, not emission.

@ ExternalWeakLinkage

ExternalWeak linkage description.

@ LinkOnceODRLinkage

Same, but only replaced by something equivalent.

const Constant * getInitializer() const

getInitializer - Return the initializer for this global variable.

bool hasInitializer() const

Definitions have initializers, declarations don't.

std::string message() const override

Return the error message as a string.

Definition InstrProf.cpp:257

static LLVM_ABI Expected< std::unique_ptr< InstrProfReader > > create(const Twine &Path, vfs::FileSystem &FS, const InstrProfCorrelator *Correlator=nullptr, const object::BuildIDFetcher *BIDFetcher=nullptr, const InstrProfCorrelator::ProfCorrelatorKind BIDFetcherCorrelatorKind=InstrProfCorrelator::ProfCorrelatorKind::NONE, std::function< void(Error)> Warn=nullptr)

Factory method to create an appropriately typed reader for the given instrprof file.

A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...

static LLVM_ABI StringRef getCanonicalName(StringRef PGOName)

Definition InstrProf.cpp:640

Error addSymbolName(StringRef SymbolName)

Error addVTableName(StringRef VTableName)

Adds VTableName as a known symbol, and inserts it to a map that tracks all vtable names.

LLVM_ABI void dumpNames(raw_ostream &OS) const

Dump the symbols in this table.

Definition InstrProf.cpp:707

LLVM_ABI Error create(object::SectionRef &Section)

Create InstrProfSymtab from an object file section which contains function PGO names.

Error addFuncName(StringRef FuncName)

The method name is kept since there are many callers.

LLVM_ABI Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames)

Initialize 'this' with the set of vtable names encoded in CompressedVTableNames.

Definition InstrProf.cpp:634

LLVM_ABI uint64_t getVTableHashFromAddress(uint64_t Address) const

Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab.

Definition InstrProf.cpp:687

LLVM_ABI uint64_t getFunctionHashFromAddress(uint64_t Address) const

Return a function's hash, or 0, if the function isn't in this SymTab.

Definition InstrProf.cpp:693

MDNode * getMetadata(unsigned KindID) const

Get the metadata of given kind attached to this Instruction.

LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)

Set the metadata of the specified kind to the specified node.

This is an important class for using LLVM in a threaded context.

LLVM_ABI ConstantAsMetadata * createConstant(Constant *C)

Return the given constant as metadata.

LLVM_ABI MDString * createString(StringRef Str)

Return the given string as metadata.

const MDOperand & getOperand(unsigned I) const

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

unsigned getNumOperands() const

Return number of MDNode operands.

static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)

A Module instance is used to store all the information related to an LLVM module.

const std::string & getSourceFileName() const

Get the module's original source file name.

MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...

LLVM_ABI uint64_t tell() const

Definition InstrProf.cpp:269

LLVM_ABI void writeByte(uint8_t V)

Definition InstrProf.cpp:272

LLVM_ABI void patch(ArrayRef< PatchItem > P)

Definition InstrProf.cpp:274

LLVM_ABI void write32(uint32_t V)

Definition InstrProf.cpp:271

support::endian::Writer LE

LLVM_ABI ProfOStream(raw_fd_ostream &FD)

Definition InstrProf.cpp:263

LLVM_ABI void write(uint64_t V)

Definition InstrProf.cpp:270

void reserve(size_type N)

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

std::pair< StringRef, StringRef > split(char Separator) const

Split into two substrings around the first occurrence of a separator character.

static constexpr size_t npos

std::string str() const

str - Get the contents as an std::string.

const unsigned char * bytes_end() const

constexpr StringRef substr(size_t Start, size_t N=npos) const

Return a reference to the substring from [Start, Start + N).

bool starts_with(StringRef Prefix) const

Check if this string starts with the given Prefix.

constexpr bool empty() const

empty - Check if the string is empty.

StringRef drop_front(size_t N=1) const

Return a StringRef equal to 'this' but with the first N elements dropped.

constexpr size_t size() const

size - Get the string size.

size_t find(char C, size_t From=0) const

Search for the first character C in the string.

const unsigned char * bytes_begin() const

Triple - Helper class for working with autoconf configuration names.

static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)

static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

LLVM_ABI LLVMContext & getContext() const

All values hold a context through their type.

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

An efficient, type-erasing, non-owning reference to a callable.

A raw_ostream that writes to a file descriptor.

uint64_t seek(uint64_t off)

Flushes the stream and repositions the underlying file descriptor position to the offset specified fr...

This class implements an extremely fast bulk output stream that can only output to a stream.

uint64_t tell() const

tell - Return the current offset with the file.

A raw_ostream that writes to an std::string.

std::string & str()

Returns the string's reference.

@ C

The default llvm calling convention, compatible with C.

initializer< Ty > init(const Ty &Val)

LLVM_ABI void compress(ArrayRef< uint8_t > Input, SmallVectorImpl< uint8_t > &CompressedBuffer, int Level=DefaultCompression)

LLVM_ABI Error decompress(ArrayRef< uint8_t > Input, uint8_t *Output, size_t &UncompressedSize)

LLVM_ABI bool isAvailable()

constexpr int BestSizeCompression

std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)

Extract a Value from Metadata, if any.

value_type byte_swap(value_type value, endianness endian)

value_type readNext(const CharT *&memory, endianness endian)

Read a value of a particular endianness from a buffer, and increment the buffer past that value.

LLVM_ABI bool is_separator(char value, Style style=Style::native)

Check whether the given char is a path separator on the host OS.

void swapByteOrder(T &Value)

LLVM_ABI IntrusiveRefCntPtr< FileSystem > getRealFileSystem()

Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.

This is an optimization pass for GlobalISel generic memory operations.

StringRef getInstrProfNameVarPrefix()

Return the name prefix of variables containing instrumented function names.

LLVM_ABI std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)

Please use getIRPGOFuncName for LLVM IR instrumentation.

Definition InstrProf.cpp:421

LLVM_ABI void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)

Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.

Definition InstrProf.cpp:1460

ArrayRef< CharT > arrayRefFromStringRef(StringRef Input)

Construct a string ref from an array ref of unsigned chars.

LLVM_ABI std::string getIRPGOFuncName(const Function &F, bool InLTO=false)

Definition InstrProf.cpp:410

StringRef getPGOFuncNameMetadataName()

void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, uint32_t K, uint32_t S)

Definition InstrProf.cpp:1156

LLVM_ABI cl::opt< bool > DoInstrProfNameCompression

LLVM_ABI StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="")

Given a PGO function name, remove the filename prefix and return the original (static) function name.

Definition InstrProf.cpp:452

auto partition_point(R &&Range, Predicate P)

Binary search for the first iterator in a range where a predicate is false.

uint64_t decodeULEB128(const uint8_t *p, unsigned *n=nullptr, const uint8_t *end=nullptr, const char **error=nullptr)

Utility function to decode a ULEB128 value.

LLVM_ABI void createPGONameMetadata(GlobalObject &GO, StringRef PGOName)

Create the PGOName metadata if a global object's PGO name is different from its mangled name.

Definition InstrProf.cpp:1464

INSTR_PROF_VISIBILITY ValueProfRecord * getValueProfRecordNext(ValueProfRecord *VPR)

Use this method to advance to the next This ValueProfRecord.

Definition InstrProf.cpp:602

LLVM_ABI std::pair< StringRef, StringRef > getParsedIRPGOName(StringRef IRPGOName)

Definition InstrProf.cpp:445

LLVM_ABI MDNode * getPGOFuncNameMetadata(const Function &F)

Return the PGOFuncName meta data associated with a function.

Definition InstrProf.cpp:1440

static std::unique_ptr< ValueProfData > allocValueProfData(uint32_t TotalSize)

Definition InstrProf.cpp:1247

MDNode * mayHaveValueProfileOfKind(const Instruction &Inst, InstrProfValueKind ValueKind)

Definition InstrProf.cpp:1378

struct llvm::ValueProfData ValueProfData

Per-function header/control data structure for value profiling data in indexed format.

LLVM_ABI std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)

Return the name of the profile section corresponding to IPSK.

Definition InstrProf.cpp:238

cl::opt< bool > EnableVTableProfileUse("enable-vtable-profile-use", cl::init(false), cl::desc("If ThinLTO and WPD is enabled and this option is true, vtable " "profiles will be used by ICP pass for more efficient indirect " "call sequence. If false, type profiles won't be used."))

uint64_t getInstrMaxCountValue()

Return the max count value. We reserver a few large values for special use.

LLVM_ABI bool needsComdatForCounter(const GlobalObject &GV, const Module &M)

Check if we can use Comdat for profile variables.

Definition InstrProf.cpp:1468

auto dyn_cast_or_null(const Y &Val)

LLVM_ABI std::string getPGOName(const GlobalVariable &V, bool InLTO=false)

Definition InstrProf.cpp:437

LLVM_ABI GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)

Create and return the global variable for function name used in PGO instrumentation.

Definition InstrProf.cpp:524

FunctionAddr VTableAddr uintptr_t uintptr_t Version

LLVM_ABI void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)

Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...

Definition InstrProf.cpp:1334

INSTR_PROF_VISIBILITY uint32_t getValueProfDataSize(ValueProfRecordClosure *Closure)

Return the total size in bytes of the on-disk value profile data given the data stored in Record.

Definition InstrProf.cpp:624

LLVM_ABI Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)

Produce Result string with the same format described above.

Definition InstrProf.cpp:759

void sort(IteratorTy Start, IteratorTy End)

LLVM_ABI Error readAndDecodeStrings(StringRef NameStrings, std::function< Error(StringRef)> NameCallback)

NameStrings is a string composed of one or more possibly encoded sub-strings.

Definition InstrProf.cpp:576

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

FunctionAddr NumValueSites[IPVK_Last+1]

FunctionAddr VTableAddr Count

LLVM_ABI StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)

Return the initializer in string of the PGO name var NameVar.

Definition InstrProf.cpp:752

std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed=nullptr)

Multiply two unsigned integers, X and Y, and add the unsigned integer, A to the product.

INSTR_PROF_VISIBILITY ValueProfRecord * getFirstValueProfRecord(ValueProfData *VPD)

Return the first ValueProfRecord instance.

Definition InstrProf.cpp:613

StringRef getInstrProfNameSeparator()

Return the marker used to separate PGO names during serialization.

LLVM_ABI SmallVector< InstrProfValueData, 4 > getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)

Extract the value profile data from Inst and returns them if Inst is annotated with value profile dat...

Definition InstrProf.cpp:1402

INSTR_PROF_VISIBILITY ValueProfData * serializeValueProfDataFrom(ValueProfRecordClosure *Closure, ValueProfData *DstData)

Extract value profile data of a function from the Closure and serialize the data into DstData if it i...

Definition InstrProf.cpp:669

INSTR_PROF_VISIBILITY InstrProfValueData * getValueProfRecordValueData(ValueProfRecord *VPR)

Return the pointer to the start of value data array.

Definition InstrProf.cpp:581

format_object< Ts... > format(const char *Fmt, const Ts &... Vals)

These are helper functions used to produce formatted output.

static std::string getIRPGOObjectName(const GlobalObject &GO, bool InLTO, MDNode *PGONameMetadata)

Definition InstrProf.cpp:391

Error make_error(ArgTs &&... Args)

Make a Error instance representing failure using the given error info type.

std::string join(IteratorT Begin, IteratorT End, StringRef Separator)

Joins the strings in the range [Begin, End), adding Separator between the elements.

FunctionAddr VTableAddr uintptr_t uintptr_t Data

@ unable_to_correlate_profile

@ raw_profile_version_mismatch

@ counter_value_too_large

@ missing_correlation_info

@ value_site_count_mismatch

@ unexpected_correlation_info

std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingMultiply(T X, T Y, bool *ResultOverflowed=nullptr)

Multiply two unsigned integers, X and Y, of type T.

LLVM_ABI const std::error_category & instrprof_category()

Definition InstrProf.cpp:193

LLVM_ABI Error collectVTableStrings(ArrayRef< GlobalVariable * > VTables, std::string &Result, bool doCompression)

Definition InstrProf.cpp:769

auto count(R &&Range, const E &Element)

Wrapper function around std::count to count the number of times an element Element occurs in the give...

static StringRef getStrippedSourceFileName(const GlobalObject &GO)

Definition InstrProf.cpp:344

ArrayRef(const T &OneElt) -> ArrayRef< T >

uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind)

Definition InstrProf.cpp:1140

OutputIt copy(R &&Range, OutputIt Out)

LLVM_ABI bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)

Check if we can safely rename this Comdat function.

Definition InstrProf.cpp:1516

LLVM_ABI void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)

Definition InstrProf.cpp:1539

constexpr char GlobalIdentifierDelimiter

LLVM_ABI Error collectGlobalObjectNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)

Given a vector of strings (names of global objects like functions or, virtual tables) NameStrs,...

Definition InstrProf.cpp:714

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

void erase_if(Container &C, UnaryPredicate P)

Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...

void setPGOFuncVisibility(Module &M, GlobalVariable *FuncNameVar)

Definition InstrProf.cpp:487

INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t getValueProfRecordNumValueData(ValueProfRecord *This)

Return the total number of value data for This record.

Definition InstrProf.cpp:590

unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)

Utility function to encode a ULEB128 value to an output stream.

uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, uint32_t S)

Definition InstrProf.cpp:1150

static ValueProfRecordClosure InstrProfRecordClosure

Definition InstrProf.cpp:1168

LLVM_ABI std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)

Return the name of the global variable used to store a function name in PGO instrumentation.

Definition InstrProf.cpp:464

static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix)

Definition InstrProf.cpp:329

static std::optional< std::string > lookupPGONameFromMetadata(MDNode *MD)

Definition InstrProf.cpp:370

std::enable_if_t< std::is_unsigned_v< T >, T > SaturatingAdd(T X, T Y, bool *ResultOverflowed=nullptr)

Add two unsigned integers, X and Y, of type T.

LLVM_ABI bool isGPUProfTarget(const Module &M)

Determines whether module targets a GPU eligable for PGO instrumentation.

Definition InstrProf.cpp:482

LLVM_ABI bool isIRPGOFlagSet(const Module *M)

Check if INSTR_PROF_RAW_VERSION_VAR is defined.

Definition InstrProf.cpp:1494

StringRef getPGONameMetadataName()

void consumeError(Error Err)

Consume a Error without doing anything.

const uint64_t NOMORE_ICP_MAGICNUM

Magic number in the value profile metadata showing a target has been promoted for the instruction and...

StringRef toStringRef(bool B)

Construct a string ref from a boolean.

uint32_t getNumValueKindsInstrProf(const void *Record)

ValueProfRecordClosure Interface implementation for InstrProfRecord class.

Definition InstrProf.cpp:1136

ValueProfData * allocValueProfDataInstrProf(size_t TotalSizeInBytes)

Definition InstrProf.cpp:1162

uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind)

Definition InstrProf.cpp:1145

static std::string getIRPGONameForGlobalObject(const GlobalObject &GO, GlobalValue::LinkageTypes Linkage, StringRef FileName)

Definition InstrProf.cpp:364

cl::opt< bool > EnableVTableValueProfiling("enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison."))

std::array< double, IPVK_Last - IPVK_First+1 > ValueCounts

Profiling information for a single function.

LLVM_ABI void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)

Compute the overlap of value profile counts.

Definition InstrProf.cpp:828

InstrProfRecord()=default

std::vector< uint64_t > Counts

ArrayRef< InstrProfValueData > getValueArrayForSite(uint32_t ValueKind, uint32_t Site) const

Return the array of profiled values at Site.

CountPseudoKind getCountPseudoKind() const

LLVM_ABI void accumulateCounts(CountSumOrPercent &Sum) const

Compute the sums of all counts and store in Sum.

Definition InstrProf.cpp:779

uint32_t getNumValueSites(uint32_t ValueKind) const

Return the number of instrumented sites for ValueKind.

void setPseudoCount(CountPseudoKind Kind)

LLVM_ABI void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)

Merge the counts in Other into this one.

Definition InstrProf.cpp:957

LLVM_ABI void addValueData(uint32_t ValueKind, uint32_t Site, ArrayRef< InstrProfValueData > VData, InstrProfSymtab *SymTab)

Add ValueData for ValueKind at value Site.

Definition InstrProf.cpp:1052

LLVM_ABI void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)

Compute the overlap b/w this IntrprofRecord and Other.

Definition InstrProf.cpp:846

std::vector< uint8_t > BitmapBytes

LLVM_ABI void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)

Scale up profile counts (including value profile data) by a factor of (N / D).

Definition InstrProf.cpp:1020

InstrProfValueSiteRecord()=default

void sortByTargetValues()

Sort ValueData ascending by Value.

std::vector< InstrProfValueData > ValueData

Value profiling data pairs at a given value site.

LLVM_ABI void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)

Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.

Definition InstrProf.cpp:897

LLVM_ABI void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)

Compute the overlap b/w this record and Input record.

Definition InstrProf.cpp:797

LLVM_ABI void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)

Scale up value profile data counts by N (Numerator) / D (Denominator).

Definition InstrProf.cpp:926

static LLVM_ABI const char * ValueProfile

LLVM_ABI void addOneMismatch(const CountSumOrPercent &MismatchFunc)

Definition InstrProf.cpp:1584

static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)

LLVM_ABI Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)

Definition InstrProf.cpp:1556

LLVM_ABI void dump(raw_fd_ostream &OS) const

Definition InstrProf.cpp:1603

CountSumOrPercent Overlap

LLVM_ABI void addOneUnique(const CountSumOrPercent &UniqueFunc)

Definition InstrProf.cpp:1594

const std::string * BaseFilename

const std::string * TestFilename

CountSumOrPercent Mismatch

static LLVM_ABI void createBPFunctionNodes(ArrayRef< TemporalProfTraceTy > Traces, std::vector< BPFunctionNode > &Nodes, bool RemoveOutlierUNs=true)

Use a set of temporal profile traces to create a list of balanced partitioning function nodes used by...

Definition InstrProf.cpp:1071

Per-function header/control data structure for value profiling data in indexed format.