LLVM: include/llvm/ProfileData/InstrProf.h Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#ifndef LLVM_PROFILEDATA_INSTRPROF_H
16#define LLVM_PROFILEDATA_INSTRPROF_H
17
39#include
40#include
41#include
42#include
43#include
44#include
45#include
46#include <system_error>
47#include
48#include
49
50namespace llvm {
51
59
60
61
66
67
68
70public:
73
80
81
83
84
85
90
92#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind,
98 return std::numeric_limits<uint64_t>::max() - 2;
99}
100
101
102
103
104
105
108 bool AddSegmentInfo = true);
111
115
116
117
121
122
124
125
127
128
130
131
133
134
136
143
144
146
148
149
150
151
153 return "__llvm_coverage_mapping";
154}
156
157
158
159
161 return "__llvm_coverage_names";
162}
164
170 return "__llvm_profile_register_functions";
171}
172
173
174
176 return "__llvm_profile_register_function";
177}
178
179
180
182 return "__llvm_profile_register_names_function";
183}
184
185
186
188
190
191
192
193
197
198
199
201 return "__llvm_profile_runtime_user";
202}
203
207
211
212
214
215
216
218
219
220
221
222
223
227
228
229
230
231
235
236
237
238
240
241
242
243LLVM_ABI std::pair<StringRef, StringRef>
245
246
247
248
249
252
253
254
255
256
258 StringRef PGOFuncName);
259
260
261
262
263
266 StringRef PGOFuncName);
272
274 StringRef FileName = "");
275
276
277
278
279
280
281
282
283
284
286 bool doCompression,
287 std::string &Result);
288
289
290
291
292
293
295 std::string &Result,
296 bool doCompression = true);
299 std::string &Result, bool doCompression);
304
305
306
307
309 bool CheckAddressTaken = false);
310
312#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value,
314};
316
317
318
323
324
325
326
331
332
333
334
335
339 bool GetNoICPValue = false);
340
342
344
345
347
348LLVM_ABI std::string getPGOName(const GlobalVariable &V, bool InLTO = false);
349
350
351
352
353
354
356
357
358
360
361
362
364
365
366
367
368
370 StringRef NameStrings, std::function<Error(StringRef)> NameCallback);
371
372
375
377
379
381
383
385
387
389
391
394};
395
397
426
427
428
435
436
437
441 std::vector &Nodes,
442 bool RemoveOutlierUNs = true);
443};
444
448
450public:
452 : Err(Err), Msg(ErrStr.str()) {
454 }
455
456 std::string message() const override;
457
459
463
465 const std::string &getMessage() const { return Msg; }
466
467
468
469
470 static std::pair<instrprof_error, std::string> take(Error E) {
472 std::string Msg = "";
475 Err = IPE.get();
477 });
478 return {Err, Msg};
479 }
480
482
483private:
485 std::string Msg;
486};
487
489
492}
493
499
500
501
502
503
504
506public:
507 using AddrHashMap = std::vector<std::pair<uint64_t, uint64_t>>;
513
514private:
515 using AddrIntervalMap =
524
525
527
528 mutable std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
529
530
531 mutable std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
533
534
535
536
538
539
541
543
544 AddrIntervalMap VTableAddrMap;
545
546
547
548 mutable bool Sorted = false;
549
550 static StringRef getExternalSymbol() { return "** External Symbol **"; }
551
552
553
554
555
556
557
558 Error addFuncWithName(Function &F, StringRef PGOFuncName, bool AddCanonical);
559
560
561
562
563
564
565 Error addVTableWithName(GlobalVariable &V, StringRef PGOVTableName);
566
567
568
569
570
571
572
573 inline void finalizeSymtab() const;
574
575public:
577
578
579
584
585
586
587
588
589
590
592
593
594
595
597
598
599
600
601
603
604
605
608
609
610
612
613
614
615
616
617
618
620 bool AddCanonical = true);
621
622
623
624 template
625 Error create(const NameIterRange &IterRange);
626
627
628
629
630 template <typename FuncNameIterRange, typename VTableNameIterRange>
631 Error create(const FuncNameIterRange &FuncIterRange,
632 const VTableNameIterRange &VTableIterRange);
633
634
636 if (SymbolName.empty())
638 "symbol name is empty");
639
640
641
642 auto Ins = NameTab.insert(SymbolName);
643 if (Ins.second) {
644 MD5NameMap.push_back(std::make_pair(
646 Sorted = false;
647 }
649 }
650
651
652
654
655
656
659 return E;
660
661
662
663 VTableNames.insert(VTableName);
665 }
666
667 const std::vector<std::pair<uint64_t, Function *>> &getIDToNameMap() const {
668 return MD5FuncMap;
669 }
670
672
673
674
676 AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val));
677 }
678
679
680
681
683 VTableAddrMap.insert(StartAddr, EndAddr, MD5Val);
684 }
685
686
688
689
691
692
693
694
697
698
699
701
702
703
704
706
707
709 return Symbol == InstrProfSymtab::getExternalSymbol();
710 }
711
712
714
715
716
718
719
721
722
724};
725
727 Data = D;
728 Address = BaseAddr;
730}
731
732template
734 for (auto Name : IterRange)
736 return E;
737
738 finalizeSymtab();
740}
741
742template <typename FuncNameIterRange, typename VTableNameIterRange>
744 const VTableNameIterRange &VTableIterRange) {
745
746
747
748 for (StringRef Name : FuncIterRange)
750 return E;
751
752 for (StringRef VTableName : VTableIterRange)
754 return E;
755
756 finalizeSymtab();
758}
759
760void InstrProfSymtab::finalizeSymtab() const {
761 if (Sorted)
762 return;
766 AddrToMD5Map.erase(llvm::unique(AddrToMD5Map), AddrToMD5Map.end());
767 Sorted = true;
768}
769
773 return InstrProfSymtab::getExternalSymbol();
774 return Ret;
775}
776
778 finalizeSymtab();
780 [](const std::pair<uint64_t, StringRef> &LHS,
782 if (Result != MD5NameMap.end() && Result->first == MD5Hash)
783 return Result->second;
785}
786
788 finalizeSymtab();
790 [](const std::pair<uint64_t, Function *> &LHS,
792 if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash)
793 return Result->second;
794 return nullptr;
795}
796
798 return MD5VTableMap.lookup(MD5Hash);
799}
800
801
802
806 std::array<double, IPVK_Last - IPVK_First + 1> ValueCounts = {};
813};
814
815
818
820
822
832
834
836
841
846
848 double Sum2) {
849 if (Sum1 < 1.0f || Sum2 < 1.0f)
850 return 0.0f;
851 return std::min(Val1 / Sum1, Val2 / Sum2);
852 }
853};
854
855
856
861
863
865
869
870
874 return L.Value < R.Value;
875 });
876 }
877
879
880
881
884
887
888
891};
892
893
897
906 ValueData(RHS.ValueData
907 ? std::make_unique(*RHS.ValueData)
908 : nullptr) {}
913 if (.ValueData) {
914 ValueData = nullptr;
915 return *this;
916 }
917 if (!ValueData)
918 ValueData = std::make_unique(*RHS.ValueData);
919 else
920 *ValueData = *RHS.ValueData;
921 return *this;
922 }
923
924
925
927
929
930
932
933
936
937
939
940
941
945
946
947
950
951
952
955
956
958 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
959 for (auto &SR : getValueSitesForKind(Kind))
960 SR.sortByCount();
961 }
962
963
968
969
971
972
974
975
978
979
983
1007
1008private:
1009 using ValueProfData = std::array<std::vector,
1010 IPVK_Last - IPVK_First + 1>;
1011 std::unique_ptr ValueData;
1012
1014 getValueSitesForKind(uint32_t ValueKind) {
1015
1016
1017
1018 auto AR = const_cast<const InstrProfRecord *>(this)->getValueSitesForKind(
1019 ValueKind);
1022 }
1024 getValueSitesForKind(uint32_t ValueKind) const {
1025 if (!ValueData)
1026 return {};
1027 assert(IPVK_First <= ValueKind && ValueKind <= IPVK_Last &&
1028 "Unknown value kind!");
1029 return (*ValueData)[ValueKind - IPVK_First];
1030 }
1031
1032 std::vector &
1033 getOrCreateValueSitesForKind(uint32_t ValueKind) {
1034 if (!ValueData)
1035 ValueData = std::make_unique();
1036 assert(IPVK_First <= ValueKind && ValueKind <= IPVK_Last &&
1037 "Unknown value kind!");
1038 return (*ValueData)[ValueKind - IPVK_First];
1039 }
1040
1041
1042 uint64_t remapValue(uint64_t Value, uint32_t ValueKind,
1043 InstrProfSymtab *SymTab);
1044
1045
1046
1047 void mergeValueProfData(uint32_t ValkeKind, InstrProfRecord &Src,
1048 uint64_t Weight,
1050
1051
1052 void scaleValueProfData(uint32_t ValueKind, uint64_t N, uint64_t D,
1054};
1055
1059
1060
1062
1064
1067 std::vector<uint64_t> Counts)
1070 std::vector<uint64_t> Counts,
1074
1081};
1082
1085 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1086 NumValueKinds += !(getValueSitesForKind(Kind).empty());
1087 return NumValueKinds;
1088}
1089
1092 for (const auto &SR : getValueSitesForKind(ValueKind))
1093 N += SR.ValueData.size();
1094 return N;
1095}
1096
1098 return getValueSitesForKind(ValueKind).size();
1099}
1100
1103 return getValueSitesForKind(ValueKind)[Site].ValueData;
1104}
1105
1108 return;
1109 getOrCreateValueSitesForKind(ValueKind).reserve(NumValueSites);
1110}
1111
1112
1113#define INSTR_PROF_VALUE_PROF_DATA
1115
1118 ValueData, [](const InstrProfValueData &L, const InstrProfValueData &R) {
1119 return L.Count > R.Count;
1120 });
1121
1125}
1126
1128
1133
1135 switch (Type) {
1138 }
1140}
1141
1143
1145
1146
1147
1148
1150
1151
1152
1153
1155
1156
1158
1159
1161
1163
1164
1166
1168
1170
1172
1174
1176
1178
1180
1182};
1184
1186
1188
1189
1190
1191
1194
1195
1196
1200
1201
1202
1208
1209
1210
1211
1212
1213
1215
1216
1217
1219
1220
1221
1223};
1224
1225
1226
1227
1235
1236
1237
1239
1241
1243
1244
1245
1247
1249
1251
1254 };
1255
1256
1258
1260
1263
1264 void operator delete(void *ptr) { ::operator delete(ptr); }
1265
1268 NumSumFields * sizeof(uint64_t);
1269 }
1270
1272 return reinterpret_cast<const uint64_t *>(this + 1);
1273 }
1274
1276 return reinterpret_cast<uint64_t *>(this + 1);
1277 }
1278
1280 return reinterpret_cast<const Entry *>(
1282 }
1283
1287
1291
1295
1297
1304};
1305
1307 return std::unique_ptr
1309}
1310
1311}
1312
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1330
1335
1339
1340
1341
1342
1343
1344
1346#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
1348};
1349
1351#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Type Name;
1353};
1354
1355
1356
1357
1358
1360#define INSTR_PROF_RAW_HEADER(Type, Name, Init) const Type Name;
1362};
1363
1364}
1365
1366
1368
1369
1370
1372
1373}
1374#endif
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file defines the DenseMap class.
#define INSTR_PROF_QUOTE(x)
#define INSTR_PROF_RAW_MAGIC_32
#define INSTR_PROF_MAX_NUM_VAL_PER_SITE
#define INSTR_PROF_RAW_VERSION
#define INSTR_PROF_PROFILE_BITMAP_BIAS_VAR
#define INSTR_PROF_INDEX_VERSION
#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR
#define INSTR_PROF_VALUE_PROF_FUNC_STR
#define INSTR_PROF_RAW_MAGIC_64
#define INSTR_PROF_PROFILE_RUNTIME_VAR
#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR
This file implements a coalescing interval map for small objects.
Machine Check Debug Module
StringSet - A set-like wrapper for the StringMap.
The Input class is used to parse a yaml document into in-memory structs and vectors.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Base class for user error types.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
LinkageTypes
An enumeration for the kinds of linkage for global values.
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition InstrProf.h:458
static std::pair< instrprof_error, std::string > take(Error E)
Consume an Error and return the raw enum value contained within it, and the optional error message.
Definition InstrProf.h:470
const std::string & getMessage() const
Definition InstrProf.h:465
static char ID
Definition InstrProf.h:481
instrprof_error get() const
Definition InstrProf.h:464
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition InstrProf.h:460
std::string message() const override
Return the error message as a string.
InstrProfError(instrprof_error Err, const Twine &ErrStr=Twine())
Definition InstrProf.h:451
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
Definition InstrProf.h:505
StringRef getFuncOrVarName(uint64_t ValMD5Hash) const
Return name of functions or global variables from the name's md5 hash value.
Definition InstrProf.h:777
static LLVM_ABI StringRef getCanonicalName(StringRef PGOName)
static bool isExternalSymbol(const StringRef &Symbol)
True if Symbol is the value used to represent external symbols.
Definition InstrProf.h:708
const std::vector< std::pair< uint64_t, Function * > > & getIDToNameMap() const
Definition InstrProf.h:667
Error addSymbolName(StringRef SymbolName)
Definition InstrProf.h:635
LLVM_ABI StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize) const
Return function's PGO name from the function name's symbol address in the object file.
InstrProfSymtab & operator=(InstrProfSymtab &&)=delete
GlobalVariable * getGlobalVariable(uint64_t MD5Hash) const
Return the global variable corresponding to md5 hash.
Definition InstrProf.h:797
StringRef getFuncOrVarNameIfDefined(uint64_t ValMD5Hash) const
Just like getFuncOrVarName, except that it will return literal string 'External Symbol' if the functi...
Definition InstrProf.h:770
void mapAddress(uint64_t Addr, uint64_t MD5Val)
Map a function address to its name's MD5 hash.
Definition InstrProf.h:675
Error addVTableName(StringRef VTableName)
Adds VTableName as a known symbol, and inserts it to a map that tracks all vtable names.
Definition InstrProf.h:657
std::vector< std::pair< uint64_t, uint64_t > > AddrHashMap
Definition InstrProf.h:507
LLVM_ABI void dumpNames(raw_ostream &OS) const
Dump the symbols in this table.
StringRef getNameData() const
Return the name section data.
Definition InstrProf.h:720
LLVM_ABI Error create(object::SectionRef &Section)
Create InstrProfSymtab from an object file section which contains function PGO names.
Error addFuncName(StringRef FuncName)
The method name is kept since there are many callers.
Definition InstrProf.h:653
void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val)
Map the address range (i.e., [start_address, end_address)) of a variable to its names' MD5 hash.
Definition InstrProf.h:682
InstrProfSymtab()
Definition InstrProf.h:576
LLVM_ABI Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames)
Initialize 'this' with the set of vtable names encoded in CompressedVTableNames.
const StringSet & getVTableNames() const
Definition InstrProf.h:671
LLVM_ABI uint64_t getVTableHashFromAddress(uint64_t Address) const
Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab.
Function * getFunction(uint64_t FuncMD5Hash) const
Return function from the name's md5 hash. Return nullptr if not found.
Definition InstrProf.h:787
InstrProfSymtab & operator=(const InstrProfSymtab &)=delete
InstrProfSymtab(InstrProfSymtab &&)=delete
InstrProfSymtab(const InstrProfSymtab &)=delete
LLVM_ABI uint64_t getFunctionHashFromAddress(uint64_t Address) const
Return a function's hash, or 0, if the function isn't in this SymTab.
typename Sizer::Allocator Allocator
A Module instance is used to store all the information related to an LLVM module.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
raw_ostream & OS
Definition InstrProf.h:87
LLVM_ABI uint64_t tell() const
LLVM_ABI void writeByte(uint8_t V)
LLVM_ABI void patch(ArrayRef< PatchItem > P)
LLVM_ABI void write32(uint32_t V)
bool IsFDOStream
Definition InstrProf.h:86
support::endian::Writer LE
Definition InstrProf.h:88
LLVM_ABI ProfOStream(raw_fd_ostream &FD)
LLVM_ABI void write(uint64_t V)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
constexpr bool empty() const
empty - Check if the string is empty.
StringSet - A wrapper for StringMap that provides set-like functionality.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
An efficient, type-erasing, non-owning reference to a callable.
This is a value type class that represents a single section in the list of sections in the object fil...
A raw_ostream that writes to a file descriptor.
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition InstrProf.h:494
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
Definition InstrProf.h:1306
uint64_t ComputeHash(StringRef K)
Definition InstrProf.h:1187
const uint64_t Version
Definition InstrProf.h:1183
HashT
Definition InstrProf.h:1129
@ MD5
Definition InstrProf.h:1130
@ Last
Definition InstrProf.h:1131
const uint64_t Magic
Definition InstrProf.h:1142
const HashT HashType
Definition InstrProf.h:1185
ProfVersion
Definition InstrProf.h:1144
@ Version13
Definition InstrProf.h:1179
@ Version5
Definition InstrProf.h:1162
@ CurrentVersion
Definition InstrProf.h:1181
@ Version9
Definition InstrProf.h:1171
@ Version6
Definition InstrProf.h:1165
@ Version7
Definition InstrProf.h:1167
@ Version3
Definition InstrProf.h:1157
@ Version10
Definition InstrProf.h:1173
@ Version4
Definition InstrProf.h:1160
@ Version11
Definition InstrProf.h:1175
@ Version12
Definition InstrProf.h:1177
@ Version2
Definition InstrProf.h:1154
@ Version8
Definition InstrProf.h:1169
@ Version1
Definition InstrProf.h:1149
Definition InstrProf.h:1313
const uint64_t Version
Definition InstrProf.h:1329
uint64_t getMagic< uint32_t >()
Definition InstrProf.h:1336
uint64_t getMagic< uint64_t >()
Definition InstrProf.h:1332
constexpr size_t NameSize
uint64_t MD5Hash(const FunctionId &Obj)
This is an optimization pass for GlobalISel generic memory operations.
FunctionAddr VTableAddr Value
Definition InstrProf.h:137
void stable_sort(R &&Range)
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition InstrProf.h:123
LLVM_ABI std::string getPGOFuncName(const Function &F, bool InLTO=false, uint64_t Version=INSTR_PROF_INDEX_VERSION)
Please use getIRPGOFuncName for LLVM IR instrumentation.
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
Definition InstrProf.h:194
LLVM_ABI void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName)
Create the PGOFuncName meta data if PGOFuncName is different from function's raw name.
LLVM_ABI std::string getIRPGOFuncName(const Function &F, bool InLTO=false)
std::error_code make_error_code(BitcodeError E)
StringRef getPGOFuncNameMetadataName()
Definition InstrProf.h:341
StringRef getCoverageMappingVarName()
Return the name of a covarage mapping variable (internal linkage) for each instrumented source module...
Definition InstrProf.h:152
void handleAllErrors(Error E, HandlerTs &&... Handlers)
Behaves the same as handleErrors, except that by contract all errors must be handled by the given han...
StringRef getInstrProfBitmapVarPrefix()
Return the name prefix of profile bitmap variables.
Definition InstrProf.h:135
LLVM_ABI cl::opt< bool > DoInstrProfNameCompression
LLVM_ABI StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName="")
Given a PGO function name, remove the filename prefix and return the original (static) function name.
FuncHash
Definition InstrProf.h:78
LLVM_ABI void createPGONameMetadata(GlobalObject &GO, StringRef PGOName)
Create the PGOName metadata if a global object's PGO name is different from its mangled name.
StringRef getInstrProfVTableNamesVarName()
Definition InstrProf.h:147
LLVM_ABI std::pair< StringRef, StringRef > getParsedIRPGOName(StringRef IRPGOName)
auto unique(Range &&R, Predicate P)
LLVM_ABI MDNode * getPGOFuncNameMetadata(const Function &F)
Return the PGOFuncName meta data associated with a function.
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
Definition InstrProf.h:129
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
Definition InstrProf.h:160
LLVM_ABI std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
uint64_t getInstrMaxCountValue()
Return the max count value. We reserver a few large values for special use.
Definition InstrProf.h:97
LLVM_ABI bool needsComdatForCounter(const GlobalObject &GV, const Module &M)
Check if we can use Comdat for profile variables.
LLVM_ABI std::string getPGOName(const GlobalVariable &V, bool InLTO=false)
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
Definition InstrProf.h:189
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
Definition InstrProf.h:138
StringRef getInstrProfCounterBiasVarName()
Definition InstrProf.h:204
LLVM_ABI GlobalVariable * createPGOFuncNameVar(Function &F, StringRef PGOFuncName)
Create and return the global variable for function name used in PGO instrumentation.
LLVM_ABI void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
Definition InstrProf.h:200
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Definition InstrProf.h:169
LLVM_ABI Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)
Produce Result string with the same format described above.
void sort(IteratorTy Start, IteratorTy End)
InstrProfSectKind
Definition InstrProf.h:91
LLVM_ABI Error readAndDecodeStrings(StringRef NameStrings, std::function< Error(StringRef)> NameCallback)
NameStrings is a string composed of one or more possibly encoded sub-strings.
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
Definition InstrProf.h:132
FunctionAddr NumValueSites[IPVK_Last+1]
Definition InstrProf.h:93
LLVM_ABI StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
StringRef getInstrProfBitmapBiasVarName()
Definition InstrProf.h:208
StringRef getInstrProfNameSeparator()
Return the marker used to separate PGO names during serialization.
Definition InstrProf.h:213
LLVM_ABI SmallVector< InstrProfValueData, 4 > getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst and returns them if Inst is annotated with value profile dat...
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
Definition InstrProf.h:118
MutableArrayRef(T &OneElt) -> MutableArrayRef< T >
Error make_error(ArgTs &&... Args)
Make a Error instance representing failure using the given error info type.
@ LLVM_MARK_AS_BITMASK_ENUM
FunctionAddr VTableAddr uintptr_t uintptr_t Data
Definition InstrProf.h:189
instrprof_error
Definition InstrProf.h:398
@ zlib_unavailable
Definition InstrProf.h:422
@ success
Definition InstrProf.h:399
@ unable_to_correlate_profile
Definition InstrProf.h:411
@ raw_profile_version_mismatch
Definition InstrProf.h:423
@ counter_value_too_large
Definition InstrProf.h:424
@ uncompress_failed
Definition InstrProf.h:420
@ missing_correlation_info
Definition InstrProf.h:409
@ counter_overflow
Definition InstrProf.h:417
@ value_site_count_mismatch
Definition InstrProf.h:418
@ malformed
Definition InstrProf.h:408
@ too_large
Definition InstrProf.h:406
@ compress_failed
Definition InstrProf.h:419
@ truncated
Definition InstrProf.h:407
@ unknown_function
Definition InstrProf.h:412
@ count_mismatch
Definition InstrProf.h:415
@ invalid_prof
Definition InstrProf.h:413
@ hash_mismatch
Definition InstrProf.h:414
@ unsupported_hash_type
Definition InstrProf.h:405
@ unrecognized_format
Definition InstrProf.h:401
@ empty_raw_profile
Definition InstrProf.h:421
@ bitmap_mismatch
Definition InstrProf.h:416
@ unexpected_correlation_info
Definition InstrProf.h:410
InstrProfValueKind
Definition InstrProf.h:311
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
Definition InstrProf.h:181
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
LLVM_ABI const std::error_category & instrprof_category()
LLVM_ABI Error collectVTableStrings(ArrayRef< GlobalVariable * > VTables, std::string &Result, bool doCompression)
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
LLVM_ABI void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
LLVM_ABI Error collectGlobalObjectNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (names of global objects like functions or, virtual tables) NameStrs,...
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
Definition InstrProf.h:112
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Definition InstrProf.h:175
LLVM_ABI std::string getPGOFuncNameVarName(StringRef FuncName, GlobalValue::LinkageTypes Linkage)
Return the name of the global variable used to store a function name in PGO instrumentation.
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
Definition InstrProf.h:145
LogicalResult success(bool IsSuccess=true)
Utility function to generate a LogicalResult.
LLVM_ABI bool isGPUProfTarget(const Module &M)
Determines whether module targets a GPU eligable for PGO instrumentation.
LLVM_ABI bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
StringRef getPGONameMetadataName()
Definition InstrProf.h:343
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
Definition InstrProf.h:141
StringRef getInstrProfVTableVarPrefix()
Return the name prefix of variables containing virtual table profile data.
Definition InstrProf.h:126
InstrProfKind
An enum describing the attributes of an instrumented profile.
Definition InstrProf.h:373
@ ContextSensitive
Definition InstrProf.h:382
@ FunctionEntryOnly
Definition InstrProf.h:386
@ LoopEntriesInstrumentation
Definition InstrProf.h:392
@ FunctionEntryInstrumentation
Definition InstrProf.h:380
@ TemporalProfile
Definition InstrProf.h:390
@ IRInstrumentation
Definition InstrProf.h:378
@ SingleByteCoverage
Definition InstrProf.h:384
@ FrontendInstrumentation
Definition InstrProf.h:376
@ MemProf
Definition InstrProf.h:388
Implement std::hash so that hash_code can be used in STL containers.
Definition InstrProf.h:803
CountSumOrPercent()=default
void reset()
Definition InstrProf.h:808
std::array< double, IPVK_Last - IPVK_First+1 > ValueCounts
Definition InstrProf.h:806
double CountSum
Definition InstrProf.h:805
uint64_t NumEntries
Definition InstrProf.h:804
Definition InstrProf.h:1229
uint64_t Cutoff
The required percentile of total execution count.
Definition InstrProf.h:1230
uint64_t NumBlocks
Number of blocks >= the minumum execution count.
Definition InstrProf.h:1233
uint64_t MinBlockCount
The minimum execution count for this percentile.
Definition InstrProf.h:1232
Definition InstrProf.h:1228
const Entry * getCutoffEntryBase() const
Definition InstrProf.h:1279
uint64_t get(SummaryFieldKind K) const
Definition InstrProf.h:1288
void set(SummaryFieldKind K, uint64_t V)
Definition InstrProf.h:1292
Entry * getCutoffEntryBase()
Definition InstrProf.h:1284
void setEntry(uint32_t I, const ProfileSummaryEntry &E)
Definition InstrProf.h:1298
uint64_t * getSummaryDataBase()
Definition InstrProf.h:1275
SummaryFieldKind
Definition InstrProf.h:1238
@ TotalNumFunctions
The total number of functions instrumented.
Definition InstrProf.h:1240
@ TotalNumBlocks
Total number of instrumented blocks/edges.
Definition InstrProf.h:1242
@ MaxFunctionCount
The maximal execution count among all functions.
Definition InstrProf.h:1246
@ TotalBlockCount
The sum of all instrumented block counts.
Definition InstrProf.h:1252
@ MaxBlockCount
Max block count of the program.
Definition InstrProf.h:1248
@ MaxInternalBlockCount
Max internal block count of the program (excluding entry blocks).
Definition InstrProf.h:1250
@ NumKinds
Definition InstrProf.h:1253
const uint64_t * getSummaryDataBase() const
Definition InstrProf.h:1271
static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries)
Definition InstrProf.h:1266
uint64_t NumSummaryFields
Definition InstrProf.h:1257
const Entry & getEntry(uint32_t I) const
Definition InstrProf.h:1296
Summary(uint32_t Size)
Definition InstrProf.h:1262
uint64_t NumCutoffEntries
Definition InstrProf.h:1259
Profiling information for a single function.
Definition InstrProf.h:894
LLVM_ABI void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap of value profile counts.
InstrProfRecord()=default
std::vector< uint64_t > Counts
Definition InstrProf.h:895
ArrayRef< InstrProfValueData > getValueArrayForSite(uint32_t ValueKind, uint32_t Site) const
Return the array of profiled values at Site.
Definition InstrProf.h:1102
CountPseudoKind getCountPseudoKind() const
Definition InstrProf.h:993
InstrProfRecord(std::vector< uint64_t > Counts)
Definition InstrProf.h:899
LLVM_ABI void accumulateCounts(CountSumOrPercent &Sum) const
Compute the sums of all counts and store in Sum.
uint32_t getNumValueSites(uint32_t ValueKind) const
Return the number of instrumented sites for ValueKind.
Definition InstrProf.h:1097
uint32_t getNumValueKinds() const
Return the number of value profile kinds with non-zero number of profile sites.
Definition InstrProf.h:1083
void setPseudoCount(CountPseudoKind Kind)
Definition InstrProf.h:1001
InstrProfRecord(InstrProfRecord &&)=default
uint32_t getNumValueData(uint32_t ValueKind) const
Return the total number of ValueData for ValueKind.
Definition InstrProf.h:1090
LLVM_ABI void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
LLVM_ABI void addValueData(uint32_t ValueKind, uint32_t Site, ArrayRef< InstrProfValueData > VData, InstrProfSymtab *SymTab)
Add ValueData for ValueKind at value Site.
CountPseudoKind
Definition InstrProf.h:984
@ PseudoWarm
Definition InstrProf.h:987
@ NotPseudo
Definition InstrProf.h:985
@ PseudoHot
Definition InstrProf.h:986
InstrProfRecord & operator=(const InstrProfRecord &RHS)
Definition InstrProf.h:910
void clearValueData()
Clear value data entries.
Definition InstrProf.h:970
InstrProfRecord(const InstrProfRecord &RHS)
Definition InstrProf.h:904
InstrProfRecord(std::vector< uint64_t > Counts, std::vector< uint8_t > BitmapBytes)
Definition InstrProf.h:900
void reserveSites(uint32_t ValueKind, uint32_t NumValueSites)
Reserve space for NumValueSites sites.
Definition InstrProf.h:1106
LLVM_ABI void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
void sortValueData()
Sort value profile data (per site) by count.
Definition InstrProf.h:957
std::vector< uint8_t > BitmapBytes
Definition InstrProf.h:896
PseudoCountVal
Definition InstrProf.h:989
@ WarmFunctionVal
Definition InstrProf.h:991
@ HotFunctionVal
Definition InstrProf.h:990
InstrProfRecord & operator=(InstrProfRecord &&)=default
void Clear()
Clear value data entries and edge counters.
Definition InstrProf.h:964
LLVM_ABI void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
Definition InstrProf.h:862
void sortByCount()
Sort ValueData Descending by Count.
Definition InstrProf.h:1116
InstrProfValueSiteRecord(std::vector< InstrProfValueData > &&VD)
Definition InstrProf.h:867
InstrProfValueSiteRecord()=default
void sortByTargetValues()
Sort ValueData ascending by Value.
Definition InstrProf.h:871
std::vector< InstrProfValueData > ValueData
Value profiling data pairs at a given value site.
Definition InstrProf.h:864
LLVM_ABI void merge(InstrProfValueSiteRecord &Input, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge data from another InstrProfValueSiteRecord Optionally scale merged counts by Weight.
LLVM_ABI void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap)
Compute the overlap b/w this record and Input record.
LLVM_ABI void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up value profile data counts by N (Numerator) / D (Denominator).
static bool hasCSFlagInHash(uint64_t FuncHash)
Definition InstrProf.h:1075
NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector< uint64_t > Counts)
Definition InstrProf.h:1066
NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector< uint64_t > Counts, std::vector< uint8_t > BitmapBytes)
Definition InstrProf.h:1069
StringRef Name
Definition InstrProf.h:1057
uint64_t Hash
Definition InstrProf.h:1058
static void setCSFlagInHash(uint64_t &FuncHash)
Definition InstrProf.h:1078
static constexpr uint64_t FUNC_HASH_MASK
Definition InstrProf.h:1061
NamedInstrProfRecord()=default
static constexpr unsigned CS_FLAG_IN_FUNC_HASH
Definition InstrProf.h:1063
Definition InstrProf.h:857
uint64_t ValueCutoff
Definition InstrProf.h:858
const std::string NameFilter
Definition InstrProf.h:859
Definition InstrProf.h:816
LLVM_ABI void addOneMismatch(const CountSumOrPercent &MismatchFunc)
static double score(uint64_t Val1, uint64_t Val2, double Sum1, double Sum2)
Definition InstrProf.h:847
LLVM_ABI Error accumulateCounts(const std::string &BaseFilename, const std::string &TestFilename, bool IsCS)
LLVM_ABI void dump(raw_fd_ostream &OS) const
CountSumOrPercent Overlap
Definition InstrProf.h:823
CountSumOrPercent Base
Definition InstrProf.h:819
bool Valid
Definition InstrProf.h:831
uint64_t FuncHash
Definition InstrProf.h:830
OverlapStatsLevel
Definition InstrProf.h:817
@ FunctionLevel
Definition InstrProf.h:817
@ ProgramLevel
Definition InstrProf.h:817
LLVM_ABI void addOneUnique(const CountSumOrPercent &UniqueFunc)
const std::string * BaseFilename
Definition InstrProf.h:827
const std::string * TestFilename
Definition InstrProf.h:828
void setFuncInfo(StringRef Name, uint64_t Hash)
Definition InstrProf.h:837
CountSumOrPercent Unique
Definition InstrProf.h:825
CountSumOrPercent Mismatch
Definition InstrProf.h:824
StringRef FuncName
Definition InstrProf.h:829
OverlapStatsLevel Level
Definition InstrProf.h:826
OverlapStats(OverlapStatsLevel L=ProgramLevel)
Definition InstrProf.h:833
CountSumOrPercent Test
Definition InstrProf.h:821
Definition InstrProf.h:62
uint64_t Pos
Definition InstrProf.h:63
ArrayRef< uint64_t > D
Definition InstrProf.h:64
Definition InstrProf.h:1345
Definition InstrProf.h:1350
InstrProfValueData
Definition InstrProf.h:873
static LLVM_ABI void createBPFunctionNodes(ArrayRef< TemporalProfTraceTy > Traces, std::vector< BPFunctionNode > &Nodes, bool RemoveOutlierUNs=true)
Use a set of temporal profile traces to create a list of balanced partitioning function nodes used by...
std::vector< uint64_t > FunctionNameRefs
Definition InstrProf.h:430
TemporalProfTraceTy(std::initializer_list< uint64_t > Trace={}, uint64_t Weight=1)
Definition InstrProf.h:432
uint64_t Weight
Definition InstrProf.h:431
Per-function header/control data structure for value profiling data in indexed format.
Definition InstrProf.h:424
Function object to check whether the first component of a container supported by std::get (like std::...
Adapter to write values to a stream in a particular byte order.