LLVM: lib/Transforms/IPO/SampleProfileProbe.cpp Source File (original) (raw)
1
2
3
4
5
6
7
8
9
10
11
12
33#include <unordered_set>
34#include
35
36using namespace llvm;
37#define DEBUG_TYPE "pseudo-probe"
38
40 "Number of probes that have an artificial debug line");
41
44 cl::desc("Do pseudo probe verification"));
45
47 "verify-pseudo-probe-funcs", cl::Hidden,
48 cl::desc("The option to specify the name of the functions to verify."));
49
52 cl::desc("Update pseudo probe distribution factor"));
53
56 const DILocation *InlinedAt = DIL ? DIL->getInlinedAt() : nullptr;
57 while (InlinedAt) {
58 Hash ^= MD5Hash(std::to_string(InlinedAt->getLine()));
59 Hash ^= MD5Hash(std::to_string(InlinedAt->getColumn()));
60 auto Name = InlinedAt->getSubprogramLinkageName();
62 InlinedAt = InlinedAt->getInlinedAt();
63 }
64 return Hash;
65}
66
70
71bool PseudoProbeVerifier::shouldVerifyFunction(const Function *F) {
72
73 if (F->isDeclaration())
74 return false;
75
76
77 if (F->hasAvailableExternallyLinkage())
78 return false;
79
80 static std::unordered_setstd::string VerifyFuncNames(
82 return VerifyFuncNames.empty() || VerifyFuncNames.count(F->getName().str());
83}
84
87 PIC.registerAfterPassCallback(
90 });
91 }
92}
93
94
96 std::string Banner =
97 "\n*** Pseudo Probe Verification After " + PassID.str() + " ***\n";
98 dbgs() << Banner;
107 else
109}
110
114}
115
119}
120
122 if (!shouldVerifyFunction(F))
123 return;
125 for (const auto &BB : *F)
126 collectProbeFactors(&BB, ProbeFactors);
127 verifyProbeFactors(F, ProbeFactors);
128}
129
131 const Function *F = L->getHeader()->getParent();
133}
134
135void PseudoProbeVerifier::collectProbeFactors(const BasicBlock *Block,
137 for (const auto &I : *Block) {
138 if (std::optional Probe = extractProbe(I)) {
140 ProbeFactors[{Probe->Id, Hash}] += Probe->Factor;
141 }
142 }
143}
144
145void PseudoProbeVerifier::verifyProbeFactors(
147 bool BannerPrinted = false;
148 auto &PrevProbeFactors = FunctionProbeFactors[F->getName()];
149 for (const auto &I : ProbeFactors) {
150 float CurProbeFactor = I.second;
151 auto [It, Inserted] = PrevProbeFactors.try_emplace(I.first);
152 if (!Inserted) {
153 float PrevProbeFactor = It->second;
154 if (std::abs(CurProbeFactor - PrevProbeFactor) >
155 DistributionFactorVariance) {
156 if (!BannerPrinted) {
157 dbgs() << "Function " << F->getName() << ":\n";
158 BannerPrinted = true;
159 }
160 dbgs() << "Probe " << I.first.first << "\tprevious factor "
161 << format("%0.2f", PrevProbeFactor) << "\tcurrent factor "
162 << format("%0.2f", CurProbeFactor) << "\n";
163 }
164 }
165
166
167 It->second = I.second;
168 }
169}
170
172 BlockProbeIds.clear();
173 CallProbeIds.clear();
175
178 computeBlocksToIgnore(BlocksToIgnore, BlocksAndCallsToIgnore);
179
180 computeProbeId(BlocksToIgnore, BlocksAndCallsToIgnore);
181 computeCFGHash(BlocksToIgnore);
182}
183
184
185
186
187
188
189
190
191
192
193void SampleProfileProber::computeBlocksToIgnore(
196
198 findUnreachableBlocks(BlocksAndCallsToIgnore);
199
200 BlocksToIgnore.insert_range(BlocksAndCallsToIgnore);
201
202
203
204
205
206
207
208 findInvokeNormalDests(BlocksToIgnore);
209}
210
211
212void SampleProfileProber::findUnreachableBlocks(
214 for (auto &BB : *F) {
215 if (&BB != &F->getEntryBlock() && pred_size(&BB) == 0)
216 BlocksToIgnore.insert(&BB);
217 }
218}
219
220
221
222void SampleProfileProber::findInvokeNormalDests(
224 for (auto &BB : *F) {
225 auto *TI = BB.getTerminator();
227 auto *ND = II->getNormalDest();
228 InvokeNormalDests.insert(ND);
229
230
231
235 InvokeNormalDests.insert(Pred);
236 ND = Pred;
237 } else
238 break;
239 }
240 }
241 }
242}
243
244
245
246
247
248
249const Instruction *SampleProfileProber::getOriginalTerminator(
253 return getOriginalTerminator(II->getNormalDest(), BlocksToIgnore);
254 } else if (succ_size(Head) == 1 &&
256
257 return getOriginalTerminator(*succ_begin(Head), BlocksToIgnore);
258 }
259 return TI;
260}
261
262
263
264
265
266void SampleProfileProber::computeCFGHash(
268 std::vector<uint8_t> Indexes;
269 JamCRC JC;
270 for (auto &BB : *F) {
271 if (BlocksToIgnore.contains(&BB))
272 continue;
273
274 auto *TI = getOriginalTerminator(&BB, BlocksToIgnore);
275 for (unsigned I = 0, E = TI->getNumSuccessors(); I != E; ++I) {
276 auto *Succ = TI->getSuccessor(I);
277 auto Index = getBlockId(Succ);
278
279 if (Index == 0)
280 continue;
281 for (int J = 0; J < 4; J++)
282 Indexes.push_back((uint8_t)(Index >> (J * 8)));
283 }
284 }
285
287
288 FunctionHash = (uint64_t)CallProbeIds.size() << 48 |
289 (uint64_t)Indexes.size() << 32 | JC.getCRC();
290
291 FunctionHash &= 0x0FFFFFFFFFFFFFFF;
292 assert(FunctionHash && "Function checksum should not be zero");
293 LLVM_DEBUG(dbgs() << "\nFunction Hash Computation for " << F->getName()
294 << ":\n"
295 << " CRC = " << JC.getCRC() << ", Edges = "
296 << Indexes.size() << ", ICSites = " << CallProbeIds.size()
297 << ", Hash = " << FunctionHash << "\n");
298}
299
300void SampleProfileProber::computeProbeId(
303 LLVMContext &Ctx = F->getContext();
304 Module *M = F->getParent();
305
306 for (auto &BB : *F) {
307 if (!BlocksToIgnore.contains(&BB))
308 BlockProbeIds[&BB] = ++LastProbeId;
309
310 if (BlocksAndCallsToIgnore.contains(&BB))
311 continue;
312 for (auto &I : BB) {
314 continue;
315
316
317
318 if (LastProbeId >= 0xFFFF) {
319 std::string Msg = "Pseudo instrumentation incomplete for " +
320 std::string(F->getName()) + " because it's too large";
322 DiagnosticInfoSampleProfile(M->getName().data(), Msg, DS_Warning));
323 return;
324 }
325
326 CallProbeIds[&I] = ++LastProbeId;
327 }
328 }
329}
330
331uint32_t SampleProfileProber::getBlockId(const BasicBlock *BB) const {
332 auto I = BlockProbeIds.find(const_cast<BasicBlock *>(BB));
333 return I == BlockProbeIds.end() ? 0 : I->second;
334}
335
336uint32_t SampleProfileProber::getCallsiteId(const Instruction *Call) const {
337 auto Iter = CallProbeIds.find(const_cast<Instruction *>(Call));
338 return Iter == CallProbeIds.end() ? 0 : Iter->second;
339}
340
342 Module *M = F.getParent();
344
345
346
348 if (auto *SP = F.getSubprogram()) {
349 FName = SP->getLinkageName();
350 if (FName.empty())
351 FName = SP->getName();
352 }
354
355
356
357
358
359
362 "Expecting pseudo probe or call instructions");
363 if (->getDebugLoc()) {
364 if (auto *SP = F.getSubprogram()) {
366 I->setDebugLoc(DIL);
367 ArtificialDbgLine++;
369 dbgs() << "\nIn Function " << F.getName()
370 << " Probe gets an artificial debug line\n";
371 I->dump();
372 });
373 }
374 }
375 };
376
377
378 for (auto &I : BlockProbeIds) {
381
382
383
384
385
386
387 auto HasValidDbgLine = [](Instruction *J) {
388 return (J) && !J->isLifetimeStartOrEnd() && J->getDebugLoc();
389 };
390
392 while (J != BB->getTerminator() && !HasValidDbgLine(J)) {
394 }
395
397 assert(Builder.GetInsertPoint() != BB->end() &&
398 "Cannot get the probing point");
401 Value *Args[] = {Builder.getInt64(Guid), Builder.getInt64(Index),
402 Builder.getInt32(0),
404 auto *Probe = Builder.CreateCall(ProbeFn, Args);
405 AssignDebugLoc(Probe);
406
407
408 if (auto DIL = Probe->getDebugLoc()) {
409 if (DIL->getDiscriminator()) {
410 DIL = DIL->cloneWithDiscriminator(0);
411 Probe->setDebugLoc(DIL);
412 }
413 }
414 }
415
416
417
418
419 for (auto &I : CallProbeIds) {
425 AssignDebugLoc(Call);
426 if (auto DIL = Call->getDebugLoc()) {
427
428
429
432 DIL->getBaseDiscriminator());
433 DIL = DIL->cloneWithDiscriminator(V);
434 Call->setDebugLoc(DIL);
435 }
436 }
437
438
439
440
441
442
443 auto Hash = getFunctionHash();
446 assert(NMD && "llvm.pseudo_probe_desc should be pre-created");
447 NMD->addOperand(MD);
448}
449
452
453
454
456
457 for (auto &F : M) {
458 if (F.isDeclaration())
459 continue;
462 }
463
465}
466
467void PseudoProbeUpdatePass::runOnFunction(Function &F,
470 auto BBProfileCount = [&BFI](BasicBlock *BB) {
472 };
473
474
478 if (std::optional Probe = extractProbe(I)) {
480 ProbeFactors[{Probe->Id, Hash}] += BBProfileCount(&Block);
481 }
482 }
483 }
484
485
488 if (std::optional Probe = extractProbe(I)) {
490 float Sum = ProbeFactors[{Probe->Id, Hash}];
491 if (Sum != 0)
493 }
494 }
495 }
496}
497
501 for (auto &F : M) {
502 if (F.isDeclaration())
503 continue;
507 }
508 }
510}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Module.h This file contains the declarations for the Module class.
Legalize the Machine IR a function s Machine IR
Machine Check Debug Module
uint64_t IntrinsicInst * II
FunctionAnalysisManager FAM
PassInstrumentationCallbacks PIC
static cl::opt< bool > UpdatePseudoProbe("update-pseudo-probe", cl::init(true), cl::Hidden, cl::desc("Update pseudo probe distribution factor"))
static cl::opt< bool > VerifyPseudoProbe("verify-pseudo-probe", cl::init(false), cl::Hidden, cl::desc("Do pseudo probe verification"))
static cl::list< std::string > VerifyPseudoProbeFuncList("verify-pseudo-probe-funcs", cl::Hidden, cl::desc("The option to specify the name of the functions to verify."))
static uint64_t computeCallStackHash(const Instruction &Inst)
Definition SampleProfileProbe.cpp:67
static uint64_t getCallStackHash(const DILocation *DIL)
Definition SampleProfileProbe.cpp:54
This file provides the interface for the pseudo probe implementation for AutoFDO.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
LLVM Basic Block Representation.
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
LLVM_ABI std::optional< uint64_t > getBlockProfileCount(const BasicBlock *BB, bool AllowSynthetic=false) const
Returns the estimated profile count of BB.
Implements a dense probed hash-table based set.
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI void update(ArrayRef< uint8_t > Data)
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
A node in the call graph.
An SCC of the call graph.
Represents a single loop in the control flow graph.
LLVM_ABI MDNode * createPseudoProbeDesc(uint64_t GUID, uint64_t Hash, StringRef FName)
Return metadata containing the pseudo probe descriptor for a function.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
A Module instance is used to store all the information related to an LLVM module.
This class manages callbacks registration, as well as provides a way for PassInstrumentation to pass ...
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Definition SampleProfileProbe.cpp:498
LLVM_ABI void registerCallbacks(PassInstrumentationCallbacks &PIC)
Definition SampleProfileProbe.cpp:85
LLVM_ABI void runAfterPass(StringRef PassID, Any IR)
Definition SampleProfileProbe.cpp:95
LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Definition SampleProfileProbe.cpp:450
Sample profile pseudo prober.
LLVM_ABI SampleProfileProber(Function &F)
Definition SampleProfileProbe.cpp:171
LLVM_ABI void instrumentOneFunc(Function &F, TargetMachine *TM)
Definition SampleProfileProbe.cpp:341
StringRef - Represent a constant reference to a string, i.e.
std::string str() const
str - Get the contents as an std::string.
constexpr bool empty() const
empty - Check if the string is empty.
Primary interface to the complete machine description for the target machine.
The instances of the Type class are immutable: once they are created, they are never changed.
LLVM Value Representation.
std::pair< iterator, bool > insert(const ValueT &V)
void insert_range(Range &&R)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
@ BasicBlock
Various leaf nodes.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
initializer< Ty > init(const Ty &Val)
uint64_t MD5Hash(const FunctionId &Obj)
friend class Instruction
Iterator for Instructions in a `BasicBlock.
This is an optimization pass for GlobalISel generic memory operations.
T any_cast(const Any &Value)
static constexpr uint64_t PseudoProbeFullDistributionFactor
decltype(auto) dyn_cast(const From &Val)
dyn_cast - Return the argument parameter cast to the specified type.
auto pred_size(const MachineBasicBlock *BB)
InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy
Provide the FunctionAnalysisManager to Module proxy.
LLVM_ABI void setProbeDistributionFactor(Instruction &Inst, float Factor)
LLVM_ABI std::optional< PseudoProbe > extractProbe(const Instruction &Inst)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
auto succ_size(const MachineBasicBlock *BB)
static void computeEHOnlyBlocks(FunctionT &F, DenseSet< BlockT * > &EHBlocks)
Compute a list of blocks that are only reachable via EH paths.
bool isa(const From &Val)
isa - Return true if the parameter to the template is an instance of one of the template type argu...
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
RNSuccIterator< NodeRef, BlockT, RegionT > succ_begin(NodeRef Node)
auto pred_begin(const MachineBasicBlock *BB)
decltype(auto) cast(const From &Val)
cast - Return the argument parameter cast to the specified type.
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
AnalysisManager< Module > ModuleAnalysisManager
Convenience typedef for the Module analysis manager.
constexpr const char * PseudoProbeDescMetadataName
std::unordered_map< std::pair< uint64_t, uint64_t >, float, pair_hash< uint64_t, uint64_t > > ProbeFactorMap
static constexpr uint8_t FullDistributionFactor
static uint32_t packProbeData(uint32_t Index, uint32_t Type, uint32_t Flags, uint32_t Factor, std::optional< uint32_t > DwarfBaseDiscriminator)