LLVM: lib/Transforms/IPO/SampleProfile.cpp File Reference (original) (raw)
Go to the source code of this file.
| Namespaces | |
|---|---|
| namespace | llvm |
| This is an optimization pass for GlobalISel generic memory operations. |
| Macros | |
|---|---|
| #define | DEBUG_TYPE "sample-profile" |
| #define | CSINLINE_DEBUG DEBUG_TYPE "-inline" |
| Functions | |
|---|---|
| STATISTIC (NumCSInlined, "Number of functions inlined with context sensitive profile") | |
| STATISTIC (NumCSNotInlined, "Number of functions not inlined with context sensitive profile") | |
| STATISTIC (NumMismatchedProfile, "Number of functions with CFG mismatched profile") | |
| STATISTIC (NumMatchedProfile, "Number of functions with CFG matched profile") | |
| STATISTIC (NumDuplicatedInlinesite, "Number of inlined callsites with a partial distribution factor") | |
| STATISTIC (NumCSInlinedHitMinLimit, "Number of functions with FDO inline stopped due to min size limit") | |
| STATISTIC (NumCSInlinedHitMaxLimit, "Number of functions with FDO inline stopped due to max size limit") | |
| STATISTIC (NumCSInlinedHitGrowthLimit, "Number of functions with FDO inline stopped due to growth size limit") | |
| cl::opt< bool > | llvm::SortProfiledSCC ("sort-profiled-scc-member", cl::init(true), cl::Hidden, cl::desc("Sort profiled recursion by edge weights.")) |
| cl::opt< int > | llvm::ProfileInlineGrowthLimit ("sample-profile-inline-growth-limit", cl::Hidden, cl::init(12), cl::desc("The size growth ratio limit for proirity-based sample profile " "loader inlining.")) |
| cl::opt< int > | llvm::ProfileInlineLimitMin ("sample-profile-inline-limit-min", cl::Hidden, cl::init(100), cl::desc("The lower bound of size growth limit for " "proirity-based sample profile loader inlining.")) |
| cl::opt< int > | llvm::ProfileInlineLimitMax ("sample-profile-inline-limit-max", cl::Hidden, cl::init(10000), cl::desc("The upper bound of size growth limit for " "proirity-based sample profile loader inlining.")) |
| cl::opt< int > | llvm::SampleHotCallSiteThreshold ("sample-profile-hot-inline-threshold", cl::Hidden, cl::init(3000), cl::desc("Hot callsite threshold for proirity-based sample profile loader " "inlining.")) |
| cl::opt< int > | llvm::SampleColdCallSiteThreshold ("sample-profile-cold-inline-threshold", cl::Hidden, cl::init(45), cl::desc("Threshold for inlining cold callsites")) |
| static bool | doesHistoryAllowICP (const Instruction &Inst, StringRef Candidate) |
| Check whether the indirect call promotion history of Inst allows the promotion for Candidate. | |
| static void | updateIDTMetaData (Instruction &Inst, const SmallVectorImpl< InstrProfValueData > &CallTargets, uint64_t Sum) |
| Update indirect call target profile metadata for Inst. | |
| static SmallVector< InstrProfValueData, 2 > | GetSortedValueDataFromCallTargets (const SampleRecord::CallTargetMap &M) |
| Returns the sorted CallTargetMap M by count in descending order. |
| Variables | |
|---|---|
| static cl::opt< std::string > | llvm::SampleProfileFile ("sample-profile-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile file loaded by -sample-profile"), cl::Hidden) |
| static cl::opt< std::string > | llvm::SampleProfileRemappingFile ("sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden) |
| cl::opt< bool > | llvm::SalvageStaleProfile ("salvage-stale-profile", cl::Hidden, cl::init(false), cl::desc("Salvage stale profile by fuzzy matching and use the remapped " "location for sample profile query.")) |
| cl::opt< bool > | llvm::SalvageUnusedProfile ("salvage-unused-profile", cl::Hidden, cl::init(false), cl::desc("Salvage unused profile by matching with new " "functions on call graph.")) |
| cl::opt< bool > | llvm::ReportProfileStaleness ("report-profile-staleness", cl::Hidden, cl::init(false), cl::desc("Compute and report stale profile statistical metrics.")) |
| cl::opt< bool > | llvm::PersistProfileStaleness ("persist-profile-staleness", cl::Hidden, cl::init(false), cl::desc("Compute stale profile statistical metrics and write it into the " "native object file(.llvm_stats section).")) |
| static cl::opt< bool > | llvm::ProfileSampleAccurate ("profile-sample-accurate", cl::Hidden, cl::init(false), cl::desc("If the sample profile is accurate, we will mark all un-sampled " "callsite and function as having 0 samples. Otherwise, treat " "un-sampled callsites and functions conservatively as unknown. ")) |
| static cl::opt< bool > | llvm::ProfileSampleBlockAccurate ("profile-sample-block-accurate", cl::Hidden, cl::init(false), cl::desc("If the sample profile is accurate, we will mark all un-sampled " "branches and calls as having 0 samples. Otherwise, treat " "them conservatively as unknown. ")) |
| static cl::opt< bool > | llvm::ProfileAccurateForSymsInList ("profile-accurate-for-symsinlist", cl::Hidden, cl::init(true), cl::desc("For symbols in profile symbol list, regard their profiles to " "be accurate. It may be overridden by profile-sample-accurate. ")) |
| static cl::opt< bool > | llvm::ProfileMergeInlinee ("sample-profile-merge-inlinee", cl::Hidden, cl::init(true), cl::desc("Merge past inlinee's profile to outline version if sample " "profile loader decided not to inline a call site. It will " "only be enabled when top-down order of profile loading is " "enabled. ")) |
| static cl::opt< bool > | llvm::ProfileTopDownLoad ("sample-profile-top-down-load", cl::Hidden, cl::init(true), cl::desc("Do profile annotation and inlining for functions in top-down " "order of call graph during sample profile loading. It only " "works for new pass manager. ")) |
| static cl::opt< bool > | llvm::UseProfiledCallGraph ("use-profiled-call-graph", cl::init(true), cl::Hidden, cl::desc("Process functions in a top-down order " "defined by the profiled call graph when " "-sample-profile-top-down-load is on.")) |
| static cl::opt< bool > | llvm::ProfileSizeInline ("sample-profile-inline-size", cl::Hidden, cl::init(false), cl::desc("Inline cold call sites in profile loader if it's beneficial " "for code size.")) |
| static cl::opt< bool > | llvm::DisableSampleLoaderInlining ("disable-sample-loader-inlining", cl::Hidden, cl::init(false), cl::desc("If true, artificially skip inline transformation in sample-loader " "pass, and merge (or scale) profiles (as configured by " "--sample-profile-merge-inlinee).")) |
| static cl::opt< unsigned > | ProfileICPRelativeHotness ("sample-profile-icp-relative-hotness", cl::Hidden, cl::init(25), cl::desc("Relative hotness percentage threshold for indirect " "call promotion in proirity-based sample profile loader inlining.")) |
| static cl::opt< unsigned > | ProfileICPRelativeHotnessSkip ("sample-profile-icp-relative-hotness-skip", cl::Hidden, cl::init(1), cl::desc("Skip relative hotness check for ICP up to given number of targets.")) |
| static cl::opt< unsigned > | HotFuncCutoffForStalenessError ("hot-func-cutoff-for-staleness-error", cl::Hidden, cl::init(800000), cl::desc("A function is considered hot for staleness error check if its " "total sample count is above the specified percentile")) |
| static cl::opt< unsigned > | MinfuncsForStalenessError ("min-functions-for-staleness-error", cl::Hidden, cl::init(50), cl::desc("Skip the check if the number of hot functions is smaller than " "the specified number.")) |
| static cl::opt< unsigned > | PrecentMismatchForStalenessError ("precent-mismatch-for-staleness-error", cl::Hidden, cl::init(80), cl::desc("Reject the profile if the mismatch percent is higher than the " "given number.")) |
| static cl::opt< bool > | CallsitePrioritizedInline ("sample-profile-prioritized-inline", cl::Hidden, cl::desc("Use call site prioritized inlining for sample profile loader. " "Currently only CSSPGO is supported.")) |
| static cl::opt< bool > | UsePreInlinerDecision ("sample-profile-use-preinliner", cl::Hidden, cl::desc("Use the preinliner decisions stored in profile context.")) |
| static cl::opt< bool > | AllowRecursiveInline ("sample-profile-recursive-inline", cl::Hidden, cl::desc("Allow sample loader inliner to inline recursive calls.")) |
| static cl::opt< bool > | RemoveProbeAfterProfileAnnotation ("sample-profile-remove-probe", cl::Hidden, cl::init(false), cl::desc("Remove pseudo-probe after sample profile annotation.")) |
| static cl::opt< std::string > | ProfileInlineReplayFile ("sample-profile-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc("Optimization remarks file containing inline remarks to be replayed " "by inlining from sample profile loader."), cl::Hidden) |
| static cl::opt< ReplayInlinerSettings::Scope > | ProfileInlineReplayScope ("sample-profile-inline-replay-scope", cl::init(ReplayInlinerSettings::Scope::Function), cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module")), cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during sample profile inlining."), cl::Hidden) |
| static cl::opt< ReplayInlinerSettings::Fallback > | ProfileInlineReplayFallback ("sample-profile-inline-replay-fallback", cl::init(ReplayInlinerSettings::Fallback::Original), cl::values(clEnumValN(ReplayInlinerSettings::Fallback::Original, "Original", "All decisions not in replay send to original advisor (default)"), clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline, "AlwaysInline", "All decisions not in replay are inlined"), clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline", "All decisions not in replay are not inlined")), cl::desc("How sample profile inline replay treats sites that don't come " "from the replay. Original: defers to original advisor, " "AlwaysInline: inline all sites not in replay, NeverInline: " "inline no sites not in replay"), cl::Hidden) |
| static cl::opt< CallSiteFormat::Format > | ProfileInlineReplayFormat ("sample-profile-inline-replay-format", cl::init(CallSiteFormat::Format::LineColumnDiscriminator), cl::values(clEnumValN(CallSiteFormat::Format::Line, "Line", ""), clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", ":"), clEnumValN(CallSiteFormat::Format::LineDiscriminator, "LineDiscriminator", "."), clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, "LineColumnDiscriminator", ":. (default)")), cl::desc("How sample profile inline replay file is formatted"), cl::Hidden) |
| static cl::opt< unsigned > | MaxNumPromotions ("sample-profile-icp-max-prom", cl::init(3), cl::Hidden, cl::desc("Max number of promotions for a single indirect " "call callsite in sample profile loader")) |
| static cl::opt< bool > | OverwriteExistingWeights ("overwrite-existing-weights", cl::Hidden, cl::init(false), cl::desc("Ignore existing branch weights on IR and always overwrite.")) |
| static cl::opt< bool > | AnnotateSampleProfileInlinePhase ("annotate-sample-profile-inline-phase", cl::Hidden, cl::init(false), cl::desc("Annotate LTO phase (prelink / postlink), or main (no LTO) for " "sample-profile inline pass name.")) |
◆ CSINLINE_DEBUG
◆ DEBUG_TYPE
#define DEBUG_TYPE "sample-profile"
◆ doesHistoryAllowICP()
◆ GetSortedValueDataFromCallTargets()
◆ STATISTIC() [1/8]
| STATISTIC | ( | NumCSInlined | , |
|---|---|---|---|
| "Number of functions inlined with context sensitive profile" | ) |
◆ STATISTIC() [2/8]
| STATISTIC | ( | NumCSInlinedHitGrowthLimit | , |
|---|---|---|---|
| "Number of functions with FDO inline stopped due to growth size limit" | ) |
◆ STATISTIC() [3/8]
| STATISTIC | ( | NumCSInlinedHitMaxLimit | , |
|---|---|---|---|
| "Number of functions with FDO inline stopped due to max size limit" | ) |
◆ STATISTIC() [4/8]
| STATISTIC | ( | NumCSInlinedHitMinLimit | , |
|---|---|---|---|
| "Number of functions with FDO inline stopped due to min size limit" | ) |
◆ STATISTIC() [5/8]
| STATISTIC | ( | NumCSNotInlined | , |
|---|---|---|---|
| "Number of functions not inlined with context sensitive profile" | ) |
◆ STATISTIC() [6/8]
| STATISTIC | ( | NumDuplicatedInlinesite | , |
|---|---|---|---|
| "Number of inlined callsites with a partial distribution factor" | ) |
◆ STATISTIC() [7/8]
| STATISTIC | ( | NumMatchedProfile | , |
|---|---|---|---|
| "Number of functions with CFG matched profile" | ) |
◆ STATISTIC() [8/8]
| STATISTIC | ( | NumMismatchedProfile | , |
|---|---|---|---|
| "Number of functions with CFG mismatched profile" | ) |
◆ updateIDTMetaData()
Update indirect call target profile metadata for Inst.
Usually Sum is the sum of counts of all the targets for Inst. If it is 0, it means updateIDTMetaData is used to mark a certain target to be promoted already. If it is not zero, we expect to use it to update the total count in the value profile.
Definition at line 839 of file SampleProfile.cpp.
References llvm::annotateValueSite(), assert(), llvm::Count, llvm::Data, llvm::SmallVectorImpl< T >::emplace_back(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::getValueProfDataFromInst(), llvm::MaxNumPromotions, llvm::NOMORE_ICP_MAGICNUM, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::sort(), and llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::try_emplace().
◆ AllowRecursiveInline
| cl::opt< bool > AllowRecursiveInline("sample-profile-recursive-inline", cl::Hidden, cl::desc("Allow sample loader inliner to inline recursive calls.")) ( "sample-profile-recursive-inline" , cl::Hidden , cl::desc("Allow sample loader inliner to inline recursive calls.") ) | static |
|---|
◆ AnnotateSampleProfileInlinePhase
| cl::opt< bool > AnnotateSampleProfileInlinePhase("annotate-sample-profile-inline-phase", cl::Hidden, cl::init(false), cl::desc("Annotate LTO phase (prelink / postlink), or main (no LTO) for " "sample-profile inline pass name.")) ( "annotate-sample-profile-inline-phase" , cl::Hidden , cl::init(false) , cl::desc("Annotate LTO phase (prelink / postlink), or main (no LTO) for " "sample-profile inline pass name.") ) | static |
|---|
◆ CallsitePrioritizedInline
| cl::opt< bool > CallsitePrioritizedInline("sample-profile-prioritized-inline", cl::Hidden, cl::desc("Use call site prioritized inlining for sample profile loader. " "Currently only CSSPGO is supported.")) ( "sample-profile-prioritized-inline" , cl::Hidden , cl::desc("Use call site prioritized inlining for sample profile loader. " "Currently only CSSPGO is supported.") ) | static |
|---|
◆ HotFuncCutoffForStalenessError
| cl::opt< unsigned > HotFuncCutoffForStalenessError("hot-func-cutoff-for-staleness-error", cl::Hidden, cl::init(800000), cl::desc("A function is considered hot for staleness error check if its " "total sample count is above the specified percentile")) ( "hot-func-cutoff-for-staleness-error" , cl::Hidden , cl::init(800000) , cl::desc("A function is considered hot for staleness error check if its " "total sample count is above the specified percentile") ) | static |
|---|
◆ MaxNumPromotions
| cl::opt< unsigned > MaxNumPromotions("sample-profile-icp-max-prom", cl::init(3), cl::Hidden, cl::desc("Max number of promotions for a single indirect " "call callsite in sample profile loader")) ( "sample-profile-icp-max-prom" , cl::init(3) , cl::Hidden , cl::desc("Max number of promotions for a single indirect " "call callsite in sample profile loader") ) | static |
|---|
◆ MinfuncsForStalenessError
◆ OverwriteExistingWeights
| cl::opt< bool > OverwriteExistingWeights("overwrite-existing-weights", cl::Hidden, cl::init(false), cl::desc("Ignore existing branch weights on IR and always overwrite.")) ( "overwrite-existing-weights" , cl::Hidden , cl::init(false) , cl::desc("Ignore existing branch weights on IR and always overwrite.") ) | static |
|---|
◆ PrecentMismatchForStalenessError
| cl::opt< unsigned > PrecentMismatchForStalenessError("precent-mismatch-for-staleness-error", cl::Hidden, cl::init(80), cl::desc("Reject the profile if the mismatch percent is higher than the " "given number.")) ( "precent-mismatch-for-staleness-error" , cl::Hidden , cl::init(80) , cl::desc("Reject the profile if the mismatch percent is higher than the " "given number.") ) | static |
|---|
◆ ProfileICPRelativeHotness
| cl::opt< unsigned > ProfileICPRelativeHotness("sample-profile-icp-relative-hotness", cl::Hidden, cl::init(25), cl::desc( "Relative hotness percentage threshold for indirect " "call promotion in proirity-based sample profile loader inlining.")) ( "sample-profile-icp-relative-hotness" , cl::Hidden , cl::init(25) , cl::desc( "Relative hotness percentage threshold for indirect " "call promotion in proirity-based sample profile loader inlining.") ) | static |
|---|
◆ ProfileICPRelativeHotnessSkip
| cl::opt< unsigned > ProfileICPRelativeHotnessSkip("sample-profile-icp-relative-hotness-skip", cl::Hidden, cl::init(1), cl::desc( "Skip relative hotness check for ICP up to given number of targets.")) ( "sample-profile-icp-relative-hotness-skip" , cl::Hidden , cl::init(1) , cl::desc( "Skip relative hotness check for ICP up to given number of targets.") ) | static |
|---|
◆ ProfileInlineReplayFallback
| cl::opt< ReplayInlinerSettings::Fallback > ProfileInlineReplayFallback("sample-profile-inline-replay-fallback", cl::init(ReplayInlinerSettings::Fallback::Original), cl::values( clEnumValN( ReplayInlinerSettings::Fallback::Original, "Original", "All decisions not in replay send to original advisor (default)"), clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline, "AlwaysInline", "All decisions not in replay are inlined"), clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline", "All decisions not in replay are not inlined")), cl::desc("How sample profile inline replay treats sites that don't come " "from the replay. Original: defers to original advisor, " "AlwaysInline: inline all sites not in replay, NeverInline: " "inline no sites not in replay"), cl::Hidden) ( "sample-profile-inline-replay-fallback" , cl::init(ReplayInlinerSettings::Fallback::Original) , cl::values( clEnumValN( ReplayInlinerSettings::Fallback::Original, "Original", "All decisions not in replay send to original advisor (default)"), clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline, "AlwaysInline", "All decisions not in replay are inlined"), clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline", "All decisions not in replay are not inlined")) , cl::desc("How sample profile inline replay treats sites that don't come " "from the replay. Original: defers to original advisor, " "AlwaysInline: inline all sites not in replay, NeverInline: " "inline no sites not in replay") , cl::Hidden ) | static |
|---|
◆ ProfileInlineReplayFile
| cl::opt< std::string > ProfileInlineReplayFile("sample-profile-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc( "Optimization remarks file containing inline remarks to be replayed " "by inlining from sample profile loader."), cl::Hidden) ( "sample-profile-inline-replay" , cl::init("") , cl::value_desc("filename") , cl::desc( "Optimization remarks file containing inline remarks to be replayed " "by inlining from sample profile loader.") , cl::Hidden ) | static |
|---|
◆ ProfileInlineReplayFormat
| cl::opt< CallSiteFormat::Format > ProfileInlineReplayFormat("sample-profile-inline-replay-format", cl::init(CallSiteFormat::Format::LineColumnDiscriminator), cl::values( clEnumValN(CallSiteFormat::Format::Line, "Line", ""), clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", ":"), clEnumValN(CallSiteFormat::Format::LineDiscriminator, "LineDiscriminator", "."), clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, "LineColumnDiscriminator", ":. (default)")), cl::desc("How sample profile inline replay file is formatted"), cl::Hidden) ( "sample-profile-inline-replay-format" , cl::init(CallSiteFormat::Format::LineColumnDiscriminator) , cl::values( clEnumValN(CallSiteFormat::Format::Line, "Line", ""), clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", ":"), clEnumValN(CallSiteFormat::Format::LineDiscriminator, "LineDiscriminator", "."), clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, "LineColumnDiscriminator", ":. (default)")) , cl::desc("How sample profile inline replay file is formatted") , cl::Hidden ) | static |
|---|
◆ ProfileInlineReplayScope
| cl::opt< ReplayInlinerSettings::Scope > ProfileInlineReplayScope("sample-profile-inline-replay-scope", cl::init(ReplayInlinerSettings::Scope::Function), cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module")), cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during sample profile inlining."), cl::Hidden) ( "sample-profile-inline-replay-scope" , cl::init(ReplayInlinerSettings::Scope::Function) , cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module")) , cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during sample profile inlining.") , cl::Hidden ) | static |
|---|
◆ RemoveProbeAfterProfileAnnotation
| cl::opt< bool > RemoveProbeAfterProfileAnnotation("sample-profile-remove-probe", cl::Hidden, cl::init(false), cl::desc("Remove pseudo-probe after sample profile annotation.")) ( "sample-profile-remove-probe" , cl::Hidden , cl::init(false) , cl::desc("Remove pseudo-probe after sample profile annotation.") ) | static |
|---|
◆ UsePreInlinerDecision
| cl::opt< bool > UsePreInlinerDecision("sample-profile-use-preinliner", cl::Hidden, cl::desc("Use the preinliner decisions stored in profile context.")) ( "sample-profile-use-preinliner" , cl::Hidden , cl::desc("Use the preinliner decisions stored in profile context.") ) | static |
|---|