ProfileVerify.cpp Source File (original) (raw)

29using namespace llvm;

35 cl::desc("Also inject (if missing) and verify MD_prof for "

36 "`select` instructions"));

39 cl::desc("Generate weights with small values for tests."));

42 "profcheck-default-select-true-weight", cl::init(2U),

43 cl::desc("When annotating `select` instructions, this value will be used "

44 "for the first ('true') case."));

46 "profcheck-default-select-false-weight", cl::init(3U),

47 cl::desc("When annotating `select` instructions, this value will be used "

48 "for the second ('false') case."));

49namespace {

50class ProfileInjector {

54public:

56 getTerminatorBenefitingFromMDProf(const BasicBlock &BB) {

58 return nullptr;

62 ? Term

63 : nullptr;

64 }

66 static Instruction *getTerminatorBenefitingFromMDProf(BasicBlock &BB) {

68 getTerminatorBenefitingFromMDProf(const_cast<const BasicBlock &>(BB)));

69 }

72 bool inject();

73};

75bool isAsmOnly(const Function &F) {

76 if (.hasFnAttribute(Attribute::AttrKind::Naked))

77 return false;

78 for (const auto &BB : F)

81 if (!CB || !CB->isInlineAsm())

82 return false;

83 }

84 return true;

85}

86}

90bool ProfileInjector::inject() {

92 if (isAsmOnly(F))

93 return false;

100

101

102

103 auto &BPI = FAM.getResult(F);

104

105

106

107

108

109

110

111

112 if (.getEntryCount(true))

114

115

116 if (F.getEntryCount(true)->getCount() == 0)

117 return false;

119

120

121

122 uint32_t WeightsForTestOffset = 0;

123 for (auto &BB : F) {

125 for (auto &I : BB) {

127 if (SI->getCondition()->getType()->isVectorTy())

128 continue;

129 if (I.getMetadata(LLVMContext::MD_prof))

130 continue;

132 false);

133 }

134 }

135 }

136 auto *Term = getTerminatorBenefitingFromMDProf(BB);

137 if (!Term || Term->getMetadata(LLVMContext::MD_prof))

138 continue;

139 SmallVector Probs;

140

141 SmallVector<uint32_t> Weights;

142 Weights.reserve(Term->getNumSuccessors());

144 static const std::array Primes{3, 5, 7, 11, 13, 17, 19, 23, 29, 31,

145 37, 41, 43, 47, 53, 59, 61, 67, 71};

146 for (uint32_t I = 0, E = Term->getNumSuccessors(); I < E; ++I)

148 Primes[(WeightsForTestOffset + I) % Primes.size()]);

149 ++WeightsForTestOffset;

150 } else {

152 for (auto I = 0U, E = Term->getNumSuccessors(); I < E; ++I)

153 Probs.emplace_back(BPI.getEdgeProbability(&BB, Term->getSuccessor(I)));

154

156 [](const BranchProbability &P) {

157 return P.isUnknown();

158 }) == Probs.end() &&

159 "All branch probabilities should be valid");

160 const auto *FirstZeroDenominator =

161 find_if(Probs, [](const BranchProbability &P) {

162 return P.getDenominator() == 0;

163 });

164 (void)FirstZeroDenominator;

165 assert(FirstZeroDenominator == Probs.end());

166 const auto *FirstNonZeroNumerator = find_if(

167 Probs, [](const BranchProbability &P) { return .isZero(); });

168 assert(FirstNonZeroNumerator != Probs.end());

169 DynamicAPInt LCM(Probs[0].getDenominator());

170 DynamicAPInt GCD(FirstNonZeroNumerator->getNumerator());

171 for (const auto &Prob : drop_begin(Probs)) {

172 if (!Prob.getNumerator())

173 continue;

174 LCM = llvm::lcm(LCM, DynamicAPInt(Prob.getDenominator()));

175 GCD = llvm::gcd(GCD, DynamicAPInt(Prob.getNumerator()));

176 }

177 for (const auto &Prob : Probs) {

178 DynamicAPInt W =

179 (Prob.getNumerator() * LCM / GCD) / Prob.getDenominator();

180 Weights.emplace_back(static_cast<uint32_t>((int64_t)W));

181 }

182 }

185 }

187}

188

191 ProfileInjector PI(F, FAM);

192 if (!PI.inject())

194

196}

197

200 auto PopulateIgnoreList = [&](StringRef GVName) {

201 if (const auto *CT = M.getGlobalVariable(GVName))

202 if (const auto *CA =

204 for (const auto &Elt : CA->operands())

206 if (CS->getNumOperands() >= 2 && CS->getOperand(1))

208 CS->getOperand(1)->stripPointerCasts()))

209 IgnoreList.insert(F);

210 };

211 PopulateIgnoreList("llvm.global_ctors");

212 PopulateIgnoreList("llvm.global_dtors");

213

214

215

216

220 return PVP.run(F, FAM);

221 }

223 };

224

226}

227

230

231 if (isAsmOnly(F))

233 if (IgnoreList.contains(&F))

235

236 const auto EntryCount = F.getEntryCount(true);

237 if (!EntryCount) {

238 auto *MD = F.getMetadata(LLVMContext::MD_prof);

240 F.getContext().emitError("Profile verification failed: function entry "

241 "count missing (set to 0 if cold)");

243 }

244 } else if (EntryCount->getCount() == 0) {

246 }

247 for (const auto &BB : F) {

249 for (const auto &I : BB)

251 if (SI->getCondition()->getType()->isVectorTy())

252 continue;

253 if (I.getMetadata(LLVMContext::MD_prof))

254 continue;

255 F.getContext().emitError(

256 "Profile verification failed: select annotation missing");

257 }

258 }

259 if (const auto *Term =

260 ProfileInjector::getTerminatorBenefitingFromMDProf(BB))

261 if ( Term ->getMetadata(LLVMContext::MD_prof))

262 F.getContext().emitError(

263 "Profile verification failed: branch annotation missing");

264 }

266}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

amdgpu aa AMDGPU Address space based Alias Analysis Wrapper

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

This file contains the declarations for the subclasses of Constant, which represent the different fla...

Module.h This file contains the declarations for the Module class.

This header defines various interfaces for pass management in LLVM.

FunctionAnalysisManager FAM

ModuleAnalysisManager MAM

This file contains the declarations for profiling metadata utility functions.

static cl::opt< bool > WeightsForTest("profcheck-weights-for-test", cl::init(false), cl::desc("Generate weights with small values for tests."))

static cl::opt< uint32_t > SelectFalseWeight("profcheck-default-select-false-weight", cl::init(3U), cl::desc("When annotating `select` instructions, this value will be used " "for the second ('false') case."))

static cl::opt< int64_t > DefaultFunctionEntryCount("profcheck-default-function-entry-count", cl::init(1000))

static cl::opt< bool > AnnotateSelect("profcheck-annotate-select", cl::init(true), cl::desc("Also inject (if missing) and verify MD_prof for " "`select` instructions"))

static cl::opt< uint32_t > SelectTrueWeight("profcheck-default-select-true-weight", cl::init(2U), cl::desc("When annotating `select` instructions, this value will be used " "for the first ('true') case."))

PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)

Get the result of an analysis pass for a given IR unit.

LLVM Basic Block Representation.

LLVM_ABI iterator_range< filter_iterator< BasicBlock::const_iterator, std::function< bool(const Instruction &)> > > instructionsWithoutDebug(bool SkipPseudoOp=true) const

Return a const iterator range over the instructions in the block, skipping any debug instructions.

const Instruction * getTerminator() const LLVM_READONLY

Returns the terminator instruction if the block is well formed or null if the block is not well forme...

LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)

Runs the function pass across every function in the module.

A Module instance is used to store all the information related to an LLVM module.

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM)

Definition ProfileVerify.cpp:189

Checks that MD_prof is present on every instruction that supports it.

reference emplace_back(ArgTypes &&... Args)

void reserve(size_type N)

StringRef - Represent a constant reference to a string, i.e.

Pass manager infrastructure for declaring and invalidating analyses.

@ BasicBlock

Various leaf nodes.

initializer< Ty > init(const Ty &Val)

friend class Instruction

Iterator for Instructions in a `BasicBlock.

This is an optimization pass for GlobalISel generic memory operations.

auto drop_begin(T &&RangeOrContainer, size_t N=1)

Return a range covering RangeOrContainer with the first N elements excluded.

LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt gcd(const DynamicAPInt &A, const DynamicAPInt &B)

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

ModuleToFunctionPassAdaptor createModuleToFunctionPassAdaptor(FunctionPassT &&Pass, bool EagerlyInvalidate=false)

A function to deduce a function pass type and wrap it in the templated adaptor.

LLVM_ABI bool isExplicitlyUnknownProfileMetadata(const MDNode &MD)

auto dyn_cast_if_present(const Y &Val)

dyn_cast_if_present - Functionally identical to dyn_cast, except that a null (or none in the case ...

LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected, bool ElideAllZero=false)

Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...

auto succ_size(const MachineBasicBlock *BB)

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

auto drop_end(T &&RangeOrContainer, size_t N=1)

Return a range covering RangeOrContainer with the last N elements excluded.

LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt lcm(const DynamicAPInt &A, const DynamicAPInt &B)

Returns the least common multiple of A and B.

auto find_if(R &&Range, UnaryPredicate P)

Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.

AnalysisManager< Function > FunctionAnalysisManager

Convenience typedef for the Function analysis manager.

AnalysisManager< Module > ModuleAnalysisManager

Convenience typedef for the Module analysis manager.

A CRTP mix-in to automatically provide informational APIs needed for passes.