ProfDataUtils.cpp Source File (original) (raw)

25using namespace llvm;

42static constexpr unsigned MinBWOps = 3;

45static constexpr unsigned MinVPOps = 5;

50 unsigned MinOps) {

53 if (!ProfData || !Name || MinOps < 2)

54 return false;

57 if (NOps < MinOps)

58 return false;

61 if (!ProfDataName)

62 return false;

64 return ProfDataName->getString() == Name;

65}

67template <typename T,

68 typename = typename std::enable_if<std::is_arithmetic_v>>

75 assert(WeightsIdx < NOps && "Weights Index must be less than NOps.");

76 Weights.resize(NOps - WeightsIdx);

78 for (unsigned Idx = WeightsIdx, E = NOps; Idx != E; ++Idx) {

81 assert(Weight && "Malformed branch_weight in MD_prof node");

83 "Too many bits for MD_prof branch_weight");

84 Weights[Idx - WeightsIdx] = Weight->getZExtValue();

85 }

86}

93 if (Max > UINT_MAX) {

97 } else {

99 }

100 return Ret;

101}

102

104#if defined(LLVM_ENABLE_PROFCHECK)

106#else

108#endif

109);

115 "synthetic_function_entry_count";

118

120 return I.hasMetadata(LLVMContext::MD_prof);

121}

122

126

130

132 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);

134}

135

137 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);

138

140 return true;

141

142

144}

145

149

151 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);

153}

154

157 return false;

159

160

161

162 assert(ProfDataName == nullptr ||

164 return ProfDataName != nullptr;

165}

166

170

174

176 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);

178 return nullptr;

179 return ProfileData;

180}

181

184 if (ProfileData && getNumBranchWeights(*ProfileData) == I.getNumSuccessors())

185 return ProfileData;

186 return nullptr;

187}

188

193

198

202 return false;

204 return true;

205}

206

209 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);

211}

212

215 assert((I.getOpcode() == Instruction::Br ||

216 I.getOpcode() == Instruction::Select) &&

217 "Looking for branch weights on something besides branch, select, or "

218 "switch");

219

221 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);

223 return false;

224

225 if (Weights.size() > 2)

226 return false;

227

228 TrueVal = Weights[0];

229 FalseVal = Weights[1];

230 return true;

231}

232

235 TotalVal = 0;

236 if (!ProfileData)

237 return false;

238

240 if (!ProfDataName)

241 return false;

242

247 TotalVal += V->getValue().getZExtValue();

248 }

249 return true;

250 }

251

255 ->getValue()

256 .getZExtValue();

257 return true;

258 }

259 return false;

260}

261

265

269 I.setMetadata(

270 LLVMContext::MD_prof,

272 {MDB.createString(MDProfLabels::UnknownBranchWeightsMarker),

273 MDB.createString(PassName)}));

274}

275

279 F = F ? F : I.getFunction();

280 assert(F && "Either pass a instruction attached to a Function, or explicitly "

281 "pass the Function that it will be attached to");

282 if (std::optionalFunction::ProfileCount EC = F->getEntryCount();

283 EC && EC->getCount() > 0)

285}

286

290 F.setMetadata(

291 LLVMContext::MD_prof,

293 {MDB.createString(MDProfLabels::UnknownBranchWeightsMarker),

294 MDB.createString(PassName)}));

295}

296

302

304 auto *MD = I.getMetadata(LLVMContext::MD_prof);

305 if (!MD)

306 return false;

308}

309

311 bool IsExpected, bool ElideAllZero) {

314 I.setMetadata(LLVMContext::MD_prof, nullptr);

315 return;

316 }

317

320 I.setMetadata(LLVMContext::MD_prof, BranchWeights);

321}

322

324 bool IsExpected, bool ElideAllZero) {

326}

327

330 std::optional<uint64_t> KnownMaxCount) {

331 uint64_t MaxCount = KnownMaxCount.has_value() ? KnownMaxCount.value()

333 assert(MaxCount > 0 && "Bad max count");

336 for (const auto &ECI : Weights)

338 return DownscaledWeights;

339}

340

342 assert(T != 0 && "Caller should guarantee");

343 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);

344 if (ProfileData == nullptr)

345 return;

346

348 if (!ProfDataName ||

351 return;

352

354 return;

355

357

360 Vals.push_back(ProfileData->getOperand(0));

361 APInt APS(128, S), APT(128, T);

363 ProfileData->getNumOperands() > 0) {

364

368 ->getValue()

369 .getZExtValue());

370 Val *= APS;

374 for (unsigned Idx = 1; Idx < ProfileData->getNumOperands(); Idx += 2) {

375

376 Vals.push_back(ProfileData->getOperand(Idx));

379 ->getValue()

380 .getZExtValue();

381

383 Vals.push_back(ProfileData->getOperand(Idx + 1));

384 continue;

385 }

386

388 Val *= APS;

391 }

392 I.setMetadata(LLVMContext::MD_prof, MDNode::get(C, Vals));

393}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

This file contains the declarations for the subclasses of Constant, which represent the different fla...

static constexpr unsigned MinVPOps

Definition ProfDataUtils.cpp:45

static cl::opt< bool > ElideAllZeroBranchWeights("elide-all-zero-branch-weights", cl::init(true))

static bool isTargetMD(const MDNode *ProfData, const char *Name, unsigned MinOps)

Definition ProfDataUtils.cpp:49

static void extractFromBranchWeightMD(const MDNode *ProfileData, SmallVectorImpl< T > &Weights)

Definition ProfDataUtils.cpp:69

static bool hasCountTypeMD(const Instruction &I)

Definition ProfDataUtils.cpp:136

static constexpr unsigned MinBWOps

Definition ProfDataUtils.cpp:42

This file contains the declarations for profiling metadata utility functions.

This file defines the SmallVector class.

static const char PassName[]

Class for arbitrary precision integers.

LLVM_ABI APInt udiv(const APInt &RHS) const

Unsigned division operation.

unsigned getActiveBits() const

Compute the number of active bits in the value.

uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const

If this value is smaller than the specified limit, return it, otherwise return the limit value.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

This is the shared class of boolean and integer constants.

uint64_t getZExtValue() const

Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...

const APInt & getValue() const

Return the constant as an APInt value reference.

This is an important class for using LLVM in a threaded context.

LLVM_ABI ConstantAsMetadata * createConstant(Constant *C)

Return the given constant as metadata.

LLVM_ABI MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight, bool IsExpected=false)

Return metadata containing two branch weights.

const MDOperand & getOperand(unsigned I) const

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

unsigned getNumOperands() const

Return number of MDNode operands.

bool equalsStr(StringRef Str) const

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

void reserve(size_type N)

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)

static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)

LLVM Value Representation.

@ C

The default llvm calling convention, compatible with C.

initializer< Ty > init(const Ty &Val)

std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)

Extract a Value from Metadata, if any.

std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)

Extract a Value from Metadata.

This is an optimization pass for GlobalISel generic memory operations.

bool all_of(R &&range, UnaryPredicate P)

Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.

LLVM_ABI bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalWeights)

Retrieve the total of all weights from MD_prof data.

Definition ProfDataUtils.cpp:233

LLVM_ABI unsigned getBranchWeightOffset(const MDNode *ProfileData)

Return the offset to the first branch weight data.

Definition ProfDataUtils.cpp:167

LLVM_ABI void setExplicitlyUnknownBranchWeightsIfProfiled(Instruction &I, StringRef PassName, const Function *F=nullptr)

Like setExplicitlyUnknownBranchWeights(...), but only sets unknown branch weights in the new instruct...

Definition ProfDataUtils.cpp:276

LLVM_ABI bool isBranchWeightMD(const MDNode *ProfileData)

Checks if an MDNode contains Branch Weight Metadata.

Definition ProfDataUtils.cpp:123

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

LLVM_ABI bool isExplicitlyUnknownProfileMetadata(const MDNode &MD)

Definition ProfDataUtils.cpp:297

LLVM_ABI MDNode * getBranchWeightMDNode(const Instruction &I)

Get the branch weights metadata node.

Definition ProfDataUtils.cpp:175

void append_range(Container &C, Range &&R)

Wrapper function to append range R to container C.

LLVM_ABI void setExplicitlyUnknownBranchWeights(Instruction &I, StringRef PassName)

Specify that the branch weights for this terminator cannot be known at compile time.

Definition ProfDataUtils.cpp:266

LLVM_ABI bool hasBranchWeightOrigin(const Instruction &I)

Check if Branch Weight Metadata has an "expected" field from an llvm.expect* intrinsic.

Definition ProfDataUtils.cpp:150

LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected, bool ElideAllZero=false)

Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...

Definition ProfDataUtils.cpp:310

int countl_zero(T Val)

Count number of 0's from the most significant bit to the least stopping at the first 1.

LLVM_ABI SmallVector< uint32_t > fitWeights(ArrayRef< uint64_t > Weights)

Push the weights right to fit in uint32_t.

Definition ProfDataUtils.cpp:89

LLVM_ABI MDNode * getValidBranchWeightMDNode(const Instruction &I)

Get the valid branch weights metadata node.

Definition ProfDataUtils.cpp:182

LLVM_ABI bool hasValidBranchWeightMD(const Instruction &I)

Checks if an instructions has valid Branch Weight Metadata.

Definition ProfDataUtils.cpp:146

FunctionAddr VTableAddr Count

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

LLVM_ABI void setExplicitlyUnknownFunctionEntryCount(Function &F, StringRef PassName)

Analogous to setExplicitlyUnknownBranchWeights, but for functions and their entry counts.

Definition ProfDataUtils.cpp:287

LLVM_ABI bool isValueProfileMD(const MDNode *ProfileData)

Checks if an MDNode contains value profiling Metadata.

Definition ProfDataUtils.cpp:127

LLVM_ABI unsigned getNumBranchWeights(const MDNode &ProfileData)

Definition ProfDataUtils.cpp:171

LLVM_ABI void extractFromBranchWeightMD32(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)

Faster version of extractBranchWeights() that skips checks and must only be called with "branch_weigh...

Definition ProfDataUtils.cpp:189

LLVM_ABI bool hasExplicitlyUnknownBranchWeights(const Instruction &I)

Definition ProfDataUtils.cpp:303

LLVM_ABI bool hasProfMD(const Instruction &I)

Checks if an Instruction has MD_prof Metadata.

Definition ProfDataUtils.cpp:119

auto max_element(R &&Range)

Provide wrappers to std::max_element which take ranges instead of having to pass begin/end explicitly...

LLVM_ABI bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)

Extract branch weights from MD_prof metadata.

Definition ProfDataUtils.cpp:199

LLVM_ABI const char * LLVMLoopEstimatedTripCount

Profile-based loop metadata that should be accessed only by using llvm::getLoopEstimatedTripCount and...

Definition ProfDataUtils.cpp:117

uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale)

Scale an individual branch count.

LLVM_ABI bool hasBranchWeightMD(const Instruction &I)

Checks if an instructions has Branch Weight Metadata.

Definition ProfDataUtils.cpp:131

LLVM_ABI void setFittedBranchWeights(Instruction &I, ArrayRef< uint64_t > Weights, bool IsExpected, bool ElideAllZero=false)

Variant of setBranchWeights where the Weights will be fit first to uint32_t by shifting right.

Definition ProfDataUtils.cpp:323

uint64_t calculateCountScale(uint64_t MaxCount)

Calculate what to divide by to scale counts.

LLVM_ABI SmallVector< uint32_t > downscaleWeights(ArrayRef< uint64_t > Weights, std::optional< uint64_t > KnownMaxCount=std::nullopt)

downscale the given weights preserving the ratio.

Definition ProfDataUtils.cpp:329

const uint64_t NOMORE_ICP_MAGICNUM

Magic number in the value profile metadata showing a target has been promoted for the instruction and...

LLVM_ABI void scaleProfData(Instruction &I, uint64_t S, uint64_t T)

Scaling the profile data attached to 'I' using the ratio of S/T.

Definition ProfDataUtils.cpp:341

LLVM_ABI void extractFromBranchWeightMD64(const MDNode *ProfileData, SmallVectorImpl< uint64_t > &Weights)

Faster version of extractBranchWeights() that skips checks and must only be called with "branch_weigh...

Definition ProfDataUtils.cpp:194

static LLVM_ABI const char * ExpectedBranchWeights

static LLVM_ABI const char * SyntheticFunctionEntryCount

static LLVM_ABI const char * UnknownBranchWeightsMarker

static LLVM_ABI const char * ValueProfile

static LLVM_ABI const char * FunctionEntryCount

static LLVM_ABI const char * BranchWeights