ProfDataUtils.cpp Source File (original) (raw)

23using namespace llvm;

25namespace {

42constexpr unsigned MinBWOps = 3;

45constexpr unsigned MinVPOps = 5;

49bool isTargetMD(const MDNode *ProfData, const char *Name, unsigned MinOps) {

52 if (!ProfData || Name || MinOps < 2)

53 return false;

56 if (NOps < MinOps)

57 return false;

59 auto *ProfDataName = dyn_cast(ProfData->getOperand(0));

60 if (!ProfDataName)

61 return false;

63 return ProfDataName->getString() == Name;

64}

66template <typename T,

67 typename = typename std::enable_if<std::is_arithmetic_v>>

68static void extractFromBranchWeightMD(const MDNode *ProfileData,

74 assert(WeightsIdx < NOps && "Weights Index must be less than NOps.");

75 Weights.resize(NOps - WeightsIdx);

77 for (unsigned Idx = WeightsIdx, E = NOps; Idx != E; ++Idx) {

79 mdconst::dyn_extract(ProfileData->getOperand(Idx));

80 assert(Weight && "Malformed branch_weight in MD_prof node");

82 "Too many bits for MD_prof branch_weight");

84 }

85}

87}

89namespace llvm {

92 return I.hasMetadata(LLVMContext::MD_prof);

93}

96 return isTargetMD(ProfileData, "branch_weights", MinBWOps);

97}

100 return isTargetMD(ProfileData, "VP", MinVPOps);

101}

102

104 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);

106}

107

109 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);

110

112 return true;

113

114

116}

117

120}

121

123 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);

125}

126

129 return false;

130 auto *ProfDataName = dyn_cast(ProfileData->getOperand(1));

131

132

133

134 assert(ProfDataName == nullptr || ProfDataName->getString() == "expected");

135 return ProfDataName != nullptr;

136}

137

140}

141

144}

145

147 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);

149 return nullptr;

150 return ProfileData;

151}

152

155 if (ProfileData && getNumBranchWeights(*ProfileData) == I.getNumSuccessors())

156 return ProfileData;

157 return nullptr;

158}

159

162 extractFromBranchWeightMD(ProfileData, Weights);

163}

164

167 extractFromBranchWeightMD(ProfileData, Weights);

168}

169

173 return false;

174 extractFromBranchWeightMD(ProfileData, Weights);

175 return true;

176}

177

180 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);

182}

183

186 assert((I.getOpcode() == Instruction::Br ||

187 I.getOpcode() == Instruction::Select) &&

188 "Looking for branch weights on something besides branch, select, or "

189 "switch");

190

192 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);

194 return false;

195

196 if (Weights.size() > 2)

197 return false;

198

199 TrueVal = Weights[0];

200 FalseVal = Weights[1];

201 return true;

202}

203

205 TotalVal = 0;

206 if (!ProfileData)

207 return false;

208

209 auto *ProfDataName = dyn_cast(ProfileData->getOperand(0));

210 if (!ProfDataName)

211 return false;

212

213 if (ProfDataName->getString() == "branch_weights") {

216 auto *V = mdconst::extract(ProfileData->getOperand(Idx));

217 TotalVal += V->getValue().getZExtValue();

218 }

219 return true;

220 }

221

222 if (ProfDataName->getString() == "VP" && ProfileData->getNumOperands() > 3) {

223 TotalVal = mdconst::dyn_extract(ProfileData->getOperand(2))

224 ->getValue()

225 .getZExtValue();

226 return true;

227 }

228 return false;

229}

230

233}

234

236 bool IsExpected) {

239 I.setMetadata(LLVMContext::MD_prof, BranchWeights);

240}

241

243 assert(T != 0 && "Caller should guarantee");

244 auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);

245 if (ProfileData == nullptr)

246 return;

247

248 auto *ProfDataName = dyn_cast(ProfileData->getOperand(0));

249 if (!ProfDataName || (ProfDataName->getString() != "branch_weights" &&

250 ProfDataName->getString() != "VP"))

251 return;

252

254 return;

255

257

261 APInt APS(128, S), APT(128, T);

262 if (ProfDataName->getString() == "branch_weights" &&

264

266 mdconst::dyn_extract(

268 ->getValue()

269 .getZExtValue());

270 Val *= APS;

273 } else if (ProfDataName->getString() == "VP")

274 for (unsigned i = 1; i < ProfileData->getNumOperands(); i += 2) {

275

278 mdconst::dyn_extract(ProfileData->getOperand(i + 1))

279 ->getValue()

280 .getZExtValue();

281

284 continue;

285 }

286

287 APInt Val(128, Count);

288 Val *= APS;

291 }

292 I.setMetadata(LLVMContext::MD_prof, MDNode::get(C, Vals));

293}

294

295}

This file contains the declarations for the subclasses of Constant, which represent the different fla...

Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx

This file contains the declarations for profiling metadata utility functions.

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

This file defines the SmallVector class.

Class for arbitrary precision integers.

APInt udiv(const APInt &RHS) const

Unsigned division operation.

unsigned getActiveBits() const

Compute the number of active bits in the value.

uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const

If this value is smaller than the specified limit, return it, otherwise return the limit value.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

This is the shared class of boolean and integer constants.

uint64_t getZExtValue() const

Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...

const APInt & getValue() const

Return the constant as an APInt value reference.

This is an important class for using LLVM in a threaded context.

ConstantAsMetadata * createConstant(Constant *C)

Return the given constant as metadata.

MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight, bool IsExpected=false)

Return metadata containing two branch weights.

const MDOperand & getOperand(unsigned I) const

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

unsigned getNumOperands() const

Return number of MDNode operands.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

static IntegerType * getInt32Ty(LLVMContext &C)

static IntegerType * getInt64Ty(LLVMContext &C)

@ C

The default llvm calling convention, compatible with C.

This is an optimization pass for GlobalISel generic memory operations.

bool extractProfTotalWeight(const MDNode *ProfileData, uint64_t &TotalWeights)

Retrieve the total of all weights from MD_prof data.

unsigned getBranchWeightOffset(const MDNode *ProfileData)

Return the offset to the first branch weight data.

bool isBranchWeightMD(const MDNode *ProfileData)

Checks if an MDNode contains Branch Weight Metadata.

MDNode * getBranchWeightMDNode(const Instruction &I)

Get the branch weights metadata node.

bool hasBranchWeightOrigin(const Instruction &I)

Check if Branch Weight Metadata has an "expected" field from an llvm.expect* intrinsic.

void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected)

Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...

MDNode * getValidBranchWeightMDNode(const Instruction &I)

Get the valid branch weights metadata node.

bool hasValidBranchWeightMD(const Instruction &I)

Checks if an instructions has valid Branch Weight Metadata.

bool isValueProfileMD(const MDNode *ProfileData)

bool hasCountTypeMD(const Instruction &I)

unsigned getNumBranchWeights(const MDNode &ProfileData)

void extractFromBranchWeightMD32(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)

Faster version of extractBranchWeights() that skips checks and must only be called with "branch_weigh...

bool hasProfMD(const Instruction &I)

Checks if an Instruction has MD_prof Metadata.

bool extractBranchWeights(const MDNode *ProfileData, SmallVectorImpl< uint32_t > &Weights)

Extract branch weights from MD_prof metadata.

bool hasBranchWeightMD(const Instruction &I)

Checks if an instructions has Branch Weight Metadata.

const uint64_t NOMORE_ICP_MAGICNUM

Magic number in the value profile metadata showing a target has been promoted for the instruction and...

void scaleProfData(Instruction &I, uint64_t S, uint64_t T)

Scaling the profile data attached to 'I' using the ratio of S/T.

void extractFromBranchWeightMD64(const MDNode *ProfileData, SmallVectorImpl< uint64_t > &Weights)

Faster version of extractBranchWeights() that skips checks and must only be called with "branch_weigh...