PGOCtxProfFlattening.cpp Source File (original) (raw)

40using namespace llvm;

42#define DEBUG_TYPE "ctx_prof_flatten"

44namespace {

52 F.setEntryCount(RawCounters[0]);

55 for (auto &BB : F) {

56 for (auto &I : BB)

58 uint64_t TrueCount, FalseCount = 0;

60 continue;

62 std::max(TrueCount, FalseCount));

63 }

65 continue;

68 continue;

70 setProfMetadata(BB.getTerminator(), ProfileHolder, MaxCount);

71 }

72}

74[[maybe_unused]] bool areAllBBsReachable(const Function &F,

78 F, [&](const BasicBlock &BB) { return DT.isReachableFromEntry(&BB); });

79}

81void clearColdFunctionProfile(Function &F) {

82 for (auto &BB : F)

83 BB.getTerminator()->setMetadata(LLVMContext::MD_prof, nullptr);

84 F.setEntryCount(0U);

85}

87void removeInstrumentation(Function &F) {

88 for (auto &BB : F)

91 I.eraseFromParent();

92}

94void annotateIndirectCall(

99 auto FIt = FlatProf.find(Idx);

100 if (FIt == FlatProf.end())

101 return;

102 const auto &Targets = FIt->second;

105 for (auto &[Guid, Count] : Targets) {

106 Data.push_back({Guid, Count});

108 }

109

111 [](const InstrProfValueData &A, const InstrProfValueData &B) {

112 return A.Count > B.Count;

113 });

115 InstrProfValueKind::IPVK_IndirectCallTarget,

116 Data.size());

117 LLVM_DEBUG(dbgs() << "[ctxprof] flat indirect call prof: " << CB

118 << CB.getMetadata(LLVMContext::MD_prof) << "\n");

119}

120

121

122

123

126 for (auto &F : M) {

127 if (F.isDeclaration())

128 continue;

130 if (FlatProfIter == FlatIndCalls.end())

131 continue;

132 const auto &FlatProf = FlatProfIter->second;

133 for (auto &BB : F) {

134 for (auto &I : BB) {

137 continue;

139 annotateIndirectCall(M, *CB, FlatProf, *Ins);

140 }

141 }

142 }

143}

144

145}

146

149

150

151

152

153

154

156 if (IsPreThinlink)

157 return;

158 for (auto &F : M)

159 removeInstrumentation(F);

160 });

162

163

164

165 if (!IsPreThinlink && !CtxProf.isInSpecializedModule())

167

168 if (IsPreThinlink)

169 annotateIndirectCalls(M, CtxProf);

170 const auto FlattenedProfile = CtxProf.flatten();

171

172 for (auto &F : M) {

173 if (F.isDeclaration())

174 continue;

175

176 assert(areAllBBsReachable(

178 .getManager()) &&

179 "Function has unreachable basic blocks. The expectation was that "

180 "DCE was run before.");

181

183

184 if (It == FlattenedProfile.end())

185 clearColdFunctionProfile(F);

186 else

187 assignProfileData(F, It->second);

188 }

190

191

192

193 for (auto &C : FlattenedProfile) {

194 PB.addEntryCount(C.second[0]);

196 PB.addInternalCount(V);

197 }

198

199 M.setProfileSummary(PB.getSummary()->getMD(M.getContext()),

203 MAM.invalidate(M, PA);

205 PSI.refresh(PB.getSummary());

207}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

This file provides various utilities for inspecting and working with the control flow graph in LLVM I...

Module.h This file contains the declarations for the Module class.

This header defines various interfaces for pass management in LLVM.

This file provides the interface for IR based instrumentation passes ( (profile-gen,...

FunctionAnalysisManager FAM

ModuleAnalysisManager MAM

PassBuilder PB(Machine, PassOpts->PTO, std::nullopt, &PIC)

This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

bool empty() const

empty - Check if the array is empty.

static LLVM_ABI uint64_t getGUID(const Function &F)

LLVM Basic Block Representation.

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

LLVM_ABI bool isIndirectCall() const

Return true if the callsite is an indirect call.

uint64_t getZExtValue() const

Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...

static LLVM_ABI InstrProfCallsite * getCallsiteInstrumentation(CallBase &CB)

Get the instruction instrumenting a callsite, or nullptr if that cannot be found.

PGOContextualProfile Result

iterator find(const_arg_type_t< KeyT > Val)

Analysis pass which computes a DominatorTree.

This represents the llvm.instrprof.callsite intrinsic.

LLVM_ABI ConstantInt * getIndex() const

MDNode * getMetadata(unsigned KindID) const

Get the metadata of given kind attached to this Instruction.

A Module instance is used to store all the information related to an LLVM module.

LLVM_ABI const CtxProfFlatIndirectCallProfile flattenVirtCalls() const

PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)

Definition PGOCtxProfFlattening.cpp:147

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

PreservedAnalyses & abandon()

Mark an analysis as abandoned.

LLVM_ABI bool getSelectInstrProfile(SelectInst &SI, uint64_t &TrueCount, uint64_t &FalseCount) const

LLVM_ABI bool getOutgoingBranchWeights(BasicBlock &BB, SmallVectorImpl< uint64_t > &Profile, uint64_t &MaxCount) const

An analysis pass based on the new PM to deliver ProfileSummaryInfo.

static LLVM_ABI const ArrayRef< uint32_t > DefaultCutoffs

A vector of useful cutoff values for detailed summary.

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

Pass manager infrastructure for declaring and invalidating analyses.

@ C

The default llvm calling convention, compatible with C.

This is an optimization pass for GlobalISel generic memory operations.

auto drop_begin(T &&RangeOrContainer, size_t N=1)

Return a range covering RangeOrContainer with the first N elements excluded.

bool all_of(R &&range, UnaryPredicate P)

Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.

LLVM_ABI void setProfMetadata(Instruction *TI, ArrayRef< uint64_t > EdgeCounts, uint64_t MaxCount)

detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy

Provide the FunctionAnalysisManager to Module proxy.

LLVM_ABI void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)

Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...

void sort(IteratorTy Start, IteratorTy End)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

FunctionAddr VTableAddr Count

auto succ_size(const MachineBasicBlock *BB)

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

FunctionAddr VTableAddr uintptr_t uintptr_t Data

AnalysisManager< Function > FunctionAnalysisManager

Convenience typedef for the Function analysis manager.

AnalysisManager< Module > ModuleAnalysisManager

Convenience typedef for the Module analysis manager.