InlineOrder.cpp Source File (original) (raw)

21using namespace llvm;

23#define DEBUG_TYPE "inline-order"

29 cl::desc("Choose the priority mode to use in module inline"),

31 "Use callee size priority."),

33 "Use inline cost priority."),

35 "Use cost-benefit ratio."),

40 cl::desc("The cost threshold for call sites that get inlined without the "

41 "cost-benefit analysis"));

43namespace {

51 .getCachedResult(

52 *CB.getParent()->getParent()->getParent());

57 };

60 };

63 };

67 bool RemarksEnabled =

68 Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(

70 return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,

71 GetBFI, PSI, RemarksEnabled ? &ORE : nullptr);

72}

74class SizePriority {

75public:

76 SizePriority() = default;

78 const InlineParams &) {

80 Size = Callee->getInstructionCount();

81 }

83 static bool isMoreDesirable(const SizePriority &P1, const SizePriority &P2) {

84 return P1.Size < P2.Size;

85 }

87private:

88 unsigned Size = UINT_MAX;

89};

91class CostPriority {

92public:

93 CostPriority() = default;

95 const InlineParams &Params) {

96 auto IC = getInlineCostWrapper(const_cast<CallBase &>(*CB), FAM, Params);

97 if (IC.isVariable())

98 Cost = IC.getCost();

99 else

100 Cost = IC.isNever() ? INT_MAX : INT_MIN;

101 }

102

103 static bool isMoreDesirable(const CostPriority &P1, const CostPriority &P2) {

104 return P1.Cost < P2.Cost;

105 }

106

107private:

108 int Cost = INT_MAX;

109};

110

111class CostBenefitPriority {

112public:

113 CostBenefitPriority() = default;

115 const InlineParams &Params) {

116 auto IC = getInlineCostWrapper(const_cast<CallBase &>(*CB), FAM, Params);

117 if (IC.isVariable())

118 Cost = IC.getCost();

119 else

120 Cost = IC.isNever() ? INT_MAX : INT_MIN;

121 StaticBonusApplied = IC.getStaticBonusApplied();

122 CostBenefit = IC.getCostBenefit();

123 }

124

125 static bool isMoreDesirable(const CostBenefitPriority &P1,

126 const CostBenefitPriority &P2) {

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142 bool P1ReducesCallerSize =

144 bool P2ReducesCallerSize =

146 if (P1ReducesCallerSize || P2ReducesCallerSize) {

147

148

149 if (P1ReducesCallerSize != P2ReducesCallerSize)

150 return P1ReducesCallerSize;

151

152

153

154 return P1.Cost < P2.Cost;

155 }

156

157 bool P1HasCB = P1.CostBenefit.has_value();

158 bool P2HasCB = P2.CostBenefit.has_value();

159 if (P1HasCB || P2HasCB) {

160

161

162 if (P1HasCB != P2HasCB)

163 return P1HasCB;

164

165

166

167 APInt LHS = P1.CostBenefit->getBenefit() * P2.CostBenefit->getCost();

168 APInt RHS = P2.CostBenefit->getBenefit() * P1.CostBenefit->getCost();

170 }

171

172

173 return P1.Cost < P2.Cost;

174 }

175

176private:

177 int Cost = INT_MAX;

178 int StaticBonusApplied = 0;

179 std::optional CostBenefit;

180};

181

182class MLPriority {

183public:

184 MLPriority() = default;

186 const InlineParams &Params) {

187 auto IC = getInlineCostWrapper(const_cast<CallBase &>(*CB), FAM, Params);

188 if (IC.isVariable())

189 Cost = IC.getCost();

190 else

191 Cost = IC.isNever() ? INT_MAX : INT_MIN;

192 }

193

194 static bool isMoreDesirable(const MLPriority &P1, const MLPriority &P2) {

195 return P1.Cost < P2.Cost;

196 }

197

198private:

199 int Cost = INT_MAX;

200};

201

202template

203class PriorityInlineOrder : public InlineOrder<std::pair<CallBase *, int>> {

204 using T = std::pair<CallBase *, int>;

205

206 bool hasLowerPriority(const CallBase *L, const CallBase *R) const {

207 const auto I1 = Priorities.find(L);

208 const auto I2 = Priorities.find(R);

209 assert(I1 != Priorities.end() && I2 != Priorities.end());

210 return PriorityT::isMoreDesirable(I2->second, I1->second);

211 }

212

213 bool updateAndCheckDecreased(const CallBase *CB) {

214 auto It = Priorities.find(CB);

215 const auto OldPriority = It->second;

216 It->second = PriorityT(CB, FAM, Params);

217 const auto NewPriority = It->second;

218 return PriorityT::isMoreDesirable(OldPriority, NewPriority);

219 }

220

221

222

223

224

225

226

227

228 void pop_heap_adjust() {

229 std::pop_heap(Heap.begin(), Heap.end(), isLess);

230 while (updateAndCheckDecreased(Heap.back())) {

231 std::push_heap(Heap.begin(), Heap.end(), isLess);

232 std::pop_heap(Heap.begin(), Heap.end(), isLess);

233 }

234 }

235

236public:

238 : FAM(FAM), Params(Params) {

239 isLess = [&](const CallBase *L, const CallBase *R) {

240 return hasLowerPriority(L, R);

241 };

242 }

243

244 size_t size() override { return Heap.size(); }

245

246 void push(const T &Elt) override {

247 CallBase *CB = Elt.first;

248 const int InlineHistoryID = Elt.second;

249

250 Heap.push_back(CB);

251 Priorities[CB] = PriorityT(CB, FAM, Params);

252 std::push_heap(Heap.begin(), Heap.end(), isLess);

253 InlineHistoryMap[CB] = InlineHistoryID;

254 }

255

256 T pop() override {

258 pop_heap_adjust();

259

260 CallBase *CB = Heap.pop_back_val();

261 T Result = std::make_pair(CB, InlineHistoryMap[CB]);

262 InlineHistoryMap.erase(CB);

264 }

265

266 void erase_if(function_ref<bool(T)> Pred) override {

267 auto PredWrapper = [=](CallBase *CB) -> bool {

268 return Pred(std::make_pair(CB, InlineHistoryMap[CB]));

269 };

271 std::make_heap(Heap.begin(), Heap.end(), isLess);

272 }

273

274private:

276 std::function<bool(const CallBase *L, const CallBase *R)> isLess;

277 DenseMap<CallBase *, int> InlineHistoryMap;

278 DenseMap<const CallBase *, PriorityT> Priorities;

280 const InlineParams &Params;

281};

282

283}

284

286

287std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>

293 LLVM_DEBUG(dbgs() << " Current used priority: Size priority ---- \n");

294 return std::make_unique<PriorityInlineOrder>(FAM, Params);

295

297 LLVM_DEBUG(dbgs() << " Current used priority: Cost priority ---- \n");

298 return std::make_unique<PriorityInlineOrder>(FAM, Params);

299

302 dbgs() << " Current used priority: cost-benefit priority ---- \n");

303 return std::make_unique<PriorityInlineOrder>(FAM,

304 Params);

306 LLVM_DEBUG(dbgs() << " Current used priority: ML priority ---- \n");

307 return std::make_unique<PriorityInlineOrder>(FAM, Params);

308 }

309 return nullptr;

310}

311

312std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>

316 LLVM_DEBUG(dbgs() << " Current used priority: plugin ---- \n");

318 M);

319 }

321}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

#define clEnumValN(ENUMVAL, FLAGNAME, DESC)

This is the interface for a simple mod/ref and alias analysis over globals.

static cl::opt< int > ModuleInlinerTopPriorityThreshold("module-inliner-top-priority-threshold", cl::Hidden, cl::init(0), cl::desc("The cost threshold for call sites that get inlined without the " "cost-benefit analysis"))

InlinePriorityMode

Definition InlineOrder.cpp:25

@ Cost

Definition InlineOrder.cpp:25

@ CostBenefit

Definition InlineOrder.cpp:25

@ Size

Definition InlineOrder.cpp:25

@ ML

Definition InlineOrder.cpp:25

static cl::opt< InlinePriorityMode > UseInlinePriority("inline-priority-mode", cl::init(InlinePriorityMode::Size), cl::Hidden, cl::desc("Choose the priority mode to use in module inline"), cl::values(clEnumValN(InlinePriorityMode::Size, "size", "Use callee size priority."), clEnumValN(InlinePriorityMode::Cost, "cost", "Use inline cost priority."), clEnumValN(InlinePriorityMode::CostBenefit, "cost-benefit", "Use cost-benefit ratio."), clEnumValN(InlinePriorityMode::ML, "ml", "Use ML.")))

FunctionAnalysisManager FAM

ModuleAnalysisManager MAM

This pass exposes codegen information to IR-level passes.

A function analysis which provides an AssumptionCache.

A cache of @llvm.assume calls within a function.

Analysis pass which computes BlockFrequencyInfo.

BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

Function * getCalledFunction() const

Returns the function called, or null if this is an indirect function invocation or the function signa...

LLVM_ABI Function * getCaller()

Helper to get the caller (the parent function).

Represents the cost of inlining a function.

A Module instance is used to store all the information related to an LLVM module.

Used for dynamically loading instances of InlineOrder as plugins.

static LLVM_ABI AnalysisKey Key

Analysis providing profile information.

Analysis pass providing the TargetTransformInfo.

Analysis pass providing the TargetLibraryInfo.

Provides information about what library functions are available for the current target.

const ParentTy * getParent() const

ValuesClass values(OptsTy... Options)

Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...

initializer< Ty > init(const Ty &Val)

This is an optimization pass for GlobalISel generic memory operations.

auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)

Get the size of a range.

OuterAnalysisManagerProxy< ModuleAnalysisManager, Function > ModuleAnalysisManagerFunctionProxy

Provide the ModuleAnalysisManager to Function proxy.

LLVM_ABI std::unique_ptr< InlineOrder< std::pair< CallBase *, int > > > getDefaultInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params, ModuleAnalysisManager &MAM, Module &M)

Definition InlineOrder.cpp:288

LLVM_ABI std::unique_ptr< InlineOrder< std::pair< CallBase *, int > > > getInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params, ModuleAnalysisManager &MAM, Module &M)

Definition InlineOrder.cpp:313

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

LLVM_ABI InlineCost getInlineCost(CallBase &Call, const InlineParams &Params, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< const TargetLibraryInfo &(Function &)> GetTLI, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr, function_ref< EphemeralValuesCache &(Function &)> GetEphValuesCache=nullptr)

Get an InlineCost object representing the cost of inlining this callsite.

void erase_if(Container &C, UnaryPredicate P)

Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...

AnalysisManager< Function > FunctionAnalysisManager

Convenience typedef for the Function analysis manager.

AnalysisManager< Module > ModuleAnalysisManager

Convenience typedef for the Module analysis manager.

A special type used by analysis passes to provide an address that identifies that particular analysis...

Thresholds to tune inline cost analysis.