MLInlineAdvisor.cpp Source File (original) (raw)

37using namespace llvm;

40 "inliner-interactive-channel-base", cl::Hidden,

42 "Base file path for the interactive mode. The incoming filename should "

43 "have the name .in, while the "

44 "outgoing name should be .out"));

46 (Twine("In interactive mode, also send the default policy decision: ") +

48 .str();

59 "if-caller-not-cold", "if the caller is not cold")));

67#if defined(LLVM_HAVE_TF_AOT_INLINERSIZEMODEL)

69#include "InlinerSizeModel.h"

71#else

73#endif

75std::unique_ptr

77 std::function<bool(CallBase &)> GetDefaultAdvice) {

80 return nullptr;

81 auto RunnerFactory = [&](const std::vector &InputFeatures)

82 -> std::unique_ptr {

83 std::unique_ptr AOTRunner;

85 AOTRunner = std::make_unique<ReleaseModeModelRunner>(

88 else {

89 AOTRunner = std::make_unique(

93 }

94 return AOTRunner;

95 };

96 return std::make_unique(M, MAM, RunnerFactory,

97 GetDefaultAdvice);

98}

100#define DEBUG_TYPE "inline-ml"

101

103 "ml-advisor-size-increase-threshold", cl::Hidden,

104 cl::desc("Maximum factor by which expected native size may increase before "

105 "blocking any further inlining."),

107

109 "ml-advisor-keep-fpi-cache", cl::Hidden,

111 "For test - keep the ML Inline advisor's FunctionPropertiesInfo cache"),

113

115

116static std::vector FeatureMap{

117#define POPULATE_NAMES(DTYPE, SHAPE, NAME, __) TensorSpec::createSpec(#NAME, SHAPE),

118

120

121

123#undef POPULATE_NAMES

124};

125

127}

128

136

139 if (Function *Callee = CS->getCalledFunction()) {

140 if (!Callee->isDeclaration()) {

141 return CS;

142 }

143 }

144 return nullptr;

145}

146

149 std::function<

150 std::unique_ptr(const std::vector &)>

151 GetModelRunner,

158 InitialIRSize(getModuleIRSize()), CurrentIRSize(InitialIRSize),

160

161

162

163

164

165

167 for (auto I = scc_begin(&CGraph); .isAtEnd(); ++I) {

168 const std::vector<CallGraphNode *> &CGNodes = *I;

169 unsigned Level = 0;

170 for (auto *CGNode : CGNodes) {

171 Function *F = CGNode->getFunction();

172 if ( || F->isDeclaration())

173 continue;

176 auto *Called = CS->getCalledFunction();

177 auto Pos = FunctionLevels.find(&CG.get(*Called));

178

179

180

181 if (Pos == FunctionLevels.end())

182 continue;

183 Level = std::max(Level, Pos->second + 1);

184 }

185 }

186 }

187 for (auto *CGNode : CGNodes) {

188 Function *F = CGNode->getFunction();

189 if (F && ->isDeclaration())

190 FunctionLevels[&CG.get(*F)] = Level;

191 }

192 }

193 for (auto KVP : FunctionLevels) {

194 AllNodes.insert(KVP.first);

195 EdgeCount += getLocalCalls(KVP.first->getFunction());

196 }

197 NodeCount = AllNodes.size();

198

200 if (!IR2VecVocabResult->isValid()) {

201 M.getContext().emitError("IR2VecVocabAnalysis is not valid");

202 return;

203 }

204

205 auto IR2VecDim = IR2VecVocabResult->getDimension();

210 }

213

216 M.getContext().emitError("Could not create model runner");

217 return;

218 }

221}

222

224 return CG.lookup(F) ? FunctionLevels.at(CG.lookup(F)) : 0;

225}

226

228 if (!CurSCC || ForceStop)

229 return;

230 FPICache.clear();

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248 while (!NodesInLastSCC.empty()) {

249 const auto *N = *NodesInLastSCC.begin();

251 NodesInLastSCC.erase(N);

253 const auto NLevel = FunctionLevels.at(N);

254 for (const auto &E : *(*N)) {

255 const auto *AdjNode = &E.getNode();

256 assert(!AdjNode->isDead() && !AdjNode->getFunction().isDeclaration());

257 auto I = AllNodes.insert(AdjNode);

258

259 if (I.second) {

260 ++NodeCount;

261 NodesInLastSCC.insert(AdjNode);

262 FunctionLevels[AdjNode] = NLevel;

263 }

264 }

265 }

266

267 EdgeCount -= EdgesOfLastSeenNodes;

268 EdgesOfLastSeenNodes = 0;

269

270

271

272 assert(NodesInLastSCC.empty());

273 for (const auto &N : *CurSCC)

274 NodesInLastSCC.insert(&N);

275}

276

278

280 FPICache.clear();

281 if (!CurSCC || ForceStop)

282 return;

283

284

285

286 EdgesOfLastSeenNodes = 0;

287

288

291 EdgesOfLastSeenNodes += getLocalCalls(N->getFunction());

292 }

293

294

295 for (const auto &N : *CurSCC) {

297 auto I = NodesInLastSCC.insert(&N);

298 if (I.second)

299 EdgesOfLastSeenNodes += getLocalCalls(N.getFunction());

300 }

301 assert(NodeCount >= NodesInLastSCC.size());

302 assert(EdgeCount >= EdgesOfLastSeenNodes);

303}

304

308

309

310

311

312

313

315 bool CalleeWasDeleted) {

319

320 {

324 FAM.invalidate(*Caller, PA);

325 }

327 if (Caller == Callee) {

328 assert(!CalleeWasDeleted);

329

330

335

336 } else {

337 int64_t IRSizeAfter =

340

341

342

343

344

345

346 int64_t NewCallerAndCalleeEdges =

348

349

350

351

352 if (CalleeWasDeleted) {

353 --NodeCount;

354 NodesInLastSCC.erase(CG.lookup(*Callee));

355 DeadFunctions.insert(Callee);

356 } else {

357 NewCallerAndCalleeEdges +=

359 }

361 }

363 ForceStop = true;

364

365 assert(CurrentIRSize >= 0 && EdgeCount >= 0 && NodeCount >= 0);

366}

367

368int64_t MLInlineAdvisor::getModuleIRSize() const {

369 int64_t Ret = 0;

370 for (auto &F : M)

371 if (.isDeclaration())

373 return Ret;

374}

375

377 auto InsertPair = FPICache.try_emplace(&F);

378 if (!InsertPair.second)

379 return InsertPair.first->second;

381 return InsertPair.first->second;

382}

383

385 if (auto Skip = getSkipAdviceIfUnreachableCallsite(CB))

386 return Skip;

387

390

393 };

396

398 if (!PSI.isFunctionEntryCold(&Caller)) {

399

400

401

402

403

404 return ForceStop ? std::make_unique(this, CB, ORE,

406 : std::make_unique(this, CB, ORE,

408 }

409 }

411

412

413

414

416 &Caller == &Callee)

418

419 bool Mandatory =

421

422

423

424 if (ForceStop) {

425 ORE.emit([&] {

427 << "Won't attempt inlining because module size grew too much.";

428 });

429 return std::make_unique(this, CB, ORE, Mandatory);

430 }

431

432 int CostEstimate = 0;

433 if (!Mandatory) {

434 auto IsCallSiteInlinable =

436 if (!IsCallSiteInlinable) {

437

438

439

440 return std::make_unique(this, CB, ORE, false);

441 }

442 CostEstimate = *IsCallSiteInlinable;

443 }

444

445 const auto CostFeatures =

447 if (!CostFeatures) {

448 return std::make_unique(this, CB, ORE, false);

449 }

450

451 if (Mandatory)

453

454 auto NumCtantParams = 0;

457 }

458

461

462 *ModelRunner->getTensor<int64_t>(FeatureIndex::callee_basic_block_count) =

463 CalleeBefore.BasicBlockCount;

464 *ModelRunner->getTensor<int64_t>(FeatureIndex::callsite_height) =

466 *ModelRunner->getTensor<int64_t>(FeatureIndex::node_count) = NodeCount;

467 *ModelRunner->getTensor<int64_t>(FeatureIndex::nr_ctant_params) =

468 NumCtantParams;

469 *ModelRunner->getTensor<int64_t>(FeatureIndex::edge_count) = EdgeCount;

470 *ModelRunner->getTensor<int64_t>(FeatureIndex::caller_users) =

471 CallerBefore.Uses;

473 FeatureIndex::caller_conditionally_executed_blocks) =

474 CallerBefore.BlocksReachedFromConditionalInstruction;

475 *ModelRunner->getTensor<int64_t>(FeatureIndex::caller_basic_block_count) =

476 CallerBefore.BasicBlockCount;

478 FeatureIndex::callee_conditionally_executed_blocks) =

479 CalleeBefore.BlocksReachedFromConditionalInstruction;

480 *ModelRunner->getTensor<int64_t>(FeatureIndex::callee_users) =

481 CalleeBefore.Uses;

482 *ModelRunner->getTensor<int64_t>(FeatureIndex::cost_estimate) = CostEstimate;

483 *ModelRunner->getTensor<int64_t>(FeatureIndex::is_callee_avail_external) =

484 Callee.hasAvailableExternallyLinkage();

485 *ModelRunner->getTensor<int64_t>(FeatureIndex::is_caller_avail_external) =

486 Caller.hasAvailableExternallyLinkage();

487

488 if (UseIR2Vec) {

489

490

491

495 [](double Val) { return static_cast(Val); });

496 };

497

498 setEmbedding(CalleeBefore.getFunctionEmbedding(),

500 setEmbedding(CallerBefore.getFunctionEmbedding(),

502 }

503

504

505 for (size_t I = 0;

509 }

510

515}

516

517std::unique_ptr

520 return std::make_unique(

521 this, CB, ORE, static_cast<bool>(ModelRunner->evaluate<int64_t>()));

522}

523

524std::unique_ptr

525MLInlineAdvisor::getSkipAdviceIfUnreachableCallsite(CallBase &CB) {

527 .isReachableFromEntry(CB.getParent()))

528 return std::make_unique(this, CB, getCallerORE(CB), false);

529 return nullptr;

530}

531

533 bool Advice) {

534

535 if (auto Skip = getSkipAdviceIfUnreachableCallsite(CB))

536 return Skip;

537 if (Advice && !ForceStop)

539

540

541

542

543

544 return std::make_unique(this, CB, getCallerORE(CB), Advice);

545}

546

547std::unique_ptr

549 return std::make_unique(this, CB, getCallerORE(CB), true);

550}

551

552void MLInlineAdvisor::print(raw_ostream &OS) const {

553 OS << "[MLInlineAdvisor] Nodes: " << NodeCount << " Edges: " << EdgeCount

554 << " EdgesOfLastSeenNodes: " << EdgesOfLastSeenNodes << "\n";

555 OS << "[MLInlineAdvisor] FPI:\n";

556 for (auto I : FPICache) {

557 OS << I.first->getName() << ":\n";

558 I.second.print(OS);

559 OS << "\n";

560 }

561 OS << "\n";

562 OS << "[MLInlineAdvisor] FuncLevels:\n";

563 for (auto I : FunctionLevels)

564 OS << (DeadFunctions.contains(&I.first->getFunction())

565 ? ""

566 : I.first->getFunction().getName())

567 << " : " << I.second << "\n";

568

569 OS << "\n";

570}

571

574 bool Recommendation)

579 ? 0

582 PreInlineCallerFPI(Advisor->getCachedFPI(*Caller)) {

583 if (Recommendation)

585}

586

587void MLInlineAdvice::reportContextForRemark(

589 using namespace ore;

591 for (size_t I = 0; I < getAdvisor()->getFeatureMap().size(); ++I)

592 OR << NV(getAdvisor()->getFeatureMap()[I].name(),

593 *getAdvisor()->getModelRunner().getTensor<int64_t>(I));

595}

596

598 FPU->finish(FAM);

599}

600

602 ORE.emit([&]() {

604 reportContextForRemark(R);

605 return R;

606 });

607 getAdvisor()->onSuccessfulInlining(*this, false);

608}

609

611 ORE.emit([&]() {

614 reportContextForRemark(R);

615 return R;

616 });

617 getAdvisor()->onSuccessfulInlining(*this, true);

618}

619

622 getAdvisor()->getCachedFPI(*Caller) = PreInlineCallerFPI;

623 ORE.emit([&]() {

626 reportContextForRemark(R);

627 return R;

628 });

629}

632 ORE.emit([&]() {

634 reportContextForRemark(R);

635 return R;

636 });

637}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

Expand Atomic instructions

This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...

#define clEnumValN(ENUMVAL, FLAGNAME, DESC)

Module.h This file contains the declarations for the Module class.

This header defines various interfaces for pass management in LLVM.

#define INLINE_COST_FEATURE_ITERATOR(M)

#define INLINE_FEATURE_ITERATOR(M)

Implements a lazy call graph analysis and related passes for the new pass manager.

static cl::opt< bool > KeepFPICache("ml-advisor-keep-fpi-cache", cl::Hidden, cl::desc("For test - keep the ML Inline advisor's FunctionPropertiesInfo cache"), cl::init(false))

static cl::opt< std::string > ModelSelector("ml-inliner-model-selector", cl::Hidden, cl::init(""))

CallBase * getInlinableCS(Instruction &I)

Definition MLInlineAdvisor.cpp:137

NoopSavedModelImpl CompiledModelType

Definition MLInlineAdvisor.cpp:72

SkipMLPolicyCriteria

Definition MLInlineAdvisor.cpp:53

@ Never

Definition MLInlineAdvisor.cpp:53

@ IfCallerIsNotCold

Definition MLInlineAdvisor.cpp:53

static cl::opt< std::string > InteractiveChannelBaseName("inliner-interactive-channel-base", cl::Hidden, cl::desc("Base file path for the interactive mode. The incoming filename should " "have the name .in, while the " "outgoing name should be .out"))

#define POPULATE_NAMES(DTYPE, SHAPE, NAME, __)

static cl::opt< bool > StopImmediatelyForTest("ml-inliner-stop-immediately", cl::Hidden)

static cl::opt< float > SizeIncreaseThreshold("ml-advisor-size-increase-threshold", cl::Hidden, cl::desc("Maximum factor by which expected native size may increase before " "blocking any further inlining."), cl::init(2.0))

static const std::string InclDefaultMsg

Definition MLInlineAdvisor.cpp:45

static cl::opt< SkipMLPolicyCriteria > SkipPolicy("ml-inliner-skip-policy", cl::Hidden, cl::init(SkipMLPolicyCriteria::Never), cl::values(clEnumValN(SkipMLPolicyCriteria::Never, "never", "never"), clEnumValN(SkipMLPolicyCriteria::IfCallerIsNotCold, "if-caller-not-cold", "if the caller is not cold")))

static cl::opt< bool > InteractiveIncludeDefault("inliner-interactive-include-default", cl::Hidden, cl::desc(InclDefaultMsg))

FunctionAnalysisManager FAM

ModuleAnalysisManager MAM

This builds on the llvm/ADT/GraphTraits.h file to find the strongly connected components (SCCs) of a ...

This pass exposes codegen information to IR-level passes.

A function analysis which provides an AssumptionCache.

A cache of @llvm.assume calls within a function.

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

Function * getCalledFunction() const

Returns the function called, or null if this is an indirect function invocation or the function signa...

User::op_iterator arg_begin()

Return the iterator pointing to the beginning of the argument list.

User::op_iterator arg_end()

Return the iterator pointing to the end of the argument list.

LLVM_ABI Function * getCaller()

Helper to get the caller (the parent function).

The basic data container for the call graph of a Module of IR.

Common features for diagnostics dealing with optimization remarks that are used by both IR and MIR pa...

Analysis pass which computes a DominatorTree.

int64_t DirectCallsToDefinedFunctions

Number of direct calls made from this function to other functions defined in this module.

This analysis provides the vocabulary for IR2Vec.

Function *const Caller

Caller and Callee are pre-inlining.

const BasicBlock *const Block

OptimizationRemarkEmitter & ORE

InlineAdvisor *const Advisor

LLVM_ABI InlineAdvice(InlineAdvisor *Advisor, CallBase &CB, OptimizationRemarkEmitter &ORE, bool IsInliningRecommended)

bool isInliningRecommended() const

Get the inlining recommendation.

OptimizationRemarkEmitter & getCallerORE(CallBase &CB)

FunctionAnalysisManager & FAM

static MandatoryInliningKind getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, OptimizationRemarkEmitter &ORE)

InlineAdvisor(InlineAdvisor &&)=delete

InlineResult is basically true or false.

An analysis pass which computes the call graph for a module.

A node in the call graph.

An SCC of the call graph.

Analysis pass that exposes the LoopInfo for a function.

InlineAdvice that tracks changes post inlining.

void updateCachedCallerFPI(FunctionAnalysisManager &FAM) const

Definition MLInlineAdvisor.cpp:597

const int64_t CallerIRSize

MLInlineAdvice(MLInlineAdvisor *Advisor, CallBase &CB, OptimizationRemarkEmitter &ORE, bool Recommendation)

Definition MLInlineAdvisor.cpp:572

const int64_t CalleeIRSize

void recordInliningImpl() override

Definition MLInlineAdvisor.cpp:601

Function * getCaller() const

const int64_t CallerAndCalleeEdges

void recordUnsuccessfulInliningImpl(const InlineResult &Result) override

Definition MLInlineAdvisor.cpp:620

Function * getCallee() const

void recordInliningWithCalleeDeletedImpl() override

Definition MLInlineAdvisor.cpp:610

void recordUnattemptedInliningImpl() override

Definition MLInlineAdvisor.cpp:630

const std::vector< TensorSpec > & getFeatureMap() const

std::unique_ptr< MLModelRunner > ModelRunner

FunctionPropertiesInfo & getCachedFPI(Function &) const

Definition MLInlineAdvisor.cpp:376

void onPassExit(LazyCallGraph::SCC *SCC) override

This must be called when the Inliner pass is exited, as function passes may be run subsequently.

Definition MLInlineAdvisor.cpp:277

void onSuccessfulInlining(const MLInlineAdvice &Advice, bool CalleeWasDeleted)

Definition MLInlineAdvisor.cpp:314

static const std::vector< TensorSpec > & getInitialFeatureMap()

Definition MLInlineAdvisor.cpp:114

virtual std::unique_ptr< MLInlineAdvice > getMandatoryAdviceImpl(CallBase &CB)

Definition MLInlineAdvisor.cpp:548

void onPassEntry(LazyCallGraph::SCC *SCC) override

This must be called when the Inliner pass is entered, to allow the InlineAdvisor update internal stat...

Definition MLInlineAdvisor.cpp:227

MLInlineAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function< std::unique_ptr< MLModelRunner >(const std::vector< TensorSpec > &)> GetModelRunner, std::function< bool(CallBase &)> GetDefaultAdvice)

Definition MLInlineAdvisor.cpp:147

int64_t getLocalCalls(Function &F)

Definition MLInlineAdvisor.cpp:305

std::vector< TensorSpec > FeatureMap

virtual std::unique_ptr< MLInlineAdvice > getAdviceFromModel(CallBase &CB, OptimizationRemarkEmitter &ORE)

Definition MLInlineAdvisor.cpp:518

int64_t getIRSize(Function &F) const

std::function< bool(CallBase &)> GetDefaultAdvice

std::unique_ptr< InlineAdvice > getAdviceImpl(CallBase &CB) override

Definition MLInlineAdvisor.cpp:384

std::unique_ptr< InlineAdvice > getMandatoryAdvice(CallBase &CB, bool Advice) override

Definition MLInlineAdvisor.cpp:532

unsigned getInitialFunctionLevel(const Function &F) const

Definition MLInlineAdvisor.cpp:223

A Module instance is used to store all the information related to an LLVM module.

A mock class satisfying the interface expected by ReleaseModeModelRunner for its TGen parameter.

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

PreservedAnalyses & abandon()

Mark an analysis as abandoned.

An analysis pass based on the new PM to deliver ProfileSummaryInfo.

Analysis pass providing the TargetTransformInfo.

static TensorSpec createSpec(const std::string &Name, const std::vector< int64_t > &Shape, int Port=0)

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

bool contains(const_arg_type_t< ValueT > V) const

Check if the set contains the given element.

const ParentTy * getParent() const

This class implements an extremely fast bulk output stream that can only output to a stream.

ValuesClass values(OptsTy... Options)

Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...

initializer< Ty > init(const Ty &Val)

Add a small namespace to avoid name clashes with the classes used in the streaming interface.

This is an optimization pass for GlobalISel generic memory operations.

constexpr FeatureIndex inlineCostFeatureToMlFeature(InlineCostFeatureIndex Feature)

LLVM_ABI const char *const DefaultDecisionName

Definition MLInlineAdvisor.cpp:132

bool isEmbeddedModelEvaluatorValid()

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

scc_iterator< T > scc_begin(const T &G)

Construct the begin iterator for a deduced graph type T.

LLVM_ABI std::unique_ptr< InlineAdvisor > getReleaseModeAdvisor(Module &M, ModuleAnalysisManager &MAM, std::function< bool(CallBase &)> GetDefaultAdvice)

Definition MLInlineAdvisor.cpp:76

InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy

Provide the FunctionAnalysisManager to Module proxy.

OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)

Wrapper function around std::transform to apply a function to a range and store the result elsewhere.

LLVM_ABI const TensorSpec DefaultDecisionSpec

Definition MLInlineAdvisor.cpp:133

LLVM_ABI const char *const DecisionName

Definition MLInlineAdvisor.cpp:129

@ Never

Never set the bit.

static const std::vector< TensorSpec > InputFeatures

LLVM_ABI std::optional< InlineCostFeatures > getInliningCostFeatures(CallBase &Call, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, function_ref< const TargetLibraryInfo &(Function &)> GetTLI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)

Get the expanded cost features.

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

LLVM_ABI const TensorSpec InlineDecisionSpec

Definition MLInlineAdvisor.cpp:130

LLVM_ABI const char *const RewardName

Definition MLInlineAdvisor.cpp:135

LLVM_ABI std::optional< int > getInliningCostEstimate(CallBase &Call, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, function_ref< const TargetLibraryInfo &(Function &)> GetTLI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)

Get the cost estimate ignoring thresholds.

AnalysisManager< Function > FunctionAnalysisManager

Convenience typedef for the Function analysis manager.

AnalysisManager< Module > ModuleAnalysisManager

Convenience typedef for the Module analysis manager.

ReleaseModeModelRunner - production mode implementation of the MLModelRunner.

Embedding is a datatype that wraps std::vector.