BasicBlockSections.cpp Source File (original) (raw)

86#include

88using namespace llvm;

95 "bbsections-cold-text-prefix",

96 cl::desc("The text prefix to use for cold basic block clusters"),

100 "bbsections-detect-source-drift",

101 cl::desc("This checks if there is a fdo instr. profile hash "

102 "mismatch for this function"),

104

105namespace {

106

108public:

109 static char ID;

110

112

115 }

116

117 StringRef getPassName() const override {

118 return "Basic Block Sections Analysis";

119 }

120

121 void getAnalysisUsage(AnalysisUsage &AU) const override;

122

123

124

125 bool runOnMachineFunction(MachineFunction &MF) override;

126

127private:

128 bool handleBBSections(MachineFunction &MF);

129 bool handleBBAddrMap(MachineFunction &MF);

130};

131

132}

133

134char BasicBlockSections::ID = 0;

136 BasicBlockSections, "bbsections-prepare",

137 "Prepares for basic block sections, by splitting functions "

138 "into clusters of basic blocks.",

139 false, false)

142 "Prepares for basic block sections, by splitting functions "

143 "into clusters of basic blocks.",

145

146

147

148static void

153 for (auto &MBB : MF) {

154 auto NextMBBI = std::next(MBB.getIterator());

155 auto *FTMBB = PreLayoutFallThroughs[MBB.getNumber()];

156

157

158

159

160

161

162 if (FTMBB && (MBB.isEndSection() || &*NextMBBI != FTMBB))

163 TII->insertUnconditionalBranch(MBB, FTMBB, MBB.findBranchDebugLoc());

164

165

166

167 if (MBB.isEndSection())

168 continue;

169

170

171

172 Cond.clear();

175 continue;

176 MBB.updateTerminator(FTMBB);

177 }

178}

179

180

181

182

188 if (!OptWeightInfo)

189 return BBClusterInfos;

190 auto BlockWeights = OptWeightInfo->BlockWeights;

191 auto EdgeWeights = OptWeightInfo->EdgeWeights;

192

194 if (MF.size() <= 2) {

195 for (auto &MBB : MF) {

196 if (MBB.isEntryBlock() || BlockWeights[&MBB] > 0) {

198 }

199 }

200 } else {

203 std::vector<const MachineBasicBlock *> OrigOrder;

204 OrigOrder.reserve(MF.size());

206

207

209

210

211 for (auto &MBB : MF) {

212 auto NonDbgInsts =

214 int NumInsts = std::distance(NonDbgInsts.begin(), NonDbgInsts.end());

215 BlockSizes[MBB.getNumber()] = 4 * NumInsts;

216 BlockCounts[MBB.getNumber()] = BlockWeights[&MBB];

217 OrigOrder.push_back(&MBB);

218 }

219

220

221 for (auto &MBB : MF) {

222 for (auto *Succ : MBB.successors()) {

223 auto EdgeWeight = EdgeWeights[std::make_pair(&MBB, Succ)];

225 static_cast<uint64_t>(Succ->getNumber()),

226 EdgeWeight});

227 }

228 }

229

230

231 auto Result = computeExtTspLayout(BlockSizes, BlockCounts, JumpCounts);

233 auto Block = OrigOrder[R];

234 if (Block->isEntryBlock() || BlockWeights[Block] > 0)

236 }

237 }

238

239

240 if (!HotMBBs.empty()) {

241 unsigned CurrentPosition = 0;

242 for (auto &MBB : HotMBBs) {

243 if (MBB->getBBID()) {

244 BBClusterInfos.push_back({*(MBB->getBBID()), 0, CurrentPosition++});

245 }

246 }

247 }

248 return BBClusterInfos;

249}

250

251

252

253

254

255

256

257

258

259

260static void

264

265

266

267 std::optional EHPadsSectionID;

268

269 for (auto &MBB : MF) {

270

271

272

274

275

276

277

278 MBB.setSectionID(MBB.getNumber());

279 } else {

280 auto I = FuncClusterInfo.find(*MBB.getBBID());

281 if (I != FuncClusterInfo.end()) {

282 MBB.setSectionID(I->second.ClusterID);

283 } else {

285 *MBB.getParent()->getSubtarget().getInstrInfo();

286

287 if (TII.isMBBSafeToSplitToCold(MBB)) {

288

289

291 }

292 }

293 }

294

295 if (MBB.isEHPad() && EHPadsSectionID != MBB.getSectionID() &&

297

298

299

301 : MBB.getSectionID();

302 }

303 }

304

305

306

308 for (auto &MBB : MF)

309 if (MBB.isEHPad())

310 MBB.setSectionID(*EHPadsSectionID);

311}

312

317 for (auto &MBB : MF)

318 PreLayoutFallThroughs[MBB.getNumber()] =

319 MBB.getFallThrough(false);

320

321 MF.sort(MBBCmp);

323 "Entry block should not be displaced by basic block sections");

324

325

327

328

329

330

332}

333

334

335

336

337

339 std::optional CurrentSection;

341 if (MBB.empty() || MBB.getSectionID() == CurrentSection)

342 return false;

343 CurrentSection = MBB.getSectionID();

344 return true;

345 };

346

347 for (auto &MBB : MF) {

348 if (IsFirstNonEmptyBBInSection(MBB) && MBB.isEHPad()) {

350 while (->isEHLabel())

351 ++MI;

353 }

354 }

355}

356

359 return false;

360

361 const char MetadataName[] = "instr_prof_hash_mismatch";

363 if (Existing) {

365 for (const auto &N : Tuple->operands())

366 if (N.equalsStr(MetadataName))

367 return true;

368 }

369

370 return false;

371}

372

373

374

375bool BasicBlockSections::handleBBSections(MachineFunction &MF) {

378 return false;

379

380

381

382

383

384

385

388 return false;

389

393 if (auto *BMI = getAnalysisIfAvailable()) {

395 } else {

396 ClusterInfo = getAnalysis()

397 .getClusterInfoForFunction(MF.getName());

398 }

400 return false;

401 for (auto &BBClusterInfo : ClusterInfo) {

402 FuncClusterInfo.try_emplace(BBClusterInfo.BBID, BBClusterInfo);

403 }

404 }

405

406

407

409

412

413 const MachineBasicBlock &EntryBB = MF.front();

414 auto EntryBBSectionID = EntryBB.getSectionID();

415

416

417

418

419

420

421

422 auto MBBSectionOrder = [EntryBBSectionID](const MBBSectionID &LHS,

423 const MBBSectionID &RHS) {

424

425

426 if (LHS == EntryBBSectionID || RHS == EntryBBSectionID)

427 return LHS == EntryBBSectionID;

428 return LHS.Type == RHS.Type ? LHS.Number < RHS.Number : LHS.Type < RHS.Type;

429 };

430

431

432

433

434

435

436

437 auto Comparator = [&](const MachineBasicBlock &X,

438 const MachineBasicBlock &Y) {

439 auto XSectionID = X.getSectionID();

440 auto YSectionID = Y.getSectionID();

441 if (XSectionID != YSectionID)

442 return MBBSectionOrder(XSectionID, YSectionID);

443

444 if (&X == &EntryBB || &Y == &EntryBB)

445 return &X == &EntryBB;

446

447

448 if (XSectionID.Type == MBBSectionID::SectionType::Default)

449 return FuncClusterInfo.lookup(*X.getBBID()).PositionInCluster <

450 FuncClusterInfo.lookup(*Y.getBBID()).PositionInCluster;

451 return X.getNumber() < Y.getNumber();

452 };

453

456 return true;

457}

458

459

460

461

462

463bool BasicBlockSections::handleBBAddrMap(MachineFunction &MF) {

465 return false;

467 return true;

468}

469

470bool BasicBlockSections::runOnMachineFunction(MachineFunction &MF) {

471

472 auto R1 = handleBBSections(MF);

473

474 auto R2 = handleBBAddrMap(MF);

475

476

477 if (auto *WP = getAnalysisIfAvailable())

478 WP->getDomTree().updateBlockNumbers();

479 if (auto *WP = getAnalysisIfAvailable())

480 WP->getPostDomTree().updateBlockNumbers();

481

482 return R1 || R2;

483}

484

485void BasicBlockSections::getAnalysisUsage(AnalysisUsage &AU) const {

487 AU.addRequired();

492}

493

495 return new BasicBlockSections();

496}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

const TargetInstrInfo & TII

static void assignSections(MachineFunction &MF, const DenseMap< UniqueBBID, BBClusterInfo > &FuncClusterInfo)

Definition BasicBlockSections.cpp:261

static cl::opt< bool > BBSectionsDetectSourceDrift("bbsections-detect-source-drift", cl::desc("This checks if there is a fdo instr. profile hash " "mismatch for this function"), cl::init(true), cl::Hidden)

bbsections Prepares for basic block by splitting functions into clusters of basic static false void updateBranches(MachineFunction &MF, const SmallVector< MachineBasicBlock * > &PreLayoutFallThroughs)

Definition BasicBlockSections.cpp:149

static SmallVector< BBClusterInfo > createBBClusterInfoForFunction(MachineFunction &MF, const BasicBlockMatchingAndInference &BMI)

Definition BasicBlockSections.cpp:184

Declares methods and data structures for code layout algorithms.

#define INITIALIZE_PASS_DEPENDENCY(depName)

#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)

#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)

const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB

const SmallVectorImpl< MachineOperand > & Cond

This file defines the SmallVector class.

static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")

static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")

AnalysisUsage & addUsedIfAvailable()

Add the specified Pass class to the set of analyses used by this pass.

AnalysisUsage & addRequired()

void setPreservesAll()

Set by analyses that do not transform their input at all.

std::optional< WeightInfo > getWeightInfo(StringRef FuncName) const

ValueT lookup(const_arg_type_t< KeyT > Val) const

lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...

iterator find(const_arg_type_t< KeyT > Val)

std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)

MDNode * getMetadata(unsigned KindID) const

Get the current metadata attachments for the given kind, if any.

MBBSectionID getSectionID() const

Returns the section ID of this basic block.

MachineInstrBundleIterator< MachineInstr > iterator

MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...

void getAnalysisUsage(AnalysisUsage &AU) const override

getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.

void setBBSectionsType(BasicBlockSection V)

const TargetSubtargetInfo & getSubtarget() const

getSubtarget - Return the subtarget for which this machine code is being compiled.

StringRef getName() const

getName - Return the name of the corresponding LLVM function.

bool hasBBSections() const

Returns true if this function has basic block sections enabled.

Function & getFunction()

Return the LLVM function that this machine code represents.

unsigned getNumBlockIDs() const

getNumBlockIDs - Return the number of MBB ID's allocated.

void RenumberBlocks(MachineBasicBlock *MBBFrom=nullptr)

RenumberBlocks - This discards all of the MachineBasicBlock numbers and recomputes them.

const MachineBasicBlock & front() const

void assignBeginEndSections()

Assign IsBeginSection IsEndSection fields for basic blocks in this function.

const TargetMachine & getTarget() const

getTarget - Return the target machine this machine code is compiled with

static LLVM_ABI PassRegistry * getPassRegistry()

getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

TargetInstrInfo - Interface to description of machine instruction set.

virtual void insertNoop(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI) const

Insert a noop into the instruction stream at the specified point.

llvm::BasicBlockSection getBBSectionsType() const

If basic blocks should be emitted into their own section, corresponding to -fbasic-block-sections.

virtual const TargetInstrInfo * getInstrInfo() const

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

initializer< Ty > init(const Ty &Val)

This is an optimization pass for GlobalISel generic memory operations.

LLVM_ABI MachineFunctionPass * createBasicBlockSectionsPass()

createBasicBlockSections Pass - This pass assigns sections to machine basic blocks and is enabled wit...

Definition BasicBlockSections.cpp:494

LLVM_ABI void initializeBasicBlockSectionsPass(PassRegistry &)

auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)

Construct a range iterator which begins at It and moves forwards until End is reached,...

bool hasInstrProfHashMismatch(MachineFunction &MF)

This checks if the source of this function has drifted since this binary was profiled previously.

Definition BasicBlockSections.cpp:357

SmallPtrSet< SUnit *, 8 > ClusterInfo

Keep record of which SUnit are in the same cluster group.

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

void avoidZeroOffsetLandingPad(MachineFunction &MF)

Definition BasicBlockSections.cpp:338

cl::opt< std::string > BBSectionsColdTextPrefix

function_ref< bool(const MachineBasicBlock &, const MachineBasicBlock &)> MachineBasicBlockComparator

void sortBasicBlocksAndUpdateBranches(MachineFunction &MF, MachineBasicBlockComparator MBBCmp)

Definition BasicBlockSections.cpp:313

LLVM_ABI static const MBBSectionID ExceptionSectionID

LLVM_ABI static const MBBSectionID ColdSectionID