VPlanTransforms.h Source File (original) (raw)

13#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H

14#define LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H

22namespace llvm {

41 template <typename... ArgsTy>

43 typename std::remove_reference::type &...Args) {

44 bool Res = Transform(Plan, Args...);

47 return Res;

48 }

51 template <typename... ArgsTy>

53 typename std::remove_reference::type &...Args) {

54 Fn(Plan, Args...);

57 }

100

104

105

107 bool HasUncountableExit);

108

109

110

112 bool RequiresScalarEpilogueCheck,

113 bool TailFolded);

114

115

118 ElementCount MinProfitableTripCount, bool RequiresScalarEpilogue,

119 bool TailFolded, bool CheckNeededWithTailFolding, Loop *OrigLoop,

121

122

123

126 bool RequiresScalarEpilogue, ElementCount EpilogueVF, unsigned EpilogueUF,

127 unsigned MainLoopStep, unsigned EpilogueLoopStep, ScalarEvolution &SE);

128

129

130

132

133

134

135

137 bool AddBranchWeights);

138

139

140

141

145 GetIntOrFpInductionDescriptor,

147

148

149

150

152

153

154

155

156

157

158

159

160

162

163

164

165

166

168

169

171

172

174

175

176

177

178

180

181

182

184 unsigned BestUF,

186

187

188

189

191

192

193

194

195

197

198

199

200

201

202

203

204

205

207 bool UseActiveLaneMaskForControlFlow,

209

210

211

212 static void

215

216

217

218 static void

221

222

223

224

225

226

227

228

229

230

231

232

233

236 const std::function<bool(BasicBlock *)> &BlockNeedsPredication);

237

238

239

240

241

242

243 static void

245 const std::optional &MaxEVLSafeElements);

246

247

248

249

253 &InterleaveGroups,

254 VPRecipeBuilder &RecipeBuilder, const bool &ScalarEpilogueAllowed);

255

256

258

259

260

261

262

263

264

269

270

272

273

274

275

276

277

278

279

280

281

282

283

285

286

288

289

290

291

292

295

296

298

299

300

302

303

304 static void cse(VPlan &Plan);

305

306

307

308

309 static void

313

314

316

317

318

319

321

322

323

324

326 const Loop *L);

327

328

329

330

331

333 const Loop *L);

334

335

336

337 static void

339 unsigned BestUF,

341

342

345 bool TailByMasking,

346 bool RequiresScalarEpilogue);

347

348

349

352

353

354

355

357

358

361

362

363

364

365

368

369

370

371

372

373

374

377

378

379

380

381

382

383

386

387

388

389 static void

391 std::optional VScaleForTuning);

392

393

394

395

396 static void

399

400

401

402

403

405};

406

407}

408

409#endif

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

#define LLVM_ABI_FOR_TEST

static constexpr uint32_t MinItersBypassWeights[]

ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))

const SmallVectorImpl< MachineOperand > & Cond

This file declares the class VPlanVerifier, which contains utility functions to check the consistency...

This file contains the declarations of the Vectorization Plan base classes:

LLVM Basic Block Representation.

A struct for saving information about induction variables.

The group of interleaved loads/stores sharing the same stride and close to each other.

This class emits a version of the loop where run-time checks ensure that may-alias pointers can't ove...

Represents a single loop in the control flow graph.

This class implements a map that also provides access to all stored values in a deterministic order.

An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...

The main scalar evolution driver.

A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...

Provides information about what library functions are available for the current target.

The instances of the Type class are immutable: once they are created, they are never changed.

VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.

VPlan-based builder utility analogous to IRBuilder.

Helper class to create VPRecipies from IR instructions.

VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...

LLVM Value Representation.

An efficient, type-erasing, non-owning reference to a callable.

This is an optimization pass for GlobalISel generic memory operations.

LLVM_ABI_FOR_TEST cl::opt< bool > VerifyEachVPlan

LLVM_ABI_FOR_TEST bool verifyVPlanIsValid(const VPlan &Plan, bool VerifyLate=false)

Verify invariants for general VPlans.

LLVM_ABI_FOR_TEST cl::opt< bool > EnableWideActiveLaneMask

@ DataAndControlFlowWithoutRuntimeCheck

Use predicate to control both data and control flow, but modify the trip count so that a runtime over...

A range of powers-of-2 vectorization factors with fixed start and adjustable end.

Struct to hold various analysis needed for cost computations.

Definition VPlanTransforms.h:38

static void hoistPredicatedLoads(VPlan &Plan, ScalarEvolution &SE, const Loop *L)

Hoist predicated loads from the same address to the loop entry block, if they are guaranteed to execu...

static void sinkPredicatedStores(VPlan &Plan, ScalarEvolution &SE, const Loop *L)

Sink predicated stores to the same address with complementary predicates (P and NOT P) to an uncondit...

static void materializeBroadcasts(VPlan &Plan)

Add explicit broadcasts for live-ins and VPValues defined in Plan's entry block if they are used as v...

static void materializePacksAndUnpacks(VPlan &Plan)

Add explicit Build[Struct]Vector recipes to Pack multiple scalar values into vectors and Unpack recip...

static bool handleMultiUseReductions(VPlan &Plan)

Try to legalize reductions with multiple in-loop uses.

static LLVM_ABI_FOR_TEST std::unique_ptr< VPlan > buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy, DebugLoc IVDL, PredicatedScalarEvolution &PSE, LoopVersioning *LVer=nullptr)

Create a base VPlan0, serving as the common starting point for all later candidates.

static void materializeBackedgeTakenCount(VPlan &Plan, VPBasicBlock *VectorPH)

Materialize the backedge-taken count to be computed explicitly using VPInstructions.

static void optimizeInductionExitUsers(VPlan &Plan, DenseMap< VPValue *, VPValue * > &EndValues, ScalarEvolution &SE)

If there's a single exit block, optimize its phi recipes that use exiting IV values by feeding them p...

static void hoistInvariantLoads(VPlan &Plan)

Hoist single-scalar loads with invariant addresses out of the vector loop to the preheader,...

static LLVM_ABI_FOR_TEST void handleEarlyExits(VPlan &Plan, bool HasUncountableExit)

Update Plan to account for all early exits.

static void canonicalizeEVLLoops(VPlan &Plan)

Transform EVL loops to use variable-length stepping after region dissolution.

static void dropPoisonGeneratingRecipes(VPlan &Plan, const std::function< bool(BasicBlock *)> &BlockNeedsPredication)

Drop poison flags from recipes that may generate a poison value that is used after vectorization,...

static void createAndOptimizeReplicateRegions(VPlan &Plan)

Wrap predicated VPReplicateRecipes with a mask operand in an if-then region block and remove the mask...

static void createInterleaveGroups(VPlan &Plan, const SmallPtrSetImpl< const InterleaveGroup< Instruction > * > &InterleaveGroups, VPRecipeBuilder &RecipeBuilder, const bool &ScalarEpilogueAllowed)

static bool runPass(bool(*Transform)(VPlan &, ArgsTy...), VPlan &Plan, typename std::remove_reference< ArgsTy >::type &...Args)

Helper to run a VPlan transform Transform on VPlan, forwarding extra arguments to the transform.

Definition VPlanTransforms.h:42

static void addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF, std::optional< unsigned > VScaleForTuning)

Add branch weight metadata, if the Plan's middle block is terminated by a BranchOnCond recipe.

static void narrowInterleaveGroups(VPlan &Plan, ElementCount VF, TypeSize VectorRegWidth)

Try to convert a plan with interleave groups with VF elements to a plan with the interleave groups re...

static void unrollByUF(VPlan &Plan, unsigned UF)

Explicitly unroll Plan by UF.

static DenseMap< const SCEV *, Value * > expandSCEVs(VPlan &Plan, ScalarEvolution &SE)

Expand VPExpandSCEVRecipes in Plan's entry block.

static void convertToConcreteRecipes(VPlan &Plan)

Lower abstract recipes to concrete ones, that can be codegen'd.

static void runPass(void(*Fn)(VPlan &, ArgsTy...), VPlan &Plan, typename std::remove_reference< ArgsTy >::type &...Args)

Helper to run a VPlan transform Transform on VPlan, forwarding extra arguments to the transform.

Definition VPlanTransforms.h:52

static void addMinimumIterationCheck(VPlan &Plan, ElementCount VF, unsigned UF, ElementCount MinProfitableTripCount, bool RequiresScalarEpilogue, bool TailFolded, bool CheckNeededWithTailFolding, Loop *OrigLoop, const uint32_t *MinItersBypassWeights, DebugLoc DL, ScalarEvolution &SE)

static void convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx, VFRange &Range)

This function converts initial recipes to the abstract recipes and clamps Range based on cost model f...

static void materializeConstantVectorTripCount(VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)

static LLVM_ABI_FOR_TEST bool tryToConvertVPInstructionsToVPRecipes(VPlan &Plan, function_ref< const InductionDescriptor *(PHINode *)> GetIntOrFpInductionDescriptor, const TargetLibraryInfo &TLI)

Replaces the VPInstructions in Plan with corresponding widen recipes.

static void addExitUsersForFirstOrderRecurrences(VPlan &Plan, VFRange &Range)

Handle users in the exit block for first order reductions in the original exit block.

static DenseMap< VPBasicBlock *, VPValue * > introduceMasksAndLinearize(VPlan &Plan, bool FoldTail)

Predicate and linearize the control-flow in the only loop region of Plan.

static void addExplicitVectorLength(VPlan &Plan, const std::optional< unsigned > &MaxEVLSafeElements)

Add a VPEVLBasedIVPHIRecipe and related recipes to Plan and replaces all uses except the canonical IV...

static void replaceSymbolicStrides(VPlan &Plan, PredicatedScalarEvolution &PSE, const DenseMap< Value *, const SCEV * > &StridesMap)

Replace symbolic strides from StridesMap in Plan with constants when possible.

static bool handleMaxMinNumReductions(VPlan &Plan)

Check if Plan contains any FMaxNum or FMinNum reductions.

static void removeBranchOnConst(VPlan &Plan)

Remove BranchOnCond recipes with true or false conditions together with removing dead edges to their ...

static LLVM_ABI_FOR_TEST void createLoopRegions(VPlan &Plan)

Replace loops in Plan's flat CFG with VPRegionBlocks, turning Plan's flat CFG into a hierarchical CFG...

static void removeDeadRecipes(VPlan &Plan)

Remove dead recipes from Plan.

static void attachCheckBlock(VPlan &Plan, Value *Cond, BasicBlock *CheckBlock, bool AddBranchWeights)

Wrap runtime check block CheckBlock in a VPIRBB and Cond in a VPValue and connect the block to Plan,...

static void materializeVectorTripCount(VPlan &Plan, VPBasicBlock *VectorPHVPBB, bool TailByMasking, bool RequiresScalarEpilogue)

Materialize vector trip count computations to a set of VPInstructions.

static void simplifyRecipes(VPlan &Plan)

Perform instcombine-like simplifications on recipes in Plan.

static void handleUncountableEarlyExit(VPBasicBlock *EarlyExitingVPBB, VPBasicBlock *EarlyExitVPBB, VPlan &Plan, VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB)

Update Plan to account for the uncountable early exit from EarlyExitingVPBB to EarlyExitVPBB by.

static void replicateByVF(VPlan &Plan, ElementCount VF)

Replace each replicating VPReplicateRecipe and VPInstruction outside of any replicate region in Plan ...

static void clearReductionWrapFlags(VPlan &Plan)

Clear NSW/NUW flags from reduction instructions if necessary.

static void cse(VPlan &Plan)

Perform common-subexpression-elimination on Plan.

static void addActiveLaneMask(VPlan &Plan, bool UseActiveLaneMaskForControlFlow, bool DataAndControlFlowWithoutRuntimeCheck)

Replace (ICMP_ULE, wide canonical IV, backedge-taken-count) checks with an (active-lane-mask recipe,...

static LLVM_ABI_FOR_TEST void optimize(VPlan &Plan)

Apply VPlan-to-VPlan optimizations to Plan, including induction recipe optimizations,...

static void dissolveLoopRegions(VPlan &Plan)

Replace loop regions with explicit CFG.

static void truncateToMinimalBitwidths(VPlan &Plan, const MapVector< Instruction *, uint64_t > &MinBWs)

Insert truncates and extends for any truncated recipe.

static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder)

Try to have all users of fixed-order recurrences appear after the recipe defining their previous valu...

static void optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)

Optimize Plan based on BestVF and BestUF.

static void materializeVFAndVFxUF(VPlan &Plan, VPBasicBlock *VectorPH, ElementCount VF)

Materialize VF and VFxUF to be computed explicitly using VPInstructions.

static void addMinimumVectorEpilogueIterationCheck(VPlan &Plan, Value *TripCount, Value *VectorTripCount, bool RequiresScalarEpilogue, ElementCount EpilogueVF, unsigned EpilogueUF, unsigned MainLoopStep, unsigned EpilogueLoopStep, ScalarEvolution &SE)

Add a check to Plan to see if the epilogue vector loop should be executed.

static void updateScalarResumePhis(VPlan &Plan, DenseMap< VPValue *, VPValue * > &IVEndValues)

Update the resume phis in the scalar preheader after creating wide recipes for first-order recurrence...

static LLVM_ABI_FOR_TEST void addMiddleCheck(VPlan &Plan, bool RequiresScalarEpilogueCheck, bool TailFolded)

If a check is needed to guard executing the scalar epilogue loop, it will be added to the middle bloc...