VPlanUtils.cpp Source File (original) (raw)

17using namespace llvm;

21 return all_of(Def->users(),

22 [Def](const VPUser *U) { return U->usesFirstLaneOnly(Def); });

23}

26 return all_of(Def->users(),

27 [Def](const VPUser *U) { return U->usesFirstPartOnly(Def); });

28}

31 return all_of(Def->users(),

32 [Def](const VPUser *U) { return U->usesScalars(Def); });

33}

47 return Expanded;

48}

52 return true;

54 auto IsWideCanonicalIV = [](VPValue *A) {

58 };

62 auto m_CanonicalScalarIVSteps =

68 (match(A, m_CanonicalScalarIVSteps) || IsWideCanonicalIV(A));

71 if (match(V, m_ICmp(m_CanonicalScalarIVSteps,

74 "Non-scalar VF using scalar IV steps for header mask?");

75 return true;

76 }

80}

84 if (V->isLiveIn()) {

85 if (Value *LiveIn = V->getLiveInIRValue())

86 return SE.getSCEV(LiveIn);

88 }

95 if (!L)

100 })

101 .Case(

106 const SCEV *Start =

109 })

116

119 Scale, IV->getType())));

120 })

128 Step);

129 })

131 if (R->getOpcode() != Instruction::GetElementPtr)

133

137

144 }

145

147 ->getSourceElementType();

148 return SE.getGEPExpr(Base, IndexExprs, SrcElementTy);

149 })

151}

152

153

154

157 return true;

158 switch (Opcode) {

159 case Instruction::GetElementPtr:

160 case Instruction::ICmp:

161 case Instruction::FCmp:

162 case Instruction::Select:

166 return true;

167 default:

168 return false;

169 }

170}

171

173

175 return true;

176

178 const VPRegionBlock *RegionOfR = Rep->getRegion();

179

180

181

183 return false;

184 return Rep->isSingleScalar() || (preservesUniformity(Rep->getOpcode()) &&

186 }

193 }

195 return VPI->isSingleScalar() || VPI->isVectorToScalar() ||

199 return !RR->isPartialReduction();

202 return true;

204 return Expr->isSingleScalar();

205

206

208}

209

211

212 if (V->isLiveIn())

213 return true;

214

216 if (R && V->isDefinedOutsideLoopRegions()) {

217 if (match(V->getDefiningRecipe(),

219 return false;

221 }

222

223 auto *CanonicalIV =

224 R->getParent()->getEnclosingLoopRegion()->getCanonicalIV();

225

226 if (V == CanonicalIV || V == CanonicalIV->getBackedgeValue())

227 return true;

228

231 .Case([](const auto *R) {

232

233

234

235 return R->isSingleScalar() &&

236 (!R->mayHaveSideEffects() ||

239 })

240 .Case([](const auto *R) {

243 })

244 .Case([](const auto *VPI) {

245 return (VPI->isScalarCast() &&

249 })

250 .Case([](const auto *R) {

251

253 })

255

256 return false;

257 });

258}

259

267

269 if (!R)

270 return 1;

272 return RR->getVFScaleFactor();

274 return RR->getVFScaleFactor();

276 return ER->getVFScaleFactor();

280 "getting scaling factor of reduction-start-vector not implemented yet");

281 return 1;

282}

283

284std::optional<VPValue *>

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

334 VPValue *UncountableCondition = nullptr;

335 if ( match (Region->getExitingBasicBlock()->getTerminator(),

338 return std::nullopt;

339

341 Worklist.push_back(UncountableCondition);

342 while (!Worklist.empty()) {

344

345

346 if (V->isDefinedOutsideLoopRegions())

347 continue;

348

349

350

351

352 if (V->getNumUsers() > 1)

353 return std::nullopt;

354

356

360 Recipes.push_back(V->getDefiningRecipe());

362

363

364 if (Load->isMasked())

365 return std::nullopt;

366

369 return std::nullopt;

370

374 } else

375 return std::nullopt;

376 }

377

378 return UncountableCondition;

379}

380

384 if (!VPBB)

385 return false;

386

387

388

389 if (auto *R = VPBB->getParent())

390 return !R->isReplicator() && !VPBB->hasPredecessors();

391

392

393

396}

397

406

407std::optional

410 if (!M)

411 return std::nullopt;

413

414 if (MDNode *NoAliasMD = M->getMetadata(LLVMContext::MD_noalias))

415 Loc.AATags.NoAlias = NoAliasMD;

416 if (MDNode *AliasScopeMD = M->getMetadata(LLVMContext::MD_alias_scope))

417 Loc.AATags.Scope = AliasScopeMD;

418 return Loc;

419}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

This file provides utility analysis objects describing memory locations.

This file implements the TypeSwitch template, which mimics a switch() statement whose cases are type ...

This file implements dominator tree analysis for a single level of a VPlan's H-CFG.

static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)

Returns the opcode of Values or ~0 if they do not all agree.

static bool preservesUniformity(unsigned Opcode)

Returns true if Opcode preserves uniformity, i.e., if all operands are uniform, the result will also ...

Definition VPlanUtils.cpp:155

static const uint32_t IV[8]

bool dominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const

dominates - Returns true iff A dominates B.

Represents a single loop in the control flow graph.

Representation for a specific memory location.

This class represents an analyzed expression in the program.

LLVM_ABI bool isOne() const

Return true if the expression is a constant one.

LLVM_ABI Type * getType() const

Return the LLVM type of this SCEV expression.

The main scalar evolution driver.

LLVM_ABI const SCEV * getSCEV(Value *V)

Return a SCEV expression for the full generality of the specified expression.

const SCEV * getOne(Type *Ty)

Return a SCEV for the constant 1 of a specific type.

LLVM_ABI const SCEV * getAddRecExpr(const SCEV *Start, const SCEV *Step, const Loop *L, SCEV::NoWrapFlags Flags)

Get an add recurrence expression for the specified loop.

LLVM_ABI const SCEV * getCouldNotCompute()

LLVM_ABI const SCEV * getGEPExpr(GEPOperator *GEP, ArrayRef< const SCEV * > IndexExprs)

Returns an expression for a GEP.

LLVM_ABI const SCEV * getMulExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)

Get a canonical multiply expression, or something simpler if possible.

LLVM_ABI const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)

Get a canonical add expression, or something simpler if possible.

LLVM_ABI const SCEV * getTruncateOrSignExtend(const SCEV *V, Type *Ty, unsigned Depth=0)

Return a SCEV corresponding to a conversion of the input value to the specified type.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

This class implements a switch-like dispatch statement for a value of 'T' using dyn_cast functionalit...

TypeSwitch< T, ResultT > & Case(CallableT &&caseFn)

Add a case on the given type.

The instances of the Type class are immutable: once they are created, they are never changed.

VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.

void appendRecipe(VPRecipeBase *Recipe)

Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.

A recipe for vectorizing a phi-node as a sequence of mask-based select instructions.

VPBlockBase is the building block of the Hierarchical Control-Flow Graph.

size_t getNumSuccessors() const

const VPBlocksTy & getPredecessors() const

const VPBlocksTy & getSuccessors() const

static bool isLatch(const VPBlockBase *VPB, const VPDominatorTree &VPDT)

Returns true if VPB is a loop latch, using isHeader().

Definition VPlanUtils.cpp:398

static bool isHeader(const VPBlockBase *VPB, const VPDominatorTree &VPDT)

Returns true if VPB is a loop header, based on regions or VPDT in their absence.

Definition VPlanUtils.cpp:381

Canonical scalar induction phi of the vector loop.

A recipe for converting the input value IV value to the corresponding value of an IV with different s...

Template specialization of the standard LLVM dominator tree utility for VPBlockBases.

Recipe to expand a SCEV expression.

@ ReductionStartVector

Start vector for reductions with 3 operands: the original start value, the identity value for the red...

VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.

VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...

bool isReplicator() const

An indicator whether this region is to generate multiple replicated instances of output IR correspond...

VPCanonicalIVPHIRecipe * getCanonicalIV()

Returns the canonical induction recipe of the region.

VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...

A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...

This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...

This is the base class of the VPlan Def/Use graph, used for modeling the data flow into,...

VPRecipeBase * getDefiningRecipe()

Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...

bool isLiveIn() const

Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.

A recipe to compute a pointer to the last element of each part of a widened memory access for widened...

A recipe to compute the pointers for widened memory accesses of IndexTy.

A recipe for handling GEP instructions.

A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...

VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...

VPBasicBlock * getEntry()

VPValue & getVF()

Returns the VF of the vector loop region.

VPValue * getTripCount() const

The trip count of the original loop.

VPValue * getBackedgeTakenCount() const

LLVM_ABI_FOR_TEST VPRegionBlock * getVectorLoopRegion()

Returns the VPRegionBlock of the vector loop.

VPValue * getOrAddLiveIn(Value *V)

Gets the live-in VPValue for V or adds a new live-in (if none exists yet) for V.

bool hasScalarVFOnly() const

LLVM Value Representation.

OneUse_match< SubPat > m_OneUse(const SubPat &SP)

bool match(Val *V, const Pattern &P)

specificval_ty m_Specific(const Value *V)

Match if we have a specific specified value.

cst_pred_ty< is_one > m_One()

Match an integer 1 or a vector with all elements equal to 1.

CmpClass_match< LHS, RHS, ICmpInst > m_ICmp(CmpPredicate &Pred, const LHS &L, const RHS &R)

VPInstruction_match< VPInstruction::AnyOf > m_AnyOf()

AllRecipe_commutative_match< Instruction::Or, Op0_t, Op1_t > m_c_BinaryOr(const Op0_t &Op0, const Op1_t &Op1)

VPScalarIVSteps_match< Op0_t, Op1_t, Op2_t > m_ScalarIVSteps(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2)

GEPLikeRecipe_match< Op0_t, Op1_t > m_GetElementPtr(const Op0_t &Op0, const Op1_t &Op1)

VPInstruction_match< VPInstruction::ActiveLaneMask, Op0_t, Op1_t, Op2_t > m_ActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2)

live_in_vpvalue m_LiveIn()

class_match< VPValue > m_VPValue()

Match an arbitrary VPValue and ignore it.

bind_ty< VPInstruction > m_VPInstruction(VPInstruction *&V)

Match a VPInstruction, capturing if we match.

VPInstruction_match< VPInstruction::BranchOnCond > m_BranchOnCond()

bool isSingleScalar(const VPValue *VPV)

Returns true if VPV is a single scalar, either because it produces the same value for all lanes or on...

Definition VPlanUtils.cpp:172

bool isUniformAcrossVFsAndUFs(VPValue *V)

Checks if V is uniform across all VF lanes and UF parts.

Definition VPlanUtils.cpp:210

VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr)

Get or create a VPValue that corresponds to the expansion of Expr.

Definition VPlanUtils.cpp:35

VPBasicBlock * getFirstLoopHeader(VPlan &Plan, VPDominatorTree &VPDT)

Returns the header block of the first, top-level loop, or null if none exist.

Definition VPlanUtils.cpp:260

bool onlyFirstPartUsed(const VPValue *Def)

Returns true if only the first part of Def is used.

Definition VPlanUtils.cpp:25

std::optional< MemoryLocation > getMemoryLocation(const VPRecipeBase &R)

Return a MemoryLocation for R with noalias metadata populated from R, if the recipe is supported and ...

Definition VPlanUtils.cpp:408

bool onlyFirstLaneUsed(const VPValue *Def)

Returns true if only the first lane of Def is used.

Definition VPlanUtils.cpp:20

bool onlyScalarValuesUsed(const VPValue *Def)

Returns true if only scalar values of Def are used by all users.

Definition VPlanUtils.cpp:30

unsigned getVFScaleFactor(VPRecipeBase *R)

Get the VF scaling factor applied to the recipe's output, if the recipe has one.

Definition VPlanUtils.cpp:268

bool isHeaderMask(const VPValue *V, const VPlan &Plan)

Return true if V is a header mask in Plan.

Definition VPlanUtils.cpp:50

LLVM_ABI_FOR_TEST std::optional< VPValue * > getRecipesForUncountableExit(VPlan &Plan, SmallVectorImpl< VPRecipeBase * > &Recipes, SmallVectorImpl< VPRecipeBase * > &GEPs)

Returns the VPValue representing the uncountable exit comparison used by AnyOf if the recipes it depe...

Definition VPlanUtils.cpp:285

const SCEV * getSCEVExprForVPValue(const VPValue *V, ScalarEvolution &SE, const Loop *L=nullptr)

Return the SCEV expression for V.

Definition VPlanUtils.cpp:82

This is an optimization pass for GlobalISel generic memory operations.

auto drop_begin(T &&RangeOrContainer, size_t N=1)

Return a range covering RangeOrContainer with the first N elements excluded.

bool all_of(R &&range, UnaryPredicate P)

Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)

Returns an iterator range to traverse the graph starting at G in depth-first order.

bool any_of(R &&range, UnaryPredicate P)

Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

ArrayRef(const T &OneElt) -> ArrayRef< T >

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

auto find_if(R &&Range, UnaryPredicate P)

Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.

@ Default

The result values are uniform if and only if all operands are uniform.

constexpr detail::IsaCheckPredicate< Types... > IsaPred

Function object wrapper for the llvm::isa type check.

A recipe for widening select instructions.