VPlanHelpers.h Source File (original) (raw)

15#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLANHELPERS_H

16#define LLVM_TRANSFORMS_VECTORIZE_VPLANHELPERS_H

29namespace llvm {

51 int64_t Step);

64 return End.getKnownMinValue() <= Start.getKnownMinValue();

65 }

70 "Both Start and End should have the same scalable flag");

72 "Expected Start to be a power of 2");

74 "Expected End to be a power of 2");

75 }

80 ElementCount> {

83 public:

91 VF *= 2;

92 return *this;

93 }

94 };

101};

102

103

104

105

106

107

108

109

111public:

112

114

115

117

118

119

120

122 };

123

124private:

125

126 unsigned Lane;

127

128

130

131public:

132 VPLane(unsigned Lane) : Lane(Lane) {}

133 VPLane(unsigned Lane, Kind LaneKind) : Lane(Lane), LaneKind(LaneKind) {}

134

136

139 "trying to extract with invalid offset");

141 Kind LaneKind;

143

144

146 else

148 return VPLane(LaneOffset, LaneKind);

149 }

150

154

155

156

159 "can only get known lane from the beginning");

160 return Lane;

161 }

162

163

164

166

167

169

170

172

173

175 switch (LaneKind) {

178 "ScalableLast can only be used with scalable VFs");

180 default:

182 "Cannot extract lane larger than VF");

183 return Lane;

184 }

185 }

186};

187

188

189

194 Type *CanonicalIVTy);

195

197

198

200

201

202

203

204 std::optional Lane;

205

213

214

215

217

218

220

222 return Data.VPV2Vector.contains(Def);

223 }

224

226 auto I = Data.VPV2Scalars.find(Def);

227 if (I == Data.VPV2Scalars.end())

228 return false;

229 unsigned CacheIdx = Lane.mapToCacheIndex(VF);

230 return CacheIdx < I->second.size() && I->second[CacheIdx];

231 }

232

233

234

236 if (IsScalar) {

238 return;

239 }

241 "scalar values must be stored as (0, 0)");

242 Data.VPV2Vector[Def] = V;

243 }

244

245

247 assert(Data.VPV2Vector.contains(Def) && "need to overwrite existing value");

248 Data.VPV2Vector[Def] = V;

249 }

250

251

253 auto &Scalars = Data.VPV2Scalars[Def];

254 unsigned CacheIdx = Lane.mapToCacheIndex(VF);

255 if (Scalars.size() <= CacheIdx)

256 Scalars.resize(CacheIdx + 1);

257 assert(!Scalars[CacheIdx] && "should overwrite existing value");

258 Scalars[CacheIdx] = V;

259 }

260

261

263 auto Iter = Data.VPV2Scalars.find(Def);

264 assert(Iter != Data.VPV2Scalars.end() &&

265 "need to overwrite existing value");

266 unsigned CacheIdx = Lane.mapToCacheIndex(VF);

267 assert(CacheIdx < Iter->second.size() &&

268 "need to overwrite existing value");

269 Iter->second[CacheIdx] = V;

270 }

271

272

274

275

276

279

280

281

283

285

286

287

289

290

291

293

294

295

297

298

300

304

305

307

308

309

311

312

314

315

317

318

320

321

323

324

326};

327

328

339

346

347

348

350

351

352

354

355

356

358

359

361

362

363

364

366

367

368

369

370

374 bool AlwaysIncludeReplicatingR = false);

375};

376

377

378

379

380

381

382

384

385

387

389

390

391 unsigned NextSlot = 0;

392

393

394

395 std::unique_ptr MST;

396

397

399

400

401 const Module *M = nullptr;

402

403 void assignName(const VPValue *V);

406 std::string getName(const Value *V);

407

408public:

410 if (Plan) {

411 assignNames(*Plan);

413 M = ScalarHeader->getIRBasicBlock()->getModule();

414 }

415 }

416

417

418

419

421

422

424 if (MDNames.empty() && M)

425 M->getContext().getMDKindNames(MDNames);

426 return MDNames;

427 }

428

429

431};

432

433#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

434

435

438 const VPlan &Plan;

439 unsigned Depth = 0;

440 unsigned TabWidth = 2;

441 std::string Indent;

442 unsigned BID = 0;

444

446

447

448 void bumpIndent(int b) { Indent = std::string((Depth += b) * TabWidth, ' '); }

449

450

452

453

454

456

457

458

460

461

463

465 return BlockID.count(Block) ? BlockID[Block] : BlockID[Block] = BID++;

466 }

467

469

471

472

474 const Twine &Label);

475

476public:

478 : OS(O), Plan(P), SlotTracker(&P) {}

479

481};

482#endif

483

484

485

488}

489

490#endif

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

#define LLVM_DUMP_METHOD

Mark debug helper function definitions like dump() that should not be stripped from debug builds.

#define LLVM_ABI_FOR_TEST

This file defines the DenseMap class.

This file defines an InstructionCost class that is used when calculating the cost of an instruction,...

This file defines the SmallPtrSet class.

This file defines the SmallVector class.

This pass exposes codegen information to IR-level passes.

This file implements dominator tree analysis for a single level of a VPlan's H-CFG.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

A cache of @llvm.assume calls within a function.

LLVM Basic Block Representation.

Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.

Common base class shared among various IRBuilders.

InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...

This is an important class for using LLVM in a threaded context.

LoopVectorizationCostModel - estimates the expected speedups due to vectorization.

Represents a single loop in the control flow graph.

A Module instance is used to store all the information related to an LLVM module.

This class represents an analyzed expression in the program.

The main scalar evolution driver.

SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringMap - This is an unconventional map that is specialized for handling keys that are "strings",...

Provides information about what library functions are available for the current target.

This pass provides access to the codegen interfaces that are needed for IR-level transformations.

TargetCostKind

The kind of cost model.

PartialReductionExtendKind

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

The instances of the Type class are immutable: once they are created, they are never changed.

Iterator to iterate over vectorization factors in a VFRange.

Definition VPlanHelpers.h:80

ElementCount operator*() const

Definition VPlanHelpers.h:88

iterator & operator++()

Definition VPlanHelpers.h:90

iterator(ElementCount VF)

Definition VPlanHelpers.h:84

bool operator==(const iterator &Other) const

Definition VPlanHelpers.h:86

VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.

VPBlockBase is the building block of the Hierarchical Control-Flow Graph.

Template specialization of the standard LLVM dominator tree utility for VPBlockBases.

In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...

Definition VPlanHelpers.h:110

static VPLane getLastLaneForVF(const ElementCount &VF)

Definition VPlanHelpers.h:151

Value * getAsRuntimeExpr(IRBuilderBase &Builder, const ElementCount &VF) const

Returns an expression describing the lane index that can be used at runtime.

VPLane(unsigned Lane, Kind LaneKind)

Definition VPlanHelpers.h:133

Kind getKind() const

Returns the Kind of lane offset.

Definition VPlanHelpers.h:168

static VPLane getLaneFromEnd(const ElementCount &VF, unsigned Offset)

Definition VPlanHelpers.h:137

bool isFirstLane() const

Returns true if this is the first lane of the whole vector.

Definition VPlanHelpers.h:171

VPLane(unsigned Lane)

Definition VPlanHelpers.h:132

unsigned getKnownLane() const

Returns a compile-time known value for the lane index and asserts if the lane can only be calculated ...

Definition VPlanHelpers.h:157

static VPLane getFirstLane()

Definition VPlanHelpers.h:135

Kind

Kind describes how to interpret Lane.

Definition VPlanHelpers.h:113

@ ScalableLast

For ScalableLast, Lane is the offset from the start of the last N-element subvector in a scalable vec...

Definition VPlanHelpers.h:121

@ First

For First, Lane is the index into the first N elements of a fixed-vector <N x > or a scalable v...

Definition VPlanHelpers.h:116

unsigned mapToCacheIndex(const ElementCount &VF) const

Maps the lane to a cache index based on VF.

Definition VPlanHelpers.h:174

VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...

This class can be used to assign names to VPValues.

Definition VPlanHelpers.h:383

ArrayRef< StringRef > getMDNames()

Returns the cached metadata kind names.

Definition VPlanHelpers.h:423

std::string getOrCreateName(const VPValue *V) const

Returns the name assigned to V, if there is one, otherwise try to construct one from the underlying v...

const Module * getModule() const

Returns the cached Module pointer.

Definition VPlanHelpers.h:430

VPSlotTracker(const VPlan *Plan=nullptr)

Definition VPlanHelpers.h:409

An analysis for type-inference for VPValues.

This is the base class of the VPlan Def/Use graph, used for modeling the data flow into,...

VPlanPrinter(raw_ostream &O, const VPlan &P)

Definition VPlanHelpers.h:477

LLVM_DUMP_METHOD void dump()

VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...

VPIRBasicBlock * getScalarHeader() const

Return the VPIRBasicBlock wrapping the header of the scalar loop.

LLVM Value Representation.

constexpr bool isScalable() const

Returns whether the quantity is scaled by a runtime quantity (vscale).

constexpr ScalarTy getKnownMinValue() const

Returns the minimum value this quantity can represent.

CRTP base class which implements the entire standard iterator facade in terms of a minimal subset of ...

This class implements an extremely fast bulk output stream that can only output to a stream.

This is an optimization pass for GlobalISel generic memory operations.

Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)

Return the runtime value for VF.

bool isVectorizedTy(Type *Ty)

Returns true if Ty is a vector type or a struct of vector types where all vector types share the same...

constexpr bool isPowerOf2_32(uint32_t Value)

Return true if the argument is a power of two > 0.

bool canConstantBeExtended(const APInt *C, Type *NarrowType, TTI::PartialReductionExtendKind ExtKind)

Check if a constant CI can be safely treated as having been extended from a narrower type with the gi...

FunctionAddr VTableAddr uintptr_t uintptr_t Data

Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)

Return a value for Step multiplied by VF.

iterator end()

Definition VPlanHelpers.h:97

const ElementCount Start

Definition VPlanHelpers.h:58

ElementCount End

Definition VPlanHelpers.h:61

iterator begin()

Definition VPlanHelpers.h:96

bool isEmpty() const

Definition VPlanHelpers.h:63

VFRange(const ElementCount &Start, const ElementCount &End)

Definition VPlanHelpers.h:67

unsigned getPredBlockCostDivisor(BasicBlock *BB) const

const Loop * L

Definition VPlanHelpers.h:338

LLVMContext & LLVMCtx

Definition VPlanHelpers.h:333

ScalarEvolution & SE

Definition VPlanHelpers.h:337

LoopVectorizationCostModel & CM

Definition VPlanHelpers.h:334

TargetTransformInfo::OperandValueInfo getOperandInfo(VPValue *V) const

Returns the OperandInfo for V, if it is a live-in.

bool isLegacyUniformAfterVectorization(Instruction *I, ElementCount VF) const

Return true if I is considered uniform-after-vectorization in the legacy cost model for VF.

bool skipCostComputation(Instruction *UI, bool IsVector) const

Return true if the cost for UI shouldn't be computed, e.g.

InstructionCost getScalarizationOverhead(Type *ResultTy, ArrayRef< const VPValue * > Operands, ElementCount VF, bool AlwaysIncludeReplicatingR=false)

Estimate the overhead of scalarizing a recipe with result type ResultTy and Operands with VF.

InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const

Return the cost for UI with VF using the legacy cost model as fallback until computing the cost of al...

VPCostContext(const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI, const VPlan &Plan, LoopVectorizationCostModel &CM, TargetTransformInfo::TargetCostKind CostKind, ScalarEvolution &SE, const Loop *L)

Definition VPlanHelpers.h:340

TargetTransformInfo::TargetCostKind CostKind

Definition VPlanHelpers.h:336

VPTypeAnalysis Types

Definition VPlanHelpers.h:332

const TargetLibraryInfo & TLI

Definition VPlanHelpers.h:331

const TargetTransformInfo & TTI

Definition VPlanHelpers.h:330

SmallPtrSet< Instruction *, 8 > SkipCostComputation

Definition VPlanHelpers.h:335

BasicBlock * PrevBB

The previous IR BasicBlock created or used.

Definition VPlanHelpers.h:288

VPBasicBlock * PrevVPBB

The previous VPBasicBlock visited. Initially set to null.

Definition VPlanHelpers.h:284

BasicBlock * ExitBB

The last IR BasicBlock in the output IR.

Definition VPlanHelpers.h:292

CFGState(DominatorTree *DT)

Definition VPlanHelpers.h:301

SmallDenseMap< const VPBasicBlock *, BasicBlock * > VPBB2IRBB

A mapping of each VPBasicBlock to the corresponding BasicBlock.

Definition VPlanHelpers.h:296

DomTreeUpdater DTU

Updater for the DominatorTree.

Definition VPlanHelpers.h:299

Definition VPlanHelpers.h:206

DenseMap< const VPValue *, SmallVector< Value *, 4 > > VPV2Scalars

Definition VPlanHelpers.h:211

DenseMap< const VPValue *, Value * > VPV2Vector

Definition VPlanHelpers.h:209

LoopInfo * LI

Hold a pointer to LoopInfo to register new basic blocks in the loop.

Definition VPlanHelpers.h:306

void reset(const VPValue *Def, Value *V)

Reset an existing vector value for Def and a given Part.

Definition VPlanHelpers.h:246

VPTypeAnalysis TypeAnalysis

VPlan-based type analysis.

Definition VPlanHelpers.h:322

struct llvm::VPTransformState::DataState Data

Value * get(const VPValue *Def, bool IsScalar=false)

Get the generated vector Value for a given VPValue Def if IsScalar is false, otherwise return the gen...

VPTransformState(const TargetTransformInfo *TTI, ElementCount VF, LoopInfo *LI, DominatorTree *DT, AssumptionCache *AC, IRBuilderBase &Builder, VPlan *Plan, Loop *CurrentParentLoop, Type *CanonicalIVTy)

std::optional< VPLane > Lane

Hold the index to generate specific scalar instructions.

Definition VPlanHelpers.h:204

void set(const VPValue *Def, Value *V, const VPLane &Lane)

Set the generated scalar V for Def and the given Lane.

Definition VPlanHelpers.h:252

IRBuilderBase & Builder

Hold a reference to the IRBuilder used to generate output IR code.

Definition VPlanHelpers.h:313

bool hasScalarValue(const VPValue *Def, VPLane Lane)

Definition VPlanHelpers.h:225

const TargetTransformInfo * TTI