LoopVectorizationLegality.h Source File (original) (raw)

26#ifndef LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONLEGALITY_H

27#define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONLEGALITY_H

34namespace llvm {

61 enum HintKind {

62 HK_WIDTH,

63 HK_INTERLEAVE,

64 HK_FORCE,

65 HK_ISVECTORIZED,

66 HK_PREDICATE,

67 HK_SCALABLE

68 };

71 struct Hint {

72 const char *Name;

73 unsigned Value;

74 HintKind Kind;

76 Hint(const char *Name, unsigned Value, HintKind Kind)

77 : Name(Name), Value(Value), Kind(Kind) {}

79 bool validate(unsigned Val);

80 };

83 Hint Width;

86 Hint Interleave;

89 Hint Force;

92 Hint IsVectorized;

95 Hint Predicate;

98 Hint Scalable;

100

101 static StringRef Prefix() { return "llvm.loop."; }

102

103

104 bool PotentiallyUnsafe = false;

105

106public:

112

114

116

118

119

120

122 };

123

127

128

130

132 bool VectorizeOnlyWhenForced) const;

133

134

136

141

143 if (Interleave.Value)

144 return Interleave.Value;

145

146

148 return 1;

149 return 0;

150 }

152 unsigned getPredicate() const { return Predicate.Value; }

159

160

164

165

166

168

169

170

171

172

173

175

177

178

179

180

181

183 }

184

186

187private:

188

189 void getHintsFromMetadata();

190

191

193

194

195 const Loop *TheLoop;

196

197

199

200

201

202

203

204 void reportDisallowedVectorization(const StringRef DebugMsg,

207 const Loop *L) const;

208};

209

210

211

212

213

214

215

216

217

218

219

220

221

223public:

224

226 if (I && !ExactFPMathInst)

227 ExactFPMathInst = I;

228 }

229

231

232private:

234};

235

236

237

238

247

248

249

250

251

252

253

254

255

256

257

258

259

260

262public:

269 : TheLoop(L), LI(LI), PSE(PSE), TTI(TTI), TLI(TLI), DT(DT), LAIs(LAIs),

270 ORE(ORE), Requirements(R), Hints(H), DB(DB), AC(AC),

271 AllowRuntimeSCEVChecks(AllowRuntimeSCEVChecks), AA(AA) {}

272

273

274

276

277

278

280

281

282

284

285

286

287

288

289

290

291

293

294

295

296

298

299

300

302

303

304

306

307

309

310

312

313

314

317 "only reductions have recurrence descriptors");

318 return Reductions.find(PN)->second;

319 }

320

321

323

324

326

327

329

330

331

333

334

335

337

338

340

341

342

344

345

346

348

349

350

351

353

354

355

356

358

359

361

362

364

365

366

368

369

370

371

372

373

374

375

376

377

378

380

381

382

384

385

386

388

389

390

391

393

394

396 return LAI->getRuntimePointerChecking();

397 }

398

400

402 return LAI->getDepChecker().isSafeForAnyVectorWidth() &&

403 LAI->getDepChecker().isSafeForAnyStoreLoadForwardDistances();

404 }

405

407 return LAI->getDepChecker().getMaxSafeVectorWidthInBits();

408 }

409

410

411

413

414

416 return UncountableExitingBB;

417 }

418

419

420

421

422

424 return UncountableExitWithSideEffects;

425 }

426

427

429 return LAI->getDepChecker().isSafeForAnyStoreLoadForwardDistances();

430 }

431

432

433

435 return LAI->getDepChecker().getStoreLoadForwardSafeDistanceInBits();

436 }

437

438

439

441 return MaskedOp.contains(I);

442 }

443

444

445

447

448 unsigned getNumStores() const { return LAI->getNumStores(); }

449 unsigned getNumLoads() const { return LAI->getNumLoads(); }

450

451

452

453

456 if (HGram.Load == I || HGram.Update == I || HGram.Store == I)

457 return &HGram;

458

459 return std::nullopt;

460 }

461

462

464

465

468 return PotentiallyFaultingLoads;

469 }

470

474

476

478

480

482

484

485

486

488 return CountableExitingBlocks;

489 }

490

491private:

492

493

494

495

496

497

498

499 bool canVectorizeLoopNestCFG(Loop *Lp, bool UseVPlanNativePath);

500

501

502

503

504 bool setupOuterLoopInductions();

505

506

507

508

509

510

511

512 bool canVectorizeLoopCFG(Loop *Lp, bool UseVPlanNativePath);

513

514

515

516

517 bool canVectorizeInstrs();

518

519

521

522

523

524

525

526 bool canVectorizeMemory();

527

528

529

530

531 bool canVectorizeIndirectUnsafeDependences();

532

533

534

535 bool canVectorizeWithIfConvert();

536

537

538

539 bool canVectorizeOuterLoop();

540

541

542

543

544

545

546

547

548

549

550

551

552

553

554

555

556

557

558 bool isVectorizableEarlyExitLoop();

559

560

561

562

563

564

565

566

567

568

569

570

571

572

573

574

575

576

577

578

579

580

581

582

583

584

585

586

587

588

589

590

591

592

593

594

595

596

597

598

599

600

601

602

603

604

605

606

607

608

609

610

611

612

613

614

615

616

617

618

619

620

621

622

623 bool canUncountableExitConditionLoadBeMoved(BasicBlock *ExitingBlock);

624

625

626

627

628

629

630

631

632 bool

635

636

637

638

641

642

643 Loop *TheLoop;

644

645

647

648

649

650

651

652

654

655

657

658

660

661

663

664

666

668

669

671

672

673

674

675

676 PHINode *PrimaryInduction = nullptr;

677

678

680

681

682

683

685

686

687

688

689

691

692

694

695

697

698

699

701

702

704

705

707

708

709

711

712

713

715

716

717

718

720

721

722

723

725

726

728

729

730 bool AllowRuntimeSCEVChecks;

731

732

733

735

736

737

738

739

740 bool VecCallVariantsFound = false;

741

742

743

745

746

747

748 BasicBlock *UncountableExitingBB = nullptr;

749

750

751

752 bool UncountableExitWithSideEffects = false;

753};

754

755}

756

757#endif

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

This file implements a map that provides insertion order iteration.

A cache of @llvm.assume calls within a function.

LLVM Basic Block Representation.

BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...

Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.

static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)

A struct for saving information about induction variables.

Class to represent integer types.

An instruction for reading from memory.

Drive the analysis of memory accesses in the loop.

MapVector< PHINode *, InductionDescriptor > InductionList

InductionList saves induction variables and maps them to the induction descriptor.

Definition LoopVectorizationLegality.h:279

unsigned getNumStores() const

Definition LoopVectorizationLegality.h:448

LoopInfo * getLoopInfo() const

Definition LoopVectorizationLegality.h:477

bool isInvariantStoreOfReduction(StoreInst *SI)

Returns True if given store is a final invariant store of one of the reductions found in the loop.

bool hasVectorCallVariants() const

Returns true if there is at least one function call in the loop which has a vectorized variant availa...

Definition LoopVectorizationLegality.h:446

uint64_t getMaxSafeVectorWidthInBits() const

Definition LoopVectorizationLegality.h:406

const RecurrenceDescriptor & getRecurrenceDescriptor(PHINode *PN) const

Returns the recurrence descriptor associated with a given phi node PN, expecting one to exist.

Definition LoopVectorizationLegality.h:315

const SmallPtrSetImpl< const Instruction * > & getPotentiallyFaultingLoads() const

Returns potentially faulting loads.

Definition LoopVectorizationLegality.h:467

RecurrenceSet & getFixedOrderRecurrences()

Return the fixed-order recurrences found in the loop.

Definition LoopVectorizationLegality.h:325

uint64_t getMaxStoreLoadForwardSafeDistanceInBits() const

Return safe power-of-2 number of elements, which do not prevent store-load forwarding and safe to ope...

Definition LoopVectorizationLegality.h:434

bool isInvariantAddressOfReduction(Value *V)

Returns True if given address is invariant and is used to store recurrent expression.

DominatorTree * getDominatorTree() const

Definition LoopVectorizationLegality.h:483

bool canVectorize(bool UseVPlanNativePath)

Returns true if it is legal to vectorize this loop.

PredicatedScalarEvolution * getPredicatedScalarEvolution() const

Definition LoopVectorizationLegality.h:471

bool blockNeedsPredication(const BasicBlock *BB) const

Return true if the block BB needs to be predicated in order for the loop to be vectorized.

int isConsecutivePtr(Type *AccessTy, Value *Ptr) const

Check if this pointer is consecutive when vectorizing.

AssumptionCache * getAssumptionCache() const

Definition LoopVectorizationLegality.h:479

std::optional< const HistogramInfo * > getHistogramInfo(Instruction *I) const

Returns a HistogramInfo* for the given instruction if it was determined to be part of a load -> updat...

Definition LoopVectorizationLegality.h:454

SmallPtrSet< const PHINode *, 8 > RecurrenceSet

RecurrenceSet contains the phi nodes that are recurrences other than inductions and reductions.

Definition LoopVectorizationLegality.h:283

bool hasUncountableExitWithSideEffects() const

Returns true if this is an early exit loop with state-changing or potentially-faulting operations and...

Definition LoopVectorizationLegality.h:423

bool canVectorizeFPMath(bool EnableStrictReductions)

Returns true if it is legal to vectorize the FP math operations in this loop.

bool isReductionVariable(PHINode *PN) const

Returns True if PN is a reduction variable in this loop.

Definition LoopVectorizationLegality.h:360

bool isFixedOrderRecurrence(const PHINode *Phi) const

Returns True if Phi is a fixed-order recurrence in this loop.

IntegerType * getWidestInductionType()

Returns the widest induction type.

Definition LoopVectorizationLegality.h:328

const InductionDescriptor * getPointerInductionDescriptor(PHINode *Phi) const

Returns a pointer to the induction descriptor, if Phi is pointer induction.

const InductionDescriptor * getIntOrFpInductionDescriptor(PHINode *Phi) const

Returns a pointer to the induction descriptor, if Phi is an integer or floating point induction.

bool isInductionPhi(const Value *V) const

Returns True if V is a Phi node of an induction variable in this loop.

PHINode * getPrimaryInduction()

Returns the primary induction variable.

Definition LoopVectorizationLegality.h:308

const SmallVector< BasicBlock *, 4 > & getCountableExitingBlocks() const

Returns all exiting blocks with a countable exit, i.e.

Definition LoopVectorizationLegality.h:487

bool isUniform(Value *V, ElementCount VF) const

Returns true if value V is uniform across VF lanes, when VF is provided, and otherwise if V is invari...

const InductionList & getInductionVars() const

Returns the induction variables found in the loop.

Definition LoopVectorizationLegality.h:322

bool isInvariant(Value *V) const

Returns true if V is invariant across all loop iterations according to SCEV.

const ReductionList & getReductionVars() const

Returns the reduction variables found in the loop.

Definition LoopVectorizationLegality.h:311

bool isSafeForAnyStoreLoadForwardDistances() const

Return true if there is store-load forwarding dependencies.

Definition LoopVectorizationLegality.h:428

bool isSafeForAnyVectorWidth() const

Definition LoopVectorizationLegality.h:401

unsigned getNumLoads() const

Definition LoopVectorizationLegality.h:449

bool canFoldTailByMasking() const

Return true if we can vectorize this loop while folding its tail by masking.

void prepareToFoldTailByMasking()

Mark all respective loads/stores for masking.

bool hasUncountableEarlyExit() const

Returns true if the loop has exactly one uncountable early exit, i.e.

Definition LoopVectorizationLegality.h:412

bool hasHistograms() const

Returns a list of all known histogram operations in the loop.

Definition LoopVectorizationLegality.h:463

const LoopAccessInfo * getLAI() const

Definition LoopVectorizationLegality.h:399

MapVector< PHINode *, RecurrenceDescriptor > ReductionList

ReductionList contains the reduction descriptors for all of the reductions that were found in the loo...

Definition LoopVectorizationLegality.h:275

ScalarEvolution * getScalarEvolution() const

Definition LoopVectorizationLegality.h:481

bool isUniformMemOp(Instruction &I, ElementCount VF) const

A uniform memory op is a load or store which accesses the same memory location on all VF lanes,...

BasicBlock * getUncountableEarlyExitingBlock() const

Returns the uncountable early exiting block, if there is exactly one.

Definition LoopVectorizationLegality.h:415

Loop * getLoop() const

Definition LoopVectorizationLegality.h:475

LoopVectorizationLegality(Loop *L, PredicatedScalarEvolution &PSE, DominatorTree *DT, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, Function *F, LoopAccessInfoManager &LAIs, LoopInfo *LI, OptimizationRemarkEmitter *ORE, LoopVectorizationRequirements *R, LoopVectorizeHints *H, DemandedBits *DB, AssumptionCache *AC, bool AllowRuntimeSCEVChecks, AAResults *AA)

Definition LoopVectorizationLegality.h:263

bool isMaskRequired(const Instruction *I) const

Returns true if vector representation of the instruction I requires mask.

Definition LoopVectorizationLegality.h:440

const RuntimePointerChecking * getRuntimePointerChecking() const

Returns the information that we collected about runtime memory check.

Definition LoopVectorizationLegality.h:395

bool isInductionVariable(const Value *V) const

Returns True if V can be considered as an induction variable in this loop.

bool isCastedInductionVariable(const Value *V) const

Returns True if V is a cast that is part of an induction def-use chain, and had been proven to be red...

This holds vectorization requirements that must be verified late in the process.

Definition LoopVectorizationLegality.h:222

Instruction * getExactFPInst()

Definition LoopVectorizationLegality.h:230

void addExactFPMathInst(Instruction *I)

Track the 1st floating-point instruction that can not be reassociated.

Definition LoopVectorizationLegality.h:225

Utility class for getting and setting loop vectorizer hints in the form of loop metadata.

Definition LoopVectorizationLegality.h:60

ScalableForceKind

Definition LoopVectorizationLegality.h:113

@ SK_PreferScalable

Vectorize loops using scalable vectors or fixed-width vectors, but favor scalable vectors when the co...

Definition LoopVectorizationLegality.h:121

@ SK_Unspecified

Not selected.

Definition LoopVectorizationLegality.h:115

@ SK_FixedWidthOnly

Disables vectorization with scalable vectors.

Definition LoopVectorizationLegality.h:117

bool isScalableVectorizationDisabled() const

Definition LoopVectorizationLegality.h:161

enum ForceKind getForce() const

Definition LoopVectorizationLegality.h:153

bool allowVectorization(Function *F, Loop *L, bool VectorizeOnlyWhenForced) const

bool allowReordering() const

When enabling loop hints are provided we allow the vectorizer to change the order of operations that ...

void emitRemarkWithHints() const

Dumps all the hint information.

bool isPotentiallyUnsafe() const

Definition LoopVectorizationLegality.h:176

ElementCount getWidth() const

Definition LoopVectorizationLegality.h:137

ForceKind

Definition LoopVectorizationLegality.h:107

@ FK_Enabled

Forcing enabled.

Definition LoopVectorizationLegality.h:110

@ FK_Undefined

Not selected.

Definition LoopVectorizationLegality.h:108

@ FK_Disabled

Forcing disabled.

Definition LoopVectorizationLegality.h:109

unsigned getPredicate() const

Definition LoopVectorizationLegality.h:152

void setAlreadyVectorized()

Mark the loop L as already vectorized by setting the width to 1.

LoopVectorizeHints(const Loop *L, bool InterleaveOnlyWhenForced, OptimizationRemarkEmitter &ORE, const TargetTransformInfo *TTI=nullptr)

const char * vectorizeAnalysisPassName() const

If hints are provided that force vectorization, use the AlwaysPrint pass name to force the frontend t...

unsigned getInterleave() const

Definition LoopVectorizationLegality.h:142

unsigned getIsVectorized() const

Definition LoopVectorizationLegality.h:151

void setPotentiallyUnsafe()

Definition LoopVectorizationLegality.h:185

Represents a single loop in the control flow graph.

This class implements a map that also provides access to all stored values in a deterministic order.

An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...

Analysis providing profile information.

The RecurrenceDescriptor is used to identify recurrences variables in a loop.

Holds information about the memory runtime legality checks to verify that a group of pointers do not ...

The main scalar evolution driver.

A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...

SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

An instruction for storing to memory.

StringRef - Represent a constant reference to a string, i.e.

Provides information about what library functions are available for the current target.

This pass provides access to the codegen interfaces that are needed for IR-level transformations.

The instances of the Type class are immutable: once they are created, they are never changed.

LLVM Value Representation.

Abstract Attribute helper functions.

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

This is an optimization pass for GlobalISel generic memory operations.

FunctionAddr VTableAddr Value

LLVM_ABI bool hasDisableAllTransformsHint(const Loop *L)

Look for the loop attribute that disables all transformation heuristic.

LLVM_ABI TransformationMode hasUnrollTransformation(const Loop *L)

@ TM_Disable

The transformation should not be applied.

This holds details about a histogram operation – a load -> update -> store sequence where each lane i...

Definition LoopVectorizationLegality.h:239

LoadInst * Load

Definition LoopVectorizationLegality.h:240

StoreInst * Store

Definition LoopVectorizationLegality.h:242

HistogramInfo(LoadInst *Load, Instruction *Update, StoreInst *Store)

Definition LoopVectorizationLegality.h:244

Instruction * Update

Definition LoopVectorizationLegality.h:241