VPlanRecipes.cpp Source File (original) (raw)

38#include

40using namespace llvm;

44namespace llvm {

46}

49#define LV_NAME "loop-vectorize"

50#define DEBUG_TYPE LV_NAME

54 case VPInstructionSC:

55 return cast(this)->opcodeMayReadOrWriteFromMemory();

56 case VPInterleaveSC:

57 return cast(this)->getNumStoreOperands() > 0;

58 case VPWidenStoreEVLSC:

59 case VPWidenStoreSC:

60 return true;

61 case VPReplicateSC:

62 return cast(getVPSingleValue()->getUnderlyingValue())

63 ->mayWriteToMemory();

64 case VPWidenCallSC:

65 return !cast(this)

66 ->getCalledScalarFunction()

67 ->onlyReadsMemory();

68 case VPWidenIntrinsicSC:

69 return cast(this)->mayWriteToMemory();

70 case VPBranchOnMaskSC:

71 case VPScalarIVStepsSC:

72 case VPPredInstPHISC:

73 return false;

74 case VPBlendSC:

75 case VPReductionEVLSC:

76 case VPReductionSC:

77 case VPVectorPointerSC:

78 case VPWidenCanonicalIVSC:

79 case VPWidenCastSC:

80 case VPWidenGEPSC:

81 case VPWidenIntOrFpInductionSC:

82 case VPWidenLoadEVLSC:

83 case VPWidenLoadSC:

84 case VPWidenPHISC:

85 case VPWidenSC:

86 case VPWidenEVLSC:

87 case VPWidenSelectSC: {

89 dyn_cast_or_null(getVPSingleValue()->getUnderlyingValue());

90 (void)I;

91 assert(( || ->mayWriteToMemory()) &&

92 "underlying instruction may write to memory");

93 return false;

94 }

95 default:

96 return true;

97 }

98}

102 case VPInstructionSC:

103 return cast(this)->opcodeMayReadOrWriteFromMemory();

104 case VPWidenLoadEVLSC:

105 case VPWidenLoadSC:

106 return true;

107 case VPReplicateSC:

108 return cast(getVPSingleValue()->getUnderlyingValue())

109 ->mayReadFromMemory();

110 case VPWidenCallSC:

111 return !cast(this)

112 ->getCalledScalarFunction()

113 ->onlyWritesMemory();

114 case VPWidenIntrinsicSC:

115 return cast(this)->mayReadFromMemory();

116 case VPBranchOnMaskSC:

117 case VPPredInstPHISC:

118 case VPScalarIVStepsSC:

119 case VPWidenStoreEVLSC:

120 case VPWidenStoreSC:

121 return false;

122 case VPBlendSC:

123 case VPReductionEVLSC:

124 case VPReductionSC:

125 case VPVectorPointerSC:

126 case VPWidenCanonicalIVSC:

127 case VPWidenCastSC:

128 case VPWidenGEPSC:

129 case VPWidenIntOrFpInductionSC:

130 case VPWidenPHISC:

131 case VPWidenSC:

132 case VPWidenEVLSC:

133 case VPWidenSelectSC: {

135 dyn_cast_or_null(getVPSingleValue()->getUnderlyingValue());

136 (void)I;

137 assert(( || ->mayReadFromMemory()) &&

138 "underlying instruction may read from memory");

139 return false;

140 }

141 default:

142 return true;

143 }

144}

145

148 case VPDerivedIVSC:

149 case VPPredInstPHISC:

150 case VPScalarCastSC:

151 case VPReverseVectorPointerSC:

152 return false;

153 case VPInstructionSC:

155 case VPWidenCallSC: {

156 Function *Fn = cast(this)->getCalledScalarFunction();

158 }

159 case VPWidenIntrinsicSC:

160 return cast(this)->mayHaveSideEffects();

161 case VPBlendSC:

162 case VPReductionEVLSC:

163 case VPReductionSC:

164 case VPScalarIVStepsSC:

165 case VPVectorPointerSC:

166 case VPWidenCanonicalIVSC:

167 case VPWidenCastSC:

168 case VPWidenGEPSC:

169 case VPWidenIntOrFpInductionSC:

170 case VPWidenPHISC:

171 case VPWidenPointerInductionSC:

172 case VPWidenSC:

173 case VPWidenEVLSC:

174 case VPWidenSelectSC: {

176 dyn_cast_or_null(getVPSingleValue()->getUnderlyingValue());

177 (void)I;

178 assert(( || ->mayHaveSideEffects()) &&

179 "underlying instruction has side-effects");

180 return false;

181 }

182 case VPInterleaveSC:

184 case VPWidenLoadEVLSC:

185 case VPWidenLoadSC:

186 case VPWidenStoreEVLSC:

187 case VPWidenStoreSC:

189 cast(this)->getIngredient().mayHaveSideEffects() ==

191 "mayHaveSideffects result for ingredient differs from this "

192 "implementation");

194 case VPReplicateSC: {

195 auto *R = cast(this);

196 return R->getUnderlyingInstr()->mayHaveSideEffects();

197 }

198 default:

199 return true;

200 }

201}

202

204 assert(!Parent && "Recipe already in some VPBasicBlock");

206 "Insertion position not in any VPBasicBlock");

208}

209

212 assert(!Parent && "Recipe already in some VPBasicBlock");

213 assert(I == BB.end() || I->getParent() == &BB);

215}

216

218 assert(!Parent && "Recipe already in some VPBasicBlock");

220 "Insertion position not in any VPBasicBlock");

222}

223

227 Parent = nullptr;

228}

229

233}

234

238}

239

244}

245

247

248

249

250

252 if (auto *S = dyn_cast(this))

253 UI = dyn_cast_or_null(S->getUnderlyingValue());

254 else if (auto *IG = dyn_cast(this))

255 UI = IG->getInsertPos();

256 else if (auto *WidenMem = dyn_cast(this))

257 UI = &WidenMem->getIngredient();

258

261 RecipeCost = 0;

262 } else {

267 }

268

270 dbgs() << "Cost of " << RecipeCost << " for VF " << VF << ": ";

272 });

273 return RecipeCost;

274}

275

279}

280

284 std::optional Opcode = std::nullopt;

286 if (auto *WidenR = dyn_cast(BinOpR))

287 Opcode = std::make_optional(WidenR->getOpcode());

288

291

297

299

300 if (!R)

302 auto *WidenCastR = dyn_cast(R);

303 if (!WidenCastR)

305 if (WidenCastR->getOpcode() == Instruction::CastOps::ZExt)

307 if (WidenCastR->getOpcode() == Instruction::CastOps::SExt)

310 };

311

313 PhiType, VF, GetExtendKind(ExtAR),

314 GetExtendKind(ExtBR), Opcode);

315}

316

319 auto &Builder = State.Builder;

320

322 "Unhandled partial reduction opcode");

323

326 assert(PhiVal && BinOpVal && "Phi and Mul must be set");

327

329

330 CallInst *V = Builder.CreateIntrinsic(

331 RetTy, Intrinsic::experimental_vector_partial_reduce_add,

332 {PhiVal, BinOpVal}, nullptr, "partial.reduce");

333

334 State.set(this, V);

335}

336

337#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

340 O << Indent << "PARTIAL-REDUCE ";

344}

345#endif

346

348 assert(OpType == OperationType::FPMathOp &&

349 "recipe doesn't have fast math flags");

358 return Res;

359}

360

361#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

363#endif

364

365template

368 if (U.getNumOperands() == PartOpIdx + 1)

369 return U.getOperand(PartOpIdx);

370 return nullptr;

371}

372

373template

375 if (auto *UnrollPartOp = getUnrollPartOperand(U))

376 return cast(UnrollPartOp->getLiveInIRValue())->getZExtValue();

377 return 0;

378}

379

384 Pred, DL),

385 Opcode(Opcode), Name(Name.str()) {

386 assert(Opcode == Instruction::ICmp &&

387 "only ICmp predicates supported at the moment");

388}

389

391 std::initializer_list<VPValue *> Operands,

394 Opcode(Opcode), Name(Name.str()) {

395

396 assert(isFPMathOp() && "this op can't take fast-math flags");

397}

398

399bool VPInstruction::doesGeneratePerAllLanes() const {

401}

402

403bool VPInstruction::canGenerateScalarForFirstLane() const {

405 return true;

407 return true;

408 switch (Opcode) {

409 case Instruction::ICmp:

410 case Instruction::Select:

418 return true;

419 default:

420 return false;

421 }

422}

423

425 const VPLane &Lane) {

427

429 "only PtrAdd opcodes are supported for now");

432}

433

436

441 auto *Res =

443 if (auto *I = dyn_cast(Res))

445 return Res;

446 }

447

452 }

453 case Instruction::ICmp: {

458 }

459 case Instruction::Select: {

465 }

467

469

471

472

473

476 Name);

477

480 return Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask,

481 {PredTy, ScalarTC->getType()},

482 {VIVElem0, ScalarTC}, nullptr, Name);

483 }

485

486

487

488

489

490

491

492

493

494

495

496

498 if (!V1->getType()->isVectorTy())

499 return V1;

502 }

511 }

513

514

516

518 "Requested vector length should be an integer.");

519

522

524 State.Builder.getInt32Ty(), Intrinsic::experimental_get_vector_length,

525 {AVL, VFArg, State.Builder.getTrue()});

526 return EVL;

527 }

531 assert(Part != 0 && "Must have a positive part");

532

533

537 }

540

541

542

547

549 return CondBr;

550

554 return CondBr;

555 }

557

561

562

564 VPRegionBlock *TopRegion = Plan->getVectorLoopRegion();

566

567

568

569

570

571

576 return CondBr;

577 }

579

580

581 auto *PhiR = cast(getOperand(0));

582 auto *OrigPhi = cast(PhiR->getUnderlyingValue());

583

585

587

588 Type *PhiTy = OrigPhi->getType();

589

590

593 for (unsigned Part = 0; Part < UF; ++Part)

594 RdxParts[Part] = State.get(getOperand(1 + Part), PhiR->isInLoop());

595

596

597

598

599

602 for (unsigned Part = 0; Part < UF; ++Part)

603 RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);

604 }

605

606 Value *ReducedPartRdx = RdxParts[0];

609 Op = Instruction::Or;

610

611 if (PhiR->isOrdered()) {

612 ReducedPartRdx = RdxParts[UF - 1];

613 } else {

614

617 for (unsigned Part = 1; Part < UF; ++Part) {

618 Value *RdxPart = RdxParts[Part];

619 if (Op != Instruction::ICmp && Op != Instruction::FCmp)

623 ReducedPartRdx =

625 else

626 ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);

627 }

628 }

629

630

631

635 !PhiR->isInLoop()) {

636 ReducedPartRdx =

637 createReduction(Builder, RdxDesc, ReducedPartRdx, OrigPhi);

638

639

641 ReducedPartRdx = RdxDesc.isSigned()

642 ? Builder.CreateSExt(ReducedPartRdx, PhiTy)

643 : Builder.CreateZExt(ReducedPartRdx, PhiTy);

644 }

645

646 return ReducedPartRdx;

647 }

650 unsigned Offset = CI->getZExtValue();

651 assert(Offset > 0 && "Offset from end must be positive");

655 "invalid offset to extract from");

656

658 } else {

659 assert(Offset <= 1 && "invalid offset to extract from");

661 }

662 if (isa(Res))

664 return Res;

665 }

670 }

673 "can only generate first lane for PtrAdd");

677 }

679 Value *IncomingFromVPlanPred =

681 Value *IncomingFromOtherPreds =

683 auto *NewPhi =

686 State.CFG

688 NewPhi->addIncoming(IncomingFromVPlanPred, VPlanPred);

690 if (OtherPred == VPlanPred)

691 continue;

692 NewPhi->addIncoming(IncomingFromOtherPreds, OtherPred);

693 }

694 return NewPhi;

695 }

699 }

700

701 default:

703 }

704}

705

710}

711

714}

715

716#if !defined(NDEBUG)

717bool VPInstruction::isFPMathOp() const {

718

719

720 return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||

721 Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||

722 Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||

723 Opcode == Instruction::FCmp || Opcode == Instruction::Select;

724}

725#endif

726

728 assert(!State.Lane && "VPInstruction executing an Lane");

731 getOpcode() == Instruction::Select) &&

732 "Recipe not a FPMathOp but has fast-math flags?");

736 bool GeneratesPerFirstLaneOnly = canGenerateScalarForFirstLane() &&

739 bool GeneratesPerAllLanes = doesGeneratePerAllLanes();

740 if (GeneratesPerAllLanes) {

742 Lane != NumLanes; ++Lane) {

743 Value *GeneratedValue = generatePerLane(State, VPLane(Lane));

744 assert(GeneratedValue && "generatePerLane must produce a value");

745 State.set(this, GeneratedValue, VPLane(Lane));

746 }

747 return;

748 }

749

750 Value *GeneratedValue = generate(State);

752 return;

753 assert(GeneratedValue && "generate must produce a value");

755 (GeneratedValue->getType()->isVectorTy() == !GeneratesPerFirstLaneOnly ||

757 "scalar value but not only first lane defined");

758 State.set(this, GeneratedValue,

759 GeneratesPerFirstLaneOnly);

760}

761

764 return false;

766 case Instruction::ICmp:

767 case Instruction::Select:

776 return false;

777 default:

778 return true;

779 }

780}

781

786

788 default:

789 return false;

790 case Instruction::ICmp:

791 case Instruction::Select:

792 case Instruction::Or:

794

803 return true;

804 };

806}

807

812

814 default:

815 return false;

816 case Instruction::ICmp:

817 case Instruction::Select:

822 return true;

823 };

825}

826

827#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

831}

832

835 O << Indent << "EMIT ";

836

839 O << " = ";

840 }

841

844 O << "not";

845 break;

847 O << "combined load";

848 break;

850 O << "combined store";

851 break;

853 O << "active lane mask";

854 break;

856 O << "resume-phi";

857 break;

859 O << "EXPLICIT-VECTOR-LENGTH";

860 break;

862 O << "first-order splice";

863 break;

865 O << "branch-on-cond";

866 break;

868 O << "TC > VF ? TC - VF : 0";

869 break;

871 O << "VF * Part +";

872 break;

874 O << "branch-on-count";

875 break;

877 O << "extract-from-end";

878 break;

880 O << "compute-reduction-result";

881 break;

883 O << "logical-and";

884 break;

886 O << "ptradd";

887 break;

889 O << "any-of";

890 break;

891 default:

893 }

894

897

899 O << ", !dbg ";

901 }

902}

903#endif

904

907 "Only PHINodes can have extra operands");

916

917

920 auto *Phi = cast(&I);

921

922

923 if (Phi->getBasicBlockIndex(PredBB) == -1)

924 Phi->addIncoming(V, PredBB);

925 else

926 Phi->setIncomingValueForBlock(PredBB, V);

927 }

928

929

930

932}

933

936

937

938 return 0;

939}

940

943 "can only add exiting operands to phi nodes");

951 {Exiting,

952 Plan.getOrAddLiveIn(ConstantInt::get(IntegerType::get(Ctx, 32), 1))});

953 }

955}

956

957#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

960 O << Indent << "IR " << I;

961

963 O << " (extra operand" << (getNumOperands() > 1 ? "s" : "") << ": ";

967 O << " from ";

969 });

970 O << ")";

971 }

972}

973#endif

974

978

980

984

985

986

988 Arg = State.get(I.value(), VPLane(0));

989 else

991 Args.push_back(Arg);

992 }

993

994 assert(Variant != nullptr && "Can't create vector function.");

995

998 if (CI)

999 CI->getOperandBundlesAsDefs(OpBundles);

1000

1003

1004 if (!V->getType()->isVoidTy())

1005 State.set(this, V);

1007}

1008

1014}

1015

1016#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1019 O << Indent << "WIDEN-CALL ";

1020

1023 O << "void ";

1024 else {

1026 O << " = ";

1027 }

1028

1029 O << "call";

1031 O << " @" << CalledFn->getName() << "(";

1034 });

1035 O << ")";

1036

1037 O << " (using library function";

1039 O << ": " << Variant->getName();

1040 O << ")";

1041}

1042#endif

1043

1047

1049

1054

1055

1058 State.TTI))

1059 Arg = State.get(I.value(), VPLane(0));

1060 else

1063 State.TTI))

1065 Args.push_back(Arg);

1066 }

1067

1068

1073 "Can't retrieve vector intrinsic or vector-predication intrinsics.");

1074

1077 if (CI)

1078 CI->getOperandBundlesAsDefs(OpBundles);

1079

1081

1083

1084 if (!V->getType()->isVoidTy())

1085 State.set(this, V);

1087}

1088

1091

1092

1093

1094

1095

1098 auto *V = Op->getUnderlyingValue();

1099 if (!V) {

1100

1101

1102

1103

1104

1107 continue;

1108 }

1110 Arguments.push_back(UI->getArgOperand(Idx));

1111 continue;

1112 }

1114 break;

1115 }

1117 }

1118

1124

1125

1131}

1132

1135}

1136

1139

1140

1143}

1144

1145#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1148 O << Indent << "WIDEN-INTRINSIC ";

1150 O << "void ";

1151 } else {

1153 O << " = ";

1154 }

1155

1156 O << "call";

1159

1162 });

1163 O << ")";

1164}

1165#endif

1166

1170

1174

1175

1176

1177

1180 Mask = State.get(VPMask);

1182 Mask =

1184

1185

1186

1187 if (Opcode == Instruction::Sub)

1188 IncAmt = Builder.CreateNeg(IncAmt);

1189 else

1190 assert(Opcode == Instruction::Add && "only add or sub supported for now");

1191

1193 {VTy, IncAmt->getType()},

1194 {Address, IncAmt, Mask});

1195}

1196

1199

1200

1201

1202

1203

1204 assert(VF.isVector() && "Invalid VF for histogram cost");

1209

1210

1211

1216

1219 }

1220

1221

1226 {PtrTy, IncTy, MaskTy});

1227

1228

1231}

1232

1233#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1236 O << Indent << "WIDEN-HISTOGRAM buckets: ";

1238

1239 if (Opcode == Instruction::Sub)

1240 O << ", dec: ";

1241 else {

1242 assert(Opcode == Instruction::Add);

1243 O << ", inc: ";

1244 }

1246

1248 O << ", mask: ";

1250 }

1251}

1252

1255 O << Indent << "WIDEN-SELECT ";

1257 O << " = select ";

1260 O << ", ";

1262 O << ", ";

1264 O << (isInvariantCond() ? " (condition is loop invariant)" : "");

1265}

1266#endif

1267

1270

1271

1272

1273

1274

1275 auto *InvarCond =

1277

1282 State.set(this, Sel);

1283 if (isa(Sel))

1284 setFlags(cast(Sel));

1286}

1287

1294

1299 match(this, m_LogicalOr(m_VPValue(Op0), m_VPValue(Op1))))) {

1300

1301

1302 const auto [Op1VK, Op1VP] = Ctx.getOperandInfo(Op0);

1303 const auto [Op2VK, Op2VP] = Ctx.getOperandInfo(Op1);

1304

1307 [](VPValue *Op) { return Op->getUnderlyingValue(); }))

1308 Operands.append(SI->op_begin(), SI->op_end());

1309 bool IsLogicalOr = match(this, m_LogicalOr(m_VPValue(Op0), m_VPValue(Op1)));

1311 IsLogicalOr ? Instruction::Or : Instruction::And, VectorTy,

1312 Ctx.CostKind, {Op1VK, Op1VP}, {Op2VK, Op2VP}, Operands, SI);

1313 }

1314

1316 if (!ScalarCond)

1318

1320 if (auto *Cmp = dyn_cast(SI->getCondition()))

1321 Pred = Cmp->getPredicate();

1323 Instruction::Select, VectorTy, CondTy, Pred, Ctx.CostKind,

1324 {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None}, SI);

1325}

1326

1327VPRecipeWithIRFlags::FastMathFlagsTy::FastMathFlagsTy(

1330 NoNaNs = FMF.noNaNs();

1331 NoInfs = FMF.noInfs();

1336}

1337

1338#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1340 switch (OpType) {

1341 case OperationType::Cmp:

1343 break;

1344 case OperationType::DisjointOp:

1346 O << " disjoint";

1347 break;

1348 case OperationType::PossiblyExactOp:

1350 O << " exact";

1351 break;

1352 case OperationType::OverflowingBinOp:

1354 O << " nuw";

1356 O << " nsw";

1357 break;

1358 case OperationType::FPMathOp:

1360 break;

1361 case OperationType::GEPOp:

1363 O << " inbounds";

1365 O << " nusw";

1367 O << " nuw";

1368 break;

1369 case OperationType::NonNegOp:

1371 O << " nneg";

1372 break;

1373 case OperationType::Other:

1374 break;

1375 }

1377 O << " ";

1378}

1379#endif

1380

1383 auto &Builder = State.Builder;

1384 switch (Opcode) {

1385 case Instruction::Call:

1386 case Instruction::Br:

1387 case Instruction::PHI:

1388 case Instruction::GetElementPtr:

1389 case Instruction::Select:

1390 llvm_unreachable("This instruction is handled by a different recipe.");

1391 case Instruction::UDiv:

1392 case Instruction::SDiv:

1393 case Instruction::SRem:

1394 case Instruction::URem:

1395 case Instruction::Add:

1396 case Instruction::FAdd:

1397 case Instruction::Sub:

1398 case Instruction::FSub:

1399 case Instruction::FNeg:

1400 case Instruction::Mul:

1401 case Instruction::FMul:

1402 case Instruction::FDiv:

1403 case Instruction::FRem:

1404 case Instruction::Shl:

1405 case Instruction::LShr:

1406 case Instruction::AShr:

1407 case Instruction::And:

1408 case Instruction::Or:

1409 case Instruction::Xor: {

1410

1414

1416

1417 if (auto *VecOp = dyn_cast(V))

1419

1420

1421 State.set(this, V);

1423 break;

1424 }

1425 case Instruction::Freeze: {

1427

1429 State.set(this, Freeze);

1430 break;

1431 }

1432 case Instruction::ICmp:

1433 case Instruction::FCmp: {

1434

1435 bool FCmp = Opcode == Instruction::FCmp;

1439 if (FCmp) {

1440

1444 } else {

1446 }

1447 State.set(this, C);

1449 break;

1450 }

1451 default:

1452

1453 LLVM_DEBUG(dbgs() << "LV: Found an unhandled opcode : "

1456 }

1457

1458#if !defined(NDEBUG)

1459

1460

1463 "inferred type and type from generated instructions do not match");

1464#endif

1465}

1466

1469 switch (Opcode) {

1470 case Instruction::FNeg: {

1473 Opcode, VectorTy, Ctx.CostKind,

1474 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},

1475 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None});

1476 }

1477

1478 case Instruction::UDiv:

1479 case Instruction::SDiv:

1480 case Instruction::SRem:

1481 case Instruction::URem:

1482

1484 case Instruction::Add:

1485 case Instruction::FAdd:

1486 case Instruction::Sub:

1487 case Instruction::FSub:

1488 case Instruction::Mul:

1489 case Instruction::FMul:

1490 case Instruction::FDiv:

1491 case Instruction::FRem:

1492 case Instruction::Shl:

1493 case Instruction::LShr:

1494 case Instruction::AShr:

1495 case Instruction::And:

1496 case Instruction::Or:

1497 case Instruction::Xor: {

1499

1500

1503 if (RHS->isLiveIn())

1505

1511

1513 if (CtxI)

1516 Opcode, VectorTy, Ctx.CostKind,

1517 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},

1519 }

1520 case Instruction::Freeze: {

1521

1525 }

1526 case Instruction::ICmp:

1527 case Instruction::FCmp: {

1532 {TTI::OK_AnyValue, TTI::OP_None},

1533 {TTI::OK_AnyValue, TTI::OP_None}, CtxI);

1534 }

1535 default:

1537 }

1538}

1539

1542

1544 llvm_unreachable("Unsupported opcode in VPWidenEVLRecipe::execute");

1545

1547

1549 "VPWidenEVLRecipe should not be used for scalars");

1550

1552 Value *EVLArg = State.get(EVL, true);

1556

1561 }

1562

1566

1567

1568 if (isa(VPInst))

1569 setFlags(cast(VPInst));

1570

1571 State.set(this, VPInst);

1574}

1575

1576#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1579 O << Indent << "WIDEN ";

1584}

1585

1588 O << Indent << "WIDEN ";

1593}

1594#endif

1595

1598 auto &Builder = State.Builder;

1599

1605 State.set(this, Cast);

1607 if (auto *CastOp = dyn_cast(Cast))

1609}

1610

1613

1614

1615

1617 return 0;

1618

1622 if (isa(R))

1624 if (const auto *ReplicateRecipe = dyn_cast(R))

1627 const auto *WidenMemoryRecipe = dyn_cast(R);

1628 if (WidenMemoryRecipe == nullptr)

1630 if (!WidenMemoryRecipe->isConsecutive())

1632 if (WidenMemoryRecipe->isReverse())

1634 if (WidenMemoryRecipe->isMasked())

1637 };

1638

1641

1642 if ((Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) &&

1644 if (auto *StoreRecipe = dyn_cast(*user_begin()))

1645 CCH = ComputeCCH(StoreRecipe);

1646 }

1647

1648 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||

1649 Opcode == Instruction::FPExt) {

1654 }

1655

1656 auto *SrcTy =

1659

1661 Opcode, DestTy, SrcTy, CCH, Ctx.CostKind,

1663}

1664

1665#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1668 O << Indent << "WIDEN-CAST ";

1674}

1675#endif

1676

1680}

1681

1682

1683

1684

1685

1689 assert(VF.isVector() && "only vector VFs are supported");

1690

1691

1692 auto *ValVTy = cast(Val->getType());

1693 ElementCount VLen = ValVTy->getElementCount();

1694

1697 "Induction Step must be an integer or FP");

1698 assert(Step->getType() == STy && "Step has wrong type");

1699

1701

1702

1705 Type *InitVecValSTy =

1708 }

1710

1714

1715

1716 Step = Builder.CreateMul(InitVec, Step);

1717 return Builder.CreateAdd(Val, Step, "induction");

1718 }

1719

1720

1721 assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&

1722 "Binary Opcode should be specified for FP induction");

1723 InitVec = Builder.CreateUIToFP(InitVec, ValVTy);

1724

1727 return Builder.CreateBinOp(BinOp, Val, MulOp, "induction");

1728}

1729

1730

1731

1734 : ConstantFP::get(Ty, C);

1735}

1736

1738 assert(!State.Lane && "Int or FP induction being replicated.");

1739

1745 "Types must match");

1747

1748

1749

1751

1752

1754 if (ID.getInductionBinOp() && isa(ID.getInductionBinOp()))

1755 Builder.setFastMathFlags(ID.getInductionBinOp()->getFastMathFlags());

1756

1757

1759

1760 assert((isa(EntryVal) || isa(EntryVal)) &&

1761 "Expected either an induction phi-node or a truncate of it!");

1762

1763

1764 auto CurrIP = Builder.saveIP();

1767 if (isa(EntryVal)) {

1768 assert(Start->getType()->isIntegerTy() &&

1769 "Truncation requires an integer type");

1770 auto *TruncType = cast(EntryVal->getType());

1771 Step = Builder.CreateTrunc(Step, TruncType);

1772 Start = Builder.CreateCast(Instruction::Trunc, Start, TruncType);

1773 }

1774

1776 Value *SteppedStart = getStepVector(SplatStart, Step, ID.getInductionOpcode(),

1778

1779

1780

1784 AddOp = Instruction::Add;

1785 MulOp = Instruction::Mul;

1786 } else {

1787 AddOp = ID.getInductionOpcode();

1788 MulOp = Instruction::FMul;

1789 }

1790

1793

1794

1795 SplatVF = State.get(SplatVFOperand);

1796 } else {

1797

1798

1802 RuntimeVF = Builder.CreateUIToFP(RuntimeVF, StepType);

1803 else

1806

1807

1809 }

1810

1812

1813

1814

1818 State.set(this, VecInd);

1819

1820 Instruction *LastInduction = cast(

1821 Builder.CreateBinOp(AddOp, VecInd, SplatVF, "vec.ind.next"));

1822 if (isa(EntryVal))

1823 State.addMetadata(LastInduction, EntryVal);

1825

1826 VecInd->addIncoming(SteppedStart, VectorPH);

1827

1828

1829

1830

1831

1832 VecInd->addIncoming(LastInduction, VectorPH);

1833}

1834

1835#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1838 O << Indent;

1840 O << " = WIDEN-INDUCTION ";

1842

1844 O << " (truncated to " << *TI->getType() << ")";

1845}

1846#endif

1847

1849

1850

1851

1853 return false;

1856 auto *CanIV = cast(&*getParent()->begin());

1857 return StartC && StartC->isZero() && StepC && StepC->isOne() &&

1859}

1860

1861#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1864 O << Indent;

1866 O << " = DERIVED-IV ";

1868 O << " + ";

1870 O << " * ";

1872}

1873#endif

1874

1876

1880

1881

1882

1883

1887

1888

1890 assert(BaseIVTy == Step->getType() && "Types of BaseIV and Step must match!");

1891

1892

1893

1897 AddOp = Instruction::Add;

1898 MulOp = Instruction::Mul;

1899 } else {

1900 AddOp = InductionOpcode;

1901 MulOp = Instruction::FMul;

1902 }

1903

1904

1905

1907

1908 Type *IntStepTy =

1910 Type *VecIVTy = nullptr;

1911 Value *UnitStepVec = nullptr, *SplatStep = nullptr, *SplatIV = nullptr;

1912 if (!FirstLaneOnly && State.VF.isScalable()) {

1914 UnitStepVec =

1918 }

1919

1920 unsigned StartLane = 0;

1922 if (State.Lane) {

1923 StartLane = State.Lane->getKnownLane();

1924 EndLane = StartLane + 1;

1925 }

1926 Value *StartIdx0 =

1928

1929 if (!FirstLaneOnly && State.VF.isScalable()) {

1931 auto *InitVec = Builder.CreateAdd(SplatStartIdx, UnitStepVec);

1933 InitVec = Builder.CreateSIToFP(InitVec, VecIVTy);

1934 auto *Mul = Builder.CreateBinOp(MulOp, InitVec, SplatStep);

1936 State.set(this, Add);

1937

1938

1939

1940 }

1941

1943 StartIdx0 = Builder.CreateSIToFP(StartIdx0, BaseIVTy);

1944

1945 for (unsigned Lane = StartLane; Lane < EndLane; ++Lane) {

1948

1949

1951 "Expected StartIdx to be folded to a constant when VF is not "

1952 "scalable");

1953 auto *Mul = Builder.CreateBinOp(MulOp, StartIdx, Step);

1956 }

1957}

1958

1959#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1962 O << Indent;

1964 O << " = SCALAR-STEPS ";

1966}

1967#endif

1968

1972

1973

1974

1975

1976

1977

1978 if (areAllOperandsInvariant()) {

1979

1980

1981

1982

1983

1984

1985

1986

1987

1988

1989

1990

1994

1995 auto *NewGEP = State.Builder.CreateGEP(GEP->getSourceElementType(), Ops[0],

1996 ArrayRef(Ops).drop_front(), "",

2001 } else {

2002

2003

2004

2005

2008

2009

2010

2014 if (isIndexLoopInvariant(I - 1))

2016 else

2018 }

2019

2020

2021

2024 assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) &&

2025 "NewGEP is not a pointer vector");

2026 State.set(this, NewGEP);

2028 }

2029}

2030

2031#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2034 O << Indent << "WIDEN-GEP ";

2035 O << (isPointerLoopInvariant() ? "Inv" : "Var");

2037 O << "[" << (isIndexLoopInvariant(I) ? "Inv" : "Var") << "]";

2038

2039 O << " ";

2041 O << " = getelementptr";

2044}

2045#endif

2046

2049

2050

2052 return IsScalable && (IsReverse || CurrentPart > 0)

2053 ? DL.getIndexType(Builder.getPtrTy(0))

2055}

2056

2058 auto &Builder = State.Builder;

2062 CurrentPart, Builder);

2063

2064

2066 if (IndexTy != RunTimeVF->getType())

2068

2070 ConstantInt::get(IndexTy, -(int64_t)CurrentPart), RunTimeVF);

2071

2072 Value *LastLane = Builder.CreateSub(ConstantInt::get(IndexTy, 1), RunTimeVF);

2074 Value *ResultPtr =

2076 ResultPtr = Builder.CreateGEP(IndexedTy, ResultPtr, LastLane, "",

2078

2079 State.set(this, ResultPtr, true);

2080}

2081

2082#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2085 O << Indent;

2087 O << " = reverse-vector-pointer";

2090}

2091#endif

2092

2094 auto &Builder = State.Builder;

2098 CurrentPart, Builder);

2100

2102 Value *ResultPtr =

2104

2105 State.set(this, ResultPtr, true);

2106}

2107

2108#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2111 O << Indent;

2113 O << " = vector-pointer ";

2114

2116}

2117#endif

2118

2122

2123

2124

2125

2126

2127

2128

2130

2131

2132

2133

2134

2135

2136

2137

2139 Value *Result = nullptr;

2140 for (unsigned In = 0; In < NumIncoming; ++In) {

2141

2142

2144 if (In == 0)

2145 Result = In0;

2146 else {

2147

2148

2151 }

2152 }

2153 State.set(this, Result, OnlyFirstLaneUsed);

2154}

2155

2158

2159

2162

2168}

2169

2170#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2173 O << Indent << "BLEND ";

2175 O << " =";

2177

2178

2179 O << " ";

2181 } else {

2183 O << " ";

2185 if (I == 0)

2186 continue;

2187 O << "/";

2189 }

2190 }

2191}

2192#endif

2193

2195 assert(!State.Lane && "Reduction being replicated.");

2198

2207

2211 else

2216

2219 }

2221 Value *NextInChain;

2222 if (IsOrdered) {

2224 NewRed =

2226 else

2229 PrevInChain = NewRed;

2230 NextInChain = NewRed;

2231 } else {

2232 PrevInChain = State.get(getChainOp(), true);

2236 NewRed, PrevInChain);

2237 else

2240 }

2241 State.set(this, NextInChain, true);

2242}

2243

2245 assert(!State.Lane && "Reduction being replicated.");

2246

2247 auto &Builder = State.Builder;

2248

2252

2257

2259 VBuilder.setEVL(EVL);

2261

2263 Mask = State.get(CondOp);

2264 else

2267

2271 } else {

2274 NewRed = createMinMaxOp(Builder, Kind, NewRed, Prev);

2275 else

2277 NewRed, Prev);

2278 }

2279 State.set(this, NewRed, true);

2280}

2281

2286 auto *VectorTy = cast(toVectorTy(ElementTy, VF));

2287 unsigned Opcode = RdxDesc.getOpcode();

2288

2289

2293 "Any-of reduction not implemented in VPlan-based cost model currently.");

2295 (!cast(getOperand(0))->isInLoop() ||

2297 "In-loop reduction not implemented in VPlan-based cost model currently.");

2298

2300 "Inferred type and recurrence type mismatch.");

2301

2302

2309 }

2310

2313}

2314

2315#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2318 O << Indent << "REDUCE ";

2320 O << " = ";

2322 O << " +";

2328 O << ", ";

2330 }

2331 O << ")";

2333 O << " (with final reduction value stored in invariant address sank "

2334 "outside of loop)";

2335}

2336

2340 O << Indent << "REDUCE ";

2342 O << " = ";

2344 O << " +";

2349 O << ", ";

2352 O << ", ";

2354 }

2355 O << ")";

2357 O << " (with final reduction value stored in invariant address sank "

2358 "outside of loop)";

2359}

2360#endif

2361

2363

2364

2366 if (auto *PredR = dyn_cast(U))

2367 return any_of(PredR->users(), [PredR](const VPUser *U) {

2368 return !U->usesScalars(PredR);

2369 });

2370 return false;

2371 });

2372}

2373

2377

2378

2381}

2382

2383#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2386 O << Indent << (IsUniform ? "CLONE " : "REPLICATE ");

2387

2390 O << " = ";

2391 }

2393 O << "call";

2395 O << "@" << CB->getCalledFunction()->getName() << "(";

2399 });

2400 O << ")";

2401 } else {

2405 }

2406

2408 O << " (S->V)";

2409}

2410#endif

2411

2415 "Codegen only implemented for first lane.");

2416 switch (Opcode) {

2417 case Instruction::SExt:

2418 case Instruction::ZExt:

2419 case Instruction::Trunc: {

2420

2423 }

2424 default:

2426 }

2427}

2428

2430 State.set(this, generate(State), VPLane(0));

2431}

2432

2433#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2436 O << Indent << "SCALAR-CAST ";

2440 O << " to " << *ResultTy;

2441}

2442#endif

2443

2445 assert(State.Lane && "Branch on Mask works only on single instance.");

2446

2447 unsigned Lane = State.Lane->getKnownLane();

2448

2449 Value *ConditionBit = nullptr;

2451 if (BlockInMask) {

2452 ConditionBit = State.get(BlockInMask);

2456 } else

2458

2459

2460

2462 assert(isa(CurrentTerminator) &&

2463 "Expected to replace unreachable terminator with conditional branch.");

2467}

2468

2471

2472

2473

2474 return 0;

2475}

2476

2479 assert(State.Lane && "Predicated instruction PHI works per instance.");

2484 assert(PredicatingBB && "Predicated block has no single predecessor.");

2486 "operand must be VPReplicateRecipe");

2487

2488

2489

2490

2491

2492

2493

2499 VPhi->addIncoming(IEI, PredicatedBB);

2501 State.reset(this, VPhi);

2502 else

2503 State.set(this, VPhi);

2504

2505

2507 } else {

2509 return;

2510

2514 PredicatingBB);

2515 Phi->addIncoming(ScalarPredInst, PredicatedBB);

2517 State.reset(this, Phi, *State.Lane);

2518 else

2519 State.set(this, Phi, *State.Lane);

2520

2521

2523 }

2524}

2525

2526#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2529 O << Indent << "PHI-PREDICATED-INSTRUCTION ";

2531 O << " = ";

2533}

2534#endif

2535

2539 const Align Alignment =

2541 unsigned AS =

2543

2545

2546

2547

2550 "Inconsecutive memory access should not have the order.");

2555 }

2556

2561 } else {

2566 }

2568 return Cost;

2569

2570 return Cost +=

2572 cast(Ty), {}, Ctx.CostKind, 0);

2573}

2574

2576 auto *LI = cast(&Ingredient);

2577

2582

2583 auto &Builder = State.Builder;

2585 Value *Mask = nullptr;

2586 if (auto *VPMask = getMask()) {

2587

2588

2589 Mask = State.get(VPMask);

2592 }

2593

2596 if (CreateGather) {

2598 "wide.masked.gather");

2599 } else if (Mask) {

2600 NewLI =

2603 } else {

2605 }

2606

2610 State.set(this, NewLI);

2611}

2612

2613#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2616 O << Indent << "WIDEN ";

2618 O << " = load ";

2620}

2621#endif

2622

2623

2624

2628 Value *AllTrueMask =

2630 return Builder.CreateIntrinsic(ValTy, Intrinsic::experimental_vp_reverse,

2631 {Operand, AllTrueMask, EVL}, nullptr, Name);

2632}

2633

2635 auto *LI = cast(&Ingredient);

2636

2641

2642 auto &Builder = State.Builder;

2647 Value *Mask = nullptr;

2649 Mask = State.get(VPMask);

2651 Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");

2652 } else {

2654 }

2655

2656 if (CreateGather) {

2657 NewLI =

2658 Builder.CreateIntrinsic(DataTy, Intrinsic::vp_gather, {Addr, Mask, EVL},

2659 nullptr, "wide.masked.gather");

2660 } else {

2664 Instruction::Load, DataTy, Addr, "vp.op.load"));

2665 }

2672 State.set(this, Res);

2673}

2674

2679

2680

2681

2682

2683

2684

2686 const Align Alignment =

2688 unsigned AS =

2693 return Cost;

2694

2696 cast(Ty), {}, Ctx.CostKind,

2697 0);

2698}

2699

2700#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2703 O << Indent << "WIDEN ";

2705 O << " = vp.load ";

2707}

2708#endif

2709

2711 auto *SI = cast(&Ingredient);

2712

2716

2717 auto &Builder = State.Builder;

2719

2720 Value *Mask = nullptr;

2721 if (auto *VPMask = getMask()) {

2722

2723

2724 Mask = State.get(VPMask);

2727 }

2728

2729 Value *StoredVal = State.get(StoredVPValue);

2731

2732

2734

2735

2736 }

2739 if (CreateScatter)

2741 else if (Mask)

2743 else

2746}

2747

2748#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2751 O << Indent << "WIDEN store ";

2753}

2754#endif

2755

2757 auto *SI = cast(&Ingredient);

2758

2762

2763 auto &Builder = State.Builder;

2765

2767 Value *StoredVal = State.get(StoredValue);

2770 StoredVal = createReverseEVL(Builder, StoredVal, EVL, "vp.reverse");

2771 Value *Mask = nullptr;

2773 Mask = State.get(VPMask);

2775 Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");

2776 } else {

2778 }

2780 if (CreateScatter) {

2782 Intrinsic::vp_scatter,

2783 {StoredVal, Addr, Mask, EVL});

2784 } else {

2789 {StoredVal, Addr}));

2790 }

2794}

2795

2800

2801

2802

2803

2804

2805

2807 const Align Alignment =

2809 unsigned AS =

2814 return Cost;

2815

2817 cast(Ty), {}, Ctx.CostKind,

2818 0);

2819}

2820

2821#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2824 O << Indent << "WIDEN vp.store ";

2826}

2827#endif

2828

2831

2832 auto VF = DstVTy->getElementCount();

2833 auto *SrcVecTy = cast(V->getType());

2834 assert(VF == SrcVecTy->getElementCount() && "Vector dimensions do not match");

2835 Type *SrcElemTy = SrcVecTy->getElementType();

2836 Type *DstElemTy = DstVTy->getElementType();

2837 assert((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) &&

2838 "Vector elements must have same size");

2839

2840

2843 }

2844

2845

2846

2847

2849 "Only one type should be a pointer type");

2851 "Only one type should be a floating point type");

2852 Type *IntTy =

2857}

2858

2859

2860

2863 unsigned Factor = Vals.size();

2864 assert(Factor > 1 && "Tried to interleave invalid number of vectors");

2865

2867#ifndef NDEBUG

2868 for (Value *Val : Vals)

2869 assert(Val->getType() == VecTy && "Tried to interleave mismatched types");

2870#endif

2871

2872

2873

2874 if (VecTy->isScalableTy()) {

2876 return Builder.CreateIntrinsic(WideVecTy, Intrinsic::vector_interleave2,

2877 Vals,

2878 nullptr, Name);

2879 }

2880

2881

2883

2884

2885 const unsigned NumElts = VecTy->getElementCount().getFixedValue();

2888}

2889

2890

2891

2892

2893

2894

2895

2896

2897

2898

2899

2900

2901

2902

2903

2904

2905

2906

2907

2908

2909

2910

2911

2912

2913

2914

2915

2916

2917

2919 assert(!State.Lane && "Interleave group being replicated.");

2922

2923

2925 unsigned InterleaveFactor = Group->getFactor();

2926 auto *VecTy = VectorType::get(ScalarTy, State.VF * InterleaveFactor);

2927

2928

2931 "Reversed masked interleave-group not supported.");

2932

2935 if (auto *I = dyn_cast(ResAddr))

2937

2938

2939

2940

2941

2943 Value *RuntimeVF =

2950

2951 bool InBounds = false;

2952 if (auto *Gep = dyn_cast(ResAddr->stripPointerCasts()))

2953 InBounds = Gep->isInBounds();

2954 ResAddr = State.Builder.CreateGEP(ScalarTy, ResAddr, Index, "", InBounds);

2955 }

2956

2959

2960 auto CreateGroupMask = [&BlockInMask, &State,

2961 &InterleaveFactor](Value *MaskForGaps) -> Value * {

2963 assert(!MaskForGaps && "Interleaved groups with gaps are not supported.");

2964 assert(InterleaveFactor == 2 &&

2965 "Unsupported deinterleave factor for scalable vectors");

2966 auto *ResBlockInMask = State.get(BlockInMask);

2971 MaskTy, Intrinsic::vector_interleave2, Ops,

2972 nullptr, "interleaved.mask");

2973 }

2974

2975 if (!BlockInMask)

2976 return MaskForGaps;

2977

2978 Value *ResBlockInMask = State.get(BlockInMask);

2980 ResBlockInMask,

2982 "interleaved.mask");

2984 ShuffledMask, MaskForGaps)

2985 : ShuffledMask;

2986 };

2987

2988 const DataLayout &DL = Instr->getDataLayout();

2989

2990 if (isa(Instr)) {

2991 Value *MaskForGaps = nullptr;

2992 if (NeedsMaskForGaps) {

2995 assert(MaskForGaps && "Mask for Gaps is required but it is null");

2996 }

2997

2999 if (BlockInMask || MaskForGaps) {

3000 Value *GroupMask = CreateGroupMask(MaskForGaps);

3002 Group->getAlign(), GroupMask,

3003 PoisonVec, "wide.masked.vec");

3004 } else

3006 Group->getAlign(), "wide.vec");

3008

3011 if (VecTy->isScalableTy()) {

3012 assert(InterleaveFactor == 2 &&

3013 "Unsupported deinterleave factor for scalable vectors");

3014

3015

3016

3018 Intrinsic::vector_deinterleave2, VecTy, NewLoad,

3019 nullptr, "strided.vec");

3020 unsigned J = 0;

3021 for (unsigned I = 0; I < InterleaveFactor; ++I) {

3023

3024 if (!Member)

3025 continue;

3026

3028

3029 if (Member->getType() != ScalarTy) {

3031 StridedVec =

3033 }

3034

3037

3038 State.set(VPDefs[J], StridedVec);

3039 ++J;

3040 }

3041

3042 return;

3043 }

3044

3045

3046

3047 unsigned J = 0;

3048 for (unsigned I = 0; I < InterleaveFactor; ++I) {

3050

3051

3052 if (!Member)

3053 continue;

3054

3055 auto StrideMask =

3057 Value *StridedVec =

3059

3060

3061 if (Member->getType() != ScalarTy) {

3062 assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");

3064 StridedVec =

3066 }

3067

3070

3071 State.set(VPDefs[J], StridedVec);

3072 ++J;

3073 }

3074 return;

3075 }

3076

3077

3079

3080

3081 Value *MaskForGaps =

3084 "masking gaps for scalable vectors is not yet supported.");

3086

3088 unsigned StoredIdx = 0;

3089 for (unsigned i = 0; i < InterleaveFactor; i++) {

3091 "Fail to get a member from an interleaved store group");

3093

3094

3095 if (!Member) {

3098 continue;

3099 }

3100

3101 Value *StoredVec = State.get(StoredValues[StoredIdx]);

3102 ++StoredIdx;

3103

3106

3107

3108

3109 if (StoredVec->getType() != SubVT)

3111

3112 StoredVecs.push_back(StoredVec);

3113 }

3114

3115

3118 if (BlockInMask || MaskForGaps) {

3119 Value *GroupMask = CreateGroupMask(MaskForGaps);

3121 IVec, ResAddr, Group->getAlign(), GroupMask);

3122 } else

3123 NewStoreInstr =

3125

3127}

3128

3129#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3132 O << Indent << "INTERLEAVE-GROUP with factor " << IG->getFactor() << " at ";

3133 IG->getInsertPos()->printAsOperand(O, false);

3134 O << ", ";

3137 if (Mask) {

3138 O << ", ";

3140 }

3141

3142 unsigned OpIdx = 0;

3143 for (unsigned i = 0; i < IG->getFactor(); ++i) {

3144 if (!IG->getMember(i))

3145 continue;

3147 O << "\n" << Indent << " store ";

3149 O << " to index " << i;

3150 } else {

3151 O << "\n" << Indent << " ";

3153 O << " = load from index " << i;

3154 }

3155 ++OpIdx;

3156 }

3157}

3158#endif

3159

3163

3164 unsigned InsertPosIdx = 0;

3165 for (unsigned Idx = 0; IG->getFactor(); ++Idx)

3166 if (auto *Member = IG->getMember(Idx)) {

3167 if (Member == InsertPos)

3168 break;

3169 InsertPosIdx++;

3170 }

3174 auto *VectorTy = cast(toVectorTy(ValTy, VF));

3176

3177 unsigned InterleaveFactor = IG->getFactor();

3178 auto *WideVecTy = VectorType::get(ValTy, VF * InterleaveFactor);

3179

3180

3182 for (unsigned IF = 0; IF < InterleaveFactor; IF++)

3183 if (IG->getMember(IF))

3185

3186

3188 InsertPos->getOpcode(), WideVecTy, IG->getFactor(), Indices,

3189 IG->getAlign(), AS, Ctx.CostKind, getMask(), NeedsMaskForGaps);

3190

3191 if (!IG->isReverse())

3192 return Cost;

3193

3194 return Cost + IG->getNumMembers() *

3196 VectorTy, std::nullopt, Ctx.CostKind,

3197 0);

3198}

3199

3200#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3203 O << Indent << "EMIT ";

3205 O << " = CANONICAL-INDUCTION ";

3207}

3208#endif

3209

3211 return IsScalarAfterVectorization &&

3213}

3214

3218 "Not a pointer induction according to InductionDescriptor!");

3220 "Unexpected type.");

3222 "Recipe should have been replaced");

3223

3225

3226

3228 Type *ScStValueType = ScalarStartValue->getType();

3229

3231 PHINode *NewPointerPhi = nullptr;

3232 if (CurrentPart == 0) {

3233 auto *IVR = cast(&getParent()

3234 ->getPlan()

3235 ->getVectorLoopRegion()

3236 ->getEntryBasicBlock()

3237 ->front());

3238 PHINode *CanonicalIV = cast(State.get(IVR, true));

3239 NewPointerPhi = PHINode::Create(ScStValueType, 2, "pointer.phi",

3241 NewPointerPhi->addIncoming(ScalarStartValue, VectorPH);

3243 } else {

3244

3245

3246 auto *GEP =

3248 NewPointerPhi = cast(GEP->getPointerOperand());

3249 }

3250

3251

3256

3257

3258

3259

3260

3261 if (CurrentPart == 0) {

3262

3263

3265 Value *NumUnrolledElems =

3266 State.Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, UF));

3267

3270 State.Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind",

3271 InductionLoc);

3272

3273 NewPointerPhi->addIncoming(InductionGEP, VectorPH);

3274 }

3275

3276

3277

3280 RuntimeVF, ConstantInt::get(PhiType, CurrentPart));

3281 Value *StartOffset =

3283

3286

3288 "scalar step must be the same across all parts");

3292 State.VF, ScalarStepValue)),

3293 "vector.gep");

3294 State.set(this, GEP);

3295}

3296

3297#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3301 "unexpected number of operands");

3302 O << Indent << "EMIT ";

3304 O << " = WIDEN-POINTER-INDUCTION ";

3306 O << ", ";

3309 O << ", ";

3311 O << ", ";

3313 }

3314}

3315#endif

3316

3318 assert(!State.Lane && "cannot be used in per-lane");

3320

3321

3322

3323

3326 "Results must match");

3327 return;

3328 }

3329

3332

3333 Value *Res = Exp.expandCodeFor(Expr, Expr->getType(),

3336 State.set(this, Res, VPLane(0));

3337}

3338

3339#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3342 O << Indent << "EMIT ";

3344 O << " = EXPAND SCEV " << *Expr;

3345}

3346#endif

3347

3354 ? CanonicalIV

3359 VStep =

3361 }

3362 Value *CanonicalVectorIV = Builder.CreateAdd(VStart, VStep, "vec.iv");

3363 State.set(this, CanonicalVectorIV);

3364}

3365

3366#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3369 O << Indent << "EMIT ";

3371 O << " = WIDEN-CANONICAL-INDUCTION ";

3373}

3374#endif

3375

3377 auto &Builder = State.Builder;

3378

3380

3382 ? VectorInit->getType()

3384

3388 auto *One = ConstantInt::get(IdxTy, 1);

3391 auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, State.VF);

3392 auto *LastIdx = Builder.CreateSub(RuntimeVF, One);

3394 PoisonValue::get(VecTy), VectorInit, LastIdx, "vector.recur.init");

3395 }

3396

3397

3400 Phi->addIncoming(VectorInit, VectorPH);

3401 State.set(this, Phi);

3402}

3403

3409

3412

3414 std::iota(Mask.begin(), Mask.end(), VF.getKnownMinValue() - 1);

3415 Type *VectorTy =

3417

3419 cast(VectorTy), Mask, Ctx.CostKind,

3421}

3422

3423#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3426 O << Indent << "FIRST-ORDER-RECURRENCE-PHI ";

3428 O << " = phi ";

3430}

3431#endif

3432

3434 auto &Builder = State.Builder;

3435

3436

3437

3439

3440

3441

3444

3445

3446

3447

3448

3449 bool ScalarPHI = State.VF.isScalar() || IsInLoop;

3450 Type *VecTy =

3452

3455 "recipe must be in the vector loop header");

3458 State.set(this, Phi, IsInLoop);

3459

3461

3462 Value *Iden = nullptr;

3465

3468

3469 if (ScalarPHI) {

3470 Iden = StartV;

3471 } else {

3474 StartV = Iden = State.get(StartVPV);

3475 }

3477

3478

3479

3480

3481

3482

3483

3484

3485

3486 Iden = StartV;

3487 if (!ScalarPHI) {

3491 }

3492 } else {

3495

3496 if (!ScalarPHI) {

3497 if (CurrentPart == 0) {

3498

3499

3500

3506 } else {

3508 }

3509 }

3510 }

3511

3512 Phi = cast(State.get(this, IsInLoop));

3513 Value *StartVal = (CurrentPart == 0) ? StartV : Iden;

3514 Phi->addIncoming(StartVal, VectorPH);

3515}

3516

3517#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3520 O << Indent << "WIDEN-REDUCTION-PHI ";

3521

3523 O << " = phi ";

3525 if (VFScaleFactor != 1)

3526 O << " (VF scaled by 1/" << VFScaleFactor << ")";

3527}

3528#endif

3529

3532 "Non-native vplans are not expected to have VPWidenPHIRecipes.");

3533

3538 State.set(this, VecPhi);

3539}

3540

3541#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3544 O << Indent << "WIDEN-PHI ";

3545

3547

3548

3549

3550

3551 if (getNumOperands() != OriginalPhi->getNumOperands()) {

3553 return;

3554 }

3555

3557 O << " = phi ";

3559}

3560#endif

3561

3562

3563

3569 Phi->addIncoming(StartMask, VectorPH);

3571 State.set(this, Phi);

3572}

3573

3574#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3577 O << Indent << "ACTIVE-LANE-MASK-PHI ";

3578

3580 O << " = phi ";

3582}

3583#endif

3584

3585#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3588 O << Indent << "EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ";

3589

3591 O << " = phi ";

3593}

3594#endif

3595

3600 Phi->addIncoming(Start, VectorPH);

3602 State.set(this, Phi, true);

3603}

3604

3605#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3608 O << Indent << "SCALAR-PHI ";

3610 O << " = phi ";

3612}

3613#endif

AMDGPU Lower Kernel Arguments

AMDGPU Register Bank Select

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx

This file provides a LoopVectorizationPlanner class.

cl::opt< unsigned > ForceTargetInstructionCost("force-target-instruction-cost", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's expected cost for " "an instruction to a single constant value. Mostly " "useful for getting consistent testing."))

mir Rename Register Operands

static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)

Return the first found DebugLoc that has a DILocation, given a range of instructions.

const SmallVectorImpl< MachineOperand > & Cond

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

This file defines the SmallVector class.

static SymbolRef::Type getType(const Symbol *Sym)

static Instruction * createReverseEVL(IRBuilderBase &Builder, Value *Operand, Value *EVL, const Twine &Name)

Use all-true mask for reverse rather than actual mask, as it avoids a dependence w/o affecting the re...

static Value * interleaveVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vals, const Twine &Name)

Return a vector containing interleaved elements from multiple smaller input vectors.

static Value * createBitOrPointerCast(IRBuilderBase &Builder, Value *V, VectorType *DstVTy, const DataLayout &DL)

cl::opt< unsigned > ForceTargetInstructionCost

static Value * getStepVector(Value *Val, Value *Step, Instruction::BinaryOps BinOp, ElementCount VF, IRBuilderBase &Builder)

This function adds (0 * Step, 1 * Step, 2 * Step, ...) to each vector element of Val.

static Type * getGEPIndexTy(bool IsScalable, bool IsReverse, unsigned CurrentPart, IRBuilderBase &Builder)

static Constant * getSignedIntOrFpConstant(Type *Ty, int64_t C)

A helper function that returns an integer or floating-point constant with value C.

This file contains the declarations of the Vectorization Plan base classes:

static const uint32_t IV[8]

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

static Attribute getWithAlignment(LLVMContext &Context, Align Alignment)

Return a uniquified Attribute object that has the specific alignment set.

LLVM Basic Block Representation.

const_iterator getFirstInsertionPt() const

Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...

InstListType::const_iterator getFirstNonPHIIt() const

Iterator returning form of getFirstNonPHI.

const BasicBlock * getSinglePredecessor() const

Return the predecessor of this block if it has a single predecessor block.

const DataLayout & getDataLayout() const

Get the data layout of the module this basic block belongs to.

InstListType::iterator iterator

Instruction iterators...

const Instruction * getTerminator() const LLVM_READONLY

Returns the terminator instruction if the block is well formed or null if the block is not well forme...

const Module * getModule() const

Return the module owning the function this basic block belongs to, or nullptr if the function does no...

Conditional or Unconditional Branch instruction.

static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)

void setSuccessor(unsigned idx, BasicBlock *NewSucc)

void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)

Adds the attribute to the indicated argument.

This class represents a function call, abstracting a target machine's calling convention.

static bool isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy, const DataLayout &DL)

Check whether a bitcast, inttoptr, or ptrtoint cast between these types is valid and a no-op.

Predicate

This enumeration lists the possible predicates for CmpInst subclasses.

@ ICMP_UGT

unsigned greater than

@ ICMP_ULT

unsigned less than

static StringRef getPredicateName(Predicate P)

This is the shared class of boolean and integer constants.

static ConstantInt * getSigned(IntegerType *Ty, int64_t V)

Return a ConstantInt with the specified value for the specified type.

uint64_t getZExtValue() const

Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...

This is an important base class in LLVM.

This class represents an Operation in the Expression.

A parsed version of the target data layout string in and methods for querying it.

constexpr bool isVector() const

One or more elements.

constexpr bool isScalar() const

Exactly one element.

Convenience struct for specifying and reasoning about fast-math flags.

void setAllowContract(bool B=true)

bool noSignedZeros() const

void setAllowReciprocal(bool B=true)

bool allowReciprocal() const

void print(raw_ostream &O) const

Print fast-math flags to O.

void setNoSignedZeros(bool B=true)

bool allowReassoc() const

Flag queries.

void setNoNaNs(bool B=true)

void setAllowReassoc(bool B=true)

Flag setters.

void setApproxFunc(bool B=true)

void setNoInfs(bool B=true)

bool allowContract() const

Class to represent function types.

Type * getParamType(unsigned i) const

Parameter type accessors.

ArrayRef< Type * > params() const

FunctionType * getFunctionType() const

Returns the FunctionType for me.

bool willReturn() const

Determine if the function will return.

bool doesNotThrow() const

Determine if the function cannot unwind.

Type * getReturnType() const

Returns the type of the ret val.

bool hasNoUnsignedSignedWrap() const

bool hasNoUnsignedWrap() const

static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)

Common base class shared among various IRBuilders.

ConstantInt * getInt1(bool V)

Get a constant value representing either true or false.

Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")

IntegerType * getInt1Ty()

Fetch the type representing a single bit.

Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")

Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")

LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)

Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")

Create a ZExt or Trunc from the integer value V to DestTy.

Value * CreateVectorSplice(Value *V1, Value *V2, int64_t Imm, const Twine &Name="")

Return a vector splice intrinsic if using scalable vectors, otherwise return a shufflevector.

Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")

Return a vector value that contains.

Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")

ConstantInt * getTrue()

Get the constant value for i1 true.

CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")

Create a call to Masked Load intrinsic.

Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)

BasicBlock::iterator GetInsertPoint() const

Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")

Value * CreateFreeze(Value *V, const Twine &Name="")

IntegerType * getInt32Ty()

Fetch the type representing a 32-bit integer.

Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())

Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr, FMFSource FMFSource={})

Value * CreateUIToFP(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)

BasicBlock * GetInsertBlock() const

void setFastMathFlags(FastMathFlags NewFMF)

Set the fast-math flags to be used with generated fp-math operators.

Value * CreateVectorReverse(Value *V, const Twine &Name="")

Return a vector value that contains the vector V reversed.

Value * CreateFCmpFMF(CmpInst::Predicate P, Value *LHS, Value *RHS, FMFSource FMFSource, const Twine &Name="", MDNode *FPMathTag=nullptr)

Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())

Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)

CallInst * CreateOrReduce(Value *Src)

Create a vector int OR reduction intrinsic of the source vector.

InsertPoint saveIP() const

Returns the current insert point.

CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")

Create a call to intrinsic ID with Args, mangled using Types.

ConstantInt * getInt32(uint32_t C)

Get a constant 32-bit value.

Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")

Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)

PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")

Value * CreateNot(Value *V, const Twine &Name="")

Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)

Create a conditional 'br Cond, TrueDest, FalseDest' instruction.

Value * CreateNAryOp(unsigned Opc, ArrayRef< Value * > Ops, const Twine &Name="", MDNode *FPMathTag=nullptr)

Create either a UnaryOperator or BinaryOperator depending on Opc.

Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)

Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")

LLVMContext & getContext() const

CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)

Create a call to Masked Store intrinsic.

Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)

Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)

PointerType * getPtrTy(unsigned AddrSpace=0)

Fetch the type representing a pointer.

Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)

Value * CreateLogicalAnd(Value *Cond1, Value *Cond2, const Twine &Name="")

void restoreIP(InsertPoint IP)

Sets the current insert point to a previously-saved location.

void SetInsertPoint(BasicBlock *TheBB)

This specifies that created instructions should be appended to the end of the specified block.

StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)

Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)

IntegerType * getInt8Ty()

Fetch the type representing an 8-bit integer.

Value * CreateStepVector(Type *DstType, const Twine &Name="")

Creates a vector of type DstType with the linear sequence <0, 1, ...>

Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

CallInst * CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment, Value *Mask=nullptr)

Create a call to Masked Scatter intrinsic.

CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")

Create a call to Masked Gather intrinsic.

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

A struct for saving information about induction variables.

@ IK_PtrInduction

Pointer induction var. Step = C.

This instruction inserts a single (scalar) element into a VectorType value.

VectorType * getType() const

Overload to return most specific vector type.

static InstructionCost getInvalid(CostType Val=0)

void insertBefore(Instruction *InsertPos)

Insert an unlinked instruction into a basic block immediately before the specified instruction.

InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

FastMathFlags getFastMathFlags() const LLVM_READONLY

Convenience function for getting all the fast-math flags, which must be an operator which supports th...

const char * getOpcodeName() const

unsigned getOpcode() const

Returns a member of one of the enums like Instruction::Add.

void setDebugLoc(DebugLoc Loc)

Set the debug location information for this instruction.

static IntegerType * get(LLVMContext &C, unsigned NumBits)

This static method is the primary way of constructing an IntegerType.

The group of interleaved loads/stores sharing the same stride and close to each other.

uint32_t getFactor() const

InstTy * getMember(uint32_t Index) const

Get the member with the given index Index.

InstTy * getInsertPos() const

void addMetadata(InstTy *NewInst) const

Add metadata (e.g.

This is an important class for using LLVM in a threaded context.

BlockT * getHeader() const

void print(raw_ostream &OS, const SlotIndexes *=nullptr, bool IsStandalone=true) const

A Module instance is used to store all the information related to an LLVM module.

void addIncoming(Value *V, BasicBlock *BB)

Add an incoming value to the end of the PHI list.

static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)

Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...

static PoisonValue * get(Type *T)

Static factory methods - Return an 'poison' object of the specified type.

The RecurrenceDescriptor is used to identify recurrences variables in a loop.

FastMathFlags getFastMathFlags() const

static unsigned getOpcode(RecurKind Kind)

Returns the opcode corresponding to the RecurrenceKind.

Type * getRecurrenceType() const

Returns the type of the recurrence.

TrackingVH< Value > getRecurrenceStartValue() const

static bool isAnyOfRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...

static bool isFindLastIVRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...

bool isSigned() const

Returns true if all source operands of the recurrence are SExtInsts.

RecurKind getRecurrenceKind() const

StoreInst * IntermediateStore

Reductions may store temporary or final result to an invariant address.

static bool isMinMaxRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is any min/max kind.

This class uses information about analyze scalars to rewrite expressions in canonical form.

Type * getType() const

Return the LLVM type of this SCEV expression.

This class represents the LLVM 'select' instruction.

This class provides computation of slot numbers for LLVM Assembly writing.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, OperandValueInfo Op1Info={OK_AnyValue, OP_None}, OperandValueInfo Op2Info={OK_AnyValue, OP_None}, const Instruction *I=nullptr) const

InstructionCost getAddressComputationCost(Type *Ty, ScalarEvolution *SE=nullptr, const SCEV *Ptr=nullptr) const

InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, OperandValueInfo OpdInfo={OK_AnyValue, OP_None}, const Instruction *I=nullptr) const

InstructionCost getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef< unsigned > Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, bool UseMaskForCond=false, bool UseMaskForGaps=false) const

InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind) const

InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty, std::optional< FastMathFlags > FMF, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput) const

Calculate the cost of vector reduction intrinsics.

InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH, TTI::TargetCostKind CostKind=TTI::TCK_SizeAndLatency, const Instruction *I=nullptr) const

static OperandValueInfo getOperandInfo(const Value *V)

Collect properties of V used in cost analysis, e.g. OP_PowerOf2.

InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF=FastMathFlags(), TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput) const

InstructionCost getArithmeticInstrCost(unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, TTI::OperandValueInfo Opd1Info={TTI::OK_AnyValue, TTI::OP_None}, TTI::OperandValueInfo Opd2Info={TTI::OK_AnyValue, TTI::OP_None}, ArrayRef< const Value * > Args={}, const Instruction *CxtI=nullptr, const TargetLibraryInfo *TLibInfo=nullptr) const

This is an approximation of reciprocal throughput of a math/logic op.

InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput) const

InstructionCost getShuffleCost(ShuffleKind Kind, VectorType *Tp, ArrayRef< int > Mask={}, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, int Index=0, VectorType *SubTp=nullptr, ArrayRef< const Value * > Args={}, const Instruction *CxtI=nullptr) const

InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, Align Alignment, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, const Instruction *I=nullptr) const

@ TCC_Free

Expected to fold away in lowering.

InstructionCost getPartialReductionCost(unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType, ElementCount VF, PartialReductionExtendKind OpAExtend, PartialReductionExtendKind OpBExtend, std::optional< unsigned > BinOp=std::nullopt) const

@ SK_Splice

Concatenates elements from the first input vector with elements of the second input vector.

@ SK_Reverse

Reverse the order of the vector.

InstructionCost getCallInstrCost(Function *F, Type *RetTy, ArrayRef< Type * > Tys, TTI::TargetCostKind CostKind=TTI::TCK_SizeAndLatency) const

InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind=TTI::TCK_SizeAndLatency, const Instruction *I=nullptr) const

CastContextHint

Represents a hint about the context in which a cast is used.

@ Reversed

The cast is used with a reversed load/store.

@ Masked

The cast is used with a masked load/store.

@ None

The cast is not used with a load/store of any kind.

@ Normal

The cast is used with a normal load/store.

@ Interleave

The cast is used with an interleaved load/store.

@ GatherScatter

The cast is used with a gather/scatter.

This class represents a truncation of integer types.

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

The instances of the Type class are immutable: once they are created, they are never changed.

bool isVectorTy() const

True if this is an instance of VectorType.

bool isPointerTy() const

True if this is an instance of PointerType.

static IntegerType * getInt1Ty(LLVMContext &C)

static IntegerType * getIntNTy(LLVMContext &C, unsigned N)

unsigned getScalarSizeInBits() const LLVM_READONLY

If this is a vector type, return the getPrimitiveSizeInBits value for the element type.

static Type * getVoidTy(LLVMContext &C)

LLVMContext & getContext() const

Return the LLVMContext in which this type was uniqued.

bool isFloatingPointTy() const

Return true if this is one of the floating-point types.

bool isIntegerTy() const

True if this is an instance of IntegerType.

TypeID getTypeID() const

Return the type id for the type.

bool isVoidTy() const

Return true if this is 'void'.

Type * getScalarType() const

If this is a vector type, return the element type, otherwise return 'this'.

value_op_iterator value_op_end()

Value * getOperand(unsigned i) const

value_op_iterator value_op_begin()

void execute(VPTransformState &State) override

Generate the active lane mask phi of the vector loop.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.

RecipeListTy & getRecipeList()

Returns a reference to the list of recipes.

void insert(VPRecipeBase *Recipe, iterator InsertPt)

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenMemoryRecipe.

VPValue * getIncomingValue(unsigned Idx) const

Return incoming value number Idx.

VPValue * getMask(unsigned Idx) const

Return mask number Idx.

unsigned getNumIncomingValues() const

Return the number of incoming values, taking into account when normalized the first incoming value wi...

void execute(VPTransformState &State) override

Generate the phi/select nodes.

bool isNormalized() const

A normalized blend is one that has an odd number of operands, whereby the first operand does not have...

VPBlockBase is the building block of the Hierarchical Control-Flow Graph.

VPRegionBlock * getParent()

const VPBasicBlock * getExitingBasicBlock() const

const VPBlocksTy & getPredecessors() const

const VPBasicBlock * getEntryBasicBlock() const

VPValue * getMask() const

Return the mask used by this recipe.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPBranchOnMaskRecipe.

void execute(VPTransformState &State) override

Generate the extraction of the appropriate bit from the block mask and the conditional branch.

VPlan-based builder utility analogous to IRBuilder.

VPInstruction * createNaryOp(unsigned Opcode, ArrayRef< VPValue * > Operands, Instruction *Inst=nullptr, const Twine &Name="")

Create an N-ary operation with Opcode, Operands and set Inst as its underlying Instruction.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

This class augments a recipe with a set of VPValues defined by the recipe.

void dump() const

Dump the VPDef to stderr (for debugging).

unsigned getNumDefinedValues() const

Returns the number of values defined by the VPDef.

ArrayRef< VPValue * > definedValues()

Returns an ArrayRef of the values defined by the VPDef.

VPValue * getVPSingleValue()

Returns the only VPValue defined by the VPDef.

VPValue * getVPValue(unsigned I)

Returns the VPValue with index I defined by the VPDef.

unsigned getVPDefID() const

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

VPValue * getStepValue() const

VPValue * getStartValue() const

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

Generate a canonical vector induction variable of the vector loop, with.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

Produce a vectorized histogram operation.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPHistogramRecipe.

VPValue * getMask() const

Return the mask operand if one was provided, or a null pointer if all lanes should be executed uncond...

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Instruction & getInstruction() const

void execute(VPTransformState &State) override

The method which generates the output IR instructions that correspond to this VPRecipe,...

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPIRInstruction.

void extractLastLaneOfOperand(VPBuilder &Builder)

Update the recipes single operand to the last lane of the operand using Builder.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

bool opcodeMayReadOrWriteFromMemory() const

Returns true if the underlying opcode may read from or write to memory.

LLVM_DUMP_METHOD void dump() const

Print the VPInstruction to dbgs() (for debugging).

unsigned getOpcode() const

bool onlyFirstPartUsed(const VPValue *Op) const override

Returns true if the recipe only uses the first part of operand Op.

@ ResumePhi

Creates a scalar phi in a leaf VPBB with a single predecessor in VPlan.

@ FirstOrderRecurrenceSplice

@ CanonicalIVIncrementForPart

@ CalculateTripCountMinusVF

bool isVectorToScalar() const

Returns true if this VPInstruction produces a scalar value from a vector, e.g.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the VPInstruction to O.

bool onlyFirstLaneUsed(const VPValue *Op) const override

Returns true if the recipe only uses the first lane of operand Op.

bool isSingleScalar() const

Returns true if this VPInstruction's operands are single scalars and the result is also a single scal...

void execute(VPTransformState &State) override

Generate the instruction.

VPValue * getAddr() const

Return the address accessed by this recipe.

VPValue * getMask() const

Return the mask used by this recipe.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

Generate the wide load or store, and shuffles.

ArrayRef< VPValue * > getStoredValues() const

Return the VPValues stored by this interleave group.

Instruction * getInsertPos() const

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPInterleaveRecipe.

unsigned getNumStoreOperands() const

Returns the number of stored operands of this interleave group.

static bool isVPIntrinsic(Intrinsic::ID)

In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...

static VPLane getLastLaneForVF(const ElementCount &VF)

static VPLane getLaneFromEnd(const ElementCount &VF, unsigned Offset)

static VPLane getFirstLane()

void execute(VPTransformState &State) override

Generate the reduction in the loop.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPPartialReductionRecipe.

unsigned getOpcode() const

Get the binary op's opcode.

void execute(VPTransformState &State) override

Generates phi nodes for live-outs (from a replicate region) as needed to retain SSA form.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.

bool mayReadFromMemory() const

Returns true if the recipe may read from memory.

bool mayHaveSideEffects() const

Returns true if the recipe may have side-effects.

bool mayWriteToMemory() const

Returns true if the recipe may write to memory.

virtual InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const

Compute the cost of this recipe either using a recipe's specialized implementation or using the legac...

VPBasicBlock * getParent()

DebugLoc getDebugLoc() const

Returns the debug location of the recipe.

void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)

Unlink this recipe and insert into BB before I.

void insertBefore(VPRecipeBase *InsertPos)

Insert an unlinked recipe into a basic block immediately before the specified recipe.

void insertAfter(VPRecipeBase *InsertPos)

Insert an unlinked Recipe into a basic block immediately after the specified Recipe.

iplist< VPRecipeBase >::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

InstructionCost cost(ElementCount VF, VPCostContext &Ctx)

Return the cost of this recipe, taking into account if the cost computation should be skipped and the...

void removeFromParent()

This method unlinks 'this' from the containing basic block, but does not delete it.

void moveAfter(VPRecipeBase *MovePos)

Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...

Class to record LLVM IR flag for a recipe along with it.

NonNegFlagsTy NonNegFlags

GEPNoWrapFlags getGEPNoWrapFlags() const

void setFlags(Instruction *I) const

Set the IR flags for I.

bool hasFastMathFlags() const

Returns true if the recipe has fast-math flags.

DisjointFlagsTy DisjointFlags

bool hasNoUnsignedWrap() const

void printFlags(raw_ostream &O) const

CmpInst::Predicate getPredicate() const

bool hasNoSignedWrap() const

FastMathFlags getFastMathFlags() const

void execute(VPTransformState &State) override

Generate the reduction in the loop.

VPValue * getEVL() const

The VPValue of the explicit vector length.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

Generate the phi/select nodes.

bool isConditional() const

Return true if the in-loop reduction is conditional.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of VPReductionRecipe.

VPValue * getVecOp() const

The VPValue of the vector value to be reduced.

const RecurrenceDescriptor & getRecurrenceDescriptor() const

Return the recurrence decriptor for the in-loop reduction.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

VPValue * getCondOp() const

The VPValue of the condition for the block.

bool isOrdered() const

Return true if the in-loop reduction is ordered.

VPValue * getChainOp() const

The VPValue of the scalar Chain being accumulated.

void execute(VPTransformState &State) override

Generate the reduction in the loop.

VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...

const VPBlockBase * getEntry() const

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPReplicateRecipe.

unsigned getOpcode() const

bool shouldPack() const

Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

The method which generates the output IR instructions that correspond to this VPRecipe,...

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

VPValue * getStepValue() const

void execute(VPTransformState &State) override

Generate the scalarized versions of the phi node as needed by their users.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

Generate the phi/select nodes.

Instruction * getUnderlyingInstr()

Returns the underlying instruction.

LLVM_DUMP_METHOD void dump() const

Print this VPSingleDefRecipe to dbgs() (for debugging).

This class can be used to assign names to VPValues.

LLVMContext & getContext()

Return the LLVMContext used by the analysis.

Type * inferScalarType(const VPValue *V)

Infer the type of V. Returns the scalar type of V.

VPValue * getUnrollPartOperand(VPUser &U) const

Return the VPValue operand containing the unroll part or null if there is no such operand.

unsigned getUnrollPart(VPUser &U) const

Return the unroll part.

This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...

void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const

Print the operands to O.

void setOperand(unsigned I, VPValue *New)

unsigned getNumOperands() const

operand_iterator op_begin()

VPValue * getOperand(unsigned N) const

virtual bool onlyFirstLaneUsed(const VPValue *Op) const

Returns true if the VPUser only uses the first lane of operand Op.

bool isDefinedOutsideLoopRegions() const

Returns true if the VPValue is defined outside any loop region.

VPRecipeBase * getDefiningRecipe()

Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...

void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const

friend class VPInstruction

bool hasMoreThanOneUniqueUser() const

Returns true if the value has more than one unique user.

Value * getUnderlyingValue() const

Return the underlying Value attached to this VPValue.

user_iterator user_begin()

unsigned getNumUsers() const

Value * getLiveInIRValue()

Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.

bool isLiveIn() const

Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

The method which generates the output IR instructions that correspond to this VPRecipe,...

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Function * getCalledScalarFunction() const

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenCallRecipe.

void execute(VPTransformState &State) override

Produce a widened version of the call instruction.

operand_range arg_operands()

void execute(VPTransformState &State) override

Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Type * getResultType() const

Returns the result type of the cast.

void execute(VPTransformState &State) override

Produce widened copies of the cast.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenCastRecipe.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override final

Print the recipe.

void execute(VPTransformState &State) override final

Produce a vp-intrinsic using the opcode and operands of the recipe, processing EVL elements.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

Generate the gep nodes.

PHINode * getPHINode() const

VPValue * getStepValue()

Returns the step value of the induction.

const InductionDescriptor & getInductionDescriptor() const

Returns the induction descriptor for the recipe.

TruncInst * getTruncInst()

Returns the first defined value as TruncInst, if it is one or nullptr otherwise.

void execute(VPTransformState &State) override

Generate the vectorized and scalarized versions of the phi node as needed by their users.

Type * getScalarType() const

Returns the scalar type of the induction.

bool isCanonical() const

Returns true if the induction is canonical, i.e.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

VPValue * getSplatVFValue()

bool onlyFirstLaneUsed(const VPValue *Op) const override

Returns true if the VPUser only uses the first lane of operand Op.

StringRef getIntrinsicName() const

Return to name of the intrinsic as string.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Type * getResultType() const

Return the scalar return type of the intrinsic.

void execute(VPTransformState &State) override

Produce a widened version of the vector intrinsic.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this vector intrinsic.

bool IsMasked

Whether the memory access is masked.

bool Reverse

Whether the consecutive accessed addresses are in reverse order.

bool isConsecutive() const

Return whether the loaded-from / stored-to addresses are consecutive.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenMemoryRecipe.

bool Consecutive

Whether the accessed addresses are consecutive.

VPValue * getMask() const

Return the mask used by this recipe.

VPValue * getAddr() const

Return the address accessed by this recipe.

bool isReverse() const

Return whether the consecutive loaded/stored addresses are in reverse order.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

Generate the phi/select nodes.

bool onlyScalarsGenerated(bool IsScalable)

Returns true if only scalar values will be generated.

VPValue * getFirstUnrolledPartOperand()

Returns the VPValue representing the value of this induction at the first unrolled part,...

void execute(VPTransformState &State) override

Generate vector values for the pointer induction.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenRecipe.

void execute(VPTransformState &State) override

Produce a widened instruction using the opcode and operands of the recipe, processing State....

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

unsigned getOpcode() const

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

void setName(const Twine &Name)

Change the name of the value.

const Value * stripPointerCasts() const

Strip off pointer casts, all-zero GEPs and address space casts.

LLVMContext & getContext() const

All values hold a context through their type.

StringRef getName() const

Return a constant reference to the value's name.

VectorBuilder & setEVL(Value *NewExplicitVectorLength)

VectorBuilder & setMask(Value *NewMask)

Value * createVectorInstruction(unsigned Opcode, Type *ReturnTy, ArrayRef< Value * > VecOpArray, const Twine &Name=Twine())

Base class of all SIMD vector types.

ElementCount getElementCount() const

Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...

static VectorType * get(Type *ElementType, ElementCount EC)

This static method is the primary way to construct an VectorType.

static VectorType * getDoubleElementsVectorType(VectorType *VTy)

This static method returns a VectorType with twice as many elements as the input type and the same el...

Type * getElementType() const

constexpr bool isScalable() const

Returns whether the quantity is scaled by a runtime quantity (vscale).

constexpr ScalarTy getKnownMinValue() const

Returns the minimum value this quantity can represent.

constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const

We do not provide the '/' operator here because division for polynomial types does not work in the sa...

const ParentTy * getParent() const

self_iterator getIterator()

base_list_type::iterator iterator

iterator erase(iterator where)

pointer remove(iterator &IT)

This class implements an extremely fast bulk output stream that can only output to a stream.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

@ C

The default llvm calling convention, compatible with C.

Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})

Look up the Function declaration of the intrinsic id in the Module M.

StringRef getBaseName(ID id)

Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....

bool match(Val *V, const Pattern &P)

auto m_LogicalOr()

Matches L || R where L and R are arbitrary values.

auto m_LogicalAnd()

Matches L && R where L and R are arbitrary values.

bool isUniformAfterVectorization(const VPValue *VPV)

Returns true if VPV is uniform after vectorization.

bool onlyFirstPartUsed(const VPValue *Def)

Returns true if only the first part of Def is used.

bool onlyFirstLaneUsed(const VPValue *Def)

Returns true if only the first lane of Def is used.

This is an optimization pass for GlobalISel generic memory operations.

void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)

Replace the instruction specified by BI with the instruction specified by I.

Value * createSimpleReduction(IRBuilderBase &B, Value *Src, RecurKind RdxKind)

Create a reduction of the given vector.

bool all_of(R &&range, UnaryPredicate P)

Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.

unsigned getLoadStoreAddressSpace(const Value *I)

A helper function that returns the address space of the pointer operand of load or store instruction.

Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)

Returns the min/max intrinsic used when expanding a min/max reduction.

auto enumerate(FirstRange &&First, RestRanges &&...Rest)

Given two or more input ranges, returns a new range whose values are tuples (A, B,...

const Value * getLoadStorePointerOperand(const Value *V)

A helper function that returns the pointer operand of a load or store instruction.

Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)

Return the runtime value for VF.

iterator_range< T > make_range(T x, T y)

Convenience function for iterating over sub-ranges.

void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)

Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)

Concatenate a list of vectors.

Align getLoadStoreAlignment(const Value *I)

A helper function that returns the alignment of load or store instruction.

Value * createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left, Value *Right)

Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.

bool any_of(R &&range, UnaryPredicate P)

Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.

Constant * createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, const InterleaveGroup< Instruction > &Group)

Create a mask that filters the members of an interleave group where there are gaps.

llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)

Create a stride shuffle mask.

cl::opt< bool > EnableVPlanNativePath("enable-vplan-native-path", cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))

llvm::SmallVector< int, 16 > createReplicatedMask(unsigned ReplicationFactor, unsigned VF)

Create a mask with replicated elements.

raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

bool isPointerTy(const Type *T)

Value * createOrderedReduction(IRBuilderBase &B, const RecurrenceDescriptor &Desc, Value *Src, Value *Start)

Create an ordered reduction intrinsic using the given recurrence descriptor Desc.

Value * createReduction(IRBuilderBase &B, const RecurrenceDescriptor &Desc, Value *Src, PHINode *OrigPhi=nullptr)

Create a generic reduction using a recurrence descriptor Desc Fast-math-flags are propagated using th...

llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)

Create an interleave shuffle mask.

RecurKind

These are the kinds of recurrences that we support.

@ Mul

Product of integers.

@ SMax

Signed integer max implemented in terms of select(cmp()).

bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx, const TargetTransformInfo *TTI)

Identifies if the vector form of the intrinsic has a scalar operand.

Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF)

Given information about an recurrence kind, return the identity for the @llvm.vector....

DWARFExpression::Operation Op

Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)

Return a value for Step multiplied by VF.

auto predecessors(const MachineBasicBlock *BB)

bool is_contained(R &&Range, const E &Element)

Returns true if Element is found in Range.

Type * getLoadStoreType(const Value *I)

A helper function that returns the type of a load or store instruction.

Type * toVectorTy(Type *Scalar, ElementCount EC)

A helper function for converting Scalar types to vector types.

bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx, const TargetTransformInfo *TTI)

Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...

This struct is a compact representation of a valid (non-zero power of two) alignment.

Struct to hold various analysis needed for cost computations.

TargetTransformInfo::OperandValueInfo getOperandInfo(VPValue *V) const

Returns the OperandInfo for V, if it is a live-in.

bool skipCostComputation(Instruction *UI, bool IsVector) const

Return true if the cost for UI shouldn't be computed, e.g.

InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const

Return the cost for UI with VF using the legacy cost model as fallback until computing the cost of al...

TargetTransformInfo::TargetCostKind CostKind

const TargetLibraryInfo & TLI

const TargetTransformInfo & TTI

SmallPtrSet< Instruction *, 8 > SkipCostComputation

void execute(VPTransformState &State) override

Generate the phi nodes.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this first-order recurrence phi recipe.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

BasicBlock * PrevBB

The previous IR BasicBlock created or used.

SmallDenseMap< VPBasicBlock *, BasicBlock * > VPBB2IRBB

A mapping of each VPBasicBlock to the corresponding BasicBlock.

BasicBlock * getPreheaderBBFor(VPRecipeBase *R)

Returns the BasicBlock* mapped to the pre-header of the loop region containing R.

VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...

bool hasScalarValue(VPValue *Def, VPLane Lane)

bool hasVectorValue(VPValue *Def)

DenseMap< const SCEV *, Value * > ExpandedSCEVs

Map SCEVs to their expanded values.

VPTypeAnalysis TypeAnalysis

VPlan-based type analysis.

void addMetadata(Value *To, Instruction *From)

Add metadata from one instruction to another.

Value * get(VPValue *Def, bool IsScalar=false)

Get the generated vector Value for a given VPValue Def if IsScalar is false, otherwise return the gen...

struct llvm::VPTransformState::CFGState CFG

std::optional< VPLane > Lane

Hold the index to generate specific scalar instructions.

IRBuilderBase & Builder

Hold a reference to the IRBuilder used to generate output IR code.

const TargetTransformInfo * TTI

Target Transform Info.

void reset(VPValue *Def, Value *V)

Reset an existing vector value for Def and a given Part.

ElementCount VF

The chosen Vectorization Factor of the loop being vectorized.

void setDebugLocFrom(DebugLoc DL)

Set the debug location in the builder using the debug location DL.

Loop * CurrentParentLoop

The parent loop object for the current scope, or nullptr.

void set(VPValue *Def, Value *V, bool IsScalar=false)

Set the generated vector Value for a given VPValue, if IsScalar is false.

void execute(VPTransformState &State) override

Generate the wide load or gather.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenLoadEVLRecipe.

VPValue * getEVL() const

Return the EVL operand.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

Generate a wide load or gather.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

bool isInvariantCond() const

VPValue * getCond() const

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenSelectRecipe.

void execute(VPTransformState &State) override

Produce a widened version of the select instruction.

VPValue * getStoredValue() const

Return the address accessed by this recipe.

void execute(VPTransformState &State) override

Generate the wide store or scatter.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenStoreEVLRecipe.

VPValue * getEVL() const

Return the EVL operand.

void execute(VPTransformState &State) override

Generate a wide store or scatter.

VPValue * getStoredValue() const

Return the value stored by this recipe.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.