VPlanRecipes.cpp Source File (original) (raw)

37#include

39using namespace llvm;

43namespace llvm {

45}

48#define LV_NAME "loop-vectorize"

49#define DEBUG_TYPE LV_NAME

53 case VPInstructionSC:

54 return cast(this)->opcodeMayReadOrWriteFromMemory();

55 case VPInterleaveSC:

56 return cast(this)->getNumStoreOperands() > 0;

57 case VPWidenStoreEVLSC:

58 case VPWidenStoreSC:

59 return true;

60 case VPReplicateSC:

61 return cast(getVPSingleValue()->getUnderlyingValue())

62 ->mayWriteToMemory();

63 case VPWidenCallSC:

64 return !cast(this)

65 ->getCalledScalarFunction()

66 ->onlyReadsMemory();

67 case VPWidenIntrinsicSC:

68 return cast(this)->mayWriteToMemory();

69 case VPBranchOnMaskSC:

70 case VPScalarIVStepsSC:

71 case VPPredInstPHISC:

72 return false;

73 case VPBlendSC:

74 case VPReductionEVLSC:

75 case VPReductionSC:

76 case VPVectorPointerSC:

77 case VPWidenCanonicalIVSC:

78 case VPWidenCastSC:

79 case VPWidenGEPSC:

80 case VPWidenIntOrFpInductionSC:

81 case VPWidenLoadEVLSC:

82 case VPWidenLoadSC:

83 case VPWidenPHISC:

84 case VPWidenSC:

85 case VPWidenEVLSC:

86 case VPWidenSelectSC: {

88 dyn_cast_or_null(getVPSingleValue()->getUnderlyingValue());

89 (void)I;

90 assert(( || ->mayWriteToMemory()) &&

91 "underlying instruction may write to memory");

92 return false;

93 }

94 default:

95 return true;

96 }

97}

101 case VPInstructionSC:

102 return cast(this)->opcodeMayReadOrWriteFromMemory();

103 case VPWidenLoadEVLSC:

104 case VPWidenLoadSC:

105 return true;

106 case VPReplicateSC:

107 return cast(getVPSingleValue()->getUnderlyingValue())

108 ->mayReadFromMemory();

109 case VPWidenCallSC:

110 return !cast(this)

111 ->getCalledScalarFunction()

112 ->onlyWritesMemory();

113 case VPWidenIntrinsicSC:

114 return cast(this)->mayReadFromMemory();

115 case VPBranchOnMaskSC:

116 case VPPredInstPHISC:

117 case VPScalarIVStepsSC:

118 case VPWidenStoreEVLSC:

119 case VPWidenStoreSC:

120 return false;

121 case VPBlendSC:

122 case VPReductionEVLSC:

123 case VPReductionSC:

124 case VPVectorPointerSC:

125 case VPWidenCanonicalIVSC:

126 case VPWidenCastSC:

127 case VPWidenGEPSC:

128 case VPWidenIntOrFpInductionSC:

129 case VPWidenPHISC:

130 case VPWidenSC:

131 case VPWidenEVLSC:

132 case VPWidenSelectSC: {

134 dyn_cast_or_null(getVPSingleValue()->getUnderlyingValue());

135 (void)I;

136 assert(( || ->mayReadFromMemory()) &&

137 "underlying instruction may read from memory");

138 return false;

139 }

140 default:

141 return true;

142 }

143}

144

147 case VPDerivedIVSC:

148 case VPPredInstPHISC:

149 case VPScalarCastSC:

150 case VPReverseVectorPointerSC:

151 return false;

152 case VPInstructionSC:

154 case VPWidenCallSC: {

155 Function *Fn = cast(this)->getCalledScalarFunction();

157 }

158 case VPWidenIntrinsicSC:

159 return cast(this)->mayHaveSideEffects();

160 case VPBlendSC:

161 case VPReductionEVLSC:

162 case VPReductionSC:

163 case VPScalarIVStepsSC:

164 case VPVectorPointerSC:

165 case VPWidenCanonicalIVSC:

166 case VPWidenCastSC:

167 case VPWidenGEPSC:

168 case VPWidenIntOrFpInductionSC:

169 case VPWidenPHISC:

170 case VPWidenPointerInductionSC:

171 case VPWidenSC:

172 case VPWidenEVLSC:

173 case VPWidenSelectSC: {

175 dyn_cast_or_null(getVPSingleValue()->getUnderlyingValue());

176 (void)I;

177 assert(( || ->mayHaveSideEffects()) &&

178 "underlying instruction has side-effects");

179 return false;

180 }

181 case VPInterleaveSC:

183 case VPWidenLoadEVLSC:

184 case VPWidenLoadSC:

185 case VPWidenStoreEVLSC:

186 case VPWidenStoreSC:

188 cast(this)->getIngredient().mayHaveSideEffects() ==

190 "mayHaveSideffects result for ingredient differs from this "

191 "implementation");

193 case VPReplicateSC: {

194 auto *R = cast(this);

195 return R->getUnderlyingInstr()->mayHaveSideEffects();

196 }

197 default:

198 return true;

199 }

200}

201

203 assert(!Parent && "Recipe already in some VPBasicBlock");

205 "Insertion position not in any VPBasicBlock");

207}

208

211 assert(!Parent && "Recipe already in some VPBasicBlock");

212 assert(I == BB.end() || I->getParent() == &BB);

214}

215

217 assert(!Parent && "Recipe already in some VPBasicBlock");

219 "Insertion position not in any VPBasicBlock");

221}

222

226 Parent = nullptr;

227}

228

232}

233

237}

238

243}

244

246

247

248

249

251 if (auto *S = dyn_cast(this))

252 UI = dyn_cast_or_null(S->getUnderlyingValue());

253 else if (auto *IG = dyn_cast(this))

254 UI = IG->getInsertPos();

255 else if (auto *WidenMem = dyn_cast(this))

256 UI = &WidenMem->getIngredient();

257

260 RecipeCost = 0;

261 } else {

266 }

267

269 dbgs() << "Cost of " << RecipeCost << " for VF " << VF << ": ";

271 });

272 return RecipeCost;

273}

274

278}

279

283 std::optional Opcode = std::nullopt;

285 if (auto *WidenR = dyn_cast(BinOpR))

286 Opcode = std::make_optional(WidenR->getOpcode());

287

290

296

298

299 if (!R)

301 auto *WidenCastR = dyn_cast(R);

302 if (!WidenCastR)

304 if (WidenCastR->getOpcode() == Instruction::CastOps::ZExt)

306 if (WidenCastR->getOpcode() == Instruction::CastOps::SExt)

309 };

310

312 PhiType, VF, GetExtendKind(ExtAR),

313 GetExtendKind(ExtBR), Opcode);

314}

315

318 auto &Builder = State.Builder;

319

321 "Unhandled partial reduction opcode");

322

325 assert(PhiVal && BinOpVal && "Phi and Mul must be set");

326

328

329 CallInst *V = Builder.CreateIntrinsic(

330 RetTy, Intrinsic::experimental_vector_partial_reduce_add,

331 {PhiVal, BinOpVal}, nullptr, "partial.reduce");

332

333 State.set(this, V);

334}

335

336#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

339 O << Indent << "PARTIAL-REDUCE ";

343}

344#endif

345

347 assert(OpType == OperationType::FPMathOp &&

348 "recipe doesn't have fast math flags");

357 return Res;

358}

359

360#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

362#endif

363

364template

367 if (U.getNumOperands() == PartOpIdx + 1)

368 return U.getOperand(PartOpIdx);

369 return nullptr;

370}

371

372template

374 if (auto *UnrollPartOp = getUnrollPartOperand(U))

375 return cast(UnrollPartOp->getLiveInIRValue())->getZExtValue();

376 return 0;

377}

378

383 Pred, DL),

384 Opcode(Opcode), Name(Name.str()) {

385 assert(Opcode == Instruction::ICmp &&

386 "only ICmp predicates supported at the moment");

387}

388

390 std::initializer_list<VPValue *> Operands,

393 Opcode(Opcode), Name(Name.str()) {

394

395 assert(isFPMathOp() && "this op can't take fast-math flags");

396}

397

398bool VPInstruction::doesGeneratePerAllLanes() const {

400}

401

402bool VPInstruction::canGenerateScalarForFirstLane() const {

404 return true;

406 return true;

407 switch (Opcode) {

408 case Instruction::ICmp:

409 case Instruction::Select:

417 return true;

418 default:

419 return false;

420 }

421}

422

424 const VPLane &Lane) {

426

428 "only PtrAdd opcodes are supported for now");

431}

432

435

440 auto *Res =

442 if (auto *I = dyn_cast(Res))

444 return Res;

445 }

446

451 }

452 case Instruction::ICmp: {

457 }

458 case Instruction::Select: {

464 }

466

468

470

471

472

475 Name);

476

479 return Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask,

480 {PredTy, ScalarTC->getType()},

481 {VIVElem0, ScalarTC}, nullptr, Name);

482 }

484

485

486

487

488

489

490

491

492

493

494

495

497 if (!V1->getType()->isVectorTy())

498 return V1;

501 }

510 }

512

513

515

517 "Requested vector length should be an integer.");

518

521

523 State.Builder.getInt32Ty(), Intrinsic::experimental_get_vector_length,

524 {AVL, VFArg, State.Builder.getTrue()});

525 return EVL;

526 }

530 assert(Part != 0 && "Must have a positive part");

531

532

536 }

539

540

541

546

548 return CondBr;

549

553 return CondBr;

554 }

556

560

561

563 VPRegionBlock *TopRegion = Plan->getVectorLoopRegion();

565

566

567

568

569

570

575 return CondBr;

576 }

578

579

580 auto *PhiR = cast(getOperand(0));

581 auto *OrigPhi = cast(PhiR->getUnderlyingValue());

582

584

586

587 Type *PhiTy = OrigPhi->getType();

588

589

592 for (unsigned Part = 0; Part < UF; ++Part)

593 RdxParts[Part] = State.get(getOperand(1 + Part), PhiR->isInLoop());

594

595

596

597

598

601 for (unsigned Part = 0; Part < UF; ++Part)

602 RdxParts[Part] = Builder.CreateTrunc(RdxParts[Part], RdxVecTy);

603 }

604

605 Value *ReducedPartRdx = RdxParts[0];

608 Op = Instruction::Or;

609

610 if (PhiR->isOrdered()) {

611 ReducedPartRdx = RdxParts[UF - 1];

612 } else {

613

616 for (unsigned Part = 1; Part < UF; ++Part) {

617 Value *RdxPart = RdxParts[Part];

618 if (Op != Instruction::ICmp && Op != Instruction::FCmp)

622 ReducedPartRdx =

624 else

625 ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);

626 }

627 }

628

629

630

634 !PhiR->isInLoop()) {

635 ReducedPartRdx =

636 createReduction(Builder, RdxDesc, ReducedPartRdx, OrigPhi);

637

638

640 ReducedPartRdx = RdxDesc.isSigned()

641 ? Builder.CreateSExt(ReducedPartRdx, PhiTy)

642 : Builder.CreateZExt(ReducedPartRdx, PhiTy);

643 }

644

645 return ReducedPartRdx;

646 }

649 unsigned Offset = CI->getZExtValue();

650 assert(Offset > 0 && "Offset from end must be positive");

654 "invalid offset to extract from");

655

657 } else {

658 assert(Offset <= 1 && "invalid offset to extract from");

660 }

661 if (isa(Res))

663 return Res;

664 }

669 }

672 "can only generate first lane for PtrAdd");

676 }

678 Value *IncomingFromVPlanPred =

680 Value *IncomingFromOtherPreds =

682 auto *NewPhi =

685 State.CFG

687 NewPhi->addIncoming(IncomingFromVPlanPred, VPlanPred);

689 if (OtherPred == VPlanPred)

690 continue;

691 NewPhi->addIncoming(IncomingFromOtherPreds, OtherPred);

692 }

693 return NewPhi;

694 }

698 }

699

700 default:

702 }

703}

704

709}

710

713}

714

715#if !defined(NDEBUG)

716bool VPInstruction::isFPMathOp() const {

717

718

719 return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||

720 Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||

721 Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||

722 Opcode == Instruction::FCmp || Opcode == Instruction::Select;

723}

724#endif

725

727 assert(!State.Lane && "VPInstruction executing an Lane");

730 getOpcode() == Instruction::Select) &&

731 "Recipe not a FPMathOp but has fast-math flags?");

735 bool GeneratesPerFirstLaneOnly = canGenerateScalarForFirstLane() &&

738 bool GeneratesPerAllLanes = doesGeneratePerAllLanes();

739 if (GeneratesPerAllLanes) {

741 Lane != NumLanes; ++Lane) {

742 Value *GeneratedValue = generatePerLane(State, VPLane(Lane));

743 assert(GeneratedValue && "generatePerLane must produce a value");

744 State.set(this, GeneratedValue, VPLane(Lane));

745 }

746 return;

747 }

748

749 Value *GeneratedValue = generate(State);

751 return;

752 assert(GeneratedValue && "generate must produce a value");

754 (GeneratedValue->getType()->isVectorTy() == !GeneratesPerFirstLaneOnly ||

756 "scalar value but not only first lane defined");

757 State.set(this, GeneratedValue,

758 GeneratesPerFirstLaneOnly);

759}

760

763 return false;

765 case Instruction::ICmp:

766 case Instruction::Select:

775 return false;

776 default:

777 return true;

778 }

779}

780

785

787 default:

788 return false;

789 case Instruction::ICmp:

790 case Instruction::Select:

791 case Instruction::Or:

793

802 return true;

803 };

805}

806

811

813 default:

814 return false;

815 case Instruction::ICmp:

816 case Instruction::Select:

821 return true;

822 };

824}

825

826#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

830}

831

834 O << Indent << "EMIT ";

835

838 O << " = ";

839 }

840

843 O << "not";

844 break;

846 O << "combined load";

847 break;

849 O << "combined store";

850 break;

852 O << "active lane mask";

853 break;

855 O << "resume-phi";

856 break;

858 O << "EXPLICIT-VECTOR-LENGTH";

859 break;

861 O << "first-order splice";

862 break;

864 O << "branch-on-cond";

865 break;

867 O << "TC > VF ? TC - VF : 0";

868 break;

870 O << "VF * Part +";

871 break;

873 O << "branch-on-count";

874 break;

876 O << "extract-from-end";

877 break;

879 O << "compute-reduction-result";

880 break;

882 O << "logical-and";

883 break;

885 O << "ptradd";

886 break;

888 O << "any-of";

889 break;

890 default:

892 }

893

896

898 O << ", !dbg ";

900 }

901}

902#endif

903

906 "Only PHINodes can have extra operands");

915

916

919 auto *Phi = cast(&I);

920

921

922 if (Phi->getBasicBlockIndex(PredBB) == -1)

923 Phi->addIncoming(V, PredBB);

924 else

925 Phi->setIncomingValueForBlock(PredBB, V);

926 }

927

928

929

931}

932

935

936

937 return 0;

938}

939

940#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

943 O << Indent << "IR " << I;

944

946 O << " (extra operand" << (getNumOperands() > 1 ? "s" : "") << ": ";

950 O << " from ";

952 });

953 O << ")";

954 }

955}

956#endif

957

961

963

967

968

969

971 Arg = State.get(I.value(), VPLane(0));

972 else

974 Args.push_back(Arg);

975 }

976

977 assert(Variant != nullptr && "Can't create vector function.");

978

981 if (CI)

982 CI->getOperandBundlesAsDefs(OpBundles);

983

986

987 if (!V->getType()->isVoidTy())

988 State.set(this, V);

990}

991

997}

998

999#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1002 O << Indent << "WIDEN-CALL ";

1003

1006 O << "void ";

1007 else {

1009 O << " = ";

1010 }

1011

1012 O << "call";

1014 O << " @" << CalledFn->getName() << "(";

1017 });

1018 O << ")";

1019

1020 O << " (using library function";

1022 O << ": " << Variant->getName();

1023 O << ")";

1024}

1025#endif

1026

1030

1032

1037

1038

1041 State.TTI))

1042 Arg = State.get(I.value(), VPLane(0));

1043 else

1046 State.TTI))

1048 Args.push_back(Arg);

1049 }

1050

1051

1056 "Can't retrieve vector intrinsic or vector-predication intrinsics.");

1057

1060 if (CI)

1061 CI->getOperandBundlesAsDefs(OpBundles);

1062

1064

1066

1067 if (!V->getType()->isVoidTy())

1068 State.set(this, V);

1070}

1071

1074

1075

1076

1077

1078

1081 auto *V = Op->getUnderlyingValue();

1082 if (!V) {

1083

1084

1085

1086

1087

1090 continue;

1091 }

1093 Arguments.push_back(UI->getArgOperand(Idx));

1094 continue;

1095 }

1097 break;

1098 }

1100 }

1101

1107

1108

1114}

1115

1118}

1119

1122

1123

1126}

1127

1128#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1131 O << Indent << "WIDEN-INTRINSIC ";

1133 O << "void ";

1134 } else {

1136 O << " = ";

1137 }

1138

1139 O << "call";

1142

1145 });

1146 O << ")";

1147}

1148#endif

1149

1153

1157

1158

1159

1160

1161 Value *Mask = nullptr;

1163 Mask = State.get(VPMask);

1164 else

1165 Mask =

1167

1168

1169

1170 if (Opcode == Instruction::Sub)

1171 IncAmt = Builder.CreateNeg(IncAmt);

1172 else

1173 assert(Opcode == Instruction::Add && "only add or sub supported for now");

1174

1176 {VTy, IncAmt->getType()},

1177 {Address, IncAmt, Mask});

1178}

1179

1182

1183

1184

1185

1186

1187 assert(VF.isVector() && "Invalid VF for histogram cost");

1192

1193

1194

1199

1202 }

1203

1204

1209 {PtrTy, IncTy, MaskTy});

1210

1211

1214}

1215

1216#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1219 O << Indent << "WIDEN-HISTOGRAM buckets: ";

1221

1222 if (Opcode == Instruction::Sub)

1223 O << ", dec: ";

1224 else {

1225 assert(Opcode == Instruction::Add);

1226 O << ", inc: ";

1227 }

1229

1231 O << ", mask: ";

1233 }

1234}

1235

1238 O << Indent << "WIDEN-SELECT ";

1240 O << " = select ";

1243 O << ", ";

1245 O << ", ";

1247 O << (isInvariantCond() ? " (condition is loop invariant)" : "");

1248}

1249#endif

1250

1253

1254

1255

1256

1257

1258 auto *InvarCond =

1260

1265 State.set(this, Sel);

1266 if (isa(Sel))

1267 setFlags(cast(Sel));

1269}

1270

1277

1282 match(this, m_LogicalOr(m_VPValue(Op0), m_VPValue(Op1))))) {

1283

1284

1285 const auto [Op1VK, Op1VP] = Ctx.getOperandInfo(Op0);

1286 const auto [Op2VK, Op2VP] = Ctx.getOperandInfo(Op1);

1287

1290 [](VPValue *Op) { return Op->getUnderlyingValue(); }))

1291 Operands.append(SI->op_begin(), SI->op_end());

1292 bool IsLogicalOr = match(this, m_LogicalOr(m_VPValue(Op0), m_VPValue(Op1)));

1294 IsLogicalOr ? Instruction::Or : Instruction::And, VectorTy,

1295 Ctx.CostKind, {Op1VK, Op1VP}, {Op2VK, Op2VP}, Operands, SI);

1296 }

1297

1299 if (!ScalarCond)

1301

1303 if (auto *Cmp = dyn_cast(SI->getCondition()))

1304 Pred = Cmp->getPredicate();

1306 Instruction::Select, VectorTy, CondTy, Pred, Ctx.CostKind,

1307 {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None}, SI);

1308}

1309

1310VPRecipeWithIRFlags::FastMathFlagsTy::FastMathFlagsTy(

1313 NoNaNs = FMF.noNaNs();

1314 NoInfs = FMF.noInfs();

1319}

1320

1321#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1323 switch (OpType) {

1324 case OperationType::Cmp:

1326 break;

1327 case OperationType::DisjointOp:

1329 O << " disjoint";

1330 break;

1331 case OperationType::PossiblyExactOp:

1333 O << " exact";

1334 break;

1335 case OperationType::OverflowingBinOp:

1337 O << " nuw";

1339 O << " nsw";

1340 break;

1341 case OperationType::FPMathOp:

1343 break;

1344 case OperationType::GEPOp:

1346 O << " inbounds";

1348 O << " nusw";

1350 O << " nuw";

1351 break;

1352 case OperationType::NonNegOp:

1354 O << " nneg";

1355 break;

1356 case OperationType::Other:

1357 break;

1358 }

1360 O << " ";

1361}

1362#endif

1363

1366 auto &Builder = State.Builder;

1367 switch (Opcode) {

1368 case Instruction::Call:

1369 case Instruction::Br:

1370 case Instruction::PHI:

1371 case Instruction::GetElementPtr:

1372 case Instruction::Select:

1373 llvm_unreachable("This instruction is handled by a different recipe.");

1374 case Instruction::UDiv:

1375 case Instruction::SDiv:

1376 case Instruction::SRem:

1377 case Instruction::URem:

1378 case Instruction::Add:

1379 case Instruction::FAdd:

1380 case Instruction::Sub:

1381 case Instruction::FSub:

1382 case Instruction::FNeg:

1383 case Instruction::Mul:

1384 case Instruction::FMul:

1385 case Instruction::FDiv:

1386 case Instruction::FRem:

1387 case Instruction::Shl:

1388 case Instruction::LShr:

1389 case Instruction::AShr:

1390 case Instruction::And:

1391 case Instruction::Or:

1392 case Instruction::Xor: {

1393

1397

1399

1400 if (auto *VecOp = dyn_cast(V))

1402

1403

1404 State.set(this, V);

1406 break;

1407 }

1408 case Instruction::Freeze: {

1410

1412 State.set(this, Freeze);

1413 break;

1414 }

1415 case Instruction::ICmp:

1416 case Instruction::FCmp: {

1417

1418 bool FCmp = Opcode == Instruction::FCmp;

1422 if (FCmp) {

1423

1427 } else {

1429 }

1430 State.set(this, C);

1432 break;

1433 }

1434 default:

1435

1436 LLVM_DEBUG(dbgs() << "LV: Found an unhandled opcode : "

1439 }

1440

1441#if !defined(NDEBUG)

1442

1443

1446 "inferred type and type from generated instructions do not match");

1447#endif

1448}

1449

1452 switch (Opcode) {

1453 case Instruction::FNeg: {

1456 Opcode, VectorTy, Ctx.CostKind,

1457 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},

1458 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None});

1459 }

1460

1461 case Instruction::UDiv:

1462 case Instruction::SDiv:

1463 case Instruction::SRem:

1464 case Instruction::URem:

1465

1467 case Instruction::Add:

1468 case Instruction::FAdd:

1469 case Instruction::Sub:

1470 case Instruction::FSub:

1471 case Instruction::Mul:

1472 case Instruction::FMul:

1473 case Instruction::FDiv:

1474 case Instruction::FRem:

1475 case Instruction::Shl:

1476 case Instruction::LShr:

1477 case Instruction::AShr:

1478 case Instruction::And:

1479 case Instruction::Or:

1480 case Instruction::Xor: {

1482

1483

1486 if (RHS->isLiveIn())

1488

1494

1496 if (CtxI)

1499 Opcode, VectorTy, Ctx.CostKind,

1500 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},

1502 }

1503 case Instruction::Freeze: {

1504

1508 }

1509 case Instruction::ICmp:

1510 case Instruction::FCmp: {

1515 {TTI::OK_AnyValue, TTI::OP_None},

1516 {TTI::OK_AnyValue, TTI::OP_None}, CtxI);

1517 }

1518 default:

1520 }

1521}

1522

1525

1527 llvm_unreachable("Unsupported opcode in VPWidenEVLRecipe::execute");

1528

1530

1532 "VPWidenEVLRecipe should not be used for scalars");

1533

1535 Value *EVLArg = State.get(EVL, true);

1539

1544 }

1545

1549

1550

1551 if (isa(VPInst))

1552 setFlags(cast(VPInst));

1553

1554 State.set(this, VPInst);

1557}

1558

1559#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1562 O << Indent << "WIDEN ";

1567}

1568

1571 O << Indent << "WIDEN ";

1576}

1577#endif

1578

1581 auto &Builder = State.Builder;

1582

1588 State.set(this, Cast);

1590 if (auto *CastOp = dyn_cast(Cast))

1592}

1593

1596

1597

1598

1600 return 0;

1601

1605 if (isa(R))

1607 if (const auto *ReplicateRecipe = dyn_cast(R))

1610 const auto *WidenMemoryRecipe = dyn_cast(R);

1611 if (WidenMemoryRecipe == nullptr)

1613 if (!WidenMemoryRecipe->isConsecutive())

1615 if (WidenMemoryRecipe->isReverse())

1617 if (WidenMemoryRecipe->isMasked())

1620 };

1621

1624

1625 if ((Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) &&

1627 if (auto *StoreRecipe = dyn_cast(*user_begin()))

1628 CCH = ComputeCCH(StoreRecipe);

1629 }

1630

1631 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||

1632 Opcode == Instruction::FPExt) {

1637 }

1638

1639 auto *SrcTy =

1642

1644 Opcode, DestTy, SrcTy, CCH, Ctx.CostKind,

1646}

1647

1648#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1651 O << Indent << "WIDEN-CAST ";

1657}

1658#endif

1659

1663}

1664

1665

1666

1667

1668

1672 assert(VF.isVector() && "only vector VFs are supported");

1673

1674

1675 auto *ValVTy = cast(Val->getType());

1676 ElementCount VLen = ValVTy->getElementCount();

1677

1680 "Induction Step must be an integer or FP");

1681 assert(Step->getType() == STy && "Step has wrong type");

1682

1684

1685

1688 Type *InitVecValSTy =

1691 }

1693

1697

1698

1699 Step = Builder.CreateMul(InitVec, Step);

1700 return Builder.CreateAdd(Val, Step, "induction");

1701 }

1702

1703

1704 assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&

1705 "Binary Opcode should be specified for FP induction");

1706 InitVec = Builder.CreateUIToFP(InitVec, ValVTy);

1707

1710 return Builder.CreateBinOp(BinOp, Val, MulOp, "induction");

1711}

1712

1713

1714

1717 : ConstantFP::get(Ty, C);

1718}

1719

1721 assert(!State.Lane && "Int or FP induction being replicated.");

1722

1728 "Types must match");

1730

1731

1732

1734

1735

1737 if (ID.getInductionBinOp() && isa(ID.getInductionBinOp()))

1738 Builder.setFastMathFlags(ID.getInductionBinOp()->getFastMathFlags());

1739

1740

1742

1743 assert((isa(EntryVal) || isa(EntryVal)) &&

1744 "Expected either an induction phi-node or a truncate of it!");

1745

1746

1747 auto CurrIP = Builder.saveIP();

1750 if (isa(EntryVal)) {

1751 assert(Start->getType()->isIntegerTy() &&

1752 "Truncation requires an integer type");

1753 auto *TruncType = cast(EntryVal->getType());

1754 Step = Builder.CreateTrunc(Step, TruncType);

1755 Start = Builder.CreateCast(Instruction::Trunc, Start, TruncType);

1756 }

1757

1759 Value *SteppedStart = getStepVector(SplatStart, Step, ID.getInductionOpcode(),

1761

1762

1763

1767 AddOp = Instruction::Add;

1768 MulOp = Instruction::Mul;

1769 } else {

1770 AddOp = ID.getInductionOpcode();

1771 MulOp = Instruction::FMul;

1772 }

1773

1776

1777

1778 SplatVF = State.get(SplatVFOperand);

1779 } else {

1780

1781

1785 RuntimeVF = Builder.CreateUIToFP(RuntimeVF, StepType);

1786 else

1789

1790

1792 }

1793

1795

1796

1797

1801 State.set(this, VecInd);

1802

1803 Instruction *LastInduction = cast(

1804 Builder.CreateBinOp(AddOp, VecInd, SplatVF, "vec.ind.next"));

1805 if (isa(EntryVal))

1806 State.addMetadata(LastInduction, EntryVal);

1808

1809 VecInd->addIncoming(SteppedStart, VectorPH);

1810

1811

1812

1813

1814

1815 VecInd->addIncoming(LastInduction, VectorPH);

1816}

1817

1818#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1821 O << Indent;

1823 O << " = WIDEN-INDUCTION ";

1825

1827 O << " (truncated to " << *TI->getType() << ")";

1828}

1829#endif

1830

1832

1833

1834

1836 return false;

1839 auto *CanIV = cast(&*getParent()->begin());

1840 return StartC && StartC->isZero() && StepC && StepC->isOne() &&

1842}

1843

1844#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1847 O << Indent;

1849 O << " = DERIVED-IV ";

1851 O << " + ";

1853 O << " * ";

1855}

1856#endif

1857

1859

1863

1864

1865

1866

1870

1871

1873 assert(BaseIVTy == Step->getType() && "Types of BaseIV and Step must match!");

1874

1875

1876

1880 AddOp = Instruction::Add;

1881 MulOp = Instruction::Mul;

1882 } else {

1883 AddOp = InductionOpcode;

1884 MulOp = Instruction::FMul;

1885 }

1886

1887

1888

1890

1891 Type *IntStepTy =

1893 Type *VecIVTy = nullptr;

1894 Value *UnitStepVec = nullptr, *SplatStep = nullptr, *SplatIV = nullptr;

1895 if (!FirstLaneOnly && State.VF.isScalable()) {

1897 UnitStepVec =

1901 }

1902

1903 unsigned StartLane = 0;

1905 if (State.Lane) {

1906 StartLane = State.Lane->getKnownLane();

1907 EndLane = StartLane + 1;

1908 }

1909 Value *StartIdx0 =

1911

1912 if (!FirstLaneOnly && State.VF.isScalable()) {

1914 auto *InitVec = Builder.CreateAdd(SplatStartIdx, UnitStepVec);

1916 InitVec = Builder.CreateSIToFP(InitVec, VecIVTy);

1917 auto *Mul = Builder.CreateBinOp(MulOp, InitVec, SplatStep);

1919 State.set(this, Add);

1920

1921

1922

1923 }

1924

1926 StartIdx0 = Builder.CreateSIToFP(StartIdx0, BaseIVTy);

1927

1928 for (unsigned Lane = StartLane; Lane < EndLane; ++Lane) {

1931

1932

1934 "Expected StartIdx to be folded to a constant when VF is not "

1935 "scalable");

1936 auto *Mul = Builder.CreateBinOp(MulOp, StartIdx, Step);

1939 }

1940}

1941

1942#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1945 O << Indent;

1947 O << " = SCALAR-STEPS ";

1949}

1950#endif

1951

1955

1956

1957

1958

1959

1960

1961 if (areAllOperandsInvariant()) {

1962

1963

1964

1965

1966

1967

1968

1969

1970

1971

1972

1973

1977

1978 auto *NewGEP = State.Builder.CreateGEP(GEP->getSourceElementType(), Ops[0],

1979 ArrayRef(Ops).drop_front(), "",

1984 } else {

1985

1986

1987

1988

1991

1992

1993

1997 if (isIndexLoopInvariant(I - 1))

1999 else

2001 }

2002

2003

2004

2007 assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) &&

2008 "NewGEP is not a pointer vector");

2009 State.set(this, NewGEP);

2011 }

2012}

2013

2014#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2017 O << Indent << "WIDEN-GEP ";

2018 O << (isPointerLoopInvariant() ? "Inv" : "Var");

2020 O << "[" << (isIndexLoopInvariant(I) ? "Inv" : "Var") << "]";

2021

2022 O << " ";

2024 O << " = getelementptr";

2027}

2028#endif

2029

2032

2033

2035 return IsScalable && (IsReverse || CurrentPart > 0)

2036 ? DL.getIndexType(Builder.getPtrTy(0))

2038}

2039

2041 auto &Builder = State.Builder;

2045 CurrentPart, Builder);

2046

2047

2049 if (IndexTy != RunTimeVF->getType())

2051

2053 ConstantInt::get(IndexTy, -(int64_t)CurrentPart), RunTimeVF);

2054

2055 Value *LastLane = Builder.CreateSub(ConstantInt::get(IndexTy, 1), RunTimeVF);

2057 Value *ResultPtr =

2059 ResultPtr = Builder.CreateGEP(IndexedTy, ResultPtr, LastLane, "",

2061

2062 State.set(this, ResultPtr, true);

2063}

2064

2065#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2068 O << Indent;

2070 O << " = reverse-vector-pointer";

2073}

2074#endif

2075

2077 auto &Builder = State.Builder;

2081 CurrentPart, Builder);

2083

2085 Value *ResultPtr =

2087

2088 State.set(this, ResultPtr, true);

2089}

2090

2091#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2094 O << Indent;

2096 O << " = vector-pointer ";

2097

2099}

2100#endif

2101

2105

2106

2107

2108

2109

2110

2111

2113

2114

2115

2116

2117

2118

2119

2120

2122 Value *Result = nullptr;

2123 for (unsigned In = 0; In < NumIncoming; ++In) {

2124

2125

2127 if (In == 0)

2128 Result = In0;

2129 else {

2130

2131

2134 }

2135 }

2136 State.set(this, Result, OnlyFirstLaneUsed);

2137}

2138

2141

2142

2145

2151}

2152

2153#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2156 O << Indent << "BLEND ";

2158 O << " =";

2160

2161

2162 O << " ";

2164 } else {

2166 O << " ";

2168 if (I == 0)

2169 continue;

2170 O << "/";

2172 }

2173 }

2174}

2175#endif

2176

2178 assert(!State.Lane && "Reduction being replicated.");

2181

2190

2194 else

2199

2202 }

2204 Value *NextInChain;

2205 if (IsOrdered) {

2207 NewRed =

2209 else

2212 PrevInChain = NewRed;

2213 NextInChain = NewRed;

2214 } else {

2215 PrevInChain = State.get(getChainOp(), true);

2219 NewRed, PrevInChain);

2220 else

2223 }

2224 State.set(this, NextInChain, true);

2225}

2226

2228 assert(!State.Lane && "Reduction being replicated.");

2229

2230 auto &Builder = State.Builder;

2231

2235

2240

2242 VBuilder.setEVL(EVL);

2244

2246 Mask = State.get(CondOp);

2247 else

2250

2254 } else {

2257 NewRed = createMinMaxOp(Builder, Kind, NewRed, Prev);

2258 else

2260 NewRed, Prev);

2261 }

2262 State.set(this, NewRed, true);

2263}

2264

2269 auto *VectorTy = cast(toVectorTy(ElementTy, VF));

2270 unsigned Opcode = RdxDesc.getOpcode();

2271

2272

2276 "Any-of reduction not implemented in VPlan-based cost model currently.");

2278 (!cast(getOperand(0))->isInLoop() ||

2280 "In-loop reduction not implemented in VPlan-based cost model currently.");

2281

2283 "Inferred type and recurrence type mismatch.");

2284

2285

2292 }

2293

2296}

2297

2298#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2301 O << Indent << "REDUCE ";

2303 O << " = ";

2305 O << " +";

2311 O << ", ";

2313 }

2314 O << ")";

2316 O << " (with final reduction value stored in invariant address sank "

2317 "outside of loop)";

2318}

2319

2323 O << Indent << "REDUCE ";

2325 O << " = ";

2327 O << " +";

2332 O << ", ";

2335 O << ", ";

2337 }

2338 O << ")";

2340 O << " (with final reduction value stored in invariant address sank "

2341 "outside of loop)";

2342}

2343#endif

2344

2346

2347

2349 if (auto *PredR = dyn_cast(U))

2350 return any_of(PredR->users(), [PredR](const VPUser *U) {

2351 return !U->usesScalars(PredR);

2352 });

2353 return false;

2354 });

2355}

2356

2360

2361

2364}

2365

2366#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2369 O << Indent << (IsUniform ? "CLONE " : "REPLICATE ");

2370

2373 O << " = ";

2374 }

2376 O << "call";

2378 O << "@" << CB->getCalledFunction()->getName() << "(";

2382 });

2383 O << ")";

2384 } else {

2388 }

2389

2391 O << " (S->V)";

2392}

2393#endif

2394

2398 "Codegen only implemented for first lane.");

2399 switch (Opcode) {

2400 case Instruction::SExt:

2401 case Instruction::ZExt:

2402 case Instruction::Trunc: {

2403

2406 }

2407 default:

2409 }

2410}

2411

2413 State.set(this, generate(State), VPLane(0));

2414}

2415

2416#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2419 O << Indent << "SCALAR-CAST ";

2423 O << " to " << *ResultTy;

2424}

2425#endif

2426

2428 assert(State.Lane && "Branch on Mask works only on single instance.");

2429

2430 unsigned Lane = State.Lane->getKnownLane();

2431

2432 Value *ConditionBit = nullptr;

2434 if (BlockInMask) {

2435 ConditionBit = State.get(BlockInMask);

2439 } else

2441

2442

2443

2445 assert(isa(CurrentTerminator) &&

2446 "Expected to replace unreachable terminator with conditional branch.");

2450}

2451

2454

2455

2456

2457 return 0;

2458}

2459

2462 assert(State.Lane && "Predicated instruction PHI works per instance.");

2467 assert(PredicatingBB && "Predicated block has no single predecessor.");

2469 "operand must be VPReplicateRecipe");

2470

2471

2472

2473

2474

2475

2476

2482 VPhi->addIncoming(IEI, PredicatedBB);

2484 State.reset(this, VPhi);

2485 else

2486 State.set(this, VPhi);

2487

2488

2490 } else {

2492 return;

2493

2497 PredicatingBB);

2498 Phi->addIncoming(ScalarPredInst, PredicatedBB);

2500 State.reset(this, Phi, *State.Lane);

2501 else

2502 State.set(this, Phi, *State.Lane);

2503

2504

2506 }

2507}

2508

2509#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2512 O << Indent << "PHI-PREDICATED-INSTRUCTION ";

2514 O << " = ";

2516}

2517#endif

2518

2522 const Align Alignment =

2524 unsigned AS =

2526

2528

2529

2530

2533 "Inconsecutive memory access should not have the order.");

2538 }

2539

2544 } else {

2549 }

2551 return Cost;

2552

2553 return Cost +=

2555 cast(Ty), {}, Ctx.CostKind, 0);

2556}

2557

2559 auto *LI = cast(&Ingredient);

2560

2565

2566 auto &Builder = State.Builder;

2568 Value *Mask = nullptr;

2569 if (auto *VPMask = getMask()) {

2570

2571

2572 Mask = State.get(VPMask);

2575 }

2576

2579 if (CreateGather) {

2581 "wide.masked.gather");

2582 } else if (Mask) {

2583 NewLI =

2586 } else {

2588 }

2589

2593 State.set(this, NewLI);

2594}

2595

2596#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2599 O << Indent << "WIDEN ";

2601 O << " = load ";

2603}

2604#endif

2605

2606

2607

2611 Value *AllTrueMask =

2613 return Builder.CreateIntrinsic(ValTy, Intrinsic::experimental_vp_reverse,

2614 {Operand, AllTrueMask, EVL}, nullptr, Name);

2615}

2616

2618 auto *LI = cast(&Ingredient);

2619

2624

2625 auto &Builder = State.Builder;

2630 Value *Mask = nullptr;

2632 Mask = State.get(VPMask);

2634 Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");

2635 } else {

2637 }

2638

2639 if (CreateGather) {

2640 NewLI =

2641 Builder.CreateIntrinsic(DataTy, Intrinsic::vp_gather, {Addr, Mask, EVL},

2642 nullptr, "wide.masked.gather");

2643 } else {

2647 Instruction::Load, DataTy, Addr, "vp.op.load"));

2648 }

2655 State.set(this, Res);

2656}

2657

2662

2663

2664

2665

2666

2667

2669 const Align Alignment =

2671 unsigned AS =

2676 return Cost;

2677

2679 cast(Ty), {}, Ctx.CostKind,

2680 0);

2681}

2682

2683#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2686 O << Indent << "WIDEN ";

2688 O << " = vp.load ";

2690}

2691#endif

2692

2694 auto *SI = cast(&Ingredient);

2695

2699

2700 auto &Builder = State.Builder;

2702

2703 Value *Mask = nullptr;

2704 if (auto *VPMask = getMask()) {

2705

2706

2707 Mask = State.get(VPMask);

2710 }

2711

2712 Value *StoredVal = State.get(StoredVPValue);

2714

2715

2717

2718

2719 }

2722 if (CreateScatter)

2724 else if (Mask)

2726 else

2729}

2730

2731#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2734 O << Indent << "WIDEN store ";

2736}

2737#endif

2738

2740 auto *SI = cast(&Ingredient);

2741

2745

2746 auto &Builder = State.Builder;

2748

2750 Value *StoredVal = State.get(StoredValue);

2753 StoredVal = createReverseEVL(Builder, StoredVal, EVL, "vp.reverse");

2754 Value *Mask = nullptr;

2756 Mask = State.get(VPMask);

2758 Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");

2759 } else {

2761 }

2763 if (CreateScatter) {

2765 Intrinsic::vp_scatter,

2766 {StoredVal, Addr, Mask, EVL});

2767 } else {

2772 {StoredVal, Addr}));

2773 }

2777}

2778

2783

2784

2785

2786

2787

2788

2790 const Align Alignment =

2792 unsigned AS =

2797 return Cost;

2798

2800 cast(Ty), {}, Ctx.CostKind,

2801 0);

2802}

2803

2804#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2807 O << Indent << "WIDEN vp.store ";

2809}

2810#endif

2811

2814

2815 auto VF = DstVTy->getElementCount();

2816 auto *SrcVecTy = cast(V->getType());

2817 assert(VF == SrcVecTy->getElementCount() && "Vector dimensions do not match");

2818 Type *SrcElemTy = SrcVecTy->getElementType();

2819 Type *DstElemTy = DstVTy->getElementType();

2820 assert((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) &&

2821 "Vector elements must have same size");

2822

2823

2826 }

2827

2828

2829

2830

2832 "Only one type should be a pointer type");

2834 "Only one type should be a floating point type");

2835 Type *IntTy =

2840}

2841

2842

2843

2846 unsigned Factor = Vals.size();

2847 assert(Factor > 1 && "Tried to interleave invalid number of vectors");

2848

2850#ifndef NDEBUG

2851 for (Value *Val : Vals)

2852 assert(Val->getType() == VecTy && "Tried to interleave mismatched types");

2853#endif

2854

2855

2856

2857 if (VecTy->isScalableTy()) {

2859 "scalable vectors, must be power of 2");

2861

2862

2863 auto *InterleaveTy = cast(InterleavingValues[0]->getType());

2864 for (unsigned Midpoint = Factor / 2; Midpoint > 0; Midpoint /= 2) {

2866 for (unsigned I = 0; I < Midpoint; ++I)

2868 InterleaveTy, Intrinsic::vector_interleave2,

2869 {InterleavingValues[I], InterleavingValues[Midpoint + I]},

2870 nullptr, Name);

2871 }

2872 return InterleavingValues[0];

2873 }

2874

2875

2877

2878

2879 const unsigned NumElts = VecTy->getElementCount().getFixedValue();

2882}

2883

2884

2885

2886

2887

2888

2889

2890

2891

2892

2893

2894

2895

2896

2897

2898

2899

2900

2901

2902

2903

2904

2905

2906

2907

2908

2909

2910

2911

2913 assert(!State.Lane && "Interleave group being replicated.");

2916

2917

2919 unsigned InterleaveFactor = Group->getFactor();

2920 auto *VecTy = VectorType::get(ScalarTy, State.VF * InterleaveFactor);

2921

2922

2925 "Reversed masked interleave-group not supported.");

2926

2929 if (auto *I = dyn_cast(ResAddr))

2931

2932

2933

2934

2935

2937 Value *RuntimeVF =

2944

2945 bool InBounds = false;

2946 if (auto *Gep = dyn_cast(ResAddr->stripPointerCasts()))

2947 InBounds = Gep->isInBounds();

2948 ResAddr = State.Builder.CreateGEP(ScalarTy, ResAddr, Index, "", InBounds);

2949 }

2950

2953

2954 auto CreateGroupMask = [&BlockInMask, &State,

2955 &InterleaveFactor](Value *MaskForGaps) -> Value * {

2957 assert(!MaskForGaps && "Interleaved groups with gaps are not supported.");

2959 "Unsupported deinterleave factor for scalable vectors");

2960 auto *ResBlockInMask = State.get(BlockInMask);

2963 }

2964

2965 if (!BlockInMask)

2966 return MaskForGaps;

2967

2968 Value *ResBlockInMask = State.get(BlockInMask);

2970 ResBlockInMask,

2972 "interleaved.mask");

2974 ShuffledMask, MaskForGaps)

2975 : ShuffledMask;

2976 };

2977

2978 const DataLayout &DL = Instr->getDataLayout();

2979

2980 if (isa(Instr)) {

2981 Value *MaskForGaps = nullptr;

2982 if (NeedsMaskForGaps) {

2985 assert(MaskForGaps && "Mask for Gaps is required but it is null");

2986 }

2987

2989 if (BlockInMask || MaskForGaps) {

2990 Value *GroupMask = CreateGroupMask(MaskForGaps);

2992 Group->getAlign(), GroupMask,

2993 PoisonVec, "wide.masked.vec");

2994 } else

2996 Group->getAlign(), "wide.vec");

2998

3001 if (VecTy->isScalableTy()) {

3003 "Unsupported deinterleave factor for scalable vectors");

3004

3005

3006

3008 DeinterleavedValues[0] = NewLoad;

3009

3010

3011

3012

3013

3014

3015 for (unsigned NumVectors = 1; NumVectors < InterleaveFactor;

3016 NumVectors *= 2) {

3017

3019 for (unsigned I = 0; I < NumVectors; ++I) {

3020 auto *DiTy = DeinterleavedValues[I]->getType();

3022 Intrinsic::vector_deinterleave2, DiTy, DeinterleavedValues[I],

3023 nullptr, "strided.vec");

3024 }

3025

3026 for (unsigned I = 0; I < 2; ++I)

3027 for (unsigned J = 0; J < NumVectors; ++J)

3028 DeinterleavedValues[NumVectors * I + J] =

3030 }

3031

3032#ifndef NDEBUG

3033 for (Value *Val : DeinterleavedValues)

3034 assert(Val && "NULL Deinterleaved Value");

3035#endif

3036 for (unsigned I = 0, J = 0; I < InterleaveFactor; ++I) {

3038 Value *StridedVec = DeinterleavedValues[I];

3039 if (!Member) {

3040

3041 cast(StridedVec)->eraseFromParent();

3042 continue;

3043 }

3044

3045 if (Member->getType() != ScalarTy) {

3047 StridedVec =

3049 }

3050

3053

3054 State.set(VPDefs[J], StridedVec);

3055 ++J;

3056 }

3057

3058 return;

3059 }

3060

3061

3062

3063 unsigned J = 0;

3064 for (unsigned I = 0; I < InterleaveFactor; ++I) {

3066

3067

3068 if (!Member)

3069 continue;

3070

3071 auto StrideMask =

3073 Value *StridedVec =

3075

3076

3077 if (Member->getType() != ScalarTy) {

3078 assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");

3080 StridedVec =

3082 }

3083

3086

3087 State.set(VPDefs[J], StridedVec);

3088 ++J;

3089 }

3090 return;

3091 }

3092

3093

3095

3096

3097 Value *MaskForGaps =

3100 "masking gaps for scalable vectors is not yet supported.");

3102

3104 unsigned StoredIdx = 0;

3105 for (unsigned i = 0; i < InterleaveFactor; i++) {

3107 "Fail to get a member from an interleaved store group");

3109

3110

3111 if (!Member) {

3114 continue;

3115 }

3116

3117 Value *StoredVec = State.get(StoredValues[StoredIdx]);

3118 ++StoredIdx;

3119

3122

3123

3124

3125 if (StoredVec->getType() != SubVT)

3127

3128 StoredVecs.push_back(StoredVec);

3129 }

3130

3131

3134 if (BlockInMask || MaskForGaps) {

3135 Value *GroupMask = CreateGroupMask(MaskForGaps);

3137 IVec, ResAddr, Group->getAlign(), GroupMask);

3138 } else

3139 NewStoreInstr =

3141

3143}

3144

3145#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3148 O << Indent << "INTERLEAVE-GROUP with factor " << IG->getFactor() << " at ";

3149 IG->getInsertPos()->printAsOperand(O, false);

3150 O << ", ";

3153 if (Mask) {

3154 O << ", ";

3156 }

3157

3158 unsigned OpIdx = 0;

3159 for (unsigned i = 0; i < IG->getFactor(); ++i) {

3160 if (!IG->getMember(i))

3161 continue;

3163 O << "\n" << Indent << " store ";

3165 O << " to index " << i;

3166 } else {

3167 O << "\n" << Indent << " ";

3169 O << " = load from index " << i;

3170 }

3171 ++OpIdx;

3172 }

3173}

3174#endif

3175

3179

3180 unsigned InsertPosIdx = 0;

3181 for (unsigned Idx = 0; IG->getFactor(); ++Idx)

3182 if (auto *Member = IG->getMember(Idx)) {

3183 if (Member == InsertPos)

3184 break;

3185 InsertPosIdx++;

3186 }

3190 auto *VectorTy = cast(toVectorTy(ValTy, VF));

3192

3193 unsigned InterleaveFactor = IG->getFactor();

3194 auto *WideVecTy = VectorType::get(ValTy, VF * InterleaveFactor);

3195

3196

3198 for (unsigned IF = 0; IF < InterleaveFactor; IF++)

3199 if (IG->getMember(IF))

3201

3202

3204 InsertPos->getOpcode(), WideVecTy, IG->getFactor(), Indices,

3205 IG->getAlign(), AS, Ctx.CostKind, getMask(), NeedsMaskForGaps);

3206

3207 if (!IG->isReverse())

3208 return Cost;

3209

3210 return Cost + IG->getNumMembers() *

3212 VectorTy, std::nullopt, Ctx.CostKind,

3213 0);

3214}

3215

3216#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3219 O << Indent << "EMIT ";

3221 O << " = CANONICAL-INDUCTION ";

3223}

3224#endif

3225

3227 return IsScalarAfterVectorization &&

3229}

3230

3234 "Not a pointer induction according to InductionDescriptor!");

3236 "Unexpected type.");

3238 "Recipe should have been replaced");

3239

3241

3242

3244 Type *ScStValueType = ScalarStartValue->getType();

3245

3247 PHINode *NewPointerPhi = nullptr;

3248 if (CurrentPart == 0) {

3249 auto *IVR = cast(&getParent()

3250 ->getPlan()

3251 ->getVectorLoopRegion()

3252 ->getEntryBasicBlock()

3253 ->front());

3254 PHINode *CanonicalIV = cast(State.get(IVR, true));

3255 NewPointerPhi = PHINode::Create(ScStValueType, 2, "pointer.phi",

3257 NewPointerPhi->addIncoming(ScalarStartValue, VectorPH);

3259 } else {

3260

3261

3262 auto *GEP =

3264 NewPointerPhi = cast(GEP->getPointerOperand());

3265 }

3266

3267

3272

3273

3274

3275

3276

3277 if (CurrentPart == 0) {

3278

3279

3281 Value *NumUnrolledElems =

3282 State.Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, UF));

3283

3286 State.Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind",

3287 InductionLoc);

3288

3289 NewPointerPhi->addIncoming(InductionGEP, VectorPH);

3290 }

3291

3292

3293

3296 RuntimeVF, ConstantInt::get(PhiType, CurrentPart));

3297 Value *StartOffset =

3299

3302

3304 "scalar step must be the same across all parts");

3308 State.VF, ScalarStepValue)),

3309 "vector.gep");

3310 State.set(this, GEP);

3311}

3312

3313#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3317 "unexpected number of operands");

3318 O << Indent << "EMIT ";

3320 O << " = WIDEN-POINTER-INDUCTION ";

3322 O << ", ";

3325 O << ", ";

3327 O << ", ";

3329 }

3330}

3331#endif

3332

3334 assert(!State.Lane && "cannot be used in per-lane");

3336

3337

3338

3339

3342 "Results must match");

3343 return;

3344 }

3345

3348

3349 Value *Res = Exp.expandCodeFor(Expr, Expr->getType(),

3352 State.set(this, Res, VPLane(0));

3353}

3354

3355#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3358 O << Indent << "EMIT ";

3360 O << " = EXPAND SCEV " << *Expr;

3361}

3362#endif

3363

3370 ? CanonicalIV

3375 VStep =

3377 }

3378 Value *CanonicalVectorIV = Builder.CreateAdd(VStart, VStep, "vec.iv");

3379 State.set(this, CanonicalVectorIV);

3380}

3381

3382#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3385 O << Indent << "EMIT ";

3387 O << " = WIDEN-CANONICAL-INDUCTION ";

3389}

3390#endif

3391

3393 auto &Builder = State.Builder;

3394

3396

3398 ? VectorInit->getType()

3400

3404 auto *One = ConstantInt::get(IdxTy, 1);

3407 auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, State.VF);

3408 auto *LastIdx = Builder.CreateSub(RuntimeVF, One);

3410 PoisonValue::get(VecTy), VectorInit, LastIdx, "vector.recur.init");

3411 }

3412

3413

3416 Phi->addIncoming(VectorInit, VectorPH);

3417 State.set(this, Phi);

3418}

3419

3425

3428

3430 std::iota(Mask.begin(), Mask.end(), VF.getKnownMinValue() - 1);

3431 Type *VectorTy =

3433

3435 cast(VectorTy), Mask, Ctx.CostKind,

3437}

3438

3439#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3442 O << Indent << "FIRST-ORDER-RECURRENCE-PHI ";

3444 O << " = phi ";

3446}

3447#endif

3448

3450 auto &Builder = State.Builder;

3451

3452

3453

3455

3456

3457

3460

3461

3462

3463

3464

3465 bool ScalarPHI = State.VF.isScalar() || IsInLoop;

3466 Type *VecTy =

3468

3471 "recipe must be in the vector loop header");

3474 State.set(this, Phi, IsInLoop);

3475

3477

3478 Value *Iden = nullptr;

3481

3484

3485 if (ScalarPHI) {

3486 Iden = StartV;

3487 } else {

3490 StartV = Iden = State.get(StartVPV);

3491 }

3493

3494

3495

3496

3497

3498

3499

3500

3501

3502 Iden = StartV;

3503 if (!ScalarPHI) {

3507 }

3508 } else {

3511

3512 if (!ScalarPHI) {

3513 if (CurrentPart == 0) {

3514

3515

3516

3522 } else {

3524 }

3525 }

3526 }

3527

3528 Phi = cast(State.get(this, IsInLoop));

3529 Value *StartVal = (CurrentPart == 0) ? StartV : Iden;

3530 Phi->addIncoming(StartVal, VectorPH);

3531}

3532

3533#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3536 O << Indent << "WIDEN-REDUCTION-PHI ";

3537

3539 O << " = phi ";

3541 if (VFScaleFactor != 1)

3542 O << " (VF scaled by 1/" << VFScaleFactor << ")";

3543}

3544#endif

3545

3548 "Non-native vplans are not expected to have VPWidenPHIRecipes.");

3549

3553 State.set(this, VecPhi);

3554}

3555

3556#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3559 O << Indent << "WIDEN-PHI ";

3560

3562

3563

3564

3565

3566 if (getNumOperands() != OriginalPhi->getNumOperands()) {

3568 return;

3569 }

3570

3572 O << " = phi ";

3574}

3575#endif

3576

3577

3578

3584 Phi->addIncoming(StartMask, VectorPH);

3586 State.set(this, Phi);

3587}

3588

3589#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3592 O << Indent << "ACTIVE-LANE-MASK-PHI ";

3593

3595 O << " = phi ";

3597}

3598#endif

3599

3600#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3603 O << Indent << "EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ";

3604

3606 O << " = phi ";

3608}

3609#endif

3610

3615 Phi->addIncoming(Start, VectorPH);

3617 State.set(this, Phi, true);

3618}

3619

3620#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3623 O << Indent << "SCALAR-PHI ";

3625 O << " = phi ";

3627}

3628#endif

AMDGPU Lower Kernel Arguments

AMDGPU Register Bank Select

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx

cl::opt< unsigned > ForceTargetInstructionCost("force-target-instruction-cost", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's expected cost for " "an instruction to a single constant value. Mostly " "useful for getting consistent testing."))

mir Rename Register Operands

static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)

Return the first found DebugLoc that has a DILocation, given a range of instructions.

const SmallVectorImpl< MachineOperand > & Cond

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

This file defines the SmallVector class.

static SymbolRef::Type getType(const Symbol *Sym)

static Instruction * createReverseEVL(IRBuilderBase &Builder, Value *Operand, Value *EVL, const Twine &Name)

Use all-true mask for reverse rather than actual mask, as it avoids a dependence w/o affecting the re...

static Value * interleaveVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vals, const Twine &Name)

Return a vector containing interleaved elements from multiple smaller input vectors.

static Value * createBitOrPointerCast(IRBuilderBase &Builder, Value *V, VectorType *DstVTy, const DataLayout &DL)

cl::opt< unsigned > ForceTargetInstructionCost

static Value * getStepVector(Value *Val, Value *Step, Instruction::BinaryOps BinOp, ElementCount VF, IRBuilderBase &Builder)

This function adds (0 * Step, 1 * Step, 2 * Step, ...) to each vector element of Val.

static Type * getGEPIndexTy(bool IsScalable, bool IsReverse, unsigned CurrentPart, IRBuilderBase &Builder)

static Constant * getSignedIntOrFpConstant(Type *Ty, int64_t C)

A helper function that returns an integer or floating-point constant with value C.

This file contains the declarations of the Vectorization Plan base classes:

static const uint32_t IV[8]

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

static Attribute getWithAlignment(LLVMContext &Context, Align Alignment)

Return a uniquified Attribute object that has the specific alignment set.

LLVM Basic Block Representation.

const_iterator getFirstInsertionPt() const

Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...

InstListType::const_iterator getFirstNonPHIIt() const

Iterator returning form of getFirstNonPHI.

const BasicBlock * getSinglePredecessor() const

Return the predecessor of this block if it has a single predecessor block.

const DataLayout & getDataLayout() const

Get the data layout of the module this basic block belongs to.

InstListType::iterator iterator

Instruction iterators...

const Instruction * getTerminator() const LLVM_READONLY

Returns the terminator instruction if the block is well formed or null if the block is not well forme...

const Module * getModule() const

Return the module owning the function this basic block belongs to, or nullptr if the function does no...

Conditional or Unconditional Branch instruction.

static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)

void setSuccessor(unsigned idx, BasicBlock *NewSucc)

void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)

Adds the attribute to the indicated argument.

This class represents a function call, abstracting a target machine's calling convention.

static bool isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy, const DataLayout &DL)

Check whether a bitcast, inttoptr, or ptrtoint cast between these types is valid and a no-op.

Predicate

This enumeration lists the possible predicates for CmpInst subclasses.

@ ICMP_UGT

unsigned greater than

@ ICMP_ULT

unsigned less than

static StringRef getPredicateName(Predicate P)

This is the shared class of boolean and integer constants.

static ConstantInt * getSigned(IntegerType *Ty, int64_t V)

Return a ConstantInt with the specified value for the specified type.

uint64_t getZExtValue() const

Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...

This is an important base class in LLVM.

This class represents an Operation in the Expression.

A parsed version of the target data layout string in and methods for querying it.

constexpr bool isVector() const

One or more elements.

constexpr bool isScalar() const

Exactly one element.

Convenience struct for specifying and reasoning about fast-math flags.

void setAllowContract(bool B=true)

bool noSignedZeros() const

void setAllowReciprocal(bool B=true)

bool allowReciprocal() const

void print(raw_ostream &O) const

Print fast-math flags to O.

void setNoSignedZeros(bool B=true)

bool allowReassoc() const

Flag queries.

void setNoNaNs(bool B=true)

void setAllowReassoc(bool B=true)

Flag setters.

void setApproxFunc(bool B=true)

void setNoInfs(bool B=true)

bool allowContract() const

Class to represent function types.

Type * getParamType(unsigned i) const

Parameter type accessors.

ArrayRef< Type * > params() const

FunctionType * getFunctionType() const

Returns the FunctionType for me.

bool willReturn() const

Determine if the function will return.

bool doesNotThrow() const

Determine if the function cannot unwind.

Type * getReturnType() const

Returns the type of the ret val.

bool hasNoUnsignedSignedWrap() const

bool hasNoUnsignedWrap() const

static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)

Common base class shared among various IRBuilders.

ConstantInt * getInt1(bool V)

Get a constant value representing either true or false.

Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")

Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")

Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")

LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)

Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")

Create a ZExt or Trunc from the integer value V to DestTy.

Value * CreateVectorSplice(Value *V1, Value *V2, int64_t Imm, const Twine &Name="")

Return a vector splice intrinsic if using scalable vectors, otherwise return a shufflevector.

Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")

Return a vector value that contains.

Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")

ConstantInt * getTrue()

Get the constant value for i1 true.

CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")

Create a call to Masked Load intrinsic.

Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)

BasicBlock::iterator GetInsertPoint() const

Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")

Value * CreateFreeze(Value *V, const Twine &Name="")

IntegerType * getInt32Ty()

Fetch the type representing a 32-bit integer.

Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())

Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr, FMFSource FMFSource={})

Value * CreateUIToFP(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)

BasicBlock * GetInsertBlock() const

void setFastMathFlags(FastMathFlags NewFMF)

Set the fast-math flags to be used with generated fp-math operators.

Value * CreateVectorReverse(Value *V, const Twine &Name="")

Return a vector value that contains the vector V reversed.

Value * CreateFCmpFMF(CmpInst::Predicate P, Value *LHS, Value *RHS, FMFSource FMFSource, const Twine &Name="", MDNode *FPMathTag=nullptr)

Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())

Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)

CallInst * CreateOrReduce(Value *Src)

Create a vector int OR reduction intrinsic of the source vector.

InsertPoint saveIP() const

Returns the current insert point.

CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")

Create a call to intrinsic ID with Args, mangled using Types.

ConstantInt * getInt32(uint32_t C)

Get a constant 32-bit value.

Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")

Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)

PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")

Value * CreateNot(Value *V, const Twine &Name="")

Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)

Create a conditional 'br Cond, TrueDest, FalseDest' instruction.

Value * CreateNAryOp(unsigned Opc, ArrayRef< Value * > Ops, const Twine &Name="", MDNode *FPMathTag=nullptr)

Create either a UnaryOperator or BinaryOperator depending on Opc.

Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)

Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")

LLVMContext & getContext() const

CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)

Create a call to Masked Store intrinsic.

Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)

Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)

PointerType * getPtrTy(unsigned AddrSpace=0)

Fetch the type representing a pointer.

Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)

Value * CreateLogicalAnd(Value *Cond1, Value *Cond2, const Twine &Name="")

void restoreIP(InsertPoint IP)

Sets the current insert point to a previously-saved location.

void SetInsertPoint(BasicBlock *TheBB)

This specifies that created instructions should be appended to the end of the specified block.

StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)

Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)

IntegerType * getInt8Ty()

Fetch the type representing an 8-bit integer.

Value * CreateStepVector(Type *DstType, const Twine &Name="")

Creates a vector of type DstType with the linear sequence <0, 1, ...>

Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

CallInst * CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment, Value *Mask=nullptr)

Create a call to Masked Scatter intrinsic.

CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")

Create a call to Masked Gather intrinsic.

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

A struct for saving information about induction variables.

@ IK_PtrInduction

Pointer induction var. Step = C.

This instruction inserts a single (scalar) element into a VectorType value.

VectorType * getType() const

Overload to return most specific vector type.

static InstructionCost getInvalid(CostType Val=0)

void insertBefore(Instruction *InsertPos)

Insert an unlinked instruction into a basic block immediately before the specified instruction.

InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

FastMathFlags getFastMathFlags() const LLVM_READONLY

Convenience function for getting all the fast-math flags, which must be an operator which supports th...

const char * getOpcodeName() const

unsigned getOpcode() const

Returns a member of one of the enums like Instruction::Add.

void setDebugLoc(DebugLoc Loc)

Set the debug location information for this instruction.

static IntegerType * get(LLVMContext &C, unsigned NumBits)

This static method is the primary way of constructing an IntegerType.

The group of interleaved loads/stores sharing the same stride and close to each other.

uint32_t getFactor() const

InstTy * getMember(uint32_t Index) const

Get the member with the given index Index.

InstTy * getInsertPos() const

void addMetadata(InstTy *NewInst) const

Add metadata (e.g.

BlockT * getHeader() const

void print(raw_ostream &OS, const SlotIndexes *=nullptr, bool IsStandalone=true) const

A Module instance is used to store all the information related to an LLVM module.

void addIncoming(Value *V, BasicBlock *BB)

Add an incoming value to the end of the PHI list.

static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)

Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...

static PoisonValue * get(Type *T)

Static factory methods - Return an 'poison' object of the specified type.

The RecurrenceDescriptor is used to identify recurrences variables in a loop.

FastMathFlags getFastMathFlags() const

static unsigned getOpcode(RecurKind Kind)

Returns the opcode corresponding to the RecurrenceKind.

Type * getRecurrenceType() const

Returns the type of the recurrence.

TrackingVH< Value > getRecurrenceStartValue() const

static bool isAnyOfRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...

static bool isFindLastIVRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...

bool isSigned() const

Returns true if all source operands of the recurrence are SExtInsts.

RecurKind getRecurrenceKind() const

StoreInst * IntermediateStore

Reductions may store temporary or final result to an invariant address.

static bool isMinMaxRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is any min/max kind.

This class uses information about analyze scalars to rewrite expressions in canonical form.

Type * getType() const

Return the LLVM type of this SCEV expression.

This class represents the LLVM 'select' instruction.

This class provides computation of slot numbers for LLVM Assembly writing.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

InstructionCost getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, OperandValueInfo Op1Info={OK_AnyValue, OP_None}, OperandValueInfo Op2Info={OK_AnyValue, OP_None}, const Instruction *I=nullptr) const

InstructionCost getAddressComputationCost(Type *Ty, ScalarEvolution *SE=nullptr, const SCEV *Ptr=nullptr) const

InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, OperandValueInfo OpdInfo={OK_AnyValue, OP_None}, const Instruction *I=nullptr) const

InstructionCost getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef< unsigned > Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, bool UseMaskForCond=false, bool UseMaskForGaps=false) const

InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind) const

InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *Ty, std::optional< FastMathFlags > FMF, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput) const

Calculate the cost of vector reduction intrinsics.

InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH, TTI::TargetCostKind CostKind=TTI::TCK_SizeAndLatency, const Instruction *I=nullptr) const

static OperandValueInfo getOperandInfo(const Value *V)

Collect properties of V used in cost analysis, e.g. OP_PowerOf2.

InstructionCost getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF=FastMathFlags(), TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput) const

InstructionCost getArithmeticInstrCost(unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, TTI::OperandValueInfo Opd1Info={TTI::OK_AnyValue, TTI::OP_None}, TTI::OperandValueInfo Opd2Info={TTI::OK_AnyValue, TTI::OP_None}, ArrayRef< const Value * > Args={}, const Instruction *CxtI=nullptr, const TargetLibraryInfo *TLibInfo=nullptr) const

This is an approximation of reciprocal throughput of a math/logic op.

InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput) const

InstructionCost getShuffleCost(ShuffleKind Kind, VectorType *Tp, ArrayRef< int > Mask={}, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, int Index=0, VectorType *SubTp=nullptr, ArrayRef< const Value * > Args={}, const Instruction *CxtI=nullptr) const

InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, Align Alignment, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, const Instruction *I=nullptr) const

@ TCC_Free

Expected to fold away in lowering.

InstructionCost getPartialReductionCost(unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType, ElementCount VF, PartialReductionExtendKind OpAExtend, PartialReductionExtendKind OpBExtend, std::optional< unsigned > BinOp=std::nullopt) const

@ SK_Splice

Concatenates elements from the first input vector with elements of the second input vector.

@ SK_Reverse

Reverse the order of the vector.

InstructionCost getCallInstrCost(Function *F, Type *RetTy, ArrayRef< Type * > Tys, TTI::TargetCostKind CostKind=TTI::TCK_SizeAndLatency) const

InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind=TTI::TCK_SizeAndLatency, const Instruction *I=nullptr) const

CastContextHint

Represents a hint about the context in which a cast is used.

@ Reversed

The cast is used with a reversed load/store.

@ Masked

The cast is used with a masked load/store.

@ None

The cast is not used with a load/store of any kind.

@ Normal

The cast is used with a normal load/store.

@ Interleave

The cast is used with an interleaved load/store.

@ GatherScatter

The cast is used with a gather/scatter.

This class represents a truncation of integer types.

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

The instances of the Type class are immutable: once they are created, they are never changed.

bool isVectorTy() const

True if this is an instance of VectorType.

bool isPointerTy() const

True if this is an instance of PointerType.

static IntegerType * getInt1Ty(LLVMContext &C)

static IntegerType * getIntNTy(LLVMContext &C, unsigned N)

unsigned getScalarSizeInBits() const LLVM_READONLY

If this is a vector type, return the getPrimitiveSizeInBits value for the element type.

static Type * getVoidTy(LLVMContext &C)

LLVMContext & getContext() const

Return the LLVMContext in which this type was uniqued.

bool isFloatingPointTy() const

Return true if this is one of the floating-point types.

bool isIntegerTy() const

True if this is an instance of IntegerType.

TypeID getTypeID() const

Return the type id for the type.

bool isVoidTy() const

Return true if this is 'void'.

Type * getScalarType() const

If this is a vector type, return the element type, otherwise return 'this'.

value_op_iterator value_op_end()

Value * getOperand(unsigned i) const

value_op_iterator value_op_begin()

void execute(VPTransformState &State) override

Generate the active lane mask phi of the vector loop.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.

RecipeListTy & getRecipeList()

Returns a reference to the list of recipes.

void insert(VPRecipeBase *Recipe, iterator InsertPt)

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenMemoryRecipe.

VPValue * getIncomingValue(unsigned Idx) const

Return incoming value number Idx.

VPValue * getMask(unsigned Idx) const

Return mask number Idx.

unsigned getNumIncomingValues() const

Return the number of incoming values, taking into account when normalized the first incoming value wi...

void execute(VPTransformState &State) override

Generate the phi/select nodes.

bool isNormalized() const

A normalized blend is one that has an odd number of operands, whereby the first operand does not have...

VPBlockBase is the building block of the Hierarchical Control-Flow Graph.

VPRegionBlock * getParent()

const VPBasicBlock * getExitingBasicBlock() const

const VPBlocksTy & getPredecessors() const

const VPBasicBlock * getEntryBasicBlock() const

VPValue * getMask() const

Return the mask used by this recipe.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPBranchOnMaskRecipe.

void execute(VPTransformState &State) override

Generate the extraction of the appropriate bit from the block mask and the conditional branch.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

This class augments a recipe with a set of VPValues defined by the recipe.

void dump() const

Dump the VPDef to stderr (for debugging).

unsigned getNumDefinedValues() const

Returns the number of values defined by the VPDef.

ArrayRef< VPValue * > definedValues()

Returns an ArrayRef of the values defined by the VPDef.

VPValue * getVPSingleValue()

Returns the only VPValue defined by the VPDef.

VPValue * getVPValue(unsigned I)

Returns the VPValue with index I defined by the VPDef.

unsigned getVPDefID() const

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

VPValue * getStepValue() const

VPValue * getStartValue() const

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

Generate a canonical vector induction variable of the vector loop, with.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

Produce a vectorized histogram operation.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPHistogramRecipe.

VPValue * getMask() const

Return the mask operand if one was provided, or a null pointer if all lanes should be executed uncond...

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

The method which generates the output IR instructions that correspond to this VPRecipe,...

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPIRInstruction.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

@ ResumePhi

Creates a scalar phi in a leaf VPBB with a single predecessor in VPlan.

@ FirstOrderRecurrenceSplice

@ CanonicalIVIncrementForPart

@ CalculateTripCountMinusVF

bool opcodeMayReadOrWriteFromMemory() const

Returns true if the underlying opcode may read from or write to memory.

LLVM_DUMP_METHOD void dump() const

Print the VPInstruction to dbgs() (for debugging).

unsigned getOpcode() const

bool onlyFirstPartUsed(const VPValue *Op) const override

Returns true if the recipe only uses the first part of operand Op.

bool isVectorToScalar() const

Returns true if this VPInstruction produces a scalar value from a vector, e.g.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the VPInstruction to O.

bool onlyFirstLaneUsed(const VPValue *Op) const override

Returns true if the recipe only uses the first lane of operand Op.

bool isSingleScalar() const

Returns true if this VPInstruction's operands are single scalars and the result is also a single scal...

void execute(VPTransformState &State) override

Generate the instruction.

VPValue * getAddr() const

Return the address accessed by this recipe.

VPValue * getMask() const

Return the mask used by this recipe.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

Generate the wide load or store, and shuffles.

ArrayRef< VPValue * > getStoredValues() const

Return the VPValues stored by this interleave group.

Instruction * getInsertPos() const

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPInterleaveRecipe.

unsigned getNumStoreOperands() const

Returns the number of stored operands of this interleave group.

static bool isVPIntrinsic(Intrinsic::ID)

In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...

static VPLane getLastLaneForVF(const ElementCount &VF)

static VPLane getLaneFromEnd(const ElementCount &VF, unsigned Offset)

static VPLane getFirstLane()

void execute(VPTransformState &State) override

Generate the reduction in the loop.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPPartialReductionRecipe.

unsigned getOpcode() const

Get the binary op's opcode.

void execute(VPTransformState &State) override

Generates phi nodes for live-outs (from a replicate region) as needed to retain SSA form.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.

bool mayReadFromMemory() const

Returns true if the recipe may read from memory.

bool mayHaveSideEffects() const

Returns true if the recipe may have side-effects.

bool mayWriteToMemory() const

Returns true if the recipe may write to memory.

virtual InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const

Compute the cost of this recipe either using a recipe's specialized implementation or using the legac...

VPBasicBlock * getParent()

DebugLoc getDebugLoc() const

Returns the debug location of the recipe.

void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)

Unlink this recipe and insert into BB before I.

void insertBefore(VPRecipeBase *InsertPos)

Insert an unlinked recipe into a basic block immediately before the specified recipe.

void insertAfter(VPRecipeBase *InsertPos)

Insert an unlinked Recipe into a basic block immediately after the specified Recipe.

iplist< VPRecipeBase >::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

InstructionCost cost(ElementCount VF, VPCostContext &Ctx)

Return the cost of this recipe, taking into account if the cost computation should be skipped and the...

void removeFromParent()

This method unlinks 'this' from the containing basic block, but does not delete it.

void moveAfter(VPRecipeBase *MovePos)

Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...

Class to record LLVM IR flag for a recipe along with it.

NonNegFlagsTy NonNegFlags

GEPNoWrapFlags getGEPNoWrapFlags() const

void setFlags(Instruction *I) const

Set the IR flags for I.

bool hasFastMathFlags() const

Returns true if the recipe has fast-math flags.

DisjointFlagsTy DisjointFlags

bool hasNoUnsignedWrap() const

void printFlags(raw_ostream &O) const

CmpInst::Predicate getPredicate() const

bool hasNoSignedWrap() const

FastMathFlags getFastMathFlags() const

void execute(VPTransformState &State) override

Generate the reduction in the loop.

VPValue * getEVL() const

The VPValue of the explicit vector length.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

Generate the phi/select nodes.

bool isConditional() const

Return true if the in-loop reduction is conditional.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of VPReductionRecipe.

VPValue * getVecOp() const

The VPValue of the vector value to be reduced.

const RecurrenceDescriptor & getRecurrenceDescriptor() const

Return the recurrence decriptor for the in-loop reduction.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

VPValue * getCondOp() const

The VPValue of the condition for the block.

bool isOrdered() const

Return true if the in-loop reduction is ordered.

VPValue * getChainOp() const

The VPValue of the scalar Chain being accumulated.

void execute(VPTransformState &State) override

Generate the reduction in the loop.

VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...

const VPBlockBase * getEntry() const

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPReplicateRecipe.

unsigned getOpcode() const

bool shouldPack() const

Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

The method which generates the output IR instructions that correspond to this VPRecipe,...

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

VPValue * getStepValue() const

void execute(VPTransformState &State) override

Generate the scalarized versions of the phi node as needed by their users.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

Generate the phi/select nodes.

Instruction * getUnderlyingInstr()

Returns the underlying instruction.

LLVM_DUMP_METHOD void dump() const

Print this VPSingleDefRecipe to dbgs() (for debugging).

This class can be used to assign names to VPValues.

LLVMContext & getContext()

Return the LLVMContext used by the analysis.

Type * inferScalarType(const VPValue *V)

Infer the type of V. Returns the scalar type of V.

VPValue * getUnrollPartOperand(VPUser &U) const

Return the VPValue operand containing the unroll part or null if there is no such operand.

unsigned getUnrollPart(VPUser &U) const

Return the unroll part.

This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...

void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const

Print the operands to O.

unsigned getNumOperands() const

operand_iterator op_begin()

VPValue * getOperand(unsigned N) const

virtual bool onlyFirstLaneUsed(const VPValue *Op) const

Returns true if the VPUser only uses the first lane of operand Op.

bool isDefinedOutsideLoopRegions() const

Returns true if the VPValue is defined outside any loop region.

VPRecipeBase * getDefiningRecipe()

Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...

void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const

friend class VPInstruction

bool hasMoreThanOneUniqueUser() const

Returns true if the value has more than one unique user.

Value * getUnderlyingValue() const

Return the underlying Value attached to this VPValue.

user_iterator user_begin()

unsigned getNumUsers() const

Value * getLiveInIRValue()

Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.

bool isLiveIn() const

Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

The method which generates the output IR instructions that correspond to this VPRecipe,...

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Function * getCalledScalarFunction() const

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenCallRecipe.

void execute(VPTransformState &State) override

Produce a widened version of the call instruction.

operand_range arg_operands()

void execute(VPTransformState &State) override

Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Type * getResultType() const

Returns the result type of the cast.

void execute(VPTransformState &State) override

Produce widened copies of the cast.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenCastRecipe.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override final

Print the recipe.

void execute(VPTransformState &State) override final

Produce a vp-intrinsic using the opcode and operands of the recipe, processing EVL elements.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

Generate the gep nodes.

PHINode * getPHINode() const

VPValue * getStepValue()

Returns the step value of the induction.

const InductionDescriptor & getInductionDescriptor() const

Returns the induction descriptor for the recipe.

TruncInst * getTruncInst()

Returns the first defined value as TruncInst, if it is one or nullptr otherwise.

void execute(VPTransformState &State) override

Generate the vectorized and scalarized versions of the phi node as needed by their users.

Type * getScalarType() const

Returns the scalar type of the induction.

bool isCanonical() const

Returns true if the induction is canonical, i.e.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

VPValue * getSplatVFValue()

bool onlyFirstLaneUsed(const VPValue *Op) const override

Returns true if the VPUser only uses the first lane of operand Op.

StringRef getIntrinsicName() const

Return to name of the intrinsic as string.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Type * getResultType() const

Return the scalar return type of the intrinsic.

void execute(VPTransformState &State) override

Produce a widened version of the vector intrinsic.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this vector intrinsic.

bool IsMasked

Whether the memory access is masked.

bool Reverse

Whether the consecutive accessed addresses are in reverse order.

bool isConsecutive() const

Return whether the loaded-from / stored-to addresses are consecutive.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenMemoryRecipe.

bool Consecutive

Whether the accessed addresses are consecutive.

VPValue * getMask() const

Return the mask used by this recipe.

VPValue * getAddr() const

Return the address accessed by this recipe.

bool isReverse() const

Return whether the consecutive loaded/stored addresses are in reverse order.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

Generate the phi/select nodes.

bool onlyScalarsGenerated(bool IsScalable)

Returns true if only scalar values will be generated.

VPValue * getFirstUnrolledPartOperand()

Returns the VPValue representing the value of this induction at the first unrolled part,...

void execute(VPTransformState &State) override

Generate vector values for the pointer induction.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenRecipe.

void execute(VPTransformState &State) override

Produce a widened instruction using the opcode and operands of the recipe, processing State....

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

unsigned getOpcode() const

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

void setName(const Twine &Name)

Change the name of the value.

const Value * stripPointerCasts() const

Strip off pointer casts, all-zero GEPs and address space casts.

LLVMContext & getContext() const

All values hold a context through their type.

StringRef getName() const

Return a constant reference to the value's name.

VectorBuilder & setEVL(Value *NewExplicitVectorLength)

VectorBuilder & setMask(Value *NewMask)

Value * createVectorInstruction(unsigned Opcode, Type *ReturnTy, ArrayRef< Value * > VecOpArray, const Twine &Name=Twine())

Base class of all SIMD vector types.

ElementCount getElementCount() const

Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...

static VectorType * get(Type *ElementType, ElementCount EC)

This static method is the primary way to construct an VectorType.

static VectorType * getDoubleElementsVectorType(VectorType *VTy)

This static method returns a VectorType with twice as many elements as the input type and the same el...

Type * getElementType() const

constexpr bool isScalable() const

Returns whether the quantity is scaled by a runtime quantity (vscale).

constexpr ScalarTy getKnownMinValue() const

Returns the minimum value this quantity can represent.

constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const

We do not provide the '/' operator here because division for polynomial types does not work in the sa...

const ParentTy * getParent() const

self_iterator getIterator()

base_list_type::iterator iterator

iterator erase(iterator where)

pointer remove(iterator &IT)

This class implements an extremely fast bulk output stream that can only output to a stream.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

@ C

The default llvm calling convention, compatible with C.

Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})

Look up the Function declaration of the intrinsic id in the Module M.

StringRef getBaseName(ID id)

Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....

bool match(Val *V, const Pattern &P)

auto m_LogicalOr()

Matches L || R where L and R are arbitrary values.

auto m_LogicalAnd()

Matches L && R where L and R are arbitrary values.

bool isUniformAfterVectorization(const VPValue *VPV)

Returns true if VPV is uniform after vectorization.

bool onlyFirstPartUsed(const VPValue *Def)

Returns true if only the first part of Def is used.

bool onlyFirstLaneUsed(const VPValue *Def)

Returns true if only the first lane of Def is used.

This is an optimization pass for GlobalISel generic memory operations.

void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)

Replace the instruction specified by BI with the instruction specified by I.

Value * createSimpleReduction(IRBuilderBase &B, Value *Src, RecurKind RdxKind)

Create a reduction of the given vector.

bool all_of(R &&range, UnaryPredicate P)

Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.

unsigned getLoadStoreAddressSpace(const Value *I)

A helper function that returns the address space of the pointer operand of load or store instruction.

Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)

Returns the min/max intrinsic used when expanding a min/max reduction.

auto enumerate(FirstRange &&First, RestRanges &&...Rest)

Given two or more input ranges, returns a new range whose values are tuples (A, B,...

const Value * getLoadStorePointerOperand(const Value *V)

A helper function that returns the pointer operand of a load or store instruction.

Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)

Return the runtime value for VF.

iterator_range< T > make_range(T x, T y)

Convenience function for iterating over sub-ranges.

void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)

Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)

Concatenate a list of vectors.

Align getLoadStoreAlignment(const Value *I)

A helper function that returns the alignment of load or store instruction.

Value * createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left, Value *Right)

Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.

bool any_of(R &&range, UnaryPredicate P)

Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.

Constant * createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, const InterleaveGroup< Instruction > &Group)

Create a mask that filters the members of an interleave group where there are gaps.

llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)

Create a stride shuffle mask.

constexpr bool isPowerOf2_32(uint32_t Value)

Return true if the argument is a power of two > 0.

cl::opt< bool > EnableVPlanNativePath("enable-vplan-native-path", cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))

llvm::SmallVector< int, 16 > createReplicatedMask(unsigned ReplicationFactor, unsigned VF)

Create a mask with replicated elements.

raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

bool isPointerTy(const Type *T)

Value * createOrderedReduction(IRBuilderBase &B, const RecurrenceDescriptor &Desc, Value *Src, Value *Start)

Create an ordered reduction intrinsic using the given recurrence descriptor Desc.

Value * createReduction(IRBuilderBase &B, const RecurrenceDescriptor &Desc, Value *Src, PHINode *OrigPhi=nullptr)

Create a generic reduction using a recurrence descriptor Desc Fast-math-flags are propagated using th...

llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)

Create an interleave shuffle mask.

RecurKind

These are the kinds of recurrences that we support.

@ Mul

Product of integers.

@ SMax

Signed integer max implemented in terms of select(cmp()).

bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx, const TargetTransformInfo *TTI)

Identifies if the vector form of the intrinsic has a scalar operand.

Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF)

Given information about an recurrence kind, return the identity for the @llvm.vector....

DWARFExpression::Operation Op

Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)

Return a value for Step multiplied by VF.

auto predecessors(const MachineBasicBlock *BB)

bool is_contained(R &&Range, const E &Element)

Returns true if Element is found in Range.

Type * getLoadStoreType(const Value *I)

A helper function that returns the type of a load or store instruction.

Type * toVectorTy(Type *Scalar, ElementCount EC)

A helper function for converting Scalar types to vector types.

bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx, const TargetTransformInfo *TTI)

Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...

This struct is a compact representation of a valid (non-zero power of two) alignment.

Struct to hold various analysis needed for cost computations.

TargetTransformInfo::OperandValueInfo getOperandInfo(VPValue *V) const

Returns the OperandInfo for V, if it is a live-in.

bool skipCostComputation(Instruction *UI, bool IsVector) const

Return true if the cost for UI shouldn't be computed, e.g.

InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const

Return the cost for UI with VF using the legacy cost model as fallback until computing the cost of al...

TargetTransformInfo::TargetCostKind CostKind

const TargetLibraryInfo & TLI

const TargetTransformInfo & TTI

SmallPtrSet< Instruction *, 8 > SkipCostComputation

void execute(VPTransformState &State) override

Generate the phi nodes.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this first-order recurrence phi recipe.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

BasicBlock * PrevBB

The previous IR BasicBlock created or used.

SmallDenseMap< VPBasicBlock *, BasicBlock * > VPBB2IRBB

A mapping of each VPBasicBlock to the corresponding BasicBlock.

BasicBlock * getPreheaderBBFor(VPRecipeBase *R)

Returns the BasicBlock* mapped to the pre-header of the loop region containing R.

VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...

bool hasScalarValue(VPValue *Def, VPLane Lane)

bool hasVectorValue(VPValue *Def)

DenseMap< const SCEV *, Value * > ExpandedSCEVs

Map SCEVs to their expanded values.

VPTypeAnalysis TypeAnalysis

VPlan-based type analysis.

void addMetadata(Value *To, Instruction *From)

Add metadata from one instruction to another.

Value * get(VPValue *Def, bool IsScalar=false)

Get the generated vector Value for a given VPValue Def if IsScalar is false, otherwise return the gen...

struct llvm::VPTransformState::CFGState CFG

std::optional< VPLane > Lane

Hold the index to generate specific scalar instructions.

IRBuilderBase & Builder

Hold a reference to the IRBuilder used to generate output IR code.

const TargetTransformInfo * TTI

Target Transform Info.

void reset(VPValue *Def, Value *V)

Reset an existing vector value for Def and a given Part.

ElementCount VF

The chosen Vectorization Factor of the loop being vectorized.

void setDebugLocFrom(DebugLoc DL)

Set the debug location in the builder using the debug location DL.

Loop * CurrentParentLoop

The parent loop object for the current scope, or nullptr.

void set(VPValue *Def, Value *V, bool IsScalar=false)

Set the generated vector Value for a given VPValue, if IsScalar is false.

void execute(VPTransformState &State) override

Generate the wide load or gather.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenLoadEVLRecipe.

VPValue * getEVL() const

Return the EVL operand.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

void execute(VPTransformState &State) override

Generate a wide load or gather.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

bool isInvariantCond() const

VPValue * getCond() const

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenSelectRecipe.

void execute(VPTransformState &State) override

Produce a widened version of the select instruction.

VPValue * getStoredValue() const

Return the address accessed by this recipe.

void execute(VPTransformState &State) override

Generate the wide store or scatter.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenStoreEVLRecipe.

VPValue * getEVL() const

Return the EVL operand.

void execute(VPTransformState &State) override

Generate a wide store or scatter.

VPValue * getStoredValue() const

Return the value stored by this recipe.

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.