VPlanRecipes.cpp Source File (original) (raw)

39#include

41using namespace llvm;

46#define LV_NAME "loop-vectorize"

47#define DEBUG_TYPE LV_NAME

51 case VPExpressionSC:

53 case VPInstructionSC: {

56 if (VPI->getOpcode() == Instruction::Load)

57 return false;

58 return VPI->opcodeMayReadOrWriteFromMemory();

59 }

60 case VPInterleaveEVLSC:

61 case VPInterleaveSC:

63 case VPWidenStoreEVLSC:

64 case VPWidenStoreSC:

65 return true;

66 case VPReplicateSC:

68 ->mayWriteToMemory();

69 case VPWidenCallSC:

71 ->getCalledScalarFunction()

72 ->onlyReadsMemory();

73 case VPWidenIntrinsicSC:

75 case VPCanonicalIVPHISC:

76 case VPBranchOnMaskSC:

77 case VPDerivedIVSC:

78 case VPFirstOrderRecurrencePHISC:

79 case VPReductionPHISC:

80 case VPScalarIVStepsSC:

81 case VPPredInstPHISC:

82 return false;

83 case VPBlendSC:

84 case VPReductionEVLSC:

85 case VPReductionSC:

86 case VPVectorPointerSC:

87 case VPWidenCanonicalIVSC:

88 case VPWidenCastSC:

89 case VPWidenGEPSC:

90 case VPWidenIntOrFpInductionSC:

91 case VPWidenLoadEVLSC:

92 case VPWidenLoadSC:

93 case VPWidenPHISC:

94 case VPWidenPointerInductionSC:

95 case VPWidenSC:

96 case VPWidenSelectSC: {

99 (void)I;

100 assert(( || ->mayWriteToMemory()) &&

101 "underlying instruction may write to memory");

102 return false;

103 }

104 default:

105 return true;

106 }

107}

108

111 case VPExpressionSC:

113 case VPInstructionSC:

115 case VPWidenLoadEVLSC:

116 case VPWidenLoadSC:

117 return true;

118 case VPReplicateSC:

120 ->mayReadFromMemory();

121 case VPWidenCallSC:

123 ->getCalledScalarFunction()

124 ->onlyWritesMemory();

125 case VPWidenIntrinsicSC:

127 case VPBranchOnMaskSC:

128 case VPDerivedIVSC:

129 case VPFirstOrderRecurrencePHISC:

130 case VPPredInstPHISC:

131 case VPScalarIVStepsSC:

132 case VPWidenStoreEVLSC:

133 case VPWidenStoreSC:

134 return false;

135 case VPBlendSC:

136 case VPReductionEVLSC:

137 case VPReductionSC:

138 case VPVectorPointerSC:

139 case VPWidenCanonicalIVSC:

140 case VPWidenCastSC:

141 case VPWidenGEPSC:

142 case VPWidenIntOrFpInductionSC:

143 case VPWidenPHISC:

144 case VPWidenPointerInductionSC:

145 case VPWidenSC:

146 case VPWidenSelectSC: {

149 (void)I;

150 assert(( || ->mayReadFromMemory()) &&

151 "underlying instruction may read from memory");

152 return false;

153 }

154 default:

155

156 return true;

157 }

158}

159

162 case VPExpressionSC:

164 case VPDerivedIVSC:

165 case VPFirstOrderRecurrencePHISC:

166 case VPPredInstPHISC:

167 case VPVectorEndPointerSC:

168 return false;

169 case VPInstructionSC: {

174 }

175 case VPWidenCallSC: {

178 }

179 case VPWidenIntrinsicSC:

181 case VPBlendSC:

182 case VPReductionEVLSC:

183 case VPReductionSC:

184 case VPScalarIVStepsSC:

185 case VPVectorPointerSC:

186 case VPWidenCanonicalIVSC:

187 case VPWidenCastSC:

188 case VPWidenGEPSC:

189 case VPWidenIntOrFpInductionSC:

190 case VPWidenPHISC:

191 case VPWidenPointerInductionSC:

192 case VPWidenSC:

193 case VPWidenSelectSC: {

196 (void)I;

197 assert(( || ->mayHaveSideEffects()) &&

198 "underlying instruction has side-effects");

199 return false;

200 }

201 case VPInterleaveEVLSC:

202 case VPInterleaveSC:

204 case VPWidenLoadEVLSC:

205 case VPWidenLoadSC:

206 case VPWidenStoreEVLSC:

207 case VPWidenStoreSC:

211 "mayHaveSideffects result for ingredient differs from this "

212 "implementation");

214 case VPReplicateSC: {

216 return R->getUnderlyingInstr()->mayHaveSideEffects();

217 }

218 default:

219 return true;

220 }

221}

222

224 assert(!Parent && "Recipe already in some VPBasicBlock");

226 "Insertion position not in any VPBasicBlock");

228}

229

232 assert(!Parent && "Recipe already in some VPBasicBlock");

233 assert(I == BB.end() || I->getParent() == &BB);

235}

236

238 assert(!Parent && "Recipe already in some VPBasicBlock");

240 "Insertion position not in any VPBasicBlock");

242}

243

247 Parent = nullptr;

248}

249

254

259

265

267

268

269

270

275 UI = IG->getInsertPos();

277 UI = &WidenMem->getIngredient();

278

280 if (UI && Ctx.skipCostComputation(UI, VF.isVector())) {

281 RecipeCost = 0;

282 } else {

286 if (UI)

288 else

290 }

291 }

292

294 dbgs() << "Cost of " << RecipeCost << " for VF " << VF << ": ";

296 });

297 return RecipeCost;

298}

299

304

309

314

316 assert(OpType == Other.OpType && "OpType must match");

317 switch (OpType) {

318 case OperationType::OverflowingBinOp:

319 WrapFlags.HasNUW &= Other.WrapFlags.HasNUW;

320 WrapFlags.HasNSW &= Other.WrapFlags.HasNSW;

321 break;

322 case OperationType::Trunc:

323 TruncFlags.HasNUW &= Other.TruncFlags.HasNUW;

324 TruncFlags.HasNSW &= Other.TruncFlags.HasNSW;

325 break;

326 case OperationType::DisjointOp:

327 DisjointFlags.IsDisjoint &= Other.DisjointFlags.IsDisjoint;

328 break;

329 case OperationType::PossiblyExactOp:

330 ExactFlags.IsExact &= Other.ExactFlags.IsExact;

331 break;

332 case OperationType::GEPOp:

334 break;

335 case OperationType::FPMathOp:

336 case OperationType::FCmp:

337 assert((OpType != OperationType::FCmp ||

338 FCmpFlags.Pred == Other.FCmpFlags.Pred) &&

339 "Cannot drop CmpPredicate");

340 getFMFsRef().NoNaNs &= Other.getFMFsRef().NoNaNs;

341 getFMFsRef().NoInfs &= Other.getFMFsRef().NoInfs;

342 break;

343 case OperationType::NonNegOp:

344 NonNegFlags.NonNeg &= Other.NonNegFlags.NonNeg;

345 break;

346 case OperationType::Cmp:

347 assert(CmpPredicate == Other.CmpPredicate && "Cannot drop CmpPredicate");

348 break;

349 case OperationType::Other:

350 assert(AllFlags == Other.AllFlags && "Cannot drop other flags");

351 break;

352 }

353}

354

356 assert((OpType == OperationType::FPMathOp || OpType == OperationType::FCmp) &&

357 "recipe doesn't have fast math flags");

358 const FastMathFlagsTy &F = getFMFsRef();

367 return Res;

368}

369

370#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

372

377 O << ", !dbg ";

378 DL.print(O);

379 }

380

383}

384#endif

385

386template

389 if (U.getNumOperands() == PartOpIdx + 1)

390 return U.getOperand(PartOpIdx);

391 return nullptr;

392}

393

394template

397 return cast(UnrollPartOp->getLiveInIRValue())->getZExtValue();

398 return 0;

399}

400

401namespace llvm {

405}

406

411 VPIRMetadata(MD), Opcode(Opcode), Name(Name.str()) {

413 "Set flags not supported for the provided opcode");

414 assert((getNumOperandsForOpcode(Opcode) == -1u ||

415 getNumOperandsForOpcode(Opcode) == getNumOperands()) &&

416 "number of operands does not match opcode");

417}

418

419#ifndef NDEBUG

420unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {

422 return 1;

423

425 return 2;

426

427 switch (Opcode) {

430 return 0;

431 case Instruction::Alloca:

432 case Instruction::ExtractValue:

433 case Instruction::Freeze:

434 case Instruction::Load:

448 return 1;

449 case Instruction::ICmp:

450 case Instruction::FCmp:

451 case Instruction::ExtractElement:

452 case Instruction::Store:

461 return 2;

462 case Instruction::Select:

466 return 3;

468 return 4;

469 case Instruction::Call:

470 case Instruction::GetElementPtr:

471 case Instruction::PHI:

472 case Instruction::Switch:

478

479 return -1u;

480 }

482}

483#endif

484

488

489bool VPInstruction::canGenerateScalarForFirstLane() const {

491 return true;

493 return true;

494 switch (Opcode) {

495 case Instruction::Freeze:

496 case Instruction::ICmp:

497 case Instruction::PHI:

498 case Instruction::Select:

507 return true;

508 default:

509 return false;

510 }

511}

512

513

514

515

516

519

520

521

522

524 BasicBlock *SecondIRSucc = State.CFG.VPBB2IRBB.lookup(SecondVPSucc);

525 BasicBlock *IRBB = State.CFG.VPBB2IRBB[VPBB];

526 BranchInst *CondBr = State.Builder.CreateCondBr(Cond, IRBB, SecondIRSucc);

527

530 return CondBr;

531}

532

534 IRBuilderBase &Builder = State.Builder;

535

540 auto *Res =

544 return Res;

545 }

546

552 }

553 case Instruction::ExtractElement: {

554 assert(State.VF.isVector() && "Only extract elements from vectors");

556 unsigned IdxToExtract =

558 return State.get(getOperand(0), VPLane(IdxToExtract));

559 }

563 }

564 case Instruction::Freeze: {

567 }

568 case Instruction::FCmp:

569 case Instruction::ICmp: {

574 }

575 case Instruction::PHI: {

577 }

578 case Instruction::Select: {

586 }

588

590

592

593

594

597 Name);

598

602 return Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask,

603 {PredTy, ScalarTC->getType()},

604 {VIVElem0, ScalarTC}, nullptr, Name);

605 }

607

608

609

610

611

612

613

614

615

616

617

618

620 if (!V1->getType()->isVectorTy())

621 return V1;

624 }

633 }

635

636

638

640 "Requested vector length should be an integer.");

641

644

646 Builder.getInt32Ty(), Intrinsic::experimental_get_vector_length,

647 {AVL, VFArg, Builder.getTrue()});

648 return EVL;

649 }

653 assert(Part != 0 && "Must have a positive part");

654

655

659 }

664 return Br;

665 }

667

672 }

675 State.VF, State.get(getOperand(0), true), "broadcast");

676 }

678

679

680 auto *StructTy =

684 for (unsigned FieldIndex = 0; FieldIndex != StructTy->getNumElements();

685 FieldIndex++) {

686 Value *ScalarValue =

689 VectorValue =

692 }

693 }

694 return Res;

695 }

703 return Res;

704 }

708 IRBuilderBase::FastMathFlagGuard FMFG(Builder);

710

711

717 }

719

720

722 auto *OrigPhi = cast(PhiR->getUnderlyingValue());

724 for (unsigned Idx = 3; Idx < getNumOperands(); ++Idx)

725 ReducedPartRdx =

727 ReducedPartRdx, "bin.rdx");

730 }

732

733

735

736 RecurKind RK = PhiR->getRecurrenceKind();

738 "Unexpected reduction kind");

739 assert(!PhiR->isInLoop() &&

740 "In-loop FindLastIV reduction is not supported yet");

741

742

743

750 else

752 for (unsigned Part = 1; Part < UF; ++Part)

753 ReducedPartRdx = createMinMaxOp(Builder, MinMaxKind, ReducedPartRdx,

755

760 }

762

763

765

766

767 RecurKind RK = PhiR->getRecurrenceKind();

769 "should be handled by ComputeFindIVResult");

770

771

772

775 for (unsigned Part = 0; Part < UF; ++Part)

776 RdxParts[Part] = State.get(getOperand(1 + Part), PhiR->isInLoop());

777

778 IRBuilderBase::FastMathFlagGuard FMFG(Builder);

781

782

783 Value *ReducedPartRdx = RdxParts[0];

784 if (PhiR->isOrdered()) {

785 ReducedPartRdx = RdxParts[UF - 1];

786 } else {

787

788 for (unsigned Part = 1; Part < UF; ++Part) {

789 Value *RdxPart = RdxParts[Part];

791 ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);

792 else {

793

794

797 ? Instruction::Add

799 ReducedPartRdx =

800 Builder.CreateBinOp(Opcode, RdxPart, ReducedPartRdx, "bin.rdx");

801 }

802 }

803 }

804

805

806

807 if (State.VF.isVector() && !PhiR->isInLoop()) {

808

809

810

812 }

813

814 return ReducedPartRdx;

815 }

823 "invalid offset to extract from");

824

826 } else {

827

828 assert(Offset <= 1 && "invalid offset to extract from");

830 }

833 return Res;

834 }

839 }

842 "can only generate first lane for PtrAdd");

846 }

852 }

858 }

862 Value *Res = nullptr;

864

865 for (unsigned Idx = 1; Idx != getNumOperands(); ++Idx) {

866 Value *VectorStart =

867 Builder.CreateMul(RuntimeVF, ConstantInt::get(IdxTy, Idx - 1));

868 Value *VectorIdx = Idx == 1

869 ? LaneToExtract

870 : Builder.CreateSub(LaneToExtract, VectorStart);

875 if (Res) {

878 } else {

879 Res = Ext;

880 }

881 }

882 return Res;

883 }

888 false, Name);

889 }

890

891

892

895 Value *Res = nullptr;

896 for (int Idx = LastOpIdx; Idx >= 0; --Idx) {

897 Value *TrailingZeros =

905 false, Name);

907 Builder.CreateMul(RuntimeVF, Builder.getInt64(Idx)), TrailingZeros);

908 if (Res) {

910 Res = Builder.CreateSelect(Cmp, Current, Res);

911 } else {

912 Res = Current;

913 }

914 }

915

916 return Res;

917 }

920 default:

922 }

923}

924

927 Type *ScalarTy = Ctx.Types.inferScalarType(this);

929 switch (Opcode) {

930 case Instruction::FNeg:

931 return Ctx.TTI.getArithmeticInstrCost(Opcode, ResultTy, Ctx.CostKind);

932 case Instruction::UDiv:

933 case Instruction::SDiv:

934 case Instruction::SRem:

935 case Instruction::URem:

936 case Instruction::Add:

937 case Instruction::FAdd:

938 case Instruction::Sub:

939 case Instruction::FSub:

940 case Instruction::Mul:

941 case Instruction::FMul:

942 case Instruction::FDiv:

943 case Instruction::FRem:

944 case Instruction::Shl:

945 case Instruction::LShr:

946 case Instruction::AShr:

947 case Instruction::And:

948 case Instruction::Or:

949 case Instruction::Xor: {

952

954

955

957 RHSInfo = Ctx.getOperandInfo(RHS);

958

962 }

963

966 if (CtxI)

968 return Ctx.TTI.getArithmeticInstrCost(

969 Opcode, ResultTy, Ctx.CostKind,

970 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},

971 RHSInfo, Operands, CtxI, &Ctx.TLI);

972 }

973 case Instruction::Freeze:

974

975 return Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, ResultTy,

976 Ctx.CostKind);

977 case Instruction::ExtractValue:

978 return Ctx.TTI.getInsertExtractValueCost(Instruction::ExtractValue,

979 Ctx.CostKind);

980 case Instruction::ICmp:

981 case Instruction::FCmp: {

982 Type *ScalarOpTy = Ctx.Types.inferScalarType(getOperand(0));

985 return Ctx.TTI.getCmpSelInstrCost(

987 Ctx.CostKind, {TTI::OK_AnyValue, TTI::OP_None},

988 {TTI::OK_AnyValue, TTI::OP_None}, CtxI);

989 }

990 }

992}

993

998

999

1000 return 0;

1001 }

1002

1004 "Should only generate a vector value or single scalar, not scalars "

1005 "for all lanes.");

1009 }

1010

1012 case Instruction::Select: {

1015 auto *CondTy = Ctx.Types.inferScalarType(getOperand(0));

1016 auto *VecTy = Ctx.Types.inferScalarType(getOperand(1));

1020 }

1021 return Ctx.TTI.getCmpSelInstrCost(Instruction::Select, VecTy, CondTy, Pred,

1022 Ctx.CostKind);

1023 }

1024 case Instruction::ExtractElement:

1027

1028

1029 return 0;

1030 }

1031

1032

1034 return Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy,

1035 Ctx.CostKind);

1036 }

1038 auto *VecTy = toVectorTy(Ctx.Types.inferScalarType(this), VF);

1039 return Ctx.TTI.getArithmeticReductionCost(

1040 Instruction::Or, cast(VecTy), std::nullopt, Ctx.CostKind);

1041 }

1043 Type *ScalarTy = Ctx.Types.inferScalarType(getOperand(0));

1045 return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, ScalarTy,

1048

1049 auto *PredTy = toVectorTy(ScalarTy, VF);

1052 {PredTy, Type::getInt1Ty(Ctx.LLVMCtx)});

1053 return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);

1054 }

1056 Type *ScalarTy = Ctx.Types.inferScalarType(getOperand(0));

1058 return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, ScalarTy,

1061

1062 auto *PredTy = toVectorTy(ScalarTy, VF);

1065 {PredTy, Type::getInt1Ty(Ctx.LLVMCtx)});

1066 InstructionCost Cost = Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);

1067

1068 Cost += Ctx.TTI.getArithmeticInstrCost(

1069 Instruction::Xor, PredTy, Ctx.CostKind,

1070 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},

1071 {TargetTransformInfo::OK_UniformConstantValue,

1072 TargetTransformInfo::OP_None});

1073

1074 Cost += Ctx.TTI.getArithmeticInstrCost(

1075 Instruction::Sub, Type::getInt64Ty(Ctx.LLVMCtx), Ctx.CostKind);

1076 return Cost;

1077 }

1079 assert(VF.isVector() && "Scalar FirstOrderRecurrenceSplice?");

1081 std::iota(Mask.begin(), Mask.end(), VF.getKnownMinValue() - 1);

1082 Type *VectorTy = toVectorTy(Ctx.Types.inferScalarType(this), VF);

1083

1088 }

1090 Type *ArgTy = Ctx.Types.inferScalarType(getOperand(0));

1091 unsigned Multiplier =

1095 {ArgTy, ArgTy});

1096 return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);

1097 }

1099 Type *Arg0Ty = Ctx.Types.inferScalarType(getOperand(0));

1103 I32Ty, {Arg0Ty, I32Ty, I1Ty});

1104 return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);

1105 }

1107

1109 return Ctx.TTI.getIndexedVectorInstrCostFromEnd(Instruction::ExtractElement,

1110 VecTy, Ctx.CostKind, 0);

1111 }

1115 [[fallthrough]];

1116 default:

1117

1118

1120 "unexpected VPInstruction witht underlying value");

1121 return 0;

1122 }

1123}

1124

1128 getOpcode() == Instruction::ExtractElement ||

1136}

1137

1140 case Instruction::PHI:

1144 return true;

1145 default:

1147 }

1148}

1149

1151 assert(!State.Lane && "VPInstruction executing an Lane");

1154 "Set flags not supported for the provided opcode");

1157 Value *GeneratedValue = generate(State);

1159 return;

1160 assert(GeneratedValue && "generate must produce a value");

1161 bool GeneratesPerFirstLaneOnly = canGenerateScalarForFirstLane() &&

1166 !GeneratesPerFirstLaneOnly) ||

1167 State.VF.isScalar()) &&

1168 "scalar value but not only first lane defined");

1169 State.set(this, GeneratedValue,

1170 GeneratesPerFirstLaneOnly);

1171}

1172

1175 return false;

1177 case Instruction::GetElementPtr:

1178 case Instruction::ExtractElement:

1179 case Instruction::Freeze:

1180 case Instruction::FCmp:

1181 case Instruction::ICmp:

1182 case Instruction::Select:

1183 case Instruction::PHI:

1210 return false;

1211 default:

1212 return true;

1213 }

1214}

1215

1220

1222 default:

1223 return false;

1224 case Instruction::ExtractElement:

1226 case Instruction::PHI:

1227 return true;

1228 case Instruction::FCmp:

1229 case Instruction::ICmp:

1230 case Instruction::Select:

1231 case Instruction::Or:

1232 case Instruction::Freeze:

1234

1244 return true;

1247

1248

1249

1254

1255 return false;

1261 };

1263}

1264

1269

1271 default:

1272 return false;

1273 case Instruction::FCmp:

1274 case Instruction::ICmp:

1275 case Instruction::Select:

1280 return true;

1281 };

1283}

1284

1285#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1290

1293 O << Indent << "EMIT" << (isSingleScalar() ? "-SCALAR" : "") << " ";

1294

1297 O << " = ";

1298 }

1299

1302 O << "not";

1303 break;

1305 O << "combined load";

1306 break;

1308 O << "combined store";

1309 break;

1311 O << "active lane mask";

1312 break;

1314 O << "EXPLICIT-VECTOR-LENGTH";

1315 break;

1317 O << "first-order splice";

1318 break;

1320 O << "branch-on-cond";

1321 break;

1323 O << "TC > VF ? TC - VF : 0";

1324 break;

1326 O << "VF * Part +";

1327 break;

1329 O << "branch-on-count";

1330 break;

1332 O << "broadcast";

1333 break;

1335 O << "buildstructvector";

1336 break;

1338 O << "buildvector";

1339 break;

1341 O << "extract-lane";

1342 break;

1344 O << "extract-last-lane";

1345 break;

1347 O << "extract-last-part";

1348 break;

1350 O << "extract-penultimate-element";

1351 break;

1353 O << "compute-anyof-result";

1354 break;

1356 O << "compute-find-iv-result";

1357 break;

1359 O << "compute-reduction-result";

1360 break;

1362 O << "logical-and";

1363 break;

1365 O << "ptradd";

1366 break;

1368 O << "wide-ptradd";

1369 break;

1371 O << "any-of";

1372 break;

1374 O << "first-active-lane";

1375 break;

1377 O << "last-active-lane";

1378 break;

1380 O << "reduction-start-vector";

1381 break;

1383 O << "resume-for-epilogue";

1384 break;

1386 O << "unpack";

1387 break;

1388 default:

1390 }

1391

1394}

1395#endif

1396

1402 Op, ResultTy);

1403 State.set(this, Cast, VPLane(0));

1404 return;

1405 }

1409 State.Builder.CreateStepVector(VectorType::get(ResultTy, State.VF));

1411 break;

1412 }

1414 Value *VScale = State.Builder.CreateVScale(ResultTy);

1415 State.set(this, VScale, true);

1416 break;

1417 }

1418

1419 default:

1421 }

1422}

1423

1424#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1427 O << Indent << "EMIT" << (isSingleScalar() ? "-SCALAR" : "") << " ";

1429 O << " = ";

1430

1433 O << "wide-iv-step ";

1435 break;

1437 O << "step-vector " << *ResultTy;

1438 break;

1440 O << "vscale " << *ResultTy;

1441 break;

1442 default:

1446 O << " to " << *ResultTy;

1447 }

1448}

1449#endif

1450

1453 PHINode *NewPhi = State.Builder.CreatePHI(

1454 State.TypeAnalysis.inferScalarType(this), 2, getName());

1457

1458

1459 NumIncoming = 1;

1460 }

1461 for (unsigned Idx = 0; Idx != NumIncoming; ++Idx) {

1465 }

1466 State.set(this, NewPhi, VPLane(0));

1467}

1468

1469#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1472 O << Indent << "EMIT" << (isSingleScalar() ? "-SCALAR" : "") << " ";

1474 O << " = phi ";

1476}

1477#endif

1478

1481 return new VPIRPhi(*Phi);

1483}

1484

1487 "PHINodes must be handled by VPIRPhi");

1488

1489

1490 State.Builder.SetInsertPoint(I.getParent(), std::next(I.getIterator()));

1491}

1492

1495

1496

1497 return 0;

1498}

1499

1503 "can only update exiting operands to phi nodes");

1507 return;

1508

1512}

1513

1514#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1517 O << Indent << "IR " << I;

1518}

1519#endif

1520

1529 auto *PredVPBB = Pred->getExitingBasicBlock();

1530 BasicBlock *PredBB = State.CFG.VPBB2IRBB[PredVPBB];

1531

1532

1533 State.Builder.SetInsertPoint(PredBB, PredBB->getFirstNonPHIIt());

1534 Value *V = State.get(ExitValue, VPLane(Lane));

1535

1536

1537 if (Phi->getBasicBlockIndex(PredBB) == -1)

1538 Phi->addIncoming(V, PredBB);

1539 else

1540 Phi->setIncomingValueForBlock(PredBB, V);

1541 }

1542

1543

1544

1545 State.Builder.SetInsertPoint(Phi->getParent(), std::next(Phi->getIterator()));

1546}

1547

1550 assert(R->getNumOperands() == R->getParent()->getNumPredecessors() &&

1551 "Number of phi operands must match number of predecessors");

1552 unsigned Position = R->getParent()->getIndexForPredecessor(IncomingBlock);

1553 R->removeOperand(Position);

1554}

1555

1556#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1561 O << "[ ";

1563 O << ", ";

1565 O << " ]";

1566 });

1567}

1568#endif

1569

1570#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1574

1576 O << " (extra operand" << (getNumOperands() > 1 ? "s" : "") << ": ";

1579 std::get<0>(Op)->printAsOperand(O, SlotTracker);

1580 O << " from ";

1581 std::get<1>(Op)->printAsOperand(O);

1582 });

1583 O << ")";

1584 }

1585}

1586#endif

1587

1589 for (const auto &[Kind, Node] : Metadata)

1590 I.setMetadata(Kind, Node);

1591}

1592

1595 for (const auto &[KindA, MDA] : Metadata) {

1596 for (const auto &[KindB, MDB] : Other.Metadata) {

1597 if (KindA == KindB && MDA == MDB) {

1598 MetadataIntersection.emplace_back(KindA, MDA);

1599 break;

1600 }

1601 }

1602 }

1603 Metadata = std::move(MetadataIntersection);

1604}

1605

1606#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1609 if (Metadata.empty() || !M)

1610 return;

1611

1613 O << " (";

1614 interleaveComma(Metadata, O, [&](const auto &KindNodePair) {

1615 auto [Kind, Node] = KindNodePair;

1616 assert(Kind < MDNames.size() && !MDNames[Kind].empty() &&

1617 "Unexpected unnamed metadata kind");

1618 O << "!" << MDNames[Kind] << " ";

1620 });

1621 O << ")";

1622}

1623#endif

1624

1626 assert(State.VF.isVector() && "not widening");

1627 assert(Variant != nullptr && "Can't create vector function.");

1628

1629 FunctionType *VFTy = Variant->getFunctionType();

1630

1634

1635

1636

1638 Arg = State.get(I.value(), VPLane(0));

1639 else

1641 Args.push_back(Arg);

1642 }

1643

1646 if (CI)

1647 CI->getOperandBundlesAsDefs(OpBundles);

1648

1649 CallInst *V = State.Builder.CreateCall(Variant, Args, OpBundles);

1652 V->setCallingConv(Variant->getCallingConv());

1653

1654 if (!V->getType()->isVoidTy())

1655 State.set(this, V);

1656}

1657

1660 return Ctx.TTI.getCallInstrCost(nullptr, Variant->getReturnType(),

1661 Variant->getFunctionType()->params(),

1662 Ctx.CostKind);

1663}

1664

1665#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1668 O << Indent << "WIDEN-CALL ";

1669

1672 O << "void ";

1673 else {

1675 O << " = ";

1676 }

1677

1678 O << "call";

1680 O << " @" << CalledFn->getName() << "(";

1683 });

1684 O << ")";

1685

1686 O << " (using library function";

1687 if (Variant->hasName())

1688 O << ": " << Variant->getName();

1689 O << ")";

1690}

1691#endif

1692

1694 assert(State.VF.isVector() && "not widening");

1695

1697

1702

1703

1706 State.TTI))

1707 Arg = State.get(I.value(), VPLane(0));

1708 else

1711 State.TTI))

1713 Args.push_back(Arg);

1714 }

1715

1716

1717 Module *M = State.Builder.GetInsertBlock()->getModule();

1721 "Can't retrieve vector intrinsic or vector-predication intrinsics.");

1722

1725 if (CI)

1726 CI->getOperandBundlesAsDefs(OpBundles);

1727

1728 CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);

1729

1732

1733 if (!V->getType()->isVoidTy())

1734 State.set(this, V);

1735}

1736

1737

1743

1744

1745

1746

1747

1749 for (const auto &[Idx, Op] : enumerate(Operands)) {

1750 auto *V = Op->getUnderlyingValue();

1751 if (!V) {

1753 Arguments.push_back(UI->getArgOperand(Idx));

1754 continue;

1755 }

1757 break;

1758 }

1760 }

1761

1762 Type *ScalarRetTy = Ctx.Types.inferScalarType(&R);

1765 for (const VPValue *Op : Operands) {

1767 ? toVectorTy(Ctx.Types.inferScalarType(Op), VF)

1768 : Ctx.Types.inferScalarType(Op));

1769 }

1770

1771

1773 R.hasFastMathFlags() ? R.getFastMathFlags() : FastMathFlags();

1778 return Ctx.TTI.getIntrinsicInstrCost(CostAttrs, Ctx.CostKind);

1779}

1780

1786

1790

1794 auto [Idx, V] = X;

1796 Idx, nullptr);

1797 });

1798}

1799

1800#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1803 O << Indent << "WIDEN-INTRINSIC ";

1804 if (ResultTy->isVoidTy()) {

1805 O << "void ";

1806 } else {

1808 O << " = ";

1809 }

1810

1811 O << "call";

1814

1817 });

1818 O << ")";

1819}

1820#endif

1821

1824

1826 Value *IncAmt = State.get(getOperand(1), true);

1828

1829

1830

1831

1832 Value *Mask = nullptr;

1834 Mask = State.get(VPMask);

1835 else

1836 Mask =

1837 Builder.CreateVectorSplat(VTy->getElementCount(), Builder.getInt1(1));

1838

1839

1840

1841 if (Opcode == Instruction::Sub)

1842 IncAmt = Builder.CreateNeg(IncAmt);

1843 else

1844 assert(Opcode == Instruction::Add && "only add or sub supported for now");

1845

1846 State.Builder.CreateIntrinsic(Intrinsic::experimental_vector_histogram_add,

1847 {VTy, IncAmt->getType()},

1848 {Address, IncAmt, Mask});

1849}

1850

1853

1854

1855

1856

1857

1858 assert(VF.isVector() && "Invalid VF for histogram cost");

1859 Type *AddressTy = Ctx.Types.inferScalarType(getOperand(0));

1861 Type *IncTy = Ctx.Types.inferScalarType(IncAmt);

1863

1864

1865

1867 Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy, Ctx.CostKind);

1870

1873 }

1874

1875

1880 {PtrTy, IncTy, MaskTy});

1881

1882

1883 return Ctx.TTI.getIntrinsicInstrCost(ICA, Ctx.CostKind) + MulCost +

1884 Ctx.TTI.getArithmeticInstrCost(Opcode, VTy, Ctx.CostKind);

1885}

1886

1887#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1890 O << Indent << "WIDEN-HISTOGRAM buckets: ";

1892

1893 if (Opcode == Instruction::Sub)

1894 O << ", dec: ";

1895 else {

1896 assert(Opcode == Instruction::Add);

1897 O << ", inc: ";

1898 }

1900

1902 O << ", mask: ";

1904 }

1905}

1906

1909 O << Indent << "WIDEN-SELECT ";

1911 O << " = select ";

1914 O << ", ";

1916 O << ", ";

1919 : "");

1920}

1921#endif

1922

1925

1928 Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1);

1929 State.set(this, Sel);

1934 }

1935}

1936

1941 Type *ScalarTy = Ctx.Types.inferScalarType(this);

1942 Type *VectorTy = toVectorTy(Ctx.Types.inferScalarType(this), VF);

1943

1948

1949

1950 const auto [Op1VK, Op1VP] = Ctx.getOperandInfo(Op0);

1951 const auto [Op2VK, Op2VP] = Ctx.getOperandInfo(Op1);

1952

1955 [](VPValue *Op) { return Op->getUnderlyingValue(); }))

1956 Operands.append(SI->op_begin(), SI->op_end());

1958 return Ctx.TTI.getArithmeticInstrCost(

1959 IsLogicalOr ? Instruction::Or : Instruction::And, VectorTy,

1960 Ctx.CostKind, {Op1VK, Op1VP}, {Op2VK, Op2VP}, Operands, SI);

1961 }

1962

1963 Type *CondTy = Ctx.Types.inferScalarType(getOperand(0));

1964 if (!ScalarCond)

1966

1969 Pred = Cmp->getPredicate();

1970 return Ctx.TTI.getCmpSelInstrCost(

1971 Instruction::Select, VectorTy, CondTy, Pred, Ctx.CostKind,

1972 {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None}, SI);

1973}

1974

1975VPIRFlags::FastMathFlagsTy::FastMathFlagsTy(const FastMathFlags &FMF) {

1977 NoNaNs = FMF.noNaNs();

1978 NoInfs = FMF.noInfs();

1983}

1984

1985#if !defined(NDEBUG)

1987 switch (OpType) {

1988 case OperationType::OverflowingBinOp:

1989 return Opcode == Instruction::Add || Opcode == Instruction::Sub ||

1990 Opcode == Instruction::Mul || Opcode == Instruction::Shl ||

1991 Opcode == VPInstruction::VPInstruction::CanonicalIVIncrementForPart;

1992 case OperationType::Trunc:

1993 return Opcode == Instruction::Trunc;

1994 case OperationType::DisjointOp:

1995 return Opcode == Instruction::Or;

1996 case OperationType::PossiblyExactOp:

1997 return Opcode == Instruction::AShr || Opcode == Instruction::LShr ||

1998 Opcode == Instruction::UDiv || Opcode == Instruction::SDiv;

1999 case OperationType::GEPOp:

2000 return Opcode == Instruction::GetElementPtr ||

2003 case OperationType::FPMathOp:

2004 return Opcode == Instruction::Call || Opcode == Instruction::FAdd ||

2005 Opcode == Instruction::FMul || Opcode == Instruction::FSub ||

2006 Opcode == Instruction::FNeg || Opcode == Instruction::FDiv ||

2007 Opcode == Instruction::FRem || Opcode == Instruction::FPExt ||

2008 Opcode == Instruction::FPTrunc || Opcode == Instruction::Select ||

2012 case OperationType::FCmp:

2013 return Opcode == Instruction::FCmp;

2014 case OperationType::NonNegOp:

2015 return Opcode == Instruction::ZExt || Opcode == Instruction::UIToFP;

2016 case OperationType::Cmp:

2017 return Opcode == Instruction::FCmp || Opcode == Instruction::ICmp;

2018 case OperationType::Other:

2019 return true;

2020 }

2022}

2023#endif

2024

2025#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2027 switch (OpType) {

2028 case OperationType::Cmp:

2030 break;

2031 case OperationType::FCmp:

2034 break;

2035 case OperationType::DisjointOp:

2037 O << " disjoint";

2038 break;

2039 case OperationType::PossiblyExactOp:

2041 O << " exact";

2042 break;

2043 case OperationType::OverflowingBinOp:

2045 O << " nuw";

2047 O << " nsw";

2048 break;

2049 case OperationType::Trunc:

2051 O << " nuw";

2053 O << " nsw";

2054 break;

2055 case OperationType::FPMathOp:

2057 break;

2058 case OperationType::GEPOp:

2060 O << " inbounds";

2061 else if (GEPFlags.hasNoUnsignedSignedWrap())

2062 O << " nusw";

2063 if (GEPFlags.hasNoUnsignedWrap())

2064 O << " nuw";

2065 break;

2066 case OperationType::NonNegOp:

2068 O << " nneg";

2069 break;

2070 case OperationType::Other:

2071 break;

2072 }

2073 O << " ";

2074}

2075#endif

2076

2078 auto &Builder = State.Builder;

2079 switch (Opcode) {

2080 case Instruction::Call:

2081 case Instruction::Br:

2082 case Instruction::PHI:

2083 case Instruction::GetElementPtr:

2084 case Instruction::Select:

2085 llvm_unreachable("This instruction is handled by a different recipe.");

2086 case Instruction::UDiv:

2087 case Instruction::SDiv:

2088 case Instruction::SRem:

2089 case Instruction::URem:

2090 case Instruction::Add:

2091 case Instruction::FAdd:

2092 case Instruction::Sub:

2093 case Instruction::FSub:

2094 case Instruction::FNeg:

2095 case Instruction::Mul:

2096 case Instruction::FMul:

2097 case Instruction::FDiv:

2098 case Instruction::FRem:

2099 case Instruction::Shl:

2100 case Instruction::LShr:

2101 case Instruction::AShr:

2102 case Instruction::And:

2103 case Instruction::Or:

2104 case Instruction::Xor: {

2105

2108 Ops.push_back(State.get(VPOp));

2109

2110 Value *V = Builder.CreateNAryOp(Opcode, Ops);

2111

2115 }

2116

2117

2118 State.set(this, V);

2119 break;

2120 }

2121 case Instruction::ExtractValue: {

2125 Value *Extract = Builder.CreateExtractValue(Op, CI->getZExtValue());

2126 State.set(this, Extract);

2127 break;

2128 }

2129 case Instruction::Freeze: {

2131 Value *Freeze = Builder.CreateFreeze(Op);

2132 State.set(this, Freeze);

2133 break;

2134 }

2135 case Instruction::ICmp:

2136 case Instruction::FCmp: {

2137

2138 bool FCmp = Opcode == Instruction::FCmp;

2142 if (FCmp) {

2144 } else {

2146 }

2150 }

2151 State.set(this, C);

2152 break;

2153 }

2154 default:

2155

2156 LLVM_DEBUG(dbgs() << "LV: Found an unhandled opcode : "

2159 }

2160

2161#if !defined(NDEBUG)

2162

2163

2165 State.get(this)->getType() &&

2166 "inferred type and type from generated instructions do not match");

2167#endif

2168}

2169

2172 switch (Opcode) {

2173 case Instruction::UDiv:

2174 case Instruction::SDiv:

2175 case Instruction::SRem:

2176 case Instruction::URem:

2177

2178

2179

2180

2181 case Instruction::FNeg:

2182 case Instruction::Add:

2183 case Instruction::FAdd:

2184 case Instruction::Sub:

2185 case Instruction::FSub:

2186 case Instruction::Mul:

2187 case Instruction::FMul:

2188 case Instruction::FDiv:

2189 case Instruction::FRem:

2190 case Instruction::Shl:

2191 case Instruction::LShr:

2192 case Instruction::AShr:

2193 case Instruction::And:

2194 case Instruction::Or:

2195 case Instruction::Xor:

2196 case Instruction::Freeze:

2197 case Instruction::ExtractValue:

2198 case Instruction::ICmp:

2199 case Instruction::FCmp:

2201 default:

2203 }

2204}

2205

2206#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2209 O << Indent << "WIDEN ";

2214}

2215#endif

2216

2218 auto &Builder = State.Builder;

2219

2220 assert(State.VF.isVector() && "Not vectorizing?");

2225 State.set(this, Cast);

2229 }

2230}

2231

2234

2235

2236

2238 return 0;

2239

2249 if (WidenMemoryRecipe == nullptr)

2251 if (!WidenMemoryRecipe->isConsecutive())

2253 if (WidenMemoryRecipe->isReverse())

2255 if (WidenMemoryRecipe->isMasked())

2258 };

2259

2262

2263 if ((Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) &&

2266 CCH = ComputeCCH(StoreRecipe);

2267 }

2268

2269 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||

2270 Opcode == Instruction::FPExt) {

2275 }

2276

2277 auto *SrcTy =

2280

2281 return Ctx.TTI.getCastInstrCost(

2282 Opcode, DestTy, SrcTy, CCH, Ctx.CostKind,

2284}

2285

2286#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2289 O << Indent << "WIDEN-CAST ";

2295}

2296#endif

2297

2300 return Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);

2301}

2302

2303

2304

2307 : ConstantFP::get(Ty, C);

2308}

2309

2310#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2313 O << Indent;

2315 O << " = WIDEN-INDUCTION";

2317 O << " ";

2319

2321 O << " (truncated to " << *TI->getType() << ")";

2322}

2323#endif

2324

2326

2327

2328

2330 return false;

2333 return StartC && StartC->isZero() && StepC && StepC->isOne() &&

2335}

2336

2337#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2340 O << Indent;

2342 O << " = DERIVED-IV ";

2344 O << " + ";

2346 O << " * ";

2348}

2349#endif

2350

2352

2356

2357

2358

2359

2363

2364

2366 assert(BaseIVTy == Step->getType() && "Types of BaseIV and Step must match!");

2367

2368

2369

2373 AddOp = Instruction::Add;

2374 MulOp = Instruction::Mul;

2375 } else {

2376 AddOp = InductionOpcode;

2377 MulOp = Instruction::FMul;

2378 }

2379

2380

2381

2383

2384 Type *IntStepTy =

2386

2387 unsigned StartLane = 0;

2388 unsigned EndLane = FirstLaneOnly ? 1 : State.VF.getKnownMinValue();

2389 if (State.Lane) {

2390 StartLane = State.Lane->getKnownLane();

2391 EndLane = StartLane + 1;

2392 }

2393 Value *StartIdx0;

2395 StartIdx0 = ConstantInt::get(IntStepTy, 0);

2396 else {

2397 StartIdx0 = State.get(getOperand(2), true);

2399 StartIdx0 =

2400 Builder.CreateMul(StartIdx0, ConstantInt::get(StartIdx0->getType(),

2402 }

2403 StartIdx0 = Builder.CreateSExtOrTrunc(StartIdx0, IntStepTy);

2404 }

2405

2407 StartIdx0 = Builder.CreateSIToFP(StartIdx0, BaseIVTy);

2408

2409 for (unsigned Lane = StartLane; Lane < EndLane; ++Lane) {

2410 Value *StartIdx = Builder.CreateBinOp(

2412

2413

2415 "Expected StartIdx to be folded to a constant when VF is not "

2416 "scalable");

2417 auto *Mul = Builder.CreateBinOp(MulOp, StartIdx, Step);

2418 auto *Add = Builder.CreateBinOp(AddOp, BaseIV, Mul);

2419 State.set(this, Add, VPLane(Lane));

2420 }

2421}

2422

2423#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2426 O << Indent;

2428 O << " = SCALAR-STEPS ";

2430}

2431#endif

2432

2437

2439 assert(State.VF.isVector() && "not widening");

2440

2441

2442

2443

2444

2445

2448 [](VPValue *Op) { return ->isDefinedOutsideLoopRegions(); }) &&

2449 "Expected at least one loop-variant operand");

2450

2451

2452

2453

2454

2455 auto *Ptr = State.get(getOperand(0), isPointerLoopInvariant());

2456

2457

2458

2462 Indices.push_back(State.get(Operand, isIndexLoopInvariant(I - 1)));

2463 }

2464

2465

2466

2467 auto *NewGEP = State.Builder.CreateGEP(getSourceElementType(), Ptr, Indices,

2469 assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) &&

2470 "NewGEP is not a pointer vector");

2471 State.set(this, NewGEP);

2472}

2473

2474#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2477 O << Indent << "WIDEN-GEP ";

2478 O << (isPointerLoopInvariant() ? "Inv" : "Var");

2480 O << "[" << (isIndexLoopInvariant(I) ? "Inv" : "Var") << "]";

2481

2482 O << " ";

2484 O << " = getelementptr";

2487}

2488#endif

2489

2491 auto &Builder = State.Builder;

2493 const DataLayout &DL = Builder.GetInsertBlock()->getDataLayout();

2494 Type *IndexTy = DL.getIndexType(State.TypeAnalysis.inferScalarType(this));

2495

2496

2498 if (IndexTy != RunTimeVF->getType())

2499 RunTimeVF = Builder.CreateZExtOrTrunc(RunTimeVF, IndexTy);

2500

2501 Value *NumElt = Builder.CreateMul(

2502 ConstantInt::get(IndexTy, Stride * (int64_t)CurrentPart), RunTimeVF);

2503

2504 Value *LastLane = Builder.CreateSub(RunTimeVF, ConstantInt::get(IndexTy, 1));

2505 if (Stride != 1)

2506 LastLane = Builder.CreateMul(ConstantInt::get(IndexTy, Stride), LastLane);

2508 Value *ResultPtr =

2509 Builder.CreateGEP(IndexedTy, Ptr, NumElt, "", getGEPNoWrapFlags());

2510 ResultPtr = Builder.CreateGEP(IndexedTy, ResultPtr, LastLane, "",

2512

2513 State.set(this, ResultPtr, true);

2514}

2515

2516#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2519 O << Indent;

2521 O << " = vector-end-pointer";

2524}

2525#endif

2526

2528 auto &Builder = State.Builder;

2530 const DataLayout &DL = Builder.GetInsertBlock()->getDataLayout();

2531 Type *IndexTy = DL.getIndexType(State.TypeAnalysis.inferScalarType(this));

2533

2537

2538 State.set(this, ResultPtr, true);

2539}

2540

2541#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2544 O << Indent;

2546 O << " = vector-pointer";

2549}

2550#endif

2551

2554

2555

2557 return Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);

2558

2559 Type *ResultTy = toVectorTy(Ctx.Types.inferScalarType(this), VF);

2562 Ctx.TTI.getCmpSelInstrCost(Instruction::Select, ResultTy, CmpTy,

2564}

2565

2566#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2569 O << Indent << "BLEND ";

2571 O << " =";

2573

2574

2575 O << " ";

2577 } else {

2579 O << " ";

2581 if (I == 0)

2582 continue;

2583 O << "/";

2585 }

2586 }

2587}

2588#endif

2589

2591 assert(!State.Lane && "Reduction being replicated.");

2594 "In-loop AnyOf reductions aren't currently supported");

2595

2600 Value *NewCond = State.get(Cond, State.VF.isScalar());

2603

2605 if (State.VF.isVector())

2606 Start = State.Builder.CreateVectorSplat(VecTy->getElementCount(), Start);

2607

2608 Value *Select = State.Builder.CreateSelect(NewCond, NewVecOp, Start);

2610 }

2612 Value *NextInChain;

2614 Value *PrevInChain = State.get(getChainOp(), true);

2615 if (State.VF.isVector())

2616 NewRed =

2618 else

2619 NewRed = State.Builder.CreateBinOp(

2621 PrevInChain, NewVecOp);

2622 PrevInChain = NewRed;

2623 NextInChain = NewRed;

2626 Value *PrevInChain = State.get(getChainOp(), false);

2627 NewRed = State.Builder.CreateIntrinsic(

2628 PrevInChain->getType(), Intrinsic::vector_partial_reduce_add,

2629 {PrevInChain, NewVecOp}, nullptr, "partial.reduce");

2630 PrevInChain = NewRed;

2631 NextInChain = NewRed;

2632 } else {

2634 "The reduction must either be ordered, partial or in-loop");

2635 Value *PrevInChain = State.get(getChainOp(), true);

2638 NextInChain = createMinMaxOp(State.Builder, Kind, NewRed, PrevInChain);

2639 else

2640 NextInChain = State.Builder.CreateBinOp(

2642 PrevInChain, NewRed);

2643 }

2645}

2646

2648 assert(!State.Lane && "Reduction being replicated.");

2649

2650 auto &Builder = State.Builder;

2651

2654

2656 Value *Prev = State.get(getChainOp(), true);

2659

2662 Mask = State.get(CondOp);

2663 else

2664 Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());

2665

2669 } else {

2672 NewRed = createMinMaxOp(Builder, Kind, NewRed, Prev);

2673 else

2674 NewRed = Builder.CreateBinOp(

2676 Prev);

2677 }

2678 State.set(this, NewRed, true);

2679}

2680

2684 Type *ElementTy = Ctx.Types.inferScalarType(this);

2688 std::optional OptionalFMF =

2690

2697 CondCost = Ctx.TTI.getCmpSelInstrCost(Instruction::Select, VectorTy,

2698 CondTy, Pred, Ctx.CostKind);

2699 }

2700 return CondCost + Ctx.TTI.getPartialReductionCost(

2701 Opcode, ElementTy, ElementTy, ElementTy, VF,

2704 Ctx.CostKind);

2705 }

2706

2707

2711 "Any-of reduction not implemented in VPlan-based cost model currently.");

2712

2713

2714

2717 return Ctx.TTI.getMinMaxReductionCost(Id, VectorTy, FMFs, Ctx.CostKind);

2718 }

2719

2720

2721

2722 return Ctx.TTI.getArithmeticReductionCost(Opcode, VectorTy, OptionalFMF,

2723 Ctx.CostKind);

2724}

2725

2727 ExpressionTypes ExpressionType,

2730 ExpressionRecipes(ExpressionRecipes), ExpressionType(ExpressionType) {

2731 assert(!ExpressionRecipes.empty() && "Nothing to combine?");

2733 none_of(ExpressionRecipes,

2735 "expression cannot contain recipes with side-effects");

2736

2737

2739 for (auto *R : ExpressionRecipes)

2740 ExpressionRecipesAsSetOfUsers.insert(R);

2741

2742

2743

2744

2746 if (R != ExpressionRecipes.back() &&

2747 any_of(R->users(), [&ExpressionRecipesAsSetOfUsers](VPUser *U) {

2748 return !ExpressionRecipesAsSetOfUsers.contains(U);

2749 })) {

2750

2751

2753 R->replaceUsesWithIf(CopyForExtUsers, [&ExpressionRecipesAsSetOfUsers](

2754 VPUser &U, unsigned) {

2755 return !ExpressionRecipesAsSetOfUsers.contains(&U);

2756 });

2758 }

2759 if (R->getParent())

2760 R->removeFromParent();

2761 }

2762

2763

2764

2765

2766

2767 for (auto *R : ExpressionRecipes) {

2768 for (const auto &[Idx, Op] : enumerate(R->operands())) {

2769 auto *Def = Op->getDefiningRecipe();

2770 if (Def && ExpressionRecipesAsSetOfUsers.contains(Def))

2771 continue;

2773 LiveInPlaceholders.push_back(new VPValue());

2774 }

2775 }

2776

2777

2778

2779 for (auto *R : ExpressionRecipes)

2780 for (auto const &[LiveIn, Tmp] : zip(operands(), LiveInPlaceholders))

2781 R->replaceUsesOfWith(LiveIn, Tmp);

2782}

2783

2785 for (auto *R : ExpressionRecipes)

2786

2787

2788 if (!R->getParent())

2789 R->insertBefore(this);

2790

2792 LiveInPlaceholders[Idx]->replaceAllUsesWith(Op);

2793

2795 ExpressionRecipes.clear();

2796}

2797

2800 Type *RedTy = Ctx.Types.inferScalarType(this);

2804 "VPExpressionRecipe only supports integer types currently.");

2807 switch (ExpressionType) {

2808 case ExpressionTypes::ExtendedReduction: {

2812

2814 ->isPartialReduction()

2815 ? Ctx.TTI.getPartialReductionCost(

2816 Opcode, Ctx.Types.inferScalarType(getOperand(0)), nullptr,

2817 RedTy, VF,

2819 ExtR->getOpcode()),

2821 : Ctx.TTI.getExtendedReductionCost(

2822 Opcode, ExtR->getOpcode() == Instruction::ZExt, RedTy,

2823 SrcVecTy, std::nullopt, Ctx.CostKind);

2824 }

2825 case ExpressionTypes::MulAccReduction:

2826 return Ctx.TTI.getMulAccReductionCost(false, Opcode, RedTy, SrcVecTy,

2827 Ctx.CostKind);

2828

2829 case ExpressionTypes::ExtNegatedMulAccReduction:

2830 assert(Opcode == Instruction::Add && "Unexpected opcode");

2831 Opcode = Instruction::Sub;

2832 [[fallthrough]];

2833 case ExpressionTypes::ExtMulAccReduction: {

2835 if (RedR->isPartialReduction()) {

2839 return Ctx.TTI.getPartialReductionCost(

2840 Opcode, Ctx.Types.inferScalarType(getOperand(0)),

2841 Ctx.Types.inferScalarType(getOperand(1)), RedTy, VF,

2843 Ext0R->getOpcode()),

2845 Ext1R->getOpcode()),

2846 Mul->getOpcode(), Ctx.CostKind);

2847 }

2848 return Ctx.TTI.getMulAccReductionCost(

2850 Instruction::ZExt,

2851 Opcode, RedTy, SrcVecTy, Ctx.CostKind);

2852 }

2853 }

2854 llvm_unreachable("Unknown VPExpressionRecipe::ExpressionTypes enum");

2855}

2856

2859 return R->mayReadFromMemory() || R->mayWriteToMemory();

2860 });

2861}

2862

2865 none_of(ExpressionRecipes,

2866 [](VPSingleDefRecipe *R) { return R->mayHaveSideEffects(); }) &&

2867 "expression cannot contain recipes with side-effects");

2868 return false;

2869}

2870

2872

2873

2875 return RR && !RR->isPartialReduction();

2876}

2877

2878#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2879

2882 O << Indent << "EXPRESSION ";

2884 O << " = ";

2887

2888 switch (ExpressionType) {

2889 case ExpressionTypes::ExtendedReduction: {

2891 O << " + " << (Red->isPartialReduction() ? "partial." : "") << "reduce.";

2894 Red->printFlags(O);

2895

2898 << *Ext0->getResultType();

2899 if (Red->isConditional()) {

2900 O << ", ";

2901 Red->getCondOp()->printAsOperand(O, SlotTracker);

2902 }

2903 O << ")";

2904 break;

2905 }

2906 case ExpressionTypes::ExtNegatedMulAccReduction: {

2908 O << " + " << (Red->isPartialReduction() ? "partial." : "") << "reduce.";

2911 << " (sub (0, mul";

2913 Mul->printFlags(O);

2914 O << "(";

2918 << *Ext0->getResultType() << "), (";

2922 << *Ext1->getResultType() << ")";

2923 if (Red->isConditional()) {

2924 O << ", ";

2925 Red->getCondOp()->printAsOperand(O, SlotTracker);

2926 }

2927 O << "))";

2928 break;

2929 }

2930 case ExpressionTypes::MulAccReduction:

2931 case ExpressionTypes::ExtMulAccReduction: {

2933 O << " + " << (Red->isPartialReduction() ? "partial." : "") << "reduce.";

2936 << " (";

2937 O << "mul";

2938 bool IsExtended = ExpressionType == ExpressionTypes::ExtMulAccReduction;

2940 : ExpressionRecipes[0]);

2941 Mul->printFlags(O);

2942 if (IsExtended)

2943 O << "(";

2945 if (IsExtended) {

2948 << *Ext0->getResultType() << "), (";

2949 } else {

2950 O << ", ";

2951 }

2953 if (IsExtended) {

2956 << *Ext1->getResultType() << ")";

2957 }

2958 if (Red->isConditional()) {

2959 O << ", ";

2960 Red->getCondOp()->printAsOperand(O, SlotTracker);

2961 }

2962 O << ")";

2963 break;

2964 }

2965 }

2966}

2967

2971 O << Indent << "PARTIAL-REDUCE ";

2972 else

2973 O << Indent << "REDUCE ";

2975 O << " = ";

2977 O << " +";

2979 O << " reduce."

2982 << " (";

2985 O << ", ";

2987 }

2988 O << ")";

2989}

2990

2993 O << Indent << "REDUCE ";

2995 O << " = ";

2997 O << " +";

2999 O << " vp.reduce."

3002 << " (";

3004 O << ", ";

3007 O << ", ";

3009 }

3010 O << ")";

3011}

3012

3013#endif

3014

3015

3016

3017

3021 assert((!Instr->getType()->isAggregateType() ||

3023 "Expected vectorizable or non-aggregate type.");

3024

3025

3026 bool IsVoidRetTy = Instr->getType()->isVoidTy();

3027

3029 if (!IsVoidRetTy) {

3030 Cloned->setName(Instr->getName() + ".cloned");

3031 Type *ResultTy = State.TypeAnalysis.inferScalarType(RepRecipe);

3032

3033

3034

3035 if (ResultTy != Cloned->getType())

3037 }

3038

3041

3044

3046 State.setDebugLocFrom(DL);

3047

3048

3049

3051 auto InputLane = Lane;

3052 VPValue *Operand = I.value();

3055 Cloned->setOperand(I.index(), State.get(Operand, InputLane));

3056 }

3057

3058

3059 State.Builder.Insert(Cloned);

3060

3061 State.set(RepRecipe, Cloned, Lane);

3062

3063

3065 State.AC->registerAssumption(II);

3066

3071 [](VPValue *Op) { return Op->isDefinedOutsideLoopRegions(); })) &&

3072 "Expected a recipe is either within a region or all of its operands "

3073 "are defined outside the vectorized region.");

3074}

3075

3078

3079 if (!State.Lane) {

3080 assert(IsSingleScalar && "VPReplicateRecipes outside replicate regions "

3081 "must have already been unrolled");

3083 return;

3084 }

3085

3087 "uniform recipe shouldn't be predicated");

3088 assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");

3090

3091 if (State.VF.isVector() && shouldPack()) {

3092 Value *WideValue =

3093 State.Lane->isFirstLane()

3095 : State.get(this);

3096 State.set(this, State.packScalarIntoVectorizedValue(this, WideValue,

3097 *State.Lane));

3098 }

3099}

3100

3102

3103

3107 return false;

3108 });

3109}

3110

3111

3112

3113

3114

3115

3117 const Loop *L) {

3121 Instruction::GetElementPtr) ||

3124 return nullptr;

3125

3126

3127

3129 if (!Opd->isDefinedOutsideLoopRegions() &&

3131 return nullptr;

3132 }

3133

3135}

3136

3137

3138

3142

3143 while (!WorkList.empty()) {

3145 if (!Cur || !Seen.insert(Cur).second)

3146 continue;

3147

3149

3150

3152 [&](unsigned I) {

3153 return Seen.contains(

3154 Blend->getIncomingValue(I)->getDefiningRecipe());

3155 }))

3156 continue;

3157

3158 for (VPUser *U : Cur->users()) {

3160 if (InterleaveR->getAddr() == Cur)

3161 return true;

3163 if (RepR->getOpcode() == Instruction::Load &&

3164 RepR->getOperand(0) == Cur)

3165 return true;

3166 if (RepR->getOpcode() == Instruction::Store &&

3167 RepR->getOperand(1) == Cur)

3168 return true;

3169 }

3171 if (MemR->getAddr() == Cur && MemR->isConsecutive())

3172 return true;

3173 }

3174 }

3175

3176

3177

3178

3179 if (Blend)

3180 continue;

3181

3183 }

3184 return false;

3185}

3186

3190

3191

3192 Ctx.SkipCostComputation.insert(UI);

3193

3196

3198 case Instruction::GetElementPtr:

3199

3200

3201

3202

3203 return 0;

3204 case Instruction::Call: {

3205 auto *CalledFn =

3207

3210 for (const VPValue *ArgOp : ArgOps)

3211 Tys.push_back(Ctx.Types.inferScalarType(ArgOp));

3212

3213 if (CalledFn->isIntrinsic())

3214

3215

3216 switch (CalledFn->getIntrinsicID()) {

3217 case Intrinsic::assume:

3218 case Intrinsic::lifetime_end:

3219 case Intrinsic::lifetime_start:

3220 case Intrinsic::sideeffect:

3221 case Intrinsic::pseudoprobe:

3222 case Intrinsic::experimental_noalias_scope_decl: {

3225 "scalarizing intrinsic should be free");

3227 }

3228 default:

3229 break;

3230 }

3231

3232 Type *ResultTy = Ctx.Types.inferScalarType(this);

3234 Ctx.TTI.getCallInstrCost(CalledFn, ResultTy, Tys, Ctx.CostKind);

3236 if (CalledFn->isIntrinsic())

3237 ScalarCallCost = std::min(

3238 ScalarCallCost,

3241 return ScalarCallCost;

3242 }

3243

3245 Ctx.getScalarizationOverhead(ResultTy, ArgOps, VF);

3246 }

3247 case Instruction::Add:

3248 case Instruction::Sub:

3249 case Instruction::FAdd:

3250 case Instruction::FSub:

3251 case Instruction::Mul:

3252 case Instruction::FMul:

3253 case Instruction::FDiv:

3254 case Instruction::FRem:

3255 case Instruction::Shl:

3256 case Instruction::LShr:

3257 case Instruction::AShr:

3258 case Instruction::And:

3259 case Instruction::Or:

3260 case Instruction::Xor:

3261 case Instruction::ICmp:

3262 case Instruction::FCmp:

3264 Ctx) *

3266 case Instruction::SDiv:

3267 case Instruction::UDiv:

3268 case Instruction::SRem:

3269 case Instruction::URem: {

3273 return ScalarCost;

3274

3276 Ctx.getScalarizationOverhead(Ctx.Types.inferScalarType(this),

3278

3279

3280 if ( getRegion ()->isReplicator())

3281 return ScalarCost;

3282

3283

3285 Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);

3286

3287

3288

3289 ScalarCost /= Ctx.getPredBlockCostDivisor(UI->getParent());

3290 return ScalarCost;

3291 }

3292 case Instruction::Load:

3293 case Instruction::Store: {

3294

3295

3297 if (ParentRegion && ParentRegion->isReplicator())

3298 break;

3299

3300 bool IsLoad = UI->getOpcode() == Instruction::Load;

3304 break;

3305

3306 Type *ValTy = Ctx.Types.inferScalarType(IsLoad ? this : getOperand(0));

3307 Type *ScalarPtrTy = Ctx.Types.inferScalarType(PtrOp);

3309 unsigned AS = cast(ScalarPtrTy)->getAddressSpace();

3311 InstructionCost ScalarMemOpCost = Ctx.TTI.getMemoryOpCost(

3312 UI->getOpcode(), ValTy, Alignment, AS, Ctx.CostKind, OpInfo);

3313

3315 bool PreferVectorizedAddressing = Ctx.TTI.prefersVectorizedAddressing();

3316 bool UsedByLoadStoreAddress =

3319 ScalarMemOpCost + Ctx.TTI.getAddressComputationCost(

3320 PtrTy, UsedByLoadStoreAddress ? nullptr : &Ctx.SE,

3321 PtrSCEV, Ctx.CostKind);

3323 return ScalarCost;

3324

3327

3328

3329

3330

3331 if (!UsedByLoadStoreAddress) {

3332 bool EfficientVectorLoadStore =

3333 Ctx.TTI.supportsEfficientVectorElementLoadStore();

3334 if (!(IsLoad && !PreferVectorizedAddressing) &&

3335 !(!IsLoad && EfficientVectorLoadStore))

3337

3338 if (!EfficientVectorLoadStore)

3339 ResultTy = Ctx.Types.inferScalarType(this);

3340 }

3341

3343 Ctx.getScalarizationOverhead(ResultTy, OpsToScalarize, VF, true);

3344 }

3345 }

3346

3347 return Ctx.getLegacyCost(UI, VF);

3348}

3349

3350#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3353 O << Indent << (IsSingleScalar ? "CLONE " : "REPLICATE ");

3354

3357 O << " = ";

3358 }

3360 O << "call";

3362 O << "@" << CB->getCalledFunction()->getName() << "(";

3366 });

3367 O << ")";

3368 } else {

3372 }

3373

3375 O << " (S->V)";

3376}

3377#endif

3378

3380 assert(State.Lane && "Branch on Mask works only on single instance.");

3381

3383 Value *ConditionBit = State.get(BlockInMask, *State.Lane);

3384

3385

3386

3387 auto *CurrentTerminator = State.CFG.PrevBB->getTerminator();

3389 "Expected to replace unreachable terminator with conditional branch.");

3390 auto CondBr =

3391 State.Builder.CreateCondBr(ConditionBit, State.CFG.PrevBB, nullptr);

3392 CondBr->setSuccessor(0, nullptr);

3393 CurrentTerminator->eraseFromParent();

3394}

3395

3398

3399

3400

3401 return 0;

3402}

3403

3405 assert(State.Lane && "Predicated instruction PHI works per instance.");

3410 assert(PredicatingBB && "Predicated block has no single predecessor.");

3412 "operand must be VPReplicateRecipe");

3413

3414

3415

3416

3417

3418

3419

3420 if (State.hasVectorValue(getOperand(0))) {

3423 "Packed operands must generate an insertelement or insertvalue");

3424

3425

3426

3427

3428

3429

3431 for (unsigned I = 0; I < StructTy->getNumContainedTypes() - 1; I++)

3433

3434 PHINode *VPhi = State.Builder.CreatePHI(VecI->getType(), 2);

3435 VPhi->addIncoming(VecI->getOperand(0), PredicatingBB);

3436 VPhi->addIncoming(VecI, PredicatedBB);

3437 if (State.hasVectorValue(this))

3438 State.reset(this, VPhi);

3439 else

3440 State.set(this, VPhi);

3441

3442

3444 } else {

3446 return;

3447

3448 Type *PredInstType = State.TypeAnalysis.inferScalarType(getOperand(0));

3449 PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2);

3451 PredicatingBB);

3452 Phi->addIncoming(ScalarPredInst, PredicatedBB);

3453 if (State.hasScalarValue(this, *State.Lane))

3454 State.reset(this, Phi, *State.Lane);

3455 else

3456 State.set(this, Phi, *State.Lane);

3457

3458

3459 State.reset(getOperand(0), Phi, *State.Lane);

3460 }

3461}

3462

3463#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3466 O << Indent << "PHI-PREDICATED-INSTRUCTION ";

3468 O << " = ";

3470}

3471#endif

3472

3477 ->getAddressSpace();

3479 ? Instruction::Load

3480 : Instruction::Store;

3481

3483

3484

3485

3487 "Inconsecutive memory access should not have the order.");

3488

3491

3492

3493

3496

3500 : Intrinsic::vp_scatter;

3501 return Ctx.TTI.getAddressComputationCost(PtrTy, nullptr, nullptr,

3502 Ctx.CostKind) +

3503 Ctx.TTI.getMemIntrinsicInstrCost(

3506 Ctx.CostKind);

3507 }

3508

3512 : Intrinsic::masked_store;

3513 Cost += Ctx.TTI.getMemIntrinsicInstrCost(

3515 } else {

3519 Cost += Ctx.TTI.getMemoryOpCost(Opcode, Ty, Alignment, AS, Ctx.CostKind,

3521 }

3523 return Cost;

3524

3525 return Cost += Ctx.TTI.getShuffleCost(

3528}

3529

3534

3535 auto &Builder = State.Builder;

3536 Value *Mask = nullptr;

3537 if (auto *VPMask = getMask()) {

3538

3539

3540 Mask = State.get(VPMask);

3542 Mask = Builder.CreateVectorReverse(Mask, "reverse");

3543 }

3544

3545 Value *Addr = State.get(getAddr(), !CreateGather);

3547 if (CreateGather) {

3548 NewLI = Builder.CreateMaskedGather(DataTy, Addr, Alignment, Mask, nullptr,

3549 "wide.masked.gather");

3550 } else if (Mask) {

3551 NewLI =

3552 Builder.CreateMaskedLoad(DataTy, Addr, Alignment, Mask,

3554 } else {

3555 NewLI = Builder.CreateAlignedLoad(DataTy, Addr, Alignment, "wide.load");

3556 }

3559 NewLI = Builder.CreateVectorReverse(NewLI, "reverse");

3560 State.set(this, NewLI);

3561}

3562

3563#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3566 O << Indent << "WIDEN ";

3568 O << " = load ";

3570}

3571#endif

3572

3573

3574

3578 Value *AllTrueMask =

3579 Builder.CreateVectorSplat(ValTy->getElementCount(), Builder.getTrue());

3580 return Builder.CreateIntrinsic(ValTy, Intrinsic::experimental_vp_reverse,

3581 {Operand, AllTrueMask, EVL}, nullptr, Name);

3582}

3583

3588

3589 auto &Builder = State.Builder;

3592 Value *Addr = State.get(getAddr(), !CreateGather);

3593 Value *Mask = nullptr;

3595 Mask = State.get(VPMask);

3597 Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");

3598 } else {

3599 Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());

3600 }

3601

3602 if (CreateGather) {

3603 NewLI =

3604 Builder.CreateIntrinsic(DataTy, Intrinsic::vp_gather, {Addr, Mask, EVL},

3605 nullptr, "wide.masked.gather");

3606 } else {

3607 NewLI = Builder.CreateIntrinsic(DataTy, Intrinsic::vp_load,

3608 {Addr, Mask, EVL}, nullptr, "vp.op.load");

3609 }

3616 State.set(this, Res);

3617}

3618

3623

3624

3625

3626

3627

3628

3631 ->getAddressSpace();

3634 Ctx.CostKind);

3636 return Cost;

3637

3638 return Cost + Ctx.TTI.getShuffleCost(

3641}

3642

3643#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3646 O << Indent << "WIDEN ";

3648 O << " = vp.load ";

3650}

3651#endif

3652

3656

3657 auto &Builder = State.Builder;

3658

3659 Value *Mask = nullptr;

3660 if (auto *VPMask = getMask()) {

3661

3662

3663 Mask = State.get(VPMask);

3665 Mask = Builder.CreateVectorReverse(Mask, "reverse");

3666 }

3667

3668 Value *StoredVal = State.get(StoredVPValue);

3670

3671

3672 StoredVal = Builder.CreateVectorReverse(StoredVal, "reverse");

3673

3674

3675 }

3676 Value *Addr = State.get(getAddr(), !CreateScatter);

3678 if (CreateScatter)

3679 NewSI = Builder.CreateMaskedScatter(StoredVal, Addr, Alignment, Mask);

3680 else if (Mask)

3681 NewSI = Builder.CreateMaskedStore(StoredVal, Addr, Alignment, Mask);

3682 else

3683 NewSI = Builder.CreateAlignedStore(StoredVal, Addr, Alignment);

3685}

3686

3687#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3690 O << Indent << "WIDEN store ";

3692}

3693#endif

3694

3698

3699 auto &Builder = State.Builder;

3700

3702 Value *StoredVal = State.get(StoredValue);

3705 StoredVal = createReverseEVL(Builder, StoredVal, EVL, "vp.reverse");

3706 Value *Mask = nullptr;

3708 Mask = State.get(VPMask);

3710 Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");

3711 } else {

3712 Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());

3713 }

3714 Value *Addr = State.get(getAddr(), !CreateScatter);

3715 if (CreateScatter) {

3717 Intrinsic::vp_scatter,

3718 {StoredVal, Addr, Mask, EVL});

3719 } else {

3721 Intrinsic::vp_store,

3722 {StoredVal, Addr, Mask, EVL});

3723 }

3727}

3728

3733

3734

3735

3736

3737

3738

3741 ->getAddressSpace();

3744 Ctx.CostKind);

3746 return Cost;

3747

3748 return Cost + Ctx.TTI.getShuffleCost(

3751}

3752

3753#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3756 O << Indent << "WIDEN vp.store ";

3758}

3759#endif

3760

3763

3764 auto VF = DstVTy->getElementCount();

3766 assert(VF == SrcVecTy->getElementCount() && "Vector dimensions do not match");

3767 Type *SrcElemTy = SrcVecTy->getElementType();

3768 Type *DstElemTy = DstVTy->getElementType();

3769 assert((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) &&

3770 "Vector elements must have same size");

3771

3772

3774 return Builder.CreateBitOrPointerCast(V, DstVTy);

3775 }

3776

3777

3778

3779

3781 "Only one type should be a pointer type");

3783 "Only one type should be a floating point type");

3784 Type *IntTy =

3787 Value *CastVal = Builder.CreateBitOrPointerCast(V, VecIntTy);

3788 return Builder.CreateBitOrPointerCast(CastVal, DstVTy);

3789}

3790

3791

3792

3794 const Twine &Name) {

3795 unsigned Factor = Vals.size();

3796 assert(Factor > 1 && "Tried to interleave invalid number of vectors");

3797

3799#ifndef NDEBUG

3800 for (Value *Val : Vals)

3801 assert(Val->getType() == VecTy && "Tried to interleave mismatched types");

3802#endif

3803

3804

3805

3806 if (VecTy->isScalableTy()) {

3807 assert(Factor <= 8 && "Unsupported interleave factor for scalable vectors");

3808 return Builder.CreateVectorInterleave(Vals, Name);

3809 }

3810

3811

3813

3814

3815 const unsigned NumElts = VecTy->getElementCount().getFixedValue();

3816 return Builder.CreateShuffleVector(

3818}

3819

3820

3821

3822

3823

3824

3825

3826

3827

3828

3829

3830

3831

3832

3833

3834

3835

3836

3837

3838

3839

3840

3841

3842

3843

3844

3845

3846

3847

3849 assert(!State.Lane && "Interleave group being replicated.");

3851 "Masking gaps for scalable vectors is not yet supported.");

3854

3855

3857 unsigned InterleaveFactor = Group->getFactor();

3858 auto *VecTy = VectorType::get(ScalarTy, State.VF * InterleaveFactor);

3859

3862 Value *ResAddr = State.get(Addr, VPLane(0));

3863

3864 auto CreateGroupMask = [&BlockInMask, &State,

3865 &InterleaveFactor](Value *MaskForGaps) -> Value * {

3866 if (State.VF.isScalable()) {

3867 assert(!MaskForGaps && "Interleaved groups with gaps are not supported.");

3868 assert(InterleaveFactor <= 8 &&

3869 "Unsupported deinterleave factor for scalable vectors");

3870 auto *ResBlockInMask = State.get(BlockInMask);

3873 }

3874

3875 if (!BlockInMask)

3876 return MaskForGaps;

3877

3878 Value *ResBlockInMask = State.get(BlockInMask);

3879 Value *ShuffledMask = State.Builder.CreateShuffleVector(

3880 ResBlockInMask,

3882 "interleaved.mask");

3883 return MaskForGaps ? State.Builder.CreateBinOp(Instruction::And,

3884 ShuffledMask, MaskForGaps)

3885 : ShuffledMask;

3886 };

3887

3888 const DataLayout &DL = Instr->getDataLayout();

3889

3891 Value *MaskForGaps = nullptr;

3893 MaskForGaps =

3895 assert(MaskForGaps && "Mask for Gaps is required but it is null");

3896 }

3897

3899 if (BlockInMask || MaskForGaps) {

3900 Value *GroupMask = CreateGroupMask(MaskForGaps);

3902 NewLoad = State.Builder.CreateMaskedLoad(VecTy, ResAddr,

3903 Group->getAlign(), GroupMask,

3904 PoisonVec, "wide.masked.vec");

3905 } else

3906 NewLoad = State.Builder.CreateAlignedLoad(VecTy, ResAddr,

3907 Group->getAlign(), "wide.vec");

3909

3911

3913 if (VecTy->isScalableTy()) {

3914

3915

3916 assert(InterleaveFactor <= 8 &&

3917 "Unsupported deinterleave factor for scalable vectors");

3918 NewLoad = State.Builder.CreateIntrinsic(

3920 NewLoad->getType(), NewLoad,

3921 nullptr, "strided.vec");

3922 }

3923

3924 auto CreateStridedVector = [&InterleaveFactor, &State,

3925 &NewLoad](unsigned Index) -> Value * {

3926 assert(Index < InterleaveFactor && "Illegal group index");

3927 if (State.VF.isScalable())

3928 return State.Builder.CreateExtractValue(NewLoad, Index);

3929

3930

3931

3932 auto StrideMask =

3933 createStrideMask(Index, InterleaveFactor, State.VF.getFixedValue());

3934 return State.Builder.CreateShuffleVector(NewLoad, StrideMask,

3935 "strided.vec");

3936 };

3937

3938 for (unsigned I = 0, J = 0; I < InterleaveFactor; ++I) {

3940

3941

3942 if (!Member)

3943 continue;

3944

3945 Value *StridedVec = CreateStridedVector(I);

3946

3947

3948 if (Member->getType() != ScalarTy) {

3950 StridedVec =

3952 }

3953

3955 StridedVec = State.Builder.CreateVectorReverse(StridedVec, "reverse");

3956

3957 State.set(VPDefs[J], StridedVec);

3958 ++J;

3959 }

3960 return;

3961 }

3962

3963

3965

3966

3967 Value *MaskForGaps =

3970 "Mismatch between NeedsMaskForGaps and MaskForGaps");

3972

3974 unsigned StoredIdx = 0;

3975 for (unsigned i = 0; i < InterleaveFactor; i++) {

3977 "Fail to get a member from an interleaved store group");

3979

3980

3981 if (!Member) {

3984 continue;

3985 }

3986

3987 Value *StoredVec = State.get(StoredValues[StoredIdx]);

3988 ++StoredIdx;

3989

3991 StoredVec = State.Builder.CreateVectorReverse(StoredVec, "reverse");

3992

3993

3994

3995 if (StoredVec->getType() != SubVT)

3997

3998 StoredVecs.push_back(StoredVec);

3999 }

4000

4001

4004 if (BlockInMask || MaskForGaps) {

4005 Value *GroupMask = CreateGroupMask(MaskForGaps);

4006 NewStoreInstr = State.Builder.CreateMaskedStore(

4007 IVec, ResAddr, Group->getAlign(), GroupMask);

4008 } else

4009 NewStoreInstr =

4010 State.Builder.CreateAlignedStore(IVec, ResAddr, Group->getAlign());

4011

4013

4015}

4016

4017#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4021 O << Indent << "INTERLEAVE-GROUP with factor " << IG->getFactor() << " at ";

4022 IG->getInsertPos()->printAsOperand(O, false);

4023 O << ", ";

4026 if (Mask) {

4027 O << ", ";

4029 }

4030

4031 unsigned OpIdx = 0;

4032 for (unsigned i = 0; i < IG->getFactor(); ++i) {

4033 if (!IG->getMember(i))

4034 continue;

4036 O << "\n" << Indent << " store ";

4038 O << " to index " << i;

4039 } else {

4040 O << "\n" << Indent << " ";

4042 O << " = load from index " << i;

4043 }

4045 }

4046}

4047#endif

4048

4050 assert(!State.Lane && "Interleave group being replicated.");

4051 assert(State.VF.isScalable() &&

4052 "Only support scalable VF for EVL tail-folding.");

4054 "Masking gaps for scalable vectors is not yet supported.");

4057

4058

4060 unsigned InterleaveFactor = Group->getFactor();

4061 assert(InterleaveFactor <= 8 &&

4062 "Unsupported deinterleave/interleave factor for scalable vectors");

4063 ElementCount WideVF = State.VF * InterleaveFactor;

4065

4067 Value *ResAddr = State.get(Addr, VPLane(0));

4069 Value *InterleaveEVL = State.Builder.CreateMul(

4070 EVL, ConstantInt::get(EVL->getType(), InterleaveFactor), "interleave.evl",

4071 true, true);

4072 LLVMContext &Ctx = State.Builder.getContext();

4073

4074 Value *GroupMask = nullptr;

4078 } else {

4079 GroupMask =

4080 State.Builder.CreateVectorSplat(WideVF, State.Builder.getTrue());

4081 }

4082

4083

4085 CallInst *NewLoad = State.Builder.CreateIntrinsic(

4086 VecTy, Intrinsic::vp_load, {ResAddr, GroupMask, InterleaveEVL}, nullptr,

4087 "wide.vp.load");

4090

4092

4094

4095

4096

4097 NewLoad = State.Builder.CreateIntrinsic(

4099 NewLoad->getType(), NewLoad,

4100 nullptr, "strided.vec");

4101

4102 const DataLayout &DL = Instr->getDataLayout();

4103 for (unsigned I = 0, J = 0; I < InterleaveFactor; ++I) {

4105

4106 if (!Member)

4107 continue;

4108

4109 Value *StridedVec = State.Builder.CreateExtractValue(NewLoad, I);

4110

4111 if (Member->getType() != ScalarTy) {

4113 StridedVec =

4115 }

4116

4117 State.set(getVPValue(J), StridedVec);

4118 ++J;

4119 }

4120 return;

4121 }

4122

4123

4125

4127

4129 const DataLayout &DL = Instr->getDataLayout();

4130 for (unsigned I = 0, StoredIdx = 0; I < InterleaveFactor; I++) {

4132

4133 if (!Member) {

4135 continue;

4136 }

4137

4138 Value *StoredVec = State.get(StoredValues[StoredIdx]);

4139

4140 if (StoredVec->getType() != SubVT)

4142

4143 StoredVecs.push_back(StoredVec);

4144 ++StoredIdx;

4145 }

4146

4147

4150 State.Builder.CreateIntrinsic(Type::getVoidTy(Ctx), Intrinsic::vp_store,

4151 {IVec, ResAddr, GroupMask, InterleaveEVL});

4154

4156

4158}

4159

4160#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4164 O << Indent << "INTERLEAVE-GROUP with factor " << IG->getFactor() << " at ";

4165 IG->getInsertPos()->printAsOperand(O, false);

4166 O << ", ";

4168 O << ", ";

4171 O << ", ";

4173 }

4174

4175 unsigned OpIdx = 0;

4176 for (unsigned i = 0; i < IG->getFactor(); ++i) {

4177 if (!IG->getMember(i))

4178 continue;

4180 O << "\n" << Indent << " vp.store ";

4182 O << " to index " << i;

4183 } else {

4184 O << "\n" << Indent << " ";

4186 O << " = vp.load from index " << i;

4187 }

4189 }

4190}

4191#endif

4192

4196

4197 unsigned InsertPosIdx = 0;

4198 for (unsigned Idx = 0; IG->getFactor(); ++Idx)

4199 if (auto *Member = IG->getMember(Idx)) {

4200 if (Member == InsertPos)

4201 break;

4202 InsertPosIdx++;

4203 }

4204 Type *ValTy = Ctx.Types.inferScalarType(

4209 ->getAddressSpace();

4210

4211 unsigned InterleaveFactor = IG->getFactor();

4212 auto *WideVecTy = VectorType::get(ValTy, VF * InterleaveFactor);

4213

4214

4216 for (unsigned IF = 0; IF < InterleaveFactor; IF++)

4217 if (IG->getMember(IF))

4219

4220

4222 InsertPos->getOpcode(), WideVecTy, IG->getFactor(), Indices,

4223 IG->getAlign(), AS, Ctx.CostKind, getMask(), NeedsMaskForGaps);

4224

4225 if (!IG->isReverse())

4226 return Cost;

4227

4228 return Cost + IG->getNumMembers() *

4230 VectorTy, VectorTy, {}, Ctx.CostKind,

4231 0);

4232}

4233

4234#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4237 O << Indent << "EMIT ";

4239 O << " = CANONICAL-INDUCTION ";

4241}

4242#endif

4243

4248

4249#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4253 "unexpected number of operands");

4254 O << Indent << "EMIT ";

4256 O << " = WIDEN-POINTER-INDUCTION ";

4258 O << ", ";

4260 O << ", ";

4263 O << ", ";

4265 O << ", ";

4267 }

4268}

4269

4272 O << Indent << "EMIT ";

4274 O << " = EXPAND SCEV " << *Expr;

4275}

4276#endif

4277

4279 Value *CanonicalIV = State.get(getOperand(0), true);

4281 IRBuilder<> Builder(State.CFG.PrevBB->getTerminator());

4284 ? CanonicalIV

4285 : Builder.CreateVectorSplat(VF, CanonicalIV, "broadcast");

4288 VStep = Builder.CreateVectorSplat(VF, VStep);

4289 VStep =

4290 Builder.CreateAdd(VStep, Builder.CreateStepVector(VStep->getType()));

4291 }

4292 Value *CanonicalVectorIV = Builder.CreateAdd(VStart, VStep, "vec.iv");

4293 State.set(this, CanonicalVectorIV);

4294}

4295

4296#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4299 O << Indent << "EMIT ";

4301 O << " = WIDEN-CANONICAL-INDUCTION ";

4303}

4304#endif

4305

4307 auto &Builder = State.Builder;

4308

4310

4311 Type *VecTy = State.VF.isScalar()

4312 ? VectorInit->getType()

4314

4316 State.CFG.VPBB2IRBB.at(getParent()->getCFGPredecessor(0));

4317 if (State.VF.isVector()) {

4319 auto *One = ConstantInt::get(IdxTy, 1);

4321 Builder.SetInsertPoint(VectorPH->getTerminator());

4322 auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, State.VF);

4323 auto *LastIdx = Builder.CreateSub(RuntimeVF, One);

4324 VectorInit = Builder.CreateInsertElement(

4325 PoisonValue::get(VecTy), VectorInit, LastIdx, "vector.recur.init");

4326 }

4327

4328

4330 Phi->insertBefore(State.CFG.PrevBB->getFirstInsertionPt());

4331 Phi->addIncoming(VectorInit, VectorPH);

4332 State.set(this, Phi);

4333}

4334

4339 return Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);

4340

4341 return 0;

4342}

4343

4344#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4347 O << Indent << "FIRST-ORDER-RECURRENCE-PHI ";

4349 O << " = phi ";

4351}

4352#endif

4353

4355

4356

4358

4359

4360

4361

4362

4364 State.CFG.VPBB2IRBB.at(getParent()->getCFGPredecessor(0));

4365 bool ScalarPHI = State.VF.isScalar() || isInLoop();

4366 Value *StartV = State.get(StartVPV, ScalarPHI);

4368

4369 BasicBlock *HeaderBB = State.CFG.PrevBB;

4370 assert(State.CurrentParentLoop->getHeader() == HeaderBB &&

4371 "recipe must be in the vector loop header");

4374 State.set(this, Phi, isInLoop());

4375

4376 Phi->addIncoming(StartV, VectorPH);

4377}

4378

4379#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4382 O << Indent << "WIDEN-REDUCTION-PHI ";

4383

4385 O << " = phi ";

4389}

4390#endif

4391

4395 Instruction *VecPhi = State.Builder.CreatePHI(VecTy, 2, Name);

4396 State.set(this, VecPhi);

4397}

4398

4399#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4402 O << Indent << "WIDEN-PHI ";

4403

4405 O << " = phi ";

4407}

4408#endif

4409

4410

4411

4414 State.CFG.VPBB2IRBB.at(getParent()->getCFGPredecessor(0));

4417 State.Builder.CreatePHI(StartMask->getType(), 2, "active.lane.mask");

4418 Phi->addIncoming(StartMask, VectorPH);

4419 State.set(this, Phi);

4420}

4421

4422#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4425 O << Indent << "ACTIVE-LANE-MASK-PHI ";

4426

4428 O << " = phi ";

4430}

4431#endif

4432

4433#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4436 O << Indent << "EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ";

4437

4439 O << " = phi ";

4441}

4442#endif

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static MCDisassembler::DecodeStatus addOperand(MCInst &Inst, const MCOperand &Opnd)

AMDGPU Lower Kernel Arguments

AMDGPU Register Bank Select

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

static std::pair< Value *, APInt > getMask(Value *WideMask, unsigned Factor, ElementCount LeafValueEC)

const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]

This file provides a LoopVectorizationPlanner class.

static const SCEV * getAddressAccessSCEV(Value *Ptr, LoopVectorizationLegality *Legal, PredicatedScalarEvolution &PSE, const Loop *TheLoop)

Gets Address Access SCEV after verifying that the access pattern is loop invariant except the inducti...

static bool isOrdered(const Instruction *I)

MachineInstr unsigned OpIdx

uint64_t IntrinsicInst * II

const SmallVectorImpl< MachineOperand > & Cond

This file defines the SmallVector class.

static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")

static SymbolRef::Type getType(const Symbol *Sym)

This file contains the declarations of different VPlan-related auxiliary helpers.

static Instruction * createReverseEVL(IRBuilderBase &Builder, Value *Operand, Value *EVL, const Twine &Name)

Use all-true mask for reverse rather than actual mask, as it avoids a dependence w/o affecting the re...

Definition VPlanRecipes.cpp:3575

static Value * interleaveVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vals, const Twine &Name)

Return a vector containing interleaved elements from multiple smaller input vectors.

Definition VPlanRecipes.cpp:3793

static InstructionCost getCostForIntrinsics(Intrinsic::ID ID, ArrayRef< const VPValue * > Operands, const VPRecipeWithIRFlags &R, ElementCount VF, VPCostContext &Ctx)

Compute the cost for the intrinsic ID with Operands, produced by R.

Definition VPlanRecipes.cpp:1738

static Value * createBitOrPointerCast(IRBuilderBase &Builder, Value *V, VectorType *DstVTy, const DataLayout &DL)

Definition VPlanRecipes.cpp:3761

SmallVector< Value *, 2 > VectorParts

Definition VPlanRecipes.cpp:44

static bool isUsedByLoadStoreAddress(const VPUser *V)

Returns true if V is used as part of the address of another load or store.

Definition VPlanRecipes.cpp:3139

static void scalarizeInstruction(const Instruction *Instr, VPReplicateRecipe *RepRecipe, const VPLane &Lane, VPTransformState &State)

A helper function to scalarize a single Instruction in the innermost loop.

Definition VPlanRecipes.cpp:3018

static Constant * getSignedIntOrFpConstant(Type *Ty, int64_t C)

A helper function that returns an integer or floating-point constant with value C.

Definition VPlanRecipes.cpp:2305

static BranchInst * createCondBranch(Value *Cond, VPBasicBlock *VPBB, VPTransformState &State)

Create a conditional branch using Cond branching to the successors of VPBB.

Definition VPlanRecipes.cpp:517

static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)

Returns the opcode of Values or ~0 if they do not all agree.

This file contains the declarations of the Vectorization Plan base classes:

static const uint32_t IV[8]

void printAsOperand(OutputBuffer &OB, Prec P=Prec::Default, bool StrictlyWorse=false) const

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

bool empty() const

empty - Check if the array is empty.

static LLVM_ABI Attribute getWithAlignment(LLVMContext &Context, Align Alignment)

Return a uniquified Attribute object that has the specific alignment set.

LLVM Basic Block Representation.

LLVM_ABI const_iterator getFirstInsertionPt() const

Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...

LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const

Returns an iterator to the first instruction in this block that is not a PHINode instruction.

LLVM_ABI const BasicBlock * getSinglePredecessor() const

Return the predecessor of this block if it has a single predecessor block.

const Instruction * getTerminator() const LLVM_READONLY

Returns the terminator instruction if the block is well formed or null if the block is not well forme...

Conditional or Unconditional Branch instruction.

void setSuccessor(unsigned idx, BasicBlock *NewSucc)

void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)

Adds the attribute to the indicated argument.

This class represents a function call, abstracting a target machine's calling convention.

static LLVM_ABI bool isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy, const DataLayout &DL)

Check whether a bitcast, inttoptr, or ptrtoint cast between these types is valid and a no-op.

static Type * makeCmpResultType(Type *opnd_type)

Create a result type for fcmp/icmp.

Predicate

This enumeration lists the possible predicates for CmpInst subclasses.

@ ICMP_UGT

unsigned greater than

@ ICMP_ULT

unsigned less than

static LLVM_ABI StringRef getPredicateName(Predicate P)

An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...

This is the shared class of boolean and integer constants.

static ConstantInt * getSigned(IntegerType *Ty, int64_t V)

Return a ConstantInt with the specified value for the specified type.

uint64_t getZExtValue() const

Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...

This is an important base class in LLVM.

static LLVM_ABI Constant * getNullValue(Type *Ty)

Constructor to create a '0' constant of arbitrary type.

A parsed version of the target data layout string in and methods for querying it.

constexpr bool isVector() const

One or more elements.

static constexpr ElementCount getScalable(ScalarTy MinVal)

static constexpr ElementCount getFixed(ScalarTy MinVal)

constexpr bool isScalar() const

Exactly one element.

Convenience struct for specifying and reasoning about fast-math flags.

LLVM_ABI void print(raw_ostream &O) const

Print fast-math flags to O.

void setAllowContract(bool B=true)

bool noSignedZeros() const

void setAllowReciprocal(bool B=true)

bool allowReciprocal() const

void setNoSignedZeros(bool B=true)

bool allowReassoc() const

Flag queries.

void setNoNaNs(bool B=true)

void setAllowReassoc(bool B=true)

Flag setters.

void setApproxFunc(bool B=true)

void setNoInfs(bool B=true)

bool allowContract() const

Class to represent function types.

Type * getParamType(unsigned i) const

Parameter type accessors.

bool willReturn() const

Determine if the function will return.

bool doesNotThrow() const

Determine if the function cannot unwind.

Type * getReturnType() const

Returns the type of the ret val.

Common base class shared among various IRBuilders.

Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")

IntegerType * getInt1Ty()

Fetch the type representing a single bit.

Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")

Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")

LLVM_ABI Value * CreateVectorSplice(Value *V1, Value *V2, int64_t Imm, const Twine &Name="")

Return a vector splice intrinsic if using scalable vectors, otherwise return a shufflevector.

LLVM_ABI Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")

Return a vector value that contains.

Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")

LLVM_ABI Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)

Value * CreateFreeze(Value *V, const Twine &Name="")

IntegerType * getInt32Ty()

Fetch the type representing a 32-bit integer.

Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())

void setFastMathFlags(FastMathFlags NewFMF)

Set the fast-math flags to be used with generated fp-math operators.

IntegerType * getInt64Ty()

Fetch the type representing a 64-bit integer.

Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")

ConstantInt * getInt64(uint64_t C)

Get a constant 64-bit value.

LLVM_ABI CallInst * CreateOrReduce(Value *Src)

Create a vector int OR reduction intrinsic of the source vector.

Value * CreateLogicalAnd(Value *Cond1, Value *Cond2, const Twine &Name="", Instruction *MDFrom=nullptr)

LLVM_ABI CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")

Create a call to intrinsic ID with Args, mangled using Types.

ConstantInt * getInt32(uint32_t C)

Get a constant 32-bit value.

Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)

Value * CreateNot(Value *V, const Twine &Name="")

Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateCountTrailingZeroElems(Type *ResTy, Value *Mask, bool ZeroIsPoison=true, const Twine &Name="")

Create a call to llvm.experimental_cttz_elts.

Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)

Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

ConstantInt * getFalse()

Get the constant value for i1 false.

Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)

Value * CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="", bool IsDisjoint=false)

Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

static InstructionCost getInvalid(CostType Val=0)

LLVM_ABI InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

const char * getOpcodeName() const

unsigned getOpcode() const

Returns a member of one of the enums like Instruction::Add.

static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)

This static method is the primary way of constructing an IntegerType.

The group of interleaved loads/stores sharing the same stride and close to each other.

uint32_t getFactor() const

InstTy * getMember(uint32_t Index) const

Get the member with the given index Index.

InstTy * getInsertPos() const

void addMetadata(InstTy *NewInst) const

Add metadata (e.g.

This is an important class for using LLVM in a threaded context.

Represents a single loop in the control flow graph.

Information for memory intrinsic cost model.

A Module instance is used to store all the information related to an LLVM module.

void addIncoming(Value *V, BasicBlock *BB)

Add an incoming value to the end of the PHI list.

static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)

Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...

static LLVM_ABI PoisonValue * get(Type *T)

Static factory methods - Return an 'poison' object of the specified type.

static bool isSignedRecurrenceKind(RecurKind Kind)

Returns true if recurrece kind is a signed redux kind.

static LLVM_ABI unsigned getOpcode(RecurKind Kind)

Returns the opcode corresponding to the RecurrenceKind.

unsigned getOpcode() const

static bool isAnyOfRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...

static bool isFindLastIVRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...

static bool isFindIVRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...

static bool isMinMaxRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is any min/max kind.

This class represents an analyzed expression in the program.

The main scalar evolution driver.

This class represents the LLVM 'select' instruction.

This class provides computation of slot numbers for LLVM Assembly writing.

std::pair< iterator, bool > insert(PtrType Ptr)

Inserts Ptr if and only if there is no element in the container equal to Ptr.

SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.

reference emplace_back(ArgTypes &&... Args)

void append(ItTy in_start, ItTy in_end)

Add the specified range to the end of the SmallVector.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

static LLVM_ABI PartialReductionExtendKind getPartialReductionExtendKind(Instruction *I)

Get the kind of extension that an instruction represents.

static LLVM_ABI OperandValueInfo getOperandInfo(const Value *V)

Collect properties of V used in cost analysis, e.g. OP_PowerOf2.

@ TCC_Free

Expected to fold away in lowering.

@ SK_Splice

Concatenates elements from the first input vector with elements of the second input vector.

@ SK_Reverse

Reverse the order of the vector.

CastContextHint

Represents a hint about the context in which a cast is used.

@ Reversed

The cast is used with a reversed load/store.

@ Masked

The cast is used with a masked load/store.

@ None

The cast is not used with a load/store of any kind.

@ Normal

The cast is used with a normal load/store.

@ Interleave

The cast is used with an interleaved load/store.

@ GatherScatter

The cast is used with a gather/scatter.

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

The instances of the Type class are immutable: once they are created, they are never changed.

static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)

bool isVectorTy() const

True if this is an instance of VectorType.

static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)

bool isPointerTy() const

True if this is an instance of PointerType.

static LLVM_ABI Type * getVoidTy(LLVMContext &C)

Type * getScalarType() const

If this is a vector type, return the element type, otherwise return 'this'.

bool isStructTy() const

True if this is an instance of StructType.

LLVMContext & getContext() const

Return the LLVMContext in which this type was uniqued.

LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY

If this is a vector type, return the getPrimitiveSizeInBits value for the element type.

static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)

bool isFloatingPointTy() const

Return true if this is one of the floating-point types.

bool isIntegerTy() const

True if this is an instance of IntegerType.

static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)

bool isVoidTy() const

Return true if this is 'void'.

value_op_iterator value_op_end()

void setOperand(unsigned i, Value *Val)

Value * getOperand(unsigned i) const

value_op_iterator value_op_begin()

void execute(VPTransformState &State) override

Generate the active lane mask phi of the vector loop.

Definition VPlanRecipes.cpp:4412

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4423

VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.

RecipeListTy & getRecipeList()

Returns a reference to the list of recipes.

void insert(VPRecipeBase *Recipe, iterator InsertPt)

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenMemoryRecipe.

Definition VPlanRecipes.cpp:2552

VPValue * getIncomingValue(unsigned Idx) const

Return incoming value number Idx.

unsigned getNumIncomingValues() const

Return the number of incoming values, taking into account when normalized the first incoming value wi...

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2567

VPBlockBase is the building block of the Hierarchical Control-Flow Graph.

const VPBlocksTy & getPredecessors() const

void printAsOperand(raw_ostream &OS, bool PrintType=false) const

const VPBlocksTy & getSuccessors() const

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPBranchOnMaskRecipe.

Definition VPlanRecipes.cpp:3396

void execute(VPTransformState &State) override

Generate the extraction of the appropriate bit from the block mask and the conditional branch.

Definition VPlanRecipes.cpp:3379

VPlan-based builder utility analogous to IRBuilder.

LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4235

This class augments a recipe with a set of VPValues defined by the recipe.

LLVM_ABI_FOR_TEST void dump() const

Dump the VPDef to stderr (for debugging).

unsigned getNumDefinedValues() const

Returns the number of values defined by the VPDef.

ArrayRef< VPValue * > definedValues()

Returns an ArrayRef of the values defined by the VPDef.

VPValue * getVPSingleValue()

Returns the only VPValue defined by the VPDef.

VPValue * getVPValue(unsigned I)

Returns the VPValue with index I defined by the VPDef.

unsigned getVPDefID() const

VPValue * getStepValue() const

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2338

VPValue * getStartValue() const

LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4434

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4270

void decompose()

Insert the recipes of the expression back into the VPlan, directly before the current recipe.

Definition VPlanRecipes.cpp:2784

bool isSingleScalar() const

Returns true if the result of this VPExpressionRecipe is a single-scalar.

Definition VPlanRecipes.cpp:2871

bool mayHaveSideEffects() const

Returns true if this expression contains recipes that may have side effects.

Definition VPlanRecipes.cpp:2863

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Compute the cost of this recipe either using a recipe's specialized implementation or using the legac...

Definition VPlanRecipes.cpp:2798

bool mayReadOrWriteMemory() const

Returns true if this expression contains recipes that may read from or write to memory.

Definition VPlanRecipes.cpp:2857

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2880

void execute(VPTransformState &State) override

Produce a vectorized histogram operation.

Definition VPlanRecipes.cpp:1822

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPHistogramRecipe.

Definition VPlanRecipes.cpp:1851

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:1888

VPValue * getMask() const

Return the mask operand if one was provided, or a null pointer if all lanes should be executed uncond...

Class to record and manage LLVM IR flags.

bool flagsValidForOpcode(unsigned Opcode) const

Returns true if the set flags are valid for Opcode.

Definition VPlanRecipes.cpp:1986

CmpInst::Predicate CmpPredicate

void printFlags(raw_ostream &O) const

Definition VPlanRecipes.cpp:2026

bool hasFastMathFlags() const

Returns true if the recipe has fast-math flags.

LLVM_ABI_FOR_TEST FastMathFlags getFastMathFlags() const

Definition VPlanRecipes.cpp:355

CmpInst::Predicate getPredicate() const

bool hasNoSignedWrap() const

void intersectFlags(const VPIRFlags &Other)

Only keep flags also present in Other.

Definition VPlanRecipes.cpp:315

GEPNoWrapFlags getGEPNoWrapFlags() const

bool hasPredicate() const

Returns true if the recipe has a comparison predicate.

DisjointFlagsTy DisjointFlags

bool hasNoUnsignedWrap() const

NonNegFlagsTy NonNegFlags

void applyFlags(Instruction &I) const

Apply the IR flags to I.

Instruction & getInstruction() const

void extractLastLaneOfLastPartOfFirstOperand(VPBuilder &Builder)

Update the recipe's first operand to the last lane of the last part of the operand using Builder.

Definition VPlanRecipes.cpp:1500

void execute(VPTransformState &State) override

The method which generates the output IR instructions that correspond to this VPRecipe,...

Definition VPlanRecipes.cpp:1485

LLVM_ABI_FOR_TEST InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPIRInstruction.

Definition VPlanRecipes.cpp:1493

VPIRInstruction(Instruction &I)

VPIRInstruction::create() should be used to create VPIRInstructions, as subclasses may need to be cre...

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:1515

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:1425

void execute(VPTransformState &State) override

Generate the instruction.

Definition VPlanRecipes.cpp:1397

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPInstruction.

Definition VPlanRecipes.cpp:994

bool doesGeneratePerAllLanes() const

Returns true if this VPInstruction generates scalar values for all lanes.

Definition VPlanRecipes.cpp:485

@ ExtractLane

Extracts a single lane (first operand) from a set of vector operands.

@ ComputeAnyOfResult

Compute the final result of a AnyOf reduction with select(cmp(),x,y), where one of (x,...

@ WideIVStep

Scale the first operand (vector step) by the second operand (scalar-step).

@ ExtractPenultimateElement

@ ResumeForEpilogue

Explicit user for the resume phi of the canonical induction in the main VPlan, used by the epilogue v...

@ Unpack

Extracts all lanes from its (non-scalable) vector operand.

@ FirstOrderRecurrenceSplice

@ ReductionStartVector

Start vector for reductions with 3 operands: the original start value, the identity value for the red...

@ BuildVector

Creates a fixed-width vector containing all operands.

@ BuildStructVector

Given operands of (the same) struct type, creates a struct of fixed- width vectors each containing a ...

@ VScale

Returns the value for vscale.

@ CanonicalIVIncrementForPart

@ CalculateTripCountMinusVF

bool opcodeMayReadOrWriteFromMemory() const

Returns true if the underlying opcode may read from or write to memory.

Definition VPlanRecipes.cpp:1173

LLVM_DUMP_METHOD void dump() const

Print the VPInstruction to dbgs() (for debugging).

Definition VPlanRecipes.cpp:1286

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the VPInstruction to O.

Definition VPlanRecipes.cpp:1291

StringRef getName() const

Returns the symbolic name assigned to the VPInstruction.

unsigned getOpcode() const

VPInstruction(unsigned Opcode, ArrayRef< VPValue * > Operands, const VPIRFlags &Flags={}, const VPIRMetadata &MD={}, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="")

Definition VPlanRecipes.cpp:407

bool usesFirstLaneOnly(const VPValue *Op) const override

Returns true if the recipe only uses the first lane of operand Op.

Definition VPlanRecipes.cpp:1216

bool isVectorToScalar() const

Returns true if this VPInstruction produces a scalar value from a vector, e.g.

Definition VPlanRecipes.cpp:1125

bool isSingleScalar() const

Returns true if this VPInstruction's operands are single scalars and the result is also a single scal...

Definition VPlanRecipes.cpp:1138

void execute(VPTransformState &State) override

Generate the instruction.

Definition VPlanRecipes.cpp:1150

bool usesFirstPartOnly(const VPValue *Op) const override

Returns true if the recipe only uses the first part of operand Op.

Definition VPlanRecipes.cpp:1265

bool needsMaskForGaps() const

Return true if the access needs a mask because of the gaps.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this recipe.

Definition VPlanRecipes.cpp:4193

Instruction * getInsertPos() const

const InterleaveGroup< Instruction > * getInterleaveGroup() const

VPValue * getMask() const

Return the mask used by this recipe.

ArrayRef< VPValue * > getStoredValues() const

Return the VPValues stored by this interleave group.

VPValue * getAddr() const

Return the address accessed by this recipe.

VPValue * getEVL() const

The VPValue of the explicit vector length.

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4161

unsigned getNumStoreOperands() const override

Returns the number of stored operands of this interleave group.

void execute(VPTransformState &State) override

Generate the wide load or store, and shuffles.

Definition VPlanRecipes.cpp:4049

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4018

unsigned getNumStoreOperands() const override

Returns the number of stored operands of this interleave group.

void execute(VPTransformState &State) override

Generate the wide load or store, and shuffles.

Definition VPlanRecipes.cpp:3848

In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...

static VPLane getLastLaneForVF(const ElementCount &VF)

static VPLane getLaneFromEnd(const ElementCount &VF, unsigned Offset)

static VPLane getFirstLane()

virtual const VPRecipeBase * getAsRecipe() const =0

Return a VPRecipeBase* to the current object.

virtual unsigned getNumIncoming() const

Returns the number of incoming values, also number of incoming blocks.

void removeIncomingValueFor(VPBlockBase *IncomingBlock) const

Removes the incoming value for IncomingBlock, which must be a predecessor.

Definition VPlanRecipes.cpp:1548

const VPBasicBlock * getIncomingBlock(unsigned Idx) const

Returns the incoming block with index Idx.

detail::zippy< llvm::detail::zip_first, VPUser::const_operand_range, const_incoming_blocks_range > incoming_values_and_blocks() const

Returns an iterator range over pairs of incoming values and corresponding incoming blocks.

VPValue * getIncomingValue(unsigned Idx) const

Returns the incoming VPValue with index Idx.

void printPhiOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const

Print the recipe.

Definition VPlanRecipes.cpp:1557

void execute(VPTransformState &State) override

Generates phi nodes for live-outs (from a replicate region) as needed to retain SSA form.

Definition VPlanRecipes.cpp:3404

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:3464

VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.

bool mayReadFromMemory() const

Returns true if the recipe may read from memory.

Definition VPlanRecipes.cpp:109

bool mayHaveSideEffects() const

Returns true if the recipe may have side-effects.

Definition VPlanRecipes.cpp:160

virtual void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const =0

Each concrete VPRecipe prints itself, without printing common information, like debug info or metadat...

VPRegionBlock * getRegion()

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override final

Print the recipe, delegating to printRecipe().

Definition VPlanRecipes.cpp:373

bool isPhi() const

Returns true for PHI-like recipes.

Definition VPlanRecipes.cpp:305

bool mayWriteToMemory() const

Returns true if the recipe may write to memory.

Definition VPlanRecipes.cpp:49

virtual InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const

Compute the cost of this recipe either using a recipe's specialized implementation or using the legac...

Definition VPlanRecipes.cpp:300

VPBasicBlock * getParent()

DebugLoc getDebugLoc() const

Returns the debug location of the recipe.

void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)

Unlink this recipe and insert into BB before I.

Definition VPlanRecipes.cpp:260

void insertBefore(VPRecipeBase *InsertPos)

Insert an unlinked recipe into a basic block immediately before the specified recipe.

Definition VPlanRecipes.cpp:223

void insertAfter(VPRecipeBase *InsertPos)

Insert an unlinked Recipe into a basic block immediately after the specified Recipe.

Definition VPlanRecipes.cpp:237

iplist< VPRecipeBase >::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

Definition VPlanRecipes.cpp:250

InstructionCost cost(ElementCount VF, VPCostContext &Ctx)

Return the cost of this recipe, taking into account if the cost computation should be skipped and the...

Definition VPlanRecipes.cpp:266

bool isScalarCast() const

Return true if the recipe is a scalar cast.

Definition VPlanRecipes.cpp:310

void removeFromParent()

This method unlinks 'this' from the containing basic block, but does not delete it.

Definition VPlanRecipes.cpp:244

void moveAfter(VPRecipeBase *MovePos)

Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...

Definition VPlanRecipes.cpp:255

VPRecipeBase(const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL=DebugLoc::getUnknown())

void execute(VPTransformState &State) override

Generate the reduction in the loop.

Definition VPlanRecipes.cpp:2647

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2991

VPValue * getEVL() const

The VPValue of the explicit vector length.

unsigned getVFScaleFactor() const

Get the factor that the VF of this recipe's output should be scaled by, or 1 if it isn't scaled.

bool isInLoop() const

Returns true if the phi is part of an in-loop reduction.

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4380

void execute(VPTransformState &State) override

Generate the phi/select nodes.

Definition VPlanRecipes.cpp:4354

bool isConditional() const

Return true if the in-loop reduction is conditional.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of VPReductionRecipe.

Definition VPlanRecipes.cpp:2681

VPValue * getVecOp() const

The VPValue of the vector value to be reduced.

VPValue * getCondOp() const

The VPValue of the condition for the block.

RecurKind getRecurrenceKind() const

Return the recurrence kind for the in-loop reduction.

bool isPartialReduction() const

Returns true if the reduction outputs a vector with a scaled down VF.

VPValue * getChainOp() const

The VPValue of the scalar Chain being accumulated.

bool isInLoop() const

Returns true if the reduction is in-loop.

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2968

void execute(VPTransformState &State) override

Generate the reduction in the loop.

Definition VPlanRecipes.cpp:2590

VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...

bool isReplicator() const

An indicator whether this region is to generate multiple replicated instances of output IR correspond...

VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...

void execute(VPTransformState &State) override

Generate replicas of the desired Ingredient.

Definition VPlanRecipes.cpp:3076

bool isSingleScalar() const

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPReplicateRecipe.

Definition VPlanRecipes.cpp:3187

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:3351

unsigned getOpcode() const

bool shouldPack() const

Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.

Definition VPlanRecipes.cpp:3101

VPValue * getStepValue() const

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2424

void execute(VPTransformState &State) override

Generate the scalarized versions of the phi node as needed by their users.

Definition VPlanRecipes.cpp:2351

VPSingleDef is a base class for recipes for modeling a sequence of one or more output IR that define ...

Instruction * getUnderlyingInstr()

Returns the underlying instruction.

LLVM_ABI_FOR_TEST LLVM_DUMP_METHOD void dump() const

Print this VPSingleDefRecipe to dbgs() (for debugging).

Definition VPlanRecipes.cpp:371

VPSingleDefRecipe(const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL=DebugLoc::getUnknown())

This class can be used to assign names to VPValues.

Type * inferScalarType(const VPValue *V)

Infer the type of V. Returns the scalar type of V.

Helper to access the operand that contains the unroll part for this recipe after unrolling.

VPValue * getUnrollPartOperand(const VPUser &U) const

Return the VPValue operand containing the unroll part or null if there is no such operand.

Definition VPlanRecipes.cpp:388

unsigned getUnrollPart(const VPUser &U) const

Return the unroll part.

Definition VPlanRecipes.cpp:395

This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...

void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const

Print the operands to O.

void setOperand(unsigned I, VPValue *New)

unsigned getNumOperands() const

operand_iterator op_begin()

VPValue * getOperand(unsigned N) const

virtual bool usesFirstLaneOnly(const VPValue *Op) const

Returns true if the VPUser only uses the first lane of operand Op.

This is the base class of the VPlan Def/Use graph, used for modeling the data flow into,...

bool isDefinedOutsideLoopRegions() const

Returns true if the VPValue is defined outside any loop.

VPRecipeBase * getDefiningRecipe()

Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...

friend class VPExpressionRecipe

void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const

bool hasMoreThanOneUniqueUser() const

Returns true if the value has more than one unique user.

Value * getLiveInIRValue() const

Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.

Value * getUnderlyingValue() const

Return the underlying Value attached to this VPValue.

VPValue(const unsigned char SC, Value *UV=nullptr, VPDef *Def=nullptr)

void replaceAllUsesWith(VPValue *New)

user_iterator user_begin()

unsigned getNumUsers() const

bool isLiveIn() const

Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.

void execute(VPTransformState &State) override

The method which generates the output IR instructions that correspond to this VPRecipe,...

Definition VPlanRecipes.cpp:2490

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2517

Type * getSourceElementType() const

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2542

void execute(VPTransformState &State) override

The method which generates the output IR instructions that correspond to this VPRecipe,...

Definition VPlanRecipes.cpp:2527

Function * getCalledScalarFunction() const

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenCallRecipe.

Definition VPlanRecipes.cpp:1658

void execute(VPTransformState &State) override

Produce a widened version of the call instruction.

Definition VPlanRecipes.cpp:1625

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:1666

void execute(VPTransformState &State) override

Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...

Definition VPlanRecipes.cpp:4278

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4297

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2287

Type * getResultType() const

Returns the result type of the cast.

void execute(VPTransformState &State) override

Produce widened copies of the cast.

Definition VPlanRecipes.cpp:2217

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenCastRecipe.

Definition VPlanRecipes.cpp:2232

void execute(VPTransformState &State) override

Generate the gep nodes.

Definition VPlanRecipes.cpp:2438

Type * getSourceElementType() const

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2475

bool usesFirstLaneOnly(const VPValue *Op) const override

Returns true if the recipe only uses the first lane of operand Op.

Definition VPlanRecipes.cpp:2433

VPValue * getStepValue()

Returns the step value of the induction.

TruncInst * getTruncInst()

Returns the first defined value as TruncInst, if it is one or nullptr otherwise.

Type * getScalarType() const

Returns the scalar type of the induction.

bool isCanonical() const

Returns true if the induction is canonical, i.e.

Definition VPlanRecipes.cpp:2325

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2311

Intrinsic::ID getVectorIntrinsicID() const

Return the ID of the intrinsic.

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:1801

StringRef getIntrinsicName() const

Return to name of the intrinsic as string.

Definition VPlanRecipes.cpp:1787

bool usesFirstLaneOnly(const VPValue *Op) const override

Returns true if the VPUser only uses the first lane of operand Op.

Definition VPlanRecipes.cpp:1791

Type * getResultType() const

Return the scalar return type of the intrinsic.

void execute(VPTransformState &State) override

Produce a widened version of the vector intrinsic.

Definition VPlanRecipes.cpp:1693

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this vector intrinsic.

Definition VPlanRecipes.cpp:1781

bool IsMasked

Whether the memory access is masked.

bool Reverse

Whether the consecutive accessed addresses are in reverse order.

bool isConsecutive() const

Return whether the loaded-from / stored-to addresses are consecutive.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenMemoryRecipe.

Definition VPlanRecipes.cpp:3473

bool Consecutive

Whether the accessed addresses are consecutive.

VPValue * getMask() const

Return the mask used by this recipe.

Align Alignment

Alignment information for this memory access.

VPValue * getAddr() const

Return the address accessed by this recipe.

bool isReverse() const

Return whether the consecutive loaded/stored addresses are in reverse order.

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4400

void execute(VPTransformState &State) override

Generate the phi/select nodes.

Definition VPlanRecipes.cpp:4392

bool onlyScalarsGenerated(bool IsScalable)

Returns true if only scalar values will be generated.

Definition VPlanRecipes.cpp:4244

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4250

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenRecipe.

Definition VPlanRecipes.cpp:2170

void execute(VPTransformState &State) override

Produce a widened instruction using the opcode and operands of the recipe, processing State....

Definition VPlanRecipes.cpp:2077

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2207

LLVM_ABI_FOR_TEST VPRegionBlock * getVectorLoopRegion()

Returns the VPRegionBlock of the vector loop.

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

LLVM_ABI void setName(const Twine &Name)

Change the name of the value.

LLVM_ABI LLVMContext & getContext() const

All values hold a context through their type.

void mutateType(Type *Ty)

Mutate the type of this Value to be of the specified type.

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

Base class of all SIMD vector types.

ElementCount getElementCount() const

Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...

static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)

This static method is the primary way to construct an VectorType.

Type * getElementType() const

constexpr ScalarTy getFixedValue() const

constexpr bool isScalable() const

Returns whether the quantity is scaled by a runtime quantity (vscale).

constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const

constexpr ScalarTy getKnownMinValue() const

Returns the minimum value this quantity can represent.

constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const

We do not provide the '/' operator here because division for polynomial types does not work in the sa...

const ParentTy * getParent() const

self_iterator getIterator()

typename base_list_type::iterator iterator

iterator erase(iterator where)

pointer remove(iterator &IT)

This class implements an extremely fast bulk output stream that can only output to a stream.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

constexpr std::underlying_type_t< E > Mask()

Get a bitmask with 1s in all places up to the high-order bit of E's largest value.

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

@ C

The default llvm calling convention, compatible with C.

LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})

Look up the Function declaration of the intrinsic id in the Module M.

LLVM_ABI Intrinsic::ID getDeinterleaveIntrinsicID(unsigned Factor)

Returns the corresponding llvm.vector.deinterleaveN intrinsic for factor N.

LLVM_ABI StringRef getBaseName(ID id)

Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....

bool match(Val *V, const Pattern &P)

auto m_LogicalOr()

Matches L || R where L and R are arbitrary values.

class_match< CmpInst > m_Cmp()

Matches any compare instruction and ignore it.

auto m_LogicalAnd()

Matches L && R where L and R are arbitrary values.

GEPLikeRecipe_match< Op0_t, Op1_t > m_GetElementPtr(const Op0_t &Op0, const Op1_t &Op1)

class_match< VPValue > m_VPValue()

Match an arbitrary VPValue and ignore it.

NodeAddr< DefNode * > Def

bool isSingleScalar(const VPValue *VPV)

Returns true if VPV is a single scalar, either because it produces the same value for all lanes or on...

bool onlyFirstPartUsed(const VPValue *Def)

Returns true if only the first part of Def is used.

bool onlyFirstLaneUsed(const VPValue *Def)

Returns true if only the first lane of Def is used.

bool onlyScalarValuesUsed(const VPValue *Def)

Returns true if only scalar values of Def are used by all users.

const SCEV * getSCEVExprForVPValue(const VPValue *V, ScalarEvolution &SE, const Loop *L=nullptr)

Return the SCEV expression for V.

This is an optimization pass for GlobalISel generic memory operations.

auto drop_begin(T &&RangeOrContainer, size_t N=1)

Return a range covering RangeOrContainer with the first N elements excluded.

LLVM_ABI Value * createSimpleReduction(IRBuilderBase &B, Value *Src, RecurKind RdxKind)

Create a reduction of the given vector.

detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)

zip iterator for two or more iteratable types.

FunctionAddr VTableAddr Value

LLVM_ABI Value * createFindLastIVReduction(IRBuilderBase &B, Value *Src, RecurKind RdxKind, Value *Start, Value *Sentinel)

Create a reduction of the given vector Src for a reduction of the kind RecurKind::FindLastIV.

bool all_of(R &&range, UnaryPredicate P)

Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.

LLVM_ABI Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)

Returns the min/max intrinsic used when expanding a min/max reduction.

auto enumerate(FirstRange &&First, RestRanges &&...Rest)

Given two or more input ranges, returns a new range whose values are tuples (A, B,...

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

const Value * getLoadStorePointerOperand(const Value *V)

A helper function that returns the pointer operand of a load or store instruction.

Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)

Return the runtime value for VF.

auto dyn_cast_if_present(const Y &Val)

dyn_cast_if_present - Functionally identical to dyn_cast, except that a null (or none in the case ...

iterator_range< T > make_range(T x, T y)

Convenience function for iterating over sub-ranges.

void append_range(Container &C, Range &&R)

Wrapper function to append range R to container C.

void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)

auto cast_or_null(const Y &Val)

LLVM_ABI Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)

Concatenate a list of vectors.

Align getLoadStoreAlignment(const Value *I)

A helper function that returns the alignment of load or store instruction.

bool isa_and_nonnull(const Y &Val)

LLVM_ABI Value * createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left, Value *Right)

Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.

auto dyn_cast_or_null(const Y &Val)

bool any_of(R &&range, UnaryPredicate P)

Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.

LLVM_ABI Constant * createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, const InterleaveGroup< Instruction > &Group)

Create a mask that filters the members of an interleave group where there are gaps.

LLVM_ABI llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)

Create a stride shuffle mask.

auto reverse(ContainerTy &&C)

LLVM_ABI llvm::SmallVector< int, 16 > createReplicatedMask(unsigned ReplicationFactor, unsigned VF)

Create a mask with replicated elements.

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

bool none_of(R &&Range, UnaryPredicate P)

Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.

SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)

Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...

Type * toVectorizedTy(Type *Ty, ElementCount EC)

A helper for converting to vectorized types.

cl::opt< unsigned > ForceTargetInstructionCost

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

auto drop_end(T &&RangeOrContainer, size_t N=1)

Return a range covering RangeOrContainer with the last N elements excluded.

bool canVectorizeTy(Type *Ty)

Returns true if Ty is a valid vector element type, void, or an unpacked literal struct where all elem...

LLVM_ABI llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)

Create an interleave shuffle mask.

RecurKind

These are the kinds of recurrences that we support.

@ UMin

Unsigned integer min implemented in terms of select(cmp()).

@ Mul

Product of integers.

@ SMax

Signed integer max implemented in terms of select(cmp()).

@ SMin

Signed integer min implemented in terms of select(cmp()).

@ Sub

Subtraction of integers.

@ UMax

Unsigned integer max implemented in terms of select(cmp()).

LLVM_ABI bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx, const TargetTransformInfo *TTI)

Identifies if the vector form of the intrinsic has a scalar operand.

LLVM_ABI Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF)

Given information about an recurrence kind, return the identity for the @llvm.vector....

DWARFExpression::Operation Op

Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)

Return a value for Step multiplied by VF.

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

bool is_contained(R &&Range, const E &Element)

Returns true if Element is found in Range.

Type * getLoadStoreType(const Value *I)

A helper function that returns the type of a load or store instruction.

LLVM_ABI Value * createOrderedReduction(IRBuilderBase &B, RecurKind RdxKind, Value *Src, Value *Start)

Create an ordered reduction intrinsic using the given recurrence kind RdxKind.

auto seq(T Begin, T End)

Iterate over an integral type from Begin up to - but not including - End.

@ Increment

Incrementally increasing token ID.

Type * toVectorTy(Type *Scalar, ElementCount EC)

A helper function for converting Scalar types to vector types.

LLVM_ABI Value * createAnyOfReduction(IRBuilderBase &B, Value *Src, Value *InitVal, PHINode *OrigPhi)

Create a reduction of the given vector Src for a reduction of kind RecurKind::AnyOf.

LLVM_ABI bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx, const TargetTransformInfo *TTI)

Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...

This struct is a compact representation of a valid (non-zero power of two) alignment.

Struct to hold various analysis needed for cost computations.

void execute(VPTransformState &State) override

Generate the phi nodes.

Definition VPlanRecipes.cpp:4306

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this first-order recurrence phi recipe.

Definition VPlanRecipes.cpp:4336

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4345

An overlay for VPIRInstructions wrapping PHI nodes enabling convenient use cast/dyn_cast/isa and exec...

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:1571

void execute(VPTransformState &State) override

The method which generates the output IR instructions that correspond to this VPRecipe,...

Definition VPlanRecipes.cpp:1521

void execute(VPTransformState &State) override

Generate the instruction.

Definition VPlanRecipes.cpp:1451

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:1470

A pure-virtual common base class for recipes defining a single VPValue and using IR flags.

InstructionCost getCostForRecipeWithOpcode(unsigned Opcode, ElementCount VF, VPCostContext &Ctx) const

Compute the cost for this recipe for VF, using Opcode and Ctx.

Definition VPlanRecipes.cpp:925

VPRecipeWithIRFlags(const unsigned char SC, ArrayRef< VPValue * > Operands, const VPIRFlags &Flags, DebugLoc DL=DebugLoc::getUnknown())

VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...

VPTypeAnalysis TypeAnalysis

VPlan-based type analysis.

Value * get(const VPValue *Def, bool IsScalar=false)

Get the generated vector Value for a given VPValue Def if IsScalar is false, otherwise return the gen...

IRBuilderBase & Builder

Hold a reference to the IRBuilder used to generate output IR code.

ElementCount VF

The chosen Vectorization Factor of the loop being vectorized.

void execute(VPTransformState &State) override

Generate the wide load or gather.

Definition VPlanRecipes.cpp:3584

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:3644

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenLoadEVLRecipe.

Definition VPlanRecipes.cpp:3619

VPValue * getEVL() const

Return the EVL operand.

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:3564

void execute(VPTransformState &State) override

Generate a wide load or gather.

Definition VPlanRecipes.cpp:3530

VPValue * getCond() const

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenSelectRecipe.

Definition VPlanRecipes.cpp:1937

void execute(VPTransformState &State) override

Produce a widened version of the select instruction.

Definition VPlanRecipes.cpp:1923

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:1907

VPValue * getStoredValue() const

Return the address accessed by this recipe.

void execute(VPTransformState &State) override

Generate the wide store or scatter.

Definition VPlanRecipes.cpp:3695

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:3754

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenStoreEVLRecipe.

Definition VPlanRecipes.cpp:3729

VPValue * getEVL() const

Return the EVL operand.

void execute(VPTransformState &State) override

Generate a wide store or scatter.

Definition VPlanRecipes.cpp:3653

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:3688

VPValue * getStoredValue() const

Return the value stored by this recipe.