VPlanRecipes.cpp Source File (original) (raw)

39#include

41using namespace llvm;

46#define LV_NAME "loop-vectorize"

47#define DEBUG_TYPE LV_NAME

51 case VPExpressionSC:

53 case VPInstructionSC: {

56 if (VPI->getOpcode() == Instruction::Load)

57 return false;

58 return VPI->opcodeMayReadOrWriteFromMemory();

59 }

60 case VPInterleaveEVLSC:

61 case VPInterleaveSC:

63 case VPWidenStoreEVLSC:

64 case VPWidenStoreSC:

65 return true;

66 case VPReplicateSC:

68 ->mayWriteToMemory();

69 case VPWidenCallSC:

71 ->getCalledScalarFunction()

72 ->onlyReadsMemory();

73 case VPWidenIntrinsicSC:

75 case VPCanonicalIVPHISC:

76 case VPBranchOnMaskSC:

77 case VPDerivedIVSC:

78 case VPFirstOrderRecurrencePHISC:

79 case VPReductionPHISC:

80 case VPScalarIVStepsSC:

81 case VPPredInstPHISC:

82 return false;

83 case VPBlendSC:

84 case VPReductionEVLSC:

85 case VPReductionSC:

86 case VPVectorPointerSC:

87 case VPWidenCanonicalIVSC:

88 case VPWidenCastSC:

89 case VPWidenGEPSC:

90 case VPWidenIntOrFpInductionSC:

91 case VPWidenLoadEVLSC:

92 case VPWidenLoadSC:

93 case VPWidenPHISC:

94 case VPWidenPointerInductionSC:

95 case VPWidenSC:

96 case VPWidenSelectSC: {

99 (void)I;

100 assert(( || ->mayWriteToMemory()) &&

101 "underlying instruction may write to memory");

102 return false;

103 }

104 default:

105 return true;

106 }

107}

108

111 case VPExpressionSC:

113 case VPInstructionSC:

115 case VPWidenLoadEVLSC:

116 case VPWidenLoadSC:

117 return true;

118 case VPReplicateSC:

120 ->mayReadFromMemory();

121 case VPWidenCallSC:

123 ->getCalledScalarFunction()

124 ->onlyWritesMemory();

125 case VPWidenIntrinsicSC:

127 case VPBranchOnMaskSC:

128 case VPDerivedIVSC:

129 case VPFirstOrderRecurrencePHISC:

130 case VPPredInstPHISC:

131 case VPScalarIVStepsSC:

132 case VPWidenStoreEVLSC:

133 case VPWidenStoreSC:

134 return false;

135 case VPBlendSC:

136 case VPReductionEVLSC:

137 case VPReductionSC:

138 case VPVectorPointerSC:

139 case VPWidenCanonicalIVSC:

140 case VPWidenCastSC:

141 case VPWidenGEPSC:

142 case VPWidenIntOrFpInductionSC:

143 case VPWidenPHISC:

144 case VPWidenPointerInductionSC:

145 case VPWidenSC:

146 case VPWidenSelectSC: {

149 (void)I;

150 assert(( || ->mayReadFromMemory()) &&

151 "underlying instruction may read from memory");

152 return false;

153 }

154 default:

155

156 return true;

157 }

158}

159

162 case VPExpressionSC:

164 case VPDerivedIVSC:

165 case VPFirstOrderRecurrencePHISC:

166 case VPPredInstPHISC:

167 case VPVectorEndPointerSC:

168 return false;

169 case VPInstructionSC: {

174 }

175 case VPWidenCallSC: {

178 }

179 case VPWidenIntrinsicSC:

181 case VPBlendSC:

182 case VPReductionEVLSC:

183 case VPReductionSC:

184 case VPScalarIVStepsSC:

185 case VPVectorPointerSC:

186 case VPWidenCanonicalIVSC:

187 case VPWidenCastSC:

188 case VPWidenGEPSC:

189 case VPWidenIntOrFpInductionSC:

190 case VPWidenPHISC:

191 case VPWidenPointerInductionSC:

192 case VPWidenSC:

193 case VPWidenSelectSC: {

196 (void)I;

197 assert(( || ->mayHaveSideEffects()) &&

198 "underlying instruction has side-effects");

199 return false;

200 }

201 case VPInterleaveEVLSC:

202 case VPInterleaveSC:

204 case VPWidenLoadEVLSC:

205 case VPWidenLoadSC:

206 case VPWidenStoreEVLSC:

207 case VPWidenStoreSC:

211 "mayHaveSideffects result for ingredient differs from this "

212 "implementation");

214 case VPReplicateSC: {

216 return R->getUnderlyingInstr()->mayHaveSideEffects();

217 }

218 default:

219 return true;

220 }

221}

222

224 assert(!Parent && "Recipe already in some VPBasicBlock");

226 "Insertion position not in any VPBasicBlock");

228}

229

232 assert(!Parent && "Recipe already in some VPBasicBlock");

233 assert(I == BB.end() || I->getParent() == &BB);

235}

236

238 assert(!Parent && "Recipe already in some VPBasicBlock");

240 "Insertion position not in any VPBasicBlock");

242}

243

247 Parent = nullptr;

248}

249

254

259

265

267

268

269

270

275 UI = IG->getInsertPos();

277 UI = &WidenMem->getIngredient();

278

280 if (UI && Ctx.skipCostComputation(UI, VF.isVector())) {

281 RecipeCost = 0;

282 } else {

286 if (UI)

288 else

290 }

291 }

292

294 dbgs() << "Cost of " << RecipeCost << " for VF " << VF << ": ";

296 });

297 return RecipeCost;

298}

299

304

309

314

316 assert(OpType == Other.OpType && "OpType must match");

317 switch (OpType) {

318 case OperationType::OverflowingBinOp:

319 WrapFlags.HasNUW &= Other.WrapFlags.HasNUW;

320 WrapFlags.HasNSW &= Other.WrapFlags.HasNSW;

321 break;

322 case OperationType::Trunc:

323 TruncFlags.HasNUW &= Other.TruncFlags.HasNUW;

324 TruncFlags.HasNSW &= Other.TruncFlags.HasNSW;

325 break;

326 case OperationType::DisjointOp:

327 DisjointFlags.IsDisjoint &= Other.DisjointFlags.IsDisjoint;

328 break;

329 case OperationType::PossiblyExactOp:

330 ExactFlags.IsExact &= Other.ExactFlags.IsExact;

331 break;

332 case OperationType::GEPOp:

334 break;

335 case OperationType::FPMathOp:

336 case OperationType::FCmp:

337 assert((OpType != OperationType::FCmp ||

338 FCmpFlags.Pred == Other.FCmpFlags.Pred) &&

339 "Cannot drop CmpPredicate");

340 getFMFsRef().NoNaNs &= Other.getFMFsRef().NoNaNs;

341 getFMFsRef().NoInfs &= Other.getFMFsRef().NoInfs;

342 break;

343 case OperationType::NonNegOp:

344 NonNegFlags.NonNeg &= Other.NonNegFlags.NonNeg;

345 break;

346 case OperationType::Cmp:

347 assert(CmpPredicate == Other.CmpPredicate && "Cannot drop CmpPredicate");

348 break;

349 case OperationType::Other:

350 assert(AllFlags == Other.AllFlags && "Cannot drop other flags");

351 break;

352 }

353}

354

356 assert((OpType == OperationType::FPMathOp || OpType == OperationType::FCmp) &&

357 "recipe doesn't have fast math flags");

358 const FastMathFlagsTy &F = getFMFsRef();

367 return Res;

368}

369

370#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

372

377 O << ", !dbg ";

378 DL.print(O);

379 }

380

383}

384#endif

385

386template

389 if (U.getNumOperands() == PartOpIdx + 1)

390 return U.getOperand(PartOpIdx);

391 return nullptr;

392}

393

394template

397 return cast(UnrollPartOp->getLiveInIRValue())->getZExtValue();

398 return 0;

399}

400

401namespace llvm {

405}

406

411 VPIRMetadata(MD), Opcode(Opcode), Name(Name.str()) {

413 "Set flags not supported for the provided opcode");

414 assert((getNumOperandsForOpcode(Opcode) == -1u ||

415 getNumOperandsForOpcode(Opcode) == getNumOperands()) &&

416 "number of operands does not match opcode");

417}

418

419#ifndef NDEBUG

420unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {

422 return 1;

423

425 return 2;

426

427 switch (Opcode) {

430 return 0;

431 case Instruction::Alloca:

432 case Instruction::ExtractValue:

433 case Instruction::Freeze:

434 case Instruction::Load:

448 return 1;

449 case Instruction::ICmp:

450 case Instruction::FCmp:

451 case Instruction::ExtractElement:

452 case Instruction::Store:

461 return 2;

462 case Instruction::Select:

466 return 3;

468 return 4;

469 case Instruction::Call:

470 case Instruction::GetElementPtr:

471 case Instruction::PHI:

472 case Instruction::Switch:

478

479 return -1u;

480 }

482}

483#endif

484

488

489bool VPInstruction::canGenerateScalarForFirstLane() const {

491 return true;

493 return true;

494 switch (Opcode) {

495 case Instruction::Freeze:

496 case Instruction::ICmp:

497 case Instruction::PHI:

498 case Instruction::Select:

507 return true;

508 default:

509 return false;

510 }

511}

512

513

514

515

516

519

520

521

522

524 BasicBlock *SecondIRSucc = State.CFG.VPBB2IRBB.lookup(SecondVPSucc);

525 BasicBlock *IRBB = State.CFG.VPBB2IRBB[VPBB];

526 BranchInst *CondBr = State.Builder.CreateCondBr(Cond, IRBB, SecondIRSucc);

527

530 return CondBr;

531}

532

534 IRBuilderBase &Builder = State.Builder;

535

540 auto *Res =

544 return Res;

545 }

546

552 }

553 case Instruction::ExtractElement: {

554 assert(State.VF.isVector() && "Only extract elements from vectors");

556 unsigned IdxToExtract =

558 return State.get(getOperand(0), VPLane(IdxToExtract));

559 }

563 }

564 case Instruction::Freeze: {

567 }

568 case Instruction::FCmp:

569 case Instruction::ICmp: {

574 }

575 case Instruction::PHI: {

577 }

578 case Instruction::Select: {

586 }

588

590

592

593

594

597 Name);

598

602 return Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask,

603 {PredTy, ScalarTC->getType()},

604 {VIVElem0, ScalarTC}, nullptr, Name);

605 }

607

608

609

610

611

612

613

614

615

616

617

618

620 if (!V1->getType()->isVectorTy())

621 return V1;

624 }

633 }

635

636

638

640 "Requested vector length should be an integer.");

641

644

646 Builder.getInt32Ty(), Intrinsic::experimental_get_vector_length,

647 {AVL, VFArg, Builder.getTrue()});

648 return EVL;

649 }

653 assert(Part != 0 && "Must have a positive part");

654

655

659 }

664 return Br;

665 }

667

672 }

675 State.VF, State.get(getOperand(0), true), "broadcast");

676 }

678

679

680 auto *StructTy =

684 for (unsigned FieldIndex = 0; FieldIndex != StructTy->getNumElements();

685 FieldIndex++) {

686 Value *ScalarValue =

689 VectorValue =

692 }

693 }

694 return Res;

695 }

703 return Res;

704 }

708 IRBuilderBase::FastMathFlagGuard FMFG(Builder);

710

711

717 }

719

720

722 auto *OrigPhi = cast(PhiR->getUnderlyingValue());

724 for (unsigned Idx = 3; Idx < getNumOperands(); ++Idx)

725 ReducedPartRdx =

727 ReducedPartRdx, "bin.rdx");

730 }

732

733

735

736 RecurKind RK = PhiR->getRecurrenceKind();

738 "Unexpected reduction kind");

739 assert(!PhiR->isInLoop() &&

740 "In-loop FindLastIV reduction is not supported yet");

741

742

743

750 else

752 for (unsigned Part = 1; Part < UF; ++Part)

753 ReducedPartRdx = createMinMaxOp(Builder, MinMaxKind, ReducedPartRdx,

755

760 }

762

763

765

766

767 RecurKind RK = PhiR->getRecurrenceKind();

769 "should be handled by ComputeFindIVResult");

770

771

772

775 for (unsigned Part = 0; Part < UF; ++Part)

776 RdxParts[Part] = State.get(getOperand(1 + Part), PhiR->isInLoop());

777

778 IRBuilderBase::FastMathFlagGuard FMFG(Builder);

781

782

783 Value *ReducedPartRdx = RdxParts[0];

784 if (PhiR->isOrdered()) {

785 ReducedPartRdx = RdxParts[UF - 1];

786 } else {

787

788 for (unsigned Part = 1; Part < UF; ++Part) {

789 Value *RdxPart = RdxParts[Part];

791 ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);

792 else {

793

794

797 ? Instruction::Add

799 ReducedPartRdx =

800 Builder.CreateBinOp(Opcode, RdxPart, ReducedPartRdx, "bin.rdx");

801 }

802 }

803 }

804

805

806

807 if (State.VF.isVector() && !PhiR->isInLoop()) {

808

809

810

812 }

813

814 return ReducedPartRdx;

815 }

823 "invalid offset to extract from");

824

826 } else {

827

828 assert(Offset <= 1 && "invalid offset to extract from");

830 }

833 return Res;

834 }

839 }

842 "can only generate first lane for PtrAdd");

846 }

852 }

858 }

862 Value *Res = nullptr;

864

865 for (unsigned Idx = 1; Idx != getNumOperands(); ++Idx) {

866 Value *VectorStart =

867 Builder.CreateMul(RuntimeVF, ConstantInt::get(IdxTy, Idx - 1));

868 Value *VectorIdx = Idx == 1

869 ? LaneToExtract

870 : Builder.CreateSub(LaneToExtract, VectorStart);

875 if (Res) {

878 } else {

879 Res = Ext;

880 }

881 }

882 return Res;

883 }

888 false, Name);

889 }

890

891

892

895 Value *Res = nullptr;

896 for (int Idx = LastOpIdx; Idx >= 0; --Idx) {

897 Value *TrailingZeros =

905 false, Name);

907 Builder.CreateMul(RuntimeVF, Builder.getInt64(Idx)), TrailingZeros);

908 if (Res) {

910 Res = Builder.CreateSelect(Cmp, Current, Res);

911 } else {

912 Res = Current;

913 }

914 }

915

916 return Res;

917 }

920 default:

922 }

923}

924

927 Type *ScalarTy = Ctx.Types.inferScalarType(this);

929 switch (Opcode) {

930 case Instruction::FNeg:

931 return Ctx.TTI.getArithmeticInstrCost(Opcode, ResultTy, Ctx.CostKind);

932 case Instruction::UDiv:

933 case Instruction::SDiv:

934 case Instruction::SRem:

935 case Instruction::URem:

936 case Instruction::Add:

937 case Instruction::FAdd:

938 case Instruction::Sub:

939 case Instruction::FSub:

940 case Instruction::Mul:

941 case Instruction::FMul:

942 case Instruction::FDiv:

943 case Instruction::FRem:

944 case Instruction::Shl:

945 case Instruction::LShr:

946 case Instruction::AShr:

947 case Instruction::And:

948 case Instruction::Or:

949 case Instruction::Xor: {

952

954

955

957 RHSInfo = Ctx.getOperandInfo(RHS);

958

962 }

963

966 if (CtxI)

968 return Ctx.TTI.getArithmeticInstrCost(

969 Opcode, ResultTy, Ctx.CostKind,

970 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},

971 RHSInfo, Operands, CtxI, &Ctx.TLI);

972 }

973 case Instruction::Freeze:

974

975 return Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, ResultTy,

976 Ctx.CostKind);

977 case Instruction::ExtractValue:

978 return Ctx.TTI.getInsertExtractValueCost(Instruction::ExtractValue,

979 Ctx.CostKind);

980 case Instruction::ICmp:

981 case Instruction::FCmp: {

982 Type *ScalarOpTy = Ctx.Types.inferScalarType(getOperand(0));

985 return Ctx.TTI.getCmpSelInstrCost(

987 Ctx.CostKind, {TTI::OK_AnyValue, TTI::OP_None},

988 {TTI::OK_AnyValue, TTI::OP_None}, CtxI);

989 }

990 }

992}

993

998

999

1000 return 0;

1001 }

1002

1004 "Should only generate a vector value or single scalar, not scalars "

1005 "for all lanes.");

1009 }

1010

1012 case Instruction::Select: {

1015 auto *CondTy = Ctx.Types.inferScalarType(getOperand(0));

1016 auto *VecTy = Ctx.Types.inferScalarType(getOperand(1));

1020 }

1021 return Ctx.TTI.getCmpSelInstrCost(Instruction::Select, VecTy, CondTy, Pred,

1022 Ctx.CostKind);

1023 }

1024 case Instruction::ExtractElement:

1027

1028

1029 return 0;

1030 }

1031

1032

1034 return Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy,

1035 Ctx.CostKind);

1036 }

1038 auto *VecTy = toVectorTy(Ctx.Types.inferScalarType(this), VF);

1039 return Ctx.TTI.getArithmeticReductionCost(

1040 Instruction::Or, cast(VecTy), std::nullopt, Ctx.CostKind);

1041 }

1043 Type *ScalarTy = Ctx.Types.inferScalarType(getOperand(0));

1045 return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, ScalarTy,

1048

1049 auto *PredTy = toVectorTy(ScalarTy, VF);

1052 {PredTy, Type::getInt1Ty(Ctx.LLVMCtx)});

1053 return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);

1054 }

1056 Type *ScalarTy = Ctx.Types.inferScalarType(getOperand(0));

1058 return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, ScalarTy,

1061

1062 auto *PredTy = toVectorTy(ScalarTy, VF);

1065 {PredTy, Type::getInt1Ty(Ctx.LLVMCtx)});

1066 InstructionCost Cost = Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);

1067

1068 Cost += Ctx.TTI.getArithmeticInstrCost(

1069 Instruction::Xor, PredTy, Ctx.CostKind,

1070 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},

1071 {TargetTransformInfo::OK_UniformConstantValue,

1072 TargetTransformInfo::OP_None});

1073

1074 Cost += Ctx.TTI.getArithmeticInstrCost(

1075 Instruction::Sub, Type::getInt64Ty(Ctx.LLVMCtx), Ctx.CostKind);

1076 return Cost;

1077 }

1079 assert(VF.isVector() && "Scalar FirstOrderRecurrenceSplice?");

1081 std::iota(Mask.begin(), Mask.end(), VF.getKnownMinValue() - 1);

1082 Type *VectorTy = toVectorTy(Ctx.Types.inferScalarType(this), VF);

1083

1088 }

1090 Type *ArgTy = Ctx.Types.inferScalarType(getOperand(0));

1091 unsigned Multiplier =

1095 {ArgTy, ArgTy});

1096 return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);

1097 }

1099 Type *Arg0Ty = Ctx.Types.inferScalarType(getOperand(0));

1103 I32Ty, {Arg0Ty, I32Ty, I1Ty});

1104 return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);

1105 }

1107

1109 return Ctx.TTI.getIndexedVectorInstrCostFromEnd(Instruction::ExtractElement,

1110 VecTy, Ctx.CostKind, 0);

1111 }

1115 [[fallthrough]];

1116 default:

1117

1118

1120 "unexpected VPInstruction witht underlying value");

1121 return 0;

1122 }

1123}

1124

1128 getOpcode() == Instruction::ExtractElement ||

1136}

1137

1140 case Instruction::PHI:

1144 return true;

1145 default:

1147 }

1148}

1149

1151 assert(!State.Lane && "VPInstruction executing an Lane");

1154 "Set flags not supported for the provided opcode");

1157 Value *GeneratedValue = generate(State);

1159 return;

1160 assert(GeneratedValue && "generate must produce a value");

1161 bool GeneratesPerFirstLaneOnly = canGenerateScalarForFirstLane() &&

1166 !GeneratesPerFirstLaneOnly) ||

1167 State.VF.isScalar()) &&

1168 "scalar value but not only first lane defined");

1169 State.set(this, GeneratedValue,

1170 GeneratesPerFirstLaneOnly);

1171}

1172

1175 return false;

1177 case Instruction::GetElementPtr:

1178 case Instruction::ExtractElement:

1179 case Instruction::Freeze:

1180 case Instruction::FCmp:

1181 case Instruction::ICmp:

1182 case Instruction::Select:

1183 case Instruction::PHI:

1210 return false;

1211 default:

1212 return true;

1213 }

1214}

1215

1220

1222 default:

1223 return false;

1224 case Instruction::ExtractElement:

1226 case Instruction::PHI:

1227 return true;

1228 case Instruction::FCmp:

1229 case Instruction::ICmp:

1230 case Instruction::Select:

1231 case Instruction::Or:

1232 case Instruction::Freeze:

1234

1244 return true;

1247

1248

1249

1254

1255 return false;

1261 };

1263}

1264

1269

1271 default:

1272 return false;

1273 case Instruction::FCmp:

1274 case Instruction::ICmp:

1275 case Instruction::Select:

1280 return true;

1281 };

1283}

1284

1285#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1290

1293 O << Indent << "EMIT" << (isSingleScalar() ? "-SCALAR" : "") << " ";

1294

1297 O << " = ";

1298 }

1299

1302 O << "not";

1303 break;

1305 O << "combined load";

1306 break;

1308 O << "combined store";

1309 break;

1311 O << "active lane mask";

1312 break;

1314 O << "EXPLICIT-VECTOR-LENGTH";

1315 break;

1317 O << "first-order splice";

1318 break;

1320 O << "branch-on-cond";

1321 break;

1323 O << "TC > VF ? TC - VF : 0";

1324 break;

1326 O << "VF * Part +";

1327 break;

1329 O << "branch-on-count";

1330 break;

1332 O << "broadcast";

1333 break;

1335 O << "buildstructvector";

1336 break;

1338 O << "buildvector";

1339 break;

1341 O << "extract-lane";

1342 break;

1344 O << "extract-last-lane";

1345 break;

1347 O << "extract-last-part";

1348 break;

1350 O << "extract-penultimate-element";

1351 break;

1353 O << "compute-anyof-result";

1354 break;

1356 O << "compute-find-iv-result";

1357 break;

1359 O << "compute-reduction-result";

1360 break;

1362 O << "logical-and";

1363 break;

1365 O << "ptradd";

1366 break;

1368 O << "wide-ptradd";

1369 break;

1371 O << "any-of";

1372 break;

1374 O << "first-active-lane";

1375 break;

1377 O << "last-active-lane";

1378 break;

1380 O << "reduction-start-vector";

1381 break;

1383 O << "resume-for-epilogue";

1384 break;

1386 O << "unpack";

1387 break;

1388 default:

1390 }

1391

1394}

1395#endif

1396

1402 Op, ResultTy);

1403 State.set(this, Cast, VPLane(0));

1404 return;

1405 }

1409 State.Builder.CreateStepVector(VectorType::get(ResultTy, State.VF));

1411 break;

1412 }

1414 Value *VScale = State.Builder.CreateVScale(ResultTy);

1415 State.set(this, VScale, true);

1416 break;

1417 }

1418

1419 default:

1421 }

1422}

1423

1424#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1427 O << Indent << "EMIT" << (isSingleScalar() ? "-SCALAR" : "") << " ";

1429 O << " = ";

1430

1433 O << "wide-iv-step ";

1435 break;

1437 O << "step-vector " << *ResultTy;

1438 break;

1440 O << "vscale " << *ResultTy;

1441 break;

1442 default:

1446 O << " to " << *ResultTy;

1447 }

1448}

1449#endif

1450

1453 PHINode *NewPhi = State.Builder.CreatePHI(

1454 State.TypeAnalysis.inferScalarType(this), 2, getName());

1457

1458

1459 NumIncoming = 1;

1460 }

1461 for (unsigned Idx = 0; Idx != NumIncoming; ++Idx) {

1465 }

1466 State.set(this, NewPhi, VPLane(0));

1467}

1468

1469#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1472 O << Indent << "EMIT" << (isSingleScalar() ? "-SCALAR" : "") << " ";

1474 O << " = phi ";

1476}

1477#endif

1478

1481 return new VPIRPhi(*Phi);

1483}

1484

1487 "PHINodes must be handled by VPIRPhi");

1488

1489

1490 State.Builder.SetInsertPoint(I.getParent(), std::next(I.getIterator()));

1491}

1492

1495

1496

1497 return 0;

1498}

1499

1503 "can only update exiting operands to phi nodes");

1507 return;

1508

1512}

1513

1514#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1517 O << Indent << "IR " << I;

1518}

1519#endif

1520

1529 auto *PredVPBB = Pred->getExitingBasicBlock();

1530 BasicBlock *PredBB = State.CFG.VPBB2IRBB[PredVPBB];

1531

1532

1533 State.Builder.SetInsertPoint(PredBB, PredBB->getFirstNonPHIIt());

1534 Value *V = State.get(ExitValue, VPLane(Lane));

1535

1536

1537 if (Phi->getBasicBlockIndex(PredBB) == -1)

1538 Phi->addIncoming(V, PredBB);

1539 else

1540 Phi->setIncomingValueForBlock(PredBB, V);

1541 }

1542

1543

1544

1545 State.Builder.SetInsertPoint(Phi->getParent(), std::next(Phi->getIterator()));

1546}

1547

1550 assert(R->getNumOperands() == R->getParent()->getNumPredecessors() &&

1551 "Number of phi operands must match number of predecessors");

1552 unsigned Position = R->getParent()->getIndexForPredecessor(IncomingBlock);

1553 R->removeOperand(Position);

1554}

1555

1556#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1561 O << "[ ";

1563 O << ", ";

1565 O << " ]";

1566 });

1567}

1568#endif

1569

1570#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1574

1576 O << " (extra operand" << (getNumOperands() > 1 ? "s" : "") << ": ";

1579 std::get<0>(Op)->printAsOperand(O, SlotTracker);

1580 O << " from ";

1581 std::get<1>(Op)->printAsOperand(O);

1582 });

1583 O << ")";

1584 }

1585}

1586#endif

1587

1589 for (const auto &[Kind, Node] : Metadata)

1590 I.setMetadata(Kind, Node);

1591}

1592

1595 for (const auto &[KindA, MDA] : Metadata) {

1596 for (const auto &[KindB, MDB] : Other.Metadata) {

1597 if (KindA == KindB && MDA == MDB) {

1598 MetadataIntersection.emplace_back(KindA, MDA);

1599 break;

1600 }

1601 }

1602 }

1603 Metadata = std::move(MetadataIntersection);

1604}

1605

1606#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1609 if (Metadata.empty() || !M)

1610 return;

1611

1613 O << " (";

1614 interleaveComma(Metadata, O, [&](const auto &KindNodePair) {

1615 auto [Kind, Node] = KindNodePair;

1616 assert(Kind < MDNames.size() && !MDNames[Kind].empty() &&

1617 "Unexpected unnamed metadata kind");

1618 O << "!" << MDNames[Kind] << " ";

1620 });

1621 O << ")";

1622}

1623#endif

1624

1626 assert(State.VF.isVector() && "not widening");

1627 assert(Variant != nullptr && "Can't create vector function.");

1628

1629 FunctionType *VFTy = Variant->getFunctionType();

1630

1634

1635

1636

1638 Arg = State.get(I.value(), VPLane(0));

1639 else

1641 Args.push_back(Arg);

1642 }

1643

1646 if (CI)

1647 CI->getOperandBundlesAsDefs(OpBundles);

1648

1649 CallInst *V = State.Builder.CreateCall(Variant, Args, OpBundles);

1652 V->setCallingConv(Variant->getCallingConv());

1653

1654 if (!V->getType()->isVoidTy())

1655 State.set(this, V);

1656}

1657

1660 return Ctx.TTI.getCallInstrCost(nullptr, Variant->getReturnType(),

1661 Variant->getFunctionType()->params(),

1662 Ctx.CostKind);

1663}

1664

1665#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1668 O << Indent << "WIDEN-CALL ";

1669

1672 O << "void ";

1673 else {

1675 O << " = ";

1676 }

1677

1678 O << "call";

1680 O << " @" << CalledFn->getName() << "(";

1683 });

1684 O << ")";

1685

1686 O << " (using library function";

1687 if (Variant->hasName())

1688 O << ": " << Variant->getName();

1689 O << ")";

1690}

1691#endif

1692

1694 assert(State.VF.isVector() && "not widening");

1695

1697

1702

1703

1706 State.TTI))

1707 Arg = State.get(I.value(), VPLane(0));

1708 else

1711 State.TTI))

1713 Args.push_back(Arg);

1714 }

1715

1716

1717 Module *M = State.Builder.GetInsertBlock()->getModule();

1721 "Can't retrieve vector intrinsic or vector-predication intrinsics.");

1722

1725 if (CI)

1726 CI->getOperandBundlesAsDefs(OpBundles);

1727

1728 CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);

1729

1732

1733 if (!V->getType()->isVoidTy())

1734 State.set(this, V);

1735}

1736

1737

1743

1744

1745

1746

1747

1749 for (const auto &[Idx, Op] : enumerate(Operands)) {

1750 auto *V = Op->getUnderlyingValue();

1751 if (!V) {

1753 Arguments.push_back(UI->getArgOperand(Idx));

1754 continue;

1755 }

1757 break;

1758 }

1760 }

1761

1762 Type *ScalarRetTy = Ctx.Types.inferScalarType(&R);

1765 for (const VPValue *Op : Operands) {

1767 ? toVectorTy(Ctx.Types.inferScalarType(Op), VF)

1768 : Ctx.Types.inferScalarType(Op));

1769 }

1770

1771

1773 R.hasFastMathFlags() ? R.getFastMathFlags() : FastMathFlags();

1778 return Ctx.TTI.getIntrinsicInstrCost(CostAttrs, Ctx.CostKind);

1779}

1780

1786

1790

1794 auto [Idx, V] = X;

1796 Idx, nullptr);

1797 });

1798}

1799

1800#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1803 O << Indent << "WIDEN-INTRINSIC ";

1804 if (ResultTy->isVoidTy()) {

1805 O << "void ";

1806 } else {

1808 O << " = ";

1809 }

1810

1811 O << "call";

1814

1817 });

1818 O << ")";

1819}

1820#endif

1821

1824

1826 Value *IncAmt = State.get(getOperand(1), true);

1828

1829

1830

1831

1832 Value *Mask = nullptr;

1834 Mask = State.get(VPMask);

1835 else

1836 Mask =

1837 Builder.CreateVectorSplat(VTy->getElementCount(), Builder.getInt1(1));

1838

1839

1840

1841 if (Opcode == Instruction::Sub)

1842 IncAmt = Builder.CreateNeg(IncAmt);

1843 else

1844 assert(Opcode == Instruction::Add && "only add or sub supported for now");

1845

1846 State.Builder.CreateIntrinsic(Intrinsic::experimental_vector_histogram_add,

1847 {VTy, IncAmt->getType()},

1848 {Address, IncAmt, Mask});

1849}

1850

1853

1854

1855

1856

1857

1858 assert(VF.isVector() && "Invalid VF for histogram cost");

1859 Type *AddressTy = Ctx.Types.inferScalarType(getOperand(0));

1861 Type *IncTy = Ctx.Types.inferScalarType(IncAmt);

1863

1864

1865

1867 Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy, Ctx.CostKind);

1870

1873 }

1874

1875

1880 {PtrTy, IncTy, MaskTy});

1881

1882

1883 return Ctx.TTI.getIntrinsicInstrCost(ICA, Ctx.CostKind) + MulCost +

1884 Ctx.TTI.getArithmeticInstrCost(Opcode, VTy, Ctx.CostKind);

1885}

1886

1887#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

1890 O << Indent << "WIDEN-HISTOGRAM buckets: ";

1892

1893 if (Opcode == Instruction::Sub)

1894 O << ", dec: ";

1895 else {

1896 assert(Opcode == Instruction::Add);

1897 O << ", inc: ";

1898 }

1900

1902 O << ", mask: ";

1904 }

1905}

1906

1909 O << Indent << "WIDEN-SELECT ";

1911 O << " = select";

1914 O << ", ";

1916 O << ", ";

1919 : "");

1920}

1921#endif

1922

1925

1928 Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1);

1929 State.set(this, Sel);

1934 }

1935}

1936

1941 Type *ScalarTy = Ctx.Types.inferScalarType(this);

1942 Type *VectorTy = toVectorTy(Ctx.Types.inferScalarType(this), VF);

1943

1948

1949

1950 const auto [Op1VK, Op1VP] = Ctx.getOperandInfo(Op0);

1951 const auto [Op2VK, Op2VP] = Ctx.getOperandInfo(Op1);

1952

1955 [](VPValue *Op) { return Op->getUnderlyingValue(); }))

1956 Operands.append(SI->op_begin(), SI->op_end());

1958 return Ctx.TTI.getArithmeticInstrCost(

1959 IsLogicalOr ? Instruction::Or : Instruction::And, VectorTy,

1960 Ctx.CostKind, {Op1VK, Op1VP}, {Op2VK, Op2VP}, Operands, SI);

1961 }

1962

1963 Type *CondTy = Ctx.Types.inferScalarType(getOperand(0));

1964 if (!ScalarCond)

1966

1969 Pred = Cmp->getPredicate();

1970 return Ctx.TTI.getCmpSelInstrCost(

1971 Instruction::Select, VectorTy, CondTy, Pred, Ctx.CostKind,

1972 {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None}, SI);

1973}

1974

1975VPIRFlags::FastMathFlagsTy::FastMathFlagsTy(const FastMathFlags &FMF) {

1977 NoNaNs = FMF.noNaNs();

1978 NoInfs = FMF.noInfs();

1983}

1984

1985#if !defined(NDEBUG)

1987 switch (OpType) {

1988 case OperationType::OverflowingBinOp:

1989 return Opcode == Instruction::Add || Opcode == Instruction::Sub ||

1990 Opcode == Instruction::Mul || Opcode == Instruction::Shl ||

1991 Opcode == VPInstruction::VPInstruction::CanonicalIVIncrementForPart;

1992 case OperationType::Trunc:

1993 return Opcode == Instruction::Trunc;

1994 case OperationType::DisjointOp:

1995 return Opcode == Instruction::Or;

1996 case OperationType::PossiblyExactOp:

1997 return Opcode == Instruction::AShr || Opcode == Instruction::LShr ||

1998 Opcode == Instruction::UDiv || Opcode == Instruction::SDiv;

1999 case OperationType::GEPOp:

2000 return Opcode == Instruction::GetElementPtr ||

2003 case OperationType::FPMathOp:

2004 return Opcode == Instruction::Call || Opcode == Instruction::FAdd ||

2005 Opcode == Instruction::FMul || Opcode == Instruction::FSub ||

2006 Opcode == Instruction::FNeg || Opcode == Instruction::FDiv ||

2007 Opcode == Instruction::FRem || Opcode == Instruction::FPExt ||

2008 Opcode == Instruction::FPTrunc || Opcode == Instruction::Select ||

2012 case OperationType::FCmp:

2013 return Opcode == Instruction::FCmp;

2014 case OperationType::NonNegOp:

2015 return Opcode == Instruction::ZExt || Opcode == Instruction::UIToFP;

2016 case OperationType::Cmp:

2017 return Opcode == Instruction::FCmp || Opcode == Instruction::ICmp;

2018 case OperationType::Other:

2019 return true;

2020 }

2022}

2023#endif

2024

2025#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2027 switch (OpType) {

2028 case OperationType::Cmp:

2030 break;

2031 case OperationType::FCmp:

2034 break;

2035 case OperationType::DisjointOp:

2037 O << " disjoint";

2038 break;

2039 case OperationType::PossiblyExactOp:

2041 O << " exact";

2042 break;

2043 case OperationType::OverflowingBinOp:

2045 O << " nuw";

2047 O << " nsw";

2048 break;

2049 case OperationType::Trunc:

2051 O << " nuw";

2053 O << " nsw";

2054 break;

2055 case OperationType::FPMathOp:

2057 break;

2058 case OperationType::GEPOp:

2060 O << " inbounds";

2061 else if (GEPFlags.hasNoUnsignedSignedWrap())

2062 O << " nusw";

2063 if (GEPFlags.hasNoUnsignedWrap())

2064 O << " nuw";

2065 break;

2066 case OperationType::NonNegOp:

2068 O << " nneg";

2069 break;

2070 case OperationType::Other:

2071 break;

2072 }

2073 O << " ";

2074}

2075#endif

2076

2078 auto &Builder = State.Builder;

2079 switch (Opcode) {

2080 case Instruction::Call:

2081 case Instruction::Br:

2082 case Instruction::PHI:

2083 case Instruction::GetElementPtr:

2084 case Instruction::Select:

2085 llvm_unreachable("This instruction is handled by a different recipe.");

2086 case Instruction::UDiv:

2087 case Instruction::SDiv:

2088 case Instruction::SRem:

2089 case Instruction::URem:

2090 case Instruction::Add:

2091 case Instruction::FAdd:

2092 case Instruction::Sub:

2093 case Instruction::FSub:

2094 case Instruction::FNeg:

2095 case Instruction::Mul:

2096 case Instruction::FMul:

2097 case Instruction::FDiv:

2098 case Instruction::FRem:

2099 case Instruction::Shl:

2100 case Instruction::LShr:

2101 case Instruction::AShr:

2102 case Instruction::And:

2103 case Instruction::Or:

2104 case Instruction::Xor: {

2105

2108 Ops.push_back(State.get(VPOp));

2109

2110 Value *V = Builder.CreateNAryOp(Opcode, Ops);

2111

2115 }

2116

2117

2118 State.set(this, V);

2119 break;

2120 }

2121 case Instruction::ExtractValue: {

2125 Value *Extract = Builder.CreateExtractValue(Op, CI->getZExtValue());

2126 State.set(this, Extract);

2127 break;

2128 }

2129 case Instruction::Freeze: {

2131 Value *Freeze = Builder.CreateFreeze(Op);

2132 State.set(this, Freeze);

2133 break;

2134 }

2135 case Instruction::ICmp:

2136 case Instruction::FCmp: {

2137

2138 bool FCmp = Opcode == Instruction::FCmp;

2142 if (FCmp) {

2144 } else {

2146 }

2150 }

2151 State.set(this, C);

2152 break;

2153 }

2154 default:

2155

2156 LLVM_DEBUG(dbgs() << "LV: Found an unhandled opcode : "

2159 }

2160

2161#if !defined(NDEBUG)

2162

2163

2165 State.get(this)->getType() &&

2166 "inferred type and type from generated instructions do not match");

2167#endif

2168}

2169

2172 switch (Opcode) {

2173 case Instruction::UDiv:

2174 case Instruction::SDiv:

2175 case Instruction::SRem:

2176 case Instruction::URem:

2177

2178

2179

2180

2181 case Instruction::FNeg:

2182 case Instruction::Add:

2183 case Instruction::FAdd:

2184 case Instruction::Sub:

2185 case Instruction::FSub:

2186 case Instruction::Mul:

2187 case Instruction::FMul:

2188 case Instruction::FDiv:

2189 case Instruction::FRem:

2190 case Instruction::Shl:

2191 case Instruction::LShr:

2192 case Instruction::AShr:

2193 case Instruction::And:

2194 case Instruction::Or:

2195 case Instruction::Xor:

2196 case Instruction::Freeze:

2197 case Instruction::ExtractValue:

2198 case Instruction::ICmp:

2199 case Instruction::FCmp:

2201 default:

2203 }

2204}

2205

2206#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2209 O << Indent << "WIDEN ";

2214}

2215#endif

2216

2218 auto &Builder = State.Builder;

2219

2220 assert(State.VF.isVector() && "Not vectorizing?");

2225 State.set(this, Cast);

2229 }

2230}

2231

2234

2235

2236

2238 return 0;

2239

2249 if (WidenMemoryRecipe == nullptr)

2251 if (!WidenMemoryRecipe->isConsecutive())

2253 if (WidenMemoryRecipe->isReverse())

2255 if (WidenMemoryRecipe->isMasked())

2258 };

2259

2262

2263 if ((Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) &&

2266 CCH = ComputeCCH(StoreRecipe);

2267 }

2268

2269 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||

2270 Opcode == Instruction::FPExt) {

2275 }

2276

2277 auto *SrcTy =

2280

2281 return Ctx.TTI.getCastInstrCost(

2282 Opcode, DestTy, SrcTy, CCH, Ctx.CostKind,

2284}

2285

2286#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2289 O << Indent << "WIDEN-CAST ";

2295}

2296#endif

2297

2300 return Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);

2301}

2302

2303

2304

2307 : ConstantFP::get(Ty, C);

2308}

2309

2310#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2313 O << Indent;

2315 O << " = WIDEN-INDUCTION";

2317 O << " ";

2319

2321 O << " (truncated to " << *TI->getType() << ")";

2322}

2323#endif

2324

2326

2327

2328

2330 return false;

2333 return StartC && StartC->isZero() && StepC && StepC->isOne() &&

2335}

2336

2337#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2340 O << Indent;

2342 O << " = DERIVED-IV ";

2344 O << " + ";

2346 O << " * ";

2348}

2349#endif

2350

2352

2356

2357

2358

2359

2363

2364

2366 assert(BaseIVTy == Step->getType() && "Types of BaseIV and Step must match!");

2367

2368

2369

2373 AddOp = Instruction::Add;

2374 MulOp = Instruction::Mul;

2375 } else {

2376 AddOp = InductionOpcode;

2377 MulOp = Instruction::FMul;

2378 }

2379

2380

2381

2383

2384 Type *IntStepTy =

2386

2387 unsigned StartLane = 0;

2388 unsigned EndLane = FirstLaneOnly ? 1 : State.VF.getKnownMinValue();

2389 if (State.Lane) {

2390 StartLane = State.Lane->getKnownLane();

2391 EndLane = StartLane + 1;

2392 }

2393 Value *StartIdx0;

2395 StartIdx0 = ConstantInt::get(IntStepTy, 0);

2396 else {

2397 StartIdx0 = State.get(getOperand(2), true);

2399 StartIdx0 =

2400 Builder.CreateMul(StartIdx0, ConstantInt::get(StartIdx0->getType(),

2402 }

2403 StartIdx0 = Builder.CreateSExtOrTrunc(StartIdx0, IntStepTy);

2404 }

2405

2407 StartIdx0 = Builder.CreateSIToFP(StartIdx0, BaseIVTy);

2408

2409 for (unsigned Lane = StartLane; Lane < EndLane; ++Lane) {

2410 Value *StartIdx = Builder.CreateBinOp(

2412

2413

2415 "Expected StartIdx to be folded to a constant when VF is not "

2416 "scalable");

2417 auto *Mul = Builder.CreateBinOp(MulOp, StartIdx, Step);

2418 auto *Add = Builder.CreateBinOp(AddOp, BaseIV, Mul);

2419 State.set(this, Add, VPLane(Lane));

2420 }

2421}

2422

2423#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2426 O << Indent;

2428 O << " = SCALAR-STEPS ";

2430}

2431#endif

2432

2437

2439 assert(State.VF.isVector() && "not widening");

2440

2441

2442

2443

2444

2445

2448 [](VPValue *Op) { return ->isDefinedOutsideLoopRegions(); }) &&

2449 "Expected at least one loop-variant operand");

2450

2451

2452

2453

2454

2455 auto *Ptr = State.get(getOperand(0), isPointerLoopInvariant());

2456

2457

2458

2462 Indices.push_back(State.get(Operand, isIndexLoopInvariant(I - 1)));

2463 }

2464

2465

2466

2467 auto *NewGEP = State.Builder.CreateGEP(getSourceElementType(), Ptr, Indices,

2469 assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) &&

2470 "NewGEP is not a pointer vector");

2471 State.set(this, NewGEP);

2472}

2473

2474#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2477 O << Indent << "WIDEN-GEP ";

2478 O << (isPointerLoopInvariant() ? "Inv" : "Var");

2480 O << "[" << (isIndexLoopInvariant(I) ? "Inv" : "Var") << "]";

2481

2482 O << " ";

2484 O << " = getelementptr";

2487}

2488#endif

2489

2491 auto &Builder = State.Builder;

2493 const DataLayout &DL = Builder.GetInsertBlock()->getDataLayout();

2494 Type *IndexTy = DL.getIndexType(State.TypeAnalysis.inferScalarType(this));

2495

2496

2498 if (IndexTy != RunTimeVF->getType())

2499 RunTimeVF = Builder.CreateZExtOrTrunc(RunTimeVF, IndexTy);

2500

2501 Value *NumElt = Builder.CreateMul(

2502 ConstantInt::get(IndexTy, Stride * (int64_t)CurrentPart), RunTimeVF);

2503

2504 Value *LastLane = Builder.CreateSub(RunTimeVF, ConstantInt::get(IndexTy, 1));

2505 if (Stride != 1)

2506 LastLane =

2509 Value *ResultPtr =

2510 Builder.CreateGEP(IndexedTy, Ptr, NumElt, "", getGEPNoWrapFlags());

2511 ResultPtr = Builder.CreateGEP(IndexedTy, ResultPtr, LastLane, "",

2513

2514 State.set(this, ResultPtr, true);

2515}

2516

2517#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2520 O << Indent;

2522 O << " = vector-end-pointer";

2525}

2526#endif

2527

2529 auto &Builder = State.Builder;

2531 const DataLayout &DL = Builder.GetInsertBlock()->getDataLayout();

2532 Type *IndexTy = DL.getIndexType(State.TypeAnalysis.inferScalarType(this));

2534

2538

2539 State.set(this, ResultPtr, true);

2540}

2541

2542#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2545 O << Indent;

2547 O << " = vector-pointer";

2550}

2551#endif

2552

2555 Type *ResultTy = toVectorTy(Ctx.Types.inferScalarType(this), VF);

2558 Ctx.TTI.getCmpSelInstrCost(Instruction::Select, ResultTy, CmpTy,

2560}

2561

2562#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2565 O << Indent << "BLEND ";

2567 O << " =";

2569

2570

2571 O << " ";

2573 } else {

2575 O << " ";

2577 if (I == 0)

2578 continue;

2579 O << "/";

2581 }

2582 }

2583}

2584#endif

2585

2587 assert(!State.Lane && "Reduction being replicated.");

2590 "In-loop AnyOf reductions aren't currently supported");

2591

2596 Value *NewCond = State.get(Cond, State.VF.isScalar());

2599

2601 if (State.VF.isVector())

2602 Start = State.Builder.CreateVectorSplat(VecTy->getElementCount(), Start);

2603

2604 Value *Select = State.Builder.CreateSelect(NewCond, NewVecOp, Start);

2606 }

2608 Value *NextInChain;

2610 Value *PrevInChain = State.get(getChainOp(), true);

2611 if (State.VF.isVector())

2612 NewRed =

2614 else

2615 NewRed = State.Builder.CreateBinOp(

2617 PrevInChain, NewVecOp);

2618 PrevInChain = NewRed;

2619 NextInChain = NewRed;

2622 Value *PrevInChain = State.get(getChainOp(), false);

2623 NewRed = State.Builder.CreateIntrinsic(

2624 PrevInChain->getType(), Intrinsic::vector_partial_reduce_add,

2625 {PrevInChain, NewVecOp}, nullptr, "partial.reduce");

2626 PrevInChain = NewRed;

2627 NextInChain = NewRed;

2628 } else {

2630 "The reduction must either be ordered, partial or in-loop");

2631 Value *PrevInChain = State.get(getChainOp(), true);

2634 NextInChain = createMinMaxOp(State.Builder, Kind, NewRed, PrevInChain);

2635 else

2636 NextInChain = State.Builder.CreateBinOp(

2638 PrevInChain, NewRed);

2639 }

2641}

2642

2644 assert(!State.Lane && "Reduction being replicated.");

2645

2646 auto &Builder = State.Builder;

2647

2650

2652 Value *Prev = State.get(getChainOp(), true);

2655

2658 Mask = State.get(CondOp);

2659 else

2660 Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());

2661

2665 } else {

2668 NewRed = createMinMaxOp(Builder, Kind, NewRed, Prev);

2669 else

2670 NewRed = Builder.CreateBinOp(

2672 Prev);

2673 }

2674 State.set(this, NewRed, true);

2675}

2676

2680 Type *ElementTy = Ctx.Types.inferScalarType(this);

2684 std::optional OptionalFMF =

2686

2693 CondCost = Ctx.TTI.getCmpSelInstrCost(Instruction::Select, VectorTy,

2694 CondTy, Pred, Ctx.CostKind);

2695 }

2696 return CondCost + Ctx.TTI.getPartialReductionCost(

2697 Opcode, ElementTy, ElementTy, ElementTy, VF,

2700 Ctx.CostKind);

2701 }

2702

2703

2707 "Any-of reduction not implemented in VPlan-based cost model currently.");

2708

2709

2710

2713 return Ctx.TTI.getMinMaxReductionCost(Id, VectorTy, FMFs, Ctx.CostKind);

2714 }

2715

2716

2717

2718 return Ctx.TTI.getArithmeticReductionCost(Opcode, VectorTy, OptionalFMF,

2719 Ctx.CostKind);

2720}

2721

2723 ExpressionTypes ExpressionType,

2726 ExpressionRecipes(ExpressionRecipes), ExpressionType(ExpressionType) {

2727 assert(!ExpressionRecipes.empty() && "Nothing to combine?");

2729 none_of(ExpressionRecipes,

2731 "expression cannot contain recipes with side-effects");

2732

2733

2735 for (auto *R : ExpressionRecipes)

2736 ExpressionRecipesAsSetOfUsers.insert(R);

2737

2738

2739

2740

2742 if (R != ExpressionRecipes.back() &&

2743 any_of(R->users(), [&ExpressionRecipesAsSetOfUsers](VPUser *U) {

2744 return !ExpressionRecipesAsSetOfUsers.contains(U);

2745 })) {

2746

2747

2749 R->replaceUsesWithIf(CopyForExtUsers, [&ExpressionRecipesAsSetOfUsers](

2750 VPUser &U, unsigned) {

2751 return !ExpressionRecipesAsSetOfUsers.contains(&U);

2752 });

2754 }

2755 if (R->getParent())

2756 R->removeFromParent();

2757 }

2758

2759

2760

2761

2762

2763 for (auto *R : ExpressionRecipes) {

2764 for (const auto &[Idx, Op] : enumerate(R->operands())) {

2765 auto *Def = Op->getDefiningRecipe();

2766 if (Def && ExpressionRecipesAsSetOfUsers.contains(Def))

2767 continue;

2769 LiveInPlaceholders.push_back(new VPValue());

2770 }

2771 }

2772

2773

2774

2775 for (auto *R : ExpressionRecipes)

2776 for (auto const &[LiveIn, Tmp] : zip(operands(), LiveInPlaceholders))

2777 R->replaceUsesOfWith(LiveIn, Tmp);

2778}

2779

2781 for (auto *R : ExpressionRecipes)

2782

2783

2784 if (!R->getParent())

2785 R->insertBefore(this);

2786

2788 LiveInPlaceholders[Idx]->replaceAllUsesWith(Op);

2789

2791 ExpressionRecipes.clear();

2792}

2793

2796 Type *RedTy = Ctx.Types.inferScalarType(this);

2800 "VPExpressionRecipe only supports integer types currently.");

2803 switch (ExpressionType) {

2804 case ExpressionTypes::ExtendedReduction: {

2808

2810 ->isPartialReduction()

2811 ? Ctx.TTI.getPartialReductionCost(

2812 Opcode, Ctx.Types.inferScalarType(getOperand(0)), nullptr,

2813 RedTy, VF,

2815 ExtR->getOpcode()),

2817 : Ctx.TTI.getExtendedReductionCost(

2818 Opcode, ExtR->getOpcode() == Instruction::ZExt, RedTy,

2819 SrcVecTy, std::nullopt, Ctx.CostKind);

2820 }

2821 case ExpressionTypes::MulAccReduction:

2822 return Ctx.TTI.getMulAccReductionCost(false, Opcode, RedTy, SrcVecTy,

2823 Ctx.CostKind);

2824

2825 case ExpressionTypes::ExtNegatedMulAccReduction:

2826 assert(Opcode == Instruction::Add && "Unexpected opcode");

2827 Opcode = Instruction::Sub;

2828 [[fallthrough]];

2829 case ExpressionTypes::ExtMulAccReduction: {

2831 if (RedR->isPartialReduction()) {

2835 return Ctx.TTI.getPartialReductionCost(

2836 Opcode, Ctx.Types.inferScalarType(getOperand(0)),

2837 Ctx.Types.inferScalarType(getOperand(1)), RedTy, VF,

2839 Ext0R->getOpcode()),

2841 Ext1R->getOpcode()),

2842 Mul->getOpcode(), Ctx.CostKind);

2843 }

2844 return Ctx.TTI.getMulAccReductionCost(

2846 Instruction::ZExt,

2847 Opcode, RedTy, SrcVecTy, Ctx.CostKind);

2848 }

2849 }

2850 llvm_unreachable("Unknown VPExpressionRecipe::ExpressionTypes enum");

2851}

2852

2855 return R->mayReadFromMemory() || R->mayWriteToMemory();

2856 });

2857}

2858

2861 none_of(ExpressionRecipes,

2862 [](VPSingleDefRecipe *R) { return R->mayHaveSideEffects(); }) &&

2863 "expression cannot contain recipes with side-effects");

2864 return false;

2865}

2866

2868

2869

2871 return RR && !RR->isPartialReduction();

2872}

2873

2874#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

2875

2878 O << Indent << "EXPRESSION ";

2880 O << " = ";

2883

2884 switch (ExpressionType) {

2885 case ExpressionTypes::ExtendedReduction: {

2887 O << " + " << (Red->isPartialReduction() ? "partial." : "") << "reduce.";

2890 Red->printFlags(O);

2891

2894 << *Ext0->getResultType();

2895 if (Red->isConditional()) {

2896 O << ", ";

2897 Red->getCondOp()->printAsOperand(O, SlotTracker);

2898 }

2899 O << ")";

2900 break;

2901 }

2902 case ExpressionTypes::ExtNegatedMulAccReduction: {

2904 O << " + " << (Red->isPartialReduction() ? "partial." : "") << "reduce.";

2907 << " (sub (0, mul";

2909 Mul->printFlags(O);

2910 O << "(";

2914 << *Ext0->getResultType() << "), (";

2918 << *Ext1->getResultType() << ")";

2919 if (Red->isConditional()) {

2920 O << ", ";

2921 Red->getCondOp()->printAsOperand(O, SlotTracker);

2922 }

2923 O << "))";

2924 break;

2925 }

2926 case ExpressionTypes::MulAccReduction:

2927 case ExpressionTypes::ExtMulAccReduction: {

2929 O << " + " << (Red->isPartialReduction() ? "partial." : "") << "reduce.";

2932 << " (";

2933 O << "mul";

2934 bool IsExtended = ExpressionType == ExpressionTypes::ExtMulAccReduction;

2936 : ExpressionRecipes[0]);

2937 Mul->printFlags(O);

2938 if (IsExtended)

2939 O << "(";

2941 if (IsExtended) {

2944 << *Ext0->getResultType() << "), (";

2945 } else {

2946 O << ", ";

2947 }

2949 if (IsExtended) {

2952 << *Ext1->getResultType() << ")";

2953 }

2954 if (Red->isConditional()) {

2955 O << ", ";

2956 Red->getCondOp()->printAsOperand(O, SlotTracker);

2957 }

2958 O << ")";

2959 break;

2960 }

2961 }

2962}

2963

2967 O << Indent << "PARTIAL-REDUCE ";

2968 else

2969 O << Indent << "REDUCE ";

2971 O << " = ";

2973 O << " +";

2975 O << " reduce."

2978 << " (";

2981 O << ", ";

2983 }

2984 O << ")";

2985}

2986

2989 O << Indent << "REDUCE ";

2991 O << " = ";

2993 O << " +";

2995 O << " vp.reduce."

2998 << " (";

3000 O << ", ";

3003 O << ", ";

3005 }

3006 O << ")";

3007}

3008

3009#endif

3010

3011

3012

3013

3017 assert((!Instr->getType()->isAggregateType() ||

3019 "Expected vectorizable or non-aggregate type.");

3020

3021

3022 bool IsVoidRetTy = Instr->getType()->isVoidTy();

3023

3025 if (!IsVoidRetTy) {

3026 Cloned->setName(Instr->getName() + ".cloned");

3027 Type *ResultTy = State.TypeAnalysis.inferScalarType(RepRecipe);

3028

3029

3030

3031 if (ResultTy != Cloned->getType())

3033 }

3034

3037

3040

3042 State.setDebugLocFrom(DL);

3043

3044

3045

3047 auto InputLane = Lane;

3048 VPValue *Operand = I.value();

3051 Cloned->setOperand(I.index(), State.get(Operand, InputLane));

3052 }

3053

3054

3055 State.Builder.Insert(Cloned);

3056

3057 State.set(RepRecipe, Cloned, Lane);

3058

3059

3061 State.AC->registerAssumption(II);

3062

3067 [](VPValue *Op) { return Op->isDefinedOutsideLoopRegions(); })) &&

3068 "Expected a recipe is either within a region or all of its operands "

3069 "are defined outside the vectorized region.");

3070}

3071

3074

3075 if (!State.Lane) {

3076 assert(IsSingleScalar && "VPReplicateRecipes outside replicate regions "

3077 "must have already been unrolled");

3079 return;

3080 }

3081

3083 "uniform recipe shouldn't be predicated");

3084 assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");

3086

3087 if (State.VF.isVector() && shouldPack()) {

3088 Value *WideValue =

3089 State.Lane->isFirstLane()

3091 : State.get(this);

3092 State.set(this, State.packScalarIntoVectorizedValue(this, WideValue,

3093 *State.Lane));

3094 }

3095}

3096

3098

3099

3103 return false;

3104 });

3105}

3106

3107

3108

3109

3110

3111

3113 const Loop *L) {

3117 Instruction::GetElementPtr) ||

3120 return nullptr;

3121

3122

3123

3125 if (!Opd->isDefinedOutsideLoopRegions() &&

3127 return nullptr;

3128 }

3129

3131}

3132

3133

3134

3138

3139 while (!WorkList.empty()) {

3141 if (!Cur || !Seen.insert(Cur).second)

3142 continue;

3143

3145

3146

3148 [&](unsigned I) {

3149 return Seen.contains(

3150 Blend->getIncomingValue(I)->getDefiningRecipe());

3151 }))

3152 continue;

3153

3154 for (VPUser *U : Cur->users()) {

3156 if (InterleaveR->getAddr() == Cur)

3157 return true;

3159 if (RepR->getOpcode() == Instruction::Load &&

3160 RepR->getOperand(0) == Cur)

3161 return true;

3162 if (RepR->getOpcode() == Instruction::Store &&

3163 RepR->getOperand(1) == Cur)

3164 return true;

3165 }

3167 if (MemR->getAddr() == Cur && MemR->isConsecutive())

3168 return true;

3169 }

3170 }

3171

3172

3173

3174

3175 if (Blend)

3176 continue;

3177

3179 }

3180 return false;

3181}

3182

3186

3187

3188 Ctx.SkipCostComputation.insert(UI);

3189

3192

3194 case Instruction::GetElementPtr:

3195

3196

3197

3198

3199 return 0;

3200 case Instruction::Call: {

3201 auto *CalledFn =

3203

3206 for (const VPValue *ArgOp : ArgOps)

3207 Tys.push_back(Ctx.Types.inferScalarType(ArgOp));

3208

3209 if (CalledFn->isIntrinsic())

3210

3211

3212 switch (CalledFn->getIntrinsicID()) {

3213 case Intrinsic::assume:

3214 case Intrinsic::lifetime_end:

3215 case Intrinsic::lifetime_start:

3216 case Intrinsic::sideeffect:

3217 case Intrinsic::pseudoprobe:

3218 case Intrinsic::experimental_noalias_scope_decl: {

3221 "scalarizing intrinsic should be free");

3223 }

3224 default:

3225 break;

3226 }

3227

3228 Type *ResultTy = Ctx.Types.inferScalarType(this);

3230 Ctx.TTI.getCallInstrCost(CalledFn, ResultTy, Tys, Ctx.CostKind);

3232 if (CalledFn->isIntrinsic())

3233 ScalarCallCost = std::min(

3234 ScalarCallCost,

3237 return ScalarCallCost;

3238 }

3239

3241 Ctx.getScalarizationOverhead(ResultTy, ArgOps, VF);

3242 }

3243 case Instruction::Add:

3244 case Instruction::Sub:

3245 case Instruction::FAdd:

3246 case Instruction::FSub:

3247 case Instruction::Mul:

3248 case Instruction::FMul:

3249 case Instruction::FDiv:

3250 case Instruction::FRem:

3251 case Instruction::Shl:

3252 case Instruction::LShr:

3253 case Instruction::AShr:

3254 case Instruction::And:

3255 case Instruction::Or:

3256 case Instruction::Xor:

3257 case Instruction::ICmp:

3258 case Instruction::FCmp:

3260 Ctx) *

3262 case Instruction::SDiv:

3263 case Instruction::UDiv:

3264 case Instruction::SRem:

3265 case Instruction::URem: {

3269 return ScalarCost;

3270

3272 Ctx.getScalarizationOverhead(Ctx.Types.inferScalarType(this),

3274

3275

3276 if ( getRegion ()->isReplicator())

3277 return ScalarCost;

3278

3279

3281 Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);

3282

3283

3284

3285 ScalarCost /= Ctx.getPredBlockCostDivisor(UI->getParent());

3286 return ScalarCost;

3287 }

3288 case Instruction::Load:

3289 case Instruction::Store: {

3290

3291

3293 if (ParentRegion && ParentRegion->isReplicator())

3294 break;

3295

3296 bool IsLoad = UI->getOpcode() == Instruction::Load;

3300 break;

3301

3302 Type *ValTy = Ctx.Types.inferScalarType(IsLoad ? this : getOperand(0));

3303 Type *ScalarPtrTy = Ctx.Types.inferScalarType(PtrOp);

3305 unsigned AS = cast(ScalarPtrTy)->getAddressSpace();

3307 InstructionCost ScalarMemOpCost = Ctx.TTI.getMemoryOpCost(

3308 UI->getOpcode(), ValTy, Alignment, AS, Ctx.CostKind, OpInfo);

3309

3311 bool PreferVectorizedAddressing = Ctx.TTI.prefersVectorizedAddressing();

3312 bool UsedByLoadStoreAddress =

3315 ScalarMemOpCost + Ctx.TTI.getAddressComputationCost(

3316 PtrTy, UsedByLoadStoreAddress ? nullptr : &Ctx.SE,

3317 PtrSCEV, Ctx.CostKind);

3319 return ScalarCost;

3320

3323

3324

3325

3326

3327 if (!UsedByLoadStoreAddress) {

3328 bool EfficientVectorLoadStore =

3329 Ctx.TTI.supportsEfficientVectorElementLoadStore();

3330 if (!(IsLoad && !PreferVectorizedAddressing) &&

3331 !(!IsLoad && EfficientVectorLoadStore))

3333

3334 if (!EfficientVectorLoadStore)

3335 ResultTy = Ctx.Types.inferScalarType(this);

3336 }

3337

3339 Ctx.getScalarizationOverhead(ResultTy, OpsToScalarize, VF, true);

3340 }

3341 }

3342

3343 return Ctx.getLegacyCost(UI, VF);

3344}

3345

3346#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3349 O << Indent << (IsSingleScalar ? "CLONE " : "REPLICATE ");

3350

3353 O << " = ";

3354 }

3356 O << "call";

3358 O << "@" << CB->getCalledFunction()->getName() << "(";

3362 });

3363 O << ")";

3364 } else {

3368 }

3369

3371 O << " (S->V)";

3372}

3373#endif

3374

3376 assert(State.Lane && "Branch on Mask works only on single instance.");

3377

3379 Value *ConditionBit = State.get(BlockInMask, *State.Lane);

3380

3381

3382

3383 auto *CurrentTerminator = State.CFG.PrevBB->getTerminator();

3385 "Expected to replace unreachable terminator with conditional branch.");

3386 auto CondBr =

3387 State.Builder.CreateCondBr(ConditionBit, State.CFG.PrevBB, nullptr);

3388 CondBr->setSuccessor(0, nullptr);

3389 CurrentTerminator->eraseFromParent();

3390}

3391

3394

3395

3396

3397 return 0;

3398}

3399

3401 assert(State.Lane && "Predicated instruction PHI works per instance.");

3406 assert(PredicatingBB && "Predicated block has no single predecessor.");

3408 "operand must be VPReplicateRecipe");

3409

3410

3411

3412

3413

3414

3415

3416 if (State.hasVectorValue(getOperand(0))) {

3419 "Packed operands must generate an insertelement or insertvalue");

3420

3421

3422

3423

3424

3425

3427 for (unsigned I = 0; I < StructTy->getNumContainedTypes() - 1; I++)

3429

3430 PHINode *VPhi = State.Builder.CreatePHI(VecI->getType(), 2);

3431 VPhi->addIncoming(VecI->getOperand(0), PredicatingBB);

3432 VPhi->addIncoming(VecI, PredicatedBB);

3433 if (State.hasVectorValue(this))

3434 State.reset(this, VPhi);

3435 else

3436 State.set(this, VPhi);

3437

3438

3440 } else {

3442 return;

3443

3444 Type *PredInstType = State.TypeAnalysis.inferScalarType(getOperand(0));

3445 PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2);

3447 PredicatingBB);

3448 Phi->addIncoming(ScalarPredInst, PredicatedBB);

3449 if (State.hasScalarValue(this, *State.Lane))

3450 State.reset(this, Phi, *State.Lane);

3451 else

3452 State.set(this, Phi, *State.Lane);

3453

3454

3455 State.reset(getOperand(0), Phi, *State.Lane);

3456 }

3457}

3458

3459#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3462 O << Indent << "PHI-PREDICATED-INSTRUCTION ";

3464 O << " = ";

3466}

3467#endif

3468

3473 ->getAddressSpace();

3475 ? Instruction::Load

3476 : Instruction::Store;

3477

3479

3480

3481

3483 "Inconsecutive memory access should not have the order.");

3484

3487

3488

3489

3492

3496 : Intrinsic::vp_scatter;

3497 return Ctx.TTI.getAddressComputationCost(PtrTy, nullptr, nullptr,

3498 Ctx.CostKind) +

3499 Ctx.TTI.getMemIntrinsicInstrCost(

3502 Ctx.CostKind);

3503 }

3504

3508 : Intrinsic::masked_store;

3509 Cost += Ctx.TTI.getMemIntrinsicInstrCost(

3511 } else {

3515 Cost += Ctx.TTI.getMemoryOpCost(Opcode, Ty, Alignment, AS, Ctx.CostKind,

3517 }

3519 return Cost;

3520

3521 return Cost += Ctx.TTI.getShuffleCost(

3524}

3525

3530

3531 auto &Builder = State.Builder;

3532 Value *Mask = nullptr;

3533 if (auto *VPMask = getMask()) {

3534

3535

3536 Mask = State.get(VPMask);

3538 Mask = Builder.CreateVectorReverse(Mask, "reverse");

3539 }

3540

3541 Value *Addr = State.get(getAddr(), !CreateGather);

3543 if (CreateGather) {

3544 NewLI = Builder.CreateMaskedGather(DataTy, Addr, Alignment, Mask, nullptr,

3545 "wide.masked.gather");

3546 } else if (Mask) {

3547 NewLI =

3548 Builder.CreateMaskedLoad(DataTy, Addr, Alignment, Mask,

3550 } else {

3551 NewLI = Builder.CreateAlignedLoad(DataTy, Addr, Alignment, "wide.load");

3552 }

3555 NewLI = Builder.CreateVectorReverse(NewLI, "reverse");

3556 State.set(this, NewLI);

3557}

3558

3559#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3562 O << Indent << "WIDEN ";

3564 O << " = load ";

3566}

3567#endif

3568

3569

3570

3574 Value *AllTrueMask =

3575 Builder.CreateVectorSplat(ValTy->getElementCount(), Builder.getTrue());

3576 return Builder.CreateIntrinsic(ValTy, Intrinsic::experimental_vp_reverse,

3577 {Operand, AllTrueMask, EVL}, nullptr, Name);

3578}

3579

3584

3585 auto &Builder = State.Builder;

3588 Value *Addr = State.get(getAddr(), !CreateGather);

3589 Value *Mask = nullptr;

3591 Mask = State.get(VPMask);

3593 Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");

3594 } else {

3595 Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());

3596 }

3597

3598 if (CreateGather) {

3599 NewLI =

3600 Builder.CreateIntrinsic(DataTy, Intrinsic::vp_gather, {Addr, Mask, EVL},

3601 nullptr, "wide.masked.gather");

3602 } else {

3603 NewLI = Builder.CreateIntrinsic(DataTy, Intrinsic::vp_load,

3604 {Addr, Mask, EVL}, nullptr, "vp.op.load");

3605 }

3612 State.set(this, Res);

3613}

3614

3619

3620

3621

3622

3623

3624

3627 ->getAddressSpace();

3630 Ctx.CostKind);

3632 return Cost;

3633

3634 return Cost + Ctx.TTI.getShuffleCost(

3637}

3638

3639#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3642 O << Indent << "WIDEN ";

3644 O << " = vp.load ";

3646}

3647#endif

3648

3652

3653 auto &Builder = State.Builder;

3654

3655 Value *Mask = nullptr;

3656 if (auto *VPMask = getMask()) {

3657

3658

3659 Mask = State.get(VPMask);

3661 Mask = Builder.CreateVectorReverse(Mask, "reverse");

3662 }

3663

3664 Value *StoredVal = State.get(StoredVPValue);

3666

3667

3668 StoredVal = Builder.CreateVectorReverse(StoredVal, "reverse");

3669

3670

3671 }

3672 Value *Addr = State.get(getAddr(), !CreateScatter);

3674 if (CreateScatter)

3675 NewSI = Builder.CreateMaskedScatter(StoredVal, Addr, Alignment, Mask);

3676 else if (Mask)

3677 NewSI = Builder.CreateMaskedStore(StoredVal, Addr, Alignment, Mask);

3678 else

3679 NewSI = Builder.CreateAlignedStore(StoredVal, Addr, Alignment);

3681}

3682

3683#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3686 O << Indent << "WIDEN store ";

3688}

3689#endif

3690

3694

3695 auto &Builder = State.Builder;

3696

3698 Value *StoredVal = State.get(StoredValue);

3701 StoredVal = createReverseEVL(Builder, StoredVal, EVL, "vp.reverse");

3702 Value *Mask = nullptr;

3704 Mask = State.get(VPMask);

3706 Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");

3707 } else {

3708 Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());

3709 }

3710 Value *Addr = State.get(getAddr(), !CreateScatter);

3711 if (CreateScatter) {

3713 Intrinsic::vp_scatter,

3714 {StoredVal, Addr, Mask, EVL});

3715 } else {

3717 Intrinsic::vp_store,

3718 {StoredVal, Addr, Mask, EVL});

3719 }

3723}

3724

3729

3730

3731

3732

3733

3734

3737 ->getAddressSpace();

3740 Ctx.CostKind);

3742 return Cost;

3743

3744 return Cost + Ctx.TTI.getShuffleCost(

3747}

3748

3749#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

3752 O << Indent << "WIDEN vp.store ";

3754}

3755#endif

3756

3759

3760 auto VF = DstVTy->getElementCount();

3762 assert(VF == SrcVecTy->getElementCount() && "Vector dimensions do not match");

3763 Type *SrcElemTy = SrcVecTy->getElementType();

3764 Type *DstElemTy = DstVTy->getElementType();

3765 assert((DL.getTypeSizeInBits(SrcElemTy) == DL.getTypeSizeInBits(DstElemTy)) &&

3766 "Vector elements must have same size");

3767

3768

3770 return Builder.CreateBitOrPointerCast(V, DstVTy);

3771 }

3772

3773

3774

3775

3777 "Only one type should be a pointer type");

3779 "Only one type should be a floating point type");

3780 Type *IntTy =

3783 Value *CastVal = Builder.CreateBitOrPointerCast(V, VecIntTy);

3784 return Builder.CreateBitOrPointerCast(CastVal, DstVTy);

3785}

3786

3787

3788

3790 const Twine &Name) {

3791 unsigned Factor = Vals.size();

3792 assert(Factor > 1 && "Tried to interleave invalid number of vectors");

3793

3795#ifndef NDEBUG

3796 for (Value *Val : Vals)

3797 assert(Val->getType() == VecTy && "Tried to interleave mismatched types");

3798#endif

3799

3800

3801

3802 if (VecTy->isScalableTy()) {

3803 assert(Factor <= 8 && "Unsupported interleave factor for scalable vectors");

3804 return Builder.CreateVectorInterleave(Vals, Name);

3805 }

3806

3807

3809

3810

3811 const unsigned NumElts = VecTy->getElementCount().getFixedValue();

3812 return Builder.CreateShuffleVector(

3814}

3815

3816

3817

3818

3819

3820

3821

3822

3823

3824

3825

3826

3827

3828

3829

3830

3831

3832

3833

3834

3835

3836

3837

3838

3839

3840

3841

3842

3843

3845 assert(!State.Lane && "Interleave group being replicated.");

3847 "Masking gaps for scalable vectors is not yet supported.");

3850

3851

3853 unsigned InterleaveFactor = Group->getFactor();

3854 auto *VecTy = VectorType::get(ScalarTy, State.VF * InterleaveFactor);

3855

3858 Value *ResAddr = State.get(Addr, VPLane(0));

3859

3860 auto CreateGroupMask = [&BlockInMask, &State,

3861 &InterleaveFactor](Value *MaskForGaps) -> Value * {

3862 if (State.VF.isScalable()) {

3863 assert(!MaskForGaps && "Interleaved groups with gaps are not supported.");

3864 assert(InterleaveFactor <= 8 &&

3865 "Unsupported deinterleave factor for scalable vectors");

3866 auto *ResBlockInMask = State.get(BlockInMask);

3869 }

3870

3871 if (!BlockInMask)

3872 return MaskForGaps;

3873

3874 Value *ResBlockInMask = State.get(BlockInMask);

3875 Value *ShuffledMask = State.Builder.CreateShuffleVector(

3876 ResBlockInMask,

3878 "interleaved.mask");

3879 return MaskForGaps ? State.Builder.CreateBinOp(Instruction::And,

3880 ShuffledMask, MaskForGaps)

3881 : ShuffledMask;

3882 };

3883

3884 const DataLayout &DL = Instr->getDataLayout();

3885

3887 Value *MaskForGaps = nullptr;

3889 MaskForGaps =

3891 assert(MaskForGaps && "Mask for Gaps is required but it is null");

3892 }

3893

3895 if (BlockInMask || MaskForGaps) {

3896 Value *GroupMask = CreateGroupMask(MaskForGaps);

3898 NewLoad = State.Builder.CreateMaskedLoad(VecTy, ResAddr,

3899 Group->getAlign(), GroupMask,

3900 PoisonVec, "wide.masked.vec");

3901 } else

3902 NewLoad = State.Builder.CreateAlignedLoad(VecTy, ResAddr,

3903 Group->getAlign(), "wide.vec");

3905

3907

3909 if (VecTy->isScalableTy()) {

3910

3911

3912 assert(InterleaveFactor <= 8 &&

3913 "Unsupported deinterleave factor for scalable vectors");

3914 NewLoad = State.Builder.CreateIntrinsic(

3916 NewLoad->getType(), NewLoad,

3917 nullptr, "strided.vec");

3918 }

3919

3920 auto CreateStridedVector = [&InterleaveFactor, &State,

3921 &NewLoad](unsigned Index) -> Value * {

3922 assert(Index < InterleaveFactor && "Illegal group index");

3923 if (State.VF.isScalable())

3924 return State.Builder.CreateExtractValue(NewLoad, Index);

3925

3926

3927

3928 auto StrideMask =

3929 createStrideMask(Index, InterleaveFactor, State.VF.getFixedValue());

3930 return State.Builder.CreateShuffleVector(NewLoad, StrideMask,

3931 "strided.vec");

3932 };

3933

3934 for (unsigned I = 0, J = 0; I < InterleaveFactor; ++I) {

3936

3937

3938 if (!Member)

3939 continue;

3940

3941 Value *StridedVec = CreateStridedVector(I);

3942

3943

3944 if (Member->getType() != ScalarTy) {

3946 StridedVec =

3948 }

3949

3951 StridedVec = State.Builder.CreateVectorReverse(StridedVec, "reverse");

3952

3953 State.set(VPDefs[J], StridedVec);

3954 ++J;

3955 }

3956 return;

3957 }

3958

3959

3961

3962

3963 Value *MaskForGaps =

3966 "Mismatch between NeedsMaskForGaps and MaskForGaps");

3968

3970 unsigned StoredIdx = 0;

3971 for (unsigned i = 0; i < InterleaveFactor; i++) {

3973 "Fail to get a member from an interleaved store group");

3975

3976

3977 if (!Member) {

3980 continue;

3981 }

3982

3983 Value *StoredVec = State.get(StoredValues[StoredIdx]);

3984 ++StoredIdx;

3985

3987 StoredVec = State.Builder.CreateVectorReverse(StoredVec, "reverse");

3988

3989

3990

3991 if (StoredVec->getType() != SubVT)

3993

3994 StoredVecs.push_back(StoredVec);

3995 }

3996

3997

4000 if (BlockInMask || MaskForGaps) {

4001 Value *GroupMask = CreateGroupMask(MaskForGaps);

4002 NewStoreInstr = State.Builder.CreateMaskedStore(

4003 IVec, ResAddr, Group->getAlign(), GroupMask);

4004 } else

4005 NewStoreInstr =

4006 State.Builder.CreateAlignedStore(IVec, ResAddr, Group->getAlign());

4007

4009

4011}

4012

4013#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4017 O << Indent << "INTERLEAVE-GROUP with factor " << IG->getFactor() << " at ";

4018 IG->getInsertPos()->printAsOperand(O, false);

4019 O << ", ";

4022 if (Mask) {

4023 O << ", ";

4025 }

4026

4027 unsigned OpIdx = 0;

4028 for (unsigned i = 0; i < IG->getFactor(); ++i) {

4029 if (!IG->getMember(i))

4030 continue;

4032 O << "\n" << Indent << " store ";

4034 O << " to index " << i;

4035 } else {

4036 O << "\n" << Indent << " ";

4038 O << " = load from index " << i;

4039 }

4041 }

4042}

4043#endif

4044

4046 assert(!State.Lane && "Interleave group being replicated.");

4047 assert(State.VF.isScalable() &&

4048 "Only support scalable VF for EVL tail-folding.");

4050 "Masking gaps for scalable vectors is not yet supported.");

4053

4054

4056 unsigned InterleaveFactor = Group->getFactor();

4057 assert(InterleaveFactor <= 8 &&

4058 "Unsupported deinterleave/interleave factor for scalable vectors");

4059 ElementCount WideVF = State.VF * InterleaveFactor;

4061

4063 Value *ResAddr = State.get(Addr, VPLane(0));

4065 Value *InterleaveEVL = State.Builder.CreateMul(

4066 EVL, ConstantInt::get(EVL->getType(), InterleaveFactor), "interleave.evl",

4067 true, true);

4068 LLVMContext &Ctx = State.Builder.getContext();

4069

4070 Value *GroupMask = nullptr;

4074 } else {

4075 GroupMask =

4076 State.Builder.CreateVectorSplat(WideVF, State.Builder.getTrue());

4077 }

4078

4079

4081 CallInst *NewLoad = State.Builder.CreateIntrinsic(

4082 VecTy, Intrinsic::vp_load, {ResAddr, GroupMask, InterleaveEVL}, nullptr,

4083 "wide.vp.load");

4086

4088

4090

4091

4092

4093 NewLoad = State.Builder.CreateIntrinsic(

4095 NewLoad->getType(), NewLoad,

4096 nullptr, "strided.vec");

4097

4098 const DataLayout &DL = Instr->getDataLayout();

4099 for (unsigned I = 0, J = 0; I < InterleaveFactor; ++I) {

4101

4102 if (!Member)

4103 continue;

4104

4105 Value *StridedVec = State.Builder.CreateExtractValue(NewLoad, I);

4106

4107 if (Member->getType() != ScalarTy) {

4109 StridedVec =

4111 }

4112

4113 State.set(getVPValue(J), StridedVec);

4114 ++J;

4115 }

4116 return;

4117 }

4118

4119

4121

4123

4125 const DataLayout &DL = Instr->getDataLayout();

4126 for (unsigned I = 0, StoredIdx = 0; I < InterleaveFactor; I++) {

4128

4129 if (!Member) {

4131 continue;

4132 }

4133

4134 Value *StoredVec = State.get(StoredValues[StoredIdx]);

4135

4136 if (StoredVec->getType() != SubVT)

4138

4139 StoredVecs.push_back(StoredVec);

4140 ++StoredIdx;

4141 }

4142

4143

4146 State.Builder.CreateIntrinsic(Type::getVoidTy(Ctx), Intrinsic::vp_store,

4147 {IVec, ResAddr, GroupMask, InterleaveEVL});

4150

4152

4154}

4155

4156#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4160 O << Indent << "INTERLEAVE-GROUP with factor " << IG->getFactor() << " at ";

4161 IG->getInsertPos()->printAsOperand(O, false);

4162 O << ", ";

4164 O << ", ";

4167 O << ", ";

4169 }

4170

4171 unsigned OpIdx = 0;

4172 for (unsigned i = 0; i < IG->getFactor(); ++i) {

4173 if (!IG->getMember(i))

4174 continue;

4176 O << "\n" << Indent << " vp.store ";

4178 O << " to index " << i;

4179 } else {

4180 O << "\n" << Indent << " ";

4182 O << " = vp.load from index " << i;

4183 }

4185 }

4186}

4187#endif

4188

4192

4193 unsigned InsertPosIdx = 0;

4194 for (unsigned Idx = 0; IG->getFactor(); ++Idx)

4195 if (auto *Member = IG->getMember(Idx)) {

4196 if (Member == InsertPos)

4197 break;

4198 InsertPosIdx++;

4199 }

4200 Type *ValTy = Ctx.Types.inferScalarType(

4205 ->getAddressSpace();

4206

4207 unsigned InterleaveFactor = IG->getFactor();

4208 auto *WideVecTy = VectorType::get(ValTy, VF * InterleaveFactor);

4209

4210

4212 for (unsigned IF = 0; IF < InterleaveFactor; IF++)

4213 if (IG->getMember(IF))

4215

4216

4218 InsertPos->getOpcode(), WideVecTy, IG->getFactor(), Indices,

4219 IG->getAlign(), AS, Ctx.CostKind, getMask(), NeedsMaskForGaps);

4220

4221 if (!IG->isReverse())

4222 return Cost;

4223

4224 return Cost + IG->getNumMembers() *

4226 VectorTy, VectorTy, {}, Ctx.CostKind,

4227 0);

4228}

4229

4230#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4233 O << Indent << "EMIT ";

4235 O << " = CANONICAL-INDUCTION ";

4237}

4238#endif

4239

4244

4245#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4249 "unexpected number of operands");

4250 O << Indent << "EMIT ";

4252 O << " = WIDEN-POINTER-INDUCTION ";

4254 O << ", ";

4256 O << ", ";

4259 O << ", ";

4261 O << ", ";

4263 }

4264}

4265

4268 O << Indent << "EMIT ";

4270 O << " = EXPAND SCEV " << *Expr;

4271}

4272#endif

4273

4275 Value *CanonicalIV = State.get(getOperand(0), true);

4277 IRBuilder<> Builder(State.CFG.PrevBB->getTerminator());

4280 ? CanonicalIV

4281 : Builder.CreateVectorSplat(VF, CanonicalIV, "broadcast");

4284 VStep = Builder.CreateVectorSplat(VF, VStep);

4285 VStep =

4286 Builder.CreateAdd(VStep, Builder.CreateStepVector(VStep->getType()));

4287 }

4288 Value *CanonicalVectorIV = Builder.CreateAdd(VStart, VStep, "vec.iv");

4289 State.set(this, CanonicalVectorIV);

4290}

4291

4292#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4295 O << Indent << "EMIT ";

4297 O << " = WIDEN-CANONICAL-INDUCTION ";

4299}

4300#endif

4301

4303 auto &Builder = State.Builder;

4304

4306

4307 Type *VecTy = State.VF.isScalar()

4308 ? VectorInit->getType()

4310

4312 State.CFG.VPBB2IRBB.at(getParent()->getCFGPredecessor(0));

4313 if (State.VF.isVector()) {

4315 auto *One = ConstantInt::get(IdxTy, 1);

4317 Builder.SetInsertPoint(VectorPH->getTerminator());

4318 auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, State.VF);

4319 auto *LastIdx = Builder.CreateSub(RuntimeVF, One);

4320 VectorInit = Builder.CreateInsertElement(

4321 PoisonValue::get(VecTy), VectorInit, LastIdx, "vector.recur.init");

4322 }

4323

4324

4326 Phi->insertBefore(State.CFG.PrevBB->getFirstInsertionPt());

4327 Phi->addIncoming(VectorInit, VectorPH);

4328 State.set(this, Phi);

4329}

4330

4335 return Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);

4336

4337 return 0;

4338}

4339

4340#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4343 O << Indent << "FIRST-ORDER-RECURRENCE-PHI ";

4345 O << " = phi ";

4347}

4348#endif

4349

4351

4352

4354

4355

4356

4357

4358

4360 State.CFG.VPBB2IRBB.at(getParent()->getCFGPredecessor(0));

4361 bool ScalarPHI = State.VF.isScalar() || isInLoop();

4362 Value *StartV = State.get(StartVPV, ScalarPHI);

4364

4365 BasicBlock *HeaderBB = State.CFG.PrevBB;

4366 assert(State.CurrentParentLoop->getHeader() == HeaderBB &&

4367 "recipe must be in the vector loop header");

4370 State.set(this, Phi, isInLoop());

4371

4372 Phi->addIncoming(StartV, VectorPH);

4373}

4374

4375#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4378 O << Indent << "WIDEN-REDUCTION-PHI ";

4379

4381 O << " = phi ";

4385}

4386#endif

4387

4391 Instruction *VecPhi = State.Builder.CreatePHI(VecTy, 2, Name);

4392 State.set(this, VecPhi);

4393}

4394

4395#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4398 O << Indent << "WIDEN-PHI ";

4399

4401 O << " = phi ";

4403}

4404#endif

4405

4408 State.CFG.VPBB2IRBB.at(getParent()->getCFGPredecessor(0));

4411 State.Builder.CreatePHI(StartMask->getType(), 2, "active.lane.mask");

4412 Phi->addIncoming(StartMask, VectorPH);

4413 State.set(this, Phi);

4414}

4415

4416#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4419 O << Indent << "ACTIVE-LANE-MASK-PHI ";

4420

4422 O << " = phi ";

4424}

4425#endif

4426

4427#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)

4430 O << Indent << "EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ";

4431

4433 O << " = phi ";

4435}

4436#endif

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static MCDisassembler::DecodeStatus addOperand(MCInst &Inst, const MCOperand &Opnd)

AMDGPU Lower Kernel Arguments

AMDGPU Register Bank Select

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

static std::pair< Value *, APInt > getMask(Value *WideMask, unsigned Factor, ElementCount LeafValueEC)

const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]

This file provides a LoopVectorizationPlanner class.

static const SCEV * getAddressAccessSCEV(Value *Ptr, LoopVectorizationLegality *Legal, PredicatedScalarEvolution &PSE, const Loop *TheLoop)

Gets Address Access SCEV after verifying that the access pattern is loop invariant except the inducti...

static bool isOrdered(const Instruction *I)

MachineInstr unsigned OpIdx

uint64_t IntrinsicInst * II

const SmallVectorImpl< MachineOperand > & Cond

This file defines the SmallVector class.

static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")

static SymbolRef::Type getType(const Symbol *Sym)

This file contains the declarations of different VPlan-related auxiliary helpers.

static Instruction * createReverseEVL(IRBuilderBase &Builder, Value *Operand, Value *EVL, const Twine &Name)

Use all-true mask for reverse rather than actual mask, as it avoids a dependence w/o affecting the re...

Definition VPlanRecipes.cpp:3571

static Value * interleaveVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vals, const Twine &Name)

Return a vector containing interleaved elements from multiple smaller input vectors.

Definition VPlanRecipes.cpp:3789

static InstructionCost getCostForIntrinsics(Intrinsic::ID ID, ArrayRef< const VPValue * > Operands, const VPRecipeWithIRFlags &R, ElementCount VF, VPCostContext &Ctx)

Compute the cost for the intrinsic ID with Operands, produced by R.

Definition VPlanRecipes.cpp:1738

static Value * createBitOrPointerCast(IRBuilderBase &Builder, Value *V, VectorType *DstVTy, const DataLayout &DL)

Definition VPlanRecipes.cpp:3757

SmallVector< Value *, 2 > VectorParts

Definition VPlanRecipes.cpp:44

static bool isUsedByLoadStoreAddress(const VPUser *V)

Returns true if V is used as part of the address of another load or store.

Definition VPlanRecipes.cpp:3135

static void scalarizeInstruction(const Instruction *Instr, VPReplicateRecipe *RepRecipe, const VPLane &Lane, VPTransformState &State)

A helper function to scalarize a single Instruction in the innermost loop.

Definition VPlanRecipes.cpp:3014

static Constant * getSignedIntOrFpConstant(Type *Ty, int64_t C)

A helper function that returns an integer or floating-point constant with value C.

Definition VPlanRecipes.cpp:2305

static BranchInst * createCondBranch(Value *Cond, VPBasicBlock *VPBB, VPTransformState &State)

Create a conditional branch using Cond branching to the successors of VPBB.

Definition VPlanRecipes.cpp:517

static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)

Returns the opcode of Values or ~0 if they do not all agree.

This file contains the declarations of the Vectorization Plan base classes:

static const uint32_t IV[8]

void printAsOperand(OutputBuffer &OB, Prec P=Prec::Default, bool StrictlyWorse=false) const

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

bool empty() const

empty - Check if the array is empty.

static LLVM_ABI Attribute getWithAlignment(LLVMContext &Context, Align Alignment)

Return a uniquified Attribute object that has the specific alignment set.

LLVM Basic Block Representation.

LLVM_ABI const_iterator getFirstInsertionPt() const

Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...

LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const

Returns an iterator to the first instruction in this block that is not a PHINode instruction.

LLVM_ABI const BasicBlock * getSinglePredecessor() const

Return the predecessor of this block if it has a single predecessor block.

const Instruction * getTerminator() const LLVM_READONLY

Returns the terminator instruction if the block is well formed or null if the block is not well forme...

Conditional or Unconditional Branch instruction.

void setSuccessor(unsigned idx, BasicBlock *NewSucc)

void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)

Adds the attribute to the indicated argument.

This class represents a function call, abstracting a target machine's calling convention.

static LLVM_ABI bool isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy, const DataLayout &DL)

Check whether a bitcast, inttoptr, or ptrtoint cast between these types is valid and a no-op.

static Type * makeCmpResultType(Type *opnd_type)

Create a result type for fcmp/icmp.

Predicate

This enumeration lists the possible predicates for CmpInst subclasses.

@ ICMP_UGT

unsigned greater than

@ ICMP_ULT

unsigned less than

static LLVM_ABI StringRef getPredicateName(Predicate P)

An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...

This is the shared class of boolean and integer constants.

static ConstantInt * getSigned(IntegerType *Ty, int64_t V)

Return a ConstantInt with the specified value for the specified type.

uint64_t getZExtValue() const

Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...

This is an important base class in LLVM.

static LLVM_ABI Constant * getNullValue(Type *Ty)

Constructor to create a '0' constant of arbitrary type.

A parsed version of the target data layout string in and methods for querying it.

constexpr bool isVector() const

One or more elements.

static constexpr ElementCount getScalable(ScalarTy MinVal)

static constexpr ElementCount getFixed(ScalarTy MinVal)

constexpr bool isScalar() const

Exactly one element.

Convenience struct for specifying and reasoning about fast-math flags.

LLVM_ABI void print(raw_ostream &O) const

Print fast-math flags to O.

void setAllowContract(bool B=true)

bool noSignedZeros() const

void setAllowReciprocal(bool B=true)

bool allowReciprocal() const

void setNoSignedZeros(bool B=true)

bool allowReassoc() const

Flag queries.

void setNoNaNs(bool B=true)

void setAllowReassoc(bool B=true)

Flag setters.

void setApproxFunc(bool B=true)

void setNoInfs(bool B=true)

bool allowContract() const

Class to represent function types.

Type * getParamType(unsigned i) const

Parameter type accessors.

bool willReturn() const

Determine if the function will return.

bool doesNotThrow() const

Determine if the function cannot unwind.

Type * getReturnType() const

Returns the type of the ret val.

Common base class shared among various IRBuilders.

Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")

IntegerType * getInt1Ty()

Fetch the type representing a single bit.

Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")

Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")

LLVM_ABI Value * CreateVectorSplice(Value *V1, Value *V2, int64_t Imm, const Twine &Name="")

Return a vector splice intrinsic if using scalable vectors, otherwise return a shufflevector.

LLVM_ABI Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")

Return a vector value that contains.

Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")

LLVM_ABI Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)

Value * CreateFreeze(Value *V, const Twine &Name="")

IntegerType * getInt32Ty()

Fetch the type representing a 32-bit integer.

Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())

void setFastMathFlags(FastMathFlags NewFMF)

Set the fast-math flags to be used with generated fp-math operators.

IntegerType * getInt64Ty()

Fetch the type representing a 64-bit integer.

Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")

ConstantInt * getInt64(uint64_t C)

Get a constant 64-bit value.

LLVM_ABI CallInst * CreateOrReduce(Value *Src)

Create a vector int OR reduction intrinsic of the source vector.

Value * CreateLogicalAnd(Value *Cond1, Value *Cond2, const Twine &Name="", Instruction *MDFrom=nullptr)

LLVM_ABI CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")

Create a call to intrinsic ID with Args, mangled using Types.

ConstantInt * getInt32(uint32_t C)

Get a constant 32-bit value.

Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)

Value * CreateNot(Value *V, const Twine &Name="")

Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateCountTrailingZeroElems(Type *ResTy, Value *Mask, bool ZeroIsPoison=true, const Twine &Name="")

Create a call to llvm.experimental_cttz_elts.

Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)

Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

ConstantInt * getFalse()

Get the constant value for i1 false.

Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)

Value * CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="", bool IsDisjoint=false)

Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

static InstructionCost getInvalid(CostType Val=0)

LLVM_ABI InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

const char * getOpcodeName() const

unsigned getOpcode() const

Returns a member of one of the enums like Instruction::Add.

static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)

This static method is the primary way of constructing an IntegerType.

The group of interleaved loads/stores sharing the same stride and close to each other.

uint32_t getFactor() const

InstTy * getMember(uint32_t Index) const

Get the member with the given index Index.

InstTy * getInsertPos() const

void addMetadata(InstTy *NewInst) const

Add metadata (e.g.

This is an important class for using LLVM in a threaded context.

Represents a single loop in the control flow graph.

Information for memory intrinsic cost model.

A Module instance is used to store all the information related to an LLVM module.

void addIncoming(Value *V, BasicBlock *BB)

Add an incoming value to the end of the PHI list.

static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)

Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...

static LLVM_ABI PoisonValue * get(Type *T)

Static factory methods - Return an 'poison' object of the specified type.

static bool isSignedRecurrenceKind(RecurKind Kind)

Returns true if recurrece kind is a signed redux kind.

static LLVM_ABI unsigned getOpcode(RecurKind Kind)

Returns the opcode corresponding to the RecurrenceKind.

unsigned getOpcode() const

static bool isAnyOfRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...

static bool isFindLastIVRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...

static bool isFindIVRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...

static bool isMinMaxRecurrenceKind(RecurKind Kind)

Returns true if the recurrence kind is any min/max kind.

This class represents an analyzed expression in the program.

The main scalar evolution driver.

This class represents the LLVM 'select' instruction.

This class provides computation of slot numbers for LLVM Assembly writing.

std::pair< iterator, bool > insert(PtrType Ptr)

Inserts Ptr if and only if there is no element in the container equal to Ptr.

SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.

reference emplace_back(ArgTypes &&... Args)

void append(ItTy in_start, ItTy in_end)

Add the specified range to the end of the SmallVector.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

static LLVM_ABI PartialReductionExtendKind getPartialReductionExtendKind(Instruction *I)

Get the kind of extension that an instruction represents.

static LLVM_ABI OperandValueInfo getOperandInfo(const Value *V)

Collect properties of V used in cost analysis, e.g. OP_PowerOf2.

@ TCC_Free

Expected to fold away in lowering.

@ SK_Splice

Concatenates elements from the first input vector with elements of the second input vector.

@ SK_Reverse

Reverse the order of the vector.

CastContextHint

Represents a hint about the context in which a cast is used.

@ Reversed

The cast is used with a reversed load/store.

@ Masked

The cast is used with a masked load/store.

@ None

The cast is not used with a load/store of any kind.

@ Normal

The cast is used with a normal load/store.

@ Interleave

The cast is used with an interleaved load/store.

@ GatherScatter

The cast is used with a gather/scatter.

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

The instances of the Type class are immutable: once they are created, they are never changed.

static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)

bool isVectorTy() const

True if this is an instance of VectorType.

static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)

bool isPointerTy() const

True if this is an instance of PointerType.

static LLVM_ABI Type * getVoidTy(LLVMContext &C)

Type * getScalarType() const

If this is a vector type, return the element type, otherwise return 'this'.

bool isStructTy() const

True if this is an instance of StructType.

LLVMContext & getContext() const

Return the LLVMContext in which this type was uniqued.

LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY

If this is a vector type, return the getPrimitiveSizeInBits value for the element type.

static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)

bool isFloatingPointTy() const

Return true if this is one of the floating-point types.

bool isIntegerTy() const

True if this is an instance of IntegerType.

static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)

bool isVoidTy() const

Return true if this is 'void'.

value_op_iterator value_op_end()

void setOperand(unsigned i, Value *Val)

Value * getOperand(unsigned i) const

value_op_iterator value_op_begin()

void execute(VPTransformState &State) override

Generate the active lane mask phi of the vector loop.

Definition VPlanRecipes.cpp:4406

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4417

VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.

RecipeListTy & getRecipeList()

Returns a reference to the list of recipes.

void insert(VPRecipeBase *Recipe, iterator InsertPt)

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenMemoryRecipe.

Definition VPlanRecipes.cpp:2553

VPValue * getIncomingValue(unsigned Idx) const

Return incoming value number Idx.

unsigned getNumIncomingValues() const

Return the number of incoming values, taking into account when normalized the first incoming value wi...

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2563

VPBlockBase is the building block of the Hierarchical Control-Flow Graph.

const VPBlocksTy & getPredecessors() const

void printAsOperand(raw_ostream &OS, bool PrintType=false) const

const VPBlocksTy & getSuccessors() const

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPBranchOnMaskRecipe.

Definition VPlanRecipes.cpp:3392

void execute(VPTransformState &State) override

Generate the extraction of the appropriate bit from the block mask and the conditional branch.

Definition VPlanRecipes.cpp:3375

VPlan-based builder utility analogous to IRBuilder.

LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4231

This class augments a recipe with a set of VPValues defined by the recipe.

LLVM_ABI_FOR_TEST void dump() const

Dump the VPDef to stderr (for debugging).

unsigned getNumDefinedValues() const

Returns the number of values defined by the VPDef.

ArrayRef< VPValue * > definedValues()

Returns an ArrayRef of the values defined by the VPDef.

VPValue * getVPSingleValue()

Returns the only VPValue defined by the VPDef.

VPValue * getVPValue(unsigned I)

Returns the VPValue with index I defined by the VPDef.

unsigned getVPDefID() const

VPValue * getStepValue() const

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2338

VPValue * getStartValue() const

LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4428

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4266

void decompose()

Insert the recipes of the expression back into the VPlan, directly before the current recipe.

Definition VPlanRecipes.cpp:2780

bool isSingleScalar() const

Returns true if the result of this VPExpressionRecipe is a single-scalar.

Definition VPlanRecipes.cpp:2867

bool mayHaveSideEffects() const

Returns true if this expression contains recipes that may have side effects.

Definition VPlanRecipes.cpp:2859

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Compute the cost of this recipe either using a recipe's specialized implementation or using the legac...

Definition VPlanRecipes.cpp:2794

bool mayReadOrWriteMemory() const

Returns true if this expression contains recipes that may read from or write to memory.

Definition VPlanRecipes.cpp:2853

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2876

void execute(VPTransformState &State) override

Produce a vectorized histogram operation.

Definition VPlanRecipes.cpp:1822

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPHistogramRecipe.

Definition VPlanRecipes.cpp:1851

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:1888

VPValue * getMask() const

Return the mask operand if one was provided, or a null pointer if all lanes should be executed uncond...

Class to record and manage LLVM IR flags.

LLVM_ABI_FOR_TEST bool flagsValidForOpcode(unsigned Opcode) const

Returns true if the set flags are valid for Opcode.

Definition VPlanRecipes.cpp:1986

CmpInst::Predicate CmpPredicate

void printFlags(raw_ostream &O) const

Definition VPlanRecipes.cpp:2026

bool hasFastMathFlags() const

Returns true if the recipe has fast-math flags.

LLVM_ABI_FOR_TEST FastMathFlags getFastMathFlags() const

Definition VPlanRecipes.cpp:355

CmpInst::Predicate getPredicate() const

bool hasNoSignedWrap() const

void intersectFlags(const VPIRFlags &Other)

Only keep flags also present in Other.

Definition VPlanRecipes.cpp:315

GEPNoWrapFlags getGEPNoWrapFlags() const

bool hasPredicate() const

Returns true if the recipe has a comparison predicate.

DisjointFlagsTy DisjointFlags

bool hasNoUnsignedWrap() const

NonNegFlagsTy NonNegFlags

void applyFlags(Instruction &I) const

Apply the IR flags to I.

Instruction & getInstruction() const

void extractLastLaneOfLastPartOfFirstOperand(VPBuilder &Builder)

Update the recipe's first operand to the last lane of the last part of the operand using Builder.

Definition VPlanRecipes.cpp:1500

void execute(VPTransformState &State) override

The method which generates the output IR instructions that correspond to this VPRecipe,...

Definition VPlanRecipes.cpp:1485

LLVM_ABI_FOR_TEST InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPIRInstruction.

Definition VPlanRecipes.cpp:1493

VPIRInstruction(Instruction &I)

VPIRInstruction::create() should be used to create VPIRInstructions, as subclasses may need to be cre...

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:1515

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:1425

void execute(VPTransformState &State) override

Generate the instruction.

Definition VPlanRecipes.cpp:1397

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPInstruction.

Definition VPlanRecipes.cpp:994

bool doesGeneratePerAllLanes() const

Returns true if this VPInstruction generates scalar values for all lanes.

Definition VPlanRecipes.cpp:485

@ ExtractLane

Extracts a single lane (first operand) from a set of vector operands.

@ ComputeAnyOfResult

Compute the final result of a AnyOf reduction with select(cmp(),x,y), where one of (x,...

@ WideIVStep

Scale the first operand (vector step) by the second operand (scalar-step).

@ ExtractPenultimateElement

@ ResumeForEpilogue

Explicit user for the resume phi of the canonical induction in the main VPlan, used by the epilogue v...

@ Unpack

Extracts all lanes from its (non-scalable) vector operand.

@ FirstOrderRecurrenceSplice

@ ReductionStartVector

Start vector for reductions with 3 operands: the original start value, the identity value for the red...

@ BuildVector

Creates a fixed-width vector containing all operands.

@ BuildStructVector

Given operands of (the same) struct type, creates a struct of fixed- width vectors each containing a ...

@ VScale

Returns the value for vscale.

@ CanonicalIVIncrementForPart

@ CalculateTripCountMinusVF

bool opcodeMayReadOrWriteFromMemory() const

Returns true if the underlying opcode may read from or write to memory.

Definition VPlanRecipes.cpp:1173

LLVM_DUMP_METHOD void dump() const

Print the VPInstruction to dbgs() (for debugging).

Definition VPlanRecipes.cpp:1286

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the VPInstruction to O.

Definition VPlanRecipes.cpp:1291

StringRef getName() const

Returns the symbolic name assigned to the VPInstruction.

unsigned getOpcode() const

VPInstruction(unsigned Opcode, ArrayRef< VPValue * > Operands, const VPIRFlags &Flags={}, const VPIRMetadata &MD={}, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="")

Definition VPlanRecipes.cpp:407

bool usesFirstLaneOnly(const VPValue *Op) const override

Returns true if the recipe only uses the first lane of operand Op.

Definition VPlanRecipes.cpp:1216

bool isVectorToScalar() const

Returns true if this VPInstruction produces a scalar value from a vector, e.g.

Definition VPlanRecipes.cpp:1125

bool isSingleScalar() const

Returns true if this VPInstruction's operands are single scalars and the result is also a single scal...

Definition VPlanRecipes.cpp:1138

void execute(VPTransformState &State) override

Generate the instruction.

Definition VPlanRecipes.cpp:1150

bool usesFirstPartOnly(const VPValue *Op) const override

Returns true if the recipe only uses the first part of operand Op.

Definition VPlanRecipes.cpp:1265

bool needsMaskForGaps() const

Return true if the access needs a mask because of the gaps.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this recipe.

Definition VPlanRecipes.cpp:4189

Instruction * getInsertPos() const

const InterleaveGroup< Instruction > * getInterleaveGroup() const

VPValue * getMask() const

Return the mask used by this recipe.

ArrayRef< VPValue * > getStoredValues() const

Return the VPValues stored by this interleave group.

VPValue * getAddr() const

Return the address accessed by this recipe.

VPValue * getEVL() const

The VPValue of the explicit vector length.

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4157

unsigned getNumStoreOperands() const override

Returns the number of stored operands of this interleave group.

void execute(VPTransformState &State) override

Generate the wide load or store, and shuffles.

Definition VPlanRecipes.cpp:4045

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4014

unsigned getNumStoreOperands() const override

Returns the number of stored operands of this interleave group.

void execute(VPTransformState &State) override

Generate the wide load or store, and shuffles.

Definition VPlanRecipes.cpp:3844

In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...

static VPLane getLastLaneForVF(const ElementCount &VF)

static VPLane getLaneFromEnd(const ElementCount &VF, unsigned Offset)

static VPLane getFirstLane()

virtual const VPRecipeBase * getAsRecipe() const =0

Return a VPRecipeBase* to the current object.

virtual unsigned getNumIncoming() const

Returns the number of incoming values, also number of incoming blocks.

void removeIncomingValueFor(VPBlockBase *IncomingBlock) const

Removes the incoming value for IncomingBlock, which must be a predecessor.

Definition VPlanRecipes.cpp:1548

const VPBasicBlock * getIncomingBlock(unsigned Idx) const

Returns the incoming block with index Idx.

detail::zippy< llvm::detail::zip_first, VPUser::const_operand_range, const_incoming_blocks_range > incoming_values_and_blocks() const

Returns an iterator range over pairs of incoming values and corresponding incoming blocks.

VPValue * getIncomingValue(unsigned Idx) const

Returns the incoming VPValue with index Idx.

void printPhiOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const

Print the recipe.

Definition VPlanRecipes.cpp:1557

void execute(VPTransformState &State) override

Generates phi nodes for live-outs (from a replicate region) as needed to retain SSA form.

Definition VPlanRecipes.cpp:3400

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:3460

VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.

bool mayReadFromMemory() const

Returns true if the recipe may read from memory.

Definition VPlanRecipes.cpp:109

bool mayHaveSideEffects() const

Returns true if the recipe may have side-effects.

Definition VPlanRecipes.cpp:160

virtual void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const =0

Each concrete VPRecipe prints itself, without printing common information, like debug info or metadat...

VPRegionBlock * getRegion()

void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override final

Print the recipe, delegating to printRecipe().

Definition VPlanRecipes.cpp:373

bool isPhi() const

Returns true for PHI-like recipes.

Definition VPlanRecipes.cpp:305

bool mayWriteToMemory() const

Returns true if the recipe may write to memory.

Definition VPlanRecipes.cpp:49

virtual InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const

Compute the cost of this recipe either using a recipe's specialized implementation or using the legac...

Definition VPlanRecipes.cpp:300

VPBasicBlock * getParent()

DebugLoc getDebugLoc() const

Returns the debug location of the recipe.

void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)

Unlink this recipe and insert into BB before I.

Definition VPlanRecipes.cpp:260

void insertBefore(VPRecipeBase *InsertPos)

Insert an unlinked recipe into a basic block immediately before the specified recipe.

Definition VPlanRecipes.cpp:223

void insertAfter(VPRecipeBase *InsertPos)

Insert an unlinked Recipe into a basic block immediately after the specified Recipe.

Definition VPlanRecipes.cpp:237

iplist< VPRecipeBase >::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

Definition VPlanRecipes.cpp:250

InstructionCost cost(ElementCount VF, VPCostContext &Ctx)

Return the cost of this recipe, taking into account if the cost computation should be skipped and the...

Definition VPlanRecipes.cpp:266

bool isScalarCast() const

Return true if the recipe is a scalar cast.

Definition VPlanRecipes.cpp:310

void removeFromParent()

This method unlinks 'this' from the containing basic block, but does not delete it.

Definition VPlanRecipes.cpp:244

void moveAfter(VPRecipeBase *MovePos)

Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...

Definition VPlanRecipes.cpp:255

VPRecipeBase(const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL=DebugLoc::getUnknown())

void execute(VPTransformState &State) override

Generate the reduction in the loop.

Definition VPlanRecipes.cpp:2643

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2987

VPValue * getEVL() const

The VPValue of the explicit vector length.

unsigned getVFScaleFactor() const

Get the factor that the VF of this recipe's output should be scaled by, or 1 if it isn't scaled.

bool isInLoop() const

Returns true if the phi is part of an in-loop reduction.

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4376

void execute(VPTransformState &State) override

Generate the phi/select nodes.

Definition VPlanRecipes.cpp:4350

bool isConditional() const

Return true if the in-loop reduction is conditional.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of VPReductionRecipe.

Definition VPlanRecipes.cpp:2677

VPValue * getVecOp() const

The VPValue of the vector value to be reduced.

VPValue * getCondOp() const

The VPValue of the condition for the block.

RecurKind getRecurrenceKind() const

Return the recurrence kind for the in-loop reduction.

bool isPartialReduction() const

Returns true if the reduction outputs a vector with a scaled down VF.

VPValue * getChainOp() const

The VPValue of the scalar Chain being accumulated.

bool isInLoop() const

Returns true if the reduction is in-loop.

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2964

void execute(VPTransformState &State) override

Generate the reduction in the loop.

Definition VPlanRecipes.cpp:2586

VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...

bool isReplicator() const

An indicator whether this region is to generate multiple replicated instances of output IR correspond...

VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...

void execute(VPTransformState &State) override

Generate replicas of the desired Ingredient.

Definition VPlanRecipes.cpp:3072

bool isSingleScalar() const

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPReplicateRecipe.

Definition VPlanRecipes.cpp:3183

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:3347

unsigned getOpcode() const

bool shouldPack() const

Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.

Definition VPlanRecipes.cpp:3097

VPValue * getStepValue() const

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2424

void execute(VPTransformState &State) override

Generate the scalarized versions of the phi node as needed by their users.

Definition VPlanRecipes.cpp:2351

VPSingleDef is a base class for recipes for modeling a sequence of one or more output IR that define ...

Instruction * getUnderlyingInstr()

Returns the underlying instruction.

LLVM_ABI_FOR_TEST LLVM_DUMP_METHOD void dump() const

Print this VPSingleDefRecipe to dbgs() (for debugging).

Definition VPlanRecipes.cpp:371

VPSingleDefRecipe(const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL=DebugLoc::getUnknown())

This class can be used to assign names to VPValues.

Type * inferScalarType(const VPValue *V)

Infer the type of V. Returns the scalar type of V.

Helper to access the operand that contains the unroll part for this recipe after unrolling.

VPValue * getUnrollPartOperand(const VPUser &U) const

Return the VPValue operand containing the unroll part or null if there is no such operand.

Definition VPlanRecipes.cpp:388

unsigned getUnrollPart(const VPUser &U) const

Return the unroll part.

Definition VPlanRecipes.cpp:395

This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...

void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const

Print the operands to O.

void setOperand(unsigned I, VPValue *New)

unsigned getNumOperands() const

operand_iterator op_begin()

VPValue * getOperand(unsigned N) const

virtual bool usesFirstLaneOnly(const VPValue *Op) const

Returns true if the VPUser only uses the first lane of operand Op.

This is the base class of the VPlan Def/Use graph, used for modeling the data flow into,...

bool isDefinedOutsideLoopRegions() const

Returns true if the VPValue is defined outside any loop.

VPRecipeBase * getDefiningRecipe()

Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...

friend class VPExpressionRecipe

void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const

bool hasMoreThanOneUniqueUser() const

Returns true if the value has more than one unique user.

Value * getLiveInIRValue() const

Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.

Value * getUnderlyingValue() const

Return the underlying Value attached to this VPValue.

VPValue(const unsigned char SC, Value *UV=nullptr, VPDef *Def=nullptr)

void replaceAllUsesWith(VPValue *New)

user_iterator user_begin()

unsigned getNumUsers() const

bool isLiveIn() const

Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.

void execute(VPTransformState &State) override

The method which generates the output IR instructions that correspond to this VPRecipe,...

Definition VPlanRecipes.cpp:2490

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2518

Type * getSourceElementType() const

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2543

void execute(VPTransformState &State) override

The method which generates the output IR instructions that correspond to this VPRecipe,...

Definition VPlanRecipes.cpp:2528

Function * getCalledScalarFunction() const

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenCallRecipe.

Definition VPlanRecipes.cpp:1658

void execute(VPTransformState &State) override

Produce a widened version of the call instruction.

Definition VPlanRecipes.cpp:1625

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:1666

void execute(VPTransformState &State) override

Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...

Definition VPlanRecipes.cpp:4274

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4293

LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2287

Type * getResultType() const

Returns the result type of the cast.

LLVM_ABI_FOR_TEST void execute(VPTransformState &State) override

Produce widened copies of the cast.

Definition VPlanRecipes.cpp:2217

LLVM_ABI_FOR_TEST InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenCastRecipe.

Definition VPlanRecipes.cpp:2232

void execute(VPTransformState &State) override

Generate the gep nodes.

Definition VPlanRecipes.cpp:2438

Type * getSourceElementType() const

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2475

bool usesFirstLaneOnly(const VPValue *Op) const override

Returns true if the recipe only uses the first lane of operand Op.

Definition VPlanRecipes.cpp:2433

VPValue * getStepValue()

Returns the step value of the induction.

TruncInst * getTruncInst()

Returns the first defined value as TruncInst, if it is one or nullptr otherwise.

Type * getScalarType() const

Returns the scalar type of the induction.

bool isCanonical() const

Returns true if the induction is canonical, i.e.

Definition VPlanRecipes.cpp:2325

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2311

Intrinsic::ID getVectorIntrinsicID() const

Return the ID of the intrinsic.

LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:1801

StringRef getIntrinsicName() const

Return to name of the intrinsic as string.

Definition VPlanRecipes.cpp:1787

LLVM_ABI_FOR_TEST bool usesFirstLaneOnly(const VPValue *Op) const override

Returns true if the VPUser only uses the first lane of operand Op.

Definition VPlanRecipes.cpp:1791

Type * getResultType() const

Return the scalar return type of the intrinsic.

LLVM_ABI_FOR_TEST void execute(VPTransformState &State) override

Produce a widened version of the vector intrinsic.

Definition VPlanRecipes.cpp:1693

LLVM_ABI_FOR_TEST InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this vector intrinsic.

Definition VPlanRecipes.cpp:1781

bool IsMasked

Whether the memory access is masked.

bool Reverse

Whether the consecutive accessed addresses are in reverse order.

bool isConsecutive() const

Return whether the loaded-from / stored-to addresses are consecutive.

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenMemoryRecipe.

Definition VPlanRecipes.cpp:3469

bool Consecutive

Whether the accessed addresses are consecutive.

VPValue * getMask() const

Return the mask used by this recipe.

Align Alignment

Alignment information for this memory access.

VPValue * getAddr() const

Return the address accessed by this recipe.

bool isReverse() const

Return whether the consecutive loaded/stored addresses are in reverse order.

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4396

void execute(VPTransformState &State) override

Generate the phi/select nodes.

Definition VPlanRecipes.cpp:4388

bool onlyScalarsGenerated(bool IsScalable)

Returns true if only scalar values will be generated.

Definition VPlanRecipes.cpp:4240

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4246

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenRecipe.

Definition VPlanRecipes.cpp:2170

void execute(VPTransformState &State) override

Produce a widened instruction using the opcode and operands of the recipe, processing State....

Definition VPlanRecipes.cpp:2077

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:2207

LLVM_ABI_FOR_TEST VPRegionBlock * getVectorLoopRegion()

Returns the VPRegionBlock of the vector loop.

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

LLVM_ABI void setName(const Twine &Name)

Change the name of the value.

LLVM_ABI LLVMContext & getContext() const

All values hold a context through their type.

void mutateType(Type *Ty)

Mutate the type of this Value to be of the specified type.

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

Base class of all SIMD vector types.

ElementCount getElementCount() const

Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...

static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)

This static method is the primary way to construct an VectorType.

Type * getElementType() const

constexpr ScalarTy getFixedValue() const

constexpr bool isScalable() const

Returns whether the quantity is scaled by a runtime quantity (vscale).

constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const

constexpr ScalarTy getKnownMinValue() const

Returns the minimum value this quantity can represent.

constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const

We do not provide the '/' operator here because division for polynomial types does not work in the sa...

const ParentTy * getParent() const

self_iterator getIterator()

typename base_list_type::iterator iterator

iterator erase(iterator where)

pointer remove(iterator &IT)

This class implements an extremely fast bulk output stream that can only output to a stream.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

constexpr std::underlying_type_t< E > Mask()

Get a bitmask with 1s in all places up to the high-order bit of E's largest value.

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

@ C

The default llvm calling convention, compatible with C.

LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})

Look up the Function declaration of the intrinsic id in the Module M.

LLVM_ABI Intrinsic::ID getDeinterleaveIntrinsicID(unsigned Factor)

Returns the corresponding llvm.vector.deinterleaveN intrinsic for factor N.

LLVM_ABI StringRef getBaseName(ID id)

Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....

bool match(Val *V, const Pattern &P)

auto m_LogicalOr()

Matches L || R where L and R are arbitrary values.

class_match< CmpInst > m_Cmp()

Matches any compare instruction and ignore it.

auto m_LogicalAnd()

Matches L && R where L and R are arbitrary values.

GEPLikeRecipe_match< Op0_t, Op1_t > m_GetElementPtr(const Op0_t &Op0, const Op1_t &Op1)

class_match< VPValue > m_VPValue()

Match an arbitrary VPValue and ignore it.

NodeAddr< DefNode * > Def

bool isSingleScalar(const VPValue *VPV)

Returns true if VPV is a single scalar, either because it produces the same value for all lanes or on...

bool onlyFirstPartUsed(const VPValue *Def)

Returns true if only the first part of Def is used.

bool onlyFirstLaneUsed(const VPValue *Def)

Returns true if only the first lane of Def is used.

bool onlyScalarValuesUsed(const VPValue *Def)

Returns true if only scalar values of Def are used by all users.

const SCEV * getSCEVExprForVPValue(const VPValue *V, ScalarEvolution &SE, const Loop *L=nullptr)

Return the SCEV expression for V.

This is an optimization pass for GlobalISel generic memory operations.

auto drop_begin(T &&RangeOrContainer, size_t N=1)

Return a range covering RangeOrContainer with the first N elements excluded.

LLVM_ABI Value * createSimpleReduction(IRBuilderBase &B, Value *Src, RecurKind RdxKind)

Create a reduction of the given vector.

detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)

zip iterator for two or more iteratable types.

FunctionAddr VTableAddr Value

LLVM_ABI Value * createFindLastIVReduction(IRBuilderBase &B, Value *Src, RecurKind RdxKind, Value *Start, Value *Sentinel)

Create a reduction of the given vector Src for a reduction of the kind RecurKind::FindLastIV.

bool all_of(R &&range, UnaryPredicate P)

Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.

LLVM_ABI Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)

Returns the min/max intrinsic used when expanding a min/max reduction.

auto enumerate(FirstRange &&First, RestRanges &&...Rest)

Given two or more input ranges, returns a new range whose values are tuples (A, B,...

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

const Value * getLoadStorePointerOperand(const Value *V)

A helper function that returns the pointer operand of a load or store instruction.

Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)

Return the runtime value for VF.

auto dyn_cast_if_present(const Y &Val)

dyn_cast_if_present - Functionally identical to dyn_cast, except that a null (or none in the case ...

iterator_range< T > make_range(T x, T y)

Convenience function for iterating over sub-ranges.

void append_range(Container &C, Range &&R)

Wrapper function to append range R to container C.

void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)

auto cast_or_null(const Y &Val)

LLVM_ABI Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)

Concatenate a list of vectors.

Align getLoadStoreAlignment(const Value *I)

A helper function that returns the alignment of load or store instruction.

bool isa_and_nonnull(const Y &Val)

LLVM_ABI Value * createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left, Value *Right)

Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.

auto dyn_cast_or_null(const Y &Val)

bool any_of(R &&range, UnaryPredicate P)

Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.

LLVM_ABI Constant * createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, const InterleaveGroup< Instruction > &Group)

Create a mask that filters the members of an interleave group where there are gaps.

LLVM_ABI llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)

Create a stride shuffle mask.

auto reverse(ContainerTy &&C)

LLVM_ABI llvm::SmallVector< int, 16 > createReplicatedMask(unsigned ReplicationFactor, unsigned VF)

Create a mask with replicated elements.

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

bool none_of(R &&Range, UnaryPredicate P)

Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.

SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)

Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...

Type * toVectorizedTy(Type *Ty, ElementCount EC)

A helper for converting to vectorized types.

cl::opt< unsigned > ForceTargetInstructionCost

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

auto drop_end(T &&RangeOrContainer, size_t N=1)

Return a range covering RangeOrContainer with the last N elements excluded.

bool canVectorizeTy(Type *Ty)

Returns true if Ty is a valid vector element type, void, or an unpacked literal struct where all elem...

LLVM_ABI llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)

Create an interleave shuffle mask.

RecurKind

These are the kinds of recurrences that we support.

@ UMin

Unsigned integer min implemented in terms of select(cmp()).

@ Mul

Product of integers.

@ SMax

Signed integer max implemented in terms of select(cmp()).

@ SMin

Signed integer min implemented in terms of select(cmp()).

@ Sub

Subtraction of integers.

@ UMax

Unsigned integer max implemented in terms of select(cmp()).

LLVM_ABI bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx, const TargetTransformInfo *TTI)

Identifies if the vector form of the intrinsic has a scalar operand.

LLVM_ABI Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF)

Given information about an recurrence kind, return the identity for the @llvm.vector....

DWARFExpression::Operation Op

Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)

Return a value for Step multiplied by VF.

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

bool is_contained(R &&Range, const E &Element)

Returns true if Element is found in Range.

Type * getLoadStoreType(const Value *I)

A helper function that returns the type of a load or store instruction.

LLVM_ABI Value * createOrderedReduction(IRBuilderBase &B, RecurKind RdxKind, Value *Src, Value *Start)

Create an ordered reduction intrinsic using the given recurrence kind RdxKind.

auto seq(T Begin, T End)

Iterate over an integral type from Begin up to - but not including - End.

@ Increment

Incrementally increasing token ID.

Type * toVectorTy(Type *Scalar, ElementCount EC)

A helper function for converting Scalar types to vector types.

LLVM_ABI Value * createAnyOfReduction(IRBuilderBase &B, Value *Src, Value *InitVal, PHINode *OrigPhi)

Create a reduction of the given vector Src for a reduction of kind RecurKind::AnyOf.

LLVM_ABI bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx, const TargetTransformInfo *TTI)

Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...

This struct is a compact representation of a valid (non-zero power of two) alignment.

Struct to hold various analysis needed for cost computations.

void execute(VPTransformState &State) override

Generate the phi nodes.

Definition VPlanRecipes.cpp:4302

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this first-order recurrence phi recipe.

Definition VPlanRecipes.cpp:4332

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:4341

An overlay for VPIRInstructions wrapping PHI nodes enabling convenient use cast/dyn_cast/isa and exec...

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:1571

void execute(VPTransformState &State) override

The method which generates the output IR instructions that correspond to this VPRecipe,...

Definition VPlanRecipes.cpp:1521

void execute(VPTransformState &State) override

Generate the instruction.

Definition VPlanRecipes.cpp:1451

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:1470

A pure-virtual common base class for recipes defining a single VPValue and using IR flags.

InstructionCost getCostForRecipeWithOpcode(unsigned Opcode, ElementCount VF, VPCostContext &Ctx) const

Compute the cost for this recipe for VF, using Opcode and Ctx.

Definition VPlanRecipes.cpp:925

VPRecipeWithIRFlags(const unsigned char SC, ArrayRef< VPValue * > Operands, const VPIRFlags &Flags, DebugLoc DL=DebugLoc::getUnknown())

VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...

VPTypeAnalysis TypeAnalysis

VPlan-based type analysis.

Value * get(const VPValue *Def, bool IsScalar=false)

Get the generated vector Value for a given VPValue Def if IsScalar is false, otherwise return the gen...

IRBuilderBase & Builder

Hold a reference to the IRBuilder used to generate output IR code.

ElementCount VF

The chosen Vectorization Factor of the loop being vectorized.

LLVM_ABI_FOR_TEST void execute(VPTransformState &State) override

Generate the wide load or gather.

Definition VPlanRecipes.cpp:3580

LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:3640

LLVM_ABI_FOR_TEST InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenLoadEVLRecipe.

Definition VPlanRecipes.cpp:3615

VPValue * getEVL() const

Return the EVL operand.

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:3560

void execute(VPTransformState &State) override

Generate a wide load or gather.

Definition VPlanRecipes.cpp:3526

VPValue * getCond() const

InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenSelectRecipe.

Definition VPlanRecipes.cpp:1937

void execute(VPTransformState &State) override

Produce a widened version of the select instruction.

Definition VPlanRecipes.cpp:1923

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:1907

VPValue * getStoredValue() const

Return the address accessed by this recipe.

LLVM_ABI_FOR_TEST void execute(VPTransformState &State) override

Generate the wide store or scatter.

Definition VPlanRecipes.cpp:3691

LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:3750

LLVM_ABI_FOR_TEST InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override

Return the cost of this VPWidenStoreEVLRecipe.

Definition VPlanRecipes.cpp:3725

VPValue * getEVL() const

Return the EVL operand.

void execute(VPTransformState &State) override

Generate a wide store or scatter.

Definition VPlanRecipes.cpp:3649

void printRecipe(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override

Print the recipe.

Definition VPlanRecipes.cpp:3684

VPValue * getStoredValue() const

Return the value stored by this recipe.