VVPISelLowering.cpp Source File (original) (raw)

18using namespace llvm;

20#define DEBUG_TYPE "ve-lower"

26 CDAG.getConstant(Op.getValueType().getVectorNumElements(), MVT::i32);

33 unsigned Opc = Op.getOpcode();

34 auto LoRes = CDAG.getNode(Opc, MVT::v256i1, {LoA, LoB});

35 auto HiRes = CDAG.getNode(Opc, MVT::v256i1, {HiA, HiB});

36 return CDAG.getPack(MVT::v512i1, LoRes, HiRes, AVL);

37}

41 const unsigned Opcode = Op->getOpcode();

43 if (!VVPOpcodeOpt)

45 unsigned VVPOpcode = *VVPOpcodeOpt;

51 switch (VVPOpcode) {

52 case VEISD::VVP_LOAD:

53 case VEISD::VVP_STORE:

55 case VEISD::VVP_GATHER:

56 case VEISD::VVP_SCATTER:

58 }

67 if (FromVP) {

71 if (MaskIdx)

72 Mask = Op->getOperand(*MaskIdx);

73 if (AVLIdx)

74 AVL = Op->getOperand(*AVLIdx);

75 }

78 if (!AVL)

80 if (!Mask)

85 return CDAG.getNode(VVPOpcode, LegalVecVT, {Op->getOperand(0), Mask, AVL});

87 return CDAG.getNode(VVPOpcode, LegalVecVT,

88 {Op->getOperand(0), Op->getOperand(1), Mask, AVL});

92 SDValue VectorV = Op->getOperand(SrcHasStart ? 1 : 0);

94 VectorV, Mask, AVL, Op->getFlags());

95 }

97 switch (VVPOpcode) {

98 default:

100 case VEISD::VVP_FFMA: {

101

102

103 auto X = Op->getOperand(2);

104 auto Y = Op->getOperand(0);

105 auto Z = Op->getOperand(1);

106 return CDAG.getNode(VVPOpcode, LegalVecVT, {X, Y, Z, Mask, AVL});

107 }

108 case VEISD::VVP_SELECT: {

109 auto Mask = Op->getOperand(0);

110 auto OnTrue = Op->getOperand(1);

111 auto OnFalse = Op->getOperand(2);

112 return CDAG.getNode(VVPOpcode, LegalVecVT, {OnTrue, OnFalse, Mask, AVL});

113 }

114 case VEISD::VVP_SETCC: {

116 auto LHS = Op->getOperand(0);

117 auto RHS = Op->getOperand(1);

118 auto Pred = Op->getOperand(2);

119 return CDAG.getNode(VVPOpcode, LegalResVT, {LHS, RHS, Pred, Mask, AVL});

120 }

121 }

122}

123

127 const bool IsLoad = (VVPOpc == VEISD::VVP_LOAD);

128

129

134

136

138

140

143

144

145 if (!AVL)

146 AVL = CDAG.getConstant(DataVT.getVectorNumElements(), MVT::i32);

147

148

149 if (!Mask)

151

152 if (IsLoad) {

154 Packing, DataVT.getVectorElementType().getSimpleVT());

155

156 auto NewLoadV = CDAG.getNode(VEISD::VVP_LOAD, {LegalDataVT, MVT::Other},

157 {Chain, BasePtr, StrideV, Mask, AVL});

158

159 if (!PassThru || PassThru->isUndef())

160 return NewLoadV;

161

162

163 SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, DataVT,

164 {NewLoadV, PassThru, Mask, AVL});

165 SDValue NewLoadChainV = SDValue(NewLoadV.getNode(), 1);

166

167

169 }

170

171

172 assert(VVPOpc == VEISD::VVP_STORE);

175

176

177

179 return CDAG.getNode(VEISD::VVP_STORE, Op.getNode()->getVTList(),

180 {Chain, Data, BasePtr, StrideV, Mask, AVL});

181}

182

186 assert((VVPOC == VEISD::VVP_LOAD) || (VVPOC == VEISD::VVP_STORE));

187

190 "Can only split packed load/store");

192

194 "Should have been folded in lowering to VVP layer");

195

196

202

203 unsigned ChainResIdx = PackData ? 0 : 1;

204

206

207 SDValue UpperPartAVL;

209

210

212

213

215 UpperPartAVL = SplitTM.AVL;

216

217

219

220

222

223

224 if (PackData) {

226 CDAG.getUnpack(SplitDataVT, PackData, Part, SplitTM.AVL);

228 }

229

230

231

232

233

236

237

240

241 if (PackData) {

242

243 PartOps[(int)Part] = CDAG.getNode(VVPOC, MVT::Other, OpVec);

244 } else {

245

246 PartOps[(int)Part] =

247 CDAG.getNode(VVPOC, {SplitDataVT, MVT::Other}, OpVec);

248 }

249 }

250

251

256

257

258 if (PackData)

259 return FusedChains;

260

261

262 MVT PackedVT =

266

267 return CDAG.getMergeValues({PackedVals, FusedChains});

268}

269

274 MVT LegalDataVT =

276

285 if (PassThru && PassThru->isUndef())

287

288 bool IsScatter = (bool)StoredValue;

289

290

291 if (!AVL)

293

294

295 if (!Mask)

297

300 if (IsScatter)

301 return CDAG.getNode(VEISD::VVP_SCATTER, MVT::Other,

302 {Chain, StoredValue, AddressVec, Mask, AVL});

303

304

305 SDValue NewLoadV = CDAG.getNode(VEISD::VVP_GATHER, {LegalDataVT, MVT::Other},

306 {Chain, AddressVec, Mask, AVL});

307

308 if (!PassThru)

309 return NewLoadV;

310

311

312 SDValue DataV = CDAG.getNode(VEISD::VVP_SELECT, LegalDataVT,

313 {NewLoadV, PassThru, Mask, AVL});

316}

317

320 LLVM_DEBUG(dbgs() << "::legalizeInternalLoadStoreOp\n";);

322

323

326

328}

329

332 LLVM_DEBUG(dbgs() << "::legalizeInternalVectorOp\n";);

334

335

336 switch (Op->getOpcode()) {

337 case VEISD::VVP_LOAD:

338 case VEISD::VVP_STORE:

340 }

341

342 EVT IdiomVT = Op.getValueType();

346

347

349}

350

353

354 auto AVLPos = getAVLPos(Op->getOpcode());

356

359 SDValue PackedAVL = AVLPair.first;

360 assert(!AVLPair.second && "Expecting non pack-legalized oepration");

361

362

364

365 SDValue UpperPartAVL;

367

368

370

372 UpperPartAVL = SplitTM.AVL;

373

374

376 for (unsigned i = 0; i < Op.getNumOperands(); ++i) {

377 if (AVLPos && ((int)i) == *AVLPos)

378 continue;

379 if (MaskPos && ((int)i) == *MaskPos)

380 continue;

381

382

383 auto PackedOperand = Op.getOperand(i);

384 auto UnpackedOpVT = splitVectorType(PackedOperand.getSimpleValueType());

386 CDAG.getUnpack(UnpackedOpVT, PackedOperand, Part, SplitTM.AVL);

388 }

389

390

393

394 PartOps[(int)Part] =

395 CDAG.getNode(Op.getOpcode(), ResVT, OpVec, Op->getFlags());

396 }

397

398

401}

402

406

408 return Op;

409

410

413 return Op;

414

415

420 "TODO Shift predication from EVL into Mask");

421

423 LegalAVL = CDAG.getConstant((ConstAVL->getZExtValue() + 1) / 2, MVT::i32);

424 } else {

425 auto ConstOne = CDAG.getConstant(1, MVT::i32);

426 auto PlusOne = CDAG.getNode(ISD::ADD, MVT::i32, {AVL, ConstOne});

427 LegalAVL = CDAG.getNode(ISD::SRL, MVT::i32, {PlusOne, ConstOne});

428 }

429 }

430

432

433

434 int NumOp = Op->getNumOperands();

435 auto AVLPos = getAVLPos(Op->getOpcode());

436 std::vector FixedOperands;

437 for (int i = 0; i < NumOp; ++i) {

438 if (AVLPos && (i == *AVLPos)) {

439 FixedOperands.push_back(AnnotatedLegalAVL);

440 continue;

441 }

442 FixedOperands.push_back(Op->getOperand(i));

443 }

444

445

446 auto Flags = Op->getFlags();

448 CDAG.getNode(Op->getOpcode(), Op->getVTList(), FixedOperands, Flags);

449 return NewN;

450}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")

static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")

MVT getVectorElementType() const

bool isUndef() const

Returns true if the node type is UNDEF or POISON.

Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.

SDNode * getNode() const

get the SDNode which holds the desired result

This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...

LLVMContext * getContext() const

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const

For types supported by the target, this is an identity function.

LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const

Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...

SDValue getSplitPtrOffset(SDValue Ptr, SDValue ByteStride, PackElem Part) const

SDValue getMergeValues(ArrayRef< SDValue > Values) const

} Packing

SDValue getConstantMask(Packing Packing, bool AllTrue) const

SDValue getGatherScatterAddress(SDValue BasePtr, SDValue Scale, SDValue Index, SDValue Mask, SDValue AVL) const

SDValue getLegalReductionOpVVP(unsigned VVPOpcode, EVT ResVT, SDValue StartV, SDValue VectorV, SDValue Mask, SDValue AVL, SDNodeFlags Flags) const

} getNode

SDValue getNode(unsigned OC, SDVTList VTL, ArrayRef< SDValue > OpV, std::optional< SDNodeFlags > Flags=std::nullopt) const

getNode {

SDValue annotateLegalAVL(SDValue AVL) const

SDValue getUnpack(EVT DestVT, SDValue Vec, PackElem Part, SDValue AVL) const

} Legalizing getNode

SelectionDAG * getDAG() const

SDValue getPack(EVT DestVT, SDValue LoVec, SDValue HiVec, SDValue AVL) const

SDValue getConstant(uint64_t Val, EVT VT, bool IsTarget=false, bool IsOpaque=false) const

SDValue getSplitPtrStride(SDValue PackStride) const

VETargetMasks getTargetSplitMask(SDValue RawMask, SDValue RawAVL, PackElem Part) const

SDValue splitMaskArithmetic(SDValue Op, SelectionDAG &DAG) const

Definition VVPISelLowering.cpp:22

SDValue lowerVVP_GATHER_SCATTER(SDValue Op, VECustomDAG &) const

Definition VVPISelLowering.cpp:270

SDValue lowerToVVP(SDValue Op, SelectionDAG &DAG) const

} Custom Inserter

Definition VVPISelLowering.cpp:39

SDValue lowerVVP_LOAD_STORE(SDValue Op, VECustomDAG &) const

Definition VVPISelLowering.cpp:124

SDValue legalizePackedAVL(SDValue Op, VECustomDAG &CDAG) const

Definition VVPISelLowering.cpp:403

SDValue splitPackedLoadStore(SDValue Op, VECustomDAG &CDAG) const

Definition VVPISelLowering.cpp:183

SDValue legalizeInternalVectorOp(SDValue Op, SelectionDAG &DAG) const

Definition VVPISelLowering.cpp:330

SDValue splitVectorOp(SDValue Op, VECustomDAG &CDAG) const

Definition VVPISelLowering.cpp:351

SDValue legalizeInternalLoadStoreOp(SDValue Op, VECustomDAG &CDAG) const

Definition VVPISelLowering.cpp:318

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

@ ADD

Simple integer binary arithmetic operators.

@ TokenFactor

TokenFactor - This node takes multiple tokens as input and produces a single token result.

LLVM_ABI std::optional< unsigned > getVPMaskIdx(unsigned Opcode)

The operand position of the vector mask.

LLVM_ABI std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)

The operand position of the explicit vector length parameter.

LLVM_ABI bool isVPOpcode(unsigned Opcode)

Whether this is a vector-predicated Opcode.

This is an optimization pass for GlobalISel generic memory operations.

bool isVVPReductionOp(unsigned Opcode)

bool isPackedVectorType(EVT SomeVT)

bool supportsPackedMode(unsigned Opcode, EVT IdiomVT)

std::optional< int > getAVLPos(unsigned Opc)

The VE backend uses a two-staged process to lower and legalize vector instructions:

SDValue getGatherScatterScale(SDValue Op)

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

SDValue getStoredValue(SDValue Op)

bool isVVPBinaryOp(unsigned VVPOpcode)

std::optional< EVT > getIdiomaticVectorType(SDNode *Op)

} AVL Functions

SDValue getNodeChain(SDValue Op)

Node Properties {.

SDValue getNodeAVL(SDValue Op)

} Node Properties

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

bool isLegalAVL(SDValue AVL)

MVT splitVectorType(MVT VT)

SDValue getNodePassthru(SDValue Op)

FunctionAddr VTableAddr uintptr_t uintptr_t Data

bool maySafelyIgnoreMask(SDValue Op)

bool isVVPOrVEC(unsigned Opcode)

MVT getLegalVectorType(Packing P, MVT ElemVT)

SDValue getMemoryPtr(SDValue Op)

std::optional< int > getMaskPos(unsigned Opc)

std::pair< SDValue, bool > getAnnotatedNodeAVL(SDValue Op)

DWARFExpression::Operation Op

bool hasReductionStartParam(unsigned OPC)

SDValue getGatherScatterIndex(SDValue Op)

Packing getTypePacking(EVT VT)

std::optional< unsigned > getVVPOpcode(unsigned Opcode)

bool isVVPUnaryOp(unsigned VVPOpcode)

SDValue getNodeMask(SDValue Op)

SDValue getLoadStoreStride(SDValue Op, VECustomDAG &CDAG)

bool isSimple() const

Test if the given EVT is simple (as opposed to being extended).

MVT getSimpleVT() const

Return the SimpleValueType held in the specified simple EVT.

EVT getVectorElementType() const

Given a vector type, return the type of each element.

unsigned getVectorNumElements() const

Given a vector type, return the number of elements it contains.