VECustomDAG.cpp Source File (original) (raw)

17#ifndef DEBUG_TYPE

18#define DEBUG_TYPE "vecustomdag"

19#endif

21namespace llvm {

52 switch (Op.getOpcode()) {

53 default:

54 return false;

59 }

60}

63std::optional getVVPOpcode(unsigned Opcode) {

64 switch (Opcode) {

65 case ISD::MLOAD:

66 return VEISD::VVP_LOAD;

67 case ISD::MSTORE:

68 return VEISD::VVP_STORE;

69#define HANDLE_VP_TO_VVP(VPOPC, VVPNAME) \

70 case ISD::VPOPC: \

71 return VEISD::VVPNAME;

72#define ADD_VVP_OP(VVPNAME, SDNAME) \

73 case VEISD::VVPNAME: \

74 case ISD::SDNAME: \

75 return VEISD::VVPNAME;

76#include "VVPNodes.def"

78 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:

79 return VEISD::VVP_LOAD;

80 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:

81 return VEISD::VVP_STORE;

82 }

83 return std::nullopt;

84}

88 auto Opc = VVPOpc.value_or(Op->getOpcode());

90 switch (Opc) {

91 case VEISD::VVP_SDIV:

92 case VEISD::VVP_UDIV:

93 case VEISD::VVP_FDIV:

94 case VEISD::VVP_SELECT:

95 return false;

97 default:

98 return true;

99 }

100}

101

105 switch (Opcode) {

106 default:

107 return false;

108

109 case VEISD::VEC_BROADCAST:

110 return true;

111#define REGISTER_PACKED(VVP_NAME) case VEISD::VVP_NAME:

112#include "VVPNodes.def"

113 return IsPackedOp && !IsMaskOp;

114 }

115}

116

118 switch (Opc) {

119 case VEISD::VEC_PACK:

120 case VEISD::VEC_UNPACK_LO:

121 case VEISD::VEC_UNPACK_HI:

122 return true;

123 }

124 return false;

125}

126

128 switch (Opcode) {

129 case VEISD::VEC_BROADCAST:

130#define ADD_VVP_OP(VVPNAME, ...) case VEISD::VVPNAME:

131#include "VVPNodes.def"

132 return true;

133 }

134 return false;

135}

136

138 switch (VVPOpcode) {

139#define ADD_UNARY_VVP_OP(VVPNAME, ...) \

140 case VEISD::VVPNAME: \

141 return true;

142#include "VVPNodes.def"

143 }

144 return false;

145}

146

148 switch (VVPOpcode) {

149#define ADD_BINARY_VVP_OP(VVPNAME, ...) \

150 case VEISD::VVPNAME: \

151 return true;

152#include "VVPNodes.def"

153 }

154 return false;

155}

156

158 switch (Opcode) {

159#define ADD_REDUCE_VVP_OP(VVP_NAME, SDNAME) case VEISD::VVP_NAME:

160#include "VVPNodes.def"

161 return true;

162 }

163 return false;

164}

165

166

168

170 if (PosOpt)

171 return *PosOpt;

172

173

175 return 3;

176

177

178 switch (Opc) {

179 case VEISD::VEC_BROADCAST:

180 return 1;

181 case VEISD::VVP_SELECT:

182 return 3;

183 case VEISD::VVP_LOAD:

184 return 4;

185 case VEISD::VVP_STORE:

186 return 5;

187 }

188

189 return std::nullopt;

190}

191

193

195 if (PosOpt)

196 return *PosOpt;

197

198

200 return 2;

201

202

203 switch (Opc) {

204 case ISD::MSTORE:

205 return 4;

206 case ISD::MLOAD:

207 return 3;

208 case VEISD::VVP_SELECT:

209 return 2;

210 }

211

212 return std::nullopt;

213}

214

216

217

218

221 return MemN->getChain();

222

223 switch (Op->getOpcode()) {

224 case VEISD::VVP_LOAD:

225 case VEISD::VVP_STORE:

226 return Op->getOperand(0);

227 }

229}

230

233 return MemN->getBasePtr();

234

235 switch (Op->getOpcode()) {

236 case VEISD::VVP_LOAD:

237 return Op->getOperand(1);

238 case VEISD::VVP_STORE:

239 return Op->getOperand(2);

240 }

242}

243

245 unsigned OC = Op->getOpcode();

246

247

249 return MemN->getMemoryVT();

250

251 switch (OC) {

252

253 case ISD::SELECT:

259 return Op->getValueType(0);

260 }

261

262

263 unsigned OriginalOC = OC;

265 OC = *VVPOpc;

266

269 .getValueType();

270

271 switch (OC) {

272 default:

273 case VEISD::VVP_SETCC:

274 return Op->getOperand(0).getValueType();

275

276 case VEISD::VVP_SELECT:

277#define ADD_BINARY_VVP_OP(VVP_NAME, ...) case VEISD::VVP_NAME:

278#include "VVPNodes.def"

279 return Op->getValueType(0);

280

281 case VEISD::VVP_LOAD:

282 return Op->getValueType(0);

283

284 case VEISD::VVP_STORE:

285 return Op->getOperand(1)->getValueType(0);

286

287

288 case VEISD::VEC_BROADCAST:

289 return Op->getValueType(0);

290 }

291}

292

294 switch (Op->getOpcode()) {

295 case VEISD::VVP_STORE:

296 return Op->getOperand(3);

297 case VEISD::VVP_LOAD:

298 return Op->getOperand(2);

299 }

300

302 return StoreN->getStride();

304 return StoreN->getStride();

305

307

308

310 ->getVectorElementType()

311 .getStoreSize();

312 return CDAG.getConstant(ElemStride, MVT::i64);

313 }

315}

316

319 return N->getIndex();

321 return N->getIndex();

323}

324

327 return N->getScale();

329 return N->getScale();

331}

332

334 switch (Op->getOpcode()) {

335 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:

336 case VEISD::VVP_STORE:

337 return Op->getOperand(1);

338 }

340 return StoreN->getValue();

342 return StoreN->getValue();

344 return StoreN->getValue();

346 return StoreN->getValue();

348 return StoreN->getValue();

350 return StoreN->getValue();

352}

353

356 return N->getPassThru();

358 return N->getPassThru();

359

361}

362

364

366 return true;

367 return false;

368}

369

371 assert(!IsMask && "Mask reduction isel");

372

373 switch (VVPOC) {

374#define HANDLE_VVP_REDUCE_TO_SCALAR(VVP_RED_ISD, REDUCE_ISD) \

375 case VEISD::VVP_RED_ISD: \

376 return ISD::REDUCE_ISD;

377#include "VVPNodes.def"

378 default:

379 break;

380 }

381 llvm_unreachable("Cannot not scalarize this reduction Opcode!");

382}

383

384

385

387 auto PosOpt = getAVLPos(Op->getOpcode());

388 return PosOpt ? Op->getOperand(*PosOpt) : SDValue();

389}

390

393 return PosOpt ? Op->getOperand(*PosOpt) : SDValue();

394}

395

398 if (!AVL)

399 return {SDValue(), true};

402 return {AVL, false};

403}

404

406 bool IsOpaque) const {

407 return DAG.getConstant(Val, DL, VT, IsTarget, IsOpaque);

408}

409

412

413

414 auto TrueVal = DAG.getAllOnesConstant(DL, MVT::i32);

415 auto AVL = getConstant(MaskVT.getVectorNumElements(), MVT::i32);

416 auto Res = getNode(VEISD::VEC_BROADCAST, MaskVT, {TrueVal, AVL});

417 if (AllTrue)

418 return Res;

419

420 return DAG.getNOT(DL, Res, Res.getValueType());

421}

422

425

428 BcConst->getSExtValue() != 0);

429

430

431 auto ScalarBoolVT = Scalar.getSimpleValueType();

432 assert(ScalarBoolVT == MVT::i32);

433

434

435 SDValue CmpElem = DAG.getSExtOrTrunc(Scalar, DL, MVT::i32);

438

439

441 DAG.getNode(VEISD::VEC_BROADCAST, DL, CmpVecTy, {CmpElem, AVL});

443 getBroadcast(CmpVecTy, {DAG.getConstant(0, DL, ScalarBoolVT)}, AVL);

444

446

447

448

450}

451

455 auto ScaVT = Scalar.getValueType();

456

459

461

462

463

464

465 if (ScaVT == MVT::f32) {

466 Scalar = getNode(VEISD::REPL_F32, MVT::i64, Scalar);

467 } else if (ScaVT == MVT::i32) {

468 Scalar = getNode(VEISD::REPL_I32, MVT::i64, Scalar);

469 }

470 }

471

472 return getNode(VEISD::VEC_BROADCAST, ResultVT, {Scalar, AVL});

473}

474

480

484

485

486 unsigned OC =

487 (Part == PackElem::Lo) ? VEISD::VEC_UNPACK_LO : VEISD::VEC_UNPACK_HI;

488 return DAG.getNode(OC, DL, DestVT, Vec, AVL);

489}

490

494

495

496 return DAG.getNode(VEISD::VEC_PACK, DL, DestVT, LoVec, HiVec, AVL);

497}

498

501

506 else

507 NewAVL = RawAVL;

509

511

512

514 if (!RawMask)

516 else

517 NewMask = getUnpack(MVT::v256i1, RawMask, Part, NewAVL);

518

520}

521

524

525

527 return Ptr;

529}

530

533 return getConstant(2 * ConstBytes->getSExtValue(), MVT::i64);

535}

536

540 EVT IndexVT = Index.getValueType();

541

542

545 ScaledIndex = Index;

546 else {

548 ScaledIndex =

549 getNode(VEISD::VVP_MUL, IndexVT, {Index, ScaleBroadcast, Mask, AVL});

550 }

551

552

554 return ScaledIndex;

555

556

558 auto ResPtr =

559 getNode(VEISD::VVP_ADD, IndexVT, {BaseBroadcast, ScaledIndex, Mask, AVL});

560 return ResPtr;

561}

562

567

568

569

572 assert(!IsMaskReduction && "TODO Implement");

573 auto AttachStartValue = [&](SDValue ReductionResV) {

574 if (!scalarizeStartParam)

575 return ReductionResV;

577 return getNode(ScalarOC, ResVT, {StartV, ReductionResV});

578 };

579

580

581 if (!scalarizeStartParam && StartV) {

583 return AttachStartValue(

584 getNode(VVPOpcode, ResVT, {StartV, VectorV, Mask, AVL}, Flags));

585 } else

586 return AttachStartValue(

587 getNode(VVPOpcode, ResVT, {VectorV, Mask, AVL}, Flags));

588}

589

590}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

bool isVector() const

Return true if this is a vector value type.

static MVT getVectorVT(MVT VT, unsigned NumElements)

MVT getVectorElementType() const

This is an abstract virtual class for memory operations.

Represents one node in the SelectionDAG.

unsigned getOpcode() const

Return the SelectionDAG opcode value for this node.

const SDValue & getOperand(unsigned Num) const

Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.

SDNode * getNode() const

get the SDNode which holds the desired result

EVT getValueType() const

Return the ValueType of the referenced return value.

SDValue getSplitPtrOffset(SDValue Ptr, SDValue ByteStride, PackElem Part) const

Definition VECustomDAG.cpp:522

SDValue getBroadcast(EVT ResultVT, SDValue Scalar, SDValue AVL) const

Definition VECustomDAG.cpp:452

SDValue getConstantMask(Packing Packing, bool AllTrue) const

Definition VECustomDAG.cpp:410

SDValue getGatherScatterAddress(SDValue BasePtr, SDValue Scale, SDValue Index, SDValue Mask, SDValue AVL) const

Definition VECustomDAG.cpp:537

SDValue getMaskBroadcast(EVT ResultVT, SDValue Scalar, SDValue AVL) const

Definition VECustomDAG.cpp:423

SDValue getLegalReductionOpVVP(unsigned VVPOpcode, EVT ResVT, SDValue StartV, SDValue VectorV, SDValue Mask, SDValue AVL, SDNodeFlags Flags) const

} getNode

Definition VECustomDAG.cpp:563

SDValue getNode(unsigned OC, SDVTList VTL, ArrayRef< SDValue > OpV, std::optional< SDNodeFlags > Flags=std::nullopt) const

getNode {

SDValue annotateLegalAVL(SDValue AVL) const

Definition VECustomDAG.cpp:475

SDValue getUnpack(EVT DestVT, SDValue Vec, PackElem Part, SDValue AVL) const

} Legalizing getNode

Definition VECustomDAG.cpp:481

SDValue getPack(EVT DestVT, SDValue LoVec, SDValue HiVec, SDValue AVL) const

Definition VECustomDAG.cpp:491

SDValue getConstant(uint64_t Val, EVT VT, bool IsTarget=false, bool IsOpaque=false) const

Definition VECustomDAG.cpp:405

SDValue getSplitPtrStride(SDValue PackStride) const

Definition VECustomDAG.cpp:531

VETargetMasks getTargetSplitMask(SDValue RawMask, SDValue RawAVL, PackElem Part) const

Definition VECustomDAG.cpp:499

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

@ ADD

Simple integer binary arithmetic operators.

@ CONCAT_VECTORS

CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...

@ SCALAR_TO_VECTOR

SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...

@ SELECT

Select(COND, TRUEVAL, FALSEVAL).

@ SHL

Shift and rotation operations.

@ VECTOR_SHUFFLE

VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.

@ EXTRACT_SUBVECTOR

EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.

@ AND

Bitwise operators - logical and, logical or, logical xor.

@ BUILD_VECTOR

BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...

LLVM_ABI std::optional< unsigned > getVPMaskIdx(unsigned Opcode)

The operand position of the vector mask.

LLVM_ABI std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)

The operand position of the explicit vector length parameter.

LLVM_ABI bool isVPReduction(unsigned Opcode)

Whether this is a vector-predicated reduction opcode.

This is an optimization pass for GlobalISel generic memory operations.

bool isVVPReductionOp(unsigned Opcode)

Definition VECustomDAG.cpp:157

bool isPackedVectorType(EVT SomeVT)

Definition VECustomDAG.cpp:23

bool supportsPackedMode(unsigned Opcode, EVT IdiomVT)

Definition VECustomDAG.cpp:102

std::optional< int > getAVLPos(unsigned Opc)

The VE backend uses a two-staged process to lower and legalize vector instructions:

Definition VECustomDAG.cpp:167

LLVM_ABI bool isNullConstant(SDValue V)

Returns true if V is a constant integer zero.

SDValue getGatherScatterScale(SDValue Op)

Definition VECustomDAG.cpp:325

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

SDValue getStoredValue(SDValue Op)

Definition VECustomDAG.cpp:333

bool isVVPBinaryOp(unsigned VVPOpcode)

Definition VECustomDAG.cpp:147

std::optional< EVT > getIdiomaticVectorType(SDNode *Op)

} AVL Functions

Definition VECustomDAG.cpp:244

SDValue getNodeChain(SDValue Op)

Node Properties {.

Definition VECustomDAG.cpp:219

static const unsigned StandardVectorWidth

static const unsigned PackedVectorWidth

bool isMaskArithmetic(SDValue Op)

Definition VECustomDAG.cpp:51

SDValue getNodeAVL(SDValue Op)

} Node Properties

Definition VECustomDAG.cpp:386

bool isMaskType(EVT SomeVT)

Definition VECustomDAG.cpp:45

bool isLegalAVL(SDValue AVL)

Definition VECustomDAG.cpp:215

MVT splitVectorType(MVT VT)

Definition VECustomDAG.cpp:29

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

SDValue getNodePassthru(SDValue Op)

Definition VECustomDAG.cpp:354

bool maySafelyIgnoreMask(SDValue Op)

Definition VECustomDAG.cpp:86

bool isVVPOrVEC(unsigned Opcode)

Definition VECustomDAG.cpp:127

MVT getLegalVectorType(Packing P, MVT ElemVT)

Definition VECustomDAG.cpp:35

SDValue getMemoryPtr(SDValue Op)

Definition VECustomDAG.cpp:231

std::optional< int > getMaskPos(unsigned Opc)

Definition VECustomDAG.cpp:192

bool isPackingSupportOpcode(unsigned Opc)

Definition VECustomDAG.cpp:117

std::pair< SDValue, bool > getAnnotatedNodeAVL(SDValue Op)

Definition VECustomDAG.cpp:396

DWARFExpression::Operation Op

LLVM_ABI bool isOneConstant(SDValue V)

Returns true if V is a constant integer one.

bool hasReductionStartParam(unsigned OPC)

Definition VECustomDAG.cpp:363

SDValue getGatherScatterIndex(SDValue Op)

Definition VECustomDAG.cpp:317

Packing getTypePacking(EVT VT)

Definition VECustomDAG.cpp:40

unsigned getScalarReductionOpcode(unsigned VVPOC, bool IsMask)

Definition VECustomDAG.cpp:370

std::optional< unsigned > getVVPOpcode(unsigned Opcode)

Definition VECustomDAG.cpp:63

bool isVVPUnaryOp(unsigned VVPOpcode)

Definition VECustomDAG.cpp:137

SDValue getNodeMask(SDValue Op)

Definition VECustomDAG.cpp:391

SDValue getLoadStoreStride(SDValue Op, VECustomDAG &CDAG)

Definition VECustomDAG.cpp:293

bool isVector() const

Return true if this is a vector value type.

EVT getVectorElementType() const

Given a vector type, return the type of each element.

unsigned getVectorNumElements() const

Given a vector type, return the number of elements it contains.

These are IR-level optimization flags that may be propagated to SDNodes.