LegalizeTypesGeneric.cpp Source File (original) (raw)

23using namespace llvm;

25#define DEBUG_TYPE "legalize-types"

34void DAGTypeLegalizer::ExpandRes_MERGE_VALUES(SDNode *N, unsigned ResNo,

36 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);

37 GetExpandedOp(Op, Lo, Hi);

38}

41 EVT OutVT = N->getValueType(0);

42 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);

43 SDValue InOp = N->getOperand(0);

45 SDLoc dl(N);

48 switch (getTypeAction(InVT)) {

51 break;

54 llvm_unreachable("Bitcast of a promotion-needing float should never need"

55 "expansion");

57 SplitInteger(GetSoftenedFloat(InOp), Lo, Hi);

58 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);

59 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);

60 return;

63 auto &DL = DAG.getDataLayout();

65 GetExpandedOp(InOp, Lo, Hi);

66 if (TLI.hasBigEndianPartOrdering(InVT, DL) !=

67 TLI.hasBigEndianPartOrdering(OutVT, DL))

69 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);

70 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);

71 return;

72 }

74 GetSplitVector(InOp, Lo, Hi);

75 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))

77 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);

78 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);

79 return;

82 SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi);

83 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);

84 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);

85 return;

87 report_fatal_error("Scalarization of scalable vectors is not supported.");

90 InOp = GetWidenedVector(InOp);

91 EVT LoVT, HiVT;

92 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(InVT);

93 std::tie(Lo, Hi) = DAG.SplitVector(InOp, dl, LoVT, HiVT);

94 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))

96 Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);

97 Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);

98 return;

99 }

100 }

101

103

104

105 unsigned NumElems = 2;

106 EVT ElemVT = NOutVT;

107 EVT NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);

108

109

110 while (!isTypeLegal(NVT)) {

111 unsigned NewSizeInBits = ElemVT.getSizeInBits() / 2;

112

113 if (NewSizeInBits < 8)

114 break;

115 NumElems *= 2;

118 }

119

120 if (isTypeLegal(NVT)) {

121 SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp);

122

124 for (unsigned i = 0; i < NumElems; ++i)

126 CastInOp, DAG.getVectorIdxConstant(i, dl)));

127

128

129 unsigned Slot = 0;

130 for (unsigned e = Vals.size(); e - Slot > 2; Slot += 2, e += 1) {

131

132

135

136 if (DAG.getDataLayout().isBigEndian())

138

143 }

146

147 if (DAG.getDataLayout().isBigEndian())

149

150 return;

151 }

152 }

153

154

156

157

158

159

160

161

162 Align InAlign = DAG.getReducedAlign(InVT, false);

163 Align NOutAlign = DAG.getReducedAlign(NOutVT, false);

164 Align Align = std::max(InAlign, NOutAlign);

167 MachinePointerInfo PtrInfo =

169

170

171 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo);

172

173

174 Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, PtrInfo, NOutAlign);

175

176

177 unsigned IncrementSize = NOutVT.getSizeInBits() / 8;

179 DAG.getMemBasePlusOffset(StackPtr, TypeSize::getFixed(IncrementSize), dl);

180

181

182 Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr,

183 PtrInfo.getWithOffset(IncrementSize), NOutAlign);

184

185

186 if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))

188}

189

190void DAGTypeLegalizer::ExpandRes_BUILD_PAIR(SDNode *N, SDValue &Lo,

192

193 Lo = N->getOperand(0);

194 Hi = N->getOperand(1);

195}

196

197void DAGTypeLegalizer::ExpandRes_EXTRACT_ELEMENT(SDNode *N, SDValue &Lo,

199 GetExpandedOp(N->getOperand(0), Lo, Hi);

200 SDValue Part = N->getConstantOperandVal(1) ? Hi : Lo;

201

203 "Type twice as big as expanded type not itself expanded!");

204

205 GetPairElements(Part, Lo, Hi);

206}

207

208void DAGTypeLegalizer::ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo,

210 SDValue OldVec = N->getOperand(0);

213 SDLoc dl(N);

214

215

216

217 EVT OldVT = N->getValueType(0);

218 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);

219

220 if (OldVT != OldEltVT) {

221

222

223

224 assert(OldEltVT.bitsLT(OldVT) && "Result type smaller then element type!");

225 EVT NVecVT = EVT::getVectorVT(*DAG.getContext(), OldVT, OldEltCount);

227 }

228

229 SDValue NewVec = DAG.getNode(

230 ISD::BITCAST, dl,

231 EVT::getVectorVT(*DAG.getContext(), NewVT, OldEltCount * 2), OldVec);

232

233

234 SDValue Idx = N->getOperand(1);

235

238

242

243 if (DAG.getDataLayout().isBigEndian())

245}

246

247void DAGTypeLegalizer::ExpandRes_NormalLoad(SDNode *N, SDValue &Lo,

250 SDLoc dl(N);

251

253 assert(->isAtomic() && "Atomics can not be split");

254 EVT ValueVT = LD->getValueType(0);

255 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT);

258 AAMDNodes AAInfo = LD->getAAInfo();

259

261

262 Lo = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getPointerInfo(),

263 LD->getBaseAlign(), LD->getMemOperand()->getFlags(), AAInfo);

264

265

266 unsigned IncrementSize = NVT.getSizeInBits() / 8;

267 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));

268 Hi = DAG.getLoad(NVT, dl, Chain, Ptr,

269 LD->getPointerInfo().getWithOffset(IncrementSize),

270 LD->getBaseAlign(), LD->getMemOperand()->getFlags(), AAInfo);

271

272

273

275 Hi.getValue(1));

276

277

278 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))

280

281

282

283 ReplaceValueWith(SDValue(N, 1), Chain);

284}

285

287 EVT OVT = N->getValueType(0);

288 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);

291 SDLoc dl(N);

292 const unsigned Align = N->getConstantOperandVal(3);

293

294 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), Align);

295 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, N->getOperand(2), 0);

297

298

299 if (TLI.hasBigEndianPartOrdering(OVT, DAG.getDataLayout()))

301

302

303

304 ReplaceValueWith(SDValue(N, 1), Chain);

305}

306

307

308

309

310

311

312void DAGTypeLegalizer::IntegerToVector(SDValue Op, unsigned NumElements,

314 EVT EltVT) {

315 assert(Op.getValueType().isInteger());

316 SDLoc DL(Op);

318

319 if (NumElements > 1) {

320 NumElements >>= 1;

321 SplitInteger(Op, Parts[0], Parts[1]);

322 if (DAG.getDataLayout().isBigEndian())

324 IntegerToVector(Parts[0], NumElements, Ops, EltVT);

325 IntegerToVector(Parts[1], NumElements, Ops, EltVT);

326 } else {

327 Ops.push_back(DAG.getNode(ISD::BITCAST, DL, EltVT, Op));

328 }

329}

330

331SDValue DAGTypeLegalizer::ExpandOp_BITCAST(SDNode *N) {

332 SDLoc dl(N);

333 if (N->getValueType(0).isVector() &&

334 N->getOperand(0).getValueType().isInteger()) {

335

336

337

338

339

340

341

342

343

344 unsigned NumElts = 2;

345 EVT OVT = N->getOperand(0).getValueType();

347 TLI.getTypeToTransformTo(*DAG.getContext(), OVT),

348 NumElts);

349 if (!isTypeLegal(NVT)) {

350

351

352 NumElts = N->getValueType(0).getVectorNumElements();

353 NVT = N->getValueType(0);

354 }

355

358

359 SDValue Vec = DAG.getBuildVector(NVT, dl, ArrayRef(Ops.data(), NumElts));

360 return DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), Vec);

361 }

362

363

364 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));

365}

366

367SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) {

368

371 EVT OldVT = N->getOperand(0).getValueType();

372 EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);

373 SDLoc dl(N);

374

376 "BUILD_VECTOR operand type doesn't match vector element type!");

377

382 GetExpandedOp(V, Lo, Hi);

384 }

385 }

386

387

388

390 NewElts.reserve(NumElts*2);

391

392 for (unsigned i = 0; i < NumElts; ++i) {

394 GetExpandedOp(N->getOperand(i), Lo, Hi);

395 if (DAG.getDataLayout().isBigEndian())

399 }

400

402 SDValue NewVec = DAG.getBuildVector(NewVecVT, dl, NewElts);

403

404

405 return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec);

406}

407

408SDValue DAGTypeLegalizer::ExpandOp_EXTRACT_ELEMENT(SDNode *N) {

410 GetExpandedOp(N->getOperand(0), Lo, Hi);

411 return N->getConstantOperandVal(1) ? Hi : Lo;

412}

413

414

415

416SDValue DAGTypeLegalizer::ExpandOp_FAKE_USE(SDNode *N) {

419 GetExpandedOp(N->getOperand(1), Lo, Hi);

420 SDValue LoUse = DAG.getNode(ISD::FAKE_USE, SDLoc(), MVT::Other, Chain, Lo);

421 DAG.UpdateNodeOperands(N, LoUse, Hi);

423}

424

425SDValue DAGTypeLegalizer::ExpandOp_INSERT_VECTOR_ELT(SDNode *N) {

426

427 EVT VecVT = N->getValueType(0);

429 SDLoc dl(N);

430

431 SDValue Val = N->getOperand(1);

433 EVT NewEVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldEVT);

434

436 "Inserted element type doesn't match vector element type!");

437

438

439

440 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEVT, NumElts*2);

442 NewVecVT, N->getOperand(0));

443

445 GetExpandedOp(Val, Lo, Hi);

446 if (DAG.getDataLayout().isBigEndian())

448

456

457

458 return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec);

459}

460

461SDValue DAGTypeLegalizer::ExpandOp_SCALAR_TO_VECTOR(SDNode *N) {

462 SDLoc dl(N);

463 EVT VT = N->getValueType(0);

465 "SCALAR_TO_VECTOR operand type doesn't match vector element type!");

468 Ops[0] = N->getOperand(0);

470 for (unsigned i = 1; i < NumElts; ++i)

471 Ops[i] = UndefVal;

472 return DAG.getBuildVector(VT, dl, Ops);

473}

474

475SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) {

477 assert(OpNo == 1 && "Can only expand the stored value so far");

478 SDLoc dl(N);

479

481 assert(!St->isAtomic() && "Atomics can not be split");

483 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT);

486 AAMDNodes AAInfo = St->getAAInfo();

487

489 unsigned IncrementSize = NVT.getSizeInBits() / 8;

490

493

494 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))

496

500

501 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(IncrementSize));

502 Hi = DAG.getStore(

505

507}

508

509

510

511

512

513

514

515

516

517

518

519void DAGTypeLegalizer::SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo,

521 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);

522 GetSplitOp(Op, Lo, Hi);

523}

524

527 SDLoc dl(N);

528 unsigned Opcode = N->getOpcode();

529 GetSplitOp(N->getOperand(1), LL, LH);

530 GetSplitOp(N->getOperand(2), RL, RH);

531

534 if (Cond.getValueType().isVector()) {

535 if (SDValue Res = WidenVSELECTMask(N))

536 std::tie(CL, CH) = DAG.SplitVector(Res, dl);

537

538

539 else if (getTypeAction(Cond.getValueType()) ==

541 GetSplitVector(Cond, CL, CH);

542

543

545

546

547

548 EVT CondLHSVT = Cond.getOperand(0).getValueType();

549 if (Cond.getValueType().getVectorElementType() == MVT::i1 &&

550 isTypeLegal(CondLHSVT) &&

551 getSetCCResultType(CondLHSVT) == Cond.getValueType())

552 std::tie(CL, CH) = DAG.SplitVector(Cond, dl);

553 else

554 SplitVecRes_SETCC(Cond.getNode(), CL, CH);

555 } else

556 std::tie(CL, CH) = DAG.SplitVector(Cond, dl);

557 }

558

559 if (Opcode != ISD::VP_SELECT && Opcode != ISD::VP_MERGE) {

560 Lo = DAG.getNode(Opcode, dl, LL.getValueType(), CL, LL, RL);

562 return;

563 }

564

566 std::tie(EVLLo, EVLHi) =

567 DAG.SplitEVL(N->getOperand(3), N->getValueType(0), dl);

568

570 Hi = DAG.getNode(Opcode, dl, LH.getValueType(), CH, LH, RH, EVLHi);

571}

572

573void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo,

576 SDLoc dl(N);

577 GetSplitOp(N->getOperand(2), LL, LH);

578 GetSplitOp(N->getOperand(3), RL, RH);

579

581 N->getOperand(1), LL, RL, N->getOperand(4));

583 N->getOperand(1), LH, RH, N->getOperand(4));

584}

585

587 EVT LoVT, HiVT;

588 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));

589 Lo = DAG.getUNDEF(LoVT);

590 Hi = DAG.getUNDEF(HiVT);

591}

592

593void DAGTypeLegalizer::SplitVecRes_AssertZext(SDNode *N, SDValue &Lo,

596 SDLoc dl(N);

597 GetSplitOp(N->getOperand(0), L, H);

598

599 Lo = DAG.getNode(ISD::AssertZext, dl, L.getValueType(), L, N->getOperand(1));

600 Hi = DAG.getNode(ISD::AssertZext, dl, H.getValueType(), H, N->getOperand(1));

601}

602

605 SDLoc dl(N);

606 GetSplitOp(N->getOperand(0), L, H);

607

608 Lo = DAG.getNode(ISD::FREEZE, dl, L.getValueType(), L);

610}

611

612void DAGTypeLegalizer::SplitRes_ARITH_FENCE(SDNode *N, SDValue &Lo,

615 SDLoc DL(N);

616 GetSplitOp(N->getOperand(0), L, H);

617

618 Lo = DAG.getNode(ISD::ARITH_FENCE, DL, L.getValueType(), L);

619 Hi = DAG.getNode(ISD::ARITH_FENCE, DL, H.getValueType(), H);

620}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]

const SmallVectorImpl< MachineOperand > & Cond

static Type * getValueType(Value *V)

Returns the type of the given value/instruction V.

Flags getFlags() const

Return the raw flags of the source value,.

Align getBaseAlign() const

Returns alignment and volatility of the memory access.

AAMDNodes getAAInfo() const

Returns the AA info that describes the dereference.

MachineMemOperand * getMemOperand() const

Return a MachineMemOperand object describing the memory reference performed by operation.

const MachinePointerInfo & getPointerInfo() const

const SDValue & getChain() const

bool isAtomic() const

Return true if the memory operation ordering is Unordered or higher.

Represents one node in the SelectionDAG.

const SDValue & getOperand(unsigned Num) const

EVT getValueType(unsigned ResNo) const

Return the type of a specified result.

Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.

SDNode * getNode() const

get the SDNode which holds the desired result

SDValue getValue(unsigned R) const

EVT getValueType() const

Return the ValueType of the referenced return value.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

void reserve(size_type N)

void push_back(const T &Elt)

const SDValue & getBasePtr() const

const SDValue & getValue() const

@ TypeScalarizeScalableVector

static constexpr TypeSize getFixed(ScalarTy ExactSize)

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

constexpr char Align[]

Key for Kernel::Arg::Metadata::mAlign.

@ SETCC

SetCC operator - This evaluates to a true value iff the condition is true.

@ ADD

Simple integer binary arithmetic operators.

@ ANY_EXTEND

ANY_EXTEND - Used for integer types. The high bits are undefined.

@ BUILD_PAIR

BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.

@ SPLAT_VECTOR

SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.

@ EXTRACT_VECTOR_ELT

EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...

@ SELECT_CC

Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...

@ SPLAT_VECTOR_PARTS

SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...

@ FREEZE

FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...

@ INSERT_VECTOR_ELT

INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.

@ TokenFactor

TokenFactor - This node takes multiple tokens as input and produces a single token result.

bool isNormalStore(const SDNode *N)

Returns true if the specified node is a non-truncating and unindexed store.

bool isNormalLoad(const SDNode *N)

Returns true if the specified node is a non-extending and unindexed load.

This is an optimization pass for GlobalISel generic memory operations.

LLVM_ABI Value * getSplatValue(const Value *V)

Get splat value if the input is a splat vector or return nullptr.

LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

DWARFExpression::Operation Op

ArrayRef(const T &OneElt) -> ArrayRef< T >

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)

Implement std::swap in terms of BitVector swap.

TypeSize getStoreSize() const

Return the number of bytes overwritten by a store of the specified value type.

static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)

Returns the EVT that represents a vector NumElements in length, where each element is of type VT.

bool bitsLT(EVT VT) const

Return true if this has less bits than VT.

ElementCount getVectorElementCount() const

TypeSize getSizeInBits() const

Return the size of the specified value type in bits.

bool isByteSized() const

Return true if the bit size is a multiple of 8.

static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)

Returns the EVT that represents an integer with the given number of bits.

bool isVector() const

Return true if this is a vector value type.

EVT getVectorElementType() const

Given a vector type, return the type of each element.

unsigned getVectorNumElements() const

Given a vector type, return the number of elements it contains.

bool isInteger() const

Return true if this is an integer or a vector integer type.

MachinePointerInfo getWithOffset(int64_t O) const

static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)

Return a MachinePointerInfo record that refers to the specified FrameIndex.