LLVM: lib/CodeGen/SelectionDAG/LegalizeTypes.h Source File (original) (raw)

34public:

35

36

38

40

41

42

44

45

46

48

49

51

52

53 };

54private:

55

56

57

58

60

61

63 return TLI.getTypeAction(*DAG.getContext(), VT);

64 }

65

66

67 bool isTypeLegal(EVT VT) const {

68 return TLI.getTypeAction(*DAG.getContext(), VT) == TargetLowering::TypeLegal;

69 }

70

71

72 bool isSimpleLegalType(EVT VT) const {

73 return VT.isSimple() && TLI.isTypeLegal(VT);

74 }

75

76 EVT getSetCCResultType(EVT VT) const {

78 }

79

80

81 bool IgnoreNodeResults(SDNode *N) const {

82 return N->getOpcode() == ISD::TargetConstant ||

83 N->getOpcode() == ISD::Register;

84 }

85

86

87

88

89 typedef unsigned TableId;

90

91 TableId NextValueId = 1;

92

93 SmallDenseMap<SDValue, TableId, 8> ValueToIdMap;

94 SmallDenseMap<TableId, SDValue, 8> IdToValueMap;

95

96

97

98 SmallDenseMap<TableId, TableId, 8> PromotedIntegers;

99

100

101

102 SmallDenseMap<TableId, std::pair<TableId, TableId>, 8> ExpandedIntegers;

103

104

105

106 SmallDenseMap<TableId, TableId, 8> SoftenedFloats;

107

108

109

110 SmallDenseMap<TableId, TableId, 8> PromotedFloats;

111

112

113

114 SmallDenseMap<TableId, TableId, 8> SoftPromotedHalfs;

115

116

117

118 SmallDenseMap<TableId, std::pair<TableId, TableId>, 8> ExpandedFloats;

119

120

121

122 SmallDenseMap<TableId, TableId, 8> ScalarizedVectors;

123

124

125

126 SmallDenseMap<TableId, std::pair<TableId, TableId>, 8> SplitVectors;

127

128

129

130 SmallDenseMap<TableId, TableId, 8> WidenedVectors;

131

132

133

134 SmallDenseMap<TableId, TableId, 8> ReplacedValues;

135

136

137

138 SmallVector<SDNode*, 128> Worklist;

139

140 TableId getTableId(SDValue V) {

141 assert(V.getNode() && "Getting TableId on SDValue()");

142

143 auto I = ValueToIdMap.find(V);

144 if (I != ValueToIdMap.end()) {

145

146 RemapId(I->second);

147 assert(I->second && "All Ids should be nonzero");

148 return I->second;

149 }

150

151 ValueToIdMap.insert(std::make_pair(V, NextValueId));

152 IdToValueMap.insert(std::make_pair(NextValueId, V));

153 ++NextValueId;

154 assert(NextValueId != 0 &&

155 "Ran out of Ids. Increase id type size or add compactification");

156 return NextValueId - 1;

157 }

158

159 const SDValue &getSDValue(TableId &Id) {

160 RemapId(Id);

161 assert(Id && "TableId should be non-zero");

162 auto I = IdToValueMap.find(Id);

163 assert(I != IdToValueMap.end() && "cannot find Id in map");

164 return I->second;

165 }

166

167public:

169 : TLI(dag.getTargetLoweringInfo()), DAG(dag),

170 ValueTypeActions(TLI.getValueTypeActions()) {

171 }

172

173

174

175

176 bool run();

177

179 assert(Old != New && "node replaced with self");

180 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {

181 TableId NewId = getTableId(SDValue(New, i));

182 TableId OldId = getTableId(SDValue(Old, i));

183

184 if (OldId != NewId) {

185 ReplacedValues[OldId] = NewId;

186

187

188

189

190 IdToValueMap.erase(OldId);

191 PromotedIntegers.erase(OldId);

192 ExpandedIntegers.erase(OldId);

193 SoftenedFloats.erase(OldId);

194 PromotedFloats.erase(OldId);

195 SoftPromotedHalfs.erase(OldId);

196 ExpandedFloats.erase(OldId);

197 ScalarizedVectors.erase(OldId);

198 SplitVectors.erase(OldId);

199 WidenedVectors.erase(OldId);

200 }

201

202 ValueToIdMap.erase(SDValue(Old, i));

203 }

204 }

205

207

208private:

210 void AnalyzeNewValue(SDValue &Val);

211 void PerformExpensiveChecks();

212 void RemapId(TableId &Id);

213 void RemapValue(SDValue &V);

214

215

219 bool CustomLowerNode(SDNode *N, EVT VT, bool LegalizeResult);

220 bool CustomWidenLowerNode(SDNode *N, EVT VT);

221

222

223

224

225 SDValue DisintegrateMERGE_VALUES(SDNode *N, unsigned ResNo);

226

228

229 std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node);

230

232

237

238

239

240

241

242

243

244

245

246

247

248

249

250

252 TableId &PromotedId = PromotedIntegers[getTableId(Op)];

253 SDValue PromotedOp = getSDValue(PromotedId);

254 assert(PromotedOp.getNode() && "Operand wasn't promoted?");

255 return PromotedOp;

256 }

258

259

261 EVT OldVT = Op.getValueType();

262 SDLoc dl(Op);

263 Op = GetPromotedInteger(Op);

264 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), Op,

266 }

267

268

270 EVT OldVT = Op.getValueType();

271 SDLoc dl(Op);

272 Op = GetPromotedInteger(Op);

274 }

275

276

278 EVT OldVT = Op.getValueType();

279 SDLoc dl(Op);

280 Op = GetPromotedInteger(Op);

281

282 EVT VT = Op.getValueType();

283 unsigned BitsDiff = VT.getScalarSizeInBits() - OldVT.getScalarSizeInBits();

285 SDValue Shl = DAG.getNode(ISD::VP_SHL, dl, VT, Op, ShiftCst, Mask, EVL);

286 return DAG.getNode(ISD::VP_SRA, dl, VT, Shl, ShiftCst, Mask, EVL);

287 }

288

289

291 EVT OldVT = Op.getValueType();

292 SDLoc dl(Op);

293 Op = GetPromotedInteger(Op);

295 }

296

297

298

299

300

301 SDValue PromoteIntOpVectorReduction(SDNode *N, SDValue V);

302

303

304 void PromoteIntegerResult(SDNode *N, unsigned ResNo);

305 SDValue PromoteIntRes_MERGE_VALUES(SDNode *N, unsigned ResNo);

306 SDValue PromoteIntRes_AssertSext(SDNode *N);

307 SDValue PromoteIntRes_AssertZext(SDNode *N);

308 SDValue PromoteIntRes_Atomic0(AtomicSDNode *N);

309 SDValue PromoteIntRes_Atomic1(AtomicSDNode *N);

310 SDValue PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, unsigned ResNo);

311 SDValue PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N);

312 SDValue PromoteIntRes_INSERT_SUBVECTOR(SDNode *N);

313 SDValue PromoteIntRes_VECTOR_REVERSE(SDNode *N);

314 SDValue PromoteIntRes_VECTOR_SHUFFLE(SDNode *N);

315 SDValue PromoteIntRes_VECTOR_SPLICE(SDNode *N);

316 SDValue PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(SDNode *N);

317 SDValue PromoteIntRes_BUILD_VECTOR(SDNode *N);

318 SDValue PromoteIntRes_ScalarOp(SDNode *N);

319 SDValue PromoteIntRes_STEP_VECTOR(SDNode *N);

320 SDValue PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N);

321 SDValue PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N);

322 SDValue PromoteIntRes_CONCAT_VECTORS(SDNode *N);

323 SDValue PromoteIntRes_BITCAST(SDNode *N);

324 SDValue PromoteIntRes_BSWAP(SDNode *N);

325 SDValue PromoteIntRes_BITREVERSE(SDNode *N);

326 SDValue PromoteIntRes_BUILD_PAIR(SDNode *N);

327 SDValue PromoteIntRes_Constant(SDNode *N);

328 SDValue PromoteIntRes_CTLZ(SDNode *N);

329 SDValue PromoteIntRes_CTPOP_PARITY(SDNode *N);

330 SDValue PromoteIntRes_CTTZ(SDNode *N);

331 SDValue PromoteIntRes_VP_CttzElements(SDNode *N);

332 SDValue PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N);

333 SDValue PromoteIntRes_FP_TO_XINT(SDNode *N);

334 SDValue PromoteIntRes_FP_TO_XINT_SAT(SDNode *N);

335 SDValue PromoteIntRes_FP_TO_FP16_BF16(SDNode *N);

336 SDValue PromoteIntRes_STRICT_FP_TO_FP16_BF16(SDNode *N);

337 SDValue PromoteIntRes_XRINT(SDNode *N);

338 SDValue PromoteIntRes_FREEZE(SDNode *N);

339 SDValue PromoteIntRes_INT_EXTEND(SDNode *N);

340 SDValue PromoteIntRes_LOAD(LoadSDNode *N);

341 SDValue PromoteIntRes_VP_LOAD(VPLoadSDNode *N);

342 SDValue PromoteIntRes_MLOAD(MaskedLoadSDNode *N);

343 SDValue PromoteIntRes_MGATHER(MaskedGatherSDNode *N);

344 SDValue PromoteIntRes_VECTOR_COMPRESS(SDNode *N);

345 SDValue PromoteIntRes_Overflow(SDNode *N);

346 SDValue PromoteIntRes_FFREXP(SDNode *N);

347 SDValue PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo);

348 SDValue PromoteIntRes_CMP(SDNode *N);

349 SDValue PromoteIntRes_Select(SDNode *N);

350 SDValue PromoteIntRes_SELECT_CC(SDNode *N);

351 SDValue PromoteIntRes_SETCC(SDNode *N);

352 SDValue PromoteIntRes_SHL(SDNode *N);

353 SDValue PromoteIntRes_SimpleIntBinOp(SDNode *N);

354 SDValue PromoteIntRes_ZExtIntBinOp(SDNode *N);

355 SDValue PromoteIntRes_SExtIntBinOp(SDNode *N);

356 SDValue PromoteIntRes_UMINUMAX(SDNode *N);

357 SDValue PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N);

358 SDValue PromoteIntRes_SRA(SDNode *N);

359 SDValue PromoteIntRes_SRL(SDNode *N);

360 SDValue PromoteIntRes_TRUNCATE(SDNode *N);

361 SDValue PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo);

362 SDValue PromoteIntRes_UADDSUBO_CARRY(SDNode *N, unsigned ResNo);

363 SDValue PromoteIntRes_SADDSUBO_CARRY(SDNode *N, unsigned ResNo);

364 SDValue PromoteIntRes_UNDEF(SDNode *N);

365 SDValue PromoteIntRes_VAARG(SDNode *N);

366 SDValue PromoteIntRes_VSCALE(SDNode *N);

367 SDValue PromoteIntRes_XMULO(SDNode *N, unsigned ResNo);

368 template

369 SDValue PromoteIntRes_ADDSUBSHLSAT(SDNode *N);

370 SDValue PromoteIntRes_MULFIX(SDNode *N);

371 SDValue PromoteIntRes_DIVFIX(SDNode *N);

372 SDValue PromoteIntRes_GET_ROUNDING(SDNode *N);

373 SDValue PromoteIntRes_VECREDUCE(SDNode *N);

374 SDValue PromoteIntRes_VP_REDUCE(SDNode *N);

375 SDValue PromoteIntRes_ABS(SDNode *N);

376 SDValue PromoteIntRes_Rotate(SDNode *N);

377 SDValue PromoteIntRes_FunnelShift(SDNode *N);

378 SDValue PromoteIntRes_VPFunnelShift(SDNode *N);

379 SDValue PromoteIntRes_IS_FPCLASS(SDNode *N);

380 SDValue PromoteIntRes_PATCHPOINT(SDNode *N);

381 SDValue PromoteIntRes_READ_REGISTER(SDNode *N);

382 SDValue PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(SDNode *N);

383 SDValue PromoteIntRes_GET_ACTIVE_LANE_MASK(SDNode *N);

384 SDValue PromoteIntRes_PARTIAL_REDUCE_MLA(SDNode *N);

385 SDValue PromoteIntRes_LOOP_DEPENDENCE_MASK(SDNode *N);

386

387

388 bool PromoteIntegerOperand(SDNode *N, unsigned OpNo);

389 SDValue PromoteIntOp_ANY_EXTEND(SDNode *N);

390 SDValue PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N);

391 SDValue PromoteIntOp_BITCAST(SDNode *N);

392 SDValue PromoteIntOp_BUILD_PAIR(SDNode *N);

393 SDValue PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo);

394 SDValue PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo);

395 SDValue PromoteIntOp_BUILD_VECTOR(SDNode *N);

396 SDValue PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, unsigned OpNo);

397 SDValue PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N);

398 SDValue PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N);

399 SDValue PromoteIntOp_INSERT_SUBVECTOR(SDNode *N);

400 SDValue PromoteIntOp_FAKE_USE(SDNode *N);

401 SDValue PromoteIntOp_CONCAT_VECTORS(SDNode *N);

402 SDValue PromoteIntOp_ScalarOp(SDNode *N);

403 SDValue PromoteIntOp_SELECT(SDNode *N, unsigned OpNo);

404 SDValue PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo);

405 SDValue PromoteIntOp_SETCC(SDNode *N, unsigned OpNo);

406 SDValue PromoteIntOp_Shift(SDNode *N);

407 SDValue PromoteIntOp_CMP(SDNode *N);

408 SDValue PromoteIntOp_FunnelShift(SDNode *N);

409 SDValue PromoteIntOp_SIGN_EXTEND(SDNode *N);

410 SDValue PromoteIntOp_VP_SIGN_EXTEND(SDNode *N);

411 SDValue PromoteIntOp_SINT_TO_FP(SDNode *N);

412 SDValue PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N);

413 SDValue PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo);

414 SDValue PromoteIntOp_TRUNCATE(SDNode *N);

415 SDValue PromoteIntOp_UINT_TO_FP(SDNode *N);

416 SDValue PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N);

417 SDValue PromoteIntOp_ZERO_EXTEND(SDNode *N);

418 SDValue PromoteIntOp_VP_ZERO_EXTEND(SDNode *N);

419 SDValue PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo);

420 SDValue PromoteIntOp_MLOAD(MaskedLoadSDNode *N, unsigned OpNo);

421 SDValue PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);

422 SDValue PromoteIntOp_MGATHER(MaskedGatherSDNode *N, unsigned OpNo);

423 SDValue PromoteIntOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo);

424 SDValue PromoteIntOp_FRAMERETURNADDR(SDNode *N);

425 SDValue PromoteIntOp_FIX(SDNode *N);

426 SDValue PromoteIntOp_ExpOp(SDNode *N);

427 SDValue PromoteIntOp_VECREDUCE(SDNode *N);

428 SDValue PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo);

429 SDValue PromoteIntOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo);

430 SDValue PromoteIntOp_SET_ROUNDING(SDNode *N);

431 SDValue PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo);

432 SDValue PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo);

433 SDValue PromoteIntOp_WRITE_REGISTER(SDNode *N, unsigned OpNo);

434 SDValue PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo);

435 SDValue PromoteIntOp_VP_SPLICE(SDNode *N, unsigned OpNo);

436 SDValue PromoteIntOp_VECTOR_HISTOGRAM(SDNode *N, unsigned OpNo);

437 SDValue PromoteIntOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N, unsigned OpNo);

438 SDValue PromoteIntOp_GET_ACTIVE_LANE_MASK(SDNode *N);

439 SDValue PromoteIntOp_PARTIAL_REDUCE_MLA(SDNode *N);

440 SDValue PromoteIntOp_LOOP_DEPENDENCE_MASK(SDNode *N, unsigned OpNo);

441

444

445

446

447

448

449

450

451

452

453

454

457

458

459 void ExpandIntegerResult(SDNode *N, unsigned ResNo);

460 void ExpandIntRes_ANY_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);

461 void ExpandIntRes_AssertSext (SDNode *N, SDValue &Lo, SDValue &Hi);

462 void ExpandIntRes_AssertZext (SDNode *N, SDValue &Lo, SDValue &Hi);

463 void ExpandIntRes_Constant (SDNode *N, SDValue &Lo, SDValue &Hi);

464 void ExpandIntRes_ABS (SDNode *N, SDValue &Lo, SDValue &Hi);

465 void ExpandIntRes_ABD (SDNode *N, SDValue &Lo, SDValue &Hi);

466 void ExpandIntRes_CTLZ (SDNode *N, SDValue &Lo, SDValue &Hi);

467 void ExpandIntRes_CTPOP (SDNode *N, SDValue &Lo, SDValue &Hi);

468 void ExpandIntRes_CTTZ (SDNode *N, SDValue &Lo, SDValue &Hi);

469 void ExpandIntRes_LOAD (LoadSDNode *N, SDValue &Lo, SDValue &Hi);

470 void ExpandIntRes_READCOUNTER (SDNode *N, SDValue &Lo, SDValue &Hi);

471 void ExpandIntRes_SIGN_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);

472 void ExpandIntRes_SIGN_EXTEND_INREG (SDNode *N, SDValue &Lo, SDValue &Hi);

473 void ExpandIntRes_TRUNCATE (SDNode *N, SDValue &Lo, SDValue &Hi);

474 void ExpandIntRes_ZERO_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);

475 void ExpandIntRes_GET_ROUNDING (SDNode *N, SDValue &Lo, SDValue &Hi);

476 void ExpandIntRes_FP_TO_XINT (SDNode *N, SDValue &Lo, SDValue &Hi);

477 void ExpandIntRes_FP_TO_XINT_SAT (SDNode *N, SDValue &Lo, SDValue &Hi);

478 void ExpandIntRes_XROUND_XRINT (SDNode *N, SDValue &Lo, SDValue &Hi);

479

480 void ExpandIntRes_Logical (SDNode *N, SDValue &Lo, SDValue &Hi);

481 void ExpandIntRes_ADDSUB (SDNode *N, SDValue &Lo, SDValue &Hi);

482 void ExpandIntRes_ADDSUBC (SDNode *N, SDValue &Lo, SDValue &Hi);

483 void ExpandIntRes_ADDSUBE (SDNode *N, SDValue &Lo, SDValue &Hi);

484 void ExpandIntRes_UADDSUBO_CARRY (SDNode *N, SDValue &Lo, SDValue &Hi);

485 void ExpandIntRes_SADDSUBO_CARRY (SDNode *N, SDValue &Lo, SDValue &Hi);

486 void ExpandIntRes_BITREVERSE (SDNode *N, SDValue &Lo, SDValue &Hi);

487 void ExpandIntRes_BSWAP (SDNode *N, SDValue &Lo, SDValue &Hi);

488 void ExpandIntRes_PARITY (SDNode *N, SDValue &Lo, SDValue &Hi);

489 void ExpandIntRes_MUL (SDNode *N, SDValue &Lo, SDValue &Hi);

490 void ExpandIntRes_SDIV (SDNode *N, SDValue &Lo, SDValue &Hi);

491 void ExpandIntRes_SREM (SDNode *N, SDValue &Lo, SDValue &Hi);

492 void ExpandIntRes_UDIV (SDNode *N, SDValue &Lo, SDValue &Hi);

493 void ExpandIntRes_UREM (SDNode *N, SDValue &Lo, SDValue &Hi);

494 void ExpandIntRes_ShiftThroughStack (SDNode *N, SDValue &Lo, SDValue &Hi);

495 void ExpandIntRes_Shift (SDNode *N, SDValue &Lo, SDValue &Hi);

496

497 void ExpandIntRes_MINMAX (SDNode *N, SDValue &Lo, SDValue &Hi);

498

499 void ExpandIntRes_CMP (SDNode *N, SDValue &Lo, SDValue &Hi);

500 void ExpandIntRes_SETCC (SDNode *N, SDValue &Lo, SDValue &Hi);

501

502 void ExpandIntRes_SADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi);

503 void ExpandIntRes_UADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi);

504 void ExpandIntRes_XMULO (SDNode *N, SDValue &Lo, SDValue &Hi);

505 void ExpandIntRes_AVG (SDNode *N, SDValue &Lo, SDValue &Hi);

506 void ExpandIntRes_ADDSUBSAT (SDNode *N, SDValue &Lo, SDValue &Hi);

507 void ExpandIntRes_SHLSAT (SDNode *N, SDValue &Lo, SDValue &Hi);

508 void ExpandIntRes_MULFIX (SDNode *N, SDValue &Lo, SDValue &Hi);

509 void ExpandIntRes_DIVFIX (SDNode *N, SDValue &Lo, SDValue &Hi);

510

511 void ExpandIntRes_ATOMIC_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi);

512 void ExpandIntRes_VECREDUCE (SDNode *N, SDValue &Lo, SDValue &Hi);

513

514 void ExpandIntRes_Rotate (SDNode *N, SDValue &Lo, SDValue &Hi);

515 void ExpandIntRes_FunnelShift (SDNode *N, SDValue &Lo, SDValue &Hi);

516

517 void ExpandIntRes_VSCALE (SDNode *N, SDValue &Lo, SDValue &Hi);

518 void ExpandIntRes_READ_REGISTER(SDNode *N, SDValue &Lo, SDValue &Hi);

519

520 void ExpandShiftByConstant(SDNode *N, const APInt &Amt,

522 bool ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);

523 bool ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);

524

525

526 bool ExpandIntegerOperand(SDNode *N, unsigned OpNo);

527 SDValue ExpandIntOp_BR_CC(SDNode *N);

528 SDValue ExpandIntOp_SELECT_CC(SDNode *N);

529 SDValue ExpandIntOp_SETCC(SDNode *N);

530 SDValue ExpandIntOp_SETCCCARRY(SDNode *N);

531 SDValue ExpandIntOp_Shift(SDNode *N);

532 SDValue ExpandIntOp_CMP(SDNode *N);

533 SDValue ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo);

534 SDValue ExpandIntOp_TRUNCATE(SDNode *N);

535 SDValue ExpandIntOp_XINT_TO_FP(SDNode *N);

536 SDValue ExpandIntOp_RETURNADDR(SDNode *N);

537 SDValue ExpandIntOp_ATOMIC_STORE(SDNode *N);

538 SDValue ExpandIntOp_SPLAT_VECTOR(SDNode *N);

539 SDValue ExpandIntOp_STACKMAP(SDNode *N, unsigned OpNo);

540 SDValue ExpandIntOp_PATCHPOINT(SDNode *N, unsigned OpNo);

541 SDValue ExpandIntOp_VP_STRIDED(SDNode *N, unsigned OpNo);

542 SDValue ExpandIntOp_WRITE_REGISTER(SDNode *N, unsigned OpNo);

543

544 void IntegerExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,

545 ISD::CondCode &CCCode, const SDLoc &dl);

546

547

548

549

550

551

552

553

554

555

557 TableId Id = getTableId(Op);

558 auto Iter = SoftenedFloats.find(Id);

559 if (Iter == SoftenedFloats.end()) {

560 assert(isSimpleLegalType(Op.getValueType()) &&

561 "Operand wasn't converted to integer?");

562 return Op;

563 }

564 SDValue SoftenedOp = getSDValue(Iter->second);

565 assert(SoftenedOp.getNode() && "Unconverted op in SoftenedFloats?");

566 return SoftenedOp;

567 }

569

570

571 void SoftenFloatResult(SDNode *N, unsigned ResNo);

572 SDValue SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC);

573 bool SoftenFloatRes_UnaryWithTwoFPResults(

574 SDNode *N, RTLIB::Libcall LC, std::optional CallRetResNo = {});

575 SDValue SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC);

576 SDValue SoftenFloatRes_MERGE_VALUES(SDNode *N, unsigned ResNo);

577 SDValue SoftenFloatRes_ARITH_FENCE(SDNode *N);

578 SDValue SoftenFloatRes_BITCAST(SDNode *N);

579 SDValue SoftenFloatRes_BUILD_PAIR(SDNode *N);

580 SDValue SoftenFloatRes_ConstantFP(SDNode *N);

581 SDValue SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N);

582 SDValue SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo);

583 SDValue SoftenFloatRes_FABS(SDNode *N);

584 SDValue SoftenFloatRes_FACOS(SDNode *N);

585 SDValue SoftenFloatRes_FASIN(SDNode *N);

586 SDValue SoftenFloatRes_FATAN(SDNode *N);

587 SDValue SoftenFloatRes_FATAN2(SDNode *N);

588 SDValue SoftenFloatRes_FMINNUM(SDNode *N);

589 SDValue SoftenFloatRes_FMAXNUM(SDNode *N);

590 SDValue SoftenFloatRes_FMINIMUMNUM(SDNode *N);

591 SDValue SoftenFloatRes_FMAXIMUMNUM(SDNode *N);

592 SDValue SoftenFloatRes_FMINIMUM(SDNode *N);

593 SDValue SoftenFloatRes_FMAXIMUM(SDNode *N);

594 SDValue SoftenFloatRes_FADD(SDNode *N);

595 SDValue SoftenFloatRes_FCBRT(SDNode *N);

596 SDValue SoftenFloatRes_FCEIL(SDNode *N);

597 SDValue SoftenFloatRes_FCOPYSIGN(SDNode *N);

598 SDValue SoftenFloatRes_FCOS(SDNode *N);

599 SDValue SoftenFloatRes_FCOSH(SDNode *N);

600 SDValue SoftenFloatRes_FDIV(SDNode *N);

601 SDValue SoftenFloatRes_FEXP(SDNode *N);

602 SDValue SoftenFloatRes_FEXP2(SDNode *N);

603 SDValue SoftenFloatRes_FEXP10(SDNode *N);

604 SDValue SoftenFloatRes_FFLOOR(SDNode *N);

605 SDValue SoftenFloatRes_FLOG(SDNode *N);

606 SDValue SoftenFloatRes_FLOG2(SDNode *N);

607 SDValue SoftenFloatRes_FLOG10(SDNode *N);

608 SDValue SoftenFloatRes_FMA(SDNode *N);

609 SDValue SoftenFloatRes_FMUL(SDNode *N);

610 SDValue SoftenFloatRes_FNEARBYINT(SDNode *N);

611 SDValue SoftenFloatRes_FNEG(SDNode *N);

612 SDValue SoftenFloatRes_FP_EXTEND(SDNode *N);

613 SDValue SoftenFloatRes_FP16_TO_FP(SDNode *N);

614 SDValue SoftenFloatRes_BF16_TO_FP(SDNode *N);

615 SDValue SoftenFloatRes_FP_ROUND(SDNode *N);

616 SDValue SoftenFloatRes_FPOW(SDNode *N);

617 SDValue SoftenFloatRes_ExpOp(SDNode *N);

618 SDValue SoftenFloatRes_FFREXP(SDNode *N);

619 SDValue SoftenFloatRes_FSINCOS(SDNode *N);

620 SDValue SoftenFloatRes_FMODF(SDNode *N);

621 SDValue SoftenFloatRes_FREEZE(SDNode *N);

622 SDValue SoftenFloatRes_FREM(SDNode *N);

623 SDValue SoftenFloatRes_FRINT(SDNode *N);

624 SDValue SoftenFloatRes_FROUND(SDNode *N);

625 SDValue SoftenFloatRes_FROUNDEVEN(SDNode *N);

626 SDValue SoftenFloatRes_FSIN(SDNode *N);

627 SDValue SoftenFloatRes_FSINH(SDNode *N);

628 SDValue SoftenFloatRes_FSQRT(SDNode *N);

629 SDValue SoftenFloatRes_FSUB(SDNode *N);

630 SDValue SoftenFloatRes_FTAN(SDNode *N);

631 SDValue SoftenFloatRes_FTANH(SDNode *N);

632 SDValue SoftenFloatRes_FTRUNC(SDNode *N);

633 SDValue SoftenFloatRes_LOAD(SDNode *N);

634 SDValue SoftenFloatRes_ATOMIC_LOAD(SDNode *N);

635 SDValue SoftenFloatRes_SELECT(SDNode *N);

636 SDValue SoftenFloatRes_SELECT_CC(SDNode *N);

637 SDValue SoftenFloatRes_UNDEF(SDNode *N);

638 SDValue SoftenFloatRes_VAARG(SDNode *N);

639 SDValue SoftenFloatRes_XINT_TO_FP(SDNode *N);

640 SDValue SoftenFloatRes_VECREDUCE(SDNode *N);

641 SDValue SoftenFloatRes_VECREDUCE_SEQ(SDNode *N);

642

643

644 bool SoftenFloatOperand(SDNode *N, unsigned OpNo);

645 SDValue SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC);

646 SDValue SoftenFloatOp_BITCAST(SDNode *N);

647 SDValue SoftenFloatOp_BR_CC(SDNode *N);

648 SDValue SoftenFloatOp_FP_ROUND(SDNode *N);

649 SDValue SoftenFloatOp_FP_TO_XINT(SDNode *N);

650 SDValue SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N);

651 SDValue SoftenFloatOp_LROUND(SDNode *N);

652 SDValue SoftenFloatOp_LLROUND(SDNode *N);

653 SDValue SoftenFloatOp_LRINT(SDNode *N);

654 SDValue SoftenFloatOp_LLRINT(SDNode *N);

655 SDValue SoftenFloatOp_SELECT_CC(SDNode *N);

656 SDValue SoftenFloatOp_SETCC(SDNode *N);

657 SDValue SoftenFloatOp_STORE(SDNode *N, unsigned OpNo);

658 SDValue SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo);

659 SDValue SoftenFloatOp_FCOPYSIGN(SDNode *N);

660 SDValue SoftenFloatOp_FAKE_USE(SDNode *N);

661 SDValue SoftenFloatOp_STACKMAP(SDNode *N, unsigned OpNo);

662 SDValue SoftenFloatOp_PATCHPOINT(SDNode *N, unsigned OpNo);

663

664

665

666

667

668

669

670

671

672

673

676

677

678 void ExpandFloatResult(SDNode *N, unsigned ResNo);

679 void ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, SDValue &Hi);

680 void ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,

682 void ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,

684 void ExpandFloatRes_UnaryWithTwoFPResults(

685 SDNode *N, RTLIB::Libcall LC, std::optional CallRetResNo = {});

686

687

688 void ExpandFloatRes_AssertNoFPClass(SDNode *N, SDValue &Lo, SDValue &Hi);

689 void ExpandFloatRes_FABS (SDNode *N, SDValue &Lo, SDValue &Hi);

690 void ExpandFloatRes_FACOS (SDNode *N, SDValue &Lo, SDValue &Hi);

691 void ExpandFloatRes_FASIN (SDNode *N, SDValue &Lo, SDValue &Hi);

692 void ExpandFloatRes_FATAN (SDNode *N, SDValue &Lo, SDValue &Hi);

693 void ExpandFloatRes_FATAN2 (SDNode *N, SDValue &Lo, SDValue &Hi);

694 void ExpandFloatRes_FMINNUM (SDNode *N, SDValue &Lo, SDValue &Hi);

695 void ExpandFloatRes_FMAXNUM (SDNode *N, SDValue &Lo, SDValue &Hi);

696 void ExpandFloatRes_FMINIMUMNUM(SDNode *N, SDValue &Lo, SDValue &Hi);

697 void ExpandFloatRes_FMAXIMUMNUM(SDNode *N, SDValue &Lo, SDValue &Hi);

698 void ExpandFloatRes_FADD (SDNode *N, SDValue &Lo, SDValue &Hi);

699 void ExpandFloatRes_FCBRT (SDNode *N, SDValue &Lo, SDValue &Hi);

700 void ExpandFloatRes_FCEIL (SDNode *N, SDValue &Lo, SDValue &Hi);

701 void ExpandFloatRes_FCOPYSIGN (SDNode *N, SDValue &Lo, SDValue &Hi);

702 void ExpandFloatRes_FCOS (SDNode *N, SDValue &Lo, SDValue &Hi);

703 void ExpandFloatRes_FCOSH (SDNode *N, SDValue &Lo, SDValue &Hi);

704 void ExpandFloatRes_FDIV (SDNode *N, SDValue &Lo, SDValue &Hi);

705 void ExpandFloatRes_FEXP (SDNode *N, SDValue &Lo, SDValue &Hi);

706 void ExpandFloatRes_FEXP2 (SDNode *N, SDValue &Lo, SDValue &Hi);

707 void ExpandFloatRes_FEXP10 (SDNode *N, SDValue &Lo, SDValue &Hi);

708 void ExpandFloatRes_FFLOOR (SDNode *N, SDValue &Lo, SDValue &Hi);

709 void ExpandFloatRes_FLOG (SDNode *N, SDValue &Lo, SDValue &Hi);

710 void ExpandFloatRes_FLOG2 (SDNode *N, SDValue &Lo, SDValue &Hi);

711 void ExpandFloatRes_FLOG10 (SDNode *N, SDValue &Lo, SDValue &Hi);

712 void ExpandFloatRes_FMA (SDNode *N, SDValue &Lo, SDValue &Hi);

713 void ExpandFloatRes_FMUL (SDNode *N, SDValue &Lo, SDValue &Hi);

714 void ExpandFloatRes_FNEARBYINT(SDNode *N, SDValue &Lo, SDValue &Hi);

715 void ExpandFloatRes_FNEG (SDNode *N, SDValue &Lo, SDValue &Hi);

716 void ExpandFloatRes_FP_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);

717 void ExpandFloatRes_FPOW (SDNode *N, SDValue &Lo, SDValue &Hi);

718 void ExpandFloatRes_FPOWI (SDNode *N, SDValue &Lo, SDValue &Hi);

719 void ExpandFloatRes_FLDEXP (SDNode *N, SDValue &Lo, SDValue &Hi);

720 void ExpandFloatRes_FREEZE (SDNode *N, SDValue &Lo, SDValue &Hi);

721 void ExpandFloatRes_FREM (SDNode *N, SDValue &Lo, SDValue &Hi);

722 void ExpandFloatRes_FRINT (SDNode *N, SDValue &Lo, SDValue &Hi);

723 void ExpandFloatRes_FROUND (SDNode *N, SDValue &Lo, SDValue &Hi);

724 void ExpandFloatRes_FROUNDEVEN(SDNode *N, SDValue &Lo, SDValue &Hi);

725 void ExpandFloatRes_FSIN (SDNode *N, SDValue &Lo, SDValue &Hi);

726 void ExpandFloatRes_FSINH (SDNode *N, SDValue &Lo, SDValue &Hi);

727 void ExpandFloatRes_FSQRT (SDNode *N, SDValue &Lo, SDValue &Hi);

728 void ExpandFloatRes_FSUB (SDNode *N, SDValue &Lo, SDValue &Hi);

729 void ExpandFloatRes_FTAN (SDNode *N, SDValue &Lo, SDValue &Hi);

730 void ExpandFloatRes_FTANH (SDNode *N, SDValue &Lo, SDValue &Hi);

731 void ExpandFloatRes_FTRUNC (SDNode *N, SDValue &Lo, SDValue &Hi);

732 void ExpandFloatRes_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi);

733 void ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, SDValue &Hi);

734 void ExpandFloatRes_FMODF(SDNode *N);

735 void ExpandFloatRes_FSINCOS(SDNode* N);

736 void ExpandFloatRes_FSINCOSPI(SDNode* N);

737

738

739

740 bool ExpandFloatOperand(SDNode *N, unsigned OpNo);

741 SDValue ExpandFloatOp_BR_CC(SDNode *N);

742 SDValue ExpandFloatOp_FCOPYSIGN(SDNode *N);

743 SDValue ExpandFloatOp_FP_ROUND(SDNode *N);

744 SDValue ExpandFloatOp_FP_TO_XINT(SDNode *N);

745 SDValue ExpandFloatOp_LROUND(SDNode *N);

746 SDValue ExpandFloatOp_LLROUND(SDNode *N);

747 SDValue ExpandFloatOp_LRINT(SDNode *N);

748 SDValue ExpandFloatOp_LLRINT(SDNode *N);

749 SDValue ExpandFloatOp_SELECT_CC(SDNode *N);

750 SDValue ExpandFloatOp_SETCC(SDNode *N);

751 SDValue ExpandFloatOp_STORE(SDNode *N, unsigned OpNo);

752

753 void FloatExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,

754 ISD::CondCode &CCCode, const SDLoc &dl,

755 SDValue &Chain, bool IsSignaling = false);

756

757

758

759

760

762 TableId &PromotedId = PromotedFloats[getTableId(Op)];

763 SDValue PromotedOp = getSDValue(PromotedId);

764 assert(PromotedOp.getNode() && "Operand wasn't promoted?");

765 return PromotedOp;

766 }

768

769 void PromoteFloatResult(SDNode *N, unsigned ResNo);

770 SDValue PromoteFloatRes_BITCAST(SDNode *N);

771 SDValue PromoteFloatRes_FREEZE(SDNode *N);

772 SDValue PromoteFloatRes_BinOp(SDNode *N);

773 SDValue PromoteFloatRes_UnaryWithTwoFPResults(SDNode *N);

774 SDValue PromoteFloatRes_ConstantFP(SDNode *N);

775 SDValue PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N);

776 SDValue PromoteFloatRes_FCOPYSIGN(SDNode *N);

777 SDValue PromoteFloatRes_FMAD(SDNode *N);

778 SDValue PromoteFloatRes_ExpOp(SDNode *N);

779 SDValue PromoteFloatRes_FFREXP(SDNode *N);

780 SDValue PromoteFloatRes_FP_ROUND(SDNode *N);

781 SDValue PromoteFloatRes_STRICT_FP_ROUND(SDNode *N);

782 SDValue PromoteFloatRes_LOAD(SDNode *N);

783 SDValue PromoteFloatRes_ATOMIC_LOAD(SDNode *N);

784 SDValue PromoteFloatRes_SELECT(SDNode *N);

785 SDValue PromoteFloatRes_SELECT_CC(SDNode *N);

786 SDValue PromoteFloatRes_UnaryOp(SDNode *N);

787 SDValue PromoteFloatRes_AssertNoFPClass(SDNode *N);

788 SDValue PromoteFloatRes_UNDEF(SDNode *N);

789 SDValue BitcastToInt_ATOMIC_SWAP(SDNode *N);

790 SDValue PromoteFloatRes_XINT_TO_FP(SDNode *N);

791 SDValue PromoteFloatRes_VECREDUCE(SDNode *N);

792 SDValue PromoteFloatRes_VECREDUCE_SEQ(SDNode *N);

793

794 bool PromoteFloatOperand(SDNode *N, unsigned OpNo);

795 SDValue PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo);

796 SDValue PromoteFloatOp_FAKE_USE(SDNode *N, unsigned OpNo);

797 SDValue PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo);

798 SDValue PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo);

799 SDValue PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N, unsigned OpNo);

800 SDValue PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo);

801 SDValue PromoteFloatOp_AssertNoFPClass(SDNode *N, unsigned OpNo);

802 SDValue PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, unsigned OpNo);

803 SDValue PromoteFloatOp_STORE(SDNode *N, unsigned OpNo);

804 SDValue PromoteFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo);

805 SDValue PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo);

806 SDValue PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo);

807

808

809

810

811

813 TableId &PromotedId = SoftPromotedHalfs[getTableId(Op)];

814 SDValue PromotedOp = getSDValue(PromotedId);

815 assert(PromotedOp.getNode() && "Operand wasn't promoted?");

816 return PromotedOp;

817 }

818 void SetSoftPromotedHalf(SDValue Op, SDValue Result);

819

820 void SoftPromoteHalfResult(SDNode *N, unsigned ResNo);

821 SDValue SoftPromoteHalfRes_ARITH_FENCE(SDNode *N);

822 SDValue SoftPromoteHalfRes_BinOp(SDNode *N);

823 SDValue SoftPromoteHalfRes_UnaryWithTwoFPResults(SDNode *N);

824 SDValue SoftPromoteHalfRes_BITCAST(SDNode *N);

825 SDValue SoftPromoteHalfRes_ConstantFP(SDNode *N);

826 SDValue SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N);

827 SDValue SoftPromoteHalfRes_FCOPYSIGN(SDNode *N);

828 SDValue SoftPromoteHalfRes_FMAD(SDNode *N);

829 SDValue SoftPromoteHalfRes_ExpOp(SDNode *N);

830 SDValue SoftPromoteHalfRes_FFREXP(SDNode *N);

831 SDValue SoftPromoteHalfRes_FP_ROUND(SDNode *N);

832 SDValue SoftPromoteHalfRes_LOAD(SDNode *N);

833 SDValue SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N);

834 SDValue SoftPromoteHalfRes_SELECT(SDNode *N);

835 SDValue SoftPromoteHalfRes_SELECT_CC(SDNode *N);

836 SDValue SoftPromoteHalfRes_UnaryOp(SDNode *N);

837 SDValue SoftPromoteHalfRes_FABS(SDNode *N);

838 SDValue SoftPromoteHalfRes_FNEG(SDNode *N);

839 SDValue SoftPromoteHalfRes_AssertNoFPClass(SDNode *N);

840 SDValue SoftPromoteHalfRes_XINT_TO_FP(SDNode *N);

841 SDValue SoftPromoteHalfRes_UNDEF(SDNode *N);

842 SDValue SoftPromoteHalfRes_VECREDUCE(SDNode *N);

843 SDValue SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N);

844

845 bool SoftPromoteHalfOperand(SDNode *N, unsigned OpNo);

846 SDValue SoftPromoteHalfOp_BITCAST(SDNode *N);

847 SDValue SoftPromoteHalfOp_FAKE_USE(SDNode *N, unsigned OpNo);

848 SDValue SoftPromoteHalfOp_FCOPYSIGN(SDNode *N, unsigned OpNo);

849 SDValue SoftPromoteHalfOp_FP_EXTEND(SDNode *N);

850 SDValue SoftPromoteHalfOp_Op0WithStrict(SDNode *N);

851 SDValue SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N);

852 SDValue SoftPromoteHalfOp_SETCC(SDNode *N);

853 SDValue SoftPromoteHalfOp_SELECT_CC(SDNode *N, unsigned OpNo);

854 SDValue SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo);

855 SDValue SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N, unsigned OpNo);

856 SDValue SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo);

857 SDValue SoftPromoteHalfOp_PATCHPOINT(SDNode *N, unsigned OpNo);

858

859

860

861

862

863

864

865

867 TableId &ScalarizedId = ScalarizedVectors[getTableId(Op)];

868 SDValue ScalarizedOp = getSDValue(ScalarizedId);

869 assert(ScalarizedOp.getNode() && "Operand wasn't scalarized?");

870 return ScalarizedOp;

871 }

872 void SetScalarizedVector(SDValue Op, SDValue Result);

873

874

875 void ScalarizeVectorResult(SDNode *N, unsigned ResNo);

876 SDValue ScalarizeVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo);

877 SDValue ScalarizeVecRes_LOOP_DEPENDENCE_MASK(SDNode *N);

878 SDValue ScalarizeVecRes_BinOp(SDNode *N);

879 SDValue ScalarizeVecRes_CMP(SDNode *N);

880 SDValue ScalarizeVecRes_TernaryOp(SDNode *N);

881 SDValue ScalarizeVecRes_UnaryOp(SDNode *N);

882 SDValue ScalarizeVecRes_StrictFPOp(SDNode *N);

883 SDValue ScalarizeVecRes_OverflowOp(SDNode *N, unsigned ResNo);

884 SDValue ScalarizeVecRes_InregOp(SDNode *N);

885 SDValue ScalarizeVecRes_VecInregOp(SDNode *N);

886

887 SDValue ScalarizeVecRes_ADDRSPACECAST(SDNode *N);

888 SDValue ScalarizeVecRes_BITCAST(SDNode *N);

889 SDValue ScalarizeVecRes_BUILD_VECTOR(SDNode *N);

890 SDValue ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N);

891 SDValue ScalarizeVecRes_FP_ROUND(SDNode *N);

892 SDValue ScalarizeVecRes_UnaryOpWithExtraInput(SDNode *N);

893 SDValue ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N);

894 SDValue ScalarizeVecRes_LOAD(LoadSDNode *N);

895 SDValue ScalarizeVecRes_ATOMIC_LOAD(AtomicSDNode *N);

896 SDValue ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N);

897 SDValue ScalarizeVecRes_VSELECT(SDNode *N);

898 SDValue ScalarizeVecRes_SELECT(SDNode *N);

899 SDValue ScalarizeVecRes_SELECT_CC(SDNode *N);

900 SDValue ScalarizeVecRes_SETCC(SDNode *N);

901 SDValue ScalarizeVecRes_UNDEF(SDNode *N);

902 SDValue ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N);

903 SDValue ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N);

904 SDValue ScalarizeVecRes_IS_FPCLASS(SDNode *N);

905

906 SDValue ScalarizeVecRes_FIX(SDNode *N);

907 SDValue ScalarizeVecRes_UnaryOpWithTwoResults(SDNode *N, unsigned ResNo);

908

909

910 bool ScalarizeVectorOperand(SDNode *N, unsigned OpNo);

911 SDValue ScalarizeVecOp_BITCAST(SDNode *N);

912 SDValue ScalarizeVecOp_UnaryOp(SDNode *N);

913 SDValue ScalarizeVecOp_UnaryOpWithExtraInput(SDNode *N);

914 SDValue ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N);

915 SDValue ScalarizeVecOp_CONCAT_VECTORS(SDNode *N);

916 SDValue ScalarizeVecOp_INSERT_SUBVECTOR(SDNode *N, unsigned OpNo);

917 SDValue ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N);

918 SDValue ScalarizeVecOp_VSELECT(SDNode *N);

919 SDValue ScalarizeVecOp_VSETCC(SDNode *N);

920 SDValue ScalarizeVecOp_VSTRICT_FSETCC(SDNode *N, unsigned OpNo);

921 SDValue ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo);

922 SDValue ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo);

923 SDValue ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N, unsigned OpNo);

924 SDValue ScalarizeVecOp_FP_EXTEND(SDNode *N);

925 SDValue ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N);

926 SDValue ScalarizeVecOp_VECREDUCE(SDNode *N);

927 SDValue ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N);

928 SDValue ScalarizeVecOp_CMP(SDNode *N);

929 SDValue ScalarizeVecOp_FAKE_USE(SDNode *N);

930

931

932

933

934

935

936

937

938

939

940

941

944

945

946 std::pair<SDValue, SDValue> SplitMask(SDValue Mask);

947

948

949 std::pair<SDValue, SDValue> SplitMask(SDValue Mask, const SDLoc &DL);

950

951

952

953 void IncrementPointer(MemSDNode *N, EVT MemVT, MachinePointerInfo &MPI,

954 SDValue &Ptr, uint64_t *ScaledOffset = nullptr);

955

956

957 void SplitVectorResult(SDNode *N, unsigned ResNo);

958 void SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi);

959 void SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);

960 void SplitVecRes_CMP(SDNode *N, SDValue &Lo, SDValue &Hi);

961 void SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);

962 void SplitVecRes_ADDRSPACECAST(SDNode *N, SDValue &Lo, SDValue &Hi);

963 void SplitVecRes_UnaryOpWithTwoResults(SDNode *N, unsigned ResNo, SDValue &Lo,

965 void SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, SDValue &Hi);

966 void SplitVecRes_InregOp(SDNode *N, SDValue &Lo, SDValue &Hi);

967 void SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, SDValue &Hi);

968 void SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, SDValue &Hi);

969 void SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,

971

972 void SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi);

973

974 void SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi);

975 void SplitVecRes_LOOP_DEPENDENCE_MASK(SDNode *N, SDValue &Lo, SDValue &Hi);

976 void SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);

977 void SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, SDValue &Hi);

978 void SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);

979 void SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);

980 void SplitVecRes_FPOp_MultiType(SDNode *N, SDValue &Lo, SDValue &Hi);

981 void SplitVecRes_IS_FPCLASS(SDNode *N, SDValue &Lo, SDValue &Hi);

982 void SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi);

983 void SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, SDValue &Hi);

984 void SplitVecRes_VP_LOAD(VPLoadSDNode *LD, SDValue &Lo, SDValue &Hi);

985 void SplitVecRes_VP_LOAD_FF(VPLoadFFSDNode *LD, SDValue &Lo, SDValue &Hi);

986 void SplitVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *SLD, SDValue &Lo,

988 void SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, SDValue &Lo, SDValue &Hi);

989 void SplitVecRes_Gather(MemSDNode *VPGT, SDValue &Lo, SDValue &Hi,

990 bool SplitSETCC = false);

991 void SplitVecRes_VECTOR_COMPRESS(SDNode *N, SDValue &Lo, SDValue &Hi);

992 void SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo, SDValue &Hi);

993 void SplitVecRes_VP_SPLAT(SDNode *N, SDValue &Lo, SDValue &Hi);

994 void SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);

995 void SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi);

996 void SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo, SDValue &Hi);

997 void SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, SDValue &Lo,

999 void SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo, SDValue &Hi);

1000 void SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N);

1001 void SplitVecRes_VECTOR_INTERLEAVE(SDNode *N);

1002 void SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi);

1003 void SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, SDValue &Hi);

1004 void SplitVecRes_VP_SPLICE(SDNode *N, SDValue &Lo, SDValue &Hi);

1005 void SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo, SDValue &Hi);

1006 void SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo, SDValue &Hi);

1007 void SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo, SDValue &Hi);

1008

1009

1010 bool SplitVectorOperand(SDNode *N, unsigned OpNo);

1011 SDValue SplitVecOp_VSELECT(SDNode *N, unsigned OpNo);

1012 SDValue SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo);

1013 SDValue SplitVecOp_VECREDUCE_SEQ(SDNode *N);

1014 SDValue SplitVecOp_VP_REDUCE(SDNode *N, unsigned OpNo);

1015 SDValue SplitVecOp_UnaryOp(SDNode *N);

1016 SDValue SplitVecOp_TruncateHelper(SDNode *N);

1017 SDValue SplitVecOp_VECTOR_COMPRESS(SDNode *N, unsigned OpNo);

1018

1019 SDValue SplitVecOp_BITCAST(SDNode *N);

1020 SDValue SplitVecOp_INSERT_SUBVECTOR(SDNode *N, unsigned OpNo);

1021 SDValue SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N);

1022 SDValue SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N);

1023 SDValue SplitVecOp_ExtVecInRegOp(SDNode *N);

1024 SDValue SplitVecOp_FAKE_USE(SDNode *N);

1025 SDValue SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo);

1026 SDValue SplitVecOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo);

1027 SDValue SplitVecOp_VP_STRIDED_STORE(VPStridedStoreSDNode *N, unsigned OpNo);

1028 SDValue SplitVecOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo);

1029 SDValue SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo);

1030 SDValue SplitVecOp_Gather(MemSDNode *MGT, unsigned OpNo);

1031 SDValue SplitVecOp_CONCAT_VECTORS(SDNode *N);

1032 SDValue SplitVecOp_VSETCC(SDNode *N);

1033 SDValue SplitVecOp_FP_ROUND(SDNode *N);

1034 SDValue SplitVecOp_FPOpDifferentTypes(SDNode *N);

1035 SDValue SplitVecOp_CMP(SDNode *N);

1036 SDValue SplitVecOp_FP_TO_XINT_SAT(SDNode *N);

1037 SDValue SplitVecOp_VP_CttzElements(SDNode *N);

1038 SDValue SplitVecOp_VECTOR_HISTOGRAM(SDNode *N);

1039 SDValue SplitVecOp_PARTIAL_REDUCE_MLA(SDNode *N);

1040

1041

1042

1043

1044

1045

1046

1047

1048

1049

1050

1052 TableId &WidenedId = WidenedVectors[getTableId(Op)];

1053 SDValue WidenedOp = getSDValue(WidenedId);

1054 assert(WidenedOp.getNode() && "Operand wasn't widened?");

1055 return WidenedOp;

1056 }

1058

1059

1060 SDValue GetWidenedMask(SDValue Mask, ElementCount EC) {

1061

1062

1063

1064

1065

1066

1067 assert(getTypeAction(Mask.getValueType()) ==

1068 TargetLowering::TypeWidenVector &&

1069 "Unable to widen binary VP op");

1070 Mask = GetWidenedVector(Mask);

1071 assert(Mask.getValueType().getVectorElementCount() == EC &&

1072 "Unable to widen binary VP op");

1073 return Mask;

1074 }

1075

1076

1077 void WidenVectorResult(SDNode *N, unsigned ResNo);

1078 SDValue WidenVecRes_MERGE_VALUES(SDNode* N, unsigned ResNo);

1079 SDValue WidenVecRes_ADDRSPACECAST(SDNode *N);

1080 SDValue WidenVecRes_AssertZext(SDNode* N);

1081 SDValue WidenVecRes_BITCAST(SDNode* N);

1082 SDValue WidenVecRes_LOOP_DEPENDENCE_MASK(SDNode *N);

1083 SDValue WidenVecRes_BUILD_VECTOR(SDNode* N);

1084 SDValue WidenVecRes_CONCAT_VECTORS(SDNode* N);

1085 SDValue WidenVecRes_EXTEND_VECTOR_INREG(SDNode* N);

1086 SDValue WidenVecRes_EXTRACT_SUBVECTOR(SDNode* N);

1087 SDValue WidenVecRes_INSERT_SUBVECTOR(SDNode *N);

1088 SDValue WidenVecRes_INSERT_VECTOR_ELT(SDNode* N);

1089 SDValue WidenVecRes_LOAD(SDNode* N);

1090 SDValue WidenVecRes_VP_LOAD(VPLoadSDNode *N);

1091 SDValue WidenVecRes_VP_LOAD_FF(VPLoadFFSDNode *N);

1092 SDValue WidenVecRes_VP_STRIDED_LOAD(VPStridedLoadSDNode *N);

1093 SDValue WidenVecRes_VECTOR_COMPRESS(SDNode *N);

1094 SDValue WidenVecRes_MLOAD(MaskedLoadSDNode* N);

1095 SDValue WidenVecRes_MGATHER(MaskedGatherSDNode* N);

1096 SDValue WidenVecRes_VP_GATHER(VPGatherSDNode* N);

1097 SDValue WidenVecRes_ScalarOp(SDNode* N);

1098 SDValue WidenVecRes_Select(SDNode *N);

1099 SDValue WidenVSELECTMask(SDNode *N);

1100 SDValue WidenVecRes_SELECT_CC(SDNode* N);

1101 SDValue WidenVecRes_SETCC(SDNode* N);

1102 SDValue WidenVecRes_STRICT_FSETCC(SDNode* N);

1103 SDValue WidenVecRes_UNDEF(SDNode *N);

1104 SDValue WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N);

1105 SDValue WidenVecRes_VECTOR_REVERSE(SDNode *N);

1106 SDValue WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N);

1107

1108 SDValue WidenVecRes_Ternary(SDNode *N);

1109 SDValue WidenVecRes_Binary(SDNode *N);

1110 SDValue WidenVecRes_CMP(SDNode *N);

1111 SDValue WidenVecRes_BinaryCanTrap(SDNode *N);

1112 SDValue WidenVecRes_BinaryWithExtraScalarOp(SDNode *N);

1113 SDValue WidenVecRes_StrictFP(SDNode *N);

1114 SDValue WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo);

1115 SDValue WidenVecRes_Convert(SDNode *N);

1116 SDValue WidenVecRes_Convert_StrictFP(SDNode *N);

1117 SDValue WidenVecRes_FP_TO_XINT_SAT(SDNode *N);

1118 SDValue WidenVecRes_XROUND(SDNode *N);

1119 SDValue WidenVecRes_FCOPYSIGN(SDNode *N);

1120 SDValue WidenVecRes_UnarySameEltsWithScalarArg(SDNode *N);

1121 SDValue WidenVecRes_ExpOp(SDNode *N);

1122 SDValue WidenVecRes_Unary(SDNode *N);

1123 SDValue WidenVecRes_InregOp(SDNode *N);

1124 SDValue WidenVecRes_UnaryOpWithTwoResults(SDNode *N, unsigned ResNo);

1125 void ReplaceOtherWidenResults(SDNode *N, SDNode *WidenNode,

1126 unsigned WidenResNo);

1127

1128

1129 bool WidenVectorOperand(SDNode *N, unsigned OpNo);

1130 SDValue WidenVecOp_BITCAST(SDNode *N);

1131 SDValue WidenVecOp_CONCAT_VECTORS(SDNode *N);

1132 SDValue WidenVecOp_EXTEND(SDNode *N);

1133 SDValue WidenVecOp_CMP(SDNode *N);

1134 SDValue WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N);

1135 SDValue WidenVecOp_INSERT_SUBVECTOR(SDNode *N);

1136 SDValue WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N);

1137 SDValue WidenVecOp_EXTEND_VECTOR_INREG(SDNode *N);

1138 SDValue WidenVecOp_FAKE_USE(SDNode *N);

1139 SDValue WidenVecOp_STORE(SDNode* N);

1140 SDValue WidenVecOp_VP_STORE(SDNode *N, unsigned OpNo);

1141 SDValue WidenVecOp_VP_STRIDED_STORE(SDNode *N, unsigned OpNo);

1142 SDValue WidenVecOp_MSTORE(SDNode* N, unsigned OpNo);

1143 SDValue WidenVecOp_MGATHER(SDNode* N, unsigned OpNo);

1144 SDValue WidenVecOp_MSCATTER(SDNode* N, unsigned OpNo);

1145 SDValue WidenVecOp_VP_SCATTER(SDNode* N, unsigned OpNo);

1146 SDValue WidenVecOp_VP_SPLAT(SDNode *N, unsigned OpNo);

1147 SDValue WidenVecOp_SETCC(SDNode* N);

1148 SDValue WidenVecOp_STRICT_FSETCC(SDNode* N);

1149 SDValue WidenVecOp_VSELECT(SDNode *N);

1150

1151 SDValue WidenVecOp_Convert(SDNode *N);

1152 SDValue WidenVecOp_FP_TO_XINT_SAT(SDNode *N);

1153 SDValue WidenVecOp_UnrollVectorOp(SDNode *N);

1154 SDValue WidenVecOp_IS_FPCLASS(SDNode *N);

1155 SDValue WidenVecOp_VECREDUCE(SDNode *N);

1156 SDValue WidenVecOp_VECREDUCE_SEQ(SDNode *N);

1157 SDValue WidenVecOp_VP_REDUCE(SDNode *N);

1158 SDValue WidenVecOp_ExpOp(SDNode *N);

1159 SDValue WidenVecOp_VP_CttzElements(SDNode *N);

1160

1161

1162

1163

1164 SDValue UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE);

1165

1166

1167

1168

1169

1170

1171

1172

1173

1174 SDValue GenWidenVectorLoads(SmallVectorImpl &LdChain,

1175 LoadSDNode *LD);

1176

1177

1178

1179

1180

1181

1182 SDValue GenWidenVectorExtLoads(SmallVectorImpl &LdChain,

1183 LoadSDNode *LD, ISD::LoadExtType ExtType);

1184

1185

1186

1187

1188

1189 bool GenWidenVectorStores(SmallVectorImpl &StChain, StoreSDNode *ST);

1190

1191

1192

1193

1194

1195 SDValue ModifyToType(SDValue InOp, EVT NVT, bool FillWithZeroes = false);

1196

1197

1198

1199 SDValue convertMask(SDValue InMask, EVT MaskVT, EVT ToMaskVT);

1200

1201

1202

1203

1204

1205

1206

1207

1208

1210 if (Op.getValueType().isVector())

1211 GetSplitVector(Op, Lo, Hi);

1212 else if (Op.getValueType().isInteger())

1213 GetExpandedInteger(Op, Lo, Hi);

1214 else

1215 GetExpandedFloat(Op, Lo, Hi);

1216 }

1217

1218

1219

1221

1222

1223 void SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo,

1225 void SplitVecRes_AssertZext (SDNode *N, SDValue &Lo, SDValue &Hi);

1226 void SplitRes_ARITH_FENCE (SDNode *N, SDValue &Lo, SDValue &Hi);

1227 void SplitRes_Select (SDNode *N, SDValue &Lo, SDValue &Hi);

1228 void SplitRes_SELECT_CC (SDNode *N, SDValue &Lo, SDValue &Hi);

1229 void SplitRes_UNDEF (SDNode *N, SDValue &Lo, SDValue &Hi);

1230 void SplitRes_FREEZE (SDNode *N, SDValue &Lo, SDValue &Hi);

1231

1232

1233

1234

1235

1236

1237

1238

1239

1240

1242 if (Op.getValueType().isInteger())

1243 GetExpandedInteger(Op, Lo, Hi);

1244 else

1245 GetExpandedFloat(Op, Lo, Hi);

1246 }

1247

1248

1249

1250

1251 void IntegerToVector(SDValue Op, unsigned NumElements,

1252 SmallVectorImpl &Ops, EVT EltVT);

1253

1254

1255 void ExpandRes_MERGE_VALUES (SDNode *N, unsigned ResNo,

1257 void ExpandRes_BITCAST (SDNode *N, SDValue &Lo, SDValue &Hi);

1258 void ExpandRes_BUILD_PAIR (SDNode *N, SDValue &Lo, SDValue &Hi);

1259 void ExpandRes_EXTRACT_ELEMENT (SDNode *N, SDValue &Lo, SDValue &Hi);

1260 void ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi);

1261 void ExpandRes_NormalLoad (SDNode *N, SDValue &Lo, SDValue &Hi);

1262 void ExpandRes_VAARG (SDNode *N, SDValue &Lo, SDValue &Hi);

1263

1264

1265 SDValue ExpandOp_BITCAST (SDNode *N);

1266 SDValue ExpandOp_BUILD_VECTOR (SDNode *N);

1267 SDValue ExpandOp_EXTRACT_ELEMENT (SDNode *N);

1268 SDValue ExpandOp_FAKE_USE(SDNode *N);

1269 SDValue ExpandOp_INSERT_VECTOR_ELT(SDNode *N);

1270 SDValue ExpandOp_SCALAR_TO_VECTOR (SDNode *N);

1271 SDValue ExpandOp_NormalStore (SDNode *N, unsigned OpNo);

1272};