InstCombineVectorOps.cpp Source File (original) (raw)

39#include

40#include

41#include

42#include

44#define DEBUG_TYPE "instcombine"

46using namespace llvm;

49 STATISTIC(NumAggregateReconstructionsSimplified,

50 "Number of aggregate reconstructions turned into reuse of the "

51 "original aggregate");

63 return CEI || C->getSplatValue();

69 return CEI->getValue().ult(EC.getKnownMinValue());

70 }

76 return CEI;

79 return true;

82 return true;

87 return true;

92 return true;

94 return false;

95}

102 SmallVector<Instruction *, 2> Extracts;

103

104

105

106

108 for (auto *U : PN->users()) {

112 else

113 return nullptr;

114 } else if (!PHIUser) {

116 } else {

117 return nullptr;

118 }

119 }

120

121 if (!PHIUser)

122 return nullptr;

123

124

125

126

130 return nullptr;

131

132

133

136

141

142 if (PHIInVal == PHIUser) {

143

144

145

147 unsigned opId = (B0->getOperand(0) == PN) ? 1 : 0;

152

153

156 Value *newPHIUser =

158 B0->getOpcode(), FirstOp, SecondOp, B0),

160 scalarPHI->addIncoming(newPHIUser, inBB);

161 } else {

162

164

169 } else {

171 }

172

174

176 }

177 }

178

179 for (auto *E : Extracts) {

181

183 }

184

185 return &EI;

186}

187

190 uint64_t ExtIndexC;

193 return nullptr;

194

195 ElementCount NumElts =

199 bool IsBigEndian = DL.isBigEndian();

200

201

202

203 if (X->getType()->isIntegerTy()) {

205 "Expected fixed vector type for bitcast from scalar integer");

206

207

208

209

210 if (IsBigEndian)

212 unsigned ShiftAmountC = ExtIndexC * DestWidth;

213 if ((!ShiftAmountC ||

214 isDesirableIntType(X->getType()->getPrimitiveSizeInBits())) &&

216 if (ShiftAmountC)

217 X = Builder.CreateLShr(X, ShiftAmountC, "extelt.offset");

221 return new BitCastInst(Trunc, DestTy);

222 }

223 return new TruncInst(X, DestTy);

224 }

225 }

226

227 if (->getType()->isVectorTy())

228 return nullptr;

229

230

231

232

234 ElementCount NumSrcElts = SrcTy->getElementCount();

235 if (NumSrcElts == NumElts)

237 return new BitCastInst(Elt, DestTy);

238

240 "Src and Dst must be the same sort of vector type");

241

242

243

247 uint64_t InsIndexC;

250 return nullptr;

251

252

253

254

255

256 unsigned NarrowingRatio =

258

259 if (ExtIndexC / NarrowingRatio != InsIndexC) {

260

261

262

263

264

268 }

269 return nullptr;

270 }

271

272

273

274

275

276

277

278

279

280

281

282

283 unsigned Chunk = ExtIndexC % NarrowingRatio;

284 if (IsBigEndian)

285 Chunk = NarrowingRatio - 1 - Chunk;

286

287

288

289

290 bool NeedSrcBitcast = SrcTy->getScalarType()->isFloatingPointTy();

292 if (NeedSrcBitcast && NeedDestBitcast)

293 return nullptr;

294

295 unsigned SrcWidth = SrcTy->getScalarSizeInBits();

296 unsigned ShAmt = Chunk * DestWidth;

297

298

299

300

302 if (NeedSrcBitcast || NeedDestBitcast)

303 return nullptr;

304

305 if (NeedSrcBitcast) {

307 Scalar = Builder.CreateBitCast(Scalar, SrcIntTy);

308 }

309

310 if (ShAmt) {

311

313 return nullptr;

315 }

316

317 if (NeedDestBitcast) {

319 return new BitCastInst(Builder.CreateTrunc(Scalar, DestIntTy), DestTy);

320 }

321 return new TruncInst(Scalar, DestTy);

322 }

323

324 return nullptr;

325}

326

327

328

330 APInt &UnionUsedElts) {

332

333 switch (UserInstr->getOpcode()) {

334 case Instruction::ExtractElement: {

338 if (EEIIndexC && EEIIndexC->getValue().ult(VWidth)) {

340 return true;

341 }

342 break;

343 }

344 case Instruction::ShuffleVector: {

346 unsigned MaskNumElts =

348

349 for (auto I : llvm::seq(MaskNumElts)) {

351 if (MaskVal == -1u || MaskVal >= 2 * VWidth)

352 continue;

353 if (Shuffle->getOperand(0) == V && (MaskVal < VWidth))

354 UnionUsedElts.setBit(MaskVal);

356 ((MaskVal >= VWidth) && (MaskVal < 2 * VWidth)))

357 UnionUsedElts.setBit(MaskVal - VWidth);

358 }

359 return true;

360 }

361 default:

362 break;

363 }

364

365 return false;

366}

367

368

369

370

371

374

375 APInt UnionUsedElts(VWidth, 0);

376 for (const Use &U : V->uses()) {

380 } else {

382 break;

383 }

384

386 break;

387 }

388

389 return UnionUsedElts;

390}

391

392

393

394

395

397 const unsigned IndexBW = IndexC->getBitWidth();

399 return nullptr;

400 return ConstantInt::get(IndexC->getContext(),

402}

403

408 SQ.getWithInstruction(&EI)))

410

411

412

413

414

415

416

417

418

420 if (SI->getCondition()->getType()->isIntegerTy() &&

423 return R;

424

425

426

428 bool HasKnownValidIndex = false;

429 if (IndexC) {

430

433

435 unsigned NumElts = EC.getKnownMinValue();

436 HasKnownValidIndex = IndexC->getValue().ult(NumElts);

437

440

441

442 if (IID == Intrinsic::stepvector && IndexC->getValue().ult(NumElts)) {

444 unsigned BitWidth = Ty->getIntegerBitWidth();

446

447

448 if (IndexC->getValue().getActiveBits() <= BitWidth)

449 Idx = ConstantInt::get(Ty, IndexC->getValue().zextOrTrunc(BitWidth));

450 else

453 }

454 }

455

456

457

458 if (!EC.isScalable() && IndexC->getValue().uge(NumElts))

459 return nullptr;

460

462 return I;

463

464

465

467 if (Instruction *ScalarPHI = scalarizePHI(EI, Phi))

468 return ScalarPHI;

469 }

470

471

472

474 if (match(SrcVec,

477

478

479

482

484 Value *E = Builder.CreateExtractElement(X, Index);

486 }

487

488

489

492 (HasKnownValidIndex ||

494

496 Value *E0 = Builder.CreateExtractElement(X, Index);

497 Value *E1 = Builder.CreateExtractElement(Y, Index);

499 }

500

505

506 Value *E0 = Builder.CreateExtractElement(X, Index);

507 Value *E1 = Builder.CreateExtractElement(Y, Index);

510 SrcCmpInst);

511 }

512

515

516

517

522 ElementCount EC = VecType->getElementCount();

523 uint64_t IdxVal = IndexC ? IndexC->getZExtValue() : 0;

524 if (IndexC && IdxVal < EC.getKnownMinValue() && GEP->hasOneUse()) {

525

526

527

528

529

530

531

532

533 unsigned VectorOps =

535 return isa(V->getType());

536 });

537 if (VectorOps == 1) {

538 Value *NewPtr = GEP->getPointerOperand();

540 NewPtr = Builder.CreateExtractElement(NewPtr, IndexC);

541

543 for (unsigned I = 1; I != GEP->getNumOperands(); ++I) {

547 else

549 }

550

552 GEP->getSourceElementType(), NewPtr, NewOps);

554 return NewGEP;

555 }

556 }

558 int SplatIndex = getSplatIndex(SVI->getShuffleMask());

559

560

561 if (SplatIndex == 0)

564

566 std::optional SrcIdx;

567

568 if (SplatIndex != -1)

569 SrcIdx = SplatIndex;

571 SrcIdx = SVI->getMaskValue(CI->getZExtValue());

572

573 if (SrcIdx) {

575 unsigned LHSWidth =

577 ->getNumElements();

578

579 if (*SrcIdx < 0)

581 if (*SrcIdx < (int)LHSWidth)

582 Src = SVI->getOperand(0);

583 else {

584 *SrcIdx -= LHSWidth;

585 Src = SVI->getOperand(1);

586 }

589 Src, ConstantInt::get(Int64Ty, *SrcIdx, false));

590 }

591 }

593

594

595

596 if (CI->hasOneUse() && (CI->getOpcode() != Instruction::BitCast)) {

597 Value *EE = Builder.CreateExtractElement(CI->getOperand(0), Index);

599 }

600 }

601 }

602

603

604

605

606 if (IndexC) {

608 unsigned NumElts = EC.getKnownMinValue();

609

610

611

612 if (!EC.isScalable() && NumElts != 1) {

613

614

616 APInt PoisonElts(NumElts, 0);

617 APInt DemandedElts(NumElts, 0);

618 DemandedElts.setBit(IndexC->getZExtValue());

622 } else {

623

624

627 APInt PoisonElts(NumElts, 0);

629 SrcVec, DemandedElts, PoisonElts, 0 ,

630 true )) {

631 if (V != SrcVec) {

634 return &EI;

635 }

636 }

637 }

638 }

639 }

640 }

641 return nullptr;

642}

643

644

645

649 "Invalid CollectSingleShuffleElements");

651

653 Mask.assign(NumElts, -1);

654 return true;

655 }

656

657 if (V == LHS) {

658 for (unsigned i = 0; i != NumElts; ++i)

659 Mask.push_back(i);

660 return true;

661 }

662

663 if (V == RHS) {

664 for (unsigned i = 0; i != NumElts; ++i)

665 Mask.push_back(i + NumElts);

666 return true;

667 }

668

670

671 Value *VecOp = IEI->getOperand(0);

672 Value *ScalarOp = IEI->getOperand(1);

673 Value *IdxOp = IEI->getOperand(2);

674

676 return false;

678

679 if (isa(ScalarOp)) {

680

681

683

684 Mask[InsertedIdx] = -1;

685 return true;

686 }

689 unsigned ExtractedIdx =

691 unsigned NumLHSElts =

693

694

696

697

699

701 Mask[InsertedIdx % NumElts] = ExtractedIdx;

702 } else {

704 Mask[InsertedIdx % NumElts] = ExtractedIdx + NumLHSElts;

705 }

706 return true;

707 }

708 }

709 }

710 }

711 }

712

713 return false;

714}

715

716

717

718

724 unsigned NumInsElts = InsVecType->getNumElements();

725 unsigned NumExtElts = ExtVecType->getNumElements();

726

727

728 if (InsVecType->getElementType() != ExtVecType->getElementType() ||

729 NumExtElts >= NumInsElts)

730 return false;

731

733

735 return false;

736

737

738

739

740

742 for (unsigned i = 0; i < NumExtElts; ++i)

744 for (unsigned i = NumExtElts; i < NumInsElts; ++i)

746

751

752

753

754

755

756

757

758

759

760

761 if (InsertionBlock != InsElt->getParent())

762 return false;

763

764

765

766

767

768

770 return false;

771

773

774

775

776

777

778 if (ExtVecOpInst && isa<PHINode> (ExtVecOpInst))

779 WideVec->insertAfter(ExtVecOpInst->getIterator());

780 else

782

783

784

785 for (User *U : ExtVecOp->users()) {

787 if (!OldExt || OldExt->getParent() != WideVec->getParent())

788 continue;

792

793

795 }

796

797 return true;

798}

799

800

801

802

803

804

805

806

807

809

811 Value *PermittedRHS,

813 assert(V->getType()->isVectorTy() && "Invalid shuffle!");

815

817 Mask.assign(NumElts, -1);

818 return std::make_pair(

820 }

821

823 Mask.assign(NumElts, 0);

824 return std::make_pair(V, nullptr);

825 }

826

828

829 Value *VecOp = IEI->getOperand(0);

830 Value *ScalarOp = IEI->getOperand(1);

831 Value *IdxOp = IEI->getOperand(2);

832

835 unsigned ExtractedIdx =

838

839

840

841 if (EI->getOperand(0) == PermittedRHS || PermittedRHS == nullptr) {

844 assert(LR.second == nullptr || LR.second == RHS);

845

846 if (LR.first->getType() != RHS->getType()) {

847

848

850 Rerun = true;

851

852

853

854 for (unsigned i = 0; i < NumElts; ++i)

855 Mask[i] = i;

856 return std::make_pair(V, nullptr);

857 }

858

859 unsigned NumLHSElts =

861 Mask[InsertedIdx % NumElts] = NumLHSElts + ExtractedIdx;

862 return std::make_pair(LR.first, RHS);

863 }

864

865 if (VecOp == PermittedRHS) {

866

867

868 unsigned NumLHSElts =

870 ->getNumElements();

871 for (unsigned i = 0; i != NumElts; ++i)

872 Mask.push_back(i == InsertedIdx ? ExtractedIdx : NumLHSElts + i);

873 return std::make_pair(EI->getOperand(0), PermittedRHS);

874 }

875

876

877

880 Mask))

881 return std::make_pair(EI->getOperand(0), PermittedRHS);

882 }

883 }

884 }

885

886

887 for (unsigned i = 0; i != NumElts; ++i)

888 Mask.push_back(i);

889 return std::make_pair(V, nullptr);

890}

891

892

893

894

895

896

900 unsigned NumAggElts;

904 break;

907 break;

908 default:

910 }

911

912

913

914

915 assert(NumAggElts > 0 && "Aggregate should have elements.");

916 if (NumAggElts > 2)

917 return nullptr;

918

919 static constexpr auto NotFound = std::nullopt;

920 static constexpr auto FoundMismatch = nullptr;

921

922

923

925

926

927 auto KnowAllElts = [&AggElts]() {

929 };

930

932

933

934

935 static const int DepthLimit = 2 * NumAggElts;

936

937

938

940 Depth < DepthLimit && CurrIVI && !KnowAllElts();

943 auto *InsertedValue =

945 if (!InsertedValue)

946 return nullptr;

947

949

950

951 if (Indices.size() != 1)

952 return nullptr;

953

954

955

956

957 std::optional<Instruction *> &Elt = AggElts[Indices.front()];

958 Elt = Elt.value_or(InsertedValue);

959

960

961 }

962

963

964 if (!KnowAllElts())

965 return nullptr;

966

967

968

969

970

971 enum class AggregateDescription {

972

973

974 NotFound,

975

976

977

978 Found,

979

980

981

982

983

984

985 FoundMismatch

986 };

987 auto Describe = [](std::optional<Value *> SourceAggregate) {

988 if (SourceAggregate == NotFound)

989 return AggregateDescription::NotFound;

990 if (*SourceAggregate == FoundMismatch)

991 return AggregateDescription::FoundMismatch;

992 return AggregateDescription::Found;

993 };

994

995

996 bool EltDefinedInUseBB = false;

997

998

999

1000

1001

1002

1003 auto FindSourceAggregate =

1004 [&](Instruction *Elt, unsigned EltIdx, std::optional<BasicBlock *> UseBB,

1005 std::optional<BasicBlock *> PredBB) -> std::optional<Value *> {

1006

1007 if (UseBB && PredBB) {

1009 if (Elt && Elt->getParent() == *UseBB)

1010 EltDefinedInUseBB = true;

1011 }

1012

1013

1014

1016 if (!EVI)

1017 return NotFound;

1018

1019 Value *SourceAggregate = EVI->getAggregateOperand();

1020

1021

1022 if (SourceAggregate->getType() != AggTy)

1023 return FoundMismatch;

1024

1025 if (EVI->getNumIndices() != 1 || EltIdx != EVI->getIndices().front())

1026 return FoundMismatch;

1027

1028 return SourceAggregate;

1029 };

1030

1031

1032

1033

1034 auto FindCommonSourceAggregate =

1035 [&](std::optional<BasicBlock *> UseBB,

1036 std::optional<BasicBlock *> PredBB) -> std::optional<Value *> {

1037 std::optional<Value *> SourceAggregate;

1038

1040 assert(Describe(SourceAggregate) != AggregateDescription::FoundMismatch &&

1041 "We don't store nullptr in SourceAggregate!");

1042 assert((Describe(SourceAggregate) == AggregateDescription::Found) ==

1043 (I.index() != 0) &&

1044 "SourceAggregate should be valid after the first element,");

1045

1046

1047

1048

1049 std::optional<Value *> SourceAggregateForElement =

1050 FindSourceAggregate(*I.value(), I.index(), UseBB, PredBB);

1051

1052

1053

1054

1055

1056

1057 if (Describe(SourceAggregateForElement) != AggregateDescription::Found)

1058 return SourceAggregateForElement;

1059

1060

1061

1062 switch (Describe(SourceAggregate)) {

1063 case AggregateDescription::NotFound:

1064

1065 SourceAggregate = SourceAggregateForElement;

1066 continue;

1067 case AggregateDescription::Found:

1068

1069

1070 if (*SourceAggregateForElement != *SourceAggregate)

1071 return FoundMismatch;

1072 continue;

1073 case AggregateDescription::FoundMismatch:

1074 llvm_unreachable("Can't happen. We would have early-exited then.");

1075 };

1076 }

1077

1078 assert(Describe(SourceAggregate) == AggregateDescription::Found &&

1079 "Must be a valid Value");

1080 return *SourceAggregate;

1081 };

1082

1083 std::optional<Value *> SourceAggregate;

1084

1085

1086 SourceAggregate = FindCommonSourceAggregate(std::nullopt,

1087 std::nullopt);

1088 if (Describe(SourceAggregate) != AggregateDescription::NotFound) {

1089 if (Describe(SourceAggregate) == AggregateDescription::FoundMismatch)

1090 return nullptr;

1091 ++NumAggregateReconstructionsSimplified;

1093 }

1094

1095

1096

1097

1098

1099

1100

1101

1103

1104 for (const std::optional<Instruction *> &I : AggElts) {

1106

1107 if (!UseBB) {

1108 UseBB = BB;

1109 continue;

1110 }

1111

1112 if (UseBB != BB)

1113 return nullptr;

1114 }

1115

1116

1117

1118

1119 if (!UseBB)

1120 return nullptr;

1121

1122

1123

1125 return nullptr;

1126

1127

1128 static const int PredCountLimit = 64;

1129

1130

1131

1134

1135 if (Preds.size() >= PredCountLimit)

1136 return nullptr;

1138 }

1139

1140

1141

1142

1144 bool FoundSrcAgg = false;

1146 std::pair<decltype(SourceAggregates)::iterator, bool> IV =

1148

1149 if (.second)

1150 continue;

1151

1152

1153

1154

1155 SourceAggregate = FindCommonSourceAggregate(UseBB, Pred);

1156 if (Describe(SourceAggregate) == AggregateDescription::Found) {

1157 FoundSrcAgg = true;

1158 IV.first->second = *SourceAggregate;

1159 } else {

1160

1161

1163 if (!BI || !BI->isUnconditional())

1164 return nullptr;

1165 }

1166 }

1167

1168 if (!FoundSrcAgg)

1169 return nullptr;

1170

1171

1172 auto OrigBB = OrigIVI.getParent();

1173 for (auto &It : SourceAggregates) {

1174 if (Describe(It.second) == AggregateDescription::Found)

1175 continue;

1176

1177

1178 if (EltDefinedInUseBB)

1179 return nullptr;

1180

1181

1182

1183

1184

1185

1186 if (UseBB != OrigBB)

1187 return nullptr;

1188

1189

1190

1191 bool ConstAgg = true;

1192 for (auto Val : AggElts) {

1195 ConstAgg = false;

1196 break;

1197 }

1198 }

1199 if (ConstAgg)

1200 return nullptr;

1201 }

1202

1203

1204

1205 for (auto &It : SourceAggregates) {

1206 if (Describe(It.second) == AggregateDescription::Found)

1207 continue;

1208

1210 Builder.SetInsertPoint(Pred->getTerminator());

1212 for (auto [Idx, Val] : enumerate(AggElts)) {

1214 V = Builder.CreateInsertValue(V, Elt, Idx);

1215 }

1216

1217 It.second = V;

1218 }

1219

1220

1221

1222

1223

1224

1226 Builder.SetInsertPoint(UseBB, UseBB->getFirstNonPHIIt());

1227 auto *PHI =

1228 Builder.CreatePHI(AggTy, Preds.size(), OrigIVI.getName() + ".merged");

1230 PHI->addIncoming(SourceAggregates[Pred], Pred);

1231

1232 ++NumAggregateReconstructionsSimplified;

1234}

1235

1236

1237

1238

1239

1240

1241

1242

1245 I.getAggregateOperand(), I.getInsertedValueOperand(), I.getIndices(),

1246 SQ.getWithInstruction(&I)))

1248

1249 bool IsRedundant = false;

1251

1252

1253

1254

1255

1257 unsigned Depth = 0;

1258 while (V->hasOneUse() && Depth < 10) {

1261 if (!UserInsInst || U->getOperand(0) != V)

1262 break;

1263 if (UserInsInst->getIndices() == FirstIndices) {

1264 IsRedundant = true;

1265 break;

1266 }

1267 V = UserInsInst;

1269 }

1270

1271 if (IsRedundant)

1273

1275 return NewI;

1276

1277 return nullptr;

1278}

1279

1281

1282

1284 return false;

1285

1287 int VecSize =

1289

1290

1291 if (MaskSize != VecSize)

1292 return false;

1293

1294

1295

1296 for (int i = 0; i != MaskSize; ++i) {

1298 if (Elt != -1 && Elt != i && Elt != i + VecSize)

1299 return false;

1300 }

1301

1302 return true;

1303}

1304

1305

1306

1307

1309

1310

1312 return nullptr;

1313

1315

1316

1318 return nullptr;

1320

1321

1322

1323 if (NumElements == 1)

1324 return nullptr;

1325

1330

1331

1332

1333 while (CurrIE) {

1335 if (!Idx || CurrIE->getOperand(1) != SplatVal)

1336 return nullptr;

1337

1339

1340

1341

1342 if (CurrIE != &InsElt &&

1343 (!CurrIE->hasOneUse() && (NextIE != nullptr || !Idx->isZero())))

1344 return nullptr;

1345

1346 ElementPresent[Idx->getZExtValue()] = true;

1347 FirstIE = CurrIE;

1348 CurrIE = NextIE;

1349 }

1350

1351

1352 if (FirstIE == &InsElt)

1353 return nullptr;

1354

1355

1356

1357

1358

1360 if (!ElementPresent.all())

1361 return nullptr;

1362

1363

1366 Constant *Zero = ConstantInt::get(Int64Ty, 0);

1370

1371

1373 for (unsigned i = 0; i != NumElements; ++i)

1374 if (!ElementPresent[i])

1375 Mask[i] = -1;

1376

1378}

1379

1380

1381

1383

1385 if (!Shuf || !Shuf->isZeroEltSplat())

1386 return nullptr;

1387

1388

1389

1391 return nullptr;

1392

1393

1396 return nullptr;

1397

1398

1400 Value *Op0 = Shuf->getOperand(0);

1402 return nullptr;

1403

1404

1405

1406

1407

1408 unsigned NumMaskElts =

1411 for (unsigned i = 0; i != NumMaskElts; ++i)

1412 NewMask[i] = i == IdxC ? 0 : Shuf->getMaskValue(i);

1413

1415}

1416

1417

1418

1420

1422 if (!Shuf || match (Shuf->getOperand(1), m_Poison()) ||

1423 !(Shuf->isIdentityWithExtract() || Shuf->isIdentityWithPadding()))

1424 return nullptr;

1425

1426

1427

1429 return nullptr;

1430

1431

1434 return nullptr;

1435

1436

1437

1439 Value *X = Shuf->getOperand(0);

1441 return nullptr;

1442

1443

1444

1445

1446

1447 unsigned NumMaskElts =

1451 for (unsigned i = 0; i != NumMaskElts; ++i) {

1452 if (i != IdxC) {

1453

1454 NewMask[i] = OldMask[i];

1455 } else if (OldMask[i] == (int)IdxC) {

1456

1457

1458 return nullptr;

1459 } else {

1461 "Unexpected shuffle mask element for identity shuffle");

1462 NewMask[i] = IdxC;

1463 }

1464 }

1465

1467}

1468

1469

1470

1471

1472

1473

1474

1475

1476

1477

1481 if (!InsElt1 || !InsElt1->hasOneUse())

1482 return nullptr;

1483

1487 if (match(InsElt1->getOperand(0), m_Value(X)) &&

1492 Value *NewInsElt1 = Builder.CreateInsertElement(X, ScalarC, IdxC2);

1494 }

1495

1496 return nullptr;

1497}

1498

1499

1500

1503

1504

1505 if (!Inst || !Inst->hasOneUse())

1506 return nullptr;

1508

1509

1510 Constant *ShufConstVec, *InsEltScalar;

1512 if ( match (Shuf->getOperand(1), m_Constant(ShufConstVec)) ||

1515 return nullptr;

1516

1517

1518

1519

1520

1521

1523 return nullptr;

1524

1525

1526

1527

1528

1529

1530

1531

1532

1534 unsigned NumElts = Mask.size();

1537 for (unsigned I = 0; I != NumElts; ++I) {

1538 if (I == InsEltIndex) {

1539 NewShufElts[I] = InsEltScalar;

1540 NewMaskElts[I] = InsEltIndex + NumElts;

1541 } else {

1542

1544 NewMaskElts[I] = Mask[I];

1545 }

1546

1547

1548 if (!NewShufElts[I])

1549 return nullptr;

1550 }

1551

1552

1553

1557

1558

1559

1560

1562 return nullptr;

1563 unsigned NumElts =

1565

1572 return nullptr;

1575 auto ValI = std::begin(Val);

1576

1577

1578

1580 if (!Values[I]) {

1581 Values[I] = *ValI;

1582 Mask[I] = NumElts + I;

1583 }

1584 ++ValI;

1585 }

1586

1587 for (unsigned I = 0; I < NumElts; ++I) {

1588 if (!Values[I]) {

1590 Mask[I] = I;

1591 }

1592 }

1593

1594

1597 }

1598 return nullptr;

1599}

1600

1601

1602

1603

1604

1607

1608

1609

1610

1613 return nullptr;

1614

1619 CastOpcode = Instruction::FPExt;

1621 CastOpcode = Instruction::SExt;

1623 CastOpcode = Instruction::ZExt;

1624 else

1625 return nullptr;

1626

1627

1628 if (X->getType()->getScalarType() != Y->getType())

1629 return nullptr;

1630

1631

1632 Value *NewInsElt = Builder.CreateInsertElement(X, Y, InsElt.getOperand(2));

1634}

1635

1636

1637

1639 bool IsBigEndian,

1644

1645

1646

1647

1648

1649

1650

1651

1652

1653

1654

1656 Value *Scalar0, *BaseVec;

1658 if (!VTy || (VTy->getNumElements() & 1) ||

1663 return nullptr;

1664

1665

1666

1667 if (Index0 + 1 != Index1 || Index0 & 1)

1668 return nullptr;

1669

1670

1671

1674 if (IsBigEndian) {

1677 return nullptr;

1678 } else {

1681 return nullptr;

1682 }

1683

1684 Type *SrcTy = X->getType();

1685 unsigned ScalarWidth = SrcTy->getScalarSizeInBits();

1686 unsigned VecEltWidth = VTy->getScalarSizeInBits();

1687 if (ScalarWidth != VecEltWidth * 2 || ShAmt != VecEltWidth)

1688 return nullptr;

1689

1690

1692 Value *CastBaseVec = Builder.CreateBitCast(BaseVec, CastTy);

1693

1694

1695

1696 uint64_t NewIndex = IsBigEndian ? Index1 / 2 : Index0 / 2;

1697 Value *NewInsert = Builder.CreateInsertElement(CastBaseVec, X, NewIndex);

1699}

1700

1702 Value *VecOp = IE.getOperand(0);

1703 Value *ScalarOp = IE.getOperand(1);

1704 Value *IdxOp = IE.getOperand(2);

1705

1707 VecOp, ScalarOp, IdxOp, SQ.getWithInstruction(&IE)))

1709

1710

1714

1715 Value *BaseVec, *OtherScalar;

1720 isa<Constant> (OtherScalar) && OtherIndexVal > IndexC->getZExtValue()) {

1721 Value *NewIns = Builder.CreateInsertElement(BaseVec, ScalarOp, IdxOp);

1723 Builder.getInt64(OtherIndexVal));

1724 }

1725 }

1726

1727

1728

1729

1730 Value *ScalarSrc;

1735

1736

1738 Type *VecTy = VectorType::get(ScalarTy, IE.getType()->getElementCount());

1741 Value *NewInsElt = Builder.CreateInsertElement(NewUndef, ScalarSrc, IdxOp);

1742 return new BitCastInst(NewInsElt, IE.getType());

1743 }

1744

1745

1746

1754

1755

1756 Value *NewInsElt = Builder.CreateInsertElement(VecSrc, ScalarSrc, IdxOp);

1757 return new BitCastInst(NewInsElt, IE.getType());

1758 }

1759

1760

1761

1762

1763

1764 uint64_t InsertedIdx, ExtractedIdx;

1765 Value *ExtVecOp;

1771 ExtractedIdx <

1773

1774

1775

1776

1777

1778

1779

1780

1781

1782

1783

1784

1785

1786

1788 if (!Insert.hasOneUse())

1789 return true;

1791 if (!InsertUser)

1792 return true;

1793 return false;

1794 };

1795

1796

1797 if (isShuffleRootCandidate(IE)) {

1798 bool Rerun = true;

1799 while (Rerun) {

1800 Rerun = false;

1801

1805

1806

1807

1808 if (LR.first != &IE && LR.second != &IE) {

1809

1810 if (LR.second == nullptr)

1813 }

1814 }

1815 }

1816 }

1817

1819 unsigned VWidth = VecTy->getNumElements();

1820 APInt PoisonElts(VWidth, 0);

1823 PoisonElts)) {

1824 if (V != &IE)

1826 return &IE;

1827 }

1828 }

1829

1831 return Shuf;

1832

1834 return NewInsElt;

1835

1837 return Broadcast;

1838

1841

1843 return IdentityShuf;

1844

1846 return Ext;

1847

1849 return Ext;

1850

1851 return nullptr;

1852}

1853

1854

1855

1857 unsigned Depth = 5) {

1858

1860 return true;

1861

1862

1864 if () return false;

1865

1866

1867 if (->hasOneUse())

1868 return false;

1869

1870 if (Depth == 0) return false;

1871

1872 switch (I->getOpcode()) {

1873 case Instruction::UDiv:

1874 case Instruction::SDiv:

1875 case Instruction::URem:

1876 case Instruction::SRem:

1877

1878

1879

1881 return false;

1882 [[fallthrough]];

1883 case Instruction::Add:

1884 case Instruction::FAdd:

1885 case Instruction::Sub:

1886 case Instruction::FSub:

1887 case Instruction::Mul:

1888 case Instruction::FMul:

1889 case Instruction::FDiv:

1890 case Instruction::FRem:

1891 case Instruction::Shl:

1892 case Instruction::LShr:

1893 case Instruction::AShr:

1894 case Instruction::And:

1895 case Instruction::Or:

1896 case Instruction::Xor:

1897 case Instruction::ICmp:

1898 case Instruction::FCmp:

1899 case Instruction::Trunc:

1900 case Instruction::ZExt:

1901 case Instruction::SExt:

1902 case Instruction::FPToUI:

1903 case Instruction::FPToSI:

1904 case Instruction::UIToFP:

1905 case Instruction::SIToFP:

1906 case Instruction::FPTrunc:

1907 case Instruction::FPExt:

1908 case Instruction::GetElementPtr: {

1909

1910

1911 Type *ITy = I->getType();

1914 return false;

1915 for (Value *Operand : I->operands()) {

1917 return false;

1918 }

1919 return true;

1920 }

1921 case Instruction::InsertElement: {

1923 if (!CI) return false;

1925

1926

1927

1928 bool SeenOnce = false;

1929 for (int I : Mask) {

1930 if (I == ElementNumber) {

1931 if (SeenOnce)

1932 return false;

1933 SeenOnce = true;

1934 }

1935 }

1937 }

1938 }

1939 return false;

1940}

1941

1942

1943

1946 Builder.SetInsertPoint(I);

1947 switch (I->getOpcode()) {

1948 case Instruction::Add:

1949 case Instruction::FAdd:

1950 case Instruction::Sub:

1951 case Instruction::FSub:

1952 case Instruction::Mul:

1953 case Instruction::FMul:

1954 case Instruction::UDiv:

1955 case Instruction::SDiv:

1956 case Instruction::FDiv:

1957 case Instruction::URem:

1958 case Instruction::SRem:

1959 case Instruction::FRem:

1960 case Instruction::Shl:

1961 case Instruction::LShr:

1962 case Instruction::AShr:

1963 case Instruction::And:

1964 case Instruction::Or:

1965 case Instruction::Xor: {

1967 assert(NewOps.size() == 2 && "binary operator with #ops != 2");

1969 NewOps[0], NewOps[1]);

1974 }

1976 NewI->setIsExact(BO->isExact());

1977 }

1979 NewI->copyFastMathFlags(I);

1980 }

1981 return New;

1982 }

1983 case Instruction::ICmp:

1984 assert(NewOps.size() == 2 && "icmp with #ops != 2");

1985 return Builder.CreateICmp(cast(I)->getPredicate(), NewOps[0],

1986 NewOps[1]);

1987 case Instruction::FCmp:

1988 assert(NewOps.size() == 2 && "fcmp with #ops != 2");

1989 return Builder.CreateFCmp(cast(I)->getPredicate(), NewOps[0],

1990 NewOps[1]);

1991 case Instruction::Trunc:

1992 case Instruction::ZExt:

1993 case Instruction::SExt:

1994 case Instruction::FPToUI:

1995 case Instruction::FPToSI:

1996 case Instruction::UIToFP:

1997 case Instruction::SIToFP:

1998 case Instruction::FPTrunc:

1999 case Instruction::FPExt: {

2000

2001

2003 I->getType()->getScalarType(),

2005 assert(NewOps.size() == 1 && "cast with #ops != 1");

2007 DestTy);

2008 }

2009 case Instruction::GetElementPtr: {

2010 Value *Ptr = NewOps[0];

2012 return Builder.CreateGEP(cast(I)->getSourceElementType(),

2013 Ptr, Idx, "",

2015 }

2016 }

2018}

2019

2022

2023

2024 assert(V->getType()->isVectorTy() && "can't reorder non-vector elements");

2026

2029

2032

2035

2038 Mask);

2039

2041 switch (I->getOpcode()) {

2042 case Instruction::Add:

2043 case Instruction::FAdd:

2044 case Instruction::Sub:

2045 case Instruction::FSub:

2046 case Instruction::Mul:

2047 case Instruction::FMul:

2048 case Instruction::UDiv:

2049 case Instruction::SDiv:

2050 case Instruction::FDiv:

2051 case Instruction::URem:

2052 case Instruction::SRem:

2053 case Instruction::FRem:

2054 case Instruction::Shl:

2055 case Instruction::LShr:

2056 case Instruction::AShr:

2057 case Instruction::And:

2058 case Instruction::Or:

2059 case Instruction::Xor:

2060 case Instruction::ICmp:

2061 case Instruction::FCmp:

2062 case Instruction::Trunc:

2063 case Instruction::ZExt:

2064 case Instruction::SExt:

2065 case Instruction::FPToUI:

2066 case Instruction::FPToSI:

2067 case Instruction::UIToFP:

2068 case Instruction::SIToFP:

2069 case Instruction::FPTrunc:

2070 case Instruction::FPExt:

2071 case Instruction::Select:

2072 case Instruction::GetElementPtr: {

2074 bool NeedsRebuild =

2075 (Mask.size() !=

2077 for (int i = 0, e = I->getNumOperands(); i != e; ++i) {

2079

2080

2081

2082 if (I->getOperand(i)->getType()->isVectorTy())

2084 else

2085 V = I->getOperand(i);

2087 NeedsRebuild |= (V != I->getOperand(i));

2088 }

2089 if (NeedsRebuild)

2090 return buildNew(I, NewOps, Builder);

2091 return I;

2092 }

2093 case Instruction::InsertElement: {

2094 int Element = cast(I->getOperand(2))->getLimitedValue();

2095

2096

2097

2098

2099 bool Found = false;

2100 int Index = 0;

2101 for (int e = Mask.size(); Index != e; ++Index) {

2102 if (Mask[Index] == Element) {

2103 Found = true;

2104 break;

2105 }

2106 }

2107

2108

2109

2110 if (!Found)

2112

2114 Builder);

2115 Builder.SetInsertPoint(I);

2116 return Builder.CreateInsertElement(V, I->getOperand(1), Index);

2117 }

2118 }

2119 llvm_unreachable("failed to reorder elements of vector instruction!");

2120}

2121

2122

2123

2124

2125

2126

2127

2130 unsigned LHSElems =

2132 unsigned MaskElems = Mask.size();

2133 unsigned BegIdx = Mask.front();

2134 unsigned EndIdx = Mask.back();

2135 if (BegIdx > EndIdx || EndIdx >= LHSElems || EndIdx - BegIdx != MaskElems - 1)

2136 return false;

2137 for (unsigned I = 0; I != MaskElems; ++I)

2138 if (static_cast<unsigned>(Mask[I]) != BegIdx + I)

2139 return false;

2140 return true;

2141}

2142

2143

2144

2154

2155

2156

2157

2158

2163 case Instruction::Shl: {

2164

2168 Instruction::Shl, ConstantInt::get(Ty, 1), C, DL);

2169 assert(ShlOne && "Constant folding of immediate constants failed");

2170 return {Instruction::Mul, BO0, ShlOne};

2171 }

2172 break;

2173 }

2174 case Instruction::Or: {

2175

2177 return {Instruction::Add, BO0, BO1};

2178 break;

2179 }

2180 case Instruction::Sub:

2181

2184 break;

2185 default:

2186 break;

2187 }

2188 return {};

2189}

2190

2191

2192

2193

2195 assert(Shuf.isSelect() && "Must have select-equivalent shuffle");

2196

2200 unsigned NumElts = Mask.size();

2201

2202

2204 if (ShufOp && ShufOp->isSelect() &&

2205 (ShufOp->getOperand(0) == Op1 || ShufOp->getOperand(1) == Op1)) {

2208 }

2209

2211 if (!ShufOp || !ShufOp->isSelect() ||

2212 (ShufOp->getOperand(0) != Op0 && ShufOp->getOperand(1) != Op0))

2213 return nullptr;

2214

2215 Value *X = ShufOp->getOperand(0), *Y = ShufOp->getOperand(1);

2217 ShufOp->getShuffleMask(Mask1);

2218 assert(Mask1.size() == NumElts && "Vector size changed with select shuffle");

2219

2220

2221 if (Y == Op0) {

2224 }

2225

2226

2227

2228

2229

2231 for (unsigned i = 0; i != NumElts; ++i)

2232 NewMask[i] = Mask[i] < (signed)NumElts ? Mask[i] : Mask1[i];

2233

2234

2237 "Unexpected shuffle mask");

2239}

2240

2243 assert(Shuf.isSelect() && "Must have select-equivalent shuffle");

2244

2245

2246

2249 bool Op0IsBinop;

2251 Op0IsBinop = true;

2253 Op0IsBinop = false;

2254 else

2255 return nullptr;

2256

2257

2258

2259

2263 if (!IdC)

2264 return nullptr;

2265

2266 Value *X = Op0IsBinop ? Op1 : Op0;

2267

2268

2269

2270

2271

2272

2273

2274

2277 return nullptr;

2278

2279

2280

2281

2282

2286

2287 bool MightCreatePoisonOrUB =

2290 if (MightCreatePoisonOrUB)

2292

2293

2294

2297

2298

2299

2300

2303 return NewBO;

2304}

2305

2306

2307

2308

2309

2316

2317

2321 return nullptr;

2322

2323

2325 Value *NewIns = Builder.CreateInsertElement(PoisonVec, X, (uint64_t)0);

2326

2327

2328

2329

2330

2331 unsigned NumMaskElts =

2334 for (unsigned i = 0; i != NumMaskElts; ++i)

2336 NewMask[i] = Mask[i];

2337

2339}

2340

2341

2344 return nullptr;

2345

2346

2347

2351

2352

2354 return &Shuf;

2355 }

2356

2358 return I;

2359

2362 return I;

2363

2367 return nullptr;

2368

2369

2370

2371

2373 Constant *C0 = nullptr, *C1 = nullptr;

2374 bool ConstantsAreOp1;

2377 ConstantsAreOp1 = false;

2382 ConstantsAreOp1 = true;

2383 else

2384 return nullptr;

2385

2386

2389 bool DropNSW = false;

2390 if (ConstantsAreOp1 && Opc0 != Opc1) {

2391

2392

2393

2394 if (Opc0 == Instruction::Shl || Opc1 == Instruction::Shl)

2395 DropNSW = true;

2397 assert(isa(AltB0.Op1) && "Expecting constant with alt binop");

2398 Opc0 = AltB0.Opcode;

2401 assert(isa(AltB1.Op1) && "Expecting constant with alt binop");

2402 Opc1 = AltB1.Opcode;

2404 }

2405 }

2406

2407 if (Opc0 != Opc1 || !C0 || !C1)

2408 return nullptr;

2409

2410

2412

2413

2416

2417

2418

2419

2420 bool MightCreatePoisonOrUB =

2423 if (MightCreatePoisonOrUB)

2425 ConstantsAreOp1);

2426

2428 if (X == Y) {

2429

2430

2431

2432 V = X;

2433 } else {

2434

2435

2436

2438 return nullptr;

2439

2440

2441

2442

2443

2444

2445

2446 if (MightCreatePoisonOrUB && !ConstantsAreOp1)

2447 return nullptr;

2448

2449

2450

2451

2452

2453

2454

2455

2456 V = Builder.CreateShuffleVector(X, Y, Mask);

2457 }

2458

2459 Value *NewBO = ConstantsAreOp1 ? Builder.CreateBinOp(BOpc, V, NewC) :

2460 Builder.CreateBinOp(BOpc, NewC, V);

2461

2462

2463

2464

2465

2466

2468 NewI->copyIRFlags(B0);

2469 NewI->andIRFlags(B1);

2470 if (DropNSW)

2471 NewI->setHasNoSignedWrap(false);

2473 NewI->dropPoisonGeneratingFlags();

2474 }

2476}

2477

2478

2479

2480

2482 bool IsBigEndian) {

2483

2488 return nullptr;

2489

2490

2491

2492 Type *SrcType = X->getType();

2493 if (!SrcType->isVectorTy() || !SrcType->isIntOrIntVectorTy() ||

2497 return nullptr;

2498

2500 "Expected a shuffle that decreases length");

2501

2502

2503

2507 for (unsigned i = 0, e = Mask.size(); i != e; ++i) {

2509 continue;

2510 uint64_t LSBIndex = IsBigEndian ? (i + 1) * TruncRatio - 1 : i * TruncRatio;

2511 assert(LSBIndex <= INT32_MAX && "Overflowed 32-bits");

2512 if (Mask[i] != (int)LSBIndex)

2513 return nullptr;

2514 }

2515

2517}

2518

2519

2520

2521

2524

2525

2527 return nullptr;

2528

2529

2530

2534 return nullptr;

2535

2536

2537

2538 unsigned NarrowNumElts =

2540 Value *NarrowCond;

2543 NarrowNumElts ||

2545 return nullptr;

2546

2547

2548

2549

2553}

2554

2555

2561 return nullptr;

2562

2563 bool IsFNeg = S0->getOpcode() == Instruction::FNeg;

2564

2565

2569 S0->getOpcode() != S1->getOpcode() ||

2570 (!S0->hasOneUse() && ->hasOneUse()))

2571 return nullptr;

2572

2573

2576 if (IsFNeg) {

2577 NewF = UnaryOperator::CreateFNeg(NewShuf);

2578 } else {

2582 }

2585 return NewF;

2586}

2587

2588

2592 if (!Cast0)

2593 return nullptr;

2594

2595

2596

2598 switch (CastOpcode) {

2599 case Instruction::SExt:

2600 case Instruction::ZExt:

2601 case Instruction::FPToSI:

2602 case Instruction::FPToUI:

2603 case Instruction::SIToFP:

2604 case Instruction::UIToFP:

2605 break;

2606 default:

2607 return nullptr;

2608 }

2609

2613

2614

2615 if (ShufTy->getElementCount().getKnownMinValue() >

2616 ShufOpTy->getElementCount().getKnownMinValue())

2617 return nullptr;

2618

2619

2620

2623 auto *NewIns = Builder.CreateShuffleVector(Cast0->getOperand(0),

2627 }

2628

2630

2631 if (!Cast1 || Cast0->getOpcode() != Cast1->getOpcode() ||

2632 Cast0->getSrcTy() != Cast1->getSrcTy())

2633 return nullptr;

2634

2635

2637 "Expected fixed vector operands for casts and binary shuffle");

2638 if (CastSrcTy->getPrimitiveSizeInBits() > ShufOpTy->getPrimitiveSizeInBits())

2639 return nullptr;

2640

2641

2642 if (!Cast0->hasOneUse() && !Cast1->hasOneUse())

2643 return nullptr;

2644

2645

2646 Value *X = Cast0->getOperand(0);

2647 Value *Y = Cast1->getOperand(0);

2650}

2651

2652

2656 return nullptr;

2657

2658

2659

2662 X->getType()->getPrimitiveSizeInBits() ==

2665

2666

2670 return nullptr;

2671

2672

2673

2675 return nullptr;

2676

2677

2678

2679

2680

2681

2682

2683

2684

2685

2686

2687

2690 assert(NumElts < Mask.size() &&

2691 "Identity with extract must have less elements than its inputs");

2692

2693 for (unsigned i = 0; i != NumElts; ++i) {

2695 int MaskElt = Mask[i];

2696 NewMask[i] = ExtractMaskElt == PoisonMaskElem ? ExtractMaskElt : MaskElt;

2697 }

2699}

2700

2701

2702

2708

2709 int NumElts = Mask.size();

2711

2712

2713

2714

2715

2716

2720

2723 }

2725

2726

2727 IdxC += InpNumElts;

2728

2731 }

2732

2733

2734

2735 if (NumElts != InpNumElts)

2736 return nullptr;

2737

2738

2739 auto isShufflingScalarIntoOp1 = [&](Value *&Scalar, ConstantInt *&IndexC) {

2740

2743 return false;

2744

2745

2746

2747 int NewInsIndex = -1;

2748 for (int i = 0; i != NumElts; ++i) {

2749

2750 if (Mask[i] == -1)

2751 continue;

2752

2753

2754 if (Mask[i] == NumElts + i)

2755 continue;

2756

2757

2758 if (NewInsIndex != -1 || Mask[i] != IndexC->getSExtValue())

2759 return false;

2760

2761

2762 NewInsIndex = i;

2763 }

2764

2765 assert(NewInsIndex != -1 && "Did not fold shuffle with unused operand?");

2766

2767

2768 IndexC = ConstantInt::get(IndexC->getIntegerType(), NewInsIndex);

2769 return true;

2770 };

2771

2772

2773

2774

2777 if (isShufflingScalarIntoOp1(Scalar, IndexC))

2779

2780

2781

2782

2785 if (isShufflingScalarIntoOp1(Scalar, IndexC))

2787

2788 return nullptr;

2789}

2790

2792

2793

2794

2797 if (!Shuffle0 || !Shuffle0->isIdentityWithPadding() ||

2798 !Shuffle1 || !Shuffle1->isIdentityWithPadding())

2799 return nullptr;

2800

2801

2802

2803

2804

2805

2806 Value *X = Shuffle0->getOperand(0);

2807 Value *Y = Shuffle1->getOperand(0);

2808 if (X->getType() != Y->getType() ||

2814 return nullptr;

2816 match(Shuffle1->getOperand(1), m_Undef()) &&

2817 "Unexpected operand for identity shuffle");

2818

2819

2820

2821

2822

2825 assert(WideElts > NarrowElts && "Unexpected types for identity with padding");

2826

2829 for (int i = 0, e = Mask.size(); i != e; ++i) {

2830 if (Mask[i] == -1)

2831 continue;

2832

2833

2834

2835 if (Mask[i] < WideElts) {

2836 if (Shuffle0->getMaskValue(Mask[i]) == -1)

2837 continue;

2838 } else {

2839 if (Shuffle1->getMaskValue(Mask[i] - WideElts) == -1)

2840 continue;

2841 }

2842

2843

2844

2845

2846 if (Mask[i] < WideElts) {

2847 assert(Mask[i] < NarrowElts && "Unexpected shuffle mask");

2848 NewMask[i] = Mask[i];

2849 } else {

2850 assert(Mask[i] < (WideElts + NarrowElts) && "Unexpected shuffle mask");

2851 NewMask[i] = Mask[i] - (WideElts - NarrowElts);

2852 }

2853 }

2855}

2856

2857

2858

2859

2864 return nullptr;

2865

2872 return nullptr;

2873 if (X->getType() != Y->getType())

2874 return nullptr;

2875

2878 return nullptr;

2879

2880 Value *NewBO = Builder.CreateBinOp(BinOp->getOpcode(), X, Y);

2882 NewBOI->copyIRFlags(BinOp);

2883

2885}

2886

2892 SVI.getType(), ShufQuery))

2894

2896 return I;

2897

2898

2899

2902

2904 return nullptr;

2905

2908

2909

2910

2911

2912

2913

2914

2915

2918 X->getType()->isVectorTy() && X->getType() == Y->getType() &&

2919 X->getType()->getScalarSizeInBits() ==

2921 (LHS->hasOneUse() || RHS->hasOneUse())) {

2923 SVI.getName() + ".uncasted");

2925 }

2926

2928

2929

2930

2931

2932

2933

2934

2936 X->getType()->isVectorTy() && VWidth == LHSWidth) {

2937

2939 unsigned XNumElts = XType->getNumElements();

2942

2943

2945 ScaledMask, XType, ShufQuery))

2947 }

2948 }

2949

2950

2951 if (LHS == RHS) {

2953 "Shuffle with 2 undef ops not simplified?");

2955 }

2956

2957

2960 return &SVI;

2961 }

2962

2964 return I;

2965

2967 return I;

2968

2970 return I;

2971

2973 return I;

2974

2976 return I;

2977

2979 return I;

2980

2981 APInt PoisonElts(VWidth, 0);

2984 if (V != &SVI)

2986 return &SVI;

2987 }

2988

2990 return I;

2991

2992

2993

2995 return I;

2997 return I;

2998

3001

3002

3003 if (SI->getCondition()->getType()->isIntegerTy() &&

3007 return I;

3008 }

3009 }

3012 return I;

3013 }

3014 }

3015

3019 }

3020

3021

3022

3023

3024

3025

3026

3027

3028

3029

3030

3031

3032

3033

3034

3035

3036

3037

3038

3039

3040

3041

3042

3043

3044

3045

3046

3047

3048

3049

3050 bool MadeChange = false;

3053 unsigned MaskElems = Mask.size();

3055 unsigned VecBitWidth = DL.getTypeSizeInBits(SrcTy);

3056 unsigned SrcElemBitWidth = DL.getTypeSizeInBits(SrcTy->getElementType());

3057 assert(SrcElemBitWidth && "vector elements must have a bitwidth");

3058 unsigned SrcNumElems = SrcTy->getNumElements();

3063

3064 if (BC->use_empty())

3065 continue;

3066

3067 if (BC->hasOneUse()) {

3069 if (BC2 && isEliminableCastPair(BC, BC2))

3070 continue;

3071 }

3073 }

3075 unsigned BegIdx = Mask.front();

3076 Type *TgtTy = BC->getDestTy();

3077 unsigned TgtElemBitWidth = DL.getTypeSizeInBits(TgtTy);

3078 if (!TgtElemBitWidth)

3079 continue;

3080 unsigned TgtNumElems = VecBitWidth / TgtElemBitWidth;

3081 bool VecBitWidthsEqual = VecBitWidth == TgtNumElems * TgtElemBitWidth;

3082 bool BegIsAligned = 0 == ((SrcElemBitWidth * BegIdx) % TgtElemBitWidth);

3083 if (!VecBitWidthsEqual)

3084 continue;

3086 continue;

3088 if (!BegIsAligned) {

3089

3090

3092 for (unsigned I = 0, E = MaskElems, Idx = BegIdx; I != E; ++Idx, ++I)

3093 ShuffleMask[I] = Idx;

3094 V = Builder.CreateShuffleVector(V, ShuffleMask,

3095 SVI.getName() + ".extract");

3096 BegIdx = 0;

3097 }

3098 unsigned SrcElemsPerTgtElem = TgtElemBitWidth / SrcElemBitWidth;

3099 assert(SrcElemsPerTgtElem);

3100 BegIdx /= SrcElemsPerTgtElem;

3101 auto [It, Inserted] = NewBCs.try_emplace(CastSrcTy);

3102 if (Inserted)

3103 It->second = Builder.CreateBitCast(V, CastSrcTy, SVI.getName() + ".bc");

3104 auto *Ext = Builder.CreateExtractElement(It->second, BegIdx,

3105 SVI.getName() + ".extract");

3106

3107

3109 MadeChange = true;

3110 }

3111 }

3112

3113

3114

3115

3116

3117

3118

3119

3120

3121

3122

3123

3124

3125

3126

3127

3128

3129

3130

3131

3132

3133

3134

3135

3136

3137

3138

3139

3140

3141

3142

3143

3144

3145

3146

3147

3148

3149

3150

3151

3152

3153

3154

3155

3158 if (LHSShuffle)

3161 LHSShuffle = nullptr;

3162 if (RHSShuffle)

3164 RHSShuffle = nullptr;

3165 if (!LHSShuffle && !RHSShuffle)

3166 return MadeChange ? &SVI : nullptr;

3167

3168 Value* LHSOp0 = nullptr;

3169 Value* LHSOp1 = nullptr;

3170 Value* RHSOp0 = nullptr;

3171 unsigned LHSOp0Width = 0;

3172 unsigned RHSOp0Width = 0;

3173 if (LHSShuffle) {

3177 }

3178 if (RHSShuffle) {

3181 }

3182 Value* newLHS = LHS;

3183 Value* newRHS = RHS;

3184 if (LHSShuffle) {

3185

3187 newLHS = LHSOp0;

3188 newRHS = LHSOp1;

3189 }

3190

3191 else if (LHSOp0Width == LHSWidth) {

3192 newLHS = LHSOp0;

3193 }

3194 }

3195

3196 if (RHSShuffle && RHSOp0Width == LHSWidth) {

3197 newRHS = RHSOp0;

3198 }

3199

3200 if (LHSOp0 == RHSOp0) {

3201 newLHS = LHSOp0;

3202 newRHS = nullptr;

3203 }

3204

3205 if (newLHS == LHS && newRHS == RHS)

3206 return MadeChange ? &SVI : nullptr;

3207

3210 if (newLHS != LHS)

3212 if (RHSShuffle && newRHS != RHS)

3214

3215 unsigned newLHSWidth = (newLHS != LHS) ? LHSOp0Width : LHSWidth;

3218 int SplatElt = -1;

3219

3220

3221 for (unsigned i = 0; i < VWidth; ++i) {

3222 int eltMask;

3223 if (Mask[i] < 0) {

3224

3225 eltMask = -1;

3226 } else if (Mask[i] < (int)LHSWidth) {

3227

3228

3229

3230

3231 if (newLHS != LHS) {

3232 eltMask = LHSMask[Mask[i]];

3233

3234

3235 if (eltMask >= (int)LHSOp0Width && isa(LHSOp1))

3236 eltMask = -1;

3237 } else

3238 eltMask = Mask[i];

3239 } else {

3240

3241

3242

3243

3245 eltMask = -1;

3246

3247

3248 else if (newRHS != RHS) {

3249 eltMask = RHSMask[Mask[i]-LHSWidth];

3250

3251

3252 if (eltMask >= (int)RHSOp0Width) {

3254 "should have been check above");

3255 eltMask = -1;

3256 }

3257 } else

3258 eltMask = Mask[i]-LHSWidth;

3259

3260

3261

3262

3263

3264

3265

3266 if (eltMask >= 0 && newRHS != nullptr && newLHS != newRHS)

3267 eltMask += newLHSWidth;

3268 }

3269

3270

3271 if (eltMask >= 0) {

3272 if (SplatElt >= 0 && SplatElt != eltMask)

3274 SplatElt = eltMask;

3275 }

3276

3278 }

3279

3280

3281

3282 if (isSplat || newMask == LHSMask || newMask == RHSMask || newMask == Mask) {

3283 if (!newRHS)

3286 }

3287

3288 return MadeChange ? &SVI : nullptr;

3289}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

This file implements a class to represent arbitrary precision integral constant values and operations...

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

This file contains the declarations for the subclasses of Constant, which represent the different fla...

This file defines the DenseMap class.

This file provides internal interfaces used to implement the InstCombine.

static Instruction * foldConstantInsEltIntoShuffle(InsertElementInst &InsElt)

insertelt (shufflevector X, CVec, Mask|insertelt X, C1, CIndex1), C, CIndex --> shufflevector X,...

Definition InstCombineVectorOps.cpp:1501

static Value * evaluateInDifferentElementOrder(Value *V, ArrayRef< int > Mask, IRBuilderBase &Builder)

Definition InstCombineVectorOps.cpp:2020

static bool collectSingleShuffleElements(Value *V, Value *LHS, Value *RHS, SmallVectorImpl< int > &Mask)

If V is a shuffle of values that ONLY returns elements from either LHS or RHS, return the shuffle mas...

Definition InstCombineVectorOps.cpp:646

static ShuffleOps collectShuffleElements(Value *V, SmallVectorImpl< int > &Mask, Value *PermittedRHS, InstCombinerImpl &IC, bool &Rerun)

Definition InstCombineVectorOps.cpp:810

static APInt findDemandedEltsByAllUsers(Value *V)

Find union of elements of V demanded by all its users.

Definition InstCombineVectorOps.cpp:372

static Instruction * foldTruncInsEltPair(InsertElementInst &InsElt, bool IsBigEndian, InstCombiner::BuilderTy &Builder)

If we are inserting 2 halves of a value into adjacent elements of a vector, try to convert to a singl...

Definition InstCombineVectorOps.cpp:1638

static Instruction * foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf, const SimplifyQuery &SQ)

Definition InstCombineVectorOps.cpp:2241

static Instruction * foldIdentityPaddedShuffles(ShuffleVectorInst &Shuf)

Definition InstCombineVectorOps.cpp:2791

static Instruction * foldIdentityExtractShuffle(ShuffleVectorInst &Shuf)

Try to fold an extract subvector operation.

Definition InstCombineVectorOps.cpp:2653

static bool findDemandedEltsBySingleUser(Value *V, Instruction *UserInstr, APInt &UnionUsedElts)

Find elements of V demanded by UserInstr.

Definition InstCombineVectorOps.cpp:329

static Instruction * foldInsEltIntoSplat(InsertElementInst &InsElt)

Try to fold an insert element into an existing splat shuffle by changing the shuffle's mask to includ...

Definition InstCombineVectorOps.cpp:1382

std::pair< Value *, Value * > ShuffleOps

We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.

Definition InstCombineVectorOps.cpp:808

static Instruction * foldShuffleWithInsert(ShuffleVectorInst &Shuf, InstCombinerImpl &IC)

Try to replace a shuffle with an insertelement or try to replace a shuffle operand with the operand o...

Definition InstCombineVectorOps.cpp:2703

static Instruction * canonicalizeInsertSplat(ShuffleVectorInst &Shuf, InstCombiner::BuilderTy &Builder)

If we have an insert of a scalar to a non-zero element of an undefined vector and then shuffle that v...

Definition InstCombineVectorOps.cpp:2310

static Instruction * foldTruncShuffle(ShuffleVectorInst &Shuf, bool IsBigEndian)

Convert a narrowing shuffle of a bitcasted vector into a vector truncate.

Definition InstCombineVectorOps.cpp:2481

static bool replaceExtractElements(InsertElementInst *InsElt, ExtractElementInst *ExtElt, InstCombinerImpl &IC)

If we have insertion into a vector that is wider than the vector that we are extracting from,...

Definition InstCombineVectorOps.cpp:719

static bool cheapToScalarize(Value *V, Value *EI)

Return true if the value is cheaper to scalarize than it is to leave as a vector operation.

Definition InstCombineVectorOps.cpp:58

static Value * buildNew(Instruction *I, ArrayRef< Value * > NewOps, IRBuilderBase &Builder)

Rebuild a new instruction just like 'I' but with the new operands given.

Definition InstCombineVectorOps.cpp:1944

static bool canEvaluateShuffled(Value *V, ArrayRef< int > Mask, unsigned Depth=5)

Return true if we can evaluate the specified expression tree if the vector elements were shuffled in ...

Definition InstCombineVectorOps.cpp:1856

static Instruction * foldSelectShuffleOfSelectShuffle(ShuffleVectorInst &Shuf)

A select shuffle of a select shuffle with a shared operand can be reduced to a single select shuffle.

Definition InstCombineVectorOps.cpp:2194

static Instruction * hoistInsEltConst(InsertElementInst &InsElt2, InstCombiner::BuilderTy &Builder)

If we have an insertelement instruction feeding into another insertelement and the 2nd is inserting a...

Definition InstCombineVectorOps.cpp:1478

static Instruction * foldShuffleOfUnaryOps(ShuffleVectorInst &Shuf, InstCombiner::BuilderTy &Builder)

Canonicalize FP negate/abs after shuffle.

Definition InstCombineVectorOps.cpp:2556

static Instruction * foldCastShuffle(ShuffleVectorInst &Shuf, InstCombiner::BuilderTy &Builder)

Canonicalize casts after shuffle.

Definition InstCombineVectorOps.cpp:2589

static Instruction * narrowInsElt(InsertElementInst &InsElt, InstCombiner::BuilderTy &Builder)

If both the base vector and the inserted element are extended from the same type, do the insert eleme...

Definition InstCombineVectorOps.cpp:1605

static bool isShuffleEquivalentToSelect(ShuffleVectorInst &Shuf)

Definition InstCombineVectorOps.cpp:1280

static Instruction * foldInsSequenceIntoSplat(InsertElementInst &InsElt)

Turn a chain of inserts that splats a value into an insert + shuffle: insertelt(insertelt(insertelt(i...

Definition InstCombineVectorOps.cpp:1308

static Instruction * foldInsEltIntoIdentityShuffle(InsertElementInst &InsElt)

Try to fold an extract+insert element into an existing identity shuffle by changing the shuffle's mas...

Definition InstCombineVectorOps.cpp:1419

static ConstantInt * getPreferredVectorIndex(ConstantInt *IndexC)

Given a constant index for a extractelement or insertelement instruction, return it with the canonica...

Definition InstCombineVectorOps.cpp:396

static bool isShuffleExtractingFromLHS(ShuffleVectorInst &SVI, ArrayRef< int > Mask)

Definition InstCombineVectorOps.cpp:2128

static BinopElts getAlternateBinop(BinaryOperator *BO, const DataLayout &DL)

Binops may be transformed into binops with different opcodes and operands.

Definition InstCombineVectorOps.cpp:2159

This file provides the interface for the instcombine pass implementation.

static bool isSplat(Value *V)

Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).

uint64_t IntrinsicInst * II

const SmallVectorImpl< MachineOperand > & Cond

This file implements the SmallBitVector class.

This file defines the SmallVector class.

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")

static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")

static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)

Returns the opcode of Values or ~0 if they do not all agree.

static SDValue narrowVectorSelect(SDNode *N, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget)

If both arms of a vector select are concatenated vectors, split the select, and concatenate the resul...

static const uint32_t IV[8]

Class for arbitrary precision integers.

static APInt getAllOnes(unsigned numBits)

Return an APInt of a specified width with all bits set.

LLVM_ABI APInt zextOrTrunc(unsigned width) const

Zero extend or truncate to width.

unsigned getActiveBits() const

Compute the number of active bits in the value.

void setBit(unsigned BitPosition)

Set the given bit to 1 whose position is given as "bitPosition".

bool isAllOnes() const

Determine if all bits are set. This is true for zero-width values.

bool ult(const APInt &RHS) const

Unsigned less than comparison.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

const T & front() const

front - Get the first element.

size_t size() const

size - Get the array size.

ArrayRef< T > slice(size_t N, size_t M) const

slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.

LLVM Basic Block Representation.

LLVM_ABI const_iterator getFirstInsertionPt() const

Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...

const Function * getParent() const

Return the enclosing method, or null if none.

InstListType::iterator iterator

Instruction iterators...

BinaryOps getOpcode() const

static LLVM_ABI BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), InsertPosition InsertBefore=nullptr)

Construct a binary instruction, given the opcode and the two operands.

static BinaryOperator * CreateWithCopiedFlags(BinaryOps Opc, Value *V1, Value *V2, Value *CopyO, const Twine &Name="", InsertPosition InsertBefore=nullptr)

This class represents a no-op cast from one type to another.

static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)

static LLVM_ABI CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", InsertPosition InsertBefore=nullptr)

Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...

This class is the base class for the comparison instructions.

static LLVM_ABI CmpInst * CreateWithCopiedFlags(OtherOps Op, Predicate Pred, Value *S1, Value *S2, const Instruction *FlagsSource, const Twine &Name="", InsertPosition InsertBefore=nullptr)

Construct a compare instruction, given the opcode, the predicate, the two operands and the instructio...

OtherOps getOpcode() const

Get the opcode casted to the right type.

An abstraction over a floating-point predicate, and a pack of an integer predicate with samesign info...

static LLVM_ABI ConstantAggregateZero * get(Type *Ty)

static LLVM_ABI Constant * getShuffleVector(Constant *V1, Constant *V2, ArrayRef< int > Mask, Type *OnlyIfReducedTy=nullptr)

static LLVM_ABI Constant * getBinOpIdentity(unsigned Opcode, Type *Ty, bool AllowRHSConstant=false, bool NSZ=false)

Return the identity constant for a binary opcode.

This is the shared class of boolean and integer constants.

uint64_t getLimitedValue(uint64_t Limit=~0ULL) const

getLimitedValue - If the value is smaller than the specified limit, return it, otherwise return the l...

unsigned getBitWidth() const

getBitWidth - Return the scalar bitwidth of this constant.

uint64_t getZExtValue() const

Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...

const APInt & getValue() const

Return the constant as an APInt value reference.

static LLVM_ABI Constant * get(ArrayRef< Constant * > V)

This is an important base class in LLVM.

static LLVM_ABI Constant * getAllOnesValue(Type *Ty)

LLVM_ABI Constant * getAggregateElement(unsigned Elt) const

For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...

A parsed version of the target data layout string in and methods for querying it.

std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)

static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)

an instruction for type-safe pointer arithmetic to access elements of arrays and structs

static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)

LLVM_ABI void setNoWrapFlags(GEPNoWrapFlags NW)

Set nowrap flags for GEP instruction.

Common base class shared among various IRBuilders.

This instruction inserts a single (scalar) element into a VectorType value.

static InsertElementInst * Create(Value *Vec, Value *NewElt, Value *Idx, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)

VectorType * getType() const

Overload to return most specific vector type.

This instruction inserts a struct field of array element value into an aggregate value.

Instruction * foldOpIntoPhi(Instruction &I, PHINode *PN, bool AllowMultipleUses=false)

Given a binary operator, cast instruction, or select which has a PHI node as operand #0,...

Value * SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, APInt &PoisonElts, unsigned Depth=0, bool AllowMultipleUsers=false) override

The specified value produces a vector with any number of elements.

Instruction * foldSelectShuffle(ShuffleVectorInst &Shuf)

Try to fold shuffles that are the equivalent of a vector select.

Definition InstCombineVectorOps.cpp:2342

Instruction * FoldOpIntoSelect(Instruction &Op, SelectInst *SI, bool FoldWithMultiUse=false, bool SimplifyBothArms=false)

Given an instruction with a select as one operand and a constant as the other operand,...

Instruction * visitInsertValueInst(InsertValueInst &IV)

Try to find redundant insertvalue instructions, like the following ones: %0 = insertvalue { i8,...

Definition InstCombineVectorOps.cpp:1243

Instruction * visitInsertElementInst(InsertElementInst &IE)

Definition InstCombineVectorOps.cpp:1701

Instruction * visitExtractElementInst(ExtractElementInst &EI)

Definition InstCombineVectorOps.cpp:404

Instruction * simplifyBinOpSplats(ShuffleVectorInst &SVI)

Definition InstCombineVectorOps.cpp:2860

Instruction * foldAggregateConstructionIntoAggregateReuse(InsertValueInst &OrigIVI)

Look for chain of insertvalue's that fully define an aggregate, and trace back the values inserted,...

Definition InstCombineVectorOps.cpp:897

Instruction * visitShuffleVectorInst(ShuffleVectorInst &SVI)

Definition InstCombineVectorOps.cpp:2887

IRBuilder< TargetFolder, IRBuilderCallbackInserter > BuilderTy

An IRBuilder that automatically inserts new instructions into the worklist.

Instruction * replaceInstUsesWith(Instruction &I, Value *V)

A combiner-aware RAUW-like routine.

InstructionWorklist & Worklist

A worklist of the instructions that need to be simplified.

Instruction * InsertNewInstWith(Instruction *New, BasicBlock::iterator Old)

Same as InsertNewInstBefore, but also sets the debug loc.

void addToWorklist(Instruction *I)

Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)

Replace operand of instruction and add old operand to the worklist.

static Constant * getSafeVectorConstantForBinop(BinaryOperator::BinaryOps Opcode, Constant *In, bool IsRHSConstant)

Some binary operators require special handling to avoid poison and undefined behavior.

const SimplifyQuery & getSimplifyQuery() const

LLVM_ABI bool hasNoUnsignedWrap() const LLVM_READONLY

Determine whether the no unsigned wrap flag is set.

LLVM_ABI bool hasNoSignedWrap() const LLVM_READONLY

Determine whether the no signed wrap flag is set.

LLVM_ABI void copyIRFlags(const Value *V, bool IncludeWrapFlags=true)

Convenience method to copy supported exact, fast-math, and (optionally) wrapping flags from V to this...

LLVM_ABI const Module * getModule() const

Return the module owning the function this instruction belongs to or nullptr it the function does not...

LLVM_ABI void andIRFlags(const Value *V)

Logical 'and' of any supported wrapping, exact, and fast-math flags of V and this instruction.

Instruction * user_back()

Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...

LLVM_ABI bool isExact() const LLVM_READONLY

Determine whether the exact flag is set.

unsigned getOpcode() const

Returns a member of one of the enums like Instruction::Add.

LLVM_ABI void dropPoisonGeneratingFlags()

Drops flags that may cause this instruction to evaluate to poison despite having non-poison inputs.

A wrapper class for inspecting calls to intrinsic functions.

std::pair< iterator, bool > try_emplace(const KeyT &Key, Ts &&...Args)

void addIncoming(Value *V, BasicBlock *BB)

Add an incoming value to the end of the PHI list.

BasicBlock * getIncomingBlock(unsigned i) const

Return incoming basic block number i.

Value * getIncomingValue(unsigned i) const

Return incoming value number x.

unsigned getNumIncomingValues() const

Return the number of incoming edges.

static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)

Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...

In order to facilitate speculative execution, many instructions do not invoke immediate undefined beh...

static LLVM_ABI PoisonValue * get(Type *T)

Static factory methods - Return an 'poison' object of the specified type.

This class represents the LLVM 'select' instruction.

static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", InsertPosition InsertBefore=nullptr, const Instruction *MDFrom=nullptr)

This instruction constructs a fixed permutation of two input vectors.

bool changesLength() const

Return true if this shuffle returns a vector with a different number of elements than its source vect...

int getMaskValue(unsigned Elt) const

Return the shuffle mask value of this instruction for the given element index.

static LLVM_ABI bool isSelectMask(ArrayRef< int > Mask, int NumSrcElts)

Return true if this shuffle mask chooses elements from its source vectors without lane crossings.

VectorType * getType() const

Overload to return most specific vector type.

bool increasesLength() const

Return true if this shuffle returns a vector with a greater number of elements than its source vector...

LLVM_ABI bool isIdentityWithExtract() const

Return true if this shuffle extracts the first N elements of exactly one source vector.

static LLVM_ABI void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)

Convert the input shuffle mask operand to a vector of integers.

bool isSelect() const

Return true if this shuffle chooses elements from its source vectors without lane crossings and all o...

static LLVM_ABI bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)

Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...

static void commuteShuffleMask(MutableArrayRef< int > Mask, unsigned InVecNumElts)

Change values in a shuffle permute mask assuming the two vector operands of length InVecNumElts have ...

LLVM_ABI void commute()

Swap the operands and adjust the mask to preserve the semantics of the instruction.

This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...

bool all() const

Returns true if all bits are set.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

reference emplace_back(ArgTypes &&... Args)

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

This class represents a truncation of integer types.

The instances of the Type class are immutable: once they are created, they are never changed.

static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)

bool isVectorTy() const

True if this is an instance of VectorType.

bool isIntOrIntVectorTy() const

Return true if this is an integer type or a vector of integer types.

LLVM_ABI unsigned getStructNumElements() const

LLVM_ABI uint64_t getArrayNumElements() const

Type * getScalarType() const

If this is a vector type, return the element type, otherwise return 'this'.

LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY

Return the basic size of this type if it is a primitive type.

LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY

If this is a vector type, return the getPrimitiveSizeInBits value for the element type.

bool isFloatingPointTy() const

Return true if this is one of the floating-point types.

bool isIntegerTy() const

True if this is an instance of IntegerType.

TypeID getTypeID() const

Return the type id for the type.

static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)

static UnaryOperator * CreateWithCopiedFlags(UnaryOps Opc, Value *V, Instruction *CopyO, const Twine &Name="", InsertPosition InsertBefore=nullptr)

UnaryOps getOpcode() const

static LLVM_ABI UndefValue * get(Type *T)

Static factory methods - Return an 'undef' object of the specified type.

A Use represents the edge between a Value definition and its users.

Value * getOperand(unsigned i) const

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

LLVM_ABI const Value * DoPHITranslation(const BasicBlock *CurBB, const BasicBlock *PredBB) const

Translate PHI node to its predecessor from the given basic block.

bool hasOneUse() const

Return true if there is exactly one use of this value.

LLVM_ABI void replaceAllUsesWith(Value *V)

Change all uses of this to point to a new Value.

iterator_range< user_iterator > users()

LLVM_ABI LLVMContext & getContext() const

All values hold a context through their type.

iterator_range< use_iterator > uses()

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

ElementCount getElementCount() const

Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...

static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)

This static method is the primary way to construct an VectorType.

static LLVM_ABI bool isValidElementType(Type *ElemTy)

Return true if the specified type is valid as a element type.

Type * getElementType() const

constexpr bool isScalable() const

Returns whether the quantity is scaled by a runtime quantity (vscale).

constexpr ScalarTy getKnownMinValue() const

Returns the minimum value this quantity can represent.

const ParentTy * getParent() const

self_iterator getIterator()

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

@ C

The default llvm calling convention, compatible with C.

@ BasicBlock

Various leaf nodes.

LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})

Look up the Function declaration of the intrinsic id in the Module M.

BinaryOpc_match< LHS, RHS, false > m_BinOp(unsigned Opcode, const LHS &L, const RHS &R)

SpecificConstantMatch m_ZeroInt()

Convenience matchers for specific integer values.

BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)

Matches a register negated by a G_SUB.

OneUse_match< SubPat > m_OneUse(const SubPat &SP)

class_match< PoisonValue > m_Poison()

Match an arbitrary poison constant.

class_match< BinaryOperator > m_BinOp()

Match an arbitrary binary operation and ignore it.

class_match< Constant > m_Constant()

Match an arbitrary Constant and ignore it.

CastInst_match< OpTy, TruncInst > m_Trunc(const OpTy &Op)

Matches Trunc.

specific_intval< false > m_SpecificInt(const APInt &V)

Match a specific integer value or vector with all elements equal to the value.

bool match(Val *V, const Pattern &P)

specificval_ty m_Specific(const Value *V)

Match if we have a specific specified value.

TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)

Matches ExtractElementInst.

class_match< ConstantInt > m_ConstantInt()

Match an arbitrary ConstantInt and ignore it.

IntrinsicID_match m_Intrinsic()

Match intrinsic calls like this: m_IntrinsicIntrinsic::fabs(m_Value(X))

ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)

Matches SelectInst.

TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)

Matches ShuffleVectorInst independently of mask value.

CastInst_match< OpTy, FPExtInst > m_FPExt(const OpTy &Op)

OneOps_match< OpTy, Instruction::Load > m_Load(const OpTy &Op)

Matches LoadInst.

CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)

Matches ZExt.

class_match< CmpInst > m_Cmp()

Matches any compare instruction and ignore it.

match_immconstant_ty m_ImmConstant()

Match an arbitrary immediate Constant and ignore it.

CastOperator_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)

Matches BitCast.

class_match< UnaryOperator > m_UnOp()

Match an arbitrary unary operation and ignore it.

class_match< Value > m_Value()

Match an arbitrary value and ignore it.

BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)

FNeg_match< OpTy > m_FNeg(const OpTy &X)

Match 'fneg X' as 'fsub -0.0, X'.

auto m_Undef()

Match an arbitrary undef constant.

CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)

Matches SExt.

is_zero m_Zero()

Match any null constant or a vector with all elements equal to 0.

ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)

Matches InsertElementInst.

m_Intrinsic_Ty< Opnd0 >::Ty m_FAbs(const Opnd0 &Op0)

match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)

Combine two pattern matchers matching L || R.

friend class Instruction

Iterator for Instructions in a `BasicBlock.

This is an optimization pass for GlobalISel generic memory operations.

FunctionAddr VTableAddr Value

auto enumerate(FirstRange &&First, RestRanges &&...Rest)

Given two or more input ranges, returns a new range whose values are tuples (A, B,...

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

LLVM_ABI llvm::SmallVector< int, 16 > createUnaryMask(ArrayRef< int > Mask, unsigned NumElts)

Given a shuffle mask for a binary shuffle, create the equivalent shuffle mask assuming both operands ...

LLVM_ABI Value * simplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef< int > Mask, Type *RetTy, const SimplifyQuery &Q)

Given operands for a ShuffleVectorInst, fold the result or return null.

auto dyn_cast_or_null(const Y &Val)

constexpr bool isPowerOf2_32(uint32_t Value)

Return true if the argument is a power of two > 0.

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

LLVM_ABI Value * simplifyInsertValueInst(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const SimplifyQuery &Q)

Given operands for an InsertValueInst, fold the result or return null.

LLVM_ABI Constant * ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS, Constant *RHS, const DataLayout &DL)

Attempt to constant fold a binary operation with the specified operands.

constexpr int PoisonMaskElem

LLVM_ABI Value * findScalarElement(Value *V, unsigned EltNo)

Given a vector and an element number, see if the scalar value is already around as a register,...

DWARFExpression::Operation Op

bool isSafeToSpeculativelyExecuteWithVariableReplaced(const Instruction *I, bool IgnoreUBImplyingAttrs=true)

Don't use information from its non-constant operands.

LLVM_ABI Value * simplifyInsertElementInst(Value *Vec, Value *Elt, Value *Idx, const SimplifyQuery &Q)

Given operands for an InsertElement, fold the result or return null.

constexpr unsigned BitWidth

auto count_if(R &&Range, UnaryPredicate P)

Wrapper function around std::count_if to count the number of times an element satisfying a given pred...

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

LLVM_ABI bool isKnownNeverNaN(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)

Return true if the floating-point scalar value is not a NaN or if the floating-point vector value has...

auto predecessors(const MachineBasicBlock *BB)

bool is_contained(R &&Range, const E &Element)

Returns true if Element is found in Range.

bool pred_empty(const BasicBlock *BB)

auto seq(T Begin, T End)

Iterate over an integral type from Begin up to - but not including - End.

LLVM_ABI bool isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)

Returns true if V cannot be poison, but may be undef.

LLVM_ABI Value * simplifyExtractElementInst(Value *Vec, Value *Idx, const SimplifyQuery &Q)

Given operands for an ExtractElementInst, fold the result or return null.

LLVM_ABI bool scaleShuffleMaskElts(unsigned NumDstElts, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)

Attempt to narrow/widen the Mask shuffle mask to the NumDstElts target width.

LLVM_ABI int getSplatIndex(ArrayRef< int > Mask)

If all non-negative Mask elements are the same value, return that value.

void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)

Implement std::swap in terms of BitVector swap.

These are the ingredients in an alternate form binary operator as described below.

Definition InstCombineVectorOps.cpp:2145

BinopElts(BinaryOperator::BinaryOps Opc=(BinaryOperator::BinaryOps) 0, Value *V0=nullptr, Value *V1=nullptr)

Definition InstCombineVectorOps.cpp:2149

BinaryOperator::BinaryOps Opcode

Definition InstCombineVectorOps.cpp:2146

Value * Op1

Definition InstCombineVectorOps.cpp:2148

Value * Op0

Definition InstCombineVectorOps.cpp:2147

A MapVector that performs no allocations if smaller than a certain size.