ScalarizeMaskedMemIntrin.cpp Source File (original) (raw)

38#include

39#include

41using namespace llvm;

43#define DEBUG_TYPE "scalarize-masked-mem-intrin"

45namespace {

47class ScalarizeMaskedMemIntrinLegacyPass : public FunctionPass {

48public:

49 static char ID;

51 explicit ScalarizeMaskedMemIntrinLegacyPass() : FunctionPass(ID) {

54 }

58 StringRef getPassName() const override {

59 return "Scalarize Masked Memory Intrinsics";

60 }

62 void getAnalysisUsage(AnalysisUsage &AU) const override {

65 }

66};

68}

75 const DataLayout &DL, bool HasBranchDivergence,

78char ScalarizeMaskedMemIntrinLegacyPass::ID = 0;

81 "Scalarize unsupported masked memory intrinsics", false,

82 false)

86 "Scalarize unsupported masked memory intrinsics", false,

90 return new ScalarizeMaskedMemIntrinLegacyPass();

91}

95 if ()

96 return false;

99 for (unsigned i = 0; i != NumElts; ++i) {

100 Constant *CElt = C->getAggregateElement(i);

102 return false;

103 }

104

105 return true;

106}

107

109 unsigned Idx) {

110 return DL.isBigEndian() ? VectorWidth - 1 - Idx : Idx;

111}

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

147 bool &ModifiedDT) {

151

154

155 Type *EltTy = VecType->getElementType();

156

160

161 Builder.SetInsertPoint(InsertPt);

162 Builder.SetCurrentDebugLocation(CI->getDebugLoc());

163

164

166 LoadInst *NewI = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal);

171 return;

172 }

173

174

175 const Align AdjustedAlignVal =

178

179

180 Value *VResult = Src0;

181

183 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {

184 if (cast(Mask)->getAggregateElement(Idx)->isNullValue())

185 continue;

186 Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);

187 LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AdjustedAlignVal);

188 VResult = Builder.CreateInsertElement(VResult, Load, Idx);

189 }

192 return;

193 }

194

195

196

197

200 Mask->getName() + ".first");

203 nullptr, DTU);

204

206 CondBlock->setName("cond.load");

207 Builder.SetInsertPoint(CondBlock->getTerminator());

208 LoadInst *Load = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal,

209 CI->getName() + ".cond.load");

210 Load->copyMetadata(*CI);

211

213 Builder.SetInsertPoint(PostLoad, PostLoad->begin());

214 PHINode *Phi = Builder.CreatePHI(VecType, 2);

215 Phi->addIncoming(Load, CondBlock);

216 Phi->addIncoming(Src0, IfBlock);

217 Phi->takeName(CI);

218

221 ModifiedDT = true;

222 return;

223 }

224

225

226

227 Value *SclrMask = nullptr;

228 if (VectorWidth != 1 && !HasBranchDivergence) {

229 Type *SclrMaskTy = Builder.getIntNTy(VectorWidth);

230 SclrMask = Builder.CreateBitCast(Mask, SclrMaskTy, "scalar_mask");

231 }

232

233 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {

234

235

236

237

238

239

240

241

242

244 if (SclrMask != nullptr) {

247 Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),

248 Builder.getIntN(VectorWidth, 0));

249 } else {

250 Predicate = Builder.CreateExtractElement(Mask, Idx);

251 }

252

253

254

255

256

257

258

261 nullptr, DTU);

262

264 CondBlock->setName("cond.load");

265

266 Builder.SetInsertPoint(CondBlock->getTerminator());

267 Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);

268 LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AdjustedAlignVal);

269 Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx);

270

271

273 NewIfBlock->setName("else");

275 IfBlock = NewIfBlock;

276

277

278 Builder.SetInsertPoint(NewIfBlock, NewIfBlock->begin());

279 PHINode *Phi = Builder.CreatePHI(VecType, 2, "res.phi.else");

280 Phi->addIncoming(NewVResult, CondBlock);

281 Phi->addIncoming(VResult, PrevIfBlock);

282 VResult = Phi;

283 }

284

287

288 ModifiedDT = true;

289}

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

319 bool &ModifiedDT) {

323

326

327 Type *EltTy = VecType->getElementType();

328

331 Builder.SetInsertPoint(InsertPt);

332 Builder.SetCurrentDebugLocation(CI->getDebugLoc());

333

334

336 StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);

337 Store->takeName(CI);

338 Store->copyMetadata(*CI);

340 return;

341 }

342

343

344 const Align AdjustedAlignVal =

347

349 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {

350 if (cast(Mask)->getAggregateElement(Idx)->isNullValue())

351 continue;

352 Value *OneElt = Builder.CreateExtractElement(Src, Idx);

353 Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);

354 Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);

355 }

357 return;

358 }

359

360

361

362

365 Mask->getName() + ".first");

368 nullptr, DTU);

370 CondBlock->setName("cond.store");

371 Builder.SetInsertPoint(CondBlock->getTerminator());

372

373 StoreInst *Store = Builder.CreateAlignedStore(Src, Ptr, AlignVal);

374 Store->takeName(CI);

375 Store->copyMetadata(*CI);

376

378 ModifiedDT = true;

379 return;

380 }

381

382

383

384

385 Value *SclrMask = nullptr;

386 if (VectorWidth != 1 && !HasBranchDivergence) {

387 Type *SclrMaskTy = Builder.getIntNTy(VectorWidth);

388 SclrMask = Builder.CreateBitCast(Mask, SclrMaskTy, "scalar_mask");

389 }

390

391 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {

392

393

394

395

396

397

398

399

400

402 if (SclrMask != nullptr) {

405 Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),

406 Builder.getIntN(VectorWidth, 0));

407 } else {

408 Predicate = Builder.CreateExtractElement(Mask, Idx);

409 }

410

411

412

413

414

415

416

419 nullptr, DTU);

420

422 CondBlock->setName("cond.store");

423

424 Builder.SetInsertPoint(CondBlock->getTerminator());

425 Value *OneElt = Builder.CreateExtractElement(Src, Idx);

426 Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, Idx);

427 Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);

428

429

431 NewIfBlock->setName("else");

432

433 Builder.SetInsertPoint(NewIfBlock, NewIfBlock->begin());

434 }

436

437 ModifiedDT = true;

438}

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

464

465

466

467

468

470 bool HasBranchDivergence, CallInst *CI,

475

477 Type *EltTy = VecType->getElementType();

478

482 Builder.SetInsertPoint(InsertPt);

484

485 Builder.SetCurrentDebugLocation(CI->getDebugLoc());

486

487

488 Value *VResult = Src0;

489 unsigned VectorWidth = VecType->getNumElements();

490

491

493 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {

494 if (cast(Mask)->getAggregateElement(Idx)->isNullValue())

495 continue;

496 Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));

498 Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));

499 VResult =

500 Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));

501 }

504 return;

505 }

506

507

508

509

510 Value *SclrMask = nullptr;

511 if (VectorWidth != 1 && !HasBranchDivergence) {

512 Type *SclrMaskTy = Builder.getIntNTy(VectorWidth);

513 SclrMask = Builder.CreateBitCast(Mask, SclrMaskTy, "scalar_mask");

514 }

515

516 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {

517

518

519

520

521

522

523

524

525

526

528 if (SclrMask != nullptr) {

531 Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),

532 Builder.getIntN(VectorWidth, 0));

533 } else {

534 Predicate = Builder.CreateExtractElement(Mask, Idx, "Mask" + Twine(Idx));

535 }

536

537

538

539

540

541

542

545 nullptr, DTU);

546

548 CondBlock->setName("cond.load");

549

550 Builder.SetInsertPoint(CondBlock->getTerminator());

551 Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));

553 Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));

554 Value *NewVResult =

555 Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));

556

557

559 NewIfBlock->setName("else");

561 IfBlock = NewIfBlock;

562

563

564 Builder.SetInsertPoint(NewIfBlock, NewIfBlock->begin());

565 PHINode *Phi = Builder.CreatePHI(VecType, 2, "res.phi.else");

566 Phi->addIncoming(NewVResult, CondBlock);

567 Phi->addIncoming(VResult, PrevIfBlock);

568 VResult = Phi;

569 }

570

573

574 ModifiedDT = true;

575}

576

577

578

579

580

581

582

583

584

585

586

587

588

589

590

591

592

593

594

595

596

597

598

599

600

601

602

604 bool HasBranchDivergence, CallInst *CI,

609

611

615 "Vector of pointers is expected in masked scatter intrinsic");

616

619 Builder.SetInsertPoint(InsertPt);

620 Builder.SetCurrentDebugLocation(CI->getDebugLoc());

621

623 unsigned VectorWidth = SrcFVTy->getNumElements();

624

625

627 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {

628 if (cast(Mask)->getAggregateElement(Idx)->isNullValue())

629 continue;

631 Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));

632 Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));

633 Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);

634 }

636 return;

637 }

638

639

640

641 Value *SclrMask = nullptr;

642 if (VectorWidth != 1 && !HasBranchDivergence) {

643 Type *SclrMaskTy = Builder.getIntNTy(VectorWidth);

644 SclrMask = Builder.CreateBitCast(Mask, SclrMaskTy, "scalar_mask");

645 }

646

647 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {

648

649

650

651

652

653

654

655

656

658 if (SclrMask != nullptr) {

661 Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),

662 Builder.getIntN(VectorWidth, 0));

663 } else {

664 Predicate = Builder.CreateExtractElement(Mask, Idx, "Mask" + Twine(Idx));

665 }

666

667

668

669

670

671

672

675 nullptr, DTU);

676

678 CondBlock->setName("cond.store");

679

680 Builder.SetInsertPoint(CondBlock->getTerminator());

681 Value *OneElt = Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));

682 Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));

683 Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);

684

685

687 NewIfBlock->setName("else");

688

689 Builder.SetInsertPoint(NewIfBlock, NewIfBlock->begin());

690 }

692

693 ModifiedDT = true;

694}

695

697 bool HasBranchDivergence, CallInst *CI,

703

705

706 Type *EltTy = VecType->getElementType();

707

711

712 Builder.SetInsertPoint(InsertPt);

713 Builder.SetCurrentDebugLocation(CI->getDebugLoc());

714

715 unsigned VectorWidth = VecType->getNumElements();

716

717

718 Value *VResult = PassThru;

719

720

721 const Align AdjustedAlignment =

723

724

725

726

728 unsigned MemIndex = 0;

731 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {

732 Value *InsertElt;

733 if (cast(Mask)->getAggregateElement(Idx)->isNullValue()) {

735 ShuffleMask[Idx] = Idx + VectorWidth;

736 } else {

738 Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, MemIndex);

739 InsertElt = Builder.CreateAlignedLoad(EltTy, NewPtr, AdjustedAlignment,

740 "Load" + Twine(Idx));

741 ShuffleMask[Idx] = Idx;

742 ++MemIndex;

743 }

744 VResult = Builder.CreateInsertElement(VResult, InsertElt, Idx,

745 "Res" + Twine(Idx));

746 }

747 VResult = Builder.CreateShuffleVector(VResult, PassThru, ShuffleMask);

750 return;

751 }

752

753

754

755

756 Value *SclrMask = nullptr;

757 if (VectorWidth != 1 && !HasBranchDivergence) {

758 Type *SclrMaskTy = Builder.getIntNTy(VectorWidth);

759 SclrMask = Builder.CreateBitCast(Mask, SclrMaskTy, "scalar_mask");

760 }

761

762 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {

763

764

765

766

767

768

769

770

771

772

774 if (SclrMask != nullptr) {

777 Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),

778 Builder.getIntN(VectorWidth, 0));

779 } else {

780 Predicate = Builder.CreateExtractElement(Mask, Idx, "Mask" + Twine(Idx));

781 }

782

783

784

785

786

787

788

791 nullptr, DTU);

792

794 CondBlock->setName("cond.load");

795

796 Builder.SetInsertPoint(CondBlock->getTerminator());

797 LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Ptr, AdjustedAlignment);

798 Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx);

799

800

802 if ((Idx + 1) != VectorWidth)

803 NewPtr = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, 1);

804

805

807 NewIfBlock->setName("else");

809 IfBlock = NewIfBlock;

810

811

812 Builder.SetInsertPoint(NewIfBlock, NewIfBlock->begin());

813 PHINode *ResultPhi = Builder.CreatePHI(VecType, 2, "res.phi.else");

814 ResultPhi->addIncoming(NewVResult, CondBlock);

815 ResultPhi->addIncoming(VResult, PrevIfBlock);

816 VResult = ResultPhi;

817

818

819 if ((Idx + 1) != VectorWidth) {

820 PHINode *PtrPhi = Builder.CreatePHI(Ptr->getType(), 2, "ptr.phi.else");

823 Ptr = PtrPhi;

824 }

825 }

826

829

830 ModifiedDT = true;

831}

832

834 bool HasBranchDivergence, CallInst *CI,

836 bool &ModifiedDT) {

841

843

847

848 Builder.SetInsertPoint(InsertPt);

849 Builder.SetCurrentDebugLocation(CI->getDebugLoc());

850

851 Type *EltTy = VecType->getElementType();

852

853

854 const Align AdjustedAlignment =

856

857 unsigned VectorWidth = VecType->getNumElements();

858

859

861 unsigned MemIndex = 0;

862 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {

863 if (cast(Mask)->getAggregateElement(Idx)->isNullValue())

864 continue;

866 Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));

867 Value *NewPtr = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, MemIndex);

868 Builder.CreateAlignedStore(OneElt, NewPtr, AdjustedAlignment);

869 ++MemIndex;

870 }

872 return;

873 }

874

875

876

877

878 Value *SclrMask = nullptr;

879 if (VectorWidth != 1 && !HasBranchDivergence) {

880 Type *SclrMaskTy = Builder.getIntNTy(VectorWidth);

881 SclrMask = Builder.CreateBitCast(Mask, SclrMaskTy, "scalar_mask");

882 }

883

884 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {

885

886

887

888

889

890

891

892

894 if (SclrMask != nullptr) {

897 Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),

898 Builder.getIntN(VectorWidth, 0));

899 } else {

900 Predicate = Builder.CreateExtractElement(Mask, Idx, "Mask" + Twine(Idx));

901 }

902

903

904

905

906

907

908

911 nullptr, DTU);

912

914 CondBlock->setName("cond.store");

915

916 Builder.SetInsertPoint(CondBlock->getTerminator());

917 Value *OneElt = Builder.CreateExtractElement(Src, Idx);

918 Builder.CreateAlignedStore(OneElt, Ptr, AdjustedAlignment);

919

920

922 if ((Idx + 1) != VectorWidth)

923 NewPtr = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, 1);

924

925

927 NewIfBlock->setName("else");

929 IfBlock = NewIfBlock;

930

931 Builder.SetInsertPoint(NewIfBlock, NewIfBlock->begin());

932

933

934 if ((Idx + 1) != VectorWidth) {

935 PHINode *PtrPhi = Builder.CreatePHI(Ptr->getType(), 2, "ptr.phi.else");

938 Ptr = PtrPhi;

939 }

940 }

942

943 ModifiedDT = true;

944}

945

948 bool &ModifiedDT) {

949

950

955

958

961 Builder.SetInsertPoint(InsertPt);

962

963 Builder.SetCurrentDebugLocation(CI->getDebugLoc());

964

965

966 unsigned VectorWidth = AddrType->getNumElements();

967 auto CreateHistogramUpdateValue = [&](IntrinsicInst *CI, Value *Load,

971 case Intrinsic::experimental_vector_histogram_add:

972 UpdateOp = Builder.CreateAdd(Load, Inc);

973 break;

974 case Intrinsic::experimental_vector_histogram_uadd_sat:

975 UpdateOp =

976 Builder.CreateIntrinsic(Intrinsic::uadd_sat, {EltTy}, {Load, Inc});

977 break;

978 case Intrinsic::experimental_vector_histogram_umin:

979 UpdateOp = Builder.CreateIntrinsic(Intrinsic::umin, {EltTy}, {Load, Inc});

980 break;

981 case Intrinsic::experimental_vector_histogram_umax:

982 UpdateOp = Builder.CreateIntrinsic(Intrinsic::umax, {EltTy}, {Load, Inc});

983 break;

984

985 default:

987 }

988 return UpdateOp;

989 };

990

991

993 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {

994 if (cast(Mask)->getAggregateElement(Idx)->isNullValue())

995 continue;

996 Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));

997 LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));

1000 Builder.CreateStore(Update, Ptr);

1001 }

1003 return;

1004 }

1005

1006 for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) {

1008 Builder.CreateExtractElement(Mask, Idx, "Mask" + Twine(Idx));

1009

1012 nullptr, DTU);

1013

1015 CondBlock->setName("cond.histogram.update");

1016

1017 Builder.SetInsertPoint(CondBlock->getTerminator());

1018 Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));

1019 LoadInst *Load = Builder.CreateLoad(EltTy, Ptr, "Load" + Twine(Idx));

1020 Value *UpdateOp =

1022 Builder.CreateStore(UpdateOp, Ptr);

1023

1024

1026 NewIfBlock->setName("else");

1027 Builder.SetInsertPoint(NewIfBlock, NewIfBlock->begin());

1028 }

1029

1031 ModifiedDT = true;

1032}

1033

1036 std::optional DTU;

1037 if (DT)

1038 DTU.emplace(DT, DomTreeUpdater::UpdateStrategy::Lazy);

1039

1040 bool EverMadeChange = false;

1041 bool MadeChange = true;

1042 auto &DL = F.getDataLayout();

1043 bool HasBranchDivergence = TTI.hasBranchDivergence(&F);

1044 while (MadeChange) {

1045 MadeChange = false;

1047 bool ModifiedDTOnIteration = false;

1049 HasBranchDivergence, DTU ? &*DTU : nullptr);

1050

1051

1052 if (ModifiedDTOnIteration)

1053 break;

1054 }

1055

1056 EverMadeChange |= MadeChange;

1057 }

1058 return EverMadeChange;

1059}

1060

1061bool ScalarizeMaskedMemIntrinLegacyPass::runOnFunction(Function &F) {

1062 auto &TTI = getAnalysis().getTTI(F);

1063 DominatorTree *DT = nullptr;

1064 if (auto *DTWP = getAnalysisIfAvailable())

1065 DT = &DTWP->getDomTree();

1067}

1068

1069PreservedAnalyses

1080

1084 bool MadeChange = false;

1085

1087 while (CurInstIterator != BB.end()) {

1089 MadeChange |=

1091 if (ModifiedDT)

1092 return true;

1093 }

1094

1095 return MadeChange;

1096}

1097

1100 const DataLayout &DL, bool HasBranchDivergence,

1103 if (II) {

1104

1107 [](Value *V) { return isa(V->getType()); }))

1108 return false;

1109 switch (II->getIntrinsicID()) {

1110 default:

1111 break;

1112 case Intrinsic::experimental_vector_histogram_add:

1113 case Intrinsic::experimental_vector_histogram_uadd_sat:

1114 case Intrinsic::experimental_vector_histogram_umin:

1115 case Intrinsic::experimental_vector_histogram_umax:

1118 return false;

1120 return true;

1121 case Intrinsic::masked_load:

1122

1123 if (TTI.isLegalMaskedLoad(

1126 ->getAddressSpace(),

1130 return false;

1132 return true;

1133 case Intrinsic::masked_store:

1134 if (TTI.isLegalMaskedStore(

1138 ->getAddressSpace(),

1142 return false;

1144 return true;

1145 case Intrinsic::masked_gather: {

1148 if (TTI.isLegalMaskedGather(LoadTy, Alignment) &&

1150 return false;

1152 return true;

1153 }

1154 case Intrinsic::masked_scatter: {

1157 if (TTI.isLegalMaskedScatter(StoreTy, Alignment) &&

1159 Alignment))

1160 return false;

1162 return true;

1163 }

1164 case Intrinsic::masked_expandload:

1165 if (TTI.isLegalMaskedExpandLoad(

1167 CI->getAttributes().getParamAttrs(0).getAlignment().valueOrOne()))

1168 return false;

1170 return true;

1171 case Intrinsic::masked_compressstore:

1172 if (TTI.isLegalMaskedCompressStore(

1174 CI->getAttributes().getParamAttrs(1).getAlignment().valueOrOne()))

1175 return false;

1177 ModifiedDT);

1178 return true;

1179 }

1180 }

1181

1182 return false;

1183}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

This file contains the declarations for the subclasses of Constant, which represent the different fla...

static bool runOnFunction(Function &F, bool PostInlining)

static bool runImpl(Function &F, const TargetLowering &TLI, const LibcallLoweringInfo &Libcalls, AssumptionCache *AC)

uint64_t IntrinsicInst * II

#define INITIALIZE_PASS_DEPENDENCY(depName)

#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)

#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)

static void scalarizeMaskedExpandLoad(const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)

Definition ScalarizeMaskedMemIntrin.cpp:696

static void scalarizeMaskedVectorHistogram(const DataLayout &DL, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)

Definition ScalarizeMaskedMemIntrin.cpp:946

static bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, bool HasBranchDivergence, DomTreeUpdater *DTU)

Definition ScalarizeMaskedMemIntrin.cpp:1081

static void scalarizeMaskedScatter(const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)

Definition ScalarizeMaskedMemIntrin.cpp:603

static unsigned adjustForEndian(const DataLayout &DL, unsigned VectorWidth, unsigned Idx)

Definition ScalarizeMaskedMemIntrin.cpp:108

static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, bool HasBranchDivergence, DomTreeUpdater *DTU)

Definition ScalarizeMaskedMemIntrin.cpp:1098

static void scalarizeMaskedStore(const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)

Definition ScalarizeMaskedMemIntrin.cpp:317

static void scalarizeMaskedCompressStore(const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)

Definition ScalarizeMaskedMemIntrin.cpp:833

static void scalarizeMaskedGather(const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)

Definition ScalarizeMaskedMemIntrin.cpp:469

static bool runImpl(Function &F, const TargetTransformInfo &TTI, DominatorTree *DT)

Definition ScalarizeMaskedMemIntrin.cpp:1034

static bool isConstantIntVector(Value *Mask)

Definition ScalarizeMaskedMemIntrin.cpp:93

static void scalarizeMaskedLoad(const DataLayout &DL, bool HasBranchDivergence, CallInst *CI, DomTreeUpdater *DTU, bool &ModifiedDT)

Definition ScalarizeMaskedMemIntrin.cpp:145

This pass exposes codegen information to IR-level passes.

static APInt getOneBitSet(unsigned numBits, unsigned BitNo)

Return an APInt with exactly one bit set in the result.

PassT::Result * getCachedResult(IRUnitT &IR) const

Get the cached result of an analysis pass for a given IR unit.

PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)

Get the result of an analysis pass for a given IR unit.

Represent the analysis usage information of a pass.

AnalysisUsage & addRequired()

AnalysisUsage & addPreserved()

Add the specified Pass class to the set of analyses preserved by this pass.

LLVM Basic Block Representation.

iterator begin()

Instruction iterator methods.

InstListType::iterator iterator

Instruction iterators...

const Instruction * getTerminator() const LLVM_READONLY

Returns the terminator instruction if the block is well formed or null if the block is not well forme...

MaybeAlign getParamAlign(unsigned ArgNo) const

Extract the alignment for a call or parameter (0=unknown).

Value * getArgOperand(unsigned i) const

LLVM_ABI Intrinsic::ID getIntrinsicID() const

Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...

AttributeList getAttributes() const

Return the attributes for this call.

This class represents a function call, abstracting a target machine's calling convention.

This is an important base class in LLVM.

A parsed version of the target data layout string in and methods for querying it.

Analysis pass which computes a DominatorTree.

Legacy analysis pass which computes a DominatorTree.

Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.

FunctionPass class - This class is used to implement most global optimizations.

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

const DebugLoc & getDebugLoc() const

Return the debug location for this node as a DebugLoc.

LLVM_ABI InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

LLVM_ABI BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY

Return the specified successor. This instruction must be a terminator.

LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())

Copy metadata from SrcInst to this instruction.

A wrapper class for inspecting calls to intrinsic functions.

An instruction for reading from memory.

void addIncoming(Value *V, BasicBlock *BB)

Add an incoming value to the end of the PHI list.

static LLVM_ABI PassRegistry * getPassRegistry()

getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...

static LLVM_ABI PoisonValue * get(Type *T)

Static factory methods - Return an 'poison' object of the specified type.

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

PreservedAnalyses & preserve()

Mark an analysis as preserved.

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

An instruction for storing to memory.

StringRef - Represent a constant reference to a string, i.e.

Analysis pass providing the TargetTransformInfo.

Wrapper pass for TargetTransformInfo.

This pass provides access to the codegen interfaces that are needed for IR-level transformations.

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

The instances of the Type class are immutable: once they are created, they are never changed.

LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY

Return the basic size of this type if it is a primitive type.

static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)

bool isVoidTy() const

Return true if this is 'void'.

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

LLVM_ABI void setName(const Twine &Name)

Change the name of the value.

LLVM_ABI void replaceAllUsesWith(Value *V)

Change all uses of this to point to a new Value.

LLVM_ABI LLVMContext & getContext() const

All values hold a context through their type.

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

LLVM_ABI void takeName(Value *V)

Transfer the name from V to this value.

const ParentTy * getParent() const

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

@ C

The default llvm calling convention, compatible with C.

This is an optimization pass for GlobalISel generic memory operations.

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

LLVM_ABI FunctionPass * createScalarizeMaskedMemIntrinLegacyPass()

Definition ScalarizeMaskedMemIntrin.cpp:89

bool any_of(R &&range, UnaryPredicate P)

Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.

LLVM_ABI bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)

Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...

LLVM_ABI void initializeScalarizeMaskedMemIntrinLegacyPassPass(PassRegistry &)

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

constexpr int PoisonMaskElem

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

Align commonAlignment(Align A, uint64_t Offset)

Returns the alignment that satisfies both alignments.

LLVM_ABI Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)

Split the containing block at the specified instruction - everything before SplitBefore stays in the ...

AnalysisManager< Function > FunctionAnalysisManager

Convenience typedef for the Function analysis manager.

This struct is a compact representation of a valid (non-zero power of two) alignment.

Align valueOrOne() const

For convenience, returns a valid alignment or 1 if undefined.

PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)

Definition ScalarizeMaskedMemIntrin.cpp:1070