AtomicExpandPass.cpp Source File (original) (raw)

53#include

54#include

55#include

57using namespace llvm;

59#define DEBUG_TYPE "atomic-expand"

61namespace {

63class AtomicExpandImpl {

67private:

71 bool tryExpandAtomicLoad(LoadInst *LI);

72 bool expandAtomicLoadToLL(LoadInst *LI);

73 bool expandAtomicLoadToCmpXchg(LoadInst *LI);

75 bool tryExpandAtomicStore(StoreInst *SI);

76 void expandAtomicStore(StoreInst *SI);

83 void expandAtomicOpToLLSC(

87 void expandPartwordAtomicRMW(

91 void expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI);

95 static Value *insertRMWCmpXchgLoop(

101

105

107 Value *PointerOperand, Value *ValueOperand,

111 void expandAtomicLoadToLibcall(LoadInst *LI);

112 void expandAtomicStoreToLibcall(StoreInst *LI);

115

116 friend bool

119

121

122public:

124};

125

126class AtomicExpandLegacy : public FunctionPass {

127public:

128 static char ID;

129

132 }

133

135};

136

137

138struct ReplacementIRBuilder

139 : IRBuilder<InstSimplifyFolder, IRBuilderCallbackInserter> {

140 MDNode *MMRAMD = nullptr;

141

142

143

150 if (BB->getParent()->getAttributes().hasFnAttr(Attribute::StrictFP))

152

153 MMRAMD = I->getMetadata(LLVMContext::MD_mmra);

154 }

155

158 I->setMetadata(LLVMContext::MD_mmra, MMRAMD);

159 }

160};

161

162}

163

164char AtomicExpandLegacy::ID = 0;

165

167

169 "Expand Atomic instructions", false, false)

173

174

177 return DL.getTypeStoreSize(LI->getType());

178}

179

182 return DL.getTypeStoreSize(SI->getValueOperand()->getType());

183}

184

188}

189

193}

194

195

199 Source.getAllMetadata(MD);

202

203 for (auto [ID, N] : MD) {

204 switch (ID) {

205 case LLVMContext::MD_dbg:

206 case LLVMContext::MD_tbaa:

207 case LLVMContext::MD_tbaa_struct:

208 case LLVMContext::MD_alias_scope:

209 case LLVMContext::MD_noalias:

210 case LLVMContext::MD_noalias_addrspace:

211 case LLVMContext::MD_access_group:

212 case LLVMContext::MD_mmra:

214 break;

215 default:

216 if (ID == Ctx.getMDKindID("amdgpu.no.remote.memory"))

218 else if (ID == Ctx.getMDKindID("amdgpu.no.fine.grained.memory"))

220

221

222

223 break;

224 }

225 }

226}

227

228

229

230

231template

234 Align Alignment = I->getAlign();

235 return Alignment >= Size &&

237}

238

239bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {

240 auto *LI = dyn_cast(I);

241 auto *SI = dyn_cast(I);

242 auto *RMWI = dyn_cast(I);

243 auto *CASI = dyn_cast(I);

244

245 bool MadeChange = false;

246

247

248 if (LI) {

249 if (!LI->isAtomic())

250 return false;

251

253 expandAtomicLoadToLibcall(LI);

254 return true;

255 }

256

257 if (TLI->shouldCastAtomicLoadInIR(LI) ==

258 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {

259 I = LI = convertAtomicLoadToIntegerType(LI);

260 MadeChange = true;

261 }

262 } else if (SI) {

263 if (->isAtomic())

264 return false;

265

267 expandAtomicStoreToLibcall(SI);

268 return true;

269 }

270

271 if (TLI->shouldCastAtomicStoreInIR(SI) ==

272 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {

273 I = SI = convertAtomicStoreToIntegerType(SI);

274 MadeChange = true;

275 }

276 } else if (RMWI) {

278 expandAtomicRMWToLibcall(RMWI);

279 return true;

280 }

281

282 if (TLI->shouldCastAtomicRMWIInIR(RMWI) ==

283 TargetLoweringBase::AtomicExpansionKind::CastToInteger) {

284 I = RMWI = convertAtomicXchgToIntegerType(RMWI);

285 MadeChange = true;

286 }

287 } else if (CASI) {

289 expandAtomicCASToLibcall(CASI);

290 return true;

291 }

292

293

294

295 if (CASI->getCompareOperand()->getType()->isPointerTy()) {

296

297

298 I = CASI = convertCmpXchgToIntegerType(CASI);

299 MadeChange = true;

300 }

301 } else

302 return false;

303

304 if (TLI->shouldInsertFencesForAtomic(I)) {

305 auto FenceOrdering = AtomicOrdering::Monotonic;

307 FenceOrdering = LI->getOrdering();

308 LI->setOrdering(AtomicOrdering::Monotonic);

310 FenceOrdering = SI->getOrdering();

311 SI->setOrdering(AtomicOrdering::Monotonic);

314 FenceOrdering = RMWI->getOrdering();

315 RMWI->setOrdering(AtomicOrdering::Monotonic);

316 } else if (CASI &&

317 TLI->shouldExpandAtomicCmpXchgInIR(CASI) ==

318 TargetLoweringBase::AtomicExpansionKind::None &&

322

323

324

325

326 FenceOrdering = CASI->getMergedOrdering();

327 CASI->setSuccessOrdering(AtomicOrdering::Monotonic);

328 CASI->setFailureOrdering(AtomicOrdering::Monotonic);

329 }

330

331 if (FenceOrdering != AtomicOrdering::Monotonic) {

332 MadeChange |= bracketInstWithFences(I, FenceOrdering);

333 }

334 } else if (I->hasAtomicStore() &&

335 TLI->shouldInsertTrailingFenceForAtomicStore(I)) {

336 auto FenceOrdering = AtomicOrdering::Monotonic;

337 if (SI)

338 FenceOrdering = SI->getOrdering();

339 else if (RMWI)

340 FenceOrdering = RMWI->getOrdering();

341 else if (CASI && TLI->shouldExpandAtomicCmpXchgInIR(CASI) !=

342 TargetLoweringBase::AtomicExpansionKind::LLSC)

343

344 FenceOrdering = CASI->getSuccessOrdering();

345

347 if (auto TrailingFence =

348 TLI->emitTrailingFence(Builder, I, FenceOrdering)) {

349 TrailingFence->moveAfter(I);

350 MadeChange = true;

351 }

352 }

353

354 if (LI)

355 MadeChange |= tryExpandAtomicLoad(LI);

356 else if (SI)

357 MadeChange |= tryExpandAtomicStore(SI);

358 else if (RMWI) {

359

360

361

362

363

364 if (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) {

365 MadeChange = true;

366

367 } else {

368 MadeChange |= tryExpandAtomicRMW(RMWI);

369 }

370 } else if (CASI)

371 MadeChange |= tryExpandAtomicCmpXchg(CASI);

372

373 return MadeChange;

374}

375

377 const auto *Subtarget = TM->getSubtargetImpl(F);

378 if (!Subtarget->enableAtomicExpand())

379 return false;

380 TLI = Subtarget->getTargetLowering();

381 DL = &F.getDataLayout();

382

383 bool MadeChange = false;

384

387

389

391 I = Next) {

393 Next = std::next(I);

394

395 if (processAtomicInstr(&Inst)) {

396 MadeChange = true;

397

398

399 BBE = F.end();

400 }

401 }

402 }

403

404 return MadeChange;

405}

406

407bool AtomicExpandLegacy::runOnFunction(Function &F) {

408

409 auto *TPC = getAnalysisIfAvailable();

410 if (!TPC)

411 return false;

413 AtomicExpandImpl AE;

414 return AE.run(F, TM);

415}

416

418 return new AtomicExpandLegacy();

419}

420

423 AtomicExpandImpl AE;

424

425 bool Changed = AE.run(F, TM);

426 if (!Changed)

428

430}

431

432bool AtomicExpandImpl::bracketInstWithFences(Instruction *I,

434 ReplacementIRBuilder Builder(I, *DL);

435

436 auto LeadingFence = TLI->emitLeadingFence(Builder, I, Order);

437

438 auto TrailingFence = TLI->emitTrailingFence(Builder, I, Order);

439

440

441 if (TrailingFence)

442 TrailingFence->moveAfter(I);

443

444 return (LeadingFence || TrailingFence);

445}

446

447

449AtomicExpandImpl::getCorrespondingIntegerType(Type *T, const DataLayout &DL) {

450 EVT VT = TLI->getMemValueType(DL, T);

454}

455

456

457

458

459LoadInst *AtomicExpandImpl::convertAtomicLoadToIntegerType(LoadInst *LI) {

461 Type *NewTy = getCorrespondingIntegerType(LI->getType(), M->getDataLayout());

462

463 ReplacementIRBuilder Builder(LI, *DL);

464

466

467 auto *NewLI = Builder.CreateLoad(NewTy, Addr);

468 NewLI->setAlignment(LI->getAlign());

469 NewLI->setVolatile(LI->isVolatile());

471 LLVM_DEBUG(dbgs() << "Replaced " << *LI << " with " << *NewLI << "\n");

472

473 Value *NewVal = Builder.CreateBitCast(NewLI, LI->getType());

476 return NewLI;

477}

478

480AtomicExpandImpl::convertAtomicXchgToIntegerType(AtomicRMWInst *RMWI) {

482

484 Type *NewTy =

485 getCorrespondingIntegerType(RMWI->getType(), M->getDataLayout());

486

487 ReplacementIRBuilder Builder(RMWI, *DL);

488

492 ? Builder.CreatePtrToInt(Val, NewTy)

493 : Builder.CreateBitCast(Val, NewTy);

494

498 NewRMWI->setVolatile(RMWI->isVolatile());

500 LLVM_DEBUG(dbgs() << "Replaced " << *RMWI << " with " << *NewRMWI << "\n");

501

503 ? Builder.CreateIntToPtr(NewRMWI, RMWI->getType())

504 : Builder.CreateBitCast(NewRMWI, RMWI->getType());

507 return NewRMWI;

508}

509

510bool AtomicExpandImpl::tryExpandAtomicLoad(LoadInst *LI) {

511 switch (TLI->shouldExpandAtomicLoadInIR(LI)) {

513 return false;

515 expandAtomicOpToLLSC(

519 return true;

521 return expandAtomicLoadToLL(LI);

523 return expandAtomicLoadToCmpXchg(LI);

526 return true;

527 default:

529 }

530}

531

532bool AtomicExpandImpl::tryExpandAtomicStore(StoreInst *SI) {

533 switch (TLI->shouldExpandAtomicStoreInIR(SI)) {

535 return false;

537 expandAtomicStore(SI);

538 return true;

541 return true;

542 default:

544 }

545}

546

547bool AtomicExpandImpl::expandAtomicLoadToLL(LoadInst *LI) {

548 ReplacementIRBuilder Builder(LI, *DL);

549

550

551

552

553 Value *Val = TLI->emitLoadLinked(Builder, LI->getType(),

555 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);

556

559

560 return true;

561}

562

563bool AtomicExpandImpl::expandAtomicLoadToCmpXchg(LoadInst *LI) {

564 ReplacementIRBuilder Builder(LI, *DL);

568

572

573 Value *Pair = Builder.CreateAtomicCmpXchg(

574 Addr, DummyVal, DummyVal, LI->getAlign(), Order,

576 Value *Loaded = Builder.CreateExtractValue(Pair, 0, "loaded");

577

580

581 return true;

582}

583

584

585

586

587

588

589

590

591

592StoreInst *AtomicExpandImpl::convertAtomicStoreToIntegerType(StoreInst *SI) {

593 ReplacementIRBuilder Builder(SI, *DL);

594 auto *M = SI->getModule();

595 Type *NewTy = getCorrespondingIntegerType(SI->getValueOperand()->getType(),

596 M->getDataLayout());

597 Value *NewVal = Builder.CreateBitCast(SI->getValueOperand(), NewTy);

598

600

601 StoreInst *NewSI = Builder.CreateStore(NewVal, Addr);

604 NewSI->setAtomic(SI->getOrdering(), SI->getSyncScopeID());

605 LLVM_DEBUG(dbgs() << "Replaced " << *SI << " with " << *NewSI << "\n");

606 SI->eraseFromParent();

607 return NewSI;

608}

609

610void AtomicExpandImpl::expandAtomicStore(StoreInst *SI) {

611

612

613

614

615

616

617 ReplacementIRBuilder Builder(SI, *DL);

625 SI->getAlign(), RMWOrdering);

626 SI->eraseFromParent();

627

628

629 tryExpandAtomicRMW(AI);

630}

631

638

639

642 if (NeedBitcast) {

646 }

647

649 Addr, Loaded, NewVal, AddrAlign, MemOpOrder,

651 if (MetadataSrc)

653

656

657 if (NeedBitcast)

658 NewLoaded = Builder.CreateBitCast(NewLoaded, OrigTy);

659}

660

661bool AtomicExpandImpl::tryExpandAtomicRMW(AtomicRMWInst *AI) {

664 switch (Kind) {

666 return false;

668 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;

670 if (ValueSize < MinCASSize) {

671 expandPartwordAtomicRMW(AI,

673 } else {

677 };

680 }

681 return true;

682 }

684 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;

686 if (ValueSize < MinCASSize) {

687 expandPartwordAtomicRMW(AI,

689 } else {

693 ? "system"

696 ORE.emit([&]() {

698 << "A compare and swap loop was generated for an atomic "

700 << MemScope << " memory scope";

701 });

703 }

704 return true;

705 }

707 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;

709 if (ValueSize < MinCASSize) {

711

714 tryExpandAtomicRMW(widenPartwordAtomicRMW(AI));

715 return true;

716 }

717 }

718 expandAtomicRMWToMaskedIntrinsic(AI);

719 return true;

720 }

722 TLI->emitBitTestAtomicRMWIntrinsic(AI);

723 return true;

724 }

726 TLI->emitCmpArithAtomicRMWIntrinsic(AI);

727 return true;

728 }

732 TLI->emitExpandAtomicRMW(AI);

733 return true;

734 default:

736 }

737}

738

739namespace {

740

741struct PartwordMaskValues {

742

743 Type *WordType = nullptr;

745 Type *IntValueType = nullptr;

746 Value *AlignedAddr = nullptr;

747 Align AlignedAddrAlignment;

748

749 Value *ShiftAmt = nullptr;

751 Value *Inv_Mask = nullptr;

752};

753

756 auto PrintObj = [&O](auto *V) {

757 if (V)

758 O << *V;

759 else

760 O << "nullptr";

761 O << '\n';

762 };

763 O << "PartwordMaskValues {\n";

764 O << " WordType: ";

765 PrintObj(PMV.WordType);

766 O << " ValueType: ";

767 PrintObj(PMV.ValueType);

768 O << " AlignedAddr: ";

769 PrintObj(PMV.AlignedAddr);

770 O << " AlignedAddrAlignment: " << PMV.AlignedAddrAlignment.value() << '\n';

771 O << " ShiftAmt: ";

772 PrintObj(PMV.ShiftAmt);

773 O << " Mask: ";

774 PrintObj(PMV.Mask);

775 O << " Inv_Mask: ";

776 PrintObj(PMV.Inv_Mask);

777 O << "}\n";

778 return O;

779}

780

781}

782

783

784

785

786

787

788

789

790

791

792

793

794

795

796

797

798

802 unsigned MinWordSize) {

803 PartwordMaskValues PMV;

804

805 Module *M = I->getModule();

808 unsigned ValueSize = DL.getTypeStoreSize(ValueType);

809

810 PMV.ValueType = PMV.IntValueType = ValueType;

811 if (PMV.ValueType->isFloatingPointTy() || PMV.ValueType->isVectorTy())

812 PMV.IntValueType =

814

815 PMV.WordType = MinWordSize > ValueSize ? Type::getIntNTy(Ctx, MinWordSize * 8)

817 if (PMV.ValueType == PMV.WordType) {

818 PMV.AlignedAddr = Addr;

819 PMV.AlignedAddrAlignment = AddrAlign;

820 PMV.ShiftAmt = ConstantInt::get(PMV.ValueType, 0);

821 PMV.Mask = ConstantInt::get(PMV.ValueType, ~0, true);

822 return PMV;

823 }

824

825 PMV.AlignedAddrAlignment = Align(MinWordSize);

826

827 assert(ValueSize < MinWordSize);

828

830 IntegerType *IntTy = DL.getIndexType(Ctx, PtrTy->getAddressSpace());

832

833 if (AddrAlign < MinWordSize) {

835 Intrinsic::ptrmask, {PtrTy, IntTy},

836 {Addr, ConstantInt::get(IntTy, ~(uint64_t)(MinWordSize - 1))}, nullptr,

837 "AlignedAddr");

838

840 PtrLSB = Builder.CreateAnd(AddrInt, MinWordSize - 1, "PtrLSB");

841 } else {

842

843 PMV.AlignedAddr = Addr;

845 }

846

847 if (DL.isLittleEndian()) {

848

849 PMV.ShiftAmt = Builder.CreateShl(PtrLSB, 3);

850 } else {

851

852 PMV.ShiftAmt = Builder.CreateShl(

853 Builder.CreateXor(PtrLSB, MinWordSize - ValueSize), 3);

854 }

855

856 PMV.ShiftAmt = Builder.CreateTrunc(PMV.ShiftAmt, PMV.WordType, "ShiftAmt");

858 ConstantInt::get(PMV.WordType, (1 << (ValueSize * 8)) - 1), PMV.ShiftAmt,

859 "Mask");

860

861 PMV.Inv_Mask = Builder.CreateNot(PMV.Mask, "Inv_Mask");

862

863 return PMV;

864}

865

867 const PartwordMaskValues &PMV) {

868 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch");

869 if (PMV.WordType == PMV.ValueType)

870 return WideWord;

871

872 Value *Shift = Builder.CreateLShr(WideWord, PMV.ShiftAmt, "shifted");

873 Value *Trunc = Builder.CreateTrunc(Shift, PMV.IntValueType, "extracted");

874 return Builder.CreateBitCast(Trunc, PMV.ValueType);

875}

876

878 Value *Updated, const PartwordMaskValues &PMV) {

879 assert(WideWord->getType() == PMV.WordType && "Widened type mismatch");

880 assert(Updated->getType() == PMV.ValueType && "Value type mismatch");

881 if (PMV.WordType == PMV.ValueType)

882 return Updated;

883

884 Updated = Builder.CreateBitCast(Updated, PMV.IntValueType);

885

886 Value *ZExt = Builder.CreateZExt(Updated, PMV.WordType, "extended");

888 Builder.CreateShl(ZExt, PMV.ShiftAmt, "shifted", true);

889 Value *And = Builder.CreateAnd(WideWord, PMV.Inv_Mask, "unmasked");

891 return Or;

892}

893

894

895

896

900 const PartwordMaskValues &PMV) {

901

902

903

904 switch (Op) {

906 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);

907 Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, Shifted_Inc);

908 return FinalVal;

909 }

913 llvm_unreachable("Or/Xor/And handled by widenPartwordAtomicRMW");

917

919 Value *NewVal_Masked = Builder.CreateAnd(NewVal, PMV.Mask);

920 Value *Loaded_MaskOut = Builder.CreateAnd(Loaded, PMV.Inv_Mask);

921 Value *FinalVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Masked);

922 return FinalVal;

923 }

936

937

938

942 return FinalVal;

943 }

944 default:

946 }

947}

948

949

950

951

952

953

954

955

956void AtomicExpandImpl::expandPartwordAtomicRMW(

958

962 tryExpandAtomicRMW(widenPartwordAtomicRMW(AI));

963 return;

964 }

967

968 ReplacementIRBuilder Builder(AI, *DL);

969

970 PartwordMaskValues PMV =

972 AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);

973

974 Value *ValOperand_Shifted = nullptr;

977 Value *ValOp = Builder.CreateBitCast(AI->getValOperand(), PMV.IntValueType);

978 ValOperand_Shifted =

979 Builder.CreateShl(Builder.CreateZExt(ValOp, PMV.WordType), PMV.ShiftAmt,

980 "ValOperand_Shifted");

981 }

982

983 auto PerformPartwordOp = [&](IRBuilderBase &Builder, Value *Loaded) {

986 };

987

988 Value *OldResult;

990 OldResult = insertRMWCmpXchgLoop(

991 Builder, PMV.WordType, PMV.AlignedAddr, PMV.AlignedAddrAlignment,

993 } else {

995 OldResult = insertRMWLLSCLoop(Builder, PMV.WordType, PMV.AlignedAddr,

996 PMV.AlignedAddrAlignment, MemOpOrder,

997 PerformPartwordOp);

998 }

999

1003}

1004

1005

1007 ReplacementIRBuilder Builder(AI, *DL);

1009

1012 "Unable to widen operation");

1013

1014 PartwordMaskValues PMV =

1016 AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);

1017

1018 Value *ValOperand_Shifted =

1020 PMV.ShiftAmt, "ValOperand_Shifted");

1021

1022 Value *NewOperand;

1023

1025 NewOperand =

1026 Builder.CreateOr(ValOperand_Shifted, PMV.Inv_Mask, "AndOperand");

1027 else

1028 NewOperand = ValOperand_Shifted;

1029

1031 Op, PMV.AlignedAddr, NewOperand, PMV.AlignedAddrAlignment,

1033

1035

1039 return NewAI;

1040}

1041

1042bool AtomicExpandImpl::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {

1043

1044

1045

1046

1047

1048

1049

1050

1051

1052

1053

1054

1055

1056

1057

1058

1059

1060

1061

1062

1063

1064

1065

1066

1067

1068

1069

1070

1071

1072

1073

1074

1075

1076

1077

1081

1084 ReplacementIRBuilder Builder(CI, *DL);

1086

1089 auto FailureBB =

1091 auto LoopBB = BasicBlock::Create(Ctx, "partword.cmpxchg.loop", F, FailureBB);

1092

1093

1094

1095 std::prev(BB->end())->eraseFromParent();

1097

1098 PartwordMaskValues PMV =

1100 CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);

1101

1102

1103 Value *NewVal_Shifted =

1104 Builder.CreateShl(Builder.CreateZExt(NewVal, PMV.WordType), PMV.ShiftAmt);

1105 Value *Cmp_Shifted =

1107

1108

1109

1110 LoadInst *InitLoaded = Builder.CreateLoad(PMV.WordType, PMV.AlignedAddr);

1112 Value *InitLoaded_MaskOut = Builder.CreateAnd(InitLoaded, PMV.Inv_Mask);

1114

1115

1117 PHINode *Loaded_MaskOut = Builder.CreatePHI(PMV.WordType, 2);

1118 Loaded_MaskOut->addIncoming(InitLoaded_MaskOut, BB);

1119

1120

1121 Value *FullWord_NewVal = Builder.CreateOr(Loaded_MaskOut, NewVal_Shifted);

1122 Value *FullWord_Cmp = Builder.CreateOr(Loaded_MaskOut, Cmp_Shifted);

1124 PMV.AlignedAddr, FullWord_Cmp, FullWord_NewVal, PMV.AlignedAddrAlignment,

1127

1128

1129

1130

1131

1133

1136

1139 else

1141

1142

1144

1145

1146

1147 Value *OldVal_MaskOut = Builder.CreateAnd(OldVal, PMV.Inv_Mask);

1148 Value *ShouldContinue = Builder.CreateICmpNE(Loaded_MaskOut, OldVal_MaskOut);

1149 Builder.CreateCondBr(ShouldContinue, LoopBB, EndBB);

1150

1151

1152 Loaded_MaskOut->addIncoming(OldVal_MaskOut, FailureBB);

1153

1154

1156

1161

1164 return true;

1165}

1166

1167void AtomicExpandImpl::expandAtomicOpToLLSC(

1171 ReplacementIRBuilder Builder(I, *DL);

1172 Value *Loaded = insertRMWLLSCLoop(Builder, ResultType, Addr, AddrAlign,

1173 MemOpOrder, PerformOp);

1174

1175 I->replaceAllUsesWith(Loaded);

1176 I->eraseFromParent();

1177}

1178

1179void AtomicExpandImpl::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {

1180 ReplacementIRBuilder Builder(AI, *DL);

1181

1182 PartwordMaskValues PMV =

1184 AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);

1185

1186

1187

1188

1192 CastOp = Instruction::SExt;

1193

1196 PMV.ShiftAmt, "ValOperand_Shifted");

1197 Value *OldResult = TLI->emitMaskedAtomicRMWIntrinsic(

1198 Builder, AI, PMV.AlignedAddr, ValOperand_Shifted, PMV.Mask, PMV.ShiftAmt,

1203}

1204

1205void AtomicExpandImpl::expandAtomicCmpXchgToMaskedIntrinsic(

1207 ReplacementIRBuilder Builder(CI, *DL);

1208

1211 CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);

1212

1215 "CmpVal_Shifted");

1218 "NewVal_Shifted");

1219 Value *OldVal = TLI->emitMaskedAtomicCmpXchgIntrinsic(

1220 Builder, CI, PMV.AlignedAddr, CmpVal_Shifted, NewVal_Shifted, PMV.Mask,

1226 CmpVal_Shifted, Builder.CreateAnd(OldVal, PMV.Mask), "Success");

1228

1231}

1232

1233Value *AtomicExpandImpl::insertRMWLLSCLoop(

1240

1242 F->getDataLayout().getTypeStoreSize(ResultTy) &&

1243 "Expected at least natural alignment at this point.");

1244

1245

1246

1247

1248

1249

1250

1251

1252

1253

1254

1255

1256

1260

1261

1262

1263 std::prev(BB->end())->eraseFromParent();

1266

1267

1269 Value *Loaded = TLI->emitLoadLinked(Builder, ResultTy, Addr, MemOpOrder);

1270

1271 Value *NewVal = PerformOp(Builder, Loaded);

1272

1273 Value *StoreSuccess =

1274 TLI->emitStoreConditional(Builder, NewVal, Addr, MemOpOrder);

1276 StoreSuccess, ConstantInt::get(IntegerType::get(Ctx, 32), 0), "tryagain");

1277 Builder.CreateCondBr(TryAgain, LoopBB, ExitBB);

1278

1280 return Loaded;

1281}

1282

1283

1284

1285

1286

1287

1289AtomicExpandImpl::convertCmpXchgToIntegerType(AtomicCmpXchgInst *CI) {

1292 M->getDataLayout());

1293

1294 ReplacementIRBuilder Builder(CI, *DL);

1295

1297

1300

1306 LLVM_DEBUG(dbgs() << "Replaced " << *CI << " with " << *NewCI << "\n");

1307

1310

1312

1316

1319 return NewCI;

1320}

1321

1322bool AtomicExpandImpl::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {

1329

1330

1331

1332

1333 bool ShouldInsertFencesForAtomic = TLI->shouldInsertFencesForAtomic(CI);

1334 AtomicOrdering MemOpOrder = ShouldInsertFencesForAtomic

1337

1338

1339

1340

1341

1342

1343

1344

1345

1346

1347 bool HasReleasedLoadBB = !CI->isWeak() && ShouldInsertFencesForAtomic &&

1350 ->hasMinSize();

1351

1352

1353

1354 bool UseUnconditionalReleaseBarrier = F->hasMinSize() && !CI->isWeak();

1355

1356

1357

1358

1359

1360

1361

1362

1363

1364

1365

1366

1367

1368

1369

1370

1371

1372

1373

1374

1375

1376

1377

1378

1379

1380

1381

1382

1383

1384

1385

1386

1387

1388

1389

1390

1391

1392

1393

1394

1395

1396

1397

1398

1399

1400

1401

1402

1403

1406 auto NoStoreBB = BasicBlock::Create(Ctx, "cmpxchg.nostore", F, FailureBB);

1407 auto SuccessBB = BasicBlock::Create(Ctx, "cmpxchg.success", F, NoStoreBB);

1408 auto ReleasedLoadBB =

1410 auto TryStoreBB =

1412 auto ReleasingStoreBB =

1414 auto StartBB = BasicBlock::Create(Ctx, "cmpxchg.start", F, ReleasingStoreBB);

1415

1416 ReplacementIRBuilder Builder(CI, *DL);

1417

1418

1419

1420

1421 std::prev(BB->end())->eraseFromParent();

1423 if (ShouldInsertFencesForAtomic && UseUnconditionalReleaseBarrier)

1424 TLI->emitLeadingFence(Builder, CI, SuccessOrder);

1425

1426 PartwordMaskValues PMV =

1428 CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);

1430

1431

1433 Value *UnreleasedLoad =

1434 TLI->emitLoadLinked(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder);

1435 Value *UnreleasedLoadExtract =

1438 UnreleasedLoadExtract, CI->getCompareOperand(), "should_store");

1439

1440

1441

1442 Builder.CreateCondBr(ShouldStore, ReleasingStoreBB, NoStoreBB);

1443

1445 if (ShouldInsertFencesForAtomic && !UseUnconditionalReleaseBarrier)

1446 TLI->emitLeadingFence(Builder, CI, SuccessOrder);

1447 Builder.CreateBr(TryStoreBB);

1448

1450 PHINode *LoadedTryStore =

1451 Builder.CreatePHI(PMV.WordType, 2, "loaded.trystore");

1452 LoadedTryStore->addIncoming(UnreleasedLoad, ReleasingStoreBB);

1453 Value *NewValueInsert =

1455 Value *StoreSuccess = TLI->emitStoreConditional(Builder, NewValueInsert,

1456 PMV.AlignedAddr, MemOpOrder);

1458 StoreSuccess, ConstantInt::get(Type::getInt32Ty(Ctx), 0), "success");

1459 BasicBlock *RetryBB = HasReleasedLoadBB ? ReleasedLoadBB : StartBB;

1460 Builder.CreateCondBr(StoreSuccess, SuccessBB,

1461 CI->isWeak() ? FailureBB : RetryBB);

1462

1464 Value *SecondLoad;

1465 if (HasReleasedLoadBB) {

1466 SecondLoad =

1467 TLI->emitLoadLinked(Builder, PMV.WordType, PMV.AlignedAddr, MemOpOrder);

1469 ShouldStore = Builder.CreateICmpEQ(SecondLoadExtract,

1471

1472

1473

1474 Builder.CreateCondBr(ShouldStore, TryStoreBB, NoStoreBB);

1475

1476 LoadedTryStore->addIncoming(SecondLoad, ReleasedLoadBB);

1477 } else

1479

1480

1481

1483 if (ShouldInsertFencesForAtomic ||

1484 TLI->shouldInsertTrailingFenceForAtomicStore(CI))

1485 TLI->emitTrailingFence(Builder, CI, SuccessOrder);

1487

1489 PHINode *LoadedNoStore =

1490 Builder.CreatePHI(UnreleasedLoad->getType(), 2, "loaded.nostore");

1491 LoadedNoStore->addIncoming(UnreleasedLoad, StartBB);

1492 if (HasReleasedLoadBB)

1493 LoadedNoStore->addIncoming(SecondLoad, ReleasedLoadBB);

1494

1495

1496

1497

1498 TLI->emitAtomicCmpXchgNoStoreLLBalance(Builder);

1499 Builder.CreateBr(FailureBB);

1500

1502 PHINode *LoadedFailure =

1503 Builder.CreatePHI(UnreleasedLoad->getType(), 2, "loaded.failure");

1504 LoadedFailure->addIncoming(LoadedNoStore, NoStoreBB);

1506 LoadedFailure->addIncoming(LoadedTryStore, TryStoreBB);

1507 if (ShouldInsertFencesForAtomic)

1508 TLI->emitTrailingFence(Builder, CI, FailureOrder);

1510

1511

1512

1513

1514

1517 Builder.CreatePHI(UnreleasedLoad->getType(), 2, "loaded.exit");

1518 LoadedExit->addIncoming(LoadedTryStore, SuccessBB);

1519 LoadedExit->addIncoming(LoadedFailure, FailureBB);

1523

1524

1525

1526 Value *LoadedFull = LoadedExit;

1527

1530

1531

1532

1534 for (auto *User : CI->users()) {

1536 if (!EV)

1537 continue;

1538

1540 "weird extraction from { iN, i1 }");

1541

1544 else

1546

1548 }

1549

1550

1551 for (auto *EV : PrunedInsts)

1553

1555

1556

1560

1562 }

1563

1565 return true;

1566}

1567

1568bool AtomicExpandImpl::isIdempotentRMW(AtomicRMWInst *RMWI) {

1569 auto C = dyn_cast(RMWI->getValOperand());

1570 if ()

1571 return false;

1572

1574 switch (Op) {

1579 return C->isZero();

1581 return C->isMinusOne();

1582

1583 default:

1584 return false;

1585 }

1586}

1587

1588bool AtomicExpandImpl::simplifyIdempotentRMW(AtomicRMWInst *RMWI) {

1589 if (auto ResultingLoad = TLI->lowerIdempotentRMWIntoFencedLoad(RMWI)) {

1590 tryExpandAtomicLoad(ResultingLoad);

1591 return true;

1592 }

1593 return false;

1594}

1595

1596Value *AtomicExpandImpl::insertRMWCmpXchgLoop(

1604

1605

1606

1607

1608

1609

1610

1611

1612

1613

1614

1615

1616

1617

1618

1619

1623

1624

1625

1626

1627 std::prev(BB->end())->eraseFromParent();

1631

1632

1636

1637 Value *NewVal = PerformOp(Builder, Loaded);

1638

1639 Value *NewLoaded = nullptr;

1641

1642 CreateCmpXchg(Builder, Addr, Loaded, NewVal, AddrAlign,

1645 : MemOpOrder,

1646 SSID, Success, NewLoaded, MetadataSrc);

1648

1650

1652

1654 return NewLoaded;

1655}

1656

1657bool AtomicExpandImpl::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) {

1658 unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;

1660

1661 switch (TLI->shouldExpandAtomicCmpXchgInIR(CI)) {

1662 default:

1663 llvm_unreachable("Unhandled case in tryExpandAtomicCmpXchg");

1665 if (ValueSize < MinCASSize)

1666 return expandPartwordCmpXchg(CI);

1667 return false;

1669 return expandAtomicCmpXchg(CI);

1670 }

1672 expandAtomicCmpXchgToMaskedIntrinsic(CI);

1673 return true;

1677 TLI->emitExpandAtomicCmpXchg(CI);

1678 return true;

1679 }

1680 }

1681}

1682

1683

1686 ReplacementIRBuilder Builder(AI, AI->getDataLayout());

1687 Builder.setIsFPConstrained(

1689

1690

1691

1692 Value *Loaded = AtomicExpandImpl::insertRMWCmpXchgLoop(

1696 return buildAtomicRMWValue(AI->getOperation(), Builder, Loaded,

1697 AI->getValOperand());

1698 },

1699 CreateCmpXchg, AI);

1700

1703 return true;

1704}

1705

1706

1707

1708

1709

1710

1713

1714

1715

1716

1717

1718

1719

1720 unsigned LargestSize = DL.getLargestLegalIntTypeSizeInBits() >= 64 ? 16 : 8;

1721 return Alignment >= Size &&

1723 Size <= LargestSize;

1724}

1725

1726void AtomicExpandImpl::expandAtomicLoadToLibcall(LoadInst *I) {

1728 RTLIB::ATOMIC_LOAD, RTLIB::ATOMIC_LOAD_1, RTLIB::ATOMIC_LOAD_2,

1729 RTLIB::ATOMIC_LOAD_4, RTLIB::ATOMIC_LOAD_8, RTLIB::ATOMIC_LOAD_16};

1731

1732 bool expanded = expandAtomicOpToLibcall(

1733 I, Size, I->getAlign(), I->getPointerOperand(), nullptr, nullptr,

1735 if (!expanded)

1736 report_fatal_error("expandAtomicOpToLibcall shouldn't fail for Load");

1737}

1738

1739void AtomicExpandImpl::expandAtomicStoreToLibcall(StoreInst *I) {

1741 RTLIB::ATOMIC_STORE, RTLIB::ATOMIC_STORE_1, RTLIB::ATOMIC_STORE_2,

1742 RTLIB::ATOMIC_STORE_4, RTLIB::ATOMIC_STORE_8, RTLIB::ATOMIC_STORE_16};

1744

1745 bool expanded = expandAtomicOpToLibcall(

1746 I, Size, I->getAlign(), I->getPointerOperand(), I->getValueOperand(),

1748 if (!expanded)

1749 report_fatal_error("expandAtomicOpToLibcall shouldn't fail for Store");

1750}

1751

1752void AtomicExpandImpl::expandAtomicCASToLibcall(AtomicCmpXchgInst *I) {

1754 RTLIB::ATOMIC_COMPARE_EXCHANGE, RTLIB::ATOMIC_COMPARE_EXCHANGE_1,

1755 RTLIB::ATOMIC_COMPARE_EXCHANGE_2, RTLIB::ATOMIC_COMPARE_EXCHANGE_4,

1756 RTLIB::ATOMIC_COMPARE_EXCHANGE_8, RTLIB::ATOMIC_COMPARE_EXCHANGE_16};

1758

1759 bool expanded = expandAtomicOpToLibcall(

1760 I, Size, I->getAlign(), I->getPointerOperand(), I->getNewValOperand(),

1761 I->getCompareOperand(), I->getSuccessOrdering(), I->getFailureOrdering(),

1762 Libcalls);

1763 if (!expanded)

1764 report_fatal_error("expandAtomicOpToLibcall shouldn't fail for CAS");

1765}

1766

1769 RTLIB::ATOMIC_EXCHANGE, RTLIB::ATOMIC_EXCHANGE_1,

1770 RTLIB::ATOMIC_EXCHANGE_2, RTLIB::ATOMIC_EXCHANGE_4,

1771 RTLIB::ATOMIC_EXCHANGE_8, RTLIB::ATOMIC_EXCHANGE_16};

1773 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_ADD_1,

1774 RTLIB::ATOMIC_FETCH_ADD_2, RTLIB::ATOMIC_FETCH_ADD_4,

1775 RTLIB::ATOMIC_FETCH_ADD_8, RTLIB::ATOMIC_FETCH_ADD_16};

1777 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_SUB_1,

1778 RTLIB::ATOMIC_FETCH_SUB_2, RTLIB::ATOMIC_FETCH_SUB_4,

1779 RTLIB::ATOMIC_FETCH_SUB_8, RTLIB::ATOMIC_FETCH_SUB_16};

1781 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_AND_1,

1782 RTLIB::ATOMIC_FETCH_AND_2, RTLIB::ATOMIC_FETCH_AND_4,

1783 RTLIB::ATOMIC_FETCH_AND_8, RTLIB::ATOMIC_FETCH_AND_16};

1785 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_OR_1,

1786 RTLIB::ATOMIC_FETCH_OR_2, RTLIB::ATOMIC_FETCH_OR_4,

1787 RTLIB::ATOMIC_FETCH_OR_8, RTLIB::ATOMIC_FETCH_OR_16};

1789 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_XOR_1,

1790 RTLIB::ATOMIC_FETCH_XOR_2, RTLIB::ATOMIC_FETCH_XOR_4,

1791 RTLIB::ATOMIC_FETCH_XOR_8, RTLIB::ATOMIC_FETCH_XOR_16};

1793 RTLIB::UNKNOWN_LIBCALL, RTLIB::ATOMIC_FETCH_NAND_1,

1794 RTLIB::ATOMIC_FETCH_NAND_2, RTLIB::ATOMIC_FETCH_NAND_4,

1795 RTLIB::ATOMIC_FETCH_NAND_8, RTLIB::ATOMIC_FETCH_NAND_16};

1796

1797 switch (Op) {

1801 return ArrayRef(LibcallsXchg);

1803 return ArrayRef(LibcallsAdd);

1805 return ArrayRef(LibcallsSub);

1807 return ArrayRef(LibcallsAnd);

1809 return ArrayRef(LibcallsOr);

1811 return ArrayRef(LibcallsXor);

1813 return ArrayRef(LibcallsNand);

1826

1827 return {};

1828 }

1830}

1831

1832void AtomicExpandImpl::expandAtomicRMWToLibcall(AtomicRMWInst *I) {

1834

1836

1838 if (!Libcalls.empty())

1839 Success = expandAtomicOpToLibcall(

1840 I, Size, I->getAlign(), I->getPointerOperand(), I->getValOperand(),

1842

1843

1844

1845

1846

1853

1855 Addr, Loaded, NewVal, Alignment, MemOpOrder,

1857 if (MetadataSrc)

1859

1862

1863

1864 expandAtomicCASToLibcall(Pair);

1865 });

1866 }

1867}

1868

1869

1870

1871

1872

1873

1874

1875bool AtomicExpandImpl::expandAtomicOpToLibcall(

1880

1885 IRBuilder<> AllocaBuilder(&I->getFunction()->getEntryBlock().front());

1886

1889

1890 const Align AllocaAlignment = DL.getPrefTypeAlign(SizedIntTy);

1891

1892

1893

1898 Constant *Ordering2Val = nullptr;

1899 if (CASExpected) {

1901 Ordering2Val =

1903 }

1905

1907 if (UseSizedLibcall) {

1908 switch (Size) {

1909 case 1:

1910 RTLibType = Libcalls[1];

1911 break;

1912 case 2:

1913 RTLibType = Libcalls[2];

1914 break;

1915 case 4:

1916 RTLibType = Libcalls[3];

1917 break;

1918 case 8:

1919 RTLibType = Libcalls[4];

1920 break;

1921 case 16:

1922 RTLibType = Libcalls[5];

1923 break;

1924 }

1925 } else if (Libcalls[0] != RTLIB::UNKNOWN_LIBCALL) {

1926 RTLibType = Libcalls[0];

1927 } else {

1928

1929

1930 return false;

1931 }

1932

1933 if (!TLI->getLibcallName(RTLibType)) {

1934

1935 return false;

1936 }

1937

1938

1939

1940

1941

1942

1943

1944

1945

1946

1947

1948

1949

1950

1951

1952

1953

1954

1955

1956

1957

1958

1959

1960

1961

1962

1963

1964 AllocaInst *AllocaCASExpected = nullptr;

1967

1968 Type *ResultTy;

1971

1972

1973 if (!UseSizedLibcall) {

1974

1975 Args.push_back(ConstantInt::get(DL.getIntPtrType(Ctx), Size));

1976 }

1977

1978

1979

1980

1981

1982

1983 Value *PtrVal = PointerOperand;

1985 Args.push_back(PtrVal);

1986

1987

1988 if (CASExpected) {

1989 AllocaCASExpected = AllocaBuilder.CreateAlloca(CASExpected->getType());

1990 AllocaCASExpected->setAlignment(AllocaAlignment);

1992 Builder.CreateAlignedStore(CASExpected, AllocaCASExpected, AllocaAlignment);

1993 Args.push_back(AllocaCASExpected);

1994 }

1995

1996

1997 if (ValueOperand) {

1998 if (UseSizedLibcall) {

1999 Value *IntValue =

2001 Args.push_back(IntValue);

2002 } else {

2003 AllocaValue = AllocaBuilder.CreateAlloca(ValueOperand->getType());

2006 Builder.CreateAlignedStore(ValueOperand, AllocaValue, AllocaAlignment);

2007 Args.push_back(AllocaValue);

2008 }

2009 }

2010

2011

2012 if (!CASExpected && HasResult && !UseSizedLibcall) {

2013 AllocaResult = AllocaBuilder.CreateAlloca(I->getType());

2014 AllocaResult->setAlignment(AllocaAlignment);

2016 Args.push_back(AllocaResult);

2017 }

2018

2019

2020 Args.push_back(OrderingVal);

2021

2022

2023 if (Ordering2Val)

2024 Args.push_back(Ordering2Val);

2025

2026

2027 if (CASExpected) {

2030 } else if (HasResult && UseSizedLibcall)

2031 ResultTy = SizedIntTy;

2032 else

2034

2035

2037 for (Value *Arg : Args)

2038 ArgTys.push_back(Arg->getType());

2041 M->getOrInsertFunction(TLI->getLibcallName(RTLibType), FnType, Attr);

2043 Call->setAttributes(Attr);

2045

2046

2047 if (ValueOperand && !UseSizedLibcall)

2049

2050 if (CASExpected) {

2051

2052

2053 Type *FinalResultTy = I->getType();

2056 CASExpected->getType(), AllocaCASExpected, AllocaAlignment);

2060 I->replaceAllUsesWith(V);

2061 } else if (HasResult) {

2063 if (UseSizedLibcall)

2065 else {

2067 AllocaAlignment);

2069 }

2070 I->replaceAllUsesWith(V);

2071 }

2072 I->eraseFromParent();

2073 return true;

2074}

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

Expand Atomic instructions

static Value * performMaskedAtomicOp(AtomicRMWInst::BinOp Op, IRBuilderBase &Builder, Value *Loaded, Value *Shifted_Inc, Value *Inc, const PartwordMaskValues &PMV)

Emit IR to implement a masked version of a given atomicrmw operation.

static PartwordMaskValues createMaskInstrs(IRBuilderBase &Builder, Instruction *I, Type *ValueType, Value *Addr, Align AddrAlign, unsigned MinWordSize)

This is a helper function which builds instructions to provide values necessary for partword atomic o...

static bool canUseSizedAtomicCall(unsigned Size, Align Alignment, const DataLayout &DL)

static Value * extractMaskedValue(IRBuilderBase &Builder, Value *WideWord, const PartwordMaskValues &PMV)

static void createCmpXchgInstFun(IRBuilderBase &Builder, Value *Addr, Value *Loaded, Value *NewVal, Align AddrAlign, AtomicOrdering MemOpOrder, SyncScope::ID SSID, Value *&Success, Value *&NewLoaded, Instruction *MetadataSrc)

Expand Atomic static false unsigned getAtomicOpSize(LoadInst *LI)

static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I)

static Value * insertMaskedValue(IRBuilderBase &Builder, Value *WideWord, Value *Updated, const PartwordMaskValues &PMV)

static void copyMetadataForAtomic(Instruction &Dest, const Instruction &Source)

Copy metadata that's safe to preserve when widening atomics.

static ArrayRef< RTLIB::Libcall > GetRMWLibcall(AtomicRMWInst::BinOp Op)

Atomic ordering constants.

This file contains the simple types necessary to represent the attributes associated with functions a...

#define LLVM_ATTRIBUTE_UNUSED

This file contains the declarations for the subclasses of Constant, which represent the different fla...

Module.h This file contains the declarations for the Module class.

This file provides utility for Memory Model Relaxation Annotations (MMRAs).

#define INITIALIZE_PASS_DEPENDENCY(depName)

#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)

#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

This file defines the SmallVector class.

This file describes how to lower LLVM code to machine code.

Target-Independent Code Generator Pass Configuration Options pass.

an instruction to allocate memory on the stack

void setAlignment(Align Align)

A container for analyses that lazily runs them and caches their results.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

bool empty() const

empty - Check if the array is empty.

An instruction that atomically checks whether a specified value is in a memory location,...

Value * getNewValOperand()

AtomicOrdering getMergedOrdering() const

Returns a single ordering which is at least as strong as both the success and failure orderings for t...

void setWeak(bool IsWeak)

bool isVolatile() const

Return true if this is a cmpxchg from a volatile memory location.

Value * getCompareOperand()

AtomicOrdering getFailureOrdering() const

Returns the failure ordering constraint of this cmpxchg instruction.

Value * getPointerOperand()

static AtomicOrdering getStrongestFailureOrdering(AtomicOrdering SuccessOrdering)

Returns the strongest permitted ordering on failure, given the desired ordering on success.

Align getAlign() const

Return the alignment of the memory that is being allocated by the instruction.

bool isWeak() const

Return true if this cmpxchg may spuriously fail.

void setVolatile(bool V)

Specify whether this is a volatile cmpxchg.

AtomicOrdering getSuccessOrdering() const

Returns the success ordering constraint of this cmpxchg instruction.

SyncScope::ID getSyncScopeID() const

Returns the synchronization scope ID of this cmpxchg instruction.

PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)

an instruction that atomically reads a memory location, combines it with another value,...

Align getAlign() const

Return the alignment of the memory that is being allocated by the instruction.

bool isVolatile() const

Return true if this is a RMW on a volatile memory location.

BinOp

This enumeration lists the possible modifications atomicrmw can make.

@ USubCond

Subtract only if no unsigned overflow.

@ Min

*p = old <signed v ? old : v

@ USubSat

*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.

@ UIncWrap

Increment one up to a maximum value.

@ Max

*p = old >signed v ? old : v

@ UMin

*p = old <unsigned v ? old : v

@ FMin

*p = minnum(old, v) minnum matches the behavior of llvm.minnum.

@ UMax

*p = old >unsigned v ? old : v

@ FMax

*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.

@ UDecWrap

Decrement one until a minimum value or zero.

Value * getPointerOperand()

BinOp getOperation() const

SyncScope::ID getSyncScopeID() const

Returns the synchronization scope ID of this rmw instruction.

static StringRef getOperationName(BinOp Op)

AtomicOrdering getOrdering() const

Returns the ordering constraint of this rmw instruction.

AttributeList addRetAttribute(LLVMContext &C, Attribute::AttrKind Kind) const

Add a return value attribute to the list.

LLVM Basic Block Representation.

iterator begin()

Instruction iterator methods.

reverse_iterator rbegin()

static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)

Creates a new BasicBlock.

BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)

Split the basic block into two basic blocks at the specified instruction.

InstListType::reverse_iterator reverse_iterator

const Function * getParent() const

Return the enclosing method, or null if none.

This class represents a function call, abstracting a target machine's calling convention.

This is the shared class of boolean and integer constants.

static ConstantInt * getTrue(LLVMContext &Context)

static ConstantInt * getFalse(LLVMContext &Context)

This is an important base class in LLVM.

static Constant * getNullValue(Type *Ty)

Constructor to create a '0' constant of arbitrary type.

This class represents an Operation in the Expression.

A parsed version of the target data layout string in and methods for querying it.

A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...

FunctionPass class - This class is used to implement most global optimizations.

virtual bool runOnFunction(Function &F)=0

runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass.

static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)

This static method is the primary way of constructing a FunctionType.

BasicBlockListType::iterator iterator

bool hasFnAttribute(Attribute::AttrKind Kind) const

Return true if the function has the attribute.

Common base class shared among various IRBuilders.

AtomicCmpXchgInst * CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, MaybeAlign Align, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SyncScope::ID SSID=SyncScope::System)

Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")

IntegerType * getIntNTy(unsigned N)

Fetch the type representing an N-bit integer.

LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)

UnreachableInst * CreateUnreachable()

CallInst * CreateLifetimeStart(Value *Ptr, ConstantInt *Size=nullptr)

Create a lifetime.start intrinsic.

Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")

BasicBlock::iterator GetInsertPoint() const

Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")

Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)

Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr, FMFSource FMFSource={})

BasicBlock * GetInsertBlock() const

Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")

void CollectMetadataToCopy(Instruction *Src, ArrayRef< unsigned > MetadataKinds)

Collect metadata with IDs MetadataKinds from Src which should be added to all created instructions.

CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")

Create a call to intrinsic ID with Args, mangled using Types.

Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")

PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")

Value * CreateNot(Value *V, const Twine &Name="")

Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")

void setIsFPConstrained(bool IsCon)

Enable/Disable use of constrained floating point math.

Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")

BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)

Create a conditional 'br Cond, TrueDest, FalseDest' instruction.

LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)

Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...

Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)

Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)

LLVMContext & getContext() const

Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")

Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")

CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)

AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)

Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)

CallInst * CreateLifetimeEnd(Value *Ptr, ConstantInt *Size=nullptr)

Create a lifetime.end intrinsic.

Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")

BranchInst * CreateBr(BasicBlock *Dest)

Create an unconditional 'br label X' instruction.

void SetInsertPoint(BasicBlock *TheBB)

This specifies that created instructions should be appended to the end of the specified block.

StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)

Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")

Provides an 'InsertHelper' that calls a user-provided callback after performing the default insertion...

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

InstSimplifyFolder - Use InstructionSimplify to fold operations to existing values.

const Module * getModule() const

Return the module owning the function this instruction belongs to or nullptr it the function does not...

InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

const Function * getFunction() const

Return the function this instruction belongs to.

void setMetadata(unsigned KindID, MDNode *Node)

Set the metadata of the specified kind to the specified node.

const DataLayout & getDataLayout() const

Get the data layout of the module this instruction belongs to.

Class to represent integer types.

static IntegerType * get(LLVMContext &C, unsigned NumBits)

This static method is the primary way of constructing an IntegerType.

This is an important class for using LLVM in a threaded context.

unsigned getMDKindID(StringRef Name) const

getMDKindID - Return a unique non-zero ID for the specified metadata kind.

void getSyncScopeNames(SmallVectorImpl< StringRef > &SSNs) const

getSyncScopeNames - Populates client supplied SmallVector with synchronization scope names registered...

An instruction for reading from memory.

Value * getPointerOperand()

bool isVolatile() const

Return true if this is a load from a volatile memory location.

void setAtomic(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)

Sets the ordering constraint and the synchronization scope ID of this load instruction.

AtomicOrdering getOrdering() const

Returns the ordering constraint of this load instruction.

void setVolatile(bool V)

Specify whether this is a volatile load or not.

SyncScope::ID getSyncScopeID() const

Returns the synchronization scope ID of this load instruction.

Align getAlign() const

Return the alignment of the access that is being performed.

A Module instance is used to store all the information related to an LLVM module.

LLVMContext & getContext() const

Get the global data context.

void addIncoming(Value *V, BasicBlock *BB)

Add an incoming value to the end of the PHI list.

static PassRegistry * getPassRegistry()

getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...

static PointerType * getUnqual(Type *ElementType)

This constructs a pointer to an object of the specified type in the default address space (address sp...

static PoisonValue * get(Type *T)

Static factory methods - Return an 'poison' object of the specified type.

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

An instruction for storing to memory.

void setVolatile(bool V)

Specify whether this is a volatile store or not.

void setAlignment(Align Align)

void setAtomic(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)

Sets the ordering constraint and the synchronization scope ID of this store instruction.

unsigned getMaxAtomicSizeInBitsSupported() const

Returns the maximum atomic operation size (in bits) supported by the backend.

AtomicExpansionKind

Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.

This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...

Primary interface to the complete machine description for the target machine.

Target-Independent Code Generator Pass Configuration Options.

The instances of the Type class are immutable: once they are created, they are never changed.

bool isVectorTy() const

True if this is an instance of VectorType.

bool isPointerTy() const

True if this is an instance of PointerType.

static IntegerType * getInt1Ty(LLVMContext &C)

static IntegerType * getIntNTy(LLVMContext &C, unsigned N)

static Type * getVoidTy(LLVMContext &C)

bool isFloatingPointTy() const

Return true if this is one of the floating-point types.

static IntegerType * getInt32Ty(LLVMContext &C)

static IntegerType * getInt64Ty(LLVMContext &C)

TypeSize getPrimitiveSizeInBits() const LLVM_READONLY

Return the basic size of this type if it is a primitive type.

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

void replaceAllUsesWith(Value *V)

Change all uses of this to point to a new Value.

iterator_range< user_iterator > users()

LLVMContext & getContext() const

All values hold a context through their type.

An efficient, type-erasing, non-owning reference to a callable.

const ParentTy * getParent() const

self_iterator getIterator()

This class implements an extremely fast bulk output stream that can only output to a stream.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

constexpr char Args[]

Key for Kernel::Metadata::mArgs.

constexpr std::underlying_type_t< E > Mask()

Get a bitmask with 1s in all places up to the high-order bit of E's largest value.

@ C

The default llvm calling convention, compatible with C.

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

Libcall

RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.

This is an optimization pass for GlobalISel generic memory operations.

void initializeAtomicExpandLegacyPass(PassRegistry &)

bool canInstructionHaveMMRAs(const Instruction &I)

bool isReleaseOrStronger(AtomicOrdering AO)

AtomicOrderingCABI toCABI(AtomicOrdering AO)

raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

void report_fatal_error(Error Err, bool gen_crash_diag=true)

Report a serious error, calling any installed error handler.

Value * buildAtomicRMWValue(AtomicRMWInst::BinOp Op, IRBuilderBase &Builder, Value *Loaded, Value *Val)

Emit IR to implement the given atomicrmw operation on values in registers, returning the new value.

AtomicOrdering

Atomic ordering for LLVM's memory model.

bool expandAtomicRMWToCmpXchg(AtomicRMWInst *AI, CreateCmpXchgInstFun CreateCmpXchg)

Expand an atomic RMW instruction into a loop utilizing cmpxchg.

@ Or

Bitwise or logical OR of integers.

@ And

Bitwise or logical AND of integers.

raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)

bool isAcquireOrStronger(AtomicOrdering AO)

constexpr unsigned BitWidth

bool lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI)

Convert the given Cmpxchg into primitive load and compare.

bool lowerAtomicRMWInst(AtomicRMWInst *RMWI)

Convert the given RMWI into primitive load and stores, assuming that doing so is legal.

FunctionPass * createAtomicExpandLegacyPass()

AtomicExpandPass - At IR level this pass replace atomic instructions with __atomic_* library calls,...

PointerUnion< const Value *, const PseudoSourceValue * > ValueType

char & AtomicExpandID

AtomicExpandID – Lowers atomic operations in terms of either cmpxchg load-linked/store-conditional lo...

This struct is a compact representation of a valid (non-zero power of two) alignment.

TypeSize getSizeInBits() const

Return the size of the specified value type in bits.

TypeSize getStoreSizeInBits() const

Return the number of bits overwritten by a store of the specified value type.